From 026eb6834a94626c4c808d39867e1fb8fc427848 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2026 20:07:21 +0000 Subject: [PATCH 1/5] Add AT-SPI2 Linux perception engine Co-Authored-By: Matthew Lam --- Package.swift | 19 +- Sources/CAtSpi/include/catspi_shim.h | 247 +++++++++++++++ Sources/CAtSpi/module.modulemap | 7 + .../computer-use-mcp/Core/AppResolver.swift | 4 + Sources/computer-use-mcp/HealthReport.swift | 11 +- Sources/computer-use-mcp/LinuxSupport.swift | 292 ++++++++++++++++-- 6 files changed, 555 insertions(+), 25 deletions(-) create mode 100644 Sources/CAtSpi/include/catspi_shim.h create mode 100644 Sources/CAtSpi/module.modulemap diff --git a/Package.swift b/Package.swift index ced8ba3..5ab8c2b 100644 --- a/Package.swift +++ b/Package.swift @@ -15,9 +15,24 @@ let package = Package( .executableTarget( name: "computer-use-mcp", dependencies: [ - .product(name: "MCP", package: "swift-sdk") + .product(name: "MCP", package: "swift-sdk"), + .target(name: "CAtSpi", condition: .when(platforms: [.linux])), ], - path: "Sources/computer-use-mcp" + path: "Sources/computer-use-mcp", + linkerSettings: [ + .linkedLibrary("atspi", .when(platforms: [.linux])), + .linkedLibrary("gobject-2.0", .when(platforms: [.linux])), + .linkedLibrary("glib-2.0", .when(platforms: [.linux])), + .linkedLibrary("dbus-1", .when(platforms: [.linux])), + ] + ), + .systemLibrary( + name: "CAtSpi", + path: "Sources/CAtSpi", + pkgConfig: "atspi-2", + providers: [ + .apt(["libatspi2.0-dev"]) + ] ), // Deterministic GUI fixture app for the end-to-end "truth suite". // See docs/fixture-app.md. diff --git a/Sources/CAtSpi/include/catspi_shim.h b/Sources/CAtSpi/include/catspi_shim.h new file mode 100644 index 0000000..021cb29 --- /dev/null +++ b/Sources/CAtSpi/include/catspi_shim.h @@ -0,0 +1,247 @@ +#ifndef COMPUTER_USE_MCP_CATSPi_SHIM_H +#define COMPUTER_USE_MCP_CATSPi_SHIM_H + +#include +#include + +typedef struct { + int x; + int y; + int width; + int height; +} CAtSpiRect; + +static inline int catspi_init(void) { + return atspi_init(); +} + +static inline AtspiAccessible *catspi_desktop(void) { + return atspi_get_desktop(0); +} + +static inline AtspiAccessible *catspi_ref(AtspiAccessible *object) { + return object == NULL ? NULL : g_object_ref(object); +} + +static inline void catspi_unref(AtspiAccessible *object) { + if (object != NULL) { + g_object_unref(object); + } +} + +static inline void catspi_free_string(char *value) { + if (value != NULL) { + g_free(value); + } +} + +static inline AtspiAccessible *catspi_application_for_pid(guint pid) { + AtspiAccessible *desktop = catspi_desktop(); + if (desktop == NULL) { + return NULL; + } + + GError *error = NULL; + gint count = atspi_accessible_get_child_count(desktop, &error); + if (error != NULL) { + g_error_free(error); + g_object_unref(desktop); + return NULL; + } + + for (gint index = 0; index < count; index++) { + error = NULL; + AtspiAccessible *child = atspi_accessible_get_child_at_index(desktop, index, &error); + if (error != NULL) { + g_error_free(error); + continue; + } + if (child == NULL) { + continue; + } + error = NULL; + guint child_pid = atspi_accessible_get_process_id(child, &error); + if (error == NULL && child_pid == pid) { + g_object_unref(desktop); + return child; + } + if (error != NULL) { + g_error_free(error); + } + g_object_unref(child); + } + g_object_unref(desktop); + return NULL; +} + +static inline guint catspi_process_id(AtspiAccessible *object) { + GError *error = NULL; + guint pid = atspi_accessible_get_process_id(object, &error); + if (error != NULL) { + g_error_free(error); + return 0; + } + return pid; +} + +static inline char *catspi_name(AtspiAccessible *object) { + GError *error = NULL; + char *value = atspi_accessible_get_name(object, &error); + if (error != NULL) { + g_error_free(error); + } + return value; +} + +static inline char *catspi_description(AtspiAccessible *object) { + GError *error = NULL; + char *value = atspi_accessible_get_description(object, &error); + if (error != NULL) { + g_error_free(error); + } + return value; +} + +static inline char *catspi_role_name(AtspiAccessible *object) { + GError *error = NULL; + char *value = atspi_accessible_get_role_name(object, &error); + if (error != NULL) { + g_error_free(error); + } + return value; +} + +static inline int catspi_child_count(AtspiAccessible *object) { + GError *error = NULL; + gint count = atspi_accessible_get_child_count(object, &error); + if (error != NULL) { + g_error_free(error); + return 0; + } + return count; +} + +static inline AtspiAccessible *catspi_child_at_index(AtspiAccessible *object, int index) { + GError *error = NULL; + AtspiAccessible *child = atspi_accessible_get_child_at_index(object, index, &error); + if (error != NULL) { + g_error_free(error); + return NULL; + } + return child; +} + +static inline AtspiAccessible *catspi_parent(AtspiAccessible *object) { + GError *error = NULL; + AtspiAccessible *parent = atspi_accessible_get_parent(object, &error); + if (error != NULL) { + g_error_free(error); + return NULL; + } + return parent; +} + +static inline int catspi_state(AtspiAccessible *object, int state) { + AtspiStateSet *states = atspi_accessible_get_state_set(object); + if (states == NULL) { + return 0; + } + int result = atspi_state_set_contains(states, (AtspiStateType)state); + g_object_unref(states); + return result; +} + +static inline int catspi_extents(AtspiAccessible *object, CAtSpiRect *out) { + AtspiComponent *component = atspi_accessible_get_component_iface(object); + if (component == NULL || out == NULL) { + if (component != NULL) { + g_object_unref(component); + } + return 0; + } + GError *error = NULL; + AtspiRect *rect = atspi_component_get_extents(component, ATSPI_COORD_TYPE_SCREEN, &error); + if (error != NULL || rect == NULL) { + if (error != NULL) { + g_error_free(error); + } + g_object_unref(component); + return 0; + } + out->x = rect->x; + out->y = rect->y; + out->width = rect->width; + out->height = rect->height; + g_free(rect); + g_object_unref(component); + return 1; +} + +static inline int catspi_do_action(AtspiAccessible *object, int index) { + AtspiAction *action = atspi_accessible_get_action_iface(object); + if (action == NULL) { + return 0; + } + GError *error = NULL; + gboolean result = atspi_action_do_action(action, index, &error); + if (error != NULL) { + g_error_free(error); + } + g_object_unref(action); + return result; +} + +static inline char *catspi_text(AtspiAccessible *object) { + AtspiText *text = atspi_accessible_get_text_iface(object); + if (text == NULL) { + return NULL; + } + GError *error = NULL; + gint count = atspi_text_get_character_count(text, &error); + if (error != NULL || count < 1) { + if (error != NULL) { + g_error_free(error); + } + g_object_unref(text); + return NULL; + } + error = NULL; + char *value = atspi_text_get_text(text, 0, count, &error); + if (error != NULL) { + g_error_free(error); + } + g_object_unref(text); + return value; +} + +static inline int catspi_action_count(AtspiAccessible *object) { + AtspiAction *action = atspi_accessible_get_action_iface(object); + if (action == NULL) { + return 0; + } + GError *error = NULL; + gint count = atspi_action_get_n_actions(action, &error); + if (error != NULL) { + g_error_free(error); + g_object_unref(action); + return 0; + } + g_object_unref(action); + return count; +} + +static inline char *catspi_action_name(AtspiAccessible *object, int index) { + AtspiAction *action = atspi_accessible_get_action_iface(object); + if (action == NULL) { + return NULL; + } + GError *error = NULL; + char *name = atspi_action_get_name(action, index, &error); + if (error != NULL) { + g_error_free(error); + } + g_object_unref(action); + return name; +} + +#endif diff --git a/Sources/CAtSpi/module.modulemap b/Sources/CAtSpi/module.modulemap new file mode 100644 index 0000000..b737f16 --- /dev/null +++ b/Sources/CAtSpi/module.modulemap @@ -0,0 +1,7 @@ +module CAtSpi [system] { + header "include/catspi_shim.h" + link "atspi" + link "gobject-2.0" + link "glib-2.0" + export * +} diff --git a/Sources/computer-use-mcp/Core/AppResolver.swift b/Sources/computer-use-mcp/Core/AppResolver.swift index a57ed3e..7559bbb 100644 --- a/Sources/computer-use-mcp/Core/AppResolver.swift +++ b/Sources/computer-use-mcp/Core/AppResolver.swift @@ -45,6 +45,9 @@ private final class RunningApplicationsCache: @unchecked Sendable { } private static func scan() -> [NSRunningApplication] { + #if os(Linux) + return NSWorkspace.shared.runningApplications + #else let pids = allProcessIDs() guard !pids.isEmpty else { return NSWorkspace.shared.runningApplications } return pids.compactMap { pid in @@ -53,6 +56,7 @@ private final class RunningApplicationsCache: @unchecked Sendable { else { return nil } return app } + #endif } } diff --git a/Sources/computer-use-mcp/HealthReport.swift b/Sources/computer-use-mcp/HealthReport.swift index ac6fc5d..f5bd719 100644 --- a/Sources/computer-use-mcp/HealthReport.swift +++ b/Sources/computer-use-mcp/HealthReport.swift @@ -66,6 +66,7 @@ func makeHealthReport(prompt: Bool, probeCaptureService: Bool) async -> HealthRe ) #else let process = ProcessDiagnostics.current() + let accessibility = linuxAccessibilityAvailable() return HealthReport( reportVersion: 1, version: version, @@ -73,14 +74,20 @@ func makeHealthReport(prompt: Bool, probeCaptureService: Bool) async -> HealthRe bundleIdentifier: Bundle.main.bundleIdentifier, process: process, permissions: PermissionDiagnostics( - accessibility: PermissionStatus(granted: false, status: "unsupported", requiredFor: "Accessibility is unsupported on Linux"), + accessibility: PermissionStatus( + granted: accessibility, + status: accessibility ? "granted" : "not_available", + requiredFor: "AT-SPI2 accessibility bus and application trees" + ), screenRecording: PermissionStatus(granted: false, status: "unsupported", requiredFor: "Screen Recording is unsupported on Linux") ), captureService: CaptureServiceDiagnostic(status: .skipped, detail: "Screen capture is unsupported on Linux."), daemon: daemonDiagnostics(), telemetry: telemetryDiagnostics(), tccAttribution: "Linux does not use TCC.", - recommendedNextAction: "Linux reports unsupported capabilities; use version/help/health_report for diagnostics." + recommendedNextAction: accessibility + ? "AT-SPI2 perception is available; X11 input and capture remain separate Linux engine phases." + : "Start an AT-SPI2 accessibility bus and enable application accessibility, then rerun health_report." ) #endif } diff --git a/Sources/computer-use-mcp/LinuxSupport.swift b/Sources/computer-use-mcp/LinuxSupport.swift index a2e37c9..780b556 100644 --- a/Sources/computer-use-mcp/LinuxSupport.swift +++ b/Sources/computer-use-mcp/LinuxSupport.swift @@ -1,6 +1,7 @@ #if os(Linux) import Foundation import Glibc +import CAtSpi struct LinuxPeerCredentials { var pid: pid_t = 0 @@ -45,7 +46,33 @@ struct CFRange { var length: Int } -final class AXUIElement: @unchecked Sendable {} +final class LinuxAXValue: @unchecked Sendable { + enum Kind { + case range(CFRange) + case point(CGPoint) + case size(CGSize) + } + + let kind: Kind + + init(_ kind: Kind) { + self.kind = kind + } +} + +final class AXUIElement: @unchecked Sendable { + let native: UnsafeMutablePointer? + + init(native: UnsafeMutablePointer?) { + self.native = native + } + + deinit { + if let native { + catspi_unref(native) + } + } +} enum AXError: Int32 { case success = 0 @@ -95,47 +122,249 @@ let kAXAttributedStringForRangeParameterizedAttribute = "AXAttributedStringForRa let kAXStringForRangeParameterizedAttribute = "AXStringForRange" let kCFBooleanTrue = true -func AXUIElementCreateApplication(_ pid: pid_t) -> AXUIElement { AXUIElement() } -func AXUIElementCreateSystemWide() -> AXUIElement { AXUIElement() } -func AXUIElementGetTypeID() -> Int { 0 } -func AXValueGetTypeID() -> Int { 0 } -func AXValueCreate(_ type: AXValueType, _ value: UnsafeRawPointer?) -> AXValue? { nil } -func AXValueGetValue(_ value: AXValue, _ type: AXValueType, _ ptr: UnsafeMutableRawPointer) -> Bool { false } +private let linuxAXElementTypeID = 0x415845 +private let linuxAXValueTypeID = 0x415856 +private let linuxBooleanTypeID = 0x415842 +private let linuxAttributedStringTypeID = 0x415841 + +private final class AtSpiState: @unchecked Sendable { + static let shared = AtSpiState() + let lock = NSLock() + var initialized = false +} + +@discardableResult +private func ensureAtSpiInitialized() -> Bool { + let state = AtSpiState.shared + state.lock.lock() + defer { state.lock.unlock() } + if state.initialized { return true } + guard catspi_init() == 0 else { return false } + state.initialized = true + return true +} + +func linuxAccessibilityAvailable() -> Bool { + guard ensureAtSpiInitialized(), let desktop = catspi_desktop() else { return false } + catspi_unref(desktop) + return true +} + +func AXUIElementCreateApplication(_ pid: pid_t) -> AXUIElement { + guard ensureAtSpiInitialized() else { return AXUIElement(native: nil) } + return AXUIElement(native: catspi_application_for_pid(UInt32(pid))) +} +func AXUIElementCreateSystemWide() -> AXUIElement { + guard ensureAtSpiInitialized() else { return AXUIElement(native: nil) } + return AXUIElement(native: catspi_desktop()) +} +func AXUIElementGetTypeID() -> Int { linuxAXElementTypeID } +func AXValueGetTypeID() -> Int { linuxAXValueTypeID } +func AXValueCreate(_ type: AXValueType, _ value: UnsafeRawPointer?) -> AXValue? { + guard let value else { return nil } + switch type { + case .cfRange: + return LinuxAXValue(.range(value.assumingMemoryBound(to: CFRange.self).pointee)) + case .cgPoint: + return LinuxAXValue(.point(value.assumingMemoryBound(to: CGPoint.self).pointee)) + case .cgSize: + return LinuxAXValue(.size(value.assumingMemoryBound(to: CGSize.self).pointee)) + } +} +func AXValueGetValue(_ value: AXValue, _ type: AXValueType, _ ptr: UnsafeMutableRawPointer) -> Bool { + guard let value = value as? LinuxAXValue else { return false } + switch (value.kind, type) { + case let (.range(range), .cfRange): + ptr.assumingMemoryBound(to: CFRange.self).pointee = range + case let (.point(point), .cgPoint): + ptr.assumingMemoryBound(to: CGPoint.self).pointee = point + case let (.size(size), .cgSize): + ptr.assumingMemoryBound(to: CGSize.self).pointee = size + default: + return false + } + return true +} func AXUIElementSetMessagingTimeout(_ element: AXUIElement, _ timeout: Float) {} + +private func atspiRoleName(_ raw: String?) -> String { + switch raw?.lowercased() { + case "application": return "AXApplication" + case "frame", "window": return "AXWindow" + case "push button", "button": return "AXButton" + case "toggle button", "check box": return "AXCheckBox" + case "radio button": return "AXRadioButton" + case "combo box": return "AXComboBox" + case "entry", "text", "password text": return "AXTextField" + case "text area", "document text": return "AXTextArea" + case "label", "static": return "AXStaticText" + case "menu": return "AXMenu" + case "menu item": return "AXMenuItem" + case "scroll pane", "scroll bar": return "AXScrollArea" + case "table": return "AXTable" + case "list": return "AXList" + case "list item": return "AXRow" + case "tree": return "AXOutline" + case "tree item": return "AXRow" + case "tool bar": return "AXToolbar" + case "page tab list": return "AXTabGroup" + case "page tab": return "AXRadioButton" + case "dialog": return "AXDialog" + case "image": return "AXImage" + case "link": return "AXLink" + case "separator": return "AXSplitter" + case "panel", "filler", "section", "group": return "AXGroup" + default: + guard let raw, !raw.isEmpty else { return "AXUnknown" } + return "AX\(raw.split(separator: " ").map { $0.capitalized }.joined())" + } +} + +private func atspiString(_ pointer: UnsafeMutablePointer?) -> String? { + guard let pointer else { return nil } + let value = String(cString: pointer) + catspi_free_string(pointer) + return value +} + func AXUIElementCopyAttributeValue( _ element: AXUIElement, _ attribute: CFString, _ value: UnsafeMutablePointer -) -> AXError { .attributeUnsupported } +) -> AXError { + guard let native = element.native else { return .invalidUIElement } + switch attribute { + case kAXRoleAttribute: + value.pointee = atspiRoleName(atspiString(catspi_role_name(native))) + case kAXRoleDescriptionAttribute: + value.pointee = atspiString(catspi_role_name(native)) + case kAXTitleAttribute, kAXDescriptionAttribute, kAXHelpAttribute, kAXValueAttribute: + let pointer: UnsafeMutablePointer? + if attribute == kAXDescriptionAttribute { + pointer = catspi_description(native) + } else if attribute == kAXValueAttribute { + pointer = catspi_text(native) ?? catspi_name(native) + } else { + pointer = catspi_name(native) + } + value.pointee = atspiString(pointer) + case kAXChildrenAttribute, kAXWindowsAttribute: + let count = catspi_child_count(native) + var children: [Any] = [] + for index in 0.. ) -> AXError { .attributeUnsupported } func AXUIElementCopyAttributeNames( _ element: AXUIElement, _ names: UnsafeMutablePointer -) -> AXError { .attributeUnsupported } +) -> AXError { + names.pointee = [ + kAXRoleAttribute, kAXRoleDescriptionAttribute, kAXTitleAttribute, + kAXDescriptionAttribute, kAXValueAttribute, kAXChildrenAttribute, + kAXParentAttribute, kAXPositionAttribute, kAXSizeAttribute, + kAXEnabledAttribute, kAXFocusedAttribute, kAXSelectedAttribute, + ] + return .success +} func AXUIElementCopyElementAtPosition( _ element: AXUIElement, _ x: Float, _ y: Float, _ value: UnsafeMutablePointer ) -> AXError { .attributeUnsupported } func AXUIElementCopyActionNames( _ element: AXUIElement, _ names: UnsafeMutablePointer -) -> AXError { .attributeUnsupported } -func AXUIElementPerformAction(_ element: AXUIElement, _ action: CFString) -> AXError { .attributeUnsupported } +) -> AXError { + let count = catspi_action_count(element.native) + var result: [Any] = [] + for index in 0.. AXError { + let count = catspi_action_count(element.native) + for index in 0.. AXError { .attributeUnsupported } func AXUIElementIsAttributeSettable( _ element: AXUIElement, _ attribute: CFString, _ settable: UnsafeMutablePointer ) -> AXError { .attributeUnsupported } -func AXIsProcessTrusted() -> Bool { false } +func AXIsProcessTrusted() -> Bool { ensureAtSpiInitialized() } func AXIsProcessTrustedWithOptions(_ options: CFTypeRef) -> Bool { false } -func CFGetTypeID(_ value: CFTypeRef) -> Int { 0 } +func CFGetTypeID(_ value: CFTypeRef) -> Int { + if value is AXUIElement { return linuxAXElementTypeID } + if value is LinuxAXValue { return linuxAXValueTypeID } + if value is Bool { return linuxBooleanTypeID } + if value is NSAttributedString { return linuxAttributedStringTypeID } + return 0 +} func CFEqual(_ lhs: CFTypeRef, _ rhs: CFTypeRef) -> Bool { lhs as AnyObject === rhs as AnyObject } -func CFAttributedStringGetTypeID() -> Int { 0 } -func CFBooleanGetTypeID() -> Int { 0 } +func CFAttributedStringGetTypeID() -> Int { linuxAttributedStringTypeID } +func CFBooleanGetTypeID() -> Int { linuxBooleanTypeID } func AXValueGetType(_ value: AXValue) -> Int { 0 } +private func linuxProcessName(_ pid: pid_t) -> String? { + try? String(contentsOfFile: "/proc/\(pid)/comm", encoding: .utf8) + .trimmingCharacters(in: .whitespacesAndNewlines) +} + final class NSRunningApplication: @unchecked Sendable { static let current = NSRunningApplication() - var processIdentifier: pid_t { getpid() } - var localizedName: String? { nil } - var bundleIdentifier: String? { nil } + let processIdentifier: pid_t + let localizedName: String? + let bundleIdentifier: String? + let nativeApplication: AXUIElement? var bundleURL: URL? { nil } var executableURL: URL? { nil } var isTerminated: Bool { false } @@ -143,12 +372,33 @@ final class NSRunningApplication: @unchecked Sendable { var isActive: Bool { false } enum ActivationPolicy { case prohibited, regular } var activationPolicy: ActivationPolicy { .regular } - init() {} - init?(processIdentifier: pid_t) { return nil } + init() { + processIdentifier = getpid() + localizedName = linuxProcessName(processIdentifier) ?? "computer-use-mcp" + bundleIdentifier = localizedName + nativeApplication = nil + } + init?(processIdentifier: pid_t) { + guard ensureAtSpiInitialized(), let native = catspi_application_for_pid(UInt32(processIdentifier)) + else { return nil } + self.processIdentifier = processIdentifier + self.nativeApplication = AXUIElement(native: native) + self.localizedName = atspiString(catspi_name(native)) ?? linuxProcessName(processIdentifier) + self.bundleIdentifier = self.localizedName + } } final class NSWorkspace: @unchecked Sendable { static let shared = NSWorkspace() - var runningApplications: [NSRunningApplication] { [] } + var runningApplications: [NSRunningApplication] { + guard ensureAtSpiInitialized(), let desktop = catspi_desktop() else { return [] } + defer { catspi_unref(desktop) } + return (0.. Date: Wed, 22 Jul 2026 20:26:18 +0000 Subject: [PATCH 2/5] Add X11 Linux input delivery Co-Authored-By: Matthew Lam --- Package.swift | 11 + Sources/CX11/include/cx11_shim.h | 100 +++++ Sources/CX11/module.modulemap | 6 + Sources/computer-use-mcp/HealthReport.swift | 8 +- Sources/computer-use-mcp/LinuxStubs.swift | 348 +++++++++++++++++- .../HealthReportTests.swift | 1 + 6 files changed, 464 insertions(+), 10 deletions(-) create mode 100644 Sources/CX11/include/cx11_shim.h create mode 100644 Sources/CX11/module.modulemap diff --git a/Package.swift b/Package.swift index 5ab8c2b..8304692 100644 --- a/Package.swift +++ b/Package.swift @@ -17,10 +17,13 @@ let package = Package( dependencies: [ .product(name: "MCP", package: "swift-sdk"), .target(name: "CAtSpi", condition: .when(platforms: [.linux])), + .target(name: "CX11", condition: .when(platforms: [.linux])), ], path: "Sources/computer-use-mcp", linkerSettings: [ .linkedLibrary("atspi", .when(platforms: [.linux])), + .linkedLibrary("X11", .when(platforms: [.linux])), + .linkedLibrary("Xtst", .when(platforms: [.linux])), .linkedLibrary("gobject-2.0", .when(platforms: [.linux])), .linkedLibrary("glib-2.0", .when(platforms: [.linux])), .linkedLibrary("dbus-1", .when(platforms: [.linux])), @@ -34,6 +37,14 @@ let package = Package( .apt(["libatspi2.0-dev"]) ] ), + .systemLibrary( + name: "CX11", + path: "Sources/CX11", + pkgConfig: "xtst", + providers: [ + .apt(["libx11-dev", "libxtst-dev"]) + ] + ), // Deterministic GUI fixture app for the end-to-end "truth suite". // See docs/fixture-app.md. .executableTarget( diff --git a/Sources/CX11/include/cx11_shim.h b/Sources/CX11/include/cx11_shim.h new file mode 100644 index 0000000..ac6b6e9 --- /dev/null +++ b/Sources/CX11/include/cx11_shim.h @@ -0,0 +1,100 @@ +#ifndef COMPUTER_USE_MCP_CX11_SHIM_H +#define COMPUTER_USE_MCP_CX11_SHIM_H + +#include +#include +#include +#include + +typedef void *CX11DisplayRef; + +static inline CX11DisplayRef cx11_open_display(void) { + return (CX11DisplayRef)XOpenDisplay(NULL); +} + +static inline void cx11_close_display(CX11DisplayRef display) { + if (display != NULL) { + XCloseDisplay((Display *)display); + } +} + +static inline int cx11_default_screen(CX11DisplayRef display) { + return DefaultScreen((Display *)display); +} + +static inline unsigned long cx11_default_root_window(CX11DisplayRef display) { + return RootWindow((Display *)display, DefaultScreen((Display *)display)); +} + +static inline int cx11_sync(CX11DisplayRef display) { + return XSync((Display *)display, False); +} + +static inline int cx11_flush(CX11DisplayRef display) { + return XFlush((Display *)display); +} + +static inline KeySym cx11_keysym_for_name(const char *name) { + return XStringToKeysym(name); +} + +static inline KeyCode cx11_keycode_for_keysym(CX11DisplayRef display, KeySym keysym) { + return XKeysymToKeycode((Display *)display, keysym); +} + +static inline int cx11_display_keycodes( + CX11DisplayRef display, + int *min, + int *max +) { + XDisplayKeycodes((Display *)display, min, max); + return 1; +} + +static inline KeySym *cx11_keyboard_mapping( + CX11DisplayRef display, + int first_keycode, + int keycode_count, + int *keysyms_per_keycode +) { + return XGetKeyboardMapping((Display *)display, first_keycode, keycode_count, keysyms_per_keycode); +} + +static inline void cx11_free(void *ptr) { + if (ptr != NULL) { + XFree(ptr); + } +} + +static inline int cx11_change_keyboard_mapping( + CX11DisplayRef display, + int first_keycode, + int keysyms_per_keycode, + const KeySym *keysyms, + int num_codes +) { + XChangeKeyboardMapping((Display *)display, first_keycode, keysyms_per_keycode, (KeySym *)keysyms, num_codes); + return 1; +} + +static inline int cx11_xtest_available(CX11DisplayRef display) { + int event_base = 0; + int error_base = 0; + int major = 0; + int minor = 0; + return XTestQueryExtension((Display *)display, &event_base, &error_base, &major, &minor); +} + +static inline int cx11_fake_motion_event(CX11DisplayRef display, int screen, int x, int y) { + return XTestFakeMotionEvent((Display *)display, screen, x, y, 0); +} + +static inline int cx11_fake_button_event(CX11DisplayRef display, unsigned int button, int press) { + return XTestFakeButtonEvent((Display *)display, button, press, 0); +} + +static inline int cx11_fake_key_event(CX11DisplayRef display, unsigned int keycode, int press) { + return XTestFakeKeyEvent((Display *)display, keycode, press, 0); +} + +#endif diff --git a/Sources/CX11/module.modulemap b/Sources/CX11/module.modulemap new file mode 100644 index 0000000..7002693 --- /dev/null +++ b/Sources/CX11/module.modulemap @@ -0,0 +1,6 @@ +module CX11 [system] { + header "include/cx11_shim.h" + link "X11" + link "Xtst" + export * +} diff --git a/Sources/computer-use-mcp/HealthReport.swift b/Sources/computer-use-mcp/HealthReport.swift index f5bd719..8e69ec3 100644 --- a/Sources/computer-use-mcp/HealthReport.swift +++ b/Sources/computer-use-mcp/HealthReport.swift @@ -60,6 +60,7 @@ func makeHealthReport(prompt: Bool, probeCaptureService: Bool) async -> HealthRe permissions: permissions, captureService: captureService, daemon: daemon, + inputDelivery: nil, telemetry: telemetryDiagnostics(), tccAttribution: tccAttributionNote(parent: process.parent), recommendedNextAction: action @@ -67,6 +68,7 @@ func makeHealthReport(prompt: Bool, probeCaptureService: Bool) async -> HealthRe #else let process = ProcessDiagnostics.current() let accessibility = linuxAccessibilityAvailable() + let inputDelivery = linuxInputDeliveryDiagnostic() return HealthReport( reportVersion: 1, version: version, @@ -83,10 +85,13 @@ func makeHealthReport(prompt: Bool, probeCaptureService: Bool) async -> HealthRe ), captureService: CaptureServiceDiagnostic(status: .skipped, detail: "Screen capture is unsupported on Linux."), daemon: daemonDiagnostics(), + inputDelivery: inputDelivery, telemetry: telemetryDiagnostics(), tccAttribution: "Linux does not use TCC.", recommendedNextAction: accessibility - ? "AT-SPI2 perception is available; X11 input and capture remain separate Linux engine phases." + ? (inputDelivery.status == "available" + ? "AT-SPI2 perception and X11/XTest input are available; capture remains a separate Linux engine phase." + : "AT-SPI2 perception is available, but X11/XTest input is unavailable: \(inputDelivery.detail)") : "Start an AT-SPI2 accessibility bus and enable application accessibility, then rerun health_report." ) #endif @@ -185,6 +190,7 @@ struct HealthReport: Codable, Sendable { let permissions: PermissionDiagnostics let captureService: CaptureServiceDiagnostic let daemon: DaemonDiagnostics + let inputDelivery: InputDeliveryDiagnostic? /// Absent when no daemon has persisted a telemetry snapshot yet (or /// telemetry is disabled with "no_telemetry"). let telemetry: TelemetryReport? diff --git a/Sources/computer-use-mcp/LinuxStubs.swift b/Sources/computer-use-mcp/LinuxStubs.swift index 8ec7331..95980fd 100644 --- a/Sources/computer-use-mcp/LinuxStubs.swift +++ b/Sources/computer-use-mcp/LinuxStubs.swift @@ -3,6 +3,7 @@ import Foundation #if canImport(FoundationNetworking) import FoundationNetworking #endif +import CX11 import MCP enum InputTier: String { @@ -21,6 +22,12 @@ enum KeyDeliveryMode: String, Equatable { case skyLight = "tier25-skylight-sleventpostto-pid" case perPid = "tier3-cgeventpostto-pid" case globalSessionTap = "tier4-global-session-tap" + case globalXTest = "tier4-global-xtest" +} + +struct InputDeliveryDiagnostic: Codable, Sendable { + let status: String + let detail: String } enum FallbackReason: String, Equatable, Sendable { @@ -39,6 +46,7 @@ enum FallbackReason: String, Equatable, Sendable { case chainSelectionRelayUnverified = "chain-selection-relay-unverified" case chainChildActionUnverified = "chain-child-action-unverified" case chainAncestorActionUnverified = "chain-ancestor-action-unverified" + case x11GlobalInput = "x11-global-input" } struct DeliveryOutcome: Equatable { @@ -86,6 +94,13 @@ enum MouseButtonKind { struct KeyChord { let keyCode: CGKeyCode let flags: CGEventFlags + let keyString: String? + + init(keyCode: CGKeyCode, flags: CGEventFlags, keyString: String? = nil) { + self.keyCode = keyCode + self.flags = flags + self.keyString = keyString + } } enum Keymap { @@ -124,7 +139,7 @@ enum Keymap { } else { throw ToolError.invalidArguments("Unknown key \"\(key)\".") } - return KeyChord(keyCode: code, flags: flags) + return KeyChord(keyCode: code, flags: flags, keyString: key) } static func wouldInsertText(combo: String, chord: KeyChord) -> Bool { let lower = combo.lowercased() @@ -164,27 +179,275 @@ func isDroppableBackgroundDeliveryTier(_ rawTier: String) -> Bool { || rawTier == KeyDeliveryMode.perPid.rawValue } +private final class X11State: @unchecked Sendable { + static let shared = X11State() + let lock = NSLock() +} + +private func withX11Display(_ body: (CX11DisplayRef) throws -> R) throws -> R { + guard let display = cx11_open_display() else { + throw ToolError.failed( + "X11 input is unavailable because DISPLAY is not set or the X server cannot be opened." + ) + } + defer { cx11_close_display(display) } + return try body(display) +} + +private func linuxX11InputAvailability(displayName: String?) -> InputDeliveryDiagnostic { + guard let displayName, !displayName.isEmpty else { + return InputDeliveryDiagnostic( + status: "unavailable", + detail: "DISPLAY is not set, so X11/XTest input is unavailable." + ) + } + guard let display = cx11_open_display() else { + return InputDeliveryDiagnostic( + status: "unavailable", + detail: "Could not open X11 display \(displayName)." + ) + } + defer { cx11_close_display(display) } + guard cx11_xtest_available(display) != 0 else { + return InputDeliveryDiagnostic( + status: "unavailable", + detail: "XTest is not available on display \(displayName)." + ) + } + return InputDeliveryDiagnostic( + status: "available", + detail: "X11/XTest input is available on display \(displayName)." + ) +} + +func linuxInputDeliveryDiagnostic(displayName: String? = ProcessInfo.processInfo.environment["DISPLAY"]) -> InputDeliveryDiagnostic { + linuxX11InputAvailability(displayName: displayName) +} + +func linuxInputDeliveryAvailable() -> Bool { + linuxInputDeliveryDiagnostic().status == "available" +} + +private func linuxKeyToken(for chord: KeyChord) -> String? { + if let keyString = chord.keyString, !keyString.isEmpty { + return keyString + } + switch chord.keyCode { + case CGKeyCode(kVK_Return): return "Return" + case CGKeyCode(kVK_Tab): return "Tab" + case CGKeyCode(kVK_Space): return "space" + case CGKeyCode(kVK_Delete): return "Delete" + case CGKeyCode(kVK_ForwardDelete): return "ForwardDelete" + case CGKeyCode(kVK_Escape): return "Escape" + case CGKeyCode(kVK_LeftArrow): return "Left" + case CGKeyCode(kVK_RightArrow): return "Right" + case CGKeyCode(kVK_UpArrow): return "Up" + case CGKeyCode(kVK_DownArrow): return "Down" + case CGKeyCode(kVK_Home): return "Home" + case CGKeyCode(kVK_End): return "End" + case CGKeyCode(kVK_PageUp): return "PageUp" + case CGKeyCode(kVK_PageDown): return "PageDown" + default: + if chord.keyCode < 128, + let scalar = UnicodeScalar(UInt32(chord.keyCode)), + scalar.value >= 32 + { + return String(scalar) + } + return nil + } +} + +private func linuxModifierKeycodes(for flags: CGEventFlags, display: CX11DisplayRef) -> [KeyCode] { + var codes: [KeyCode] = [] + if flags.contains(.maskShift), let code = linuxKeycode(named: "Shift_L", display: display) { + codes.append(code) + } + if flags.contains(.maskControl), let code = linuxKeycode(named: "Control_L", display: display) { + codes.append(code) + } + if flags.contains(.maskAlternate), let code = linuxKeycode(named: "Alt_L", display: display) { + codes.append(code) + } + if flags.contains(.maskCommand), let code = linuxKeycode(named: "Super_L", display: display) { + codes.append(code) + } + return codes +} + +private func linuxKeycode(named name: String, display: CX11DisplayRef) -> KeyCode? { + let keysym = cx11_keysym_for_name(name) + guard keysym != 0 else { return nil } + let keycode = cx11_keycode_for_keysym(display, keysym) + return keycode == 0 ? nil : keycode +} + +private func linuxKeycode(for token: String, display: CX11DisplayRef) -> KeyCode? { + let keysym = cx11_keysym_for_name(token) + if keysym != 0 { + let keycode = cx11_keycode_for_keysym(display, keysym) + if keycode != 0 { return keycode } + } + if token.count == 1, let scalar = token.unicodeScalars.first { + let codepointKeysym = KeySym(0x0100_0000 | UInt32(scalar.value)) + let keycode = cx11_keycode_for_keysym(display, codepointKeysym) + if keycode != 0 { return keycode } + } + return nil +} + +private func linuxSendKeySequence( + display: CX11DisplayRef, + keycode: KeyCode, + modifiers: [KeyCode] +) -> Bool { + for modifier in modifiers { + guard cx11_fake_key_event(display, unsignedInt(modifier), 1) != 0 else { return false } + } + guard cx11_fake_key_event(display, unsignedInt(keycode), 1) != 0 else { return false } + guard cx11_fake_key_event(display, unsignedInt(keycode), 0) != 0 else { return false } + for modifier in modifiers.reversed() { + guard cx11_fake_key_event(display, unsignedInt(modifier), 0) != 0 else { return false } + } + return cx11_flush(display) != 0 && cx11_sync(display) != 0 +} + +private func linuxSendUnicodeScalar(_ scalar: UnicodeScalar, display: CX11DisplayRef) throws { + var minKeycode: Int32 = 0 + var maxKeycode: Int32 = 0 + guard cx11_display_keycodes(display, &minKeycode, &maxKeycode) != 0, minKeycode <= maxKeycode else { + throw ToolError.failed("Could not query the X11 keyboard map for Unicode text delivery.") + } + let spareKeycode = maxKeycode + var keysymsPerKeycode: Int32 = 0 + guard let mapping = cx11_keyboard_mapping(display, spareKeycode, 1, &keysymsPerKeycode), + keysymsPerKeycode > 0 + else { + throw ToolError.failed("Could not read the X11 keyboard map for Unicode text delivery.") + } + defer { cx11_free(mapping) } + + let saved = Array(UnsafeBufferPointer(start: mapping, count: Int(keysymsPerKeycode))) + let unicodeKeysym = KeySym(0x0100_0000 | UInt32(scalar.value)) + var replacement = Array(repeating: KeySym(0), count: Int(keysymsPerKeycode)) + replacement[0] = unicodeKeysym + replacement.withUnsafeBufferPointer { buffer in + _ = cx11_change_keyboard_mapping(display, spareKeycode, keysymsPerKeycode, buffer.baseAddress, 1) + } + guard cx11_sync(display) != 0 else { + throw ToolError.failed("Could not update the X11 keyboard map for Unicode text delivery.") + } + Thread.sleep(forTimeInterval: 0.01) + defer { + saved.withUnsafeBufferPointer { buffer in + _ = cx11_change_keyboard_mapping(display, spareKeycode, keysymsPerKeycode, buffer.baseAddress, 1) + } + _ = cx11_sync(display) + Thread.sleep(forTimeInterval: 0.01) + } + + guard linuxSendKeySequence(display: display, keycode: KeyCode(spareKeycode), modifiers: []) else { + throw ToolError.failed("Could not synthesize Unicode text on X11.") + } + Thread.sleep(forTimeInterval: 0.02) +} + +private func unsignedInt(_ keycode: KeyCode) -> UInt32 { + UInt32(keycode) +} + @discardableResult func deliverClick( at point: CGPoint, button: MouseButtonKind, clickCount: Int, context: DeliveryContext, allowGlobalCursor: Bool = false ) throws -> DeliveryOutcome { - throw ToolError.failed("Mouse click delivery is unsupported on Linux.") + guard linuxInputDeliveryAvailable() else { + throw ToolError.failed("Mouse click delivery is unavailable because X11/XTest is unavailable.") + } + try withX11Display { display in + let screen = cx11_default_screen(display) + let clickButton: UInt32 = switch button { + case .left: 1 + case .middle: 2 + case .right: 3 + } + for _ in 0.. InputTier { - throw ToolError.failed("Scroll delivery is unsupported on Linux.") + guard linuxInputDeliveryAvailable() else { + throw ToolError.failed("Scroll delivery is unavailable because X11/XTest is unavailable.") + } + try withX11Display { display in + let screen = cx11_default_screen(display) + let clicksX = max(1, abs(deltaX) / 120) + let clicksY = max(1, abs(deltaY) / 120) + guard cx11_fake_motion_event(display, screen, Int32(point.x.rounded()), Int32(point.y.rounded())) != 0 else { + throw ToolError.failed("Could not position the X11 pointer for scroll delivery.") + } + if deltaY > 0 { + for _ in 0.. 0 { + for _ in 0.. InputTier { - throw ToolError.failed("Drag delivery is unsupported on Linux.") + guard linuxInputDeliveryAvailable() else { + throw ToolError.failed("Drag delivery is unavailable because X11/XTest is unavailable.") + } + try withX11Display { display in + let screen = cx11_default_screen(display) + guard cx11_fake_motion_event(display, screen, Int32(from.x.rounded()), Int32(from.y.rounded())) != 0 else { + throw ToolError.failed("Could not position the X11 pointer for drag delivery.") + } + guard cx11_fake_button_event(display, 1, 1) != 0 else { + throw ToolError.failed("Could not press the X11 drag button.") + } + guard cx11_fake_motion_event(display, screen, Int32(to.x.rounded()), Int32(to.y.rounded())) != 0 else { + throw ToolError.failed("Could not move the X11 pointer during drag delivery.") + } + guard cx11_fake_button_event(display, 1, 0) != 0 else { + throw ToolError.failed("Could not release the X11 drag button.") + } + } + return .globalCursor } @discardableResult func typeUnicodeText(_ text: String, context: DeliveryContext) throws -> InputTier { - throw ToolError.failed("Text input is unsupported on Linux.") + guard linuxInputDeliveryAvailable() else { + throw ToolError.failed("Text input is unavailable because X11/XTest is unavailable.") + } + X11State.shared.lock.lock() + defer { X11State.shared.lock.unlock() } + try withX11Display { display in + for scalar in text.unicodeScalars { + try linuxSendUnicodeScalar(scalar, display: display) + } + } + return .globalCursor } func keyDeliveryMode(context: DeliveryContext, targetAppIsActive: Bool) throws -> KeyDeliveryMode { @@ -198,10 +461,30 @@ func keyDeliveryMode(context: DeliveryContext, targetAppIsActive: Bool) throws - } func deliverKey(_ chord: KeyChord, context: DeliveryContext, targetAppIsActive: Bool) throws -> KeyDeliveryMode { - throw ToolError.failed("Keyboard input is unsupported on Linux.") + guard linuxInputDeliveryAvailable() else { + throw ToolError.failed("Keyboard input is unavailable because X11/XTest is unavailable.") + } + guard let token = linuxKeyToken(for: chord) else { + throw ToolError.failed("Could not map the requested key to an X11 keysym.") + } + try withX11Display { display in + let modifiers = linuxModifierKeycodes(for: chord.flags, display: display) + let keycode: KeyCode + if let mapped = linuxKeycode(for: token, display: display) { + keycode = mapped + } else { + throw ToolError.failed("Could not map \(token) to an X11 keycode.") + } + guard linuxSendKeySequence(display: display, keycode: keycode, modifiers: modifiers) else { + throw ToolError.failed("Could not synthesize keyboard input on X11.") + } + } + return .globalXTest } -func syntheticFallbackReasons(context: DeliveryContext, allowGlobalCursor: Bool) -> [FallbackReason] { [] } +func syntheticFallbackReasons(context: DeliveryContext, allowGlobalCursor: Bool) -> [FallbackReason] { + [.x11GlobalInput] +} func windowID(for axWindow: AXUIElement) -> CGWindowID? { nil } func dragReleasePoint(from: CGPoint, to: CGPoint, aborted: Bool) -> CGPoint { aborted ? from : to } func unicodeTypingChunks(_ text: String) -> [[UniChar]] { @@ -234,7 +517,48 @@ func waitForImpl(_ args: [String: Value]) async throws -> CallTool.Result { throw ToolError.failed("Waiting for UI conditions is unsupported on Linux.") } func typeTextImpl(_ args: [String: Value]) async throws -> CallTool.Result { - throw ToolError.failed("Text entry is unsupported on Linux.") + let app = try resolveApp(args.requireString("app")) + try requireAccessibilityTrusted() + let text = try args.requireString("text") + try ArgumentBounds.checkStringLength(text, argument: "text", maximum: ArgumentBounds.maxTypeTextCharacters) + let confirmed = SafetyPolicy.confirmed(args) + try SafetyPolicy.check(app: app, confirmed: confirmed) + + let element: AXUIElement + let described: String + if let elementID = args.string("element_id") { + let target = try await resolveTarget(app: app, elementID: elementID) + element = target.element + described = describeTarget(target) + } else { + guard let focused = axElement(app.axApplication, kAXFocusedUIElementAttribute) else { + throw ToolError.failed( + "\(app.name) has no focused element. Pass element_id for the field to type into." + ) + } + element = focused + described = "the focused element (\(axRole(focused)))" + } + + try SafetyPolicy.checkTyping(into: element, app: app, confirmed: confirmed) + let context = DeliveryContext( + pid: app.pid, + windowNumber: nil, + windowFrame: nil, + allowGlobalCursor: false + ) + if axBool(element, kAXFocusedAttribute) != true, let frame = axFrame(element) { + _ = try deliverClick( + at: CGPoint(x: frame.midX, y: frame.midY), + button: .left, + clickCount: 1, + context: context + ) + } else { + AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue) + } + let tier = try typeUnicodeText(text, context: context) + return .text("Typed \(text.count) characters into \(described) [\(tier.rawValue)].") } func setValueImpl(_ args: [String: Value]) async throws -> CallTool.Result { throw ToolError.failed("Value editing is unsupported on Linux.") @@ -243,7 +567,13 @@ func selectTextImpl(_ args: [String: Value]) async throws -> CallTool.Result { throw ToolError.failed("Text selection is unsupported on Linux.") } func readTextImpl(_ args: [String: Value]) async throws -> CallTool.Result { - throw ToolError.failed("Text reading is unsupported on Linux.") + let app = try resolveApp(args.requireString("app")) + try requireAccessibilityTrusted() + let target = try await resolveTarget(app: app, elementID: args.requireString("element_id")) + guard let value = axString(target.element, kAXValueAttribute) else { + throw ToolError.failed("\(describeTarget(target)) has no readable text value.") + } + return .text("Text of \(describeTarget(target)) — \(value.count) chars total:\n\(value)") } func performSecondaryActionImpl(_ args: [String: Value]) async throws -> CallTool.Result { throw ToolError.failed("Secondary actions are unsupported on Linux.") diff --git a/Tests/ComputerUseMCPTests/HealthReportTests.swift b/Tests/ComputerUseMCPTests/HealthReportTests.swift index d540193..2800391 100644 --- a/Tests/ComputerUseMCPTests/HealthReportTests.swift +++ b/Tests/ComputerUseMCPTests/HealthReportTests.swift @@ -213,6 +213,7 @@ private func outcomeFields(in result: CallTool.Result) throws -> [String: Value] secretExists: false, secretContentsReported: false ), + inputDelivery: nil, telemetry: nil, tccAttribution: "mock attribution", recommendedNextAction: recommendedNextAction( From 588d4148b82ef2f5edbe32309df0345e675a7af4 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2026 20:36:15 +0000 Subject: [PATCH 3/5] Add Linux X11 capture and system tools Co-Authored-By: Matthew Lam --- Package.swift | 3 +- README.md | 11 +- Sources/CX11/include/cx11_shim.h | 145 +++++++++ Sources/CX11/module.modulemap | 1 + .../computer-use-mcp/Core/Screenshot.swift | 92 +++++- Sources/computer-use-mcp/HealthReport.swift | 7 +- Sources/computer-use-mcp/LinuxStubs.swift | 298 +++++++++++++++++- 7 files changed, 542 insertions(+), 15 deletions(-) diff --git a/Package.swift b/Package.swift index 8304692..59c14be 100644 --- a/Package.swift +++ b/Package.swift @@ -24,6 +24,7 @@ let package = Package( .linkedLibrary("atspi", .when(platforms: [.linux])), .linkedLibrary("X11", .when(platforms: [.linux])), .linkedLibrary("Xtst", .when(platforms: [.linux])), + .linkedLibrary("png", .when(platforms: [.linux])), .linkedLibrary("gobject-2.0", .when(platforms: [.linux])), .linkedLibrary("glib-2.0", .when(platforms: [.linux])), .linkedLibrary("dbus-1", .when(platforms: [.linux])), @@ -42,7 +43,7 @@ let package = Package( path: "Sources/CX11", pkgConfig: "xtst", providers: [ - .apt(["libx11-dev", "libxtst-dev"]) + .apt(["libx11-dev", "libxtst-dev", "libpng-dev"]) ] ), // Deterministic GUI fixture app for the end-to-end "truth suite". diff --git a/README.md b/README.md index f21b6ea..af1ce3d 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ and the agent can **see and operate the apps on your Mac — in the background, without hijacking your cursor or stealing focus.** > **Status:** pre-1.0, used in production by the authors. The interaction engine -> runs on macOS while the Linux build, tests, CLI, and daemon plumbing are -> supported. On macOS it runs while the Mac is unlocked (the system is kept from +> runs on macOS and X11 Linux; Linux requires an AT-SPI2 accessibility bus and +> does not support Wayland. On macOS it runs while the Mac is unlocked (the system is kept from > idle-sleeping during active sessions; if the screen locks, mutating tools pause > with a recoverable error until you unlock). @@ -521,9 +521,10 @@ productionization checklist. `allow_global_cursor: true` as an explicit fallback. - Menu key-equivalents (e.g. `cmd+a`) are reliable when the app is the key window; some apps ignore them when targeted purely in the background. -- The interaction engine is macOS-only (Accessibility, ScreenCaptureKit, and - CoreGraphics). Linux tools that require those capabilities return structured - unsupported errors; the protocol/tool layer is OS-agnostic. +- Linux supports AT-SPI2 perception, X11/XTest input, X11 screenshots, build, + tests, CLI, and daemon plumbing. Wayland is not supported; overlay, OCR, and + skills recording remain macOS-only. Other unavailable Linux capabilities + return structured errors; the protocol/tool layer is OS-agnostic. ## Development diff --git a/Sources/CX11/include/cx11_shim.h b/Sources/CX11/include/cx11_shim.h index ac6b6e9..4f09cc3 100644 --- a/Sources/CX11/include/cx11_shim.h +++ b/Sources/CX11/include/cx11_shim.h @@ -4,7 +4,11 @@ #include #include #include +#include +#include +#include #include +#include typedef void *CX11DisplayRef; @@ -97,4 +101,145 @@ static inline int cx11_fake_key_event(CX11DisplayRef display, unsigned int keyco return XTestFakeKeyEvent((Display *)display, keycode, press, 0); } +static inline unsigned char cx11_channel(unsigned long pixel, unsigned long mask) { + if (mask == 0) { + return 0; + } + unsigned long value = (pixel & mask); + unsigned long shift = 0; + while ((mask & 1UL) == 0) { + mask >>= 1; + value >>= 1; + } + unsigned long max = mask; + return (unsigned char)((value * 255UL + max / 2UL) / max); +} + +static inline int cx11_capture_root_rgba( + CX11DisplayRef display, + int x, + int y, + unsigned int width, + unsigned int height, + unsigned char **pixels +) { + if (display == NULL || pixels == NULL || width == 0 || height == 0) { + return 0; + } + Display *xdisplay = (Display *)display; + Window root = RootWindow(xdisplay, DefaultScreen(xdisplay)); + XImage *image = XGetImage(xdisplay, root, x, y, width, height, AllPlanes, ZPixmap); + if (image == NULL) { + return 0; + } + size_t size = (size_t)width * (size_t)height * 4U; + unsigned char *output = (unsigned char *)malloc(size); + if (output == NULL) { + XDestroyImage(image); + return 0; + } + for (unsigned int row = 0; row < height; row++) { + for (unsigned int column = 0; column < width; column++) { + unsigned long pixel = XGetPixel(image, column, row); + size_t offset = ((size_t)row * width + column) * 4U; + output[offset] = cx11_channel(pixel, image->red_mask); + output[offset + 1] = cx11_channel(pixel, image->green_mask); + output[offset + 2] = cx11_channel(pixel, image->blue_mask); + output[offset + 3] = 255; + } + } + XDestroyImage(image); + *pixels = output; + return 1; +} + +typedef struct { + unsigned char *data; + size_t size; + size_t capacity; +} CX11PngBuffer; + +static void cx11_png_write( + png_structp png_ptr, + png_bytep data, + png_size_t length +) { + CX11PngBuffer *buffer = (CX11PngBuffer *)png_get_io_ptr(png_ptr); + size_t required = buffer->size + (size_t)length; + if (required > buffer->capacity) { + size_t capacity = buffer->capacity == 0 ? 4096 : buffer->capacity; + while (capacity < required) { + capacity *= 2; + } + unsigned char *resized = (unsigned char *)realloc(buffer->data, capacity); + if (resized == NULL) { + png_error(png_ptr, "PNG allocation failed"); + return; + } + buffer->data = resized; + buffer->capacity = capacity; + } + memcpy(buffer->data + buffer->size, data, (size_t)length); + buffer->size = required; +} + +static void cx11_png_flush(png_structp png_ptr) { + (void)png_ptr; +} + +static inline int cx11_encode_png_rgba( + const unsigned char *pixels, + unsigned int width, + unsigned int height, + unsigned int stride, + unsigned char **png_data, + size_t *png_size +) { + if (pixels == NULL || png_data == NULL || png_size == NULL || width == 0 || height == 0) { + return 0; + } + png_structp png = png_create_write_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL); + if (png == NULL) { + return 0; + } + png_infop info = png_create_info_struct(png); + if (info == NULL) { + png_destroy_write_struct(&png, NULL); + return 0; + } + CX11PngBuffer buffer = {0}; + if (setjmp(png_jmpbuf(png)) != 0) { + free(buffer.data); + png_destroy_write_struct(&png, &info); + return 0; + } + png_set_write_fn(png, &buffer, cx11_png_write, cx11_png_flush); + png_set_IHDR( + png, + info, + width, + height, + 8, + PNG_COLOR_TYPE_RGBA, + PNG_INTERLACE_NONE, + PNG_COMPRESSION_TYPE_DEFAULT, + PNG_FILTER_TYPE_DEFAULT + ); + png_write_info(png, info); + png_bytep *rows = (png_bytep *)malloc((size_t)height * sizeof(png_bytep)); + if (rows == NULL) { + png_error(png, "PNG row allocation failed"); + } + for (unsigned int row = 0; row < height; row++) { + rows[row] = (png_bytep)(pixels + (size_t)row * stride); + } + png_write_image(png, rows); + png_write_end(png, NULL); + free(rows); + png_destroy_write_struct(&png, &info); + *png_data = buffer.data; + *png_size = buffer.size; + return 1; +} + #endif diff --git a/Sources/CX11/module.modulemap b/Sources/CX11/module.modulemap index 7002693..33daa5a 100644 --- a/Sources/CX11/module.modulemap +++ b/Sources/CX11/module.modulemap @@ -2,5 +2,6 @@ module CX11 [system] { header "include/cx11_shim.h" link "X11" link "Xtst" + link "png" export * } diff --git a/Sources/computer-use-mcp/Core/Screenshot.swift b/Sources/computer-use-mcp/Core/Screenshot.swift index c5fd474..2707b99 100644 --- a/Sources/computer-use-mcp/Core/Screenshot.swift +++ b/Sources/computer-use-mcp/Core/Screenshot.swift @@ -7,6 +7,8 @@ import CoreGraphics import ImageIO @preconcurrency import ScreenCaptureKit import UniformTypeIdentifiers +#elseif os(Linux) +import CX11 #endif struct WindowCapture { @@ -240,7 +242,95 @@ private func encodePNG(_ image: CGImage) -> Data? { return data as Data } #else +func linuxCaptureDiagnostic() -> CaptureServiceDiagnostic { + guard let displayName = ProcessInfo.processInfo.environment["DISPLAY"], !displayName.isEmpty else { + return CaptureServiceDiagnostic( + status: .skipped, + detail: "DISPLAY is not set, so X11 capture is unavailable." + ) + } + guard let display = cx11_open_display() else { + return CaptureServiceDiagnostic( + status: .skipped, + detail: "Could not open X11 display \(displayName) for capture." + ) + } + cx11_close_display(display) + return CaptureServiceDiagnostic( + status: .responsive, + detail: "X11 root-window capture is available on display \(displayName)." + ) +} + func captureWindow(pid: pid_t, title: String?, frame: CGRect, detail: ScreenshotDetail) async throws -> WindowCapture { - throw ToolError.failed("Screenshots are unsupported on Linux.") + guard frame.width > 0, frame.height > 0 else { + throw ToolError.failed("Cannot capture a window with an empty frame.") + } + guard let display = cx11_open_display() else { + throw ToolError.failed( + "X11 capture is unavailable because DISPLAY is not set or the X server cannot be opened." + ) + } + defer { cx11_close_display(display) } + + let sourceWidth = Int(frame.width.rounded(.up)) + let sourceHeight = Int(frame.height.rounded(.up)) + var sourcePixels: UnsafeMutablePointer? + guard cx11_capture_root_rgba( + display, + Int32(frame.origin.x.rounded()), + Int32(frame.origin.y.rounded()), + UInt32(sourceWidth), + UInt32(sourceHeight), + &sourcePixels + ) != 0, let sourcePixels else { + throw ToolError.failed("Could not capture the X11 root window at the requested frame.") + } + defer { cx11_free(sourcePixels) } + + let maxDimension = Int(detail.maxDimension.rounded(.down)) + let scale = min(1.0, Double(maxDimension) / Double(max(sourceWidth, sourceHeight))) + let outputWidth = max(1, Int((Double(sourceWidth) * scale).rounded())) + let outputHeight = max(1, Int((Double(sourceHeight) * scale).rounded())) + let outputStride = outputWidth * 4 + var output = [UInt8](repeating: 0, count: outputStride * outputHeight) + let source = UnsafeBufferPointer( + start: sourcePixels, + count: sourceWidth * sourceHeight * 4 + ) + for row in 0..? + var encodedSize: Int = 0 + let encodedOK = output.withUnsafeBufferPointer { buffer in + cx11_encode_png_rgba( + buffer.baseAddress, + UInt32(outputWidth), + UInt32(outputHeight), + UInt32(outputStride), + &encoded, + &encodedSize + ) + } + guard encodedOK != 0, let encoded, encodedSize > 0 else { + throw ToolError.failed("Failed to encode the X11 window screenshot as PNG.") + } + defer { cx11_free(encoded) } + return WindowCapture( + pngData: Data(bytes: encoded, count: encodedSize), + pixelWidth: outputWidth, + pixelHeight: outputHeight + ) } #endif diff --git a/Sources/computer-use-mcp/HealthReport.swift b/Sources/computer-use-mcp/HealthReport.swift index 8e69ec3..4907c00 100644 --- a/Sources/computer-use-mcp/HealthReport.swift +++ b/Sources/computer-use-mcp/HealthReport.swift @@ -69,6 +69,7 @@ func makeHealthReport(prompt: Bool, probeCaptureService: Bool) async -> HealthRe let process = ProcessDiagnostics.current() let accessibility = linuxAccessibilityAvailable() let inputDelivery = linuxInputDeliveryDiagnostic() + let captureService = linuxCaptureDiagnostic() return HealthReport( reportVersion: 1, version: version, @@ -83,14 +84,16 @@ func makeHealthReport(prompt: Bool, probeCaptureService: Bool) async -> HealthRe ), screenRecording: PermissionStatus(granted: false, status: "unsupported", requiredFor: "Screen Recording is unsupported on Linux") ), - captureService: CaptureServiceDiagnostic(status: .skipped, detail: "Screen capture is unsupported on Linux."), + captureService: captureService, daemon: daemonDiagnostics(), inputDelivery: inputDelivery, telemetry: telemetryDiagnostics(), tccAttribution: "Linux does not use TCC.", recommendedNextAction: accessibility ? (inputDelivery.status == "available" - ? "AT-SPI2 perception and X11/XTest input are available; capture remains a separate Linux engine phase." + ? (captureService.status == .responsive + ? "AT-SPI2 perception, X11/XTest input, and X11 capture are available; Wayland is unsupported." + : "AT-SPI2 perception and X11/XTest input are available, but X11 capture is unavailable: \(captureService.detail)") : "AT-SPI2 perception is available, but X11/XTest input is unavailable: \(inputDelivery.detail)") : "Start an AT-SPI2 accessibility bus and enable application accessibility, then rerun health_report." ) diff --git a/Sources/computer-use-mcp/LinuxStubs.swift b/Sources/computer-use-mcp/LinuxStubs.swift index 95980fd..cbdee4b 100644 --- a/Sources/computer-use-mcp/LinuxStubs.swift +++ b/Sources/computer-use-mcp/LinuxStubs.swift @@ -491,14 +491,159 @@ func unicodeTypingChunks(_ text: String) -> [[UniChar]] { text.map { Array(String($0).utf16) } } +private func linuxExecutable(named name: String) -> String? { + let candidates = [ + name, + "/usr/bin/\(name)", + "/usr/local/bin/\(name)", + "\(NSHomeDirectory())/.local/bin/\(name)", + ] + return candidates.first { FileManager.default.isExecutableFile(atPath: $0) } +} + +private func runLinuxProcess( + executable: String, + arguments: [String], + input: Data? = nil +) throws -> (status: Int32, output: Data, error: Data) { + let process = Process() + process.executableURL = URL(fileURLWithPath: executable) + process.arguments = arguments + let output = Pipe() + let error = Pipe() + process.standardOutput = output + process.standardError = error + if let input { + let pipe = Pipe() + process.standardInput = pipe + try process.run() + pipe.fileHandleForWriting.write(input) + pipe.fileHandleForWriting.closeFile() + } else { + try process.run() + } + process.waitUntilExit() + return ( + process.terminationStatus, + output.fileHandleForReading.readDataToEndOfFile(), + error.fileHandleForReading.readDataToEndOfFile() + ) +} + +private func launchLinuxProcess(executable: String, arguments: [String], input: Data) throws { + let process = Process() + process.executableURL = URL(fileURLWithPath: executable) + process.arguments = arguments + let pipe = Pipe() + process.standardInput = pipe + try process.run() + pipe.fileHandleForWriting.write(input) + pipe.fileHandleForWriting.closeFile() +} + func openAppImpl(_ args: [String: Value]) async throws -> CallTool.Result { - throw ToolError.failed("App launching is unsupported on Linux.") + let identifier = try args.requireString("app") + let activate = args.bool("activate") ?? false + let confirmed = SafetyPolicy.confirmed(args) + if let running = try? resolveApp(identifier) { + try SafetyPolicy.checkOpenApp( + identifier: running.name, + activate: activate, + isAlreadyRunning: true, + confirmed: confirmed + ) + return .text("\(running.name) is already running.") + } + try SafetyPolicy.checkOpenApp( + identifier: identifier, + activate: activate, + isAlreadyRunning: false, + confirmed: confirmed + ) + let executable: String + let arguments: [String] + if let gtkLaunch = linuxExecutable(named: "gtk-launch"), + !identifier.contains("/"), + identifier.hasSuffix(".desktop") + { + executable = gtkLaunch + arguments = [identifier] + } else if let path = linuxExecutable(named: identifier) { + executable = path + arguments = [] + } else if FileManager.default.isExecutableFile(atPath: identifier) { + executable = identifier + arguments = [] + } else { + throw ToolError.failed( + "No executable named \"\(identifier)\" was found on Linux. Pass a binary name or executable path." + ) + } + do { + let process = Process() + process.executableURL = URL(fileURLWithPath: executable) + process.arguments = arguments + process.environment = ProcessInfo.processInfo.environment + if process.environment?["GTK_MODULES"] == nil { + process.environment?["GTK_MODULES"] = "atk-bridge" + } + try process.run() + try? await Task.sleep(for: .milliseconds(300)) + guard process.isRunning else { + throw ToolError.failed("\(identifier) exited immediately after launch.") + } + return .text("Launched \(identifier) (pid \(process.processIdentifier)).") + } catch { + throw ToolError.failed("Launching \(identifier) failed: \(error.localizedDescription).") + } } func openURLImpl(_ args: [String: Value]) async throws -> CallTool.Result { - throw ToolError.failed("URL opening is unsupported on Linux.") + let raw = try args.requireString("url") + try requireFocusChangeAllowed( + args, + reason: "Opening a URL or file path can launch or activate its default handler." + ) + let url: URL + if let parsed = URL(string: raw), parsed.scheme != nil { + url = parsed + } else if FileManager.default.fileExists(atPath: (raw as NSString).expandingTildeInPath) { + url = URL(fileURLWithPath: (raw as NSString).expandingTildeInPath) + } else { + throw ToolError.invalidArguments("\"\(raw)\" is not a valid URL.") + } + try SafetyPolicy.checkOpenURL(url, confirmed: SafetyPolicy.confirmed(args)) + guard let opener = linuxExecutable(named: "xdg-open") else { + throw ToolError.failed("URL opening requires xdg-open on Linux.") + } + do { + let process = Process() + process.executableURL = URL(fileURLWithPath: opener) + process.arguments = [raw] + process.environment = ProcessInfo.processInfo.environment + try process.run() + return .text("Opened \(raw) with xdg-open.") + } catch { + throw ToolError.failed("Opening \(raw) failed: \(error.localizedDescription).") + } } func listWindowsImpl(_ args: [String: Value]) async throws -> CallTool.Result { - throw ToolError.failed("Window enumeration is unsupported on Linux.") + let app = try resolveApp(args.requireString("app")) + try requireAccessibilityTrusted() + try requireAppAlive(app) + let windows = axElements(app.axApplication, kAXWindowsAttribute) + guard !windows.isEmpty else { + return .text("\(app.name) has no windows right now.") + } + var lines = ["Windows of \(app.name) (pid \(app.pid)):"] + for window in windows where axRole(window) == "AXWindow" { + var parts = ["\"\(axString(window, kAXTitleAttribute) ?? "")\""] + if let frame = axFrame(window) { + parts.append("(\(Int(frame.origin.x)),\(Int(frame.origin.y)) \(Int(frame.width))x\(Int(frame.height)) pt)") + } + if axBool(window, kAXFocusedAttribute) == true { parts.append("focused") } + lines.append(" " + parts.joined(separator: " ")) + } + return .text(lines.joined(separator: "\n")) } func manageWindowImpl(_ args: [String: Value]) async throws -> CallTool.Result { throw ToolError.failed("Window management is unsupported on Linux.") @@ -506,15 +651,156 @@ func manageWindowImpl(_ args: [String: Value]) async throws -> CallTool.Result { func clickMenuItemImpl(_ args: [String: Value]) async throws -> CallTool.Result { throw ToolError.failed("Menu interaction is unsupported on Linux.") } +private func linuxClipboardCommand() -> (String, [String], [String])? { + if let xclip = linuxExecutable(named: "xclip") { + return ( + xclip, + ["-selection", "clipboard", "-o"], + ["-selection", "clipboard", "-in"] + ) + } + if let xsel = linuxExecutable(named: "xsel") { + return ( + xsel, + ["--clipboard", "--output"], + ["--clipboard", "--input"] + ) + } + return nil +} func readClipboardImpl(_ args: [String: Value]) async throws -> CallTool.Result { - throw ToolError.failed("Clipboard access is unsupported on Linux.") + guard let (command, readArguments, _) = linuxClipboardCommand() else { + throw ToolError.failed("Clipboard access requires xclip or xsel on Linux.") + } + let result = try runLinuxProcess(executable: command, arguments: readArguments) + guard result.status == 0 else { + let detail = String(data: result.error, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines) + throw ToolError.failed("Could not read the X11 clipboard\(detail.map { ": \($0)" } ?? "").") + } + let text = String(data: result.output, encoding: .utf8) ?? "" + return .text("Clipboard text (\(text.count) chars):\n\(text)") } func writeClipboardImpl(_ args: [String: Value]) async throws -> CallTool.Result { - throw ToolError.failed("Clipboard access is unsupported on Linux.") + let text = try args.requireString("text") + try ArgumentBounds.checkStringLength(text, argument: "text", maximum: ArgumentBounds.maxClipboardCharacters) + try SafetyPolicy.checkClipboardWrite(confirmed: SafetyPolicy.confirmed(args)) + guard let (command, _, writeArguments) = linuxClipboardCommand() else { + throw ToolError.failed("Clipboard access requires xclip or xsel on Linux.") + } + let previous = try? runLinuxProcess( + executable: command, + arguments: linuxClipboardCommand()!.1 + ) + let previousText: String? + if let previous, previous.status == 0 { + previousText = String(data: previous.output, encoding: .utf8) + } else { + previousText = nil + } + do { + try launchLinuxProcess( + executable: command, + arguments: writeArguments, + input: Data(text.utf8) + ) + } catch { + throw ToolError.failed("Could not write the X11 clipboard: \(error.localizedDescription).") + } + try? await Task.sleep(for: .milliseconds(80)) + let verification = try? runLinuxProcess( + executable: command, + arguments: linuxClipboardCommand()!.1 + ) + guard let verification, verification.status == 0, + String(data: verification.output, encoding: .utf8) == text + else { + if let previousText { + try? launchLinuxProcess( + executable: command, + arguments: writeArguments, + input: Data(previousText.utf8) + ) + } + throw ToolError.failed("The X11 clipboard did not accept the requested write.") + } + return .text("Replaced the clipboard with \(text.count) characters.") } func clipboardRestoreValue(committed: Bool, previous: String?) -> String? { committed ? nil : previous } func waitForImpl(_ args: [String: Value]) async throws -> CallTool.Result { - throw ToolError.failed("Waiting for UI conditions is unsupported on Linux.") + let app = try resolveApp(args.requireString("app")) + try requireAccessibilityTrusted() + let label = args.string("label") + let role = args.string("role") + let valueContains = args.string("value_contains") + guard label != nil || role != nil || valueContains != nil else { + throw ToolError.invalidArguments("Provide at least one of label, role, or value_contains.") + } + let waitForGone = args.bool("gone") ?? false + let timeout = min(60.0, max(1.0, args.number("timeout_seconds") ?? 10)) + let start = Date() + let deadline = start.addingTimeInterval(timeout) + var conditionMet = false + repeat { + let window = try? targetWindow(for: app, title: args.string("window_title")) + let found = window.map { + linuxElementExists( + in: $0.element, + role: role, + label: label, + valueContains: valueContains + ) + } ?? false + if found != waitForGone { + conditionMet = true + break + } + try? await Task.sleep(for: .milliseconds(400)) + } while Date() < deadline + let what = [ + role.map { "role \($0)" }, + label.map { "label \"\($0)\"" }, + valueContains.map { "value containing \"\($0)\"" }, + ].compactMap { $0 }.joined(separator: ", ") + let elapsed = String(format: "%.1f", Date().timeIntervalSince(start)) + let note = conditionMet + ? "Condition met after \(elapsed)s: \(what)\(waitForGone ? " is gone" : " appeared")." + : "TIMED OUT after \(Int(timeout))s waiting for \(what)\(waitForGone ? " to disappear" : ""). Current state below." + return try await stateResult( + app: app, + windowTitle: args.string("window_title"), + note: note, + screenshot: screenshotDetail(args) + ) +} + +private func linuxElementExists( + in root: AXUIElement, + role: String?, + label: String?, + valueContains: String?, + depth: Int = 0 +) -> Bool { + if depth <= 14 { + let matchesRole = role.map { axRole(root).lowercased() == $0.lowercased() } ?? true + let query = label?.lowercased() + let title = axString(root, kAXTitleAttribute)?.lowercased() + let description = axString(root, kAXDescriptionAttribute)?.lowercased() + let value = axString(root, kAXValueAttribute)?.lowercased() + let matchesLabel = query.map { + (title?.contains($0) ?? false) || (description?.contains($0) ?? false) + } ?? true + let matchesValue = valueContains.map { value?.contains($0.lowercased()) ?? false } ?? true + if matchesRole && matchesLabel && matchesValue { return true } + } + return axElements(root, kAXChildrenAttribute).contains { + linuxElementExists( + in: $0, + role: role, + label: label, + valueContains: valueContains, + depth: depth + 1 + ) + } } func typeTextImpl(_ args: [String: Value]) async throws -> CallTool.Result { let app = try resolveApp(args.requireString("app")) From 4c91a066ed1cfa4b8f836f31ff29a123c58c1e54 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2026 20:41:35 +0000 Subject: [PATCH 4/5] Fix CI Linux dependencies and health report portability Co-Authored-By: Matthew Lam --- .github/workflows/ci.yml | 10 ++++++++++ Sources/computer-use-mcp/HealthReport.swift | 5 +++++ Sources/computer-use-mcp/LinuxStubs.swift | 5 ----- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ce3007..b89bf18 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,16 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install Linux interaction dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + libatspi2.0-dev \ + libx11-dev \ + libxtst-dev \ + libxext-dev \ + libpng-dev + - name: Set up Swift uses: swift-actions/setup-swift@v2 with: diff --git a/Sources/computer-use-mcp/HealthReport.swift b/Sources/computer-use-mcp/HealthReport.swift index 4907c00..34c710d 100644 --- a/Sources/computer-use-mcp/HealthReport.swift +++ b/Sources/computer-use-mcp/HealthReport.swift @@ -9,6 +9,11 @@ import ScreenCaptureKit import Glibc #endif +struct InputDeliveryDiagnostic: Codable, Sendable { + let status: String + let detail: String +} + func makeHealthReport(prompt: Bool, probeCaptureService: Bool) async -> HealthReport { #if os(macOS) let accessibility: Bool diff --git a/Sources/computer-use-mcp/LinuxStubs.swift b/Sources/computer-use-mcp/LinuxStubs.swift index cbdee4b..2bffe7c 100644 --- a/Sources/computer-use-mcp/LinuxStubs.swift +++ b/Sources/computer-use-mcp/LinuxStubs.swift @@ -25,11 +25,6 @@ enum KeyDeliveryMode: String, Equatable { case globalXTest = "tier4-global-xtest" } -struct InputDeliveryDiagnostic: Codable, Sendable { - let status: String - let detail: String -} - enum FallbackReason: String, Equatable, Sendable { case axActionUnsupported = "ax-action-unsupported" case windowNumberUnresolved = "window-number-unresolved" From 868a39bfb5b3f02c86fdca3dc6dc355f567d855e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2026 21:01:59 +0000 Subject: [PATCH 5/5] Harden Linux capture and X11 keyboard focus Co-Authored-By: Matthew Lam --- Sources/CX11/include/cx11_shim.h | 192 ++++++++++++++++++ .../computer-use-mcp/Core/Screenshot.swift | 32 ++- Sources/computer-use-mcp/HealthReport.swift | 6 +- Sources/computer-use-mcp/LinuxStubs.swift | 28 ++- 4 files changed, 248 insertions(+), 10 deletions(-) diff --git a/Sources/CX11/include/cx11_shim.h b/Sources/CX11/include/cx11_shim.h index 4f09cc3..ae734be 100644 --- a/Sources/CX11/include/cx11_shim.h +++ b/Sources/CX11/include/cx11_shim.h @@ -2,9 +2,11 @@ #define COMPUTER_USE_MCP_CX11_SHIM_H #include +#include #include #include #include +#include #include #include #include @@ -101,6 +103,183 @@ static inline int cx11_fake_key_event(CX11DisplayRef display, unsigned int keyco return XTestFakeKeyEvent((Display *)display, keycode, press, 0); } +static pthread_mutex_t cx11_x_error_lock = PTHREAD_MUTEX_INITIALIZER; +static int cx11_x_error_code = 0; + +static int cx11_capture_x_error(Display *display, XErrorEvent *event) { + (void)display; + cx11_x_error_code = event->error_code; + return 0; +} + +static inline int cx11_root_width(CX11DisplayRef display) { + return display == NULL ? 0 : DisplayWidth((Display *)display, DefaultScreen((Display *)display)); +} + +static inline int cx11_root_height(CX11DisplayRef display) { + return display == NULL ? 0 : DisplayHeight((Display *)display, DefaultScreen((Display *)display)); +} + +static inline unsigned long cx11_window_pid(Display *display, Window window) { + Atom atom = XInternAtom(display, "_NET_WM_PID", True); + if (atom == None) { + return 0; + } + Atom actual_type = None; + int actual_format = 0; + unsigned long item_count = 0; + unsigned long bytes_after = 0; + unsigned char *data = NULL; + int result = XGetWindowProperty( + display, + window, + atom, + 0, + 1, + False, + XA_CARDINAL, + &actual_type, + &actual_format, + &item_count, + &bytes_after, + &data + ); + unsigned long pid = 0; + if (result == Success && data != NULL && actual_format == 32 && item_count > 0) { + pid = ((unsigned long *)data)[0]; + } + if (data != NULL) { + XFree(data); + } + return pid; +} + +static Window cx11_find_window_by_pid(Display *display, Window root, unsigned long pid, int depth) { + if (depth > 8) { + return 0; + } + if (cx11_window_pid(display, root) == pid) { + return root; + } + Window root_return = 0; + Window parent_return = 0; + Window *children = NULL; + unsigned int child_count = 0; + if (!XQueryTree( + display, + root, + &root_return, + &parent_return, + &children, + &child_count + )) { + return 0; + } + Window match = 0; + for (unsigned int index = 0; index < child_count && match == 0; index++) { + match = cx11_find_window_by_pid(display, children[index], pid, depth + 1); + } + if (children != NULL) { + XFree(children); + } + return match; +} + +static inline unsigned long cx11_window_for_pid(CX11DisplayRef display, unsigned long pid) { + if (display == NULL || pid == 0) { + return 0; + } + Display *xdisplay = (Display *)display; + return cx11_find_window_by_pid( + xdisplay, + RootWindow(xdisplay, DefaultScreen(xdisplay)), + pid, + 0 + ); +} + +static inline int cx11_activate_window(CX11DisplayRef display, unsigned long window_id) { + if (display == NULL || window_id == 0) { + return 0; + } + Display *xdisplay = (Display *)display; + Window window = (Window)window_id; + pthread_mutex_lock(&cx11_x_error_lock); + cx11_x_error_code = 0; + int (*previous_handler)(Display *, XErrorEvent *) = XSetErrorHandler(cx11_capture_x_error); + XRaiseWindow(xdisplay, window); + XSetInputFocus(xdisplay, window, RevertToParent, CurrentTime); + Atom active_window = XInternAtom(xdisplay, "_NET_ACTIVE_WINDOW", False); + Window root = RootWindow(xdisplay, DefaultScreen(xdisplay)); + if (active_window != None) { + XClientMessageEvent event; + memset(&event, 0, sizeof(event)); + event.type = ClientMessage; + event.window = window; + event.message_type = active_window; + event.format = 32; + event.data.l[0] = 1; + event.data.l[1] = CurrentTime; + XSendEvent( + xdisplay, + root, + False, + SubstructureRedirectMask | SubstructureNotifyMask, + (XEvent *)&event + ); + } + XFlush(xdisplay); + XSync(xdisplay, False); + XSetErrorHandler(previous_handler); + int error_code = cx11_x_error_code; + pthread_mutex_unlock(&cx11_x_error_lock); + if (error_code != 0) { + return 0; + } + return 1; +} + +static inline int cx11_focus_matches_pid(CX11DisplayRef display, unsigned long pid) { + if (display == NULL || pid == 0) { + return 0; + } + Display *xdisplay = (Display *)display; + Window focused = None; + int revert_to = RevertToNone; + XGetInputFocus(xdisplay, &focused, &revert_to); + if (focused == None || focused == PointerRoot) { + return 0; + } + Window root = RootWindow(xdisplay, DefaultScreen(xdisplay)); + for (Window current = focused; current != None && current != root;) { + if (cx11_window_pid(xdisplay, current) == pid) { + return 1; + } + Window parent = None; + Window *children = NULL; + unsigned int child_count = 0; + Window root_return = None; + if (!XQueryTree( + xdisplay, + current, + &root_return, + &parent, + &children, + &child_count + )) { + break; + } + if (children != NULL) { + XFree(children); + } + if (parent == current || parent == None) { + break; + } + current = parent; + } + return 0; +} + static inline unsigned char cx11_channel(unsigned long pixel, unsigned long mask) { if (mask == 0) { return 0; @@ -128,7 +307,20 @@ static inline int cx11_capture_root_rgba( } Display *xdisplay = (Display *)display; Window root = RootWindow(xdisplay, DefaultScreen(xdisplay)); + pthread_mutex_lock(&cx11_x_error_lock); + cx11_x_error_code = 0; + int (*previous_handler)(Display *, XErrorEvent *) = XSetErrorHandler(cx11_capture_x_error); XImage *image = XGetImage(xdisplay, root, x, y, width, height, AllPlanes, ZPixmap); + XSync(xdisplay, False); + XSetErrorHandler(previous_handler); + int error_code = cx11_x_error_code; + pthread_mutex_unlock(&cx11_x_error_lock); + if (error_code != 0) { + if (image != NULL) { + XDestroyImage(image); + } + return -error_code; + } if (image == NULL) { return 0; } diff --git a/Sources/computer-use-mcp/Core/Screenshot.swift b/Sources/computer-use-mcp/Core/Screenshot.swift index 2707b99..5c93ac1 100644 --- a/Sources/computer-use-mcp/Core/Screenshot.swift +++ b/Sources/computer-use-mcp/Core/Screenshot.swift @@ -273,17 +273,37 @@ func captureWindow(pid: pid_t, title: String?, frame: CGRect, detail: Screenshot } defer { cx11_close_display(display) } - let sourceWidth = Int(frame.width.rounded(.up)) - let sourceHeight = Int(frame.height.rounded(.up)) + let rootWidth = CGFloat(cx11_root_width(display)) + let rootHeight = CGFloat(cx11_root_height(display)) + let requestedMinX = frame.origin.x.rounded(.towardZero) + let requestedMinY = frame.origin.y.rounded(.towardZero) + let requestedMaxX = (frame.origin.x + frame.width).rounded(.up) + let requestedMaxY = (frame.origin.y + frame.height).rounded(.up) + let captureMinX = max(0, min(requestedMinX, rootWidth)) + let captureMinY = max(0, min(requestedMinY, rootHeight)) + let captureMaxX = max(0, min(requestedMaxX, rootWidth)) + let captureMaxY = max(0, min(requestedMaxY, rootHeight)) + guard captureMaxX > captureMinX, captureMaxY > captureMinY else { + throw ToolError.failed("The requested window frame is entirely outside the X11 screen.") + } + let sourceWidth = Int(captureMaxX - captureMinX) + let sourceHeight = Int(captureMaxY - captureMinY) var sourcePixels: UnsafeMutablePointer? - guard cx11_capture_root_rgba( + let captureResult = cx11_capture_root_rgba( display, - Int32(frame.origin.x.rounded()), - Int32(frame.origin.y.rounded()), + Int32(captureMinX), + Int32(captureMinY), UInt32(sourceWidth), UInt32(sourceHeight), &sourcePixels - ) != 0, let sourcePixels else { + ) + guard captureResult != 0, let sourcePixels else { + if captureResult < 0 { + throw ToolError.failed( + "X11 rejected the screenshot request (error code \(-captureResult)); " + + "the window may extend outside the screen." + ) + } throw ToolError.failed("Could not capture the X11 root window at the requested frame.") } defer { cx11_free(sourcePixels) } diff --git a/Sources/computer-use-mcp/HealthReport.swift b/Sources/computer-use-mcp/HealthReport.swift index 34c710d..724efb3 100644 --- a/Sources/computer-use-mcp/HealthReport.swift +++ b/Sources/computer-use-mcp/HealthReport.swift @@ -87,7 +87,11 @@ func makeHealthReport(prompt: Bool, probeCaptureService: Bool) async -> HealthRe status: accessibility ? "granted" : "not_available", requiredFor: "AT-SPI2 accessibility bus and application trees" ), - screenRecording: PermissionStatus(granted: false, status: "unsupported", requiredFor: "Screen Recording is unsupported on Linux") + screenRecording: PermissionStatus( + granted: false, + status: "unsupported", + requiredFor: "X11 display access provides screenshots; macOS Screen Recording permission is not applicable" + ) ), captureService: captureService, daemon: daemonDiagnostics(), diff --git a/Sources/computer-use-mcp/LinuxStubs.swift b/Sources/computer-use-mcp/LinuxStubs.swift index 2bffe7c..36faca4 100644 --- a/Sources/computer-use-mcp/LinuxStubs.swift +++ b/Sources/computer-use-mcp/LinuxStubs.swift @@ -189,6 +189,28 @@ private func withX11Display(_ body: (CX11DisplayRef) throws -> R) throws -> R return try body(display) } +private func ensureLinuxX11Focus(pid: pid_t) throws { + try withX11Display { display in + let window = cx11_window_for_pid(display, UInt(pid)) + guard window != 0 else { + throw ToolError.failed( + "Could not find an X11 window for pid \(pid); refusing to send global keyboard input." + ) + } + guard cx11_activate_window(display, window) != 0 else { + throw ToolError.failed( + "Could not activate the X11 window for pid \(pid); refusing to send global keyboard input." + ) + } + Thread.sleep(forTimeInterval: 0.05) + guard cx11_focus_matches_pid(display, UInt(pid)) != 0 else { + throw ToolError.failed( + "X11 focus did not reach the target app (pid \(pid)); refusing to send global keyboard input." + ) + } + } +} + private func linuxX11InputAvailability(displayName: String?) -> InputDeliveryDiagnostic { guard let displayName, !displayName.isEmpty else { return InputDeliveryDiagnostic( @@ -435,6 +457,7 @@ func typeUnicodeText(_ text: String, context: DeliveryContext) throws -> InputTi guard linuxInputDeliveryAvailable() else { throw ToolError.failed("Text input is unavailable because X11/XTest is unavailable.") } + try ensureLinuxX11Focus(pid: context.pid) X11State.shared.lock.lock() defer { X11State.shared.lock.unlock() } try withX11Display { display in @@ -462,6 +485,7 @@ func deliverKey(_ chord: KeyChord, context: DeliveryContext, targetAppIsActive: guard let token = linuxKeyToken(for: chord) else { throw ToolError.failed("Could not map the requested key to an X11 keysym.") } + try ensureLinuxX11Focus(pid: context.pid) try withX11Display { display in let modifiers = linuxModifierKeycodes(for: chord.flags, display: display) let keycode: KeyCode @@ -828,15 +852,13 @@ func typeTextImpl(_ args: [String: Value]) async throws -> CallTool.Result { windowFrame: nil, allowGlobalCursor: false ) - if axBool(element, kAXFocusedAttribute) != true, let frame = axFrame(element) { + if let frame = axFrame(element) { _ = try deliverClick( at: CGPoint(x: frame.midX, y: frame.midY), button: .left, clickCount: 1, context: context ) - } else { - AXUIElementSetAttributeValue(element, kAXFocusedAttribute as CFString, kCFBooleanTrue) } let tier = try typeUnicodeText(text, context: context) return .text("Typed \(text.count) characters into \(described) [\(tier.rawValue)].")