From 3cd48d43486dff8d967e2c377a60248aef300331 Mon Sep 17 00:00:00 2001 From: kitakkun <48154936+kitakkun@users.noreply.github.com> Date: Sun, 19 Jul 2026 05:45:18 +0900 Subject: [PATCH 01/17] feat(mirror): add Device Mirror host plugin Mirrors Android emulator/device and booted iOS simulator screens in a host-only plugin. The mirror view supports click-to-tap and drag-to-swipe, plus buttons for screenshots, hardware keys, and text input. MCP tools (listDevices, captureScreenshot, tap, swipe, pressButton, inputText) let an AI agent drive the device in the screenshot pixel coordinate space. Android is driven via adb; iOS screenshots use `xcrun simctl`, and iOS input events require idb, failing with an actionable message when it is not installed. --- jetwhale-plugins/mirror/host/build.gradle.kts | 39 +++ .../mirror/host/CaptureScreenshotCommand.kt | 36 +++ .../plugins/mirror/host/DeviceController.kt | 206 ++++++++++++++ .../plugins/mirror/host/InputTextCommand.kt | 27 ++ .../plugins/mirror/host/ListDevicesCommand.kt | 37 +++ .../mirror/host/MirrorHostPluginFactory.kt | 154 +++++++++++ .../plugins/mirror/host/MirrorMcpCommands.kt | 18 ++ .../plugins/mirror/host/MirrorScreen.kt | 258 ++++++++++++++++++ .../plugins/mirror/host/PressButtonCommand.kt | 27 ++ .../jetwhale/plugins/mirror/host/Shell.kt | 82 ++++++ .../plugins/mirror/host/SwipeCommand.kt | 37 +++ .../plugins/mirror/host/TapCommand.kt | 28 ++ .../META-INF/jetwhale/plugin-manifest.json | 16 ++ .../main/resources/icons/mirror_filled.svg | 1 + .../main/resources/icons/mirror_outlined.svg | 1 + settings.gradle.kts | 2 + 16 files changed, 969 insertions(+) create mode 100644 jetwhale-plugins/mirror/host/build.gradle.kts create mode 100644 jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/CaptureScreenshotCommand.kt create mode 100644 jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt create mode 100644 jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/InputTextCommand.kt create mode 100644 jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/ListDevicesCommand.kt create mode 100644 jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt create mode 100644 jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorMcpCommands.kt create mode 100644 jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt create mode 100644 jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/PressButtonCommand.kt create mode 100644 jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/Shell.kt create mode 100644 jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/SwipeCommand.kt create mode 100644 jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/TapCommand.kt create mode 100644 jetwhale-plugins/mirror/host/src/main/resources/META-INF/jetwhale/plugin-manifest.json create mode 100644 jetwhale-plugins/mirror/host/src/main/resources/icons/mirror_filled.svg create mode 100644 jetwhale-plugins/mirror/host/src/main/resources/icons/mirror_outlined.svg diff --git a/jetwhale-plugins/mirror/host/build.gradle.kts b/jetwhale-plugins/mirror/host/build.gradle.kts new file mode 100644 index 000000000..737b2f629 --- /dev/null +++ b/jetwhale-plugins/mirror/host/build.gradle.kts @@ -0,0 +1,39 @@ +plugins { + alias(libs.plugins.jvm) + alias(libs.plugins.composeCompiler) + alias(libs.plugins.jetbrainsCompose) + // Provides packagePlugin / installPlugin / stageDevPlugin / runJetWhale / runJetWhaleHot (published). + alias(libs.plugins.jetwhalePlugin) + // In-repo only: adds runJetWhaleLocal, which launches the local :jetwhale-host:app project. + alias(libs.plugins.jetwhaleHostLaunch) + alias(libs.plugins.publish) +} + +// Distinct group so this module's coordinates don't collide with the other `host` plugin modules. +group = "com.kitakkun.jetwhale.plugins.mirror" + +jetwhalePlugin { + // Unique name so the packaged plugin jar doesn't collide with the other plugin modules (also + // project name "host") in ~/.jetwhale/plugins/ or the dev staging directory. + pluginArchiveName.set("jetwhale-device-mirror") +} + +dependencies { + // Provided by the host at runtime, so compileOnly: these must be neither bundled into the + // plugin jar nor listed in its dependency manifest. + compileOnly(projects.jetwhaleHostSdk) + compileOnly(compose.desktop.currentOs) + compileOnly(libs.material3) + compileOnly(libs.kotlinxSerializationJson) + testImplementation(projects.jetwhaleHostSdk) + testImplementation(libs.kotlinTest) + testImplementation(libs.kotlinxSerializationJson) + testImplementation(compose.desktop.currentOs) + testImplementation(libs.material3) +} + +jetwhalePublish { + artifactId = "jetwhale-device-mirror" + name = "JetWhale Device Mirror" + description = "JetWhale host plugin that mirrors Android emulator / iOS simulator screens and drives them interactively or via MCP." +} diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/CaptureScreenshotCommand.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/CaptureScreenshotCommand.kt new file mode 100644 index 000000000..a8730eadb --- /dev/null +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/CaptureScreenshotCommand.kt @@ -0,0 +1,36 @@ +package com.kitakkun.jetwhale.plugins.mirror.host + +import com.kitakkun.jetwhale.host.sdk.ExperimentalJetWhaleApi +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArgumentException +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArguments +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpCommand +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put +import org.jetbrains.skia.Image as SkiaImage + +@OptIn(ExperimentalJetWhaleApi::class) +internal class CaptureScreenshotCommand( + private val resolveDevice: (deviceId: String?) -> MirrorDevice, +) : JetWhaleMcpCommand() { + override val name = "$TOOL_PREFIX.captureScreenshot" + override val description = "Captures the device screen as a PNG file and returns its absolute path plus the pixel dimensions. Coordinates passed to the tap/swipe tools are in this pixel coordinate space. Read the file to see the screen." + + private val deviceId by stringOrNull("Target device id from $TOOL_PREFIX.listDevices; omitted = the device selected in the mirror UI.") + + override suspend fun execute(arguments: JetWhaleMcpArguments): String { + val device = resolveDevice(arguments[deviceId]) + val png = try { + device.controller.captureScreenshot() + } catch (e: DeviceControlException) { + throw JetWhaleMcpArgumentException(e.message ?: "screenshot failed") + } + val file = screenshotFile(device) + file.writeBytes(png) + val image = SkiaImage.makeFromEncoded(png) + return buildJsonObject { + put("path", file.absolutePath) + put("widthPx", image.width) + put("heightPx", image.height) + }.toString() + } +} diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt new file mode 100644 index 000000000..7e1a0062b --- /dev/null +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt @@ -0,0 +1,206 @@ +package com.kitakkun.jetwhale.plugins.mirror.host + +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import java.io.File +import kotlin.io.path.createTempFile +import kotlin.io.path.deleteIfExists +import kotlin.io.path.readBytes + +enum class DevicePlatform { ANDROID, IOS } + +enum class DeviceButton { HOME, BACK, POWER, VOLUME_UP, VOLUME_DOWN } + +/** One connected Android emulator/device or booted iOS simulator. */ +data class MirrorDevice( + val id: String, + val name: String, + val platform: DevicePlatform, + val controller: DeviceController, +) + +/** + * Drives one device: screenshots and input events. All coordinates are in **screenshot pixels** + * (the controller converts to the platform's native input unit internally), so on-screen positions + * in a captured frame map 1:1 to input coordinates. + */ +interface DeviceController { + suspend fun captureScreenshot(): ByteArray + + suspend fun tap(x: Int, y: Int) + + suspend fun swipe(fromX: Int, fromY: Int, toX: Int, toY: Int, durationMillis: Int) + + suspend fun pressButton(button: DeviceButton) + + suspend fun inputText(text: String) +} + +internal class AndroidDeviceController( + private val adbPath: String, + private val serial: String, +) : DeviceController { + override suspend fun captureScreenshot(): ByteArray { + // exec-out keeps the PNG stream binary-safe (no pty CRLF mangling like `shell` would do). + return runCommandChecked(adbPath, "-s", serial, "exec-out", "screencap", "-p").stdout + } + + override suspend fun tap(x: Int, y: Int) { + runCommandChecked(adbPath, "-s", serial, "shell", "input", "tap", "$x", "$y") + } + + override suspend fun swipe(fromX: Int, fromY: Int, toX: Int, toY: Int, durationMillis: Int) { + runCommandChecked(adbPath, "-s", serial, "shell", "input", "swipe", "$fromX", "$fromY", "$toX", "$toY", "$durationMillis") + } + + override suspend fun pressButton(button: DeviceButton) { + val keycode = when (button) { + DeviceButton.HOME -> "KEYCODE_HOME" + DeviceButton.BACK -> "KEYCODE_BACK" + DeviceButton.POWER -> "KEYCODE_POWER" + DeviceButton.VOLUME_UP -> "KEYCODE_VOLUME_UP" + DeviceButton.VOLUME_DOWN -> "KEYCODE_VOLUME_DOWN" + } + runCommandChecked(adbPath, "-s", serial, "shell", "input", "keyevent", keycode) + } + + override suspend fun inputText(text: String) { + // `adb shell` re-parses arguments through the device shell, so shell metacharacters must + // be escaped; `input text` additionally requires spaces encoded as %s. + val escaped = text + .replace(Regex("""([\\'"`$&*()\[\]{}+|<>;?~#!])"""), """\\$1""") + .replace(" ", "%s") + runCommandChecked(adbPath, "-s", serial, "shell", "input", "text", escaped) + } +} + +/** + * iOS simulator controller. Screenshots go through `xcrun simctl`; input events need Facebook's + * `idb` companion tool (simctl has no input API) and fail with a clear message when it is absent. + */ +internal class IosDeviceController( + private val udid: String, + private val idbPath: String?, +) : DeviceController { + // Pixels-per-point factor of the simulator screen, fetched lazily from `idb describe`. + private var cachedPixelsPerPoint: Double? = null + + override suspend fun captureScreenshot(): ByteArray { + val tempFile = createTempFile(prefix = "jetwhale-mirror-", suffix = ".png") + try { + runCommandChecked("xcrun", "simctl", "io", udid, "screenshot", tempFile.toAbsolutePath().toString()) + return tempFile.readBytes() + } finally { + tempFile.deleteIfExists() + } + } + + override suspend fun tap(x: Int, y: Int) { + val idb = requireIdb() + val scale = pixelsPerPoint() + runCommandChecked(idb, "ui", "tap", "--udid", udid, "${(x / scale).toInt()}", "${(y / scale).toInt()}") + } + + override suspend fun swipe(fromX: Int, fromY: Int, toX: Int, toY: Int, durationMillis: Int) { + val idb = requireIdb() + val scale = pixelsPerPoint() + runCommandChecked( + idb, "ui", "swipe", "--udid", udid, + "--duration", "${durationMillis / 1000.0}", + "${(fromX / scale).toInt()}", "${(fromY / scale).toInt()}", + "${(toX / scale).toInt()}", "${(toY / scale).toInt()}", + ) + } + + override suspend fun pressButton(button: DeviceButton) { + val idbButton = when (button) { + DeviceButton.HOME -> "HOME" + DeviceButton.POWER -> "LOCK" + DeviceButton.BACK -> throw DeviceControlException("iOS has no BACK button") + DeviceButton.VOLUME_UP, DeviceButton.VOLUME_DOWN -> + throw DeviceControlException("volume buttons are not controllable on the iOS simulator") + } + runCommandChecked(requireIdb(), "ui", "button", "--udid", udid, idbButton) + } + + override suspend fun inputText(text: String) { + runCommandChecked(requireIdb(), "ui", "text", "--udid", udid, text) + } + + private fun requireIdb(): String = idbPath + ?: throw DeviceControlException("iOS simulator input requires idb (https://fbidb.io). Install it with: brew install idb-companion && pipx install fb-idb") + + private suspend fun pixelsPerPoint(): Double { + cachedPixelsPerPoint?.let { return it } + val output = runCommandChecked(requireIdb(), "describe", "--udid", udid, "--json").stdoutText + val screen = try { + Json.parseToJsonElement(output).jsonObject["screen_dimensions"]!!.jsonObject + } catch (e: Exception) { + throw DeviceControlException("could not read screen dimensions from 'idb describe'", e) + } + val density = screen["density"]?.jsonPrimitive?.content?.toDoubleOrNull() + ?: throw DeviceControlException("'idb describe' reported no screen density") + cachedPixelsPerPoint = density + return density + } +} + +/** Lists connected Android devices/emulators and booted iOS simulators. */ +internal class DeviceDiscovery { + private val adbPath: String by lazy { findAdbPath() } + private val idbPath: String? by lazy { findIdbPath() } + + suspend fun discoverDevices(): List = discoverAndroidDevices() + discoverIosSimulators() + + private suspend fun discoverAndroidDevices(): List { + val result = try { + runCommandChecked(adbPath, "devices", "-l") + } catch (_: DeviceControlException) { + return emptyList() + } + return result.stdoutText.lineSequence() + .drop(1) // "List of devices attached" header + .filter { it.isNotBlank() } + .mapNotNull { line -> + val tokens = line.trim().split(Regex("\\s+")) + val serial = tokens.getOrNull(0) ?: return@mapNotNull null + if (tokens.getOrNull(1) != "device") return@mapNotNull null + val model = tokens.firstOrNull { it.startsWith("model:") }?.removePrefix("model:")?.replace('_', ' ') + MirrorDevice( + id = serial, + name = model ?: serial, + platform = DevicePlatform.ANDROID, + controller = AndroidDeviceController(adbPath = adbPath, serial = serial), + ) + } + .toList() + } + + private suspend fun discoverIosSimulators(): List { + if (!File("/usr/bin/xcrun").exists()) return emptyList() + val result = try { + runCommandChecked("xcrun", "simctl", "list", "devices", "booted", "-j") + } catch (_: DeviceControlException) { + return emptyList() + } + val devicesByRuntime = try { + Json.parseToJsonElement(result.stdoutText).jsonObject["devices"]?.jsonObject ?: return emptyList() + } catch (_: Exception) { + return emptyList() + } + return devicesByRuntime.values.flatMap { runtimeDevices -> + (runtimeDevices as? kotlinx.serialization.json.JsonArray).orEmpty().mapNotNull { element -> + val device = element.jsonObject + if (device["state"]?.jsonPrimitive?.content != "Booted") return@mapNotNull null + val udid = device["udid"]?.jsonPrimitive?.content ?: return@mapNotNull null + MirrorDevice( + id = udid, + name = device["name"]?.jsonPrimitive?.content ?: udid, + platform = DevicePlatform.IOS, + controller = IosDeviceController(udid = udid, idbPath = idbPath), + ) + } + } + } +} diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/InputTextCommand.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/InputTextCommand.kt new file mode 100644 index 000000000..33f8854e9 --- /dev/null +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/InputTextCommand.kt @@ -0,0 +1,27 @@ +package com.kitakkun.jetwhale.plugins.mirror.host + +import com.kitakkun.jetwhale.host.sdk.ExperimentalJetWhaleApi +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArgumentException +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArguments +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpCommand + +@OptIn(ExperimentalJetWhaleApi::class) +internal class InputTextCommand( + private val resolveDevice: (deviceId: String?) -> MirrorDevice, +) : JetWhaleMcpCommand() { + override val name = "$TOOL_PREFIX.inputText" + override val description = "Types text into the currently focused input field on the device. Tap the field first to focus it." + + private val deviceId by stringOrNull("Target device id from $TOOL_PREFIX.listDevices; omitted = the device selected in the mirror UI.") + private val text by string("The text to type. Android only supports ASCII text here.") + + override suspend fun execute(arguments: JetWhaleMcpArguments): String { + val device = resolveDevice(arguments[deviceId]) + try { + device.controller.inputText(arguments[text]) + } catch (e: DeviceControlException) { + throw JetWhaleMcpArgumentException(e.message ?: "text input failed") + } + return okJson() + } +} diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/ListDevicesCommand.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/ListDevicesCommand.kt new file mode 100644 index 000000000..c776c68a9 --- /dev/null +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/ListDevicesCommand.kt @@ -0,0 +1,37 @@ +package com.kitakkun.jetwhale.plugins.mirror.host + +import com.kitakkun.jetwhale.host.sdk.ExperimentalJetWhaleApi +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArguments +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpCommand +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put + +@OptIn(ExperimentalJetWhaleApi::class) +internal class ListDevicesCommand( + private val refreshDevices: suspend () -> List, + private val selectedDeviceId: () -> String?, +) : JetWhaleMcpCommand() { + override val name = "$TOOL_PREFIX.listDevices" + override val description = "Lists connected Android emulators/devices and booted iOS simulators available for mirroring and input control. Use the returned deviceId with the other $TOOL_PREFIX tools; tools with deviceId omitted target the device selected in the mirror UI." + + override suspend fun execute(arguments: JetWhaleMcpArguments): String { + val devices = refreshDevices() + val selected = selectedDeviceId() + return buildJsonObject { + put( + "devices", + JsonArray( + devices.map { device -> + buildJsonObject { + put("deviceId", device.id) + put("name", device.name) + put("platform", device.platform.name) + put("selected", device.id == selected) + } + }, + ), + ) + }.toString() + } +} diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt new file mode 100644 index 000000000..d61f7e0b6 --- /dev/null +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt @@ -0,0 +1,154 @@ +package com.kitakkun.jetwhale.plugins.mirror.host + +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateListOf +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.setValue +import androidx.compose.runtime.snapshots.SnapshotStateList +import androidx.compose.ui.graphics.ImageBitmap +import androidx.compose.ui.graphics.toComposeImageBitmap +import com.kitakkun.jetwhale.host.sdk.ExperimentalJetWhaleApi +import com.kitakkun.jetwhale.host.sdk.JetWhaleHostPlugin +import com.kitakkun.jetwhale.host.sdk.JetWhaleHostPluginFactory +import com.kitakkun.jetwhale.host.sdk.JetWhaleHostPluginUi +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArgumentException +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpCapablePlugin +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpCommand +import kotlinx.coroutines.delay +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import org.jetbrains.skia.Image as SkiaImage + +// Instantiated by the host via the fully-qualified name declared in plugin-manifest.json. +@Suppress("UNUSED") +class MirrorHostPluginFactory : JetWhaleHostPluginFactory { + override fun createPlugin(): JetWhaleHostPlugin = MirrorHostPlugin() +} + +private const val DEVICE_POLL_INTERVAL_MILLIS = 3_000L +private const val FRAME_INTERVAL_MILLIS = 350L + +@OptIn(ExperimentalJetWhaleApi::class) +private class MirrorHostPlugin : + JetWhaleHostPlugin(), + JetWhaleHostPluginUi, + JetWhaleMcpCapablePlugin { + + private val discovery = DeviceDiscovery() + + private val devices: SnapshotStateList = mutableStateListOf() + private var selectedDeviceId by mutableStateOf(null) + private var mirroringEnabled by mutableStateOf(true) + private var latestFrame by mutableStateOf(null) + private var lastError by mutableStateOf(null) + + private val selectedDevice: MirrorDevice? get() = devices.firstOrNull { it.id == selectedDeviceId } + + override fun onCreate() { + pluginScope.launch { + while (isActive) { + refreshDevices() + delay(DEVICE_POLL_INTERVAL_MILLIS) + } + } + // The mirror is a screenshot poll loop: cheap, tool-free, and identical for both platforms. + pluginScope.launch { + while (isActive) { + val device = selectedDevice + if (mirroringEnabled && device != null) { + try { + val png = device.controller.captureScreenshot() + latestFrame = SkiaImage.makeFromEncoded(png).toComposeImageBitmap() + lastError = null + } catch (e: DeviceControlException) { + lastError = e.message + } + delay(FRAME_INTERVAL_MILLIS) + } else { + delay(DEVICE_POLL_INTERVAL_MILLIS) + } + } + } + } + + private suspend fun refreshDevices(): List { + val discovered = discovery.discoverDevices() + devices.apply { + clear() + addAll(discovered) + } + if (devices.none { it.id == selectedDeviceId }) { + selectedDeviceId = devices.firstOrNull()?.id + latestFrame = null + } + return discovered + } + + // Runs a device-control action from a UI callback, surfacing failures in the status bar. + private fun runControl(action: suspend () -> Unit) { + pluginScope.launch { + try { + action() + lastError = null + } catch (e: DeviceControlException) { + lastError = e.message + } + } + } + + @Composable + override fun Content() { + MirrorScreen( + devices = devices, + selectedDeviceId = selectedDeviceId, + onSelectDevice = { id -> + selectedDeviceId = id + latestFrame = null + }, + frame = latestFrame, + mirroringEnabled = mirroringEnabled, + onToggleMirroring = { mirroringEnabled = it }, + errorMessage = lastError, + onRefreshDevices = { pluginScope.launch { refreshDevices() } }, + onTap = { x, y -> selectedDevice?.let { runControl { it.controller.tap(x, y) } } }, + onSwipe = { fromX, fromY, toX, toY -> + selectedDevice?.let { runControl { it.controller.swipe(fromX, fromY, toX, toY, durationMillis = 200) } } + }, + onPressButton = { button -> selectedDevice?.let { runControl { it.controller.pressButton(button) } } }, + onInputText = { text -> selectedDevice?.let { runControl { it.controller.inputText(text) } } }, + onSaveScreenshot = { selectedDevice?.let { runControl { saveScreenshotToDisk(it) } } }, + ) + } + + private suspend fun saveScreenshotToDisk(device: MirrorDevice) { + val png = device.controller.captureScreenshot() + val file = screenshotFile(device) + file.writeBytes(png) + lastError = "Saved screenshot: ${file.absolutePath}" + } + + // ------------------------------------------------------------------------- + // JetWhaleMcpCapablePlugin + // ------------------------------------------------------------------------- + + // MCP tools address devices by id; with the id omitted they fall back to the device selected + // in the mirror UI so the AI can simply operate "the visible device". + private fun resolveDevice(deviceId: String?): MirrorDevice { + if (deviceId == null) { + return selectedDevice + ?: throw JetWhaleMcpArgumentException("no device connected; boot an emulator/simulator and call $TOOL_PREFIX.listDevices") + } + return devices.firstOrNull { it.id == deviceId } + ?: throw JetWhaleMcpArgumentException("unknown deviceId: $deviceId (call $TOOL_PREFIX.listDevices)") + } + + override val mcpCommands: List = listOf( + ListDevicesCommand(refreshDevices = ::refreshDevices, selectedDeviceId = { selectedDeviceId }), + CaptureScreenshotCommand(resolveDevice = ::resolveDevice), + TapCommand(resolveDevice = ::resolveDevice), + SwipeCommand(resolveDevice = ::resolveDevice), + PressButtonCommand(resolveDevice = ::resolveDevice), + InputTextCommand(resolveDevice = ::resolveDevice), + ) +} diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorMcpCommands.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorMcpCommands.kt new file mode 100644 index 000000000..1ce03375d --- /dev/null +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorMcpCommands.kt @@ -0,0 +1,18 @@ +package com.kitakkun.jetwhale.plugins.mirror.host + +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put +import java.io.File +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter + +internal const val TOOL_PREFIX = "com.kitakkun.jetwhale.mirror" + +internal fun okJson(): String = buildJsonObject { put("ok", true) }.toString() + +/** Where saved device screenshots land, both for the UI button and the MCP tool. */ +internal fun screenshotFile(device: MirrorDevice): File { + val dir = File(System.getProperty("java.io.tmpdir"), "jetwhale-mirror").apply { mkdirs() } + val timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss-SSS")) + return File(dir, "${device.platform.name.lowercase()}-$timestamp.png") +} diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt new file mode 100644 index 000000000..5ba1b61aa --- /dev/null +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt @@ -0,0 +1,258 @@ +package com.kitakkun.jetwhale.plugins.mirror.host + +import androidx.compose.foundation.Image +import androidx.compose.foundation.gestures.detectDragGestures +import androidx.compose.foundation.gestures.detectTapGestures +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.aspectRatio +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.horizontalScroll +import androidx.compose.material3.Button +import androidx.compose.material3.DropdownMenuItem +import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.material3.ExposedDropdownMenuBox +import androidx.compose.material3.ExposedDropdownMenuDefaults +import androidx.compose.material3.IconButton +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedButton +import androidx.compose.material3.OutlinedTextField +import androidx.compose.material3.Scaffold +import androidx.compose.material3.Switch +import androidx.compose.material3.Text +import androidx.compose.material3.TopAppBar +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.graphics.ImageBitmap +import androidx.compose.ui.input.pointer.pointerInput +import androidx.compose.ui.unit.dp + +@OptIn(ExperimentalMaterial3Api::class) +@Composable +internal fun MirrorScreen( + devices: List, + selectedDeviceId: String?, + onSelectDevice: (String) -> Unit, + frame: ImageBitmap?, + mirroringEnabled: Boolean, + onToggleMirroring: (Boolean) -> Unit, + errorMessage: String?, + onRefreshDevices: () -> Unit, + onTap: (x: Int, y: Int) -> Unit, + onSwipe: (fromX: Int, fromY: Int, toX: Int, toY: Int) -> Unit, + onPressButton: (DeviceButton) -> Unit, + onInputText: (String) -> Unit, + onSaveScreenshot: () -> Unit, +) { + val selectedDevice = devices.firstOrNull { it.id == selectedDeviceId } + Scaffold( + topBar = { TopAppBar(title = { Text("Device Mirror") }) }, + ) { innerPadding -> + Column( + modifier = Modifier + .fillMaxSize() + .padding(innerPadding) + .padding(16.dp), + verticalArrangement = Arrangement.spacedBy(12.dp), + ) { + DeviceSelectorRow( + devices = devices, + selectedDevice = selectedDevice, + onSelectDevice = onSelectDevice, + onRefreshDevices = onRefreshDevices, + mirroringEnabled = mirroringEnabled, + onToggleMirroring = onToggleMirroring, + ) + ControlButtonsRow( + platform = selectedDevice?.platform, + enabled = selectedDevice != null, + onPressButton = onPressButton, + onSaveScreenshot = onSaveScreenshot, + ) + TextInputRow(enabled = selectedDevice != null, onInputText = onInputText) + errorMessage?.let { + Text(it, style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.error) + } + MirrorFrame( + frame = frame, + hasDevice = selectedDevice != null, + onTap = onTap, + onSwipe = onSwipe, + modifier = Modifier + .fillMaxWidth() + .weight(1f), + ) + } + } +} + +@OptIn(ExperimentalMaterial3Api::class) +@Composable +private fun DeviceSelectorRow( + devices: List, + selectedDevice: MirrorDevice?, + onSelectDevice: (String) -> Unit, + onRefreshDevices: () -> Unit, + mirroringEnabled: Boolean, + onToggleMirroring: (Boolean) -> Unit, +) { + Row( + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + var expanded by remember { mutableStateOf(false) } + ExposedDropdownMenuBox( + expanded = expanded, + onExpandedChange = { expanded = it }, + modifier = Modifier.weight(1f), + ) { + OutlinedTextField( + value = selectedDevice?.let { "${it.name} (${it.platform})" } ?: "No device", + onValueChange = {}, + readOnly = true, + singleLine = true, + trailingIcon = { ExposedDropdownMenuDefaults.TrailingIcon(expanded = expanded) }, + modifier = Modifier.menuAnchor().fillMaxWidth(), + ) + ExposedDropdownMenu(expanded = expanded, onDismissRequest = { expanded = false }) { + devices.forEach { device -> + DropdownMenuItem( + text = { Text("${device.name} (${device.platform})") }, + onClick = { + onSelectDevice(device.id) + expanded = false + }, + ) + } + } + } + OutlinedButton(onClick = onRefreshDevices) { Text("Refresh") } + Text("Mirror", style = MaterialTheme.typography.labelMedium) + Switch(checked = mirroringEnabled, onCheckedChange = onToggleMirroring) + } +} + +@Composable +private fun ControlButtonsRow( + platform: DevicePlatform?, + enabled: Boolean, + onPressButton: (DeviceButton) -> Unit, + onSaveScreenshot: () -> Unit, +) { + Row( + modifier = Modifier.horizontalScroll(rememberScrollState()), + horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + Button(onClick = onSaveScreenshot, enabled = enabled) { Text("Screenshot") } + val buttons = when (platform) { + DevicePlatform.IOS -> listOf(DeviceButton.HOME, DeviceButton.POWER) + else -> DeviceButton.entries + } + buttons.forEach { button -> + OutlinedButton(onClick = { onPressButton(button) }, enabled = enabled) { + Text( + when (button) { + DeviceButton.HOME -> "Home" + DeviceButton.BACK -> "Back" + DeviceButton.POWER -> "Power" + DeviceButton.VOLUME_UP -> "Vol +" + DeviceButton.VOLUME_DOWN -> "Vol -" + }, + ) + } + } + } +} + +@Composable +private fun TextInputRow( + enabled: Boolean, + onInputText: (String) -> Unit, +) { + Row( + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + var text by remember { mutableStateOf("") } + OutlinedTextField( + value = text, + onValueChange = { text = it }, + singleLine = true, + placeholder = { Text("Type text to send to the device") }, + modifier = Modifier.weight(1f), + ) + Button( + onClick = { + onInputText(text) + text = "" + }, + enabled = enabled && text.isNotEmpty(), + ) { + Text("Send") + } + } +} + +@Composable +private fun MirrorFrame( + frame: ImageBitmap?, + hasDevice: Boolean, + onTap: (x: Int, y: Int) -> Unit, + onSwipe: (fromX: Int, fromY: Int, toX: Int, toY: Int) -> Unit, + modifier: Modifier = Modifier, +) { + Box(modifier = modifier, contentAlignment = Alignment.Center) { + if (frame == null) { + Text( + if (hasDevice) "Waiting for the first frame…" else "Boot an Android emulator or iOS simulator to start mirroring.", + style = MaterialTheme.typography.bodyMedium, + ) + return@Box + } + // aspectRatio makes the image composable's bounds coincide exactly with the drawn frame, + // so pointer offsets scale linearly to device pixels. + Image( + bitmap = frame, + contentDescription = "Mirrored device screen", + modifier = Modifier + .aspectRatio(frame.width.toFloat() / frame.height.toFloat()) + .pointerInput(frame.width, frame.height) { + detectTapGestures { offset -> + val scale = frame.width.toFloat() / size.width + onTap((offset.x * scale).toInt(), (offset.y * scale).toInt()) + } + } + .pointerInput(frame.width, frame.height) { + var dragStart = Offset.Zero + var dragEnd = Offset.Zero + detectDragGestures( + onDragStart = { offset -> + dragStart = offset + dragEnd = offset + }, + onDrag = { change, _ -> dragEnd = change.position }, + onDragEnd = { + val scale = frame.width.toFloat() / size.width + onSwipe( + (dragStart.x * scale).toInt(), + (dragStart.y * scale).toInt(), + (dragEnd.x * scale).toInt(), + (dragEnd.y * scale).toInt(), + ) + }, + ) + }, + ) + } +} diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/PressButtonCommand.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/PressButtonCommand.kt new file mode 100644 index 000000000..6c99a52ed --- /dev/null +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/PressButtonCommand.kt @@ -0,0 +1,27 @@ +package com.kitakkun.jetwhale.plugins.mirror.host + +import com.kitakkun.jetwhale.host.sdk.ExperimentalJetWhaleApi +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArgumentException +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArguments +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpCommand + +@OptIn(ExperimentalJetWhaleApi::class) +internal class PressButtonCommand( + private val resolveDevice: (deviceId: String?) -> MirrorDevice, +) : JetWhaleMcpCommand() { + override val name = "$TOOL_PREFIX.pressButton" + override val description = "Presses a hardware button on the device. Android supports all buttons; the iOS simulator supports HOME and POWER (BACK and volume buttons fail there)." + + private val deviceId by stringOrNull("Target device id from $TOOL_PREFIX.listDevices; omitted = the device selected in the mirror UI.") + private val button by enum("The hardware button to press.", DeviceButton.entries) + + override suspend fun execute(arguments: JetWhaleMcpArguments): String { + val device = resolveDevice(arguments[deviceId]) + try { + device.controller.pressButton(arguments[button]) + } catch (e: DeviceControlException) { + throw JetWhaleMcpArgumentException(e.message ?: "button press failed") + } + return okJson() + } +} diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/Shell.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/Shell.kt new file mode 100644 index 000000000..38fa7da06 --- /dev/null +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/Shell.kt @@ -0,0 +1,82 @@ +package com.kitakkun.jetwhale.plugins.mirror.host + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.io.File + +/** A device-control operation failed or is not supported; [message] is caller-facing. */ +class DeviceControlException(message: String, cause: Throwable? = null) : Exception(message, cause) + +internal class CommandResult( + val exitCode: Int, + val stdout: ByteArray, + val stderr: String, +) { + val stdoutText: String get() = stdout.decodeToString() +} + +/** + * Runs an external command and captures stdout as raw bytes (screenshots arrive as binary PNG + * data on stdout) and stderr as text. + */ +internal suspend fun runCommand(vararg command: String): CommandResult = withContext(Dispatchers.IO) { + val process = try { + ProcessBuilder(*command).start() + } catch (e: java.io.IOException) { + throw DeviceControlException("failed to launch '${command.first()}': ${e.message}", e) + } + // Drain stderr concurrently so neither pipe can fill up and deadlock the process. + var stderr = "" + val stderrThread = Thread { + stderr = process.errorStream.bufferedReader().readText() + }.apply { start() } + val stdout = process.inputStream.readBytes() + val exitCode = process.waitFor() + stderrThread.join() + CommandResult(exitCode = exitCode, stdout = stdout, stderr = stderr) +} + +/** Runs [command] and returns stdout text, throwing [DeviceControlException] on a non-zero exit. */ +internal suspend fun runCommandChecked(vararg command: String): CommandResult { + val result = runCommand(*command) + if (result.exitCode != 0) { + val detail = result.stderr.ifBlank { result.stdoutText }.trim().take(500) + throw DeviceControlException("'${command.joinToString(" ")}' failed (exit ${result.exitCode}): $detail") + } + return result +} + +/** + * Finds the absolute path to the adb executable by checking common installation locations, + * falling back to "adb" on the PATH. + */ +internal fun findAdbPath(): String { + val homeDir = System.getProperty("user.home") + val androidHome = System.getenv("ANDROID_HOME") + val androidSdkRoot = System.getenv("ANDROID_SDK_ROOT") + + val candidatePaths = listOfNotNull( + "/usr/bin/adb", + "/usr/local/bin/adb", + homeDir?.let { "$it/Android/Sdk/platform-tools/adb" }, + homeDir?.let { "$it/Library/Android/sdk/platform-tools/adb" }, + androidHome?.let { "$it/platform-tools/adb" }, + androidSdkRoot?.let { "$it/platform-tools/adb" }, + ) + + return candidatePaths.firstOrNull { path -> + File(path).exists() && File(path).canExecute() + } ?: "adb" +} + +/** Finds the idb executable (used to drive iOS simulator input), or null when not installed. */ +internal fun findIdbPath(): String? { + val candidatePaths = listOf( + "/usr/local/bin/idb", + "/opt/homebrew/bin/idb", + ) + candidatePaths.firstOrNull { File(it).canExecute() }?.let { return it } + // Fall back to a PATH lookup. + val pathDirs = System.getenv("PATH")?.split(File.pathSeparator).orEmpty() + return pathDirs.map { File(it, "idb") }.firstOrNull { it.canExecute() }?.absolutePath +} diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/SwipeCommand.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/SwipeCommand.kt new file mode 100644 index 000000000..6f17090de --- /dev/null +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/SwipeCommand.kt @@ -0,0 +1,37 @@ +package com.kitakkun.jetwhale.plugins.mirror.host + +import com.kitakkun.jetwhale.host.sdk.ExperimentalJetWhaleApi +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArgumentException +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArguments +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpCommand + +@OptIn(ExperimentalJetWhaleApi::class) +internal class SwipeCommand( + private val resolveDevice: (deviceId: String?) -> MirrorDevice, +) : JetWhaleMcpCommand() { + override val name = "$TOOL_PREFIX.swipe" + override val description = "Swipes on the device screen from one position to another, in the pixel coordinate space of $TOOL_PREFIX.captureScreenshot. Use it for scrolling and drag gestures." + + private val deviceId by stringOrNull("Target device id from $TOOL_PREFIX.listDevices; omitted = the device selected in the mirror UI.") + private val fromX by int("Start horizontal position in screenshot pixels.") + private val fromY by int("Start vertical position in screenshot pixels.") + private val toX by int("End horizontal position in screenshot pixels.") + private val toY by int("End vertical position in screenshot pixels.") + private val durationMillis by intOrNull("Gesture duration in milliseconds; omitted = 300.") + + override suspend fun execute(arguments: JetWhaleMcpArguments): String { + val device = resolveDevice(arguments[deviceId]) + try { + device.controller.swipe( + fromX = arguments[fromX], + fromY = arguments[fromY], + toX = arguments[toX], + toY = arguments[toY], + durationMillis = arguments[durationMillis] ?: 300, + ) + } catch (e: DeviceControlException) { + throw JetWhaleMcpArgumentException(e.message ?: "swipe failed") + } + return okJson() + } +} diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/TapCommand.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/TapCommand.kt new file mode 100644 index 000000000..3dc132dcc --- /dev/null +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/TapCommand.kt @@ -0,0 +1,28 @@ +package com.kitakkun.jetwhale.plugins.mirror.host + +import com.kitakkun.jetwhale.host.sdk.ExperimentalJetWhaleApi +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArgumentException +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArguments +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpCommand + +@OptIn(ExperimentalJetWhaleApi::class) +internal class TapCommand( + private val resolveDevice: (deviceId: String?) -> MirrorDevice, +) : JetWhaleMcpCommand() { + override val name = "$TOOL_PREFIX.tap" + override val description = "Taps the device screen at the given position, in the pixel coordinate space of $TOOL_PREFIX.captureScreenshot." + + private val deviceId by stringOrNull("Target device id from $TOOL_PREFIX.listDevices; omitted = the device selected in the mirror UI.") + private val x by int("Horizontal position in screenshot pixels.") + private val y by int("Vertical position in screenshot pixels.") + + override suspend fun execute(arguments: JetWhaleMcpArguments): String { + val device = resolveDevice(arguments[deviceId]) + try { + device.controller.tap(arguments[x], arguments[y]) + } catch (e: DeviceControlException) { + throw JetWhaleMcpArgumentException(e.message ?: "tap failed") + } + return okJson() + } +} diff --git a/jetwhale-plugins/mirror/host/src/main/resources/META-INF/jetwhale/plugin-manifest.json b/jetwhale-plugins/mirror/host/src/main/resources/META-INF/jetwhale/plugin-manifest.json new file mode 100644 index 000000000..58d070785 --- /dev/null +++ b/jetwhale-plugins/mirror/host/src/main/resources/META-INF/jetwhale/plugin-manifest.json @@ -0,0 +1,16 @@ +{ + "$schema": "../../../../../../../../../../schemas/plugin-manifest.schema.json", + "plugins": [ + { + "pluginId": "com.kitakkun.jetwhale.mirror", + "pluginName": "Device Mirror", + "version": "1.0.0", + "factoryClass": "com.kitakkun.jetwhale.plugins.mirror.host.MirrorHostPluginFactory", + "requiresAgent": false, + "icon": { + "activePath": "icons/mirror_filled.svg", + "inactivePath": "icons/mirror_outlined.svg" + } + } + ] +} diff --git a/jetwhale-plugins/mirror/host/src/main/resources/icons/mirror_filled.svg b/jetwhale-plugins/mirror/host/src/main/resources/icons/mirror_filled.svg new file mode 100644 index 000000000..c9c524f32 --- /dev/null +++ b/jetwhale-plugins/mirror/host/src/main/resources/icons/mirror_filled.svg @@ -0,0 +1 @@ + diff --git a/jetwhale-plugins/mirror/host/src/main/resources/icons/mirror_outlined.svg b/jetwhale-plugins/mirror/host/src/main/resources/icons/mirror_outlined.svg new file mode 100644 index 000000000..50b9f933c --- /dev/null +++ b/jetwhale-plugins/mirror/host/src/main/resources/icons/mirror_outlined.svg @@ -0,0 +1 @@ + diff --git a/settings.gradle.kts b/settings.gradle.kts index 14d0d35b4..3670e3195 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -51,6 +51,8 @@ include(":jetwhale-plugins:network:agent-ktor") include(":jetwhale-plugins:network:agent-okhttp") include(":jetwhale-plugins:network:host") +include(":jetwhale-plugins:mirror:host") + include(":test-annotations") include(":demo:shared") From 4ac98367b6b4302d3013740ff672a7b1808476bf Mon Sep 17 00:00:00 2001 From: kitakkun <48154936+kitakkun@users.noreply.github.com> Date: Sun, 19 Jul 2026 05:49:10 +0900 Subject: [PATCH 02/17] style(mirror): apply spotless formatting --- .../kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt | 3 +++ .../com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt index 7e1a0062b..b34451470 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt @@ -116,8 +116,11 @@ internal class IosDeviceController( override suspend fun pressButton(button: DeviceButton) { val idbButton = when (button) { DeviceButton.HOME -> "HOME" + DeviceButton.POWER -> "LOCK" + DeviceButton.BACK -> throw DeviceControlException("iOS has no BACK button") + DeviceButton.VOLUME_UP, DeviceButton.VOLUME_DOWN -> throw DeviceControlException("volume buttons are not controllable on the iOS simulator") } diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt index 5ba1b61aa..54bc76ad1 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt @@ -3,6 +3,7 @@ package com.kitakkun.jetwhale.plugins.mirror.host import androidx.compose.foundation.Image import androidx.compose.foundation.gestures.detectDragGestures import androidx.compose.foundation.gestures.detectTapGestures +import androidx.compose.foundation.horizontalScroll import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column @@ -12,7 +13,6 @@ import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.padding import androidx.compose.foundation.rememberScrollState -import androidx.compose.foundation.horizontalScroll import androidx.compose.material3.Button import androidx.compose.material3.DropdownMenuItem import androidx.compose.material3.ExperimentalMaterial3Api From 97d403c56cf94d51aec455fb585e90891ba03a5a Mon Sep 17 00:00:00 2001 From: kitakkun <48154936+kitakkun@users.noreply.github.com> Date: Sun, 19 Jul 2026 05:57:39 +0900 Subject: [PATCH 03/17] perf(mirror): pace the frame loop by capture latency A fixed 350ms sleep per frame capped mirroring at ~1-2fps on top of the capture round trip. Capture continuously with a minimal gap instead, and back off only after a failed capture. --- .../plugins/mirror/host/MirrorHostPluginFactory.kt | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt index d61f7e0b6..e0ad0dc0d 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt @@ -27,7 +27,13 @@ class MirrorHostPluginFactory : JetWhaleHostPluginFactory { } private const val DEVICE_POLL_INTERVAL_MILLIS = 3_000L -private const val FRAME_INTERVAL_MILLIS = 350L + +// The frame loop is paced by the screenshot capture itself (~100-300ms per adb/simctl round +// trip); this small gap only keeps a failing capture from busy-looping. +private const val FRAME_INTERVAL_MILLIS = 16L + +// Back off after a failed capture so a dead device doesn't spam error processes. +private const val FRAME_RETRY_INTERVAL_MILLIS = 1_000L @OptIn(ExperimentalJetWhaleApi::class) private class MirrorHostPlugin : @@ -61,10 +67,11 @@ private class MirrorHostPlugin : val png = device.controller.captureScreenshot() latestFrame = SkiaImage.makeFromEncoded(png).toComposeImageBitmap() lastError = null + delay(FRAME_INTERVAL_MILLIS) } catch (e: DeviceControlException) { lastError = e.message + delay(FRAME_RETRY_INTERVAL_MILLIS) } - delay(FRAME_INTERVAL_MILLIS) } else { delay(DEVICE_POLL_INTERVAL_MILLIS) } From ff78fa9cf723b8001c3af15d867311041ab21c5b Mon Sep 17 00:00:00 2001 From: kitakkun <48154936+kitakkun@users.noreply.github.com> Date: Sun, 19 Jul 2026 06:01:48 +0900 Subject: [PATCH 04/17] feat(mirror): add BACKSPACE and ENTER keys Text could be typed but not deleted or confirmed. Android maps them to KEYCODE_DEL / KEYCODE_ENTER; iOS sends USB HID usage codes via `idb ui key`. Exposed as control-row buttons and through the existing pressButton MCP tool. --- .../plugins/mirror/host/DeviceController.kt | 22 +++++++++++++++---- .../plugins/mirror/host/MirrorScreen.kt | 4 +++- .../plugins/mirror/host/PressButtonCommand.kt | 2 +- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt index b34451470..06dcb4195 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt @@ -10,7 +10,7 @@ import kotlin.io.path.readBytes enum class DevicePlatform { ANDROID, IOS } -enum class DeviceButton { HOME, BACK, POWER, VOLUME_UP, VOLUME_DOWN } +enum class DeviceButton { HOME, BACK, POWER, VOLUME_UP, VOLUME_DOWN, BACKSPACE, ENTER } /** One connected Android emulator/device or booted iOS simulator. */ data class MirrorDevice( @@ -61,6 +61,8 @@ internal class AndroidDeviceController( DeviceButton.POWER -> "KEYCODE_POWER" DeviceButton.VOLUME_UP -> "KEYCODE_VOLUME_UP" DeviceButton.VOLUME_DOWN -> "KEYCODE_VOLUME_DOWN" + DeviceButton.BACKSPACE -> "KEYCODE_DEL" + DeviceButton.ENTER -> "KEYCODE_ENTER" } runCommandChecked(adbPath, "-s", serial, "shell", "input", "keyevent", keycode) } @@ -114,19 +116,31 @@ internal class IosDeviceController( } override suspend fun pressButton(button: DeviceButton) { - val idbButton = when (button) { - DeviceButton.HOME -> "HOME" + when (button) { + DeviceButton.HOME -> pressHardwareButton("HOME") - DeviceButton.POWER -> "LOCK" + DeviceButton.POWER -> pressHardwareButton("LOCK") + + // Keyboard keys go through `ui key` with USB HID usage codes. + DeviceButton.BACKSPACE -> pressKey(hidCode = 42) + + DeviceButton.ENTER -> pressKey(hidCode = 40) DeviceButton.BACK -> throw DeviceControlException("iOS has no BACK button") DeviceButton.VOLUME_UP, DeviceButton.VOLUME_DOWN -> throw DeviceControlException("volume buttons are not controllable on the iOS simulator") } + } + + private suspend fun pressHardwareButton(idbButton: String) { runCommandChecked(requireIdb(), "ui", "button", "--udid", udid, idbButton) } + private suspend fun pressKey(hidCode: Int) { + runCommandChecked(requireIdb(), "ui", "key", "--udid", udid, "$hidCode") + } + override suspend fun inputText(text: String) { runCommandChecked(requireIdb(), "ui", "text", "--udid", udid, text) } diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt index 54bc76ad1..1e7816d0e 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt @@ -156,7 +156,7 @@ private fun ControlButtonsRow( ) { Button(onClick = onSaveScreenshot, enabled = enabled) { Text("Screenshot") } val buttons = when (platform) { - DevicePlatform.IOS -> listOf(DeviceButton.HOME, DeviceButton.POWER) + DevicePlatform.IOS -> listOf(DeviceButton.HOME, DeviceButton.POWER, DeviceButton.BACKSPACE, DeviceButton.ENTER) else -> DeviceButton.entries } buttons.forEach { button -> @@ -168,6 +168,8 @@ private fun ControlButtonsRow( DeviceButton.POWER -> "Power" DeviceButton.VOLUME_UP -> "Vol +" DeviceButton.VOLUME_DOWN -> "Vol -" + DeviceButton.BACKSPACE -> "⌫" + DeviceButton.ENTER -> "⏎" }, ) } diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/PressButtonCommand.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/PressButtonCommand.kt index 6c99a52ed..2319b723d 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/PressButtonCommand.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/PressButtonCommand.kt @@ -10,7 +10,7 @@ internal class PressButtonCommand( private val resolveDevice: (deviceId: String?) -> MirrorDevice, ) : JetWhaleMcpCommand() { override val name = "$TOOL_PREFIX.pressButton" - override val description = "Presses a hardware button on the device. Android supports all buttons; the iOS simulator supports HOME and POWER (BACK and volume buttons fail there)." + override val description = "Presses a hardware button or keyboard key on the device. BACKSPACE deletes one character in the focused field and ENTER confirms it. Android supports all buttons; the iOS simulator supports HOME, POWER, BACKSPACE, and ENTER (BACK and volume buttons fail there)." private val deviceId by stringOrNull("Target device id from $TOOL_PREFIX.listDevices; omitted = the device selected in the mirror UI.") private val button by enum("The hardware button to press.", DeviceButton.entries) From 0a34907b560730273fdfd025dad6a114759ee201 Mon Sep 17 00:00:00 2001 From: kitakkun <48154936+kitakkun@users.noreply.github.com> Date: Sun, 19 Jul 2026 06:06:44 +0900 Subject: [PATCH 05/17] feat(mirror): add screen recording and side-by-side multi-device mirroring Recording: Android records on-device via `adb shell screenrecord` (SIGINT to finalize the mp4, then pull), iOS via `xcrun simctl io recordVideo` (SIGINT to the local process). Exposed as a Record/Stop button and startRecording/stopRecording MCP tools; one recording at a time. Mirroring now runs one capture loop per connected device and shows all devices side by side. Taps and swipes go to the frame under the cursor (and select that device); the button/text controls target the selected, highlighted device. --- .../plugins/mirror/host/DeviceController.kt | 55 ++++++ .../mirror/host/MirrorHostPluginFactory.kt | 103 +++++++---- .../plugins/mirror/host/MirrorMcpCommands.kt | 9 +- .../plugins/mirror/host/MirrorScreen.kt | 163 ++++++++++-------- .../mirror/host/StartRecordingCommand.kt | 27 +++ .../mirror/host/StopRecordingCommand.kt | 28 +++ 6 files changed, 283 insertions(+), 102 deletions(-) create mode 100644 jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/StartRecordingCommand.kt create mode 100644 jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/StopRecordingCommand.kt diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt index 06dcb4195..a6a4a4d14 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt @@ -1,9 +1,13 @@ package com.kitakkun.jetwhale.plugins.mirror.host +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.withContext import kotlinx.serialization.json.Json import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive import java.io.File +import java.util.concurrent.TimeUnit import kotlin.io.path.createTempFile import kotlin.io.path.deleteIfExists import kotlin.io.path.readBytes @@ -35,6 +39,14 @@ interface DeviceController { suspend fun pressButton(button: DeviceButton) suspend fun inputText(text: String) + + /** Starts recording the screen into [outputFile] (mp4/mov). Stop it via the returned handle. */ + suspend fun startRecording(outputFile: File): DeviceRecording +} + +/** One in-progress screen recording; [stop] finalizes and returns the video file. */ +interface DeviceRecording { + suspend fun stop(): File } internal class AndroidDeviceController( @@ -75,6 +87,31 @@ internal class AndroidDeviceController( .replace(" ", "%s") runCommandChecked(adbPath, "-s", serial, "shell", "input", "text", escaped) } + + override suspend fun startRecording(outputFile: File): DeviceRecording { + // screenrecord writes on-device; the file is pulled after a clean SIGINT shutdown. + val remotePath = "/sdcard/${outputFile.name}" + val process = withContext(Dispatchers.IO) { + ProcessBuilder(adbPath, "-s", serial, "shell", "screenrecord", "--time-limit", "180", remotePath).start() + } + return object : DeviceRecording { + override suspend fun stop(): File = withContext(Dispatchers.IO) { + // SIGINT lets screenrecord finalize the mp4 moov atom; killing the local adb + // client instead would leave an unplayable file. Exit code is ignored: the + // process has already exited when the 180s time limit was hit. + runCommand(adbPath, "-s", serial, "shell", "pkill", "-INT", "screenrecord") + process.waitFor(10, TimeUnit.SECONDS) + // The device flushes the file asynchronously after the process exits. + delay(500) + try { + runCommandChecked(adbPath, "-s", serial, "pull", remotePath, outputFile.absolutePath) + } finally { + runCommand(adbPath, "-s", serial, "shell", "rm", "-f", remotePath) + } + outputFile + } + } + } } /** @@ -145,6 +182,24 @@ internal class IosDeviceController( runCommandChecked(requireIdb(), "ui", "text", "--udid", udid, text) } + override suspend fun startRecording(outputFile: File): DeviceRecording { + val process = withContext(Dispatchers.IO) { + ProcessBuilder("xcrun", "simctl", "io", udid, "recordVideo", "--codec=h264", "--force", outputFile.absolutePath).start() + } + return object : DeviceRecording { + override suspend fun stop(): File = withContext(Dispatchers.IO) { + // recordVideo finalizes the file only on SIGINT (its ctrl-c path); + // Process.destroy() sends SIGTERM, which leaves the video unplayable. + runCommand("kill", "-INT", "${process.pid()}") + if (!process.waitFor(15, TimeUnit.SECONDS)) process.destroyForcibly() + if (!outputFile.exists() || outputFile.length() == 0L) { + throw DeviceControlException("recording failed: ${outputFile.absolutePath} was not written") + } + outputFile + } + } + } + private fun requireIdb(): String = idbPath ?: throw DeviceControlException("iOS simulator input requires idb (https://fbidb.io). Install it with: brew install idb-companion && pipx install fb-idb") diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt index e0ad0dc0d..00e2dd359 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt @@ -3,9 +3,11 @@ package com.kitakkun.jetwhale.plugins.mirror.host import androidx.compose.runtime.Composable import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateListOf +import androidx.compose.runtime.mutableStateMapOf import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.setValue import androidx.compose.runtime.snapshots.SnapshotStateList +import androidx.compose.runtime.snapshots.SnapshotStateMap import androidx.compose.ui.graphics.ImageBitmap import androidx.compose.ui.graphics.toComposeImageBitmap import com.kitakkun.jetwhale.host.sdk.ExperimentalJetWhaleApi @@ -15,9 +17,13 @@ import com.kitakkun.jetwhale.host.sdk.JetWhaleHostPluginUi import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArgumentException import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpCapablePlugin import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpCommand +import kotlinx.coroutines.Job +import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.delay import kotlinx.coroutines.isActive import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock import org.jetbrains.skia.Image as SkiaImage // Instantiated by the host via the fully-qualified name declared in plugin-manifest.json. @@ -46,8 +52,16 @@ private class MirrorHostPlugin : private val devices: SnapshotStateList = mutableStateListOf() private var selectedDeviceId by mutableStateOf(null) private var mirroringEnabled by mutableStateOf(true) - private var latestFrame by mutableStateOf(null) + + // All connected devices are mirrored concurrently, one capture loop and frame slot each. + private val frames: SnapshotStateMap = mutableStateMapOf() + private val mirrorJobs = mutableMapOf() + private val refreshMutex = Mutex() + private var lastError by mutableStateOf(null) + private var activeRecording by mutableStateOf(null) + + private class ActiveRecording(val deviceId: String, val recording: DeviceRecording) private val selectedDevice: MirrorDevice? get() = devices.firstOrNull { it.id == selectedDeviceId } @@ -58,28 +72,9 @@ private class MirrorHostPlugin : delay(DEVICE_POLL_INTERVAL_MILLIS) } } - // The mirror is a screenshot poll loop: cheap, tool-free, and identical for both platforms. - pluginScope.launch { - while (isActive) { - val device = selectedDevice - if (mirroringEnabled && device != null) { - try { - val png = device.controller.captureScreenshot() - latestFrame = SkiaImage.makeFromEncoded(png).toComposeImageBitmap() - lastError = null - delay(FRAME_INTERVAL_MILLIS) - } catch (e: DeviceControlException) { - lastError = e.message - delay(FRAME_RETRY_INTERVAL_MILLIS) - } - } else { - delay(DEVICE_POLL_INTERVAL_MILLIS) - } - } - } } - private suspend fun refreshDevices(): List { + private suspend fun refreshDevices(): List = refreshMutex.withLock { val discovered = discovery.discoverDevices() devices.apply { clear() @@ -87,9 +82,36 @@ private class MirrorHostPlugin : } if (devices.none { it.id == selectedDeviceId }) { selectedDeviceId = devices.firstOrNull()?.id - latestFrame = null } - return discovered + val discoveredIds = discovered.map { it.id }.toSet() + mirrorJobs.keys.filter { it !in discoveredIds }.forEach { id -> + mirrorJobs.remove(id)?.cancel() + frames.remove(id) + } + discovered.forEach { device -> + if (mirrorJobs[device.id]?.isActive != true) { + mirrorJobs[device.id] = pluginScope.launch { mirrorLoop(device) } + } + } + discovered + } + + // The mirror is a screenshot poll loop: cheap, tool-free, and identical for both platforms. + private suspend fun mirrorLoop(device: MirrorDevice) { + while (currentCoroutineContext().isActive) { + if (!mirroringEnabled) { + delay(DEVICE_POLL_INTERVAL_MILLIS) + continue + } + try { + val png = device.controller.captureScreenshot() + frames[device.id] = SkiaImage.makeFromEncoded(png).toComposeImageBitmap() + delay(FRAME_INTERVAL_MILLIS) + } catch (e: DeviceControlException) { + lastError = "${device.name}: ${e.message}" + delay(FRAME_RETRY_INTERVAL_MILLIS) + } + } } // Runs a device-control action from a UI callback, surfacing failures in the status bar. @@ -109,22 +131,27 @@ private class MirrorHostPlugin : MirrorScreen( devices = devices, selectedDeviceId = selectedDeviceId, - onSelectDevice = { id -> - selectedDeviceId = id - latestFrame = null - }, - frame = latestFrame, + onSelectDevice = { id -> selectedDeviceId = id }, + frames = frames, mirroringEnabled = mirroringEnabled, onToggleMirroring = { mirroringEnabled = it }, errorMessage = lastError, onRefreshDevices = { pluginScope.launch { refreshDevices() } }, - onTap = { x, y -> selectedDevice?.let { runControl { it.controller.tap(x, y) } } }, - onSwipe = { fromX, fromY, toX, toY -> - selectedDevice?.let { runControl { it.controller.swipe(fromX, fromY, toX, toY, durationMillis = 200) } } + onTap = { device, x, y -> runControl { device.controller.tap(x, y) } }, + onSwipe = { device, fromX, fromY, toX, toY -> + runControl { device.controller.swipe(fromX, fromY, toX, toY, durationMillis = 200) } }, onPressButton = { button -> selectedDevice?.let { runControl { it.controller.pressButton(button) } } }, onInputText = { text -> selectedDevice?.let { runControl { it.controller.inputText(text) } } }, onSaveScreenshot = { selectedDevice?.let { runControl { saveScreenshotToDisk(it) } } }, + isRecording = activeRecording != null, + onToggleRecording = { + if (activeRecording != null) { + runControl { lastError = "Saved recording: ${stopRecording().absolutePath}" } + } else { + selectedDevice?.let { runControl { startRecording(it) } } + } + }, ) } @@ -135,6 +162,18 @@ private class MirrorHostPlugin : lastError = "Saved screenshot: ${file.absolutePath}" } + private suspend fun startRecording(device: MirrorDevice) { + if (activeRecording != null) throw DeviceControlException("a recording is already in progress; stop it first") + val recording = device.controller.startRecording(recordingFile(device)) + activeRecording = ActiveRecording(deviceId = device.id, recording = recording) + } + + private suspend fun stopRecording(): java.io.File { + val active = activeRecording ?: throw DeviceControlException("no recording in progress") + activeRecording = null + return active.recording.stop() + } + // ------------------------------------------------------------------------- // JetWhaleMcpCapablePlugin // ------------------------------------------------------------------------- @@ -157,5 +196,7 @@ private class MirrorHostPlugin : SwipeCommand(resolveDevice = ::resolveDevice), PressButtonCommand(resolveDevice = ::resolveDevice), InputTextCommand(resolveDevice = ::resolveDevice), + StartRecordingCommand(resolveDevice = ::resolveDevice, startRecording = ::startRecording), + StopRecordingCommand(stopRecording = ::stopRecording), ) } diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorMcpCommands.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorMcpCommands.kt index 1ce03375d..4e8649668 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorMcpCommands.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorMcpCommands.kt @@ -11,8 +11,13 @@ internal const val TOOL_PREFIX = "com.kitakkun.jetwhale.mirror" internal fun okJson(): String = buildJsonObject { put("ok", true) }.toString() /** Where saved device screenshots land, both for the UI button and the MCP tool. */ -internal fun screenshotFile(device: MirrorDevice): File { +internal fun screenshotFile(device: MirrorDevice): File = captureFile(device, extension = "png") + +/** Where saved screen recordings land, both for the UI button and the MCP tool. */ +internal fun recordingFile(device: MirrorDevice): File = captureFile(device, extension = "mp4") + +private fun captureFile(device: MirrorDevice, extension: String): File { val dir = File(System.getProperty("java.io.tmpdir"), "jetwhale-mirror").apply { mkdirs() } val timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss-SSS")) - return File(dir, "${device.platform.name.lowercase()}-$timestamp.png") + return File(dir, "${device.platform.name.lowercase()}-$timestamp.$extension") } diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt index 1e7816d0e..a4a904f1b 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt @@ -1,6 +1,7 @@ package com.kitakkun.jetwhale.plugins.mirror.host import androidx.compose.foundation.Image +import androidx.compose.foundation.border import androidx.compose.foundation.gestures.detectDragGestures import androidx.compose.foundation.gestures.detectTapGestures import androidx.compose.foundation.horizontalScroll @@ -9,16 +10,14 @@ import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.aspectRatio +import androidx.compose.foundation.layout.fillMaxHeight import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.padding import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.material3.Button -import androidx.compose.material3.DropdownMenuItem import androidx.compose.material3.ExperimentalMaterial3Api -import androidx.compose.material3.ExposedDropdownMenuBox -import androidx.compose.material3.ExposedDropdownMenuDefaults -import androidx.compose.material3.IconButton import androidx.compose.material3.MaterialTheme import androidx.compose.material3.OutlinedButton import androidx.compose.material3.OutlinedTextField @@ -34,6 +33,7 @@ import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.graphics.Color import androidx.compose.ui.graphics.ImageBitmap import androidx.compose.ui.input.pointer.pointerInput import androidx.compose.ui.unit.dp @@ -44,16 +44,18 @@ internal fun MirrorScreen( devices: List, selectedDeviceId: String?, onSelectDevice: (String) -> Unit, - frame: ImageBitmap?, + frames: Map, mirroringEnabled: Boolean, onToggleMirroring: (Boolean) -> Unit, errorMessage: String?, onRefreshDevices: () -> Unit, - onTap: (x: Int, y: Int) -> Unit, - onSwipe: (fromX: Int, fromY: Int, toX: Int, toY: Int) -> Unit, + onTap: (device: MirrorDevice, x: Int, y: Int) -> Unit, + onSwipe: (device: MirrorDevice, fromX: Int, fromY: Int, toX: Int, toY: Int) -> Unit, onPressButton: (DeviceButton) -> Unit, onInputText: (String) -> Unit, onSaveScreenshot: () -> Unit, + isRecording: Boolean, + onToggleRecording: () -> Unit, ) { val selectedDevice = devices.firstOrNull { it.id == selectedDeviceId } Scaffold( @@ -66,77 +68,71 @@ internal fun MirrorScreen( .padding(16.dp), verticalArrangement = Arrangement.spacedBy(12.dp), ) { - DeviceSelectorRow( - devices = devices, - selectedDevice = selectedDevice, - onSelectDevice = onSelectDevice, - onRefreshDevices = onRefreshDevices, + MirrorToolbar( mirroringEnabled = mirroringEnabled, onToggleMirroring = onToggleMirroring, + onRefreshDevices = onRefreshDevices, ) ControlButtonsRow( platform = selectedDevice?.platform, enabled = selectedDevice != null, onPressButton = onPressButton, onSaveScreenshot = onSaveScreenshot, + isRecording = isRecording, + onToggleRecording = onToggleRecording, ) TextInputRow(enabled = selectedDevice != null, onInputText = onInputText) errorMessage?.let { Text(it, style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.error) } - MirrorFrame( - frame = frame, - hasDevice = selectedDevice != null, - onTap = onTap, - onSwipe = onSwipe, - modifier = Modifier - .fillMaxWidth() - .weight(1f), - ) + if (devices.isEmpty()) { + Box( + modifier = Modifier + .fillMaxWidth() + .weight(1f), + contentAlignment = Alignment.Center, + ) { + Text( + "Boot an Android emulator or iOS simulator to start mirroring.", + style = MaterialTheme.typography.bodyMedium, + ) + } + } else { + // All connected devices side by side; the highlighted one receives the + // button/text controls, while taps and swipes go to the frame under the cursor. + Row( + modifier = Modifier + .fillMaxWidth() + .weight(1f) + .horizontalScroll(rememberScrollState()), + horizontalArrangement = Arrangement.spacedBy(12.dp), + ) { + devices.forEach { device -> + DeviceMirrorPane( + device = device, + frame = frames[device.id], + isSelected = device.id == selectedDeviceId, + onSelect = { onSelectDevice(device.id) }, + onTap = onTap, + onSwipe = onSwipe, + ) + } + } + } } } } -@OptIn(ExperimentalMaterial3Api::class) @Composable -private fun DeviceSelectorRow( - devices: List, - selectedDevice: MirrorDevice?, - onSelectDevice: (String) -> Unit, - onRefreshDevices: () -> Unit, +private fun MirrorToolbar( mirroringEnabled: Boolean, onToggleMirroring: (Boolean) -> Unit, + onRefreshDevices: () -> Unit, ) { Row( verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp), ) { - var expanded by remember { mutableStateOf(false) } - ExposedDropdownMenuBox( - expanded = expanded, - onExpandedChange = { expanded = it }, - modifier = Modifier.weight(1f), - ) { - OutlinedTextField( - value = selectedDevice?.let { "${it.name} (${it.platform})" } ?: "No device", - onValueChange = {}, - readOnly = true, - singleLine = true, - trailingIcon = { ExposedDropdownMenuDefaults.TrailingIcon(expanded = expanded) }, - modifier = Modifier.menuAnchor().fillMaxWidth(), - ) - ExposedDropdownMenu(expanded = expanded, onDismissRequest = { expanded = false }) { - devices.forEach { device -> - DropdownMenuItem( - text = { Text("${device.name} (${device.platform})") }, - onClick = { - onSelectDevice(device.id) - expanded = false - }, - ) - } - } - } OutlinedButton(onClick = onRefreshDevices) { Text("Refresh") } Text("Mirror", style = MaterialTheme.typography.labelMedium) Switch(checked = mirroringEnabled, onCheckedChange = onToggleMirroring) @@ -149,12 +145,17 @@ private fun ControlButtonsRow( enabled: Boolean, onPressButton: (DeviceButton) -> Unit, onSaveScreenshot: () -> Unit, + isRecording: Boolean, + onToggleRecording: () -> Unit, ) { Row( modifier = Modifier.horizontalScroll(rememberScrollState()), horizontalArrangement = Arrangement.spacedBy(8.dp), ) { Button(onClick = onSaveScreenshot, enabled = enabled) { Text("Screenshot") } + Button(onClick = onToggleRecording, enabled = enabled || isRecording) { + Text(if (isRecording) "Stop Rec" else "Record") + } val buttons = when (platform) { DevicePlatform.IOS -> listOf(DeviceButton.HOME, DeviceButton.POWER, DeviceButton.BACKSPACE, DeviceButton.ENTER) else -> DeviceButton.entries @@ -191,7 +192,7 @@ private fun TextInputRow( value = text, onValueChange = { text = it }, singleLine = true, - placeholder = { Text("Type text to send to the device") }, + placeholder = { Text("Type text to send to the selected device") }, modifier = Modifier.weight(1f), ) Button( @@ -207,39 +208,62 @@ private fun TextInputRow( } @Composable -private fun MirrorFrame( +private fun DeviceMirrorPane( + device: MirrorDevice, frame: ImageBitmap?, - hasDevice: Boolean, - onTap: (x: Int, y: Int) -> Unit, - onSwipe: (fromX: Int, fromY: Int, toX: Int, toY: Int) -> Unit, - modifier: Modifier = Modifier, + isSelected: Boolean, + onSelect: () -> Unit, + onTap: (device: MirrorDevice, x: Int, y: Int) -> Unit, + onSwipe: (device: MirrorDevice, fromX: Int, fromY: Int, toX: Int, toY: Int) -> Unit, ) { - Box(modifier = modifier, contentAlignment = Alignment.Center) { + Column( + modifier = Modifier.fillMaxHeight(), + verticalArrangement = Arrangement.spacedBy(4.dp), + horizontalAlignment = Alignment.CenterHorizontally, + ) { + Text( + text = "${device.name} (${device.platform})", + style = MaterialTheme.typography.labelMedium, + color = if (isSelected) MaterialTheme.colorScheme.primary else MaterialTheme.colorScheme.onSurfaceVariant, + ) if (frame == null) { - Text( - if (hasDevice) "Waiting for the first frame…" else "Boot an Android emulator or iOS simulator to start mirroring.", - style = MaterialTheme.typography.bodyMedium, - ) - return@Box + Box( + modifier = Modifier + .weight(1f) + .aspectRatio(9f / 16f) + .border(1.dp, MaterialTheme.colorScheme.outlineVariant, RoundedCornerShape(4.dp)), + contentAlignment = Alignment.Center, + ) { + Text("Waiting…", style = MaterialTheme.typography.bodySmall) + } + return@Column } // aspectRatio makes the image composable's bounds coincide exactly with the drawn frame, // so pointer offsets scale linearly to device pixels. Image( bitmap = frame, - contentDescription = "Mirrored device screen", + contentDescription = "Mirrored screen of ${device.name}", modifier = Modifier + .weight(1f) .aspectRatio(frame.width.toFloat() / frame.height.toFloat()) - .pointerInput(frame.width, frame.height) { + .border( + width = 2.dp, + color = if (isSelected) MaterialTheme.colorScheme.primary else Color.Transparent, + shape = RoundedCornerShape(4.dp), + ) + .pointerInput(device.id, frame.width, frame.height) { detectTapGestures { offset -> + onSelect() val scale = frame.width.toFloat() / size.width - onTap((offset.x * scale).toInt(), (offset.y * scale).toInt()) + onTap(device, (offset.x * scale).toInt(), (offset.y * scale).toInt()) } } - .pointerInput(frame.width, frame.height) { + .pointerInput(device.id, frame.width, frame.height) { var dragStart = Offset.Zero var dragEnd = Offset.Zero detectDragGestures( onDragStart = { offset -> + onSelect() dragStart = offset dragEnd = offset }, @@ -247,6 +271,7 @@ private fun MirrorFrame( onDragEnd = { val scale = frame.width.toFloat() / size.width onSwipe( + device, (dragStart.x * scale).toInt(), (dragStart.y * scale).toInt(), (dragEnd.x * scale).toInt(), diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/StartRecordingCommand.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/StartRecordingCommand.kt new file mode 100644 index 000000000..fea89c332 --- /dev/null +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/StartRecordingCommand.kt @@ -0,0 +1,27 @@ +package com.kitakkun.jetwhale.plugins.mirror.host + +import com.kitakkun.jetwhale.host.sdk.ExperimentalJetWhaleApi +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArgumentException +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArguments +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpCommand + +@OptIn(ExperimentalJetWhaleApi::class) +internal class StartRecordingCommand( + private val resolveDevice: (deviceId: String?) -> MirrorDevice, + private val startRecording: suspend (MirrorDevice) -> Unit, +) : JetWhaleMcpCommand() { + override val name = "$TOOL_PREFIX.startRecording" + override val description = "Starts recording the device screen to an mp4 file. Only one recording can run at a time; finish with $TOOL_PREFIX.stopRecording (Android stops automatically after 180 seconds)." + + private val deviceId by stringOrNull("Target device id from $TOOL_PREFIX.listDevices; omitted = the device selected in the mirror UI.") + + override suspend fun execute(arguments: JetWhaleMcpArguments): String { + val device = resolveDevice(arguments[deviceId]) + try { + startRecording(device) + } catch (e: DeviceControlException) { + throw JetWhaleMcpArgumentException(e.message ?: "recording start failed") + } + return okJson() + } +} diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/StopRecordingCommand.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/StopRecordingCommand.kt new file mode 100644 index 000000000..989f12a9d --- /dev/null +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/StopRecordingCommand.kt @@ -0,0 +1,28 @@ +package com.kitakkun.jetwhale.plugins.mirror.host + +import com.kitakkun.jetwhale.host.sdk.ExperimentalJetWhaleApi +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArgumentException +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArguments +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpCommand +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put +import java.io.File + +@OptIn(ExperimentalJetWhaleApi::class) +internal class StopRecordingCommand( + private val stopRecording: suspend () -> File, +) : JetWhaleMcpCommand() { + override val name = "$TOOL_PREFIX.stopRecording" + override val description = "Stops the in-progress screen recording started by $TOOL_PREFIX.startRecording and returns the absolute path of the finished mp4 file." + + override suspend fun execute(arguments: JetWhaleMcpArguments): String { + val file = try { + stopRecording() + } catch (e: DeviceControlException) { + throw JetWhaleMcpArgumentException(e.message ?: "recording stop failed") + } + return buildJsonObject { + put("path", file.absolutePath) + }.toString() + } +} From e16f966363b456fb69f8651e4ae00478fe4f825e Mon Sep 17 00:00:00 2001 From: kitakkun <48154936+kitakkun@users.noreply.github.com> Date: Sun, 19 Jul 2026 13:35:46 +0900 Subject: [PATCH 06/17] perf(mirror): throttle unselected devices and skip unchanged frames Every connected device was captured and PNG-decoded at full rate, which scaled CPU linearly with device count even on static screens. Unselected devices now refresh at 500ms, and a frame whose PNG bytes match the previous capture skips decode and recomposition entirely. --- .../mirror/host/MirrorHostPluginFactory.kt | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt index 00e2dd359..79a65214d 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt @@ -34,9 +34,13 @@ class MirrorHostPluginFactory : JetWhaleHostPluginFactory { private const val DEVICE_POLL_INTERVAL_MILLIS = 3_000L -// The frame loop is paced by the screenshot capture itself (~100-300ms per adb/simctl round -// trip); this small gap only keeps a failing capture from busy-looping. -private const val FRAME_INTERVAL_MILLIS = 16L +// The selected device's frame loop is paced by the screenshot capture itself (~100-300ms per +// adb/simctl round trip); this small gap only keeps a failing capture from busy-looping. +private const val SELECTED_FRAME_INTERVAL_MILLIS = 16L + +// Unselected devices stay visible but refresh slowly, so mirroring many devices at once does +// not multiply the process-spawn/decode cost. +private const val BACKGROUND_FRAME_INTERVAL_MILLIS = 500L // Back off after a failed capture so a dead device doesn't spam error processes. private const val FRAME_RETRY_INTERVAL_MILLIS = 1_000L @@ -98,6 +102,7 @@ private class MirrorHostPlugin : // The mirror is a screenshot poll loop: cheap, tool-free, and identical for both platforms. private suspend fun mirrorLoop(device: MirrorDevice) { + var lastPngHash = 0 while (currentCoroutineContext().isActive) { if (!mirroringEnabled) { delay(DEVICE_POLL_INTERVAL_MILLIS) @@ -105,8 +110,14 @@ private class MirrorHostPlugin : } try { val png = device.controller.captureScreenshot() - frames[device.id] = SkiaImage.makeFromEncoded(png).toComposeImageBitmap() - delay(FRAME_INTERVAL_MILLIS) + // A static screen produces byte-identical PNGs; skip the decode and the + // recomposition it would trigger. + val pngHash = png.contentHashCode() + if (pngHash != lastPngHash) { + frames[device.id] = SkiaImage.makeFromEncoded(png).toComposeImageBitmap() + lastPngHash = pngHash + } + delay(if (device.id == selectedDeviceId) SELECTED_FRAME_INTERVAL_MILLIS else BACKGROUND_FRAME_INTERVAL_MILLIS) } catch (e: DeviceControlException) { lastError = "${device.name}: ${e.message}" delay(FRAME_RETRY_INTERVAL_MILLIS) From 0efa332e915704dacf36a49603cd0cf234de8651 Mon Sep 17 00:00:00 2001 From: kitakkun <48154936+kitakkun@users.noreply.github.com> Date: Sun, 19 Jul 2026 13:38:20 +0900 Subject: [PATCH 07/17] perf(mirror): repaint frames in the draw phase instead of recomposing Frame pixels were read during composition, so every captured frame recomposed and re-laid-out the whole screen. Composition now depends only on the frame dimensions (derivedStateOf, changes on rotation at most); pixels are read inside drawBehind, so a new frame invalidates just that pane's draw pass. --- .../plugins/mirror/host/MirrorScreen.kt | 46 +++++++++++++------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt index a4a904f1b..8002fad1e 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt @@ -1,6 +1,5 @@ package com.kitakkun.jetwhale.plugins.mirror.host -import androidx.compose.foundation.Image import androidx.compose.foundation.border import androidx.compose.foundation.gestures.detectDragGestures import androidx.compose.foundation.gestures.detectTapGestures @@ -26,17 +25,21 @@ import androidx.compose.material3.Switch import androidx.compose.material3.Text import androidx.compose.material3.TopAppBar import androidx.compose.runtime.Composable +import androidx.compose.runtime.derivedStateOf import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.drawBehind import androidx.compose.ui.geometry.Offset import androidx.compose.ui.graphics.Color import androidx.compose.ui.graphics.ImageBitmap import androidx.compose.ui.input.pointer.pointerInput +import androidx.compose.ui.unit.IntSize import androidx.compose.ui.unit.dp +import kotlin.math.roundToInt @OptIn(ExperimentalMaterial3Api::class) @Composable @@ -110,7 +113,7 @@ internal fun MirrorScreen( devices.forEach { device -> DeviceMirrorPane( device = device, - frame = frames[device.id], + frames = frames, isSelected = device.id == selectedDeviceId, onSelect = { onSelectDevice(device.id) }, onTap = onTap, @@ -210,12 +213,18 @@ private fun TextInputRow( @Composable private fun DeviceMirrorPane( device: MirrorDevice, - frame: ImageBitmap?, + frames: Map, isSelected: Boolean, onSelect: () -> Unit, onTap: (device: MirrorDevice, x: Int, y: Int) -> Unit, onSwipe: (device: MirrorDevice, fromX: Int, fromY: Int, toX: Int, toY: Int) -> Unit, ) { + // Composition only depends on the frame's dimensions (via derivedStateOf), which change on + // rotation at most; the pixels are read inside the draw phase, so a new frame triggers just a + // repaint of this pane instead of recomposing and re-laying-out the whole screen. + val frameSize by remember(device.id) { + derivedStateOf { frames[device.id]?.let { IntSize(it.width, it.height) } } + } Column( modifier = Modifier.fillMaxHeight(), verticalArrangement = Arrangement.spacedBy(4.dp), @@ -226,7 +235,8 @@ private fun DeviceMirrorPane( style = MaterialTheme.typography.labelMedium, color = if (isSelected) MaterialTheme.colorScheme.primary else MaterialTheme.colorScheme.onSurfaceVariant, ) - if (frame == null) { + val size = frameSize + if (size == null) { Box( modifier = Modifier .weight(1f) @@ -238,27 +248,34 @@ private fun DeviceMirrorPane( } return@Column } - // aspectRatio makes the image composable's bounds coincide exactly with the drawn frame, - // so pointer offsets scale linearly to device pixels. - Image( - bitmap = frame, - contentDescription = "Mirrored screen of ${device.name}", + // aspectRatio makes this pane's bounds coincide exactly with the drawn frame, so pointer + // offsets scale linearly to device pixels. + Box( modifier = Modifier .weight(1f) - .aspectRatio(frame.width.toFloat() / frame.height.toFloat()) + .aspectRatio(size.width.toFloat() / size.height.toFloat()) + .drawBehind { + frames[device.id]?.let { frame -> + drawImage( + image = frame, + dstSize = IntSize(this.size.width.roundToInt(), this.size.height.roundToInt()), + ) + } + } .border( width = 2.dp, color = if (isSelected) MaterialTheme.colorScheme.primary else Color.Transparent, shape = RoundedCornerShape(4.dp), ) - .pointerInput(device.id, frame.width, frame.height) { + .pointerInput(device.id) { detectTapGestures { offset -> onSelect() - val scale = frame.width.toFloat() / size.width + val frame = frames[device.id] ?: return@detectTapGestures + val scale = frame.width.toFloat() / this.size.width onTap(device, (offset.x * scale).toInt(), (offset.y * scale).toInt()) } } - .pointerInput(device.id, frame.width, frame.height) { + .pointerInput(device.id) { var dragStart = Offset.Zero var dragEnd = Offset.Zero detectDragGestures( @@ -269,7 +286,8 @@ private fun DeviceMirrorPane( }, onDrag = { change, _ -> dragEnd = change.position }, onDragEnd = { - val scale = frame.width.toFloat() / size.width + val frame = frames[device.id] ?: return@detectDragGestures + val scale = frame.width.toFloat() / this.size.width onSwipe( device, (dragStart.x * scale).toInt(), From 9470135594b1d9d6436395b57dd6df160615a06f Mon Sep 17 00:00:00 2001 From: kitakkun <48154936+kitakkun@users.noreply.github.com> Date: Sun, 19 Jul 2026 13:43:22 +0900 Subject: [PATCH 08/17] fix(mirror): address Copilot review findings - Drain stderr with a coroutine instead of spawning a Thread per command - Escape literal % in Android inputText (input text uses % as escape prefix) - Validate tap/swipe coordinates (>= 0) and swipe duration (> 0) upfront with caller-facing errors - Add unit tests for MCP command forwarding, defaults, validation, and JSON shape --- .../plugins/mirror/host/DeviceController.kt | 4 +- .../jetwhale/plugins/mirror/host/Shell.kt | 9 +- .../plugins/mirror/host/SwipeCommand.kt | 14 ++- .../plugins/mirror/host/TapCommand.kt | 5 +- .../mirror/host/MirrorMcpCommandsTest.kt | 112 ++++++++++++++++++ 5 files changed, 131 insertions(+), 13 deletions(-) create mode 100644 jetwhale-plugins/mirror/host/src/test/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorMcpCommandsTest.kt diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt index a6a4a4d14..67c79313a 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt @@ -81,9 +81,11 @@ internal class AndroidDeviceController( override suspend fun inputText(text: String) { // `adb shell` re-parses arguments through the device shell, so shell metacharacters must - // be escaped; `input text` additionally requires spaces encoded as %s. + // be escaped; `input text` additionally treats % as an escape prefix (space is sent as + // %s), so literal percent signs must be escaped before spaces are encoded. val escaped = text .replace(Regex("""([\\'"`$&*()\[\]{}+|<>;?~#!])"""), """\\$1""") + .replace("%", "\\%") .replace(" ", "%s") runCommandChecked(adbPath, "-s", serial, "shell", "input", "text", escaped) } diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/Shell.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/Shell.kt index 38fa7da06..87f7856e8 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/Shell.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/Shell.kt @@ -1,6 +1,7 @@ package com.kitakkun.jetwhale.plugins.mirror.host import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async import kotlinx.coroutines.withContext import java.io.File @@ -26,14 +27,10 @@ internal suspend fun runCommand(vararg command: String): CommandResult = withCon throw DeviceControlException("failed to launch '${command.first()}': ${e.message}", e) } // Drain stderr concurrently so neither pipe can fill up and deadlock the process. - var stderr = "" - val stderrThread = Thread { - stderr = process.errorStream.bufferedReader().readText() - }.apply { start() } + val stderr = async { process.errorStream.bufferedReader().readText() } val stdout = process.inputStream.readBytes() val exitCode = process.waitFor() - stderrThread.join() - CommandResult(exitCode = exitCode, stdout = stdout, stderr = stderr) + CommandResult(exitCode = exitCode, stdout = stdout, stderr = stderr.await()) } /** Runs [command] and returns stdout text, throwing [DeviceControlException] on a non-zero exit. */ diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/SwipeCommand.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/SwipeCommand.kt index 6f17090de..b94877446 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/SwipeCommand.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/SwipeCommand.kt @@ -21,13 +21,17 @@ internal class SwipeCommand( override suspend fun execute(arguments: JetWhaleMcpArguments): String { val device = resolveDevice(arguments[deviceId]) + val coordinates = listOf(arguments[fromX], arguments[fromY], arguments[toX], arguments[toY]) + if (coordinates.any { it < 0 }) throw JetWhaleMcpArgumentException("coordinates must be >= 0 (got $coordinates)") + val durationMillis = arguments[durationMillis] ?: 300 + if (durationMillis <= 0) throw JetWhaleMcpArgumentException("durationMillis must be > 0 (got $durationMillis)") try { device.controller.swipe( - fromX = arguments[fromX], - fromY = arguments[fromY], - toX = arguments[toX], - toY = arguments[toY], - durationMillis = arguments[durationMillis] ?: 300, + fromX = coordinates[0], + fromY = coordinates[1], + toX = coordinates[2], + toY = coordinates[3], + durationMillis = durationMillis, ) } catch (e: DeviceControlException) { throw JetWhaleMcpArgumentException(e.message ?: "swipe failed") diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/TapCommand.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/TapCommand.kt index 3dc132dcc..27edfa946 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/TapCommand.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/TapCommand.kt @@ -18,8 +18,11 @@ internal class TapCommand( override suspend fun execute(arguments: JetWhaleMcpArguments): String { val device = resolveDevice(arguments[deviceId]) + val x = arguments[x] + val y = arguments[y] + if (x < 0 || y < 0) throw JetWhaleMcpArgumentException("coordinates must be >= 0 (got x=$x, y=$y)") try { - device.controller.tap(arguments[x], arguments[y]) + device.controller.tap(x, y) } catch (e: DeviceControlException) { throw JetWhaleMcpArgumentException(e.message ?: "tap failed") } diff --git a/jetwhale-plugins/mirror/host/src/test/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorMcpCommandsTest.kt b/jetwhale-plugins/mirror/host/src/test/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorMcpCommandsTest.kt new file mode 100644 index 000000000..384e6f0a4 --- /dev/null +++ b/jetwhale-plugins/mirror/host/src/test/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorMcpCommandsTest.kt @@ -0,0 +1,112 @@ +package com.kitakkun.jetwhale.plugins.mirror.host + +import com.kitakkun.jetwhale.host.sdk.ExperimentalJetWhaleApi +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArgumentException +import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArguments +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import java.io.File +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +@OptIn(ExperimentalJetWhaleApi::class) +class MirrorMcpCommandsTest { + private class FakeDeviceController : DeviceController { + val taps = mutableListOf>() + val swipes = mutableListOf>() + + override suspend fun captureScreenshot(): ByteArray = ByteArray(0) + + override suspend fun tap(x: Int, y: Int) { + taps += x to y + } + + override suspend fun swipe(fromX: Int, fromY: Int, toX: Int, toY: Int, durationMillis: Int) { + swipes += listOf(fromX, fromY, toX, toY, durationMillis) + } + + override suspend fun pressButton(button: DeviceButton) = Unit + + override suspend fun inputText(text: String) = Unit + + override suspend fun startRecording(outputFile: File): DeviceRecording = object : DeviceRecording { + override suspend fun stop(): File = outputFile + } + } + + private val controller = FakeDeviceController() + private val device = MirrorDevice(id = "emulator-5554", name = "Pixel 7", platform = DevicePlatform.ANDROID, controller = controller) + + @Test + fun `tap forwards coordinates to the device`() = runBlocking { + val result = TapCommand(resolveDevice = { device }).execute(JetWhaleMcpArguments(mapOf("x" to "10", "y" to "20"))) + assertEquals(listOf(10 to 20), controller.taps) + assertTrue(Json.parseToJsonElement(result).jsonObject["ok"]!!.jsonPrimitive.content.toBoolean()) + } + + @Test + fun `tap rejects negative coordinates`(): Unit = runBlocking { + assertFailsWith { + TapCommand(resolveDevice = { device }).execute(JetWhaleMcpArguments(mapOf("x" to "-1", "y" to "20"))) + } + assertTrue(controller.taps.isEmpty()) + } + + @Test + fun `swipe defaults duration to 300ms`() = runBlocking { + SwipeCommand(resolveDevice = { device }).execute( + JetWhaleMcpArguments(mapOf("fromX" to "0", "fromY" to "1", "toX" to "2", "toY" to "3")), + ) + assertEquals(listOf(listOf(0, 1, 2, 3, 300)), controller.swipes) + } + + @Test + fun `swipe rejects negative coordinates`(): Unit = runBlocking { + assertFailsWith { + SwipeCommand(resolveDevice = { device }).execute( + JetWhaleMcpArguments(mapOf("fromX" to "0", "fromY" to "-5", "toX" to "2", "toY" to "3")), + ) + } + assertTrue(controller.swipes.isEmpty()) + } + + @Test + fun `swipe rejects non-positive duration`(): Unit = runBlocking { + assertFailsWith { + SwipeCommand(resolveDevice = { device }).execute( + JetWhaleMcpArguments(mapOf("fromX" to "0", "fromY" to "1", "toX" to "2", "toY" to "3", "durationMillis" to "0")), + ) + } + assertTrue(controller.swipes.isEmpty()) + } + + @Test + fun `listDevices reports ids, platforms, and the selection`() = runBlocking { + val result = ListDevicesCommand(refreshDevices = { listOf(device) }, selectedDeviceId = { device.id }).execute(JetWhaleMcpArguments(emptyMap())) + val devices = Json.parseToJsonElement(result).jsonObject["devices"]!!.jsonArray + val entry = devices.single().jsonObject + assertEquals("emulator-5554", entry["deviceId"]!!.jsonPrimitive.content) + assertEquals("ANDROID", entry["platform"]!!.jsonPrimitive.content) + assertEquals("true", entry["selected"]!!.jsonPrimitive.content) + } + + @Test + fun `stopRecording returns the finished file path`() = runBlocking { + val file = File("/tmp/jetwhale-mirror-test.mp4") + val result = StopRecordingCommand(stopRecording = { file }).execute(JetWhaleMcpArguments(emptyMap())) + assertEquals(file.absolutePath, Json.parseToJsonElement(result).jsonObject["path"]!!.jsonPrimitive.content) + } + + @Test + fun `unknown device id surfaces as an argument error`(): Unit = runBlocking { + assertFailsWith { + TapCommand(resolveDevice = { throw JetWhaleMcpArgumentException("unknown deviceId") }) + .execute(JetWhaleMcpArguments(mapOf("deviceId" to "nope", "x" to "1", "y" to "1"))) + } + } +} From 56ea3100cadebd2608e1b3fab44622a6faae7c0c Mon Sep 17 00:00:00 2001 From: kitakkun <48154936+kitakkun@users.noreply.github.com> Date: Sun, 19 Jul 2026 13:49:36 +0900 Subject: [PATCH 09/17] feat(mirror): compact icon toolbar and direct key input with IME buffering The stacked toolbar rows squeezed the mirror panes into a fraction of the window. Controls now sit in one slim glyph-icon row (with hover tooltips) and the TopAppBar is gone. The Send field is replaced by a direct key-input field: committed characters are forwarded to the selected device as typed, IME composition (e.g. Japanese conversion) stays buffered until committed and is then sent as one string, and Backspace/Enter on an empty buffer are sent as device keys. --- .../plugins/mirror/host/MirrorScreen.kt | 326 +++++++++++------- 1 file changed, 202 insertions(+), 124 deletions(-) diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt index 8002fad1e..91630aac4 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt @@ -1,5 +1,8 @@ package com.kitakkun.jetwhale.plugins.mirror.host +import androidx.compose.foundation.ExperimentalFoundationApi +import androidx.compose.foundation.TooltipArea +import androidx.compose.foundation.background import androidx.compose.foundation.border import androidx.compose.foundation.gestures.detectDragGestures import androidx.compose.foundation.gestures.detectTapGestures @@ -13,17 +16,14 @@ import androidx.compose.foundation.layout.fillMaxHeight import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width import androidx.compose.foundation.rememberScrollState import androidx.compose.foundation.shape.RoundedCornerShape -import androidx.compose.material3.Button -import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.foundation.text.BasicTextField +import androidx.compose.material3.IconButton import androidx.compose.material3.MaterialTheme -import androidx.compose.material3.OutlinedButton -import androidx.compose.material3.OutlinedTextField -import androidx.compose.material3.Scaffold -import androidx.compose.material3.Switch import androidx.compose.material3.Text -import androidx.compose.material3.TopAppBar import androidx.compose.runtime.Composable import androidx.compose.runtime.derivedStateOf import androidx.compose.runtime.getValue @@ -36,12 +36,18 @@ import androidx.compose.ui.draw.drawBehind import androidx.compose.ui.geometry.Offset import androidx.compose.ui.graphics.Color import androidx.compose.ui.graphics.ImageBitmap +import androidx.compose.ui.graphics.SolidColor +import androidx.compose.ui.input.key.Key +import androidx.compose.ui.input.key.KeyEventType +import androidx.compose.ui.input.key.key +import androidx.compose.ui.input.key.onPreviewKeyEvent +import androidx.compose.ui.input.key.type import androidx.compose.ui.input.pointer.pointerInput +import androidx.compose.ui.text.input.TextFieldValue import androidx.compose.ui.unit.IntSize import androidx.compose.ui.unit.dp import kotlin.math.roundToInt -@OptIn(ExperimentalMaterial3Api::class) @Composable internal fun MirrorScreen( devices: List, @@ -61,155 +67,227 @@ internal fun MirrorScreen( onToggleRecording: () -> Unit, ) { val selectedDevice = devices.firstOrNull { it.id == selectedDeviceId } - Scaffold( - topBar = { TopAppBar(title = { Text("Device Mirror") }) }, - ) { innerPadding -> - Column( - modifier = Modifier - .fillMaxSize() - .padding(innerPadding) - .padding(16.dp), - verticalArrangement = Arrangement.spacedBy(12.dp), - ) { - MirrorToolbar( - mirroringEnabled = mirroringEnabled, - onToggleMirroring = onToggleMirroring, - onRefreshDevices = onRefreshDevices, - ) - ControlButtonsRow( - platform = selectedDevice?.platform, - enabled = selectedDevice != null, - onPressButton = onPressButton, - onSaveScreenshot = onSaveScreenshot, - isRecording = isRecording, - onToggleRecording = onToggleRecording, - ) - TextInputRow(enabled = selectedDevice != null, onInputText = onInputText) - errorMessage?.let { - Text(it, style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.error) + Column( + modifier = Modifier + .fillMaxSize() + .padding(8.dp), + verticalArrangement = Arrangement.spacedBy(6.dp), + ) { + MirrorToolbar( + platform = selectedDevice?.platform, + enabled = selectedDevice != null, + mirroringEnabled = mirroringEnabled, + onToggleMirroring = onToggleMirroring, + onRefreshDevices = onRefreshDevices, + onPressButton = onPressButton, + onSaveScreenshot = onSaveScreenshot, + isRecording = isRecording, + onToggleRecording = onToggleRecording, + onInputText = onInputText, + ) + errorMessage?.let { + Text(it, style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.error) + } + if (devices.isEmpty()) { + Box( + modifier = Modifier + .fillMaxWidth() + .weight(1f), + contentAlignment = Alignment.Center, + ) { + Text( + "Boot an Android emulator or iOS simulator to start mirroring.", + style = MaterialTheme.typography.bodyMedium, + ) } - if (devices.isEmpty()) { - Box( - modifier = Modifier - .fillMaxWidth() - .weight(1f), - contentAlignment = Alignment.Center, - ) { - Text( - "Boot an Android emulator or iOS simulator to start mirroring.", - style = MaterialTheme.typography.bodyMedium, + } else { + // All connected devices side by side; the highlighted one receives the + // button/text controls, while taps and swipes go to the frame under the cursor. + Row( + modifier = Modifier + .fillMaxWidth() + .weight(1f) + .horizontalScroll(rememberScrollState()), + horizontalArrangement = Arrangement.spacedBy(12.dp), + ) { + devices.forEach { device -> + DeviceMirrorPane( + device = device, + frames = frames, + isSelected = device.id == selectedDeviceId, + onSelect = { onSelectDevice(device.id) }, + onTap = onTap, + onSwipe = onSwipe, ) } - } else { - // All connected devices side by side; the highlighted one receives the - // button/text controls, while taps and swipes go to the frame under the cursor. - Row( - modifier = Modifier - .fillMaxWidth() - .weight(1f) - .horizontalScroll(rememberScrollState()), - horizontalArrangement = Arrangement.spacedBy(12.dp), - ) { - devices.forEach { device -> - DeviceMirrorPane( - device = device, - frames = frames, - isSelected = device.id == selectedDeviceId, - onSelect = { onSelectDevice(device.id) }, - onTap = onTap, - onSwipe = onSwipe, - ) - } - } } } } } +// One slim icon row: device controls on the left, the direct key-input field on the right. @Composable private fun MirrorToolbar( + platform: DevicePlatform?, + enabled: Boolean, mirroringEnabled: Boolean, onToggleMirroring: (Boolean) -> Unit, onRefreshDevices: () -> Unit, -) { - Row( - verticalAlignment = Alignment.CenterVertically, - horizontalArrangement = Arrangement.spacedBy(8.dp), - ) { - OutlinedButton(onClick = onRefreshDevices) { Text("Refresh") } - Text("Mirror", style = MaterialTheme.typography.labelMedium) - Switch(checked = mirroringEnabled, onCheckedChange = onToggleMirroring) - } -} - -@Composable -private fun ControlButtonsRow( - platform: DevicePlatform?, - enabled: Boolean, onPressButton: (DeviceButton) -> Unit, onSaveScreenshot: () -> Unit, isRecording: Boolean, onToggleRecording: () -> Unit, + onInputText: (String) -> Unit, ) { Row( - modifier = Modifier.horizontalScroll(rememberScrollState()), - horizontalArrangement = Arrangement.spacedBy(8.dp), + modifier = Modifier + .fillMaxWidth() + .horizontalScroll(rememberScrollState()), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(2.dp), ) { - Button(onClick = onSaveScreenshot, enabled = enabled) { Text("Screenshot") } - Button(onClick = onToggleRecording, enabled = enabled || isRecording) { - Text(if (isRecording) "Stop Rec" else "Record") + ToolbarIconButton(onClick = onRefreshDevices, tooltip = "Refresh devices") { + GlyphIcon("⟳") } - val buttons = when (platform) { - DevicePlatform.IOS -> listOf(DeviceButton.HOME, DeviceButton.POWER, DeviceButton.BACKSPACE, DeviceButton.ENTER) - else -> DeviceButton.entries + ToolbarIconButton(onClick = { onToggleMirroring(!mirroringEnabled) }, tooltip = "Toggle mirroring") { + GlyphIcon(if (mirroringEnabled) "⏸" else "▶") } - buttons.forEach { button -> - OutlinedButton(onClick = { onPressButton(button) }, enabled = enabled) { - Text( - when (button) { - DeviceButton.HOME -> "Home" - DeviceButton.BACK -> "Back" - DeviceButton.POWER -> "Power" - DeviceButton.VOLUME_UP -> "Vol +" - DeviceButton.VOLUME_DOWN -> "Vol -" - DeviceButton.BACKSPACE -> "⌫" - DeviceButton.ENTER -> "⏎" - }, - ) + ToolbarIconButton(onClick = onSaveScreenshot, enabled = enabled, tooltip = "Save screenshot") { + GlyphIcon("📷") + } + ToolbarIconButton(onClick = onToggleRecording, enabled = enabled || isRecording, tooltip = "Record screen") { + GlyphIcon(if (isRecording) "⏹" else "⏺", tint = if (isRecording) MaterialTheme.colorScheme.error else null) + } + ToolbarIconButton(onClick = { onPressButton(DeviceButton.HOME) }, enabled = enabled, tooltip = "Home") { + GlyphIcon("⌂") + } + if (platform != DevicePlatform.IOS) { + ToolbarIconButton(onClick = { onPressButton(DeviceButton.BACK) }, enabled = enabled, tooltip = "Back") { + GlyphIcon("←") + } + } + ToolbarIconButton(onClick = { onPressButton(DeviceButton.POWER) }, enabled = enabled, tooltip = "Power") { + GlyphIcon("⏻") + } + if (platform != DevicePlatform.IOS) { + ToolbarIconButton(onClick = { onPressButton(DeviceButton.VOLUME_UP) }, enabled = enabled, tooltip = "Volume up") { + GlyphIcon("🔊") + } + ToolbarIconButton(onClick = { onPressButton(DeviceButton.VOLUME_DOWN) }, enabled = enabled, tooltip = "Volume down") { + GlyphIcon("🔉") } } + DirectKeyInputField( + enabled = enabled, + onSendText = onInputText, + onSendKey = onPressButton, + ) } } +@OptIn(ExperimentalFoundationApi::class) @Composable -private fun TextInputRow( - enabled: Boolean, - onInputText: (String) -> Unit, +private fun ToolbarIconButton( + onClick: () -> Unit, + tooltip: String, + enabled: Boolean = true, + content: @Composable () -> Unit, ) { - Row( - verticalAlignment = Alignment.CenterVertically, - horizontalArrangement = Arrangement.spacedBy(8.dp), - ) { - var text by remember { mutableStateOf("") } - OutlinedTextField( - value = text, - onValueChange = { text = it }, - singleLine = true, - placeholder = { Text("Type text to send to the selected device") }, - modifier = Modifier.weight(1f), - ) - Button( - onClick = { - onInputText(text) - text = "" - }, - enabled = enabled && text.isNotEmpty(), + TooltipArea(tooltip = { TooltipBubble(tooltip) }) { + IconButton( + onClick = onClick, + enabled = enabled, + modifier = Modifier.size(28.dp), ) { - Text("Send") + content() } } } +@Composable +private fun TooltipBubble(text: String) { + Text( + text = text, + style = MaterialTheme.typography.labelSmall, + color = MaterialTheme.colorScheme.onSurface, + modifier = Modifier + .border(1.dp, MaterialTheme.colorScheme.outlineVariant, RoundedCornerShape(4.dp)) + .background(MaterialTheme.colorScheme.surface, RoundedCornerShape(4.dp)) + .padding(horizontal = 6.dp, vertical = 3.dp), + ) +} + +@Composable +private fun GlyphIcon(glyph: String, tint: Color? = null) { + Text( + text = glyph, + style = MaterialTheme.typography.bodyMedium, + color = tint ?: MaterialTheme.colorScheme.onSurface, + ) +} + +/** + * Sends keystrokes to the selected device as they are typed. Plain characters are committed by + * the platform immediately and forwarded right away; IME composition (e.g. Japanese conversion) + * keeps the text buffered in the field and is forwarded as one string once the conversion is + * committed. Backspace/Enter on an empty buffer are sent as device keys. + */ +@Composable +private fun DirectKeyInputField( + enabled: Boolean, + onSendText: (String) -> Unit, + onSendKey: (DeviceButton) -> Unit, +) { + var value by remember { mutableStateOf(TextFieldValue("")) } + BasicTextField( + value = value, + onValueChange = { new -> + if (new.composition == null && new.text.isNotEmpty()) { + onSendText(new.text) + value = TextFieldValue("") + } else { + value = new + } + }, + enabled = enabled, + singleLine = true, + textStyle = MaterialTheme.typography.bodySmall.copy(color = MaterialTheme.colorScheme.onSurface), + cursorBrush = SolidColor(MaterialTheme.colorScheme.primary), + modifier = Modifier + .width(220.dp) + .border(1.dp, MaterialTheme.colorScheme.outlineVariant, RoundedCornerShape(4.dp)) + .padding(horizontal = 8.dp, vertical = 5.dp) + .onPreviewKeyEvent { event -> + if (event.type != KeyEventType.KeyDown || value.text.isNotEmpty()) return@onPreviewKeyEvent false + when (event.key) { + Key.Backspace -> { + onSendKey(DeviceButton.BACKSPACE) + true + } + + Key.Enter -> { + onSendKey(DeviceButton.ENTER) + true + } + + else -> false + } + }, + decorationBox = { innerTextField -> + Box(contentAlignment = Alignment.CenterStart) { + if (value.text.isEmpty()) { + Text( + "⌨ Type here to send keys", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + innerTextField() + } + }, + ) +} + @Composable private fun DeviceMirrorPane( device: MirrorDevice, @@ -232,7 +310,7 @@ private fun DeviceMirrorPane( ) { Text( text = "${device.name} (${device.platform})", - style = MaterialTheme.typography.labelMedium, + style = MaterialTheme.typography.labelSmall, color = if (isSelected) MaterialTheme.colorScheme.primary else MaterialTheme.colorScheme.onSurfaceVariant, ) val size = frameSize From eb032954628125d5667551a2c8675be7fc9bea8f Mon Sep 17 00:00:00 2001 From: kitakkun <48154936+kitakkun@users.noreply.github.com> Date: Sun, 19 Jul 2026 13:52:17 +0900 Subject: [PATCH 10/17] feat(mirror): decode a live H.264 stream via JavaCV/FFmpeg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Screenshot polling topped out at a few fps. Each device's mirror loop now prefers a raw H.264 stream — `adb exec-out screenrecord` on Android (reopened at its 180s cap), `idb video-stream` on iOS — decoded with FFmpegFrameGrabber, falling back to screenshot polling when streaming is unavailable or fails before producing a frame. Unselected devices still decode but publish only every 15th frame. The recording stop now pkills by output path so it cannot kill the mirror stream's screenrecord. The ffmpeg natives are bundled for the build machine's platform only. --- gradle/libs.versions.toml | 4 ++ jetwhale-plugins/mirror/host/build.gradle.kts | 22 ++++++ .../plugins/mirror/host/DeviceController.kt | 28 +++++++- .../mirror/host/MirrorHostPluginFactory.kt | 69 ++++++++++++++++++- .../mirror/host/MirrorMcpCommandsTest.kt | 2 + 5 files changed, 121 insertions(+), 4 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 4a28566ad..9b3705bbd 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -11,6 +11,8 @@ kotlinxCollectionsImmutable = "0.5.1" kotlinxCoroutines = "1.11.0" byteBuddy = "1.18.11" javaKeyring = "1.0.4" +javacv = "1.5.10" +bytedecoFfmpeg = "6.1.1-1.5.10" kotlinxDatetime = "0.8.0" # android @@ -58,6 +60,8 @@ kotlinTest = { module = "org.jetbrains.kotlin:kotlin-test", version.ref = "kotli # kotlinx kotlinxSerializationCore = { module = "org.jetbrains.kotlinx:kotlinx-serialization-core", version.ref = "kotlinxSerializationJson" } kotlinxSerializationJson = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" } +javacv = { module = "org.bytedeco:javacv", version.ref = "javacv" } +bytedecoFfmpeg = { module = "org.bytedeco:ffmpeg", version.ref = "bytedecoFfmpeg" } kotlinxCollectionsImmutable = { module = "org.jetbrains.kotlinx:kotlinx-collections-immutable", version.ref = "kotlinxCollectionsImmutable" } kotlinxCoroutinesCore = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "kotlinxCoroutines" } byteBuddyAgent = { module = "net.bytebuddy:byte-buddy-agent", version.ref = "byteBuddy" } diff --git a/jetwhale-plugins/mirror/host/build.gradle.kts b/jetwhale-plugins/mirror/host/build.gradle.kts index 737b2f629..77df563f9 100644 --- a/jetwhale-plugins/mirror/host/build.gradle.kts +++ b/jetwhale-plugins/mirror/host/build.gradle.kts @@ -25,6 +25,16 @@ dependencies { compileOnly(compose.desktop.currentOs) compileOnly(libs.material3) compileOnly(libs.kotlinxSerializationJson) + // H.264 mirror-stream decoding (adb screenrecord / idb video-stream). Bundled into the + // plugin jar; the ffmpeg natives are restricted to the build machine's platform to keep the + // artifact from carrying every OS's binaries. + implementation(libs.javacv) { + // javacv declares every bytedeco preset; only the ffmpeg bindings are used here. + isTransitive = false + } + implementation("org.bytedeco:javacpp:${libs.versions.javacv.get()}") + implementation(libs.bytedecoFfmpeg) + implementation("org.bytedeco:ffmpeg:${libs.versions.bytedecoFfmpeg.get()}:${currentFfmpegClassifier()}") testImplementation(projects.jetwhaleHostSdk) testImplementation(libs.kotlinTest) testImplementation(libs.kotlinxSerializationJson) @@ -32,6 +42,18 @@ dependencies { testImplementation(libs.material3) } +fun currentFfmpegClassifier(): String { + val osName = System.getProperty("os.name").lowercase() + val arch = System.getProperty("os.arch").lowercase() + return when { + osName.contains("mac") && arch == "aarch64" -> "macosx-arm64" + osName.contains("mac") -> "macosx-x86_64" + osName.contains("windows") -> "windows-x86_64" + arch == "aarch64" -> "linux-arm64" + else -> "linux-x86_64" + } +} + jetwhalePublish { artifactId = "jetwhale-device-mirror" name = "JetWhale Device Mirror" diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt index 67c79313a..d50564623 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt @@ -42,6 +42,13 @@ interface DeviceController { /** Starts recording the screen into [outputFile] (mp4/mov). Stop it via the returned handle. */ suspend fun startRecording(outputFile: File): DeviceRecording + + /** + * Opens a process that writes a raw H.264 stream of the device screen to stdout, or null + * when the platform/tooling cannot stream (the mirror then falls back to screenshot polling). + * The stream may end on its own (e.g. screenrecord's time limit); reopen to continue. + */ + suspend fun openVideoStreamProcess(): Process? } /** One in-progress screen recording; [stop] finalizes and returns the video file. */ @@ -90,6 +97,11 @@ internal class AndroidDeviceController( runCommandChecked(adbPath, "-s", serial, "shell", "input", "text", escaped) } + override suspend fun openVideoStreamProcess(): Process? = withContext(Dispatchers.IO) { + // screenrecord caps a session at 180s; the caller reopens the stream when it ends. + ProcessBuilder(adbPath, "-s", serial, "exec-out", "screenrecord", "--output-format=h264", "--time-limit", "180", "-").start() + } + override suspend fun startRecording(outputFile: File): DeviceRecording { // screenrecord writes on-device; the file is pulled after a clean SIGINT shutdown. val remotePath = "/sdcard/${outputFile.name}" @@ -99,9 +111,11 @@ internal class AndroidDeviceController( return object : DeviceRecording { override suspend fun stop(): File = withContext(Dispatchers.IO) { // SIGINT lets screenrecord finalize the mp4 moov atom; killing the local adb - // client instead would leave an unplayable file. Exit code is ignored: the - // process has already exited when the 180s time limit was hit. - runCommand(adbPath, "-s", serial, "shell", "pkill", "-INT", "screenrecord") + // client instead would leave an unplayable file. The -f pattern targets only + // the recorder instance, not the screenrecord that streams the live mirror. + // Exit code is ignored: the process has already exited when the 180s time + // limit was hit. + runCommand(adbPath, "-s", serial, "shell", "pkill", "-INT", "-f", remotePath) process.waitFor(10, TimeUnit.SECONDS) // The device flushes the file asynchronously after the process exits. delay(500) @@ -184,6 +198,14 @@ internal class IosDeviceController( runCommandChecked(requireIdb(), "ui", "text", "--udid", udid, text) } + override suspend fun openVideoStreamProcess(): Process? { + // simctl cannot stream; idb's video-stream is the only live H.264 source for simulators. + val idb = idbPath ?: return null + return withContext(Dispatchers.IO) { + ProcessBuilder(idb, "video-stream", "--udid", udid, "--format", "h264").start() + } + } + override suspend fun startRecording(outputFile: File): DeviceRecording { val process = withContext(Dispatchers.IO) { ProcessBuilder("xcrun", "simctl", "io", udid, "recordVideo", "--codec=h264", "--force", outputFile.absolutePath).start() diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt index 79a65214d..49afae638 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt @@ -17,6 +17,7 @@ import com.kitakkun.jetwhale.host.sdk.JetWhaleHostPluginUi import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArgumentException import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpCapablePlugin import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpCommand +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.delay @@ -24,6 +25,10 @@ import kotlinx.coroutines.isActive import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import org.bytedeco.javacv.FFmpegFrameGrabber +import org.bytedeco.javacv.Java2DFrameConverter +import java.util.concurrent.ConcurrentHashMap import org.jetbrains.skia.Image as SkiaImage // Instantiated by the host via the fully-qualified name declared in plugin-manifest.json. @@ -45,6 +50,9 @@ private const val BACKGROUND_FRAME_INTERVAL_MILLIS = 500L // Back off after a failed capture so a dead device doesn't spam error processes. private const val FRAME_RETRY_INTERVAL_MILLIS = 1_000L +// Unselected devices publish only every Nth decoded stream frame (~2fps at 30fps input). +private const val BACKGROUND_STREAM_FRAME_STRIDE = 15 + @OptIn(ExperimentalJetWhaleApi::class) private class MirrorHostPlugin : JetWhaleHostPlugin(), @@ -60,6 +68,10 @@ private class MirrorHostPlugin : // All connected devices are mirrored concurrently, one capture loop and frame slot each. private val frames: SnapshotStateMap = mutableStateMapOf() private val mirrorJobs = mutableMapOf() + + // Live decoder subprocesses; grabImage() blocks the IO thread, so cancellation must kill the + // process to unblock it. + private val streamProcesses = ConcurrentHashMap() private val refreshMutex = Mutex() private var lastError by mutableStateOf(null) @@ -78,6 +90,11 @@ private class MirrorHostPlugin : } } + override fun onDispose() { + streamProcesses.values.forEach { it.destroyForcibly() } + streamProcesses.clear() + } + private suspend fun refreshDevices(): List = refreshMutex.withLock { val discovered = discovery.discoverDevices() devices.apply { @@ -90,6 +107,7 @@ private class MirrorHostPlugin : val discoveredIds = discovered.map { it.id }.toSet() mirrorJobs.keys.filter { it !in discoveredIds }.forEach { id -> mirrorJobs.remove(id)?.cancel() + streamProcesses.remove(id)?.destroyForcibly() frames.remove(id) } discovered.forEach { device -> @@ -100,14 +118,29 @@ private class MirrorHostPlugin : discovered } - // The mirror is a screenshot poll loop: cheap, tool-free, and identical for both platforms. + // Mirrors one device: preferably by decoding a live H.264 stream, falling back to screenshot + // polling when the platform/tooling cannot stream or the stream setup fails. private suspend fun mirrorLoop(device: MirrorDevice) { + var streamingBroken = false var lastPngHash = 0 while (currentCoroutineContext().isActive) { if (!mirroringEnabled) { delay(DEVICE_POLL_INTERVAL_MILLIS) continue } + if (!streamingBroken) { + val process = try { + device.controller.openVideoStreamProcess() + } catch (_: DeviceControlException) { + null + } + if (process != null) { + // Returns when the stream ends (screenrecord's time limit) — reopen right + // away; if it produced nothing, the stream path doesn't work on this device. + if (decodeVideoStream(device, process)) continue + } + streamingBroken = true + } try { val png = device.controller.captureScreenshot() // A static screen produces byte-identical PNGs; skip the decode and the @@ -125,6 +158,40 @@ private class MirrorHostPlugin : } } + /** + * Decodes H.264 frames from [process]'s stdout into [frames] until the stream ends or the + * loop is cancelled. Returns whether at least one frame was decoded. + */ + private suspend fun decodeVideoStream(device: MirrorDevice, process: Process): Boolean = withContext(Dispatchers.IO) { + streamProcesses[device.id] = process + var produced = false + val grabber = FFmpegFrameGrabber(process.inputStream, 0) + grabber.format = "h264" + val converter = Java2DFrameConverter() + try { + grabber.start() + var frameIndex = 0 + while (currentCoroutineContext().isActive && mirroringEnabled) { + val frame = grabber.grabImage() ?: break + produced = true + frameIndex++ + // Unselected devices decode (H.264 requires it) but convert/publish only a + // subset of frames to keep the multi-device cost down. + if (device.id == selectedDeviceId || frameIndex % BACKGROUND_STREAM_FRAME_STRIDE == 0) { + val image = converter.convert(frame) ?: continue + frames[device.id] = image.toComposeImageBitmap() + } + } + } catch (e: Exception) { + if (!produced) lastError = "${device.name}: video stream failed (${e.message}); falling back to screenshots" + } finally { + runCatching { grabber.close() } + streamProcesses.remove(device.id) + process.destroyForcibly() + } + produced + } + // Runs a device-control action from a UI callback, surfacing failures in the status bar. private fun runControl(action: suspend () -> Unit) { pluginScope.launch { diff --git a/jetwhale-plugins/mirror/host/src/test/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorMcpCommandsTest.kt b/jetwhale-plugins/mirror/host/src/test/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorMcpCommandsTest.kt index 384e6f0a4..17bbccca9 100644 --- a/jetwhale-plugins/mirror/host/src/test/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorMcpCommandsTest.kt +++ b/jetwhale-plugins/mirror/host/src/test/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorMcpCommandsTest.kt @@ -37,6 +37,8 @@ class MirrorMcpCommandsTest { override suspend fun startRecording(outputFile: File): DeviceRecording = object : DeviceRecording { override suspend fun stop(): File = outputFile } + + override suspend fun openVideoStreamProcess(): Process? = null } private val controller = FakeDeviceController() From e32e904e98016b0c18db791a3a1a57d6fb0dc9dc Mon Sep 17 00:00:00 2001 From: kitakkun <48154936+kitakkun@users.noreply.github.com> Date: Sun, 19 Jul 2026 13:56:14 +0900 Subject: [PATCH 11/17] fix(mirror): time out stream startup so a silent stream falls back FFmpegFrameGrabber.start() blocks in format probing for as long as the stream process keeps stdout open without writing (e.g. idb without a working companion), leaving the pane stuck on Waiting. A watchdog kills the process if no frame arrives within 7s, unblocking the probe so the loop falls back to screenshot polling. --- .../plugins/mirror/host/MirrorHostPluginFactory.kt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt index 49afae638..c85a957d4 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt @@ -53,6 +53,9 @@ private const val FRAME_RETRY_INTERVAL_MILLIS = 1_000L // Unselected devices publish only every Nth decoded stream frame (~2fps at 30fps input). private const val BACKGROUND_STREAM_FRAME_STRIDE = 15 +// A stream that has not produced a frame by this deadline is treated as broken. +private const val STREAM_START_TIMEOUT_MILLIS = 7_000L + @OptIn(ExperimentalJetWhaleApi::class) private class MirrorHostPlugin : JetWhaleHostPlugin(), @@ -168,12 +171,21 @@ private class MirrorHostPlugin : val grabber = FFmpegFrameGrabber(process.inputStream, 0) grabber.format = "h264" val converter = Java2DFrameConverter() + // grabber.start() blocks in format probing for as long as the process keeps its stdout + // open without writing (e.g. idb without a working companion). The watchdog kills the + // process to unblock it so the loop can fall back to screenshot polling. + val firstFrameSeen = java.util.concurrent.atomic.AtomicBoolean(false) + val watchdog = pluginScope.launch { + delay(STREAM_START_TIMEOUT_MILLIS) + if (!firstFrameSeen.get()) process.destroyForcibly() + } try { grabber.start() var frameIndex = 0 while (currentCoroutineContext().isActive && mirroringEnabled) { val frame = grabber.grabImage() ?: break produced = true + firstFrameSeen.set(true) frameIndex++ // Unselected devices decode (H.264 requires it) but convert/publish only a // subset of frames to keep the multi-device cost down. @@ -185,6 +197,7 @@ private class MirrorHostPlugin : } catch (e: Exception) { if (!produced) lastError = "${device.name}: video stream failed (${e.message}); falling back to screenshots" } finally { + watchdog.cancel() runCatching { grabber.close() } streamProcesses.remove(device.id) process.destroyForcibly() From ff239cf662cdbf50ee319047c67bb998b5988cd3 Mon Sep 17 00:00:00 2001 From: kitakkun <48154936+kitakkun@users.noreply.github.com> Date: Sun, 19 Jul 2026 14:16:02 +0900 Subject: [PATCH 12/17] perf(mirror): cut stream latency with low-delay ffmpeg options and publish caps The default ffmpeg input path buffers and probes for seconds, and converting every decoded frame let the pipe back up, so the mirror drifted ever further behind real time. Disable input buffering (nobuffer/low_delay, small probe window) and always drain the decoder while throttling conversion/publication by wall-clock (~30fps selected, ~2fps unselected). --- .../mirror/host/MirrorHostPluginFactory.kt | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt index c85a957d4..6968bbca1 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorHostPluginFactory.kt @@ -50,8 +50,10 @@ private const val BACKGROUND_FRAME_INTERVAL_MILLIS = 500L // Back off after a failed capture so a dead device doesn't spam error processes. private const val FRAME_RETRY_INTERVAL_MILLIS = 1_000L -// Unselected devices publish only every Nth decoded stream frame (~2fps at 30fps input). -private const val BACKGROUND_STREAM_FRAME_STRIDE = 15 +// Publish caps for decoded stream frames: decoding always keeps up with the stream, but +// conversion + publication is throttled by wall-clock (~30fps selected, ~2fps unselected). +private const val SELECTED_PUBLISH_GAP_MILLIS = 33L +private const val BACKGROUND_PUBLISH_GAP_MILLIS = 500L // A stream that has not produced a frame by this deadline is treated as broken. private const val STREAM_START_TIMEOUT_MILLIS = 7_000L @@ -170,6 +172,13 @@ private class MirrorHostPlugin : var produced = false val grabber = FFmpegFrameGrabber(process.inputStream, 0) grabber.format = "h264" + // Live-stream tuning: ffmpeg's defaults buffer input and probe the format for seconds, + // which shows up as a laggy, ever-delayed mirror. A raw H.264 stream needs almost no + // probing, and buffering is pure latency here. + grabber.setOption("fflags", "nobuffer") + grabber.setOption("flags", "low_delay") + grabber.setOption("probesize", "65536") + grabber.setOption("analyzeduration", "500000") val converter = Java2DFrameConverter() // grabber.start() blocks in format probing for as long as the process keeps its stdout // open without writing (e.g. idb without a working companion). The watchdog kills the @@ -181,17 +190,21 @@ private class MirrorHostPlugin : } try { grabber.start() - var frameIndex = 0 + var lastPublishNanos = 0L while (currentCoroutineContext().isActive && mirroringEnabled) { val frame = grabber.grabImage() ?: break produced = true firstFrameSeen.set(true) - frameIndex++ - // Unselected devices decode (H.264 requires it) but convert/publish only a - // subset of frames to keep the multi-device cost down. - if (device.id == selectedDeviceId || frameIndex % BACKGROUND_STREAM_FRAME_STRIDE == 0) { + // Decoding must keep up with the stream (or frames back up in the pipe and the + // mirror falls ever further behind), but converting/publishing every decoded + // frame is what actually costs: cap it by wall-clock instead, and give + // unselected devices a much lower cap. + val minPublishGapMillis = if (device.id == selectedDeviceId) SELECTED_PUBLISH_GAP_MILLIS else BACKGROUND_PUBLISH_GAP_MILLIS + val now = System.nanoTime() + if (now - lastPublishNanos >= minPublishGapMillis * 1_000_000) { val image = converter.convert(frame) ?: continue frames[device.id] = image.toComposeImageBitmap() + lastPublishNanos = now } } } catch (e: Exception) { From e2f308d99eda338a3e0161a9bb25a377778adced Mon Sep 17 00:00:00 2001 From: kitakkun <48154936+kitakkun@users.noreply.github.com> Date: Sun, 19 Jul 2026 14:17:13 +0900 Subject: [PATCH 13/17] perf(mirror): stream the iOS simulator at 30fps Matches the decode-side publish cap; a faster stream only spends encode/decode CPU on frames that would be dropped anyway. --- .../kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt index d50564623..a07dc6ae6 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/DeviceController.kt @@ -202,7 +202,9 @@ internal class IosDeviceController( // simctl cannot stream; idb's video-stream is the only live H.264 source for simulators. val idb = idbPath ?: return null return withContext(Dispatchers.IO) { - ProcessBuilder(idb, "video-stream", "--udid", udid, "--format", "h264").start() + // 30fps matches the publish cap on the decode side; streaming faster only costs + // encode/decode CPU for frames that would be dropped anyway. + ProcessBuilder(idb, "video-stream", "--udid", udid, "--format", "h264", "--fps", "30").start() } } From 9396a6baf518cbe99f5667ea40d204753fc58955 Mon Sep 17 00:00:00 2001 From: kitakkun <48154936+kitakkun@users.noreply.github.com> Date: Sun, 19 Jul 2026 14:25:25 +0900 Subject: [PATCH 14/17] fix(mirror): buffer all key input and send on Enter so IME works Sending each committed chunk immediately and clearing the field reset the macOS IME session, making Japanese input impossible. The field now purely buffers (never touching an active composition), sends the whole buffer on Enter, and forwards Enter/Backspace as device keys only when the buffer is empty and no composition is active. --- .../plugins/mirror/host/MirrorScreen.kt | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt index 91630aac4..eebe78b2b 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt @@ -227,10 +227,9 @@ private fun GlyphIcon(glyph: String, tint: Color? = null) { } /** - * Sends keystrokes to the selected device as they are typed. Plain characters are committed by - * the platform immediately and forwarded right away; IME composition (e.g. Japanese conversion) - * keeps the text buffered in the field and is forwarded as one string once the conversion is - * committed. Backspace/Enter on an empty buffer are sent as device keys. + * Buffers typed text — including IME composition such as Japanese conversion, which must never + * be interfered with mid-session — and sends the whole buffer to the selected device on Enter. + * Backspace/Enter on an empty buffer are sent as device keys instead. */ @Composable private fun DirectKeyInputField( @@ -241,14 +240,7 @@ private fun DirectKeyInputField( var value by remember { mutableStateOf(TextFieldValue("")) } BasicTextField( value = value, - onValueChange = { new -> - if (new.composition == null && new.text.isNotEmpty()) { - onSendText(new.text) - value = TextFieldValue("") - } else { - value = new - } - }, + onValueChange = { value = it }, enabled = enabled, singleLine = true, textStyle = MaterialTheme.typography.bodySmall.copy(color = MaterialTheme.colorScheme.onSurface), @@ -258,18 +250,27 @@ private fun DirectKeyInputField( .border(1.dp, MaterialTheme.colorScheme.outlineVariant, RoundedCornerShape(4.dp)) .padding(horizontal = 8.dp, vertical = 5.dp) .onPreviewKeyEvent { event -> - if (event.type != KeyEventType.KeyDown || value.text.isNotEmpty()) return@onPreviewKeyEvent false - when (event.key) { - Key.Backspace -> { - onSendKey(DeviceButton.BACKSPACE) + if (event.type != KeyEventType.KeyDown) return@onPreviewKeyEvent false + // While the IME is composing, every key (including Enter, which commits the + // conversion) belongs to the IME — never intercept. + if (value.composition != null) return@onPreviewKeyEvent false + when { + event.key == Key.Enter && value.text.isNotEmpty() -> { + onSendText(value.text) + value = TextFieldValue("") true } - Key.Enter -> { + event.key == Key.Enter -> { onSendKey(DeviceButton.ENTER) true } + event.key == Key.Backspace && value.text.isEmpty() -> { + onSendKey(DeviceButton.BACKSPACE) + true + } + else -> false } }, @@ -277,7 +278,7 @@ private fun DirectKeyInputField( Box(contentAlignment = Alignment.CenterStart) { if (value.text.isEmpty()) { Text( - "⌨ Type here to send keys", + "⌨ Type, Enter to send", style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.onSurfaceVariant, ) From 591aa28cb14d5d559e73d9e3bc712d050a42d5a6 Mon Sep 17 00:00:00 2001 From: kitakkun <48154936+kitakkun@users.noreply.github.com> Date: Sun, 19 Jul 2026 14:32:21 +0900 Subject: [PATCH 15/17] fix(mirror): render IME composition via TextFieldState The TextFieldValue-based BasicTextField does not display in-progress IME composition on desktop, so Japanese conversion was invisible while typing. Migrate the key-input field to the TextFieldState overload, which renders the composition inline; the Enter-to-send and empty-buffer device-key behavior is unchanged. --- .../plugins/mirror/host/MirrorScreen.kt | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt index eebe78b2b..f8c7176cb 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt @@ -21,15 +21,16 @@ import androidx.compose.foundation.layout.width import androidx.compose.foundation.rememberScrollState import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.foundation.text.BasicTextField +import androidx.compose.foundation.text.input.TextFieldLineLimits +import androidx.compose.foundation.text.input.clearText +import androidx.compose.foundation.text.input.rememberTextFieldState import androidx.compose.material3.IconButton import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.derivedStateOf import androidx.compose.runtime.getValue -import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember -import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.drawBehind @@ -43,7 +44,6 @@ import androidx.compose.ui.input.key.key import androidx.compose.ui.input.key.onPreviewKeyEvent import androidx.compose.ui.input.key.type import androidx.compose.ui.input.pointer.pointerInput -import androidx.compose.ui.text.input.TextFieldValue import androidx.compose.ui.unit.IntSize import androidx.compose.ui.unit.dp import kotlin.math.roundToInt @@ -237,12 +237,13 @@ private fun DirectKeyInputField( onSendText: (String) -> Unit, onSendKey: (DeviceButton) -> Unit, ) { - var value by remember { mutableStateOf(TextFieldValue("")) } + // TextFieldState (not the TextFieldValue overload): the value-based field does not render + // in-progress IME composition on desktop, so Japanese conversion was invisible while typing. + val state = rememberTextFieldState() BasicTextField( - value = value, - onValueChange = { value = it }, + state = state, enabled = enabled, - singleLine = true, + lineLimits = TextFieldLineLimits.SingleLine, textStyle = MaterialTheme.typography.bodySmall.copy(color = MaterialTheme.colorScheme.onSurface), cursorBrush = SolidColor(MaterialTheme.colorScheme.primary), modifier = Modifier @@ -253,11 +254,11 @@ private fun DirectKeyInputField( if (event.type != KeyEventType.KeyDown) return@onPreviewKeyEvent false // While the IME is composing, every key (including Enter, which commits the // conversion) belongs to the IME — never intercept. - if (value.composition != null) return@onPreviewKeyEvent false + if (state.composition != null) return@onPreviewKeyEvent false when { - event.key == Key.Enter && value.text.isNotEmpty() -> { - onSendText(value.text) - value = TextFieldValue("") + event.key == Key.Enter && state.text.isNotEmpty() -> { + onSendText(state.text.toString()) + state.clearText() true } @@ -266,7 +267,7 @@ private fun DirectKeyInputField( true } - event.key == Key.Backspace && value.text.isEmpty() -> { + event.key == Key.Backspace && state.text.isEmpty() -> { onSendKey(DeviceButton.BACKSPACE) true } @@ -274,9 +275,9 @@ private fun DirectKeyInputField( else -> false } }, - decorationBox = { innerTextField -> + decorator = { innerTextField -> Box(contentAlignment = Alignment.CenterStart) { - if (value.text.isEmpty()) { + if (state.text.isEmpty()) { Text( "⌨ Type, Enter to send", style = MaterialTheme.typography.bodySmall, From ea947870a99ae76bcb146f8eb17d7aed444f3772 Mon Sep 17 00:00:00 2001 From: kitakkun <48154936+kitakkun@users.noreply.github.com> Date: Sun, 19 Jul 2026 14:44:50 +0900 Subject: [PATCH 16/17] fix(mirror): route Enter through the IME-aware keyboard action A raw key handler fires before the IME, so it stole the Enter that commits a Japanese conversion and sent the half-composed buffer. Enter now goes through KeyboardOptions(imeAction = Send) + onKeyboardAction, which only fires for an Enter the IME did not consume; only Backspace on an empty, non-composing buffer is still intercepted as a device key. --- .../plugins/mirror/host/MirrorScreen.kt | 47 ++++++++++--------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt index f8c7176cb..8c10d9188 100644 --- a/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt +++ b/jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorScreen.kt @@ -21,6 +21,7 @@ import androidx.compose.foundation.layout.width import androidx.compose.foundation.rememberScrollState import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.foundation.text.BasicTextField +import androidx.compose.foundation.text.KeyboardOptions import androidx.compose.foundation.text.input.TextFieldLineLimits import androidx.compose.foundation.text.input.clearText import androidx.compose.foundation.text.input.rememberTextFieldState @@ -44,6 +45,7 @@ import androidx.compose.ui.input.key.key import androidx.compose.ui.input.key.onPreviewKeyEvent import androidx.compose.ui.input.key.type import androidx.compose.ui.input.pointer.pointerInput +import androidx.compose.ui.text.input.ImeAction import androidx.compose.ui.unit.IntSize import androidx.compose.ui.unit.dp import kotlin.math.roundToInt @@ -244,6 +246,18 @@ private fun DirectKeyInputField( state = state, enabled = enabled, lineLimits = TextFieldLineLimits.SingleLine, + // Enter is routed through the IME-aware keyboard-action pipeline instead of raw key + // interception: a key handler fires before the IME and would steal the Enter that + // commits a Japanese conversion, but the keyboard action only fires for a "free" Enter. + keyboardOptions = KeyboardOptions(imeAction = ImeAction.Send), + onKeyboardAction = { + if (state.text.isNotEmpty()) { + onSendText(state.text.toString()) + state.clearText() + } else { + onSendKey(DeviceButton.ENTER) + } + }, textStyle = MaterialTheme.typography.bodySmall.copy(color = MaterialTheme.colorScheme.onSurface), cursorBrush = SolidColor(MaterialTheme.colorScheme.primary), modifier = Modifier @@ -251,28 +265,17 @@ private fun DirectKeyInputField( .border(1.dp, MaterialTheme.colorScheme.outlineVariant, RoundedCornerShape(4.dp)) .padding(horizontal = 8.dp, vertical = 5.dp) .onPreviewKeyEvent { event -> - if (event.type != KeyEventType.KeyDown) return@onPreviewKeyEvent false - // While the IME is composing, every key (including Enter, which commits the - // conversion) belongs to the IME — never intercept. - if (state.composition != null) return@onPreviewKeyEvent false - when { - event.key == Key.Enter && state.text.isNotEmpty() -> { - onSendText(state.text.toString()) - state.clearText() - true - } - - event.key == Key.Enter -> { - onSendKey(DeviceButton.ENTER) - true - } - - event.key == Key.Backspace && state.text.isEmpty() -> { - onSendKey(DeviceButton.BACKSPACE) - true - } - - else -> false + // Only Backspace on an empty, non-composing buffer is intercepted — the IME has + // nothing in flight then, so forwarding it to the device is safe. + if (event.type == KeyEventType.KeyDown && + event.key == Key.Backspace && + state.text.isEmpty() && + state.composition == null + ) { + onSendKey(DeviceButton.BACKSPACE) + true + } else { + false } }, decorator = { innerTextField -> From bf57f45e604333021c6a211458f4004816caca9b Mon Sep 17 00:00:00 2001 From: kitakkun <48154936+kitakkun@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:55:23 +0900 Subject: [PATCH 17/17] test(mirror): adopt the JsonObject-based JetWhaleMcpArguments The SDK's JetWhaleMcpArguments now wraps a JsonObject instead of a Map; migrate the command tests accordingly. --- .../mirror/host/MirrorMcpCommandsTest.kt | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/jetwhale-plugins/mirror/host/src/test/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorMcpCommandsTest.kt b/jetwhale-plugins/mirror/host/src/test/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorMcpCommandsTest.kt index 17bbccca9..3bd97b2f9 100644 --- a/jetwhale-plugins/mirror/host/src/test/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorMcpCommandsTest.kt +++ b/jetwhale-plugins/mirror/host/src/test/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/MirrorMcpCommandsTest.kt @@ -5,6 +5,9 @@ import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArgumentException import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArguments import kotlinx.coroutines.runBlocking import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive @@ -41,12 +44,14 @@ class MirrorMcpCommandsTest { override suspend fun openVideoStreamProcess(): Process? = null } + private fun args(vararg pairs: Pair) = JetWhaleMcpArguments(JsonObject(pairs.toMap())) + private val controller = FakeDeviceController() private val device = MirrorDevice(id = "emulator-5554", name = "Pixel 7", platform = DevicePlatform.ANDROID, controller = controller) @Test fun `tap forwards coordinates to the device`() = runBlocking { - val result = TapCommand(resolveDevice = { device }).execute(JetWhaleMcpArguments(mapOf("x" to "10", "y" to "20"))) + val result = TapCommand(resolveDevice = { device }).execute(args("x" to JsonPrimitive(10), "y" to JsonPrimitive(20))) assertEquals(listOf(10 to 20), controller.taps) assertTrue(Json.parseToJsonElement(result).jsonObject["ok"]!!.jsonPrimitive.content.toBoolean()) } @@ -54,7 +59,7 @@ class MirrorMcpCommandsTest { @Test fun `tap rejects negative coordinates`(): Unit = runBlocking { assertFailsWith { - TapCommand(resolveDevice = { device }).execute(JetWhaleMcpArguments(mapOf("x" to "-1", "y" to "20"))) + TapCommand(resolveDevice = { device }).execute(args("x" to JsonPrimitive(-1), "y" to JsonPrimitive(20))) } assertTrue(controller.taps.isEmpty()) } @@ -62,7 +67,7 @@ class MirrorMcpCommandsTest { @Test fun `swipe defaults duration to 300ms`() = runBlocking { SwipeCommand(resolveDevice = { device }).execute( - JetWhaleMcpArguments(mapOf("fromX" to "0", "fromY" to "1", "toX" to "2", "toY" to "3")), + args("fromX" to JsonPrimitive(0), "fromY" to JsonPrimitive(1), "toX" to JsonPrimitive(2), "toY" to JsonPrimitive(3)), ) assertEquals(listOf(listOf(0, 1, 2, 3, 300)), controller.swipes) } @@ -71,7 +76,7 @@ class MirrorMcpCommandsTest { fun `swipe rejects negative coordinates`(): Unit = runBlocking { assertFailsWith { SwipeCommand(resolveDevice = { device }).execute( - JetWhaleMcpArguments(mapOf("fromX" to "0", "fromY" to "-5", "toX" to "2", "toY" to "3")), + args("fromX" to JsonPrimitive(0), "fromY" to JsonPrimitive(-5), "toX" to JsonPrimitive(2), "toY" to JsonPrimitive(3)), ) } assertTrue(controller.swipes.isEmpty()) @@ -81,7 +86,7 @@ class MirrorMcpCommandsTest { fun `swipe rejects non-positive duration`(): Unit = runBlocking { assertFailsWith { SwipeCommand(resolveDevice = { device }).execute( - JetWhaleMcpArguments(mapOf("fromX" to "0", "fromY" to "1", "toX" to "2", "toY" to "3", "durationMillis" to "0")), + args("fromX" to JsonPrimitive(0), "fromY" to JsonPrimitive(1), "toX" to JsonPrimitive(2), "toY" to JsonPrimitive(3), "durationMillis" to JsonPrimitive(0)), ) } assertTrue(controller.swipes.isEmpty()) @@ -89,7 +94,7 @@ class MirrorMcpCommandsTest { @Test fun `listDevices reports ids, platforms, and the selection`() = runBlocking { - val result = ListDevicesCommand(refreshDevices = { listOf(device) }, selectedDeviceId = { device.id }).execute(JetWhaleMcpArguments(emptyMap())) + val result = ListDevicesCommand(refreshDevices = { listOf(device) }, selectedDeviceId = { device.id }).execute(args()) val devices = Json.parseToJsonElement(result).jsonObject["devices"]!!.jsonArray val entry = devices.single().jsonObject assertEquals("emulator-5554", entry["deviceId"]!!.jsonPrimitive.content) @@ -100,7 +105,7 @@ class MirrorMcpCommandsTest { @Test fun `stopRecording returns the finished file path`() = runBlocking { val file = File("/tmp/jetwhale-mirror-test.mp4") - val result = StopRecordingCommand(stopRecording = { file }).execute(JetWhaleMcpArguments(emptyMap())) + val result = StopRecordingCommand(stopRecording = { file }).execute(args()) assertEquals(file.absolutePath, Json.parseToJsonElement(result).jsonObject["path"]!!.jsonPrimitive.content) } @@ -108,7 +113,7 @@ class MirrorMcpCommandsTest { fun `unknown device id surfaces as an argument error`(): Unit = runBlocking { assertFailsWith { TapCommand(resolveDevice = { throw JetWhaleMcpArgumentException("unknown deviceId") }) - .execute(JetWhaleMcpArguments(mapOf("deviceId" to "nope", "x" to "1", "y" to "1"))) + .execute(args("deviceId" to JsonPrimitive("nope"), "x" to JsonPrimitive(1), "y" to JsonPrimitive(1))) } } }