feat(mirror): add Device Mirror host plugin

gradle/libs.versions.toml

@@ -11,6 +11,8 @@ kotlinxCollectionsImmutable = "0.5.1"
 kotlinxCoroutines = "1.11.0"
 byteBuddy = "1.18.11"
 javaKeyring = "1.0.4"
+javacv = "1.5.10"
+bytedecoFfmpeg = "6.1.1-1.5.10"
 kotlinxDatetime = "0.8.0"
 
 # android
@@ -58,6 +60,8 @@ kotlinTest = { module = "org.jetbrains.kotlin:kotlin-test", version.ref = "kotli
 # kotlinx
 kotlinxSerializationCore = { module = "org.jetbrains.kotlinx:kotlinx-serialization-core", version.ref = "kotlinxSerializationJson" }
 kotlinxSerializationJson = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" }
+javacv = { module = "org.bytedeco:javacv", version.ref = "javacv" }
+bytedecoFfmpeg = { module = "org.bytedeco:ffmpeg", version.ref = "bytedecoFfmpeg" }
 kotlinxCollectionsImmutable = { module = "org.jetbrains.kotlinx:kotlinx-collections-immutable", version.ref = "kotlinxCollectionsImmutable" }
 kotlinxCoroutinesCore = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "kotlinxCoroutines" }
 byteBuddyAgent = { module = "net.bytebuddy:byte-buddy-agent", version.ref = "byteBuddy" }

jetwhale-plugins/mirror/host/build.gradle.kts

@@ -0,0 +1,61 @@
+plugins {
+    alias(libs.plugins.jvm)
+    alias(libs.plugins.composeCompiler)
+    alias(libs.plugins.jetbrainsCompose)
+    // Provides packagePlugin / installPlugin / stageDevPlugin / runJetWhale / runJetWhaleHot (published).
+    alias(libs.plugins.jetwhalePlugin)
+    // In-repo only: adds runJetWhaleLocal, which launches the local :jetwhale-host:app project.
+    alias(libs.plugins.jetwhaleHostLaunch)
+    alias(libs.plugins.publish)
+}
+
+// Distinct group so this module's coordinates don't collide with the other `host` plugin modules.
+group = "com.kitakkun.jetwhale.plugins.mirror"
+
+jetwhalePlugin {
+    // Unique name so the packaged plugin jar doesn't collide with the other plugin modules (also
+    // project name "host") in ~/.jetwhale/plugins/ or the dev staging directory.
+    pluginArchiveName.set("jetwhale-device-mirror")
+}
+
+dependencies {
+    // Provided by the host at runtime, so compileOnly: these must be neither bundled into the
+    // plugin jar nor listed in its dependency manifest.
+    compileOnly(projects.jetwhaleHostSdk)
+    compileOnly(compose.desktop.currentOs)
+    compileOnly(libs.material3)
+    compileOnly(libs.kotlinxSerializationJson)
+    // H.264 mirror-stream decoding (adb screenrecord / idb video-stream). Bundled into the
+    // plugin jar; the ffmpeg natives are restricted to the build machine's platform to keep the
+    // artifact from carrying every OS's binaries.
+    implementation(libs.javacv) {
+        // javacv declares every bytedeco preset; only the ffmpeg bindings are used here.
+        isTransitive = false
+    }
+    implementation("org.bytedeco:javacpp:${libs.versions.javacv.get()}")
+    implementation(libs.bytedecoFfmpeg)
+    implementation("org.bytedeco:ffmpeg:${libs.versions.bytedecoFfmpeg.get()}:${currentFfmpegClassifier()}")
+    testImplementation(projects.jetwhaleHostSdk)
+    testImplementation(libs.kotlinTest)
+    testImplementation(libs.kotlinxSerializationJson)
+    testImplementation(compose.desktop.currentOs)
+    testImplementation(libs.material3)
+}
+
+fun currentFfmpegClassifier(): String {
+    val osName = System.getProperty("os.name").lowercase()
+    val arch = System.getProperty("os.arch").lowercase()
+    return when {
+        osName.contains("mac") && arch == "aarch64" -> "macosx-arm64"
+        osName.contains("mac") -> "macosx-x86_64"
+        osName.contains("windows") -> "windows-x86_64"
+        arch == "aarch64" -> "linux-arm64"
+        else -> "linux-x86_64"
+    }
+}
+
+jetwhalePublish {
+    artifactId = "jetwhale-device-mirror"
+    name = "JetWhale Device Mirror"
+    description = "JetWhale host plugin that mirrors Android emulator / iOS simulator screens and drives them interactively or via MCP."
+}

jetwhale-plugins/mirror/host/src/main/kotlin/com/kitakkun/jetwhale/plugins/mirror/host/CaptureScreenshotCommand.kt

@@ -0,0 +1,36 @@
+package com.kitakkun.jetwhale.plugins.mirror.host
+
+import com.kitakkun.jetwhale.host.sdk.ExperimentalJetWhaleApi
+import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArgumentException
+import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpArguments
+import com.kitakkun.jetwhale.host.sdk.JetWhaleMcpCommand
+import kotlinx.serialization.json.buildJsonObject
+import kotlinx.serialization.json.put
+import org.jetbrains.skia.Image as SkiaImage
+
+@OptIn(ExperimentalJetWhaleApi::class)
+internal class CaptureScreenshotCommand(
+    private val resolveDevice: (deviceId: String?) -> MirrorDevice,
+) : JetWhaleMcpCommand() {
+    override val name = "$TOOL_PREFIX.captureScreenshot"
+    override val description = "Captures the device screen as a PNG file and returns its absolute path plus the pixel dimensions. Coordinates passed to the tap/swipe tools are in this pixel coordinate space. Read the file to see the screen."
+
+    private val deviceId by stringOrNull("Target device id from $TOOL_PREFIX.listDevices; omitted = the device selected in the mirror UI.")
+
+    override suspend fun execute(arguments: JetWhaleMcpArguments): String {
+        val device = resolveDevice(arguments[deviceId])
+        val png = try {
+            device.controller.captureScreenshot()
+        } catch (e: DeviceControlException) {
+            throw JetWhaleMcpArgumentException(e.message ?: "screenshot failed")
+        }
+        val file = screenshotFile(device)
+        file.writeBytes(png)
+        val image = SkiaImage.makeFromEncoded(png)
+        return buildJsonObject {
+            put("path", file.absolutePath)
+            put("widthPx", image.width)
+            put("heightPx", image.height)
+        }.toString()
+    }
+}