Merge pull request #22 from devswha/fix/browser-panel-keys

fix(browser): forward editing and navigation keys through the remote keyboard

Author: devswha

server/modules/automation/browser-sidecar-keys.test.ts

@@ -0,0 +1,31 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import { toPuppeteerKeyInput } from './browser-sidecar.js';
+
+test('maps editing and navigation keys to Puppeteer key inputs', () => {
+  for (const key of [
+    'Backspace', 'Delete', 'ArrowLeft', 'ArrowUp', 'ArrowRight', 'ArrowDown',
+    'Home', 'End', 'PageUp', 'PageDown', 'Tab', 'Enter', 'Escape', 'Shift',
+    'Control', 'Alt', 'Meta',
+  ]) {
+    assert.equal(toPuppeteerKeyInput(key), key);
+  }
+});
+
+test('maps DOM key aliases and printable characters to Puppeteer key inputs', () => {
+  assert.equal(toPuppeteerKeyInput('OS'), 'Meta');
+  assert.equal(toPuppeteerKeyInput('Esc'), 'Escape');
+  assert.equal(toPuppeteerKeyInput(' '), 'Space');
+  assert.equal(toPuppeteerKeyInput('a'), 'a');
+  assert.equal(toPuppeteerKeyInput('7'), '7');
+});
+
+test('maps function keys and rejects IME keys', () => {
+  for (let index = 1; index <= 12; index += 1) {
+    assert.equal(toPuppeteerKeyInput(`F${index}`), `F${index}`);
+  }
+  for (const key of ['Dead', 'Unidentified', 'Process']) {
+    assert.equal(toPuppeteerKeyInput(key), null);
+  }
+});

server/modules/automation/browser-sidecar.ts

@@ -6,6 +6,7 @@ import { mkdir } from 'node:fs/promises';
 import { homedir } from 'node:os';
 import { join } from 'node:path';
 import readline from 'node:readline';
+import { pathToFileURL } from 'node:url';
 
 import {
   Browser as BrowserBinary,
@@ -46,6 +47,24 @@ import {
 } from './browser-protocol.js';
 import { normalizeAutomationUrl } from './automation-url.js';
 
+const PUPPETEER_KEY_NAMES = new Set([
+  'Backspace', 'Tab', 'Enter', 'Escape', 'Shift', 'Control', 'Alt', 'Meta',
+  'CapsLock', 'Delete', 'Insert', 'Home', 'End', 'PageUp', 'PageDown',
+  'ArrowLeft', 'ArrowUp', 'ArrowRight', 'ArrowDown', 'NumLock', 'ScrollLock',
+  'Pause', 'PrintScreen', 'ContextMenu',
+]);
+
+export function toPuppeteerKeyInput(key: string, code?: string): string | null {
+  void code;
+  if (key === 'OS') return 'Meta';
+  if (key === 'Scroll') return 'ScrollLock';
+  if (key === 'Esc') return 'Escape';
+  if (key === 'Del') return 'Delete';
+  if (key === ' ') return 'Space';
+  if (PUPPETEER_KEY_NAMES.has(key) || /^F(?:[1-9]|1[0-2])$/.test(key) || key.length === 1) return key;
+  return null;
+}
+
 type Tab = {
   id: string;
   page: Page;
@@ -335,12 +354,18 @@ class BrowserRuntime {
     } else if (input.kind === 'text') {
       await cdp.send('Input.insertText', { text: input.text });
     } else {
-      await cdp.send('Input.dispatchKeyEvent', {
-        type: input.event === 'down' ? 'keyDown' : 'keyUp',
-        key: input.key,
-        code: input.code ?? input.key,
-        modifiers: input.modifiers ?? 0,
-      });
+      const keyInput = toPuppeteerKeyInput(input.key, input.code);
+      if (keyInput) {
+        if (input.event === 'down') await tab.page.keyboard.down(keyInput as KeyInput);
+        else await tab.page.keyboard.up(keyInput as KeyInput);
+      } else {
+        await cdp.send('Input.dispatchKeyEvent', {
+          type: input.event === 'down' ? 'keyDown' : 'keyUp',
+          key: input.key,
+          code: input.code ?? input.key,
+          modifiers: input.modifiers ?? 0,
+        });
+      }
     }
     return { accepted: true };
   }
@@ -770,15 +795,19 @@ function enqueue(frame: BrowserRequestFrame): void {
   globalRequestQueue = globalRequestQueue.then(() => handle(frame)).catch(reportQueueError);
 }
 
-readline.createInterface({ input: process.stdin, crlfDelay: Infinity }).on('line', (line) => {
-  try {
-    for (const frame of decoder.push(`${line}\n`)) {
-      if (frame.kind !== 'request') throw new Error('Only request frames are accepted.');
-      enqueue(frame);
+function runBrowserSidecarEntrypoint(): void {
+  readline.createInterface({ input: process.stdin, crlfDelay: Infinity }).on('line', (line) => {
+    try {
+      for (const frame of decoder.push(`${line}\n`)) {
+        if (frame.kind !== 'request') throw new Error('Only request frames are accepted.');
+        enqueue(frame);
+      }
+    } catch (error) {
+      reportQueueError(error);
     }
-  } catch (error) {
-    reportQueueError(error);
-  }
-});
+  });
+
+  emit('ready', { protocolVersion: BROWSER_PROTOCOL_VERSION });
+}
 
-emit('ready', { protocolVersion: BROWSER_PROTOCOL_VERSION });
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) runBrowserSidecarEntrypoint();

src/components/workspace/view/BrowserPanel.tsx

@@ -314,6 +314,7 @@ export default function BrowserPanel({ sessionId, navigationRequest, onNavigatio
   };
 
   const handleKeyDown = (event: KeyboardEvent<HTMLDivElement>) => {
+    if (event.nativeEvent.isComposing) return;
     if (event.metaKey || event.ctrlKey || event.altKey || event.key.length !== 1) {
       event.preventDefault();
       sendInput({ kind: 'key', event: 'down', key: event.key, code: event.code, modifiers: (event.altKey ? 1 : 0) | (event.ctrlKey ? 2 : 0) | (event.metaKey ? 4 : 0) | (event.shiftKey ? 8 : 0) });
@@ -469,6 +470,7 @@ export default function BrowserPanel({ sessionId, navigationRequest, onNavigatio
               });
             }}
             onPointerDown={(event) => {
+              surfaceRef.current?.focus();
               const point = framePoint(event);
               if (!point) return;
               event.currentTarget.setPointerCapture(event.pointerId);