From 0611863c785b56dba1927e9a5d347772bcf3b8c4 Mon Sep 17 00:00:00 2001 From: Ali Al Dallal Date: Sun, 30 Aug 2026 20:19:45 -0400 Subject: [PATCH] =?UTF-8?q?feat:=20board=20=E2=8C=98V=20lands=20clipboard?= =?UTF-8?q?=20files=20and=20screenshots=20(goal=200255)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Board-level paste of a files payload was an explicit no-op, recorded as 'real paths unreachable via the web Clipboard API'. True for the web API -- but the host pasteboard has them: a new clipboard.ReadFileURLs (JXA public.file-url per pasteboard item, same bridge as Types) feeds AtlasService.ReadPasteboardFilePaths (fail-closed nil on any error, undo-exempt read door), and the paste door routes the answer through landFiles for full drop parity -- Finder ⌘C lands exactly what dropping the file would. With no paths (a screenshot bitmap, a non-Mac host), the image File's own bytes land as an image object through the image tool's existing commit door at the pointer. Rider fix the gate itself caught: clipboard.WatchChanges no longer fires on a transition to empty text -- pbcopy's clear-then-set exposes a transient empty pasteboard to a concurrent poll (measured live at the watcher's own 20ms interval), and a non-text clipboard reads as empty too; both were spurious captures. e2e drives the bitmap branch window-level under the clipboard lock (plain text written to the real pasteboard first so the host path read honestly answers empty); a PATH-stripped unit pins the adapter's fail-closed error. The OS-bound half (real pasteboard flavors, real WKWebView ⌘V) enters testing.md's manual registry. Known-refused and deferred with a home (BACKLOG 0P0-PROMISE): file-promise drags (the post-screenshot floating thumbnail) are refused upstream by the windowing layer's NSFilenamesPboardType-only registration. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012im1JxQQV2ahnXzZDdVmZq --- .claude/rules/testing.md | 12 +++++ .../services/atlassvc/atlasservice.ts | 14 +++++ frontend/e2e/atlas-image-tool.spec.ts | 53 +++++++++++++++++++ frontend/src/atlas/useAtlasPaste.ts | 38 ++++++++++--- internal/adapters/clipboard/clipboard.go | 43 ++++++++++++++- internal/adapters/clipboard/clipboard_test.go | 15 ++++++ .../atlassvc/atlasservice_filedrop.go | 17 ++++++ internal/services/atlassvc/atlasundo_doors.go | 1 + userdocs/concepts/atlas.md | 11 ++-- userdocs/llms-full.txt | 11 ++-- 10 files changed, 198 insertions(+), 17 deletions(-) diff --git a/.claude/rules/testing.md b/.claude/rules/testing.md index ebd9fb8f..da03b145 100644 --- a/.claude/rules/testing.md +++ b/.claude/rules/testing.md @@ -279,6 +279,18 @@ at: verify by installing the ntfy Android app, subscribing to a paired device's URL, parking an approval, confirming the phone receives it backgrounded, and tapping lands on Review. + - **Board ⌘V with a REAL pasteboard: screenshot bitmap and Finder + ⌘C** (goal 0255, `ReadPasteboardFilePaths`/`clipboard.ReadFileURLs`) + — real pasteboard file flavors and WKWebView's own ⌘V event + delivery are both OS-bound (the e2e drives a synthesized + files-carrying paste and the fail-closed empty-paths branch, never + the real gesture). Verify on an installed build: ⌃⇧⌘4 a region + (clipboard screenshot) → ⌘V on the board lands the image at the + pointer; Finder ⌘C a .png → ⌘V lands an image object mirroring + the REAL file path; Finder ⌘C a .md → ⌘V lands a card, same as + dropping it. Known-refused, not a defect: dragging the + post-screenshot floating thumbnail shows the no-entry cursor + (upstream file-promise gap, BACKLOG 0P0-PROMISE). **Tests drive user primitives, not synthetic events.** An interaction test reaches behavior through the same primitives a user has — real diff --git a/frontend/bindings/github.com/alicoding/mill/internal/services/atlassvc/atlasservice.ts b/frontend/bindings/github.com/alicoding/mill/internal/services/atlassvc/atlasservice.ts index a48c1884..fcb35538 100644 --- a/frontend/bindings/github.com/alicoding/mill/internal/services/atlassvc/atlasservice.ts +++ b/frontend/bindings/github.com/alicoding/mill/internal/services/atlassvc/atlasservice.ts @@ -712,6 +712,20 @@ export function PromoteNote(noteID: string, kindID: string, title: string): $Can return $Call.ByID(881716522, noteID, kindID, title); } +/** + * ReadPasteboardFilePaths returns the real absolute paths of any files + * on the OS pasteboard (a Finder ⌘C) -- the half of a copied-file + * paste the web Clipboard API structurally can't deliver (it exposes + * bytes, never paths), so the board's paste door asks the host and + * routes the answer through the same landing pipeline a drop uses + * (goal 0255). Fail-closed: no osascript, no file flavor, or any + * error at all is an empty list -- the paste gesture then falls back + * to the pasted bytes or a no-op, never an error the user sees. + */ +export function ReadPasteboardFilePaths(): $CancellablePromise { + return $Call.ByID(1957268351); +} + /** * Redo re-applies the UI actor's last undone mark, forward, through the * same doors (ADR-0044 decision 3: "redo is the inverse's inverse, diff --git a/frontend/e2e/atlas-image-tool.spec.ts b/frontend/e2e/atlas-image-tool.spec.ts index e9e0b046..bb3353b1 100644 --- a/frontend/e2e/atlas-image-tool.spec.ts +++ b/frontend/e2e/atlas-image-tool.spec.ts @@ -1,10 +1,12 @@ import fs from 'node:fs' import os from 'node:os' import path from 'node:path' +import { execSync } from 'node:child_process' import { test, expect } from './fixtures/server' import { dragResizeHandle, nonSeededBoardObjects, openCard } from './fixtures/atlasBoard' import { deleteViaPageMenu } from './fixtures/atlasPage' import { contextMenu } from './fixtures/contextMenu' +import { withClipboardLock } from './fixtures/clipboardLock' import { ATLAS_KIND_TOPIC, selectKind } from './fixtures/kindPicker' // The image tool (goal 0169 slice 2, re-pointed by goal 0179 S1's own @@ -223,3 +225,54 @@ test('an image object can be resized by its own handle, and the size persists ac await menu.getByText('Delete', { exact: true }).click() await expect(reloaded).toHaveCount(0) }) + +// Regression (goal 0255): board-level ⌘V of a screenshot bitmap was an +// explicit no-op -- the window paste door returned on ANY files +// payload, recorded at the time as "real paths unreachable via the web +// Clipboard API". The door now asks the HOST pasteboard for real file +// paths first (Finder ⌘C gets full drop parity) and falls back to the +// bitmap's own bytes through the image tool's commit door. This test +// drives the bitmap fallback with NO popover open anywhere: the real +// pasteboard is first given plain text (lock held -- the host +// path-read touches the one real macOS pasteboard) so that read +// honestly answers empty, then a files-carrying paste is dispatched at +// the window. +test('pasting a screenshot bitmap directly on the board lands an image object', async ({ page }) => { + await withClipboardLock(async () => { + if (process.platform === 'darwin') { + execSync('pbcopy', { input: 'goal-0255-plain-text-baseline' }) + } + await page.goto('/') + await page.getByRole('link', { name: 'Atlas' }).click() + const board = page.getByTestId('atlas-board') + await expect(board).toBeVisible() + // Position gesture, not an interaction: the paste anchors at the + // last known pointer position (atlas-paste-convert.spec.ts's own + // convention -- nothing is clicked, so nothing needs actionability). + const bb = await board.boundingBox() + if (!bb) throw new Error('no board box') + // eslint-disable-next-line no-restricted-syntax -- pure pointer positioning, no interaction to check + await page.mouse.move(bb.x + bb.width * 0.55, bb.y + bb.height * 0.65) + + await page.evaluate((base64) => { + const bin = atob(base64) + const bytes = new Uint8Array(bin.length) + for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i) + const file = new File([bytes], 'screenshot.png', { type: 'image/png' }) + const dt = new DataTransfer() + dt.items.add(file) + window.dispatchEvent(new ClipboardEvent('paste', { clipboardData: dt, bubbles: true, cancelable: true })) + }, ONE_PIXEL_PNG_BASE64) + + const object = imageObjects(page) + await expect(object).toHaveCount(1) + await expect(object.locator('img')).toBeVisible() + // Nothing became a card, and no popover was involved. + await expect(page.getByTestId('atlas-note-card').filter({ hasText: 'Pasted image' })).toHaveCount(0) + await expect(page.getByTestId('atlas-image-input')).toHaveCount(0) + + await object.click() + await page.keyboard.press('Delete') + await expect(object).toHaveCount(0) + }) +}) diff --git a/frontend/src/atlas/useAtlasPaste.ts b/frontend/src/atlas/useAtlasPaste.ts index 309d5bed..849af5ef 100644 --- a/frontend/src/atlas/useAtlasPaste.ts +++ b/frontend/src/atlas/useAtlasPaste.ts @@ -4,6 +4,8 @@ import type { PasteResult } from '../../bindings/github.com/alicoding/mill/inter import { refreshAtlas } from './atlasStore' import { frameContainingPoint } from './atlasFramePoint' import { localPathFromPastedText } from './atlasCreateHelpers' +import { readClipboardImageFile } from '../shared/clipboardRead' +import { imageTool } from './tools/imageTool' import { modalSurfaceOpen } from '../shared/modalGate' import type { FrameBox } from './useAtlasDragFiling' @@ -70,14 +72,34 @@ export function useAtlasPaste({ topLevelBoxes, screenToFlowPosition, viewedID, o if (isEditableTarget(document.activeElement)) return const data = e.clipboardData if (!data) return - // A copied FILE (a Finder ⌘C) is meant to behave exactly like a - // file drop (LOCKED design §2b) -- real absolute paths for a - // pasted file are not resolvable through the standard Clipboard - // API (the same sandboxing that motivated the native drag-and-drop - // door for files), so this door is TEXT/HTML only for now; a - // copied file currently falls through as a no-op rather than - // faking a path. - if (data.files.length > 0) return + // A copied FILE (a Finder ⌘C) or a screenshot bitmap arrives as + // files, whose real paths the web Clipboard API structurally + // never exposes -- so the HOST pasteboard supplies them + // (ReadPasteboardFilePaths, goal 0255) and the paste lands with + // full drop parity through landFiles. No paths (a pure bitmap, + // or a non-Mac host) falls back to the image File's own bytes + // through the image tool's commit door. The File ref is taken + // synchronously: clipboardData is transient after the handler + // returns. + if (data.files.length > 0) { + const imageFile = readClipboardImageFile(data) + e.preventDefault() + const filesAnchor = lastMouse.current ?? { x: window.innerWidth / 2, y: window.innerHeight / 2 } + void AtlasService.ReadPasteboardFilePaths() + .then((paths) => { + if (paths && paths.length > 0) return stateRef.current.landFiles(paths, filesAnchor) + if (!imageFile) return + const { topLevelBoxes: boxes, screenToFlowPosition: toFlow, viewedID: viewed } = stateRef.current + const pos = toFlow(filesAnchor) + const parent = frameContainingPoint(boxes, pos) ?? viewed + return imageTool + .commit({ file: imageFile, title: 'Pasted image' }) + .then((artifact) => AtlasService.CreateBoardObject('image', { mirrorPath: artifact.mirrorPath, title: artifact.title }, { X: pos.x, Y: pos.y }, parent)) + .then(() => refreshAtlas()) + }) + .catch((err) => console.error('pasted file landing failed', err)) + return + } const html = data.getData('text/html') const text = data.getData('text/plain') if (!html && !text) return diff --git a/internal/adapters/clipboard/clipboard.go b/internal/adapters/clipboard/clipboard.go index 9e6594de..4f32903b 100644 --- a/internal/adapters/clipboard/clipboard.go +++ b/internal/adapters/clipboard/clipboard.go @@ -163,7 +163,15 @@ func WatchChanges(interval time.Duration, fn func(text string)) (stop func()) { } if text != last { last = text - fn(text) + // A transition to EMPTY never fires: pbcopy's own + // clear-then-set exposes a transient empty pasteboard + // to a concurrent poll (measured live: baseline -> + // "" -> new value at a 20ms interval), and a + // non-text clipboard (an image copy) also reads as + // empty -- neither is a text change worth capturing. + if text != "" { + fn(text) + } } case <-done: return @@ -194,6 +202,39 @@ func Types() ([]string, error) { return types, nil } +// ReadFileURLs returns the absolute filesystem paths of any files on +// the pasteboard (a Finder ⌘C), via the same JXA/NSPasteboard bridge +// Types uses -- the web Clipboard API exposes a pasted file's BYTES +// but never its real path, so the board's paste door has to ask the +// host for the paths a file drop would have delivered. Empty (with the +// error) wherever osascript is absent or no file flavor exists -- +// callers treat any error as "no files", never a user-facing failure. +func ReadFileURLs() ([]string, error) { + ctx, cancel := context.WithTimeout(context.Background(), cmdTimeout) + defer cancel() + const script = `ObjC.import('AppKit'); +const pb = $.NSPasteboard.generalPasteboard; +const out = []; +const items = pb.pasteboardItems; +for (let i = 0; i < items.count; i++) { + const s = items.objectAtIndex(i).stringForType('public.file-url'); + if (!s.isNil()) { + const u = $.NSURL.URLWithString(s); + if (!u.isNil() && !u.path.isNil()) out.push(ObjC.unwrap(u.path)); + } +} +JSON.stringify(out)` + out, err := exec.CommandContext(ctx, "osascript", "-l", "JavaScript", "-e", script).Output() + if err != nil { + return nil, fmt.Errorf("osascript pasteboard file urls failed: %w", err) + } + var paths []string + if err := json.Unmarshal(out, &paths); err != nil { + return nil, fmt.Errorf("decode pasteboard file urls: %w", err) + } + return paths, nil +} + // concealedTypes are the nspasteboard.org convention's own markers // (https://nspasteboard.org) a password manager or transient-content // source sets on the pasteboard to declare "don't record this in a diff --git a/internal/adapters/clipboard/clipboard_test.go b/internal/adapters/clipboard/clipboard_test.go index 047e79c0..8ecb3b41 100644 --- a/internal/adapters/clipboard/clipboard_test.go +++ b/internal/adapters/clipboard/clipboard_test.go @@ -173,3 +173,18 @@ func TestConsumeSelfWrite_NoMatchLeavesMarkerIntact(t *testing.T) { t.Error("ConsumeSelfWrite() = false for the actual self-written text after a non-matching call, want true (marker must survive a non-match)") } } + +// ReadFileURLs must fail closed where osascript is unavailable (a CI +// runner, a stripped PATH) -- the paste door treats any error as "no +// files on the pasteboard", so the error must actually surface rather +// than a panic or a fabricated result. +func TestReadFileURLs_FailsClosedWithoutOsascript(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + paths, err := ReadFileURLs() + if err == nil { + t.Fatal("ReadFileURLs() with no osascript on PATH: expected an error") + } + if paths != nil { + t.Fatalf("ReadFileURLs() with no osascript on PATH: paths = %v, want nil", paths) + } +} diff --git a/internal/services/atlassvc/atlasservice_filedrop.go b/internal/services/atlassvc/atlasservice_filedrop.go index 11711cea..473ccd3b 100644 --- a/internal/services/atlassvc/atlasservice_filedrop.go +++ b/internal/services/atlassvc/atlasservice_filedrop.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/alicoding/mill/internal/adapters/clipboard" "github.com/alicoding/mill/internal/adapters/markdown" "github.com/alicoding/mill/internal/adapters/windowing" "github.com/alicoding/mill/internal/domain/atlas" @@ -288,6 +289,22 @@ func (a *AtlasService) ConvertHTMLToMarkdown(html string) (string, error) { return markdown.ToMarkdown(html) } +// ReadPasteboardFilePaths returns the real absolute paths of any files +// on the OS pasteboard (a Finder ⌘C) -- the half of a copied-file +// paste the web Clipboard API structurally can't deliver (it exposes +// bytes, never paths), so the board's paste door asks the host and +// routes the answer through the same landing pipeline a drop uses +// (goal 0255). Fail-closed: no osascript, no file flavor, or any +// error at all is an empty list -- the paste gesture then falls back +// to the pasted bytes or a no-op, never an error the user sees. +func (a *AtlasService) ReadPasteboardFilePaths() []string { + paths, err := clipboard.ReadFileURLs() + if err != nil { + return nil + } + return paths +} + // WireFileDropWindow relays window's own native OS file-drop events // (main.go's EnableFileDrop) to the frontend as FileDropEventName -- // main.go's only call into this file, right after the window itself is diff --git a/internal/services/atlassvc/atlasundo_doors.go b/internal/services/atlassvc/atlasundo_doors.go index 6da7dc35..8b3e4ac9 100644 --- a/internal/services/atlassvc/atlasundo_doors.go +++ b/internal/services/atlassvc/atlasundo_doors.go @@ -127,6 +127,7 @@ var notMutationDoors = map[string]string{ "Perspectives": "read", "PickDiagramFile": "native file picker, no board-state change", "PickFolder": "native folder picker, no board-state change", "PickImageFile": "native file picker, no board-state change", "PreviewClipbridgeReply": "read/preview", "RenderNoteMarkdown": "pure conversion, no state", + "ReadPasteboardFilePaths": "read/pasteboard, no board-state change", "ResolveFileDropRoute": "read/route decision", "RevealCardMirror": "OS reveal-in-Finder, no board-state change", "RevealSpaceFolder": "OS reveal-in-Finder, no board-state change", "ScanFolder": "read/scan", "SpaceBundleContext": "read/export text", "SpaceContextEnvelope": "read/export text", "SpaceLinksList": "read/export text", diff --git a/userdocs/concepts/atlas.md b/userdocs/concepts/atlas.md index e6f2a965..feba2a20 100644 --- a/userdocs/concepts/atlas.md +++ b/userdocs/concepts/atlas.md @@ -51,10 +51,13 @@ connected by links, grouped into areas you can drill into. ## Drawing and images -- **Images and ink live on the board, not inside a card.** Pick Image - in the toolbar, choose a file, or paste a screenshot — it lands at - its own size, right where you put it. Pasting an image file's path - as text works too: the board shows the image from that file. +- **Images and ink live on the board, not inside a card.** Take a + screenshot to the clipboard and press ⌘V on the board — the image + lands at your pointer, at its own size. Copy a file in Finder and + paste it, and it lands exactly the way dropping it would: an image + as an image, a diagram as a diagram, a document as a card. The + Image button in the toolbar offers a file picker and a paste zone + too, and pasting an image file's path as text also works. Dropping an image file onto the board does the same. Pick Pencil and drag to draw; lift and draw again for the next stroke, no interruption. Either one is a thing in space you can move, select, and delete, and ink stays visually on diff --git a/userdocs/llms-full.txt b/userdocs/llms-full.txt index 9d59c6ef..aee55383 100644 --- a/userdocs/llms-full.txt +++ b/userdocs/llms-full.txt @@ -361,10 +361,13 @@ connected by links, grouped into areas you can drill into. ## Drawing and images -- **Images and ink live on the board, not inside a card.** Pick Image - in the toolbar, choose a file, or paste a screenshot — it lands at - its own size, right where you put it. Pasting an image file's path - as text works too: the board shows the image from that file. +- **Images and ink live on the board, not inside a card.** Take a + screenshot to the clipboard and press ⌘V on the board — the image + lands at your pointer, at its own size. Copy a file in Finder and + paste it, and it lands exactly the way dropping it would: an image + as an image, a diagram as a diagram, a document as a card. The + Image button in the toolbar offers a file picker and a paste zone + too, and pasting an image file's path as text also works. Dropping an image file onto the board does the same. Pick Pencil and drag to draw; lift and draw again for the next stroke, no interruption. Either one is a thing in space you can move, select, and delete, and ink stays visually on