diff --git a/CONTEXT.md b/CONTEXT.md index 0aa6f23..7b935b8 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -10,6 +10,10 @@ A local-first voice dictation app for macOS. The user holds a key, speaks, and t One complete act of speaking and having the resulting text placed at the cursor. Begins when the user presses the push-to-talk key and ends when the text lands. _Avoid_: Utterance, recording, session +**File transcription**: +A second, distinct mode (#253): the user drops or picks an existing audio file and gets a transcript back, rather than speaking live at the cursor. Shares the transcription engine and its resident model with Dictation, but nothing else - no cursor, no cleanup rules, no break placement, no push-to-talk. One file transcribes at a time; a long one can transiently delay a concurrent Dictation, since both use the same resident model. +_Avoid_: Batch transcription, offline transcription (accurate but not the term used here), import + **Dictation latency budget**: The time from the end of speech to the text arriving at the cursor, which the product commits to keeping under one second. It ends where the user can see it end, so the cost of placing the text is inside the budget rather than outside it. _Avoid_: Response time, turnaround diff --git a/docs/testing/file-transcription-253.md b/docs/testing/file-transcription-253.md new file mode 100644 index 0000000..4c90cb1 --- /dev/null +++ b/docs/testing/file-transcription-253.md @@ -0,0 +1,56 @@ +# Manual check: drop-a-file transcription (#253) + +CI covers the queue coordinator (`fileTranscriptionQueue.test.js`) and the +`transcribeFile` request/reply shape (`transcriptionHelper.test.js`) against +fakes. What it can't drive: the real Swift command, drag-and-drop, the file +picker, or a real audio file actually decoding. + +**The Swift side (`transcribeFile` in `native/transcription-helper`) is +compile-verified only** - it was never executed in the session that wrote it; +every attempt to run `transcription-helper` there got killed by the sandbox +within seconds. Run `npm run build:transcription-helper` before testing any +of this - the checked-in `resources/bin/transcription-helper` predates the +change. + +## Steps + +1. **Basic drop.** Open the **Files** tab. Drag a short `.wav` or `.m4a` + file (a Voice Memo works) onto the drop zone - it highlights on + dragover, and the file appears in the queue as "Transcribing…" then + "Done" with a transcript. +2. **The file picker.** Click **Choose files…**, pick 2-3 files at once + (multi-select) in different formats - `.wav`, `.m4a`, `.mp3` if you + have one handy. All three should queue and process one at a time, in + the order picked - confirm only one row ever shows "Transcribing…" at + once. +3. **A long file.** Drop something over ~30s (FluidAudio's own streaming + threshold) - a longer Voice Memo or an exported meeting recording. + Confirm it still produces a full transcript, not just the first ~30s - + this is what confirms the file-URL `transcribe()` overload is really + auto-chunking rather than silently truncating. +4. **A bad file.** Drop something that isn't audio (a `.txt`, a `.jpg`). + The row should land on "Failed" with a readable error, not crash the + queue or the app - and the next queued file (if any) should still run. +5. **Copy / Save.** On a done job, click **Copy** - paste somewhere to + confirm the clipboard has the transcript. Click **Save** - a `.txt` + with the same base name should appear next to the source file; open it + to confirm the text matches. Save again - it overwrites without asking + (deliberate, see the #253 PR). +6. **Retry / Remove.** On a failed job, **Retry** re-queues it. On any + job that isn't currently transcribing, **Remove** drops it from the + list. Confirm **Remove** is unavailable (or a no-op) on the row that's + actively transcribing. +7. **Clear finished.** With a mix of done/failed/queued jobs, **Clear + finished** should drop the done and failed rows and leave anything + still queued or transcribing. +8. **Concurrency with live dictation - the known tradeoff.** Drop a longish + file, and while it's transcribing, hold push-to-talk and dictate + normally. Confirm what actually happens: does the live dictation wait + behind the file job (expected, per the #253 PR), or does something worse + happen (a crash, a lost dictation, the overlay hanging)? This is the one + behaviour the PR flagged as accepted-but-unverified. +9. **No persistence across restart.** Quit and relaunch with jobs still in + the queue - it should come back empty. This is deliberate for v1 (no + backing store yet - see #253's PR for the "ties into #136" follow-up), + confirm it's not silently losing an in-flight job's *file* (the source + file on disk is never touched, only the in-app queue state resets). diff --git a/electron/fileTranscriptionQueue.js b/electron/fileTranscriptionQueue.js new file mode 100644 index 0000000..24079e5 --- /dev/null +++ b/electron/fileTranscriptionQueue.js @@ -0,0 +1,132 @@ +// #253: drop-a-file transcription. A small FIFO queue over the same +// resident transcription-helper live dictation uses (see transcribeFile() +// in transcriptionHelper.js) - one job at a time, since the helper only +// ever runs one transcription at a time regardless (it reads one +// newline-delimited request, blocks until the model replies, reads the +// next). A long file job can delay a concurrent live dictation; that is a +// known, accepted tradeoff for now (see the #253 PR), not something this +// module tries to work around - a second resident model would double the +// memory #257 exists to cut down. +// +// Pure and adapter-injected, same ports-and-adapters shape as +// dictationCoordinator.js: `transcription` is asserted, `onChange` is +// optional and fires with a full snapshot after every state change so a +// caller (main.js) can just re-render from the latest array rather than +// diffing. + +function createFileTranscriptionQueue(options = {}) { + const { transcription, language = { get: () => "auto" }, onChange = () => {} } = options; + + if (!transcription || typeof transcription.transcribeFile !== "function") { + throw new Error("createFileTranscriptionQueue requires transcription.transcribeFile()"); + } + + let jobs = []; + let nextId = 1; + let processing = false; + + function snapshot() { + return jobs.map((job) => ({ ...job })); + } + + function notify() { + try { + onChange(snapshot()); + } catch { + // A listener must never break the queue. + } + } + + function baseName(filePath) { + const parts = String(filePath).split("/"); + return parts[parts.length - 1] || filePath; + } + + // Runs until the queue is drained. Only one call is ever "live" at a + // time - the `processing` guard means a second addFiles() while a job is + // in flight just returns, and the in-flight call's own loop picks up + // whatever got added. + async function drain() { + if (processing) return; + processing = true; + try { + let job; + while ((job = jobs.find((candidate) => candidate.status === "queued"))) { + job.status = "transcribing"; + job.startedAt = Date.now(); + notify(); + try { + job.text = await transcription.transcribeFile(job.path, language.get()); + job.status = "done"; + } catch (error) { + job.status = "failed"; + job.error = error instanceof Error ? error.message : String(error); + } + job.finishedAt = Date.now(); + notify(); + } + } finally { + processing = false; + } + } + + function addFiles(paths) { + const added = []; + for (const filePath of Array.isArray(paths) ? paths : []) { + if (typeof filePath !== "string" || !filePath) continue; + const job = { + id: String(nextId++), + path: filePath, + name: baseName(filePath), + status: "queued", + text: null, + error: null, + queuedAt: Date.now(), + startedAt: null, + finishedAt: null, + }; + jobs.push(job); + added.push(job.id); + } + if (added.length > 0) { + notify(); + void drain(); + } + return added; + } + + function retry(id) { + const job = jobs.find((candidate) => candidate.id === id); + if (!job || job.status !== "failed") return false; + job.status = "queued"; + job.text = null; + job.error = null; + job.startedAt = null; + job.finishedAt = null; + notify(); + void drain(); + return true; + } + + function remove(id) { + const index = jobs.findIndex((candidate) => candidate.id === id); + if (index === -1 || jobs[index].status === "transcribing") return false; + jobs.splice(index, 1); + notify(); + return true; + } + + function clearFinished() { + const before = jobs.length; + jobs = jobs.filter((job) => job.status === "queued" || job.status === "transcribing"); + if (jobs.length !== before) notify(); + } + + function getJobs() { + return snapshot(); + } + + return { addFiles, retry, remove, clearFinished, getJobs }; +} + +module.exports = { createFileTranscriptionQueue }; diff --git a/electron/fileTranscriptionQueue.test.js b/electron/fileTranscriptionQueue.test.js new file mode 100644 index 0000000..7631a48 --- /dev/null +++ b/electron/fileTranscriptionQueue.test.js @@ -0,0 +1,210 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); +const { createFileTranscriptionQueue } = require("./fileTranscriptionQueue"); + +function deferred() { + let resolve, reject; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +function nextTurn() { + return new Promise((resolve) => setImmediate(resolve)); +} + +test("requires a transcription.transcribeFile adapter", () => { + assert.throws(() => createFileTranscriptionQueue({}), /transcribeFile/); + assert.throws(() => createFileTranscriptionQueue({ transcription: {} }), /transcribeFile/); +}); + +test("addFiles queues jobs and returns their ids", () => { + const queue = createFileTranscriptionQueue({ + transcription: { transcribeFile: async () => "text" }, + }); + const ids = queue.addFiles(["/a/one.wav", "/a/two.m4a"]); + assert.equal(ids.length, 2); + const jobs = queue.getJobs(); + assert.equal(jobs.length, 2); + assert.equal(jobs[0].name, "one.wav"); + assert.equal(jobs[1].name, "two.m4a"); +}); + +test("addFiles ignores non-string / empty entries without throwing", () => { + const queue = createFileTranscriptionQueue({ + transcription: { transcribeFile: async () => "text" }, + }); + const ids = queue.addFiles(["/a/one.wav", "", null, 42, undefined]); + assert.equal(ids.length, 1); + assert.equal(queue.getJobs().length, 1); +}); + +test("processes one file at a time, in FIFO order", async () => { + const calls = []; + const first = deferred(); + const transcribeFile = (path) => { + calls.push(path); + if (calls.length === 1) return first.promise; + return Promise.resolve(`text for ${path}`); + }; + const seen = []; + const queue = createFileTranscriptionQueue({ + transcription: { transcribeFile }, + onChange: (jobs) => seen.push(jobs.map((j) => j.status)), + }); + + queue.addFiles(["/a.wav", "/b.wav"]); + await nextTurn(); + + // Only the first job has started - the second is still queued behind it. + assert.deepEqual(calls, ["/a.wav"]); + assert.deepEqual( + queue.getJobs().map((j) => j.status), + ["transcribing", "queued"], + ); + + first.resolve("text for /a.wav"); + await nextTurn(); + await nextTurn(); + + assert.deepEqual(calls, ["/a.wav", "/b.wav"]); + assert.deepEqual( + queue.getJobs().map((j) => j.status), + ["done", "done"], + ); + assert.equal(queue.getJobs()[0].text, "text for /a.wav"); + assert.equal(queue.getJobs()[1].text, "text for /b.wav"); +}); + +test("a failed job records the error and does not block the rest of the queue", async () => { + const queue = createFileTranscriptionQueue({ + transcription: { + transcribeFile: async (path) => { + if (path === "/bad.wav") throw new Error("no file at /bad.wav"); + return "ok text"; + }, + }, + }); + + const [, secondId] = queue.addFiles(["/bad.wav", "/good.wav"]); + await nextTurn(); + await nextTurn(); + + const jobs = queue.getJobs(); + assert.equal(jobs[0].status, "failed"); + assert.equal(jobs[0].error, "no file at /bad.wav"); + const second = jobs.find((j) => j.id === secondId); + assert.equal(second.status, "done"); + assert.equal(second.text, "ok text"); +}); + +test("the configured language is forwarded to transcribeFile", async () => { + const seen = []; + const queue = createFileTranscriptionQueue({ + transcription: { transcribeFile: async (_path, lang) => { seen.push(lang); return "x"; } }, + language: { get: () => "fr" }, + }); + queue.addFiles(["/a.wav"]); + await nextTurn(); + assert.deepEqual(seen, ["fr"]); +}); + +test("retry re-queues a failed job and clears its error", async () => { + let attempt = 0; + const queue = createFileTranscriptionQueue({ + transcription: { + transcribeFile: async () => { + attempt += 1; + if (attempt === 1) throw new Error("first attempt failed"); + return "second time lucky"; + }, + }, + }); + + const [id] = queue.addFiles(["/a.wav"]); + await nextTurn(); + assert.equal(queue.getJobs()[0].status, "failed"); + + assert.equal(queue.retry(id), true); + await nextTurn(); + + const job = queue.getJobs()[0]; + assert.equal(job.status, "done"); + assert.equal(job.text, "second time lucky"); + assert.equal(job.error, null); +}); + +test("retry refuses a job that is not failed", async () => { + const queue = createFileTranscriptionQueue({ + transcription: { transcribeFile: async () => "text" }, + }); + const [id] = queue.addFiles(["/a.wav"]); + await nextTurn(); + assert.equal(queue.getJobs()[0].status, "done"); + assert.equal(queue.retry(id), false); +}); + +test("remove drops a queued or finished job but refuses one in flight", async () => { + const first = deferred(); + const queue = createFileTranscriptionQueue({ + transcription: { transcribeFile: () => first.promise }, + }); + const [id] = queue.addFiles(["/a.wav"]); + await nextTurn(); + assert.equal(queue.getJobs()[0].status, "transcribing"); + assert.equal(queue.remove(id), false); + + first.resolve("done text"); + await nextTurn(); + assert.equal(queue.remove(id), true); + assert.equal(queue.getJobs().length, 0); +}); + +test("clearFinished drops done and failed jobs, leaves queued/transcribing alone", async () => { + const second = deferred(); + const transcribeFile = (path) => (path === "/slow.wav" ? second.promise : Promise.reject(new Error("nope"))); + const queue = createFileTranscriptionQueue({ transcription: { transcribeFile } }); + + queue.addFiles(["/fails.wav", "/slow.wav"]); + await nextTurn(); + await nextTurn(); + + assert.deepEqual( + queue.getJobs().map((j) => j.status), + ["failed", "transcribing"], + ); + queue.clearFinished(); + assert.deepEqual( + queue.getJobs().map((j) => j.status), + ["transcribing"], + ); + + second.resolve("ok"); + await nextTurn(); +}); + +test("onChange fires with a full snapshot on every transition", async () => { + const snapshots = []; + const queue = createFileTranscriptionQueue({ + transcription: { transcribeFile: async () => "text" }, + onChange: (jobs) => snapshots.push(jobs.map((j) => j.status)), + }); + queue.addFiles(["/a.wav"]); + await nextTurn(); + // queued (from addFiles) -> transcribing -> done + assert.deepEqual(snapshots, [["queued"], ["transcribing"], ["done"]]); +}); + +test("a listener that throws never breaks the queue", async () => { + const queue = createFileTranscriptionQueue({ + transcription: { transcribeFile: async () => "text" }, + onChange: () => { + throw new Error("boom"); + }, + }); + queue.addFiles(["/a.wav"]); + await nextTurn(); + assert.equal(queue.getJobs()[0].status, "done"); +}); diff --git a/electron/main.js b/electron/main.js index 840a4f4..fa0634e 100644 --- a/electron/main.js +++ b/electron/main.js @@ -35,6 +35,7 @@ const { createBundleIdReader } = require("./appBundleId"); const { createBreakPlacementHttpAdapter } = require("./breakPlacementHttpAdapter"); const { createDictationIntake } = require("./dictationCoordinator"); const { createVoiceEditIntake } = require("./voiceEditCoordinator"); +const { createFileTranscriptionQueue } = require("./fileTranscriptionQueue"); const { createVocabularyCache } = require("./vocabularyCache"); const { createHeldResultController } = require("./heldResultController"); const { createPushToTalkCoordinator } = require("./pushToTalkCoordinator"); @@ -245,6 +246,18 @@ const voiceEditIntake = createVoiceEditIntake({ onDiagnostic: (name, value) => console.log(`[voice-edit] ${name}: ${JSON.stringify(value)}`), }); +// #253: drop-a-file transcription, a second entry point onto the same +// resident transcription-helper. See fileTranscriptionQueue.js for why a +// long file job can transiently delay a concurrent live dictation - a +// known tradeoff, not something worked around here. +const fileTranscriptionQueue = createFileTranscriptionQueue({ + transcription, + language: { get: () => (settingsStore ? settingsStore.get().inputLanguage : "en") }, + onChange: (jobs) => { + if (win && !win.isDestroyed()) win.webContents.send("file-transcription:queue", jobs); + }, +}); + // A selection longer than this is almost never a deliberate edit target, // and reading a huge AX value at key-down is exactly the case the // injection engine's own max-chars guard exists to avoid. @@ -1136,6 +1149,53 @@ ipcMain.handle("settings:pick-post-process-script", async () => { return settingsStore.setPostProcessScriptPath(result.filePaths[0]); }); +// #253: drop-a-file transcription. addFiles is idempotent-safe to call with +// whatever the renderer resolved from a drop or the file picker below - a +// non-string/empty entry is silently skipped by the queue, not an error. +ipcMain.handle("file-transcription:add", (event, filePaths) => { + return fileTranscriptionQueue.addFiles(Array.isArray(filePaths) ? filePaths : []); +}); + +ipcMain.handle("file-transcription:choose-files", async () => { + if (!win) return []; + const result = await dialog.showOpenDialog(win, { + properties: ["openFile", "multiSelections"], + filters: [ + { name: "Audio", extensions: ["wav", "aiff", "aif", "caf", "m4a", "mp4", "mp3", "aac", "flac"] }, + { name: "All Files", extensions: ["*"] }, + ], + }); + if (result.canceled) return []; + return fileTranscriptionQueue.addFiles(result.filePaths); +}); + +ipcMain.handle("file-transcription:get-queue", () => fileTranscriptionQueue.getJobs()); +ipcMain.handle("file-transcription:retry", (event, id) => fileTranscriptionQueue.retry(id)); +ipcMain.handle("file-transcription:remove", (event, id) => fileTranscriptionQueue.remove(id)); +ipcMain.handle("file-transcription:clear-finished", () => fileTranscriptionQueue.clearFinished()); + +ipcMain.handle("file-transcription:copy", (event, id) => { + const job = fileTranscriptionQueue.getJobs().find((candidate) => candidate.id === id); + if (!job || typeof job.text !== "string") return { ok: false, reason: "no transcript to copy" }; + clipboard.writeText(job.text); + return { ok: true }; +}); + +// #253: "save alongside the file" - same directory, same name, .txt +// extension. An explicit user action (they clicked Save), so a repeat +// click overwriting its own earlier save is expected, not guarded against. +ipcMain.handle("file-transcription:save", (event, id) => { + const job = fileTranscriptionQueue.getJobs().find((candidate) => candidate.id === id); + if (!job || typeof job.text !== "string") return { ok: false, reason: "no transcript to save" }; + const outPath = path.join(path.dirname(job.path), `${path.basename(job.path, path.extname(job.path))}.txt`); + try { + fs.writeFileSync(outPath, job.text, "utf8"); + return { ok: true, path: outPath }; + } catch (error) { + return { ok: false, reason: error instanceof Error ? error.message : String(error) }; + } +}); + // #19: pick an app from disk instead of hunting down its bundle id by // hand. Returns { bundleId, name } for the renderer to add, or null if the // dialog was cancelled; a bundle with no readable identifier rejects. diff --git a/electron/preload.js b/electron/preload.js index 71a0672..74d7c09 100644 --- a/electron/preload.js +++ b/electron/preload.js @@ -1,4 +1,4 @@ -const { contextBridge, ipcRenderer } = require("electron"); +const { contextBridge, ipcRenderer, webUtils } = require("electron"); // Main asks the shell to switch pages (the App menu's "Settings…" item, // the tray). Returns an unsubscribe so the renderer can clean up. @@ -32,6 +32,15 @@ function onHealthChanged(callback) { return () => ipcRenderer.removeListener("health-changed", listener); } +// #253: main pushes the full job list after every queue change - a drop, a +// retry, a job finishing - so the renderer just re-renders from the array +// rather than tracking a diff. +function onFileTranscriptionQueue(callback) { + const listener = (_event, jobs) => callback(jobs); + ipcRenderer.on("file-transcription:queue", listener); + return () => ipcRenderer.removeListener("file-transcription:queue", listener); +} + // #136: main pushes the full history list whenever it changes (a new // dictation, a remove, a clear) - the History page re-renders from the // snapshot rather than diffing. @@ -87,6 +96,25 @@ contextBridge.exposeInMainWorld("openstream", { getStatus: () => ipcRenderer.invoke("vocabulary:get-status"), chooseFolder: () => ipcRenderer.invoke("vocabulary:choose-folder"), }, + // #253: drop-a-file transcription. + fileTranscription: { + add: (filePaths) => ipcRenderer.invoke("file-transcription:add", filePaths), + chooseFiles: () => ipcRenderer.invoke("file-transcription:choose-files"), + getQueue: () => ipcRenderer.invoke("file-transcription:get-queue"), + retry: (id) => ipcRenderer.invoke("file-transcription:retry", id), + remove: (id) => ipcRenderer.invoke("file-transcription:remove", id), + clearFinished: () => ipcRenderer.invoke("file-transcription:clear-finished"), + copy: (id) => ipcRenderer.invoke("file-transcription:copy", id), + save: (id) => ipcRenderer.invoke("file-transcription:save", id), + }, + // #253: resolves a dropped File object to its real filesystem path. + // webUtils.getPathForFile supersedes the old (deprecated, removed) + // File.path augmentation - only callable from the preload/main context, + // which is why this exists as a bridge method rather than something the + // renderer reaches for directly. + files: { + getPathForFile: (file) => webUtils.getPathForFile(file), + }, recordingHistory: { get: () => ipcRenderer.invoke("recording-history:get"), remove: (id) => ipcRenderer.invoke("recording-history:remove", id), @@ -97,5 +125,6 @@ contextBridge.exposeInMainWorld("openstream", { onDictationState, onSetupProgress, onHealthChanged, + onFileTranscriptionQueue, onRecordingHistory, }); diff --git a/electron/transcriptionHelper.js b/electron/transcriptionHelper.js index 4030306..dbfd3d5 100644 --- a/electron/transcriptionHelper.js +++ b/electron/transcriptionHelper.js @@ -27,10 +27,16 @@ const FAILURE_WINDOW_MS = 12000; // this long is a failure worth surfacing, not worth waiting on. const REQUEST_TIMEOUT_MS = 30000; +// #253: a dropped file can be an hour-long meeting recording - a different +// budget entirely from a live dictation. Generous rather than tuned: the +// point is "don't time out a real job", not a tight ceiling. +const FILE_REQUEST_TIMEOUT_MS = 30 * 60 * 1000; + function createTranscriptionHelper({ spawnProcess = spawn, restartDelayMs = RESTART_DELAY_MS, requestTimeoutMs = REQUEST_TIMEOUT_MS, + fileRequestTimeoutMs = FILE_REQUEST_TIMEOUT_MS, setRequestTimer = setTimeout, clearRequestTimer = clearTimeout, setRestartTimer = setTimeout, @@ -186,7 +192,7 @@ function createTranscriptionHelper({ return readyPromise; } - function request(cmd, payload = {}) { + function request(cmd, payload = {}, timeoutMs = requestTimeoutMs) { return new Promise((resolve, reject) => { if (!child) { reject(new Error("transcription-helper is not running")); @@ -194,8 +200,8 @@ function createTranscriptionHelper({ } const id = String(nextId++); const timer = setRequestTimer(() => { - settle(id, null, new Error(`transcription-helper ${cmd} request timed out after ${requestTimeoutMs}ms`)); - }, requestTimeoutMs); + settle(id, null, new Error(`transcription-helper ${cmd} request timed out after ${timeoutMs}ms`)); + }, timeoutMs); pending.set(id, { resolve, reject, timer }); child.stdin.write(`${JSON.stringify({ id, cmd, ...payload })}\n`); }); @@ -222,6 +228,21 @@ function createTranscriptionHelper({ return reply.text.trim(); } + // #253: dropped-file transcription. Reads straight from disk on the Swift + // side (see transcribeFile in main.swift) - no base64 round-trip, and a + // far longer timeout since the file could be long. + async function transcribeFile(filePath, language) { + const payload = { path: filePath }; + if (typeof language === "string" && language) payload.lang = language; + const reply = await request("transcribeFile", payload, fileRequestTimeoutMs); + if (reply.status !== "ok" || typeof reply.text !== "string") { + throw new Error( + `transcription-helper file transcription failed${reply.reason ? `: ${reply.reason}` : ""}`, + ); + } + return reply.text.trim(); + } + // #254: a user-driven restart of a failed helper - clear the crash-loop // count so it isn't still "failed" the moment it comes back. function restart() { @@ -230,7 +251,7 @@ function createTranscriptionHelper({ start(); } - return { start, stop, restart, isReady, whenReady, status, onStatusChange, transcribe }; + return { start, stop, restart, isReady, whenReady, status, onStatusChange, transcribe, transcribeFile }; } module.exports = { diff --git a/electron/transcriptionHelper.test.js b/electron/transcriptionHelper.test.js index 7906d15..926dd77 100644 --- a/electron/transcriptionHelper.test.js +++ b/electron/transcriptionHelper.test.js @@ -114,6 +114,64 @@ test("transcribe rejects on an error reply, carrying the reason", async () => { helper.stop(); }); +test("#253: transcribeFile sends a path field, not wav, and returns the trimmed text", async () => { + const child = fakeProcess(); + const requests = readRequests(child); + const helper = createTranscriptionHelper({ spawnProcess: () => child }); + helper.start(); + child.stdout.write('{"event":"ready"}\n'); + + const promise = helper.transcribeFile("/Users/me/Desktop/meeting.m4a", "en"); + await nextTurn(); + + assert.equal(requests[0].cmd, "transcribeFile"); + assert.equal(requests[0].path, "/Users/me/Desktop/meeting.m4a"); + assert.equal(requests[0].lang, "en"); + assert.equal("wav" in requests[0], false); + + child.stdout.write('{"id":"1","status":"ok","text":" a long transcript. ","ms":48213}\n'); + assert.equal(await promise, "a long transcript."); + helper.stop(); +}); + +test("#253: transcribeFile omits lang when none is passed, and rejects on an error reply", async () => { + const child = fakeProcess(); + const requests = readRequests(child); + const helper = createTranscriptionHelper({ spawnProcess: () => child }); + helper.start(); + child.stdout.write('{"event":"ready"}\n'); + + const promise = helper.transcribeFile("/tmp/clip.wav"); + await nextTurn(); + assert.equal("lang" in requests[0], false); + + child.stdout.write('{"id":"1","status":"error","reason":"no file at /tmp/clip.wav"}\n'); + await assert.rejects(promise, /no file at \/tmp\/clip\.wav/); + helper.stop(); +}); + +test("#253: transcribeFile uses the longer file timeout, not the live-dictation one", async () => { + const child = fakeProcess(); + readRequests(child); + // A short live-dictation timeout and a slightly longer file timeout, both + // tiny so the test doesn't actually wait real minutes. + const helper = createTranscriptionHelper({ + spawnProcess: () => child, + requestTimeoutMs: 5, + fileRequestTimeoutMs: 20, + }); + helper.start(); + child.stdout.write('{"event":"ready"}\n'); + + const promise = helper.transcribeFile("/tmp/long.wav"); + // Still pending past the live-dictation timeout... + await new Promise((resolve) => setTimeout(resolve, 10)); + // ...but resolves fine within the file timeout. + child.stdout.write('{"id":"1","status":"ok","text":"done","ms":15}\n'); + assert.equal(await promise, "done"); + helper.stop(); +}); + test("a request in flight rejects if the helper exits, and the ready gate resets", async () => { const child = fakeProcess(); readRequests(child); diff --git a/native/transcription-helper/Sources/transcription-helper/main.swift b/native/transcription-helper/Sources/transcription-helper/main.swift index bb732a1..ef2a28b 100644 --- a/native/transcription-helper/Sources/transcription-helper/main.swift +++ b/native/transcription-helper/Sources/transcription-helper/main.swift @@ -12,8 +12,21 @@ import FluidAudio // <- {"id":"1","status":"ok","text":"...","ms":312} // <- {"id":"1","status":"error","reason":"..."} // -> {"id":"2","cmd":"ping"} <- {"id":"2","status":"ok"} +// -> {"id":"3","cmd":"transcribeFile","path":"/abs/path.m4a","lang":"auto"} +// <- {"id":"3","status":"ok","text":"...","ms":48213} +// <- {"id":"3","status":"error","reason":"..."} // startup: {"event":"ready"} once the model is loaded, or // {"event":"error","message":"..."} then exit(1) if it can't load. +// +// #253: "transcribeFile" is the drop-a-file path - unlike "transcribe" it +// reads straight from disk (no base64 round-trip; the file is already +// there and could be long) and hands the URL to AVAudioFile-backed +// decoding, which resamples whatever format AVFoundation can open and +// auto-chunks past ASRConfig.default's 30s streamingThreshold. No ffmpeg, +// no manual chunking - FluidAudio already does both for the file-URL +// overload of transcribe(). Runs on the same resident model as live +// dictation, so a long file transcription can delay a concurrent +// push-to-talk until it finishes - accepted for now, see #253's PR. func elog(_ message: String) { FileHandle.standardError.write(Data(("transcription-helper: " + message + "\n").utf8)) @@ -75,6 +88,12 @@ case .failure(let error): emit(["event": "ready"]) elog("ready") +// Shared by "transcribe" and "transcribeFile" (#253). +func languageHint(_ object: [String: Any]) -> Language? { + guard let code = object["lang"] as? String, code != "auto" else { return nil } + return Language(rawValue: code) +} + // MARK: - Command loop while let line = readLine(strippingNewline: true) { @@ -101,14 +120,7 @@ while let line = readLine(strippingNewline: true) { continue } - // #252: `lang` is a script hint for v3's token filter, not a hard - // setting - the model auto-detects the content either way. "auto" or - // a missing/unknown code means no hint (pure auto-detect); a valid - // code biases the decoder towards that language's script. - let language: Language? = { - guard let code = object["lang"] as? String, code != "auto" else { return nil } - return Language(rawValue: code) - }() + let language = languageHint(object) let scratch = FileManager.default.temporaryDirectory .appendingPathComponent("openstream-dictation-\(id).wav") @@ -141,6 +153,44 @@ while let line = readLine(strippingNewline: true) { elog("transcription failed: \(error)") } + case "transcribeFile": + // #253: dropped-file transcription. No base64, no temp copy, no + // manual chunking - the file is already on disk, and the file-URL + // overload of transcribe() resamples via AVAudioFile (whatever + // format AVFoundation can open: WAV, AIFF, CAF, M4A/AAC, MP3, …) + // and auto-chunks past the 30s streaming threshold on its own. + guard let path = object["path"] as? String, !path.isEmpty else { + emit(["id": id, "status": "error", "reason": "transcribeFile needs a non-empty \"path\" field"]) + continue + } + let url = URL(fileURLWithPath: path) + guard FileManager.default.fileExists(atPath: url.path) else { + emit(["id": id, "status": "error", "reason": "no file at \(path)"]) + continue + } + + let language = languageHint(object) + let startedAt = Date() + let outcome = blocking { () -> Result in + do { + var state = TdtDecoderState.make(decoderLayers: await asr.decoderLayerCount) + let result = try await asr.transcribe(url, decoderState: &state, language: language) + return .success(result.text) + } catch { + return .failure(error) + } + } + + switch outcome { + case .success(let text): + let ms = Int(Date().timeIntervalSince(startedAt) * 1000) + emit(["id": id, "status": "ok", "text": text, "ms": ms]) + elog("transcribed file \(url.lastPathComponent) in \(ms)ms") + case .failure(let error): + emit(["id": id, "status": "error", "reason": "\(error)"]) + elog("file transcription failed for \(url.lastPathComponent): \(error)") + } + default: emit(["id": id, "status": "error", "reason": "unknown command \"\(command)\""]) } diff --git a/src/App.tsx b/src/App.tsx index a8d3127..e84e836 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -2,6 +2,7 @@ import { useEffect, useState } from "react"; import { coercePage, TAB_PAGES, type Page, type TabPage } from "./nav"; import Home from "./pages/Home"; import History from "./pages/History"; +import FileTranscription from "./pages/FileTranscription"; import Commands from "./pages/Commands"; import Settings from "./pages/Settings"; import Permissions from "./pages/Permissions"; @@ -10,6 +11,7 @@ import Setup from "./pages/Setup"; const TAB_LABELS: Record = { home: "Home", history: "History", + files: "Files", commands: "Commands", settings: "Settings", }; @@ -44,6 +46,7 @@ export default function App() { {page === "settings" && } {page === "history" && } + {page === "files" && } {page === "commands" && } {page === "setup" && setPage("home")} />} {page === "permissions" && setPage("home")} />} diff --git a/src/components/Icons.tsx b/src/components/Icons.tsx index b586015..4c3f783 100644 --- a/src/components/Icons.tsx +++ b/src/components/Icons.tsx @@ -80,3 +80,7 @@ export const CommandsIcon = svg( export const CheckIcon = svg(, 3); export const ChevronDownIcon = svg(, 2.5); + +export const FileIcon = svg( + +); diff --git a/src/index.css b/src/index.css index 3a03f5c..fc44419 100644 --- a/src/index.css +++ b/src/index.css @@ -839,6 +839,60 @@ select.field option { color: initial; } .settings-advanced[open] > summary::before { content: "\2304\00a0\00a0"; } .settings-advanced .field { max-width: 240px; } +/* --- file transcription page (#253) ------------------------------------- */ + +.drop-zone { + display: flex; + flex-direction: column; + align-items: center; + gap: var(--space-2); + padding: var(--space-7) var(--space-4); + border: 2px dashed var(--line-hi); + border-radius: var(--r); + background: var(--screen-2); + text-align: center; + transition: border-color 0.12s ease, background 0.12s ease; +} +.drop-zone--active { + border-color: var(--acc); + background: var(--acc-tint); +} +.drop-zone__icon { + width: 28px; + height: 28px; + color: var(--faint); + margin-bottom: 2px; +} +.drop-zone--active .drop-zone__icon { color: var(--acc); } +.drop-zone__title { + margin: 0; + font-size: 14px; + font-weight: 600; + color: var(--fg-hi); +} +.drop-zone__desc { + margin: 0; + font-size: 12px; + color: var(--muted); +} + +.queue-row__name { + color: var(--fg-hi); + text-shadow: var(--shadow-text); +} +.queue-row__error { + display: block; + font-size: 11.5px; + color: var(--err); + margin-top: 1px; +} +.queue-row__actions { + display: flex; + align-items: center; + gap: 4px; + flex: 0 0 auto; +} + /* --- Reduce Transparency fallback -------------------------------------- The clear glass depends on the vibrancy material and a desktop behind it. When the user turns on Reduce Transparency, macOS flattens the diff --git a/src/nav.ts b/src/nav.ts index fc5dc32..0ffdeff 100644 --- a/src/nav.ts +++ b/src/nav.ts @@ -4,11 +4,11 @@ // "permissions" and "setup" are reachable views but not toolbar tabs - the // app navigates to them when a grant is missing / a model is downloading. -export type Page = "home" | "history" | "commands" | "settings" | "permissions" | "setup"; +export type Page = "home" | "history" | "files" | "commands" | "settings" | "permissions" | "setup"; -export const PAGES = ["home", "history", "commands", "settings", "permissions", "setup"] as const; +export const PAGES = ["home", "history", "files", "commands", "settings", "permissions", "setup"] as const; -export const TAB_PAGES = ["home", "history", "commands", "settings"] as const; +export const TAB_PAGES = ["home", "history", "files", "commands", "settings"] as const; export type TabPage = (typeof TAB_PAGES)[number]; export const DEFAULT_PAGE: Page = "home"; diff --git a/src/openstreamBridge.d.ts b/src/openstreamBridge.d.ts index 9efdbd4..bbdf52a 100644 --- a/src/openstreamBridge.d.ts +++ b/src/openstreamBridge.d.ts @@ -85,6 +85,23 @@ export type VocabularyStatus = { scannedAt: Date | null; }; +// #253: one dropped/chosen file's place in the transcription queue. +export type FileTranscriptionStatus = "queued" | "transcribing" | "done" | "failed"; + +export type FileTranscriptionJob = { + id: string; + path: string; + name: string; + status: FileTranscriptionStatus; + text: string | null; + error: string | null; + queuedAt: number; + startedAt: number | null; + finishedAt: number | null; +}; + +export type FileTranscriptionActionResult = { ok: true; path?: string } | { ok: false; reason: string }; + // Exposed by preload.js via contextBridge - see the comment there for what // this is expected to grow into. declare global { @@ -129,6 +146,19 @@ declare global { getStatus(): Promise; chooseFolder(): Promise; }; + fileTranscription: { + add(filePaths: string[]): Promise; + chooseFiles(): Promise; + getQueue(): Promise; + retry(id: string): Promise; + remove(id: string): Promise; + clearFinished(): Promise; + copy(id: string): Promise; + save(id: string): Promise; + }; + files: { + getPathForFile(file: File): string; + }; recordingHistory: { get(): Promise; remove(id: string): Promise; @@ -139,6 +169,7 @@ declare global { onDictationState(callback: (state: "idle" | "recording" | "transcribing") => void): () => void; onSetupProgress(callback: (progress: SetupProgress) => void): () => void; onHealthChanged(callback: () => void): () => void; + onFileTranscriptionQueue(callback: (jobs: FileTranscriptionJob[]) => void): () => void; onRecordingHistory(callback: (entries: RecordingHistoryEntry[]) => void): () => void; }; } diff --git a/src/pages/FileTranscription.tsx b/src/pages/FileTranscription.tsx new file mode 100644 index 0000000..e645e4e --- /dev/null +++ b/src/pages/FileTranscription.tsx @@ -0,0 +1,159 @@ +import { useEffect, useState } from "react"; +import type { FileTranscriptionJob } from "../openstreamBridge"; +import StatusPill, { type PillTone } from "../components/StatusPill"; +import { FileIcon } from "../components/Icons"; + +function statusPill(status: FileTranscriptionJob["status"]): { tone: PillTone; label: string } { + switch (status) { + case "queued": + return { tone: "muted", label: "Queued" }; + case "transcribing": + return { tone: "wait", label: "Transcribing…" }; + case "done": + return { tone: "ok", label: "Done" }; + case "failed": + return { tone: "err", label: "Failed" }; + } +} + +function elapsedLabel(job: FileTranscriptionJob, now: number): string | null { + if (!job.startedAt) return null; + const end = job.finishedAt ?? now; + const seconds = Math.max(0, Math.round((end - job.startedAt) / 1000)); + if (seconds < 60) return `${seconds}s`; + return `${Math.floor(seconds / 60)}m ${seconds % 60}s`; +} + +export default function FileTranscription() { + const [jobs, setJobs] = useState([]); + const [dragging, setDragging] = useState(false); + const [message, setMessage] = useState(null); + const [now, setNow] = useState(() => Date.now()); + + useEffect(() => { + window.openstream.fileTranscription.getQueue().then(setJobs); + return window.openstream.onFileTranscriptionQueue(setJobs); + }, []); + + // The elapsed-time readout on an in-flight job needs its own clock - the + // job list itself only changes on a status transition, not every second. + useEffect(() => { + if (!jobs.some((job) => job.status === "transcribing")) return; + const timer = setInterval(() => setNow(Date.now()), 1000); + return () => clearInterval(timer); + }, [jobs]); + + function handleDrop(event: React.DragEvent) { + event.preventDefault(); + setDragging(false); + const paths = Array.from(event.dataTransfer.files) + .map((file) => window.openstream.files.getPathForFile(file)) + .filter((filePath) => filePath.length > 0); + if (paths.length > 0) void window.openstream.fileTranscription.add(paths); + } + + async function copyJob(id: string) { + const result = await window.openstream.fileTranscription.copy(id); + setMessage(result.ok ? "Copied to the clipboard." : result.reason); + } + + async function saveJob(id: string) { + const result = await window.openstream.fileTranscription.save(id); + setMessage(result.ok ? `Saved to ${result.path}` : result.reason); + } + + const hasFinished = jobs.some((job) => job.status === "done" || job.status === "failed"); + + return ( +
+
{ + event.preventDefault(); + setDragging(true); + }} + onDragLeave={() => setDragging(false)} + onDrop={handleDrop} + > + +

Drop audio files here

+

A meeting recording, a voice memo, a lecture - or

+ +
+ + {jobs.length > 0 && ( +
+
Queue
+ {jobs.map((job) => { + const pill = statusPill(job.status); + const time = elapsedLabel(job, now); + return ( +
+ + {job.name} + {time && {time}} + {job.status === "failed" && job.error && {job.error}} + + +
+ {job.status === "done" && ( + <> + + + + )} + {job.status === "failed" && ( + + )} + {job.status !== "transcribing" && ( + + )} +
+
+ ); + })} + {hasFinished && ( +
+ + +
+ )} +
+ )} + + {message &&

{message}

} + +

+ One file transcribes at a time. A long file can briefly delay a live dictation started while it's running. +

+
+ ); +}