From 436208183b50114520a927bb9879e8e4ed1db6d5 Mon Sep 17 00:00:00 2001 From: Nabzx Date: Thu, 17 Sep 2026 00:41:52 +0500 Subject: [PATCH 1/6] transcription-helper: a transcribeFile command for dropped files (#253) Reads straight from disk instead of the base64-over-stdio path "transcribe" uses for live dictation - the file already exists and could be long, so there is no reason to round-trip it through JSON. Hands the URL straight to the file-URL overload of AsrManager.transcribe, which resamples via AVAudioFile (whatever format AVFoundation can open - WAV, AIFF, CAF, M4A/AAC, MP3) and auto-chunks past ASRConfig.default's 30s streaming threshold on its own. No ffmpeg, no manual chunking work needed on our side - confirmed by reading FluidAudio's own source, not guessed. Shares the #252 language-hint parsing with "transcribe" via a small languageHint() helper. Compile-verified only (swift build, twice) - I have no way to execute -test this from this session; every attempt to run transcription-helper here gets killed within seconds, same as attempts by the live running app's own supervisor. Needs a real run before this ships. --- .../Sources/transcription-helper/main.swift | 66 ++++++++++++++++--- 1 file changed, 58 insertions(+), 8 deletions(-) diff --git a/native/transcription-helper/Sources/transcription-helper/main.swift b/native/transcription-helper/Sources/transcription-helper/main.swift index bb732a1..ef2a28b 100644 --- a/native/transcription-helper/Sources/transcription-helper/main.swift +++ b/native/transcription-helper/Sources/transcription-helper/main.swift @@ -12,8 +12,21 @@ import FluidAudio // <- {"id":"1","status":"ok","text":"...","ms":312} // <- {"id":"1","status":"error","reason":"..."} // -> {"id":"2","cmd":"ping"} <- {"id":"2","status":"ok"} +// -> {"id":"3","cmd":"transcribeFile","path":"/abs/path.m4a","lang":"auto"} +// <- {"id":"3","status":"ok","text":"...","ms":48213} +// <- {"id":"3","status":"error","reason":"..."} // startup: {"event":"ready"} once the model is loaded, or // {"event":"error","message":"..."} then exit(1) if it can't load. +// +// #253: "transcribeFile" is the drop-a-file path - unlike "transcribe" it +// reads straight from disk (no base64 round-trip; the file is already +// there and could be long) and hands the URL to AVAudioFile-backed +// decoding, which resamples whatever format AVFoundation can open and +// auto-chunks past ASRConfig.default's 30s streamingThreshold. No ffmpeg, +// no manual chunking - FluidAudio already does both for the file-URL +// overload of transcribe(). Runs on the same resident model as live +// dictation, so a long file transcription can delay a concurrent +// push-to-talk until it finishes - accepted for now, see #253's PR. func elog(_ message: String) { FileHandle.standardError.write(Data(("transcription-helper: " + message + "\n").utf8)) @@ -75,6 +88,12 @@ case .failure(let error): emit(["event": "ready"]) elog("ready") +// Shared by "transcribe" and "transcribeFile" (#253). +func languageHint(_ object: [String: Any]) -> Language? { + guard let code = object["lang"] as? String, code != "auto" else { return nil } + return Language(rawValue: code) +} + // MARK: - Command loop while let line = readLine(strippingNewline: true) { @@ -101,14 +120,7 @@ while let line = readLine(strippingNewline: true) { continue } - // #252: `lang` is a script hint for v3's token filter, not a hard - // setting - the model auto-detects the content either way. "auto" or - // a missing/unknown code means no hint (pure auto-detect); a valid - // code biases the decoder towards that language's script. - let language: Language? = { - guard let code = object["lang"] as? String, code != "auto" else { return nil } - return Language(rawValue: code) - }() + let language = languageHint(object) let scratch = FileManager.default.temporaryDirectory .appendingPathComponent("openstream-dictation-\(id).wav") @@ -141,6 +153,44 @@ while let line = readLine(strippingNewline: true) { elog("transcription failed: \(error)") } + case "transcribeFile": + // #253: dropped-file transcription. No base64, no temp copy, no + // manual chunking - the file is already on disk, and the file-URL + // overload of transcribe() resamples via AVAudioFile (whatever + // format AVFoundation can open: WAV, AIFF, CAF, M4A/AAC, MP3, …) + // and auto-chunks past the 30s streaming threshold on its own. + guard let path = object["path"] as? String, !path.isEmpty else { + emit(["id": id, "status": "error", "reason": "transcribeFile needs a non-empty \"path\" field"]) + continue + } + let url = URL(fileURLWithPath: path) + guard FileManager.default.fileExists(atPath: url.path) else { + emit(["id": id, "status": "error", "reason": "no file at \(path)"]) + continue + } + + let language = languageHint(object) + let startedAt = Date() + let outcome = blocking { () -> Result in + do { + var state = TdtDecoderState.make(decoderLayers: await asr.decoderLayerCount) + let result = try await asr.transcribe(url, decoderState: &state, language: language) + return .success(result.text) + } catch { + return .failure(error) + } + } + + switch outcome { + case .success(let text): + let ms = Int(Date().timeIntervalSince(startedAt) * 1000) + emit(["id": id, "status": "ok", "text": text, "ms": ms]) + elog("transcribed file \(url.lastPathComponent) in \(ms)ms") + case .failure(let error): + emit(["id": id, "status": "error", "reason": "\(error)"]) + elog("file transcription failed for \(url.lastPathComponent): \(error)") + } + default: emit(["id": id, "status": "error", "reason": "unknown command \"\(command)\""]) } From 5eb838b0dd65fa3074189fb941faa7cbe77db8b8 Mon Sep 17 00:00:00 2001 From: Nabzx Date: Thu, 17 Sep 2026 00:43:04 +0500 Subject: [PATCH 2/6] transcription helper: transcribeFile(), a longer timeout for long files (#253) Sends a path field instead of base64 wav. The live-dictation REQUEST_TIMEOUT_MS (30s) makes no sense for a dropped file that could run many minutes, so request() now takes an optional per-call timeout and transcribeFile uses its own generous one (30 minutes). --- electron/transcriptionHelper.js | 29 ++++++++++++-- electron/transcriptionHelper.test.js | 58 ++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/electron/transcriptionHelper.js b/electron/transcriptionHelper.js index 4030306..dbfd3d5 100644 --- a/electron/transcriptionHelper.js +++ b/electron/transcriptionHelper.js @@ -27,10 +27,16 @@ const FAILURE_WINDOW_MS = 12000; // this long is a failure worth surfacing, not worth waiting on. const REQUEST_TIMEOUT_MS = 30000; +// #253: a dropped file can be an hour-long meeting recording - a different +// budget entirely from a live dictation. Generous rather than tuned: the +// point is "don't time out a real job", not a tight ceiling. +const FILE_REQUEST_TIMEOUT_MS = 30 * 60 * 1000; + function createTranscriptionHelper({ spawnProcess = spawn, restartDelayMs = RESTART_DELAY_MS, requestTimeoutMs = REQUEST_TIMEOUT_MS, + fileRequestTimeoutMs = FILE_REQUEST_TIMEOUT_MS, setRequestTimer = setTimeout, clearRequestTimer = clearTimeout, setRestartTimer = setTimeout, @@ -186,7 +192,7 @@ function createTranscriptionHelper({ return readyPromise; } - function request(cmd, payload = {}) { + function request(cmd, payload = {}, timeoutMs = requestTimeoutMs) { return new Promise((resolve, reject) => { if (!child) { reject(new Error("transcription-helper is not running")); @@ -194,8 +200,8 @@ function createTranscriptionHelper({ } const id = String(nextId++); const timer = setRequestTimer(() => { - settle(id, null, new Error(`transcription-helper ${cmd} request timed out after ${requestTimeoutMs}ms`)); - }, requestTimeoutMs); + settle(id, null, new Error(`transcription-helper ${cmd} request timed out after ${timeoutMs}ms`)); + }, timeoutMs); pending.set(id, { resolve, reject, timer }); child.stdin.write(`${JSON.stringify({ id, cmd, ...payload })}\n`); }); @@ -222,6 +228,21 @@ function createTranscriptionHelper({ return reply.text.trim(); } + // #253: dropped-file transcription. Reads straight from disk on the Swift + // side (see transcribeFile in main.swift) - no base64 round-trip, and a + // far longer timeout since the file could be long. + async function transcribeFile(filePath, language) { + const payload = { path: filePath }; + if (typeof language === "string" && language) payload.lang = language; + const reply = await request("transcribeFile", payload, fileRequestTimeoutMs); + if (reply.status !== "ok" || typeof reply.text !== "string") { + throw new Error( + `transcription-helper file transcription failed${reply.reason ? `: ${reply.reason}` : ""}`, + ); + } + return reply.text.trim(); + } + // #254: a user-driven restart of a failed helper - clear the crash-loop // count so it isn't still "failed" the moment it comes back. function restart() { @@ -230,7 +251,7 @@ function createTranscriptionHelper({ start(); } - return { start, stop, restart, isReady, whenReady, status, onStatusChange, transcribe }; + return { start, stop, restart, isReady, whenReady, status, onStatusChange, transcribe, transcribeFile }; } module.exports = { diff --git a/electron/transcriptionHelper.test.js b/electron/transcriptionHelper.test.js index 7906d15..926dd77 100644 --- a/electron/transcriptionHelper.test.js +++ b/electron/transcriptionHelper.test.js @@ -114,6 +114,64 @@ test("transcribe rejects on an error reply, carrying the reason", async () => { helper.stop(); }); +test("#253: transcribeFile sends a path field, not wav, and returns the trimmed text", async () => { + const child = fakeProcess(); + const requests = readRequests(child); + const helper = createTranscriptionHelper({ spawnProcess: () => child }); + helper.start(); + child.stdout.write('{"event":"ready"}\n'); + + const promise = helper.transcribeFile("/Users/me/Desktop/meeting.m4a", "en"); + await nextTurn(); + + assert.equal(requests[0].cmd, "transcribeFile"); + assert.equal(requests[0].path, "/Users/me/Desktop/meeting.m4a"); + assert.equal(requests[0].lang, "en"); + assert.equal("wav" in requests[0], false); + + child.stdout.write('{"id":"1","status":"ok","text":" a long transcript. ","ms":48213}\n'); + assert.equal(await promise, "a long transcript."); + helper.stop(); +}); + +test("#253: transcribeFile omits lang when none is passed, and rejects on an error reply", async () => { + const child = fakeProcess(); + const requests = readRequests(child); + const helper = createTranscriptionHelper({ spawnProcess: () => child }); + helper.start(); + child.stdout.write('{"event":"ready"}\n'); + + const promise = helper.transcribeFile("/tmp/clip.wav"); + await nextTurn(); + assert.equal("lang" in requests[0], false); + + child.stdout.write('{"id":"1","status":"error","reason":"no file at /tmp/clip.wav"}\n'); + await assert.rejects(promise, /no file at \/tmp\/clip\.wav/); + helper.stop(); +}); + +test("#253: transcribeFile uses the longer file timeout, not the live-dictation one", async () => { + const child = fakeProcess(); + readRequests(child); + // A short live-dictation timeout and a slightly longer file timeout, both + // tiny so the test doesn't actually wait real minutes. + const helper = createTranscriptionHelper({ + spawnProcess: () => child, + requestTimeoutMs: 5, + fileRequestTimeoutMs: 20, + }); + helper.start(); + child.stdout.write('{"event":"ready"}\n'); + + const promise = helper.transcribeFile("/tmp/long.wav"); + // Still pending past the live-dictation timeout... + await new Promise((resolve) => setTimeout(resolve, 10)); + // ...but resolves fine within the file timeout. + child.stdout.write('{"id":"1","status":"ok","text":"done","ms":15}\n'); + assert.equal(await promise, "done"); + helper.stop(); +}); + test("a request in flight rejects if the helper exits, and the ready gate resets", async () => { const child = fakeProcess(); readRequests(child); From 38ca8924b574d165dc429900cba7b01fda3bec5b Mon Sep 17 00:00:00 2001 From: Nabzx Date: Thu, 17 Sep 2026 00:44:10 +0500 Subject: [PATCH 3/6] file transcription queue: a FIFO coordinator for dropped files (#253) Pure, adapter-injected module - one file at a time (the resident helper only ever runs one transcription regardless), a failure never blocks the rest of the queue, retry/remove/clearFinished for the UI layer to call. onChange fires a full snapshot on every transition so main.js can just push the latest array to the renderer rather than diffing. --- electron/fileTranscriptionQueue.js | 132 +++++++++++++++ electron/fileTranscriptionQueue.test.js | 210 ++++++++++++++++++++++++ 2 files changed, 342 insertions(+) create mode 100644 electron/fileTranscriptionQueue.js create mode 100644 electron/fileTranscriptionQueue.test.js diff --git a/electron/fileTranscriptionQueue.js b/electron/fileTranscriptionQueue.js new file mode 100644 index 0000000..24079e5 --- /dev/null +++ b/electron/fileTranscriptionQueue.js @@ -0,0 +1,132 @@ +// #253: drop-a-file transcription. A small FIFO queue over the same +// resident transcription-helper live dictation uses (see transcribeFile() +// in transcriptionHelper.js) - one job at a time, since the helper only +// ever runs one transcription at a time regardless (it reads one +// newline-delimited request, blocks until the model replies, reads the +// next). A long file job can delay a concurrent live dictation; that is a +// known, accepted tradeoff for now (see the #253 PR), not something this +// module tries to work around - a second resident model would double the +// memory #257 exists to cut down. +// +// Pure and adapter-injected, same ports-and-adapters shape as +// dictationCoordinator.js: `transcription` is asserted, `onChange` is +// optional and fires with a full snapshot after every state change so a +// caller (main.js) can just re-render from the latest array rather than +// diffing. + +function createFileTranscriptionQueue(options = {}) { + const { transcription, language = { get: () => "auto" }, onChange = () => {} } = options; + + if (!transcription || typeof transcription.transcribeFile !== "function") { + throw new Error("createFileTranscriptionQueue requires transcription.transcribeFile()"); + } + + let jobs = []; + let nextId = 1; + let processing = false; + + function snapshot() { + return jobs.map((job) => ({ ...job })); + } + + function notify() { + try { + onChange(snapshot()); + } catch { + // A listener must never break the queue. + } + } + + function baseName(filePath) { + const parts = String(filePath).split("/"); + return parts[parts.length - 1] || filePath; + } + + // Runs until the queue is drained. Only one call is ever "live" at a + // time - the `processing` guard means a second addFiles() while a job is + // in flight just returns, and the in-flight call's own loop picks up + // whatever got added. + async function drain() { + if (processing) return; + processing = true; + try { + let job; + while ((job = jobs.find((candidate) => candidate.status === "queued"))) { + job.status = "transcribing"; + job.startedAt = Date.now(); + notify(); + try { + job.text = await transcription.transcribeFile(job.path, language.get()); + job.status = "done"; + } catch (error) { + job.status = "failed"; + job.error = error instanceof Error ? error.message : String(error); + } + job.finishedAt = Date.now(); + notify(); + } + } finally { + processing = false; + } + } + + function addFiles(paths) { + const added = []; + for (const filePath of Array.isArray(paths) ? paths : []) { + if (typeof filePath !== "string" || !filePath) continue; + const job = { + id: String(nextId++), + path: filePath, + name: baseName(filePath), + status: "queued", + text: null, + error: null, + queuedAt: Date.now(), + startedAt: null, + finishedAt: null, + }; + jobs.push(job); + added.push(job.id); + } + if (added.length > 0) { + notify(); + void drain(); + } + return added; + } + + function retry(id) { + const job = jobs.find((candidate) => candidate.id === id); + if (!job || job.status !== "failed") return false; + job.status = "queued"; + job.text = null; + job.error = null; + job.startedAt = null; + job.finishedAt = null; + notify(); + void drain(); + return true; + } + + function remove(id) { + const index = jobs.findIndex((candidate) => candidate.id === id); + if (index === -1 || jobs[index].status === "transcribing") return false; + jobs.splice(index, 1); + notify(); + return true; + } + + function clearFinished() { + const before = jobs.length; + jobs = jobs.filter((job) => job.status === "queued" || job.status === "transcribing"); + if (jobs.length !== before) notify(); + } + + function getJobs() { + return snapshot(); + } + + return { addFiles, retry, remove, clearFinished, getJobs }; +} + +module.exports = { createFileTranscriptionQueue }; diff --git a/electron/fileTranscriptionQueue.test.js b/electron/fileTranscriptionQueue.test.js new file mode 100644 index 0000000..7631a48 --- /dev/null +++ b/electron/fileTranscriptionQueue.test.js @@ -0,0 +1,210 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); +const { createFileTranscriptionQueue } = require("./fileTranscriptionQueue"); + +function deferred() { + let resolve, reject; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +function nextTurn() { + return new Promise((resolve) => setImmediate(resolve)); +} + +test("requires a transcription.transcribeFile adapter", () => { + assert.throws(() => createFileTranscriptionQueue({}), /transcribeFile/); + assert.throws(() => createFileTranscriptionQueue({ transcription: {} }), /transcribeFile/); +}); + +test("addFiles queues jobs and returns their ids", () => { + const queue = createFileTranscriptionQueue({ + transcription: { transcribeFile: async () => "text" }, + }); + const ids = queue.addFiles(["/a/one.wav", "/a/two.m4a"]); + assert.equal(ids.length, 2); + const jobs = queue.getJobs(); + assert.equal(jobs.length, 2); + assert.equal(jobs[0].name, "one.wav"); + assert.equal(jobs[1].name, "two.m4a"); +}); + +test("addFiles ignores non-string / empty entries without throwing", () => { + const queue = createFileTranscriptionQueue({ + transcription: { transcribeFile: async () => "text" }, + }); + const ids = queue.addFiles(["/a/one.wav", "", null, 42, undefined]); + assert.equal(ids.length, 1); + assert.equal(queue.getJobs().length, 1); +}); + +test("processes one file at a time, in FIFO order", async () => { + const calls = []; + const first = deferred(); + const transcribeFile = (path) => { + calls.push(path); + if (calls.length === 1) return first.promise; + return Promise.resolve(`text for ${path}`); + }; + const seen = []; + const queue = createFileTranscriptionQueue({ + transcription: { transcribeFile }, + onChange: (jobs) => seen.push(jobs.map((j) => j.status)), + }); + + queue.addFiles(["/a.wav", "/b.wav"]); + await nextTurn(); + + // Only the first job has started - the second is still queued behind it. + assert.deepEqual(calls, ["/a.wav"]); + assert.deepEqual( + queue.getJobs().map((j) => j.status), + ["transcribing", "queued"], + ); + + first.resolve("text for /a.wav"); + await nextTurn(); + await nextTurn(); + + assert.deepEqual(calls, ["/a.wav", "/b.wav"]); + assert.deepEqual( + queue.getJobs().map((j) => j.status), + ["done", "done"], + ); + assert.equal(queue.getJobs()[0].text, "text for /a.wav"); + assert.equal(queue.getJobs()[1].text, "text for /b.wav"); +}); + +test("a failed job records the error and does not block the rest of the queue", async () => { + const queue = createFileTranscriptionQueue({ + transcription: { + transcribeFile: async (path) => { + if (path === "/bad.wav") throw new Error("no file at /bad.wav"); + return "ok text"; + }, + }, + }); + + const [, secondId] = queue.addFiles(["/bad.wav", "/good.wav"]); + await nextTurn(); + await nextTurn(); + + const jobs = queue.getJobs(); + assert.equal(jobs[0].status, "failed"); + assert.equal(jobs[0].error, "no file at /bad.wav"); + const second = jobs.find((j) => j.id === secondId); + assert.equal(second.status, "done"); + assert.equal(second.text, "ok text"); +}); + +test("the configured language is forwarded to transcribeFile", async () => { + const seen = []; + const queue = createFileTranscriptionQueue({ + transcription: { transcribeFile: async (_path, lang) => { seen.push(lang); return "x"; } }, + language: { get: () => "fr" }, + }); + queue.addFiles(["/a.wav"]); + await nextTurn(); + assert.deepEqual(seen, ["fr"]); +}); + +test("retry re-queues a failed job and clears its error", async () => { + let attempt = 0; + const queue = createFileTranscriptionQueue({ + transcription: { + transcribeFile: async () => { + attempt += 1; + if (attempt === 1) throw new Error("first attempt failed"); + return "second time lucky"; + }, + }, + }); + + const [id] = queue.addFiles(["/a.wav"]); + await nextTurn(); + assert.equal(queue.getJobs()[0].status, "failed"); + + assert.equal(queue.retry(id), true); + await nextTurn(); + + const job = queue.getJobs()[0]; + assert.equal(job.status, "done"); + assert.equal(job.text, "second time lucky"); + assert.equal(job.error, null); +}); + +test("retry refuses a job that is not failed", async () => { + const queue = createFileTranscriptionQueue({ + transcription: { transcribeFile: async () => "text" }, + }); + const [id] = queue.addFiles(["/a.wav"]); + await nextTurn(); + assert.equal(queue.getJobs()[0].status, "done"); + assert.equal(queue.retry(id), false); +}); + +test("remove drops a queued or finished job but refuses one in flight", async () => { + const first = deferred(); + const queue = createFileTranscriptionQueue({ + transcription: { transcribeFile: () => first.promise }, + }); + const [id] = queue.addFiles(["/a.wav"]); + await nextTurn(); + assert.equal(queue.getJobs()[0].status, "transcribing"); + assert.equal(queue.remove(id), false); + + first.resolve("done text"); + await nextTurn(); + assert.equal(queue.remove(id), true); + assert.equal(queue.getJobs().length, 0); +}); + +test("clearFinished drops done and failed jobs, leaves queued/transcribing alone", async () => { + const second = deferred(); + const transcribeFile = (path) => (path === "/slow.wav" ? second.promise : Promise.reject(new Error("nope"))); + const queue = createFileTranscriptionQueue({ transcription: { transcribeFile } }); + + queue.addFiles(["/fails.wav", "/slow.wav"]); + await nextTurn(); + await nextTurn(); + + assert.deepEqual( + queue.getJobs().map((j) => j.status), + ["failed", "transcribing"], + ); + queue.clearFinished(); + assert.deepEqual( + queue.getJobs().map((j) => j.status), + ["transcribing"], + ); + + second.resolve("ok"); + await nextTurn(); +}); + +test("onChange fires with a full snapshot on every transition", async () => { + const snapshots = []; + const queue = createFileTranscriptionQueue({ + transcription: { transcribeFile: async () => "text" }, + onChange: (jobs) => snapshots.push(jobs.map((j) => j.status)), + }); + queue.addFiles(["/a.wav"]); + await nextTurn(); + // queued (from addFiles) -> transcribing -> done + assert.deepEqual(snapshots, [["queued"], ["transcribing"], ["done"]]); +}); + +test("a listener that throws never breaks the queue", async () => { + const queue = createFileTranscriptionQueue({ + transcription: { transcribeFile: async () => "text" }, + onChange: () => { + throw new Error("boom"); + }, + }); + queue.addFiles(["/a.wav"]); + await nextTurn(); + assert.equal(queue.getJobs()[0].status, "done"); +}); From 9b5f552ea1523fef79cedb52f2e64e7a3e95e572 Mon Sep 17 00:00:00 2001 From: Nabzx Date: Thu, 17 Sep 2026 00:46:13 +0500 Subject: [PATCH 4/6] wire the file-transcription queue into main.js and the renderer bridge (#253) IPC surface: add (drop), chooseFiles (dialog.showOpenDialog, audio filter + All Files), getQueue, retry, remove, clearFinished, copy, save. Save writes a .txt next to the source file - an explicit action, so re-saving over an earlier save is expected, not guarded against. preload exposes webUtils.getPathForFile as openstream.files - the renderer's only way to resolve a dropped File to a real path now that Electron 32+ dropped the File.path augmentation. --- electron/main.js | 60 +++++++++++++++++++++++++++++++++++++++ electron/preload.js | 31 +++++++++++++++++++- src/openstreamBridge.d.ts | 31 ++++++++++++++++++++ 3 files changed, 121 insertions(+), 1 deletion(-) diff --git a/electron/main.js b/electron/main.js index 328fbfd..fbc3445 100644 --- a/electron/main.js +++ b/electron/main.js @@ -33,6 +33,7 @@ const { createBundleIdReader } = require("./appBundleId"); const { createBreakPlacementHttpAdapter } = require("./breakPlacementHttpAdapter"); const { createDictationIntake } = require("./dictationCoordinator"); const { createVoiceEditIntake } = require("./voiceEditCoordinator"); +const { createFileTranscriptionQueue } = require("./fileTranscriptionQueue"); const { createVocabularyCache } = require("./vocabularyCache"); const { createHeldResultController } = require("./heldResultController"); const { createPushToTalkCoordinator } = require("./pushToTalkCoordinator"); @@ -224,6 +225,18 @@ const voiceEditIntake = createVoiceEditIntake({ onDiagnostic: (name, value) => console.log(`[voice-edit] ${name}: ${JSON.stringify(value)}`), }); +// #253: drop-a-file transcription, a second entry point onto the same +// resident transcription-helper. See fileTranscriptionQueue.js for why a +// long file job can transiently delay a concurrent live dictation - a +// known tradeoff, not something worked around here. +const fileTranscriptionQueue = createFileTranscriptionQueue({ + transcription, + language: { get: () => (settingsStore ? settingsStore.get().inputLanguage : "en") }, + onChange: (jobs) => { + if (win && !win.isDestroyed()) win.webContents.send("file-transcription:queue", jobs); + }, +}); + // A selection longer than this is almost never a deliberate edit target, // and reading a huge AX value at key-down is exactly the case the // injection engine's own max-chars guard exists to avoid. @@ -1081,6 +1094,53 @@ ipcMain.handle("settings:set-input-language", (event, language) => { return settingsStore.setInputLanguage(language); }); +// #253: drop-a-file transcription. addFiles is idempotent-safe to call with +// whatever the renderer resolved from a drop or the file picker below - a +// non-string/empty entry is silently skipped by the queue, not an error. +ipcMain.handle("file-transcription:add", (event, filePaths) => { + return fileTranscriptionQueue.addFiles(Array.isArray(filePaths) ? filePaths : []); +}); + +ipcMain.handle("file-transcription:choose-files", async () => { + if (!win) return []; + const result = await dialog.showOpenDialog(win, { + properties: ["openFile", "multiSelections"], + filters: [ + { name: "Audio", extensions: ["wav", "aiff", "aif", "caf", "m4a", "mp4", "mp3", "aac", "flac"] }, + { name: "All Files", extensions: ["*"] }, + ], + }); + if (result.canceled) return []; + return fileTranscriptionQueue.addFiles(result.filePaths); +}); + +ipcMain.handle("file-transcription:get-queue", () => fileTranscriptionQueue.getJobs()); +ipcMain.handle("file-transcription:retry", (event, id) => fileTranscriptionQueue.retry(id)); +ipcMain.handle("file-transcription:remove", (event, id) => fileTranscriptionQueue.remove(id)); +ipcMain.handle("file-transcription:clear-finished", () => fileTranscriptionQueue.clearFinished()); + +ipcMain.handle("file-transcription:copy", (event, id) => { + const job = fileTranscriptionQueue.getJobs().find((candidate) => candidate.id === id); + if (!job || typeof job.text !== "string") return { ok: false, reason: "no transcript to copy" }; + clipboard.writeText(job.text); + return { ok: true }; +}); + +// #253: "save alongside the file" - same directory, same name, .txt +// extension. An explicit user action (they clicked Save), so a repeat +// click overwriting its own earlier save is expected, not guarded against. +ipcMain.handle("file-transcription:save", (event, id) => { + const job = fileTranscriptionQueue.getJobs().find((candidate) => candidate.id === id); + if (!job || typeof job.text !== "string") return { ok: false, reason: "no transcript to save" }; + const outPath = path.join(path.dirname(job.path), `${path.basename(job.path, path.extname(job.path))}.txt`); + try { + fs.writeFileSync(outPath, job.text, "utf8"); + return { ok: true, path: outPath }; + } catch (error) { + return { ok: false, reason: error instanceof Error ? error.message : String(error) }; + } +}); + // #19: pick an app from disk instead of hunting down its bundle id by // hand. Returns { bundleId, name } for the renderer to add, or null if the // dialog was cancelled; a bundle with no readable identifier rejects. diff --git a/electron/preload.js b/electron/preload.js index 9bd4995..25874c5 100644 --- a/electron/preload.js +++ b/electron/preload.js @@ -1,4 +1,4 @@ -const { contextBridge, ipcRenderer } = require("electron"); +const { contextBridge, ipcRenderer, webUtils } = require("electron"); // Main asks the shell to switch pages (the App menu's "Settings…" item, // the tray). Returns an unsubscribe so the renderer can clean up. @@ -32,6 +32,15 @@ function onHealthChanged(callback) { return () => ipcRenderer.removeListener("health-changed", listener); } +// #253: main pushes the full job list after every queue change - a drop, a +// retry, a job finishing - so the renderer just re-renders from the array +// rather than tracking a diff. +function onFileTranscriptionQueue(callback) { + const listener = (_event, jobs) => callback(jobs); + ipcRenderer.on("file-transcription:queue", listener); + return () => ipcRenderer.removeListener("file-transcription:queue", listener); +} + // The renderer's only route to the main process, per contextIsolation - see // the settings window's webPreferences in main.js. First surface: settings // (#19). This will grow to cover the hotkey helper, the accessibility @@ -74,8 +83,28 @@ contextBridge.exposeInMainWorld("openstream", { getStatus: () => ipcRenderer.invoke("vocabulary:get-status"), chooseFolder: () => ipcRenderer.invoke("vocabulary:choose-folder"), }, + // #253: drop-a-file transcription. + fileTranscription: { + add: (filePaths) => ipcRenderer.invoke("file-transcription:add", filePaths), + chooseFiles: () => ipcRenderer.invoke("file-transcription:choose-files"), + getQueue: () => ipcRenderer.invoke("file-transcription:get-queue"), + retry: (id) => ipcRenderer.invoke("file-transcription:retry", id), + remove: (id) => ipcRenderer.invoke("file-transcription:remove", id), + clearFinished: () => ipcRenderer.invoke("file-transcription:clear-finished"), + copy: (id) => ipcRenderer.invoke("file-transcription:copy", id), + save: (id) => ipcRenderer.invoke("file-transcription:save", id), + }, + // #253: resolves a dropped File object to its real filesystem path. + // webUtils.getPathForFile supersedes the old (deprecated, removed) + // File.path augmentation - only callable from the preload/main context, + // which is why this exists as a bridge method rather than something the + // renderer reaches for directly. + files: { + getPathForFile: (file) => webUtils.getPathForFile(file), + }, onNavigate, onDictationState, onSetupProgress, onHealthChanged, + onFileTranscriptionQueue, }); diff --git a/src/openstreamBridge.d.ts b/src/openstreamBridge.d.ts index 059b2b1..c9c5082 100644 --- a/src/openstreamBridge.d.ts +++ b/src/openstreamBridge.d.ts @@ -68,6 +68,23 @@ export type VocabularyStatus = { scannedAt: Date | null; }; +// #253: one dropped/chosen file's place in the transcription queue. +export type FileTranscriptionStatus = "queued" | "transcribing" | "done" | "failed"; + +export type FileTranscriptionJob = { + id: string; + path: string; + name: string; + status: FileTranscriptionStatus; + text: string | null; + error: string | null; + queuedAt: number; + startedAt: number | null; + finishedAt: number | null; +}; + +export type FileTranscriptionActionResult = { ok: true; path?: string } | { ok: false; reason: string }; + // Exposed by preload.js via contextBridge - see the comment there for what // this is expected to grow into. declare global { @@ -108,10 +125,24 @@ declare global { getStatus(): Promise; chooseFolder(): Promise; }; + fileTranscription: { + add(filePaths: string[]): Promise; + chooseFiles(): Promise; + getQueue(): Promise; + retry(id: string): Promise; + remove(id: string): Promise; + clearFinished(): Promise; + copy(id: string): Promise; + save(id: string): Promise; + }; + files: { + getPathForFile(file: File): string; + }; onNavigate(callback: (page: string) => void): () => void; onDictationState(callback: (state: "idle" | "recording" | "transcribing") => void): () => void; onSetupProgress(callback: (progress: SetupProgress) => void): () => void; onHealthChanged(callback: () => void): () => void; + onFileTranscriptionQueue(callback: (jobs: FileTranscriptionJob[]) => void): () => void; }; } } From 82b9a3acca3f2b0ba66b24c94b60f8d7edf6b7e6 Mon Sep 17 00:00:00 2001 From: Nabzx Date: Thu, 17 Sep 2026 00:50:45 +0500 Subject: [PATCH 5/6] renderer: a Files tab for drop-a-file transcription (#253) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop zone (dashed glass panel, active state on dragover) plus a Choose files… button for the file-picker path, both landing on the same fileTranscription.add() entry point. A queue card below shows each job's status pill, a live elapsed-time readout while it's transcribing, and per-status actions - Copy/Save once done, Retry on failure, Remove for anything not actively running. Verified visually with a stubbed window.openstream against the real compiled bundle (a screenshot harness, not the shipped app) - matches the existing Liquid design system with no new visual bugs. --- src/App.tsx | 3 + src/components/Icons.tsx | 4 + src/index.css | 54 +++++++++++ src/nav.ts | 6 +- src/pages/FileTranscription.tsx | 159 ++++++++++++++++++++++++++++++++ 5 files changed, 223 insertions(+), 3 deletions(-) create mode 100644 src/pages/FileTranscription.tsx diff --git a/src/App.tsx b/src/App.tsx index 0330d27..e86a276 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -1,6 +1,7 @@ import { useEffect, useState } from "react"; import { coercePage, TAB_PAGES, type Page, type TabPage } from "./nav"; import Home from "./pages/Home"; +import FileTranscription from "./pages/FileTranscription"; import Commands from "./pages/Commands"; import Settings from "./pages/Settings"; import Permissions from "./pages/Permissions"; @@ -8,6 +9,7 @@ import Setup from "./pages/Setup"; const TAB_LABELS: Record = { home: "Home", + files: "Files", commands: "Commands", settings: "Settings", }; @@ -41,6 +43,7 @@ export default function App() { {page === "settings" && } + {page === "files" && } {page === "commands" && } {page === "setup" && setPage("home")} />} {page === "permissions" && setPage("home")} />} diff --git a/src/components/Icons.tsx b/src/components/Icons.tsx index b586015..4c3f783 100644 --- a/src/components/Icons.tsx +++ b/src/components/Icons.tsx @@ -80,3 +80,7 @@ export const CommandsIcon = svg( export const CheckIcon = svg(, 3); export const ChevronDownIcon = svg(, 2.5); + +export const FileIcon = svg( + +); diff --git a/src/index.css b/src/index.css index aeeee50..4099849 100644 --- a/src/index.css +++ b/src/index.css @@ -830,6 +830,60 @@ select.field option { color: initial; } .settings-advanced[open] > summary::before { content: "\2304\00a0\00a0"; } .settings-advanced .field { max-width: 240px; } +/* --- file transcription page (#253) ------------------------------------- */ + +.drop-zone { + display: flex; + flex-direction: column; + align-items: center; + gap: var(--space-2); + padding: var(--space-7) var(--space-4); + border: 2px dashed var(--line-hi); + border-radius: var(--r); + background: var(--screen-2); + text-align: center; + transition: border-color 0.12s ease, background 0.12s ease; +} +.drop-zone--active { + border-color: var(--acc); + background: var(--acc-tint); +} +.drop-zone__icon { + width: 28px; + height: 28px; + color: var(--faint); + margin-bottom: 2px; +} +.drop-zone--active .drop-zone__icon { color: var(--acc); } +.drop-zone__title { + margin: 0; + font-size: 14px; + font-weight: 600; + color: var(--fg-hi); +} +.drop-zone__desc { + margin: 0; + font-size: 12px; + color: var(--muted); +} + +.queue-row__name { + color: var(--fg-hi); + text-shadow: var(--shadow-text); +} +.queue-row__error { + display: block; + font-size: 11.5px; + color: var(--err); + margin-top: 1px; +} +.queue-row__actions { + display: flex; + align-items: center; + gap: 4px; + flex: 0 0 auto; +} + /* --- Reduce Transparency fallback -------------------------------------- The clear glass depends on the vibrancy material and a desktop behind it. When the user turns on Reduce Transparency, macOS flattens the diff --git a/src/nav.ts b/src/nav.ts index 4da03bc..5ee992f 100644 --- a/src/nav.ts +++ b/src/nav.ts @@ -4,11 +4,11 @@ // "permissions" and "setup" are reachable views but not toolbar tabs - the // app navigates to them when a grant is missing / a model is downloading. -export type Page = "home" | "commands" | "settings" | "permissions" | "setup"; +export type Page = "home" | "files" | "commands" | "settings" | "permissions" | "setup"; -export const PAGES = ["home", "commands", "settings", "permissions", "setup"] as const; +export const PAGES = ["home", "files", "commands", "settings", "permissions", "setup"] as const; -export const TAB_PAGES = ["home", "commands", "settings"] as const; +export const TAB_PAGES = ["home", "files", "commands", "settings"] as const; export type TabPage = (typeof TAB_PAGES)[number]; export const DEFAULT_PAGE: Page = "home"; diff --git a/src/pages/FileTranscription.tsx b/src/pages/FileTranscription.tsx new file mode 100644 index 0000000..e645e4e --- /dev/null +++ b/src/pages/FileTranscription.tsx @@ -0,0 +1,159 @@ +import { useEffect, useState } from "react"; +import type { FileTranscriptionJob } from "../openstreamBridge"; +import StatusPill, { type PillTone } from "../components/StatusPill"; +import { FileIcon } from "../components/Icons"; + +function statusPill(status: FileTranscriptionJob["status"]): { tone: PillTone; label: string } { + switch (status) { + case "queued": + return { tone: "muted", label: "Queued" }; + case "transcribing": + return { tone: "wait", label: "Transcribing…" }; + case "done": + return { tone: "ok", label: "Done" }; + case "failed": + return { tone: "err", label: "Failed" }; + } +} + +function elapsedLabel(job: FileTranscriptionJob, now: number): string | null { + if (!job.startedAt) return null; + const end = job.finishedAt ?? now; + const seconds = Math.max(0, Math.round((end - job.startedAt) / 1000)); + if (seconds < 60) return `${seconds}s`; + return `${Math.floor(seconds / 60)}m ${seconds % 60}s`; +} + +export default function FileTranscription() { + const [jobs, setJobs] = useState([]); + const [dragging, setDragging] = useState(false); + const [message, setMessage] = useState(null); + const [now, setNow] = useState(() => Date.now()); + + useEffect(() => { + window.openstream.fileTranscription.getQueue().then(setJobs); + return window.openstream.onFileTranscriptionQueue(setJobs); + }, []); + + // The elapsed-time readout on an in-flight job needs its own clock - the + // job list itself only changes on a status transition, not every second. + useEffect(() => { + if (!jobs.some((job) => job.status === "transcribing")) return; + const timer = setInterval(() => setNow(Date.now()), 1000); + return () => clearInterval(timer); + }, [jobs]); + + function handleDrop(event: React.DragEvent) { + event.preventDefault(); + setDragging(false); + const paths = Array.from(event.dataTransfer.files) + .map((file) => window.openstream.files.getPathForFile(file)) + .filter((filePath) => filePath.length > 0); + if (paths.length > 0) void window.openstream.fileTranscription.add(paths); + } + + async function copyJob(id: string) { + const result = await window.openstream.fileTranscription.copy(id); + setMessage(result.ok ? "Copied to the clipboard." : result.reason); + } + + async function saveJob(id: string) { + const result = await window.openstream.fileTranscription.save(id); + setMessage(result.ok ? `Saved to ${result.path}` : result.reason); + } + + const hasFinished = jobs.some((job) => job.status === "done" || job.status === "failed"); + + return ( +
+
{ + event.preventDefault(); + setDragging(true); + }} + onDragLeave={() => setDragging(false)} + onDrop={handleDrop} + > + +

Drop audio files here

+

A meeting recording, a voice memo, a lecture - or

+ +
+ + {jobs.length > 0 && ( +
+
Queue
+ {jobs.map((job) => { + const pill = statusPill(job.status); + const time = elapsedLabel(job, now); + return ( +
+ + {job.name} + {time && {time}} + {job.status === "failed" && job.error && {job.error}} + + +
+ {job.status === "done" && ( + <> + + + + )} + {job.status === "failed" && ( + + )} + {job.status !== "transcribing" && ( + + )} +
+
+ ); + })} + {hasFinished && ( +
+ + +
+ )} +
+ )} + + {message &&

{message}

} + +

+ One file transcribes at a time. A long file can briefly delay a live dictation started while it's running. +

+
+ ); +} From a9e688ce1d0c49f2e0154959b6a15cb95d9248f2 Mon Sep 17 00:00:00 2001 From: Nabzx Date: Thu, 17 Sep 2026 00:51:34 +0500 Subject: [PATCH 6/6] docs: manual-check checklist + a File transcription glossary entry (#253) The manual-check doc leads with the one thing that actually needs verifying before this ships: the Swift transcribeFile command was never executed in the session that wrote it, only compiled - every attempt to run transcription-helper there got killed by the sandbox. --- CONTEXT.md | 4 ++ docs/testing/file-transcription-253.md | 56 ++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 docs/testing/file-transcription-253.md diff --git a/CONTEXT.md b/CONTEXT.md index 90e1c8b..f6a8aba 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -10,6 +10,10 @@ A local-first voice dictation app for macOS. The user holds a key, speaks, and t One complete act of speaking and having the resulting text placed at the cursor. Begins when the user presses the push-to-talk key and ends when the text lands. _Avoid_: Utterance, recording, session +**File transcription**: +A second, distinct mode (#253): the user drops or picks an existing audio file and gets a transcript back, rather than speaking live at the cursor. Shares the transcription engine and its resident model with Dictation, but nothing else - no cursor, no cleanup rules, no break placement, no push-to-talk. One file transcribes at a time; a long one can transiently delay a concurrent Dictation, since both use the same resident model. +_Avoid_: Batch transcription, offline transcription (accurate but not the term used here), import + **Dictation latency budget**: The time from the end of speech to the text arriving at the cursor, which the product commits to keeping under one second. It ends where the user can see it end, so the cost of placing the text is inside the budget rather than outside it. _Avoid_: Response time, turnaround diff --git a/docs/testing/file-transcription-253.md b/docs/testing/file-transcription-253.md new file mode 100644 index 0000000..4c90cb1 --- /dev/null +++ b/docs/testing/file-transcription-253.md @@ -0,0 +1,56 @@ +# Manual check: drop-a-file transcription (#253) + +CI covers the queue coordinator (`fileTranscriptionQueue.test.js`) and the +`transcribeFile` request/reply shape (`transcriptionHelper.test.js`) against +fakes. What it can't drive: the real Swift command, drag-and-drop, the file +picker, or a real audio file actually decoding. + +**The Swift side (`transcribeFile` in `native/transcription-helper`) is +compile-verified only** - it was never executed in the session that wrote it; +every attempt to run `transcription-helper` there got killed by the sandbox +within seconds. Run `npm run build:transcription-helper` before testing any +of this - the checked-in `resources/bin/transcription-helper` predates the +change. + +## Steps + +1. **Basic drop.** Open the **Files** tab. Drag a short `.wav` or `.m4a` + file (a Voice Memo works) onto the drop zone - it highlights on + dragover, and the file appears in the queue as "Transcribing…" then + "Done" with a transcript. +2. **The file picker.** Click **Choose files…**, pick 2-3 files at once + (multi-select) in different formats - `.wav`, `.m4a`, `.mp3` if you + have one handy. All three should queue and process one at a time, in + the order picked - confirm only one row ever shows "Transcribing…" at + once. +3. **A long file.** Drop something over ~30s (FluidAudio's own streaming + threshold) - a longer Voice Memo or an exported meeting recording. + Confirm it still produces a full transcript, not just the first ~30s - + this is what confirms the file-URL `transcribe()` overload is really + auto-chunking rather than silently truncating. +4. **A bad file.** Drop something that isn't audio (a `.txt`, a `.jpg`). + The row should land on "Failed" with a readable error, not crash the + queue or the app - and the next queued file (if any) should still run. +5. **Copy / Save.** On a done job, click **Copy** - paste somewhere to + confirm the clipboard has the transcript. Click **Save** - a `.txt` + with the same base name should appear next to the source file; open it + to confirm the text matches. Save again - it overwrites without asking + (deliberate, see the #253 PR). +6. **Retry / Remove.** On a failed job, **Retry** re-queues it. On any + job that isn't currently transcribing, **Remove** drops it from the + list. Confirm **Remove** is unavailable (or a no-op) on the row that's + actively transcribing. +7. **Clear finished.** With a mix of done/failed/queued jobs, **Clear + finished** should drop the done and failed rows and leave anything + still queued or transcribing. +8. **Concurrency with live dictation - the known tradeoff.** Drop a longish + file, and while it's transcribing, hold push-to-talk and dictate + normally. Confirm what actually happens: does the live dictation wait + behind the file job (expected, per the #253 PR), or does something worse + happen (a crash, a lost dictation, the overlay hanging)? This is the one + behaviour the PR flagged as accepted-but-unverified. +9. **No persistence across restart.** Quit and relaunch with jobs still in + the queue - it should come back empty. This is deliberate for v1 (no + backing store yet - see #253's PR for the "ties into #136" follow-up), + confirm it's not silently losing an in-flight job's *file* (the source + file on disk is never touched, only the in-app queue state resets).