From cdbdc61e12ed39f2eb775b946055278068fb17f2 Mon Sep 17 00:00:00 2001 From: Colin Swaney Date: Mon, 10 Aug 2026 13:01:51 -0400 Subject: [PATCH 1/3] fix(lib): allow slow CPU inference through the proxy Non-streaming proxied requests inherited the shared client's 30s read timeout, cutting off CPU transcription. Add a bounded PROXY_TIMEOUT (60s) and surface a read timeout as a 504 with CPU/GPU guidance. --- lib/src/blackfish/server/asgi.py | 37 +++++++++++++++++++------ lib/src/blackfish/server/http_client.py | 6 ++++ lib/tests/unit/test_http_client.py | 6 ++++ 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/lib/src/blackfish/server/asgi.py b/lib/src/blackfish/server/asgi.py index 0ae94cbd..40d08e48 100644 --- a/lib/src/blackfish/server/asgi.py +++ b/lib/src/blackfish/server/asgi.py @@ -9,7 +9,11 @@ from os import urandom import json import httpx -from blackfish.server.http_client import create_http_client, STREAM_TIMEOUT +from blackfish.server.http_client import ( + create_http_client, + PROXY_TIMEOUT, + STREAM_TIMEOUT, +) from datetime import datetime from dataclasses import dataclass from collections.abc import AsyncGenerator @@ -1806,9 +1810,13 @@ async def delete_job( async def asyncpost( - client: httpx.AsyncClient, url: str, data: Any, headers: Any + client: httpx.AsyncClient, + url: str, + data: Any, + headers: Any, + timeout: Any = httpx.USE_CLIENT_DEFAULT, ) -> Any: - response = await client.post(url, content=data, headers=headers) + response = await client.post(url, content=data, headers=headers, timeout=timeout) return response.json() @@ -1870,12 +1878,23 @@ async def generator() -> AsyncGenerator: # type: ignore return Stream(generator) else: - res = await asyncpost( - state.http_client, - url, - json.dumps(data), - {"Content-Type": "application/json"}, - ) + try: + res = await asyncpost( + state.http_client, + url, + json.dumps(data), + {"Content-Type": "application/json"}, + timeout=PROXY_TIMEOUT, + ) + except httpx.ReadTimeout: + raise HTTPException( + status_code=504, + detail=( + "The service took too long to respond. This can happen " + "when running inference on CPU (no GPU); try a shorter " + "audio clip or a GPU-backed service." + ), + ) return res diff --git a/lib/src/blackfish/server/http_client.py b/lib/src/blackfish/server/http_client.py index c2ea8359..0d5530e2 100644 --- a/lib/src/blackfish/server/http_client.py +++ b/lib/src/blackfish/server/http_client.py @@ -17,6 +17,12 @@ # between chunks, so the read timeout is disabled for the streaming proxy. STREAM_TIMEOUT = httpx.Timeout(connect=5.0, read=None, write=30.0, pool=5.0) +# Non-streaming proxied inference (e.g. CPU transcription) can take longer than +# the default 30s read timeout to return a single response. The service UI is +# meant for short clips, so 60s is a generous-but-bounded ceiling; a read +# timeout here surfaces as a 504 with an actionable message. +PROXY_TIMEOUT = httpx.Timeout(connect=5.0, read=60.0, write=30.0, pool=5.0) + def create_http_client() -> httpx.AsyncClient: return httpx.AsyncClient(timeout=_DEFAULT_TIMEOUT, limits=_DEFAULT_LIMITS) diff --git a/lib/tests/unit/test_http_client.py b/lib/tests/unit/test_http_client.py index 5fc8f75f..99acc6bb 100644 --- a/lib/tests/unit/test_http_client.py +++ b/lib/tests/unit/test_http_client.py @@ -28,6 +28,12 @@ def test_stream_timeout_disables_read_deadline() -> None: assert hc.STREAM_TIMEOUT.read is None +def test_proxy_timeout_allows_slow_cpu_inference() -> None: + # Non-streaming proxied inference (e.g. CPU transcription) needs more than + # the default 30s read timeout, but stays bounded for the short-clip UI. + assert hc.PROXY_TIMEOUT.read == 60.0 + + @pytest.mark.anyio async def test_client_can_be_closed() -> None: client = hc.create_http_client() From 92666747897db891e1386b3aa8349e553d9c723e Mon Sep 17 00:00:00 2001 From: Colin Swaney Date: Mon, 10 Aug 2026 13:01:51 -0400 Subject: [PATCH 2/3] fix(web): handle transcription timeout and stale responses Surface the backend 504 as a clear timeout toast, and guard against a cancelled request's late success repopulating the output box. --- .../components/SpeechRecognitionContainer.jsx | 22 +++++-- .../routes/speech-recognition/lib/requests.js | 12 +++- .../speech-recognition/lib/requests.test.js | 62 +++++++++++++++++++ 3 files changed, 91 insertions(+), 5 deletions(-) create mode 100644 web/src/routes/speech-recognition/lib/requests.test.js diff --git a/web/src/routes/speech-recognition/components/SpeechRecognitionContainer.jsx b/web/src/routes/speech-recognition/components/SpeechRecognitionContainer.jsx index 6a56450e..7de9a526 100644 --- a/web/src/routes/speech-recognition/components/SpeechRecognitionContainer.jsx +++ b/web/src/routes/speech-recognition/components/SpeechRecognitionContainer.jsx @@ -41,16 +41,30 @@ function SpeechRecognitionContainer({ true, controller.signal, ); + // A cancelled request's fetch can still resolve successfully after + // abort() (the work was already in flight); its stale result must not + // repopulate the box. Check the signal directly so this holds whether or + // not a newer request has since replaced abortRef. + if (controller.signal.aborted) return; setOutput(res.text.trim()); } catch (err) { // A cancelled request is expected — leave the output untouched. Surface a // real failure (network error, service returned an error) to the user. if (err.name !== "AbortError") { console.error("Transcription error:", err); - setError({ - message: "Transcription failed", - detail: err.message || "The service may be unavailable.", - }); + if (err.status === 504) { + setError({ + message: "Transcription timed out", + detail: + "The service took too long to respond. This is common on CPU " + + "(no GPU) — try a shorter audio clip or a GPU-backed service.", + }); + } else { + setError({ + message: "Transcription failed", + detail: err.message || "The service may be unavailable.", + }); + } } } finally { unregister(); diff --git a/web/src/routes/speech-recognition/lib/requests.js b/web/src/routes/speech-recognition/lib/requests.js index cfdffc73..870868e3 100644 --- a/web/src/routes/speech-recognition/lib/requests.js +++ b/web/src/routes/speech-recognition/lib/requests.js @@ -21,7 +21,17 @@ export async function callSpeechRecognitionInference(service, audioPath, params, signal, }); if (!res.ok) { - throw new Error("Failed to call the service"); // activate the closest `error.js` Error Boundary + // Preserve the backend's status and message so callers can distinguish a + // timeout (504) from other failures and show an actionable message. + let detail; + try { + detail = (await res.json()).detail; + } catch { + // Response had no JSON body; fall back to a generic message below. + } + const err = new Error(detail || "Failed to call the service"); + err.status = res.status; + throw err; } return res.json(); diff --git a/web/src/routes/speech-recognition/lib/requests.test.js b/web/src/routes/speech-recognition/lib/requests.test.js new file mode 100644 index 00000000..992f3f44 --- /dev/null +++ b/web/src/routes/speech-recognition/lib/requests.test.js @@ -0,0 +1,62 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { callSpeechRecognitionInference } from "./requests"; + +const service = { port: 8080, mount: "/mnt/audio", id: "svc-1" }; +const params = { language: { name: "English" } }; + +describe("callSpeechRecognitionInference", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("returns the parsed body on success", async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ text: " hello " }), + }); + vi.stubGlobal("fetch", fetchMock); + + const res = await callSpeechRecognitionInference( + service, + "/mnt/audio/clip.wav", + params, + true, + ); + + expect(res).toEqual({ text: " hello " }); + }); + + it("surfaces the status and backend detail on a 504 timeout", async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: false, + status: 504, + json: async () => ({ detail: "The service took too long to respond." }), + }); + vi.stubGlobal("fetch", fetchMock); + + await expect( + callSpeechRecognitionInference(service, "/mnt/audio/clip.wav", params, true), + ).rejects.toMatchObject({ + status: 504, + message: "The service took too long to respond.", + }); + }); + + it("falls back to a generic message when the error body has no JSON", async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: false, + status: 500, + json: async () => { + throw new Error("not json"); + }, + }); + vi.stubGlobal("fetch", fetchMock); + + await expect( + callSpeechRecognitionInference(service, "/mnt/audio/clip.wav", params, true), + ).rejects.toMatchObject({ + status: 500, + message: "Failed to call the service", + }); + }); +}); From a86941a6f5a8f998ef4708cdc6d91fbb2e568c2b Mon Sep 17 00:00:00 2001 From: Colin Swaney Date: Wed, 12 Aug 2026 09:26:13 -0400 Subject: [PATCH 3/3] refactor(web): share error parsing and reuse backend timeout copy Lift parseErrorResponse into the shared lib so speech-recognition and text-generation don't diverge; reuse the backend's 504 detail in the timeout toast. Chain the proxy ReadTimeout -> 504 with `from e`. --- lib/src/blackfish/server/asgi.py | 4 ++-- web/src/lib/requests.js | 21 +++++++++++++++++++ .../components/SpeechRecognitionContainer.jsx | 6 ++++-- .../routes/speech-recognition/lib/requests.js | 11 ++-------- .../routes/text-generation/lib/requests.js | 20 +++--------------- 5 files changed, 32 insertions(+), 30 deletions(-) diff --git a/lib/src/blackfish/server/asgi.py b/lib/src/blackfish/server/asgi.py index 40d08e48..5f662b12 100644 --- a/lib/src/blackfish/server/asgi.py +++ b/lib/src/blackfish/server/asgi.py @@ -1886,7 +1886,7 @@ async def generator() -> AsyncGenerator: # type: ignore {"Content-Type": "application/json"}, timeout=PROXY_TIMEOUT, ) - except httpx.ReadTimeout: + except httpx.ReadTimeout as e: raise HTTPException( status_code=504, detail=( @@ -1894,7 +1894,7 @@ async def generator() -> AsyncGenerator: # type: ignore "when running inference on CPU (no GPU); try a shorter " "audio clip or a GPU-backed service." ), - ) + ) from e return res diff --git a/web/src/lib/requests.js b/web/src/lib/requests.js index 158bf909..a15f37cc 100644 --- a/web/src/lib/requests.js +++ b/web/src/lib/requests.js @@ -1,6 +1,27 @@ import { blackfishApiURL } from "../config"; import { dirname } from "./pathUtils"; +/** + * Parse an error response into an Error carrying the backend's message and + * status, so callers can distinguish failures (e.g. a 504 timeout) and reuse + * the server-provided detail rather than hardcoding their own copy. + * @param {Response} res - The fetch response object. + * @param {string} [fallback] - Message to use when the body has no JSON detail. + * @returns {Promise} An error with `.message` and `.status` set. + */ +export async function parseErrorResponse(res, fallback = "Request failed.") { + let message = fallback; + try { + const errorBody = await res.json(); + message = errorBody.detail || errorBody.message || fallback; + } catch { + // Response body may not be JSON; keep the fallback message. + } + const error = new Error(message); + error.status = res.status; + return error; +} + /* Return a list of local files with resolved path */ export async function fetchFiles(path) { const res = await fetch(`${blackfishApiURL}/api/${path}`) diff --git a/web/src/routes/speech-recognition/components/SpeechRecognitionContainer.jsx b/web/src/routes/speech-recognition/components/SpeechRecognitionContainer.jsx index 7de9a526..16e5f3ed 100644 --- a/web/src/routes/speech-recognition/components/SpeechRecognitionContainer.jsx +++ b/web/src/routes/speech-recognition/components/SpeechRecognitionContainer.jsx @@ -55,9 +55,11 @@ function SpeechRecognitionContainer({ if (err.status === 504) { setError({ message: "Transcription timed out", + // Reuse the backend's detail so the guidance stays in one place. detail: - "The service took too long to respond. This is common on CPU " + - "(no GPU) — try a shorter audio clip or a GPU-backed service.", + err.message || + "The service took too long to respond. This is common on " + + "CPU (no GPU) — try a shorter audio clip or a GPU-backed service.", }); } else { setError({ diff --git a/web/src/routes/speech-recognition/lib/requests.js b/web/src/routes/speech-recognition/lib/requests.js index 870868e3..753e0aef 100644 --- a/web/src/routes/speech-recognition/lib/requests.js +++ b/web/src/routes/speech-recognition/lib/requests.js @@ -1,4 +1,5 @@ import { blackfishApiURL } from "@/config"; +import { parseErrorResponse } from "@/lib/requests"; /** Call a service with given ID. Pass `signal` to allow cancellation. */ export async function callSpeechRecognitionInference(service, audioPath, params, use_proxy=false, signal=undefined) { @@ -23,15 +24,7 @@ export async function callSpeechRecognitionInference(service, audioPath, params, if (!res.ok) { // Preserve the backend's status and message so callers can distinguish a // timeout (504) from other failures and show an actionable message. - let detail; - try { - detail = (await res.json()).detail; - } catch { - // Response had no JSON body; fall back to a generic message below. - } - const err = new Error(detail || "Failed to call the service"); - err.status = res.status; - throw err; + throw await parseErrorResponse(res, "Failed to call the service"); } return res.json(); diff --git a/web/src/routes/text-generation/lib/requests.js b/web/src/routes/text-generation/lib/requests.js index db526ee1..b1dc5230 100644 --- a/web/src/routes/text-generation/lib/requests.js +++ b/web/src/routes/text-generation/lib/requests.js @@ -1,22 +1,8 @@ import { blackfishApiURL } from "@/config"; +import { parseErrorResponse as parseError } from "@/lib/requests"; -/** - * Parse an error response from a streaming request. - * @param {Response} res - The fetch response object. - * @returns {Promise} An error with the parsed message and status code. - */ -async function parseErrorResponse(res) { - let errorMessage = "Stream request failed."; - try { - const errorBody = await res.json(); - errorMessage = errorBody.detail || errorBody.message || errorMessage; - } catch { - // Response body may not be JSON - } - const error = new Error(errorMessage); - error.status = res.status; - return error; -} +/** Parse an error response from a streaming request. */ +const parseErrorResponse = (res) => parseError(res, "Stream request failed."); /** * Parse a single SSE chunk from a streaming response.