diff --git a/lib/src/blackfish/server/asgi.py b/lib/src/blackfish/server/asgi.py index 0ae94cbd..5f662b12 100644 --- a/lib/src/blackfish/server/asgi.py +++ b/lib/src/blackfish/server/asgi.py @@ -9,7 +9,11 @@ from os import urandom import json import httpx -from blackfish.server.http_client import create_http_client, STREAM_TIMEOUT +from blackfish.server.http_client import ( + create_http_client, + PROXY_TIMEOUT, + STREAM_TIMEOUT, +) from datetime import datetime from dataclasses import dataclass from collections.abc import AsyncGenerator @@ -1806,9 +1810,13 @@ async def delete_job( async def asyncpost( - client: httpx.AsyncClient, url: str, data: Any, headers: Any + client: httpx.AsyncClient, + url: str, + data: Any, + headers: Any, + timeout: Any = httpx.USE_CLIENT_DEFAULT, ) -> Any: - response = await client.post(url, content=data, headers=headers) + response = await client.post(url, content=data, headers=headers, timeout=timeout) return response.json() @@ -1870,12 +1878,23 @@ async def generator() -> AsyncGenerator: # type: ignore return Stream(generator) else: - res = await asyncpost( - state.http_client, - url, - json.dumps(data), - {"Content-Type": "application/json"}, - ) + try: + res = await asyncpost( + state.http_client, + url, + json.dumps(data), + {"Content-Type": "application/json"}, + timeout=PROXY_TIMEOUT, + ) + except httpx.ReadTimeout as e: + raise HTTPException( + status_code=504, + detail=( + "The service took too long to respond. This can happen " + "when running inference on CPU (no GPU); try a shorter " + "audio clip or a GPU-backed service." + ), + ) from e return res diff --git a/lib/src/blackfish/server/http_client.py b/lib/src/blackfish/server/http_client.py index c2ea8359..0d5530e2 100644 --- a/lib/src/blackfish/server/http_client.py +++ b/lib/src/blackfish/server/http_client.py @@ -17,6 +17,12 @@ # between chunks, so the read timeout is disabled for the streaming proxy. STREAM_TIMEOUT = httpx.Timeout(connect=5.0, read=None, write=30.0, pool=5.0) +# Non-streaming proxied inference (e.g. CPU transcription) can take longer than +# the default 30s read timeout to return a single response. The service UI is +# meant for short clips, so 60s is a generous-but-bounded ceiling; a read +# timeout here surfaces as a 504 with an actionable message. +PROXY_TIMEOUT = httpx.Timeout(connect=5.0, read=60.0, write=30.0, pool=5.0) + def create_http_client() -> httpx.AsyncClient: return httpx.AsyncClient(timeout=_DEFAULT_TIMEOUT, limits=_DEFAULT_LIMITS) diff --git a/lib/tests/unit/test_http_client.py b/lib/tests/unit/test_http_client.py index 5fc8f75f..99acc6bb 100644 --- a/lib/tests/unit/test_http_client.py +++ b/lib/tests/unit/test_http_client.py @@ -28,6 +28,12 @@ def test_stream_timeout_disables_read_deadline() -> None: assert hc.STREAM_TIMEOUT.read is None +def test_proxy_timeout_allows_slow_cpu_inference() -> None: + # Non-streaming proxied inference (e.g. CPU transcription) needs more than + # the default 30s read timeout, but stays bounded for the short-clip UI. + assert hc.PROXY_TIMEOUT.read == 60.0 + + @pytest.mark.anyio async def test_client_can_be_closed() -> None: client = hc.create_http_client() diff --git a/web/src/lib/requests.js b/web/src/lib/requests.js index 158bf909..a15f37cc 100644 --- a/web/src/lib/requests.js +++ b/web/src/lib/requests.js @@ -1,6 +1,27 @@ import { blackfishApiURL } from "../config"; import { dirname } from "./pathUtils"; +/** + * Parse an error response into an Error carrying the backend's message and + * status, so callers can distinguish failures (e.g. a 504 timeout) and reuse + * the server-provided detail rather than hardcoding their own copy. + * @param {Response} res - The fetch response object. + * @param {string} [fallback] - Message to use when the body has no JSON detail. + * @returns {Promise} An error with `.message` and `.status` set. + */ +export async function parseErrorResponse(res, fallback = "Request failed.") { + let message = fallback; + try { + const errorBody = await res.json(); + message = errorBody.detail || errorBody.message || fallback; + } catch { + // Response body may not be JSON; keep the fallback message. + } + const error = new Error(message); + error.status = res.status; + return error; +} + /* Return a list of local files with resolved path */ export async function fetchFiles(path) { const res = await fetch(`${blackfishApiURL}/api/${path}`) diff --git a/web/src/routes/speech-recognition/components/SpeechRecognitionContainer.jsx b/web/src/routes/speech-recognition/components/SpeechRecognitionContainer.jsx index 6a56450e..16e5f3ed 100644 --- a/web/src/routes/speech-recognition/components/SpeechRecognitionContainer.jsx +++ b/web/src/routes/speech-recognition/components/SpeechRecognitionContainer.jsx @@ -41,16 +41,32 @@ function SpeechRecognitionContainer({ true, controller.signal, ); + // A cancelled request's fetch can still resolve successfully after + // abort() (the work was already in flight); its stale result must not + // repopulate the box. Check the signal directly so this holds whether or + // not a newer request has since replaced abortRef. + if (controller.signal.aborted) return; setOutput(res.text.trim()); } catch (err) { // A cancelled request is expected — leave the output untouched. Surface a // real failure (network error, service returned an error) to the user. if (err.name !== "AbortError") { console.error("Transcription error:", err); - setError({ - message: "Transcription failed", - detail: err.message || "The service may be unavailable.", - }); + if (err.status === 504) { + setError({ + message: "Transcription timed out", + // Reuse the backend's detail so the guidance stays in one place. + detail: + err.message || + "The service took too long to respond. This is common on " + + "CPU (no GPU) — try a shorter audio clip or a GPU-backed service.", + }); + } else { + setError({ + message: "Transcription failed", + detail: err.message || "The service may be unavailable.", + }); + } } } finally { unregister(); diff --git a/web/src/routes/speech-recognition/lib/requests.js b/web/src/routes/speech-recognition/lib/requests.js index cfdffc73..753e0aef 100644 --- a/web/src/routes/speech-recognition/lib/requests.js +++ b/web/src/routes/speech-recognition/lib/requests.js @@ -1,4 +1,5 @@ import { blackfishApiURL } from "@/config"; +import { parseErrorResponse } from "@/lib/requests"; /** Call a service with given ID. Pass `signal` to allow cancellation. */ export async function callSpeechRecognitionInference(service, audioPath, params, use_proxy=false, signal=undefined) { @@ -21,7 +22,9 @@ export async function callSpeechRecognitionInference(service, audioPath, params, signal, }); if (!res.ok) { - throw new Error("Failed to call the service"); // activate the closest `error.js` Error Boundary + // Preserve the backend's status and message so callers can distinguish a + // timeout (504) from other failures and show an actionable message. + throw await parseErrorResponse(res, "Failed to call the service"); } return res.json(); diff --git a/web/src/routes/speech-recognition/lib/requests.test.js b/web/src/routes/speech-recognition/lib/requests.test.js new file mode 100644 index 00000000..992f3f44 --- /dev/null +++ b/web/src/routes/speech-recognition/lib/requests.test.js @@ -0,0 +1,62 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { callSpeechRecognitionInference } from "./requests"; + +const service = { port: 8080, mount: "/mnt/audio", id: "svc-1" }; +const params = { language: { name: "English" } }; + +describe("callSpeechRecognitionInference", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("returns the parsed body on success", async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ text: " hello " }), + }); + vi.stubGlobal("fetch", fetchMock); + + const res = await callSpeechRecognitionInference( + service, + "/mnt/audio/clip.wav", + params, + true, + ); + + expect(res).toEqual({ text: " hello " }); + }); + + it("surfaces the status and backend detail on a 504 timeout", async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: false, + status: 504, + json: async () => ({ detail: "The service took too long to respond." }), + }); + vi.stubGlobal("fetch", fetchMock); + + await expect( + callSpeechRecognitionInference(service, "/mnt/audio/clip.wav", params, true), + ).rejects.toMatchObject({ + status: 504, + message: "The service took too long to respond.", + }); + }); + + it("falls back to a generic message when the error body has no JSON", async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: false, + status: 500, + json: async () => { + throw new Error("not json"); + }, + }); + vi.stubGlobal("fetch", fetchMock); + + await expect( + callSpeechRecognitionInference(service, "/mnt/audio/clip.wav", params, true), + ).rejects.toMatchObject({ + status: 500, + message: "Failed to call the service", + }); + }); +}); diff --git a/web/src/routes/text-generation/lib/requests.js b/web/src/routes/text-generation/lib/requests.js index db526ee1..b1dc5230 100644 --- a/web/src/routes/text-generation/lib/requests.js +++ b/web/src/routes/text-generation/lib/requests.js @@ -1,22 +1,8 @@ import { blackfishApiURL } from "@/config"; +import { parseErrorResponse as parseError } from "@/lib/requests"; -/** - * Parse an error response from a streaming request. - * @param {Response} res - The fetch response object. - * @returns {Promise} An error with the parsed message and status code. - */ -async function parseErrorResponse(res) { - let errorMessage = "Stream request failed."; - try { - const errorBody = await res.json(); - errorMessage = errorBody.detail || errorBody.message || errorMessage; - } catch { - // Response body may not be JSON - } - const error = new Error(errorMessage); - error.status = res.status; - return error; -} +/** Parse an error response from a streaming request. */ +const parseErrorResponse = (res) => parseError(res, "Stream request failed."); /** * Parse a single SSE chunk from a streaming response.