fix: allow slow CPU transcription through the proxy

lib/src/blackfish/server/asgi.py

@@ -9,7 +9,11 @@
 from os import urandom
 import json
 import httpx
-from blackfish.server.http_client import create_http_client, STREAM_TIMEOUT
+from blackfish.server.http_client import (
+    create_http_client,
+    PROXY_TIMEOUT,
+    STREAM_TIMEOUT,
+)
 from datetime import datetime
 from dataclasses import dataclass
 from collections.abc import AsyncGenerator
@@ -1806,9 +1810,13 @@ async def delete_job(
 
 
 async def asyncpost(
-    client: httpx.AsyncClient, url: str, data: Any, headers: Any
+    client: httpx.AsyncClient,
+    url: str,
+    data: Any,
+    headers: Any,
+    timeout: Any = httpx.USE_CLIENT_DEFAULT,
 ) -> Any:
-    response = await client.post(url, content=data, headers=headers)
+    response = await client.post(url, content=data, headers=headers, timeout=timeout)
     return response.json()
 
 
@@ -1870,12 +1878,23 @@ async def generator() -> AsyncGenerator:  # type: ignore
 
         return Stream(generator)
     else:
-        res = await asyncpost(
-            state.http_client,
-            url,
-            json.dumps(data),
-            {"Content-Type": "application/json"},
-        )
+        try:
+            res = await asyncpost(
+                state.http_client,
+                url,
+                json.dumps(data),
+                {"Content-Type": "application/json"},
+                timeout=PROXY_TIMEOUT,
+            )
+        except httpx.ReadTimeout as e:
+            raise HTTPException(
+                status_code=504,
+                detail=(
+                    "The service took too long to respond. This can happen "
+                    "when running inference on CPU (no GPU); try a shorter "
+                    "audio clip or a GPU-backed service."
+                ),
+            ) from e
         return res
 
 

lib/src/blackfish/server/http_client.py

@@ -17,6 +17,12 @@
 # between chunks, so the read timeout is disabled for the streaming proxy.
 STREAM_TIMEOUT = httpx.Timeout(connect=5.0, read=None, write=30.0, pool=5.0)
 
+# Non-streaming proxied inference (e.g. CPU transcription) can take longer than
+# the default 30s read timeout to return a single response. The service UI is
+# meant for short clips, so 60s is a generous-but-bounded ceiling; a read
+# timeout here surfaces as a 504 with an actionable message.
+PROXY_TIMEOUT = httpx.Timeout(connect=5.0, read=60.0, write=30.0, pool=5.0)
+
 
 def create_http_client() -> httpx.AsyncClient:
     return httpx.AsyncClient(timeout=_DEFAULT_TIMEOUT, limits=_DEFAULT_LIMITS)

lib/tests/unit/test_http_client.py

@@ -28,6 +28,12 @@ def test_stream_timeout_disables_read_deadline() -> None:
     assert hc.STREAM_TIMEOUT.read is None
 
 
+def test_proxy_timeout_allows_slow_cpu_inference() -> None:
+    # Non-streaming proxied inference (e.g. CPU transcription) needs more than
+    # the default 30s read timeout, but stays bounded for the short-clip UI.
+    assert hc.PROXY_TIMEOUT.read == 60.0
+
+
 @pytest.mark.anyio
 async def test_client_can_be_closed() -> None:
     client = hc.create_http_client()

web/src/lib/requests.js

@@ -1,6 +1,27 @@
 import { blackfishApiURL } from "../config";
 import { dirname } from "./pathUtils";
 
+/**
+ * Parse an error response into an Error carrying the backend's message and
+ * status, so callers can distinguish failures (e.g. a 504 timeout) and reuse
+ * the server-provided detail rather than hardcoding their own copy.
+ * @param {Response} res - The fetch response object.
+ * @param {string} [fallback] - Message to use when the body has no JSON detail.
+ * @returns {Promise<Error>} An error with `.message` and `.status` set.
+ */
+export async function parseErrorResponse(res, fallback = "Request failed.") {
+  let message = fallback;
+  try {
+    const errorBody = await res.json();
+    message = errorBody.detail || errorBody.message || fallback;
+  } catch {
+    // Response body may not be JSON; keep the fallback message.
+  }
+  const error = new Error(message);
+  error.status = res.status;
+  return error;
+}
+
 /* Return a list of local files with resolved path */
 export async function fetchFiles(path) {
   const res = await fetch(`${blackfishApiURL}/api/${path}`)

web/src/routes/speech-recognition/components/SpeechRecognitionContainer.jsx

@@ -41,16 +41,32 @@ function SpeechRecognitionContainer({
         true,
         controller.signal,
       );
+      // A cancelled request's fetch can still resolve successfully after
+      // abort() (the work was already in flight); its stale result must not
+      // repopulate the box. Check the signal directly so this holds whether or
+      // not a newer request has since replaced abortRef.
+      if (controller.signal.aborted) return;
       setOutput(res.text.trim());
     } catch (err) {
       // A cancelled request is expected — leave the output untouched. Surface a
       // real failure (network error, service returned an error) to the user.
       if (err.name !== "AbortError") {
         console.error("Transcription error:", err);
-        setError({
-          message: "Transcription failed",
-          detail: err.message || "The service may be unavailable.",
-        });
+        if (err.status === 504) {
+          setError({
+            message: "Transcription timed out",
+            // Reuse the backend's detail so the guidance stays in one place.
+            detail:
+              err.message ||
+              "The service took too long to respond. This is common on " +
+                "CPU (no GPU) — try a shorter audio clip or a GPU-backed service.",
+          });
+        } else {
+          setError({
+            message: "Transcription failed",
+            detail: err.message || "The service may be unavailable.",
+          });
+        }
       }
     } finally {
       unregister();

web/src/routes/speech-recognition/lib/requests.js

@@ -1,4 +1,5 @@
 import { blackfishApiURL } from "@/config";
+import { parseErrorResponse } from "@/lib/requests";
 
 /** Call a service with given ID. Pass `signal` to allow cancellation. */
 export async function callSpeechRecognitionInference(service, audioPath, params, use_proxy=false, signal=undefined) {
@@ -21,7 +22,9 @@ export async function callSpeechRecognitionInference(service, audioPath, params,
     signal,
   });
   if (!res.ok) {
-    throw new Error("Failed to call the service"); // activate the closest `error.js` Error Boundary
+    // Preserve the backend's status and message so callers can distinguish a
+    // timeout (504) from other failures and show an actionable message.
+    throw await parseErrorResponse(res, "Failed to call the service");
   }
 
   return res.json();

web/src/routes/speech-recognition/lib/requests.test.js

@@ -0,0 +1,62 @@
+import { describe, it, expect, vi, afterEach } from "vitest";
+import { callSpeechRecognitionInference } from "./requests";
+
+const service = { port: 8080, mount: "/mnt/audio", id: "svc-1" };
+const params = { language: { name: "English" } };
+
+describe("callSpeechRecognitionInference", () => {
+  afterEach(() => {
+    vi.unstubAllGlobals();
+  });
+
+  it("returns the parsed body on success", async () => {
+    const fetchMock = vi.fn().mockResolvedValue({
+      ok: true,
+      json: async () => ({ text: " hello " }),
+    });
+    vi.stubGlobal("fetch", fetchMock);
+
+    const res = await callSpeechRecognitionInference(
+      service,
+      "/mnt/audio/clip.wav",
+      params,
+      true,
+    );
+
+    expect(res).toEqual({ text: " hello " });
+  });
+
+  it("surfaces the status and backend detail on a 504 timeout", async () => {
+    const fetchMock = vi.fn().mockResolvedValue({
+      ok: false,
+      status: 504,
+      json: async () => ({ detail: "The service took too long to respond." }),
+    });
+    vi.stubGlobal("fetch", fetchMock);
+
+    await expect(
+      callSpeechRecognitionInference(service, "/mnt/audio/clip.wav", params, true),
+    ).rejects.toMatchObject({
+      status: 504,
+      message: "The service took too long to respond.",
+    });
+  });
+
+  it("falls back to a generic message when the error body has no JSON", async () => {
+    const fetchMock = vi.fn().mockResolvedValue({
+      ok: false,
+      status: 500,
+      json: async () => {
+        throw new Error("not json");
+      },
+    });
+    vi.stubGlobal("fetch", fetchMock);
+
+    await expect(
+      callSpeechRecognitionInference(service, "/mnt/audio/clip.wav", params, true),
+    ).rejects.toMatchObject({
+      status: 500,
+      message: "Failed to call the service",
+    });
+  });
+});

web/src/routes/text-generation/lib/requests.js

@@ -1,22 +1,8 @@
 import { blackfishApiURL } from "@/config";
+import { parseErrorResponse as parseError } from "@/lib/requests";
 
-/**
- * Parse an error response from a streaming request.
- * @param {Response} res - The fetch response object.
- * @returns {Promise<Error>} An error with the parsed message and status code.
- */
-async function parseErrorResponse(res) {
-  let errorMessage = "Stream request failed.";
-  try {
-    const errorBody = await res.json();
-    errorMessage = errorBody.detail || errorBody.message || errorMessage;
-  } catch {
-    // Response body may not be JSON
-  }
-  const error = new Error(errorMessage);
-  error.status = res.status;
-  return error;
-}
+/** Parse an error response from a streaming request. */
+const parseErrorResponse = (res) => parseError(res, "Stream request failed.");
 
 /**
  * Parse a single SSE chunk from a streaming response.