From 01c4fc9ab7a258584e744228a4678cff4d7bd3a3 Mon Sep 17 00:00:00 2001 From: nraffa Date: Tue, 7 Apr 2026 13:03:43 +1000 Subject: [PATCH 1/5] feat(backend): add speech-to-text endpoint for voice input feature Add a new POST /speech-to-text/transcribe endpoint that accepts audio uploads and returns transcribed text via Google Cloud Speech-to-Text v2.Feature is behind a configuration toggle (disabled by default), following the same pattern as CV upload. --- backend/.env.example | 5 + backend/app/app_config.py | 6 + backend/app/server.py | 16 ++ backend/app/speech_to_text/__init__.py | 0 backend/app/speech_to_text/constants.py | 23 +++ backend/app/speech_to_text/errors.py | 15 ++ backend/app/speech_to_text/routes.py | 142 +++++++++++++++ backend/app/speech_to_text/service.py | 86 ++++++++++ backend/app/speech_to_text/test_routes.py | 191 +++++++++++++++++++++ backend/app/speech_to_text/test_service.py | 109 ++++++++++++ backend/app/speech_to_text/types.py | 12 ++ backend/conftest.py | 1 + backend/poetry.lock | 157 +++++++++++++++-- backend/pyproject.toml | 1 + config/default.json | 3 + config/inject-config.py | 6 + 16 files changed, 756 insertions(+), 17 deletions(-) create mode 100644 backend/app/speech_to_text/__init__.py create mode 100644 backend/app/speech_to_text/constants.py create mode 100644 backend/app/speech_to_text/errors.py create mode 100644 backend/app/speech_to_text/routes.py create mode 100644 backend/app/speech_to_text/service.py create mode 100644 backend/app/speech_to_text/test_routes.py create mode 100644 backend/app/speech_to_text/test_service.py create mode 100644 backend/app/speech_to_text/types.py diff --git a/backend/.env.example b/backend/.env.example index cb12c7c1..66cc2b98 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -63,6 +63,11 @@ BACKEND_CV_MAX_UPLOADS_PER_USER= BACKEND_CV_RATE_LIMIT_PER_MINUTE= +# Speech-to-text (voice input) feature +GLOBAL_ENABLE_SPEECH_TO_TEXT=false +# Required when speech-to-text is enabled: the GCP project ID +GOOGLE_CLOUD_PROJECT= + # Locales BACKEND_LANGUAGE_CONFIG='{"default_locale":"en-US","available_locales":[{"locale":"en-US","date_format":"MM/DD/YYYY"}]}' diff --git a/backend/app/app_config.py b/backend/app/app_config.py index df76305c..9797ea00 100644 --- a/backend/app/app_config.py +++ b/backend/app/app_config.py @@ -73,6 +73,12 @@ class ApplicationConfig(BaseModel): When disabled, CV upload routes will not be registered and CV indexes will not be created. """ + enable_speech_to_text: bool = False + """ + A flag to enable or disable the Speech-to-Text (voice input) feature. + When disabled, speech-to-text routes will not be registered. + """ + # CV storage and upload limits cv_storage_bucket: Optional[str] = "" cv_max_uploads_per_user: Optional[int] = Field(default=DEFAULT_MAX_UPLOADS_PER_USER, gt=0) diff --git a/backend/app/server.py b/backend/app/server.py index f956becc..c7739a44 100644 --- a/backend/app/server.py +++ b/backend/app/server.py @@ -21,6 +21,7 @@ from app.metrics.routes.routes import add_metrics_routes from app.analytics.routes import add_analytics_routes from app.teveta.routes import add_teveta_routes +from app.speech_to_text.routes import add_speech_to_text_routes from app.sentry_init import init_sentry, set_sentry_contexts from app.server_dependencies.db_dependencies import CompassDBProvider from app.users.auth import Authentication, ApiKeyAuth @@ -226,6 +227,11 @@ def setup_sentry(): _enable_cv_upload = _enable_cv_upload_str.lower() == "true" logger.info(f"GLOBAL_ENABLE_CV_UPLOAD: {_enable_cv_upload}") +# Speech-to-text feature flag - defaults to False if not set +_enable_speech_to_text_str = os.getenv("GLOBAL_ENABLE_SPEECH_TO_TEXT", "false") +_enable_speech_to_text = _enable_speech_to_text_str.lower() == "true" +logger.info(f"GLOBAL_ENABLE_SPEECH_TO_TEXT: {_enable_speech_to_text}") + application_config = ApplicationConfig( environment_name=os.getenv("TARGET_ENVIRONMENT_NAME"), version_info=load_version_info(), @@ -237,6 +243,7 @@ def setup_sentry(): features=backend_features_config, experience_pipeline_config=experience_pipeline_config, enable_cv_upload=_enable_cv_upload, + enable_speech_to_text=_enable_speech_to_text, cv_storage_bucket=os.getenv("BACKEND_CV_STORAGE_BUCKET", ""), cv_max_uploads_per_user=os.getenv("BACKEND_CV_MAX_UPLOADS_PER_USER") or DEFAULT_MAX_UPLOADS_PER_USER, cv_rate_limit_per_minute=os.getenv("BACKEND_CV_RATE_LIMIT_PER_MINUTE") or DEFAULT_RATE_LIMIT_PER_MINUTE, @@ -473,6 +480,15 @@ async def lifespan(_app: FastAPI): ############################################ add_poc_routes(app, auth) +############################################ +# Add speech-to-text routes (conditionally) +############################################ +if _enable_speech_to_text: + add_speech_to_text_routes(app, auth) + logger.info("Speech-to-text routes registered") +else: + logger.info("Speech-to-text routes skipped (GLOBAL_ENABLE_SPEECH_TO_TEXT is not enabled)") + ############################################ # Add other features routes ############################################ diff --git a/backend/app/speech_to_text/__init__.py b/backend/app/speech_to_text/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/speech_to_text/constants.py b/backend/app/speech_to_text/constants.py new file mode 100644 index 00000000..1f37a3b4 --- /dev/null +++ b/backend/app/speech_to_text/constants.py @@ -0,0 +1,23 @@ +# Maximum audio file size: 10 MB +MAX_AUDIO_SIZE_BYTES = 10 * 1024 * 1024 + +# Maximum multipart overhead for early Content-Length rejection +MAX_MULTIPART_OVERHEAD_BYTES = 1024 + +# Allowed MIME types from browser MediaRecorder and phone recordings +ALLOWED_AUDIO_MIME_TYPES = { + "audio/webm", + "audio/webm;codecs=opus", + "audio/ogg", + "audio/ogg;codecs=opus", + "audio/mp4", + "audio/mpeg", + "audio/wav", + "audio/x-m4a", + "audio/aac", + "video/mp4", + "video/webm", +} + +# Default language for transcription +DEFAULT_LANGUAGE_CODE = "en-US" diff --git a/backend/app/speech_to_text/errors.py b/backend/app/speech_to_text/errors.py new file mode 100644 index 00000000..e1ef5178 --- /dev/null +++ b/backend/app/speech_to_text/errors.py @@ -0,0 +1,15 @@ +class TranscriptionError(Exception): + """Base exception for speech-to-text transcription errors.""" + + +class EmptyTranscriptionError(TranscriptionError): + """Raised when Google STT returns no transcription results.""" + def __init__(self): + super().__init__("No transcription results returned for the provided audio") + + +class SpeechToTextServiceError(TranscriptionError): + """Raised when the Google Cloud STT API call fails.""" + def __init__(self, detail: str): + super().__init__(f"Speech-to-text service error: {detail}") + self.detail = detail diff --git a/backend/app/speech_to_text/routes.py b/backend/app/speech_to_text/routes.py new file mode 100644 index 00000000..b90cf88a --- /dev/null +++ b/backend/app/speech_to_text/routes.py @@ -0,0 +1,142 @@ +import asyncio +import logging +from http import HTTPStatus + +from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile + +from app.constants.errors import HTTPErrorResponse +from app.speech_to_text.constants import ( + ALLOWED_AUDIO_MIME_TYPES, + DEFAULT_LANGUAGE_CODE, + MAX_AUDIO_SIZE_BYTES, + MAX_MULTIPART_OVERHEAD_BYTES, +) +from app.speech_to_text.errors import EmptyTranscriptionError, SpeechToTextServiceError +from app.speech_to_text.service import GoogleSpeechToTextService, ISpeechToTextService +from app.speech_to_text.types import TranscriptionResponse +from app.users.auth import Authentication, UserInfo + +logger = logging.getLogger(__name__) + +_stt_service_lock = asyncio.Lock() +_stt_service_singleton: ISpeechToTextService | None = None + + +async def _get_speech_to_text_service() -> ISpeechToTextService: + global _stt_service_singleton # pylint: disable=global-statement + if _stt_service_singleton is None: + async with _stt_service_lock: + if _stt_service_singleton is None: + _stt_service_singleton = GoogleSpeechToTextService() + return _stt_service_singleton + + +# Public alias for dependency override in tests +get_speech_to_text_service = _get_speech_to_text_service + + +def _validate_request_size_header(request: Request): + """Validate Content-Length before reading the body to fail fast on oversized uploads.""" + content_length_header = request.headers.get("content-length") + if content_length_header is None: + return + try: + content_length = int(content_length_header) + except ValueError: + return + if content_length > (MAX_AUDIO_SIZE_BYTES + MAX_MULTIPART_OVERHEAD_BYTES): + logger.warning( + "413 via header-check: content_length=%s limit=%s", + content_length, + MAX_AUDIO_SIZE_BYTES, + ) + raise HTTPException( + status_code=HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + detail="Audio file exceeds maximum allowed size", + ) + + +def add_speech_to_text_routes(app, authentication: Authentication): + """Register speech-to-text routes on the given FastAPI app.""" + router = APIRouter(prefix="/speech-to-text", tags=["speech-to-text"]) + + @router.post( + path="/transcribe", + status_code=HTTPStatus.OK, + response_model=TranscriptionResponse, + responses={ + HTTPStatus.BAD_REQUEST: {"model": HTTPErrorResponse}, + HTTPStatus.UNSUPPORTED_MEDIA_TYPE: {"model": HTTPErrorResponse}, + HTTPStatus.REQUEST_ENTITY_TOO_LARGE: {"model": HTTPErrorResponse}, + HTTPStatus.UNPROCESSABLE_ENTITY: {"model": HTTPErrorResponse}, + HTTPStatus.BAD_GATEWAY: {"model": HTTPErrorResponse}, + HTTPStatus.INTERNAL_SERVER_ERROR: {"model": HTTPErrorResponse}, + }, + name="transcribe audio", + description="Transcribe audio input to text using Google Cloud Speech-to-Text.", + ) + async def _transcribe_audio( + request: Request, + audio: UploadFile = File(..., description="Audio file recorded in the browser"), + language: str = Form(default=DEFAULT_LANGUAGE_CODE), + user_info: UserInfo = Depends(authentication.get_user_info()), + service: ISpeechToTextService = Depends(get_speech_to_text_service), + ) -> TranscriptionResponse: + # Validate size early via Content-Length header + _validate_request_size_header(request) + + # Validate MIME type — normalize by stripping spaces around semicolons + raw_content_type = (audio.content_type or "").strip() + # Check both the base type (e.g. "audio/webm") and the full type with params normalized + # (e.g. "audio/webm;codecs=opus") to handle browsers that add spaces after ";" + content_type = raw_content_type.split(";")[0].strip() + normalized_content_type = ";".join(part.strip() for part in raw_content_type.split(";")) + if content_type not in ALLOWED_AUDIO_MIME_TYPES and normalized_content_type not in ALLOWED_AUDIO_MIME_TYPES: + raise HTTPException( + status_code=HTTPStatus.UNSUPPORTED_MEDIA_TYPE, + detail="Unsupported audio format", + ) + + # Read and validate file size + audio_bytes = await audio.read() + if len(audio_bytes) > MAX_AUDIO_SIZE_BYTES: + raise HTTPException( + status_code=HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + detail="Audio file exceeds maximum allowed size", + ) + + if len(audio_bytes) == 0: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST, + detail="Audio file is empty", + ) + + logger.info( + "Transcribing audio {user_id=%s, size_bytes=%s, content_type='%s', language='%s'}", + user_info.user_id, + len(audio_bytes), + audio.content_type, + language, + ) + + try: + return await service.transcribe(audio_bytes=audio_bytes, language_code=language) + except EmptyTranscriptionError as exc: + raise HTTPException( + status_code=HTTPStatus.UNPROCESSABLE_ENTITY, + detail="No transcription results returned for the provided audio", + ) from exc + except SpeechToTextServiceError as e: + logger.error("Speech-to-text service error", exc_info=True) + raise HTTPException( + status_code=HTTPStatus.BAD_GATEWAY, + detail="Upstream transcription service unavailable", + ) from e + except Exception as e: # pylint: disable=broad-except + logger.exception("Unexpected error during transcription") + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail="An unexpected error occurred during transcription", + ) from e + + app.include_router(router) diff --git a/backend/app/speech_to_text/service.py b/backend/app/speech_to_text/service.py new file mode 100644 index 00000000..fa60b59f --- /dev/null +++ b/backend/app/speech_to_text/service.py @@ -0,0 +1,86 @@ +import asyncio +import logging +import os +from abc import ABC, abstractmethod + +from google.cloud.speech_v2 import SpeechClient +from google.cloud.speech_v2.types import cloud_speech + +from app.speech_to_text.errors import EmptyTranscriptionError, SpeechToTextServiceError +from app.speech_to_text.types import TranscriptionResponse +from common_libs.retry import Retry + + +class ISpeechToTextService(ABC): + """Interface for speech-to-text transcription services.""" + + @abstractmethod + async def transcribe(self, *, audio_bytes: bytes, language_code: str) -> TranscriptionResponse: + """ + Transcribe audio bytes to text. + + :param audio_bytes: The raw audio bytes to transcribe. + :param language_code: The BCP-47 language code for transcription (e.g. "en-US"). + :return: The transcription response containing the text and language code. + :raises EmptyTranscriptionError: If no transcription results are returned. + :raises SpeechToTextServiceError: If the Google Cloud STT API call fails. + """ + raise NotImplementedError() + + +class GoogleSpeechToTextService(ISpeechToTextService): + """Google Cloud Speech-to-Text v2 implementation.""" + + def __init__(self): + self._logger = logging.getLogger(self.__class__.__name__) + + self._project_id = os.getenv("GOOGLE_CLOUD_PROJECT") + if not self._project_id: + raise ValueError("GOOGLE_CLOUD_PROJECT environment variable is not set") + + self._region = os.getenv("VERTEX_API_REGION", "us-central1") + + # Create the client with regional endpoint + self._client = SpeechClient( + client_options={"api_endpoint": f"{self._region}-speech.googleapis.com"} + ) + + async def transcribe(self, *, audio_bytes: bytes, language_code: str) -> TranscriptionResponse: + recognizer = f"projects/{self._project_id}/locations/{self._region}/recognizers/_" + + config = cloud_speech.RecognitionConfig( + auto_decoding_config=cloud_speech.AutoDetectDecodingConfig(), + language_codes=[language_code], + model="long", + ) + + request = cloud_speech.RecognizeRequest( + recognizer=recognizer, + config=config, + content=audio_bytes, + ) + + try: + response = await Retry[cloud_speech.RecognizeResponse].call_with_exponential_backoff( + callback=lambda: asyncio.to_thread(self._client.recognize, request=request), + logger=self._logger, + ) + except Exception as e: + self._logger.error("Speech-to-text API call failed", exc_info=True) + raise SpeechToTextServiceError(detail=str(e)) from e + + # Concatenate all transcript alternatives + transcript_parts = [] + for result in response.results: + if result.alternatives: + transcript_parts.append(result.alternatives[0].transcript) + + text = " ".join(transcript_parts).strip() + + if not text: + raise EmptyTranscriptionError() + + return TranscriptionResponse( + text=text, + language_code=language_code, + ) diff --git a/backend/app/speech_to_text/test_routes.py b/backend/app/speech_to_text/test_routes.py new file mode 100644 index 00000000..4a5ac5ff --- /dev/null +++ b/backend/app/speech_to_text/test_routes.py @@ -0,0 +1,191 @@ +import io +from http import HTTPStatus +from unittest.mock import AsyncMock + +from fastapi import FastAPI +from starlette.testclient import TestClient + +from app.speech_to_text.errors import EmptyTranscriptionError, SpeechToTextServiceError +from app.speech_to_text.routes import add_speech_to_text_routes, get_speech_to_text_service +from app.speech_to_text.service import ISpeechToTextService +from app.speech_to_text.types import TranscriptionResponse +from common_libs.test_utilities.mock_auth import MockAuth, UnauthenticatedMockAuth + + +class _MockSpeechToTextService(ISpeechToTextService): + """Mock implementation for testing.""" + + def __init__(self): + self.transcribe_mock = AsyncMock() + + async def transcribe(self, *, audio_bytes: bytes, language_code: str) -> TranscriptionResponse: + return await self.transcribe_mock(audio_bytes=audio_bytes, language_code=language_code) + + +def _given_audio_file(content: bytes = b"fake-audio-data", content_type: str = "audio/webm"): + """Create a mock audio file for upload.""" + return ("audio", ("recording.webm", io.BytesIO(content), content_type)) + + +def _create_test_client(auth=None): + """Set up a test client with mocked dependencies.""" + if auth is None: + auth = MockAuth() + mock_service = _MockSpeechToTextService() + app = FastAPI() + app.dependency_overrides[get_speech_to_text_service] = lambda: mock_service + add_speech_to_text_routes(app, authentication=auth) + client = TestClient(app) + return client, mock_service, auth + + +class TestTranscribeAudioEndpoint: + """Tests for POST /speech-to-text/transcribe""" + + def test_respond_with_status_ok_and_transcription_on_success(self): + """Test successful transcription with default language.""" + # GIVEN a valid audio file + given_audio = _given_audio_file() + # AND a service that returns a successful transcription + client, mock_service, _auth = _create_test_client() + given_response = TranscriptionResponse(text="hello world", language_code="en-US") + mock_service.transcribe_mock.return_value = given_response + + # WHEN the transcribe endpoint is called + actual_response = client.post("/speech-to-text/transcribe", files=[given_audio], data={"language": "en-US"}) + + # THEN expect the response status to be OK + assert actual_response.status_code == HTTPStatus.OK + # AND the response body to contain the transcribed text + actual_body = actual_response.json() + assert actual_body["text"] == "hello world" + assert actual_body["language_code"] == "en-US" + + def test_respond_with_status_ok_when_explicit_language_provided(self): + """Test successful transcription with an explicit language.""" + # GIVEN a valid audio file + given_audio = _given_audio_file() + # AND an explicit language + given_language = "sw-KE" + # AND a service that returns a successful transcription + client, mock_service, _auth = _create_test_client() + given_response = TranscriptionResponse(text="habari", language_code=given_language) + mock_service.transcribe_mock.return_value = given_response + + # WHEN the transcribe endpoint is called with the given language + actual_response = client.post( + "/speech-to-text/transcribe", + files=[given_audio], + data={"language": given_language}, + ) + + # THEN expect the response status to be OK + assert actual_response.status_code == HTTPStatus.OK + # AND the language code to match the given language + assert actual_response.json()["language_code"] == given_language + + def test_respond_with_status_unsupported_media_type_for_invalid_mime(self): + """Test that unsupported audio formats are rejected.""" + # GIVEN an audio file with an unsupported MIME type + given_audio = _given_audio_file(content_type="application/pdf") + client, _mock_service, _auth = _create_test_client() + + # WHEN the transcribe endpoint is called + actual_response = client.post("/speech-to-text/transcribe", files=[given_audio], data={"language": "en-US"}) + + # THEN expect the response status to be UNSUPPORTED_MEDIA_TYPE + assert actual_response.status_code == HTTPStatus.UNSUPPORTED_MEDIA_TYPE + + def test_respond_with_status_ok_when_mime_type_has_space_after_semicolon(self): + """Test that MIME types with spaces after semicolons are accepted.""" + # GIVEN an audio file with a space after the semicolon in the MIME type + given_audio = _given_audio_file(content_type="audio/webm; codecs=opus") + # AND a service that returns a successful transcription + client, mock_service, _auth = _create_test_client() + given_response = TranscriptionResponse(text="hello", language_code="en-US") + mock_service.transcribe_mock.return_value = given_response + + # WHEN the transcribe endpoint is called + actual_response = client.post("/speech-to-text/transcribe", files=[given_audio], data={"language": "en-US"}) + + # THEN expect the response status to be OK + assert actual_response.status_code == HTTPStatus.OK + + def test_respond_with_status_entity_too_large_for_oversized_audio(self): + """Test that oversized audio files are rejected.""" + # GIVEN an audio file that exceeds the maximum size + given_large_content = b"x" * (10 * 1024 * 1024 + 1) + given_audio = _given_audio_file(content=given_large_content) + client, _mock_service, _auth = _create_test_client() + + # WHEN the transcribe endpoint is called + actual_response = client.post("/speech-to-text/transcribe", files=[given_audio], data={"language": "en-US"}) + + # THEN expect the response status to be REQUEST_ENTITY_TOO_LARGE + assert actual_response.status_code == HTTPStatus.REQUEST_ENTITY_TOO_LARGE + + def test_respond_with_status_unprocessable_entity_when_transcription_is_empty(self): + """Test that empty transcription results return 422.""" + # GIVEN a valid audio file + given_audio = _given_audio_file() + # AND a service that raises EmptyTranscriptionError + client, mock_service, _auth = _create_test_client() + mock_service.transcribe_mock.side_effect = EmptyTranscriptionError() + + # WHEN the transcribe endpoint is called + actual_response = client.post("/speech-to-text/transcribe", files=[given_audio], data={"language": "en-US"}) + + # THEN expect the response status to be UNPROCESSABLE_ENTITY + assert actual_response.status_code == HTTPStatus.UNPROCESSABLE_ENTITY + + def test_respond_with_status_bad_gateway_when_service_fails(self): + """Test that Google STT API failures return 502.""" + # GIVEN a valid audio file + given_audio = _given_audio_file() + # AND a service that raises SpeechToTextServiceError + client, mock_service, _auth = _create_test_client() + mock_service.transcribe_mock.side_effect = SpeechToTextServiceError(detail="API unavailable") + + # WHEN the transcribe endpoint is called + actual_response = client.post("/speech-to-text/transcribe", files=[given_audio], data={"language": "en-US"}) + + # THEN expect the response status to be BAD_GATEWAY + assert actual_response.status_code == HTTPStatus.BAD_GATEWAY + + def test_respond_with_status_internal_server_error_on_unexpected_error(self): + """Test that unexpected errors return 500.""" + # GIVEN a valid audio file + given_audio = _given_audio_file() + # AND a service that raises an unexpected error + client, mock_service, _auth = _create_test_client() + mock_service.transcribe_mock.side_effect = RuntimeError("something broke") + + # WHEN the transcribe endpoint is called + actual_response = client.post("/speech-to-text/transcribe", files=[given_audio], data={"language": "en-US"}) + + # THEN expect the response status to be INTERNAL_SERVER_ERROR + assert actual_response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR + + def test_respond_with_status_unauthorized_when_not_authenticated(self): + """Test that unauthenticated requests are rejected.""" + # GIVEN an unauthenticated user + given_audio = _given_audio_file() + client, _mock_service, _auth = _create_test_client(auth=UnauthenticatedMockAuth()) + + # WHEN the transcribe endpoint is called + actual_response = client.post("/speech-to-text/transcribe", files=[given_audio], data={"language": "en-US"}) + + # THEN expect the response status to be UNAUTHORIZED + assert actual_response.status_code == HTTPStatus.UNAUTHORIZED + + def test_respond_with_status_bad_request_for_empty_audio(self): + """Test that empty audio files are rejected.""" + # GIVEN an empty audio file + given_audio = _given_audio_file(content=b"") + client, _mock_service, _auth = _create_test_client() + + # WHEN the transcribe endpoint is called + actual_response = client.post("/speech-to-text/transcribe", files=[given_audio], data={"language": "en-US"}) + + # THEN expect the response status to be BAD_REQUEST + assert actual_response.status_code == HTTPStatus.BAD_REQUEST diff --git a/backend/app/speech_to_text/test_service.py b/backend/app/speech_to_text/test_service.py new file mode 100644 index 00000000..aa8a6e4d --- /dev/null +++ b/backend/app/speech_to_text/test_service.py @@ -0,0 +1,109 @@ +from unittest.mock import MagicMock, patch + +import pytest + +from app.speech_to_text.errors import EmptyTranscriptionError, SpeechToTextServiceError +from app.speech_to_text.service import GoogleSpeechToTextService + + +def _mock_recognize_response(transcripts: list[str]): + """Create a mock RecognizeResponse with the given transcript texts.""" + response = MagicMock() + results = [] + for text in transcripts: + alternative = MagicMock() + alternative.transcript = text + result = MagicMock() + result.alternatives = [alternative] + results.append(result) + response.results = results + return response + + +def _mock_empty_recognize_response(): + """Create a mock RecognizeResponse with no results.""" + response = MagicMock() + response.results = [] + return response + + +@pytest.fixture() +def _set_env_vars(monkeypatch): + """Set required environment variables for GoogleSpeechToTextService.""" + monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "test-project") + monkeypatch.setenv("VERTEX_API_REGION", "us-central1") + + +@pytest.mark.usefixtures("_set_env_vars") +class TestGoogleSpeechToTextService: + """Tests for GoogleSpeechToTextService""" + + @pytest.mark.asyncio + @patch("app.speech_to_text.service.SpeechClient") + async def test_transcribe_returns_concatenated_text_on_success(self, mock_speech_client_class): + """Test that transcription results are concatenated correctly.""" + # GIVEN a service with a mocked SpeechClient + given_mock_client = MagicMock() + mock_speech_client_class.return_value = given_mock_client + # AND the client returns a response with multiple transcript parts + given_response = _mock_recognize_response(["hello", "world"]) + given_mock_client.recognize.return_value = given_response + # AND a valid audio input + given_audio_bytes = b"fake-audio" + given_language = "en-US" + + service = GoogleSpeechToTextService() + + # WHEN transcribe is called + actual_result = await service.transcribe(audio_bytes=given_audio_bytes, language_code=given_language) + + # THEN expect the text to be the concatenated transcripts + assert actual_result.text == "hello world" + # AND the language code to match the given language + assert actual_result.language_code == given_language + + @pytest.mark.asyncio + @patch("app.speech_to_text.service.SpeechClient") + async def test_transcribe_raises_empty_transcription_error_when_no_results(self, mock_speech_client_class): + """Test that EmptyTranscriptionError is raised when STT returns no results.""" + # GIVEN a service with a mocked SpeechClient + given_mock_client = MagicMock() + mock_speech_client_class.return_value = given_mock_client + # AND the client returns an empty response + given_mock_client.recognize.return_value = _mock_empty_recognize_response() + given_audio_bytes = b"fake-audio" + + service = GoogleSpeechToTextService() + + # WHEN transcribe is called + # THEN expect EmptyTranscriptionError to be raised + with pytest.raises(EmptyTranscriptionError): + await service.transcribe(audio_bytes=given_audio_bytes, language_code="en-US") + + @pytest.mark.asyncio + @patch("app.speech_to_text.service.SpeechClient") + async def test_transcribe_raises_service_error_when_api_fails(self, mock_speech_client_class): + """Test that SpeechToTextServiceError is raised when the API call fails.""" + # GIVEN a service with a mocked SpeechClient + given_mock_client = MagicMock() + mock_speech_client_class.return_value = given_mock_client + # AND the client raises an exception + given_mock_client.recognize.side_effect = Exception("API unavailable") + given_audio_bytes = b"fake-audio" + + service = GoogleSpeechToTextService() + + # WHEN transcribe is called + # THEN expect SpeechToTextServiceError to be raised + with pytest.raises(SpeechToTextServiceError): + await service.transcribe(audio_bytes=given_audio_bytes, language_code="en-US") + + def test_init_raises_when_google_cloud_project_not_set(self, monkeypatch): + """Test that initialization fails without GOOGLE_CLOUD_PROJECT.""" + # GIVEN GOOGLE_CLOUD_PROJECT is not set + monkeypatch.delenv("GOOGLE_CLOUD_PROJECT", raising=False) + + # WHEN the service is initialized + # THEN expect a ValueError + with pytest.raises(ValueError, match="GOOGLE_CLOUD_PROJECT"): + GoogleSpeechToTextService() diff --git a/backend/app/speech_to_text/types.py b/backend/app/speech_to_text/types.py new file mode 100644 index 00000000..7a86d30e --- /dev/null +++ b/backend/app/speech_to_text/types.py @@ -0,0 +1,12 @@ +from pydantic import BaseModel, Field + + +class TranscriptionResponse(BaseModel): + """Response returned after successful speech-to-text transcription.""" + + class Config: + """Pydantic model configuration.""" + extra = "forbid" + + text: str = Field(description="The transcribed text from the audio input") + language_code: str = Field(description="The language code used for transcription") diff --git a/backend/conftest.py b/backend/conftest.py index 32753182..008cfd0e 100644 --- a/backend/conftest.py +++ b/backend/conftest.py @@ -29,6 +29,7 @@ cv_storage_bucket="foo-bucket", features={}, enable_cv_upload=True, + enable_speech_to_text=False, language_config=LanguageConfig( default_locale=Locale.EN_US, available_locales=[LocaleDateFormatEntry(locale=Locale.EN_US, date_format="MM/DD/YYYY")] diff --git a/backend/poetry.lock b/backend/poetry.lock index 022982ca..7914a10b 100644 --- a/backend/poetry.lock +++ b/backend/poetry.lock @@ -1505,6 +1505,31 @@ grpc-google-iam-v1 = ">=0.12.4,<1.0.0dev" proto-plus = ">=1.22.3,<2.0.0dev" protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" +[[package]] +name = "google-cloud-speech" +version = "2.38.0" +description = "Google Cloud Speech API client library" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "google_cloud_speech-2.38.0-py3-none-any.whl", hash = "sha256:dbccb340a750a409b0e70c48c16c8d7d5d48a87c70cce2add50f3d571f5375a0"}, + {file = "google_cloud_speech-2.38.0.tar.gz", hash = "sha256:1854b51cbb7957273b6ba61f4a6cf49dec8d09ec450991587897e50267eaca51"}, +] + +[package.dependencies] +google-api-core = {version = ">=2.11.0,<3.0.0", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0" +grpcio = [ + {version = ">=1.75.1,<2.0.0", markers = "python_version >= \"3.14\""}, + {version = ">=1.33.2,<2.0.0", markers = "python_version < \"3.14\""}, +] +proto-plus = [ + {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, + {version = ">=1.22.3,<2.0.0", markers = "python_version < \"3.13\""}, +] +protobuf = ">=4.25.8,<8.0.0" + [[package]] name = "google-cloud-storage" version = "2.19.0" @@ -1616,7 +1641,7 @@ version = "1.26.0" description = "GenAI Python SDK" optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "google_genai-1.26.0-py3-none-any.whl", hash = "sha256:a050de052ee6e68654ba7cdb97028a576ad7108d0ecc9257c69bcc555498e9a2"}, {file = "google_genai-1.26.0.tar.gz", hash = "sha256:d7b019ac98ca07888caa6121a953eb65db20f78370d8ae06aec29fb534534dc8"}, @@ -1745,11 +1770,89 @@ files = [ {file = "grpcio-1.64.1-cp39-cp39-win_amd64.whl", hash = "sha256:ed6091fa0adcc7e4ff944090cf203a52da35c37a130efa564ded02b7aff63bcd"}, {file = "grpcio-1.64.1.tar.gz", hash = "sha256:8d51dd1c59d5fa0f34266b80a3805ec29a1f26425c2a54736133f6d87fc4968a"}, ] -markers = {dev = "platform_python_implementation != \"PyPy\""} +markers = {main = "python_version <= \"3.13\"", dev = "platform_python_implementation != \"PyPy\" and python_version <= \"3.13\""} [package.extras] protobuf = ["grpcio-tools (>=1.64.1)"] +[[package]] +name = "grpcio" +version = "1.80.0" +description = "HTTP/2-based RPC framework" +optional = false +python-versions = ">=3.9" +groups = ["main", "dev"] +files = [ + {file = "grpcio-1.80.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:886457a7768e408cdce226ad1ca67d2958917d306523a0e21e1a2fdaa75c9c9c"}, + {file = "grpcio-1.80.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:7b641fc3f1dc647bfd80bd713addc68f6d145956f64677e56d9ebafc0bd72388"}, + {file = "grpcio-1.80.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:33eb763f18f006dc7fee1e69831d38d23f5eccd15b2e0f92a13ee1d9242e5e02"}, + {file = "grpcio-1.80.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:52d143637e3872633fc7dd7c3c6a1c84e396b359f3a72e215f8bf69fd82084fc"}, + {file = "grpcio-1.80.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c51bf8ac4575af2e0678bccfb07e47321fc7acb5049b4482832c5c195e04e13a"}, + {file = "grpcio-1.80.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:50a9871536d71c4fba24ee856abc03a87764570f0c457dd8db0b4018f379fed9"}, + {file = "grpcio-1.80.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:a72d84ad0514db063e21887fbacd1fd7acb4d494a564cae22227cd45c7fbf199"}, + {file = "grpcio-1.80.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f7691a6788ad9196872f95716df5bc643ebba13c97140b7a5ee5c8e75d1dea81"}, + {file = "grpcio-1.80.0-cp310-cp310-win32.whl", hash = "sha256:46c2390b59d67f84e882694d489f5b45707c657832d7934859ceb8c33f467069"}, + {file = "grpcio-1.80.0-cp310-cp310-win_amd64.whl", hash = "sha256:dc053420fc75749c961e2a4c906398d7c15725d36ccc04ae6d16093167223b58"}, + {file = "grpcio-1.80.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:dfab85db094068ff42e2a3563f60ab3dddcc9d6488a35abf0132daec13209c8a"}, + {file = "grpcio-1.80.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:5c07e82e822e1161354e32da2662f741a4944ea955f9f580ec8fb409dd6f6060"}, + {file = "grpcio-1.80.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba0915d51fd4ced2db5ff719f84e270afe0e2d4c45a7bdb1e8d036e4502928c2"}, + {file = "grpcio-1.80.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:3cb8130ba457d2aa09fa6b7c3ed6b6e4e6a2685fce63cb803d479576c4d80e21"}, + {file = "grpcio-1.80.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:09e5e478b3d14afd23f12e49e8b44c8684ac3c5f08561c43a5b9691c54d136ab"}, + {file = "grpcio-1.80.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:00168469238b022500e486c1c33916acf2f2a9b2c022202cf8a1885d2e3073c1"}, + {file = "grpcio-1.80.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8502122a3cc1714038e39a0b071acb1207ca7844208d5ea0d091317555ee7106"}, + {file = "grpcio-1.80.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ce1794f4ea6cc3ca29463f42d665c32ba1b964b48958a66497917fe9069f26e6"}, + {file = "grpcio-1.80.0-cp311-cp311-win32.whl", hash = "sha256:51b4a7189b0bef2aa30adce3c78f09c83526cf3dddb24c6a96555e3b97340440"}, + {file = "grpcio-1.80.0-cp311-cp311-win_amd64.whl", hash = "sha256:02e64bb0bb2da14d947a49e6f120a75e947250aebe65f9629b62bb1f5c14e6e9"}, + {file = "grpcio-1.80.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:c624cc9f1008361014378c9d776de7182b11fe8b2e5a81bc69f23a295f2a1ad0"}, + {file = "grpcio-1.80.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:f49eddcac43c3bf350c0385366a58f36bed8cc2c0ec35ef7b74b49e56552c0c2"}, + {file = "grpcio-1.80.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d334591df610ab94714048e0d5b4f3dd5ad1bee74dfec11eee344220077a79de"}, + {file = "grpcio-1.80.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:0cb517eb1d0d0aaf1d87af7cc5b801d686557c1d88b2619f5e31fab3c2315921"}, + {file = "grpcio-1.80.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4e78c4ac0d97dc2e569b2f4bcbbb447491167cb358d1a389fc4af71ab6f70411"}, + {file = "grpcio-1.80.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2ed770b4c06984f3b47eb0517b1c69ad0b84ef3f40128f51448433be904634cd"}, + {file = "grpcio-1.80.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:256507e2f524092f1473071a05e65a5b10d84b82e3ff24c5b571513cfaa61e2f"}, + {file = "grpcio-1.80.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9a6284a5d907c37db53350645567c522be314bac859a64a7a5ca63b77bb7958f"}, + {file = "grpcio-1.80.0-cp312-cp312-win32.whl", hash = "sha256:c71309cfce2f22be26aa4a847357c502db6c621f1a49825ae98aa0907595b193"}, + {file = "grpcio-1.80.0-cp312-cp312-win_amd64.whl", hash = "sha256:9fe648599c0e37594c4809d81a9e77bd138cc82eb8baa71b6a86af65426723ff"}, + {file = "grpcio-1.80.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:e9e408fc016dffd20661f0126c53d8a31c2821b5c13c5d67a0f5ed5de93319ad"}, + {file = "grpcio-1.80.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:92d787312e613754d4d8b9ca6d3297e69994a7912a32fa38c4c4e01c272974b0"}, + {file = "grpcio-1.80.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8ac393b58aa16991a2f1144ec578084d544038c12242da3a215966b512904d0f"}, + {file = "grpcio-1.80.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:68e5851ac4b9afe07e7f84483803ad167852570d65326b34d54ca560bfa53fb6"}, + {file = "grpcio-1.80.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:873ff5d17d68992ef6605330127425d2fc4e77e612fa3c3e0ed4e668685e3140"}, + {file = "grpcio-1.80.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2bea16af2750fd0a899bf1abd9022244418b55d1f37da2202249ba4ba673838d"}, + {file = "grpcio-1.80.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba0db34f7e1d803a878284cd70e4c63cb6ae2510ba51937bf8f45ba997cefcf7"}, + {file = "grpcio-1.80.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8eb613f02d34721f1acf3626dfdb3545bd3c8505b0e52bf8b5710a28d02e8aa7"}, + {file = "grpcio-1.80.0-cp313-cp313-win32.whl", hash = "sha256:93b6f823810720912fd131f561f91f5fed0fda372b6b7028a2681b8194d5d294"}, + {file = "grpcio-1.80.0-cp313-cp313-win_amd64.whl", hash = "sha256:e172cf795a3ba5246d3529e4d34c53db70e888fa582a8ffebd2e6e48bc0cba50"}, + {file = "grpcio-1.80.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:3d4147a97c8344d065d01bbf8b6acec2cf86fb0400d40696c8bdad34a64ffc0e"}, + {file = "grpcio-1.80.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d8e11f167935b3eb089ac9038e1a063e6d7dbe995c0bb4a661e614583352e76f"}, + {file = "grpcio-1.80.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f14b618fc30de822681ee986cfdcc2d9327229dc4c98aed16896761cacd468b9"}, + {file = "grpcio-1.80.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4ed39fbdcf9b87370f6e8df4e39ca7b38b3e5e9d1b0013c7b6be9639d6578d14"}, + {file = "grpcio-1.80.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2dcc70e9f0ba987526e8e8603a610fb4f460e42899e74e7a518bf3c68fe1bf05"}, + {file = "grpcio-1.80.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:448c884b668b868562b1bda833c5fce6272d26e1926ec46747cda05741d302c1"}, + {file = "grpcio-1.80.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a1dc80fe55685b4a543555e6eef975303b36c8db1023b1599b094b92aa77965f"}, + {file = "grpcio-1.80.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:31b9ac4ad1aa28ffee5503821fafd09e4da0a261ce1c1281c6c8da0423c83b6e"}, + {file = "grpcio-1.80.0-cp314-cp314-win32.whl", hash = "sha256:367ce30ba67d05e0592470428f0ec1c31714cab9ef19b8f2e37be1f4c7d32fae"}, + {file = "grpcio-1.80.0-cp314-cp314-win_amd64.whl", hash = "sha256:3b01e1f5464c583d2f567b2e46ff0d516ef979978f72091fd81f5ab7fa6e2e7f"}, + {file = "grpcio-1.80.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:aacdfb4ed3eb919ca997504d27e03d5dba403c85130b8ed450308590a738f7a4"}, + {file = "grpcio-1.80.0-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:a361c20ec1ccd3c3953d20fb6d7b4125093bdd10dff44c5e2bbb39e58917cedc"}, + {file = "grpcio-1.80.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:43168871f170d1e4ed16ae03d10cd21efa29f190e710a624cee7e5ae07da6f4f"}, + {file = "grpcio-1.80.0-cp39-cp39-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1b97cd29a8eda100b559b455331c487a80915b6ea6bd91cf3e89836c4ee8d957"}, + {file = "grpcio-1.80.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bac1d573dfa84ce59a5547073e28fa7326d53352adda6912e362da0b917fcef4"}, + {file = "grpcio-1.80.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4560cf0e86514595dbbd330cd65b7afad4b5c4b8c4905c041cfffa138d45e6fd"}, + {file = "grpcio-1.80.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:ec0a592e926071b4abad50c1495cd0d0d513324b3ff5e7267067c33ba27506e4"}, + {file = "grpcio-1.80.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:deb10a1528473c11f72a0939eed36d83e847d7cbb63e8cc5611fb7a912d38614"}, + {file = "grpcio-1.80.0-cp39-cp39-win32.whl", hash = "sha256:627fb7312171cdc52828bd6fac8d7028ff2a64b89f1957b6f3416caa2218d141"}, + {file = "grpcio-1.80.0-cp39-cp39-win_amd64.whl", hash = "sha256:05d55e1798756282cddd52d56c896b3e7d673e3a8798c2f1cd05ba249a3bb4de"}, + {file = "grpcio-1.80.0.tar.gz", hash = "sha256:29aca15edd0688c22ba01d7cc01cb000d72b2033f4a3c72a81a19b56fd143257"}, +] +markers = {main = "python_version >= \"3.14\"", dev = "platform_python_implementation != \"PyPy\" and python_version >= \"3.14\""} + +[package.dependencies] +typing-extensions = ">=4.12,<5.0" + +[package.extras] +protobuf = ["grpcio-tools (>=1.80.0)"] + [[package]] name = "grpcio-status" version = "1.62.2" @@ -2250,8 +2353,8 @@ files = [ [package.dependencies] click = ">=8.1.7" numpy = [ - {version = ">=1.26", markers = "python_version == \"3.12\""}, {version = ">=2.1.0", markers = "python_version >= \"3.13\""}, + {version = ">=1.26", markers = "python_version == \"3.12\""}, {version = ">=1.24", markers = "python_version < \"3.12\""}, ] onnxruntime = {version = ">=1.17.0", markers = "python_version > \"3.9\""} @@ -3187,6 +3290,7 @@ description = "Beautiful, Pythonic protocol buffers." optional = false python-versions = ">=3.6" groups = ["main", "dev"] +markers = "python_version < \"3.13\"" files = [ {file = "proto-plus-1.23.0.tar.gz", hash = "sha256:89075171ef11988b3fa157f5dbd8b9cf09d65fffee97e29ce403cd8defba19d2"}, {file = "proto_plus-1.23.0-py3-none-any.whl", hash = "sha256:a829c79e619e1cf632de091013a4173deed13a55f326ef84f05af6f50ff4c82c"}, @@ -3198,25 +3302,44 @@ protobuf = ">=3.19.0,<5.0.0dev" [package.extras] testing = ["google-api-core[grpc] (>=1.31.5)"] +[[package]] +name = "proto-plus" +version = "1.27.2" +description = "Beautiful, Pythonic protocol buffers" +optional = false +python-versions = ">=3.9" +groups = ["main", "dev"] +markers = "python_version >= \"3.13\"" +files = [ + {file = "proto_plus-1.27.2-py3-none-any.whl", hash = "sha256:6432f75893d3b9e70b9c412f1d2f03f65b11fb164b793d14ae2ca01821d22718"}, + {file = "proto_plus-1.27.2.tar.gz", hash = "sha256:b2adde53adadf75737c44d3dcb0104fde65250dfc83ad59168b4aa3e574b6a24"}, +] + +[package.dependencies] +protobuf = ">=4.25.8,<8.0.0" + +[package.extras] +testing = ["google-api-core (>=1.31.5)"] + [[package]] name = "protobuf" -version = "4.25.3" +version = "4.25.9" description = "" optional = false python-versions = ">=3.8" groups = ["main", "dev"] files = [ - {file = "protobuf-4.25.3-cp310-abi3-win32.whl", hash = "sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa"}, - {file = "protobuf-4.25.3-cp310-abi3-win_amd64.whl", hash = "sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8"}, - {file = "protobuf-4.25.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c"}, - {file = "protobuf-4.25.3-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019"}, - {file = "protobuf-4.25.3-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d"}, - {file = "protobuf-4.25.3-cp38-cp38-win32.whl", hash = "sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2"}, - {file = "protobuf-4.25.3-cp38-cp38-win_amd64.whl", hash = "sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4"}, - {file = "protobuf-4.25.3-cp39-cp39-win32.whl", hash = "sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4"}, - {file = "protobuf-4.25.3-cp39-cp39-win_amd64.whl", hash = "sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c"}, - {file = "protobuf-4.25.3-py3-none-any.whl", hash = "sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9"}, - {file = "protobuf-4.25.3.tar.gz", hash = "sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c"}, + {file = "protobuf-4.25.9-cp310-abi3-win32.whl", hash = "sha256:bde396f568b0b46fc8fbfe9f02facf25b6755b2578a3b8ac61e74b9d69499e03"}, + {file = "protobuf-4.25.9-cp310-abi3-win_amd64.whl", hash = "sha256:3683c05154252206f7cb2d371626514b3708199d9bcf683b503dabf3a2e38e06"}, + {file = "protobuf-4.25.9-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:9560813560e6ee72c11ca8873878bdb7ee003c96a57ebb013245fe84e2540904"}, + {file = "protobuf-4.25.9-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:999146ef02e7fa6a692477badd1528bcd7268df211852a3df2d834ba2b480791"}, + {file = "protobuf-4.25.9-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:438c636de8fb706a0de94a12a268ef1ae8f5ba5ae655a7671fcda5968ba3c9be"}, + {file = "protobuf-4.25.9-cp38-cp38-win32.whl", hash = "sha256:7f7c1abcea3fc215918fba67a2d2a80fbcccc0f84159610eb187e9bbe6f939ee"}, + {file = "protobuf-4.25.9-cp38-cp38-win_amd64.whl", hash = "sha256:79faf4e5a80b231d94dcf3a0a2917ccbacf0f586f12c9b9c91794b41b913a853"}, + {file = "protobuf-4.25.9-cp39-cp39-win32.whl", hash = "sha256:9481e80e8cffb1c492c68e7c4e6726f4ad02eebc4fa97ead7beebeaa3639511d"}, + {file = "protobuf-4.25.9-cp39-cp39-win_amd64.whl", hash = "sha256:b1d467352de666dc1b6d5740b6319d9c08cab7b21b452501e4ee5b0ac5156780"}, + {file = "protobuf-4.25.9-py3-none-any.whl", hash = "sha256:d49b615e7c935194ac161f0965699ac84df6112c378e05ec53da65d2e4cbb6d4"}, + {file = "protobuf-4.25.9.tar.gz", hash = "sha256:b0dc7e7c68de8b1ce831dacb12fb407e838edbb8b6cc0dc3a2a6b4cbf6de9cff"}, ] [[package]] @@ -4301,7 +4424,7 @@ version = "8.5.0" description = "Retry code until it succeeds" optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "tenacity-8.5.0-py3-none-any.whl", hash = "sha256:b594c2a5945830c267ce6b79a166228323ed52718f30302c1359836112346687"}, {file = "tenacity-8.5.0.tar.gz", hash = "sha256:8bc6c0c8a09b31e6cad13c47afbed1a567518250a9a171418582ed8d9c20ca78"}, @@ -5048,4 +5171,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "0b4409c9c9f0485fc90a0ecfc35812a95d7023c605da05b5b899668d9e5406e0" +content-hash = "2ab9f2d443d7395c8e3fa55d33af80c8953c8dc625d9adef0c0b0e1f0db2d05f" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 566b307f..bfe469e5 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -14,6 +14,7 @@ uvicorn = ">=0.23.2,<0.24.0" python-dotenv = "^1.0.1" deprecated="^1.2.14" google-cloud-dlp="^3.16.0" +google-cloud-speech="^2.27.0" fix-busted-json=">=0.0.17,<1.0.0" json-repair=">=0.44.1,<1.0.0" motor = "^3.4.0" diff --git a/config/default.json b/config/default.json index 7ea7ecc7..2b272c99 100644 --- a/config/default.json +++ b/config/default.json @@ -104,6 +104,9 @@ "gtmContainerNumericId": "246009778", "enabled": true }, + "speechToText": { + "enabled": false + }, "sensitiveData": { "fields": { "name": { diff --git a/config/inject-config.py b/config/inject-config.py index 25823d44..31b86206 100644 --- a/config/inject-config.py +++ b/config/inject-config.py @@ -22,6 +22,9 @@ # CV Upload "GLOBAL_ENABLE_CV_UPLOAD": "cv.enabled", + # Speech-to-Text + "GLOBAL_ENABLE_SPEECH_TO_TEXT": "speechToText.enabled", + # Add more backend env vars here } @@ -47,6 +50,9 @@ # CV Upload "GLOBAL_ENABLE_CV_UPLOAD": "cv.enabled", + # Speech-to-Text + "GLOBAL_ENABLE_SPEECH_TO_TEXT": "speechToText.enabled", + # i18n "FRONTEND_DEFAULT_LOCALE": "i18n.ui.defaultLocale", "FRONTEND_SUPPORTED_LOCALES": "i18n.ui.supportedLocales", From 8e3f29531d336327712a0b43f3bbbb00e7ee0774 Mon Sep 17 00:00:00 2001 From: nraffa Date: Wed, 8 Apr 2026 09:26:24 +1000 Subject: [PATCH 2/5] feat(frontend): add voice input UI with real-time transcription Add mic button to chat input that uses Web Speech API for real-time interim text display while recording audio via MediaRecorder. On stop, audio is sent to the backend STT endpoint for final transcription. Existing text in the input is preserved and new transcription is appended. Feature is behind the GLOBAL_ENABLE_SPEECH_TO_TEXT toggle. --- config/default.json | 2 +- .../src/_test_utilities/envServiceMock.ts | 1 + .../ChatMessageField/ChatMessageField.tsx | 101 ++++++- .../ChatMessageField.test.tsx.snap | 2 +- frontend-new/src/envService.ts | 5 + .../src/i18n/locales/en-GB/translation.json | 12 +- .../src/i18n/locales/en-US/translation.json | 10 +- .../src/i18n/locales/es-AR/translation.json | 12 +- .../src/i18n/locales/es-ES/translation.json | 12 +- .../src/i18n/locales/ny-ZM/translation.json | 10 +- .../src/i18n/locales/sw-KE/translation.json | 10 +- .../src/speechToText/SpeechToTextService.ts | 58 ++++ .../speechToText/SpeechToTextService.types.ts | 4 + .../src/speechToText/speechRecognition.d.ts | 45 +++ .../src/speechToText/useSpeechToText.ts | 286 ++++++++++++++++++ 15 files changed, 552 insertions(+), 18 deletions(-) create mode 100644 frontend-new/src/speechToText/SpeechToTextService.ts create mode 100644 frontend-new/src/speechToText/SpeechToTextService.types.ts create mode 100644 frontend-new/src/speechToText/speechRecognition.d.ts create mode 100644 frontend-new/src/speechToText/useSpeechToText.ts diff --git a/config/default.json b/config/default.json index 2b272c99..f4392886 100644 --- a/config/default.json +++ b/config/default.json @@ -105,7 +105,7 @@ "enabled": true }, "speechToText": { - "enabled": false + "enabled": true }, "sensitiveData": { "fields": { diff --git a/frontend-new/src/_test_utilities/envServiceMock.ts b/frontend-new/src/_test_utilities/envServiceMock.ts index fea807ed..d3e0a1db 100644 --- a/frontend-new/src/_test_utilities/envServiceMock.ts +++ b/frontend-new/src/_test_utilities/envServiceMock.ts @@ -9,6 +9,7 @@ jest.mock("src/envService", () => ({ getMetricsEnabled: jest.fn(() => "true"), getMetricsConfig: jest.fn(() => ""), getCvUploadEnabled: jest.fn(() => "true"), + getSpeechToTextEnabled: jest.fn(() => "false"), getNewSessionEnabled: jest.fn(() => false), getSocialAuthDisabled: jest.fn(() => "false"), getSupportedLocales: jest.fn(() => JSON.stringify(["en-US"])), diff --git a/frontend-new/src/chat/ChatMessageField/ChatMessageField.tsx b/frontend-new/src/chat/ChatMessageField/ChatMessageField.tsx index ed568181..760ceafd 100644 --- a/frontend-new/src/chat/ChatMessageField/ChatMessageField.tsx +++ b/frontend-new/src/chat/ChatMessageField/ChatMessageField.tsx @@ -1,7 +1,9 @@ import React, { KeyboardEvent, MouseEvent, useCallback, useContext, useEffect, useMemo, useState } from "react"; -import { Box, IconButton, InputAdornment, styled, TextField, Typography, useTheme } from "@mui/material"; +import { Box, CircularProgress, IconButton, InputAdornment, styled, TextField, Typography, useTheme } from "@mui/material"; import AddIcon from "@mui/icons-material/Add"; import ArrowUpwardSharpIcon from "@mui/icons-material/ArrowUpwardSharp"; +import MicIcon from "@mui/icons-material/Mic"; +import StopIcon from "@mui/icons-material/Stop"; import UploadFileIcon from "@mui/icons-material/UploadFile"; import { AnimatePresence, motion } from "framer-motion"; import { CV_UPLOAD_ERROR_I18N_KEYS, getCvUploadErrorMessageFromHttpStatus } from "../CVUploadErrorHandling"; @@ -10,7 +12,8 @@ import { MenuItemConfig } from "src/theme/ContextMenu/menuItemConfig.types"; import { IsOnlineContext } from "src/app/isOnlineProvider/IsOnlineProvider"; import { ConversationPhase } from "src/chat/chatProgressbar/types"; import AnimatedDotBadge from "src/theme/AnimatedDotBadge/AnimatedDotBadge"; -import { getCvUploadEnabled } from "src/envService"; +import { getCvUploadEnabled, getSpeechToTextEnabled } from "src/envService"; +import { useSpeechToText } from "src/speechToText/useSpeechToText"; import CVService from "src/CV/CVService/CVService"; import { CVListItem } from "src/CV/CVService/CVService.types"; import authenticationStateService from "src/auth/services/AuthenticationState.service"; @@ -52,6 +55,9 @@ export const DATA_TEST_ID = { CHAT_MESSAGE_FIELD_PLUS_BUTTON: `chat-message-field-plus-button-${uniqueId}`, CHAT_MESSAGE_FIELD_PLUS_ICON: `chat-message-field-plus-icon-${uniqueId}`, CHAT_MESSAGE_FIELD_HIDDEN_FILE_INPUT: `chat-message-field-hidden-file-input-${uniqueId}`, + CHAT_MESSAGE_FIELD_MIC_BUTTON: `chat-message-field-mic-button-${uniqueId}`, + CHAT_MESSAGE_FIELD_MIC_ICON: `chat-message-field-mic-icon-${uniqueId}`, + CHAT_MESSAGE_FIELD_STOP_ICON: `chat-message-field-stop-icon-${uniqueId}`, }; export const MENU_ITEM_ID = { @@ -73,6 +79,8 @@ export const PLACEHOLDER_TEXTS = { OFFLINE: "chat.chatMessageField.placeholders.offline", DEFAULT: "chat.chatMessageField.placeholders.default", UPLOADING: "chat.chatMessageField.placeholders.uploading", + TRANSCRIBING: "chat.chatMessageField.placeholders.transcribing", + RECORDING: "chat.chatMessageField.placeholders.recording", } as const; export const CHARACTER_LIMIT_ERROR_MESSAGES = { @@ -142,6 +150,32 @@ const ChatMessageField: React.FC = (props) => { const isCvUploadEnabled = getCvUploadEnabled().toLowerCase() === "true"; const showCvUpload = props.showCvUpload !== false; + const isSttEnabled = getSpeechToTextEnabled().toLowerCase() === "true"; + + const handleTranscriptionComplete = useCallback( + (text: string) => { + // Append to existing message or set as new message + const newMessage = message.trim().length > 0 ? `${message} ${text}` : text; + setMessage(newMessage); + if (newMessage.trim().length > CHAT_MESSAGE_MAX_LENGTH) { + setErrorMessage(t(CHARACTER_LIMIT_ERROR_MESSAGES.MESSAGE_LIMIT, { max: CHAT_MESSAGE_MAX_LENGTH })); + } else { + setErrorMessage(""); + } + }, + [message, t] + ); + + const { + status: sttStatus, + interimText, + error: sttError, + startRecording, + stopRecording, + isSupported: isSttSupported, + } = useSpeechToText({ + onTranscriptionComplete: handleTranscriptionComplete, + }); // Show the dot badge whenever in COLLECT_EXPERIENCES and not yet seen useEffect(() => { @@ -436,7 +470,20 @@ const ChatMessageField: React.FC = (props) => { return theme.palette.text.secondary; }, [message, theme.palette.error.main, theme.palette.warning.main, theme.palette.text.secondary]); + // Show STT errors in the error message area + useEffect(() => { + if (sttError) { + setErrorMessage(sttError); + } + }, [sttError]); + const placeHolder = useMemo(() => { + if (sttStatus === "transcribing") { + return t(PLACEHOLDER_TEXTS.TRANSCRIBING); + } + if (sttStatus === "recording" && !interimText) { + return t(PLACEHOLDER_TEXTS.RECORDING); + } if (props.isChatFinished) { return t(PLACEHOLDER_TEXTS.CHAT_FINISHED); } @@ -457,6 +504,8 @@ const ChatMessageField: React.FC = (props) => { } return t(PLACEHOLDER_TEXTS.DEFAULT); }, [ + sttStatus, + interimText, props.aiIsTyping, props.isChatFinished, props.isUploadingCv, @@ -474,15 +523,17 @@ const ChatMessageField: React.FC = (props) => { props.isInputDisabled || props.isUploadingCv || !isOnline || + sttStatus === "recording" || + sttStatus === "transcribing" || message.trim().length === 0 || message.trim().length > CHAT_MESSAGE_MAX_LENGTH // Only disable the send button when over the limit ); - }, [props.isChatFinished, props.aiIsTyping, props.isInputDisabled, props.isUploadingCv, isOnline, message]); + }, [props.isChatFinished, props.aiIsTyping, props.isInputDisabled, props.isUploadingCv, isOnline, message, sttStatus]); // Check if the input field should be disabled const inputIsDisabled = useCallback(() => { - return props.isChatFinished || props.aiIsTyping || props.isInputDisabled || props.isUploadingCv || !isOnline; - }, [props.isChatFinished, props.aiIsTyping, props.isInputDisabled, props.isUploadingCv, isOnline]); + return props.isChatFinished || props.aiIsTyping || props.isInputDisabled || props.isUploadingCv || !isOnline || sttStatus === "transcribing"; + }, [props.isChatFinished, props.aiIsTyping, props.isInputDisabled, props.isUploadingCv, isOnline, sttStatus]); const contextMenuItems: MenuItemConfig[] = menuView === "main" @@ -557,7 +608,7 @@ const ChatMessageField: React.FC = (props) => { fullWidth multiline maxRows={maxRows} - value={message} + value={sttStatus === "recording" ? (message ? `${message} ${interimText}` : interimText) : message} disabled={inputIsDisabled()} onChange={handleChange} onKeyDown={handleKeyDown} @@ -566,6 +617,7 @@ const ChatMessageField: React.FC = (props) => { helperText={errorMessage} fillColor={props.fillColor} InputProps={{ + readOnly: sttStatus === "recording", startAdornment: ( @@ -597,7 +649,42 @@ const ChatMessageField: React.FC = (props) => { ), endAdornment: ( - + + {isSttEnabled && isSttSupported && ( + event.stopPropagation()} + title={ + sttStatus === "recording" + ? t("chat.chatMessageField.stopRecordingTooltip") + : t("chat.chatMessageField.startRecordingTooltip") + } + sx={{ + backgroundColor: + sttStatus === "recording" ? theme.palette.error.main : "transparent", + "&:hover": { + backgroundColor: + sttStatus === "recording" ? theme.palette.error.dark : theme.palette.action.hover, + }, + }} + > + {sttStatus === "transcribing" ? ( + + ) : sttStatus === "recording" ? ( + + ) : ( + + )} + + )}