diff --git a/backend/.env.example b/backend/.env.example index cb12c7c1..66cc2b98 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -63,6 +63,11 @@ BACKEND_CV_MAX_UPLOADS_PER_USER= BACKEND_CV_RATE_LIMIT_PER_MINUTE= +# Speech-to-text (voice input) feature +GLOBAL_ENABLE_SPEECH_TO_TEXT=false +# Required when speech-to-text is enabled: the GCP project ID +GOOGLE_CLOUD_PROJECT= + # Locales BACKEND_LANGUAGE_CONFIG='{"default_locale":"en-US","available_locales":[{"locale":"en-US","date_format":"MM/DD/YYYY"}]}' diff --git a/backend/app/app_config.py b/backend/app/app_config.py index df76305c..80ff448a 100644 --- a/backend/app/app_config.py +++ b/backend/app/app_config.py @@ -73,6 +73,18 @@ class ApplicationConfig(BaseModel): When disabled, CV upload routes will not be registered and CV indexes will not be created. """ + enable_speech_to_text: bool = False + """ + A flag to enable or disable the Speech-to-Text (voice input) feature. + When disabled, speech-to-text routes will not be registered. + """ + + enable_text_to_speech: bool = False + """ + A flag to enable or disable the Text-to-Speech (read-aloud) feature. + When disabled, text-to-speech routes will not be registered. + """ + # CV storage and upload limits cv_storage_bucket: Optional[str] = "" cv_max_uploads_per_user: Optional[int] = Field(default=DEFAULT_MAX_UPLOADS_PER_USER, gt=0) diff --git a/backend/app/server.py b/backend/app/server.py index f956becc..17446924 100644 --- a/backend/app/server.py +++ b/backend/app/server.py @@ -21,6 +21,8 @@ from app.metrics.routes.routes import add_metrics_routes from app.analytics.routes import add_analytics_routes from app.teveta.routes import add_teveta_routes +from app.speech_to_text.routes import add_speech_to_text_routes +from app.text_to_speech.routes import add_text_to_speech_routes from app.sentry_init import init_sentry, set_sentry_contexts from app.server_dependencies.db_dependencies import CompassDBProvider from app.users.auth import Authentication, ApiKeyAuth @@ -226,6 +228,16 @@ def setup_sentry(): _enable_cv_upload = _enable_cv_upload_str.lower() == "true" logger.info(f"GLOBAL_ENABLE_CV_UPLOAD: {_enable_cv_upload}") +# Speech-to-text feature flag - defaults to False if not set +_enable_speech_to_text_str = os.getenv("GLOBAL_ENABLE_SPEECH_TO_TEXT", "false") +_enable_speech_to_text = _enable_speech_to_text_str.lower() == "true" +logger.info(f"GLOBAL_ENABLE_SPEECH_TO_TEXT: {_enable_speech_to_text}") + +# Text-to-speech feature flag - defaults to False if not set +_enable_text_to_speech_str = os.getenv("GLOBAL_ENABLE_TEXT_TO_SPEECH", "false") +_enable_text_to_speech = _enable_text_to_speech_str.lower() == "true" +logger.info(f"GLOBAL_ENABLE_TEXT_TO_SPEECH: {_enable_text_to_speech}") + application_config = ApplicationConfig( environment_name=os.getenv("TARGET_ENVIRONMENT_NAME"), version_info=load_version_info(), @@ -237,6 +249,8 @@ def setup_sentry(): features=backend_features_config, experience_pipeline_config=experience_pipeline_config, enable_cv_upload=_enable_cv_upload, + enable_speech_to_text=_enable_speech_to_text, + enable_text_to_speech=_enable_text_to_speech, cv_storage_bucket=os.getenv("BACKEND_CV_STORAGE_BUCKET", ""), cv_max_uploads_per_user=os.getenv("BACKEND_CV_MAX_UPLOADS_PER_USER") or DEFAULT_MAX_UPLOADS_PER_USER, cv_rate_limit_per_minute=os.getenv("BACKEND_CV_RATE_LIMIT_PER_MINUTE") or DEFAULT_RATE_LIMIT_PER_MINUTE, @@ -473,6 +487,24 @@ async def lifespan(_app: FastAPI): ############################################ add_poc_routes(app, auth) +############################################ +# Add speech-to-text routes (conditionally) +############################################ +if _enable_speech_to_text: + add_speech_to_text_routes(app, auth) + logger.info("Speech-to-text routes registered") +else: + logger.info("Speech-to-text routes skipped (GLOBAL_ENABLE_SPEECH_TO_TEXT is not enabled)") + +############################################ +# Add text-to-speech routes (conditionally) +############################################ +if _enable_text_to_speech: + add_text_to_speech_routes(app, auth) + logger.info("Text-to-speech routes registered") +else: + logger.info("Text-to-speech routes skipped (GLOBAL_ENABLE_TEXT_TO_SPEECH is not enabled)") + ############################################ # Add other features routes ############################################ diff --git a/backend/app/speech_to_text/__init__.py b/backend/app/speech_to_text/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/speech_to_text/constants.py b/backend/app/speech_to_text/constants.py new file mode 100644 index 00000000..1f37a3b4 --- /dev/null +++ b/backend/app/speech_to_text/constants.py @@ -0,0 +1,23 @@ +# Maximum audio file size: 10 MB +MAX_AUDIO_SIZE_BYTES = 10 * 1024 * 1024 + +# Maximum multipart overhead for early Content-Length rejection +MAX_MULTIPART_OVERHEAD_BYTES = 1024 + +# Allowed MIME types from browser MediaRecorder and phone recordings +ALLOWED_AUDIO_MIME_TYPES = { + "audio/webm", + "audio/webm;codecs=opus", + "audio/ogg", + "audio/ogg;codecs=opus", + "audio/mp4", + "audio/mpeg", + "audio/wav", + "audio/x-m4a", + "audio/aac", + "video/mp4", + "video/webm", +} + +# Default language for transcription +DEFAULT_LANGUAGE_CODE = "en-US" diff --git a/backend/app/speech_to_text/errors.py b/backend/app/speech_to_text/errors.py new file mode 100644 index 00000000..e1ef5178 --- /dev/null +++ b/backend/app/speech_to_text/errors.py @@ -0,0 +1,15 @@ +class TranscriptionError(Exception): + """Base exception for speech-to-text transcription errors.""" + + +class EmptyTranscriptionError(TranscriptionError): + """Raised when Google STT returns no transcription results.""" + def __init__(self): + super().__init__("No transcription results returned for the provided audio") + + +class SpeechToTextServiceError(TranscriptionError): + """Raised when the Google Cloud STT API call fails.""" + def __init__(self, detail: str): + super().__init__(f"Speech-to-text service error: {detail}") + self.detail = detail diff --git a/backend/app/speech_to_text/routes.py b/backend/app/speech_to_text/routes.py new file mode 100644 index 00000000..b90cf88a --- /dev/null +++ b/backend/app/speech_to_text/routes.py @@ -0,0 +1,142 @@ +import asyncio +import logging +from http import HTTPStatus + +from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile + +from app.constants.errors import HTTPErrorResponse +from app.speech_to_text.constants import ( + ALLOWED_AUDIO_MIME_TYPES, + DEFAULT_LANGUAGE_CODE, + MAX_AUDIO_SIZE_BYTES, + MAX_MULTIPART_OVERHEAD_BYTES, +) +from app.speech_to_text.errors import EmptyTranscriptionError, SpeechToTextServiceError +from app.speech_to_text.service import GoogleSpeechToTextService, ISpeechToTextService +from app.speech_to_text.types import TranscriptionResponse +from app.users.auth import Authentication, UserInfo + +logger = logging.getLogger(__name__) + +_stt_service_lock = asyncio.Lock() +_stt_service_singleton: ISpeechToTextService | None = None + + +async def _get_speech_to_text_service() -> ISpeechToTextService: + global _stt_service_singleton # pylint: disable=global-statement + if _stt_service_singleton is None: + async with _stt_service_lock: + if _stt_service_singleton is None: + _stt_service_singleton = GoogleSpeechToTextService() + return _stt_service_singleton + + +# Public alias for dependency override in tests +get_speech_to_text_service = _get_speech_to_text_service + + +def _validate_request_size_header(request: Request): + """Validate Content-Length before reading the body to fail fast on oversized uploads.""" + content_length_header = request.headers.get("content-length") + if content_length_header is None: + return + try: + content_length = int(content_length_header) + except ValueError: + return + if content_length > (MAX_AUDIO_SIZE_BYTES + MAX_MULTIPART_OVERHEAD_BYTES): + logger.warning( + "413 via header-check: content_length=%s limit=%s", + content_length, + MAX_AUDIO_SIZE_BYTES, + ) + raise HTTPException( + status_code=HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + detail="Audio file exceeds maximum allowed size", + ) + + +def add_speech_to_text_routes(app, authentication: Authentication): + """Register speech-to-text routes on the given FastAPI app.""" + router = APIRouter(prefix="/speech-to-text", tags=["speech-to-text"]) + + @router.post( + path="/transcribe", + status_code=HTTPStatus.OK, + response_model=TranscriptionResponse, + responses={ + HTTPStatus.BAD_REQUEST: {"model": HTTPErrorResponse}, + HTTPStatus.UNSUPPORTED_MEDIA_TYPE: {"model": HTTPErrorResponse}, + HTTPStatus.REQUEST_ENTITY_TOO_LARGE: {"model": HTTPErrorResponse}, + HTTPStatus.UNPROCESSABLE_ENTITY: {"model": HTTPErrorResponse}, + HTTPStatus.BAD_GATEWAY: {"model": HTTPErrorResponse}, + HTTPStatus.INTERNAL_SERVER_ERROR: {"model": HTTPErrorResponse}, + }, + name="transcribe audio", + description="Transcribe audio input to text using Google Cloud Speech-to-Text.", + ) + async def _transcribe_audio( + request: Request, + audio: UploadFile = File(..., description="Audio file recorded in the browser"), + language: str = Form(default=DEFAULT_LANGUAGE_CODE), + user_info: UserInfo = Depends(authentication.get_user_info()), + service: ISpeechToTextService = Depends(get_speech_to_text_service), + ) -> TranscriptionResponse: + # Validate size early via Content-Length header + _validate_request_size_header(request) + + # Validate MIME type — normalize by stripping spaces around semicolons + raw_content_type = (audio.content_type or "").strip() + # Check both the base type (e.g. "audio/webm") and the full type with params normalized + # (e.g. "audio/webm;codecs=opus") to handle browsers that add spaces after ";" + content_type = raw_content_type.split(";")[0].strip() + normalized_content_type = ";".join(part.strip() for part in raw_content_type.split(";")) + if content_type not in ALLOWED_AUDIO_MIME_TYPES and normalized_content_type not in ALLOWED_AUDIO_MIME_TYPES: + raise HTTPException( + status_code=HTTPStatus.UNSUPPORTED_MEDIA_TYPE, + detail="Unsupported audio format", + ) + + # Read and validate file size + audio_bytes = await audio.read() + if len(audio_bytes) > MAX_AUDIO_SIZE_BYTES: + raise HTTPException( + status_code=HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + detail="Audio file exceeds maximum allowed size", + ) + + if len(audio_bytes) == 0: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST, + detail="Audio file is empty", + ) + + logger.info( + "Transcribing audio {user_id=%s, size_bytes=%s, content_type='%s', language='%s'}", + user_info.user_id, + len(audio_bytes), + audio.content_type, + language, + ) + + try: + return await service.transcribe(audio_bytes=audio_bytes, language_code=language) + except EmptyTranscriptionError as exc: + raise HTTPException( + status_code=HTTPStatus.UNPROCESSABLE_ENTITY, + detail="No transcription results returned for the provided audio", + ) from exc + except SpeechToTextServiceError as e: + logger.error("Speech-to-text service error", exc_info=True) + raise HTTPException( + status_code=HTTPStatus.BAD_GATEWAY, + detail="Upstream transcription service unavailable", + ) from e + except Exception as e: # pylint: disable=broad-except + logger.exception("Unexpected error during transcription") + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail="An unexpected error occurred during transcription", + ) from e + + app.include_router(router) diff --git a/backend/app/speech_to_text/service.py b/backend/app/speech_to_text/service.py new file mode 100644 index 00000000..fa60b59f --- /dev/null +++ b/backend/app/speech_to_text/service.py @@ -0,0 +1,86 @@ +import asyncio +import logging +import os +from abc import ABC, abstractmethod + +from google.cloud.speech_v2 import SpeechClient +from google.cloud.speech_v2.types import cloud_speech + +from app.speech_to_text.errors import EmptyTranscriptionError, SpeechToTextServiceError +from app.speech_to_text.types import TranscriptionResponse +from common_libs.retry import Retry + + +class ISpeechToTextService(ABC): + """Interface for speech-to-text transcription services.""" + + @abstractmethod + async def transcribe(self, *, audio_bytes: bytes, language_code: str) -> TranscriptionResponse: + """ + Transcribe audio bytes to text. + + :param audio_bytes: The raw audio bytes to transcribe. + :param language_code: The BCP-47 language code for transcription (e.g. "en-US"). + :return: The transcription response containing the text and language code. + :raises EmptyTranscriptionError: If no transcription results are returned. + :raises SpeechToTextServiceError: If the Google Cloud STT API call fails. + """ + raise NotImplementedError() + + +class GoogleSpeechToTextService(ISpeechToTextService): + """Google Cloud Speech-to-Text v2 implementation.""" + + def __init__(self): + self._logger = logging.getLogger(self.__class__.__name__) + + self._project_id = os.getenv("GOOGLE_CLOUD_PROJECT") + if not self._project_id: + raise ValueError("GOOGLE_CLOUD_PROJECT environment variable is not set") + + self._region = os.getenv("VERTEX_API_REGION", "us-central1") + + # Create the client with regional endpoint + self._client = SpeechClient( + client_options={"api_endpoint": f"{self._region}-speech.googleapis.com"} + ) + + async def transcribe(self, *, audio_bytes: bytes, language_code: str) -> TranscriptionResponse: + recognizer = f"projects/{self._project_id}/locations/{self._region}/recognizers/_" + + config = cloud_speech.RecognitionConfig( + auto_decoding_config=cloud_speech.AutoDetectDecodingConfig(), + language_codes=[language_code], + model="long", + ) + + request = cloud_speech.RecognizeRequest( + recognizer=recognizer, + config=config, + content=audio_bytes, + ) + + try: + response = await Retry[cloud_speech.RecognizeResponse].call_with_exponential_backoff( + callback=lambda: asyncio.to_thread(self._client.recognize, request=request), + logger=self._logger, + ) + except Exception as e: + self._logger.error("Speech-to-text API call failed", exc_info=True) + raise SpeechToTextServiceError(detail=str(e)) from e + + # Concatenate all transcript alternatives + transcript_parts = [] + for result in response.results: + if result.alternatives: + transcript_parts.append(result.alternatives[0].transcript) + + text = " ".join(transcript_parts).strip() + + if not text: + raise EmptyTranscriptionError() + + return TranscriptionResponse( + text=text, + language_code=language_code, + ) diff --git a/backend/app/speech_to_text/test_routes.py b/backend/app/speech_to_text/test_routes.py new file mode 100644 index 00000000..4a5ac5ff --- /dev/null +++ b/backend/app/speech_to_text/test_routes.py @@ -0,0 +1,191 @@ +import io +from http import HTTPStatus +from unittest.mock import AsyncMock + +from fastapi import FastAPI +from starlette.testclient import TestClient + +from app.speech_to_text.errors import EmptyTranscriptionError, SpeechToTextServiceError +from app.speech_to_text.routes import add_speech_to_text_routes, get_speech_to_text_service +from app.speech_to_text.service import ISpeechToTextService +from app.speech_to_text.types import TranscriptionResponse +from common_libs.test_utilities.mock_auth import MockAuth, UnauthenticatedMockAuth + + +class _MockSpeechToTextService(ISpeechToTextService): + """Mock implementation for testing.""" + + def __init__(self): + self.transcribe_mock = AsyncMock() + + async def transcribe(self, *, audio_bytes: bytes, language_code: str) -> TranscriptionResponse: + return await self.transcribe_mock(audio_bytes=audio_bytes, language_code=language_code) + + +def _given_audio_file(content: bytes = b"fake-audio-data", content_type: str = "audio/webm"): + """Create a mock audio file for upload.""" + return ("audio", ("recording.webm", io.BytesIO(content), content_type)) + + +def _create_test_client(auth=None): + """Set up a test client with mocked dependencies.""" + if auth is None: + auth = MockAuth() + mock_service = _MockSpeechToTextService() + app = FastAPI() + app.dependency_overrides[get_speech_to_text_service] = lambda: mock_service + add_speech_to_text_routes(app, authentication=auth) + client = TestClient(app) + return client, mock_service, auth + + +class TestTranscribeAudioEndpoint: + """Tests for POST /speech-to-text/transcribe""" + + def test_respond_with_status_ok_and_transcription_on_success(self): + """Test successful transcription with default language.""" + # GIVEN a valid audio file + given_audio = _given_audio_file() + # AND a service that returns a successful transcription + client, mock_service, _auth = _create_test_client() + given_response = TranscriptionResponse(text="hello world", language_code="en-US") + mock_service.transcribe_mock.return_value = given_response + + # WHEN the transcribe endpoint is called + actual_response = client.post("/speech-to-text/transcribe", files=[given_audio], data={"language": "en-US"}) + + # THEN expect the response status to be OK + assert actual_response.status_code == HTTPStatus.OK + # AND the response body to contain the transcribed text + actual_body = actual_response.json() + assert actual_body["text"] == "hello world" + assert actual_body["language_code"] == "en-US" + + def test_respond_with_status_ok_when_explicit_language_provided(self): + """Test successful transcription with an explicit language.""" + # GIVEN a valid audio file + given_audio = _given_audio_file() + # AND an explicit language + given_language = "sw-KE" + # AND a service that returns a successful transcription + client, mock_service, _auth = _create_test_client() + given_response = TranscriptionResponse(text="habari", language_code=given_language) + mock_service.transcribe_mock.return_value = given_response + + # WHEN the transcribe endpoint is called with the given language + actual_response = client.post( + "/speech-to-text/transcribe", + files=[given_audio], + data={"language": given_language}, + ) + + # THEN expect the response status to be OK + assert actual_response.status_code == HTTPStatus.OK + # AND the language code to match the given language + assert actual_response.json()["language_code"] == given_language + + def test_respond_with_status_unsupported_media_type_for_invalid_mime(self): + """Test that unsupported audio formats are rejected.""" + # GIVEN an audio file with an unsupported MIME type + given_audio = _given_audio_file(content_type="application/pdf") + client, _mock_service, _auth = _create_test_client() + + # WHEN the transcribe endpoint is called + actual_response = client.post("/speech-to-text/transcribe", files=[given_audio], data={"language": "en-US"}) + + # THEN expect the response status to be UNSUPPORTED_MEDIA_TYPE + assert actual_response.status_code == HTTPStatus.UNSUPPORTED_MEDIA_TYPE + + def test_respond_with_status_ok_when_mime_type_has_space_after_semicolon(self): + """Test that MIME types with spaces after semicolons are accepted.""" + # GIVEN an audio file with a space after the semicolon in the MIME type + given_audio = _given_audio_file(content_type="audio/webm; codecs=opus") + # AND a service that returns a successful transcription + client, mock_service, _auth = _create_test_client() + given_response = TranscriptionResponse(text="hello", language_code="en-US") + mock_service.transcribe_mock.return_value = given_response + + # WHEN the transcribe endpoint is called + actual_response = client.post("/speech-to-text/transcribe", files=[given_audio], data={"language": "en-US"}) + + # THEN expect the response status to be OK + assert actual_response.status_code == HTTPStatus.OK + + def test_respond_with_status_entity_too_large_for_oversized_audio(self): + """Test that oversized audio files are rejected.""" + # GIVEN an audio file that exceeds the maximum size + given_large_content = b"x" * (10 * 1024 * 1024 + 1) + given_audio = _given_audio_file(content=given_large_content) + client, _mock_service, _auth = _create_test_client() + + # WHEN the transcribe endpoint is called + actual_response = client.post("/speech-to-text/transcribe", files=[given_audio], data={"language": "en-US"}) + + # THEN expect the response status to be REQUEST_ENTITY_TOO_LARGE + assert actual_response.status_code == HTTPStatus.REQUEST_ENTITY_TOO_LARGE + + def test_respond_with_status_unprocessable_entity_when_transcription_is_empty(self): + """Test that empty transcription results return 422.""" + # GIVEN a valid audio file + given_audio = _given_audio_file() + # AND a service that raises EmptyTranscriptionError + client, mock_service, _auth = _create_test_client() + mock_service.transcribe_mock.side_effect = EmptyTranscriptionError() + + # WHEN the transcribe endpoint is called + actual_response = client.post("/speech-to-text/transcribe", files=[given_audio], data={"language": "en-US"}) + + # THEN expect the response status to be UNPROCESSABLE_ENTITY + assert actual_response.status_code == HTTPStatus.UNPROCESSABLE_ENTITY + + def test_respond_with_status_bad_gateway_when_service_fails(self): + """Test that Google STT API failures return 502.""" + # GIVEN a valid audio file + given_audio = _given_audio_file() + # AND a service that raises SpeechToTextServiceError + client, mock_service, _auth = _create_test_client() + mock_service.transcribe_mock.side_effect = SpeechToTextServiceError(detail="API unavailable") + + # WHEN the transcribe endpoint is called + actual_response = client.post("/speech-to-text/transcribe", files=[given_audio], data={"language": "en-US"}) + + # THEN expect the response status to be BAD_GATEWAY + assert actual_response.status_code == HTTPStatus.BAD_GATEWAY + + def test_respond_with_status_internal_server_error_on_unexpected_error(self): + """Test that unexpected errors return 500.""" + # GIVEN a valid audio file + given_audio = _given_audio_file() + # AND a service that raises an unexpected error + client, mock_service, _auth = _create_test_client() + mock_service.transcribe_mock.side_effect = RuntimeError("something broke") + + # WHEN the transcribe endpoint is called + actual_response = client.post("/speech-to-text/transcribe", files=[given_audio], data={"language": "en-US"}) + + # THEN expect the response status to be INTERNAL_SERVER_ERROR + assert actual_response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR + + def test_respond_with_status_unauthorized_when_not_authenticated(self): + """Test that unauthenticated requests are rejected.""" + # GIVEN an unauthenticated user + given_audio = _given_audio_file() + client, _mock_service, _auth = _create_test_client(auth=UnauthenticatedMockAuth()) + + # WHEN the transcribe endpoint is called + actual_response = client.post("/speech-to-text/transcribe", files=[given_audio], data={"language": "en-US"}) + + # THEN expect the response status to be UNAUTHORIZED + assert actual_response.status_code == HTTPStatus.UNAUTHORIZED + + def test_respond_with_status_bad_request_for_empty_audio(self): + """Test that empty audio files are rejected.""" + # GIVEN an empty audio file + given_audio = _given_audio_file(content=b"") + client, _mock_service, _auth = _create_test_client() + + # WHEN the transcribe endpoint is called + actual_response = client.post("/speech-to-text/transcribe", files=[given_audio], data={"language": "en-US"}) + + # THEN expect the response status to be BAD_REQUEST + assert actual_response.status_code == HTTPStatus.BAD_REQUEST diff --git a/backend/app/speech_to_text/test_service.py b/backend/app/speech_to_text/test_service.py new file mode 100644 index 00000000..aa8a6e4d --- /dev/null +++ b/backend/app/speech_to_text/test_service.py @@ -0,0 +1,109 @@ +from unittest.mock import MagicMock, patch + +import pytest + +from app.speech_to_text.errors import EmptyTranscriptionError, SpeechToTextServiceError +from app.speech_to_text.service import GoogleSpeechToTextService + + +def _mock_recognize_response(transcripts: list[str]): + """Create a mock RecognizeResponse with the given transcript texts.""" + response = MagicMock() + results = [] + for text in transcripts: + alternative = MagicMock() + alternative.transcript = text + result = MagicMock() + result.alternatives = [alternative] + results.append(result) + response.results = results + return response + + +def _mock_empty_recognize_response(): + """Create a mock RecognizeResponse with no results.""" + response = MagicMock() + response.results = [] + return response + + +@pytest.fixture() +def _set_env_vars(monkeypatch): + """Set required environment variables for GoogleSpeechToTextService.""" + monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "test-project") + monkeypatch.setenv("VERTEX_API_REGION", "us-central1") + + +@pytest.mark.usefixtures("_set_env_vars") +class TestGoogleSpeechToTextService: + """Tests for GoogleSpeechToTextService""" + + @pytest.mark.asyncio + @patch("app.speech_to_text.service.SpeechClient") + async def test_transcribe_returns_concatenated_text_on_success(self, mock_speech_client_class): + """Test that transcription results are concatenated correctly.""" + # GIVEN a service with a mocked SpeechClient + given_mock_client = MagicMock() + mock_speech_client_class.return_value = given_mock_client + # AND the client returns a response with multiple transcript parts + given_response = _mock_recognize_response(["hello", "world"]) + given_mock_client.recognize.return_value = given_response + # AND a valid audio input + given_audio_bytes = b"fake-audio" + given_language = "en-US" + + service = GoogleSpeechToTextService() + + # WHEN transcribe is called + actual_result = await service.transcribe(audio_bytes=given_audio_bytes, language_code=given_language) + + # THEN expect the text to be the concatenated transcripts + assert actual_result.text == "hello world" + # AND the language code to match the given language + assert actual_result.language_code == given_language + + @pytest.mark.asyncio + @patch("app.speech_to_text.service.SpeechClient") + async def test_transcribe_raises_empty_transcription_error_when_no_results(self, mock_speech_client_class): + """Test that EmptyTranscriptionError is raised when STT returns no results.""" + # GIVEN a service with a mocked SpeechClient + given_mock_client = MagicMock() + mock_speech_client_class.return_value = given_mock_client + # AND the client returns an empty response + given_mock_client.recognize.return_value = _mock_empty_recognize_response() + given_audio_bytes = b"fake-audio" + + service = GoogleSpeechToTextService() + + # WHEN transcribe is called + # THEN expect EmptyTranscriptionError to be raised + with pytest.raises(EmptyTranscriptionError): + await service.transcribe(audio_bytes=given_audio_bytes, language_code="en-US") + + @pytest.mark.asyncio + @patch("app.speech_to_text.service.SpeechClient") + async def test_transcribe_raises_service_error_when_api_fails(self, mock_speech_client_class): + """Test that SpeechToTextServiceError is raised when the API call fails.""" + # GIVEN a service with a mocked SpeechClient + given_mock_client = MagicMock() + mock_speech_client_class.return_value = given_mock_client + # AND the client raises an exception + given_mock_client.recognize.side_effect = Exception("API unavailable") + given_audio_bytes = b"fake-audio" + + service = GoogleSpeechToTextService() + + # WHEN transcribe is called + # THEN expect SpeechToTextServiceError to be raised + with pytest.raises(SpeechToTextServiceError): + await service.transcribe(audio_bytes=given_audio_bytes, language_code="en-US") + + def test_init_raises_when_google_cloud_project_not_set(self, monkeypatch): + """Test that initialization fails without GOOGLE_CLOUD_PROJECT.""" + # GIVEN GOOGLE_CLOUD_PROJECT is not set + monkeypatch.delenv("GOOGLE_CLOUD_PROJECT", raising=False) + + # WHEN the service is initialized + # THEN expect a ValueError + with pytest.raises(ValueError, match="GOOGLE_CLOUD_PROJECT"): + GoogleSpeechToTextService() diff --git a/backend/app/speech_to_text/types.py b/backend/app/speech_to_text/types.py new file mode 100644 index 00000000..7a86d30e --- /dev/null +++ b/backend/app/speech_to_text/types.py @@ -0,0 +1,12 @@ +from pydantic import BaseModel, Field + + +class TranscriptionResponse(BaseModel): + """Response returned after successful speech-to-text transcription.""" + + class Config: + """Pydantic model configuration.""" + extra = "forbid" + + text: str = Field(description="The transcribed text from the audio input") + language_code: str = Field(description="The language code used for transcription") diff --git a/backend/app/text_to_speech/__init__.py b/backend/app/text_to_speech/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/text_to_speech/constants.py b/backend/app/text_to_speech/constants.py new file mode 100644 index 00000000..0c3398e3 --- /dev/null +++ b/backend/app/text_to_speech/constants.py @@ -0,0 +1,19 @@ +# Maximum text length for synthesis +MAX_TEXT_LENGTH = 5000 + +# Default language for synthesis +DEFAULT_LANGUAGE_CODE = "en-US" + +# Map application language codes to Google Cloud TTS voice names. +# Chirp 3 HD voices provide the most natural-sounding speech. +# For languages without a Chirp 3 HD voice, fall back to English. +LANGUAGE_TO_VOICE: dict[str, str] = { + "en-US": "en-US-Chirp3-HD-Achernar", + "en-GB": "en-GB-Chirp3-HD-Achernar", + "ny-ZM": "en-US-Chirp3-HD-Achernar", # Chichewa not supported; fall back to English + "es-ES": "es-ES-Chirp3-HD-Achernar", + "es-AR": "es-ES-Chirp3-HD-Achernar", + "sw-KE": "sw-KE-Chirp3-HD-Achernar", +} + +DEFAULT_VOICE_NAME = "en-US-Chirp3-HD-Achernar" diff --git a/backend/app/text_to_speech/errors.py b/backend/app/text_to_speech/errors.py new file mode 100644 index 00000000..4eda7e24 --- /dev/null +++ b/backend/app/text_to_speech/errors.py @@ -0,0 +1,15 @@ +class TextToSpeechError(Exception): + """Base exception for text-to-speech synthesis errors.""" + + +class EmptySynthesisError(TextToSpeechError): + """Raised when Google TTS returns no audio content.""" + def __init__(self): + super().__init__("No audio content returned for the provided text") + + +class TextToSpeechServiceError(TextToSpeechError): + """Raised when the Google Cloud TTS API call fails.""" + def __init__(self, detail: str): + super().__init__(f"Text-to-speech service error: {detail}") + self.detail = detail diff --git a/backend/app/text_to_speech/routes.py b/backend/app/text_to_speech/routes.py new file mode 100644 index 00000000..51d004c4 --- /dev/null +++ b/backend/app/text_to_speech/routes.py @@ -0,0 +1,97 @@ +import asyncio +import logging +from http import HTTPStatus + +from fastapi import APIRouter, Depends, HTTPException +from starlette.responses import Response + +from app.constants.errors import HTTPErrorResponse +from app.text_to_speech.constants import MAX_TEXT_LENGTH +from app.text_to_speech.errors import EmptySynthesisError, TextToSpeechServiceError +from app.text_to_speech.service import GoogleTextToSpeechService, ITextToSpeechService +from app.text_to_speech.types import SynthesizeRequest +from app.users.auth import Authentication, UserInfo + +logger = logging.getLogger(__name__) + +_tts_service_lock = asyncio.Lock() +_tts_service_singleton: ITextToSpeechService | None = None + + +async def _get_text_to_speech_service() -> ITextToSpeechService: + global _tts_service_singleton # pylint: disable=global-statement + if _tts_service_singleton is None: + async with _tts_service_lock: + if _tts_service_singleton is None: + _tts_service_singleton = GoogleTextToSpeechService() + return _tts_service_singleton + + +# Public alias for dependency override in tests +get_text_to_speech_service = _get_text_to_speech_service + + +def add_text_to_speech_routes(app, authentication: Authentication): + """Register text-to-speech routes on the given FastAPI app.""" + router = APIRouter(prefix="/text-to-speech", tags=["text-to-speech"]) + + @router.post( + path="/synthesize", + status_code=HTTPStatus.OK, + responses={ + HTTPStatus.BAD_REQUEST: {"model": HTTPErrorResponse}, + HTTPStatus.UNPROCESSABLE_ENTITY: {"model": HTTPErrorResponse}, + HTTPStatus.BAD_GATEWAY: {"model": HTTPErrorResponse}, + HTTPStatus.INTERNAL_SERVER_ERROR: {"model": HTTPErrorResponse}, + }, + name="synthesize speech", + description="Synthesize text into speech using Google Cloud Text-to-Speech.", + ) + async def _synthesize_speech( + body: SynthesizeRequest, + user_info: UserInfo = Depends(authentication.get_user_info()), + service: ITextToSpeechService = Depends(get_text_to_speech_service), + ) -> Response: + # Validate empty text + if not body.text or not body.text.strip(): + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST, + detail="Text is empty", + ) + + # Validate text length + if len(body.text) > MAX_TEXT_LENGTH: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST, + detail=f"Text length {len(body.text)} exceeds maximum allowed length of {MAX_TEXT_LENGTH}", + ) + + logger.info( + "Synthesizing speech {user_id=%s, text_length=%s, language='%s'}", + user_info.user_id, + len(body.text), + body.language, + ) + + try: + audio_bytes = await service.synthesize(text=body.text, language_code=body.language) + return Response(content=audio_bytes, media_type="audio/mpeg") + except EmptySynthesisError as exc: + raise HTTPException( + status_code=HTTPStatus.UNPROCESSABLE_ENTITY, + detail="No audio content returned for the provided text", + ) from exc + except TextToSpeechServiceError as e: + logger.error("Text-to-speech service error", exc_info=True) + raise HTTPException( + status_code=HTTPStatus.BAD_GATEWAY, + detail="Upstream synthesis service unavailable", + ) from e + except Exception as e: # pylint: disable=broad-except + logger.exception("Unexpected error during synthesis") + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail="An unexpected error occurred during synthesis", + ) from e + + app.include_router(router) diff --git a/backend/app/text_to_speech/service.py b/backend/app/text_to_speech/service.py new file mode 100644 index 00000000..a042a956 --- /dev/null +++ b/backend/app/text_to_speech/service.py @@ -0,0 +1,77 @@ +import asyncio +import logging +import os +from abc import ABC, abstractmethod + +from google.cloud import texttospeech + +from app.text_to_speech.constants import DEFAULT_VOICE_NAME, LANGUAGE_TO_VOICE +from app.text_to_speech.errors import EmptySynthesisError, TextToSpeechServiceError +from common_libs.retry import Retry + + +class ITextToSpeechService(ABC): + """Interface for text-to-speech synthesis services.""" + + @abstractmethod + async def synthesize(self, *, text: str, language_code: str) -> bytes: + """ + Synthesize text into audio bytes. + + :param text: The text to synthesize into speech. + :param language_code: The BCP-47 language code for synthesis (e.g. "en-US"). + :return: The synthesized audio bytes in MP3 format. + :raises EmptySynthesisError: If no audio content is returned. + :raises TextToSpeechServiceError: If the Google Cloud TTS API call fails. + """ + raise NotImplementedError() + + +class GoogleTextToSpeechService(ITextToSpeechService): + """Google Cloud Text-to-Speech implementation.""" + + def __init__(self): + self._logger = logging.getLogger(self.__class__.__name__) + + # Validate that the GCP project is configured (required for authentication context) + if not os.getenv("GOOGLE_CLOUD_PROJECT"): + raise ValueError("GOOGLE_CLOUD_PROJECT environment variable is not set") + + self._client = texttospeech.TextToSpeechClient() + + async def synthesize(self, *, text: str, language_code: str) -> bytes: + # Look up the voice name for the given language, falling back to default + voice_name = LANGUAGE_TO_VOICE.get(language_code, DEFAULT_VOICE_NAME) + + # Extract the language code prefix from the voice name (e.g. "en-US" from "en-US-Chirp3-HD-Achernar") + voice_language_code = "-".join(voice_name.split("-")[:2]) + + synthesis_input = texttospeech.SynthesisInput(text=text) + + voice_params = texttospeech.VoiceSelectionParams( + language_code=voice_language_code, + name=voice_name, + ) + + audio_config = texttospeech.AudioConfig( + audio_encoding=texttospeech.AudioEncoding.MP3, + ) + + try: + response = await Retry[texttospeech.SynthesizeSpeechResponse].call_with_exponential_backoff( + callback=lambda: asyncio.to_thread( + self._client.synthesize_speech, + input=synthesis_input, + voice=voice_params, + audio_config=audio_config, + ), + logger=self._logger, + ) + except Exception as e: + self._logger.error("Text-to-speech API call failed", exc_info=True) + raise TextToSpeechServiceError(detail=str(e)) from e + + if not response.audio_content: + raise EmptySynthesisError() + + return response.audio_content diff --git a/backend/app/text_to_speech/test_routes.py b/backend/app/text_to_speech/test_routes.py new file mode 100644 index 00000000..c8c4deb1 --- /dev/null +++ b/backend/app/text_to_speech/test_routes.py @@ -0,0 +1,189 @@ +from http import HTTPStatus +from unittest.mock import AsyncMock + +from fastapi import FastAPI +from starlette.testclient import TestClient + +from app.text_to_speech.constants import MAX_TEXT_LENGTH +from app.text_to_speech.errors import EmptySynthesisError, TextToSpeechServiceError +from app.text_to_speech.routes import add_text_to_speech_routes, get_text_to_speech_service +from app.text_to_speech.service import ITextToSpeechService +from common_libs.test_utilities.mock_auth import MockAuth, UnauthenticatedMockAuth + + +class _MockTextToSpeechService(ITextToSpeechService): + """Mock implementation for testing.""" + + def __init__(self): + self.synthesize_mock = AsyncMock() + + async def synthesize(self, *, text: str, language_code: str) -> bytes: + return await self.synthesize_mock(text=text, language_code=language_code) + + +def _create_test_client(auth=None): + """Set up a test client with mocked dependencies.""" + if auth is None: + auth = MockAuth() + mock_service = _MockTextToSpeechService() + app = FastAPI() + app.dependency_overrides[get_text_to_speech_service] = lambda: mock_service + add_text_to_speech_routes(app, authentication=auth) + client = TestClient(app) + return client, mock_service, auth + + +class TestSynthesizeSpeechEndpoint: + """Tests for POST /text-to-speech/synthesize""" + + def test_respond_with_status_ok_and_audio_on_success(self): + """Test successful synthesis returns audio bytes.""" + # GIVEN a valid text input + given_text = "Hello world" + # AND a service that returns audio bytes + client, mock_service, _auth = _create_test_client() + given_audio_bytes = b"fake-audio-content" + mock_service.synthesize_mock.return_value = given_audio_bytes + + # WHEN the synthesize endpoint is called + actual_response = client.post( + "/text-to-speech/synthesize", + json={"text": given_text, "language": "en-US"}, + ) + + # THEN expect the response status to be OK + assert actual_response.status_code == HTTPStatus.OK + # AND the response content type to be audio/mpeg + assert actual_response.headers["content-type"] == "audio/mpeg" + # AND the response body to contain the audio bytes + assert actual_response.content == given_audio_bytes + + def test_respond_with_status_ok_with_custom_language(self): + """Test successful synthesis passes the language to the service.""" + # GIVEN a valid text input + given_text = "Habari" + # AND an explicit language + given_language = "sw-KE" + # AND a service that returns audio bytes + client, mock_service, _auth = _create_test_client() + given_audio_bytes = b"fake-audio-content" + mock_service.synthesize_mock.return_value = given_audio_bytes + + # WHEN the synthesize endpoint is called with the given language + actual_response = client.post( + "/text-to-speech/synthesize", + json={"text": given_text, "language": given_language}, + ) + + # THEN expect the response status to be OK + assert actual_response.status_code == HTTPStatus.OK + # AND the service to have been called with the given language + mock_service.synthesize_mock.assert_called_once_with(text=given_text, language_code=given_language) + + def test_respond_with_bad_request_when_text_is_empty(self): + """Test that empty text is rejected.""" + # GIVEN an empty text input + client, _mock_service, _auth = _create_test_client() + + # WHEN the synthesize endpoint is called with empty text + actual_response = client.post( + "/text-to-speech/synthesize", + json={"text": "", "language": "en-US"}, + ) + + # THEN expect the response status to be BAD_REQUEST + assert actual_response.status_code == HTTPStatus.BAD_REQUEST + + def test_respond_with_bad_request_when_text_is_whitespace(self): + """Test that whitespace-only text is rejected.""" + # GIVEN a whitespace-only text input + client, _mock_service, _auth = _create_test_client() + + # WHEN the synthesize endpoint is called with whitespace text + actual_response = client.post( + "/text-to-speech/synthesize", + json={"text": " ", "language": "en-US"}, + ) + + # THEN expect the response status to be BAD_REQUEST + assert actual_response.status_code == HTTPStatus.BAD_REQUEST + + def test_respond_with_bad_request_when_text_too_long(self): + """Test that text exceeding MAX_TEXT_LENGTH is rejected.""" + # GIVEN a text input that exceeds the maximum length + given_text = "x" * (MAX_TEXT_LENGTH + 1) + client, _mock_service, _auth = _create_test_client() + + # WHEN the synthesize endpoint is called with the oversized text + actual_response = client.post( + "/text-to-speech/synthesize", + json={"text": given_text, "language": "en-US"}, + ) + + # THEN expect the response status to be BAD_REQUEST + assert actual_response.status_code == HTTPStatus.BAD_REQUEST + + def test_respond_with_unprocessable_entity_when_no_audio_returned(self): + """Test that empty synthesis results return 422.""" + # GIVEN a valid text input + given_text = "Hello world" + # AND a service that raises EmptySynthesisError + client, mock_service, _auth = _create_test_client() + mock_service.synthesize_mock.side_effect = EmptySynthesisError() + + # WHEN the synthesize endpoint is called + actual_response = client.post( + "/text-to-speech/synthesize", + json={"text": given_text, "language": "en-US"}, + ) + + # THEN expect the response status to be UNPROCESSABLE_ENTITY + assert actual_response.status_code == HTTPStatus.UNPROCESSABLE_ENTITY + + def test_respond_with_bad_gateway_when_service_fails(self): + """Test that Google TTS API failures return 502.""" + # GIVEN a valid text input + given_text = "Hello world" + # AND a service that raises TextToSpeechServiceError + client, mock_service, _auth = _create_test_client() + mock_service.synthesize_mock.side_effect = TextToSpeechServiceError(detail="API unavailable") + + # WHEN the synthesize endpoint is called + actual_response = client.post( + "/text-to-speech/synthesize", + json={"text": given_text, "language": "en-US"}, + ) + + # THEN expect the response status to be BAD_GATEWAY + assert actual_response.status_code == HTTPStatus.BAD_GATEWAY + + def test_respond_with_internal_error_on_unexpected_exception(self): + """Test that unexpected errors return 500.""" + # GIVEN a valid text input + given_text = "Hello world" + # AND a service that raises an unexpected error + client, mock_service, _auth = _create_test_client() + mock_service.synthesize_mock.side_effect = RuntimeError("something broke") + + # WHEN the synthesize endpoint is called + actual_response = client.post( + "/text-to-speech/synthesize", + json={"text": given_text, "language": "en-US"}, + ) + + # THEN expect the response status to be INTERNAL_SERVER_ERROR + assert actual_response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR + + def test_respond_with_unauthorized_when_not_authenticated(self): + """Test that unauthenticated requests are rejected.""" + # GIVEN an unauthenticated user + client, _mock_service, _auth = _create_test_client(auth=UnauthenticatedMockAuth()) + + # WHEN the synthesize endpoint is called + actual_response = client.post( + "/text-to-speech/synthesize", + json={"text": "Hello world", "language": "en-US"}, + ) + + # THEN expect the response status to be UNAUTHORIZED + assert actual_response.status_code == HTTPStatus.UNAUTHORIZED diff --git a/backend/app/text_to_speech/test_service.py b/backend/app/text_to_speech/test_service.py new file mode 100644 index 00000000..80cc4479 --- /dev/null +++ b/backend/app/text_to_speech/test_service.py @@ -0,0 +1,150 @@ +from unittest.mock import MagicMock, patch + +import pytest + +from app.text_to_speech.errors import EmptySynthesisError, TextToSpeechServiceError +from app.text_to_speech.service import GoogleTextToSpeechService + + +def _mock_synthesize_response(audio_content: bytes): + """Create a mock SynthesizeSpeechResponse with the given audio content.""" + response = MagicMock() + response.audio_content = audio_content + return response + + +def _mock_empty_synthesize_response(): + """Create a mock SynthesizeSpeechResponse with no audio content.""" + response = MagicMock() + response.audio_content = b"" + return response + + +@pytest.fixture() +def _set_env_vars(monkeypatch): + """Set required environment variables for GoogleTextToSpeechService.""" + monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "test-project") + + +@pytest.mark.usefixtures("_set_env_vars") +class TestGoogleTextToSpeechService: + """Tests for GoogleTextToSpeechService""" + + @pytest.mark.asyncio + @patch("app.text_to_speech.service.texttospeech.TextToSpeechClient") + async def test_synthesize_returns_audio_bytes_on_success(self, mock_tts_client_class): + """Test that synthesize returns audio bytes on successful API call.""" + # GIVEN a service with a mocked TextToSpeechClient + given_mock_client = MagicMock() + mock_tts_client_class.return_value = given_mock_client + # AND the client returns a response with audio content + given_audio_content = b"fake-mp3-audio-data" + given_response = _mock_synthesize_response(given_audio_content) + given_mock_client.synthesize_speech.return_value = given_response + # AND a valid text input + given_text = "Hello world" + given_language = "en-US" + + service = GoogleTextToSpeechService() + + # WHEN synthesize is called + actual_result = await service.synthesize(text=given_text, language_code=given_language) + + # THEN expect the result to be the audio bytes + assert actual_result == given_audio_content + + @pytest.mark.asyncio + @patch("app.text_to_speech.service.texttospeech.TextToSpeechClient") + async def test_synthesize_raises_empty_synthesis_error_when_no_audio(self, mock_tts_client_class): + """Test that EmptySynthesisError is raised when TTS returns no audio content.""" + # GIVEN a service with a mocked TextToSpeechClient + given_mock_client = MagicMock() + mock_tts_client_class.return_value = given_mock_client + # AND the client returns an empty response + given_mock_client.synthesize_speech.return_value = _mock_empty_synthesize_response() + given_text = "Hello world" + + service = GoogleTextToSpeechService() + + # WHEN synthesize is called + # THEN expect EmptySynthesisError to be raised + with pytest.raises(EmptySynthesisError): + await service.synthesize(text=given_text, language_code="en-US") + + @pytest.mark.asyncio + @patch("app.text_to_speech.service.texttospeech.TextToSpeechClient") + async def test_synthesize_raises_service_error_on_api_failure(self, mock_tts_client_class): + """Test that TextToSpeechServiceError is raised when the API call fails.""" + # GIVEN a service with a mocked TextToSpeechClient + given_mock_client = MagicMock() + mock_tts_client_class.return_value = given_mock_client + # AND the client raises an exception + given_mock_client.synthesize_speech.side_effect = Exception("API unavailable") + given_text = "Hello world" + + service = GoogleTextToSpeechService() + + # WHEN synthesize is called + # THEN expect TextToSpeechServiceError to be raised + with pytest.raises(TextToSpeechServiceError): + await service.synthesize(text=given_text, language_code="en-US") + + def test_constructor_raises_value_error_when_project_not_set(self, monkeypatch): + """Test that initialization fails without GOOGLE_CLOUD_PROJECT.""" + # GIVEN GOOGLE_CLOUD_PROJECT is not set + monkeypatch.delenv("GOOGLE_CLOUD_PROJECT", raising=False) + + # WHEN the service is initialized + # THEN expect a ValueError + with pytest.raises(ValueError, match="GOOGLE_CLOUD_PROJECT"): + GoogleTextToSpeechService() + + @pytest.mark.asyncio + @patch("app.text_to_speech.service.texttospeech.TextToSpeechClient") + async def test_synthesize_uses_correct_voice_for_language(self, mock_tts_client_class): + """Test that the correct Chirp 3 HD voice is selected for a known language.""" + # GIVEN a service with a mocked TextToSpeechClient + given_mock_client = MagicMock() + mock_tts_client_class.return_value = given_mock_client + # AND the client returns a valid response + given_response = _mock_synthesize_response(b"audio-data") + given_mock_client.synthesize_speech.return_value = given_response + # AND a Swahili language code + given_language = "sw-KE" + + service = GoogleTextToSpeechService() + + # WHEN synthesize is called with the given language + await service.synthesize(text="Habari", language_code=given_language) + + # THEN expect the client to have been called with the correct voice name + actual_call_kwargs = given_mock_client.synthesize_speech.call_args + actual_voice_params = actual_call_kwargs.kwargs.get("voice") or actual_call_kwargs[1].get("voice") + assert actual_voice_params.name == "sw-KE-Chirp3-HD-Achernar" + # AND the language code to match the voice prefix + assert actual_voice_params.language_code == "sw-KE" + + @pytest.mark.asyncio + @patch("app.text_to_speech.service.texttospeech.TextToSpeechClient") + async def test_synthesize_falls_back_to_default_voice_for_unknown_language(self, mock_tts_client_class): + """Test that an unknown language falls back to the default voice.""" + # GIVEN a service with a mocked TextToSpeechClient + given_mock_client = MagicMock() + mock_tts_client_class.return_value = given_mock_client + # AND the client returns a valid response + given_response = _mock_synthesize_response(b"audio-data") + given_mock_client.synthesize_speech.return_value = given_response + # AND an unknown language code + given_language = "fr-FR" + + service = GoogleTextToSpeechService() + + # WHEN synthesize is called with the unknown language + await service.synthesize(text="Bonjour", language_code=given_language) + + # THEN expect the client to have been called with the default voice name + actual_call_kwargs = given_mock_client.synthesize_speech.call_args + actual_voice_params = actual_call_kwargs.kwargs.get("voice") or actual_call_kwargs[1].get("voice") + assert actual_voice_params.name == "en-US-Chirp3-HD-Achernar" + # AND the language code to be the default voice prefix + assert actual_voice_params.language_code == "en-US" diff --git a/backend/app/text_to_speech/types.py b/backend/app/text_to_speech/types.py new file mode 100644 index 00000000..e3fbdb96 --- /dev/null +++ b/backend/app/text_to_speech/types.py @@ -0,0 +1,10 @@ +from pydantic import BaseModel, ConfigDict, Field + + +class SynthesizeRequest(BaseModel): + """Request body for text-to-speech synthesis.""" + + model_config = ConfigDict(extra="forbid") + + text: str = Field(description="The text to synthesize into speech") + language: str = Field(default="en-US", description="The BCP-47 language code") diff --git a/backend/conftest.py b/backend/conftest.py index 32753182..008cfd0e 100644 --- a/backend/conftest.py +++ b/backend/conftest.py @@ -29,6 +29,7 @@ cv_storage_bucket="foo-bucket", features={}, enable_cv_upload=True, + enable_speech_to_text=False, language_config=LanguageConfig( default_locale=Locale.EN_US, available_locales=[LocaleDateFormatEntry(locale=Locale.EN_US, date_format="MM/DD/YYYY")] diff --git a/backend/poetry.lock b/backend/poetry.lock index 022982ca..ab6bd92a 100644 --- a/backend/poetry.lock +++ b/backend/poetry.lock @@ -1505,6 +1505,31 @@ grpc-google-iam-v1 = ">=0.12.4,<1.0.0dev" proto-plus = ">=1.22.3,<2.0.0dev" protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" +[[package]] +name = "google-cloud-speech" +version = "2.38.0" +description = "Google Cloud Speech API client library" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "google_cloud_speech-2.38.0-py3-none-any.whl", hash = "sha256:dbccb340a750a409b0e70c48c16c8d7d5d48a87c70cce2add50f3d571f5375a0"}, + {file = "google_cloud_speech-2.38.0.tar.gz", hash = "sha256:1854b51cbb7957273b6ba61f4a6cf49dec8d09ec450991587897e50267eaca51"}, +] + +[package.dependencies] +google-api-core = {version = ">=2.11.0,<3.0.0", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0" +grpcio = [ + {version = ">=1.75.1,<2.0.0", markers = "python_version >= \"3.14\""}, + {version = ">=1.33.2,<2.0.0", markers = "python_version < \"3.14\""}, +] +proto-plus = [ + {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, + {version = ">=1.22.3,<2.0.0", markers = "python_version < \"3.13\""}, +] +protobuf = ">=4.25.8,<8.0.0" + [[package]] name = "google-cloud-storage" version = "2.19.0" @@ -1529,6 +1554,31 @@ requests = ">=2.18.0,<3.0.0dev" protobuf = ["protobuf (<6.0.0dev)"] tracing = ["opentelemetry-api (>=1.1.0)"] +[[package]] +name = "google-cloud-texttospeech" +version = "2.36.0" +description = "Google Cloud Texttospeech API client library" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "google_cloud_texttospeech-2.36.0-py3-none-any.whl", hash = "sha256:03f76162543e9d77ecbab823c1cc3728c42ef40547353bcfdbd9ac0e71cb8121"}, + {file = "google_cloud_texttospeech-2.36.0.tar.gz", hash = "sha256:6c605af7e4774c1bac99fcaaf4538f152b10bba7738a23f42184557f444dc6b7"}, +] + +[package.dependencies] +google-api-core = {version = ">=2.11.0,<3.0.0", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0" +grpcio = [ + {version = ">=1.75.1,<2.0.0", markers = "python_version >= \"3.14\""}, + {version = ">=1.33.2,<2.0.0"}, +] +proto-plus = [ + {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, + {version = ">=1.22.3,<2.0.0"}, +] +protobuf = ">=4.25.8,<8.0.0" + [[package]] name = "google-crc32c" version = "1.5.0" @@ -1616,7 +1666,7 @@ version = "1.26.0" description = "GenAI Python SDK" optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "google_genai-1.26.0-py3-none-any.whl", hash = "sha256:a050de052ee6e68654ba7cdb97028a576ad7108d0ecc9257c69bcc555498e9a2"}, {file = "google_genai-1.26.0.tar.gz", hash = "sha256:d7b019ac98ca07888caa6121a953eb65db20f78370d8ae06aec29fb534534dc8"}, @@ -1692,63 +1742,81 @@ protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4 [[package]] name = "grpcio" -version = "1.64.1" +version = "1.80.0" description = "HTTP/2-based RPC framework" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "grpcio-1.64.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:55697ecec192bc3f2f3cc13a295ab670f51de29884ca9ae6cd6247df55df2502"}, - {file = "grpcio-1.64.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:3b64ae304c175671efdaa7ec9ae2cc36996b681eb63ca39c464958396697daff"}, - {file = "grpcio-1.64.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:bac71b4b28bc9af61efcdc7630b166440bbfbaa80940c9a697271b5e1dabbc61"}, - {file = "grpcio-1.64.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c024ffc22d6dc59000faf8ad781696d81e8e38f4078cb0f2630b4a3cf231a90"}, - {file = "grpcio-1.64.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7cd5c1325f6808b8ae31657d281aadb2a51ac11ab081ae335f4f7fc44c1721d"}, - {file = "grpcio-1.64.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0a2813093ddb27418a4c99f9b1c223fab0b053157176a64cc9db0f4557b69bd9"}, - {file = "grpcio-1.64.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2981c7365a9353f9b5c864595c510c983251b1ab403e05b1ccc70a3d9541a73b"}, - {file = "grpcio-1.64.1-cp310-cp310-win32.whl", hash = "sha256:1262402af5a511c245c3ae918167eca57342c72320dffae5d9b51840c4b2f86d"}, - {file = "grpcio-1.64.1-cp310-cp310-win_amd64.whl", hash = "sha256:19264fc964576ddb065368cae953f8d0514ecc6cb3da8903766d9fb9d4554c33"}, - {file = "grpcio-1.64.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:58b1041e7c870bb30ee41d3090cbd6f0851f30ae4eb68228955d973d3efa2e61"}, - {file = "grpcio-1.64.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:bbc5b1d78a7822b0a84c6f8917faa986c1a744e65d762ef6d8be9d75677af2ca"}, - {file = "grpcio-1.64.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:5841dd1f284bd1b3d8a6eca3a7f062b06f1eec09b184397e1d1d43447e89a7ae"}, - {file = "grpcio-1.64.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8caee47e970b92b3dd948371230fcceb80d3f2277b3bf7fbd7c0564e7d39068e"}, - {file = "grpcio-1.64.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73819689c169417a4f978e562d24f2def2be75739c4bed1992435d007819da1b"}, - {file = "grpcio-1.64.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6503b64c8b2dfad299749cad1b595c650c91e5b2c8a1b775380fcf8d2cbba1e9"}, - {file = "grpcio-1.64.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1de403fc1305fd96cfa75e83be3dee8538f2413a6b1685b8452301c7ba33c294"}, - {file = "grpcio-1.64.1-cp311-cp311-win32.whl", hash = "sha256:d4d29cc612e1332237877dfa7fe687157973aab1d63bd0f84cf06692f04c0367"}, - {file = "grpcio-1.64.1-cp311-cp311-win_amd64.whl", hash = "sha256:5e56462b05a6f860b72f0fa50dca06d5b26543a4e88d0396259a07dc30f4e5aa"}, - {file = "grpcio-1.64.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:4657d24c8063e6095f850b68f2d1ba3b39f2b287a38242dcabc166453e950c59"}, - {file = "grpcio-1.64.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:62b4e6eb7bf901719fce0ca83e3ed474ae5022bb3827b0a501e056458c51c0a1"}, - {file = "grpcio-1.64.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:ee73a2f5ca4ba44fa33b4d7d2c71e2c8a9e9f78d53f6507ad68e7d2ad5f64a22"}, - {file = "grpcio-1.64.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:198908f9b22e2672a998870355e226a725aeab327ac4e6ff3a1399792ece4762"}, - {file = "grpcio-1.64.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39b9d0acaa8d835a6566c640f48b50054f422d03e77e49716d4c4e8e279665a1"}, - {file = "grpcio-1.64.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:5e42634a989c3aa6049f132266faf6b949ec2a6f7d302dbb5c15395b77d757eb"}, - {file = "grpcio-1.64.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b1a82e0b9b3022799c336e1fc0f6210adc019ae84efb7321d668129d28ee1efb"}, - {file = "grpcio-1.64.1-cp312-cp312-win32.whl", hash = "sha256:55260032b95c49bee69a423c2f5365baa9369d2f7d233e933564d8a47b893027"}, - {file = "grpcio-1.64.1-cp312-cp312-win_amd64.whl", hash = "sha256:c1a786ac592b47573a5bb7e35665c08064a5d77ab88a076eec11f8ae86b3e3f6"}, - {file = "grpcio-1.64.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:a011ac6c03cfe162ff2b727bcb530567826cec85eb8d4ad2bfb4bd023287a52d"}, - {file = "grpcio-1.64.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:4d6dab6124225496010bd22690f2d9bd35c7cbb267b3f14e7a3eb05c911325d4"}, - {file = "grpcio-1.64.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:a5e771d0252e871ce194d0fdcafd13971f1aae0ddacc5f25615030d5df55c3a2"}, - {file = "grpcio-1.64.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2c3c1b90ab93fed424e454e93c0ed0b9d552bdf1b0929712b094f5ecfe7a23ad"}, - {file = "grpcio-1.64.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20405cb8b13fd779135df23fabadc53b86522d0f1cba8cca0e87968587f50650"}, - {file = "grpcio-1.64.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0cc79c982ccb2feec8aad0e8fb0d168bcbca85bc77b080d0d3c5f2f15c24ea8f"}, - {file = "grpcio-1.64.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a3a035c37ce7565b8f4f35ff683a4db34d24e53dc487e47438e434eb3f701b2a"}, - {file = "grpcio-1.64.1-cp38-cp38-win32.whl", hash = "sha256:1257b76748612aca0f89beec7fa0615727fd6f2a1ad580a9638816a4b2eb18fd"}, - {file = "grpcio-1.64.1-cp38-cp38-win_amd64.whl", hash = "sha256:0a12ddb1678ebc6a84ec6b0487feac020ee2b1659cbe69b80f06dbffdb249122"}, - {file = "grpcio-1.64.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:75dbbf415026d2862192fe1b28d71f209e2fd87079d98470db90bebe57b33179"}, - {file = "grpcio-1.64.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e3d9f8d1221baa0ced7ec7322a981e28deb23749c76eeeb3d33e18b72935ab62"}, - {file = "grpcio-1.64.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:5f8b75f64d5d324c565b263c67dbe4f0af595635bbdd93bb1a88189fc62ed2e5"}, - {file = "grpcio-1.64.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c84ad903d0d94311a2b7eea608da163dace97c5fe9412ea311e72c3684925602"}, - {file = "grpcio-1.64.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:940e3ec884520155f68a3b712d045e077d61c520a195d1a5932c531f11883489"}, - {file = "grpcio-1.64.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f10193c69fc9d3d726e83bbf0f3d316f1847c3071c8c93d8090cf5f326b14309"}, - {file = "grpcio-1.64.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ac15b6c2c80a4d1338b04d42a02d376a53395ddf0ec9ab157cbaf44191f3ffdd"}, - {file = "grpcio-1.64.1-cp39-cp39-win32.whl", hash = "sha256:03b43d0ccf99c557ec671c7dede64f023c7da9bb632ac65dbc57f166e4970040"}, - {file = "grpcio-1.64.1-cp39-cp39-win_amd64.whl", hash = "sha256:ed6091fa0adcc7e4ff944090cf203a52da35c37a130efa564ded02b7aff63bcd"}, - {file = "grpcio-1.64.1.tar.gz", hash = "sha256:8d51dd1c59d5fa0f34266b80a3805ec29a1f26425c2a54736133f6d87fc4968a"}, + {file = "grpcio-1.80.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:886457a7768e408cdce226ad1ca67d2958917d306523a0e21e1a2fdaa75c9c9c"}, + {file = "grpcio-1.80.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:7b641fc3f1dc647bfd80bd713addc68f6d145956f64677e56d9ebafc0bd72388"}, + {file = "grpcio-1.80.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:33eb763f18f006dc7fee1e69831d38d23f5eccd15b2e0f92a13ee1d9242e5e02"}, + {file = "grpcio-1.80.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:52d143637e3872633fc7dd7c3c6a1c84e396b359f3a72e215f8bf69fd82084fc"}, + {file = "grpcio-1.80.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c51bf8ac4575af2e0678bccfb07e47321fc7acb5049b4482832c5c195e04e13a"}, + {file = "grpcio-1.80.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:50a9871536d71c4fba24ee856abc03a87764570f0c457dd8db0b4018f379fed9"}, + {file = "grpcio-1.80.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:a72d84ad0514db063e21887fbacd1fd7acb4d494a564cae22227cd45c7fbf199"}, + {file = "grpcio-1.80.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f7691a6788ad9196872f95716df5bc643ebba13c97140b7a5ee5c8e75d1dea81"}, + {file = "grpcio-1.80.0-cp310-cp310-win32.whl", hash = "sha256:46c2390b59d67f84e882694d489f5b45707c657832d7934859ceb8c33f467069"}, + {file = "grpcio-1.80.0-cp310-cp310-win_amd64.whl", hash = "sha256:dc053420fc75749c961e2a4c906398d7c15725d36ccc04ae6d16093167223b58"}, + {file = "grpcio-1.80.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:dfab85db094068ff42e2a3563f60ab3dddcc9d6488a35abf0132daec13209c8a"}, + {file = "grpcio-1.80.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:5c07e82e822e1161354e32da2662f741a4944ea955f9f580ec8fb409dd6f6060"}, + {file = "grpcio-1.80.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba0915d51fd4ced2db5ff719f84e270afe0e2d4c45a7bdb1e8d036e4502928c2"}, + {file = "grpcio-1.80.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:3cb8130ba457d2aa09fa6b7c3ed6b6e4e6a2685fce63cb803d479576c4d80e21"}, + {file = "grpcio-1.80.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:09e5e478b3d14afd23f12e49e8b44c8684ac3c5f08561c43a5b9691c54d136ab"}, + {file = "grpcio-1.80.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:00168469238b022500e486c1c33916acf2f2a9b2c022202cf8a1885d2e3073c1"}, + {file = "grpcio-1.80.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8502122a3cc1714038e39a0b071acb1207ca7844208d5ea0d091317555ee7106"}, + {file = "grpcio-1.80.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ce1794f4ea6cc3ca29463f42d665c32ba1b964b48958a66497917fe9069f26e6"}, + {file = "grpcio-1.80.0-cp311-cp311-win32.whl", hash = "sha256:51b4a7189b0bef2aa30adce3c78f09c83526cf3dddb24c6a96555e3b97340440"}, + {file = "grpcio-1.80.0-cp311-cp311-win_amd64.whl", hash = "sha256:02e64bb0bb2da14d947a49e6f120a75e947250aebe65f9629b62bb1f5c14e6e9"}, + {file = "grpcio-1.80.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:c624cc9f1008361014378c9d776de7182b11fe8b2e5a81bc69f23a295f2a1ad0"}, + {file = "grpcio-1.80.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:f49eddcac43c3bf350c0385366a58f36bed8cc2c0ec35ef7b74b49e56552c0c2"}, + {file = "grpcio-1.80.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d334591df610ab94714048e0d5b4f3dd5ad1bee74dfec11eee344220077a79de"}, + {file = "grpcio-1.80.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:0cb517eb1d0d0aaf1d87af7cc5b801d686557c1d88b2619f5e31fab3c2315921"}, + {file = "grpcio-1.80.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4e78c4ac0d97dc2e569b2f4bcbbb447491167cb358d1a389fc4af71ab6f70411"}, + {file = "grpcio-1.80.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2ed770b4c06984f3b47eb0517b1c69ad0b84ef3f40128f51448433be904634cd"}, + {file = "grpcio-1.80.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:256507e2f524092f1473071a05e65a5b10d84b82e3ff24c5b571513cfaa61e2f"}, + {file = "grpcio-1.80.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9a6284a5d907c37db53350645567c522be314bac859a64a7a5ca63b77bb7958f"}, + {file = "grpcio-1.80.0-cp312-cp312-win32.whl", hash = "sha256:c71309cfce2f22be26aa4a847357c502db6c621f1a49825ae98aa0907595b193"}, + {file = "grpcio-1.80.0-cp312-cp312-win_amd64.whl", hash = "sha256:9fe648599c0e37594c4809d81a9e77bd138cc82eb8baa71b6a86af65426723ff"}, + {file = "grpcio-1.80.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:e9e408fc016dffd20661f0126c53d8a31c2821b5c13c5d67a0f5ed5de93319ad"}, + {file = "grpcio-1.80.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:92d787312e613754d4d8b9ca6d3297e69994a7912a32fa38c4c4e01c272974b0"}, + {file = "grpcio-1.80.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8ac393b58aa16991a2f1144ec578084d544038c12242da3a215966b512904d0f"}, + {file = "grpcio-1.80.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:68e5851ac4b9afe07e7f84483803ad167852570d65326b34d54ca560bfa53fb6"}, + {file = "grpcio-1.80.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:873ff5d17d68992ef6605330127425d2fc4e77e612fa3c3e0ed4e668685e3140"}, + {file = "grpcio-1.80.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2bea16af2750fd0a899bf1abd9022244418b55d1f37da2202249ba4ba673838d"}, + {file = "grpcio-1.80.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba0db34f7e1d803a878284cd70e4c63cb6ae2510ba51937bf8f45ba997cefcf7"}, + {file = "grpcio-1.80.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8eb613f02d34721f1acf3626dfdb3545bd3c8505b0e52bf8b5710a28d02e8aa7"}, + {file = "grpcio-1.80.0-cp313-cp313-win32.whl", hash = "sha256:93b6f823810720912fd131f561f91f5fed0fda372b6b7028a2681b8194d5d294"}, + {file = "grpcio-1.80.0-cp313-cp313-win_amd64.whl", hash = "sha256:e172cf795a3ba5246d3529e4d34c53db70e888fa582a8ffebd2e6e48bc0cba50"}, + {file = "grpcio-1.80.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:3d4147a97c8344d065d01bbf8b6acec2cf86fb0400d40696c8bdad34a64ffc0e"}, + {file = "grpcio-1.80.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d8e11f167935b3eb089ac9038e1a063e6d7dbe995c0bb4a661e614583352e76f"}, + {file = "grpcio-1.80.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f14b618fc30de822681ee986cfdcc2d9327229dc4c98aed16896761cacd468b9"}, + {file = "grpcio-1.80.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4ed39fbdcf9b87370f6e8df4e39ca7b38b3e5e9d1b0013c7b6be9639d6578d14"}, + {file = "grpcio-1.80.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2dcc70e9f0ba987526e8e8603a610fb4f460e42899e74e7a518bf3c68fe1bf05"}, + {file = "grpcio-1.80.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:448c884b668b868562b1bda833c5fce6272d26e1926ec46747cda05741d302c1"}, + {file = "grpcio-1.80.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a1dc80fe55685b4a543555e6eef975303b36c8db1023b1599b094b92aa77965f"}, + {file = "grpcio-1.80.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:31b9ac4ad1aa28ffee5503821fafd09e4da0a261ce1c1281c6c8da0423c83b6e"}, + {file = "grpcio-1.80.0-cp314-cp314-win32.whl", hash = "sha256:367ce30ba67d05e0592470428f0ec1c31714cab9ef19b8f2e37be1f4c7d32fae"}, + {file = "grpcio-1.80.0-cp314-cp314-win_amd64.whl", hash = "sha256:3b01e1f5464c583d2f567b2e46ff0d516ef979978f72091fd81f5ab7fa6e2e7f"}, + {file = "grpcio-1.80.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:aacdfb4ed3eb919ca997504d27e03d5dba403c85130b8ed450308590a738f7a4"}, + {file = "grpcio-1.80.0-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:a361c20ec1ccd3c3953d20fb6d7b4125093bdd10dff44c5e2bbb39e58917cedc"}, + {file = "grpcio-1.80.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:43168871f170d1e4ed16ae03d10cd21efa29f190e710a624cee7e5ae07da6f4f"}, + {file = "grpcio-1.80.0-cp39-cp39-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:1b97cd29a8eda100b559b455331c487a80915b6ea6bd91cf3e89836c4ee8d957"}, + {file = "grpcio-1.80.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bac1d573dfa84ce59a5547073e28fa7326d53352adda6912e362da0b917fcef4"}, + {file = "grpcio-1.80.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4560cf0e86514595dbbd330cd65b7afad4b5c4b8c4905c041cfffa138d45e6fd"}, + {file = "grpcio-1.80.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:ec0a592e926071b4abad50c1495cd0d0d513324b3ff5e7267067c33ba27506e4"}, + {file = "grpcio-1.80.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:deb10a1528473c11f72a0939eed36d83e847d7cbb63e8cc5611fb7a912d38614"}, + {file = "grpcio-1.80.0-cp39-cp39-win32.whl", hash = "sha256:627fb7312171cdc52828bd6fac8d7028ff2a64b89f1957b6f3416caa2218d141"}, + {file = "grpcio-1.80.0-cp39-cp39-win_amd64.whl", hash = "sha256:05d55e1798756282cddd52d56c896b3e7d673e3a8798c2f1cd05ba249a3bb4de"}, + {file = "grpcio-1.80.0.tar.gz", hash = "sha256:29aca15edd0688c22ba01d7cc01cb000d72b2033f4a3c72a81a19b56fd143257"}, ] markers = {dev = "platform_python_implementation != \"PyPy\""} +[package.dependencies] +typing-extensions = ">=4.12,<5.0" + [package.extras] -protobuf = ["grpcio-tools (>=1.64.1)"] +protobuf = ["grpcio-tools (>=1.80.0)"] [[package]] name = "grpcio-status" @@ -2250,8 +2318,8 @@ files = [ [package.dependencies] click = ">=8.1.7" numpy = [ - {version = ">=1.26", markers = "python_version == \"3.12\""}, {version = ">=2.1.0", markers = "python_version >= \"3.13\""}, + {version = ">=1.26", markers = "python_version == \"3.12\""}, {version = ">=1.24", markers = "python_version < \"3.12\""}, ] onnxruntime = {version = ">=1.17.0", markers = "python_version > \"3.9\""} @@ -3182,41 +3250,41 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "proto-plus" -version = "1.23.0" -description = "Beautiful, Pythonic protocol buffers." +version = "1.27.2" +description = "Beautiful, Pythonic protocol buffers" optional = false -python-versions = ">=3.6" +python-versions = ">=3.9" groups = ["main", "dev"] files = [ - {file = "proto-plus-1.23.0.tar.gz", hash = "sha256:89075171ef11988b3fa157f5dbd8b9cf09d65fffee97e29ce403cd8defba19d2"}, - {file = "proto_plus-1.23.0-py3-none-any.whl", hash = "sha256:a829c79e619e1cf632de091013a4173deed13a55f326ef84f05af6f50ff4c82c"}, + {file = "proto_plus-1.27.2-py3-none-any.whl", hash = "sha256:6432f75893d3b9e70b9c412f1d2f03f65b11fb164b793d14ae2ca01821d22718"}, + {file = "proto_plus-1.27.2.tar.gz", hash = "sha256:b2adde53adadf75737c44d3dcb0104fde65250dfc83ad59168b4aa3e574b6a24"}, ] [package.dependencies] -protobuf = ">=3.19.0,<5.0.0dev" +protobuf = ">=4.25.8,<8.0.0" [package.extras] -testing = ["google-api-core[grpc] (>=1.31.5)"] +testing = ["google-api-core (>=1.31.5)"] [[package]] name = "protobuf" -version = "4.25.3" +version = "4.25.9" description = "" optional = false python-versions = ">=3.8" groups = ["main", "dev"] files = [ - {file = "protobuf-4.25.3-cp310-abi3-win32.whl", hash = "sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa"}, - {file = "protobuf-4.25.3-cp310-abi3-win_amd64.whl", hash = "sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8"}, - {file = "protobuf-4.25.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c"}, - {file = "protobuf-4.25.3-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019"}, - {file = "protobuf-4.25.3-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d"}, - {file = "protobuf-4.25.3-cp38-cp38-win32.whl", hash = "sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2"}, - {file = "protobuf-4.25.3-cp38-cp38-win_amd64.whl", hash = "sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4"}, - {file = "protobuf-4.25.3-cp39-cp39-win32.whl", hash = "sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4"}, - {file = "protobuf-4.25.3-cp39-cp39-win_amd64.whl", hash = "sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c"}, - {file = "protobuf-4.25.3-py3-none-any.whl", hash = "sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9"}, - {file = "protobuf-4.25.3.tar.gz", hash = "sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c"}, + {file = "protobuf-4.25.9-cp310-abi3-win32.whl", hash = "sha256:bde396f568b0b46fc8fbfe9f02facf25b6755b2578a3b8ac61e74b9d69499e03"}, + {file = "protobuf-4.25.9-cp310-abi3-win_amd64.whl", hash = "sha256:3683c05154252206f7cb2d371626514b3708199d9bcf683b503dabf3a2e38e06"}, + {file = "protobuf-4.25.9-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:9560813560e6ee72c11ca8873878bdb7ee003c96a57ebb013245fe84e2540904"}, + {file = "protobuf-4.25.9-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:999146ef02e7fa6a692477badd1528bcd7268df211852a3df2d834ba2b480791"}, + {file = "protobuf-4.25.9-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:438c636de8fb706a0de94a12a268ef1ae8f5ba5ae655a7671fcda5968ba3c9be"}, + {file = "protobuf-4.25.9-cp38-cp38-win32.whl", hash = "sha256:7f7c1abcea3fc215918fba67a2d2a80fbcccc0f84159610eb187e9bbe6f939ee"}, + {file = "protobuf-4.25.9-cp38-cp38-win_amd64.whl", hash = "sha256:79faf4e5a80b231d94dcf3a0a2917ccbacf0f586f12c9b9c91794b41b913a853"}, + {file = "protobuf-4.25.9-cp39-cp39-win32.whl", hash = "sha256:9481e80e8cffb1c492c68e7c4e6726f4ad02eebc4fa97ead7beebeaa3639511d"}, + {file = "protobuf-4.25.9-cp39-cp39-win_amd64.whl", hash = "sha256:b1d467352de666dc1b6d5740b6319d9c08cab7b21b452501e4ee5b0ac5156780"}, + {file = "protobuf-4.25.9-py3-none-any.whl", hash = "sha256:d49b615e7c935194ac161f0965699ac84df6112c378e05ec53da65d2e4cbb6d4"}, + {file = "protobuf-4.25.9.tar.gz", hash = "sha256:b0dc7e7c68de8b1ce831dacb12fb407e838edbb8b6cc0dc3a2a6b4cbf6de9cff"}, ] [[package]] @@ -4301,7 +4369,7 @@ version = "8.5.0" description = "Retry code until it succeeds" optional = false python-versions = ">=3.8" -groups = ["dev"] +groups = ["main", "dev"] files = [ {file = "tenacity-8.5.0-py3-none-any.whl", hash = "sha256:b594c2a5945830c267ce6b79a166228323ed52718f30302c1359836112346687"}, {file = "tenacity-8.5.0.tar.gz", hash = "sha256:8bc6c0c8a09b31e6cad13c47afbed1a567518250a9a171418582ed8d9c20ca78"}, @@ -5048,4 +5116,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "0b4409c9c9f0485fc90a0ecfc35812a95d7023c605da05b5b899668d9e5406e0" +content-hash = "b140ce19e4c1b3ee39f1754489c06d9891e4254281f1a5d3ff70ad8715ecbfbe" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 566b307f..55f894de 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -14,6 +14,8 @@ uvicorn = ">=0.23.2,<0.24.0" python-dotenv = "^1.0.1" deprecated="^1.2.14" google-cloud-dlp="^3.16.0" +google-cloud-speech="^2.27.0" +google-cloud-texttospeech="^2.21.0" fix-busted-json=">=0.0.17,<1.0.0" json-repair=">=0.44.1,<1.0.0" motor = "^3.4.0" diff --git a/config/default.json b/config/default.json index 7ea7ecc7..c632ced5 100644 --- a/config/default.json +++ b/config/default.json @@ -104,6 +104,12 @@ "gtmContainerNumericId": "246009778", "enabled": true }, + "speechToText": { + "enabled": false + }, + "textToSpeech": { + "enabled": false + }, "sensitiveData": { "fields": { "name": { diff --git a/config/inject-config.py b/config/inject-config.py index 25823d44..28ba0993 100644 --- a/config/inject-config.py +++ b/config/inject-config.py @@ -22,6 +22,12 @@ # CV Upload "GLOBAL_ENABLE_CV_UPLOAD": "cv.enabled", + # Speech-to-Text + "GLOBAL_ENABLE_SPEECH_TO_TEXT": "speechToText.enabled", + + # Text-to-Speech + "GLOBAL_ENABLE_TEXT_TO_SPEECH": "textToSpeech.enabled", + # Add more backend env vars here } @@ -47,6 +53,12 @@ # CV Upload "GLOBAL_ENABLE_CV_UPLOAD": "cv.enabled", + # Speech-to-Text + "GLOBAL_ENABLE_SPEECH_TO_TEXT": "speechToText.enabled", + + # Text-to-Speech + "GLOBAL_ENABLE_TEXT_TO_SPEECH": "textToSpeech.enabled", + # i18n "FRONTEND_DEFAULT_LOCALE": "i18n.ui.defaultLocale", "FRONTEND_SUPPORTED_LOCALES": "i18n.ui.supportedLocales", diff --git a/config/njira.json b/config/njira.json index e66b0ef7..8d90eb2a 100644 --- a/config/njira.json +++ b/config/njira.json @@ -84,6 +84,12 @@ ] } }, + "speechToText": { + "enabled": true + }, + "textToSpeech": { + "enabled": true + }, "modules": { "enabled": true, "list": [ diff --git a/frontend-new/src/_test_utilities/envServiceMock.ts b/frontend-new/src/_test_utilities/envServiceMock.ts index fea807ed..522fc342 100644 --- a/frontend-new/src/_test_utilities/envServiceMock.ts +++ b/frontend-new/src/_test_utilities/envServiceMock.ts @@ -9,6 +9,8 @@ jest.mock("src/envService", () => ({ getMetricsEnabled: jest.fn(() => "true"), getMetricsConfig: jest.fn(() => ""), getCvUploadEnabled: jest.fn(() => "true"), + getSpeechToTextEnabled: jest.fn(() => "false"), + getTextToSpeechEnabled: jest.fn(() => "false"), getNewSessionEnabled: jest.fn(() => false), getSocialAuthDisabled: jest.fn(() => "false"), getSupportedLocales: jest.fn(() => JSON.stringify(["en-US"])), diff --git a/frontend-new/src/chat/ChatMessageField/ChatMessageField.tsx b/frontend-new/src/chat/ChatMessageField/ChatMessageField.tsx index ed568181..1391a030 100644 --- a/frontend-new/src/chat/ChatMessageField/ChatMessageField.tsx +++ b/frontend-new/src/chat/ChatMessageField/ChatMessageField.tsx @@ -1,7 +1,18 @@ import React, { KeyboardEvent, MouseEvent, useCallback, useContext, useEffect, useMemo, useState } from "react"; -import { Box, IconButton, InputAdornment, styled, TextField, Typography, useTheme } from "@mui/material"; +import { + Box, + CircularProgress, + IconButton, + InputAdornment, + styled, + TextField, + Typography, + useTheme, +} from "@mui/material"; import AddIcon from "@mui/icons-material/Add"; import ArrowUpwardSharpIcon from "@mui/icons-material/ArrowUpwardSharp"; +import MicIcon from "@mui/icons-material/Mic"; +import StopIcon from "@mui/icons-material/Stop"; import UploadFileIcon from "@mui/icons-material/UploadFile"; import { AnimatePresence, motion } from "framer-motion"; import { CV_UPLOAD_ERROR_I18N_KEYS, getCvUploadErrorMessageFromHttpStatus } from "../CVUploadErrorHandling"; @@ -10,7 +21,8 @@ import { MenuItemConfig } from "src/theme/ContextMenu/menuItemConfig.types"; import { IsOnlineContext } from "src/app/isOnlineProvider/IsOnlineProvider"; import { ConversationPhase } from "src/chat/chatProgressbar/types"; import AnimatedDotBadge from "src/theme/AnimatedDotBadge/AnimatedDotBadge"; -import { getCvUploadEnabled } from "src/envService"; +import { getCvUploadEnabled, getSpeechToTextEnabled } from "src/envService"; +import { useSpeechToText } from "src/speechToText/useSpeechToText"; import CVService from "src/CV/CVService/CVService"; import { CVListItem } from "src/CV/CVService/CVService.types"; import authenticationStateService from "src/auth/services/AuthenticationState.service"; @@ -52,6 +64,9 @@ export const DATA_TEST_ID = { CHAT_MESSAGE_FIELD_PLUS_BUTTON: `chat-message-field-plus-button-${uniqueId}`, CHAT_MESSAGE_FIELD_PLUS_ICON: `chat-message-field-plus-icon-${uniqueId}`, CHAT_MESSAGE_FIELD_HIDDEN_FILE_INPUT: `chat-message-field-hidden-file-input-${uniqueId}`, + CHAT_MESSAGE_FIELD_MIC_BUTTON: `chat-message-field-mic-button-${uniqueId}`, + CHAT_MESSAGE_FIELD_MIC_ICON: `chat-message-field-mic-icon-${uniqueId}`, + CHAT_MESSAGE_FIELD_STOP_ICON: `chat-message-field-stop-icon-${uniqueId}`, }; export const MENU_ITEM_ID = { @@ -73,6 +88,8 @@ export const PLACEHOLDER_TEXTS = { OFFLINE: "chat.chatMessageField.placeholders.offline", DEFAULT: "chat.chatMessageField.placeholders.default", UPLOADING: "chat.chatMessageField.placeholders.uploading", + TRANSCRIBING: "chat.chatMessageField.placeholders.transcribing", + RECORDING: "chat.chatMessageField.placeholders.recording", } as const; export const CHARACTER_LIMIT_ERROR_MESSAGES = { @@ -142,6 +159,35 @@ const ChatMessageField: React.FC = (props) => { const isCvUploadEnabled = getCvUploadEnabled().toLowerCase() === "true"; const showCvUpload = props.showCvUpload !== false; + const isSttEnabled = getSpeechToTextEnabled().toLowerCase() === "true"; + + const handleTranscriptionComplete = useCallback( + (text: string) => { + // Filter disallowed characters from transcription (same filter as typed input) + const filteredText = text.replace(DISALLOWED_CHARACTERS, ""); + if (!filteredText.trim()) return; + // Append to existing message or set as new message + const newMessage = message.trim().length > 0 ? `${message} ${filteredText}` : filteredText; + setMessage(newMessage); + if (newMessage.trim().length > CHAT_MESSAGE_MAX_LENGTH) { + setErrorMessage(t(CHARACTER_LIMIT_ERROR_MESSAGES.MESSAGE_LIMIT, { max: CHAT_MESSAGE_MAX_LENGTH })); + } else { + setErrorMessage(""); + } + }, + [message, t] + ); + + const { + status: sttStatus, + interimText, + error: sttError, + startRecording, + stopRecording, + isSupported: isSttSupported, + } = useSpeechToText({ + onTranscriptionComplete: handleTranscriptionComplete, + }); // Show the dot badge whenever in COLLECT_EXPERIENCES and not yet seen useEffect(() => { @@ -436,7 +482,20 @@ const ChatMessageField: React.FC = (props) => { return theme.palette.text.secondary; }, [message, theme.palette.error.main, theme.palette.warning.main, theme.palette.text.secondary]); + // Show STT errors in the error message area + useEffect(() => { + if (sttError) { + setErrorMessage(sttError); + } + }, [sttError]); + const placeHolder = useMemo(() => { + if (sttStatus === "transcribing") { + return t(PLACEHOLDER_TEXTS.TRANSCRIBING); + } + if (sttStatus === "recording" && !interimText) { + return t(PLACEHOLDER_TEXTS.RECORDING); + } if (props.isChatFinished) { return t(PLACEHOLDER_TEXTS.CHAT_FINISHED); } @@ -457,6 +516,8 @@ const ChatMessageField: React.FC = (props) => { } return t(PLACEHOLDER_TEXTS.DEFAULT); }, [ + sttStatus, + interimText, props.aiIsTyping, props.isChatFinished, props.isUploadingCv, @@ -474,15 +535,32 @@ const ChatMessageField: React.FC = (props) => { props.isInputDisabled || props.isUploadingCv || !isOnline || + sttStatus === "recording" || + sttStatus === "transcribing" || message.trim().length === 0 || message.trim().length > CHAT_MESSAGE_MAX_LENGTH // Only disable the send button when over the limit ); - }, [props.isChatFinished, props.aiIsTyping, props.isInputDisabled, props.isUploadingCv, isOnline, message]); + }, [ + props.isChatFinished, + props.aiIsTyping, + props.isInputDisabled, + props.isUploadingCv, + isOnline, + message, + sttStatus, + ]); // Check if the input field should be disabled const inputIsDisabled = useCallback(() => { - return props.isChatFinished || props.aiIsTyping || props.isInputDisabled || props.isUploadingCv || !isOnline; - }, [props.isChatFinished, props.aiIsTyping, props.isInputDisabled, props.isUploadingCv, isOnline]); + return ( + props.isChatFinished || + props.aiIsTyping || + props.isInputDisabled || + props.isUploadingCv || + !isOnline || + sttStatus === "transcribing" + ); + }, [props.isChatFinished, props.aiIsTyping, props.isInputDisabled, props.isUploadingCv, isOnline, sttStatus]); const contextMenuItems: MenuItemConfig[] = menuView === "main" @@ -557,7 +635,7 @@ const ChatMessageField: React.FC = (props) => { fullWidth multiline maxRows={maxRows} - value={message} + value={sttStatus === "recording" ? (message ? `${message} ${interimText}` : interimText) : message} disabled={inputIsDisabled()} onChange={handleChange} onKeyDown={handleKeyDown} @@ -566,6 +644,7 @@ const ChatMessageField: React.FC = (props) => { helperText={errorMessage} fillColor={props.fillColor} InputProps={{ + readOnly: sttStatus === "recording", startAdornment: ( @@ -597,7 +676,41 @@ const ChatMessageField: React.FC = (props) => { ), endAdornment: ( - + + {isSttEnabled && isSttSupported && ( + event.stopPropagation()} + title={ + sttStatus === "recording" + ? t("chat.chatMessageField.stopRecordingTooltip") + : t("chat.chatMessageField.startRecordingTooltip") + } + sx={{ + backgroundColor: sttStatus === "recording" ? theme.palette.error.main : "transparent", + "&:hover": { + backgroundColor: + sttStatus === "recording" ? theme.palette.error.dark : theme.palette.action.hover, + }, + }} + > + {sttStatus === "transcribing" ? ( + + ) : sttStatus === "recording" ? ( + + ) : ( + + )} + + )}