feat(CORE-236): add voice input (STT) and read-aloud (TTS)

backend/.env.example

@@ -63,6 +63,11 @@ BACKEND_CV_MAX_UPLOADS_PER_USER=<INTEGER>
 BACKEND_CV_RATE_LIMIT_PER_MINUTE=<INTEGER>
 
 
+# Speech-to-text (voice input) feature
+GLOBAL_ENABLE_SPEECH_TO_TEXT=false
+# Required when speech-to-text is enabled: the GCP project ID
+GOOGLE_CLOUD_PROJECT=<GCP_PROJECT_ID>
+
 # Locales
 BACKEND_LANGUAGE_CONFIG='{"default_locale":"en-US","available_locales":[{"locale":"en-US","date_format":"MM/DD/YYYY"}]}'
 

backend/app/app_config.py

@@ -73,6 +73,18 @@ class ApplicationConfig(BaseModel):
     When disabled, CV upload routes will not be registered and CV indexes will not be created.
     """
 
+    enable_speech_to_text: bool = False
+    """
+    A flag to enable or disable the Speech-to-Text (voice input) feature.
+    When disabled, speech-to-text routes will not be registered.
+    """
+
+    enable_text_to_speech: bool = False
+    """
+    A flag to enable or disable the Text-to-Speech (read-aloud) feature.
+    When disabled, text-to-speech routes will not be registered.
+    """
+
     # CV storage and upload limits
     cv_storage_bucket: Optional[str] = ""
     cv_max_uploads_per_user: Optional[int] = Field(default=DEFAULT_MAX_UPLOADS_PER_USER, gt=0)

backend/app/server.py

@@ -21,6 +21,8 @@
 from app.metrics.routes.routes import add_metrics_routes
 from app.analytics.routes import add_analytics_routes
 from app.teveta.routes import add_teveta_routes
+from app.speech_to_text.routes import add_speech_to_text_routes
+from app.text_to_speech.routes import add_text_to_speech_routes
 from app.sentry_init import init_sentry, set_sentry_contexts
 from app.server_dependencies.db_dependencies import CompassDBProvider
 from app.users.auth import Authentication, ApiKeyAuth
@@ -226,6 +228,16 @@ def setup_sentry():
 _enable_cv_upload = _enable_cv_upload_str.lower() == "true"
 logger.info(f"GLOBAL_ENABLE_CV_UPLOAD: {_enable_cv_upload}")
 
+# Speech-to-text feature flag - defaults to False if not set
+_enable_speech_to_text_str = os.getenv("GLOBAL_ENABLE_SPEECH_TO_TEXT", "false")
+_enable_speech_to_text = _enable_speech_to_text_str.lower() == "true"
+logger.info(f"GLOBAL_ENABLE_SPEECH_TO_TEXT: {_enable_speech_to_text}")
+
+# Text-to-speech feature flag - defaults to False if not set
+_enable_text_to_speech_str = os.getenv("GLOBAL_ENABLE_TEXT_TO_SPEECH", "false")
+_enable_text_to_speech = _enable_text_to_speech_str.lower() == "true"
+logger.info(f"GLOBAL_ENABLE_TEXT_TO_SPEECH: {_enable_text_to_speech}")
+
 application_config = ApplicationConfig(
     environment_name=os.getenv("TARGET_ENVIRONMENT_NAME"),
     version_info=load_version_info(),
@@ -237,6 +249,8 @@ def setup_sentry():
     features=backend_features_config,
     experience_pipeline_config=experience_pipeline_config,
     enable_cv_upload=_enable_cv_upload,
+    enable_speech_to_text=_enable_speech_to_text,
+    enable_text_to_speech=_enable_text_to_speech,
     cv_storage_bucket=os.getenv("BACKEND_CV_STORAGE_BUCKET", ""),
     cv_max_uploads_per_user=os.getenv("BACKEND_CV_MAX_UPLOADS_PER_USER") or DEFAULT_MAX_UPLOADS_PER_USER,
     cv_rate_limit_per_minute=os.getenv("BACKEND_CV_RATE_LIMIT_PER_MINUTE") or DEFAULT_RATE_LIMIT_PER_MINUTE,
@@ -473,6 +487,24 @@ async def lifespan(_app: FastAPI):
 ############################################
 add_poc_routes(app, auth)
 
+############################################
+# Add speech-to-text routes (conditionally)
+############################################
+if _enable_speech_to_text:
+    add_speech_to_text_routes(app, auth)
+    logger.info("Speech-to-text routes registered")
+else:
+    logger.info("Speech-to-text routes skipped (GLOBAL_ENABLE_SPEECH_TO_TEXT is not enabled)")
+
+############################################
+# Add text-to-speech routes (conditionally)
+############################################
+if _enable_text_to_speech:
+    add_text_to_speech_routes(app, auth)
+    logger.info("Text-to-speech routes registered")
+else:
+    logger.info("Text-to-speech routes skipped (GLOBAL_ENABLE_TEXT_TO_SPEECH is not enabled)")
+
 ############################################
 # Add other features routes
 ############################################

backend/app/speech_to_text/constants.py

@@ -0,0 +1,23 @@
+# Maximum audio file size: 10 MB
+MAX_AUDIO_SIZE_BYTES = 10 * 1024 * 1024
+
+# Maximum multipart overhead for early Content-Length rejection
+MAX_MULTIPART_OVERHEAD_BYTES = 1024
+
+# Allowed MIME types from browser MediaRecorder and phone recordings
+ALLOWED_AUDIO_MIME_TYPES = {
+    "audio/webm",
+    "audio/webm;codecs=opus",
+    "audio/ogg",
+    "audio/ogg;codecs=opus",
+    "audio/mp4",
+    "audio/mpeg",
+    "audio/wav",
+    "audio/x-m4a",
+    "audio/aac",
+    "video/mp4",
+    "video/webm",
+}
+
+# Default language for transcription
+DEFAULT_LANGUAGE_CODE = "en-US"

backend/app/speech_to_text/errors.py

@@ -0,0 +1,15 @@
+class TranscriptionError(Exception):
+    """Base exception for speech-to-text transcription errors."""
+
+
+class EmptyTranscriptionError(TranscriptionError):
+    """Raised when Google STT returns no transcription results."""
+    def __init__(self):
+        super().__init__("No transcription results returned for the provided audio")
+
+
+class SpeechToTextServiceError(TranscriptionError):
+    """Raised when the Google Cloud STT API call fails."""
+    def __init__(self, detail: str):
+        super().__init__(f"Speech-to-text service error: {detail}")
+        self.detail = detail

backend/app/speech_to_text/routes.py

@@ -0,0 +1,142 @@
+import asyncio
+import logging
+from http import HTTPStatus
+
+from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile
+
+from app.constants.errors import HTTPErrorResponse
+from app.speech_to_text.constants import (
+    ALLOWED_AUDIO_MIME_TYPES,
+    DEFAULT_LANGUAGE_CODE,
+    MAX_AUDIO_SIZE_BYTES,
+    MAX_MULTIPART_OVERHEAD_BYTES,
+)
+from app.speech_to_text.errors import EmptyTranscriptionError, SpeechToTextServiceError
+from app.speech_to_text.service import GoogleSpeechToTextService, ISpeechToTextService
+from app.speech_to_text.types import TranscriptionResponse
+from app.users.auth import Authentication, UserInfo
+
+logger = logging.getLogger(__name__)
+
+_stt_service_lock = asyncio.Lock()
+_stt_service_singleton: ISpeechToTextService | None = None
+
+
+async def _get_speech_to_text_service() -> ISpeechToTextService:
+    global _stt_service_singleton  # pylint: disable=global-statement
+    if _stt_service_singleton is None:
+        async with _stt_service_lock:
+            if _stt_service_singleton is None:
+                _stt_service_singleton = GoogleSpeechToTextService()
+    return _stt_service_singleton
+
+
+# Public alias for dependency override in tests
+get_speech_to_text_service = _get_speech_to_text_service
+
+
+def _validate_request_size_header(request: Request):
+    """Validate Content-Length before reading the body to fail fast on oversized uploads."""
+    content_length_header = request.headers.get("content-length")
+    if content_length_header is None:
+        return
+    try:
+        content_length = int(content_length_header)
+    except ValueError:
+        return
+    if content_length > (MAX_AUDIO_SIZE_BYTES + MAX_MULTIPART_OVERHEAD_BYTES):
+        logger.warning(
+            "413 via header-check: content_length=%s limit=%s",
+            content_length,
+            MAX_AUDIO_SIZE_BYTES,
+        )
+        raise HTTPException(
+            status_code=HTTPStatus.REQUEST_ENTITY_TOO_LARGE,
+            detail="Audio file exceeds maximum allowed size",
+        )
+
+
+def add_speech_to_text_routes(app, authentication: Authentication):
+    """Register speech-to-text routes on the given FastAPI app."""
+    router = APIRouter(prefix="/speech-to-text", tags=["speech-to-text"])
+
+    @router.post(
+        path="/transcribe",
+        status_code=HTTPStatus.OK,
+        response_model=TranscriptionResponse,
+        responses={
+            HTTPStatus.BAD_REQUEST: {"model": HTTPErrorResponse},
+            HTTPStatus.UNSUPPORTED_MEDIA_TYPE: {"model": HTTPErrorResponse},
+            HTTPStatus.REQUEST_ENTITY_TOO_LARGE: {"model": HTTPErrorResponse},
+            HTTPStatus.UNPROCESSABLE_ENTITY: {"model": HTTPErrorResponse},
+            HTTPStatus.BAD_GATEWAY: {"model": HTTPErrorResponse},
+            HTTPStatus.INTERNAL_SERVER_ERROR: {"model": HTTPErrorResponse},
+        },
+        name="transcribe audio",
+        description="Transcribe audio input to text using Google Cloud Speech-to-Text.",
+    )
+    async def _transcribe_audio(
+        request: Request,
+        audio: UploadFile = File(..., description="Audio file recorded in the browser"),
+        language: str = Form(default=DEFAULT_LANGUAGE_CODE),
+        user_info: UserInfo = Depends(authentication.get_user_info()),
+        service: ISpeechToTextService = Depends(get_speech_to_text_service),
+    ) -> TranscriptionResponse:
+        # Validate size early via Content-Length header
+        _validate_request_size_header(request)
+
+        # Validate MIME type — normalize by stripping spaces around semicolons
+        raw_content_type = (audio.content_type or "").strip()
+        # Check both the base type (e.g. "audio/webm") and the full type with params normalized
+        # (e.g. "audio/webm;codecs=opus") to handle browsers that add spaces after ";"
+        content_type = raw_content_type.split(";")[0].strip()
+        normalized_content_type = ";".join(part.strip() for part in raw_content_type.split(";"))
+        if content_type not in ALLOWED_AUDIO_MIME_TYPES and normalized_content_type not in ALLOWED_AUDIO_MIME_TYPES:
+            raise HTTPException(
+                status_code=HTTPStatus.UNSUPPORTED_MEDIA_TYPE,
+                detail="Unsupported audio format",
+            )
+
+        # Read and validate file size
+        audio_bytes = await audio.read()
+        if len(audio_bytes) > MAX_AUDIO_SIZE_BYTES:
+            raise HTTPException(
+                status_code=HTTPStatus.REQUEST_ENTITY_TOO_LARGE,
+                detail="Audio file exceeds maximum allowed size",
+            )
+
+        if len(audio_bytes) == 0:
+            raise HTTPException(
+                status_code=HTTPStatus.BAD_REQUEST,
+                detail="Audio file is empty",
+            )
+
+        logger.info(
+            "Transcribing audio {user_id=%s, size_bytes=%s, content_type='%s', language='%s'}",
+            user_info.user_id,
+            len(audio_bytes),
+            audio.content_type,
+            language,
+        )
+
+        try:
+            return await service.transcribe(audio_bytes=audio_bytes, language_code=language)
+        except EmptyTranscriptionError as exc:
+            raise HTTPException(
+                status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
+                detail="No transcription results returned for the provided audio",
+            ) from exc
+        except SpeechToTextServiceError as e:
+            logger.error("Speech-to-text service error", exc_info=True)
+            raise HTTPException(
+                status_code=HTTPStatus.BAD_GATEWAY,
+                detail="Upstream transcription service unavailable",
+            ) from e
+        except Exception as e:  # pylint: disable=broad-except
+            logger.exception("Unexpected error during transcription")
+            raise HTTPException(
+                status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+                detail="An unexpected error occurred during transcription",
+            ) from e
+
+    app.include_router(router)

backend/app/speech_to_text/service.py

@@ -0,0 +1,86 @@
+import asyncio
+import logging
+import os
+from abc import ABC, abstractmethod
+
+from google.cloud.speech_v2 import SpeechClient
+from google.cloud.speech_v2.types import cloud_speech
+
+from app.speech_to_text.errors import EmptyTranscriptionError, SpeechToTextServiceError
+from app.speech_to_text.types import TranscriptionResponse
+from common_libs.retry import Retry
+
+
+class ISpeechToTextService(ABC):
+    """Interface for speech-to-text transcription services."""
+
+    @abstractmethod
+    async def transcribe(self, *, audio_bytes: bytes, language_code: str) -> TranscriptionResponse:
+        """
+        Transcribe audio bytes to text.
+
+        :param audio_bytes: The raw audio bytes to transcribe.
+        :param language_code: The BCP-47 language code for transcription (e.g. "en-US").
+        :return: The transcription response containing the text and language code.
+        :raises EmptyTranscriptionError: If no transcription results are returned.
+        :raises SpeechToTextServiceError: If the Google Cloud STT API call fails.
+        """
+        raise NotImplementedError()
+
+
+class GoogleSpeechToTextService(ISpeechToTextService):
+    """Google Cloud Speech-to-Text v2 implementation."""
+
+    def __init__(self):
+        self._logger = logging.getLogger(self.__class__.__name__)
+
+        self._project_id = os.getenv("GOOGLE_CLOUD_PROJECT")
+        if not self._project_id:
+            raise ValueError("GOOGLE_CLOUD_PROJECT environment variable is not set")
+
+        self._region = os.getenv("VERTEX_API_REGION", "us-central1")
+
+        # Create the client with regional endpoint
+        self._client = SpeechClient(
+            client_options={"api_endpoint": f"{self._region}-speech.googleapis.com"}
+        )
+
+    async def transcribe(self, *, audio_bytes: bytes, language_code: str) -> TranscriptionResponse:
+        recognizer = f"projects/{self._project_id}/locations/{self._region}/recognizers/_"
+
+        config = cloud_speech.RecognitionConfig(
+            auto_decoding_config=cloud_speech.AutoDetectDecodingConfig(),
+            language_codes=[language_code],
+            model="long",
+        )
+
+        request = cloud_speech.RecognizeRequest(
+            recognizer=recognizer,
+            config=config,
+            content=audio_bytes,
+        )
+
+        try:
+            response = await Retry[cloud_speech.RecognizeResponse].call_with_exponential_backoff(
+                callback=lambda: asyncio.to_thread(self._client.recognize, request=request),
+                logger=self._logger,
+            )
+        except Exception as e:
+            self._logger.error("Speech-to-text API call failed", exc_info=True)
+            raise SpeechToTextServiceError(detail=str(e)) from e
+
+        # Concatenate all transcript alternatives
+        transcript_parts = []
+        for result in response.results:
+            if result.alternatives:
+                transcript_parts.append(result.alternatives[0].transcript)
+
+        text = " ".join(transcript_parts).strip()
+
+        if not text:
+            raise EmptyTranscriptionError()
+
+        return TranscriptionResponse(
+            text=text,
+            language_code=language_code,
+        )