[WIP] AWS Polly Agent TTS Provider

deepgram/clients/agent/v1/websocket/options.py

@@ -74,7 +74,6 @@ class Endpoint(BaseResponse):
     """
     Define a custom endpoint for the agent.
     """
-
     method: Optional[str] = field(default="POST")
     url: str = field(default="")
     headers: Optional[List[Header]] = field(
@@ -167,6 +166,24 @@ class ThinkProvider(BaseResponse):
     )
 
 
+@dataclass
+class AWSPollyCredentials(BaseResponse):
+    """
+    This class defines the credentials for AWS Polly provider.
+    """
+    type: str = field(default="IAM")  # Either "IAM" or "STS"
+    region: str = field(default="")
+    access_key_id: str = field(default="")
+    secret_access_key: str = field(default="")
+    session_token: Optional[str] = field(
+        default=None,
+        metadata=dataclass_config(exclude=lambda f: f is None)
+    )
+
+    def __getitem__(self, key):
+        return self.to_dict()[key]
+
+
 @dataclass
 class SpeakProvider(BaseResponse):
     """
@@ -175,42 +192,61 @@ class SpeakProvider(BaseResponse):
 
     type: Optional[str] = field(default="deepgram")
     """
-    Deepgram OR OpenAI model to use.
+    Provider type: deepgram, openai, aws_polly, etc.
     """
     model: Optional[str] = field(
         default="aura-2-thalia-en",
         metadata=dataclass_config(exclude=lambda f: f is None),
     )
     """
-    ElevenLabs or Cartesia model to use.
+    Deepgram OR OpenAI Model to use for TTS
     """
     model_id: Optional[str] = field(
         default=None, metadata=dataclass_config(exclude=lambda f: f is None)
     )
     """
-    Cartesia voice configuration.
+    Eleven Labs OR Cartesia Model ID to use
     """
-    voice: Optional[CartesiaVoice] = field(
+    voice: Optional[Union[CartesiaVoice, str]] = field(
         default=None, metadata=dataclass_config(exclude=lambda f: f is None)
     )
     """
-    Cartesia language.
+    Voice to use (CartesiaVoice object or string for AWS Polly voice name).
     """
     language: Optional[str] = field(
         default=None, metadata=dataclass_config(exclude=lambda f: f is None)
     )
     """
-    ElevenLabs language.
+    Cartesia language code
     """
     language_code: Optional[str] = field(
         default=None, metadata=dataclass_config(exclude=lambda f: f is None)
     )
+    """
+    Language code (e.g., en-US for AWS Polly).
+    """
+    engine: Optional[str] = field(
+        default=None, metadata=dataclass_config(exclude=lambda f: f is None)
+    )
+    """
+    Engine type (e.g., standard, neural for AWS Polly).
+    """
+    credentials: Optional[AWSPollyCredentials] = field(
+        default=None, metadata=dataclass_config(exclude=lambda f: f is None)
+    )
+    """
+    AWS Polly credentials configuration.
+    """
 
     def __getitem__(self, key):
         _dict = self.to_dict()
         if "voice" in _dict and isinstance(_dict["voice"], dict):
             _dict["voice"] = CartesiaVoice.from_dict(_dict["voice"])
+        if "credentials" in _dict and isinstance(_dict["credentials"], dict):
+            _dict["credentials"] = AWSPollyCredentials.from_dict(_dict["credentials"])
         return _dict[key]
+
+
 @dataclass
 class Think(BaseResponse):
     """
@@ -299,6 +335,8 @@ def __getitem__(self, key):
         if "speak" in _dict and isinstance(_dict["speak"], dict):
             _dict["speak"] = Speak.from_dict(_dict["speak"])
         return _dict[key]
+
+
 @dataclass
 class Input(BaseResponse):
     """
@@ -340,6 +378,7 @@ def __getitem__(self, key):
             _dict["output"] = Output.from_dict(_dict["output"])
         return _dict[key]
 
+
 @dataclass
 class SettingsOptions(BaseResponse):
     """

examples/agent/aws_polly/README.md

@@ -0,0 +1,81 @@
+# AWS Polly Speak Provider Example
+
+This example demonstrates how to use AWS Polly as the speak provider for the Deepgram Voice Agent. It shows how to configure the agent to use AWS Polly's text-to-speech service for generating voice responses.
+
+## Prerequisites
+
+1. A Deepgram API key
+2. AWS credentials (either IAM or STS)
+3. Python 3.7 or higher
+4. Required Python packages (install via `pip install -r requirements.txt`):
+
+## Environment Variables
+
+Set these credentials in your environment using the `export` command in the terminal.
+
+```env
+export DEEPGRAM_API_KEY=your_deepgram_api_key
+export AWS_ACCESS_KEY_ID=your_aws_access_key_id
+export AWS_SECRET_ACCESS_KEY=your_aws_secret_access_key
+# Only needed if using STS credentials
+export AWS_SESSION_TOKEN=your_aws_session_token
+```
+
+## Configuration Options
+
+The example demonstrates two ways to use AWS Polly:
+
+1. **IAM Credentials** (default):
+   ```python
+   options.agent.speak.provider.credentials = AWSPollyCredentials(
+       type="IAM",
+       region="us-west-2",
+       access_key_id=os.getenv("AWS_ACCESS_KEY_ID", ""),
+       secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY", "")
+   )
+   ```
+
+2. **STS Credentials** (temporary credentials):
+   ```python
+   options.agent.speak.provider.credentials = AWSPollyCredentials(
+       type="STS",
+       region="us-west-2",
+       access_key_id=os.getenv("AWS_ACCESS_KEY_ID", ""),
+       secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY", ""),
+       session_token=os.getenv("AWS_SESSION_TOKEN", "")
+   )
+   ```
+
+### AWS Polly Voice Options
+
+You can customize the AWS Polly voice by modifying these settings:
+
+```python
+options.agent.speak.provider.voice = "Matthew"  # AWS Polly voice name
+options.agent.speak.provider.language_code = "en-US"
+options.agent.speak.provider.engine = "standard"  # or "neural" for neural voices
+```
+
+Available voices and their configurations can be found in the [AWS Polly documentation](https://docs.aws.amazon.com/polly/latest/dg/voicelist.html).
+
+## Running the Example
+
+1. Set up your environment variables in `.env`
+2. Run the example:
+   ```bash
+   python main.py
+   ```
+
+## Notes
+
+- The example uses MP3 encoding for audio output since that's what AWS Polly provides
+- Make sure your AWS credentials have the necessary permissions to use AWS Polly
+- The example includes speaker playback for testing the voice output
+- The agent is configured to use Deepgram for speech-to-text and OpenAI for the language model
+
+## Troubleshooting
+
+1. If you get authentication errors, verify your AWS credentials are correct
+2. If the voice doesn't sound right, check that you've selected a valid voice name and language code
+3. If you get connection errors, ensure your Deepgram API key is valid
+4. For STS credentials, make sure the session token is current and not expired

examples/agent/aws_polly/main.py

@@ -0,0 +1,164 @@
+# Copyright 2024 Deepgram SDK contributors. All Rights Reserved.
+# Use of this source code is governed by a MIT license that can be found in the LICENSE file.
+# SPDX-License-Identifier: MIT
+
+import os
+import time
+from deepgram import (
+    DeepgramClient,
+    DeepgramClientOptions,
+    AgentWebSocketEvents,
+    AgentKeepAlive,
+)
+from deepgram.clients.agent.v1.websocket.options import (
+    SettingsOptions,
+    AWSPollyCredentials,
+    Endpoint,
+    SpeakProvider,
+    Header,
+)
+
+def main():
+    try:
+        # Initialize the Voice Agent
+        api_key = os.getenv("DEEPGRAM_API_KEY")
+        if not api_key:
+            raise ValueError("DEEPGRAM_API_KEY environment variable is not set")
+        print("API Key found")
+
+        # Initialize Deepgram client
+        config = DeepgramClientOptions(
+            options={
+                "keepalive": "true",
+                "microphone_record": "true",
+                "speaker_playback": "true",
+            },
+        )
+        deepgram = DeepgramClient(api_key, config)
+        connection = deepgram.agent.websocket.v("1")
+        print("Created WebSocket connection...")
+
+        # Define event handlers
+        def on_open(self, open_response, **kwargs):
+            print(f"\nConnection opened: {open_response}\n")
+
+        def on_welcome(self, welcome, **kwargs):
+            print(f"\nWelcome message: {welcome}\n")
+
+        def on_settings_applied(self, settings_applied, **kwargs):
+            print(f"\nSettings applied: {settings_applied}\n")
+
+        def on_conversation_text(self, conversation_text, **kwargs):
+            print(f"\nConversation: {conversation_text}\n")
+
+        def on_user_started_speaking(self, user_started_speaking, **kwargs):
+            print(f"\nUser started speaking: {user_started_speaking}\n")
+
+        def on_agent_thinking(self, agent_thinking, **kwargs):
+            print(f"\nAgent is thinking: {agent_thinking}\n")
+
+        def on_agent_started_speaking(self, agent_started_speaking, **kwargs):
+            print(f"\nAgent started speaking: {agent_started_speaking}\n")
+
+        def on_agent_audio_done(self, agent_audio_done, **kwargs):
+            print(f"\nAgent finished speaking: {agent_audio_done}\n")
+
+        def on_error(self, error, **kwargs):
+            print(f"\nError occurred: {error}\n")
+
+        def on_close(self, close, **kwargs):
+            print(f"\nConnection closed: {close}\n")
+
+        # Register event handlers
+        connection.on(AgentWebSocketEvents.Open, on_open)
+        connection.on(AgentWebSocketEvents.Welcome, on_welcome)
+        connection.on(AgentWebSocketEvents.SettingsApplied, on_settings_applied)
+        connection.on(AgentWebSocketEvents.ConversationText, on_conversation_text)
+        connection.on(AgentWebSocketEvents.UserStartedSpeaking, on_user_started_speaking)
+        connection.on(AgentWebSocketEvents.AgentThinking, on_agent_thinking)
+        connection.on(AgentWebSocketEvents.AgentStartedSpeaking, on_agent_started_speaking)
+        connection.on(AgentWebSocketEvents.AgentAudioDone, on_agent_audio_done)
+        connection.on(AgentWebSocketEvents.Error, on_error)
+        connection.on(AgentWebSocketEvents.Close, on_close)
+
+        # Configure the Agent with AWS Polly
+        options = SettingsOptions()
+
+        # [Previous configuration code remains the same...]
+        # Audio input configuration
+        options.audio.input.encoding = "linear16"
+        options.audio.input.sample_rate = 24000
+
+        # Audio output configuration
+        options.audio.output.encoding = "mp3"  # AWS Polly outputs MP3
+        options.audio.output.sample_rate = 24000
+        options.audio.output.container = "mp3"
+
+        # Agent configuration
+        options.agent.language = "en"
+
+        # Configure Listen provider (Deepgram)
+        options.agent.listen.provider.type = "deepgram"
+        options.agent.listen.provider.model = "nova-3"
+        options.agent.listen.provider.keyterms = ["hello", "goodbye"]  # Optional keyterms
+
+        # Configure Think provider (OpenAI)
+        options.agent.think.provider.type = "open_ai"
+        options.agent.think.provider.model = "gpt-4o-mini"
+        options.agent.think.prompt = "You are a friendly AI assistant."
+
+        # Configure Speak provider (AWS Polly)
+        options.agent.speak.provider = SpeakProvider(
+            type="aws_polly",
+            voice="Matthew",
+            language_code="en-US",
+            engine="standard",
+            credentials=AWSPollyCredentials(
+                type="IAM",
+                region="us-east-1",
+                access_key_id=os.getenv("AWS_ACCESS_KEY_ID", ""),
+                secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY", ""),
+            )
+        )
+
+        # Configure AWS Polly endpoint
+        options.agent.speak.endpoint = Endpoint(
+            method="POST",  # Explicitly set the method for AWS Polly
+            url="https://polly.us-east-1.amazonaws.com/v1/speech", #use the correct region
+            headers=[
+                Header(
+                    key="Content-Type",
+                    value="application/json"
+                )
+            ]
+        )
+
+        # Optional greeting message
+        options.agent.greeting = "Hello! I'm your AI assistant powered by AWS Polly."
+
+        # Start the connection
+        print("Starting connection...")
+        if not connection.start(options):
+            print("Failed to start connection")
+            return
+
+        print("Connection started successfully!")
+        print("Press Ctrl+C to exit...")
+
+        # Keep the connection alive
+        while True:
+            time.sleep(1)
+            keep_alive = AgentKeepAlive()
+            connection.send(keep_alive.to_json())
+
+    except KeyboardInterrupt:
+        print("\nExiting...")
+    except Exception as e:
+        print(f"Error: {str(e)}")
+    finally:
+        if 'connection' in locals():
+            connection.finish()
+            print("Connection closed")
+
+if __name__ == "__main__":
+    main()