From 54f1eca1592b46d14cfe86782fea4694a8af1507 Mon Sep 17 00:00:00 2001 From: John Vajda Date: Fri, 6 Jun 2025 15:45:50 -0600 Subject: [PATCH 1/2] initial commit --- .../clients/agent/v1/websocket/options.py | 63 +++++-- examples/agent/aws_polly/README.md | 81 +++++++++ examples/agent/aws_polly/main.py | 156 ++++++++++++++++++ 3 files changed, 286 insertions(+), 14 deletions(-) create mode 100644 examples/agent/aws_polly/README.md create mode 100644 examples/agent/aws_polly/main.py diff --git a/deepgram/clients/agent/v1/websocket/options.py b/deepgram/clients/agent/v1/websocket/options.py index d1b1556b..3655b66b 100644 --- a/deepgram/clients/agent/v1/websocket/options.py +++ b/deepgram/clients/agent/v1/websocket/options.py @@ -74,19 +74,14 @@ class Endpoint(BaseResponse): """ Define a custom endpoint for the agent. """ - - method: Optional[str] = field(default="POST") url: str = field(default="") - headers: Optional[List[Header]] = field( - default=None, metadata=dataclass_config(exclude=lambda f: f is None) + headers: Optional[dict] = field( + default=None, + metadata=dataclass_config(exclude=lambda f: f is None) ) def __getitem__(self, key): _dict = self.to_dict() - if "headers" in _dict: - _dict["headers"] = [ - Header.from_dict(headers) for headers in _dict["headers"] - ] return _dict[key] @@ -167,6 +162,24 @@ class ThinkProvider(BaseResponse): ) +@dataclass +class AWSPollyCredentials(BaseResponse): + """ + This class defines the credentials for AWS Polly provider. + """ + type: str = field(default="IAM") # Either "IAM" or "STS" + region: str = field(default="") + access_key_id: str = field(default="") + secret_access_key: str = field(default="") + session_token: Optional[str] = field( + default=None, + metadata=dataclass_config(exclude=lambda f: f is None) + ) + + def __getitem__(self, key): + return self.to_dict()[key] + + @dataclass class SpeakProvider(BaseResponse): """ @@ -175,42 +188,61 @@ class SpeakProvider(BaseResponse): type: Optional[str] = field(default="deepgram") """ - Deepgram OR OpenAI model to use. + Provider type: deepgram, openai, aws_polly, etc. """ model: Optional[str] = field( default="aura-2-thalia-en", metadata=dataclass_config(exclude=lambda f: f is None), ) """ - ElevenLabs or Cartesia model to use. + Deepgram OR OpenAI Model to use for TTS """ model_id: Optional[str] = field( default=None, metadata=dataclass_config(exclude=lambda f: f is None) ) """ - Cartesia voice configuration. + Eleven Labs OR Cartesia Model ID to use """ - voice: Optional[CartesiaVoice] = field( + voice: Optional[Union[CartesiaVoice, str]] = field( default=None, metadata=dataclass_config(exclude=lambda f: f is None) ) """ - Cartesia language. + Voice to use (CartesiaVoice object or string for AWS Polly voice name). """ language: Optional[str] = field( default=None, metadata=dataclass_config(exclude=lambda f: f is None) ) """ - ElevenLabs language. + Cartesia language code """ language_code: Optional[str] = field( default=None, metadata=dataclass_config(exclude=lambda f: f is None) ) + """ + Language code (e.g., en-US for AWS Polly). + """ + engine: Optional[str] = field( + default=None, metadata=dataclass_config(exclude=lambda f: f is None) + ) + """ + Engine type (e.g., standard, neural for AWS Polly). + """ + credentials: Optional[AWSPollyCredentials] = field( + default=None, metadata=dataclass_config(exclude=lambda f: f is None) + ) + """ + AWS Polly credentials configuration. + """ def __getitem__(self, key): _dict = self.to_dict() if "voice" in _dict and isinstance(_dict["voice"], dict): _dict["voice"] = CartesiaVoice.from_dict(_dict["voice"]) + if "credentials" in _dict and isinstance(_dict["credentials"], dict): + _dict["credentials"] = AWSPollyCredentials.from_dict(_dict["credentials"]) return _dict[key] + + @dataclass class Think(BaseResponse): """ @@ -299,6 +331,8 @@ def __getitem__(self, key): if "speak" in _dict and isinstance(_dict["speak"], dict): _dict["speak"] = Speak.from_dict(_dict["speak"]) return _dict[key] + + @dataclass class Input(BaseResponse): """ @@ -340,6 +374,7 @@ def __getitem__(self, key): _dict["output"] = Output.from_dict(_dict["output"]) return _dict[key] + @dataclass class SettingsOptions(BaseResponse): """ diff --git a/examples/agent/aws_polly/README.md b/examples/agent/aws_polly/README.md new file mode 100644 index 00000000..0e573370 --- /dev/null +++ b/examples/agent/aws_polly/README.md @@ -0,0 +1,81 @@ +# AWS Polly Speak Provider Example + +This example demonstrates how to use AWS Polly as the speak provider for the Deepgram Voice Agent. It shows how to configure the agent to use AWS Polly's text-to-speech service for generating voice responses. + +## Prerequisites + +1. A Deepgram API key +2. AWS credentials (either IAM or STS) +3. Python 3.7 or higher +4. Required Python packages (install via `pip install -r requirements.txt`): + +## Environment Variables + +Set these credentials in your environment using the `export` command in the terminal. + +```env +export DEEPGRAM_API_KEY=your_deepgram_api_key +export AWS_ACCESS_KEY_ID=your_aws_access_key_id +export AWS_SECRET_ACCESS_KEY=your_aws_secret_access_key +# Only needed if using STS credentials +export AWS_SESSION_TOKEN=your_aws_session_token +``` + +## Configuration Options + +The example demonstrates two ways to use AWS Polly: + +1. **IAM Credentials** (default): + ```python + options.agent.speak.provider.credentials = AWSPollyCredentials( + type="IAM", + region="us-west-2", + access_key_id=os.getenv("AWS_ACCESS_KEY_ID", ""), + secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY", "") + ) + ``` + +2. **STS Credentials** (temporary credentials): + ```python + options.agent.speak.provider.credentials = AWSPollyCredentials( + type="STS", + region="us-west-2", + access_key_id=os.getenv("AWS_ACCESS_KEY_ID", ""), + secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY", ""), + session_token=os.getenv("AWS_SESSION_TOKEN", "") + ) + ``` + +### AWS Polly Voice Options + +You can customize the AWS Polly voice by modifying these settings: + +```python +options.agent.speak.provider.voice = "Matthew" # AWS Polly voice name +options.agent.speak.provider.language_code = "en-US" +options.agent.speak.provider.engine = "standard" # or "neural" for neural voices +``` + +Available voices and their configurations can be found in the [AWS Polly documentation](https://docs.aws.amazon.com/polly/latest/dg/voicelist.html). + +## Running the Example + +1. Set up your environment variables in `.env` +2. Run the example: + ```bash + python main.py + ``` + +## Notes + +- The example uses MP3 encoding for audio output since that's what AWS Polly provides +- Make sure your AWS credentials have the necessary permissions to use AWS Polly +- The example includes speaker playback for testing the voice output +- The agent is configured to use Deepgram for speech-to-text and OpenAI for the language model + +## Troubleshooting + +1. If you get authentication errors, verify your AWS credentials are correct +2. If the voice doesn't sound right, check that you've selected a valid voice name and language code +3. If you get connection errors, ensure your Deepgram API key is valid +4. For STS credentials, make sure the session token is current and not expired \ No newline at end of file diff --git a/examples/agent/aws_polly/main.py b/examples/agent/aws_polly/main.py new file mode 100644 index 00000000..dbd7d89f --- /dev/null +++ b/examples/agent/aws_polly/main.py @@ -0,0 +1,156 @@ +# Copyright 2024 Deepgram SDK contributors. All Rights Reserved. +# Use of this source code is governed by a MIT license that can be found in the LICENSE file. +# SPDX-License-Identifier: MIT + +import os +import time +from deepgram import ( + DeepgramClient, + DeepgramClientOptions, + AgentWebSocketEvents, + AgentKeepAlive, +) +from deepgram.clients.agent.v1.websocket.options import ( + SettingsOptions, + AWSPollyCredentials, + Endpoint, + SpeakProvider, +) + +def main(): + try: + # Initialize the Voice Agent + api_key = os.getenv("DEEPGRAM_API_KEY") + if not api_key: + raise ValueError("DEEPGRAM_API_KEY environment variable is not set") + print("API Key found") + + # Initialize Deepgram client + config = DeepgramClientOptions( + options={ + "keepalive": "true", + "microphone_record": "true", + "speaker_playback": "true", + }, + ) + deepgram = DeepgramClient(api_key, config) + connection = deepgram.agent.websocket.v("1") + print("Created WebSocket connection...") + + # Define event handlers + def on_open(self, open_response, **kwargs): + print(f"\nConnection opened: {open_response}\n") + + def on_welcome(self, welcome, **kwargs): + print(f"\nWelcome message: {welcome}\n") + + def on_settings_applied(self, settings_applied, **kwargs): + print(f"\nSettings applied: {settings_applied}\n") + + def on_conversation_text(self, conversation_text, **kwargs): + print(f"\nConversation: {conversation_text}\n") + + def on_user_started_speaking(self, user_started_speaking, **kwargs): + print(f"\nUser started speaking: {user_started_speaking}\n") + + def on_agent_thinking(self, agent_thinking, **kwargs): + print(f"\nAgent is thinking: {agent_thinking}\n") + + def on_agent_started_speaking(self, agent_started_speaking, **kwargs): + print(f"\nAgent started speaking: {agent_started_speaking}\n") + + def on_agent_audio_done(self, agent_audio_done, **kwargs): + print(f"\nAgent finished speaking: {agent_audio_done}\n") + + def on_error(self, error, **kwargs): + print(f"\nError occurred: {error}\n") + + def on_close(self, close, **kwargs): + print(f"\nConnection closed: {close}\n") + + # Register event handlers + connection.on(AgentWebSocketEvents.Open, on_open) + connection.on(AgentWebSocketEvents.Welcome, on_welcome) + connection.on(AgentWebSocketEvents.SettingsApplied, on_settings_applied) + connection.on(AgentWebSocketEvents.ConversationText, on_conversation_text) + connection.on(AgentWebSocketEvents.UserStartedSpeaking, on_user_started_speaking) + connection.on(AgentWebSocketEvents.AgentThinking, on_agent_thinking) + connection.on(AgentWebSocketEvents.AgentStartedSpeaking, on_agent_started_speaking) + connection.on(AgentWebSocketEvents.AgentAudioDone, on_agent_audio_done) + connection.on(AgentWebSocketEvents.Error, on_error) + connection.on(AgentWebSocketEvents.Close, on_close) + + # Configure the Agent with AWS Polly + options = SettingsOptions() + + # [Previous configuration code remains the same...] + # Audio input configuration + options.audio.input.encoding = "linear16" + options.audio.input.sample_rate = 24000 + + # Audio output configuration + options.audio.output.encoding = "mp3" # AWS Polly outputs MP3 + options.audio.output.sample_rate = 24000 + options.audio.output.container = "mp3" + + # Agent configuration + options.agent.language = "en" + + # Configure Listen provider (Deepgram) + options.agent.listen.provider.type = "deepgram" + options.agent.listen.provider.model = "nova-3" + options.agent.listen.provider.keyterms = ["hello", "goodbye"] # Optional keyterms + + # Configure Think provider (OpenAI) + options.agent.think.provider.type = "open_ai" + options.agent.think.provider.model = "gpt-4o-mini" + options.agent.think.prompt = "You are a friendly AI assistant." + + # Configure Speak provider (AWS Polly) + options.agent.speak.provider = SpeakProvider( + type="aws_polly", + voice="Matthew", + language_code="en-US", + engine="standard", + credentials=AWSPollyCredentials( + type="IAM", + region="us-east-1", + access_key_id=os.getenv("AWS_ACCESS_KEY_ID", ""), + secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY", ""), + ) + ) + + # Configure AWS Polly endpoint + options.agent.speak.endpoint = Endpoint( + url="https://polly.us-east-1.amazonaws.com/v1/speech" # use your correct AWS region + ) + + # Optional greeting message + options.agent.greeting = "Hello! I'm your AI assistant powered by AWS Polly." + + # Start the connection + print("Starting connection...") + if not connection.start(options): + print("Failed to start connection") + return + + print("Connection started successfully!") + print("Press Ctrl+C to exit...") + + # Keep the connection alive + while True: + time.sleep(1) + keep_alive = AgentKeepAlive() + connection.send(keep_alive.to_json()) + + except KeyboardInterrupt: + print("\nExiting...") + except Exception as e: + print(f"Error: {str(e)}") + finally: + if 'connection' in locals(): + connection.finish() + print("Connection closed") + +if __name__ == "__main__": + main() \ No newline at end of file From 1866b58b1af890eaf52e10909e0411fbc06eeedb Mon Sep 17 00:00:00 2001 From: John Vajda Date: Fri, 6 Jun 2025 16:31:39 -0600 Subject: [PATCH 2/2] reverts endpoint class back --- deepgram/clients/agent/v1/websocket/options.py | 10 +++++++--- examples/agent/aws_polly/main.py | 10 +++++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/deepgram/clients/agent/v1/websocket/options.py b/deepgram/clients/agent/v1/websocket/options.py index 3655b66b..fa3ceaa1 100644 --- a/deepgram/clients/agent/v1/websocket/options.py +++ b/deepgram/clients/agent/v1/websocket/options.py @@ -74,14 +74,18 @@ class Endpoint(BaseResponse): """ Define a custom endpoint for the agent. """ + method: Optional[str] = field(default="POST") url: str = field(default="") - headers: Optional[dict] = field( - default=None, - metadata=dataclass_config(exclude=lambda f: f is None) + headers: Optional[List[Header]] = field( + default=None, metadata=dataclass_config(exclude=lambda f: f is None) ) def __getitem__(self, key): _dict = self.to_dict() + if "headers" in _dict: + _dict["headers"] = [ + Header.from_dict(headers) for headers in _dict["headers"] + ] return _dict[key] diff --git a/examples/agent/aws_polly/main.py b/examples/agent/aws_polly/main.py index dbd7d89f..129ebf93 100644 --- a/examples/agent/aws_polly/main.py +++ b/examples/agent/aws_polly/main.py @@ -15,6 +15,7 @@ AWSPollyCredentials, Endpoint, SpeakProvider, + Header, ) def main(): @@ -122,7 +123,14 @@ def on_close(self, close, **kwargs): # Configure AWS Polly endpoint options.agent.speak.endpoint = Endpoint( - url="https://polly.us-east-1.amazonaws.com/v1/speech" # use your correct AWS region + method="POST", # Explicitly set the method for AWS Polly + url="https://polly.us-east-1.amazonaws.com/v1/speech", #use the correct region + headers=[ + Header( + key="Content-Type", + value="application/json" + ) + ] ) # Optional greeting message