From 81dbc56e262ca8afd392b9cbd885a1540c2a4381 Mon Sep 17 00:00:00 2001 From: Adithya Sairamachandran Date: Tue, 24 Jun 2025 23:36:40 -0700 Subject: [PATCH 1/3] Add listeners for camera errors, network quality changes, and network connection events --- vapi.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/vapi.ts b/vapi.ts index 60d8bb8b2..437366311 100644 --- a/vapi.ts +++ b/vapi.ts @@ -55,6 +55,9 @@ type VapiEventNames = | 'message' | 'video' | 'error' + | 'camera-error' + | 'network-quality-change' + | 'network-connection' | 'daily-participant-updated'; type VapiEventListeners = { @@ -66,6 +69,9 @@ type VapiEventListeners = { video: (track: MediaStreamTrack) => void; message: (message: any) => void; error: (error: any) => void; + 'camera-error': (error: any) => void; + 'network-quality-change': (event: any) => void; + 'network-connection': (event: any) => void; 'daily-participant-updated': (participant: DailyParticipant) => void; }; @@ -266,7 +272,15 @@ export default class Vapi extends VapiEventEmitter { }); this.call.on('camera-error', (error: any) => { - this.emit('error', error); + this.emit('camera-error', error); + }); + + this.call.on('network-quality-change', (event: any) => { + this.emit('network-quality-change', event); + }); + + this.call.on('network-connection', (event: any) => { + this.emit('network-connection', event); }); this.call.on('track-started', async (e) => { From b03c53308249e4fe6c0b709ec374482db1f85e72 Mon Sep 17 00:00:00 2001 From: Adithya Sairamachandran Date: Tue, 24 Jun 2025 23:51:41 -0700 Subject: [PATCH 2/3] update api.ts, package version --- api.ts | 7046 ++++++++++++++++++++++++++++++--------------- package-lock.json | 4 +- package.json | 2 +- 3 files changed, 4686 insertions(+), 2366 deletions(-) diff --git a/api.ts b/api.ts index 3ea04dfdf..9730a5a6f 100644 --- a/api.ts +++ b/api.ts @@ -290,6 +290,16 @@ export interface AssemblyAITranscriber { */ enableUniversalStreamingApi?: boolean; /** + * This enables formatting of transcripts. Only used when `enableUniversalStreamingApi` is true. + * + * @default false + * @example false + */ + formatTurns?: boolean; + /** + * The confidence threshold to use when determining if the end of a turn has been reached. Only used when `enableUniversalStreamingApi` is true. + * + * @default 0.7 * @min 0 * @max 1 * @example 0.7 @@ -902,41 +912,6 @@ export interface DeepgramTranscriber { * @example false */ smartFormat?: boolean; - /** - * This automatically switches the transcriber's language when the customer's language changes. Defaults to false. - * - * Usage: - * - If your customers switch languages mid-call, you can set this to true. - * - * Note: - * - To detect language changes, Vapi uses a custom trained model. Languages supported (X = limited support): - * 1. Arabic - * 2. Bengali - * 3. Cantonese - * 4. Chinese - * 5. Chinese Simplified (X) - * 6. Chinese Traditional (X) - * 7. English - * 8. Farsi (X) - * 9. French - * 10. German - * 11. Haitian Creole (X) - * 12. Hindi - * 13. Italian - * 14. Japanese - * 15. Korean - * 16. Portuguese - * 17. Russian - * 18. Spanish - * 19. Thai - * 20. Urdu - * 21. Vietnamese - * - To receive `language-change-detected` webhook events, add it to `assistant.serverMessages`. - * - * @default false - * @example false - */ - codeSwitchingEnabled?: boolean; /** * If set to true, this will add mip_opt_out=true as a query parameter of all API requests. See https://developers.deepgram.com/docs/the-deepgram-model-improvement-partnership-program#want-to-opt-out * @@ -1600,6 +1575,9 @@ export interface GoogleTranscriber { provider: 'google'; /** This is the model that will be used for the transcription. */ model?: + | 'gemini-2.5-pro' + | 'gemini-2.5-flash' + | 'gemini-2.5-flash-lite' | 'gemini-2.5-pro-preview-05-06' | 'gemini-2.5-flash-preview-05-20' | 'gemini-2.5-flash-preview-04-17' @@ -1750,6 +1728,16 @@ export interface FallbackAssemblyAITranscriber { */ enableUniversalStreamingApi?: boolean; /** + * This enables formatting of transcripts. Only used when `enableUniversalStreamingApi` is true. + * + * @default false + * @example false + */ + formatTurns?: boolean; + /** + * The confidence threshold to use when determining if the end of a turn has been reached. Only used when `enableUniversalStreamingApi` is true. + * + * @default 0.7 * @min 0 * @max 1 * @example 0.7 @@ -2302,41 +2290,6 @@ export interface FallbackDeepgramTranscriber { * @example false */ smartFormat?: boolean; - /** - * This automatically switches the transcriber's language when the customer's language changes. Defaults to false. - * - * Usage: - * - If your customers switch languages mid-call, you can set this to true. - * - * Note: - * - To detect language changes, Vapi uses a custom trained model. Languages supported (X = limited support): - * 1. Arabic - * 2. Bengali - * 3. Cantonese - * 4. Chinese - * 5. Chinese Simplified (X) - * 6. Chinese Traditional (X) - * 7. English - * 8. Farsi (X) - * 9. French - * 10. German - * 11. Haitian Creole (X) - * 12. Hindi - * 13. Italian - * 14. Japanese - * 15. Korean - * 16. Portuguese - * 17. Russian - * 18. Spanish - * 19. Thai - * 20. Urdu - * 21. Vietnamese - * - To receive `language-change-detected` webhook events, add it to `assistant.serverMessages`. - * - * @default false - * @example false - */ - codeSwitchingEnabled?: boolean; /** * If set to true, this will add mip_opt_out=true as a query parameter of all API requests. See https://developers.deepgram.com/docs/the-deepgram-model-improvement-partnership-program#want-to-opt-out * @@ -2990,6 +2943,9 @@ export interface FallbackGoogleTranscriber { provider: 'google'; /** This is the model that will be used for the transcription. */ model?: + | 'gemini-2.5-pro' + | 'gemini-2.5-flash' + | 'gemini-2.5-flash-lite' | 'gemini-2.5-pro-preview-05-06' | 'gemini-2.5-flash-preview-05-20' | 'gemini-2.5-flash-preview-04-17' @@ -3903,7 +3859,9 @@ export interface SummaryPlan { * You can customize by providing any messages you want. * * Here are the template variables available: - * - {{transcript}}: The transcript of the call from `call.artifact.transcript`- {{systemPrompt}}: The system prompt of the call from `assistant.model.messages[type=system].content`- {{endedReason}}: The ended reason of the call from `call.endedReason` + * - {{transcript}}: The transcript of the call from `call.artifact.transcript` + * - {{systemPrompt}}: The system prompt of the call from `assistant.model.messages[type=system].content` + * - {{endedReason}}: The ended reason of the call from `call.endedReason` */ messages?: object[]; /** @@ -4210,6 +4168,9 @@ export interface KnowledgeBase { provider: 'google'; /** The model to use for the knowledge base */ model?: + | 'gemini-2.5-pro' + | 'gemini-2.5-flash' + | 'gemini-2.5-flash-lite' | 'gemini-2.5-pro-preview-05-06' | 'gemini-2.5-flash-preview-05-20' | 'gemini-2.5-flash-preview-04-17' @@ -4328,6 +4289,11 @@ export interface CreateSlackSendMessageToolDTO { function?: OpenAIFunction; } +export interface McpToolMetadata { + /** This is the protocol used for MCP communication. Defaults to Streamable HTTP. */ + protocol?: 'sse' | 'shttp'; +} + export interface CreateMcpToolDTO { /** * These are the messages that will be spoken to the user as the tool is running. @@ -4349,6 +4315,7 @@ export interface CreateMcpToolDTO { * - Webhook expects a response with tool call result. */ server?: Server; + metadata?: McpToolMetadata; /** * This is the function definition of the tool. * @@ -5002,6 +4969,9 @@ export interface GoogleModel { knowledgeBaseId?: string; /** This is the Google model that will be used. */ model: + | 'gemini-2.5-pro' + | 'gemini-2.5-flash' + | 'gemini-2.5-flash-lite' | 'gemini-2.5-pro-preview-05-06' | 'gemini-2.5-flash-preview-05-20' | 'gemini-2.5-flash-preview-04-17' @@ -5756,9 +5726,6 @@ export interface WorkflowOpenAIModel { | 'o1-preview-2024-09-12' | 'o1-mini' | 'o1-mini-2024-09-12' - | 'gpt-4o-realtime-preview-2024-10-01' - | 'gpt-4o-realtime-preview-2024-12-17' - | 'gpt-4o-mini-realtime-preview-2024-12-17' | 'gpt-4o-mini-2024-07-18' | 'gpt-4o-mini' | 'gpt-4o' @@ -5889,6 +5856,88 @@ export interface WorkflowAnthropicModel { maxTokens?: number; } +export interface WorkflowGoogleModel { + /** This is the provider of the model (`google`). */ + provider: 'google'; + /** + * This is the name of the model. Ex. cognitivecomputations/dolphin-mixtral-8x7b + * @maxLength 100 + */ + model: + | 'gemini-2.5-pro' + | 'gemini-2.5-flash' + | 'gemini-2.5-flash-lite' + | 'gemini-2.5-pro-preview-05-06' + | 'gemini-2.5-flash-preview-05-20' + | 'gemini-2.5-flash-preview-04-17' + | 'gemini-2.0-flash-thinking-exp' + | 'gemini-2.0-pro-exp-02-05' + | 'gemini-2.0-flash' + | 'gemini-2.0-flash-lite' + | 'gemini-2.0-flash-lite-preview-02-05' + | 'gemini-2.0-flash-exp' + | 'gemini-2.0-flash-realtime-exp' + | 'gemini-1.5-flash' + | 'gemini-1.5-flash-002' + | 'gemini-1.5-pro' + | 'gemini-1.5-pro-002' + | 'gemini-1.0-pro'; + /** + * This is the temperature of the model. + * @min 0 + * @max 2 + */ + temperature?: number; + /** + * This is the max tokens of the model. + * @min 50 + * @max 10000 + */ + maxTokens?: number; +} + +export interface WorkflowCustomModel { + /** This is the provider of the model (`custom-llm`). */ + provider: 'custom-llm'; + /** + * This determines whether metadata is sent in requests to the custom provider. + * + * - `off` will not send any metadata. payload will look like `{ messages }` + * - `variable` will send `assistant.metadata` as a variable on the payload. payload will look like `{ messages, metadata }` + * - `destructured` will send `assistant.metadata` fields directly on the payload. payload will look like `{ messages, ...metadata }` + * + * Further, `variable` and `destructured` will send `call`, `phoneNumber`, and `customer` objects in the payload. + * + * Default is `variable`. + */ + metadataSendMode?: 'off' | 'variable' | 'destructured'; + /** These is the URL we'll use for the OpenAI client's `baseURL`. Ex. https://openrouter.ai/api/v1 */ + url: string; + /** + * This sets the timeout for the connection to the custom provider without needing to stream any tokens back. Default is 20 seconds. + * @min 20 + * @max 600 + */ + timeoutSeconds?: number; + /** + * This is the name of the model. Ex. cognitivecomputations/dolphin-mixtral-8x7b + * @maxLength 100 + */ + model: string; + /** + * This is the temperature of the model. + * @min 0 + * @max 2 + */ + temperature?: number; + /** + * This is the max tokens of the model. + * @min 50 + * @max 10000 + */ + maxTokens?: number; +} + export interface GlobalNodePlan { /** * This is the flag to determine if this node is a global node @@ -5924,9 +5973,17 @@ export interface ConversationNode { * - Workflow continues. */ type: 'conversation'; - /** This is the model for the Conversation Task. */ - model?: WorkflowOpenAIModel | WorkflowAnthropicModel; - /** These are the options for the assistant's transcriber. */ + /** + * This is the model for the node. + * + * This overrides `workflow.model`. + */ + model?: WorkflowOpenAIModel | WorkflowAnthropicModel | WorkflowGoogleModel | WorkflowCustomModel; + /** + * This is the transcriber for the node. + * + * This overrides `workflow.transcriber`. + */ transcriber?: | AssemblyAITranscriber | AzureSpeechTranscriber @@ -5939,7 +5996,11 @@ export interface ConversationNode { | TalkscriberTranscriber | OpenAITranscriber | CartesiaTranscriber; - /** These are the options for the assistant's voice. */ + /** + * This is the voice for the node. + * + * This overrides `workflow.voice`. + */ voice?: | AzureVoice | CartesiaVoice @@ -6032,204 +6093,166 @@ export interface Edge { metadata?: object; } -export interface WorkflowUserEditable { - nodes: (ConversationNode | ToolNode)[]; - /** @maxLength 80 */ - name: string; - edges: Edge[]; - /** @maxLength 5000 */ - globalPrompt?: string; +export interface CompliancePlan { /** - * This is where Vapi will send webhooks. You can find all webhooks available along with their shape in ServerMessage schema. - * - * The order of precedence is: - * - * 1. tool.server - * 2. workflow.server / assistant.server - * 3. phoneNumber.server - * 4. org.server + * When this is enabled, no logs, recordings, or transcriptions will be stored. + * At the end of the call, you will still receive an end-of-call-report message to store on your server. Defaults to false. + * @example {"hipaaEnabled":false} */ - server?: Server; + hipaaEnabled?: boolean; + /** + * When this is enabled, the user will be restricted to use PCI-compliant providers, and no logs or transcripts are stored. + * At the end of the call, you will receive an end-of-call-report message to store on your server. Defaults to false. + * @example {"pciEnabled":false} + */ + pciEnabled?: boolean; } -export interface VapiModel { - /** This is the starting state for the conversation. */ - messages?: OpenAIMessage[]; +export interface StructuredDataPlan { /** - * These are the tools that the assistant can use during the call. To use existing tools, use `toolIds`. + * These are the messages used to generate the structured data. * - * Both `tools` and `toolIds` can be used together. - */ - tools?: ( - | CreateApiRequestToolDTO - | CreateBashToolDTO - | CreateComputerToolDTO - | CreateDtmfToolDTO - | CreateEndCallToolDTO - | CreateFunctionToolDTO - | CreateGoHighLevelCalendarAvailabilityToolDTO - | CreateGoHighLevelCalendarEventCreateToolDTO - | CreateGoHighLevelContactCreateToolDTO - | CreateGoHighLevelContactGetToolDTO - | CreateGoogleCalendarCheckAvailabilityToolDTO - | CreateGoogleCalendarCreateEventToolDTO - | CreateGoogleSheetsRowAppendToolDTO - | CreateMcpToolDTO - | CreateQueryToolDTO - | CreateSlackSendMessageToolDTO - | CreateSmsToolDTO - | CreateTextEditorToolDTO - | CreateTransferCallToolDTO - )[]; - /** - * These are the tools that the assistant can use during the call. To use transient tools, use `tools`. + * @default: ``` + * [ + * { + * "role": "system", + * "content": "You are an expert data extractor. You will be given a transcript of a call. Extract structured data per the JSON Schema. DO NOT return anything except the structured data.\n\nJson Schema:\\n{{schema}}\n\nOnly respond with the JSON." + * }, + * { + * "role": "user", + * "content": "Here is the transcript:\n\n{{transcript}}\n\n. Here is the ended reason of the call:\n\n{{endedReason}}\n\n" + * } + * ]``` * - * Both `tools` and `toolIds` can be used together. - */ - toolIds?: string[]; - /** These are the options for the knowledge base. */ - knowledgeBase?: CreateCustomKnowledgeBaseDTO; - /** This is the ID of the knowledge base the model will use. */ - knowledgeBaseId?: string; - provider: 'vapi'; - /** This is the workflow that will be used for the call. To use a transient workflow, use `workflow` instead. */ - workflowId?: string; - /** This is the workflow that will be used for the call. To use an existing workflow, use `workflowId` instead. */ - workflow?: WorkflowUserEditable; - /** This is the name of the model. Ex. cognitivecomputations/dolphin-mixtral-8x7b */ - model: string; - /** - * This is the temperature that will be used for calls. Default is 0 to leverage caching for lower latency. - * @min 0 - * @max 2 - */ - temperature?: number; - /** - * This is the max number of tokens that the assistant will be allowed to generate in each turn of the conversation. Default is 250. - * @min 50 - * @max 10000 + * You can customize by providing any messages you want. + * + * Here are the template variables available: + * - {{transcript}}: the transcript of the call from `call.artifact.transcript`- {{systemPrompt}}: the system prompt of the call from `assistant.model.messages[type=system].content`- {{schema}}: the schema of the structured data from `structuredDataPlan.schema`- {{endedReason}}: the ended reason of the call from `call.endedReason` */ - maxTokens?: number; + messages?: object[]; /** - * This determines whether we detect user's emotion while they speak and send it as an additional info to model. + * This determines whether structured data is generated and stored in `call.analysis.structuredData`. Defaults to false. * - * Default `false` because the model is usually are good at understanding the user's emotion from text. + * Usage: + * - If you want to extract structured data, set this to true and provide a `schema`. * * @default false */ - emotionRecognitionEnabled?: boolean; + enabled?: boolean; /** - * This sets how many turns at the start of the conversation to use a smaller, faster model from the same provider before switching to the primary model. Example, gpt-3.5-turbo if provider is openai. + * This is the schema of the structured data. The output is stored in `call.analysis.structuredData`. * - * Default is 0. + * Complete guide on JSON Schema can be found [here](https://ajv.js.org/json-schema.html#json-data-type). + */ + schema?: JsonSchema; + /** + * This is how long the request is tried before giving up. When request times out, `call.analysis.structuredData` will be empty. * - * @default 0 - * @min 0 + * Usage: + * - To guarantee the structured data is generated, set this value high. Note, this will delay the end of call report in cases where model is slow to respond. + * + * @default 5 seconds + * @min 1 + * @max 60 */ - numFastTurns?: number; + timeoutSeconds?: number; } -export interface XaiModel { - /** This is the starting state for the conversation. */ - messages?: OpenAIMessage[]; +export interface StructuredDataMultiPlan { + /** This is the key of the structured data plan in the catalog. */ + key: string; + /** This is an individual structured data plan in the catalog. */ + plan: StructuredDataPlan; +} + +export interface SuccessEvaluationPlan { /** - * These are the tools that the assistant can use during the call. To use existing tools, use `toolIds`. + * This enforces the rubric of the evaluation. The output is stored in `call.analysis.successEvaluation`. * - * Both `tools` and `toolIds` can be used together. - */ - tools?: ( - | CreateApiRequestToolDTO - | CreateBashToolDTO - | CreateComputerToolDTO - | CreateDtmfToolDTO - | CreateEndCallToolDTO - | CreateFunctionToolDTO - | CreateGoHighLevelCalendarAvailabilityToolDTO - | CreateGoHighLevelCalendarEventCreateToolDTO - | CreateGoHighLevelContactCreateToolDTO - | CreateGoHighLevelContactGetToolDTO - | CreateGoogleCalendarCheckAvailabilityToolDTO - | CreateGoogleCalendarCreateEventToolDTO - | CreateGoogleSheetsRowAppendToolDTO - | CreateMcpToolDTO - | CreateQueryToolDTO - | CreateSlackSendMessageToolDTO - | CreateSmsToolDTO - | CreateTextEditorToolDTO - | CreateTransferCallToolDTO - )[]; - /** - * These are the tools that the assistant can use during the call. To use transient tools, use `tools`. + * Options include: + * - 'NumericScale': A scale of 1 to 10. + * - 'DescriptiveScale': A scale of Excellent, Good, Fair, Poor. + * - 'Checklist': A checklist of criteria and their status. + * - 'Matrix': A grid that evaluates multiple criteria across different performance levels. + * - 'PercentageScale': A scale of 0% to 100%. + * - 'LikertScale': A scale of Strongly Agree, Agree, Neutral, Disagree, Strongly Disagree. + * - 'AutomaticRubric': Automatically break down evaluation into several criteria, each with its own score. + * - 'PassFail': A simple 'true' if call passed, 'false' if not. * - * Both `tools` and `toolIds` can be used together. - */ - toolIds?: string[]; - /** These are the options for the knowledge base. */ - knowledgeBase?: CreateCustomKnowledgeBaseDTO; - /** This is the ID of the knowledge base the model will use. */ - knowledgeBaseId?: string; - /** This is the name of the model. Ex. cognitivecomputations/dolphin-mixtral-8x7b */ - model: 'grok-beta' | 'grok-2' | 'grok-3'; - provider: 'xai'; - /** - * This is the temperature that will be used for calls. Default is 0 to leverage caching for lower latency. - * @min 0 - * @max 2 - */ - temperature?: number; - /** - * This is the max number of tokens that the assistant will be allowed to generate in each turn of the conversation. Default is 250. - * @min 50 - * @max 10000 + * Default is 'PassFail'. */ - maxTokens?: number; + rubric?: + | 'NumericScale' + | 'DescriptiveScale' + | 'Checklist' + | 'Matrix' + | 'PercentageScale' + | 'LikertScale' + | 'AutomaticRubric' + | 'PassFail'; /** - * This determines whether we detect user's emotion while they speak and send it as an additional info to model. + * These are the messages used to generate the success evaluation. * - * Default `false` because the model is usually are good at understanding the user's emotion from text. + * @default: ``` + * [ + * { + * "role": "system", + * "content": "You are an expert call evaluator. You will be given a transcript of a call and the system prompt of the AI participant. Determine if the call was successful based on the objectives inferred from the system prompt. DO NOT return anything except the result.\n\nRubric:\\n{{rubric}}\n\nOnly respond with the result." + * }, + * { + * "role": "user", + * "content": "Here is the transcript:\n\n{{transcript}}\n\n" + * }, + * { + * "role": "user", + * "content": "Here was the system prompt of the call:\n\n{{systemPrompt}}\n\n. Here is the ended reason of the call:\n\n{{endedReason}}\n\n" + * } + * ]``` * - * @default false + * You can customize by providing any messages you want. + * + * Here are the template variables available: + * - {{transcript}}: the transcript of the call from `call.artifact.transcript`- {{systemPrompt}}: the system prompt of the call from `assistant.model.messages[type=system].content`- {{rubric}}: the rubric of the success evaluation from `successEvaluationPlan.rubric`- {{endedReason}}: the ended reason of the call from `call.endedReason` */ - emotionRecognitionEnabled?: boolean; + messages?: object[]; /** - * This sets how many turns at the start of the conversation to use a smaller, faster model from the same provider before switching to the primary model. Example, gpt-3.5-turbo if provider is openai. + * This determines whether a success evaluation is generated and stored in `call.analysis.successEvaluation`. Defaults to true. * - * Default is 0. + * Usage: + * - If you want to disable the success evaluation, set this to false. * - * @default 0 - * @min 0 + * @default true */ - numFastTurns?: number; -} - -export interface ExactReplacement { + enabled?: boolean; /** - * This is the exact replacement type. You can use this to replace a specific word or phrase with a different word or phrase. + * This is how long the request is tried before giving up. When request times out, `call.analysis.successEvaluation` will be empty. * * Usage: - * - Replace "hello" with "hi": { type: 'exact', key: 'hello', value: 'hi' } - * - Replace "good morning" with "good day": { type: 'exact', key: 'good morning', value: 'good day' } - * - Replace a specific name: { type: 'exact', key: 'John Doe', value: 'Jane Smith' } - * - Replace an acronym: { type: 'exact', key: 'AI', value: 'Artificial Intelligence' } - * - Replace a company name with its phonetic pronunciation: { type: 'exact', key: 'Vapi', value: 'Vappy' } - */ - type: 'exact'; - /** - * This option let's you control whether to replace all instances of the key or only the first one. By default, it only replaces the first instance. - * Examples: - * - For { type: 'exact', key: 'hello', value: 'hi', replaceAllEnabled: false }. Before: "hello world, hello universe" | After: "hi world, hello universe" - * - For { type: 'exact', key: 'hello', value: 'hi', replaceAllEnabled: true }. Before: "hello world, hello universe" | After: "hi world, hi universe" - * @default false - * @default false + * - To guarantee the success evaluation is generated, set this value high. Note, this will delay the end of call report in cases where model is slow to respond. + * + * @default 5 seconds + * @min 1 + * @max 60 */ - replaceAllEnabled?: boolean; - /** This is the key to replace. */ - key: string; + timeoutSeconds?: number; +} + +export interface AnalysisPlan { /** - * This is the value that will replace the match. - * @maxLength 1000 + * The minimum number of messages required to run the analysis plan. + * If the number of messages is less than this, analysis will be skipped. + * @default 2 + * @min 0 */ - value: string; + minMessagesThreshold?: number; + /** This is the plan for generating the summary of the call. This outputs to `call.analysis.summary`. */ + summaryPlan?: SummaryPlan; + /** This is the plan for generating the structured data from the call. This outputs to `call.analysis.structuredData`. */ + structuredDataPlan?: StructuredDataPlan; + /** This is an array of structured data plan catalogs. Each entry includes a `key` and a `plan` for generating the structured data from the call. This outputs to `call.analysis.structuredDataMulti`. */ + structuredDataMultiPlan?: StructuredDataMultiPlan[]; + /** This is the plan for generating the success evaluation of the call. This outputs to `call.analysis.successEvaluation`. */ + successEvaluationPlan?: SuccessEvaluationPlan; } export interface RegexOption { @@ -6248,688 +6271,1797 @@ export interface RegexOption { enabled: boolean; } -export interface RegexReplacement { +export interface AssistantCustomEndpointingRule { /** - * This is the regex replacement type. You can use this to replace a word or phrase that matches a pattern. + * This endpointing rule is based on the last assistant message before customer started speaking. + * + * Flow: + * - Assistant speaks + * - Customer starts speaking + * - Customer transcription comes in + * - This rule is evaluated on the last assistant message + * - If a match is found based on `regex`, the endpointing timeout is set to `timeoutSeconds` * * Usage: - * - Replace all numbers with "some number": { type: 'regex', regex: '\\d+', value: 'some number' } - * - Replace email addresses with "[EMAIL]": { type: 'regex', regex: '\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b', value: '[EMAIL]' } - * - Replace phone numbers with a formatted version: { type: 'regex', regex: '(\\d{3})(\\d{3})(\\d{4})', value: '($1) $2-$3' } - * - Replace all instances of "color" or "colour" with "hue": { type: 'regex', regex: 'colou?r', value: 'hue' } - * - Capitalize the first letter of every sentence: { type: 'regex', regex: '(?<=\\. |^)[a-z]', value: (match) => match.toUpperCase() } + * - If you have yes/no questions in your use case like "are you interested in a loan?", you can set a shorter timeout. + * - If you have questions where the customer may pause to look up information like "what's my account number?", you can set a longer timeout. */ - type: 'regex'; + type: 'assistant'; /** - * This is the regex pattern to replace. + * This is the regex pattern to match. * * Note: - * - This works by using the `string.replace` method in Node.JS. Eg. `"hello there".replace(/hello/g, "hi")` will return `"hi there"`. + * - This works by using the `RegExp.test` method in Node.JS. Eg. `/hello/.test("hello there")` will return `true`. * * Hot tip: * - In JavaScript, escape `\` when sending the regex pattern. Eg. `"hello\sthere"` will be sent over the wire as `"hellosthere"`. Send `"hello\\sthere"` instead. + * - `RegExp.test` does substring matching, so `/cat/.test("I love cats")` will return `true`. To do full string matching, send "^cat$". */ regex: string; /** - * These are the options for the regex replacement. Defaults to all disabled. + * These are the options for the regex match. Defaults to all disabled. * * @default [] */ - options?: RegexOption[]; + regexOptions?: RegexOption[]; /** - * This is the value that will replace the match. - * @maxLength 1000 + * This is the endpointing timeout in seconds, if the rule is matched. + * @min 0 + * @max 15 */ - value: string; + timeoutSeconds: number; } -export interface FormatPlan { +export interface CustomerCustomEndpointingRule { /** - * This determines whether the chunk is formatted before being sent to the voice provider. This helps with enunciation. This includes phone numbers, emails and addresses. Default `true`. - * - * Usage: - * - To rely on the voice provider's formatting logic, set this to `false`. + * This endpointing rule is based on current customer message as they are speaking. * - * If `voice.chunkPlan.enabled` is `false`, this is automatically `false` since there's no chunk to format. + * Flow: + * - Assistant speaks + * - Customer starts speaking + * - Customer transcription comes in + * - This rule is evaluated on the current customer transcription + * - If a match is found based on `regex`, the endpointing timeout is set to `timeoutSeconds` * - * @default true - * @example true + * Usage: + * - If you want to wait longer while customer is speaking numbers, you can set a longer timeout. */ - enabled?: boolean; + type: 'customer'; /** - * This is the cutoff after which a number is converted to individual digits instead of being spoken as words. - * - * Example: - * - If cutoff 2025, "12345" is converted to "1 2 3 4 5" while "1200" is converted to "twelve hundred". + * This is the regex pattern to match. * - * Usage: - * - If your use case doesn't involve IDs like zip codes, set this to a high value. - * - If your use case involves IDs that are shorter than 5 digits, set this to a lower value. + * Note: + * - This works by using the `RegExp.test` method in Node.JS. Eg. `/hello/.test("hello there")` will return `true`. * - * @default 2025 - * @min 0 - * @example 2025 + * Hot tip: + * - In JavaScript, escape `\` when sending the regex pattern. Eg. `"hello\sthere"` will be sent over the wire as `"hellosthere"`. Send `"hello\\sthere"` instead. + * - `RegExp.test` does substring matching, so `/cat/.test("I love cats")` will return `true`. To do full string matching, send "^cat$". */ - numberToDigitsCutoff?: number; + regex: string; /** - * These are the custom replacements you can make to the chunk before it is sent to the voice provider. - * - * Usage: - * - To replace a specific word or phrase with a different word or phrase, use the `ExactReplacement` type. Eg. `{ type: 'exact', key: 'hello', value: 'hi' }` - * - To replace a word or phrase that matches a pattern, use the `RegexReplacement` type. Eg. `{ type: 'regex', regex: '\\b[a-zA-Z]{5}\\b', value: 'hi' }` + * These are the options for the regex match. Defaults to all disabled. * * @default [] */ - replacements?: (ExactReplacement | RegexReplacement)[]; + regexOptions?: RegexOption[]; /** - * List of formatters to apply. If not provided, all default formatters will be applied. - * If provided, only the specified formatters will be applied. - * Note: Some essential formatters like angle bracket removal will always be applied. - * @default undefined + * This is the endpointing timeout in seconds, if the rule is matched. + * @min 0 + * @max 15 */ - formattersEnabled?: - | 'markdown' - | 'asterisk' - | 'quote' - | 'dash' - | 'newline' - | 'colon' - | 'acronym' - | 'dollarAmount' - | 'email' - | 'date' - | 'time' - | 'distance' - | 'unit' - | 'percentage' - | 'phoneNumber' - | 'number' - | 'stripAsterisk'; + timeoutSeconds: number; } -export interface ChunkPlan { +export interface BothCustomEndpointingRule { /** - * This determines whether the model output is chunked before being sent to the voice provider. Default `true`. - * - * Usage: - * - To rely on the voice provider's audio generation logic, set this to `false`. - * - If seeing issues with quality, set this to `true`. + * This endpointing rule is based on both the last assistant message and the current customer message as they are speaking. * - * If disabled, Vapi-provided audio control tokens like will not work. + * Flow: + * - Assistant speaks + * - Customer starts speaking + * - Customer transcription comes in + * - This rule is evaluated on the last assistant message and the current customer transcription + * - If assistant message matches `assistantRegex` AND customer message matches `customerRegex`, the endpointing timeout is set to `timeoutSeconds` * - * @default true - * @example true + * Usage: + * - If you want to wait longer while customer is speaking numbers, you can set a longer timeout. */ - enabled?: boolean; + type: 'both'; /** - * This is the minimum number of characters in a chunk. + * This is the regex pattern to match the assistant's message. * - * Usage: - * - To increase quality, set this to a higher value. - * - To decrease latency, set this to a lower value. + * Note: + * - This works by using the `RegExp.test` method in Node.JS. Eg. `/hello/.test("hello there")` will return `true`. * - * @default 30 - * @min 1 - * @max 80 - * @example 30 + * Hot tip: + * - In JavaScript, escape `\` when sending the regex pattern. Eg. `"hello\sthere"` will be sent over the wire as `"hellosthere"`. Send `"hello\\sthere"` instead. + * - `RegExp.test` does substring matching, so `/cat/.test("I love cats")` will return `true`. To do full string matching, send "^cat$". */ - minCharacters?: number; + assistantRegex: string; /** - * These are the punctuations that are considered valid boundaries for a chunk to be created. - * - * Usage: - * - To increase quality, constrain to fewer boundaries. - * - To decrease latency, enable all. + * These are the options for the assistant's message regex match. Defaults to all disabled. * - * Default is automatically set to balance the trade-off between quality and latency based on the provider. - * @example ["。","，",".","!","?",";","،","۔","।","॥","|","||",",",":"] + * @default [] */ - punctuationBoundaries?: - | '。' - | '，' - | '.' - | '!' - | '?' - | ';' - | ')' - | '،' - | '۔' - | '।' - | '॥' - | '|' - | '||' - | ',' - | ':'; - /** This is the plan for formatting the chunk before it is sent to the voice provider. */ - formatPlan?: FormatPlan; -} - -export interface FallbackPlan { - /** This is the list of voices to fallback to in the event that the primary voice provider fails. */ - voices: ( - | FallbackAzureVoice - | FallbackCartesiaVoice - | FallbackHumeVoice - | FallbackCustomVoice - | FallbackDeepgramVoice - | FallbackElevenLabsVoice - | FallbackVapiVoice - | FallbackLMNTVoice - | FallbackOpenAIVoice - | FallbackPlayHTVoice - | FallbackRimeAIVoice - | FallbackSmallestAIVoice - | FallbackTavusVoice - )[]; -} - -export interface AzureVoice { + assistantRegexOptions?: RegexOption[]; + customerRegex: string; /** - * This is the flag to toggle voice caching for the assistant. - * @default true - * @example true + * These are the options for the customer's message regex match. Defaults to all disabled. + * + * @default [] */ - cachingEnabled?: boolean; - /** This is the voice provider that will be used. */ - provider: 'azure'; - /** This is the provider-specific ID that will be used. */ - voiceId: 'andrew' | 'brian' | 'emma' | string; - /** This is the plan for chunking the model output before it is sent to the voice provider. */ - chunkPlan?: ChunkPlan; + customerRegexOptions?: RegexOption[]; /** - * This is the speed multiplier that will be used. - * @min 0.5 - * @max 2 + * This is the endpointing timeout in seconds, if the rule is matched. + * @min 0 + * @max 15 */ - speed?: number; - /** This is the plan for voice provider fallbacks in the event that the primary voice provider fails. */ - fallbackPlan?: FallbackPlan; -} - -export interface CartesiaExperimentalControls { - speed?: 'slowest' | 'slow' | 'normal' | 'fast' | 'fastest' | number; - /** @example ["happiness:high"] */ - emotion?: - | 'anger:lowest' - | 'anger:low' - | 'anger:high' - | 'anger:highest' - | 'positivity:lowest' - | 'positivity:low' - | 'positivity:high' - | 'positivity:highest' - | 'surprise:lowest' - | 'surprise:low' - | 'surprise:high' - | 'surprise:highest' - | 'sadness:lowest' - | 'sadness:low' - | 'sadness:high' - | 'sadness:highest' - | 'curiosity:lowest' - | 'curiosity:low' - | 'curiosity:high' - | 'curiosity:highest'; + timeoutSeconds: number; } -export interface CartesiaVoice { +export interface VapiSmartEndpointingPlan { /** - * This is the flag to toggle voice caching for the assistant. - * @default true - * @example true + * This is the provider for the smart endpointing plan. + * @example "vapi" */ - cachingEnabled?: boolean; - /** This is the voice provider that will be used. */ - provider: 'cartesia'; - /** The ID of the particular voice you want to use. */ - voiceId: string; + provider: 'vapi' | 'livekit'; +} + +export interface LivekitSmartEndpointingPlan { /** - * This is the model that will be used. This is optional and will default to the correct model for the voiceId. - * @example "sonic-english" + * This is the provider for the smart endpointing plan. + * @example "livekit" */ - model?: 'sonic-2' | 'sonic-english' | 'sonic-multilingual' | 'sonic-preview' | 'sonic'; + provider: 'vapi' | 'livekit'; /** - * This is the language that will be used. This is optional and will default to the correct language for the voiceId. - * @example "en" + * This expression describes how long the bot will wait to start speaking based on the likelihood that the user has reached an endpoint. + * + * This is a millisecond valued function. It maps probabilities (real numbers on [0,1]) to milliseconds that the bot should wait before speaking ([0, \infty]). Any negative values that are returned are set to zero (the bot can't start talking in the past). + * + * A probability of zero represents very high confidence that the caller has stopped speaking, and would like the bot to speak to them. A probability of one represents very high confidence that the caller is still speaking. + * + * Under the hood, this is parsed into a mathjs expression. Whatever you use to write your expression needs to be valid with respect to mathjs + * + * @default "20 + 500 * sqrt(x) + 2500 * x^3" */ - language?: - | 'en' - | 'de' - | 'es' - | 'fr' - | 'ja' - | 'pt' - | 'zh' - | 'hi' - | 'it' - | 'ko' - | 'nl' - | 'pl' - | 'ru' - | 'sv' - | 'tr'; - /** Experimental controls for Cartesia voice generation */ - experimentalControls?: CartesiaExperimentalControls; - /** This is the plan for chunking the model output before it is sent to the voice provider. */ - chunkPlan?: ChunkPlan; - /** This is the plan for voice provider fallbacks in the event that the primary voice provider fails. */ - fallbackPlan?: FallbackPlan; + waitFunction?: string; } -export interface CustomVoice { +export interface TranscriptionEndpointingPlan { /** - * This is the flag to toggle voice caching for the assistant. - * @default true - * @example true + * The minimum number of seconds to wait after transcription ending with punctuation before sending a request to the model. Defaults to 0.1. + * + * This setting exists because the transcriber punctuates the transcription when it's more confident that customer has completed a thought. + * + * @default 0.1 + * @min 0 + * @max 3 + * @example 0.1 */ - cachingEnabled?: boolean; - /** This is the voice provider that will be used. Use `custom-voice` for providers that are not natively supported. */ - provider: 'custom-voice'; - /** This is the plan for chunking the model output before it is sent to the voice provider. */ - chunkPlan?: ChunkPlan; + onPunctuationSeconds?: number; /** - * This is where the voice request will be sent. + * The minimum number of seconds to wait after transcription ending without punctuation before sending a request to the model. Defaults to 1.5. * - * Request Example: + * This setting exists to catch the cases where the transcriber was not confident enough to punctuate the transcription, but the customer is done and has been silent for a long time. * - * POST https://{server.url} - * Content-Type: application/json + * @default 1.5 + * @min 0 + * @max 3 + * @example 1.5 + */ + onNoPunctuationSeconds?: number; + /** + * The minimum number of seconds to wait after transcription ending with a number before sending a request to the model. Defaults to 0.4. * - * { - * "message": { - * "type": "voice-request", - * "text": "Hello, world!", - * "sampleRate": 24000, - * ...other metadata about the call... - * } - * } + * This setting exists because the transcriber will sometimes punctuate the transcription ending with a number, even though the customer hasn't uttered the full number. This happens commonly for long numbers when the customer reads the number in chunks. * - * Response Expected: 1-channel 16-bit raw PCM audio at the sample rate specified in the request. Here is how the response will be piped to the transport: - * ``` - * response.on('data', (chunk: Buffer) => { - * outputStream.write(chunk); - * }); - * ``` + * @default 0.5 + * @min 0 + * @max 3 + * @example 0.5 */ - server: Server; - /** This is the plan for voice provider fallbacks in the event that the primary voice provider fails. */ - fallbackPlan?: FallbackPlan; + onNumberSeconds?: number; } -export interface DeepgramVoice { +export interface StartSpeakingPlan { /** - * This is the flag to toggle voice caching for the assistant. - * @default true - * @example true - */ - cachingEnabled?: boolean; - /** This is the voice provider that will be used. */ - provider: 'deepgram'; + * This is how long assistant waits before speaking. Defaults to 0.4. + * + * This is the minimum it will wait but if there is latency is the pipeline, this minimum will be exceeded. This is intended as a stopgap in case the pipeline is moving too fast. + * + * Example: + * - If model generates tokens and voice generates bytes within 100ms, the pipeline still waits 300ms before outputting speech. + * + * Usage: + * - If the customer is taking long pauses, set this to a higher value. + * - If the assistant is accidentally jumping in too much, set this to a higher value. + * + * @default 0.4 + * @min 0 + * @max 5 + * @example 0.4 + */ + waitSeconds?: number; /** - * This is the Deepgram Voice ID - * This is the provider-specific ID that will be used. + * @deprecated + * @example false */ - voiceId: - | 'asteria' - | 'luna' - | 'stella' - | 'athena' - | 'hera' - | 'orion' - | 'arcas' - | 'perseus' - | 'angus' - | 'orpheus' - | 'helios' - | 'zeus' - | 'thalia' - | 'andromeda' - | 'helena' - | 'apollo' - | 'aries' - | 'amalthea' - | 'atlas' - | 'aurora' - | 'callista' - | 'cora' - | 'cordelia' - | 'delia' - | 'draco' - | 'electra' - | 'harmonia' - | 'hermes' - | 'hyperion' - | 'iris' - | 'janus' - | 'juno' - | 'jupiter' - | 'mars' - | 'minerva' - | 'neptune' - | 'odysseus' - | 'ophelia' - | 'pandora' - | 'phoebe' - | 'pluto' - | 'saturn' - | 'selene' - | 'theia' - | 'vesta'; + smartEndpointingEnabled?: boolean | 'livekit'; /** - * This is the model that will be used. Defaults to 'aura-2' when not specified. - * @example "aura-2" + * This is the plan for smart endpointing. Pick between Vapi smart endpointing or LiveKit smart endpointing (or nothing). We strongly recommend using livekit endpointing when working in English. LiveKit endpointing is not supported in other languages, yet. + * + * If this is set, it will override and take precedence over `transcriptionEndpointingPlan`. + * This plan will still be overridden by any matching `customEndpointingRules`. */ - model?: 'aura' | 'aura-2'; + smartEndpointingPlan?: VapiSmartEndpointingPlan | LivekitSmartEndpointingPlan; /** - * If set to true, this will add mip_opt_out=true as a query parameter of all API requests. See https://developers.deepgram.com/docs/the-deepgram-model-improvement-partnership-program#want-to-opt-out + * These are the custom endpointing rules to set an endpointing timeout based on a regex on the customer's speech or the assistant's last message. * - * This will only be used if you are using your own Deepgram API key. + * Usage: + * - If you have yes/no questions like "are you interested in a loan?", you can set a shorter timeout. + * - If you have questions where the customer may pause to look up information like "what's my account number?", you can set a longer timeout. + * - If you want to wait longer while customer is enumerating a list of numbers, you can set a longer timeout. * - * @default false - * @default false - * @example false + * These rules have the highest precedence and will override both `smartEndpointingPlan` and `transcriptionEndpointingPlan` when a rule is matched. + * + * The rules are evaluated in order and the first one that matches will be used. + * + * Order of precedence for endpointing: + * 1. customEndpointingRules (if any match) + * 2. smartEndpointingPlan (if set) + * 3. transcriptionEndpointingPlan + * + * @default [] */ - mipOptOut?: boolean; - /** This is the plan for chunking the model output before it is sent to the voice provider. */ - chunkPlan?: ChunkPlan; - /** This is the plan for voice provider fallbacks in the event that the primary voice provider fails. */ - fallbackPlan?: FallbackPlan; -} - -export interface ElevenLabsVoice { + customEndpointingRules?: ( + | AssistantCustomEndpointingRule + | CustomerCustomEndpointingRule + | BothCustomEndpointingRule + )[]; /** - * This is the flag to toggle voice caching for the assistant. - * @default true - * @example true + * This determines how a customer speech is considered done (endpointing) using the transcription of customer's speech. + * + * Once an endpoint is triggered, the request is sent to `assistant.model`. + * + * Note: This plan is only used if `smartEndpointingPlan` is not set. If both are provided, `smartEndpointingPlan` takes precedence. + * This plan will also be overridden by any matching `customEndpointingRules`. */ - cachingEnabled?: boolean; - /** This is the voice provider that will be used. */ - provider: '11labs'; - /** This is the provider-specific ID that will be used. Ensure the Voice is present in your 11Labs Voice Library. */ - voiceId: - | 'burt' - | 'marissa' - | 'andrea' - | 'sarah' - | 'phillip' - | 'steve' - | 'joseph' - | 'myra' - | 'paula' - | 'ryan' - | 'drew' - | 'paul' - | 'mrb' - | 'matilda' - | 'mark' - | string; + transcriptionEndpointingPlan?: TranscriptionEndpointingPlan; +} + +export interface StopSpeakingPlan { /** - * Defines the stability for voice settings. + * This is the number of words that the customer has to say before the assistant will stop talking. + * + * Words like "stop", "actually", "no", etc. will always interrupt immediately regardless of this value. + * + * Words like "okay", "yeah", "right" will never interrupt. + * + * When set to 0, `voiceSeconds` is used in addition to the transcriptions to determine the customer has started speaking. + * + * Defaults to 0. + * + * @default 0 * @min 0 - * @max 1 - * @example 0.5 + * @max 10 + * @example 0 */ - stability?: number; + numWords?: number; /** - * Defines the similarity boost for voice settings. + * This is the seconds customer has to speak before the assistant stops talking. This uses the VAD (Voice Activity Detection) spike to determine if the customer has started speaking. + * + * Considerations: + * - A lower value might be more responsive but could potentially pick up non-speech sounds. + * - A higher value reduces false positives but might slightly delay the detection of speech onset. + * + * This is only used if `numWords` is set to 0. + * + * Defaults to 0.2 + * + * @default 0.2 * @min 0 - * @max 1 - * @example 0.75 + * @max 0.5 + * @example 0.2 */ - similarityBoost?: number; + voiceSeconds?: number; /** - * Defines the style for voice settings. + * This is the seconds to wait before the assistant will start talking again after being interrupted. + * + * Defaults to 1. + * + * @default 1 * @min 0 - * @max 1 - * @example 0 + * @max 10 + * @example 1 */ - style?: number; + backoffSeconds?: number; /** - * Defines the use speaker boost for voice settings. - * @example false + * These are the phrases that will never interrupt the assistant, even if numWords threshold is met. + * These are typically acknowledgement or backchanneling phrases. + * @default ["i understand","i see","i got it","i hear you","im listening","im with you","right","okay","ok","sure","alright","got it","understood","yeah","yes","uh-huh","mm-hmm","gotcha","mhmm","ah","yeah okay","yeah sure"] + * @example ["i understand","i see","i got it","i hear you","im listening","im with you","right","okay","ok","sure","alright","got it","understood","yeah","yes","uh-huh","mm-hmm","gotcha","mhmm","ah","yeah okay","yeah sure"] */ - useSpeakerBoost?: boolean; + acknowledgementPhrases?: string[]; /** - * Defines the speed for voice settings. - * @min 0.7 - * @max 1.2 - * @example 0.9 + * These are the phrases that will always interrupt the assistant immediately, regardless of numWords. + * These are typically phrases indicating disagreement or desire to stop. + * @default ["stop","shut","up","enough","quiet","silence","but","dont","not","no","hold","wait","cut","pause","nope","nah","nevermind","never","bad","actually"] + * @example ["stop","shut","up","enough","quiet","silence","but","dont","not","no","hold","wait","cut","pause","nope","nah","nevermind","never","bad","actually"] */ - speed?: number; + interruptionPhrases?: string[]; +} + +export interface MonitorPlan { /** - * Defines the optimize streaming latency for voice settings. Defaults to 3. - * @min 0 - * @max 4 - * @example 3 + * This determines whether the assistant's calls allow live listening. Defaults to true. + * + * Fetch `call.monitor.listenUrl` to get the live listening URL. + * + * @default true + * @example false */ - optimizeStreamingLatency?: number; + listenEnabled?: boolean; /** - * This enables the use of https://elevenlabs.io/docs/speech-synthesis/prompting#pronunciation. Defaults to false to save latency. + * This enables authentication on the `call.monitor.listenUrl`. + * + * If `listenAuthenticationEnabled` is `true`, the `call.monitor.listenUrl` will require an `Authorization: Bearer ` header. * * @default false * @example false */ - enableSsmlParsing?: boolean; + listenAuthenticationEnabled?: boolean; /** - * Defines the auto mode for voice settings. Defaults to false. + * This determines whether the assistant's calls allow live control. Defaults to true. + * + * Fetch `call.monitor.controlUrl` to get the live control URL. + * + * To use, send any control message via a POST request to `call.monitor.controlUrl`. Here are the types of controls supported: https://docs.vapi.ai/api-reference/messages/client-inbound-message + * + * @default true * @example false */ - autoMode?: boolean; + controlEnabled?: boolean; /** - * This is the model that will be used. Defaults to 'eleven_turbo_v2' if not specified. - * @example "eleven_turbo_v2_5" + * This enables authentication on the `call.monitor.controlUrl`. + * + * If `controlAuthenticationEnabled` is `true`, the `call.monitor.controlUrl` will require an `Authorization: Bearer ` header. + * + * @default false + * @example false */ - model?: - | 'eleven_multilingual_v2' - | 'eleven_turbo_v2' - | 'eleven_turbo_v2_5' - | 'eleven_flash_v2' - | 'eleven_flash_v2_5' - | 'eleven_monolingual_v1'; - /** This is the plan for chunking the model output before it is sent to the voice provider. */ - chunkPlan?: ChunkPlan; - /** This is the language (ISO 639-1) that is enforced for the model. Currently only Turbo v2.5 supports language enforcement. For other models, an error will be returned if language code is provided. */ - language?: string; - /** This is the plan for voice provider fallbacks in the event that the primary voice provider fails. */ - fallbackPlan?: FallbackPlan; + controlAuthenticationEnabled?: boolean; } -export interface HumeVoice { - /** - * This is the flag to toggle voice caching for the assistant. - * @default true - * @example true - */ - cachingEnabled?: boolean; - /** This is the voice provider that will be used. */ - provider: 'hume'; - /** - * This is the model that will be used. - * @example "octave" - */ - model?: 'octave'; - /** The ID of the particular voice you want to use. */ - voiceId: string; - /** - * Indicates whether the chosen voice is a preset Hume AI voice or a custom voice. - * @example false - */ - isCustomHumeVoice?: boolean; - /** This is the plan for chunking the model output before it is sent to the voice provider. */ - chunkPlan?: ChunkPlan; +export interface SmartDenoisingPlan { /** - * Natural language instructions describing how the synthesized speech should sound, including but not limited to tone, intonation, pacing, and accent (e.g., 'a soft, gentle voice with a strong British accent'). - * - * If a Voice is specified in the request, this description serves as acting instructions. - * If no Voice is specified, a new voice is generated based on this description. + * Whether smart denoising using Krisp is enabled. + * @default false */ - description?: string; - /** This is the plan for voice provider fallbacks in the event that the primary voice provider fails. */ - fallbackPlan?: FallbackPlan; + enabled?: boolean; } -export interface LMNTVoice { - /** - * This is the flag to toggle voice caching for the assistant. - * @default true - * @example true - */ - cachingEnabled?: boolean; - /** This is the voice provider that will be used. */ - provider: 'lmnt'; - /** This is the provider-specific ID that will be used. */ - voiceId: 'lily' | 'daniel' | string; +export interface FourierDenoisingPlan { /** - * This is the speed multiplier that will be used. - * @min 0.25 - * @max 2 - * @example null + * Whether Fourier denoising is enabled. Note that this is experimental and may not work as expected. + * @default false */ - speed?: number; - /** This is the plan for chunking the model output before it is sent to the voice provider. */ - chunkPlan?: ChunkPlan; - /** This is the plan for voice provider fallbacks in the event that the primary voice provider fails. */ - fallbackPlan?: FallbackPlan; -} - -export interface NeuphonicVoice { + enabled?: boolean; /** - * This is the flag to toggle voice caching for the assistant. + * Whether automatic media detection is enabled. When enabled, the filter will automatically + * detect consistent background TV/music/radio and switch to more aggressive filtering settings. + * Only applies when enabled is true. * @default true * @example true */ - cachingEnabled?: boolean; - /** This is the voice provider that will be used. */ - provider: 'neuphonic'; - /** This is the provider-specific ID that will be used. */ - voiceId: string; - /** - * This is the model that will be used. Defaults to 'neu_fast' if not specified. - * @example "neu_fast" - */ - model?: 'neu_hq' | 'neu_fast'; - /** - * This is the language (ISO 639-1) that is enforced for the model. - * @example "en" - */ - language: object; - /** - * This is the speed multiplier that will be used. - * @min 0.25 - * @max 2 - * @example null - */ - speed?: number; - /** This is the plan for chunking the model output before it is sent to the voice provider. */ - chunkPlan?: ChunkPlan; - /** This is the plan for voice provider fallbacks in the event that the primary voice provider fails. */ - fallbackPlan?: FallbackPlan; -} - -export interface OpenAIVoice { + mediaDetectionEnabled?: boolean; /** - * This is the flag to toggle voice caching for the assistant. - * @default true - * @example true + * Static threshold in dB used as fallback when no baseline is established. + * @min -80 + * @max 0 + * @default -35 + * @example -35 */ - cachingEnabled?: boolean; - /** This is the voice provider that will be used. */ - provider: 'openai'; + staticThreshold?: number; /** - * This is the provider-specific ID that will be used. - * Please note that ash, ballad, coral, sage, and verse may only be used with realtime models. + * How far below the rolling baseline to filter audio, in dB. + * Lower values (e.g., -10) are more aggressive, higher values (e.g., -20) are more conservative. + * @min -30 + * @max -5 + * @default -15 + * @example -15 */ - voiceId: 'alloy' | 'echo' | 'fable' | 'onyx' | 'nova' | 'shimmer' | string; - /** This is the model that will be used for text-to-speech. */ - model?: 'tts-1' | 'tts-1-hd' | 'gpt-4o-mini-tts'; + baselineOffsetDb?: number; /** - * This is a prompt that allows you to control the voice of your generated audio. - * Does not work with 'tts-1' or 'tts-1-hd' models. - * @maxLength 10000 + * Rolling window size in milliseconds for calculating the audio baseline. + * Larger windows adapt more slowly but are more stable. + * @min 1000 + * @max 30000 + * @default 3000 + * @example 3000 */ - instructions?: string; + windowSizeMs?: number; /** - * This is the speed multiplier that will be used. - * @min 0.25 - * @max 4 - * @example null + * Percentile to use for baseline calculation (1-99). + * Higher percentiles (e.g., 85) focus on louder speech, lower percentiles (e.g., 50) include quieter speech. + * @min 1 + * @max 99 + * @default 85 + * @example 85 */ - speed?: number; - /** This is the plan for chunking the model output before it is sent to the voice provider. */ - chunkPlan?: ChunkPlan; - /** This is the plan for voice provider fallbacks in the event that the primary voice provider fails. */ - fallbackPlan?: FallbackPlan; + baselinePercentile?: number; } -export interface PlayHTVoice { - /** - * This is the flag to toggle voice caching for the assistant. - * @default true - * @example true - */ - cachingEnabled?: boolean; - /** This is the voice provider that will be used. */ - provider: 'playht'; - /** This is the provider-specific ID that will be used. */ - voiceId: - | 'jennifer' - | 'melissa' - | 'will' - | 'chris' - | 'matt' - | 'jack' - | 'ruby' - | 'davis' - | 'donna' - | 'michael' - | string; - /** - * This is the speed multiplier that will be used. - * @min 0.1 - * @max 5 - * @example null - */ - speed?: number; +export interface BackgroundSpeechDenoisingPlan { + /** Whether smart denoising using Krisp is enabled. */ + smartDenoisingPlan?: SmartDenoisingPlan; /** - * A floating point number between 0, exclusive, and 2, inclusive. If equal to null or not provided, the model's default temperature will be used. The temperature parameter controls variance. Lower temperatures result in more predictable results, higher temperatures allow each run to vary more, so the voice may sound less like the baseline voice. - * @min 0.1 - * @max 2 - * @example null + * Whether Fourier denoising is enabled. Note that this is experimental and may not work as expected. + * + * This can be combined with smart denoising, and will be run afterwards. */ - temperature?: number; + fourierDenoisingPlan?: FourierDenoisingPlan; +} + +export interface WorkflowUserEditable { + nodes: (ConversationNode | ToolNode)[]; /** - * An emotion to be applied to the speech. - * @example null + * This is the transcriber for the workflow. + * + * This can be overridden at node level using `nodes[n].transcriber`. */ - emotion?: - | 'female_happy' - | 'female_sad' - | 'female_angry' - | 'female_fearful' - | 'female_disgust' - | 'female_surprised' - | 'male_happy' - | 'male_sad' - | 'male_angry' - | 'male_fearful' - | 'male_disgust' - | 'male_surprised'; + transcriber?: + | AssemblyAITranscriber + | AzureSpeechTranscriber + | CustomTranscriber + | DeepgramTranscriber + | ElevenLabsTranscriber + | GladiaTranscriber + | GoogleTranscriber + | SpeechmaticsTranscriber + | TalkscriberTranscriber + | OpenAITranscriber + | CartesiaTranscriber; /** - * A number between 1 and 6. Use lower numbers to reduce how unique your chosen voice will be compared to other voices. - * @min 1 - * @max 6 - * @example null + * This is the voice for the workflow. + * + * This can be overridden at node level using `nodes[n].voice`. */ - voiceGuidance?: number; + voice?: + | AzureVoice + | CartesiaVoice + | CustomVoice + | DeepgramVoice + | ElevenLabsVoice + | HumeVoice + | LMNTVoice + | NeuphonicVoice + | OpenAIVoice + | PlayHTVoice + | RimeAIVoice + | SmallestAIVoice + | TavusVoice + | VapiVoice + | SesameVoice; /** - * A number between 1 and 30. Use lower numbers to to reduce how strong your chosen emotion will be. Higher numbers will create a very emotional performance. - * @min 1 - * @max 30 - * @example null + * This is the plan for observability of workflow's calls. + * + * Currently, only Langfuse is supported. */ - styleGuidance?: number; - /** - * A number between 1 and 2. This number influences how closely the generated speech adheres to the input text. Use lower values to create more fluid speech, but with a higher chance of deviating from the input text. Higher numbers will make the generated speech more accurate to the input text, ensuring that the words spoken align closely with the provided text. + observabilityPlan?: LangfuseObservabilityPlan; + /** These are dynamic credentials that will be used for the workflow calls. By default, all the credentials are available for use in the call but you can supplement an additional credentials using this. Dynamic credentials override existing credentials. */ + credentials?: ( + | ({ + provider: '11labs'; + } & CreateElevenLabsCredentialDTO) + | ({ + provider: 'anthropic'; + } & CreateAnthropicCredentialDTO) + | ({ + provider: 'anyscale'; + } & CreateAnyscaleCredentialDTO) + | ({ + provider: 'assembly-ai'; + } & CreateAssemblyAICredentialDTO) + | ({ + provider: 'azure-openai'; + } & CreateAzureOpenAICredentialDTO) + | ({ + provider: 'azure'; + } & CreateAzureCredentialDTO) + | ({ + provider: 'byo-sip-trunk'; + } & CreateByoSipTrunkCredentialDTO) + | ({ + provider: 'cartesia'; + } & CreateCartesiaCredentialDTO) + | ({ + provider: 'cerebras'; + } & CreateCerebrasCredentialDTO) + | ({ + provider: 'cloudflare'; + } & CreateCloudflareCredentialDTO) + | ({ + provider: 'custom-llm'; + } & CreateCustomLLMCredentialDTO) + | ({ + provider: 'deepgram'; + } & CreateDeepgramCredentialDTO) + | ({ + provider: 'deepinfra'; + } & CreateDeepInfraCredentialDTO) + | ({ + provider: 'deep-seek'; + } & CreateDeepSeekCredentialDTO) + | ({ + provider: 'gcp'; + } & CreateGcpCredentialDTO) + | ({ + provider: 'gladia'; + } & CreateGladiaCredentialDTO) + | ({ + provider: 'gohighlevel'; + } & CreateGoHighLevelCredentialDTO) + | ({ + provider: 'google'; + } & CreateGoogleCredentialDTO) + | ({ + provider: 'groq'; + } & CreateGroqCredentialDTO) + | ({ + provider: 'inflection-ai'; + } & CreateInflectionAICredentialDTO) + | ({ + provider: 'langfuse'; + } & CreateLangfuseCredentialDTO) + | ({ + provider: 'lmnt'; + } & CreateLmntCredentialDTO) + | ({ + provider: 'make'; + } & CreateMakeCredentialDTO) + | ({ + provider: 'openai'; + } & CreateOpenAICredentialDTO) + | ({ + provider: 'openrouter'; + } & CreateOpenRouterCredentialDTO) + | ({ + provider: 'perplexity-ai'; + } & CreatePerplexityAICredentialDTO) + | ({ + provider: 'playht'; + } & CreatePlayHTCredentialDTO) + | ({ + provider: 'rime-ai'; + } & CreateRimeAICredentialDTO) + | ({ + provider: 'runpod'; + } & CreateRunpodCredentialDTO) + | ({ + provider: 's3'; + } & CreateS3CredentialDTO) + | ({ + provider: 'supabase'; + } & CreateSupabaseCredentialDTO) + | ({ + provider: 'smallest-ai'; + } & CreateSmallestAICredentialDTO) + | ({ + provider: 'tavus'; + } & CreateTavusCredentialDTO) + | ({ + provider: 'together-ai'; + } & CreateTogetherAICredentialDTO) + | ({ + provider: 'twilio'; + } & CreateTwilioCredentialDTO) + | ({ + provider: 'vonage'; + } & CreateVonageCredentialDTO) + | ({ + provider: 'webhook'; + } & CreateWebhookCredentialDTO) + | ({ + provider: 'xai'; + } & CreateXAiCredentialDTO) + | ({ + provider: 'neuphonic'; + } & CreateNeuphonicCredentialDTO) + | ({ + provider: 'hume'; + } & CreateHumeCredentialDTO) + | ({ + provider: 'mistral'; + } & CreateMistralCredentialDTO) + | ({ + provider: 'speechmatics'; + } & CreateSpeechmaticsCredentialDTO) + | ({ + provider: 'trieve'; + } & CreateTrieveCredentialDTO) + | ({ + provider: 'google.calendar.oauth2-client'; + } & CreateGoogleCalendarOAuth2ClientCredentialDTO) + | ({ + provider: 'google.calendar.oauth2-authorization'; + } & CreateGoogleCalendarOAuth2AuthorizationCredentialDTO) + | ({ + provider: 'google.sheets.oauth2-authorization'; + } & CreateGoogleSheetsOAuth2AuthorizationCredentialDTO) + | ({ + provider: 'slack.oauth2-authorization'; + } & CreateSlackOAuth2AuthorizationCredentialDTO) + | ({ + provider: 'ghl.oauth2-authorization'; + } & CreateGoHighLevelMCPCredentialDTO) + | ({ + provider: 'inworld'; + } & any) + )[]; + /** @maxLength 80 */ + name: string; + edges: Edge[]; + /** @maxLength 5000 */ + globalPrompt?: string; + /** + * This is where Vapi will send webhooks. You can find all webhooks available along with their shape in ServerMessage schema. + * + * The order of precedence is: + * + * 1. tool.server + * 2. workflow.server / assistant.server + * 3. phoneNumber.server + * 4. org.server + */ + server?: Server; + /** This is the compliance plan for the workflow. It allows you to configure HIPAA and other compliance settings. */ + compliancePlan?: CompliancePlan; + /** This is the plan for analysis of workflow's calls. Stored in `call.analysis`. */ + analysisPlan?: AnalysisPlan; + /** This is the plan for artifacts generated during workflow's calls. Stored in `call.artifact`. */ + artifactPlan?: ArtifactPlan; + /** + * This is the plan for when the workflow nodes should start talking. + * + * You should configure this if you're running into these issues: + * - The assistant is too slow to start talking after the customer is done speaking. + * - The assistant is too fast to start talking after the customer is done speaking. + * - The assistant is so fast that it's actually interrupting the customer. + */ + startSpeakingPlan?: StartSpeakingPlan; + /** + * This is the plan for when workflow nodes should stop talking on customer interruption. + * + * You should configure this if you're running into these issues: + * - The assistant is too slow to recognize customer's interruption. + * - The assistant is too fast to recognize customer's interruption. + * - The assistant is getting interrupted by phrases that are just acknowledgments. + * - The assistant is getting interrupted by background noises. + * - The assistant is not properly stopping -- it starts talking right after getting interrupted. + */ + stopSpeakingPlan?: StopSpeakingPlan; + /** + * This is the plan for real-time monitoring of the workflow's calls. + * + * Usage: + * - To enable live listening of the workflow's calls, set `monitorPlan.listenEnabled` to `true`. + * - To enable live control of the workflow's calls, set `monitorPlan.controlEnabled` to `true`. + */ + monitorPlan?: MonitorPlan; + /** + * This enables filtering of noise and background speech while the user is talking. + * + * Features: + * - Smart denoising using Krisp + * - Fourier denoising + * + * Both can be used together. Order of precedence: + * - Smart denoising + * - Fourier denoising + */ + backgroundSpeechDenoisingPlan?: BackgroundSpeechDenoisingPlan; + /** These are the credentials that will be used for the workflow calls. By default, all the credentials are available for use in the call but you can provide a subset using this. */ + credentialIds?: string[]; +} + +export interface VapiModel { + /** This is the starting state for the conversation. */ + messages?: OpenAIMessage[]; + /** + * These are the tools that the assistant can use during the call. To use existing tools, use `toolIds`. + * + * Both `tools` and `toolIds` can be used together. + */ + tools?: ( + | CreateApiRequestToolDTO + | CreateBashToolDTO + | CreateComputerToolDTO + | CreateDtmfToolDTO + | CreateEndCallToolDTO + | CreateFunctionToolDTO + | CreateGoHighLevelCalendarAvailabilityToolDTO + | CreateGoHighLevelCalendarEventCreateToolDTO + | CreateGoHighLevelContactCreateToolDTO + | CreateGoHighLevelContactGetToolDTO + | CreateGoogleCalendarCheckAvailabilityToolDTO + | CreateGoogleCalendarCreateEventToolDTO + | CreateGoogleSheetsRowAppendToolDTO + | CreateMcpToolDTO + | CreateQueryToolDTO + | CreateSlackSendMessageToolDTO + | CreateSmsToolDTO + | CreateTextEditorToolDTO + | CreateTransferCallToolDTO + )[]; + /** + * These are the tools that the assistant can use during the call. To use transient tools, use `tools`. + * + * Both `tools` and `toolIds` can be used together. + */ + toolIds?: string[]; + /** These are the options for the knowledge base. */ + knowledgeBase?: CreateCustomKnowledgeBaseDTO; + /** This is the ID of the knowledge base the model will use. */ + knowledgeBaseId?: string; + provider: 'vapi'; + /** This is the workflow that will be used for the call. To use a transient workflow, use `workflow` instead. */ + workflowId?: string; + /** This is the workflow that will be used for the call. To use an existing workflow, use `workflowId` instead. */ + workflow?: WorkflowUserEditable; + /** This is the name of the model. Ex. cognitivecomputations/dolphin-mixtral-8x7b */ + model: string; + /** + * This is the temperature that will be used for calls. Default is 0 to leverage caching for lower latency. + * @min 0 + * @max 2 + */ + temperature?: number; + /** + * This is the max number of tokens that the assistant will be allowed to generate in each turn of the conversation. Default is 250. + * @min 50 + * @max 10000 + */ + maxTokens?: number; + /** + * This determines whether we detect user's emotion while they speak and send it as an additional info to model. + * + * Default `false` because the model is usually are good at understanding the user's emotion from text. + * + * @default false + */ + emotionRecognitionEnabled?: boolean; + /** + * This sets how many turns at the start of the conversation to use a smaller, faster model from the same provider before switching to the primary model. Example, gpt-3.5-turbo if provider is openai. + * + * Default is 0. + * + * @default 0 + * @min 0 + */ + numFastTurns?: number; +} + +export interface XaiModel { + /** This is the starting state for the conversation. */ + messages?: OpenAIMessage[]; + /** + * These are the tools that the assistant can use during the call. To use existing tools, use `toolIds`. + * + * Both `tools` and `toolIds` can be used together. + */ + tools?: ( + | CreateApiRequestToolDTO + | CreateBashToolDTO + | CreateComputerToolDTO + | CreateDtmfToolDTO + | CreateEndCallToolDTO + | CreateFunctionToolDTO + | CreateGoHighLevelCalendarAvailabilityToolDTO + | CreateGoHighLevelCalendarEventCreateToolDTO + | CreateGoHighLevelContactCreateToolDTO + | CreateGoHighLevelContactGetToolDTO + | CreateGoogleCalendarCheckAvailabilityToolDTO + | CreateGoogleCalendarCreateEventToolDTO + | CreateGoogleSheetsRowAppendToolDTO + | CreateMcpToolDTO + | CreateQueryToolDTO + | CreateSlackSendMessageToolDTO + | CreateSmsToolDTO + | CreateTextEditorToolDTO + | CreateTransferCallToolDTO + )[]; + /** + * These are the tools that the assistant can use during the call. To use transient tools, use `tools`. + * + * Both `tools` and `toolIds` can be used together. + */ + toolIds?: string[]; + /** These are the options for the knowledge base. */ + knowledgeBase?: CreateCustomKnowledgeBaseDTO; + /** This is the ID of the knowledge base the model will use. */ + knowledgeBaseId?: string; + /** This is the name of the model. Ex. cognitivecomputations/dolphin-mixtral-8x7b */ + model: 'grok-beta' | 'grok-2' | 'grok-3'; + provider: 'xai'; + /** + * This is the temperature that will be used for calls. Default is 0 to leverage caching for lower latency. + * @min 0 + * @max 2 + */ + temperature?: number; + /** + * This is the max number of tokens that the assistant will be allowed to generate in each turn of the conversation. Default is 250. + * @min 50 + * @max 10000 + */ + maxTokens?: number; + /** + * This determines whether we detect user's emotion while they speak and send it as an additional info to model. + * + * Default `false` because the model is usually are good at understanding the user's emotion from text. + * + * @default false + */ + emotionRecognitionEnabled?: boolean; + /** + * This sets how many turns at the start of the conversation to use a smaller, faster model from the same provider before switching to the primary model. Example, gpt-3.5-turbo if provider is openai. + * + * Default is 0. + * + * @default 0 + * @min 0 + */ + numFastTurns?: number; +} + +export interface ExactReplacement { + /** + * This is the exact replacement type. You can use this to replace a specific word or phrase with a different word or phrase. + * + * Usage: + * - Replace "hello" with "hi": { type: 'exact', key: 'hello', value: 'hi' } + * - Replace "good morning" with "good day": { type: 'exact', key: 'good morning', value: 'good day' } + * - Replace a specific name: { type: 'exact', key: 'John Doe', value: 'Jane Smith' } + * - Replace an acronym: { type: 'exact', key: 'AI', value: 'Artificial Intelligence' } + * - Replace a company name with its phonetic pronunciation: { type: 'exact', key: 'Vapi', value: 'Vappy' } + */ + type: 'exact'; + /** + * This option let's you control whether to replace all instances of the key or only the first one. By default, it only replaces the first instance. + * Examples: + * - For { type: 'exact', key: 'hello', value: 'hi', replaceAllEnabled: false }. Before: "hello world, hello universe" | After: "hi world, hello universe" + * - For { type: 'exact', key: 'hello', value: 'hi', replaceAllEnabled: true }. Before: "hello world, hello universe" | After: "hi world, hi universe" + * @default false + * @default false + */ + replaceAllEnabled?: boolean; + /** This is the key to replace. */ + key: string; + /** + * This is the value that will replace the match. + * @maxLength 1000 + */ + value: string; +} + +export interface RegexReplacement { + /** + * This is the regex replacement type. You can use this to replace a word or phrase that matches a pattern. + * + * Usage: + * - Replace all numbers with "some number": { type: 'regex', regex: '\\d+', value: 'some number' } + * - Replace email addresses with "[EMAIL]": { type: 'regex', regex: '\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b', value: '[EMAIL]' } + * - Replace phone numbers with a formatted version: { type: 'regex', regex: '(\\d{3})(\\d{3})(\\d{4})', value: '($1) $2-$3' } + * - Replace all instances of "color" or "colour" with "hue": { type: 'regex', regex: 'colou?r', value: 'hue' } + * - Capitalize the first letter of every sentence: { type: 'regex', regex: '(?<=\\. |^)[a-z]', value: (match) => match.toUpperCase() } + */ + type: 'regex'; + /** + * This is the regex pattern to replace. + * + * Note: + * - This works by using the `string.replace` method in Node.JS. Eg. `"hello there".replace(/hello/g, "hi")` will return `"hi there"`. + * + * Hot tip: + * - In JavaScript, escape `\` when sending the regex pattern. Eg. `"hello\sthere"` will be sent over the wire as `"hellosthere"`. Send `"hello\\sthere"` instead. + */ + regex: string; + /** + * These are the options for the regex replacement. Defaults to all disabled. + * + * @default [] + */ + options?: RegexOption[]; + /** + * This is the value that will replace the match. + * @maxLength 1000 + */ + value: string; +} + +export interface FormatPlan { + /** + * This determines whether the chunk is formatted before being sent to the voice provider. This helps with enunciation. This includes phone numbers, emails and addresses. Default `true`. + * + * Usage: + * - To rely on the voice provider's formatting logic, set this to `false`. + * + * If `voice.chunkPlan.enabled` is `false`, this is automatically `false` since there's no chunk to format. + * + * @default true + * @example true + */ + enabled?: boolean; + /** + * This is the cutoff after which a number is converted to individual digits instead of being spoken as words. + * + * Example: + * - If cutoff 2025, "12345" is converted to "1 2 3 4 5" while "1200" is converted to "twelve hundred". + * + * Usage: + * - If your use case doesn't involve IDs like zip codes, set this to a high value. + * - If your use case involves IDs that are shorter than 5 digits, set this to a lower value. + * + * @default 2025 + * @min 0 + * @example 2025 + */ + numberToDigitsCutoff?: number; + /** + * These are the custom replacements you can make to the chunk before it is sent to the voice provider. + * + * Usage: + * - To replace a specific word or phrase with a different word or phrase, use the `ExactReplacement` type. Eg. `{ type: 'exact', key: 'hello', value: 'hi' }` + * - To replace a word or phrase that matches a pattern, use the `RegexReplacement` type. Eg. `{ type: 'regex', regex: '\\b[a-zA-Z]{5}\\b', value: 'hi' }` + * + * @default [] + */ + replacements?: (ExactReplacement | RegexReplacement)[]; + /** + * List of formatters to apply. If not provided, all default formatters will be applied. + * If provided, only the specified formatters will be applied. + * Note: Some essential formatters like angle bracket removal will always be applied. + * @default undefined + */ + formattersEnabled?: + | 'markdown' + | 'asterisk' + | 'quote' + | 'dash' + | 'newline' + | 'colon' + | 'acronym' + | 'dollarAmount' + | 'email' + | 'date' + | 'time' + | 'distance' + | 'unit' + | 'percentage' + | 'phoneNumber' + | 'number' + | 'stripAsterisk'; +} + +export interface ChunkPlan { + /** + * This determines whether the model output is chunked before being sent to the voice provider. Default `true`. + * + * Usage: + * - To rely on the voice provider's audio generation logic, set this to `false`. + * - If seeing issues with quality, set this to `true`. + * + * If disabled, Vapi-provided audio control tokens like will not work. + * + * @default true + * @example true + */ + enabled?: boolean; + /** + * This is the minimum number of characters in a chunk. + * + * Usage: + * - To increase quality, set this to a higher value. + * - To decrease latency, set this to a lower value. + * + * @default 30 + * @min 1 + * @max 80 + * @example 30 + */ + minCharacters?: number; + /** + * These are the punctuations that are considered valid boundaries for a chunk to be created. + * + * Usage: + * - To increase quality, constrain to fewer boundaries. + * - To decrease latency, enable all. + * + * Default is automatically set to balance the trade-off between quality and latency based on the provider. + * @example ["。","，",".","!","?",";","،","۔","।","॥","|","||",",",":"] + */ + punctuationBoundaries?: + | '。' + | '，' + | '.' + | '!' + | '?' + | ';' + | ')' + | '،' + | '۔' + | '।' + | '॥' + | '|' + | '||' + | ',' + | ':'; + /** This is the plan for formatting the chunk before it is sent to the voice provider. */ + formatPlan?: FormatPlan; +} + +export interface FallbackPlan { + /** This is the list of voices to fallback to in the event that the primary voice provider fails. */ + voices: ( + | FallbackAzureVoice + | FallbackCartesiaVoice + | FallbackHumeVoice + | FallbackCustomVoice + | FallbackDeepgramVoice + | FallbackElevenLabsVoice + | FallbackVapiVoice + | FallbackLMNTVoice + | FallbackOpenAIVoice + | FallbackPlayHTVoice + | FallbackRimeAIVoice + | FallbackSmallestAIVoice + | FallbackTavusVoice + | FallbackNeuphonicVoice + | FallbackSesameVoice + )[]; +} + +export interface AzureVoice { + /** + * This is the flag to toggle voice caching for the assistant. + * @default true + * @example true + */ + cachingEnabled?: boolean; + /** This is the voice provider that will be used. */ + provider: 'azure'; + /** This is the provider-specific ID that will be used. */ + voiceId: 'andrew' | 'brian' | 'emma' | string; + /** This is the plan for chunking the model output before it is sent to the voice provider. */ + chunkPlan?: ChunkPlan; + /** + * This is the speed multiplier that will be used. + * @min 0.5 + * @max 2 + */ + speed?: number; + /** This is the plan for voice provider fallbacks in the event that the primary voice provider fails. */ + fallbackPlan?: FallbackPlan; +} + +export interface CartesiaExperimentalControls { + speed?: 'slowest' | 'slow' | 'normal' | 'fast' | 'fastest' | number; + /** @example ["happiness:high"] */ + emotion?: + | 'anger:lowest' + | 'anger:low' + | 'anger:high' + | 'anger:highest' + | 'positivity:lowest' + | 'positivity:low' + | 'positivity:high' + | 'positivity:highest' + | 'surprise:lowest' + | 'surprise:low' + | 'surprise:high' + | 'surprise:highest' + | 'sadness:lowest' + | 'sadness:low' + | 'sadness:high' + | 'sadness:highest' + | 'curiosity:lowest' + | 'curiosity:low' + | 'curiosity:high' + | 'curiosity:highest'; +} + +export interface CartesiaVoice { + /** + * This is the flag to toggle voice caching for the assistant. + * @default true + * @example true + */ + cachingEnabled?: boolean; + /** This is the voice provider that will be used. */ + provider: 'cartesia'; + /** The ID of the particular voice you want to use. */ + voiceId: string; + /** + * This is the model that will be used. This is optional and will default to the correct model for the voiceId. + * @example "sonic-english" + */ + model?: 'sonic-2' | 'sonic-english' | 'sonic-multilingual' | 'sonic-preview' | 'sonic'; + /** + * This is the language that will be used. This is optional and will default to the correct language for the voiceId. + * @example "en" + */ + language?: + | 'en' + | 'de' + | 'es' + | 'fr' + | 'ja' + | 'pt' + | 'zh' + | 'hi' + | 'it' + | 'ko' + | 'nl' + | 'pl' + | 'ru' + | 'sv' + | 'tr'; + /** Experimental controls for Cartesia voice generation */ + experimentalControls?: CartesiaExperimentalControls; + /** This is the plan for chunking the model output before it is sent to the voice provider. */ + chunkPlan?: ChunkPlan; + /** This is the plan for voice provider fallbacks in the event that the primary voice provider fails. */ + fallbackPlan?: FallbackPlan; +} + +export interface CustomVoice { + /** + * This is the flag to toggle voice caching for the assistant. + * @default true + * @example true + */ + cachingEnabled?: boolean; + /** This is the voice provider that will be used. Use `custom-voice` for providers that are not natively supported. */ + provider: 'custom-voice'; + /** This is the plan for chunking the model output before it is sent to the voice provider. */ + chunkPlan?: ChunkPlan; + /** + * This is where the voice request will be sent. + * + * Request Example: + * + * POST https://{server.url} + * Content-Type: application/json + * + * { + * "message": { + * "type": "voice-request", + * "text": "Hello, world!", + * "sampleRate": 24000, + * ...other metadata about the call... + * } + * } + * + * Response Expected: 1-channel 16-bit raw PCM audio at the sample rate specified in the request. Here is how the response will be piped to the transport: + * ``` + * response.on('data', (chunk: Buffer) => { + * outputStream.write(chunk); + * }); + * ``` + */ + server: Server; + /** This is the plan for voice provider fallbacks in the event that the primary voice provider fails. */ + fallbackPlan?: FallbackPlan; +} + +export interface DeepgramVoice { + /** + * This is the flag to toggle voice caching for the assistant. + * @default true + * @example true + */ + cachingEnabled?: boolean; + /** This is the voice provider that will be used. */ + provider: 'deepgram'; + /** + * This is the Deepgram Voice ID + * This is the provider-specific ID that will be used. + */ + voiceId: + | 'asteria' + | 'luna' + | 'stella' + | 'athena' + | 'hera' + | 'orion' + | 'arcas' + | 'perseus' + | 'angus' + | 'orpheus' + | 'helios' + | 'zeus' + | 'thalia' + | 'andromeda' + | 'helena' + | 'apollo' + | 'aries' + | 'amalthea' + | 'atlas' + | 'aurora' + | 'callista' + | 'cora' + | 'cordelia' + | 'delia' + | 'draco' + | 'electra' + | 'harmonia' + | 'hermes' + | 'hyperion' + | 'iris' + | 'janus' + | 'juno' + | 'jupiter' + | 'mars' + | 'minerva' + | 'neptune' + | 'odysseus' + | 'ophelia' + | 'pandora' + | 'phoebe' + | 'pluto' + | 'saturn' + | 'selene' + | 'theia' + | 'vesta'; + /** + * This is the model that will be used. Defaults to 'aura-2' when not specified. + * @example "aura-2" + */ + model?: 'aura' | 'aura-2'; + /** + * If set to true, this will add mip_opt_out=true as a query parameter of all API requests. See https://developers.deepgram.com/docs/the-deepgram-model-improvement-partnership-program#want-to-opt-out + * + * This will only be used if you are using your own Deepgram API key. + * + * @default false + * @default false + * @example false + */ + mipOptOut?: boolean; + /** This is the plan for chunking the model output before it is sent to the voice provider. */ + chunkPlan?: ChunkPlan; + /** This is the plan for voice provider fallbacks in the event that the primary voice provider fails. */ + fallbackPlan?: FallbackPlan; +} + +export interface ElevenLabsVoice { + /** + * This is the flag to toggle voice caching for the assistant. + * @default true + * @example true + */ + cachingEnabled?: boolean; + /** This is the voice provider that will be used. */ + provider: '11labs'; + /** This is the provider-specific ID that will be used. Ensure the Voice is present in your 11Labs Voice Library. */ + voiceId: + | 'burt' + | 'marissa' + | 'andrea' + | 'sarah' + | 'phillip' + | 'steve' + | 'joseph' + | 'myra' + | 'paula' + | 'ryan' + | 'drew' + | 'paul' + | 'mrb' + | 'matilda' + | 'mark' + | string; + /** + * Defines the stability for voice settings. + * @min 0 + * @max 1 + * @example 0.5 + */ + stability?: number; + /** + * Defines the similarity boost for voice settings. + * @min 0 + * @max 1 + * @example 0.75 + */ + similarityBoost?: number; + /** + * Defines the style for voice settings. + * @min 0 + * @max 1 + * @example 0 + */ + style?: number; + /** + * Defines the use speaker boost for voice settings. + * @example false + */ + useSpeakerBoost?: boolean; + /** + * Defines the speed for voice settings. + * @min 0.7 + * @max 1.2 + * @example 0.9 + */ + speed?: number; + /** + * Defines the optimize streaming latency for voice settings. Defaults to 3. + * @min 0 + * @max 4 + * @example 3 + */ + optimizeStreamingLatency?: number; + /** + * This enables the use of https://elevenlabs.io/docs/speech-synthesis/prompting#pronunciation. Defaults to false to save latency. + * + * @default false + * @example false + */ + enableSsmlParsing?: boolean; + /** + * Defines the auto mode for voice settings. Defaults to false. + * @example false + */ + autoMode?: boolean; + /** + * This is the model that will be used. Defaults to 'eleven_turbo_v2' if not specified. + * @example "eleven_turbo_v2_5" + */ + model?: + | 'eleven_multilingual_v2' + | 'eleven_turbo_v2' + | 'eleven_turbo_v2_5' + | 'eleven_flash_v2' + | 'eleven_flash_v2_5' + | 'eleven_monolingual_v1'; + /** This is the plan for chunking the model output before it is sent to the voice provider. */ + chunkPlan?: ChunkPlan; + /** This is the language (ISO 639-1) that is enforced for the model. Currently only Turbo v2.5 supports language enforcement. For other models, an error will be returned if language code is provided. */ + language?: string; + /** This is the plan for voice provider fallbacks in the event that the primary voice provider fails. */ + fallbackPlan?: FallbackPlan; +} + +export interface HumeVoice { + /** + * This is the flag to toggle voice caching for the assistant. + * @default true + * @example true + */ + cachingEnabled?: boolean; + /** This is the voice provider that will be used. */ + provider: 'hume'; + /** + * This is the model that will be used. + * @example "octave" + */ + model?: 'octave'; + /** The ID of the particular voice you want to use. */ + voiceId: string; + /** + * Indicates whether the chosen voice is a preset Hume AI voice or a custom voice. + * @example false + */ + isCustomHumeVoice?: boolean; + /** This is the plan for chunking the model output before it is sent to the voice provider. */ + chunkPlan?: ChunkPlan; + /** + * Natural language instructions describing how the synthesized speech should sound, including but not limited to tone, intonation, pacing, and accent (e.g., 'a soft, gentle voice with a strong British accent'). + * + * If a Voice is specified in the request, this description serves as acting instructions. + * If no Voice is specified, a new voice is generated based on this description. + */ + description?: string; + /** This is the plan for voice provider fallbacks in the event that the primary voice provider fails. */ + fallbackPlan?: FallbackPlan; +} + +export interface LMNTVoice { + /** + * This is the flag to toggle voice caching for the assistant. + * @default true + * @example true + */ + cachingEnabled?: boolean; + /** This is the voice provider that will be used. */ + provider: 'lmnt'; + /** This is the provider-specific ID that will be used. */ + voiceId: + | 'amy' + | 'ansel' + | 'autumn' + | 'ava' + | 'brandon' + | 'caleb' + | 'cassian' + | 'chloe' + | 'dalton' + | 'daniel' + | 'dustin' + | 'elowen' + | 'evander' + | 'huxley' + | 'james' + | 'juniper' + | 'kennedy' + | 'lauren' + | 'leah' + | 'lily' + | 'lucas' + | 'magnus' + | 'miles' + | 'morgan' + | 'natalie' + | 'nathan' + | 'noah' + | 'nyssa' + | 'oliver' + | 'paige' + | 'ryan' + | 'sadie' + | 'sophie' + | 'stella' + | 'terrence' + | 'tyler' + | 'vesper' + | 'violet' + | 'warrick' + | 'zain' + | 'zeke' + | 'zoe' + | string; + /** + * This is the speed multiplier that will be used. + * @min 0.25 + * @max 2 + * @example null + */ + speed?: number; + /** + * Two letter ISO 639-1 language code. Use "auto" for auto-detection. + * @example "en" + */ + language?: + | 'aa' + | 'ab' + | 'ae' + | 'af' + | 'ak' + | 'am' + | 'an' + | 'ar' + | 'as' + | 'av' + | 'ay' + | 'az' + | 'ba' + | 'be' + | 'bg' + | 'bh' + | 'bi' + | 'bm' + | 'bn' + | 'bo' + | 'br' + | 'bs' + | 'ca' + | 'ce' + | 'ch' + | 'co' + | 'cr' + | 'cs' + | 'cu' + | 'cv' + | 'cy' + | 'da' + | 'de' + | 'dv' + | 'dz' + | 'ee' + | 'el' + | 'en' + | 'eo' + | 'es' + | 'et' + | 'eu' + | 'fa' + | 'ff' + | 'fi' + | 'fj' + | 'fo' + | 'fr' + | 'fy' + | 'ga' + | 'gd' + | 'gl' + | 'gn' + | 'gu' + | 'gv' + | 'ha' + | 'he' + | 'hi' + | 'ho' + | 'hr' + | 'ht' + | 'hu' + | 'hy' + | 'hz' + | 'ia' + | 'id' + | 'ie' + | 'ig' + | 'ii' + | 'ik' + | 'io' + | 'is' + | 'it' + | 'iu' + | 'ja' + | 'jv' + | 'ka' + | 'kg' + | 'ki' + | 'kj' + | 'kk' + | 'kl' + | 'km' + | 'kn' + | 'ko' + | 'kr' + | 'ks' + | 'ku' + | 'kv' + | 'kw' + | 'ky' + | 'la' + | 'lb' + | 'lg' + | 'li' + | 'ln' + | 'lo' + | 'lt' + | 'lu' + | 'lv' + | 'mg' + | 'mh' + | 'mi' + | 'mk' + | 'ml' + | 'mn' + | 'mr' + | 'ms' + | 'mt' + | 'my' + | 'na' + | 'nb' + | 'nd' + | 'ne' + | 'ng' + | 'nl' + | 'nn' + | 'no' + | 'nr' + | 'nv' + | 'ny' + | 'oc' + | 'oj' + | 'om' + | 'or' + | 'os' + | 'pa' + | 'pi' + | 'pl' + | 'ps' + | 'pt' + | 'qu' + | 'rm' + | 'rn' + | 'ro' + | 'ru' + | 'rw' + | 'sa' + | 'sc' + | 'sd' + | 'se' + | 'sg' + | 'si' + | 'sk' + | 'sl' + | 'sm' + | 'sn' + | 'so' + | 'sq' + | 'sr' + | 'ss' + | 'st' + | 'su' + | 'sv' + | 'sw' + | 'ta' + | 'te' + | 'tg' + | 'th' + | 'ti' + | 'tk' + | 'tl' + | 'tn' + | 'to' + | 'tr' + | 'ts' + | 'tt' + | 'tw' + | 'ty' + | 'ug' + | 'uk' + | 'ur' + | 'uz' + | 've' + | 'vi' + | 'vo' + | 'wa' + | 'wo' + | 'xh' + | 'yi' + | 'yue' + | 'yo' + | 'za' + | 'zh' + | 'zu' + | 'auto'; + /** This is the plan for chunking the model output before it is sent to the voice provider. */ + chunkPlan?: ChunkPlan; + /** This is the plan for voice provider fallbacks in the event that the primary voice provider fails. */ + fallbackPlan?: FallbackPlan; +} + +export interface NeuphonicVoice { + /** + * This is the flag to toggle voice caching for the assistant. + * @default true + * @example true + */ + cachingEnabled?: boolean; + /** This is the voice provider that will be used. */ + provider: 'neuphonic'; + /** This is the provider-specific ID that will be used. */ + voiceId: string; + /** + * This is the model that will be used. Defaults to 'neu_fast' if not specified. + * @example "neu_fast" + */ + model?: 'neu_hq' | 'neu_fast'; + /** + * This is the language (ISO 639-1) that is enforced for the model. + * @example "en" + */ + language: object; + /** + * This is the speed multiplier that will be used. + * @min 0.25 + * @max 2 + * @example null + */ + speed?: number; + /** This is the plan for chunking the model output before it is sent to the voice provider. */ + chunkPlan?: ChunkPlan; + /** This is the plan for voice provider fallbacks in the event that the primary voice provider fails. */ + fallbackPlan?: FallbackPlan; +} + +export interface OpenAIVoice { + /** + * This is the flag to toggle voice caching for the assistant. + * @default true + * @example true + */ + cachingEnabled?: boolean; + /** This is the voice provider that will be used. */ + provider: 'openai'; + /** + * This is the provider-specific ID that will be used. + * Please note that ash, ballad, coral, sage, and verse may only be used with realtime models. + */ + voiceId: 'alloy' | 'echo' | 'fable' | 'onyx' | 'nova' | 'shimmer' | string; + /** This is the model that will be used for text-to-speech. */ + model?: 'tts-1' | 'tts-1-hd' | 'gpt-4o-mini-tts'; + /** + * This is a prompt that allows you to control the voice of your generated audio. + * Does not work with 'tts-1' or 'tts-1-hd' models. + * @maxLength 10000 + */ + instructions?: string; + /** + * This is the speed multiplier that will be used. + * @min 0.25 + * @max 4 + * @example null + */ + speed?: number; + /** This is the plan for chunking the model output before it is sent to the voice provider. */ + chunkPlan?: ChunkPlan; + /** This is the plan for voice provider fallbacks in the event that the primary voice provider fails. */ + fallbackPlan?: FallbackPlan; +} + +export interface PlayHTVoice { + /** + * This is the flag to toggle voice caching for the assistant. + * @default true + * @example true + */ + cachingEnabled?: boolean; + /** This is the voice provider that will be used. */ + provider: 'playht'; + /** This is the provider-specific ID that will be used. */ + voiceId: + | 'jennifer' + | 'melissa' + | 'will' + | 'chris' + | 'matt' + | 'jack' + | 'ruby' + | 'davis' + | 'donna' + | 'michael' + | string; + /** + * This is the speed multiplier that will be used. + * @min 0.1 + * @max 5 + * @example null + */ + speed?: number; + /** + * A floating point number between 0, exclusive, and 2, inclusive. If equal to null or not provided, the model's default temperature will be used. The temperature parameter controls variance. Lower temperatures result in more predictable results, higher temperatures allow each run to vary more, so the voice may sound less like the baseline voice. + * @min 0.1 + * @max 2 + * @example null + */ + temperature?: number; + /** + * An emotion to be applied to the speech. + * @example null + */ + emotion?: + | 'female_happy' + | 'female_sad' + | 'female_angry' + | 'female_fearful' + | 'female_disgust' + | 'female_surprised' + | 'male_happy' + | 'male_sad' + | 'male_angry' + | 'male_fearful' + | 'male_disgust' + | 'male_surprised'; + /** + * A number between 1 and 6. Use lower numbers to reduce how unique your chosen voice will be compared to other voices. + * @min 1 + * @max 6 + * @example null + */ + voiceGuidance?: number; + /** + * A number between 1 and 30. Use lower numbers to to reduce how strong your chosen emotion will be. Higher numbers will create a very emotional performance. + * @min 1 + * @max 30 + * @example null + */ + styleGuidance?: number; + /** + * A number between 1 and 2. This number influences how closely the generated speech adheres to the input text. Use lower values to create more fluid speech, but with a higher chance of deviating from the input text. Higher numbers will make the generated speech more accurate to the input text, ensuring that the words spoken align closely with the provided text. * @min 1 * @max 2 * @example null @@ -7610,7 +8742,50 @@ export interface FallbackLMNTVoice { /** This is the voice provider that will be used. */ provider: 'lmnt'; /** This is the provider-specific ID that will be used. */ - voiceId: 'lily' | 'daniel' | string; + voiceId: + | 'amy' + | 'ansel' + | 'autumn' + | 'ava' + | 'brandon' + | 'caleb' + | 'cassian' + | 'chloe' + | 'dalton' + | 'daniel' + | 'dustin' + | 'elowen' + | 'evander' + | 'huxley' + | 'james' + | 'juniper' + | 'kennedy' + | 'lauren' + | 'leah' + | 'lily' + | 'lucas' + | 'magnus' + | 'miles' + | 'morgan' + | 'natalie' + | 'nathan' + | 'noah' + | 'nyssa' + | 'oliver' + | 'paige' + | 'ryan' + | 'sadie' + | 'sophie' + | 'stella' + | 'terrence' + | 'tyler' + | 'vesper' + | 'violet' + | 'warrick' + | 'zain' + | 'zeke' + | 'zoe' + | string; /** * This is the speed multiplier that will be used. * @min 0.25 @@ -7618,6 +8793,197 @@ export interface FallbackLMNTVoice { * @example null */ speed?: number; + /** + * Two letter ISO 639-1 language code. Use "auto" for auto-detection. + * @example "en" + */ + language?: + | 'aa' + | 'ab' + | 'ae' + | 'af' + | 'ak' + | 'am' + | 'an' + | 'ar' + | 'as' + | 'av' + | 'ay' + | 'az' + | 'ba' + | 'be' + | 'bg' + | 'bh' + | 'bi' + | 'bm' + | 'bn' + | 'bo' + | 'br' + | 'bs' + | 'ca' + | 'ce' + | 'ch' + | 'co' + | 'cr' + | 'cs' + | 'cu' + | 'cv' + | 'cy' + | 'da' + | 'de' + | 'dv' + | 'dz' + | 'ee' + | 'el' + | 'en' + | 'eo' + | 'es' + | 'et' + | 'eu' + | 'fa' + | 'ff' + | 'fi' + | 'fj' + | 'fo' + | 'fr' + | 'fy' + | 'ga' + | 'gd' + | 'gl' + | 'gn' + | 'gu' + | 'gv' + | 'ha' + | 'he' + | 'hi' + | 'ho' + | 'hr' + | 'ht' + | 'hu' + | 'hy' + | 'hz' + | 'ia' + | 'id' + | 'ie' + | 'ig' + | 'ii' + | 'ik' + | 'io' + | 'is' + | 'it' + | 'iu' + | 'ja' + | 'jv' + | 'ka' + | 'kg' + | 'ki' + | 'kj' + | 'kk' + | 'kl' + | 'km' + | 'kn' + | 'ko' + | 'kr' + | 'ks' + | 'ku' + | 'kv' + | 'kw' + | 'ky' + | 'la' + | 'lb' + | 'lg' + | 'li' + | 'ln' + | 'lo' + | 'lt' + | 'lu' + | 'lv' + | 'mg' + | 'mh' + | 'mi' + | 'mk' + | 'ml' + | 'mn' + | 'mr' + | 'ms' + | 'mt' + | 'my' + | 'na' + | 'nb' + | 'nd' + | 'ne' + | 'ng' + | 'nl' + | 'nn' + | 'no' + | 'nr' + | 'nv' + | 'ny' + | 'oc' + | 'oj' + | 'om' + | 'or' + | 'os' + | 'pa' + | 'pi' + | 'pl' + | 'ps' + | 'pt' + | 'qu' + | 'rm' + | 'rn' + | 'ro' + | 'ru' + | 'rw' + | 'sa' + | 'sc' + | 'sd' + | 'se' + | 'sg' + | 'si' + | 'sk' + | 'sl' + | 'sm' + | 'sn' + | 'so' + | 'sq' + | 'sr' + | 'ss' + | 'st' + | 'su' + | 'sv' + | 'sw' + | 'ta' + | 'te' + | 'tg' + | 'th' + | 'ti' + | 'tk' + | 'tl' + | 'tn' + | 'to' + | 'tr' + | 'ts' + | 'tt' + | 'tw' + | 'ty' + | 'ug' + | 'uk' + | 'ur' + | 'uz' + | 've' + | 'vi' + | 'vo' + | 'wa' + | 'wo' + | 'xh' + | 'yi' + | 'yue' + | 'yo' + | 'za' + | 'zh' + | 'zu' + | 'auto'; /** This is the plan for chunking the model output before it is sent to the voice provider. */ chunkPlan?: ChunkPlan; } @@ -8098,80 +9464,10 @@ export interface TransportConfigurationTwilio { * `dual` records each leg to a separate channel of the recording file. * The first channel of a dual-channel recording contains the parent call and the second channel contains the child call. * - * @default 'mono' - * @example "mono" - */ - recordingChannels?: 'mono' | 'dual'; -} - -export interface SmartDenoisingPlan { - /** - * Whether smart denoising using Krisp is enabled. - * @default false - */ - enabled?: boolean; -} - -export interface FourierDenoisingPlan { - /** - * Whether Fourier denoising is enabled. Note that this is experimental and may not work as expected. - * @default false - */ - enabled?: boolean; - /** - * Whether automatic media detection is enabled. When enabled, the filter will automatically - * detect consistent background TV/music/radio and switch to more aggressive filtering settings. - * Only applies when enabled is true. - * @default true - * @example true - */ - mediaDetectionEnabled?: boolean; - /** - * Static threshold in dB used as fallback when no baseline is established. - * @min -80 - * @max 0 - * @default -35 - * @example -35 - */ - staticThreshold?: number; - /** - * How far below the rolling baseline to filter audio, in dB. - * Lower values (e.g., -10) are more aggressive, higher values (e.g., -20) are more conservative. - * @min -30 - * @max -5 - * @default -15 - * @example -15 - */ - baselineOffsetDb?: number; - /** - * Rolling window size in milliseconds for calculating the audio baseline. - * Larger windows adapt more slowly but are more stable. - * @min 1000 - * @max 30000 - * @default 3000 - * @example 3000 - */ - windowSizeMs?: number; - /** - * Percentile to use for baseline calculation (1-99). - * Higher percentiles (e.g., 85) focus on louder speech, lower percentiles (e.g., 50) include quieter speech. - * @min 1 - * @max 99 - * @default 85 - * @example 85 - */ - baselinePercentile?: number; -} - -export interface BackgroundSpeechDenoisingPlan { - /** Whether smart denoising using Krisp is enabled. */ - smartDenoisingPlan?: SmartDenoisingPlan; - /** - * Whether Fourier denoising is enabled. Note that this is experimental and may not work as expected. - * - * This can be combined with smart denoising, and will be run afterwards. + * @default 'mono' + * @example "mono" */ - fourierDenoisingPlan?: FourierDenoisingPlan; + recordingChannels?: 'mono' | 'dual'; } export interface CreateAnthropicCredentialDTO { @@ -8492,7 +9788,7 @@ export interface CreateCloudflareCredentialDTO { } export interface OAuth2AuthenticationPlan { - type: 'oauth2' | 'aws-sts'; + type: 'oauth2'; /** This is the OAuth2 URL. */ url: string; /** This is the OAuth2 client ID. */ @@ -8614,6 +9910,8 @@ export interface BucketPlan { * Usage: * - If `credential.type` is `aws`, then this is required. * - If `credential.type` is `gcp`, then this is optional since GCP allows buckets to be accessed without a region but region is required for data residency requirements. Read here: https://cloud.google.com/storage/docs/request-endpoints + * + * This overrides the `credential.region` field if it is provided. */ region?: string; /** @@ -8659,7 +9957,11 @@ export interface CreateGcpCredentialDTO { * The schema is identical to the JSON that GCP outputs. */ gcpKey: GcpKey; - /** This is the bucket plan that can be provided to store call artifacts in GCP. */ + /** + * This is the region of the GCP resource. + * @maxLength 40 + */ + region?: string; bucketPlan?: BucketPlan; /** * This is the name of credential. This is just for your reference. @@ -8868,1062 +10170,969 @@ export interface SupabaseBucketPlan { | 'ap-south-1' | 'ap-southeast-1' | 'ap-northeast-1' - | 'ap-northeast-2' - | 'ap-southeast-2' - | 'sa-east-1'; - /** - * This is the S3 compatible URL for Supabase S3 - * This should look like https://.supabase.co/storage/v1/s3 - */ - url: string; - /** - * This is the Supabase S3 Access Key ID. - * The user creates this in the Supabase project Storage settings - */ - accessKeyId: string; - /** - * This is the Supabase S3 Secret Access Key. - * The user creates this in the Supabase project Storage settings along with the access key id - */ - secretAccessKey: string; - /** - * This is the Supabase S3 Bucket Name. - * The user must create this in Supabase under Storage > Buckets - * A bucket that does not exist will not be checked now, but file uploads will fail - */ - name: string; - /** - * This is the Supabase S3 Bucket Folder Path. - * The user can create this in Supabase under Storage > Buckets - * A path that does not exist will not be checked now, but file uploads will fail - * A Path is like a folder in the bucket - * Eg. If the bucket is called "my-bucket" and the path is "my-folder", the full path is "my-bucket/my-folder" - */ - path?: string; -} - -export interface CreateSupabaseCredentialDTO { - /** This is for supabase storage. */ - provider: 'supabase'; - /** - * This is the order in which this storage provider is tried during upload retries. Lower numbers are tried first in increasing order. - * @min 1 - */ - fallbackIndex?: number; - bucketPlan?: SupabaseBucketPlan; - /** - * This is the name of credential. This is just for your reference. - * @minLength 1 - * @maxLength 40 - */ - name?: string; -} - -export interface CreateSmallestAICredentialDTO { - provider: 'smallest-ai'; - /** This is not returned in the API. */ - apiKey: string; - /** - * This is the name of credential. This is just for your reference. - * @minLength 1 - * @maxLength 40 - */ - name?: string; -} - -export interface CreateTavusCredentialDTO { - provider: 'tavus'; - /** This is not returned in the API. */ - apiKey: string; - /** - * This is the name of credential. This is just for your reference. - * @minLength 1 - * @maxLength 40 - */ - name?: string; -} - -export interface CreateTogetherAICredentialDTO { - provider: 'together-ai'; - /** This is not returned in the API. */ - apiKey: string; - /** - * This is the name of credential. This is just for your reference. - * @minLength 1 - * @maxLength 40 - */ - name?: string; -} - -export interface CreateTwilioCredentialDTO { - provider: 'twilio'; - /** This is not returned in the API. */ - authToken?: string; - /** This is not returned in the API. */ - apiKey?: string; - /** This is not returned in the API. */ - apiSecret?: string; - accountSid: string; - /** - * This is the name of credential. This is just for your reference. - * @minLength 1 - * @maxLength 40 - */ - name?: string; -} - -export interface CreateVonageCredentialDTO { - provider: 'vonage'; - /** This is not returned in the API. */ - apiSecret: string; - apiKey: string; - /** - * This is the name of credential. This is just for your reference. - * @minLength 1 - * @maxLength 40 - */ - name?: string; -} - -export interface CreateWebhookCredentialDTO { - provider: 'webhook'; - /** This is the authentication plan. Currently supports OAuth2 RFC 6749. */ - authenticationPlan: OAuth2AuthenticationPlan; - /** - * This is the name of credential. This is just for your reference. - * @minLength 1 - * @maxLength 40 - */ - name?: string; -} - -export interface CreateXAiCredentialDTO { - /** This is the api key for Grok in XAi's console. Get it from here: https://console.x.ai */ - provider: 'xai'; - /** - * This is not returned in the API. - * @maxLength 10000 - */ - apiKey: string; - /** - * This is the name of credential. This is just for your reference. - * @minLength 1 - * @maxLength 40 - */ - name?: string; -} - -export interface CreateGoogleCalendarOAuth2ClientCredentialDTO { - provider: 'google.calendar.oauth2-client'; - /** - * This is the name of credential. This is just for your reference. - * @minLength 1 - * @maxLength 40 - */ - name?: string; -} - -export interface CreateGoogleCalendarOAuth2AuthorizationCredentialDTO { - provider: 'google.calendar.oauth2-authorization'; - /** The authorization ID for the OAuth2 authorization */ - authorizationId: string; - /** - * This is the name of credential. This is just for your reference. - * @minLength 1 - * @maxLength 40 - */ - name?: string; -} - -export interface CreateGoogleSheetsOAuth2AuthorizationCredentialDTO { - provider: 'google.sheets.oauth2-authorization'; - /** The authorization ID for the OAuth2 authorization */ - authorizationId: string; - /** - * This is the name of credential. This is just for your reference. - * @minLength 1 - * @maxLength 40 - */ - name?: string; -} - -export interface CreateSlackOAuth2AuthorizationCredentialDTO { - provider: 'slack.oauth2-authorization'; - /** The authorization ID for the OAuth2 authorization */ - authorizationId: string; - /** - * This is the name of credential. This is just for your reference. - * @minLength 1 - * @maxLength 40 - */ - name?: string; -} - -export interface TransferAssistantHookAction { - /** This is the type of action - must be "transfer" */ - type: 'transfer'; - /** This is the destination details for the transfer - can be a phone number or SIP URI */ - destination?: TransferDestinationNumber | TransferDestinationSip; -} - -export interface FunctionCallAssistantHookAction { - /** - * These are the messages that will be spoken to the user as the tool is running. - * - * For some tools, this is auto-filled based on special fields like `tool.destinations`. For others like the function tool, these can be custom configured. - */ - messages?: (ToolMessageStart | ToolMessageComplete | ToolMessageFailed | ToolMessageDelayed)[]; - /** The type of tool. "function" for Function tool. */ - type: 'function'; - /** - * This determines if the tool is async. - * - * If async, the assistant will move forward without waiting for your server to respond. This is useful if you just want to trigger something on your server. - * - * If sync, the assistant will wait for your server to respond. This is useful if want assistant to respond with the result from your server. - * - * Defaults to synchronous (`false`). - * @example false - */ - async?: boolean; - /** - * - * This is the server where a `tool-calls` webhook will be sent. - * - * Notes: - * - Webhook is sent to this server when a tool call is made. - * - Webhook contains the call, assistant, and phone number objects. - * - Webhook contains the variables set on the assistant. - * - Webhook is sent to the first available URL in this order: {{tool.server.url}}, {{assistant.server.url}}, {{phoneNumber.server.url}}, {{org.server.url}}. - * - Webhook expects a response with tool call result. - */ - server?: Server; - /** - * This is the function definition of the tool. - * - * For `endCall`, `transferCall`, and `dtmf` tools, this is auto-filled based on tool-specific fields like `tool.destinations`. But, even in those cases, you can provide a custom function definition for advanced use cases. - * - * An example of an advanced use case is if you want to customize the message that's spoken for `endCall` tool. You can specify a function where it returns an argument "reason". Then, in `messages` array, you can have many "request-complete" messages. One of these messages will be triggered if the `messages[].conditions` matches the "reason" argument. - */ - function?: OpenAIFunction; -} - -export interface SayAssistantHookAction { - /** This is the type of action - must be "say" */ - type: 'say'; - /** This is the message to say */ - exact: object; -} - -export interface AssistantHookFilter { + | 'ap-northeast-2' + | 'ap-southeast-2' + | 'sa-east-1'; /** - * This is the type of filter - currently only "oneOf" is supported - * @maxLength 1000 + * This is the S3 compatible URL for Supabase S3 + * This should look like https://.supabase.co/storage/v1/s3 */ - type: 'oneOf'; + url: string; /** - * This is the key to filter on (e.g. "call.endedReason") - * @maxLength 1000 + * This is the Supabase S3 Access Key ID. + * The user creates this in the Supabase project Storage settings */ - key: string; - /** This is the array of possible values to match against */ - oneOf: string[]; -} - -export interface AssistantHookCallEnding { + accessKeyId: string; /** - * This is the event that triggers this hook - * @maxLength 1000 + * This is the Supabase S3 Secret Access Key. + * The user creates this in the Supabase project Storage settings along with the access key id */ - on: 'call.ending'; - /** This is the set of actions to perform when the hook triggers */ - do: (TransferAssistantHookAction | FunctionCallAssistantHookAction)[]; - /** This is the set of filters that must match for the hook to trigger */ - filters?: AssistantHookFilter[]; -} - -export interface AssistantHookAssistantSpeechInterrupted { + secretAccessKey: string; /** - * This is the event that triggers this hook - * @maxLength 1000 + * This is the Supabase S3 Bucket Name. + * The user must create this in Supabase under Storage > Buckets + * A bucket that does not exist will not be checked now, but file uploads will fail */ - on: 'assistant.speech.interrupted'; - /** This is the set of actions to perform when the hook triggers */ - do: (TransferAssistantHookAction | FunctionCallAssistantHookAction | SayAssistantHookAction)[]; -} - -export interface AssistantHookCustomerSpeechInterrupted { + name: string; /** - * This is the event that triggers this hook - * @maxLength 1000 + * This is the Supabase S3 Bucket Folder Path. + * The user can create this in Supabase under Storage > Buckets + * A path that does not exist will not be checked now, but file uploads will fail + * A Path is like a folder in the bucket + * Eg. If the bucket is called "my-bucket" and the path is "my-folder", the full path is "my-bucket/my-folder" */ - on: 'customer.speech.interrupted'; - /** This is the set of actions to perform when the hook triggers */ - do: (TransferAssistantHookAction | FunctionCallAssistantHookAction | SayAssistantHookAction)[]; + path?: string; } -export interface VoicemailDetectionBackoffPlan { +export interface CreateSupabaseCredentialDTO { + /** This is for supabase storage. */ + provider: 'supabase'; /** - * This is the number of seconds to wait before starting the first retry attempt. - * @min 0 - * @default 5 + * This is the order in which this storage provider is tried during upload retries. Lower numbers are tried first in increasing order. + * @min 1 */ - startAtSeconds?: number; + fallbackIndex?: number; + bucketPlan?: SupabaseBucketPlan; /** - * This is the interval in seconds between retry attempts. - * @min 2.5 - * @default 5 + * This is the name of credential. This is just for your reference. + * @minLength 1 + * @maxLength 40 */ - frequencySeconds?: number; + name?: string; +} + +export interface CreateSmallestAICredentialDTO { + provider: 'smallest-ai'; + /** This is not returned in the API. */ + apiKey: string; /** - * This is the maximum number of retry attempts before giving up. - * @min 1 - * @max 10 - * @default 6 + * This is the name of credential. This is just for your reference. + * @minLength 1 + * @maxLength 40 */ - maxRetries?: number; + name?: string; } -export interface GoogleVoicemailDetectionPlan { +export interface CreateTavusCredentialDTO { + provider: 'tavus'; + /** This is not returned in the API. */ + apiKey: string; /** - * This is the maximum duration from the start of the call that we will wait for a voicemail beep, before speaking our message - * - * - If we detect a voicemail beep before this, we will speak the message at that point. - * - * - Setting too low a value means that the bot will start speaking its voicemail message too early. If it does so before the actual beep, it will get cut off. You should definitely tune this to your use case. - * - * @default 30 - * @min 0 - * @max 60 - * @min 0 - * @max 30 - * @default 30 + * This is the name of credential. This is just for your reference. + * @minLength 1 + * @maxLength 40 */ - beepMaxAwaitSeconds?: number; - /** This is the provider to use for voicemail detection. */ - provider: 'google'; - /** This is the backoff plan for the voicemail detection. */ - backoffPlan?: VoicemailDetectionBackoffPlan; + name?: string; } -export interface OpenAIVoicemailDetectionPlan { +export interface CreateTogetherAICredentialDTO { + provider: 'together-ai'; + /** This is not returned in the API. */ + apiKey: string; /** - * This is the maximum duration from the start of the call that we will wait for a voicemail beep, before speaking our message - * - * - If we detect a voicemail beep before this, we will speak the message at that point. - * - * - Setting too low a value means that the bot will start speaking its voicemail message too early. If it does so before the actual beep, it will get cut off. You should definitely tune this to your use case. - * - * @default 30 - * @min 0 - * @max 60 - * @min 0 - * @max 30 - * @default 30 + * This is the name of credential. This is just for your reference. + * @minLength 1 + * @maxLength 40 */ - beepMaxAwaitSeconds?: number; - /** This is the provider to use for voicemail detection. */ - provider: 'openai'; - /** This is the backoff plan for the voicemail detection. */ - backoffPlan?: VoicemailDetectionBackoffPlan; + name?: string; } -export interface TwilioVoicemailDetectionPlan { - /** This is the provider to use for voicemail detection. */ +export interface CreateTwilioCredentialDTO { provider: 'twilio'; + /** This is not returned in the API. */ + authToken?: string; + /** This is not returned in the API. */ + apiKey?: string; + /** This is not returned in the API. */ + apiSecret?: string; + accountSid: string; /** - * These are the AMD messages from Twilio that are considered as voicemail. Default is ['machine_end_beep', 'machine_end_silence']. - * - * @default {Array} ['machine_end_beep', 'machine_end_silence'] - * @example ["machine_end_beep","machine_end_silence"] - */ - voicemailDetectionTypes?: - | 'machine_start' - | 'human' - | 'fax' - | 'unknown' - | 'machine_end_beep' - | 'machine_end_silence' - | 'machine_end_other'; - /** - * This sets whether the assistant should detect voicemail. Defaults to true. - * - * @default true - */ - enabled?: boolean; - /** - * The number of seconds that Twilio should attempt to perform answering machine detection before timing out and returning AnsweredBy as unknown. Default is 30 seconds. - * - * Increasing this value will provide the engine more time to make a determination. This can be useful when DetectMessageEnd is provided in the MachineDetection parameter and there is an expectation of long answering machine greetings that can exceed 30 seconds. - * - * Decreasing this value will reduce the amount of time the engine has to make a determination. This can be particularly useful when the Enable option is provided in the MachineDetection parameter and you want to limit the time for initial detection. - * - * Check the [Twilio docs](https://www.twilio.com/docs/voice/answering-machine-detection#optional-api-tuning-parameters) for more info. - * - * @default 30 - * @min 3 - * @max 59 - */ - machineDetectionTimeout?: number; - /** - * The number of milliseconds that is used as the measuring stick for the length of the speech activity. Durations lower than this value will be interpreted as a human, longer as a machine. Default is 2400 milliseconds. - * - * Increasing this value will reduce the chance of a False Machine (detected machine, actually human) for a long human greeting (e.g., a business greeting) but increase the time it takes to detect a machine. - * - * Decreasing this value will reduce the chances of a False Human (detected human, actually machine) for short voicemail greetings. The value of this parameter may need to be reduced by more than 1000ms to detect very short voicemail greetings. A reduction of that significance can result in increased False Machine detections. Adjusting the MachineDetectionSpeechEndThreshold is likely the better approach for short voicemails. Decreasing MachineDetectionSpeechThreshold will also reduce the time it takes to detect a machine. - * - * Check the [Twilio docs](https://www.twilio.com/docs/voice/answering-machine-detection#optional-api-tuning-parameters) for more info. - * - * @default 2400 - * @min 1000 - * @max 6000 - */ - machineDetectionSpeechThreshold?: number; - /** - * The number of milliseconds of silence after speech activity at which point the speech activity is considered complete. Default is 1200 milliseconds. - * - * Increasing this value will typically be used to better address the short voicemail greeting scenarios. For short voicemails, there is typically 1000-2000ms of audio followed by 1200-2400ms of silence and then additional audio before the beep. Increasing the MachineDetectionSpeechEndThreshold to ~2500ms will treat the 1200-2400ms of silence as a gap in the greeting but not the end of the greeting and will result in a machine detection. The downsides of such a change include: - * - Increasing the delay for human detection by the amount you increase this parameter, e.g., a change of 1200ms to 2500ms increases human detection delay by 1300ms. - * - Cases where a human has two utterances separated by a period of silence (e.g. a "Hello", then 2000ms of silence, and another "Hello") may be interpreted as a machine. - * - * Decreasing this value will result in faster human detection. The consequence is that it can lead to increased False Human (detected human, actually machine) detections because a silence gap in a voicemail greeting (not necessarily just in short voicemail scenarios) can be incorrectly interpreted as the end of speech. - * - * Check the [Twilio docs](https://www.twilio.com/docs/voice/answering-machine-detection#optional-api-tuning-parameters) for more info. - * - * @default 1200 - * @min 500 - * @max 5000 - */ - machineDetectionSpeechEndThreshold?: number; - /** - * The number of milliseconds of initial silence after which an unknown AnsweredBy result will be returned. Default is 5000 milliseconds. - * - * Increasing this value will result in waiting for a longer period of initial silence before returning an 'unknown' AMD result. - * - * Decreasing this value will result in waiting for a shorter period of initial silence before returning an 'unknown' AMD result. - * - * Check the [Twilio docs](https://www.twilio.com/docs/voice/answering-machine-detection#optional-api-tuning-parameters) for more info. - * - * @default 5000 - * @min 2000 - * @max 10000 + * This is the name of credential. This is just for your reference. + * @minLength 1 + * @maxLength 40 */ - machineDetectionSilenceTimeout?: number; + name?: string; } -export interface VapiVoicemailDetectionPlan { +export interface CreateVonageCredentialDTO { + provider: 'vonage'; + /** This is not returned in the API. */ + apiSecret: string; + apiKey: string; /** - * This is the maximum duration from the start of the call that we will wait for a voicemail beep, before speaking our message - * - * - If we detect a voicemail beep before this, we will speak the message at that point. - * - * - Setting too low a value means that the bot will start speaking its voicemail message too early. If it does so before the actual beep, it will get cut off. You should definitely tune this to your use case. - * - * @default 30 - * @min 0 - * @max 60 - * @min 0 - * @max 30 - * @default 30 + * This is the name of credential. This is just for your reference. + * @minLength 1 + * @maxLength 40 + */ + name?: string; +} + +export interface CreateWebhookCredentialDTO { + provider: 'webhook'; + /** This is the authentication plan. Supports OAuth2 RFC 6749 and HMAC signing. */ + authenticationPlan: + | ({ + type: 'oauth2'; + } & OAuth2AuthenticationPlan) + | ({ + type: 'hmac'; + } & HMACAuthenticationPlan); + /** + * This is the name of credential. This is just for your reference. + * @minLength 1 + * @maxLength 40 */ - beepMaxAwaitSeconds?: number; - /** This is the provider to use for voicemail detection. */ - provider: 'vapi'; - /** This is the backoff plan for the voicemail detection. */ - backoffPlan?: VoicemailDetectionBackoffPlan; + name?: string; } -export interface CompliancePlan { +export interface CreateXAiCredentialDTO { + /** This is the api key for Grok in XAi's console. Get it from here: https://console.x.ai */ + provider: 'xai'; /** - * When this is enabled, no logs, recordings, or transcriptions will be stored. - * At the end of the call, you will still receive an end-of-call-report message to store on your server. Defaults to false. - * @example {"hipaaEnabled":false} + * This is not returned in the API. + * @maxLength 10000 */ - hipaaEnabled?: boolean; + apiKey: string; /** - * When this is enabled, the user will be restricted to use PCI-compliant providers, and no logs or transcripts are stored. - * At the end of the call, you will receive an end-of-call-report message to store on your server. Defaults to false. - * @example {"pciEnabled":false} + * This is the name of credential. This is just for your reference. + * @minLength 1 + * @maxLength 40 */ - pciEnabled?: boolean; + name?: string; } -export interface StructuredDataPlan { +export interface CreateGoogleCalendarOAuth2ClientCredentialDTO { + provider: 'google.calendar.oauth2-client'; /** - * These are the messages used to generate the structured data. - * - * @default: ``` - * [ - * { - * "role": "system", - * "content": "You are an expert data extractor. You will be given a transcript of a call. Extract structured data per the JSON Schema. DO NOT return anything except the structured data.\n\nJson Schema:\\n{{schema}}\n\nOnly respond with the JSON." - * }, - * { - * "role": "user", - * "content": "Here is the transcript:\n\n{{transcript}}\n\n. Here is the ended reason of the call:\n\n{{endedReason}}\n\n" - * } - * ]``` - * - * You can customize by providing any messages you want. - * - * Here are the template variables available: - * - {{transcript}}: the transcript of the call from `call.artifact.transcript`- {{systemPrompt}}: the system prompt of the call from `assistant.model.messages[type=system].content`- {{schema}}: the schema of the structured data from `structuredDataPlan.schema`- {{endedReason}}: the ended reason of the call from `call.endedReason` + * This is the name of credential. This is just for your reference. + * @minLength 1 + * @maxLength 40 */ - messages?: object[]; + name?: string; +} + +export interface CreateGoogleCalendarOAuth2AuthorizationCredentialDTO { + provider: 'google.calendar.oauth2-authorization'; + /** The authorization ID for the OAuth2 authorization */ + authorizationId: string; /** - * This determines whether structured data is generated and stored in `call.analysis.structuredData`. Defaults to false. - * - * Usage: - * - If you want to extract structured data, set this to true and provide a `schema`. - * - * @default false + * This is the name of credential. This is just for your reference. + * @minLength 1 + * @maxLength 40 */ - enabled?: boolean; + name?: string; +} + +export interface CreateGoogleSheetsOAuth2AuthorizationCredentialDTO { + provider: 'google.sheets.oauth2-authorization'; + /** The authorization ID for the OAuth2 authorization */ + authorizationId: string; /** - * This is the schema of the structured data. The output is stored in `call.analysis.structuredData`. - * - * Complete guide on JSON Schema can be found [here](https://ajv.js.org/json-schema.html#json-data-type). + * This is the name of credential. This is just for your reference. + * @minLength 1 + * @maxLength 40 */ - schema?: JsonSchema; + name?: string; +} + +export interface CreateSlackOAuth2AuthorizationCredentialDTO { + provider: 'slack.oauth2-authorization'; + /** The authorization ID for the OAuth2 authorization */ + authorizationId: string; /** - * This is how long the request is tried before giving up. When request times out, `call.analysis.structuredData` will be empty. - * - * Usage: - * - To guarantee the structured data is generated, set this value high. Note, this will delay the end of call report in cases where model is slow to respond. - * - * @default 5 seconds - * @min 1 - * @max 60 + * This is the name of credential. This is just for your reference. + * @minLength 1 + * @maxLength 40 */ - timeoutSeconds?: number; + name?: string; } -export interface StructuredDataMultiPlan { - /** This is the key of the structured data plan in the catalog. */ - key: string; - /** This is an individual structured data plan in the catalog. */ - plan: StructuredDataPlan; +export interface TransferAssistantHookAction { + /** This is the type of action - must be "transfer" */ + type: 'transfer'; + /** This is the destination details for the transfer - can be a phone number or SIP URI */ + destination?: TransferDestinationNumber | TransferDestinationSip; } -export interface SuccessEvaluationPlan { +export interface FunctionCallAssistantHookAction { /** - * This enforces the rubric of the evaluation. The output is stored in `call.analysis.successEvaluation`. - * - * Options include: - * - 'NumericScale': A scale of 1 to 10. - * - 'DescriptiveScale': A scale of Excellent, Good, Fair, Poor. - * - 'Checklist': A checklist of criteria and their status. - * - 'Matrix': A grid that evaluates multiple criteria across different performance levels. - * - 'PercentageScale': A scale of 0% to 100%. - * - 'LikertScale': A scale of Strongly Agree, Agree, Neutral, Disagree, Strongly Disagree. - * - 'AutomaticRubric': Automatically break down evaluation into several criteria, each with its own score. - * - 'PassFail': A simple 'true' if call passed, 'false' if not. + * These are the messages that will be spoken to the user as the tool is running. * - * Default is 'PassFail'. + * For some tools, this is auto-filled based on special fields like `tool.destinations`. For others like the function tool, these can be custom configured. */ - rubric?: - | 'NumericScale' - | 'DescriptiveScale' - | 'Checklist' - | 'Matrix' - | 'PercentageScale' - | 'LikertScale' - | 'AutomaticRubric' - | 'PassFail'; + messages?: (ToolMessageStart | ToolMessageComplete | ToolMessageFailed | ToolMessageDelayed)[]; + /** The type of tool. "function" for Function tool. */ + type: 'function'; /** - * These are the messages used to generate the success evaluation. + * This determines if the tool is async. * - * @default: ``` - * [ - * { - * "role": "system", - * "content": "You are an expert call evaluator. You will be given a transcript of a call and the system prompt of the AI participant. Determine if the call was successful based on the objectives inferred from the system prompt. DO NOT return anything except the result.\n\nRubric:\\n{{rubric}}\n\nOnly respond with the result." - * }, - * { - * "role": "user", - * "content": "Here is the transcript:\n\n{{transcript}}\n\n" - * }, - * { - * "role": "user", - * "content": "Here was the system prompt of the call:\n\n{{systemPrompt}}\n\n. Here is the ended reason of the call:\n\n{{endedReason}}\n\n" - * } - * ]``` + * If async, the assistant will move forward without waiting for your server to respond. This is useful if you just want to trigger something on your server. * - * You can customize by providing any messages you want. + * If sync, the assistant will wait for your server to respond. This is useful if want assistant to respond with the result from your server. * - * Here are the template variables available: - * - {{transcript}}: the transcript of the call from `call.artifact.transcript`- {{systemPrompt}}: the system prompt of the call from `assistant.model.messages[type=system].content`- {{rubric}}: the rubric of the success evaluation from `successEvaluationPlan.rubric`- {{endedReason}}: the ended reason of the call from `call.endedReason` + * Defaults to synchronous (`false`). + * @example false */ - messages?: object[]; + async?: boolean; /** - * This determines whether a success evaluation is generated and stored in `call.analysis.successEvaluation`. Defaults to true. * - * Usage: - * - If you want to disable the success evaluation, set this to false. + * This is the server where a `tool-calls` webhook will be sent. * - * @default true + * Notes: + * - Webhook is sent to this server when a tool call is made. + * - Webhook contains the call, assistant, and phone number objects. + * - Webhook contains the variables set on the assistant. + * - Webhook is sent to the first available URL in this order: {{tool.server.url}}, {{assistant.server.url}}, {{phoneNumber.server.url}}, {{org.server.url}}. + * - Webhook expects a response with tool call result. */ - enabled?: boolean; + server?: Server; /** - * This is how long the request is tried before giving up. When request times out, `call.analysis.successEvaluation` will be empty. + * This is the function definition of the tool. * - * Usage: - * - To guarantee the success evaluation is generated, set this value high. Note, this will delay the end of call report in cases where model is slow to respond. + * For `endCall`, `transferCall`, and `dtmf` tools, this is auto-filled based on tool-specific fields like `tool.destinations`. But, even in those cases, you can provide a custom function definition for advanced use cases. * - * @default 5 seconds - * @min 1 - * @max 60 + * An example of an advanced use case is if you want to customize the message that's spoken for `endCall` tool. You can specify a function where it returns an argument "reason". Then, in `messages` array, you can have many "request-complete" messages. One of these messages will be triggered if the `messages[].conditions` matches the "reason" argument. */ - timeoutSeconds?: number; + function?: OpenAIFunction; } -export interface AnalysisPlan { - /** - * The minimum number of messages required to run the analysis plan. - * If the number of messages is less than this, analysis will be skipped. - * @default 2 - * @min 0 - */ - minMessagesThreshold?: number; - /** This is the plan for generating the summary of the call. This outputs to `call.analysis.summary`. */ - summaryPlan?: SummaryPlan; - /** This is the plan for generating the structured data from the call. This outputs to `call.analysis.structuredData`. */ - structuredDataPlan?: StructuredDataPlan; - /** This is an array of structured data plan catalogs. Each entry includes a `key` and a `plan` for generating the structured data from the call. This outputs to `call.analysis.structuredDataMulti`. */ - structuredDataMultiPlan?: StructuredDataMultiPlan[]; - /** This is the plan for generating the success evaluation of the call. This outputs to `call.analysis.successEvaluation`. */ - successEvaluationPlan?: SuccessEvaluationPlan; +export interface SayAssistantHookAction { + /** This is the type of action - must be "say" */ + type: 'say'; + /** This is the message to say */ + exact: object; } -export interface MessagePlan { - /** - * This are the messages that the assistant will speak when the user hasn't responded for `idleTimeoutSeconds`. Each time the timeout is triggered, a random message will be chosen from this array. - * - * Usage: - * - If user gets distracted and doesn't respond for a while, this can be used to grab their attention. - * - If the transcriber doesn't pick up what the user said, this can be used to ask the user to repeat themselves. (From the perspective of the assistant, the conversation is idle since it didn't "hear" any user messages.) - * - * @default null (no idle message is spoken) - */ - idleMessages?: string[]; +export interface AssistantHookFilter { /** - * This determines the maximum number of times `idleMessages` can be spoken during the call. - * - * @default 3 - * @min 1 - * @max 10 + * This is the type of filter - currently only "oneOf" is supported + * @maxLength 1000 */ - idleMessageMaxSpokenCount?: number; + type: 'oneOf'; /** - * This determines whether the idle message count is reset whenever the user speaks. - * - * @default false + * This is the key to filter on (e.g. "call.endedReason") + * @maxLength 1000 */ - idleMessageResetCountOnUserSpeechEnabled?: boolean; - /** - * This is the timeout in seconds before a message from `idleMessages` is spoken. The clock starts when the assistant finishes speaking and remains active until the user speaks. - * - * @default 10 - * @min 5 - * @max 60 + key: string; + /** This is the array of possible values to match against */ + oneOf: string[]; +} + +export interface AssistantHookCallEnding { + /** + * This is the event that triggers this hook + * @maxLength 1000 */ - idleTimeoutSeconds?: number; + on: 'call.ending'; + /** This is the set of actions to perform when the hook triggers */ + do: (TransferAssistantHookAction | FunctionCallAssistantHookAction)[]; + /** This is the set of filters that must match for the hook to trigger */ + filters?: AssistantHookFilter[]; +} + +export interface AssistantHookAssistantSpeechInterrupted { /** - * This is the message that the assistant will say if the call ends due to silence. - * - * If unspecified, it will hang up without saying anything. + * This is the event that triggers this hook * @maxLength 1000 */ - silenceTimeoutMessage?: string; + on: 'assistant.speech.interrupted'; + /** This is the set of actions to perform when the hook triggers */ + do: (TransferAssistantHookAction | FunctionCallAssistantHookAction | SayAssistantHookAction)[]; } -export interface AssistantCustomEndpointingRule { +export interface AssistantHookCustomerSpeechInterrupted { /** - * This endpointing rule is based on the last assistant message before customer started speaking. - * - * Flow: - * - Assistant speaks - * - Customer starts speaking - * - Customer transcription comes in - * - This rule is evaluated on the last assistant message - * - If a match is found based on `regex`, the endpointing timeout is set to `timeoutSeconds` - * - * Usage: - * - If you have yes/no questions in your use case like "are you interested in a loan?", you can set a shorter timeout. - * - If you have questions where the customer may pause to look up information like "what's my account number?", you can set a longer timeout. + * This is the event that triggers this hook + * @maxLength 1000 */ - type: 'assistant'; + on: 'customer.speech.interrupted'; + /** This is the set of actions to perform when the hook triggers */ + do: (TransferAssistantHookAction | FunctionCallAssistantHookAction | SayAssistantHookAction)[]; +} + +export interface VoicemailDetectionBackoffPlan { /** - * This is the regex pattern to match. - * - * Note: - * - This works by using the `RegExp.test` method in Node.JS. Eg. `/hello/.test("hello there")` will return `true`. - * - * Hot tip: - * - In JavaScript, escape `\` when sending the regex pattern. Eg. `"hello\sthere"` will be sent over the wire as `"hellosthere"`. Send `"hello\\sthere"` instead. - * - `RegExp.test` does substring matching, so `/cat/.test("I love cats")` will return `true`. To do full string matching, send "^cat$". + * This is the number of seconds to wait before starting the first retry attempt. + * @min 0 + * @default 5 */ - regex: string; + startAtSeconds?: number; /** - * These are the options for the regex match. Defaults to all disabled. - * - * @default [] + * This is the interval in seconds between retry attempts. + * @min 2.5 + * @default 5 */ - regexOptions?: RegexOption[]; + frequencySeconds?: number; /** - * This is the endpointing timeout in seconds, if the rule is matched. - * @min 0 - * @max 15 + * This is the maximum number of retry attempts before giving up. + * @min 1 + * @max 10 + * @default 6 */ - timeoutSeconds: number; + maxRetries?: number; } -export interface CustomerCustomEndpointingRule { +export interface GoogleVoicemailDetectionPlan { /** - * This endpointing rule is based on current customer message as they are speaking. + * This is the maximum duration from the start of the call that we will wait for a voicemail beep, before speaking our message * - * Flow: - * - Assistant speaks - * - Customer starts speaking - * - Customer transcription comes in - * - This rule is evaluated on the current customer transcription - * - If a match is found based on `regex`, the endpointing timeout is set to `timeoutSeconds` + * - If we detect a voicemail beep before this, we will speak the message at that point. * - * Usage: - * - If you want to wait longer while customer is speaking numbers, you can set a longer timeout. + * - Setting too low a value means that the bot will start speaking its voicemail message too early. If it does so before the actual beep, it will get cut off. You should definitely tune this to your use case. + * + * @default 30 + * @min 0 + * @max 60 + * @min 0 + * @max 30 + * @default 30 */ - type: 'customer'; + beepMaxAwaitSeconds?: number; + /** This is the provider to use for voicemail detection. */ + provider: 'google'; + /** This is the backoff plan for the voicemail detection. */ + backoffPlan?: VoicemailDetectionBackoffPlan; +} + +export interface OpenAIVoicemailDetectionPlan { /** - * This is the regex pattern to match. + * This is the maximum duration from the start of the call that we will wait for a voicemail beep, before speaking our message * - * Note: - * - This works by using the `RegExp.test` method in Node.JS. Eg. `/hello/.test("hello there")` will return `true`. + * - If we detect a voicemail beep before this, we will speak the message at that point. * - * Hot tip: - * - In JavaScript, escape `\` when sending the regex pattern. Eg. `"hello\sthere"` will be sent over the wire as `"hellosthere"`. Send `"hello\\sthere"` instead. - * - `RegExp.test` does substring matching, so `/cat/.test("I love cats")` will return `true`. To do full string matching, send "^cat$". - */ - regex: string; - /** - * These are the options for the regex match. Defaults to all disabled. + * - Setting too low a value means that the bot will start speaking its voicemail message too early. If it does so before the actual beep, it will get cut off. You should definitely tune this to your use case. * - * @default [] - */ - regexOptions?: RegexOption[]; - /** - * This is the endpointing timeout in seconds, if the rule is matched. + * @default 30 * @min 0 - * @max 15 + * @max 60 + * @min 0 + * @max 30 + * @default 30 */ - timeoutSeconds: number; + beepMaxAwaitSeconds?: number; + /** This is the provider to use for voicemail detection. */ + provider: 'openai'; + /** This is the backoff plan for the voicemail detection. */ + backoffPlan?: VoicemailDetectionBackoffPlan; } -export interface BothCustomEndpointingRule { +export interface TwilioVoicemailDetectionPlan { + /** This is the provider to use for voicemail detection. */ + provider: 'twilio'; /** - * This endpointing rule is based on both the last assistant message and the current customer message as they are speaking. - * - * Flow: - * - Assistant speaks - * - Customer starts speaking - * - Customer transcription comes in - * - This rule is evaluated on the last assistant message and the current customer transcription - * - If assistant message matches `assistantRegex` AND customer message matches `customerRegex`, the endpointing timeout is set to `timeoutSeconds` + * These are the AMD messages from Twilio that are considered as voicemail. Default is ['machine_end_beep', 'machine_end_silence']. * - * Usage: - * - If you want to wait longer while customer is speaking numbers, you can set a longer timeout. + * @default {Array} ['machine_end_beep', 'machine_end_silence'] + * @example ["machine_end_beep","machine_end_silence"] */ - type: 'both'; + voicemailDetectionTypes?: + | 'machine_start' + | 'human' + | 'fax' + | 'unknown' + | 'machine_end_beep' + | 'machine_end_silence' + | 'machine_end_other'; /** - * This is the regex pattern to match the assistant's message. - * - * Note: - * - This works by using the `RegExp.test` method in Node.JS. Eg. `/hello/.test("hello there")` will return `true`. + * This sets whether the assistant should detect voicemail. Defaults to true. * - * Hot tip: - * - In JavaScript, escape `\` when sending the regex pattern. Eg. `"hello\sthere"` will be sent over the wire as `"hellosthere"`. Send `"hello\\sthere"` instead. - * - `RegExp.test` does substring matching, so `/cat/.test("I love cats")` will return `true`. To do full string matching, send "^cat$". + * @default true */ - assistantRegex: string; + enabled?: boolean; /** - * These are the options for the assistant's message regex match. Defaults to all disabled. + * The number of seconds that Twilio should attempt to perform answering machine detection before timing out and returning AnsweredBy as unknown. Default is 30 seconds. * - * @default [] + * Increasing this value will provide the engine more time to make a determination. This can be useful when DetectMessageEnd is provided in the MachineDetection parameter and there is an expectation of long answering machine greetings that can exceed 30 seconds. + * + * Decreasing this value will reduce the amount of time the engine has to make a determination. This can be particularly useful when the Enable option is provided in the MachineDetection parameter and you want to limit the time for initial detection. + * + * Check the [Twilio docs](https://www.twilio.com/docs/voice/answering-machine-detection#optional-api-tuning-parameters) for more info. + * + * @default 30 + * @min 3 + * @max 59 */ - assistantRegexOptions?: RegexOption[]; - customerRegex: string; + machineDetectionTimeout?: number; /** - * These are the options for the customer's message regex match. Defaults to all disabled. + * The number of milliseconds that is used as the measuring stick for the length of the speech activity. Durations lower than this value will be interpreted as a human, longer as a machine. Default is 2400 milliseconds. * - * @default [] + * Increasing this value will reduce the chance of a False Machine (detected machine, actually human) for a long human greeting (e.g., a business greeting) but increase the time it takes to detect a machine. + * + * Decreasing this value will reduce the chances of a False Human (detected human, actually machine) for short voicemail greetings. The value of this parameter may need to be reduced by more than 1000ms to detect very short voicemail greetings. A reduction of that significance can result in increased False Machine detections. Adjusting the MachineDetectionSpeechEndThreshold is likely the better approach for short voicemails. Decreasing MachineDetectionSpeechThreshold will also reduce the time it takes to detect a machine. + * + * Check the [Twilio docs](https://www.twilio.com/docs/voice/answering-machine-detection#optional-api-tuning-parameters) for more info. + * + * @default 2400 + * @min 1000 + * @max 6000 */ - customerRegexOptions?: RegexOption[]; + machineDetectionSpeechThreshold?: number; /** - * This is the endpointing timeout in seconds, if the rule is matched. - * @min 0 - * @max 15 + * The number of milliseconds of silence after speech activity at which point the speech activity is considered complete. Default is 1200 milliseconds. + * + * Increasing this value will typically be used to better address the short voicemail greeting scenarios. For short voicemails, there is typically 1000-2000ms of audio followed by 1200-2400ms of silence and then additional audio before the beep. Increasing the MachineDetectionSpeechEndThreshold to ~2500ms will treat the 1200-2400ms of silence as a gap in the greeting but not the end of the greeting and will result in a machine detection. The downsides of such a change include: + * - Increasing the delay for human detection by the amount you increase this parameter, e.g., a change of 1200ms to 2500ms increases human detection delay by 1300ms. + * - Cases where a human has two utterances separated by a period of silence (e.g. a "Hello", then 2000ms of silence, and another "Hello") may be interpreted as a machine. + * + * Decreasing this value will result in faster human detection. The consequence is that it can lead to increased False Human (detected human, actually machine) detections because a silence gap in a voicemail greeting (not necessarily just in short voicemail scenarios) can be incorrectly interpreted as the end of speech. + * + * Check the [Twilio docs](https://www.twilio.com/docs/voice/answering-machine-detection#optional-api-tuning-parameters) for more info. + * + * @default 1200 + * @min 500 + * @max 5000 */ - timeoutSeconds: number; -} - -export interface VapiSmartEndpointingPlan { + machineDetectionSpeechEndThreshold?: number; /** - * This is the provider for the smart endpointing plan. - * @example "vapi" + * The number of milliseconds of initial silence after which an unknown AnsweredBy result will be returned. Default is 5000 milliseconds. + * + * Increasing this value will result in waiting for a longer period of initial silence before returning an 'unknown' AMD result. + * + * Decreasing this value will result in waiting for a shorter period of initial silence before returning an 'unknown' AMD result. + * + * Check the [Twilio docs](https://www.twilio.com/docs/voice/answering-machine-detection#optional-api-tuning-parameters) for more info. + * + * @default 5000 + * @min 2000 + * @max 10000 */ - provider: 'vapi' | 'livekit'; + machineDetectionSilenceTimeout?: number; } -export interface LivekitSmartEndpointingPlan { - /** - * This is the provider for the smart endpointing plan. - * @example "livekit" - */ - provider: 'vapi' | 'livekit'; +export interface VapiVoicemailDetectionPlan { /** - * This expression describes how long the bot will wait to start speaking based on the likelihood that the user has reached an endpoint. - * - * This is a millisecond valued function. It maps probabilities (real numbers on [0,1]) to milliseconds that the bot should wait before speaking ([0, \infty]). Any negative values that are returned are set to zero (the bot can't start talking in the past). + * This is the maximum duration from the start of the call that we will wait for a voicemail beep, before speaking our message * - * A probability of zero represents very high confidence that the caller has stopped speaking, and would like the bot to speak to them. A probability of one represents very high confidence that the caller is still speaking. + * - If we detect a voicemail beep before this, we will speak the message at that point. * - * Under the hood, this is parsed into a mathjs expression. Whatever you use to write your expression needs to be valid with respect to mathjs + * - Setting too low a value means that the bot will start speaking its voicemail message too early. If it does so before the actual beep, it will get cut off. You should definitely tune this to your use case. * - * @default "20 + 500 * sqrt(x) + 2500 * x^3" + * @default 30 + * @min 0 + * @max 60 + * @min 0 + * @max 30 + * @default 30 */ - waitFunction?: string; + beepMaxAwaitSeconds?: number; + /** This is the provider to use for voicemail detection. */ + provider: 'vapi'; + /** This is the backoff plan for the voicemail detection. */ + backoffPlan?: VoicemailDetectionBackoffPlan; } -export interface TranscriptionEndpointingPlan { +export interface MessagePlan { /** - * The minimum number of seconds to wait after transcription ending with punctuation before sending a request to the model. Defaults to 0.1. + * This are the messages that the assistant will speak when the user hasn't responded for `idleTimeoutSeconds`. Each time the timeout is triggered, a random message will be chosen from this array. * - * This setting exists because the transcriber punctuates the transcription when it's more confident that customer has completed a thought. + * Usage: + * - If user gets distracted and doesn't respond for a while, this can be used to grab their attention. + * - If the transcriber doesn't pick up what the user said, this can be used to ask the user to repeat themselves. (From the perspective of the assistant, the conversation is idle since it didn't "hear" any user messages.) * - * @default 0.1 - * @min 0 - * @max 3 - * @example 0.1 + * @default null (no idle message is spoken) */ - onPunctuationSeconds?: number; + idleMessages?: string[]; /** - * The minimum number of seconds to wait after transcription ending without punctuation before sending a request to the model. Defaults to 1.5. + * This determines the maximum number of times `idleMessages` can be spoken during the call. * - * This setting exists to catch the cases where the transcriber was not confident enough to punctuate the transcription, but the customer is done and has been silent for a long time. + * @default 3 + * @min 1 + * @max 10 + */ + idleMessageMaxSpokenCount?: number; + /** + * This determines whether the idle message count is reset whenever the user speaks. * - * @default 1.5 - * @min 0 - * @max 3 - * @example 1.5 + * @default false */ - onNoPunctuationSeconds?: number; + idleMessageResetCountOnUserSpeechEnabled?: boolean; /** - * The minimum number of seconds to wait after transcription ending with a number before sending a request to the model. Defaults to 0.4. + * This is the timeout in seconds before a message from `idleMessages` is spoken. The clock starts when the assistant finishes speaking and remains active until the user speaks. * - * This setting exists because the transcriber will sometimes punctuate the transcription ending with a number, even though the customer hasn't uttered the full number. This happens commonly for long numbers when the customer reads the number in chunks. + * @default 10 + * @min 5 + * @max 60 + */ + idleTimeoutSeconds?: number; + /** + * This is the message that the assistant will say if the call ends due to silence. * - * @default 0.5 - * @min 0 - * @max 3 - * @example 0.5 + * If unspecified, it will hang up without saying anything. + * @maxLength 1000 */ - onNumberSeconds?: number; + silenceTimeoutMessage?: string; } -export interface StartSpeakingPlan { +export interface KeypadInputPlan { /** - * This is how long assistant waits before speaking. Defaults to 0.4. - * - * This is the minimum it will wait but if there is latency is the pipeline, this minimum will be exceeded. This is intended as a stopgap in case the pipeline is moving too fast. - * - * Example: - * - If model generates tokens and voice generates bytes within 100ms, the pipeline still waits 300ms before outputting speech. + * This keeps track of whether the user has enabled keypad input. + * By default, it is off. * - * Usage: - * - If the customer is taking long pauses, set this to a higher value. - * - If the assistant is accidentally jumping in too much, set this to a higher value. + * @default false + */ + enabled?: boolean; + /** + * This is the time in seconds to wait before processing the input. + * If the input is not received within this time, the input will be ignored. + * If set to "off", the input will be processed when the user enters a delimiter or immediately if no delimiter is used. * - * @default 0.4 + * @default 2 * @min 0 - * @max 5 - * @example 0.4 + * @max 10 */ - waitSeconds?: number; + timeoutSeconds?: number; /** - * @deprecated - * @example false + * This is the delimiter(s) that will be used to process the input. + * Can be '#', '*', or an empty array. */ - smartEndpointingEnabled?: boolean | 'livekit'; + delimiters?: '#' | '*' | ''; +} + +export interface CreateAssistantDTO { + /** These are the options for the assistant's transcriber. */ + transcriber?: + | AssemblyAITranscriber + | AzureSpeechTranscriber + | CustomTranscriber + | DeepgramTranscriber + | ElevenLabsTranscriber + | GladiaTranscriber + | GoogleTranscriber + | SpeechmaticsTranscriber + | TalkscriberTranscriber + | OpenAITranscriber + | CartesiaTranscriber; + /** These are the options for the assistant's LLM. */ + model?: + | AnthropicModel + | AnyscaleModel + | CerebrasModel + | CustomLLMModel + | DeepInfraModel + | DeepSeekModel + | GoogleModel + | GroqModel + | InflectionAIModel + | OpenAIModel + | OpenRouterModel + | PerplexityAIModel + | TogetherAIModel + | XaiModel; + /** These are the options for the assistant's voice. */ + voice?: + | AzureVoice + | CartesiaVoice + | CustomVoice + | DeepgramVoice + | ElevenLabsVoice + | HumeVoice + | LMNTVoice + | NeuphonicVoice + | OpenAIVoice + | PlayHTVoice + | RimeAIVoice + | SmallestAIVoice + | TavusVoice + | VapiVoice + | SesameVoice; /** - * This is the plan for smart endpointing. Pick between Vapi smart endpointing or LiveKit smart endpointing (or nothing). We strongly recommend using livekit endpointing when working in English. LiveKit endpointing is not supported in other languages, yet. + * This is the first message that the assistant will say. This can also be a URL to a containerized audio file (mp3, wav, etc.). * - * If this is set, it will override and take precedence over `transcriptionEndpointingPlan`. - * This plan will still be overridden by any matching `customEndpointingRules`. + * If unspecified, assistant will wait for user to speak and use the model to respond once they speak. + * @example "Hello! How can I help you today?" */ - smartEndpointingPlan?: VapiSmartEndpointingPlan | LivekitSmartEndpointingPlan; + firstMessage?: string; + /** @default false */ + firstMessageInterruptionsEnabled?: boolean; /** - * These are the custom endpointing rules to set an endpointing timeout based on a regex on the customer's speech or the assistant's last message. - * - * Usage: - * - If you have yes/no questions like "are you interested in a loan?", you can set a shorter timeout. - * - If you have questions where the customer may pause to look up information like "what's my account number?", you can set a longer timeout. - * - If you want to wait longer while customer is enumerating a list of numbers, you can set a longer timeout. - * - * These rules have the highest precedence and will override both `smartEndpointingPlan` and `transcriptionEndpointingPlan` when a rule is matched. - * - * The rules are evaluated in order and the first one that matches will be used. + * This is the mode for the first message. Default is 'assistant-speaks-first'. * - * Order of precedence for endpointing: - * 1. customEndpointingRules (if any match) - * 2. smartEndpointingPlan (if set) - * 3. transcriptionEndpointingPlan + * Use: + * - 'assistant-speaks-first' to have the assistant speak first. + * - 'assistant-waits-for-user' to have the assistant wait for the user to speak first. + * - 'assistant-speaks-first-with-model-generated-message' to have the assistant speak first with a message generated by the model based on the conversation state. (`assistant.model.messages` at call start, `call.messages` at squad transfer points). * - * @default [] + * @default 'assistant-speaks-first' + * @example "assistant-speaks-first" */ - customEndpointingRules?: ( - | AssistantCustomEndpointingRule - | CustomerCustomEndpointingRule - | BothCustomEndpointingRule - )[]; + firstMessageMode?: + | 'assistant-speaks-first' + | 'assistant-speaks-first-with-model-generated-message' + | 'assistant-waits-for-user'; /** - * This determines how a customer speech is considered done (endpointing) using the transcription of customer's speech. - * - * Once an endpoint is triggered, the request is sent to `assistant.model`. - * - * Note: This plan is only used if `smartEndpointingPlan` is not set. If both are provided, `smartEndpointingPlan` takes precedence. - * This plan will also be overridden by any matching `customEndpointingRules`. + * These are the settings to configure or disable voicemail detection. Alternatively, voicemail detection can be configured using the model.tools=[VoicemailTool]. + * This uses Twilio's built-in detection while the VoicemailTool relies on the model to detect if a voicemail was reached. + * You can use neither of them, one of them, or both of them. By default, Twilio built-in detection is enabled while VoicemailTool is not. */ - transcriptionEndpointingPlan?: TranscriptionEndpointingPlan; -} - -export interface StopSpeakingPlan { + voicemailDetection?: + | GoogleVoicemailDetectionPlan + | OpenAIVoicemailDetectionPlan + | TwilioVoicemailDetectionPlan + | VapiVoicemailDetectionPlan; /** - * This is the number of words that the customer has to say before the assistant will stop talking. - * - * Words like "stop", "actually", "no", etc. will always interrupt immediately regardless of this value. - * - * Words like "okay", "yeah", "right" will never interrupt. - * - * When set to 0, `voiceSeconds` is used in addition to the transcriptions to determine the customer has started speaking. - * - * Defaults to 0. - * - * @default 0 - * @min 0 - * @max 10 - * @example 0 + * These are the messages that will be sent to your Client SDKs. Default is conversation-update,function-call,hang,model-output,speech-update,status-update,transfer-update,transcript,tool-calls,user-interrupted,voice-input,workflow.node.started. You can check the shape of the messages in ClientMessage schema. + * @example ["conversation-update","function-call","hang","model-output","speech-update","status-update","transfer-update","transcript","tool-calls","user-interrupted","voice-input","workflow.node.started"] */ - numWords?: number; + clientMessages?: + | 'conversation-update' + | 'function-call' + | 'function-call-result' + | 'hang' + | 'language-changed' + | 'metadata' + | 'model-output' + | 'speech-update' + | 'status-update' + | 'transcript' + | 'tool-calls' + | 'tool-calls-result' + | 'tool.completed' + | 'transfer-update' + | 'user-interrupted' + | 'voice-input' + | 'workflow.node.started'; + /** + * These are the messages that will be sent to your Server URL. Default is conversation-update,end-of-call-report,function-call,hang,speech-update,status-update,tool-calls,transfer-destination-request,user-interrupted. You can check the shape of the messages in ServerMessage schema. + * @example ["conversation-update","end-of-call-report","function-call","hang","speech-update","status-update","tool-calls","transfer-destination-request","user-interrupted"] + */ + serverMessages?: + | 'conversation-update' + | 'end-of-call-report' + | 'function-call' + | 'hang' + | 'language-changed' + | 'language-change-detected' + | 'model-output' + | 'phone-call-control' + | 'speech-update' + | 'status-update' + | 'transcript' + | "transcript[transcriptType='final']" + | 'tool-calls' + | 'transfer-destination-request' + | 'transfer-update' + | 'user-interrupted' + | 'voice-input'; /** - * This is the seconds customer has to speak before the assistant stops talking. This uses the VAD (Voice Activity Detection) spike to determine if the customer has started speaking. + * How many seconds of silence to wait before ending the call. Defaults to 30. * - * Considerations: - * - A lower value might be more responsive but could potentially pick up non-speech sounds. - * - A higher value reduces false positives but might slightly delay the detection of speech onset. + * @default 30 + * @min 10 + * @max 3600 + * @example 30 + */ + silenceTimeoutSeconds?: number; + /** + * This is the maximum number of seconds that the call will last. When the call reaches this duration, it will be ended. * - * This is only used if `numWords` is set to 0. + * @default 600 (10 minutes) + * @min 10 + * @max 43200 + * @example 600 + */ + maxDurationSeconds?: number; + /** + * This is the background sound in the call. Default for phone calls is 'office' and default for web calls is 'off'. + * You can also provide a custom sound by providing a URL to an audio file. + */ + backgroundSound?: 'off' | 'office' | string; + /** + * This enables filtering of noise and background speech while the user is talking. * - * Defaults to 0.2 + * Default `false` while in beta. * - * @default 0.2 - * @min 0 - * @max 0.5 - * @example 0.2 + * @default false + * @deprecated + * @example false */ - voiceSeconds?: number; + backgroundDenoisingEnabled?: boolean; /** - * This is the seconds to wait before the assistant will start talking again after being interrupted. + * This determines whether the model's output is used in conversation history rather than the transcription of assistant's speech. * - * Defaults to 1. + * Default `false` while in beta. * - * @default 1 - * @min 0 - * @max 10 - * @example 1 + * @default false + * @example false */ - backoffSeconds?: number; + modelOutputInMessagesEnabled?: boolean; + /** These are the configurations to be passed to the transport providers of assistant's calls, like Twilio. You can store multiple configurations for different transport providers. For a call, only the configuration matching the call transport provider is used. */ + transportConfigurations?: TransportConfigurationTwilio[]; /** - * These are the phrases that will never interrupt the assistant, even if numWords threshold is met. - * These are typically acknowledgement or backchanneling phrases. - * @default ["i understand","i see","i got it","i hear you","im listening","im with you","right","okay","ok","sure","alright","got it","understood","yeah","yes","uh-huh","mm-hmm","gotcha","mhmm","ah","yeah okay","yeah sure"] - * @example ["i understand","i see","i got it","i hear you","im listening","im with you","right","okay","ok","sure","alright","got it","understood","yeah","yes","uh-huh","mm-hmm","gotcha","mhmm","ah","yeah okay","yeah sure"] + * This is the plan for observability of assistant's calls. + * + * Currently, only Langfuse is supported. */ - acknowledgementPhrases?: string[]; + observabilityPlan?: LangfuseObservabilityPlan; + /** These are dynamic credentials that will be used for the assistant calls. By default, all the credentials are available for use in the call but you can supplement an additional credentials using this. Dynamic credentials override existing credentials. */ + credentials?: ( + | ({ + provider: '11labs'; + } & CreateElevenLabsCredentialDTO) + | ({ + provider: 'anthropic'; + } & CreateAnthropicCredentialDTO) + | ({ + provider: 'anyscale'; + } & CreateAnyscaleCredentialDTO) + | ({ + provider: 'assembly-ai'; + } & CreateAssemblyAICredentialDTO) + | ({ + provider: 'azure-openai'; + } & CreateAzureOpenAICredentialDTO) + | ({ + provider: 'azure'; + } & CreateAzureCredentialDTO) + | ({ + provider: 'byo-sip-trunk'; + } & CreateByoSipTrunkCredentialDTO) + | ({ + provider: 'cartesia'; + } & CreateCartesiaCredentialDTO) + | ({ + provider: 'cerebras'; + } & CreateCerebrasCredentialDTO) + | ({ + provider: 'cloudflare'; + } & CreateCloudflareCredentialDTO) + | ({ + provider: 'custom-llm'; + } & CreateCustomLLMCredentialDTO) + | ({ + provider: 'deepgram'; + } & CreateDeepgramCredentialDTO) + | ({ + provider: 'deepinfra'; + } & CreateDeepInfraCredentialDTO) + | ({ + provider: 'deep-seek'; + } & CreateDeepSeekCredentialDTO) + | ({ + provider: 'gcp'; + } & CreateGcpCredentialDTO) + | ({ + provider: 'gladia'; + } & CreateGladiaCredentialDTO) + | ({ + provider: 'gohighlevel'; + } & CreateGoHighLevelCredentialDTO) + | ({ + provider: 'google'; + } & CreateGoogleCredentialDTO) + | ({ + provider: 'groq'; + } & CreateGroqCredentialDTO) + | ({ + provider: 'inflection-ai'; + } & CreateInflectionAICredentialDTO) + | ({ + provider: 'langfuse'; + } & CreateLangfuseCredentialDTO) + | ({ + provider: 'lmnt'; + } & CreateLmntCredentialDTO) + | ({ + provider: 'make'; + } & CreateMakeCredentialDTO) + | ({ + provider: 'openai'; + } & CreateOpenAICredentialDTO) + | ({ + provider: 'openrouter'; + } & CreateOpenRouterCredentialDTO) + | ({ + provider: 'perplexity-ai'; + } & CreatePerplexityAICredentialDTO) + | ({ + provider: 'playht'; + } & CreatePlayHTCredentialDTO) + | ({ + provider: 'rime-ai'; + } & CreateRimeAICredentialDTO) + | ({ + provider: 'runpod'; + } & CreateRunpodCredentialDTO) + | ({ + provider: 's3'; + } & CreateS3CredentialDTO) + | ({ + provider: 'supabase'; + } & CreateSupabaseCredentialDTO) + | ({ + provider: 'smallest-ai'; + } & CreateSmallestAICredentialDTO) + | ({ + provider: 'tavus'; + } & CreateTavusCredentialDTO) + | ({ + provider: 'together-ai'; + } & CreateTogetherAICredentialDTO) + | ({ + provider: 'twilio'; + } & CreateTwilioCredentialDTO) + | ({ + provider: 'vonage'; + } & CreateVonageCredentialDTO) + | ({ + provider: 'webhook'; + } & CreateWebhookCredentialDTO) + | ({ + provider: 'xai'; + } & CreateXAiCredentialDTO) + | ({ + provider: 'neuphonic'; + } & CreateNeuphonicCredentialDTO) + | ({ + provider: 'hume'; + } & CreateHumeCredentialDTO) + | ({ + provider: 'mistral'; + } & CreateMistralCredentialDTO) + | ({ + provider: 'speechmatics'; + } & CreateSpeechmaticsCredentialDTO) + | ({ + provider: 'trieve'; + } & CreateTrieveCredentialDTO) + | ({ + provider: 'google.calendar.oauth2-client'; + } & CreateGoogleCalendarOAuth2ClientCredentialDTO) + | ({ + provider: 'google.calendar.oauth2-authorization'; + } & CreateGoogleCalendarOAuth2AuthorizationCredentialDTO) + | ({ + provider: 'google.sheets.oauth2-authorization'; + } & CreateGoogleSheetsOAuth2AuthorizationCredentialDTO) + | ({ + provider: 'slack.oauth2-authorization'; + } & CreateSlackOAuth2AuthorizationCredentialDTO) + | ({ + provider: 'ghl.oauth2-authorization'; + } & CreateGoHighLevelMCPCredentialDTO) + | ({ + provider: 'inworld'; + } & any) + )[]; + /** This is a set of actions that will be performed on certain events. */ + hooks?: ( + | AssistantHookCallEnding + | AssistantHookAssistantSpeechInterrupted + | AssistantHookCustomerSpeechInterrupted + )[]; /** - * These are the phrases that will always interrupt the assistant immediately, regardless of numWords. - * These are typically phrases indicating disagreement or desire to stop. - * @default ["stop","shut","up","enough","quiet","silence","but","dont","not","no","hold","wait","cut","pause","nope","nah","nevermind","never","bad","actually"] - * @example ["stop","shut","up","enough","quiet","silence","but","dont","not","no","hold","wait","cut","pause","nope","nah","nevermind","never","bad","actually"] + * This is the name of the assistant. + * + * This is required when you want to transfer between assistants in a call. + * @maxLength 40 */ - interruptionPhrases?: string[]; -} - -export interface MonitorPlan { + name?: string; /** - * This determines whether the assistant's calls allow live listening. Defaults to true. - * - * Fetch `call.monitor.listenUrl` to get the live listening URL. + * This is the message that the assistant will say if the call is forwarded to voicemail. * - * @default true - * @example false + * If unspecified, it will hang up. + * @maxLength 1000 */ - listenEnabled?: boolean; + voicemailMessage?: string; /** - * This enables authentication on the `call.monitor.listenUrl`. - * - * If `listenAuthenticationEnabled` is `true`, the `call.monitor.listenUrl` will require an `Authorization: Bearer ` header. + * This is the message that the assistant will say if it ends the call. * - * @default false - * @example false + * If unspecified, it will hang up without saying anything. + * @maxLength 1000 */ - listenAuthenticationEnabled?: boolean; + endCallMessage?: string; + /** This list contains phrases that, if spoken by the assistant, will trigger the call to be hung up. Case insensitive. */ + endCallPhrases?: string[]; + compliancePlan?: CompliancePlan; + /** This is for metadata you want to store on the assistant. */ + metadata?: object; /** - * This determines whether the assistant's calls allow live control. Defaults to true. + * This enables filtering of noise and background speech while the user is talking. * - * Fetch `call.monitor.controlUrl` to get the live control URL. + * Features: + * - Smart denoising using Krisp + * - Fourier denoising * - * To use, send any control message via a POST request to `call.monitor.controlUrl`. Here are the types of controls supported: https://docs.vapi.ai/api-reference/messages/client-inbound-message + * Smart denoising can be combined with or used independently of Fourier denoising. * - * @default true - * @example false + * Order of precedence: + * - Smart denoising + * - Fourier denoising */ - controlEnabled?: boolean; + backgroundSpeechDenoisingPlan?: BackgroundSpeechDenoisingPlan; + /** This is the plan for analysis of assistant's calls. Stored in `call.analysis`. */ + analysisPlan?: AnalysisPlan; + /** This is the plan for artifacts generated during assistant's calls. Stored in `call.artifact`. */ + artifactPlan?: ArtifactPlan; /** - * This enables authentication on the `call.monitor.controlUrl`. + * This is the plan for static predefined messages that can be spoken by the assistant during the call, like `idleMessages`. * - * If `controlAuthenticationEnabled` is `true`, the `call.monitor.controlUrl` will require an `Authorization: Bearer ` header. + * Note: `firstMessage`, `voicemailMessage`, and `endCallMessage` are currently at the root level. They will be moved to `messagePlan` in the future, but will remain backwards compatible. + */ + messagePlan?: MessagePlan; + /** + * This is the plan for when the assistant should start talking. * - * @default false - * @example false + * You should configure this if you're running into these issues: + * - The assistant is too slow to start talking after the customer is done speaking. + * - The assistant is too fast to start talking after the customer is done speaking. + * - The assistant is so fast that it's actually interrupting the customer. */ - controlAuthenticationEnabled?: boolean; -} - -export interface KeypadInputPlan { + startSpeakingPlan?: StartSpeakingPlan; /** - * This keeps track of whether the user has enabled keypad input. - * By default, it is off. + * This is the plan for when assistant should stop talking on customer interruption. * - * @default false + * You should configure this if you're running into these issues: + * - The assistant is too slow to recognize customer's interruption. + * - The assistant is too fast to recognize customer's interruption. + * - The assistant is getting interrupted by phrases that are just acknowledgments. + * - The assistant is getting interrupted by background noises. + * - The assistant is not properly stopping -- it starts talking right after getting interrupted. */ - enabled?: boolean; + stopSpeakingPlan?: StopSpeakingPlan; /** - * This is the time in seconds to wait before processing the input. - * If the input is not received within this time, the input will be ignored. - * If set to "off", the input will be processed when the user enters a delimiter or immediately if no delimiter is used. + * This is the plan for real-time monitoring of the assistant's calls. * - * @default 2 - * @min 0 - * @max 10 + * Usage: + * - To enable live listening of the assistant's calls, set `monitorPlan.listenEnabled` to `true`. + * - To enable live control of the assistant's calls, set `monitorPlan.controlEnabled` to `true`. */ - timeoutSeconds?: number; + monitorPlan?: MonitorPlan; + /** These are the credentials that will be used for the assistant calls. By default, all the credentials are available for use in the call but you can provide a subset using this. */ + credentialIds?: string[]; /** - * This is the delimiter(s) that will be used to process the input. - * Can be '#', '*', or an empty array. + * This is where Vapi will send webhooks. You can find all webhooks available along with their shape in ServerMessage schema. + * + * The order of precedence is: + * + * 1. assistant.server.url + * 2. phoneNumber.serverUrl + * 3. org.serverUrl */ - delimiters?: '#' | '*' | ''; + server?: Server; + keypadInputPlan?: KeypadInputPlan; } -export interface CreateAssistantDTO { +export interface AssistantOverrides { /** These are the options for the assistant's transcriber. */ transcriber?: | AssemblyAITranscriber @@ -10071,7 +11280,15 @@ export interface CreateAssistantDTO { * You can also provide a custom sound by providing a URL to an audio file. */ backgroundSound?: 'off' | 'office' | string; - /** @example false */ + /** + * This enables filtering of noise and background speech while the user is talking. + * + * Default `false` while in beta. + * + * @default false + * @deprecated + * @example false + */ backgroundDenoisingEnabled?: boolean; /** * This determines whether the model's output is used in conversation history rather than the transcription of assistant's speech. @@ -10085,8 +11302,9 @@ export interface CreateAssistantDTO { /** These are the configurations to be passed to the transport providers of assistant's calls, like Twilio. You can store multiple configurations for different transport providers. For a call, only the configuration matching the call transport provider is used. */ transportConfigurations?: TransportConfigurationTwilio[]; /** - * This is the plan for observability configuration of assistant's calls. - * Currently supports Langfuse for tracing and monitoring. + * This is the plan for observability of assistant's calls. + * + * Currently, only Langfuse is supported. */ observabilityPlan?: LangfuseObservabilityPlan; /** These are dynamic credentials that will be used for the assistant calls. By default, all the credentials are available for use in the call but you can supplement an additional credentials using this. Dynamic credentials override existing credentials. */ @@ -10235,6 +11453,9 @@ export interface CreateAssistantDTO { | ({ provider: 'ghl.oauth2-authorization'; } & CreateGoHighLevelMCPCredentialDTO) + | ({ + provider: 'inworld'; + } & any) )[]; /** This is a set of actions that will be performed on certain events. */ hooks?: ( @@ -10242,6 +11463,16 @@ export interface CreateAssistantDTO { | AssistantHookAssistantSpeechInterrupted | AssistantHookCustomerSpeechInterrupted )[]; + /** + * These are values that will be used to replace the template variables in the assistant messages and other text-based fields. + * This uses LiquidJS syntax. https://liquidjs.com/tutorials/intro-to-liquid.html + * + * So for example, `{{ name }}` will be replaced with the value of `name` in `variableValues`. + * `{{"now" | date: "%b %d, %Y, %I:%M %p", "America/New_York"}}` will be replaced with the current date and time in New York. + * Some VAPI reserved defaults: + * - *customer* - the customer object + */ + variableValues?: object; /** * This is the name of the assistant. * @@ -10284,11 +11515,7 @@ export interface CreateAssistantDTO { backgroundSpeechDenoisingPlan?: BackgroundSpeechDenoisingPlan; /** This is the plan for analysis of assistant's calls. Stored in `call.analysis`. */ analysisPlan?: AnalysisPlan; - /** - * This is the plan for artifacts generated during assistant's calls. Stored in `call.artifact`. - * - * Note: `recordingEnabled` is currently at the root level. It will be moved to `artifactPlan` in the future, but will remain backwards compatible. - */ + /** This is the plan for artifacts generated during assistant's calls. Stored in `call.artifact`. */ artifactPlan?: ArtifactPlan; /** * This is the plan for static predefined messages that can be spoken by the assistant during the call, like `idleMessages`. @@ -10322,8 +11549,6 @@ export interface CreateAssistantDTO { * Usage: * - To enable live listening of the assistant's calls, set `monitorPlan.listenEnabled` to `true`. * - To enable live control of the assistant's calls, set `monitorPlan.controlEnabled` to `true`. - * - * Note, `serverMessages`, `clientMessages`, `serverUrl` and `serverUrlSecret` are currently at the root level but will be moved to `monitorPlan` in the future. Will remain backwards compatible */ monitorPlan?: MonitorPlan; /** These are the credentials that will be used for the assistant calls. By default, all the credentials are available for use in the call but you can provide a subset using this. */ @@ -10341,173 +11566,85 @@ export interface CreateAssistantDTO { keypadInputPlan?: KeypadInputPlan; } -export interface AssistantOverrides { - /** These are the options for the assistant's transcriber. */ - transcriber?: - | AssemblyAITranscriber - | AzureSpeechTranscriber - | CustomTranscriber - | DeepgramTranscriber - | ElevenLabsTranscriber - | GladiaTranscriber - | GoogleTranscriber - | SpeechmaticsTranscriber - | TalkscriberTranscriber - | OpenAITranscriber - | CartesiaTranscriber; - /** These are the options for the assistant's LLM. */ - model?: - | AnthropicModel - | AnyscaleModel - | CerebrasModel - | CustomLLMModel - | DeepInfraModel - | DeepSeekModel - | GoogleModel - | GroqModel - | InflectionAIModel - | OpenAIModel - | OpenRouterModel - | PerplexityAIModel - | TogetherAIModel - | XaiModel; - /** These are the options for the assistant's voice. */ - voice?: - | AzureVoice - | CartesiaVoice - | CustomVoice - | DeepgramVoice - | ElevenLabsVoice - | HumeVoice - | LMNTVoice - | NeuphonicVoice - | OpenAIVoice - | PlayHTVoice - | RimeAIVoice - | SmallestAIVoice - | TavusVoice - | VapiVoice - | SesameVoice; - /** - * This is the first message that the assistant will say. This can also be a URL to a containerized audio file (mp3, wav, etc.). - * - * If unspecified, assistant will wait for user to speak and use the model to respond once they speak. - * @example "Hello! How can I help you today?" - */ - firstMessage?: string; - /** @default false */ - firstMessageInterruptionsEnabled?: boolean; +export interface SquadMemberDTO { + /** This is the assistant that will be used for the call. To use a transient assistant, use `assistant` instead. */ + assistantId?: string | null; + /** This is the assistant that will be used for the call. To use an existing assistant, use `assistantId` instead. */ + assistant?: CreateAssistantDTO; + /** This can be used to override the assistant's settings and provide values for it's template variables. */ + assistantOverrides?: AssistantOverrides; /** - * This is the mode for the first message. Default is 'assistant-speaks-first'. - * - * Use: - * - 'assistant-speaks-first' to have the assistant speak first. - * - 'assistant-waits-for-user' to have the assistant wait for the user to speak first. - * - 'assistant-speaks-first-with-model-generated-message' to have the assistant speak first with a message generated by the model based on the conversation state. (`assistant.model.messages` at call start, `call.messages` at squad transfer points). + * These are the others assistants that this assistant can transfer to. * - * @default 'assistant-speaks-first' - * @example "assistant-speaks-first" - */ - firstMessageMode?: - | 'assistant-speaks-first' - | 'assistant-speaks-first-with-model-generated-message' - | 'assistant-waits-for-user'; - /** - * These are the settings to configure or disable voicemail detection. Alternatively, voicemail detection can be configured using the model.tools=[VoicemailTool]. - * This uses Twilio's built-in detection while the VoicemailTool relies on the model to detect if a voicemail was reached. - * You can use neither of them, one of them, or both of them. By default, Twilio built-in detection is enabled while VoicemailTool is not. - */ - voicemailDetection?: - | GoogleVoicemailDetectionPlan - | OpenAIVoicemailDetectionPlan - | TwilioVoicemailDetectionPlan - | VapiVoicemailDetectionPlan; - /** - * These are the messages that will be sent to your Client SDKs. Default is conversation-update,function-call,hang,model-output,speech-update,status-update,transfer-update,transcript,tool-calls,user-interrupted,voice-input,workflow.node.started. You can check the shape of the messages in ClientMessage schema. - * @example ["conversation-update","function-call","hang","model-output","speech-update","status-update","transfer-update","transcript","tool-calls","user-interrupted","voice-input","workflow.node.started"] - */ - clientMessages?: - | 'conversation-update' - | 'function-call' - | 'function-call-result' - | 'hang' - | 'language-changed' - | 'metadata' - | 'model-output' - | 'speech-update' - | 'status-update' - | 'transcript' - | 'tool-calls' - | 'tool-calls-result' - | 'tool.completed' - | 'transfer-update' - | 'user-interrupted' - | 'voice-input' - | 'workflow.node.started'; - /** - * These are the messages that will be sent to your Server URL. Default is conversation-update,end-of-call-report,function-call,hang,speech-update,status-update,tool-calls,transfer-destination-request,user-interrupted. You can check the shape of the messages in ServerMessage schema. - * @example ["conversation-update","end-of-call-report","function-call","hang","speech-update","status-update","tool-calls","transfer-destination-request","user-interrupted"] + * If the assistant already has transfer call tool, these destinations are just appended to existing ones. */ - serverMessages?: - | 'conversation-update' - | 'end-of-call-report' - | 'function-call' - | 'hang' - | 'language-changed' - | 'language-change-detected' - | 'model-output' - | 'phone-call-control' - | 'speech-update' - | 'status-update' - | 'transcript' - | "transcript[transcriptType='final']" - | 'tool-calls' - | 'transfer-destination-request' - | 'transfer-update' - | 'user-interrupted' - | 'voice-input'; + assistantDestinations?: TransferDestinationAssistant[]; +} + +export interface CreateSquadDTO { + /** This is the name of the squad. */ + name?: string; /** - * How many seconds of silence to wait before ending the call. Defaults to 30. + * This is the list of assistants that make up the squad. * - * @default 30 - * @min 10 - * @max 3600 - * @example 30 + * The call will start with the first assistant in the list. */ - silenceTimeoutSeconds?: number; + members: SquadMemberDTO[]; /** - * This is the maximum number of seconds that the call will last. When the call reaches this duration, it will be ended. + * This can be used to override all the assistants' settings and provide values for their template variables. * - * @default 600 (10 minutes) - * @min 10 - * @max 43200 - * @example 600 + * Both `membersOverrides` and `members[n].assistantOverrides` can be used together. First, `members[n].assistantOverrides` is applied. Then, `membersOverrides` is applied as a global override. */ - maxDurationSeconds?: number; + membersOverrides?: AssistantOverrides; +} + +export interface CreateWorkflowDTO { + nodes: (ConversationNode | ToolNode)[]; /** - * This is the background sound in the call. Default for phone calls is 'office' and default for web calls is 'off'. - * You can also provide a custom sound by providing a URL to an audio file. + * This is the transcriber for the workflow. + * + * This can be overridden at node level using `nodes[n].transcriber`. */ - backgroundSound?: 'off' | 'office' | string; - /** @example false */ - backgroundDenoisingEnabled?: boolean; + transcriber?: + | AssemblyAITranscriber + | AzureSpeechTranscriber + | CustomTranscriber + | DeepgramTranscriber + | ElevenLabsTranscriber + | GladiaTranscriber + | GoogleTranscriber + | SpeechmaticsTranscriber + | TalkscriberTranscriber + | OpenAITranscriber + | CartesiaTranscriber; /** - * This determines whether the model's output is used in conversation history rather than the transcription of assistant's speech. + * This is the voice for the workflow. * - * Default `false` while in beta. - * - * @default false - * @example false + * This can be overridden at node level using `nodes[n].voice`. */ - modelOutputInMessagesEnabled?: boolean; - /** These are the configurations to be passed to the transport providers of assistant's calls, like Twilio. You can store multiple configurations for different transport providers. For a call, only the configuration matching the call transport provider is used. */ - transportConfigurations?: TransportConfigurationTwilio[]; + voice?: + | AzureVoice + | CartesiaVoice + | CustomVoice + | DeepgramVoice + | ElevenLabsVoice + | HumeVoice + | LMNTVoice + | NeuphonicVoice + | OpenAIVoice + | PlayHTVoice + | RimeAIVoice + | SmallestAIVoice + | TavusVoice + | VapiVoice + | SesameVoice; /** - * This is the plan for observability configuration of assistant's calls. - * Currently supports Langfuse for tracing and monitoring. + * This is the plan for observability of workflow's calls. + * + * Currently, only Langfuse is supported. */ observabilityPlan?: LangfuseObservabilityPlan; - /** These are dynamic credentials that will be used for the assistant calls. By default, all the credentials are available for use in the call but you can supplement an additional credentials using this. Dynamic credentials override existing credentials. */ + /** These are dynamic credentials that will be used for the workflow calls. By default, all the credentials are available for use in the call but you can supplement an additional credentials using this. Dynamic credentials override existing credentials. */ credentials?: ( | ({ provider: '11labs'; @@ -10653,79 +11790,34 @@ export interface AssistantOverrides { | ({ provider: 'ghl.oauth2-authorization'; } & CreateGoHighLevelMCPCredentialDTO) + | ({ + provider: 'inworld'; + } & any) )[]; - /** This is a set of actions that will be performed on certain events. */ - hooks?: ( - | AssistantHookCallEnding - | AssistantHookAssistantSpeechInterrupted - | AssistantHookCustomerSpeechInterrupted - )[]; - /** - * These are values that will be used to replace the template variables in the assistant messages and other text-based fields. - * This uses LiquidJS syntax. https://liquidjs.com/tutorials/intro-to-liquid.html - * - * So for example, `{{ name }}` will be replaced with the value of `name` in `variableValues`. - * `{{"now" | date: "%b %d, %Y, %I:%M %p", "America/New_York"}}` will be replaced with the current date and time in New York. - * Some VAPI reserved defaults: - * - *customer* - the customer object - */ - variableValues?: object; - /** - * This is the name of the assistant. - * - * This is required when you want to transfer between assistants in a call. - * @maxLength 40 - */ - name?: string; + /** @maxLength 80 */ + name: string; + edges: Edge[]; + /** @maxLength 5000 */ + globalPrompt?: string; /** - * This is the message that the assistant will say if the call is forwarded to voicemail. + * This is where Vapi will send webhooks. You can find all webhooks available along with their shape in ServerMessage schema. * - * If unspecified, it will hang up. - * @maxLength 1000 - */ - voicemailMessage?: string; - /** - * This is the message that the assistant will say if it ends the call. + * The order of precedence is: * - * If unspecified, it will hang up without saying anything. - * @maxLength 1000 + * 1. tool.server + * 2. workflow.server / assistant.server + * 3. phoneNumber.server + * 4. org.server */ - endCallMessage?: string; - /** This list contains phrases that, if spoken by the assistant, will trigger the call to be hung up. Case insensitive. */ - endCallPhrases?: string[]; + server?: Server; + /** This is the compliance plan for the workflow. It allows you to configure HIPAA and other compliance settings. */ compliancePlan?: CompliancePlan; - /** This is for metadata you want to store on the assistant. */ - metadata?: object; - /** - * This enables filtering of noise and background speech while the user is talking. - * - * Features: - * - Smart denoising using Krisp - * - Fourier denoising - * - * Smart denoising can be combined with or used independently of Fourier denoising. - * - * Order of precedence: - * - Smart denoising - * - Fourier denoising - */ - backgroundSpeechDenoisingPlan?: BackgroundSpeechDenoisingPlan; - /** This is the plan for analysis of assistant's calls. Stored in `call.analysis`. */ + /** This is the plan for analysis of workflow's calls. Stored in `call.analysis`. */ analysisPlan?: AnalysisPlan; - /** - * This is the plan for artifacts generated during assistant's calls. Stored in `call.artifact`. - * - * Note: `recordingEnabled` is currently at the root level. It will be moved to `artifactPlan` in the future, but will remain backwards compatible. - */ + /** This is the plan for artifacts generated during workflow's calls. Stored in `call.artifact`. */ artifactPlan?: ArtifactPlan; /** - * This is the plan for static predefined messages that can be spoken by the assistant during the call, like `idleMessages`. - * - * Note: `firstMessage`, `voicemailMessage`, and `endCallMessage` are currently at the root level. They will be moved to `messagePlan` in the future, but will remain backwards compatible. - */ - messagePlan?: MessagePlan; - /** - * This is the plan for when the assistant should start talking. + * This is the plan for when the workflow nodes should start talking. * * You should configure this if you're running into these issues: * - The assistant is too slow to start talking after the customer is done speaking. @@ -10734,7 +11826,7 @@ export interface AssistantOverrides { */ startSpeakingPlan?: StartSpeakingPlan; /** - * This is the plan for when assistant should stop talking on customer interruption. + * This is the plan for when workflow nodes should stop talking on customer interruption. * * You should configure this if you're running into these issues: * - The assistant is too slow to recognize customer's interruption. @@ -10745,80 +11837,27 @@ export interface AssistantOverrides { */ stopSpeakingPlan?: StopSpeakingPlan; /** - * This is the plan for real-time monitoring of the assistant's calls. + * This is the plan for real-time monitoring of the workflow's calls. * * Usage: - * - To enable live listening of the assistant's calls, set `monitorPlan.listenEnabled` to `true`. - * - To enable live control of the assistant's calls, set `monitorPlan.controlEnabled` to `true`. - * - * Note, `serverMessages`, `clientMessages`, `serverUrl` and `serverUrlSecret` are currently at the root level but will be moved to `monitorPlan` in the future. Will remain backwards compatible + * - To enable live listening of the workflow's calls, set `monitorPlan.listenEnabled` to `true`. + * - To enable live control of the workflow's calls, set `monitorPlan.controlEnabled` to `true`. */ monitorPlan?: MonitorPlan; - /** These are the credentials that will be used for the assistant calls. By default, all the credentials are available for use in the call but you can provide a subset using this. */ - credentialIds?: string[]; - /** - * This is where Vapi will send webhooks. You can find all webhooks available along with their shape in ServerMessage schema. - * - * The order of precedence is: - * - * 1. assistant.server.url - * 2. phoneNumber.serverUrl - * 3. org.serverUrl - */ - server?: Server; - keypadInputPlan?: KeypadInputPlan; -} - -export interface SquadMemberDTO { - /** This is the assistant that will be used for the call. To use a transient assistant, use `assistant` instead. */ - assistantId?: string | null; - /** This is the assistant that will be used for the call. To use an existing assistant, use `assistantId` instead. */ - assistant?: CreateAssistantDTO; - /** This can be used to override the assistant's settings and provide values for it's template variables. */ - assistantOverrides?: AssistantOverrides; - /** - * These are the others assistants that this assistant can transfer to. - * - * If the assistant already has transfer call tool, these destinations are just appended to existing ones. - */ - assistantDestinations?: TransferDestinationAssistant[]; -} - -export interface CreateSquadDTO { - /** This is the name of the squad. */ - name?: string; - /** - * This is the list of assistants that make up the squad. - * - * The call will start with the first assistant in the list. - */ - members: SquadMemberDTO[]; - /** - * This can be used to override all the assistants' settings and provide values for their template variables. - * - * Both `membersOverrides` and `members[n].assistantOverrides` can be used together. First, `members[n].assistantOverrides` is applied. Then, `membersOverrides` is applied as a global override. - */ - membersOverrides?: AssistantOverrides; -} - -export interface CreateWorkflowDTO { - nodes: (ConversationNode | ToolNode)[]; - /** @maxLength 80 */ - name: string; - edges: Edge[]; - /** @maxLength 5000 */ - globalPrompt?: string; /** - * This is where Vapi will send webhooks. You can find all webhooks available along with their shape in ServerMessage schema. + * This enables filtering of noise and background speech while the user is talking. * - * The order of precedence is: + * Features: + * - Smart denoising using Krisp + * - Fourier denoising * - * 1. tool.server - * 2. workflow.server / assistant.server - * 3. phoneNumber.server - * 4. org.server + * Both can be used together. Order of precedence: + * - Smart denoising + * - Fourier denoising */ - server?: Server; + backgroundSpeechDenoisingPlan?: BackgroundSpeechDenoisingPlan; + /** These are the credentials that will be used for the workflow calls. By default, all the credentials are available for use in the call but you can provide a subset using this. */ + credentialIds?: string[]; } export interface WorkflowOverrides { @@ -11055,6 +12094,7 @@ export interface Call { | 'pipeline-error-neuphonic-voice-failed' | 'pipeline-error-hume-voice-failed' | 'pipeline-error-sesame-voice-failed' + | 'pipeline-error-inworld-voice-failed' | 'pipeline-error-tavus-video-failed' | 'call.in-progress.error-vapifault-openai-voice-failed' | 'call.in-progress.error-vapifault-cartesia-voice-failed' @@ -11068,6 +12108,7 @@ export interface Call { | 'call.in-progress.error-vapifault-neuphonic-voice-failed' | 'call.in-progress.error-vapifault-hume-voice-failed' | 'call.in-progress.error-vapifault-sesame-voice-failed' + | 'call.in-progress.error-vapifault-inworld-voice-failed' | 'call.in-progress.error-vapifault-tavus-video-failed' | 'pipeline-error-vapi-llm-failed' | 'pipeline-error-vapi-400-bad-request-validation-failed' @@ -11484,6 +12525,7 @@ export interface Call { | 'pipeline-error-deepgram-returning-500-invalid-json' | 'pipeline-error-deepgram-returning-502-network-error' | 'pipeline-error-deepgram-returning-502-bad-gateway-ehostunreach' + | 'pipeline-error-deepgram-returning-econnreset' | 'call.in-progress.error-vapifault-deepgram-returning-400-no-such-model-language-tier-combination' | 'call.in-progress.error-vapifault-deepgram-returning-401-invalid-credentials' | 'call.in-progress.error-vapifault-deepgram-returning-404-not-found' @@ -11571,6 +12613,8 @@ export interface Call { * @deprecated */ phoneCallProviderId?: string; + /** This is the campaign ID that the call belongs to. */ + campaignId?: string; /** * This is the assistant ID that will be used for the call. To use a transient assistant, use `assistant` instead. * @@ -12131,6 +13175,8 @@ export interface Chat { updatedAt: string; /** These are the costs of individual components of the chat in USD. */ costs?: (ModelCost | ChatCost)[]; + /** These are the overrides for the `assistant` or `assistantId`'s settings and template variables. */ + assistantOverrides?: AssistantOverrides; /** This is the cost of the chat in USD. */ cost?: number; } @@ -12170,6 +13216,8 @@ export interface CreateChatDTO { * Mutually exclusive with sessionId. */ previousChatId?: string; + /** These are the overrides for the `assistant` or `assistantId`'s settings and template variables. */ + assistantOverrides?: AssistantOverrides; } export interface GetChatPaginatedDTO { @@ -12286,6 +13334,8 @@ export interface OpenAIResponsesRequest { * Mutually exclusive with sessionId. */ previousChatId?: string; + /** These are the overrides for the `assistant` or `assistantId`'s settings and template variables. */ + assistantOverrides?: AssistantOverrides; } export interface ResponseOutputText { @@ -12410,6 +13460,106 @@ export interface ResponseErrorEvent { sequence_number: number; } +export interface CreateCampaignDTO { + /** + * This is the name of the campaign. This is just for your own reference. + * @example "Q2 Sales Campaign" + */ + name: string; + /** This is the assistant ID that will be used for the campaign calls. Note: Either assistantId or workflowId can be used, but not both. */ + assistantId?: string; + /** This is the workflow ID that will be used for the campaign calls. Note: Either assistantId or workflowId can be used, but not both. */ + workflowId?: string; + /** This is the phone number ID that will be used for the campaign calls. */ + phoneNumberId: string; + /** This is the schedule plan for the campaign. */ + schedulePlan: SchedulePlan; + /** These are the customers that will be called in the campaign. */ + customers: CreateCustomerDTO[]; +} + +export interface Campaign { + /** This is the status of the campaign. */ + status: 'scheduled' | 'in-progress' | 'ended'; + /** This is the explanation for how the campaign ended. */ + endedReason?: + | 'campaign.scheduled.ended-by-user' + | 'campaign.in-progress.ended-by-user' + | 'campaign.ended.success'; + /** + * This is the name of the campaign. This is just for your own reference. + * @example "Q2 Sales Campaign" + */ + name: string; + /** This is the assistant ID that will be used for the campaign calls. Note: Either assistantId or workflowId can be used, but not both. */ + assistantId?: string; + /** This is the workflow ID that will be used for the campaign calls. Note: Either assistantId or workflowId can be used, but not both. */ + workflowId?: string; + /** This is the phone number ID that will be used for the campaign calls. */ + phoneNumberId: string; + /** This is the schedule plan for the campaign. */ + schedulePlan: SchedulePlan; + /** These are the customers that will be called in the campaign. */ + customers: CreateCustomerDTO[]; + /** This is the unique identifier for the campaign. */ + id: string; + /** This is the unique identifier for the org that this campaign belongs to. */ + orgId: string; + /** + * This is the ISO 8601 date-time string of when the campaign was created. + * @format date-time + */ + createdAt: string; + /** + * This is the ISO 8601 date-time string of when the campaign was last updated. + * @format date-time + */ + updatedAt: string; + /** This is a map of call IDs to campaign call details. */ + calls: object; + /** This is the number of calls that have been scheduled. */ + callsCounterScheduled: number; + /** This is the number of calls that have been queued. */ + callsCounterQueued: number; + /** This is the number of calls that have been in progress. */ + callsCounterInProgress: number; + /** This is the number of calls whose ended reason is 'voicemail'. */ + callsCounterEndedVoicemail: number; + /** This is the number of calls that have ended. */ + callsCounterEnded: number; +} + +export interface UpdateCampaignDTO { + /** This is the name of the campaign. This is just for your own reference. */ + name?: string; + /** + * This is the assistant ID that will be used for the campaign calls. + * Can only be updated if campaign is not in progress or has ended. + */ + assistantId?: string; + /** + * This is the workflow ID that will be used for the campaign calls. + * Can only be updated if campaign is not in progress or has ended. + */ + workflowId?: string; + /** + * This is the phone number ID that will be used for the campaign calls. + * Can only be updated if campaign is not in progress or has ended. + */ + phoneNumberId?: string; + /** + * This is the schedule plan for the campaign. + * Can only be updated if campaign is not in progress or has ended. + */ + schedulePlan?: SchedulePlan; + /** + * This is the status of the campaign. + * Can only be updated to 'ended' if you want to end the campaign. + * When set to 'ended', it will delete all scheduled calls. Calls in progress will be allowed to complete. + */ + status?: 'ended'; +} + export interface Session { /** This is the unique identifier for the session. */ id: string; @@ -12563,15 +13713,54 @@ export interface GetSessionPaginatedDTO { */ updatedAtGe?: string; /** - * This will return items where the updatedAt is less than or equal to the specified value. - * @format date-time + * This will return items where the updatedAt is less than or equal to the specified value. + * @format date-time + */ + updatedAtLe?: string; +} + +export interface SessionPaginatedResponse { + results: Session[]; + metadata: PaginationMeta; +} + +export interface CreateSupportTicketDTO { + /** The category of the support request */ + category: 'bug-report' | 'feature-request' | 'doc-update' | 'pricing' | 'sales' | 'others'; + /** The subcategory for bug reports */ + bugSubcategory?: + | 'call-forwarding' + | 'tools-calling' + | 'prompting' + | 'workflows' + | 'assistant' + | 'latency' + | 'assistant-speaks-slow' + | 'assistant-doesnt-respond' + | 'endpointing' + | 'others'; + /** + * The subject/title of the support request + * @minLength 1 + * @maxLength 200 + */ + subject: string; + /** + * Detailed description of the issue or request + * @minLength 10 + * @maxLength 5000 + */ + message: string; + /** + * Call IDs and recording URLs if applicable + * @maxLength 1000 */ - updatedAtLe?: string; + callDetails?: string; } -export interface SessionPaginatedResponse { - results: Session[]; - metadata: PaginationMeta; +export interface SupportTicketResponse { + /** The unique identifier for the support ticket */ + id: string; } export interface Assistant { @@ -12722,7 +13911,15 @@ export interface Assistant { * You can also provide a custom sound by providing a URL to an audio file. */ backgroundSound?: 'off' | 'office' | string; - /** @example false */ + /** + * This enables filtering of noise and background speech while the user is talking. + * + * Default `false` while in beta. + * + * @default false + * @deprecated + * @example false + */ backgroundDenoisingEnabled?: boolean; /** * This determines whether the model's output is used in conversation history rather than the transcription of assistant's speech. @@ -12736,8 +13933,9 @@ export interface Assistant { /** These are the configurations to be passed to the transport providers of assistant's calls, like Twilio. You can store multiple configurations for different transport providers. For a call, only the configuration matching the call transport provider is used. */ transportConfigurations?: TransportConfigurationTwilio[]; /** - * This is the plan for observability configuration of assistant's calls. - * Currently supports Langfuse for tracing and monitoring. + * This is the plan for observability of assistant's calls. + * + * Currently, only Langfuse is supported. */ observabilityPlan?: LangfuseObservabilityPlan; /** These are dynamic credentials that will be used for the assistant calls. By default, all the credentials are available for use in the call but you can supplement an additional credentials using this. Dynamic credentials override existing credentials. */ @@ -12886,6 +14084,9 @@ export interface Assistant { | ({ provider: 'ghl.oauth2-authorization'; } & CreateGoHighLevelMCPCredentialDTO) + | ({ + provider: 'inworld'; + } & any) )[]; /** This is a set of actions that will be performed on certain events. */ hooks?: ( @@ -12935,11 +14136,7 @@ export interface Assistant { backgroundSpeechDenoisingPlan?: BackgroundSpeechDenoisingPlan; /** This is the plan for analysis of assistant's calls. Stored in `call.analysis`. */ analysisPlan?: AnalysisPlan; - /** - * This is the plan for artifacts generated during assistant's calls. Stored in `call.artifact`. - * - * Note: `recordingEnabled` is currently at the root level. It will be moved to `artifactPlan` in the future, but will remain backwards compatible. - */ + /** This is the plan for artifacts generated during assistant's calls. Stored in `call.artifact`. */ artifactPlan?: ArtifactPlan; /** * This is the plan for static predefined messages that can be spoken by the assistant during the call, like `idleMessages`. @@ -12973,8 +14170,6 @@ export interface Assistant { * Usage: * - To enable live listening of the assistant's calls, set `monitorPlan.listenEnabled` to `true`. * - To enable live control of the assistant's calls, set `monitorPlan.controlEnabled` to `true`. - * - * Note, `serverMessages`, `clientMessages`, `serverUrl` and `serverUrlSecret` are currently at the root level but will be moved to `monitorPlan` in the future. Will remain backwards compatible */ monitorPlan?: MonitorPlan; /** These are the credentials that will be used for the assistant calls. By default, all the credentials are available for use in the call but you can provide a subset using this. */ @@ -13165,7 +14360,15 @@ export interface UpdateAssistantDTO { * You can also provide a custom sound by providing a URL to an audio file. */ backgroundSound?: 'off' | 'office' | string; - /** @example false */ + /** + * This enables filtering of noise and background speech while the user is talking. + * + * Default `false` while in beta. + * + * @default false + * @deprecated + * @example false + */ backgroundDenoisingEnabled?: boolean; /** * This determines whether the model's output is used in conversation history rather than the transcription of assistant's speech. @@ -13179,8 +14382,9 @@ export interface UpdateAssistantDTO { /** These are the configurations to be passed to the transport providers of assistant's calls, like Twilio. You can store multiple configurations for different transport providers. For a call, only the configuration matching the call transport provider is used. */ transportConfigurations?: TransportConfigurationTwilio[]; /** - * This is the plan for observability configuration of assistant's calls. - * Currently supports Langfuse for tracing and monitoring. + * This is the plan for observability of assistant's calls. + * + * Currently, only Langfuse is supported. */ observabilityPlan?: LangfuseObservabilityPlan; /** These are dynamic credentials that will be used for the assistant calls. By default, all the credentials are available for use in the call but you can supplement an additional credentials using this. Dynamic credentials override existing credentials. */ @@ -13329,6 +14533,9 @@ export interface UpdateAssistantDTO { | ({ provider: 'ghl.oauth2-authorization'; } & CreateGoHighLevelMCPCredentialDTO) + | ({ + provider: 'inworld'; + } & any) )[]; /** This is a set of actions that will be performed on certain events. */ hooks?: ( @@ -13378,11 +14585,7 @@ export interface UpdateAssistantDTO { backgroundSpeechDenoisingPlan?: BackgroundSpeechDenoisingPlan; /** This is the plan for analysis of assistant's calls. Stored in `call.analysis`. */ analysisPlan?: AnalysisPlan; - /** - * This is the plan for artifacts generated during assistant's calls. Stored in `call.artifact`. - * - * Note: `recordingEnabled` is currently at the root level. It will be moved to `artifactPlan` in the future, but will remain backwards compatible. - */ + /** This is the plan for artifacts generated during assistant's calls. Stored in `call.artifact`. */ artifactPlan?: ArtifactPlan; /** * This is the plan for static predefined messages that can be spoken by the assistant during the call, like `idleMessages`. @@ -13416,8 +14619,6 @@ export interface UpdateAssistantDTO { * Usage: * - To enable live listening of the assistant's calls, set `monitorPlan.listenEnabled` to `true`. * - To enable live control of the assistant's calls, set `monitorPlan.controlEnabled` to `true`. - * - * Note, `serverMessages`, `clientMessages`, `serverUrl` and `serverUrlSecret` are currently at the root level but will be moved to `monitorPlan` in the future. Will remain backwards compatible */ monitorPlan?: MonitorPlan; /** These are the credentials that will be used for the assistant calls. By default, all the credentials are available for use in the call but you can provide a subset using this. */ @@ -14533,6 +15734,76 @@ export interface PhoneNumberPaginatedResponse { metadata: PaginationMeta; } +export interface ApiRequestTool { + /** + * These are the messages that will be spoken to the user as the tool is running. + * + * For some tools, this is auto-filled based on special fields like `tool.destinations`. For others like the function tool, these can be custom configured. + */ + messages?: (ToolMessageStart | ToolMessageComplete | ToolMessageFailed | ToolMessageDelayed)[]; + /** The type of tool. "apiRequest" for API request tool. */ + type: 'apiRequest'; + method: 'POST' | 'GET'; + /** + * This is the timeout in seconds for the request. Defaults to 20 seconds. + * + * @default 20 + * @min 1 + * @max 300 + * @example 20 + */ + timeoutSeconds?: number; + /** This is the unique identifier for the tool. */ + id: string; + /** This is the unique identifier for the organization that this tool belongs to. */ + orgId: string; + /** + * This is the ISO 8601 date-time string of when the tool was created. + * @format date-time + */ + createdAt: string; + /** + * This is the ISO 8601 date-time string of when the tool was last updated. + * @format date-time + */ + updatedAt: string; + /** + * This is the function definition of the tool. + * + * For `endCall`, `transferCall`, and `dtmf` tools, this is auto-filled based on tool-specific fields like `tool.destinations`. But, even in those cases, you can provide a custom function definition for advanced use cases. + * + * An example of an advanced use case is if you want to customize the message that's spoken for `endCall` tool. You can specify a function where it returns an argument "reason". Then, in `messages` array, you can have many "request-complete" messages. One of these messages will be triggered if the `messages[].conditions` matches the "reason" argument. + */ + function?: OpenAIFunction; + /** + * This is the name of the tool. This will be passed to the model. + * + * Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length of 40. + * @maxLength 40 + * @pattern /^[a-zA-Z0-9_-]{1,40}$/ + */ + name?: string; + /** + * This is the description of the tool. This will be passed to the model. + * @maxLength 1000 + */ + description?: string; + /** This is where the request will be sent. */ + url: string; + /** This is the body of the request. */ + body?: JsonSchema; + /** These are the headers to send in the request. */ + headers?: JsonSchema; + /** + * This is the backoff plan if the request fails. Defaults to undefined (the request will not be retried). + * + * @default undefined (the request will not be retried) + */ + backoffPlan?: BackoffPlan; + /** This is the plan that controls the variable extraction from the tool's response. */ + variableExtractionPlan?: VariableExtractionPlan; +} + export interface DtmfTool { /** * These are the messages that will be spoken to the user as the tool is running. @@ -15199,6 +16470,7 @@ export interface McpTool { * An example of an advanced use case is if you want to customize the message that's spoken for `endCall` tool. You can specify a function where it returns an argument "reason". Then, in `messages` array, you can have many "request-complete" messages. One of these messages will be triggered if the `messages[].conditions` matches the "reason" argument. */ function?: OpenAIFunction; + metadata?: McpToolMetadata; } export interface GoHighLevelCalendarAvailabilityTool { @@ -15547,6 +16819,60 @@ export interface CreateSmsToolDTO { function?: OpenAIFunction; } +export interface UpdateApiRequestToolDTO { + /** + * These are the messages that will be spoken to the user as the tool is running. + * + * For some tools, this is auto-filled based on special fields like `tool.destinations`. For others like the function tool, these can be custom configured. + */ + messages?: (ToolMessageStart | ToolMessageComplete | ToolMessageFailed | ToolMessageDelayed)[]; + method?: 'POST' | 'GET'; + /** + * This is the timeout in seconds for the request. Defaults to 20 seconds. + * + * @default 20 + * @min 1 + * @max 300 + * @example 20 + */ + timeoutSeconds?: number; + /** + * This is the function definition of the tool. + * + * For `endCall`, `transferCall`, and `dtmf` tools, this is auto-filled based on tool-specific fields like `tool.destinations`. But, even in those cases, you can provide a custom function definition for advanced use cases. + * + * An example of an advanced use case is if you want to customize the message that's spoken for `endCall` tool. You can specify a function where it returns an argument "reason". Then, in `messages` array, you can have many "request-complete" messages. One of these messages will be triggered if the `messages[].conditions` matches the "reason" argument. + */ + function?: OpenAIFunction; + /** + * This is the name of the tool. This will be passed to the model. + * + * Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length of 40. + * @maxLength 40 + * @pattern /^[a-zA-Z0-9_-]{1,40}$/ + */ + name?: string; + /** + * This is the description of the tool. This will be passed to the model. + * @maxLength 1000 + */ + description?: string; + /** This is where the request will be sent. */ + url?: string; + /** This is the body of the request. */ + body?: JsonSchema; + /** These are the headers to send in the request. */ + headers?: JsonSchema; + /** + * This is the backoff plan if the request fails. Defaults to undefined (the request will not be retried). + * + * @default undefined (the request will not be retried) + */ + backoffPlan?: BackoffPlan; + /** This is the plan that controls the variable extraction from the tool's response. */ + variableExtractionPlan?: VariableExtractionPlan; +} + export interface UpdateDtmfToolDTO { /** * These are the messages that will be spoken to the user as the tool is running. @@ -15847,7 +17173,105 @@ export interface UpdateGoogleCalendarCreateEventToolDTO { function?: OpenAIFunction; } -export interface UpdateGoogleSheetsRowAppendToolDTO { +export interface UpdateGoogleSheetsRowAppendToolDTO { + /** + * These are the messages that will be spoken to the user as the tool is running. + * + * For some tools, this is auto-filled based on special fields like `tool.destinations`. For others like the function tool, these can be custom configured. + */ + messages?: (ToolMessageStart | ToolMessageComplete | ToolMessageFailed | ToolMessageDelayed)[]; + /** + * This is the function definition of the tool. + * + * For `endCall`, `transferCall`, and `dtmf` tools, this is auto-filled based on tool-specific fields like `tool.destinations`. But, even in those cases, you can provide a custom function definition for advanced use cases. + * + * An example of an advanced use case is if you want to customize the message that's spoken for `endCall` tool. You can specify a function where it returns an argument "reason". Then, in `messages` array, you can have many "request-complete" messages. One of these messages will be triggered if the `messages[].conditions` matches the "reason" argument. + */ + function?: OpenAIFunction; +} + +export interface UpdateGoogleCalendarCheckAvailabilityToolDTO { + /** + * These are the messages that will be spoken to the user as the tool is running. + * + * For some tools, this is auto-filled based on special fields like `tool.destinations`. For others like the function tool, these can be custom configured. + */ + messages?: (ToolMessageStart | ToolMessageComplete | ToolMessageFailed | ToolMessageDelayed)[]; + /** + * This is the function definition of the tool. + * + * For `endCall`, `transferCall`, and `dtmf` tools, this is auto-filled based on tool-specific fields like `tool.destinations`. But, even in those cases, you can provide a custom function definition for advanced use cases. + * + * An example of an advanced use case is if you want to customize the message that's spoken for `endCall` tool. You can specify a function where it returns an argument "reason". Then, in `messages` array, you can have many "request-complete" messages. One of these messages will be triggered if the `messages[].conditions` matches the "reason" argument. + */ + function?: OpenAIFunction; +} + +export interface UpdateSlackSendMessageToolDTO { + /** + * These are the messages that will be spoken to the user as the tool is running. + * + * For some tools, this is auto-filled based on special fields like `tool.destinations`. For others like the function tool, these can be custom configured. + */ + messages?: (ToolMessageStart | ToolMessageComplete | ToolMessageFailed | ToolMessageDelayed)[]; + /** + * This is the function definition of the tool. + * + * For `endCall`, `transferCall`, and `dtmf` tools, this is auto-filled based on tool-specific fields like `tool.destinations`. But, even in those cases, you can provide a custom function definition for advanced use cases. + * + * An example of an advanced use case is if you want to customize the message that's spoken for `endCall` tool. You can specify a function where it returns an argument "reason". Then, in `messages` array, you can have many "request-complete" messages. One of these messages will be triggered if the `messages[].conditions` matches the "reason" argument. + */ + function?: OpenAIFunction; +} + +export interface UpdateSmsToolDTO { + /** + * These are the messages that will be spoken to the user as the tool is running. + * + * For some tools, this is auto-filled based on special fields like `tool.destinations`. For others like the function tool, these can be custom configured. + */ + messages?: (ToolMessageStart | ToolMessageComplete | ToolMessageFailed | ToolMessageDelayed)[]; + /** + * This is the function definition of the tool. + * + * For `endCall`, `transferCall`, and `dtmf` tools, this is auto-filled based on tool-specific fields like `tool.destinations`. But, even in those cases, you can provide a custom function definition for advanced use cases. + * + * An example of an advanced use case is if you want to customize the message that's spoken for `endCall` tool. You can specify a function where it returns an argument "reason". Then, in `messages` array, you can have many "request-complete" messages. One of these messages will be triggered if the `messages[].conditions` matches the "reason" argument. + */ + function?: OpenAIFunction; +} + +export interface UpdateMcpToolDTO { + /** + * These are the messages that will be spoken to the user as the tool is running. + * + * For some tools, this is auto-filled based on special fields like `tool.destinations`. For others like the function tool, these can be custom configured. + */ + messages?: (ToolMessageStart | ToolMessageComplete | ToolMessageFailed | ToolMessageDelayed)[]; + /** + * + * This is the server where a `tool-calls` webhook will be sent. + * + * Notes: + * - Webhook is sent to this server when a tool call is made. + * - Webhook contains the call, assistant, and phone number objects. + * - Webhook contains the variables set on the assistant. + * - Webhook is sent to the first available URL in this order: {{tool.server.url}}, {{assistant.server.url}}, {{phoneNumber.server.url}}, {{org.server.url}}. + * - Webhook expects a response with tool call result. + */ + server?: Server; + /** + * This is the function definition of the tool. + * + * For `endCall`, `transferCall`, and `dtmf` tools, this is auto-filled based on tool-specific fields like `tool.destinations`. But, even in those cases, you can provide a custom function definition for advanced use cases. + * + * An example of an advanced use case is if you want to customize the message that's spoken for `endCall` tool. You can specify a function where it returns an argument "reason". Then, in `messages` array, you can have many "request-complete" messages. One of these messages will be triggered if the `messages[].conditions` matches the "reason" argument. + */ + function?: OpenAIFunction; + metadata?: McpToolMetadata; +} + +export interface UpdateGoHighLevelCalendarAvailabilityToolDTO { /** * These are the messages that will be spoken to the user as the tool is running. * @@ -15864,7 +17288,7 @@ export interface UpdateGoogleSheetsRowAppendToolDTO { function?: OpenAIFunction; } -export interface UpdateGoogleCalendarCheckAvailabilityToolDTO { +export interface UpdateGoHighLevelCalendarEventCreateToolDTO { /** * These are the messages that will be spoken to the user as the tool is running. * @@ -15881,7 +17305,7 @@ export interface UpdateGoogleCalendarCheckAvailabilityToolDTO { function?: OpenAIFunction; } -export interface UpdateSlackSendMessageToolDTO { +export interface UpdateGoHighLevelContactCreateToolDTO { /** * These are the messages that will be spoken to the user as the tool is running. * @@ -15898,7 +17322,7 @@ export interface UpdateSlackSendMessageToolDTO { function?: OpenAIFunction; } -export interface UpdateSmsToolDTO { +export interface UpdateGoHighLevelContactGetToolDTO { /** * These are the messages that will be spoken to the user as the tool is running. * @@ -15915,394 +17339,785 @@ export interface UpdateSmsToolDTO { function?: OpenAIFunction; } -export interface UpdateMcpToolDTO { +export interface CreateFileDTO { /** - * These are the messages that will be spoken to the user as the tool is running. + * This is the File you want to upload for use with the Knowledge Base. + * @format binary + */ + file: File; +} + +export interface File { + object?: 'file'; + status?: 'processing' | 'done' | 'failed'; + /** + * This is the name of the file. This is just for your own reference. + * @maxLength 40 + */ + name?: string; + originalName?: string; + bytes?: number; + purpose?: string; + mimetype?: string; + key?: string; + path?: string; + bucket?: string; + url?: string; + parsedTextUrl?: string; + parsedTextBytes?: number; + metadata?: object; + /** This is the unique identifier for the file. */ + id: string; + /** This is the unique identifier for the org that this file belongs to. */ + orgId: string; + /** + * This is the ISO 8601 date-time string of when the file was created. + * @format date-time + */ + createdAt: string; + /** + * This is the ISO 8601 date-time string of when the file was last updated. + * @format date-time + */ + updatedAt: string; +} + +export interface UpdateFileDTO { + /** + * This is the name of the file. This is just for your own reference. + * @minLength 1 + * @maxLength 40 + */ + name?: string; +} + +export interface TrieveKnowledgeBaseSearchPlan { + /** Specifies the number of top chunks to return. This corresponds to the `page_size` parameter in Trieve. */ + topK?: number; + /** If true, stop words (specified in server/src/stop-words.txt in the git repo) will be removed. This will preserve queries that are entirely stop words. */ + removeStopWords?: boolean; + /** This is the score threshold to filter out chunks with a score below the threshold for cosine distance metric. For Manhattan Distance, Euclidean Distance, and Dot Product, it will filter out scores above the threshold distance. This threshold applies before weight and bias modifications. If not specified, this defaults to no threshold. A threshold of 0 will default to no threshold. */ + scoreThreshold?: number; + /** This is the search method used when searching for relevant chunks from the vector store. */ + searchType: 'fulltext' | 'semantic' | 'hybrid' | 'bm25'; +} + +export interface TrieveKnowledgeBase { + /** + * This knowledge base is provided by Trieve. * - * For some tools, this is auto-filled based on special fields like `tool.destinations`. For others like the function tool, these can be custom configured. + * To learn more about Trieve, visit https://trieve.ai. */ - messages?: (ToolMessageStart | ToolMessageComplete | ToolMessageFailed | ToolMessageDelayed)[]; + provider: 'trieve'; + /** This is the name of the knowledge base. */ + name?: string; /** + * This is the searching plan used when searching for relevant chunks from the vector store. * - * This is the server where a `tool-calls` webhook will be sent. + * You should configure this if you're running into these issues: + * - Too much unnecessary context is being fed as knowledge base context. + * - Not enough relevant context is being fed as knowledge base context. + */ + searchPlan?: TrieveKnowledgeBaseSearchPlan; + /** This is the plan if you want us to create/import a new vector store using Trieve. */ + createPlan?: TrieveKnowledgeBaseImport; + /** This is the id of the knowledge base. */ + id: string; + /** This is the org id of the knowledge base. */ + orgId: string; +} + +export interface CustomKnowledgeBase { + /** This knowledge base is bring your own knowledge base implementation. */ + provider: 'custom-knowledge-base'; + /** + * This is where the knowledge base request will be sent. * - * Notes: - * - Webhook is sent to this server when a tool call is made. - * - Webhook contains the call, assistant, and phone number objects. - * - Webhook contains the variables set on the assistant. - * - Webhook is sent to the first available URL in this order: {{tool.server.url}}, {{assistant.server.url}}, {{phoneNumber.server.url}}, {{org.server.url}}. - * - Webhook expects a response with tool call result. + * Request Example: + * + * POST https://{server.url} + * Content-Type: application/json + * + * { + * "messsage": { + * "type": "knowledge-base-request", + * "messages": [ + * { + * "role": "user", + * "content": "Why is ocean blue?" + * } + * ], + * ...other metadata about the call... + * } + * } + * + * Response Expected: + * ``` + * { + * "message": { + * "role": "assistant", + * "content": "The ocean is blue because water absorbs everything but blue.", + * }, // YOU CAN RETURN THE EXACT RESPONSE TO SPEAK + * "documents": [ + * { + * "content": "The ocean is blue primarily because water absorbs colors in the red part of the light spectrum and scatters the blue light, making it more visible to our eyes.", + * "similarity": 1 + * }, + * { + * "content": "Blue light is scattered more by the water molecules than other colors, enhancing the blue appearance of the ocean.", + * "similarity": .5 + * } + * ] // OR, YOU CAN RETURN AN ARRAY OF DOCUMENTS THAT WILL BE SENT TO THE MODEL + * } + * ``` */ - server?: Server; + server: Server; + /** This is the id of the knowledge base. */ + id: string; + /** This is the org id of the knowledge base. */ + orgId: string; +} + +export interface CreateTrieveKnowledgeBaseDTO { /** - * This is the function definition of the tool. + * This knowledge base is provided by Trieve. * - * For `endCall`, `transferCall`, and `dtmf` tools, this is auto-filled based on tool-specific fields like `tool.destinations`. But, even in those cases, you can provide a custom function definition for advanced use cases. + * To learn more about Trieve, visit https://trieve.ai. + */ + provider: 'trieve'; + /** This is the name of the knowledge base. */ + name?: string; + /** + * This is the searching plan used when searching for relevant chunks from the vector store. * - * An example of an advanced use case is if you want to customize the message that's spoken for `endCall` tool. You can specify a function where it returns an argument "reason". Then, in `messages` array, you can have many "request-complete" messages. One of these messages will be triggered if the `messages[].conditions` matches the "reason" argument. + * You should configure this if you're running into these issues: + * - Too much unnecessary context is being fed as knowledge base context. + * - Not enough relevant context is being fed as knowledge base context. */ - function?: OpenAIFunction; + searchPlan?: TrieveKnowledgeBaseSearchPlan; + /** This is the plan if you want us to create/import a new vector store using Trieve. */ + createPlan?: TrieveKnowledgeBaseImport; } -export interface UpdateGoHighLevelCalendarAvailabilityToolDTO { +export interface UpdateTrieveKnowledgeBaseDTO { + /** This is the name of the knowledge base. */ + name?: string; /** - * These are the messages that will be spoken to the user as the tool is running. + * This is the searching plan used when searching for relevant chunks from the vector store. * - * For some tools, this is auto-filled based on special fields like `tool.destinations`. For others like the function tool, these can be custom configured. + * You should configure this if you're running into these issues: + * - Too much unnecessary context is being fed as knowledge base context. + * - Not enough relevant context is being fed as knowledge base context. */ - messages?: (ToolMessageStart | ToolMessageComplete | ToolMessageFailed | ToolMessageDelayed)[]; + searchPlan?: TrieveKnowledgeBaseSearchPlan; + /** This is the plan if you want us to create/import a new vector store using Trieve. */ + createPlan?: TrieveKnowledgeBaseImport; +} + +export interface UpdateCustomKnowledgeBaseDTO { /** - * This is the function definition of the tool. + * This is where the knowledge base request will be sent. * - * For `endCall`, `transferCall`, and `dtmf` tools, this is auto-filled based on tool-specific fields like `tool.destinations`. But, even in those cases, you can provide a custom function definition for advanced use cases. + * Request Example: * - * An example of an advanced use case is if you want to customize the message that's spoken for `endCall` tool. You can specify a function where it returns an argument "reason". Then, in `messages` array, you can have many "request-complete" messages. One of these messages will be triggered if the `messages[].conditions` matches the "reason" argument. + * POST https://{server.url} + * Content-Type: application/json + * + * { + * "messsage": { + * "type": "knowledge-base-request", + * "messages": [ + * { + * "role": "user", + * "content": "Why is ocean blue?" + * } + * ], + * ...other metadata about the call... + * } + * } + * + * Response Expected: + * ``` + * { + * "message": { + * "role": "assistant", + * "content": "The ocean is blue because water absorbs everything but blue.", + * }, // YOU CAN RETURN THE EXACT RESPONSE TO SPEAK + * "documents": [ + * { + * "content": "The ocean is blue primarily because water absorbs colors in the red part of the light spectrum and scatters the blue light, making it more visible to our eyes.", + * "similarity": 1 + * }, + * { + * "content": "Blue light is scattered more by the water molecules than other colors, enhancing the blue appearance of the ocean.", + * "similarity": .5 + * } + * ] // OR, YOU CAN RETURN AN ARRAY OF DOCUMENTS THAT WILL BE SENT TO THE MODEL + * } + * ``` */ - function?: OpenAIFunction; + server?: Server; +} + +export interface TrieveKnowledgeBaseChunkPlan { + /** These are the file ids that will be used to create the vector store. To upload files, use the `POST /files` endpoint. */ + fileIds?: string[]; + /** These are the websites that will be used to create the vector store. */ + websites?: string[]; + /** This is an optional field which allows you to specify the number of splits you want per chunk. If not specified, the default 20 is used. However, you may want to use a different number. */ + targetSplitsPerChunk?: number; + /** This is an optional field which allows you to specify the delimiters to use when splitting the file before chunking the text. If not specified, the default [.!?\n] are used to split into sentences. However, you may want to use spaces or other delimiters. */ + splitDelimiters?: string[]; + /** This is an optional field which allows you to specify whether or not to rebalance the chunks created from the file. If not specified, the default true is used. If true, Trieve will evenly distribute remainder splits across chunks such that 66 splits with a target_splits_per_chunk of 20 will result in 3 chunks with 22 splits each. */ + rebalanceChunks?: boolean; +} + +export interface TrieveKnowledgeBaseCreate { + /** This is to create a new dataset on Trieve. */ + type: 'create'; + /** These are the chunk plans used to create the dataset. */ + chunkPlans: TrieveKnowledgeBaseChunkPlan[]; +} + +export interface TrieveKnowledgeBaseImport { + /** This is to import an existing dataset from Trieve. */ + type: 'import'; + /** This is the `datasetId` of the dataset on your Trieve account. */ + providerId: string; } -export interface UpdateGoHighLevelCalendarEventCreateToolDTO { +export interface Workflow { + nodes: (ConversationNode | ToolNode)[]; /** - * These are the messages that will be spoken to the user as the tool is running. + * This is the transcriber for the workflow. * - * For some tools, this is auto-filled based on special fields like `tool.destinations`. For others like the function tool, these can be custom configured. + * This can be overridden at node level using `nodes[n].transcriber`. */ - messages?: (ToolMessageStart | ToolMessageComplete | ToolMessageFailed | ToolMessageDelayed)[]; + transcriber?: + | AssemblyAITranscriber + | AzureSpeechTranscriber + | CustomTranscriber + | DeepgramTranscriber + | ElevenLabsTranscriber + | GladiaTranscriber + | GoogleTranscriber + | SpeechmaticsTranscriber + | TalkscriberTranscriber + | OpenAITranscriber + | CartesiaTranscriber; /** - * This is the function definition of the tool. - * - * For `endCall`, `transferCall`, and `dtmf` tools, this is auto-filled based on tool-specific fields like `tool.destinations`. But, even in those cases, you can provide a custom function definition for advanced use cases. + * This is the voice for the workflow. * - * An example of an advanced use case is if you want to customize the message that's spoken for `endCall` tool. You can specify a function where it returns an argument "reason". Then, in `messages` array, you can have many "request-complete" messages. One of these messages will be triggered if the `messages[].conditions` matches the "reason" argument. + * This can be overridden at node level using `nodes[n].voice`. */ - function?: OpenAIFunction; -} - -export interface UpdateGoHighLevelContactCreateToolDTO { + voice?: + | AzureVoice + | CartesiaVoice + | CustomVoice + | DeepgramVoice + | ElevenLabsVoice + | HumeVoice + | LMNTVoice + | NeuphonicVoice + | OpenAIVoice + | PlayHTVoice + | RimeAIVoice + | SmallestAIVoice + | TavusVoice + | VapiVoice + | SesameVoice; /** - * These are the messages that will be spoken to the user as the tool is running. + * This is the plan for observability of workflow's calls. * - * For some tools, this is auto-filled based on special fields like `tool.destinations`. For others like the function tool, these can be custom configured. + * Currently, only Langfuse is supported. */ - messages?: (ToolMessageStart | ToolMessageComplete | ToolMessageFailed | ToolMessageDelayed)[]; + observabilityPlan?: LangfuseObservabilityPlan; + /** These are dynamic credentials that will be used for the workflow calls. By default, all the credentials are available for use in the call but you can supplement an additional credentials using this. Dynamic credentials override existing credentials. */ + credentials?: ( + | ({ + provider: '11labs'; + } & CreateElevenLabsCredentialDTO) + | ({ + provider: 'anthropic'; + } & CreateAnthropicCredentialDTO) + | ({ + provider: 'anyscale'; + } & CreateAnyscaleCredentialDTO) + | ({ + provider: 'assembly-ai'; + } & CreateAssemblyAICredentialDTO) + | ({ + provider: 'azure-openai'; + } & CreateAzureOpenAICredentialDTO) + | ({ + provider: 'azure'; + } & CreateAzureCredentialDTO) + | ({ + provider: 'byo-sip-trunk'; + } & CreateByoSipTrunkCredentialDTO) + | ({ + provider: 'cartesia'; + } & CreateCartesiaCredentialDTO) + | ({ + provider: 'cerebras'; + } & CreateCerebrasCredentialDTO) + | ({ + provider: 'cloudflare'; + } & CreateCloudflareCredentialDTO) + | ({ + provider: 'custom-llm'; + } & CreateCustomLLMCredentialDTO) + | ({ + provider: 'deepgram'; + } & CreateDeepgramCredentialDTO) + | ({ + provider: 'deepinfra'; + } & CreateDeepInfraCredentialDTO) + | ({ + provider: 'deep-seek'; + } & CreateDeepSeekCredentialDTO) + | ({ + provider: 'gcp'; + } & CreateGcpCredentialDTO) + | ({ + provider: 'gladia'; + } & CreateGladiaCredentialDTO) + | ({ + provider: 'gohighlevel'; + } & CreateGoHighLevelCredentialDTO) + | ({ + provider: 'google'; + } & CreateGoogleCredentialDTO) + | ({ + provider: 'groq'; + } & CreateGroqCredentialDTO) + | ({ + provider: 'inflection-ai'; + } & CreateInflectionAICredentialDTO) + | ({ + provider: 'langfuse'; + } & CreateLangfuseCredentialDTO) + | ({ + provider: 'lmnt'; + } & CreateLmntCredentialDTO) + | ({ + provider: 'make'; + } & CreateMakeCredentialDTO) + | ({ + provider: 'openai'; + } & CreateOpenAICredentialDTO) + | ({ + provider: 'openrouter'; + } & CreateOpenRouterCredentialDTO) + | ({ + provider: 'perplexity-ai'; + } & CreatePerplexityAICredentialDTO) + | ({ + provider: 'playht'; + } & CreatePlayHTCredentialDTO) + | ({ + provider: 'rime-ai'; + } & CreateRimeAICredentialDTO) + | ({ + provider: 'runpod'; + } & CreateRunpodCredentialDTO) + | ({ + provider: 's3'; + } & CreateS3CredentialDTO) + | ({ + provider: 'supabase'; + } & CreateSupabaseCredentialDTO) + | ({ + provider: 'smallest-ai'; + } & CreateSmallestAICredentialDTO) + | ({ + provider: 'tavus'; + } & CreateTavusCredentialDTO) + | ({ + provider: 'together-ai'; + } & CreateTogetherAICredentialDTO) + | ({ + provider: 'twilio'; + } & CreateTwilioCredentialDTO) + | ({ + provider: 'vonage'; + } & CreateVonageCredentialDTO) + | ({ + provider: 'webhook'; + } & CreateWebhookCredentialDTO) + | ({ + provider: 'xai'; + } & CreateXAiCredentialDTO) + | ({ + provider: 'neuphonic'; + } & CreateNeuphonicCredentialDTO) + | ({ + provider: 'hume'; + } & CreateHumeCredentialDTO) + | ({ + provider: 'mistral'; + } & CreateMistralCredentialDTO) + | ({ + provider: 'speechmatics'; + } & CreateSpeechmaticsCredentialDTO) + | ({ + provider: 'trieve'; + } & CreateTrieveCredentialDTO) + | ({ + provider: 'google.calendar.oauth2-client'; + } & CreateGoogleCalendarOAuth2ClientCredentialDTO) + | ({ + provider: 'google.calendar.oauth2-authorization'; + } & CreateGoogleCalendarOAuth2AuthorizationCredentialDTO) + | ({ + provider: 'google.sheets.oauth2-authorization'; + } & CreateGoogleSheetsOAuth2AuthorizationCredentialDTO) + | ({ + provider: 'slack.oauth2-authorization'; + } & CreateSlackOAuth2AuthorizationCredentialDTO) + | ({ + provider: 'ghl.oauth2-authorization'; + } & CreateGoHighLevelMCPCredentialDTO) + | ({ + provider: 'inworld'; + } & any) + )[]; + id: string; + orgId: string; + /** @format date-time */ + createdAt: string; + /** @format date-time */ + updatedAt: string; + /** @maxLength 80 */ + name: string; + edges: Edge[]; + /** @maxLength 5000 */ + globalPrompt?: string; /** - * This is the function definition of the tool. + * This is where Vapi will send webhooks. You can find all webhooks available along with their shape in ServerMessage schema. * - * For `endCall`, `transferCall`, and `dtmf` tools, this is auto-filled based on tool-specific fields like `tool.destinations`. But, even in those cases, you can provide a custom function definition for advanced use cases. + * The order of precedence is: * - * An example of an advanced use case is if you want to customize the message that's spoken for `endCall` tool. You can specify a function where it returns an argument "reason". Then, in `messages` array, you can have many "request-complete" messages. One of these messages will be triggered if the `messages[].conditions` matches the "reason" argument. + * 1. tool.server + * 2. workflow.server / assistant.server + * 3. phoneNumber.server + * 4. org.server */ - function?: OpenAIFunction; -} - -export interface UpdateGoHighLevelContactGetToolDTO { + server?: Server; + /** This is the compliance plan for the workflow. It allows you to configure HIPAA and other compliance settings. */ + compliancePlan?: CompliancePlan; + /** This is the plan for analysis of workflow's calls. Stored in `call.analysis`. */ + analysisPlan?: AnalysisPlan; + /** This is the plan for artifacts generated during workflow's calls. Stored in `call.artifact`. */ + artifactPlan?: ArtifactPlan; /** - * These are the messages that will be spoken to the user as the tool is running. + * This is the plan for when the workflow nodes should start talking. * - * For some tools, this is auto-filled based on special fields like `tool.destinations`. For others like the function tool, these can be custom configured. + * You should configure this if you're running into these issues: + * - The assistant is too slow to start talking after the customer is done speaking. + * - The assistant is too fast to start talking after the customer is done speaking. + * - The assistant is so fast that it's actually interrupting the customer. */ - messages?: (ToolMessageStart | ToolMessageComplete | ToolMessageFailed | ToolMessageDelayed)[]; + startSpeakingPlan?: StartSpeakingPlan; /** - * This is the function definition of the tool. + * This is the plan for when workflow nodes should stop talking on customer interruption. * - * For `endCall`, `transferCall`, and `dtmf` tools, this is auto-filled based on tool-specific fields like `tool.destinations`. But, even in those cases, you can provide a custom function definition for advanced use cases. - * - * An example of an advanced use case is if you want to customize the message that's spoken for `endCall` tool. You can specify a function where it returns an argument "reason". Then, in `messages` array, you can have many "request-complete" messages. One of these messages will be triggered if the `messages[].conditions` matches the "reason" argument. - */ - function?: OpenAIFunction; -} - -export interface CreateFileDTO { - /** - * This is the File you want to upload for use with the Knowledge Base. - * @format binary - */ - file: File; -} - -export interface File { - object?: 'file'; - status?: 'processing' | 'done' | 'failed'; - /** - * This is the name of the file. This is just for your own reference. - * @maxLength 40 + * You should configure this if you're running into these issues: + * - The assistant is too slow to recognize customer's interruption. + * - The assistant is too fast to recognize customer's interruption. + * - The assistant is getting interrupted by phrases that are just acknowledgments. + * - The assistant is getting interrupted by background noises. + * - The assistant is not properly stopping -- it starts talking right after getting interrupted. */ - name?: string; - originalName?: string; - bytes?: number; - purpose?: string; - mimetype?: string; - key?: string; - path?: string; - bucket?: string; - url?: string; - parsedTextUrl?: string; - parsedTextBytes?: number; - metadata?: object; - /** This is the unique identifier for the file. */ - id: string; - /** This is the unique identifier for the org that this file belongs to. */ - orgId: string; + stopSpeakingPlan?: StopSpeakingPlan; /** - * This is the ISO 8601 date-time string of when the file was created. - * @format date-time + * This is the plan for real-time monitoring of the workflow's calls. + * + * Usage: + * - To enable live listening of the workflow's calls, set `monitorPlan.listenEnabled` to `true`. + * - To enable live control of the workflow's calls, set `monitorPlan.controlEnabled` to `true`. */ - createdAt: string; + monitorPlan?: MonitorPlan; /** - * This is the ISO 8601 date-time string of when the file was last updated. - * @format date-time + * This enables filtering of noise and background speech while the user is talking. + * + * Features: + * - Smart denoising using Krisp + * - Fourier denoising + * + * Both can be used together. Order of precedence: + * - Smart denoising + * - Fourier denoising */ - updatedAt: string; + backgroundSpeechDenoisingPlan?: BackgroundSpeechDenoisingPlan; + /** These are the credentials that will be used for the workflow calls. By default, all the credentials are available for use in the call but you can provide a subset using this. */ + credentialIds?: string[]; } -export interface UpdateFileDTO { +export interface UpdateWorkflowDTO { + nodes?: (ConversationNode | ToolNode)[]; /** - * This is the name of the file. This is just for your own reference. - * @minLength 1 - * @maxLength 40 + * This is the transcriber for the workflow. + * + * This can be overridden at node level using `nodes[n].transcriber`. */ - name?: string; -} - -export interface TrieveKnowledgeBaseSearchPlan { - /** Specifies the number of top chunks to return. This corresponds to the `page_size` parameter in Trieve. */ - topK?: number; - /** If true, stop words (specified in server/src/stop-words.txt in the git repo) will be removed. This will preserve queries that are entirely stop words. */ - removeStopWords?: boolean; - /** This is the score threshold to filter out chunks with a score below the threshold for cosine distance metric. For Manhattan Distance, Euclidean Distance, and Dot Product, it will filter out scores above the threshold distance. This threshold applies before weight and bias modifications. If not specified, this defaults to no threshold. A threshold of 0 will default to no threshold. */ - scoreThreshold?: number; - /** This is the search method used when searching for relevant chunks from the vector store. */ - searchType: 'fulltext' | 'semantic' | 'hybrid' | 'bm25'; -} - -export interface TrieveKnowledgeBase { + transcriber?: + | AssemblyAITranscriber + | AzureSpeechTranscriber + | CustomTranscriber + | DeepgramTranscriber + | ElevenLabsTranscriber + | GladiaTranscriber + | GoogleTranscriber + | SpeechmaticsTranscriber + | TalkscriberTranscriber + | OpenAITranscriber + | CartesiaTranscriber; /** - * This knowledge base is provided by Trieve. + * This is the voice for the workflow. * - * To learn more about Trieve, visit https://trieve.ai. + * This can be overridden at node level using `nodes[n].voice`. */ - provider: 'trieve'; - /** This is the name of the knowledge base. */ - name?: string; + voice?: + | AzureVoice + | CartesiaVoice + | CustomVoice + | DeepgramVoice + | ElevenLabsVoice + | HumeVoice + | LMNTVoice + | NeuphonicVoice + | OpenAIVoice + | PlayHTVoice + | RimeAIVoice + | SmallestAIVoice + | TavusVoice + | VapiVoice + | SesameVoice; /** - * This is the searching plan used when searching for relevant chunks from the vector store. + * This is the plan for observability of workflow's calls. * - * You should configure this if you're running into these issues: - * - Too much unnecessary context is being fed as knowledge base context. - * - Not enough relevant context is being fed as knowledge base context. + * Currently, only Langfuse is supported. */ - searchPlan?: TrieveKnowledgeBaseSearchPlan; - /** This is the plan if you want us to create/import a new vector store using Trieve. */ - createPlan?: TrieveKnowledgeBaseImport; - /** This is the id of the knowledge base. */ - id: string; - /** This is the org id of the knowledge base. */ - orgId: string; -} - -export interface CustomKnowledgeBase { - /** This knowledge base is bring your own knowledge base implementation. */ - provider: 'custom-knowledge-base'; + observabilityPlan?: LangfuseObservabilityPlan; + /** These are dynamic credentials that will be used for the workflow calls. By default, all the credentials are available for use in the call but you can supplement an additional credentials using this. Dynamic credentials override existing credentials. */ + credentials?: ( + | ({ + provider: '11labs'; + } & CreateElevenLabsCredentialDTO) + | ({ + provider: 'anthropic'; + } & CreateAnthropicCredentialDTO) + | ({ + provider: 'anyscale'; + } & CreateAnyscaleCredentialDTO) + | ({ + provider: 'assembly-ai'; + } & CreateAssemblyAICredentialDTO) + | ({ + provider: 'azure-openai'; + } & CreateAzureOpenAICredentialDTO) + | ({ + provider: 'azure'; + } & CreateAzureCredentialDTO) + | ({ + provider: 'byo-sip-trunk'; + } & CreateByoSipTrunkCredentialDTO) + | ({ + provider: 'cartesia'; + } & CreateCartesiaCredentialDTO) + | ({ + provider: 'cerebras'; + } & CreateCerebrasCredentialDTO) + | ({ + provider: 'cloudflare'; + } & CreateCloudflareCredentialDTO) + | ({ + provider: 'custom-llm'; + } & CreateCustomLLMCredentialDTO) + | ({ + provider: 'deepgram'; + } & CreateDeepgramCredentialDTO) + | ({ + provider: 'deepinfra'; + } & CreateDeepInfraCredentialDTO) + | ({ + provider: 'deep-seek'; + } & CreateDeepSeekCredentialDTO) + | ({ + provider: 'gcp'; + } & CreateGcpCredentialDTO) + | ({ + provider: 'gladia'; + } & CreateGladiaCredentialDTO) + | ({ + provider: 'gohighlevel'; + } & CreateGoHighLevelCredentialDTO) + | ({ + provider: 'google'; + } & CreateGoogleCredentialDTO) + | ({ + provider: 'groq'; + } & CreateGroqCredentialDTO) + | ({ + provider: 'inflection-ai'; + } & CreateInflectionAICredentialDTO) + | ({ + provider: 'langfuse'; + } & CreateLangfuseCredentialDTO) + | ({ + provider: 'lmnt'; + } & CreateLmntCredentialDTO) + | ({ + provider: 'make'; + } & CreateMakeCredentialDTO) + | ({ + provider: 'openai'; + } & CreateOpenAICredentialDTO) + | ({ + provider: 'openrouter'; + } & CreateOpenRouterCredentialDTO) + | ({ + provider: 'perplexity-ai'; + } & CreatePerplexityAICredentialDTO) + | ({ + provider: 'playht'; + } & CreatePlayHTCredentialDTO) + | ({ + provider: 'rime-ai'; + } & CreateRimeAICredentialDTO) + | ({ + provider: 'runpod'; + } & CreateRunpodCredentialDTO) + | ({ + provider: 's3'; + } & CreateS3CredentialDTO) + | ({ + provider: 'supabase'; + } & CreateSupabaseCredentialDTO) + | ({ + provider: 'smallest-ai'; + } & CreateSmallestAICredentialDTO) + | ({ + provider: 'tavus'; + } & CreateTavusCredentialDTO) + | ({ + provider: 'together-ai'; + } & CreateTogetherAICredentialDTO) + | ({ + provider: 'twilio'; + } & CreateTwilioCredentialDTO) + | ({ + provider: 'vonage'; + } & CreateVonageCredentialDTO) + | ({ + provider: 'webhook'; + } & CreateWebhookCredentialDTO) + | ({ + provider: 'xai'; + } & CreateXAiCredentialDTO) + | ({ + provider: 'neuphonic'; + } & CreateNeuphonicCredentialDTO) + | ({ + provider: 'hume'; + } & CreateHumeCredentialDTO) + | ({ + provider: 'mistral'; + } & CreateMistralCredentialDTO) + | ({ + provider: 'speechmatics'; + } & CreateSpeechmaticsCredentialDTO) + | ({ + provider: 'trieve'; + } & CreateTrieveCredentialDTO) + | ({ + provider: 'google.calendar.oauth2-client'; + } & CreateGoogleCalendarOAuth2ClientCredentialDTO) + | ({ + provider: 'google.calendar.oauth2-authorization'; + } & CreateGoogleCalendarOAuth2AuthorizationCredentialDTO) + | ({ + provider: 'google.sheets.oauth2-authorization'; + } & CreateGoogleSheetsOAuth2AuthorizationCredentialDTO) + | ({ + provider: 'slack.oauth2-authorization'; + } & CreateSlackOAuth2AuthorizationCredentialDTO) + | ({ + provider: 'ghl.oauth2-authorization'; + } & CreateGoHighLevelMCPCredentialDTO) + | ({ + provider: 'inworld'; + } & any) + )[]; + /** @maxLength 80 */ + name?: string; + edges?: Edge[]; + /** @maxLength 5000 */ + globalPrompt?: string; /** - * This is where the knowledge base request will be sent. - * - * Request Example: - * - * POST https://{server.url} - * Content-Type: application/json - * - * { - * "messsage": { - * "type": "knowledge-base-request", - * "messages": [ - * { - * "role": "user", - * "content": "Why is ocean blue?" - * } - * ], - * ...other metadata about the call... - * } - * } + * This is where Vapi will send webhooks. You can find all webhooks available along with their shape in ServerMessage schema. * - * Response Expected: - * ``` - * { - * "message": { - * "role": "assistant", - * "content": "The ocean is blue because water absorbs everything but blue.", - * }, // YOU CAN RETURN THE EXACT RESPONSE TO SPEAK - * "documents": [ - * { - * "content": "The ocean is blue primarily because water absorbs colors in the red part of the light spectrum and scatters the blue light, making it more visible to our eyes.", - * "similarity": 1 - * }, - * { - * "content": "Blue light is scattered more by the water molecules than other colors, enhancing the blue appearance of the ocean.", - * "similarity": .5 - * } - * ] // OR, YOU CAN RETURN AN ARRAY OF DOCUMENTS THAT WILL BE SENT TO THE MODEL - * } - * ``` - */ - server: Server; - /** This is the id of the knowledge base. */ - id: string; - /** This is the org id of the knowledge base. */ - orgId: string; -} - -export interface CreateTrieveKnowledgeBaseDTO { - /** - * This knowledge base is provided by Trieve. + * The order of precedence is: * - * To learn more about Trieve, visit https://trieve.ai. + * 1. tool.server + * 2. workflow.server / assistant.server + * 3. phoneNumber.server + * 4. org.server */ - provider: 'trieve'; - /** This is the name of the knowledge base. */ - name?: string; + server?: Server; + /** This is the compliance plan for the workflow. It allows you to configure HIPAA and other compliance settings. */ + compliancePlan?: CompliancePlan; + /** This is the plan for analysis of workflow's calls. Stored in `call.analysis`. */ + analysisPlan?: AnalysisPlan; + /** This is the plan for artifacts generated during workflow's calls. Stored in `call.artifact`. */ + artifactPlan?: ArtifactPlan; /** - * This is the searching plan used when searching for relevant chunks from the vector store. + * This is the plan for when the workflow nodes should start talking. * * You should configure this if you're running into these issues: - * - Too much unnecessary context is being fed as knowledge base context. - * - Not enough relevant context is being fed as knowledge base context. + * - The assistant is too slow to start talking after the customer is done speaking. + * - The assistant is too fast to start talking after the customer is done speaking. + * - The assistant is so fast that it's actually interrupting the customer. */ - searchPlan?: TrieveKnowledgeBaseSearchPlan; - /** This is the plan if you want us to create/import a new vector store using Trieve. */ - createPlan?: TrieveKnowledgeBaseImport; -} - -export interface UpdateTrieveKnowledgeBaseDTO { - /** This is the name of the knowledge base. */ - name?: string; + startSpeakingPlan?: StartSpeakingPlan; /** - * This is the searching plan used when searching for relevant chunks from the vector store. + * This is the plan for when workflow nodes should stop talking on customer interruption. * * You should configure this if you're running into these issues: - * - Too much unnecessary context is being fed as knowledge base context. - * - Not enough relevant context is being fed as knowledge base context. + * - The assistant is too slow to recognize customer's interruption. + * - The assistant is too fast to recognize customer's interruption. + * - The assistant is getting interrupted by phrases that are just acknowledgments. + * - The assistant is getting interrupted by background noises. + * - The assistant is not properly stopping -- it starts talking right after getting interrupted. */ - searchPlan?: TrieveKnowledgeBaseSearchPlan; - /** This is the plan if you want us to create/import a new vector store using Trieve. */ - createPlan?: TrieveKnowledgeBaseImport; -} - -export interface UpdateCustomKnowledgeBaseDTO { + stopSpeakingPlan?: StopSpeakingPlan; /** - * This is where the knowledge base request will be sent. - * - * Request Example: - * - * POST https://{server.url} - * Content-Type: application/json - * - * { - * "messsage": { - * "type": "knowledge-base-request", - * "messages": [ - * { - * "role": "user", - * "content": "Why is ocean blue?" - * } - * ], - * ...other metadata about the call... - * } - * } + * This is the plan for real-time monitoring of the workflow's calls. * - * Response Expected: - * ``` - * { - * "message": { - * "role": "assistant", - * "content": "The ocean is blue because water absorbs everything but blue.", - * }, // YOU CAN RETURN THE EXACT RESPONSE TO SPEAK - * "documents": [ - * { - * "content": "The ocean is blue primarily because water absorbs colors in the red part of the light spectrum and scatters the blue light, making it more visible to our eyes.", - * "similarity": 1 - * }, - * { - * "content": "Blue light is scattered more by the water molecules than other colors, enhancing the blue appearance of the ocean.", - * "similarity": .5 - * } - * ] // OR, YOU CAN RETURN AN ARRAY OF DOCUMENTS THAT WILL BE SENT TO THE MODEL - * } - * ``` + * Usage: + * - To enable live listening of the workflow's calls, set `monitorPlan.listenEnabled` to `true`. + * - To enable live control of the workflow's calls, set `monitorPlan.controlEnabled` to `true`. */ - server?: Server; -} - -export interface TrieveKnowledgeBaseChunkPlan { - /** These are the file ids that will be used to create the vector store. To upload files, use the `POST /files` endpoint. */ - fileIds?: string[]; - /** These are the websites that will be used to create the vector store. */ - websites?: string[]; - /** This is an optional field which allows you to specify the number of splits you want per chunk. If not specified, the default 20 is used. However, you may want to use a different number. */ - targetSplitsPerChunk?: number; - /** This is an optional field which allows you to specify the delimiters to use when splitting the file before chunking the text. If not specified, the default [.!?\n] are used to split into sentences. However, you may want to use spaces or other delimiters. */ - splitDelimiters?: string[]; - /** This is an optional field which allows you to specify whether or not to rebalance the chunks created from the file. If not specified, the default true is used. If true, Trieve will evenly distribute remainder splits across chunks such that 66 splits with a target_splits_per_chunk of 20 will result in 3 chunks with 22 splits each. */ - rebalanceChunks?: boolean; -} - -export interface TrieveKnowledgeBaseCreate { - /** This is to create a new dataset on Trieve. */ - type: 'create'; - /** These are the chunk plans used to create the dataset. */ - chunkPlans: TrieveKnowledgeBaseChunkPlan[]; -} - -export interface TrieveKnowledgeBaseImport { - /** This is to import an existing dataset from Trieve. */ - type: 'import'; - /** This is the `datasetId` of the dataset on your Trieve account. */ - providerId: string; -} - -export interface Workflow { - nodes: (ConversationNode | ToolNode)[]; - id: string; - orgId: string; - /** @format date-time */ - createdAt: string; - /** @format date-time */ - updatedAt: string; - /** @maxLength 80 */ - name: string; - edges: Edge[]; - /** @maxLength 5000 */ - globalPrompt?: string; + monitorPlan?: MonitorPlan; /** - * This is where Vapi will send webhooks. You can find all webhooks available along with their shape in ServerMessage schema. + * This enables filtering of noise and background speech while the user is talking. * - * The order of precedence is: + * Features: + * - Smart denoising using Krisp + * - Fourier denoising * - * 1. tool.server - * 2. workflow.server / assistant.server - * 3. phoneNumber.server - * 4. org.server + * Both can be used together. Order of precedence: + * - Smart denoising + * - Fourier denoising */ - server?: Server; + backgroundSpeechDenoisingPlan?: BackgroundSpeechDenoisingPlan; + /** These are the credentials that will be used for the workflow calls. By default, all the credentials are available for use in the call but you can provide a subset using this. */ + credentialIds?: string[]; } -export interface UpdateWorkflowDTO { - nodes?: (ConversationNode | ToolNode)[]; - /** @maxLength 80 */ - name?: string; - edges?: Edge[]; - /** @maxLength 5000 */ - globalPrompt?: string; - /** - * This is where Vapi will send webhooks. You can find all webhooks available along with their shape in ServerMessage schema. - * - * The order of precedence is: - * - * 1. tool.server - * 2. workflow.server / assistant.server - * 3. phoneNumber.server - * 4. org.server - */ - server?: Server; +export interface GenerateWorkflowDTO { + toolIds?: string[]; } export interface Squad { @@ -17036,6 +18851,7 @@ export interface CreateOrgDTO { server?: Server; /** * This is the concurrency limit for the org. This is the maximum number of calls that can be active at any given time. To go beyond 10, please contact us at support@vapi.ai. + * @deprecated * @min 1 * @max 10 */ @@ -17275,6 +19091,7 @@ export interface Org { server?: Server; /** * This is the concurrency limit for the org. This is the maximum number of calls that can be active at any given time. To go beyond 10, please contact us at support@vapi.ai. + * @deprecated * @min 1 * @max 10 */ @@ -17325,6 +19142,7 @@ export interface UpdateOrgDTO { server?: Server; /** * This is the concurrency limit for the org. This is the maximum number of calls that can be active at any given time. To go beyond 10, please contact us at support@vapi.ai. + * @deprecated * @min 1 * @max 10 */ @@ -17995,7 +19813,11 @@ export interface GcpCredential { * The schema is identical to the JSON that GCP outputs. */ gcpKey: GcpKey; - /** This is the bucket plan that can be provided to store call artifacts in GCP. */ + /** + * This is the region of the GCP resource. + * @maxLength 40 + */ + region?: string; bucketPlan?: BucketPlan; } @@ -18737,8 +20559,14 @@ export interface VonageCredential { export interface WebhookCredential { provider: 'webhook'; - /** This is the authentication plan. Currently supports OAuth2 RFC 6749. */ - authenticationPlan: OAuth2AuthenticationPlan; + /** This is the authentication plan. Supports OAuth2 RFC 6749 and HMAC signing. */ + authenticationPlan: + | ({ + type: 'oauth2'; + } & OAuth2AuthenticationPlan) + | ({ + type: 'hmac'; + } & HMACAuthenticationPlan); /** This is the unique identifier for the credential. */ id: string; /** This is the unique identifier for the org that this credential belongs to. */ @@ -19347,7 +21175,11 @@ export interface UpdateGcpCredentialDTO { * The schema is identical to the JSON that GCP outputs. */ gcpKey?: GcpKey; - /** This is the bucket plan that can be provided to store call artifacts in GCP. */ + /** + * This is the region of the GCP resource. + * @maxLength 40 + */ + region?: string; bucketPlan?: BucketPlan; } @@ -19682,8 +21514,14 @@ export interface UpdateVonageCredentialDTO { } export interface UpdateWebhookCredentialDTO { - /** This is the authentication plan. Currently supports OAuth2 RFC 6749. */ - authenticationPlan?: OAuth2AuthenticationPlan; + /** This is the authentication plan. Supports OAuth2 RFC 6749 and HMAC signing. */ + authenticationPlan?: + | ({ + type: 'oauth2'; + } & OAuth2AuthenticationPlan) + | ({ + type: 'hmac'; + } & HMACAuthenticationPlan); /** * This is the name of credential. This is just for your reference. * @minLength 1 @@ -19792,6 +21630,30 @@ export interface CredentialActionRequest { input: object; } +export interface HMACAuthenticationPlan { + type: 'hmac'; + /** This is the HMAC secret key used to sign requests. */ + secretKey: string; + /** This is the HMAC algorithm to use for signing. */ + algorithm: 'sha256' | 'sha512' | 'sha1'; + /** This is the header name where the signature will be sent. Defaults to 'x-signature'. */ + signatureHeader?: string; + /** This is the header name where the timestamp will be sent. Defaults to 'x-timestamp'. */ + timestampHeader?: string; + /** This is the prefix for the signature. For example, 'sha256=' for GitHub-style signatures. */ + signaturePrefix?: string; + /** Whether to include a timestamp in the signature payload. Defaults to true. */ + includeTimestamp?: boolean; + /** Custom payload format. Use {body} for request body, {timestamp} for timestamp, {method} for HTTP method, {url} for URL, {svix-id} for unique message ID. Defaults to '{timestamp}.{body}'. */ + payloadFormat?: string; + /** This is the header name where the unique message ID will be sent. Used for Svix-style webhooks. */ + messageIdHeader?: string; + /** The encoding format for the signature. Defaults to 'hex'. */ + signatureEncoding?: 'hex' | 'base64'; + /** Whether the secret key is base64-encoded and should be decoded before use. Defaults to false. */ + secretIsBase64?: boolean; +} + export interface CredentialSessionDTO { /** The type of credential to generate a session for. Only Nango user-facing providers are supported. */ provider: @@ -20058,7 +21920,8 @@ export interface VoiceLibrary { | 'rime-ai' | 'smallest-ai' | 'tavus' - | 'sesame'; + | 'sesame' + | 'inworld'; /** The ID of the voice provided by the provider. */ providerId?: string; /** The unique slug of the voice. */ @@ -20120,7 +21983,8 @@ export interface SyncVoiceLibraryDTO { | 'rime-ai' | 'smallest-ai' | 'tavus' - | 'sesame'; + | 'sesame' + | 'inworld'; } export interface CreateSesameVoiceDTO { @@ -20603,6 +22467,7 @@ export interface ServerMessageEndOfCallReport { | 'pipeline-error-neuphonic-voice-failed' | 'pipeline-error-hume-voice-failed' | 'pipeline-error-sesame-voice-failed' + | 'pipeline-error-inworld-voice-failed' | 'pipeline-error-tavus-video-failed' | 'call.in-progress.error-vapifault-openai-voice-failed' | 'call.in-progress.error-vapifault-cartesia-voice-failed' @@ -20616,6 +22481,7 @@ export interface ServerMessageEndOfCallReport { | 'call.in-progress.error-vapifault-neuphonic-voice-failed' | 'call.in-progress.error-vapifault-hume-voice-failed' | 'call.in-progress.error-vapifault-sesame-voice-failed' + | 'call.in-progress.error-vapifault-inworld-voice-failed' | 'call.in-progress.error-vapifault-tavus-video-failed' | 'pipeline-error-vapi-llm-failed' | 'pipeline-error-vapi-400-bad-request-validation-failed' @@ -21032,6 +22898,7 @@ export interface ServerMessageEndOfCallReport { | 'pipeline-error-deepgram-returning-500-invalid-json' | 'pipeline-error-deepgram-returning-502-network-error' | 'pipeline-error-deepgram-returning-502-bad-gateway-ehostunreach' + | 'pipeline-error-deepgram-returning-econnreset' | 'call.in-progress.error-vapifault-deepgram-returning-400-no-such-model-language-tier-combination' | 'call.in-progress.error-vapifault-deepgram-returning-401-invalid-credentials' | 'call.in-progress.error-vapifault-deepgram-returning-404-not-found' @@ -21330,6 +23197,7 @@ export interface ServerMessageStatusUpdate { | 'pipeline-error-neuphonic-voice-failed' | 'pipeline-error-hume-voice-failed' | 'pipeline-error-sesame-voice-failed' + | 'pipeline-error-inworld-voice-failed' | 'pipeline-error-tavus-video-failed' | 'call.in-progress.error-vapifault-openai-voice-failed' | 'call.in-progress.error-vapifault-cartesia-voice-failed' @@ -21343,6 +23211,7 @@ export interface ServerMessageStatusUpdate { | 'call.in-progress.error-vapifault-neuphonic-voice-failed' | 'call.in-progress.error-vapifault-hume-voice-failed' | 'call.in-progress.error-vapifault-sesame-voice-failed' + | 'call.in-progress.error-vapifault-inworld-voice-failed' | 'call.in-progress.error-vapifault-tavus-video-failed' | 'pipeline-error-vapi-llm-failed' | 'pipeline-error-vapi-400-bad-request-validation-failed' @@ -21759,6 +23628,7 @@ export interface ServerMessageStatusUpdate { | 'pipeline-error-deepgram-returning-500-invalid-json' | 'pipeline-error-deepgram-returning-502-network-error' | 'pipeline-error-deepgram-returning-502-bad-gateway-ehostunreach' + | 'pipeline-error-deepgram-returning-econnreset' | 'call.in-progress.error-vapifault-deepgram-returning-400-no-such-model-language-tier-combination' | 'call.in-progress.error-vapifault-deepgram-returning-401-invalid-credentials' | 'call.in-progress.error-vapifault-deepgram-returning-404-not-found' @@ -22283,6 +24153,8 @@ export interface ServerMessageResponseToolCalls { export interface ServerMessageResponseTransferDestinationRequest { /** This is the destination you'd like the call to be transferred to. */ destination?: TransferDestinationAssistant | TransferDestinationNumber | TransferDestinationSip; + /** This is the message that will be spoken to the user as the tool is running. */ + message?: ToolMessageStart | ToolMessageComplete | ToolMessageFailed | ToolMessageDelayed; /** This is the error message if the transfer should not be made. */ error?: string; } @@ -23384,6 +25256,18 @@ export class Api extends HttpClient extends HttpClient`. + * @maxLength 40 + */ + name?: string; /** * This is the page number to return. Defaults to 1. * @min 1 @@ -23503,6 +25428,47 @@ export class Api extends HttpClient`. + * @maxLength 40 + */ + name?: string; /** * This is the page number to return. Defaults to 1. * @min 1 @@ -23601,6 +25567,47 @@ export class Api extends HttpClient`. + * @maxLength 40 + */ + name?: string; /** * This is the page number to return. Defaults to 1. * @min 1 @@ -23984,6 +25991,230 @@ export class Api extends HttpClient + this.request({ + path: `/campaign`, + method: 'POST', + body: data, + secure: true, + type: ContentType.Json, + format: 'json', + ...params, + }), + + /** + * No description + * + * @tags Campaigns + * @name CampaignControllerFindAll + * @summary List Campaigns + * @request GET:/campaign + * @secure + */ + campaignControllerFindAll: ( + query?: { + /** + * This is the maximum number of items to return. Defaults to 100. + * @min 0 + * @max 1000 + */ + limit?: number; + /** + * This will return items where the createdAt is greater than the specified value. + * @format date-time + */ + createdAtGt?: string; + /** + * This will return items where the createdAt is less than the specified value. + * @format date-time + */ + createdAtLt?: string; + /** + * This will return items where the createdAt is greater than or equal to the specified value. + * @format date-time + */ + createdAtGe?: string; + /** + * This will return items where the createdAt is less than or equal to the specified value. + * @format date-time + */ + createdAtLe?: string; + /** + * This will return items where the updatedAt is greater than the specified value. + * @format date-time + */ + updatedAtGt?: string; + /** + * This will return items where the updatedAt is less than the specified value. + * @format date-time + */ + updatedAtLt?: string; + /** + * This will return items where the updatedAt is greater than or equal to the specified value. + * @format date-time + */ + updatedAtGe?: string; + /** + * This will return items where the updatedAt is less than or equal to the specified value. + * @format date-time + */ + updatedAtLe?: string; + }, + params: RequestParams = {}, + ) => + this.request({ + path: `/campaign`, + method: 'GET', + query: query, + secure: true, + format: 'json', + ...params, + }), + + /** + * No description + * + * @tags Campaigns + * @name CampaignControllerFindAllPaginated + * @summary List Campaigns with pagination + * @request GET:/campaign/paginated + * @secure + */ + campaignControllerFindAllPaginated: ( + query?: { + id?: string; + status?: 'scheduled' | 'in-progress' | 'ended'; + /** + * This is the page number to return. Defaults to 1. + * @min 1 + */ + page?: number; + /** This is the sort order for pagination. Defaults to 'DESC'. */ + sortOrder?: 'ASC' | 'DESC'; + /** + * This is the maximum number of items to return. Defaults to 100. + * @min 0 + * @max 1000 + */ + limit?: number; + /** + * This will return items where the createdAt is greater than the specified value. + * @format date-time + */ + createdAtGt?: string; + /** + * This will return items where the createdAt is less than the specified value. + * @format date-time + */ + createdAtLt?: string; + /** + * This will return items where the createdAt is greater than or equal to the specified value. + * @format date-time + */ + createdAtGe?: string; + /** + * This will return items where the createdAt is less than or equal to the specified value. + * @format date-time + */ + createdAtLe?: string; + /** + * This will return items where the updatedAt is greater than the specified value. + * @format date-time + */ + updatedAtGt?: string; + /** + * This will return items where the updatedAt is less than the specified value. + * @format date-time + */ + updatedAtLt?: string; + /** + * This will return items where the updatedAt is greater than or equal to the specified value. + * @format date-time + */ + updatedAtGe?: string; + /** + * This will return items where the updatedAt is less than or equal to the specified value. + * @format date-time + */ + updatedAtLe?: string; + }, + params: RequestParams = {}, + ) => + this.request({ + path: `/campaign/paginated`, + method: 'GET', + query: query, + secure: true, + format: 'json', + ...params, + }), + + /** + * No description + * + * @tags Campaigns + * @name CampaignControllerFindOne + * @summary Get Campaign + * @request GET:/campaign/{id} + * @secure + */ + campaignControllerFindOne: (id: string, params: RequestParams = {}) => + this.request({ + path: `/campaign/${id}`, + method: 'GET', + secure: true, + format: 'json', + ...params, + }), + + /** + * No description + * + * @tags Campaigns + * @name CampaignControllerUpdate + * @summary Update Campaign + * @request PATCH:/campaign/{id} + * @secure + */ + campaignControllerUpdate: (id: string, data: UpdateCampaignDTO, params: RequestParams = {}) => + this.request({ + path: `/campaign/${id}`, + method: 'PATCH', + body: data, + secure: true, + type: ContentType.Json, + format: 'json', + ...params, + }), + + /** + * No description + * + * @tags Campaigns + * @name CampaignControllerRemove + * @summary Delete Campaign + * @request DELETE:/campaign/{id} + * @secure + */ + campaignControllerRemove: (id: string, params: RequestParams = {}) => + this.request({ + path: `/campaign/${id}`, + method: 'DELETE', + secure: true, + format: 'json', + ...params, + }), + }; session = { /** * No description @@ -24143,6 +26374,27 @@ export class Api extends HttpClient + this.request({ + path: `/support/ticket`, + method: 'POST', + body: data, + secure: true, + type: ContentType.Json, + format: 'json', + ...params, + }), + }; assistant = { /** * No description @@ -24704,6 +26956,9 @@ export class Api extends HttpClient extends HttpClient this.request< + | ({ + type: 'apiRequest'; + } & ApiRequestTool) | ({ type: 'dtmf'; } & DtmfTool) @@ -24906,6 +27164,9 @@ export class Api extends HttpClient this.request< ( + | ({ + type: 'apiRequest'; + } & ApiRequestTool) | ({ type: 'dtmf'; } & DtmfTool) @@ -24991,6 +27252,9 @@ export class Api extends HttpClient this.request< + | ({ + type: 'apiRequest'; + } & ApiRequestTool) | ({ type: 'dtmf'; } & DtmfTool) @@ -25075,6 +27339,9 @@ export class Api extends HttpClient extends HttpClient this.request< + | ({ + type: 'apiRequest'; + } & ApiRequestTool) | ({ type: 'dtmf'; } & DtmfTool) @@ -25226,6 +27496,9 @@ export class Api extends HttpClient this.request< + | ({ + type: 'apiRequest'; + } & ApiRequestTool) | ({ type: 'dtmf'; } & DtmfTool) @@ -25715,6 +27988,29 @@ export class Api extends HttpClient + this.request({ + path: `/workflow/generate`, + method: 'POST', + body: data, + secure: true, + type: ContentType.Json, + format: 'json', + ...params, + }), }; squad = { /** @@ -27134,7 +29430,10 @@ export class Api extends HttpClient this.request< @@ -27287,7 +29586,10 @@ export class Api extends HttpClient({ path: `/credential`, @@ -27511,6 +29813,9 @@ export class Api extends HttpClient({ @@ -27682,7 +29987,10 @@ export class Api extends HttpClient({ path: `/credential/${id}`, @@ -27820,7 +30128,10 @@ export class Api extends HttpClient this.request< @@ -27973,7 +30284,10 @@ export class Api extends HttpClient({ path: `/credential/${id}`, @@ -28145,7 +30459,10 @@ export class Api extends HttpClient({ path: `/credential/${id}`, @@ -28411,7 +30728,8 @@ export class Api extends HttpClient extends HttpClient this.request({ @@ -28537,7 +30856,8 @@ export class Api extends HttpClient this.request({ diff --git a/package-lock.json b/package-lock.json index 941096a0e..f56a5b248 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@vapi-ai/web", - "version": "2.3.6", + "version": "2.3.7", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@vapi-ai/web", - "version": "2.3.6", + "version": "2.3.7", "license": "MIT", "dependencies": { "@daily-co/daily-js": "^0.79.0", diff --git a/package.json b/package.json index dfb6ea4c8..7d08f20a1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@vapi-ai/web", - "version": "2.3.6", + "version": "2.3.7", "description": "", "main": "dist/vapi.js", "types": "dist/vapi.d.ts", From 5f5e891de5f4d9839795af5cf2a952ee54da4c91 Mon Sep 17 00:00:00 2001 From: Adithya Sairamachandran Date: Wed, 25 Jun 2025 20:30:05 -0700 Subject: [PATCH 3/3] emit call-start-progress, call-start-success, and call-start-failed for more observability into Daily --- vapi.ts | 409 +++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 374 insertions(+), 35 deletions(-) diff --git a/vapi.ts b/vapi.ts index 437366311..285363c81 100644 --- a/vapi.ts +++ b/vapi.ts @@ -58,7 +58,33 @@ type VapiEventNames = | 'camera-error' | 'network-quality-change' | 'network-connection' - | 'daily-participant-updated'; + | 'daily-participant-updated' + | 'call-start-progress' + | 'call-start-success' + | 'call-start-failed'; + +interface CallStartProgressEvent { + stage: string; + status: 'started' | 'completed' | 'failed'; + duration?: number; + timestamp: string; + metadata?: Record; +} + +interface CallStartSuccessEvent { + totalDuration: number; + callId?: string; + timestamp: string; +} + +interface CallStartFailedEvent { + stage: string; + totalDuration: number; + error: string; + errorStack?: string; + timestamp: string; + context: Record; +} type VapiEventListeners = { 'call-end': () => void; @@ -73,6 +99,9 @@ type VapiEventListeners = { 'network-quality-change': (event: any) => void; 'network-connection': (event: any) => void; 'daily-participant-updated': (participant: DailyParticipant) => void; + 'call-start-progress': (event: CallStartProgressEvent) => void; + 'call-start-success': (event: CallStartSuccessEvent) => void; + 'call-start-failed': (event: CallStartFailedEvent) => void; }; async function startAudioPlayer( @@ -209,16 +238,53 @@ export default class Vapi extends VapiEventEmitter { workflow?: CreateWorkflowDTO | string, workflowOverrides?: WorkflowOverrides, ): Promise { + const startTime = Date.now(); + + // Input validation with detailed error messages if (!assistant && !squad && !workflow) { - throw new Error('Assistant or Squad or Workflow must be provided.'); + const error = new Error('Assistant or Squad or Workflow must be provided.'); + this.emit('error', { + type: 'validation-error', + stage: 'input-validation', + message: error.message, + timestamp: new Date().toISOString() + }); + throw error; } if (this.started) { + this.emit('call-start-progress', { + stage: 'validation', + status: 'failed', + timestamp: new Date().toISOString(), + metadata: { reason: 'already-started' } + }); return null; } + + this.emit('call-start-progress', { + stage: 'initialization', + status: 'started', + timestamp: new Date().toISOString(), + metadata: { + hasAssistant: !!assistant, + hasSquad: !!squad, + hasWorkflow: !!workflow + } + }); + this.started = true; try { + // Stage 1: Create web call + this.emit('call-start-progress', { + stage: 'web-call-creation', + status: 'started', + timestamp: new Date().toISOString() + }); + + const webCallStartTime = Date.now(); + const webCall = ( await client.call.callControllerCreateWebCall({ assistant: typeof assistant === 'string' ? undefined : assistant, @@ -231,8 +297,27 @@ export default class Vapi extends VapiEventEmitter { workflowOverrides, }) ).data; + + const webCallDuration = Date.now() - webCallStartTime; + this.emit('call-start-progress', { + stage: 'web-call-creation', + status: 'completed', + duration: webCallDuration, + timestamp: new Date().toISOString(), + metadata: { + callId: webCall?.id || 'unknown', + videoRecordingEnabled: webCall?.artifactPlan?.videoRecordingEnabled ?? false, + voiceProvider: webCall?.assistant?.voice?.provider || 'unknown' + } + }); if (this.call) { + this.emit('call-start-progress', { + stage: 'daily-call-object-creation', + status: 'started', + timestamp: new Date().toISOString(), + metadata: { action: 'cleanup-existing' } + }); this.cleanup(); } @@ -241,11 +326,53 @@ export default class Vapi extends VapiEventEmitter { const isVideoEnabled = webCall?.assistant?.voice?.provider === 'tavus'; - this.call = DailyIframe.createCallObject({ - audioSource: this.dailyCallObject.audioSource ?? true, - videoSource: this.dailyCallObject.videoSource ?? isVideoRecordingEnabled, - dailyConfig: this.dailyCallConfig, + // Stage 2: Create Daily call object + this.emit('call-start-progress', { + stage: 'daily-call-object-creation', + status: 'started', + timestamp: new Date().toISOString(), + metadata: { + audioSource: this.dailyCallObject.audioSource ?? true, + videoSource: this.dailyCallObject.videoSource ?? isVideoRecordingEnabled, + isVideoRecordingEnabled, + isVideoEnabled + } }); + + const dailyCallStartTime = Date.now(); + + try { + this.call = DailyIframe.createCallObject({ + audioSource: this.dailyCallObject.audioSource ?? true, + videoSource: this.dailyCallObject.videoSource ?? isVideoRecordingEnabled, + dailyConfig: this.dailyCallConfig, + }); + + const dailyCallDuration = Date.now() - dailyCallStartTime; + this.emit('call-start-progress', { + stage: 'daily-call-object-creation', + status: 'completed', + duration: dailyCallDuration, + timestamp: new Date().toISOString() + }); + } catch (error) { + const dailyCallDuration = Date.now() - dailyCallStartTime; + this.emit('call-start-progress', { + stage: 'daily-call-object-creation', + status: 'failed', + duration: dailyCallDuration, + timestamp: new Date().toISOString(), + metadata: { error: error?.toString() } + }); + this.emit('error', { + type: 'daily-call-object-creation-error', + stage: 'daily-call-object-creation', + error, + timestamp: new Date().toISOString() + }); + throw error; + } + this.call.iframe()?.style.setProperty('display', 'none'); this.call.on('left-meeting', () => { @@ -326,41 +453,182 @@ export default class Vapi extends VapiEventEmitter { destroyAudioPlayer(e.participant.session_id); }); - // Allow mobile devices to finish processing the microphone permissions - // request before joining the call and playing the assistant's audio - if (this.isMobileDevice()) { + // Stage 3: Mobile device handling and permissions + const isMobile = this.isMobileDevice(); + this.emit('call-start-progress', { + stage: 'mobile-permissions', + status: 'started', + timestamp: new Date().toISOString(), + metadata: { isMobile } + }); + + if (isMobile) { + const mobileWaitStartTime = Date.now(); await this.sleep(1000); + const mobileWaitDuration = Date.now() - mobileWaitStartTime; + this.emit('call-start-progress', { + stage: 'mobile-permissions', + status: 'completed', + duration: mobileWaitDuration, + timestamp: new Date().toISOString(), + metadata: { action: 'permissions-wait' } + }); + } else { + this.emit('call-start-progress', { + stage: 'mobile-permissions', + status: 'completed', + timestamp: new Date().toISOString(), + metadata: { action: 'skipped-not-mobile' } + }); } - await this.call.join({ - // @ts-expect-error This exists - url: webCall.webCallUrl, - subscribeToTracksAutomatically: false, + // Stage 4: Join the call + this.emit('call-start-progress', { + stage: 'daily-call-join', + status: 'started', + timestamp: new Date().toISOString() }); + + const joinStartTime = Date.now(); + + try { + await this.call.join({ + // @ts-expect-error This exists + url: webCall.webCallUrl, + subscribeToTracksAutomatically: false, + }); + + const joinDuration = Date.now() - joinStartTime; + this.emit('call-start-progress', { + stage: 'daily-call-join', + status: 'completed', + duration: joinDuration, + timestamp: new Date().toISOString() + }); + } catch (error) { + const joinDuration = Date.now() - joinStartTime; + this.emit('call-start-progress', { + stage: 'daily-call-join', + status: 'failed', + duration: joinDuration, + timestamp: new Date().toISOString(), + metadata: { error: error?.toString() } + }); + this.emit('error', { + type: 'daily-call-join-error', + stage: 'daily-call-join', + error, + duration: joinDuration, + timestamp: new Date().toISOString() + }); + throw error; + } + // Stage 5: Video recording setup (if enabled) if (isVideoRecordingEnabled) { + this.emit('call-start-progress', { + stage: 'video-recording-setup', + status: 'started', + timestamp: new Date().toISOString() + }); + const recordingRequestedTime = new Date().getTime(); + const recordingStartTime = Date.now(); - this.call.startRecording({ - width: 1280, - height: 720, - backgroundColor: '#FF1F2D3D', - layout: { - preset: 'default', - }, - }); + try { + this.call.startRecording({ + width: 1280, + height: 720, + backgroundColor: '#FF1F2D3D', + layout: { + preset: 'default', + }, + }); - this.call.on('recording-started', () => { - this.send({ - type: 'control', - control: 'say-first-message', - videoRecordingStartDelaySeconds: - (new Date().getTime() - recordingRequestedTime) / 1000, + const recordingSetupDuration = Date.now() - recordingStartTime; + this.emit('call-start-progress', { + stage: 'video-recording-setup', + status: 'completed', + duration: recordingSetupDuration, + timestamp: new Date().toISOString() + }); + + this.call.on('recording-started', () => { + const totalRecordingDelay = (new Date().getTime() - recordingRequestedTime) / 1000; + this.emit('call-start-progress', { + stage: 'video-recording-started', + status: 'completed', + timestamp: new Date().toISOString(), + metadata: { delaySeconds: totalRecordingDelay } + }); + + this.send({ + type: 'control', + control: 'say-first-message', + videoRecordingStartDelaySeconds: totalRecordingDelay, + }); + }); + } catch (error) { + const recordingSetupDuration = Date.now() - recordingStartTime; + this.emit('call-start-progress', { + stage: 'video-recording-setup', + status: 'failed', + duration: recordingSetupDuration, + timestamp: new Date().toISOString(), + metadata: { error: error?.toString() } + }); + this.emit('error', { + type: 'video-recording-setup-error', + stage: 'video-recording-setup', + error, + timestamp: new Date().toISOString() }); + // Don't throw here, video recording is optional + } + } else { + this.emit('call-start-progress', { + stage: 'video-recording-setup', + status: 'completed', + timestamp: new Date().toISOString(), + metadata: { action: 'skipped-not-enabled' } }); } - this.call.startRemoteParticipantsAudioLevelObserver(100); + // Stage 6: Audio level observer setup + this.emit('call-start-progress', { + stage: 'audio-observer-setup', + status: 'started', + timestamp: new Date().toISOString() + }); + + const audioObserverStartTime = Date.now(); + + try { + this.call.startRemoteParticipantsAudioLevelObserver(100); + const audioObserverDuration = Date.now() - audioObserverStartTime; + this.emit('call-start-progress', { + stage: 'audio-observer-setup', + status: 'completed', + duration: audioObserverDuration, + timestamp: new Date().toISOString() + }); + } catch (error) { + const audioObserverDuration = Date.now() - audioObserverStartTime; + this.emit('call-start-progress', { + stage: 'audio-observer-setup', + status: 'failed', + duration: audioObserverDuration, + timestamp: new Date().toISOString(), + metadata: { error: error?.toString() } + }); + this.emit('error', { + type: 'audio-observer-setup-error', + stage: 'audio-observer-setup', + error, + timestamp: new Date().toISOString() + }); + // Don't throw here, this is non-critical + } this.call.on('remote-participants-audio-level', (e) => { if (e) this.handleRemoteParticipantsAudioLevel(e); @@ -385,18 +653,89 @@ export default class Vapi extends VapiEventEmitter { } }); - this.call.updateInputSettings({ - audio: { - processor: { - type: 'noise-cancellation', + // Stage 7: Audio processing setup + this.emit('call-start-progress', { + stage: 'audio-processing-setup', + status: 'started', + timestamp: new Date().toISOString() + }); + + const audioProcessingStartTime = Date.now(); + + try { + this.call.updateInputSettings({ + audio: { + processor: { + type: 'noise-cancellation', + }, }, - }, + }); + + const audioProcessingDuration = Date.now() - audioProcessingStartTime; + this.emit('call-start-progress', { + stage: 'audio-processing-setup', + status: 'completed', + duration: audioProcessingDuration, + timestamp: new Date().toISOString() + }); + } catch (error) { + const audioProcessingDuration = Date.now() - audioProcessingStartTime; + this.emit('call-start-progress', { + stage: 'audio-processing-setup', + status: 'failed', + duration: audioProcessingDuration, + timestamp: new Date().toISOString(), + metadata: { error: error?.toString() } + }); + this.emit('error', { + type: 'audio-processing-setup-error', + stage: 'audio-processing-setup', + error, + timestamp: new Date().toISOString() + }); + // Don't throw here, this is non-critical + } + + const totalDuration = Date.now() - startTime; + this.emit('call-start-success', { + totalDuration, + callId: webCall?.id || 'unknown', + timestamp: new Date().toISOString() }); return webCall; } catch (e) { - console.error(e); - this.emit('error', e); + const totalDuration = Date.now() - startTime; + + this.emit('call-start-failed', { + stage: 'unknown', + totalDuration, + error: e?.toString() || 'Unknown error occurred', + errorStack: e instanceof Error ? e.stack : 'No stack trace available', + timestamp: new Date().toISOString(), + context: { + hasAssistant: !!assistant, + hasSquad: !!squad, + hasWorkflow: !!workflow, + isMobile: this.isMobileDevice() + } + }); + + // Also emit the generic error event for backward compatibility + this.emit('error', { + type: 'start-method-error', + stage: 'unknown', + error: e, + totalDuration, + timestamp: new Date().toISOString(), + context: { + hasAssistant: !!assistant, + hasSquad: !!squad, + hasWorkflow: !!workflow, + isMobile: this.isMobileDevice() + } + }); + this.cleanup(); return null; }