|
| 1 | +/** |
| 2 | + * HTTP-based text-to-speech over HTTP API (Python `HttpSpeechSynthesizer` parity). |
| 3 | + * Synced from dashscope-sdk-python v1.25.17 |
| 4 | + */ |
| 5 | +import BaseApi from '../../common/baseApi'; |
| 6 | +import { HTTP_STATUS_OK } from '../../common/consts'; |
| 7 | + |
| 8 | +/** Audio format options for HTTP speech synthesis. */ |
| 9 | +export type HttpAudioFormat = 'wav' | 'pcm' | 'mp3'; |
| 10 | + |
| 11 | +/** The result of HTTP speech synthesis. */ |
| 12 | +export class HttpSpeechSynthesisResult { |
| 13 | + private _audioData: Buffer | null; |
| 14 | + private _audioUrl: string | null; |
| 15 | + private _audioId: string | null; |
| 16 | + private _expiresAt: number | null; |
| 17 | + /** Sentence-level synthesis results (for streaming mode). */ |
| 18 | + private _sentences: Array<Record<string, unknown>>; |
| 19 | + private _response: HttpSpeechSynthesisResponse | null; |
| 20 | + |
| 21 | + constructor( |
| 22 | + audioData: Buffer | null = null, |
| 23 | + audioUrl: string | null = null, |
| 24 | + audioId: string | null = null, |
| 25 | + expiresAt: number | null = null, |
| 26 | + sentences: Array<Record<string, unknown>> = [], |
| 27 | + response: HttpSpeechSynthesisResponse | null = null, |
| 28 | + ) { |
| 29 | + this._audioData = audioData; |
| 30 | + this._audioUrl = audioUrl; |
| 31 | + this._audioId = audioId; |
| 32 | + this._expiresAt = expiresAt; |
| 33 | + this._sentences = sentences; |
| 34 | + this._response = response; |
| 35 | + } |
| 36 | + |
| 37 | + /** Get the audio data (for streaming mode). */ |
| 38 | + getAudioData(): Buffer | null { |
| 39 | + return this._audioData; |
| 40 | + } |
| 41 | + |
| 42 | + /** Get the audio URL (for non-streaming mode). */ |
| 43 | + getAudioUrl(): string | null { |
| 44 | + return this._audioUrl; |
| 45 | + } |
| 46 | + |
| 47 | + /** Get the audio ID. */ |
| 48 | + getAudioId(): string | null { |
| 49 | + return this._audioId; |
| 50 | + } |
| 51 | + |
| 52 | + /** Get the URL expiration timestamp. */ |
| 53 | + getExpiresAt(): number | null { |
| 54 | + return this._expiresAt; |
| 55 | + } |
| 56 | + |
| 57 | + /** Get the sentence-level synthesis results (for streaming mode). */ |
| 58 | + getSentences(): Array<Record<string, unknown>> { |
| 59 | + return this._sentences; |
| 60 | + } |
| 61 | + |
| 62 | + /** Get the full API response. */ |
| 63 | + getResponse(): HttpSpeechSynthesisResponse | null { |
| 64 | + return this._response; |
| 65 | + } |
| 66 | +} |
| 67 | + |
| 68 | +/** Raw HTTP synthesis envelope from the service. */ |
| 69 | +export interface HttpSpeechSynthesisResponse { |
| 70 | + request_id?: string; |
| 71 | + status_code?: number; |
| 72 | + code?: string; |
| 73 | + message?: string; |
| 74 | + output?: Record<string, unknown>; |
| 75 | + usage?: Record<string, unknown>; |
| 76 | +} |
| 77 | + |
| 78 | +/** Options for HTTP speech synthesis. */ |
| 79 | +export interface HttpSpeechSynthesisOptions { |
| 80 | + /** The speech synthesis model, e.g., 'cosyvoice-v3-flash'. */ |
| 81 | + model: string; |
| 82 | + /** The text to synthesize. */ |
| 83 | + text: string; |
| 84 | + /** The voice to use for synthesis. */ |
| 85 | + voice: string; |
| 86 | + /** Audio encoding format ('wav', 'pcm', 'mp3'). Defaults to 'wav'. */ |
| 87 | + audioFormat?: HttpAudioFormat; |
| 88 | + /** Audio sample rate in Hz. Defaults to 24000. */ |
| 89 | + sampleRate?: number; |
| 90 | + /** Whether to use streaming (SSE) mode. Defaults to false. */ |
| 91 | + stream?: boolean; |
| 92 | + /** Per-request workspace header override. */ |
| 93 | + workspace?: string; |
| 94 | + /** Per-request API key override. */ |
| 95 | + apiKey?: string; |
| 96 | + /** Custom HTTP URL if needed. */ |
| 97 | + url?: string; |
| 98 | + /** Additional parameters like volume, rate, pitch, etc. */ |
| 99 | + [key: string]: unknown; |
| 100 | +} |
| 101 | + |
| 102 | +/** SSE chunk from HTTP TTS streaming response. */ |
| 103 | +interface SseChunk { |
| 104 | + output?: { |
| 105 | + type?: string; |
| 106 | + sentence?: Record<string, unknown>; |
| 107 | + audio?: { |
| 108 | + data?: string; |
| 109 | + url?: string; |
| 110 | + id?: string; |
| 111 | + expires_at?: number; |
| 112 | + }; |
| 113 | + finish_reason?: string; |
| 114 | + }; |
| 115 | +} |
| 116 | + |
| 117 | +class HttpSpeechSynthesizer extends BaseApi { |
| 118 | + protected service = 'services/audio/tts/SpeechSynthesizer'; |
| 119 | + |
| 120 | + /** |
| 121 | + * Convert text to speech via HTTP API. |
| 122 | + * Supports both streaming (SSE) and non-streaming modes. |
| 123 | + */ |
| 124 | + async call(options: HttpSpeechSynthesisOptions): Promise<HttpSpeechSynthesisResult | AsyncGenerator<HttpSpeechSynthesisResult, void, unknown>> { |
| 125 | + const { |
| 126 | + model, |
| 127 | + text, |
| 128 | + voice, |
| 129 | + audioFormat = 'wav', |
| 130 | + sampleRate = 24000, |
| 131 | + stream = false, |
| 132 | + workspace, |
| 133 | + // Destructure and exclude from extraParams (handled by BaseApi.request) |
| 134 | + apiKey: _apiKey, |
| 135 | + url: _url, |
| 136 | + ...extraParams |
| 137 | + } = options; |
| 138 | + |
| 139 | + if (!model) throw new Error('model is required!'); |
| 140 | + if (!text) throw new Error('text is required!'); |
| 141 | + if (!voice) throw new Error('voice is required!'); |
| 142 | + |
| 143 | + // Build request body |
| 144 | + const body: Record<string, unknown> = { |
| 145 | + model, |
| 146 | + input: { |
| 147 | + text, |
| 148 | + voice, |
| 149 | + format: audioFormat, |
| 150 | + sample_rate: sampleRate, |
| 151 | + ...Object.fromEntries( |
| 152 | + Object.entries(extraParams).filter(([, v]) => v !== null && v !== undefined), |
| 153 | + ), |
| 154 | + }, |
| 155 | + }; |
| 156 | + |
| 157 | + // Prepare headers |
| 158 | + const headers: Record<string, string> = {}; |
| 159 | + if (stream) { |
| 160 | + headers['X-DashScope-SSE'] = 'enable'; |
| 161 | + } |
| 162 | + |
| 163 | + // Make the HTTP request |
| 164 | + const response = await this.request({ |
| 165 | + method: 'post', |
| 166 | + data: body, |
| 167 | + headers: Object.keys(headers).length > 0 ? headers : undefined, |
| 168 | + workspace, |
| 169 | + service: this.service, |
| 170 | + responseType: stream ? 'stream' : undefined, |
| 171 | + }); |
| 172 | + |
| 173 | + if (stream) { |
| 174 | + return this.handleStreamingResponse(response); |
| 175 | + } else { |
| 176 | + return this.handleNonStreamingResponse(response); |
| 177 | + } |
| 178 | + } |
| 179 | + |
| 180 | + private handleNonStreamingResponse(response: { status: number; data?: unknown }): HttpSpeechSynthesisResult { |
| 181 | + const output = this.extractOutput(response); |
| 182 | + const audioInfo = (output?.audio || {}) as Record<string, unknown>; |
| 183 | + |
| 184 | + return new HttpSpeechSynthesisResult( |
| 185 | + null, |
| 186 | + (audioInfo.url as string) || null, |
| 187 | + (audioInfo.id as string) || null, |
| 188 | + (audioInfo.expires_at as number) || null, |
| 189 | + [], |
| 190 | + response.data as HttpSpeechSynthesisResponse, |
| 191 | + ); |
| 192 | + } |
| 193 | + |
| 194 | + private async* handleStreamingResponse( |
| 195 | + response: { status: number; data?: unknown }, |
| 196 | + ): AsyncGenerator<HttpSpeechSynthesisResult, void, unknown> { |
| 197 | + const audioDataParts: Buffer[] = []; |
| 198 | + const sentences: Array<Record<string, unknown>> = []; |
| 199 | + |
| 200 | + // For streaming, data should be a readable stream |
| 201 | + const streamData = response.data; |
| 202 | + if (!streamData || typeof streamData !== 'object') { |
| 203 | + return; |
| 204 | + } |
| 205 | + |
| 206 | + // Handle different stream types (Node stream or AsyncIterable) |
| 207 | + const dataStream = streamData as AsyncIterable<Buffer>; |
| 208 | + |
| 209 | + let buffer = ''; |
| 210 | + try { |
| 211 | + for await (const chunk of dataStream) { |
| 212 | + const chunkStr = chunk.toString('utf-8'); |
| 213 | + buffer += chunkStr; |
| 214 | + |
| 215 | + // Process SSE events |
| 216 | + const events = buffer.split('\n\n'); |
| 217 | + buffer = events.pop() || ''; // Keep incomplete event in buffer |
| 218 | + |
| 219 | + for (const event of events) { |
| 220 | + const lines = event.split('\n'); |
| 221 | + let dataLine = ''; |
| 222 | + |
| 223 | + for (const line of lines) { |
| 224 | + if (line.startsWith('data:')) { |
| 225 | + dataLine = line.slice(5).trim(); |
| 226 | + } |
| 227 | + } |
| 228 | + |
| 229 | + if (!dataLine) continue; |
| 230 | + |
| 231 | + try { |
| 232 | + const parsed = JSON.parse(dataLine) as SseChunk; |
| 233 | + const output = parsed.output || {}; |
| 234 | + const outputType = output.type || ''; |
| 235 | + |
| 236 | + if (outputType.startsWith('sentence-')) { |
| 237 | + const sentenceInfo = output.sentence; |
| 238 | + if (sentenceInfo) { |
| 239 | + sentences.push(sentenceInfo); |
| 240 | + } |
| 241 | + |
| 242 | + const audioData = output.audio?.data; |
| 243 | + if (audioData) { |
| 244 | + const audioBytes = Buffer.from(audioData, 'base64'); |
| 245 | + audioDataParts.push(audioBytes); |
| 246 | + yield new HttpSpeechSynthesisResult( |
| 247 | + audioBytes, |
| 248 | + null, |
| 249 | + null, |
| 250 | + null, |
| 251 | + [...sentences], |
| 252 | + null, |
| 253 | + ); |
| 254 | + } |
| 255 | + } else if (output.finish_reason === 'stop') { |
| 256 | + yield this.createFinalResult(audioDataParts, sentences, output.audio || {}, parsed); |
| 257 | + } |
| 258 | + // eslint-disable-next-line @typescript-eslint/no-unused-vars |
| 259 | + } catch (_e) { |
| 260 | + // Skip malformed JSON |
| 261 | + } |
| 262 | + } |
| 263 | + } |
| 264 | + |
| 265 | + // Process any remaining data in buffer |
| 266 | + if (buffer.trim()) { |
| 267 | + const lines = buffer.split('\n'); |
| 268 | + let dataLine = ''; |
| 269 | + for (const line of lines) { |
| 270 | + if (line.startsWith('data:')) { |
| 271 | + dataLine = line.slice(5).trim(); |
| 272 | + } |
| 273 | + } |
| 274 | + if (dataLine) { |
| 275 | + try { |
| 276 | + const parsed = JSON.parse(dataLine) as SseChunk; |
| 277 | + const output = parsed.output || {}; |
| 278 | + if (output.finish_reason === 'stop') { |
| 279 | + yield this.createFinalResult(audioDataParts, sentences, output.audio || {}, parsed); |
| 280 | + } |
| 281 | + // eslint-disable-next-line @typescript-eslint/no-unused-vars |
| 282 | + } catch (_e) { |
| 283 | + // Skip malformed JSON |
| 284 | + } |
| 285 | + } |
| 286 | + } |
| 287 | + // eslint-disable-next-line @typescript-eslint/no-unused-vars |
| 288 | + } catch (_error) { |
| 289 | + // Stream error - yield final result with collected data |
| 290 | + yield new HttpSpeechSynthesisResult( |
| 291 | + audioDataParts.length > 0 ? Buffer.concat(audioDataParts) : null, |
| 292 | + null, |
| 293 | + null, |
| 294 | + null, |
| 295 | + sentences, |
| 296 | + null, |
| 297 | + ); |
| 298 | + } |
| 299 | + } |
| 300 | + |
| 301 | + private extractOutput(response: { status: number; data?: unknown }): Record<string, unknown> { |
| 302 | + if (response.status !== HTTP_STATUS_OK) { |
| 303 | + const data = response.data as { status_code?: number; code?: string; message?: string } | undefined; |
| 304 | + throw new Error( |
| 305 | + `Request failed: ${data?.status_code || response.status} ${data?.code || ''} ${data?.message || ''}`, |
| 306 | + ); |
| 307 | + } |
| 308 | + const data = response.data as { output?: Record<string, unknown> } | undefined; |
| 309 | + return data?.output || {}; |
| 310 | + } |
| 311 | + |
| 312 | + /** |
| 313 | + * Create a final HttpSpeechSynthesisResult from collected data. |
| 314 | + * Used for both regular finish events and remaining buffer processing. |
| 315 | + */ |
| 316 | + private createFinalResult( |
| 317 | + audioDataParts: Buffer[], |
| 318 | + sentences: Array<Record<string, unknown>>, |
| 319 | + audioInfo: Record<string, unknown>, |
| 320 | + parsed: SseChunk, |
| 321 | + ): HttpSpeechSynthesisResult { |
| 322 | + return new HttpSpeechSynthesisResult( |
| 323 | + audioDataParts.length > 0 ? Buffer.concat(audioDataParts) : null, |
| 324 | + (audioInfo.url as string) || null, |
| 325 | + (audioInfo.id as string) || null, |
| 326 | + (audioInfo.expires_at as number) || null, |
| 327 | + [...sentences], |
| 328 | + parsed as unknown as HttpSpeechSynthesisResponse, |
| 329 | + ); |
| 330 | + } |
| 331 | +} |
| 332 | + |
| 333 | +export default HttpSpeechSynthesizer; |
0 commit comments