|
| 1 | +"""Voice I/O for `assembly code`: speak your request, hear the reply. |
| 2 | +
|
| 3 | +The coding agent's default interactive mode (a TTY) captures one spoken turn via |
| 4 | +streaming STT and reads each assistant reply back via streaming TTS. Both legs are |
| 5 | +injected so the loop is unit-tested with fakes — no microphone, speaker, or socket. |
| 6 | +
|
| 7 | +Readback needs streaming TTS, which only the sandbox environment exposes |
| 8 | +(`tts.session.is_available`); in production, voice *input* still works and replies |
| 9 | +stay on screen as text. Microphone (STT) input works in every environment. |
| 10 | +""" |
| 11 | + |
| 12 | +from __future__ import annotations |
| 13 | + |
| 14 | +import threading |
| 15 | +from collections.abc import Callable, Iterable, Iterator |
| 16 | +from dataclasses import dataclass |
| 17 | +from typing import TYPE_CHECKING, Protocol |
| 18 | + |
| 19 | +from aai_cli.core import client, config_builder |
| 20 | +from aai_cli.core.microphone import MicrophoneSource |
| 21 | +from aai_cli.tts import session as tts_session |
| 22 | +from aai_cli.tts.audio import PcmPlayer |
| 23 | +from aai_cli.tts.session import SpeakConfig |
| 24 | + |
| 25 | +if TYPE_CHECKING: |
| 26 | + from assemblyai.streaming.v3 import StreamingParameters |
| 27 | + |
| 28 | +# The audio-device CLIError types listen() raises when no usable microphone is present; |
| 29 | +# the command degrades to typed input on these (see _exec._voice_read_line). They mirror |
| 30 | +# the error_type values core.microphone attaches to its mic-open failures. |
| 31 | +AUDIO_ERROR_TYPES = frozenset({"mic_missing", "mic_error", "audio_input_error"}) |
| 32 | + |
| 33 | +# Streaming TTS synthesizes at 24 kHz, the rate the readback player is opened at. |
| 34 | +_TTS_SAMPLE_RATE = 24000 |
| 35 | + |
| 36 | +# The streaming STT model used to transcribe a spoken turn — the same realtime default |
| 37 | +# `assembly stream` and `assembly agent-cascade` use. |
| 38 | +_SPEECH_MODEL = "u3-rt-pro" |
| 39 | + |
| 40 | + |
| 41 | +class Microphone(Protocol): |
| 42 | + """The microphone slice the listen loop drives: an iterable of PCM at a known rate.""" |
| 43 | + |
| 44 | + sample_rate: int |
| 45 | + |
| 46 | + def __iter__(self) -> Iterator[bytes]: |
| 47 | + """Yield captured PCM16 chunks until the stream ends.""" |
| 48 | + |
| 49 | + |
| 50 | +class StreamFn(Protocol): |
| 51 | + """The streaming-STT call: ``client.stream_audio`` satisfies it structurally.""" |
| 52 | + |
| 53 | + def __call__( |
| 54 | + self, |
| 55 | + api_key: str, |
| 56 | + source: Iterable[bytes], |
| 57 | + *, |
| 58 | + params: StreamingParameters, |
| 59 | + on_turn: Callable[[object], None], |
| 60 | + ) -> None: |
| 61 | + """Stream ``source`` and forward each Turn event to ``on_turn``.""" |
| 62 | + |
| 63 | + |
| 64 | +class SynthFn(Protocol): |
| 65 | + """The streaming-TTS call: ``tts.session.synthesize`` satisfies it structurally. |
| 66 | +
|
| 67 | + The return is typed ``object`` because the readback path discards it (it plays each |
| 68 | + chunk through ``on_audio`` as it arrives), which also lets a test inject a fake that |
| 69 | + returns nothing meaningful. |
| 70 | + """ |
| 71 | + |
| 72 | + def __call__( |
| 73 | + self, |
| 74 | + api_key: str, |
| 75 | + config: SpeakConfig, |
| 76 | + *, |
| 77 | + on_audio: Callable[[bytes, int], None], |
| 78 | + ) -> object: |
| 79 | + """Synthesize ``config.text``, handing each PCM chunk to ``on_audio``.""" |
| 80 | + |
| 81 | + |
| 82 | +class Player(Protocol): |
| 83 | + """The readback player: a context manager that ``feed``s PCM chunks (PcmPlayer).""" |
| 84 | + |
| 85 | + def __enter__(self) -> Player: |
| 86 | + """Enter the playback context (opens the device lazily on first feed).""" |
| 87 | + |
| 88 | + def __exit__(self, exc_type: object, *exc: object) -> object: |
| 89 | + """Drain on a clean exit, abort otherwise; never suppress.""" |
| 90 | + |
| 91 | + def feed(self, pcm: bytes, sample_rate: int) -> None: |
| 92 | + """Play one PCM chunk, opening the output device on the first call.""" |
| 93 | + |
| 94 | + |
| 95 | +def _stt_params(sample_rate: int) -> StreamingParameters: |
| 96 | + """StreamingParameters for capturing one spoken turn at ``sample_rate``. |
| 97 | +
|
| 98 | + ``format_turns`` is on so the finalized turn reads like a typed prompt (punctuated |
| 99 | + and cased) rather than raw lowercase tokens. |
| 100 | + """ |
| 101 | + merged = config_builder.merge_streaming_params( |
| 102 | + flags={"speech_model": _SPEECH_MODEL, "format_turns": True, "sample_rate": sample_rate} |
| 103 | + ) |
| 104 | + return config_builder.construct_streaming_params(merged) |
| 105 | + |
| 106 | + |
| 107 | +@dataclass |
| 108 | +class VoiceSession: |
| 109 | + """Speak-to-it / read-it-back I/O for one coding session, with injectable legs.""" |
| 110 | + |
| 111 | + api_key: str |
| 112 | + readback: bool |
| 113 | + mic_factory: Callable[[], Microphone] = MicrophoneSource |
| 114 | + stream_fn: StreamFn = client.stream_audio |
| 115 | + synth_fn: SynthFn = tts_session.synthesize |
| 116 | + player_factory: Callable[[], Player] = PcmPlayer |
| 117 | + |
| 118 | + def listen(self) -> str | None: |
| 119 | + """Capture one spoken turn and return its finalized transcript. |
| 120 | +
|
| 121 | + Returns the text of the first end-of-turn the server finalizes, or ``None`` when |
| 122 | + the microphone stream ends without one (EOF — e.g. a finite source in tests). The |
| 123 | + microphone is gated shut the moment a turn finalizes, so exactly one utterance is |
| 124 | + captured per call; a real mic blocks until you speak (Ctrl-C to quit). |
| 125 | + """ |
| 126 | + mic = self.mic_factory() |
| 127 | + done = threading.Event() |
| 128 | + captured: list[str] = [] |
| 129 | + |
| 130 | + def on_turn(event: object) -> None: |
| 131 | + text = (getattr(event, "transcript", "") or "").strip() |
| 132 | + if text and getattr(event, "end_of_turn", False): |
| 133 | + captured.append(text) |
| 134 | + done.set() |
| 135 | + |
| 136 | + def gated() -> Iterator[bytes]: |
| 137 | + for chunk in mic: |
| 138 | + if done.is_set(): |
| 139 | + return |
| 140 | + yield chunk |
| 141 | + |
| 142 | + self.stream_fn(self.api_key, gated(), params=_stt_params(mic.sample_rate), on_turn=on_turn) |
| 143 | + return " ".join(captured).strip() or None |
| 144 | + |
| 145 | + def speak(self, text: str) -> None: |
| 146 | + """Read ``text`` back via streaming TTS, when readback is available. |
| 147 | +
|
| 148 | + A no-op when readback is off (production, where streaming TTS has no host) or the |
| 149 | + text is blank — so the caller can route every assistant reply here unconditionally. |
| 150 | + """ |
| 151 | + text = text.strip() |
| 152 | + if not self.readback or not text: |
| 153 | + return |
| 154 | + config = SpeakConfig(text=text, sample_rate=_TTS_SAMPLE_RATE) |
| 155 | + with self.player_factory() as player: |
| 156 | + self.synth_fn(self.api_key, config, on_audio=player.feed) |
| 157 | + |
| 158 | + |
| 159 | +def build_voice_session(api_key: str) -> VoiceSession: |
| 160 | + """A voice session for the active environment. |
| 161 | +
|
| 162 | + Readback is enabled only where streaming TTS is available (the sandbox); microphone |
| 163 | + input is wired regardless. |
| 164 | + """ |
| 165 | + return VoiceSession(api_key=api_key, readback=tts_session.is_available()) |
0 commit comments