feat(voice): end-to-end voice pipeline with IPC integration

.github/workflows/pytest.yml

@@ -77,5 +77,11 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v4
 
+      - name: Install audio build dependencies
+        run: sudo apt-get update && sudo apt-get install -y libasound2-dev
+
       - name: Build client
         run: cargo build --verbose
+
+      - name: Run client tests
+        run: cargo test --verbose

client/Cargo.toml

@@ -14,3 +14,5 @@ serde_json = "1"
 thiserror = "1"
 futures-util = "0.3"
 tachyonfx = "=0.16.0"
+cpal = "0.18.1"
+hound = "3.5.1"

client/src/app.rs

@@ -1,6 +1,7 @@
 use crossterm::event::{MouseButton, MouseEvent, MouseEventKind};
 use ratatui::layout::{Position, Rect};
 
+use crate::audio::{Recorder, Speaker, recording_path};
 use crate::effects::Effects;
 use crate::ipc::{client::OrionClient, events::RuntimeEvent};
 use crate::theme;
@@ -27,6 +28,10 @@ pub struct App {
     // Client
     pub client: Option<OrionClient>,
 
+    // Audio — the client owns all audio hardware
+    pub recorder: Option<Recorder>,
+    pub speaker: Speaker,
+
     // Store layout bounds for mouse click target checks
     pub prompt_area: Rect,
     pub events_area: Rect,
@@ -51,6 +56,8 @@ impl App {
             cursor_position: 0,
             msg_counter: 0,
             client: None,
+            recorder: None,
+            speaker: Speaker::new(),
             prompt_area: Rect::default(),
             events_area: Rect::default(),
             events: EventLog::new(),
@@ -156,6 +163,69 @@ impl App {
         self.mode = "THINKING".into();
     }
 
+    // --- Voice (push-to-talk) ------------------------------------------
+
+    /// Toggle recording: press once to start, again to stop and send.
+    pub async fn toggle_recording(&mut self) {
+        if self.recorder.is_some() {
+            self.stop_recording().await;
+        } else {
+            self.start_recording().await;
+        }
+    }
+
+    async fn start_recording(&mut self) {
+        self.speaker.stop(); // a new recording interrupts any speech
+
+        match Recorder::start(recording_path()) {
+            Ok(recorder) => {
+                let sample_rate = recorder.sample_rate();
+                let channels = recorder.channels() as u32;
+                self.recorder = Some(recorder);
+                self.mode = "RECORDING".into();
+                self.effects.on_status_change(theme::DANGER);
+                self.events
+                    .push("VOICE_START", EventStatus::Running, "recording");
+
+                if let Some(client) = &mut self.client {
+                    let _ = client.send_voice_start(sample_rate, channels).await;
+                }
+            }
+            Err(err) => {
+                self.mode = format!("MIC ERROR: {}", err);
+                self.events.push("VOICE_START", EventStatus::Failed, err);
+            }
+        }
+    }
+
+    async fn stop_recording(&mut self) {
+        let Some(recorder) = self.recorder.take() else {
+            return;
+        };
+
+        match recorder.finish() {
+            Ok(path) => {
+                let path = path.to_string_lossy().to_string();
+                self.mode = "TRANSCRIBING".into();
+                self.events
+                    .push("VOICE_END", EventStatus::Completed, path.clone());
+
+                if let Some(client) = &mut self.client {
+                    let _ = client.send_voice_end(path).await;
+                }
+            }
+            Err(err) => {
+                self.mode = format!("REC ERROR: {}", err);
+                self.events.push("VOICE_END", EventStatus::Failed, err);
+            }
+        }
+    }
+
+    /// Interrupt any assistant speech currently playing.
+    pub fn interrupt_speech(&self) {
+        self.speaker.stop();
+    }
+
     pub fn handle_runtime_event(&mut self, event: RuntimeEvent) {
         match event {
             RuntimeEvent::Connected => self.on_connected(),
@@ -191,6 +261,8 @@ impl App {
                 self.conversation.finish_assistant_message();
                 self.events
                     .bump_last("RESPONSE", EventStatus::Completed, "response complete");
+                // Client-side TTS: speak the completed response.
+                self.speaker.speak(&self.conversation.last_assistant_text());
             }
 
             RuntimeEvent::ToolStarted { name } => {

client/src/audio.rs

@@ -0,0 +1,153 @@
+//! Client-side audio: microphone capture and speech output.
+//!
+//! Per the architecture, the *client* owns all audio hardware. [`Recorder`]
+//! captures the microphone to a 16-bit PCM WAV (via `cpal` + `hound`); the app
+//! sends that file's path to the runtime, which transcribes it. [`Speaker`]
+//! speaks the assistant's reply through the system speech engine by shelling
+//! out to `spd-say` (speech-dispatcher) — no build-time audio library required,
+//! and it degrades to silence if speech-dispatcher isn't installed.
+
+use std::path::PathBuf;
+use std::process::Command;
+use std::sync::{Arc, Mutex};
+
+use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
+
+/// Default location for the recorded clip (shared with the runtime, same host).
+pub fn recording_path() -> PathBuf {
+    std::env::temp_dir().join("orion_client_recording.wav")
+}
+
+/// An in-progress microphone recording. Dropping/finishing it stops capture.
+pub struct Recorder {
+    stream: cpal::Stream,
+    samples: Arc<Mutex<Vec<i16>>>,
+    sample_rate: u32,
+    channels: u16,
+    path: PathBuf,
+}
+
+impl Recorder {
+    /// Open the default input device and start capturing into memory.
+    pub fn start(path: PathBuf) -> Result<Self, String> {
+        let host = cpal::default_host();
+        let device = host
+            .default_input_device()
+            .ok_or_else(|| "no input device".to_string())?;
+        let supported = device.default_input_config().map_err(|e| e.to_string())?;
+
+        let sample_rate = supported.sample_rate();
+        let channels = supported.channels();
+        let sample_format = supported.sample_format();
+        let config: cpal::StreamConfig = supported.into();
+
+        let samples = Arc::new(Mutex::new(Vec::<i16>::new()));
+        let sink = samples.clone();
+        let err_fn = |err| eprintln!("audio input error: {err}");
+
+        let stream = match sample_format {
+            cpal::SampleFormat::I16 => device.build_input_stream(
+                config,
+                move |data: &[i16], _: &cpal::InputCallbackInfo| {
+                    if let Ok(mut buf) = sink.lock() {
+                        buf.extend_from_slice(data);
+                    }
+                },
+                err_fn,
+                None,
+            ),
+            cpal::SampleFormat::F32 => device.build_input_stream(
+                config,
+                move |data: &[f32], _: &cpal::InputCallbackInfo| {
+                    if let Ok(mut buf) = sink.lock() {
+                        buf.extend(data.iter().map(|s| (s.clamp(-1.0, 1.0) * 32767.0) as i16));
+                    }
+                },
+                err_fn,
+                None,
+            ),
+            other => return Err(format!("unsupported sample format: {other:?}")),
+        }
+        .map_err(|e| e.to_string())?;
+
+        stream.play().map_err(|e| e.to_string())?;
+
+        Ok(Self {
+            stream,
+            samples,
+            sample_rate,
+            channels,
+            path,
+        })
+    }
+
+    pub fn sample_rate(&self) -> u32 {
+        self.sample_rate
+    }
+
+    pub fn channels(&self) -> u16 {
+        self.channels
+    }
+
+    /// Stop capture, write the WAV, and return its path.
+    pub fn finish(self) -> Result<PathBuf, String> {
+        let Recorder {
+            stream,
+            samples,
+            sample_rate,
+            channels,
+            path,
+        } = self;
+
+        drop(stream); // stop the input stream
+
+        let samples = samples
+            .lock()
+            .map_err(|_| "audio buffer poisoned".to_string())?;
+
+        let spec = hound::WavSpec {
+            channels,
+            sample_rate,
+            bits_per_sample: 16,
+            sample_format: hound::SampleFormat::Int,
+        };
+
+        let mut writer = hound::WavWriter::create(&path, spec).map_err(|e| e.to_string())?;
+        for &sample in samples.iter() {
+            writer.write_sample(sample).map_err(|e| e.to_string())?;
+        }
+        writer.finalize().map_err(|e| e.to_string())?;
+
+        Ok(path)
+    }
+}
+
+/// Speaks assistant responses through speech-dispatcher's `spd-say` CLI.
+pub struct Speaker;
+
+impl Speaker {
+    pub fn new() -> Self {
+        Self
+    }
+
+    /// Speak `text`, cancelling anything currently being spoken first.
+    pub fn speak(&self, text: &str) {
+        let text = text.trim();
+        if text.is_empty() {
+            return;
+        }
+        // `-C` cancels current speech, so a new reply interrupts an old one.
+        let _ = Command::new("spd-say").arg("-C").arg(text).spawn();
+    }
+
+    /// Stop any in-progress speech.
+    pub fn stop(&self) {
+        let _ = Command::new("spd-say").arg("--cancel").spawn();
+    }
+}
+
+impl Default for Speaker {
+    fn default() -> Self {
+        Self::new()
+    }
+}

client/src/ipc/client.rs

@@ -31,6 +31,22 @@ impl OrionClient {
         self.session.send(&envelope).await
     }
 
+    /// Announce the start of a voice recording (metadata only).
+    pub async fn send_voice_start(
+        &mut self,
+        sample_rate: u32,
+        channels: u32,
+    ) -> Result<(), IpcError> {
+        self.session
+            .send(&Envelope::voice_start(sample_rate, channels))
+            .await
+    }
+
+    /// Finish a recording by sending the runtime the recorded file path.
+    pub async fn send_voice_end(&mut self, path: impl Into<String>) -> Result<(), IpcError> {
+        self.session.send(&Envelope::voice_end(path)).await
+    }
+
     /// Send a ping message.
     pub async fn ping(&mut self) -> Result<(), IpcError> {
         self.session.send(&Envelope::ping()).await

client/src/ipc/messages.rs

@@ -90,6 +90,33 @@ impl Envelope {
         }
     }
 
+    /// Announce the start of a voice recording session (metadata only).
+    pub fn voice_start(sample_rate: u32, channels: u32) -> Self {
+        Self {
+            version: 1,
+            id: Uuid::new_v4(),
+            correlation_id: Uuid::new_v4(),
+            message_type: MessageType::VoiceStart,
+            payload: serde_json::to_value(VoiceStartPayload {
+                sample_rate,
+                channels,
+                encoding: "pcm16".to_string(),
+            })
+            .unwrap(),
+        }
+    }
+
+    /// Finish a recording by handing the runtime the recorded file path.
+    pub fn voice_end(path: impl Into<String>) -> Self {
+        Self {
+            version: 1,
+            id: Uuid::new_v4(),
+            correlation_id: Uuid::new_v4(),
+            message_type: MessageType::VoiceEnd,
+            payload: serde_json::to_value(VoiceEndPayload { path: path.into() }).unwrap(),
+        }
+    }
+
     /// Deserialize the payload into a strongly typed struct.
     pub fn payload<T>(&self) -> serde_json::Result<T>
     where
@@ -126,8 +153,11 @@ pub struct VoiceChunkPayload {
     pub data: Vec<u8>,
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize, Default)]
-pub struct VoiceEndPayload;
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct VoiceEndPayload {
+    /// Path to the recorded audio file the runtime should transcribe.
+    pub path: String,
+}
 
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct AssistantStartPayload;

client/src/main.rs

@@ -1,4 +1,5 @@
 mod app;
+mod audio;
 mod effects;
 mod ipc;
 mod theme;
@@ -24,7 +25,9 @@ const SOCKET_PATH: &str = "/tmp/orion.sock";
 // ~30 FPS so tachyonfx effects animate smoothly instead of stepping.
 const TICK_RATE: Duration = Duration::from_millis(33);
 
-#[tokio::main]
+// Single-threaded runtime: the audio input stream is `!Send`, so the app
+// state (which owns it) must stay on one thread.
+#[tokio::main(flavor = "current_thread")]
 async fn main() -> Result<(), Box<dyn Error>> {
     // Setup terminal
     enable_raw_mode()?;
@@ -87,6 +90,14 @@ async fn main() -> Result<(), Box<dyn Error>> {
                                         KeyCode::Char('q') | KeyCode::Esc => {
                                             app.should_quit = true;
                                         }
+                                        // Push-to-talk: toggle voice recording
+                                        KeyCode::Char('v') => {
+                                            app.toggle_recording().await;
+                                        }
+                                        // Interrupt assistant speech
+                                        KeyCode::Char('s') => {
+                                            app.interrupt_speech();
+                                        }
                                         // Conversation Scrolling (Vim keys & standard navigation)
                                         KeyCode::Up | KeyCode::Char('k') => {
                                             app.conversation.scroll_up(1);

client/src/widgets/conversation.rs

@@ -161,6 +161,16 @@ impl ConversationWidget {
         self.scroll_offset = 0;
     }
 
+    /// Text of the most recent assistant message (for TTS), or empty.
+    pub fn last_assistant_text(&self) -> String {
+        self.messages
+            .iter()
+            .rev()
+            .find(|m| m.author == Author::Orion)
+            .map(|m| m.content.clone())
+            .unwrap_or_default()
+    }
+
     pub fn render(&mut self, frame: &mut Frame, area: Rect) {
         // Outer Panel Block
         let outer_block = Block::default()