diff --git a/README.md b/README.md index 6bb7083..23d02a7 100644 --- a/README.md +++ b/README.md @@ -239,15 +239,55 @@ uv run python -m orion # or: uv run orion ## Client (Rust) -The terminal client lives in `client/` and is built with Cargo: +The terminal client lives in `client/` and is built with Cargo. Start the +runtime first (it opens the IPC socket the client connects to), then: ```bash cd client cargo run ``` -It is currently a minimal placeholder; the Ratatui UI and the IPC bridge to the -runtime (see `runtime/src/orion/transport`) are upcoming. +The client is a Ratatui TUI that talks to the runtime over `/tmp/orion.sock`. +It renders the conversation, a live event stream, and Copilot-style activity +logs, with tachyonfx animations. + +### Keybindings + +| Key | Action | +|-----|--------| +| `i` | enter insert mode (type a prompt) | +| `Enter` | send the typed prompt (insert mode) | +| `Esc` | back to normal mode | +| `v` | push-to-talk: press to start recording, press again to send | +| `s` | stop the assistant's speech | +| `j`/`k`, arrows, `PgUp`/`PgDn` | scroll the conversation | +| `q` | quit | + +### Voice + +Voice is handled entirely by the client; the runtime never touches audio +hardware: + +1. Press `v` to record from your microphone, `v` again to stop. +2. The client saves a WAV and sends its path to the runtime over IPC. +3. The runtime transcribes it (Groq Whisper) and runs the normal chat pipeline. +4. The response streams back and the client speaks it aloud. + +Text-to-speech uses your system speech engine via **speech-dispatcher**. Install +it to hear responses (otherwise TTS is silently skipped): + +```bash +sudo pacman -S speech-dispatcher # Arch / EndeavourOS +# Debian/Ubuntu: sudo apt-get install -y speech-dispatcher +``` + +Building the client also needs the ALSA development headers for microphone +capture (`cpal`): + +```bash +sudo pacman -S alsa-lib # Arch / EndeavourOS +# Debian/Ubuntu: sudo apt-get install -y libasound2-dev +``` ## Tests diff --git a/client/src/app.rs b/client/src/app.rs index 8380062..02940cf 100644 --- a/client/src/app.rs +++ b/client/src/app.rs @@ -237,6 +237,23 @@ impl App { .push("DISCONNECTED", EventStatus::Failed, "runtime disconnected"); } + RuntimeEvent::UserPrompt(text) => { + let text = text.trim().to_string(); + // Ignore the echo of a prompt we already showed locally (typed); + // display it when it's new — i.e. a transcribed voice message. + if !text.is_empty() + && self.conversation.last_user_text().as_deref() != Some(text.as_str()) + { + self.msg_counter += 1; + self.conversation.add_message(Message::new( + format!("msg-{}", self.msg_counter), + Author::User, + text, + )); + self.effects.on_message(); + } + } + RuntimeEvent::AssistantStart => { self.mode = "RESPONDING".into(); self.msg_counter += 1; diff --git a/client/src/audio.rs b/client/src/audio.rs index c078380..e4949a7 100644 --- a/client/src/audio.rs +++ b/client/src/audio.rs @@ -12,12 +12,37 @@ use std::process::Command; use std::sync::{Arc, Mutex}; use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; +use cpal::{FromSample, Sample, SizedSample}; /// Default location for the recorded clip (shared with the runtime, same host). pub fn recording_path() -> PathBuf { std::env::temp_dir().join("orion_client_recording.wav") } +/// Build an input stream for sample type `T`, converting each sample to i16. +fn build_input( + device: &cpal::Device, + config: &cpal::StreamConfig, + sink: Arc>>, +) -> Result +where + T: SizedSample, + i16: FromSample, +{ + device + .build_input_stream( + config.clone(), + move |data: &[T], _: &cpal::InputCallbackInfo| { + if let Ok(mut buf) = sink.lock() { + buf.extend(data.iter().map(|&s| i16::from_sample(s))); + } + }, + |err| eprintln!("audio input error: {err}"), + None, + ) + .map_err(|e| e.to_string()) +} + /// An in-progress microphone recording. Dropping/finishing it stops capture. pub struct Recorder { stream: cpal::Stream, @@ -42,33 +67,20 @@ impl Recorder { let config: cpal::StreamConfig = supported.into(); let samples = Arc::new(Mutex::new(Vec::::new())); - let sink = samples.clone(); - let err_fn = |err| eprintln!("audio input error: {err}"); + // Accept any input sample format the device offers, converting each + // sample to i16 for the WAV. let stream = match sample_format { - cpal::SampleFormat::I16 => device.build_input_stream( - config, - move |data: &[i16], _: &cpal::InputCallbackInfo| { - if let Ok(mut buf) = sink.lock() { - buf.extend_from_slice(data); - } - }, - err_fn, - None, - ), - cpal::SampleFormat::F32 => device.build_input_stream( - config, - move |data: &[f32], _: &cpal::InputCallbackInfo| { - if let Ok(mut buf) = sink.lock() { - buf.extend(data.iter().map(|s| (s.clamp(-1.0, 1.0) * 32767.0) as i16)); - } - }, - err_fn, - None, - ), + cpal::SampleFormat::I8 => build_input::(&device, &config, samples.clone()), + cpal::SampleFormat::I16 => build_input::(&device, &config, samples.clone()), + cpal::SampleFormat::I32 => build_input::(&device, &config, samples.clone()), + cpal::SampleFormat::U8 => build_input::(&device, &config, samples.clone()), + cpal::SampleFormat::U16 => build_input::(&device, &config, samples.clone()), + cpal::SampleFormat::U32 => build_input::(&device, &config, samples.clone()), + cpal::SampleFormat::F32 => build_input::(&device, &config, samples.clone()), + cpal::SampleFormat::F64 => build_input::(&device, &config, samples.clone()), other => return Err(format!("unsupported sample format: {other:?}")), - } - .map_err(|e| e.to_string())?; + }?; stream.play().map_err(|e| e.to_string())?; diff --git a/client/src/ipc/client.rs b/client/src/ipc/client.rs index eb5fc36..df1c915 100644 --- a/client/src/ipc/client.rs +++ b/client/src/ipc/client.rs @@ -4,7 +4,10 @@ use crate::ipc::{ client_session::ClientSession, error::IpcError, events::RuntimeEvent, - messages::{AssistantChunkPayload, Envelope, ErrorPayload, MessageType, StatusPayload}, + messages::{ + AssistantChunkPayload, Envelope, ErrorPayload, MessageType, StatusPayload, + SubmitPromptPayload, + }, }; /// High-level IPC client used by the TUI. @@ -66,6 +69,13 @@ impl OrionClient { MessageType::AssistantEnd => RuntimeEvent::AssistantEnd, + // The runtime echoes the pipeline's user text back — a typed prompt + // or a transcribed voice message. Shown as the user's turn. + MessageType::SubmitPrompt => { + let payload: SubmitPromptPayload = envelope.payload()?; + RuntimeEvent::UserPrompt(payload.text) + } + MessageType::Status => { let payload: StatusPayload = envelope.payload()?; RuntimeEvent::Status(payload.message) diff --git a/client/src/ipc/client_session.rs b/client/src/ipc/client_session.rs index 887587d..506c9c6 100644 --- a/client/src/ipc/client_session.rs +++ b/client/src/ipc/client_session.rs @@ -19,6 +19,9 @@ pub struct ClientSession { id: Uuid, reader: BufReader, writer: BufWriter, + /// Persistent line buffer so a `receive()` cancelled inside a `select!` + /// resumes instead of losing partially-read bytes (cancel-safety). + read_buf: String, } impl ClientSession { @@ -31,6 +34,7 @@ impl ClientSession { id: Uuid::new_v4(), reader: BufReader::new(reader), writer: BufWriter::new(writer), + read_buf: String::new(), }) } @@ -48,14 +52,17 @@ impl ClientSession { } pub async fn receive(&mut self) -> Result { - let mut line = String::new(); + // Append into the persistent buffer. If this future is dropped by a + // `select!` (e.g. a UI tick wins), the bytes read so far remain in + // `read_buf`, and the next call resumes reading the same line — so no + // message is corrupted or lost. + let bytes = self.reader.read_line(&mut self.read_buf).await?; - let bytes = self.reader.read_line(&mut line).await?; - - if bytes == 0 { + if bytes == 0 && self.read_buf.is_empty() { return Err(IpcError::Disconnected); } + let line = std::mem::take(&mut self.read_buf); decode(&line) } diff --git a/client/src/ipc/events.rs b/client/src/ipc/events.rs index b351c72..28168da 100644 --- a/client/src/ipc/events.rs +++ b/client/src/ipc/events.rs @@ -6,18 +6,30 @@ pub enum RuntimeEvent { Connected, Disconnected, + /// The user's turn text echoed by the runtime — a typed prompt or a + /// transcribed voice message. + UserPrompt(String), + // Assistant AssistantStart, AssistantChunk(String), AssistantEnd, // Tools - ToolStarted { name: String }, - ToolFinished { name: String, success: bool }, + ToolStarted { + name: String, + }, + ToolFinished { + name: String, + success: bool, + }, // Runtime Status(String), - Error { code: String, message: String }, + Error { + code: String, + message: String, + }, // Connection Ping, @@ -25,7 +37,10 @@ pub enum RuntimeEvent { // Voice (future) VoiceStart, - VoiceChunk { sequence: u64, bytes: Vec }, + VoiceChunk { + sequence: u64, + bytes: Vec, + }, VoiceEnd, // Unknown / unsupported diff --git a/client/src/widgets/conversation.rs b/client/src/widgets/conversation.rs index c0bb999..b2232cc 100644 --- a/client/src/widgets/conversation.rs +++ b/client/src/widgets/conversation.rs @@ -161,6 +161,15 @@ impl ConversationWidget { self.scroll_offset = 0; } + /// Text of the most recent user message, if any. + pub fn last_user_text(&self) -> Option { + self.messages + .iter() + .rev() + .find(|m| m.author == Author::User) + .map(|m| m.content.clone()) + } + /// Text of the most recent assistant message (for TTS), or empty. pub fn last_assistant_text(&self) -> String { self.messages diff --git a/client/src/widgets/status.rs b/client/src/widgets/status.rs index c2f67ea..c30c4db 100644 --- a/client/src/widgets/status.rs +++ b/client/src/widgets/status.rs @@ -78,7 +78,10 @@ impl StatusWidget { sep.clone(), // 6. Keybindings info depending on mode if *input_mode == InputMode::Normal { - Span::styled("i insert · q quit", Style::default().fg(DIM)) + Span::styled( + "i insert · v talk · s stop · q quit", + Style::default().fg(DIM), + ) } else { Span::styled("esc normal · enter send", Style::default().fg(DIM)) }, diff --git a/runtime/src/orion/transport/bridge.py b/runtime/src/orion/transport/bridge.py index 1485300..1a6002f 100644 --- a/runtime/src/orion/transport/bridge.py +++ b/runtime/src/orion/transport/bridge.py @@ -25,6 +25,7 @@ SubmitPromptPayload, VoiceEndPayload, ) +from orion.services.transcript_generation import is_junk_transcript from orion.transport.session import ClientSession from orion.transport.transcription import transcribe_audio @@ -171,8 +172,9 @@ async def _handle_voice_end( transcript = (await self._transcriber(payload.path)).strip() - # Ignore empty transcriptions (silence / failed capture). - if not transcript: + # Ignore empty or hallucinated transcriptions (silence / noise) so a + # cough doesn't trigger a full agent turn. + if not transcript or is_junk_transcript(transcript): return await self._event_bus.publish(