Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 43 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -239,15 +239,55 @@ uv run python -m orion # or: uv run orion

## Client (Rust)

The terminal client lives in `client/` and is built with Cargo:
The terminal client lives in `client/` and is built with Cargo. Start the
runtime first (it opens the IPC socket the client connects to), then:

```bash
cd client
cargo run
```

It is currently a minimal placeholder; the Ratatui UI and the IPC bridge to the
runtime (see `runtime/src/orion/transport`) are upcoming.
The client is a Ratatui TUI that talks to the runtime over `/tmp/orion.sock`.
It renders the conversation, a live event stream, and Copilot-style activity
logs, with tachyonfx animations.

### Keybindings

| Key | Action |
|-----|--------|
| `i` | enter insert mode (type a prompt) |
| `Enter` | send the typed prompt (insert mode) |
| `Esc` | back to normal mode |
| `v` | push-to-talk: press to start recording, press again to send |
| `s` | stop the assistant's speech |
| `j`/`k`, arrows, `PgUp`/`PgDn` | scroll the conversation |
| `q` | quit |

### Voice

Voice is handled entirely by the client; the runtime never touches audio
hardware:

1. Press `v` to record from your microphone, `v` again to stop.
2. The client saves a WAV and sends its path to the runtime over IPC.
3. The runtime transcribes it (Groq Whisper) and runs the normal chat pipeline.
4. The response streams back and the client speaks it aloud.

Text-to-speech uses your system speech engine via **speech-dispatcher**. Install
it to hear responses (otherwise TTS is silently skipped):

```bash
sudo pacman -S speech-dispatcher # Arch / EndeavourOS
# Debian/Ubuntu: sudo apt-get install -y speech-dispatcher
```

Building the client also needs the ALSA development headers for microphone
capture (`cpal`):

```bash
sudo pacman -S alsa-lib # Arch / EndeavourOS
# Debian/Ubuntu: sudo apt-get install -y libasound2-dev
```

## Tests

Expand Down
17 changes: 17 additions & 0 deletions client/src/app.rs
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,23 @@ impl App {
.push("DISCONNECTED", EventStatus::Failed, "runtime disconnected");
}

RuntimeEvent::UserPrompt(text) => {
let text = text.trim().to_string();
// Ignore the echo of a prompt we already showed locally (typed);
// display it when it's new — i.e. a transcribed voice message.
if !text.is_empty()
&& self.conversation.last_user_text().as_deref() != Some(text.as_str())
{
self.msg_counter += 1;
self.conversation.add_message(Message::new(
format!("msg-{}", self.msg_counter),
Author::User,
text,
));
self.effects.on_message();
}
}

RuntimeEvent::AssistantStart => {
self.mode = "RESPONDING".into();
self.msg_counter += 1;
Expand Down
60 changes: 36 additions & 24 deletions client/src/audio.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,37 @@ use std::process::Command;
use std::sync::{Arc, Mutex};

use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
use cpal::{FromSample, Sample, SizedSample};

/// Default location for the recorded clip (shared with the runtime, same host).
pub fn recording_path() -> PathBuf {
std::env::temp_dir().join("orion_client_recording.wav")
}

/// Build an input stream for sample type `T`, converting each sample to i16.
fn build_input<T>(
device: &cpal::Device,
config: &cpal::StreamConfig,
sink: Arc<Mutex<Vec<i16>>>,
) -> Result<cpal::Stream, String>
where
T: SizedSample,
i16: FromSample<T>,
{
device
.build_input_stream(
config.clone(),
move |data: &[T], _: &cpal::InputCallbackInfo| {
if let Ok(mut buf) = sink.lock() {
buf.extend(data.iter().map(|&s| i16::from_sample(s)));
}
},
|err| eprintln!("audio input error: {err}"),
None,
)
.map_err(|e| e.to_string())
}

/// An in-progress microphone recording. Dropping/finishing it stops capture.
pub struct Recorder {
stream: cpal::Stream,
Expand All @@ -42,33 +67,20 @@ impl Recorder {
let config: cpal::StreamConfig = supported.into();

let samples = Arc::new(Mutex::new(Vec::<i16>::new()));
let sink = samples.clone();
let err_fn = |err| eprintln!("audio input error: {err}");

// Accept any input sample format the device offers, converting each
// sample to i16 for the WAV.
let stream = match sample_format {
cpal::SampleFormat::I16 => device.build_input_stream(
config,
move |data: &[i16], _: &cpal::InputCallbackInfo| {
if let Ok(mut buf) = sink.lock() {
buf.extend_from_slice(data);
}
},
err_fn,
None,
),
cpal::SampleFormat::F32 => device.build_input_stream(
config,
move |data: &[f32], _: &cpal::InputCallbackInfo| {
if let Ok(mut buf) = sink.lock() {
buf.extend(data.iter().map(|s| (s.clamp(-1.0, 1.0) * 32767.0) as i16));
}
},
err_fn,
None,
),
cpal::SampleFormat::I8 => build_input::<i8>(&device, &config, samples.clone()),
cpal::SampleFormat::I16 => build_input::<i16>(&device, &config, samples.clone()),
cpal::SampleFormat::I32 => build_input::<i32>(&device, &config, samples.clone()),
cpal::SampleFormat::U8 => build_input::<u8>(&device, &config, samples.clone()),
cpal::SampleFormat::U16 => build_input::<u16>(&device, &config, samples.clone()),
cpal::SampleFormat::U32 => build_input::<u32>(&device, &config, samples.clone()),
cpal::SampleFormat::F32 => build_input::<f32>(&device, &config, samples.clone()),
cpal::SampleFormat::F64 => build_input::<f64>(&device, &config, samples.clone()),
other => return Err(format!("unsupported sample format: {other:?}")),
}
.map_err(|e| e.to_string())?;
}?;

stream.play().map_err(|e| e.to_string())?;

Expand Down
12 changes: 11 additions & 1 deletion client/src/ipc/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@ use crate::ipc::{
client_session::ClientSession,
error::IpcError,
events::RuntimeEvent,
messages::{AssistantChunkPayload, Envelope, ErrorPayload, MessageType, StatusPayload},
messages::{
AssistantChunkPayload, Envelope, ErrorPayload, MessageType, StatusPayload,
SubmitPromptPayload,
},
};

/// High-level IPC client used by the TUI.
Expand Down Expand Up @@ -66,6 +69,13 @@ impl OrionClient {

MessageType::AssistantEnd => RuntimeEvent::AssistantEnd,

// The runtime echoes the pipeline's user text back — a typed prompt
// or a transcribed voice message. Shown as the user's turn.
MessageType::SubmitPrompt => {
let payload: SubmitPromptPayload = envelope.payload()?;
RuntimeEvent::UserPrompt(payload.text)
}

MessageType::Status => {
let payload: StatusPayload = envelope.payload()?;
RuntimeEvent::Status(payload.message)
Expand Down
15 changes: 11 additions & 4 deletions client/src/ipc/client_session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ pub struct ClientSession {
id: Uuid,
reader: BufReader<OwnedReadHalf>,
writer: BufWriter<OwnedWriteHalf>,
/// Persistent line buffer so a `receive()` cancelled inside a `select!`
/// resumes instead of losing partially-read bytes (cancel-safety).
read_buf: String,
}

impl ClientSession {
Expand All @@ -31,6 +34,7 @@ impl ClientSession {
id: Uuid::new_v4(),
reader: BufReader::new(reader),
writer: BufWriter::new(writer),
read_buf: String::new(),
})
}

Expand All @@ -48,14 +52,17 @@ impl ClientSession {
}

pub async fn receive(&mut self) -> Result<Envelope, IpcError> {
let mut line = String::new();
// Append into the persistent buffer. If this future is dropped by a
// `select!` (e.g. a UI tick wins), the bytes read so far remain in
// `read_buf`, and the next call resumes reading the same line — so no
// message is corrupted or lost.
let bytes = self.reader.read_line(&mut self.read_buf).await?;

let bytes = self.reader.read_line(&mut line).await?;

if bytes == 0 {
if bytes == 0 && self.read_buf.is_empty() {
return Err(IpcError::Disconnected);
}

let line = std::mem::take(&mut self.read_buf);
decode(&line)
}

Expand Down
23 changes: 19 additions & 4 deletions client/src/ipc/events.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,26 +6,41 @@ pub enum RuntimeEvent {
Connected,
Disconnected,

/// The user's turn text echoed by the runtime — a typed prompt or a
/// transcribed voice message.
UserPrompt(String),

// Assistant
AssistantStart,
AssistantChunk(String),
AssistantEnd,

// Tools
ToolStarted { name: String },
ToolFinished { name: String, success: bool },
ToolStarted {
name: String,
},
ToolFinished {
name: String,
success: bool,
},

// Runtime
Status(String),
Error { code: String, message: String },
Error {
code: String,
message: String,
},

// Connection
Ping,
Pong,

// Voice (future)
VoiceStart,
VoiceChunk { sequence: u64, bytes: Vec<u8> },
VoiceChunk {
sequence: u64,
bytes: Vec<u8>,
},
VoiceEnd,

// Unknown / unsupported
Expand Down
9 changes: 9 additions & 0 deletions client/src/widgets/conversation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,15 @@ impl ConversationWidget {
self.scroll_offset = 0;
}

/// Text of the most recent user message, if any.
pub fn last_user_text(&self) -> Option<String> {
self.messages
.iter()
.rev()
.find(|m| m.author == Author::User)
.map(|m| m.content.clone())
}

/// Text of the most recent assistant message (for TTS), or empty.
pub fn last_assistant_text(&self) -> String {
self.messages
Expand Down
5 changes: 4 additions & 1 deletion client/src/widgets/status.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,10 @@ impl StatusWidget {
sep.clone(),
// 6. Keybindings info depending on mode
if *input_mode == InputMode::Normal {
Span::styled("i insert · q quit", Style::default().fg(DIM))
Span::styled(
"i insert · v talk · s stop · q quit",
Style::default().fg(DIM),
)
} else {
Span::styled("esc normal · enter send", Style::default().fg(DIM))
},
Expand Down
6 changes: 4 additions & 2 deletions runtime/src/orion/transport/bridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
SubmitPromptPayload,
VoiceEndPayload,
)
from orion.services.transcript_generation import is_junk_transcript
from orion.transport.session import ClientSession
from orion.transport.transcription import transcribe_audio

Expand Down Expand Up @@ -171,8 +172,9 @@ async def _handle_voice_end(

transcript = (await self._transcriber(payload.path)).strip()

# Ignore empty transcriptions (silence / failed capture).
if not transcript:
# Ignore empty or hallucinated transcriptions (silence / noise) so a
# cough doesn't trigger a full agent turn.
if not transcript or is_junk_transcript(transcript):
return

await self._event_bus.publish(
Expand Down
Loading