diff --git a/apps/effectcraft/src/audio_out.rs b/apps/effectcraft/src/audio_out.rs index b4bdebb395..8f61d6d9ac 100644 --- a/apps/effectcraft/src/audio_out.rs +++ b/apps/effectcraft/src/audio_out.rs @@ -22,6 +22,14 @@ fn preferred_buffer_frames(rate: u32) -> u32 { (rate / 100).clamp(256, 2048) } +/// The native output can retain samples beyond cpal's callback timestamp estimate +/// (Windows loopback measured a truncated tail with a reported 20 ms latency). +/// Allow at least 100 ms at natural completion; this does not delay explicit Stop +/// or change the A/V sync clock. Larger reported latencies keep their full allowance. +fn natural_tail_frames(rate: u32, latency: u64) -> u64 { + latency.max(u64::from(rate).div_ceil(10)) +} + /// Wait until the feeder has mixed a block, or until preview is stopped. Starting the stream /// before that fills the device with silence, so the first real samples are heard late (#554). fn wait_to_play(feed: &AudioFeed, stop: &mpsc::Receiver<()>) -> bool { @@ -211,6 +219,10 @@ impl AudioDevice for CpalOut { fn latency_frames(&self) -> u64 { self.latency.load(Ordering::Relaxed) } + + fn drain_frames(&self) -> u64 { + natural_tail_frames(self.rate, self.latency_frames()) + } } impl Drop for CpalOut { @@ -230,6 +242,13 @@ mod tests { assert_eq!(preferred_buffer_frames(192_000), 1920); } + #[test] + fn natural_drain_allows_output_processing_beyond_the_sync_clock() { + assert_eq!(natural_tail_frames(48_000, 960), 4800); + assert_eq!(natural_tail_frames(48_000, 9600), 9600); + assert_eq!(natural_tail_frames(44_100, 0), 4410); + } + #[test] fn wait_to_play_returns_once_the_feeder_has_samples() { let feed = Arc::new(AudioFeed::default()); diff --git a/crates/engine/src/commands/keys_more.rs b/crates/engine/src/commands/keys_more.rs index c7785d08c7..f76c7d65e9 100644 --- a/crates/engine/src/commands/keys_more.rs +++ b/crates/engine/src/commands/keys_more.rs @@ -80,7 +80,8 @@ fn audio_to_keys(s: &mut Session, p: &Value) -> Result { let mut rows = vec![]; for f in f0..f1 { let t = fr.tick_of(f); - let buf = effectcraft_render::audio::mix_comp(&s.project, s.footage.as_ref(), s.expr.as_deref(), cid, t, per, RATE); + let buf = effectcraft_render::audio::mix_comp(&s.project, s.footage.as_ref(), s.expr.as_deref(), cid, t, per, RATE) + .map_err(|e| EngineError::Other(e.to_string()))?; rows.push((t, amplitudes(&buf))); } let spec = effectcraft_effects::lookup("ec.control.slider").ok_or_else(|| EngineError::Other("Slider Control is missing".into()))?; diff --git a/crates/engine/src/commands/render_queue.rs b/crates/engine/src/commands/render_queue.rs index 09a7c7d2f3..022a20af17 100644 --- a/crates/engine/src/commands/render_queue.rs +++ b/crates/engine/src/commands/render_queue.rs @@ -796,6 +796,7 @@ fn save_current_preview(s: &mut Session, p: &Value) -> Result { item.output = OutputModule::for_format(format); item.output.output = path.clone(); let job = crate::ExportJob { + cancel: None, project: &s.project, footage: s.footage.as_ref(), expr: s.expr.as_deref(), diff --git a/crates/engine/src/commands/timeline.rs b/crates/engine/src/commands/timeline.rs index 633e73c5af..8a3cce433c 100644 --- a/crates/engine/src/commands/timeline.rs +++ b/crates/engine/src/commands/timeline.rs @@ -115,6 +115,7 @@ fn prerender(s: &Session, cid: ItemId, lid: LayerId, path: &str) -> std::result: item.output.audio = AudioOutput::Off; item.output.output = path.to_string(); let job = crate::ExportJob { + cancel: None, project: &q, footage: s.footage.as_ref(), expr: s.expr.as_deref(), diff --git a/crates/engine/src/render_queue.rs b/crates/engine/src/render_queue.rs index 74b3331515..cc4181d408 100644 --- a/crates/engine/src/render_queue.rs +++ b/crates/engine/src/render_queue.rs @@ -19,6 +19,8 @@ use crate::{Event, Session}; /// One export handed to the [`Exporter`]. pub struct ExportJob<'a> { + /// Shared cancellation token, including work inside an audio block. + pub cancel: Option>, pub project: &'a Project, pub footage: &'a dyn FootageSource, pub expr: Option<&'a dyn ExprHost>, @@ -104,7 +106,7 @@ pub enum ItemUpdate { #[derive(Default)] pub struct JobShared { - pub cancel: AtomicBool, + pub cancel: Arc, pub state: Mutex, pub updates: Mutex>, } @@ -272,6 +274,7 @@ async fn run_work(w: Work, shared: &JobShared, notify: &mut dyn FnMut(&JobShared let name = w.project.item(mitem.comp).map(|i| i.name.clone()).unwrap_or_default(); let index = w.project.render_queue.iter().position(|i| i.id == mitem.id).map_or(0, |i| i + 1); let job = ExportJob { + cancel: Some(shared.cancel.clone()), project: &w.project, footage: w.footage.as_ref(), expr: w.expr.as_deref(), diff --git a/crates/export/src/encode.rs b/crates/export/src/encode.rs index e4a45561d9..bd51da3bb2 100644 --- a/crates/export/src/encode.rs +++ b/crates/export/src/encode.rs @@ -215,9 +215,33 @@ fn deinterleave(buf: &[f32], channels: usize) -> Vec> { /// The comp's audio over `n` samples from `start`, interleaved with the module's channel count /// (mono = the average of left and right). -pub(crate) fn mix(cx: &Cx, start: effectcraft_time::Tick, n: usize, sr: u32) -> Vec { - let st = effectcraft_render::audio::mix_comp(&cx.project, cx.footage, cx.expr, cx.comp, start, n, sr); - if cx.output.audio_channels == 1 { st.as_chunks::<2>().0.iter().map(|p| (p[0] + p[1]) * 0.5).collect() } else { st } +pub(crate) fn mix(cx: &Cx, start: effectcraft_time::Tick, n: usize, sr: u32) -> Result> { + let control = effectcraft_render::audio::MixControl { cancel: cx.options.cancel.clone(), ..Default::default() }; + // Video batches can span many seconds at low frame rates. Keep each mixer request bounded, + // retaining sample-aligned ticks even at rates that do not divide the tick clock (44.1 kHz). + const BLOCK: usize = 16_384; + let channels = if cx.output.audio_channels == 1 { 1 } else { 2 }; + let len = n.checked_mul(channels).ok_or_else(|| enc("audio output is too large"))?; + let mut out = Vec::new(); + out.try_reserve_exact(len).map_err(|_| enc("audio output allocation failed"))?; + for done in (0..n).step_by(BLOCK) { + let offset = i64::try_from(done).map_err(|_| enc("audio sample offset is too large"))?; + let sample = start.to_units_floor(i64::from(sr)).checked_add(offset).ok_or_else(|| enc("audio sample offset overflow"))?; + let tick = Tick::from_units(sample, i64::from(sr)); + let tick = if tick.to_units_floor(i64::from(sr)) < sample { Tick(tick.0.saturating_add(1)) } else { tick }; + let st = effectcraft_render::audio::mix_comp_with_control(&cx.project, cx.footage, cx.expr, cx.comp, tick, BLOCK.min(n - done), sr, &control).map_err( + |e| match e { + effectcraft_render::audio::AudioError::Cancelled => ExportError::Cancelled, + _ => ExportError::Encode(e.to_string()), + }, + )?; + if channels == 1 { + out.extend(st.as_chunks::<2>().0.iter().map(|p| (p[0] + p[1]) * 0.5)); + } else { + out.extend(st); + } + } + Ok(out) } /// PCM bytes of samples in `fmt` (little or big endian). @@ -334,7 +358,7 @@ pub(crate) async fn movie(job: &Cx<'_>, comp: &Comp, w: u32, h: u32, st: &mut St let n = (end - *cursor) as usize; let start = Tick::from_units(*cursor, i64::from(sr)); let start = if start.to_units_floor(i64::from(sr)) < *cursor { Tick(start.0.saturating_add(1)) } else { start }; - let buf = mix(job, start, n, sr); + let buf = mix(job, start, n, sr)?; *cursor = end; match a { Audio::Aac(e) => { @@ -384,6 +408,85 @@ pub(crate) async fn movie(job: &Cx<'_>, comp: &Comp, w: u32, h: u32, st: &mut St mod tests { use super::*; + /// Chunked mixing preserves every sample at non-dividing rates, stereo/mono, and cancellation. + #[test] + fn large_audio_mix_preserves_samples_and_cancellation() { + use effectcraft_project::{Footage, ItemId, ItemKind, LayerSource, Project, build}; + use effectcraft_render::{FootageSource, Image}; + use std::sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }; + struct Ramp { + reads: AtomicUsize, + cancel: Option>, + } + impl FootageSource for Ramp { + fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option> { + None + } + fn audio(&self, _: ItemId, _: &Footage, start: Tick, frames: usize, rate: u32) -> Option> { + if self.reads.fetch_add(1, Ordering::Relaxed) == 1 + && let Some(c) = &self.cancel + { + c.store(true, Ordering::Relaxed); + } + let first = start.to_units_floor(i64::from(rate)); + Some( + (0..frames) + .flat_map(|i| { + let x = ((first + i as i64).rem_euclid(4096) as f32) / 4096.0; + [x, x * 0.5] + }) + .collect(), + ) + } + } + let mut p = Project::default(); + let mut comp = Comp::new(16, 16, FrameRate::new(1, 1), Tick::from_seconds_f64(100.0)); + let item = p.add_item("ramp", effectcraft_color::Label::SeaFoam, None, ItemKind::Footage(Footage { has_audio: true, ..Default::default() })); + comp.layers.push(build::layer(&mut p, &comp, "ramp", LayerSource::Footage { item }, (0, 0), None)); + let cid = p.add_item("audio", effectcraft_color::Label::Sandstone, None, ItemKind::Comp(comp.into())); + for rate in [44100, 192000] { + for channels in [1, 2] { + let mut output = effectcraft_project::render_queue::OutputModule::for_format(OutputFormat::Wav); + output.audio_channels = channels; + let cancel = Arc::new(AtomicBool::new(false)); + let source = Ramp { reads: AtomicUsize::new(0), cancel: None }; + let job = crate::Job { + project: &p, + footage: &source, + expr: None, + accel: None, + comp: cid, + settings: &Default::default(), + output: &output, + path: "unused.wav", + sink: None, + options: crate::JobOptions { cancel: Some(cancel.clone()), ..Default::default() }, + nested_switches: true, + }; + let cx = Cx::new(&job); + let start = Tick(Tick::from_units(73, rate as i64).0 + 1); + let count = 1_100_003; + let actual = mix(&cx, start, count, rate).unwrap(); + assert_eq!(actual.len(), count * channels as usize); + for i in 0..count { + let x = ((73 + i) % 4096) as f32 / 4096.0; + if channels == 1 { + assert_eq!(actual[i], (x + x * 0.5) * 0.5); + } else { + assert_eq!(&actual[i * 2..i * 2 + 2], &[x, x * 0.5]); + } + } + let source = Ramp { reads: AtomicUsize::new(0), cancel: Some(cancel) }; + let job = crate::Job { footage: &source, ..job }; + assert!(matches!(mix(&Cx::new(&job), start, count, rate), Err(ExportError::Cancelled))); + assert_eq!(source.reads.load(Ordering::Relaxed), 2, "stop during the second block"); + } + } + } + #[test] fn yuv_of_primaries() { let rgba = [255u8, 255, 255, 255, 0, 0, 0, 255, 255, 0, 0, 255, 0, 255, 0, 255]; diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index eda9799588..868ef2706a 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -97,6 +97,8 @@ pub struct Job<'a> { /// Job options beyond the Render Settings and Output Module. #[derive(Default)] pub struct JobOptions<'a> { + /// Interrupt audio mixing within a batch; callback cancellation still works between batches. + pub cancel: Option>, /// The render log written next to the output. pub log: RenderLog, /// What the log calls the job (e.g. "#2 Main Comp"). diff --git a/crates/export/src/webm.rs b/crates/export/src/webm.rs index a12bd4e1a6..bb2f543949 100644 --- a/crates/export/src/webm.rs +++ b/crates/export/src/webm.rs @@ -229,13 +229,13 @@ pub(crate) async fn webm(job: &Cx<'_>, comp: &Comp, w: u32, h: u32, st: &mut Sta let mut cursor = span_start.to_units_floor(OPUS_RATE as i64); let mut pcm_left: Vec = vec![]; let mut packets = 0u64; - let mut audio_until = |until: Tick, frames: &mut Vec, finish: bool| { - let Some(enc) = opus.as_mut() else { return }; + let mut audio_until = |until: Tick, frames: &mut Vec, finish: bool| -> Result<()> { + let Some(enc) = opus.as_mut() else { return Ok(()) }; let end = until.to_units_floor(OPUS_RATE as i64); if end > cursor { let n = (end - cursor) as usize; let start = Tick(((cursor as i128 * TICKS_PER_SECOND as i128) / OPUS_RATE as i128) as i64); - pcm_left.extend(mix(job, start, n, OPUS_RATE)); + pcm_left.extend(mix(job, start, n, OPUS_RATE)?); cursor = end; } let fs = effectcraft_opusenc::OpusEncoder::FRAME_SIZE * opus_channels; @@ -250,6 +250,7 @@ pub(crate) async fn webm(job: &Cx<'_>, comp: &Comp, w: u32, h: u32, st: &mut Sta frames.push(Frame { ms: packets as i64 * 20, track: 2, key: true, prev_ms: None, data: pkt, alpha: None }); packets += 1; } + Ok(()) }; let batch = batch_size(); @@ -279,7 +280,7 @@ pub(crate) async fn webm(job: &Cx<'_>, comp: &Comp, w: u32, h: u32, st: &mut Sta } drop(pixels); let t_end = if end >= total { span_end } else { job.settings.frame_time(comp, end) }; - audio_until(t_end, &mut frames, end >= total); + audio_until(t_end, &mut frames, end >= total)?; let c = cluster(&mut frames); file.write_all(&c).map_err(io)?; seg_len += c.len() as u64; @@ -288,7 +289,7 @@ pub(crate) async fn webm(job: &Cx<'_>, comp: &Comp, w: u32, h: u32, st: &mut Sta } if total == 0 { let mut frames = vec![]; - audio_until(span_end, &mut frames, true); + audio_until(span_end, &mut frames, true)?; if !frames.is_empty() { let c = cluster(&mut frames); file.write_all(&c).map_err(io)?; @@ -347,7 +348,7 @@ pub(crate) fn audio_file(job: &Cx, comp: &Comp, aiff: bool, st: &mut State) -> R let sample = start + done as i64; let t = Tick::from_units(sample, i64::from(sr)); let t = if t.to_units_floor(i64::from(sr)) < sample { Tick(t.0.saturating_add(1)) } else { t }; - pcm.extend(pcm_bytes(&mix(job, t, k, sr), fmt, aiff)); + pcm.extend(pcm_bytes(&mix(job, t, k, sr)?, fmt, aiff)); done += k; let now = (done as u64 * total_frames / n.max(1) as u64).min(total_frames); if now > reported && st.total > 0 { diff --git a/crates/export/src/webm_av1.rs b/crates/export/src/webm_av1.rs index ca06a3e512..282a4a7f5d 100644 --- a/crates/export/src/webm_av1.rs +++ b/crates/export/src/webm_av1.rs @@ -110,13 +110,13 @@ pub(crate) async fn webm_av1(job: &Cx<'_>, comp: &Comp, w: u32, h: u32, st: &mut let mut cursor = span_start.to_units_floor(OPUS_RATE as i64); let mut pcm_left: Vec = vec![]; let mut packets = 0u64; - let mut audio_until = |until: Tick, blocks: &mut Vec, finish: bool| { - let Some(enc) = opus.as_mut() else { return }; + let mut audio_until = |until: Tick, blocks: &mut Vec, finish: bool| -> Result<()> { + let Some(enc) = opus.as_mut() else { return Ok(()) }; let end = until.to_units_floor(OPUS_RATE as i64); if end > cursor { let n = (end - cursor) as usize; let start = Tick(((cursor as i128 * TICKS_PER_SECOND as i128) / OPUS_RATE as i128) as i64); - pcm_left.extend(mix(job, start, n, OPUS_RATE)); + pcm_left.extend(mix(job, start, n, OPUS_RATE)?); cursor = end; } let fs = effectcraft_opusenc::OpusEncoder::FRAME_SIZE * opus_channels; @@ -130,6 +130,7 @@ pub(crate) async fn webm_av1(job: &Cx<'_>, comp: &Comp, w: u32, h: u32, st: &mut blocks.push(Block { ms: packets as i64 * 20, track: 2, key: true, data: pkt }); packets += 1; } + Ok(()) }; let batch = batch_size(); @@ -156,13 +157,13 @@ pub(crate) async fn webm_av1(job: &Cx<'_>, comp: &Comp, w: u32, h: u32, st: &mut } } let t_end = if end >= total { span_end } else { job.settings.frame_time(comp, end) }; - audio_until(t_end, &mut blocks, end >= total); + audio_until(t_end, &mut blocks, end >= total)?; flush_blocks(&mut blocks, &mut file, &mut seg_len)?; st.advance(end - i)?; i = end; } if total == 0 { - audio_until(span_end, &mut blocks, true); + audio_until(span_end, &mut blocks, true)?; flush_blocks(&mut blocks, &mut file, &mut seg_len)?; } let mut size = [0u8; 8]; diff --git a/crates/export/tests/render_settings.rs b/crates/export/tests/render_settings.rs index 9e9a0a18f7..acccf95151 100644 --- a/crates/export/tests/render_settings.rs +++ b/crates/export/tests/render_settings.rs @@ -40,10 +40,12 @@ fn project(rate: FrameRate, extra: impl FnOnce(&mut Project, &mut Comp)) -> (Pro type Files = Arc)>>>; +/// Select an exact export span without changing the project's timing. fn custom(start: f64, end: f64) -> RenderSettings { RenderSettings { time_span: TimeSpan::Custom { start: Tick::from_seconds_f64(start), end: Tick::from_seconds_f64(end) }, ..Default::default() } } +/// Collect finished in-memory outputs alongside the export result. fn run_with(p: &Project, cid: ItemId, s: &RenderSettings, om: &OutputModule, path: &str, options: JobOptions, cancel: bool) -> (Result, Files) { run_on(p, cid, s, om, path, options, None, cancel) } @@ -361,6 +363,7 @@ fn wav_and_aiff_channels_and_formats() { assert_eq!(u16::from_be_bytes([d[26], d[27]]), 24, "sample size"); } +/// Cancellation preserves unrelated and skipped files, including overflow destinations. #[test] fn cancelled_export_removes_only_files_it_opened() { use effectcraft_export::ExportError; @@ -476,3 +479,40 @@ fn a_job_renders_on_one_compositor() { assert_eq!(cpu.frames.load(Ordering::SeqCst), 0); assert_eq!(c[0].get_pixel(10, 44).0, [255, 0, 0, 255], "the CPU drew the red block"); } + +/// Cancellation and excessive audio work propagate through the encoder instead of producing a report. +#[test] +fn audio_mix_failures_abort_export() { + let (p, cid) = project(FrameRate::new(25, 1), |_, _| {}); + let om = OutputModule::for_format(OutputFormat::Wav); + let options = JobOptions { cancel: Some(Arc::new(std::sync::atomic::AtomicBool::new(true))), ..Default::default() }; + let (result, _) = run_with(&p, cid, &custom(0.0, 0.5), &om, "/cancel.wav", options, false); + assert_eq!(result.unwrap_err(), "cancelled"); + let (p, cid) = project(FrameRate::new(25, 1), |p, c| { + let f = effectcraft_project::Footage { has_audio: true, ..Default::default() }; + let id = p.add_item("audio", Label::SeaFoam, None, ItemKind::Footage(f)); + let mut layer = build::layer(p, c, "audio", LayerSource::Footage { item: id }, (0, 0), None); + layer.stretch = 0.000001; + c.layers.push(layer); + }); + let (result, _) = run_with(&p, cid, &custom(0.0, 0.5), &om, "/limited.wav", JobOptions::default(), false); + assert!(result.unwrap_err().contains("audio mixing exceeded its work limit")); +} + +/// Valid low-frame-rate movie exports must split large audio requests into bounded mixes. +#[test] +fn low_frame_rate_audio_exports_are_not_rejected_by_the_mix_limit() { + let (mut p, cid) = project(FrameRate::new(1, 30), |_, _| {}); + p.comp_mut(cid).unwrap().duration = Tick::from_seconds_f64(30.0); + for (format, codec) in + [(OutputFormat::Uncompressed, WebmVideoCodec::Vp9), (OutputFormat::WebM, WebmVideoCodec::Vp9), (OutputFormat::WebM, WebmVideoCodec::Av1)] + { + let mut om = OutputModule::for_format(format); + om.webm_codec = codec; + om.audio = AudioOutput::On; + om.audio_sample_rate = 48000; + let (result, files) = run_with(&p, cid, &custom(0.0, 30.0), &om, "/slow.mov", JobOptions::default(), false); + assert!(result.is_ok(), "{format:?}/{codec:?}: {result:?}"); + assert_eq!(files.lock().unwrap().len(), 1); + } +} diff --git a/crates/expr/src/audio.rs b/crates/expr/src/audio.rs new file mode 100644 index 0000000000..a89260a9e4 --- /dev/null +++ b/crates/expr/src/audio.rs @@ -0,0 +1,150 @@ +//! Exact scalar arithmetic for audio-rate expressions. Everything else uses the JS runtime. +//! No constant folding or reassociation: each sample retains the original f64 operation order. +use effectcraft_render::AudioExpression; + +enum Node { + Number(f64), + Time, + Value, + Neg(Box), + Binary(u8, Box, Box), +} + +impl Node { + /// Execute the bounded tree using the same number operations as scalar JavaScript. + fn sample(&self, time: f64, value: f64) -> f64 { + match self { + Self::Number(n) => *n, + Self::Time => time, + Self::Value => value, + Self::Neg(n) => -n.sample(time, value), + Self::Binary(op, a, b) => { + let (a, b) = (a.sample(time, value), b.sample(time, value)); + match op { + b'+' => a + b, + b'-' => a - b, + b'*' => a * b, + _ => a / b, + } + } + } + } +} +impl AudioExpression for Node { + /// Evaluate the prepared scalar, treating non-finite numbers as expression errors. + fn evaluate(&self, time: f64, value: f64) -> Option { + let n = self.sample(time, value); + n.is_finite().then_some(n) + } +} + +/// Decline unsupported syntax, legacy numeric literals, or excessive nesting before evaluation. +pub fn prepare(text: &str) -> Option> { + if text.len() > 1024 || !text.is_ascii() || text.contains("++") || text.contains("--") { + return None; + } + let mut p = Parser { text: text.as_bytes(), at: 0, nodes: 0 }; + let n = p.expression(0, 0)?; + p.space(); + if p.peek() == Some(b';') { + p.at += 1; + p.space(); + } + (p.at == p.text.len()).then(|| Box::new(n) as Box) +} + +struct Parser<'a> { + text: &'a [u8], + at: usize, + nodes: usize, +} +impl Parser<'_> { + /// Read the next byte without advancing or indexing beyond the expression. + fn peek(&self) -> Option { + self.text.get(self.at).copied() + } + /// Only ASCII JavaScript whitespace is admitted by this fast path. + fn space(&mut self) { + while self.peek().is_some_and(|c| matches!(c, b' ' | b'\t' | b'\n' | b'\r')) { + self.at += 1; + } + } + /// Precedence climbing preserves left associativity and bounds both tree size and depth. + fn expression(&mut self, min: u8, depth: usize) -> Option { + if depth > 32 || self.nodes >= 128 { + return None; + } + self.nodes += 1; + self.space(); + let mut left = match self.peek()? { + b'+' | b'-' => { + let op = self.peek()?; + self.at += 1; + let n = self.expression(3, depth + 1)?; + if op == b'-' { Node::Neg(Box::new(n)) } else { n } + } + b'(' => { + self.at += 1; + let n = self.expression(0, depth + 1)?; + self.space(); + if self.peek()? != b')' { + return None; + } + self.at += 1; + n + } + b'0'..=b'9' | b'.' => { + let start = self.at; + while self.peek().is_some_and(|c| c.is_ascii_digit()) { + self.at += 1; + } + if self.at > start + 1 && self.text.get(start) == Some(&b'0') { + return None; + } + if self.peek() == Some(b'.') { + self.at += 1; + while self.peek().is_some_and(|c| c.is_ascii_digit()) { + self.at += 1; + } + } + if matches!(self.peek(), Some(b'e' | b'E')) { + self.at += 1; + if matches!(self.peek(), Some(b'+' | b'-')) { + self.at += 1; + } + while self.peek().is_some_and(|c| c.is_ascii_digit()) { + self.at += 1; + } + } + Node::Number(std::str::from_utf8(self.text.get(start..self.at)?).ok()?.parse().ok()?) + } + _ => { + let start = self.at; + while self.peek().is_some_and(|c| c.is_ascii_alphanumeric() || c == b'_') { + self.at += 1; + } + match self.text.get(start..self.at)? { + b"time" => Node::Time, + b"value" => Node::Value, + _ => return None, + } + } + }; + loop { + self.space(); + let Some(op) = self.peek() else { break }; + let precedence = match op { + b'+' | b'-' => 1, + b'*' | b'/' => 2, + _ => break, + }; + if precedence < min { + break; + } + self.at += 1; + let right = self.expression(precedence + 1, depth + 1)?; + left = Node::Binary(op, Box::new(left), Box::new(right)); + } + Some(left) + } +} diff --git a/crates/expr/src/lib.rs b/crates/expr/src/lib.rs index 4963810c48..b94f1f53ad 100644 --- a/crates/expr/src/lib.rs +++ b/crates/expr/src/lib.rs @@ -29,6 +29,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +mod audio; pub mod host; mod noise; pub mod rewrite; @@ -67,6 +68,7 @@ fn budget_start() { /// An error once the outermost evaluation has used up its runs or its time. fn budget_left() -> Result<(), String> { if RUNS.with(|r| r.get()) >= MAX_RUNS { + runtime::fail_audio(); return Err("Error: expression reads too many other expressions' values (circular or fanning-out references?)".into()); } if runtime::out_of_time() { @@ -80,6 +82,24 @@ fn budget_left() -> Result<(), String> { pub struct Expressions; impl ExprHost for Expressions { + /// Evaluate a remap, level or effect parameter under the mix's shared interruption controls. + fn eval_audio( + &self, + ctx: &EvalCtx, + layer: &Layer, + prop: &Property, + value: &Value, + control: &effectcraft_render::audio::AudioEvalControl, + ) -> Result { + let _guard = runtime::AudioControl::enter(control); + self.eval(ctx, layer, prop, value) + } + + /// Prepare only the scalar syntax whose per-sample JavaScript semantics are exact. + fn prepare_audio(&self, text: &str) -> Option> { + audio::prepare(text) + } + fn eval(&self, ctx: &EvalCtx, layer: &Layer, prop: &Property, value: &Value) -> Result { match &prop.expr { Some(e) => evaluate(self, ctx, layer, prop, value, &e.text), @@ -198,6 +218,7 @@ fn evaluate_out( } } } + runtime::fail_audio(); Err("Error: expression needs too many property reads".into()) } diff --git a/crates/expr/src/runtime.rs b/crates/expr/src/runtime.rs index 53c2960ba8..fd3be94405 100644 --- a/crates/expr/src/runtime.rs +++ b/crates/expr/src/runtime.rs @@ -44,6 +44,7 @@ pub struct Owner { } thread_local! { + static AUDIO_CONTROL: RefCell> = const { RefCell::new(None) }; /// Never dropped: boa's own thread-local heap may be torn down first at thread exit, and /// dropping GC handles after that aborts the process. The OS reclaims it with the thread. static RUNTIME: RefCell>>> = const { RefCell::new(None) }; @@ -59,6 +60,30 @@ thread_local! { pub static STARTED: std::cell::Cell> = const { std::cell::Cell::new(None) }; } +/// Restores the previous mix context even when nested expression evaluation fails. +pub struct AudioControl(Option); +impl AudioControl { + /// Install shared cancellation, deadline and instruction allowance for one evaluation. + pub fn enter(control: &effectcraft_render::audio::AudioEvalControl) -> Self { + Self(AUDIO_CONTROL.with(|c| c.replace(Some(control.clone())))) + } +} +impl Drop for AudioControl { + /// Leave no cancellation token or exhausted allowance on the next unrelated evaluation. + fn drop(&mut self) { + AUDIO_CONTROL.with(|c| c.replace(self.0.take())); + } +} + +/// A runtime resource failure must abort audio, not retry keyframed fallback for every sample. +pub fn fail_audio() { + AUDIO_CONTROL.with(|c| { + if let Some(c) = c.borrow().as_ref() { + c.fail(); + } + }); +} + struct Runtime { ctx: Context, scripts: HashMap>, @@ -302,6 +327,9 @@ impl Runtime { // Abandoned mid-run: the context is unusable now (see `run`). None => return Run::Done(Err(TIMED_OUT.into())), }; + if result.as_ref().is_err_and(|e| matches!(e.as_engine(), Some(boa_engine::error::EngineError::RuntimeLimit(_)))) { + fail_audio(); + } let pending = FRAMES.with(|f| f.borrow().last().is_some_and(|fr| !fr.misses.is_empty())); if pending { return Run::Pending; @@ -324,7 +352,7 @@ fn evaluate_in_time(script: &Script, ctx: &mut Context) -> Option std::time::Duration { /// Whether the outermost evaluation on this thread has used up [`crate::MAX_TIME`]. pub fn out_of_time() -> bool { + if AUDIO_CONTROL.with(|c| c.borrow().as_ref().is_some_and(|c| c.interrupted())) { + return true; + } #[cfg(not(target_arch = "wasm32"))] { STARTED.with(|s| s.get()).is_some_and(|t| t.elapsed() > max_time()) diff --git a/crates/expr/tests/audio_remap.rs b/crates/expr/tests/audio_remap.rs new file mode 100644 index 0000000000..1c45c91921 --- /dev/null +++ b/crates/expr/tests/audio_remap.rs @@ -0,0 +1,288 @@ +//! Production-expression integration tests, including nested mixing and adverse work patterns. +#[path = "support/audio_workload.rs"] +mod workload; +use effectcraft_expr::Expressions; +use effectcraft_keyframe::Value; +use effectcraft_project::{Layer, Property}; +use effectcraft_render::{ + EvalCtx, ExprHost, + audio::{AudioError, MixControl, mix_comp, mix_comp_with_control}, +}; +use effectcraft_time::Tick; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; + +/// Forces the full JavaScript path as an independent oracle for prepared expressions. +struct JavaScript; +impl ExprHost for JavaScript { + /// Delegate unchanged property evaluation to the production JavaScript engine. + fn eval(&self, ctx: &EvalCtx, layer: &Layer, prop: &Property, value: &Value) -> Result { + Expressions.eval(ctx, layer, prop, value) + } +} + +/// Check numeric semantics and the boundary where expressions must fall back to JavaScript. +#[test] +fn scalar_preparation_matches_javascript_and_rejects_other_syntax() { + let (p, cid) = workload::project(1, 0, None); + let comp = p.comp(cid).unwrap(); + let layer = &comp.layers[0]; + let prop = layer.props.get("timeRemap").unwrap(); + let texts = [ + "0.25 + time * 0.5", + "time * time * .01 + value", + "(time + value) / 3 - -0.5", + "1e-3 * time;", + "value / (time-time)", + "-0", + "1e308 * 1e308 - 1e308", + "time - value - 1", + "time / 3 / 7", + "+time", + ]; + for text in texts { + let prepared = Expressions.prepare_audio(text).unwrap(); + for time in [-123.0, -0.0, 0.0, 1.0 / 48000.0, 0.1, 1.0, 179.99] { + let ctx = EvalCtx::new(&p, cid, comp, Tick::from_seconds_f64(time)); + for value in [-0.0, 0.25, 123.75] { + let actual = prepared.evaluate(ctx.time.seconds(), value).map(f64::to_bits); + let expected = effectcraft_expr::evaluate(&Expressions, &ctx, layer, prop, &Value::Scalar(value), text).ok().map(|v| v.as_f64().to_bits()); + assert_eq!(actual, expected, "{text}, time={time}, value={value}"); + } + } + } + for text in [ + "Math.sin(time)", + "time++", + "--time", + "0x10", + "012", + "time ** 2", + "var x = time; x", + "time ? 1 : 0", + "value[0]", + "time // comment", + "time +", + "/*comment*/time", + ] { + assert!(Expressions.prepare_audio(text).is_none(), "{text}"); + } + assert!(Expressions.prepare_audio(&format!("{}time{}", "(".repeat(40), ")".repeat(40))).is_none()); + assert!(Expressions.prepare_audio(&"time+".repeat(300)).is_none()); +} + +/// Compare complete mixed buffers, including layered/nested projects and a non-dividing sample rate. +#[test] +fn medium_nested_project_matches_full_runtime_and_block_boundaries() { + for (layers, depth, text) in + [(12, 0, "0.25 + time * time * 0.01"), (4, 2, "0.25 + time * 0.5"), (2, 0, "0.25 + Math.sin(time) * Math.sin(time)"), (2, 0, "value / (time-time)")] + { + let (p, cid) = workload::project(layers, depth, Some(text)); + let source = workload::Source::default(); + let expected = mix_comp(&p, &source, Some(&JavaScript), cid, Tick::ZERO, 256, 44100).unwrap(); + let actual = mix_comp(&p, &source, Some(&Expressions), cid, Tick::ZERO, 256, 44100).unwrap(); + assert_eq!(actual, expected, "{layers} layers, depth {depth}, {text}"); + let mut split = mix_comp(&p, &source, Some(&Expressions), cid, Tick::ZERO, 73, 44100).unwrap(); + let mut start = Tick::from_units(73, 44100); + if start.to_units_floor(44100) < 73 { + start.0 += 1; + } + split.extend(mix_comp(&p, &source, Some(&Expressions), cid, start, 183, 44100).unwrap()); + assert_eq!(split, actual, "block boundary: {text}"); + } +} + +/// Distinguish a reusable jump pattern from new-page traversal that must exhaust shared work. +#[test] +fn repeated_jumps_cache_pages_and_unbounded_traversal_returns_error() { + let (p, cid) = workload::project(1, 0, Some("Math.floor(time * 48000) % 2 ? 10 : 0")); + let source = workload::Source::default(); + let samples = mix_comp(&p, &source, Some(&Expressions), cid, Tick::ZERO, 1024, 48000).unwrap(); + assert_eq!(samples.len(), 2048); + assert_eq!(mix_comp(&p, &source, Some(&Expressions), cid, Tick::ZERO, usize::MAX, 48000), Err(AudioError::WorkLimit)); + assert_eq!(source.reads.load(Ordering::Relaxed), 2); + for depth in [0, 2] { + let (p, cid) = workload::project(4, depth, Some("time * 100000")); + let source = workload::Source::default(); + let control = MixControl { max_work: Some(100_000), ..Default::default() }; + assert_eq!(mix_comp_with_control(&p, &source, Some(&Expressions), cid, Tick::ZERO, 1024, 48000, &control), Err(AudioError::WorkLimit)); + assert!(source.frames.load(Ordering::Relaxed) <= 100_000); + } +} + +/// Interrupt an expensive script and reuse its thread-local runtime for a successful mix. +#[test] +fn cancellation_interrupts_running_javascript_and_does_not_poison_next_mix() { + let (p, cid) = workload::project(1, 0, Some("var n = 0; for (var i = 0; i < 1000000; i++) { n += Math.sin(i); } n")); + let cancel = Arc::new(AtomicBool::new(false)); + let signal = cancel.clone(); + let (done, completion) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + let source = workload::Source::default(); + let control = MixControl { cancel: Some(signal), ..Default::default() }; + let result = mix_comp_with_control(&p, &source, Some(&Expressions), cid, Tick::ZERO, 1024, 48000, &control); + assert_eq!(result, Err(AudioError::Cancelled)); + assert_eq!(source.reads.load(Ordering::Relaxed), 0); + let (p, cid) = workload::project(1, 0, Some("Math.sin(time)")); + assert!(mix_comp(&p, &source, Some(&Expressions), cid, Tick::ZERO, 32, 48000).is_ok()); + done.send(()).unwrap(); + }); + std::thread::sleep(std::time::Duration::from_millis(50)); + cancel.store(true, Ordering::Relaxed); + completion.recv_timeout(std::time::Duration::from_secs(3)).expect("cancellation must interrupt the script, not wait for every sample"); + worker.join().unwrap(); +} + +/// A runtime resource limit aborts the mix instead of retrying the same script per sample. +#[test] +fn expression_resource_limits_abort_audio_and_recover() { + let (p, cid) = workload::project(1, 0, Some("for (;;) {}")); + let source = workload::Source::default(); + assert_eq!(mix_comp(&p, &source, Some(&Expressions), cid, Tick::ZERO, 2, 48000), Err(AudioError::WorkLimit)); + assert_eq!(source.reads.load(Ordering::Relaxed), 0); + let (p, cid) = workload::project(1, 0, Some("Math.sin(time)")); + assert!(mix_comp(&p, &source, Some(&Expressions), cid, Tick::ZERO, 32, 48000).is_ok()); +} + +/// One deadline reaches a running script; nested layers share the same yield allowance. +#[test] +fn audio_deadline_and_yield_allowance_are_cumulative() { + for depth in [0, 2] { + let (p, cid) = workload::project(4, depth, Some("function g() { for(var i=0;i<900000;i++) {} } for(var k=0;k<900000;k++) g()")); + let source = workload::Source::default(); + for control in [ + MixControl { max_duration: Some(std::time::Duration::from_millis(50)), max_expression_yields: Some(usize::MAX), ..Default::default() }, + MixControl { max_duration: Some(std::time::Duration::from_secs(30)), max_expression_yields: Some(2), ..Default::default() }, + ] { + let start = std::time::Instant::now(); + assert_eq!(mix_comp_with_control(&p, &source, Some(&Expressions), cid, Tick::ZERO, 1600, 48000, &control), Err(AudioError::WorkLimit)); + assert!(start.elapsed() < std::time::Duration::from_secs(3)); + } + } + // Two individually small evaluations together exhaust the same allowance, rather than each + // receiving a fresh budget. A deterministic host makes this independent of machine speed. + struct Yielding; + impl ExprHost for Yielding { + /// Preserve the keyframed value outside controlled audio evaluation. + fn eval(&self, _: &EvalCtx, _: &Layer, _: &Property, v: &Value) -> Result { + Ok(v.clone()) + } + /// Charge a deterministic yield to verify sharing across evaluations. + fn eval_audio(&self, _: &EvalCtx, _: &Layer, _: &Property, v: &Value, c: &effectcraft_render::audio::AudioEvalControl) -> Result { + c.yielded(); + Ok(v.clone()) + } + } + let (p, cid) = workload::project(4, 1, Some("time")); + let control = MixControl { max_expression_yields: Some(2), ..Default::default() }; + assert_eq!(mix_comp_with_control(&p, &workload::Source::default(), Some(&Yielding), cid, Tick::ZERO, 10, 48000, &control), Err(AudioError::WorkLimit)); + let (p, cid) = workload::project(1, 0, Some("Math.sin(time)")); + assert!(mix_comp(&p, &workload::Source::default(), Some(&Expressions), cid, Tick::ZERO, 32, 48000).is_ok()); +} + +/// Put an expression on levels or a Tone parameter inside two remapped precomps. +fn parameter_project(effect: bool, text: &str, layers: usize) -> (effectcraft_project::Project, effectcraft_project::ItemId) { + use effectcraft_project::{Expression, build::Ids}; + let (mut p, cid) = workload::project(layers, 2, None); + let leaf = + p.comps().find(|(_, c)| c.layers.iter().any(|l| matches!(l.source, effectcraft_project::LayerSource::Footage { .. }))).map(|(id, _)| *id).unwrap(); + for i in 0..layers { + let expr = Some(Expression { text: text.into(), enabled: true }); + if effect { + let spec = effectcraft_effects::find("ec.audio.tone").unwrap(); + let mut group = effectcraft_effects::instantiate(spec, &mut Ids(&mut p.next_id), spec.name, [16.0, 16.0]); + let prop = group.children.iter_mut().find_map(|n| if let effectcraft_project::Node::Prop(p) = n { Some(p) } else { None }).unwrap(); + prop.expr = expr; + p.comp_mut(leaf).unwrap().layers[i].props.sub_mut("effects").unwrap().children.push(group.into()); + } else { + p.comp_mut(leaf).unwrap().layers[i].props.sub_mut("audio").unwrap().prop_mut("levels").unwrap().expr = expr; + } + } + (p, cid) +} + +/// Production parameter scripts obey resource limits; ordinary errors retain keyframed fallback. +#[test] +fn nested_audio_parameters_observe_script_limits_and_recover() { + for effect in [false, true] { + let (p, cid) = parameter_project(effect, "var n=0; for(var i=0;i<10000;i++) n+=Math.sin(i); value", 1); + let source = workload::Source::default(); + let control = MixControl { max_expression_yields: Some(0), max_duration: Some(std::time::Duration::from_secs(30)), ..Default::default() }; + assert_eq!(mix_comp_with_control(&p, &source, Some(&Expressions), cid, Tick::ZERO, 2, 48000, &control), Err(AudioError::WorkLimit), "effect={effect}"); + assert_eq!(source.reads.load(Ordering::Relaxed), 0); + // The same thread's runtime recovers; normal script errors still fall back to keys. + let (p, cid) = parameter_project(effect, "throw new Error('ordinary error')", 1); + let actual = mix_comp(&p, &source, Some(&Expressions), cid, Tick::ZERO, 32, 48000).unwrap(); + let expected = mix_comp(&p, &source, None, cid, Tick::ZERO, 32, 48000).unwrap(); + assert_eq!(actual, expected); + } +} + +/// Each layer consumes the same allowance; reaching the limit prevents subsequent evaluation. +#[test] +fn nested_audio_parameters_share_one_yield_allowance() { + #[derive(Default)] + struct Yielding(std::sync::atomic::AtomicUsize); + impl ExprHost for Yielding { + /// Preserve the keyframed value outside controlled audio evaluation. + fn eval(&self, _: &EvalCtx, _: &Layer, _: &Property, v: &Value) -> Result { + Ok(v.clone()) + } + /// Charge a deterministic yield to verify sharing across evaluations. + fn eval_audio(&self, _: &EvalCtx, _: &Layer, _: &Property, v: &Value, c: &effectcraft_render::audio::AudioEvalControl) -> Result { + self.0.fetch_add(1, Ordering::Relaxed); + c.yielded(); + Ok(v.clone()) + } + } + for effect in [false, true] { + let (p, cid) = parameter_project(effect, "value", 8); + let host = Yielding::default(); + let control = MixControl { max_expression_yields: Some(2), ..Default::default() }; + assert_eq!(mix_comp_with_control(&p, &workload::Source::default(), Some(&host), cid, Tick::ZERO, 10, 48000, &control), Err(AudioError::WorkLimit)); + assert_eq!(host.0.load(Ordering::Relaxed), 3); + } +} + +/// Cancellation interrupts a running volume/effect script and leaves its runtime reusable. +#[test] +fn cancellation_interrupts_nested_audio_parameters() { + for effect in [false, true] { + let (p, cid) = parameter_project(effect, "function g() { for(var i=0;i<900000;i++) {} } for(var k=0;k<900000;k++) g()", 1); + let cancel = Arc::new(AtomicBool::new(false)); + let signal = cancel.clone(); + let (done, completion) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + let source = workload::Source::default(); + let control = MixControl { cancel: Some(signal), max_expression_yields: Some(usize::MAX), ..Default::default() }; + assert_eq!(mix_comp_with_control(&p, &source, Some(&Expressions), cid, Tick::ZERO, 2, 48000, &control), Err(AudioError::Cancelled)); + assert_eq!(source.reads.load(Ordering::Relaxed), 0); + let (p, cid) = parameter_project(effect, "value", 1); + assert!(mix_comp(&p, &source, Some(&Expressions), cid, Tick::ZERO, 32, 48000).is_ok()); + done.send(()).unwrap(); + }); + std::thread::sleep(std::time::Duration::from_millis(50)); + cancel.store(true, Ordering::Relaxed); + completion.recv_timeout(std::time::Duration::from_secs(3)).expect("parameter script must observe cancellation"); + worker.join().unwrap(); + } +} + +/// Property-read pass exhaustion is a resource failure, including when reached from audio. +#[test] +fn property_read_pass_limits_abort_audio_and_recover() { + // A fresh host request on each replay reaches the pass limit without a costly JS loop. + let text = "globalThis.audioPassProbe = (globalThis.audioPassProbe || 0) + 1; thisLayer.transform.opacity.valueAtTime(globalThis.audioPassProbe, true)"; + let (p, cid) = workload::project(1, 0, Some(text)); + let layer = &p.comp(cid).unwrap().layers[0]; + let error = effectcraft_expr::eval_property(&p, cid, layer.id, "timeRemap", 0.0).unwrap_err(); + assert!(error.contains("too many property reads"), "must exercise the pass limit: {error}"); + let source = workload::Source::default(); + let control = MixControl { max_expression_yields: Some(usize::MAX), max_duration: Some(std::time::Duration::from_secs(30)), ..Default::default() }; + assert_eq!(mix_comp_with_control(&p, &source, Some(&Expressions), cid, Tick::ZERO, 2, 48000, &control), Err(AudioError::WorkLimit)); + assert_eq!(source.reads.load(Ordering::Relaxed), 0); + let (p, cid) = workload::project(1, 0, Some("Math.sin(time)")); + assert!(mix_comp(&p, &source, Some(&Expressions), cid, Tick::ZERO, 32, 48000).is_ok()); +} diff --git a/crates/expr/tests/audio_workloads.rs b/crates/expr/tests/audio_workloads.rs new file mode 100644 index 0000000000..1d7bba6d14 --- /dev/null +++ b/crates/expr/tests/audio_workloads.rs @@ -0,0 +1,64 @@ +//! Workload measurements are opt-in: no hardware-dependent timing assertion in ordinary CI. +#[path = "support/audio_workload.rs"] +mod workload; +use effectcraft_expr::Expressions; +use effectcraft_render::audio::mix_comp; +use effectcraft_time::Tick; +use std::{sync::atomic::Ordering, time::Instant}; + +/// Exercises medium/large projects using the production JavaScript evaluator, streaming blocks. +/// EC_AUDIO_CASE selects a case; EC_AUDIO_SECONDS and EC_AUDIO_RATE override duration/rate. +#[test] +#[ignore = "opt-in audio workload benchmark"] +fn audio_workload_matrix() { + let cases = [ + ("keyframed", 8, 0, None, 4.0), + ("typical", 4, 0, Some("0.25 + time * 0.5"), 4.0), + ("heavy", 24, 0, Some("0.25 + time * time * 0.01"), 2.0), + ("nested", 8, 2, Some("0.25 + time * 0.5"), 1.0), + ("nonlinear", 4, 0, Some("0.25 + Math.sin(time) * Math.sin(time)"), 2.0), + ("alternating", 1, 0, Some("Math.floor(time * 48000) % 2 ? 10 : 0"), 128.0 / 48000.0), + ("wandering", 1, 0, Some("time * 100000"), 128.0 / 48000.0), + ("long", 8, 0, Some("0.25 + time * 0.5"), 180.0), + ]; + let selected = std::env::var("EC_AUDIO_CASE").unwrap_or_default(); + let rate = std::env::var("EC_AUDIO_RATE").ok().and_then(|s| s.parse().ok()).unwrap_or(48000u32); + for (name, layers, depth, expression, seconds) in cases { + if (!selected.is_empty() && selected != name) || (selected.is_empty() && name == "long") { + continue; + } + let seconds = std::env::var("EC_AUDIO_SECONDS").ok().and_then(|s| s.parse().ok()).unwrap_or(seconds); + let (p, cid) = workload::project(layers, depth, expression); + let source = workload::Source::default(); + let host = Expressions; + let total = (seconds * rate as f64).round() as usize; + let mut hash = 0xcbf29ce484222325u64; + let mut worst = 0.0_f64; + let started = Instant::now(); + let mut failure = None; + for first in (0..total).step_by(1024) { + let t = Instant::now(); + let samples = mix_comp(&p, &source, Some(&host), cid, Tick::from_units(first as i64, rate as i64), 1024.min(total - first), rate); + worst = worst.max(t.elapsed().as_secs_f64()); + let samples = match samples { + Ok(s) => s, + Err(e) => { + failure = Some(e.to_string()); + break; + } + }; + for v in samples { + assert!(v.is_finite()); + hash = (hash ^ v.to_bits() as u64).wrapping_mul(0x100000001b3); + } + } + let elapsed = started.elapsed().as_secs_f64(); + if name != "wandering" { + assert!(failure.is_none(), "{name}: {failure:?}"); + } + println!( + "{}", + serde_json::json!({"error":failure,"case":name,"layers":layers,"depth":depth,"rate":rate,"seconds":seconds,"wall_seconds":elapsed,"realtime_factor":seconds/elapsed,"worst_block_ms":worst*1000.0,"reads":source.reads.load(Ordering::Relaxed),"source_frames":source.frames.load(Ordering::Relaxed),"hash":format!("{hash:016x}")}) + ); + } +} diff --git a/crates/expr/tests/support/audio_workload.rs b/crates/expr/tests/support/audio_workload.rs new file mode 100644 index 0000000000..da5756195a --- /dev/null +++ b/crates/expr/tests/support/audio_workload.rs @@ -0,0 +1,75 @@ +//! Configurable synthetic projects for exercising the real expression engine and audio mixer. +use effectcraft_color::Label; +use effectcraft_keyframe::Value; +use effectcraft_project::{ + Comp, Expression, Footage, FootageKind, ItemId, ItemKind, LayerSource, Project, + build::{self, Ids}, +}; +use effectcraft_render::{FootageSource, Image}; +use effectcraft_time::{FrameRate, Tick}; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; + +/// Counts decoded frames and reads; samples depend only on source time, never request boundaries. +#[derive(Default)] +pub struct Source { + pub reads: AtomicUsize, + pub frames: AtomicUsize, +} +impl FootageSource for Source { + /// This procedural audio source intentionally supplies no video frames. + fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option> { + None + } + /// Generate reproducible stereo samples and account for every source request. + fn audio(&self, _: ItemId, _: &Footage, start: Tick, frames: usize, rate: u32) -> Option> { + self.reads.fetch_add(1, Ordering::Relaxed); + self.frames.fetch_add(frames, Ordering::Relaxed); + let first = start.to_units_floor(rate as i64); + Some( + (0..frames) + .flat_map(|i| { + let v = ((first + i as i64).rem_euclid(97) as f32 / 97.0 - 0.5) * 0.01; + [v, -v] + }) + .collect(), + ) + } +} + +/// Build `layers` independent audio tracks wrapped in `depth` remapped precompositions. +/// All media is procedural; duration is deliberately longer than every benchmark run. +pub fn project(layers: usize, depth: usize, expression: Option<&str>) -> (Project, ItemId) { + let mut p = Project::default(); + let duration = Tick::from_seconds_f64(600.0); + let f = p.add_item( + "synthetic.wav", + Label::SeaFoam, + None, + ItemKind::Footage(Footage { kind: FootageKind::Audio, has_audio: true, duration, ..Default::default() }), + ); + let mut c = Comp::new(16, 16, FrameRate::new(24, 1), duration); + for _ in 0..layers { + let mut l = build::layer(&mut p, &c, "audio", LayerSource::Footage { item: f }, (0, 0), None); + let mut prop = Ids(&mut p.next_id).prop("timeRemap", "Time Remap", Value::Scalar(0.0)); + prop.keys = + vec![effectcraft_keyframe::Keyframe::new(Tick::ZERO, Value::Scalar(0.25)), effectcraft_keyframe::Keyframe::new(duration, Value::Scalar(300.25))]; + prop.expr = expression.map(|text| Expression { text: text.into(), enabled: true }); + l.props.children.push(prop.into()); + c.layers.push(l); + } + let mut cid = p.add_item("Audio", Label::Sandstone, None, ItemKind::Comp(c.into())); + for _ in 0..depth { + let mut c = Comp::new(16, 16, FrameRate::new(24, 1), duration); + let mut l = build::layer(&mut p, &c, "nested", LayerSource::Comp { item: cid }, (16, 16), None); + let mut prop = Ids(&mut p.next_id).prop("timeRemap", "Time Remap", Value::Scalar(0.0)); + prop.keys = + vec![effectcraft_keyframe::Keyframe::new(Tick::ZERO, Value::Scalar(0.0)), effectcraft_keyframe::Keyframe::new(duration, Value::Scalar(600.0))]; + l.props.children.push(prop.into()); + c.layers.push(l); + cid = p.add_item("Nested", Label::Sandstone, None, ItemKind::Comp(c.into())); + } + (p, cid) +} diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index bdb4e2bc33..df1d920085 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -54,6 +54,7 @@ impl Exporter for FileExporter { path: job.path, sink: self.sink.as_deref(), options: effectcraft_export::JobOptions { + cancel: job.cancel.clone(), log: job.item.log, label: job.label.clone(), storage: job.storage, diff --git a/crates/render/src/audio.rs b/crates/render/src/audio.rs index 68be63fe4f..8f4ed1d9d2 100644 --- a/crates/render/src/audio.rs +++ b/crates/render/src/audio.rs @@ -9,8 +9,9 @@ //! effects are run from a cleared state over a pre-roll of earlier audio and only the block's //! tail is kept — so preview playback and export produce identical samples. //! -//! Time-stretched layers are resampled nearest-sample; reversed (negative stretch) layers are -//! silent for now. +//! Time-stretched and time-remapped audio is linearly resampled. Time Remap uses the same +//! source-time evaluation as video, at audio sample resolution; it does not preserve pitch. +//! Reversed (negative stretch) layers are silent for now. use effectcraft_effects::Params; use effectcraft_effects::audio_fx; @@ -22,11 +23,152 @@ use crate::{EvalCtx, ExprHost, FootageSource}; const MAX_DEPTH: usize = 16; /// `frames` stereo sample frames of `comp`'s mix from comp time `start` at `rate` Hz, -/// interleaved (L R L R …). -pub fn mix_comp(project: &Project, footage: &dyn FootageSource, expr: Option<&dyn ExprHost>, comp: ItemId, start: Tick, frames: usize, rate: u32) -> Vec { +/// interleaved (L R L R …). Returns an error rather than partial audio when work is excessive. +pub fn mix_comp( + project: &Project, + footage: &dyn FootageSource, + expr: Option<&dyn ExprHost>, + comp: ItemId, + start: Tick, + frames: usize, + rate: u32, +) -> Result, AudioError> { + mix_comp_with_control(project, footage, expr, comp, start, frames, rate, &MixControl::default()) +} + +/// Failure aborts the entire block; callers must not play or export a partial mix. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AudioError { + Cancelled, + WorkLimit, +} +impl std::fmt::Display for AudioError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Cancelled => "audio mixing cancelled", + Self::WorkLimit => "audio mixing exceeded its work limit; simplify time stretching, remapping, or nested compositions", + }) + } +} +impl std::error::Error for AudioError {} + +/// Controls shared by every layer, source read and nested composition in one mix call. +#[derive(Default)] +pub struct MixControl { + /// Stop an in-flight mix, including expression evaluation when supported by the host. + pub cancel: Option>, + /// Override the default allowance of 1024 source/sample operations per output frame plus + /// 65536 for page overhead. Individual allocations are also capped at 1048576 stereo frames. + pub max_work: Option, + /// Native wall-clock allowance for the whole mix (five seconds by default). + pub max_duration: Option, + /// Shared JavaScript yield allowance (1024 yields of up to 20,000 instructions by default). + pub max_expression_yields: Option, +} + +/// Shared interruption state for every expression in one mix, including nested evaluations. +/// Clones share the sticky resource failure and yield allowance; a new sample cannot reset them. +#[derive(Clone)] +pub struct AudioEvalControl { + cancel: Option>, + remaining: std::rc::Rc>, + failed: std::rc::Rc>, + #[cfg(not(target_arch = "wasm32"))] + deadline: Option, +} +impl AudioEvalControl { + /// Capture the deadline once, before any sample or nested mix starts. + fn new(control: &MixControl) -> Self { + Self { + cancel: control.cancel.clone(), + remaining: std::rc::Rc::new(std::cell::Cell::new(control.max_expression_yields.unwrap_or(1024))), + failed: std::rc::Rc::new(std::cell::Cell::new(false)), + #[cfg(not(target_arch = "wasm32"))] + deadline: std::time::Instant::now().checked_add(control.max_duration.unwrap_or(std::time::Duration::from_secs(5))), + } + } + /// Mark a runtime resource limit; ordinary expression errors still use keyframed fallback. + pub fn fail(&self) { + self.failed.set(true); + } + /// Whether the shared resource allowance has expired (also used after script completion). + pub fn exhausted(&self) -> bool { + #[cfg(not(target_arch = "wasm32"))] + if self.deadline.is_some_and(|d| std::time::Instant::now() >= d) { + self.fail(); + } + self.failed.get() + } + /// Check cancellation and resource limits without charging another instruction chunk. + pub fn interrupted(&self) -> bool { + self.cancel.as_ref().is_some_and(|c| c.load(std::sync::atomic::Ordering::Relaxed)) || self.exhausted() + } + /// Charge one budgeted JavaScript yield; this also bounds scripts on clockless WASM builds. + pub fn yielded(&self) -> bool { + if let Some(left) = self.remaining.get().checked_sub(1) { + self.remaining.set(left); + } else { + self.fail(); + } + self.interrupted() + } +} + +struct Budget<'a> { + control: &'a MixControl, + expressions: AudioEvalControl, + remaining: std::cell::Cell, + error: std::cell::Cell>, +} +impl Budget<'_> { + /// Charge before allocating, decoding or evaluating; a failure is sticky for the whole mix. + fn charge(&self, frames: usize, footage: &dyn FootageSource) -> bool { + if self.error.get().is_some() { + return false; + } + if footage.called_off() || self.control.cancel.as_ref().is_some_and(|c| c.load(std::sync::atomic::Ordering::Relaxed)) { + self.error.set(Some(AudioError::Cancelled)); + return false; + } + if self.expressions.exhausted() || frames > 1_048_576 || frames > self.remaining.get() { + self.error.set(Some(AudioError::WorkLimit)); + return false; + } + self.remaining.set(self.remaining.get() - frames); + true + } +} + +/// Mix with cancellation and a cumulative budget; failures discard the entire output block. +#[allow(clippy::too_many_arguments)] +pub fn mix_comp_with_control( + project: &Project, + footage: &dyn FootageSource, + expr: Option<&dyn ExprHost>, + comp: ItemId, + start: Tick, + frames: usize, + rate: u32, + control: &MixControl, +) -> Result, AudioError> { + let cancellable = control.cancel.as_ref().map(|cancel| crate::CancellableFootage { inner: footage, cancel: cancel.clone() }); + let footage: &dyn FootageSource = cancellable.as_ref().map_or(footage, |source| source); + let budget = Budget { + control, + expressions: AudioEvalControl::new(control), + remaining: std::cell::Cell::new(control.max_work.unwrap_or(frames.saturating_mul(1024).saturating_add(65536))), + error: std::cell::Cell::new(None), + }; + if !budget.charge(frames, footage) { + return Err(budget.error.get().unwrap_or(AudioError::WorkLimit)); + } let mut out = vec![0.0f32; frames * 2]; - mix_into(project, footage, expr, comp, start, rate, &mut out, 1.0, 1.0, 0); - out + mix_into(project, footage, expr, comp, start, rate, &mut out, 1.0, 1.0, 0, &budget); + budget.charge(0, footage); + match budget.error.get() { + Some(e) => Err(e), + None => Ok(out), + } } /// Whether the comp has anything audible (footage with audio, a sound generator, or a precomp @@ -79,12 +221,14 @@ pub fn db_to_gain(db: f64) -> f32 { if db <= -96.0 { 0.0 } else { 10f64.powf(db / 20.0) as f32 } } +/// First sample index at or after a layer boundary. fn units_ceil(t: Tick, rate: u32) -> i64 { let n = t.0 as i128 * rate as i128; let d = TICKS_PER_SECOND as i128; (n.div_euclid(d) + i128::from(n.rem_euclid(d) != 0)) as i64 } +/// Represent a sample without rounding it back into the preceding sample. fn sample_tick(i: i64, rate: u32) -> Tick { // Keep the requested sample when the rate does not divide the tick clock. let t = Tick::from_units(i, i64::from(rate)); @@ -92,6 +236,7 @@ fn sample_tick(i: i64, rate: u32) -> Tick { } struct MixEnv<'a> { + budget: &'a Budget<'a>, project: &'a Project, footage: &'a dyn FootageSource, expr: Option<&'a dyn ExprHost>, @@ -102,6 +247,7 @@ struct MixEnv<'a> { } impl MixEnv<'_> { + /// Evaluation context at one composition sample. fn ctx(&self, t: Tick) -> EvalCtx<'_> { let mut ctx = EvalCtx::new(self.project, self.comp_id, self.comp, t); ctx.expr = self.expr; @@ -112,6 +258,9 @@ impl MixEnv<'_> { /// outside its [in, out) span. Comp sample index `k` maps to `in + out − 1 − k` when /// `reversed`. fn layer_raw(&self, l: &Layer, i0: i64, n: usize, reversed: bool) -> Vec { + if !self.budget.charge(n, self.footage) { + return Vec::new(); + } let rate = self.rate; let mut out = vec![0.0f32; n * 2]; let (in_i, out_i) = (units_ceil(l.in_point, rate), units_ceil(l.out_point, rate)); @@ -124,6 +273,9 @@ impl MixEnv<'_> { } let (ma, mb) = (in_i + out_i - b, in_i + out_i - a); let fwd = self.layer_raw(l, ma, (mb - ma) as usize, false); + if !self.budget.charge(0, self.footage) { + return out; + } for k in a..b { let j = (in_i + out_i - 1 - k - ma) as usize; let o = (k - i0) as usize; @@ -137,9 +289,25 @@ impl MixEnv<'_> { if a >= b { return out; } + if l.props.get("timeRemap").is_some() { + self.remapped_raw(l, i0, a, b, &mut out); + return out; + } let m = b - a; let speed = 100.0 / l.stretch; let src_n = ((m as f64 * speed).ceil() as usize).max(1); + if src_n > 1_048_576 { + // Paging removes the allocation limit, not the cumulative source-work allowance. + if src_n > self.budget.remaining.get() { + self.budget.error.set(Some(AudioError::WorkLimit)); + return out; + } + self.large_stretched_raw(l, i0, a, b, speed, src_n, &mut out); + return out; + } + if !self.budget.charge(src_n, self.footage) { + return out; + } let lt = l.layer_time(sample_tick(i0 + a as i64, rate)); let buf = match &l.source { LayerSource::Footage { item } => match self.project.item(*item).map(|i| &i.kind) { @@ -151,7 +319,7 @@ impl MixEnv<'_> { return out; } let mut sub = vec![0.0f32; src_n * 2]; - mix_into(self.project, self.footage, self.expr, *item, lt, rate, &mut sub, 1.0, 1.0, self.depth + 1); + mix_into(self.project, self.footage, self.expr, *item, lt, rate, &mut sub, 1.0, 1.0, self.depth + 1, self.budget); sub } _ => return out, @@ -173,13 +341,176 @@ impl MixEnv<'_> { out } - /// Effect parameters at comp time `t`. + /// Stream oversized stretched reads without allocating their whole source span. Each page + /// includes the next sample for interpolation; nested mixes share the same work budget. + #[allow(clippy::too_many_arguments)] + fn large_stretched_raw(&self, l: &Layer, i0: i64, a: usize, b: usize, speed: f64, src_n: usize, out: &mut [f32]) { + const PAGE: usize = 65536; + let first = l.layer_time(sample_tick(i0 + a as i64, self.rate)).to_units_floor(i64::from(self.rate)); + let mut cached = None; + let mut buf = Vec::new(); + for i in 0..b - a { + if !self.budget.charge(0, self.footage) { + return; + } + let x = i as f64 * speed; + let j = (x as usize).min(src_n - 1); + let page = j / PAGE * PAGE; + if cached != Some(page) { + let frames = (src_n - page).min(PAGE + 1); + if !self.budget.charge(frames, self.footage) { + return; + } + let Some(index) = i64::try_from(page).ok().and_then(|p| first.checked_add(p)) else { + self.budget.error.set(Some(AudioError::WorkLimit)); + return; + }; + let start = sample_tick(index, self.rate); + buf = match &l.source { + LayerSource::Footage { item } => match self.project.item(*item).map(|i| &i.kind) { + Some(ItemKind::Footage(f)) if f.has_audio => self.footage.audio(*item, f, start, frames, self.rate).unwrap_or_default(), + _ => return, + }, + LayerSource::Comp { item } => { + if self.project.comp_contains(*item, self.comp_id) { + return; + } + let mut sub = vec![0.0; frames * 2]; + mix_into(self.project, self.footage, self.expr, *item, start, self.rate, &mut sub, 1.0, 1.0, self.depth + 1, self.budget); + sub + } + _ => return, + }; + cached = Some(page); + } + let offset = (j - page) * 2; + let Some(pair) = buf.get(offset..offset + 2) else { return }; + let next = buf.get(offset + 2..offset + 4).unwrap_or(pair); + let fraction = ((x - j as f64) as f32).clamp(0.0, 1.0); + let Some(target) = out.get_mut((a + i) * 2..(a + i) * 2 + 2) else { return }; + for channel in 0..2 { + target[channel] = pair[channel] + (next[channel] - pair[channel]) * fraction; + } + } + } + + /// Fetch bounded source pages, rather than decoding once per output sample or allocating + /// the entire span between arbitrarily distant remap keys. Page alignment is independent + /// of the caller's buffer size, including the interpolation sample at each page boundary. + fn remapped_raw(&self, l: &Layer, i0: i64, a: usize, b: usize, out: &mut [f32]) { + const PAGE: i64 = 4096; + let Some(prop) = l.props.get("timeRemap") else { return }; + let prepared = prop.expr.as_ref().filter(|_| prop.has_expression()).and_then(|e| self.expr.and_then(|h| h.prepare_audio(&e.text))); + let mut pages: Vec<(i64, Vec)> = Vec::new(); + let mut next_page = 0; + + for i in a..b { + if !self.budget.charge(1, self.footage) { + return; + } + let Ok(offset) = i64::try_from(i) else { break }; + let Some(index) = i0.checked_add(offset) else { break }; + let time = sample_tick(index, self.rate); + let source = if let Some(prepared) = &prepared { + let base = prop.value_at(l.layer_time(time)); + if let effectcraft_keyframe::Value::Scalar(value) = base { + Tick::from_seconds_f64(prepared.evaluate(time.seconds(), value).unwrap_or(value)) + } else { + let value = self + .expr + .map(|h| h.eval_audio(&self.ctx(time), l, prop, &base, &self.budget.expressions).unwrap_or_else(|_| base.clone())) + .unwrap_or(base); + Tick::from_seconds_f64(value.as_f64()) + } + } else { + let base = prop.value_at(l.layer_time(time)); + let value = if prop.has_expression() { + self.expr.map(|h| h.eval_audio(&self.ctx(time), l, prop, &base, &self.budget.expressions).unwrap_or_else(|_| base.clone())).unwrap_or(base) + } else { + base + }; + Tick::from_seconds_f64(value.as_f64()) + }; + if !self.budget.charge(0, self.footage) { + return; + } + let numerator = i128::from(source.0) * i128::from(self.rate); + let denominator = i128::from(TICKS_PER_SECOND); + let Ok(sample) = i64::try_from(numerator.div_euclid(denominator)) else { continue }; + let Some(page) = sample.div_euclid(PAGE).checked_mul(PAGE) else { continue }; + let slot = if let Some(slot) = pages.iter().position(|(p, _)| *p == page) { + slot + } else { + let start = sample_tick(page, self.rate); + let frames = PAGE as usize + 1; + if !self.budget.charge(frames, self.footage) { + return; + } + let buf = match &l.source { + LayerSource::Footage { item } => match self.project.item(*item).map(|i| &i.kind) { + Some(ItemKind::Footage(f)) if f.has_audio => self.footage.audio(*item, f, start, frames, self.rate).unwrap_or_default(), + _ => return, + }, + LayerSource::Comp { item } => { + if self.project.comp_contains(*item, self.comp_id) { + return; + } + let mut sub = vec![0.0; frames * 2]; + mix_into(self.project, self.footage, self.expr, *item, start, self.rate, &mut sub, 1.0, 1.0, self.depth + 1, self.budget); + sub + } + _ => return, + }; + // Four pages cap retained stereo samples at about 128 KiB while retaining repeated jumps. + if pages.len() < 4 { + pages.push((page, buf)); + pages.len() - 1 + } else { + let slot = next_page; + if let Some(p) = pages.get_mut(slot) { + *p = (page, buf); + } + next_page = (next_page + 1) % 4; + slot + } + }; + let Some((_, buf)) = pages.get(slot) else { return }; + let j = (sample.rem_euclid(PAGE) as usize) * 2; + let fraction = (numerator.rem_euclid(denominator) as f64 / denominator as f64) as f32; + let Some(pair) = out.get_mut(i.saturating_mul(2)..i.saturating_mul(2).saturating_add(2)) else { break }; + for (channel, output) in pair.iter_mut().enumerate() { + let x = buf.get(j + channel).copied().unwrap_or(0.0); + let y = buf.get(j + 2 + channel).copied().unwrap_or(0.0); + *output = x + (y - x) * fraction; + } + } + } + + /// Evaluate any audio property under the same controls as per-sample Time Remap. + /// The host's scope also covers expressions reached through property references. + fn value(&self, l: &Layer, prop: &effectcraft_project::Property, t: Tick) -> effectcraft_keyframe::Value { + let base = prop.value_at(l.layer_time(t)); + if !self.budget.charge(0, self.footage) { + return base; + } + let value = if prop.has_expression() { + self.expr.map(|h| h.eval_audio(&self.ctx(t), l, prop, &base, &self.budget.expressions).unwrap_or_else(|_| base.clone())).unwrap_or(base) + } else { + base + }; + self.budget.charge(0, self.footage); + value + } + + /// Effect parameters at comp time `t`, stopping as soon as shared controls expire. fn params(&self, l: &Layer, g: &effectcraft_project::PropGroup, t: Tick) -> Params { - let ctx = self.ctx(t); let mut p = Params::default(); for c in &g.children { + if !self.budget.charge(0, self.footage) { + break; + } if let Node::Prop(pr) = c { - p.values.insert(pr.match_id.clone(), ctx.value(l, pr)); + p.values.insert(pr.match_id.clone(), self.value(l, pr, t)); } } p @@ -189,14 +520,35 @@ impl MixEnv<'_> { /// `i0 .. i0 + n`. fn layer_audio(&self, l: &Layer, i0: i64, n: usize) -> Vec { let t = sample_tick(i0, self.rate); - let fx: Vec<(&str, Params)> = audio_effect_groups(l).map(|(id, g)| (id, self.params(l, g, t))).collect(); + let mut fx = Vec::new(); + for (id, g) in audio_effect_groups(l) { + if !self.budget.charge(0, self.footage) { + return Vec::new(); + } + let params = self.params(l, g, t); + if !self.budget.charge(0, self.footage) { + return Vec::new(); + } + fx.push((id, params)); + } if fx.is_empty() { return self.layer_raw(l, i0, n, false); } let reversed = fx.iter().any(|(id, _)| audio_fx::reverses(id)); let pre_s: f64 = fx.iter().map(|(id, p)| audio_fx::preroll(id, p)).sum(); let pre = (pre_s * self.rate as f64).ceil() as usize; - let mut buf = self.layer_raw(l, i0 - pre as i64, n + pre, reversed); + let Some(total) = n.checked_add(pre).filter(|n| *n <= 1_048_576) else { + self.budget.error.set(Some(AudioError::WorkLimit)); + return Vec::new(); + }; + let Some(first) = i0.checked_sub(pre as i64) else { + self.budget.error.set(Some(AudioError::WorkLimit)); + return Vec::new(); + }; + let mut buf = self.layer_raw(l, first, total, reversed); + if !self.budget.charge(0, self.footage) { + return Vec::new(); + } let start = l.layer_time(sample_tick(i0 - pre as i64, self.rate)).seconds(); for (id, p) in &fx { audio_fx::process(id, p, &mut buf, self.rate, start); @@ -206,6 +558,7 @@ impl MixEnv<'_> { } } +/// Accumulate audible layers; nested calls share the root budget and cancellation state. #[allow(clippy::too_many_arguments)] fn mix_into( project: &Project, @@ -218,16 +571,24 @@ fn mix_into( gl: f32, gr: f32, depth: usize, + budget: &Budget, ) { let Some(comp) = project.comp(comp_id) else { return }; - if depth >= MAX_DEPTH || rate == 0 { + if depth >= MAX_DEPTH { + budget.error.set(Some(AudioError::WorkLimit)); + return; + } + if rate == 0 { return; } - let env = MixEnv { project, footage, expr, comp_id, comp, rate, depth }; + let env = MixEnv { project, footage, expr, comp_id, comp, rate, depth, budget }; let frames = out.len() / 2; let any_solo = comp.layers.iter().any(|l| l.switches.solo && audible(project, l)); let s0 = start.to_units_floor(rate as i64); for l in &comp.layers { + if !budget.charge(0, footage) { + return; + } if !audible(project, l) || (any_solo && !l.switches.solo) || l.stretch <= 0.0 { continue; } @@ -239,6 +600,9 @@ fn mix_into( } let t_a = sample_tick(s0 + a as i64, rate); let (ll, lr) = levels(&env, l, t_a); + if !budget.charge(0, footage) { + return; + } let (ll, lr) = (ll * gl, lr * gr); if ll == 0.0 && lr == 0.0 { continue; @@ -251,9 +615,10 @@ fn mix_into( } } +/// Evaluate the layer gains at the start of its audible block. fn levels(env: &MixEnv, l: &Layer, t: Tick) -> (f32, f32) { let Some(g) = l.props.sub("audio") else { return (1.0, 1.0) }; - let [a, b] = env.ctx(t).v2(l, g, "levels", [0.0, 0.0]); + let [a, b] = g.get("levels").map(|p| env.value(l, p, t).as_vec2()).unwrap_or([0.0, 0.0]); (db_to_gain(a), db_to_gain(b)) } @@ -355,7 +720,7 @@ mod tests { let cid = p.add_item("C", effectcraft_color::Label::Sandstone, None, ItemKind::Comp(comp.into())); assert!(comp_has_audio(&p, cid)); let rate = 1000; - let m = mix_comp(&p, &Dc, None, cid, Tick::from_seconds_f64(0.5), 1000, rate); + let m = mix_comp(&p, &Dc, None, cid, Tick::from_seconds_f64(0.5), 1000, rate).unwrap(); // first half: only layer b (half gain left, muted right) assert!((m[0] - 0.125).abs() < 1e-3 && m[1].abs() < 1e-6, "{} {}", m[0], m[1]); // second half (≥ 1 s): a + b @@ -405,8 +770,8 @@ mod tests { let outer_id = p.add_item("Outer", effectcraft_color::Label::Sandstone, None, ItemKind::Comp(outer.into())); let rate = 8000; let t = Tick::from_seconds_f64(0.5); - let direct = mix_comp(&p, &Tone, None, inner_id, t, 4000, rate); - let nested = mix_comp(&p, &Tone, None, outer_id, t, 4000, rate); + let direct = mix_comp(&p, &Tone, None, inner_id, t, 4000, rate).unwrap(); + let nested = mix_comp(&p, &Tone, None, outer_id, t, 4000, rate).unwrap(); assert_eq!(direct.len(), nested.len()); let worst = direct.iter().zip(&nested).map(|(a, b)| (a - b).abs()).fold(0.0f32, f32::max); assert!(worst < 1e-6, "a plain precomp changes its audio by {worst}"); @@ -432,7 +797,7 @@ mod tests { let cid = p.add_item("C", effectcraft_color::Label::Sandstone, None, ItemKind::Comp(comp.into())); let n = 2000usize; let t = Tick::from_seconds_f64(0.5); - let m = mix_comp(&p, &Tone, None, cid, t, n, rate); + let m = mix_comp(&p, &Tone, None, cid, t, n, rate).unwrap(); let speed = 100.0 / stretch; let (mut worst, mut sq) = (0.0f64, 0.0f64); for i in 0..n { diff --git a/crates/render/src/eval.rs b/crates/render/src/eval.rs index 8a193b87ac..a896820815 100644 --- a/crates/render/src/eval.rs +++ b/crates/render/src/eval.rs @@ -12,6 +12,15 @@ use effectcraft_time::Tick; pub trait ExprHost: Send + Sync { /// Evaluate the expression of `prop` (on `layer`, comp time `t`), given the keyframed value. fn eval(&self, ctx: &EvalCtx, layer: &Layer, prop: &Property, value: &Value) -> Result; + /// Prepare an exact scalar audio evaluator, or decline and use normal per-sample evaluation. + /// Implementations must reject syntax whose semantics they cannot preserve; no interpolation. + fn prepare_audio(&self, _text: &str) -> Option> { + None + } + /// Evaluate an audio property (sample remap or block parameter) under shared interruption controls. + fn eval_audio(&self, ctx: &EvalCtx, layer: &Layer, prop: &Property, value: &Value, _control: &crate::audio::AudioEvalControl) -> Result { + self.eval(ctx, layer, prop, value) + } /// Evaluate a text Expression Selector's Amount expression (on `prop`) for one unit: /// `textIndex` (1-based), `textTotal` and `selectorValue` (percent, per dimension) are /// defined. Returns the amount in percent per dimension. @@ -28,6 +37,12 @@ pub trait ExprHost: Send + Sync { } } +/// A side-effect-free numeric expression evaluated at every audio sample, with JS number semantics. +pub trait AudioExpression { + /// Comp seconds and the property's keyframed scalar value; `None` means expression error. + fn evaluate(&self, time: f64, value: f64) -> Option; +} + #[derive(Clone, Copy)] pub struct EvalCtx<'a> { pub project: &'a Project, diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index eccaed2f40..d0f83b857e 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -34,7 +34,7 @@ use effectcraft_project::{Comp, Footage, FootageKind, FrameBlend, GroupKind, Ite pub use effectcraft_raster::Image; use effectcraft_raster::{WarpOpts, composite_warp}; use effectcraft_time::{FrameRate, TICKS_PER_SECOND, Tick}; -pub use eval::{EvalCtx, ExprHost, effect_bounds, source_size}; +pub use eval::{AudioExpression, EvalCtx, ExprHost, effect_bounds, source_size}; use rayon::prelude::*; /// How a [`FootageSource`] hands over float footage (OpenEXR, float TIFF), whose values are @@ -160,7 +160,7 @@ impl FootageSource for CancellableFootage<'_> { self.read(|| self.inner.frame_at_size(item, footage, t, width, height)) } fn audio(&self, item: ItemId, footage: &Footage, t: Tick, frames: usize, rate: u32) -> Option> { - self.inner.audio(item, footage, t, frames, rate) + self.read(|| self.inner.audio(item, footage, t, frames, rate)) } fn aux(&self, item: ItemId, footage: &Footage, t: Tick) -> Option> { self.inner.aux(item, footage, t) diff --git a/crates/render/src/tests_audio_fx.rs b/crates/render/src/tests_audio_fx.rs index aced8208e7..057b2bad08 100644 --- a/crates/render/src/tests_audio_fx.rs +++ b/crates/render/src/tests_audio_fx.rs @@ -84,13 +84,168 @@ fn comp_with(id: Option<&str>, vals: &[(&str, Value)], solid: bool) -> (Project, } fn mix(p: &Project, src: &dyn FootageSource, cid: ItemId, from: f64, frames: usize) -> Vec { - mix_comp(p, src, None, cid, Tick::from_seconds_f64(from), frames, SR) + mix_comp(p, src, None, cid, Tick::from_seconds_f64(from), frames, SR).unwrap() } fn ramp(t: f64) -> (f32, f32) { (t as f32, -(t as f32)) } +/// Keys map layer seconds to source seconds; the synthetic source has distinct L/R samples. +fn remap(p: &mut Project, cid: ItemId, points: &[(f64, f64)]) { + let mut next = p.next_id; + let mut prop = Ids(&mut next).prop("timeRemap", "Time Remap", Value::Scalar(0.0)); + prop.keys = points.iter().map(|&(t, v)| effectcraft_keyframe::Keyframe::new(Tick::from_seconds_f64(t), Value::Scalar(v))).collect(); + p.next_id = next; + p.comp_mut(cid).unwrap().layers[0].props.children.push(prop.into()); +} + +#[test] +fn audio_time_remap_follows_offset_and_speed_changes_across_blocks() { + let (mut p, cid) = comp_with(None, &[], false); + remap(&mut p, cid, &[(0.0, 0.5), (0.25, 0.75), (0.75, 1.0), (1.0, 1.25)]); + let whole = mix(&p, &Signal(ramp), cid, 0.0, SR as usize); + for i in 0..SR as usize { + let t = i as f64 / SR as f64; + let expected = if t < 0.25 { + 0.5 + t + } else if t < 0.75 { + 0.75 + (t - 0.25) * 0.5 + } else { + 1.0 + t - 0.75 + }; + assert!((whole[i * 2] as f64 - expected).abs() < 1e-5, "sample {i}: {} != {expected}", whole[i * 2]); + assert!((whole[i * 2 + 1] as f64 + expected).abs() < 1e-5); + } + let mut blocked = Vec::new(); + for first in (0..SR as usize).step_by(137) { + blocked.extend(mix(&p, &Signal(ramp), cid, first as f64 / SR as f64, 137.min(SR as usize - first))); + } + assert_eq!(whole, blocked, "preview and export block sizes must not change the audio"); +} + +#[test] +fn audio_time_remap_handles_reverse_source_motion_and_layer_bounds() { + let (mut p, cid) = comp_with(None, &[], false); + remap(&mut p, cid, &[(0.0, 1.5), (1.0, 0.5)]); + let samples = mix(&p, &Signal(ramp), cid, 0.5, SR as usize); + for i in 0..SR as usize { + let expected = if i < SR as usize / 2 { 1.0 - i as f64 / SR as f64 } else { 0.0 }; + assert!((samples[i * 2] as f64 - expected).abs() < 1e-5); + } +} + +#[test] +fn audio_time_remap_evaluates_keys_in_stretched_layer_time() { + let (mut p, cid) = comp_with(None, &[], false); + remap(&mut p, cid, &[(0.0, 0.5), (1.0, 1.5)]); + let l = &mut p.comp_mut(cid).unwrap().layers[0]; + l.start_time = Tick::from_seconds_f64(0.25); + l.in_point = l.start_time; + l.out_point = Tick::from_seconds_f64(2.0); + l.stretch = 200.0; + let samples = mix(&p, &Signal(ramp), cid, 0.75, 200); + for i in 0..200 { + let expected = 0.75 + i as f64 / SR as f64 * 0.5; + assert!((samples[i * 2] as f64 - expected).abs() < 1e-5); + } +} + +#[test] +fn audio_time_remap_of_precomp_uses_nested_source_audio() { + let (mut p, inner) = comp_with(None, &[], false); + let mut outer = Comp::new(64, 64, FrameRate::new(25, 1), Tick::from_seconds_f64(1.0)); + outer.layers.push(build::layer(&mut p, &outer, "nested", LayerSource::Comp { item: inner }, (64, 64), None)); + let cid = p.add_item("Outer", effectcraft_color::Label::Sandstone, None, ItemKind::Comp(outer.into())); + remap(&mut p, cid, &[(0.0, 0.25), (1.0, 0.75)]); + let samples = mix(&p, &Signal(ramp), cid, 0.0, SR as usize); + for i in 0..SR as usize { + let expected = 0.25 + i as f64 / SR as f64 * 0.5; + assert!((samples[i * 2] as f64 - expected).abs() < 1e-5); + } +} + +#[test] +fn audio_time_remap_hold_repeats_the_selected_sample() { + let (mut p, cid) = comp_with(None, &[], false); + remap(&mut p, cid, &[(0.0, 0.75), (0.5, 1.25)]); + p.comp_mut(cid).unwrap().layers[0].props.prop_mut("timeRemap").unwrap().keys[0].out_interp = effectcraft_keyframe::Interp::Hold; + // A held picture selects a fixed audio sample (DC), not an advancing excerpt. + let samples = mix(&p, &Signal(ramp), cid, 0.0, SR as usize); + for (i, pair) in samples.as_chunks::<2>().0.iter().enumerate() { + let expected = if i < SR as usize / 2 { 0.75 } else { 1.25 }; + assert_eq!(*pair, [expected, -expected], "sample {i}"); + } +} + +#[test] +fn audio_time_remap_large_jump_keeps_source_requests_bounded() { + use std::sync::Mutex; + struct Requests(Mutex>); + impl FootageSource for Requests { + fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option> { + None + } + fn audio(&self, _: ItemId, _: &Footage, _: Tick, frames: usize, _: u32) -> Option> { + self.0.lock().unwrap().push(frames); + // Assert before allocating: a distant key must not request the intervening audio. + assert!(frames <= 4097, "unbounded source request: {frames}"); + Some(vec![0.25; frames * 2]) + } + } + let (mut p, cid) = comp_with(None, &[], false); + remap(&mut p, cid, &[(0.0, 0.0), (0.5, 1_000_000.0)]); + p.comp_mut(cid).unwrap().layers[0].props.prop_mut("timeRemap").unwrap().keys[0].out_interp = effectcraft_keyframe::Interp::Hold; + let src = Requests(Mutex::new(Vec::new())); + assert!(mix(&p, &src, cid, 0.0, SR as usize).iter().all(|v| *v == 0.25)); + assert_eq!(src.0.lock().unwrap().len(), 2, "one fetch per held source page"); +} + +#[test] +fn audio_time_remap_source_bounds_and_pages_at_44100_hz() { + let (mut p, cid) = comp_with(None, &[], false); + remap(&mut p, cid, &[(0.0, -0.25), (1.0, 2.75)]); + let rate = 44_100; + let src = Signal(|_| (0.5, -0.5)); + let whole = mix_comp(&p, &src, None, cid, Tick::ZERO, rate as usize, rate).unwrap(); + let mut blocked = Vec::new(); + for first in (0..rate as usize).step_by(137) { + blocked.extend(mix_comp(&p, &src, None, cid, Tick::from_units(first as i64, rate as i64), 137.min(rate as usize - first), rate).unwrap()); + } + assert_eq!(whole, blocked, "source page and caller boundaries must not affect samples"); + // Stay away from the one-sample interpolation transition at each source edge. + for (i, pair) in whole.as_chunks::<2>().0.iter().enumerate() { + let source = -0.25 + 3.0 * i as f64 / rate as f64; + if source < -0.001 || source > 2.001 { + assert_eq!(*pair, [0.0, 0.0], "sample {i}"); + } else if (0.001..1.999).contains(&source) { + assert_eq!(*pair, [0.5, -0.5], "sample {i}"); + } + } +} + +#[test] +fn audio_time_remap_expression_and_error_fallback() { + struct Expr(bool); + impl crate::ExprHost for Expr { + fn eval(&self, ctx: &crate::EvalCtx, _: &effectcraft_project::Layer, _: &effectcraft_project::Property, _: &Value) -> Result { + if self.0 { Ok(Value::Scalar(0.5 + ctx.time.seconds() * 0.5)) } else { Err("synthetic expression error".into()) } + } + } + let (mut p, cid) = comp_with(None, &[], false); + remap(&mut p, cid, &[(0.0, 1.25)]); + p.comp_mut(cid).unwrap().layers[0].props.prop_mut("timeRemap").unwrap().expr = + Some(effectcraft_project::Expression { text: "time / 2 + 0.5".into(), enabled: true }); + for enabled in [true, false] { + let samples = mix_comp(&p, &Signal(ramp), Some(&Expr(enabled)), cid, Tick::ZERO, SR as usize, SR).unwrap(); + for (i, pair) in samples.as_chunks::<2>().0.iter().enumerate() { + let expected = if enabled { 0.5 + i as f64 / SR as f64 * 0.5 } else { 1.25 }; + assert!((pair[0] as f64 - expected).abs() < 1e-5); + assert!((pair[1] as f64 + expected).abs() < 1e-5); + } + } +} + fn rms(m: &[f32]) -> f32 { (m.iter().map(|v| v * v).sum::() / m.len() as f32).sqrt() } @@ -253,3 +408,94 @@ fn audio_spectrum_follows_time_through_the_layer_cache() { assert!(lit(&fresh) > lit(&silent), "the tone draws more than the baseline: {} vs {}", lit(&fresh), lit(&silent)); assert_eq!(cached.data, fresh.data, "cached {} px, uncached {} px", lit(&cached), lit(&fresh)); } + +/// A normal export block can span more than the allocation cap at timelapse speeds. +/// Check every interpolated sample, including source-page edges and nested compositions. +#[test] +fn strongly_stretched_audio_reads_bounded_chunks() { + struct Indexed(std::sync::atomic::AtomicUsize); + impl FootageSource for Indexed { + fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option> { + None + } + fn audio(&self, _: ItemId, _: &Footage, start: Tick, frames: usize, rate: u32) -> Option> { + self.0.fetch_max(frames, std::sync::atomic::Ordering::Relaxed); + let first = start.to_units_floor(i64::from(rate)); + Some( + (0..frames) + .flat_map(|i| { + let v = ((first + i as i64) % 257) as f32 / 257.0; + [v, -v] + }) + .collect(), + ) + } + } + for rate in [44100, 48000] { + for speed in [65.0, 65.5355, 128.5] { + for nested in [false, true] { + let (mut p, inner) = comp_with(None, &[], false); + p.comp_mut(inner).unwrap().layers[0].out_point = Tick::from_seconds_f64(120.0); + let cid = if nested { + let mut outer = Comp::new(64, 64, FrameRate::new(25, 1), Tick::from_seconds_f64(1.0)); + let layer = build::layer(&mut p, &outer, "nested", LayerSource::Comp { item: inner }, (64, 64), None); + outer.layers.push(layer); + p.add_item("outer", effectcraft_color::Label::Sandstone, None, ItemKind::Comp(outer.into())) + } else { + inner + }; + p.comp_mut(cid).unwrap().layers[0].stretch = 100.0 / speed; + let source = Indexed(std::sync::atomic::AtomicUsize::new(0)); + let start = Tick::from_seconds_f64(0.1); + let first = p.comp(cid).unwrap().layers[0].layer_time(start).to_units_floor(i64::from(rate)) as usize; + let samples = mix_comp(&p, &source, None, cid, start, 16384, rate).unwrap(); + for i in 0..16384 { + let x = i as f64 * speed; + let j = x as usize; + let a = ((first + j) % 257) as f32 / 257.0; + let b = ((first + j + 1) % 257) as f32 / 257.0; + let expected = a + (b - a) * (x - j as f64) as f32; + assert!((samples[i * 2] - expected).abs() < 1e-6, "rate={rate} speed={speed} nested={nested} sample={i}"); + assert_eq!(samples[i * 2], -samples[i * 2 + 1]); + } + assert!(source.0.load(std::sync::atomic::Ordering::Relaxed) <= 1_048_576); + } + } + } +} + +/// Oversized stretched reads retain cumulative work limits and observe cancellation between pages. +#[test] +fn strongly_stretched_audio_retains_shared_limits() { + use crate::audio::{AudioError, MixControl, mix_comp_with_control}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + struct Stops { + cancel: Arc, + reads: AtomicUsize, + stop: bool, + } + impl FootageSource for Stops { + fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option> { + None + } + fn audio(&self, _: ItemId, _: &Footage, _: Tick, frames: usize, _: u32) -> Option> { + self.reads.fetch_add(1, Ordering::Relaxed); + if self.stop { + self.cancel.store(true, Ordering::Relaxed); + } + Some(vec![0.25; frames * 2]) + } + } + let (mut p, cid) = comp_with(None, &[], false); + p.comp_mut(cid).unwrap().layers[0].stretch = 1.0; + for stop in [false, true] { + let cancel = Arc::new(AtomicBool::new(false)); + let source = Stops { cancel: cancel.clone(), reads: AtomicUsize::new(0), stop }; + let control = MixControl { cancel: Some(cancel), max_work: if stop { None } else { Some(1_638_400 + 32_768) }, ..Default::default() }; + assert_eq!( + mix_comp_with_control(&p, &source, None, cid, Tick::ZERO, 16384, 48000, &control), + Err(if stop { AudioError::Cancelled } else { AudioError::WorkLimit }) + ); + assert_eq!(source.reads.load(Ordering::Relaxed), if stop { 1 } else { 24 }, "limits must stop source work early"); + } +} diff --git a/crates/ui-egui/src/audio.rs b/crates/ui-egui/src/audio.rs index 7972e69420..93d9351cdb 100644 --- a/crates/ui-egui/src/audio.rs +++ b/crates/ui-egui/src/audio.rs @@ -39,6 +39,12 @@ pub trait AudioDevice: Send { fn latency_frames(&self) -> u64 { 0 } + /// Conservative output-drain allowance at natural completion, in sample frames. + /// Output processing can retain a tail beyond the latency used for the sync clock. + /// Explicit Stop does not wait for this allowance. + fn drain_frames(&self) -> u64 { + self.latency_frames() + } /// Called every UI frame while playing: devices without an audio thread of their own (Web /// Audio) move queued samples to the output here. fn pump(&mut self) {} @@ -68,6 +74,7 @@ impl AudioOutput { #[derive(Default)] pub struct AudioFeed { queue: Mutex>, + error: Mutex>, /// Sample frames handed to the device (real audio only: underruns do not count). consumed: AtomicU64, /// Peak |sample| per channel since the last [`AudioFeed::take_peaks`] (f32 bits). @@ -77,6 +84,16 @@ pub struct AudioFeed { } impl AudioFeed { + /// Most recent mix failure, distinct from reaching the end of the work area. + pub fn error(&self) -> Option { + self.error.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone() + } + /// Stop feeding and preserve an actionable error for the UI. + fn fail(&self, error: String) { + *self.error.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(error); + self.finished.store(true, Ordering::Relaxed); + } + /// Fill `out` (interleaved stereo) from the queue; silence where the queue runs dry. pub fn pull(&self, out: &mut [f32]) { let mut real = 0usize; @@ -177,6 +194,8 @@ const BLOCK: usize = 1024; pub struct AudioPlayback { pub feed: Arc, device: Box, + /// Once the feed empties, keep the output open for its reported buffering latency. + drain: Mutex>, stop: Arc, thread: Option>, /// wasm32 (no threads): what the feeder thread would own, run by [`AudioPlayback::pump`]. @@ -213,7 +232,16 @@ impl AudioPlayback { let stop = Arc::new(AtomicBool::new(false)); if cfg!(target_arch = "wasm32") { // The device takes samples only in `pump`, which fills the queue first. - let mut p = AudioPlayback { feed: feed.clone(), device, stop, thread: None, feeder: Some((src, comp, span.start, mix)), rate, span }; + let mut p = AudioPlayback { + feed: feed.clone(), + device, + drain: Mutex::default(), + stop, + thread: None, + feeder: Some((src, comp, span.start, mix)), + rate, + span, + }; p.device.start(feed)?; p.pump(); return Ok(p); @@ -231,7 +259,7 @@ impl AudioPlayback { let target = queue_target(rate); while !stop.load(Ordering::Relaxed) { if feed.queued_frames() < target && !feed.finished.load(Ordering::Relaxed) { - cursor = feed_block(&src, comp, &feed, &span, cursor, rate, mix); + cursor = feed_block(&src, comp, &feed, &span, cursor, rate, mix, &stop); } else { std::thread::sleep(std::time::Duration::from_millis(4)); } @@ -240,7 +268,7 @@ impl AudioPlayback { .map_err(|e| e.to_string())? }; device.start(feed.clone())?; - Ok(AudioPlayback { feed, device, stop, thread: Some(thread), feeder: None, rate, span }) + Ok(AudioPlayback { feed, device, drain: Mutex::default(), stop, thread: Some(thread), feeder: None, rate, span }) } /// Every UI frame while playing: on wasm32 mix ahead into the feed (what the feeder thread @@ -249,7 +277,7 @@ impl AudioPlayback { if let Some((src, comp, cursor, mix)) = &mut self.feeder { let target = queue_target(self.rate); while self.feed.queued_frames() < target && !self.feed.finished.load(Ordering::Relaxed) { - *cursor = feed_block(src, *comp, &self.feed, &self.span, *cursor, self.rate, *mix); + *cursor = feed_block(src, *comp, &self.feed, &self.span, *cursor, self.rate, *mix, &self.stop); } } self.device.pump(); @@ -261,7 +289,24 @@ impl AudioPlayback { } pub fn finished(&self) -> bool { - self.feed.finished() + self.finished_at(web_time::Instant::now()) + } + + fn finished_at(&self, now: web_time::Instant) -> bool { + if !self.feed.finished() { + return false; + } + // Queue-empty means the last real samples were handed to the device. Closing + // it now discards the buffered tail. Take one conservative latency snapshot; + // explicit Stop still closes immediately, without waiting here or sleeping. + let latency = self.device.drain_frames(); + let rate = u64::from(self.rate.max(1)); + // The remainder is less than a u32 rate, so this product fits u64 and + // the nanosecond component is strictly less than one second. + let wait = std::time::Duration::new(latency / rate, ((latency % rate) * 1_000_000_000 / rate) as u32); + let mut drain = self.drain.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let (started, wait) = drain.get_or_insert((now, wait)); + now.saturating_duration_since(*started) >= *wait } pub fn stop(&mut self) { @@ -281,7 +326,8 @@ impl Drop for AudioPlayback { /// Mix one block at `cursor` into the feed; returns the next cursor (wrapping or finishing at /// the work-area end). -fn feed_block(src: &RenderSource, comp: ItemId, feed: &AudioFeed, span: &PlaySpan, cursor: i64, rate: u32, mix: u32) -> i64 { +#[allow(clippy::too_many_arguments)] +fn feed_block(src: &RenderSource, comp: ItemId, feed: &AudioFeed, span: &PlaySpan, cursor: i64, rate: u32, mix: u32, stop: &Arc) -> i64 { let mut cursor = cursor; if cursor >= span.wb { if !span.looping { @@ -291,20 +337,42 @@ fn feed_block(src: &RenderSource, comp: ItemId, feed: &AudioFeed, span: &PlaySpa cursor = span.wa; } let n = ((span.wb - cursor) as usize).min(BLOCK); - feed.push(&mix_block(src, comp, sample_time(cursor, rate), n, rate, mix)); + match mix_block(src, comp, sample_time(cursor, rate), n, rate, mix, Some(stop)) { + Ok(buf) => feed.push(&buf), + Err(e) => { + feed.fail(e.to_string()); + return cursor; + } + } cursor + n as i64 } /// `n` stereo frames of `comp`'s mix from comp time `t` at the device `rate`, mixed at `mix` Hz /// (resampled when they differ). -fn mix_block(src: &RenderSource, comp: ItemId, t: Tick, n: usize, rate: u32, mix: u32) -> Vec { - let render = - |frames: usize, at: u32| effectcraft_engine::render::audio::mix_comp(&src.project, src.footage.as_ref(), src.expr.as_deref(), comp, t, frames, at); +fn mix_block( + src: &RenderSource, + comp: ItemId, + t: Tick, + n: usize, + rate: u32, + mix: u32, + stop: Option<&Arc>, +) -> Result, effectcraft_engine::render::audio::AudioError> { + // Scrubbing runs on the UI thread: one snippet must not multiply per-expression timeouts. + let control = effectcraft_engine::render::audio::MixControl { + cancel: stop.cloned(), + max_duration: stop.is_none().then_some(std::time::Duration::from_millis(50)), + max_expression_yields: stop.is_none().then_some(32), + ..Default::default() + }; + let render = |frames: usize, at: u32| { + effectcraft_engine::render::audio::mix_comp_with_control(&src.project, src.footage.as_ref(), src.expr.as_deref(), comp, t, frames, at, &control) + }; if mix == rate { return render(n, rate); } let m = (n as u64 * mix as u64).div_ceil(rate as u64) as usize + 1; - resample_stereo(&render(m, mix), mix, rate, n) + Ok(resample_stereo(&render(m, mix)?, mix, rate, n)) } /// Audio scrubbing: an open output that plays a short, faded snippet of the comp's mix each time @@ -341,7 +409,14 @@ impl AudioScrub { let rate = self.rate; let mix = if mix_rate == 0 { rate } else { mix_rate.clamp(8000, 192_000) }; let n = (c.frame_duration().seconds().clamp(0.03, 0.1) * rate as f64) as usize; - let mut buf = mix_block(src, comp, c.frame_rate.tick_of(frame), n, rate, mix); + let mut buf = match mix_block(src, comp, c.frame_rate.tick_of(frame), n, rate, mix, None) { + Ok(buf) => buf, + Err(e) => { + self.feed.clear(); + self.feed.fail(e.to_string()); + return false; + } + }; // 3 ms fades: no clicks where snippets meet. let fade = (rate as usize * 3 / 1000).clamp(1, n / 2 + 1); let frames = buf.len() / 2; @@ -351,6 +426,7 @@ impl AudioScrub { buf[2 * i + 1] *= g; } self.feed.clear(); + *self.feed.error.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = None; self.feed.push(&buf); true } @@ -485,6 +561,49 @@ mod tests { assert_eq!(f.take_peaks(), [0.0, 0.0]); } + /// Emptying the feed means handed to the device, not audible yet. Dropping the + /// output at that point cut the last ~40--60 ms from Windows loopback captures. + #[test] + fn natural_completion_waits_for_device_tail() { + struct Buffered; + impl AudioDevice for Buffered { + fn sample_rate(&self) -> u32 { + 48000 + } + fn start(&mut self, _: Arc) -> Result<(), String> { + Ok(()) + } + fn stop(&mut self) {} + fn latency_frames(&self) -> u64 { + 4800 + } + } + let feed = Arc::new(AudioFeed::default()); + feed.push(&[0.5; 960]); + feed.finished.store(true, Ordering::Relaxed); + let mut playback = AudioPlayback { + feed: feed.clone(), + device: Box::new(Buffered), + drain: Mutex::default(), + stop: Arc::new(AtomicBool::new(false)), + thread: None, + feeder: None, + rate: 48000, + span: PlaySpan { start: 0, wa: 0, wb: 480, looping: false }, + }; + assert!(!playback.finished(), "queued samples are not finished"); + feed.pull(&mut [0.0; 960]); + assert!(feed.finished(), "the last samples were handed to the device"); + let now = web_time::Instant::now(); + assert!(!playback.finished_at(now), "keep the output open until those samples are audible"); + assert!(!playback.finished_at(now + std::time::Duration::from_millis(99))); + assert!(playback.finished_at(now + std::time::Duration::from_millis(100)), "natural completion is bounded by the device latency"); + // Explicit stop remains immediate; waiting for a natural tail never blocks it. + let start = std::time::Instant::now(); + playback.stop(); + assert!(start.elapsed() < std::time::Duration::from_millis(50)); + } + /// #408: starting a preview with sound returns at once even when reading the footage takes /// long (the first read of a long FLAC decodes all of it): the first block was mixed on the /// calling (UI) thread. The sound arrives from the feeder thread, and the clock waits for it. diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 72016cc537..bdc63d34ce 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -1073,6 +1073,9 @@ impl EffectcraftApp { let mix = self.session.prefs.audio.preview_sample_rate; if let Some(s) = &mut self.scrub { s.play(&src, comp, t, mix, now); + if let Some(error) = s.feed.error() { + self.ui.status = error; + } } } @@ -1296,6 +1299,11 @@ impl EffectcraftApp { } // Audio clock drives playback: show the frame under the audible sample. if let Some(a) = &self.audio { + if let Some(error) = a.feed.error() { + self.ui.status = error; + self.stop(); + return; + } if a.finished() { if video { self.session.set_time(fr.tick_of(wb)); diff --git a/crates/ui-egui/tests/ui_audio_scrub.rs b/crates/ui-egui/tests/ui_audio_scrub.rs index 02fefb14f7..626c8cd304 100644 --- a/crates/ui-egui/tests/ui_audio_scrub.rs +++ b/crates/ui-egui/tests/ui_audio_scrub.rs @@ -192,3 +192,83 @@ fn preview_with_audio_sounds_once_rendering_keeps_up() { } assert!(h.state().audio.is_some() && fake.0.lock().unwrap().0.is_some(), "the sound started within 2 s, at frame {last} of 900"); } + +/// Drive a real Cmd-drag from the current-time indicator, leaving the pointer held for recovery. +fn begin_scrub(h: &mut Harness<'_, EffectcraftApp>) -> egui::Pos2 { + let cti = h.state().auto.find("timeline.cti").unwrap().clone(); + let from = pos2(cti.rect[0] + cti.rect[2] / 2.0, cti.rect[1] + cti.rect[3] / 2.0); + h.event(Event::ModifiersChanged(Modifiers::COMMAND)); + h.event(Event::PointerMoved(from)); + h.step(); + h.event(Event::PointerButton { pos: from, button: egui::PointerButton::Primary, pressed: true, modifiers: Modifiers::COMMAND }); + h.step(); + let to = from + egui::vec2(40.0, 0.0); + h.event(Event::PointerMoved(to)); + h.step(); + h.step(); + to +} + +/// Real scrub gestures report both source-work and script-limit errors, draw the toast, and recover. +#[test] +fn scrub_reports_excessive_audio_work() { + for script in [false, true] { + let fake = Fake::default(); + let (mut app, cid) = setup(true, &fake); + // A failed new scrub must also discard the previous successful snippet. + app.scrub_audio(cid, Tick::ZERO, 0.0); + assert!(fake.0.lock().unwrap().0.as_ref().unwrap().queued_frames() > 0); + if script { + app.session.expr = effectcraft_host::session().expr; + let p = Arc::make_mut(&mut app.session.project); + let mut prop = effectcraft_project::build::Ids(&mut p.next_id).prop("timeRemap", "Time Remap", effectcraft_keyframe::Value::Scalar(0.0)); + prop.expr = Some(effectcraft_project::Expression { + text: "function g() { for (var i=0;i<900000;i++) {} } for (var k=0;k<900000;k++) g()".into(), + enabled: true, + }); + p.comp_mut(cid).unwrap().layers[0].props.children.push(prop.into()); + } else { + Arc::make_mut(&mut app.session.project).comp_mut(cid).unwrap().layers[0].stretch = 0.000001; + } + // Check the same feed before harness startup can close an idle scrub device. + app.scrub_audio(cid, Tick::from_units(1, 30), 0.1); + let feed = fake.0.lock().unwrap().0.clone().unwrap(); + assert_eq!(feed.queued_frames(), 0); + assert!(feed.error().is_some()); + // Use a separate app/device so the gesture cannot inherit the direct call's error. + let project = app.session.project.clone(); + let expr = app.session.expr.clone(); + drop(app); + let fake = Fake::default(); + let (mut app, fresh_cid) = setup(true, &fake); + assert_eq!(fresh_cid, cid); + app.session.project = project; + app.session.expr = expr; + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_| app); + h.run_steps(3); + assert!(h.state().ui.status.is_empty(), "the drag starts without a prior error"); + assert!(fake.0.lock().unwrap().0.is_none(), "the drag starts with no prior audio feed"); + let start = std::time::Instant::now(); + let at = begin_scrub(&mut h); + assert!(start.elapsed() < std::time::Duration::from_secs(3), "scrub must not retry per-sample timeouts"); + let error = "audio mixing exceeded its work limit; simplify time stretching, remapping, or nested compositions"; + assert!(h.state().ui.status.contains(error)); + assert!(h.output().shapes.iter().any(|s| matches!(&s.shape, egui::Shape::Text(t) if t.galley.text().contains(error))), "visible error toast"); + assert!(h.state().session.time() > Tick::ZERO, "the gesture changed the current time"); + let feed = fake.0.lock().unwrap().0.clone().unwrap(); + assert_eq!(feed.queued_frames(), 0); + assert!(feed.error().is_some()); + let layer = &mut Arc::make_mut(&mut h.state_mut().session.project).comp_mut(cid).unwrap().layers[0]; + layer.stretch = 100.0; + if let Some(prop) = layer.props.prop_mut("timeRemap") { + prop.expr = None; + } + let to = at + egui::vec2(40.0, 0.0); + h.event(Event::PointerMoved(to)); + h.step(); + assert!(feed.queued_frames() > 0); + assert!(feed.error().is_none(), "a later successful mix recovers in the same session"); + h.event(Event::PointerButton { pos: to, button: egui::PointerButton::Primary, pressed: false, modifiers: Modifiers::COMMAND }); + h.step(); + } +}