diff --git a/crates/engine/src/commands/frontend.rs b/crates/engine/src/commands/frontend.rs index 54623266..296160d1 100644 --- a/crates/engine/src/commands/frontend.rs +++ b/crates/engine/src/commands/frontend.rs @@ -46,6 +46,14 @@ pub fn specs() -> Vec { ), fe!("playback.cacheWhenIdle", "Cache Frames When Idle", ["Composition", "Preview"], None, "{value?}", always), fe!("playback.audio", "Audio", ["Composition", "Preview"], None, "{value?: include audio in previews}", always), + fe!( + "playback.scrubAudio", + "Scrub Audio", + [], + None, + "{time? (s, default the current time)} — plays one frame of the comp's audio there (Ctrl/Cmd-drag the current time)", + has_comp + ), // Viewer. fe!("view.zoomIn", "Zoom In", ["View"], Some("."), "{}", has_comp), fe!("view.zoomOut", "Zoom Out", ["View"], Some(","), "{}", has_comp), diff --git a/crates/ui-egui/src/audio.rs b/crates/ui-egui/src/audio.rs index 7e24adfe..1456f601 100644 --- a/crates/ui-egui/src/audio.rs +++ b/crates/ui-egui/src/audio.rs @@ -14,6 +14,9 @@ //! than drifting. //! //! The feed also tracks per-channel peaks for the Audio panel's VU meters ([`Meter`]). +//! +//! **Scrubbing:** Ctrl/Cmd-dragging the current-time indicator plays a short snippet of the mix +//! at each new frame ([`AudioScrub`], `playback.scrubAudio`), as in After Effects. use std::collections::VecDeque; use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; @@ -93,6 +96,13 @@ impl AudioFeed { } } + /// Drop what is still queued (scrubbing replaces the snippet still playing). + pub fn clear(&self) { + if let Ok(mut q) = self.queue.lock() { + q.clear(); + } + } + pub fn push(&self, samples: &[f32]) { if let Ok(mut q) = self.queue.lock() { q.extend(samples.iter().copied()); @@ -278,18 +288,82 @@ fn feed_block(src: &RenderSource, comp: ItemId, feed: &AudioFeed, span: &PlaySpa cursor = span.wa; } let n = ((span.wb - cursor) as usize).min(BLOCK); - let t = sample_time(cursor, rate); - let buf = if mix == rate { - effectcraft_engine::render::audio::mix_comp(&src.project, src.footage.as_ref(), src.expr.as_deref(), comp, t, n, rate) - } else { - let m = (n as u64 * mix as u64).div_ceil(rate as u64) as usize + 1; - let b = effectcraft_engine::render::audio::mix_comp(&src.project, src.footage.as_ref(), src.expr.as_deref(), comp, t, m, mix); - resample_stereo(&b, mix, rate, n) - }; - feed.push(&buf); + feed.push(&mix_block(src, comp, sample_time(cursor, rate), n, rate, mix)); cursor + n as i64 } +/// `n` stereo frames of `comp`'s mix from comp time `t` at the device `rate`, mixed at `mix` Hz +/// (resampled when they differ). +fn mix_block(src: &RenderSource, comp: ItemId, t: Tick, n: usize, rate: u32, mix: u32) -> Vec { + let render = + |frames: usize, at: u32| effectcraft_engine::render::audio::mix_comp(&src.project, src.footage.as_ref(), src.expr.as_deref(), comp, t, frames, at); + if mix == rate { + return render(n, rate); + } + let m = (n as u64 * mix as u64).div_ceil(rate as u64) as usize + 1; + resample_stereo(&render(m, mix), mix, rate, n) +} + +/// Audio scrubbing: an open output that plays a short, faded snippet of the comp's mix each time +/// the current time moves to another frame (each snippet replaces what is still queued, so the +/// sound follows the pointer without lagging behind). +pub struct AudioScrub { + pub feed: Arc, + device: Box, + pub rate: u32, + /// The comp and frame last played (holding still repeats nothing). + last: Option<(ItemId, i64)>, + /// When it last played (UI seconds): an idle scrub closes the device. + pub last_used: f64, +} + +impl AudioScrub { + pub fn open(mut device: Box, now: f64) -> Result { + let rate = device.sample_rate().max(8000); + let feed = Arc::new(AudioFeed::default()); + device.start(feed.clone())?; + Ok(AudioScrub { feed, device, rate, last: None, last_used: now }) + } + + /// Play the frame of `comp` at `t` (one frame long, 30–100 ms) unless it was the last one + /// played. Returns whether a snippet was queued. + pub fn play(&mut self, src: &RenderSource, comp: ItemId, t: Tick, mix_rate: u32, now: f64) -> bool { + self.last_used = now; + let Some(c) = src.project.comp(comp) else { return false }; + let frame = c.frame_rate.frame_at(t); + if self.last == Some((comp, frame)) { + return false; + } + self.last = Some((comp, frame)); + let rate = self.rate; + let mix = if mix_rate == 0 { rate } else { mix_rate.clamp(8000, 192_000) }; + let n = (c.frame_duration().seconds().clamp(0.03, 0.1) * rate as f64) as usize; + let mut buf = mix_block(src, comp, c.frame_rate.tick_of(frame), n, rate, mix); + // 3 ms fades: no clicks where snippets meet. + let fade = (rate as usize * 3 / 1000).clamp(1, n / 2 + 1); + let frames = buf.len() / 2; + for i in 0..frames { + let g = (i.min(frames - 1 - i) as f32 / fade as f32).min(1.0); + buf[2 * i] *= g; + buf[2 * i + 1] *= g; + } + self.feed.clear(); + self.feed.push(&buf); + true + } + + /// Every UI frame while open (Web Audio moves queued samples to the output here). + pub fn pump(&mut self) { + self.device.pump(); + } +} + +impl Drop for AudioScrub { + fn drop(&mut self) { + self.device.stop(); + } +} + /// Linear resampling of interleaved stereo from `from` Hz to `n` frames at `to` Hz. pub fn resample_stereo(src: &[f32], from: u32, to: u32, n: usize) -> Vec { let frames = src.len() / 2; diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index a2ac5cd6..3704dfbe 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -211,6 +211,8 @@ pub struct EffectcraftApp { pub integrated_titlebar: bool, /// Audio preview while playing (audio clock drives playback). pub audio: Option, + /// Audio scrubbing's open output (Ctrl/Cmd-drag the current time, `playback.scrubAudio`). + pub scrub: Option, /// Audio panel VU meters. pub meter: audio::Meter, /// Waveform peak summaries per footage item (see `panels::waveform`). @@ -276,6 +278,7 @@ impl EffectcraftApp { dialog_state: Default::default(), integrated_titlebar: false, audio: None, + scrub: None, meter: Default::default(), waveforms: Default::default(), last_reveal: None, @@ -677,6 +680,30 @@ impl EffectcraftApp { } } + /// Audio scrubbing: play the frame of `comp` at `t` (see [`audio::AudioScrub`]), opening the + /// output on first use. Silent comps and missing devices do nothing. + pub fn scrub_audio(&mut self, comp: ItemId, t: Tick, now: f64) { + if self.playback.playing || !effectcraft_engine::render::audio::comp_has_audio(&self.session.project, comp) { + return; + } + if self.scrub.is_none() { + let out = audio::AudioOutput::from_prefs(&self.session.prefs); + let Some(dev) = self.hooks.audio_device.as_ref().and_then(|f| f(&out)) else { return }; + match audio::AudioScrub::open(dev, now) { + Ok(s) => self.scrub = Some(s), + Err(e) => { + log::warn!("audio scrubbing: {e}"); + return; + } + } + } + let src = self.render_source(); + let mix = self.session.prefs.audio.preview_sample_rate; + if let Some(s) = &mut self.scrub { + s.play(&src, comp, t, mix, now); + } + } + pub fn stop(&mut self) { let was = std::mem::replace(&mut self.playback.playing, false); self.audio = None; @@ -845,6 +872,14 @@ impl EffectcraftApp { if let Some(a) = &mut self.audio { a.pump(); self.meter.update(a.feed.take_peaks(), now); + } else if let Some(s) = &mut self.scrub { + // Scrubbing: the meters follow the snippets; an idle second closes the output. + s.pump(); + self.meter.update(s.feed.take_peaks(), now); + if now - s.last_used > 1.0 { + self.scrub = None; + } + ctx.request_repaint_after(std::time::Duration::from_millis(50)); } else if self.meter.active() { self.meter.update([0.0; 2], now); ctx.request_repaint(); diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index cc6e1f94..01aa82e2 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -562,6 +562,13 @@ pub fn frontend(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, p: Valu Value::Null } // Window ▸ : dock (or bring forward) the panel the script built. + // Audio scrubbing: one frame of audio at `time` (default the current time). + "playback.scrubAudio" => { + let cid = app.session.active_comp_id().ok_or("no composition")?; + let t = p.get("time").and_then(Value::as_f64).map_or(app.session.time(), effectcraft_engine::time::Tick::from_seconds_f64); + app.scrub_audio(cid, t, now); + json!({"playing": app.scrub.is_some()}) + } // View ▸ New Viewer. "view.newViewer" => json!({"viewer": crate::panels::viewers::new_viewer(app)}), "window.scriptPanel" => { diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index f467b489..a9d81173 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -1048,6 +1048,15 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let tt = Tick::from_seconds_f64(secs); app.session.set_time(tt); app.stop(); + // Ctrl/Cmd-drag scrubs the audio too (After Effects). + if rresp.dragged() + && ui.input(|i| i.modifiers.command) + && let Some(cid) = app.session.active_comp_id() + { + let now = ui.input(|i| i.time); + let t = app.session.time(); + app.scrub_audio(cid, t, now); + } } // ---- column headers (right-click: show/hide columns; click the name header: Source/Layer Name). diff --git a/crates/ui-egui/tests/ui_audio_scrub.rs b/crates/ui-egui/tests/ui_audio_scrub.rs new file mode 100644 index 00000000..caba66d7 --- /dev/null +++ b/crates/ui-egui/tests/ui_audio_scrub.rs @@ -0,0 +1,109 @@ +//! Audio scrubbing: Ctrl/Cmd-dragging the current time plays one frame of the comp's mix at each +//! new frame (each snippet replacing what is still queued); holding still repeats nothing; silent +//! comps open no output; an idle scrub closes it. + +use std::sync::{Arc, Mutex}; + +use effectcraft_engine::Session; +use effectcraft_engine::project::ItemId; +use effectcraft_engine::time::Tick; +use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::audio::{AudioDevice, AudioFeed}; +use egui::{Event, Modifiers, pos2}; +use egui_kittest::Harness; +use serde_json::json; + +/// An output that records the feed it is given and whether it was stopped. +#[derive(Clone, Default)] +struct Fake(Arc>, bool)>>); + +impl AudioDevice for Fake { + fn sample_rate(&self) -> u32 { + 48000 + } + fn start(&mut self, feed: Arc) -> Result<(), String> { + self.0.lock().unwrap().0 = Some(feed); + Ok(()) + } + fn stop(&mut self) { + self.0.lock().unwrap().1 = true; + } +} + +/// A 30 fps comp whose solid sounds a tone (Audio switch on). +fn setup(audible: bool, fake: &Fake) -> (EffectcraftApp, ItemId) { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "A", "width": 64, "height": 36, "frameRate": 30, "duration": 2})).unwrap(); + let cid = s.active_comp_id().unwrap(); + let l = s.execute("layer.newSolid", json!({"color": "#808080"})).unwrap()["layer"].as_u64().unwrap(); + if audible { + s.execute("effect.apply", json!({"layers": [l], "effect": "ec.audio.tone"})).unwrap(); + s.execute("layer.setSwitch", json!({"layers": [l], "switch": "audio", "value": true})).unwrap(); + } + let mut app = EffectcraftApp::new(s); + let f = fake.clone(); + app.hooks.audio_device = Some(Box::new(move |_| Some(Box::new(f.clone()) as Box))); + (app, cid) +} + +#[test] +fn scrubbing_plays_one_frame_per_new_frame() { + let fake = Fake::default(); + let (mut app, cid) = setup(true, &fake); + let t = |s: f64| Tick::from_seconds_f64(s); + app.scrub_audio(cid, t(0.5), 0.0); + let feed = fake.0.lock().unwrap().0.clone().expect("the output opened"); + // One 30 fps frame at 48 kHz, and not silence. + assert_eq!(feed.queued_frames(), 1600); + let mut out = vec![0.0f32; 3200]; + feed.pull(&mut out); + assert!(out.iter().any(|v| v.abs() > 0.01), "the tone is there"); + // Faded in: the first sample is silent. + assert_eq!(out[0], 0.0); + // The same frame again plays nothing; the next frame replaces what is left. + app.scrub_audio(cid, t(0.51), 0.1); + assert_eq!(feed.queued_frames(), 0); + feed.push(&[0.0; 1000]); + app.scrub_audio(cid, t(0.6), 0.2); + assert_eq!(feed.queued_frames(), 1600, "replaced, not piled up"); + // A silent comp opens nothing. + let silent = Fake::default(); + let (mut quiet, qid) = setup(false, &silent); + quiet.scrub_audio(qid, t(0.5), 0.0); + assert!(silent.0.lock().unwrap().0.is_none() && quiet.scrub.is_none()); +} + +#[test] +fn cmd_dragging_the_current_time_scrubs_and_an_idle_scrub_closes() { + let fake = Fake::default(); + let (app, _) = setup(true, &fake); + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_| app); + h.run_steps(3); + let cti = h.state().auto.find("timeline.cti").expect("current time indicator").clone(); + let from = pos2(cti.rect[0] + cti.rect[2] / 2.0, cti.rect[1] + cti.rect[3] / 2.0); + let cmd = Modifiers::COMMAND; + h.event(Event::ModifiersChanged(cmd)); + h.event(Event::PointerMoved(from)); + h.step(); + h.event(Event::PointerButton { pos: from, button: egui::PointerButton::Primary, pressed: true, modifiers: cmd }); + h.step(); + for k in 1..=4 { + h.event(Event::PointerMoved(from + egui::vec2(40.0 * k as f32, 0.0))); + h.step(); + } + assert!(h.state().scrub.is_some(), "Cmd-drag opened the scrub output"); + assert!(fake.0.lock().unwrap().0.as_ref().is_some_and(|f| f.queued_frames() > 0)); + h.event(Event::PointerButton { pos: from + egui::vec2(160.0, 0.0), button: egui::PointerButton::Primary, pressed: false, modifiers: cmd }); + h.step(); + h.event(Event::ModifiersChanged(Modifiers::NONE)); + h.step(); + // An idle second closes the output. + for _ in 0..90 { + h.step(); + std::thread::sleep(std::time::Duration::from_millis(15)); + if h.state().scrub.is_none() { + break; + } + } + assert!(h.state().scrub.is_none() && fake.0.lock().unwrap().1, "closed and stopped"); +} diff --git a/docs/parity.md b/docs/parity.md index 90d6367c..79711127 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -422,6 +422,12 @@ layers); the Graph Editor has Auto-Select Graph Type (the default: the speed gra spatial properties are shown, else the value graph; choosing Value or Speed turns it off). Graph Editor keys are small squares whatever their interpolation, as in After Effects. +Since (M13.34): audio scrubbing. Ctrl-dragging the current-time indicator (Cmd on macOS) plays +one frame (30–100 ms) of the comp's mix at each new frame, as in After Effects. Snippets fade in +and out and replace what is still queued, so the sound follows the pointer without lagging. +Holding still repeats nothing, the Audio panel meters follow, and the output closes after an idle +second. Agents call `playback.scrubAudio {time}`. + Since (M13.33): a precomp layer's bar shows its comp's markers, as outlined, read-only markers. They are mapped through the layer's timing (start, stretch, time remapping, Responsive Design stretch) and shown within its In–Out range. Hovering names them ("beat (marker in Pre)"), and a @@ -758,7 +764,7 @@ are pinned bit for bit by golden hashes (`crates/effects/tests/particle_golden.r 17. ~~Content-keyed RAM preview frames, so an edit keeps the frames of comps it doesn't touch.~~ (M4.14) 18. ~~View ▸ New Viewer (several Composition viewers)~~ (M13.32); ~~nested comp markers on the - precomp layer bar~~ (M13.33), audio scrubbing. + precomp layer bar~~ (M13.33), ~~audio scrubbing~~ (M13.34). At the original audit the engine underneath (keyframes, expressions, shape operators, text animators, Classic 3D, all 38 blend modes, the render queue) was already deep, and most of what was