Merge pull request #57 from storytold/m13-audio-scrub

M13.34: audio scrubbing (Ctrl/Cmd-drag the current time)

crates/engine/src/commands/frontend.rs

@@ -46,6 +46,14 @@ pub fn specs() -> Vec<CommandSpec> {
         ),
         fe!("playback.cacheWhenIdle", "Cache Frames When Idle", ["Composition", "Preview"], None, "{value?}", always),
         fe!("playback.audio", "Audio", ["Composition", "Preview"], None, "{value?: include audio in previews}", always),
+        fe!(
+            "playback.scrubAudio",
+            "Scrub Audio",
+            [],
+            None,
+            "{time? (s, default the current time)} — plays one frame of the comp's audio there (Ctrl/Cmd-drag the current time)",
+            has_comp
+        ),
         // Viewer.
         fe!("view.zoomIn", "Zoom In", ["View"], Some("."), "{}", has_comp),
         fe!("view.zoomOut", "Zoom Out", ["View"], Some(","), "{}", has_comp),

crates/ui-egui/src/audio.rs

@@ -14,6 +14,9 @@
 //! than drifting.
 //!
 //! The feed also tracks per-channel peaks for the Audio panel's VU meters ([`Meter`]).
+//!
+//! **Scrubbing:** Ctrl/Cmd-dragging the current-time indicator plays a short snippet of the mix
+//! at each new frame ([`AudioScrub`], `playback.scrubAudio`), as in After Effects.
 
 use std::collections::VecDeque;
 use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
@@ -93,6 +96,13 @@ impl AudioFeed {
         }
     }
 
+    /// Drop what is still queued (scrubbing replaces the snippet still playing).
+    pub fn clear(&self) {
+        if let Ok(mut q) = self.queue.lock() {
+            q.clear();
+        }
+    }
+
     pub fn push(&self, samples: &[f32]) {
         if let Ok(mut q) = self.queue.lock() {
             q.extend(samples.iter().copied());
@@ -278,18 +288,82 @@ fn feed_block(src: &RenderSource, comp: ItemId, feed: &AudioFeed, span: &PlaySpa
         cursor = span.wa;
     }
     let n = ((span.wb - cursor) as usize).min(BLOCK);
-    let t = sample_time(cursor, rate);
-    let buf = if mix == rate {
-        effectcraft_engine::render::audio::mix_comp(&src.project, src.footage.as_ref(), src.expr.as_deref(), comp, t, n, rate)
-    } else {
-        let m = (n as u64 * mix as u64).div_ceil(rate as u64) as usize + 1;
-        let b = effectcraft_engine::render::audio::mix_comp(&src.project, src.footage.as_ref(), src.expr.as_deref(), comp, t, m, mix);
-        resample_stereo(&b, mix, rate, n)
-    };
-    feed.push(&buf);
+    feed.push(&mix_block(src, comp, sample_time(cursor, rate), n, rate, mix));
     cursor + n as i64
 }
 
+/// `n` stereo frames of `comp`'s mix from comp time `t` at the device `rate`, mixed at `mix` Hz
+/// (resampled when they differ).
+fn mix_block(src: &RenderSource, comp: ItemId, t: Tick, n: usize, rate: u32, mix: u32) -> Vec<f32> {
+    let render =
+        |frames: usize, at: u32| effectcraft_engine::render::audio::mix_comp(&src.project, src.footage.as_ref(), src.expr.as_deref(), comp, t, frames, at);
+    if mix == rate {
+        return render(n, rate);
+    }
+    let m = (n as u64 * mix as u64).div_ceil(rate as u64) as usize + 1;
+    resample_stereo(&render(m, mix), mix, rate, n)
+}
+
+/// Audio scrubbing: an open output that plays a short, faded snippet of the comp's mix each time
+/// the current time moves to another frame (each snippet replaces what is still queued, so the
+/// sound follows the pointer without lagging behind).
+pub struct AudioScrub {
+    pub feed: Arc<AudioFeed>,
+    device: Box<dyn AudioDevice>,
+    pub rate: u32,
+    /// The comp and frame last played (holding still repeats nothing).
+    last: Option<(ItemId, i64)>,
+    /// When it last played (UI seconds): an idle scrub closes the device.
+    pub last_used: f64,
+}
+
+impl AudioScrub {
+    pub fn open(mut device: Box<dyn AudioDevice>, now: f64) -> Result<AudioScrub, String> {
+        let rate = device.sample_rate().max(8000);
+        let feed = Arc::new(AudioFeed::default());
+        device.start(feed.clone())?;
+        Ok(AudioScrub { feed, device, rate, last: None, last_used: now })
+    }
+
+    /// Play the frame of `comp` at `t` (one frame long, 30–100 ms) unless it was the last one
+    /// played. Returns whether a snippet was queued.
+    pub fn play(&mut self, src: &RenderSource, comp: ItemId, t: Tick, mix_rate: u32, now: f64) -> bool {
+        self.last_used = now;
+        let Some(c) = src.project.comp(comp) else { return false };
+        let frame = c.frame_rate.frame_at(t);
+        if self.last == Some((comp, frame)) {
+            return false;
+        }
+        self.last = Some((comp, frame));
+        let rate = self.rate;
+        let mix = if mix_rate == 0 { rate } else { mix_rate.clamp(8000, 192_000) };
+        let n = (c.frame_duration().seconds().clamp(0.03, 0.1) * rate as f64) as usize;
+        let mut buf = mix_block(src, comp, c.frame_rate.tick_of(frame), n, rate, mix);
+        // 3 ms fades: no clicks where snippets meet.
+        let fade = (rate as usize * 3 / 1000).clamp(1, n / 2 + 1);
+        let frames = buf.len() / 2;
+        for i in 0..frames {
+            let g = (i.min(frames - 1 - i) as f32 / fade as f32).min(1.0);
+            buf[2 * i] *= g;
+            buf[2 * i + 1] *= g;
+        }
+        self.feed.clear();
+        self.feed.push(&buf);
+        true
+    }
+
+    /// Every UI frame while open (Web Audio moves queued samples to the output here).
+    pub fn pump(&mut self) {
+        self.device.pump();
+    }
+}
+
+impl Drop for AudioScrub {
+    fn drop(&mut self) {
+        self.device.stop();
+    }
+}
+
 /// Linear resampling of interleaved stereo from `from` Hz to `n` frames at `to` Hz.
 pub fn resample_stereo(src: &[f32], from: u32, to: u32, n: usize) -> Vec<f32> {
     let frames = src.len() / 2;

crates/ui-egui/src/lib.rs

@@ -211,6 +211,8 @@ pub struct EffectcraftApp {
     pub integrated_titlebar: bool,
     /// Audio preview while playing (audio clock drives playback).
     pub audio: Option<audio::AudioPlayback>,
+    /// Audio scrubbing's open output (Ctrl/Cmd-drag the current time, `playback.scrubAudio`).
+    pub scrub: Option<audio::AudioScrub>,
     /// Audio panel VU meters.
     pub meter: audio::Meter,
     /// Waveform peak summaries per footage item (see `panels::waveform`).
@@ -276,6 +278,7 @@ impl EffectcraftApp {
             dialog_state: Default::default(),
             integrated_titlebar: false,
             audio: None,
+            scrub: None,
             meter: Default::default(),
             waveforms: Default::default(),
             last_reveal: None,
@@ -677,6 +680,30 @@ impl EffectcraftApp {
         }
     }
 
+    /// Audio scrubbing: play the frame of `comp` at `t` (see [`audio::AudioScrub`]), opening the
+    /// output on first use. Silent comps and missing devices do nothing.
+    pub fn scrub_audio(&mut self, comp: ItemId, t: Tick, now: f64) {
+        if self.playback.playing || !effectcraft_engine::render::audio::comp_has_audio(&self.session.project, comp) {
+            return;
+        }
+        if self.scrub.is_none() {
+            let out = audio::AudioOutput::from_prefs(&self.session.prefs);
+            let Some(dev) = self.hooks.audio_device.as_ref().and_then(|f| f(&out)) else { return };
+            match audio::AudioScrub::open(dev, now) {
+                Ok(s) => self.scrub = Some(s),
+                Err(e) => {
+                    log::warn!("audio scrubbing: {e}");
+                    return;
+                }
+            }
+        }
+        let src = self.render_source();
+        let mix = self.session.prefs.audio.preview_sample_rate;
+        if let Some(s) = &mut self.scrub {
+            s.play(&src, comp, t, mix, now);
+        }
+    }
+
     pub fn stop(&mut self) {
         let was = std::mem::replace(&mut self.playback.playing, false);
         self.audio = None;
@@ -845,6 +872,14 @@ impl EffectcraftApp {
         if let Some(a) = &mut self.audio {
             a.pump();
             self.meter.update(a.feed.take_peaks(), now);
+        } else if let Some(s) = &mut self.scrub {
+            // Scrubbing: the meters follow the snippets; an idle second closes the output.
+            s.pump();
+            self.meter.update(s.feed.take_peaks(), now);
+            if now - s.last_used > 1.0 {
+                self.scrub = None;
+            }
+            ctx.request_repaint_after(std::time::Duration::from_millis(50));
         } else if self.meter.active() {
             self.meter.update([0.0; 2], now);
             ctx.request_repaint();

crates/ui-egui/src/menus.rs

@@ -562,6 +562,13 @@ pub fn frontend(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, p: Valu
             Value::Null
         }
         // Window ▸ <ScriptUI panel>: dock (or bring forward) the panel the script built.
+        // Audio scrubbing: one frame of audio at `time` (default the current time).
+        "playback.scrubAudio" => {
+            let cid = app.session.active_comp_id().ok_or("no composition")?;
+            let t = p.get("time").and_then(Value::as_f64).map_or(app.session.time(), effectcraft_engine::time::Tick::from_seconds_f64);
+            app.scrub_audio(cid, t, now);
+            json!({"playing": app.scrub.is_some()})
+        }
         // View ▸ New Viewer.
         "view.newViewer" => json!({"viewer": crate::panels::viewers::new_viewer(app)}),
         "window.scriptPanel" => {

crates/ui-egui/src/panels/timeline.rs

@@ -1048,6 +1048,15 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
         let tt = Tick::from_seconds_f64(secs);
         app.session.set_time(tt);
         app.stop();
+        // Ctrl/Cmd-drag scrubs the audio too (After Effects).
+        if rresp.dragged()
+            && ui.input(|i| i.modifiers.command)
+            && let Some(cid) = app.session.active_comp_id()
+        {
+            let now = ui.input(|i| i.time);
+            let t = app.session.time();
+            app.scrub_audio(cid, t, now);
+        }
     }
 
     // ---- column headers (right-click: show/hide columns; click the name header: Source/Layer Name).

crates/ui-egui/tests/ui_audio_scrub.rs

@@ -0,0 +1,109 @@
+//! Audio scrubbing: Ctrl/Cmd-dragging the current time plays one frame of the comp's mix at each
+//! new frame (each snippet replacing what is still queued); holding still repeats nothing; silent
+//! comps open no output; an idle scrub closes it.
+
+use std::sync::{Arc, Mutex};
+
+use effectcraft_engine::Session;
+use effectcraft_engine::project::ItemId;
+use effectcraft_engine::time::Tick;
+use effectcraft_ui_egui::EffectcraftApp;
+use effectcraft_ui_egui::audio::{AudioDevice, AudioFeed};
+use egui::{Event, Modifiers, pos2};
+use egui_kittest::Harness;
+use serde_json::json;
+
+/// An output that records the feed it is given and whether it was stopped.
+#[derive(Clone, Default)]
+struct Fake(Arc<Mutex<(Option<Arc<AudioFeed>>, bool)>>);
+
+impl AudioDevice for Fake {
+    fn sample_rate(&self) -> u32 {
+        48000
+    }
+    fn start(&mut self, feed: Arc<AudioFeed>) -> Result<(), String> {
+        self.0.lock().unwrap().0 = Some(feed);
+        Ok(())
+    }
+    fn stop(&mut self) {
+        self.0.lock().unwrap().1 = true;
+    }
+}
+
+/// A 30 fps comp whose solid sounds a tone (Audio switch on).
+fn setup(audible: bool, fake: &Fake) -> (EffectcraftApp, ItemId) {
+    let mut s = Session::default();
+    s.execute("comp.new", json!({"name": "A", "width": 64, "height": 36, "frameRate": 30, "duration": 2})).unwrap();
+    let cid = s.active_comp_id().unwrap();
+    let l = s.execute("layer.newSolid", json!({"color": "#808080"})).unwrap()["layer"].as_u64().unwrap();
+    if audible {
+        s.execute("effect.apply", json!({"layers": [l], "effect": "ec.audio.tone"})).unwrap();
+        s.execute("layer.setSwitch", json!({"layers": [l], "switch": "audio", "value": true})).unwrap();
+    }
+    let mut app = EffectcraftApp::new(s);
+    let f = fake.clone();
+    app.hooks.audio_device = Some(Box::new(move |_| Some(Box::new(f.clone()) as Box<dyn AudioDevice>)));
+    (app, cid)
+}
+
+#[test]
+fn scrubbing_plays_one_frame_per_new_frame() {
+    let fake = Fake::default();
+    let (mut app, cid) = setup(true, &fake);
+    let t = |s: f64| Tick::from_seconds_f64(s);
+    app.scrub_audio(cid, t(0.5), 0.0);
+    let feed = fake.0.lock().unwrap().0.clone().expect("the output opened");
+    // One 30 fps frame at 48 kHz, and not silence.
+    assert_eq!(feed.queued_frames(), 1600);
+    let mut out = vec![0.0f32; 3200];
+    feed.pull(&mut out);
+    assert!(out.iter().any(|v| v.abs() > 0.01), "the tone is there");
+    // Faded in: the first sample is silent.
+    assert_eq!(out[0], 0.0);
+    // The same frame again plays nothing; the next frame replaces what is left.
+    app.scrub_audio(cid, t(0.51), 0.1);
+    assert_eq!(feed.queued_frames(), 0);
+    feed.push(&[0.0; 1000]);
+    app.scrub_audio(cid, t(0.6), 0.2);
+    assert_eq!(feed.queued_frames(), 1600, "replaced, not piled up");
+    // A silent comp opens nothing.
+    let silent = Fake::default();
+    let (mut quiet, qid) = setup(false, &silent);
+    quiet.scrub_audio(qid, t(0.5), 0.0);
+    assert!(silent.0.lock().unwrap().0.is_none() && quiet.scrub.is_none());
+}
+
+#[test]
+fn cmd_dragging_the_current_time_scrubs_and_an_idle_scrub_closes() {
+    let fake = Fake::default();
+    let (app, _) = setup(true, &fake);
+    let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_| app);
+    h.run_steps(3);
+    let cti = h.state().auto.find("timeline.cti").expect("current time indicator").clone();
+    let from = pos2(cti.rect[0] + cti.rect[2] / 2.0, cti.rect[1] + cti.rect[3] / 2.0);
+    let cmd = Modifiers::COMMAND;
+    h.event(Event::ModifiersChanged(cmd));
+    h.event(Event::PointerMoved(from));
+    h.step();
+    h.event(Event::PointerButton { pos: from, button: egui::PointerButton::Primary, pressed: true, modifiers: cmd });
+    h.step();
+    for k in 1..=4 {
+        h.event(Event::PointerMoved(from + egui::vec2(40.0 * k as f32, 0.0)));
+        h.step();
+    }
+    assert!(h.state().scrub.is_some(), "Cmd-drag opened the scrub output");
+    assert!(fake.0.lock().unwrap().0.as_ref().is_some_and(|f| f.queued_frames() > 0));
+    h.event(Event::PointerButton { pos: from + egui::vec2(160.0, 0.0), button: egui::PointerButton::Primary, pressed: false, modifiers: cmd });
+    h.step();
+    h.event(Event::ModifiersChanged(Modifiers::NONE));
+    h.step();
+    // An idle second closes the output.
+    for _ in 0..90 {
+        h.step();
+        std::thread::sleep(std::time::Duration::from_millis(15));
+        if h.state().scrub.is_none() {
+            break;
+        }
+    }
+    assert!(h.state().scrub.is_none() && fake.0.lock().unwrap().1, "closed and stopped");
+}

docs/parity.md

@@ -422,6 +422,12 @@ layers); the Graph Editor has Auto-Select Graph Type (the default: the speed gra
 spatial properties are shown, else the value graph; choosing Value or Speed turns it off). Graph
 Editor keys are small squares whatever their interpolation, as in After Effects.
 
+Since (M13.34): audio scrubbing. Ctrl-dragging the current-time indicator (Cmd on macOS) plays
+one frame (30–100 ms) of the comp's mix at each new frame, as in After Effects. Snippets fade in
+and out and replace what is still queued, so the sound follows the pointer without lagging.
+Holding still repeats nothing, the Audio panel meters follow, and the output closes after an idle
+second. Agents call `playback.scrubAudio {time}`.
+
 Since (M13.33): a precomp layer's bar shows its comp's markers, as outlined, read-only markers.
 They are mapped through the layer's timing (start, stretch, time remapping, Responsive Design
 stretch) and shown within its In–Out range. Hovering names them ("beat (marker in Pre)"), and a
@@ -758,7 +764,7 @@ are pinned bit for bit by golden hashes (`crates/effects/tests/particle_golden.r
 17. ~~Content-keyed RAM preview frames, so an edit keeps the frames of comps it doesn't touch.~~
     (M4.14)
 18. ~~View ▸ New Viewer (several Composition viewers)~~ (M13.32); ~~nested comp markers on the
-    precomp layer bar~~ (M13.33), audio scrubbing.
+    precomp layer bar~~ (M13.33), ~~audio scrubbing~~ (M13.34).
 
 At the original audit the engine underneath (keyframes, expressions, shape operators, text
 animators, Classic 3D, all 38 blend modes, the render queue) was already deep, and most of what was