diff --git a/Cargo.lock b/Cargo.lock index 9d3c76d3..32b3a154 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,8 +97,8 @@ dependencies = [ "accesskit_consumer 0.35.0", "hashbrown 0.16.1", "static_assertions", - "windows", - "windows-core", + "windows 0.62.2", + "windows-core 0.62.2", ] [[package]] @@ -149,6 +149,28 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "alsa" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43" +dependencies = [ + "alsa-sys", + "bitflags 2.13.2", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "android-activity" version = "0.6.1" @@ -158,7 +180,7 @@ dependencies = [ "android-properties", "bitflags 2.13.2", "cc", - "jni", + "jni 0.22.4", "libc", "log", "ndk", @@ -737,6 +759,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "1.0.5" @@ -869,12 +897,52 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" +[[package]] +name = "coreaudio-rs" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aae284fbaf7d27aa0e292f7677dfbe26503b0d555026f702940805a630eac17" +dependencies = [ + "bitflags 1.3.2", + "libc", + "objc2-audio-toolbox", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", +] + [[package]] name = "cow-utils" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "417bef24afe1460300965a25ff4a24b8b45ad011948302ec221e8a0a81eb2c79" +[[package]] +name = "cpal" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbd307f43cc2a697e2d1f8bc7a1d824b5269e052209e28883e5bc04d095aaa3f" +dependencies = [ + "alsa", + "coreaudio-rs", + "dasp_sample", + "jni 0.21.1", + "js-sys", + "libc", + "mach2", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "objc2-audio-toolbox", + "objc2-core-audio", + "objc2-core-audio-types", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows 0.54.0", +] + [[package]] name = "cpu-time" version = "1.0.0" @@ -954,6 +1022,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "deranged" version = "0.5.8" @@ -1053,6 +1127,7 @@ dependencies = [ name = "effectcraft" version = "0.1.0" dependencies = [ + "cpal", "effectcraft-engine", "effectcraft-host", "effectcraft-ui-egui", @@ -2126,7 +2201,7 @@ dependencies = [ "log", "presser", "thiserror 2.0.21", - "windows", + "windows 0.62.2", ] [[package]] @@ -2354,6 +2429,22 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + [[package]] name = "jni" version = "0.22.4" @@ -2577,6 +2668,15 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "mach2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" +dependencies = [ + "libc", +] + [[package]] name = "memchr" version = "2.8.3" @@ -2738,6 +2838,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "num-integer" version = "0.1.47" @@ -2843,6 +2954,21 @@ dependencies = [ "objc2-foundation 0.3.2", ] +[[package]] +name = "objc2-audio-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08" +dependencies = [ + "bitflags 2.13.2", + "libc", + "objc2 0.6.4", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-cloud-kit" version = "0.2.2" @@ -2867,6 +2993,28 @@ dependencies = [ "objc2-foundation 0.2.2", ] +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2 0.6.4", + "objc2-core-audio-types", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags 2.13.2", + "objc2 0.6.4", +] + [[package]] name = "objc2-core-data" version = "0.2.2" @@ -5067,9 +5215,9 @@ dependencies = [ "web-sys", "wgpu-naga-bridge", "wgpu-types", - "windows", - "windows-core", - "windows-result", + "windows 0.62.2", + "windows-core 0.62.2", + "windows-result 0.4.1", ] [[package]] @@ -5129,6 +5277,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" +dependencies = [ + "windows-core 0.54.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows" version = "0.62.2" @@ -5136,7 +5294,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" dependencies = [ "windows-collections", - "windows-core", + "windows-core 0.62.2", "windows-future", "windows-numerics", ] @@ -5147,7 +5305,17 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" dependencies = [ - "windows-core", + "windows-core 0.62.2", +] + +[[package]] +name = "windows-core" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" +dependencies = [ + "windows-result 0.1.2", + "windows-targets 0.52.6", ] [[package]] @@ -5159,7 +5327,7 @@ dependencies = [ "windows-implement", "windows-interface", "windows-link", - "windows-result", + "windows-result 0.4.1", "windows-strings", ] @@ -5169,7 +5337,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" dependencies = [ - "windows-core", + "windows-core 0.62.2", "windows-link", "windows-threading", ] @@ -5208,10 +5376,19 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" dependencies = [ - "windows-core", + "windows-core 0.62.2", "windows-link", ] +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -5230,6 +5407,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -5266,6 +5452,21 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -5308,6 +5509,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -5320,6 +5527,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -5332,6 +5545,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -5356,6 +5575,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -5368,6 +5593,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -5380,6 +5611,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -5392,6 +5629,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" diff --git a/Cargo.toml b/Cargo.toml index bd1d4b20..ab4eef12 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ egui = "0.36" eframe = { version = "0.36", default-features = false } egui_kittest = "0.36" rfd = "0.17" +cpal = "0.16" # Local co-development against a FilmCraft checkout next to this repo: uncomment. # [patch."https://github.com/storytold/filmcraft"] diff --git a/apps/effectcraft/Cargo.toml b/apps/effectcraft/Cargo.toml index 3240c78d..5a2165aa 100644 --- a/apps/effectcraft/Cargo.toml +++ b/apps/effectcraft/Cargo.toml @@ -17,6 +17,7 @@ egui = { workspace = true } eframe = { workspace = true, features = ["default_fonts", "wgpu", "accesskit", "wayland", "x11"] } serde_json = { workspace = true } rfd = { workspace = true } +cpal = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] # Same winit as eframe: macOS event-loop options (no app activation when driven by an agent). diff --git a/apps/effectcraft/src/audio_out.rs b/apps/effectcraft/src/audio_out.rs new file mode 100644 index 00000000..bef9ff2e --- /dev/null +++ b/apps/effectcraft/src/audio_out.rs @@ -0,0 +1,139 @@ +//! Audio preview output through cpal (the default output device). +//! +//! cpal streams are not `Send` on every platform, so each preview runs its stream on a small +//! dedicated thread that owns it until stopped. The callback pulls interleaved stereo from the +//! frontend's [`AudioFeed`] and maps it onto the device's channels (mono: L+R average; extra +//! channels: silence), converting to the device's sample format. + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::mpsc; +use std::thread::JoinHandle; + +use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; +use cpal::{FromSample, SizedSample}; +use effectcraft_ui_egui::audio::{AudioDevice, AudioFeed}; + +pub struct CpalOut { + rate: u32, + latency: Arc, + stop: Option>, + thread: Option>, +} + +/// Open the default output device (`None` without one). +pub fn open() -> Option> { + let host = cpal::default_host(); + let dev = host.default_output_device()?; + let cfg = dev.default_output_config().ok()?; + Some(Box::new(CpalOut { rate: cfg.sample_rate().0, latency: Arc::new(AtomicU64::new(0)), stop: None, thread: None })) +} + +fn run>( + dev: &cpal::Device, + cfg: &cpal::StreamConfig, + feed: Arc, + latency: Arc, +) -> Result { + let ch = cfg.channels as usize; + let rate = cfg.sample_rate.0 as f64; + let mut stereo: Vec = Vec::new(); + dev.build_output_stream( + cfg, + move |out: &mut [T], info: &cpal::OutputCallbackInfo| { + let frames = out.len() / ch.max(1); + stereo.resize(frames * 2, 0.0); + feed.pull(&mut stereo); + for (f, s) in out.chunks_mut(ch.max(1)).zip(stereo.chunks_exact(2)) { + for (c, o) in f.iter_mut().enumerate() { + let v = match (ch, c) { + (1, _) => (s[0] + s[1]) * 0.5, + (_, 0) => s[0], + (_, 1) => s[1], + _ => 0.0, + }; + *o = T::from_sample(v.clamp(-1.0, 1.0)); + } + } + // Frames between "handed over" and "audible": device latency plus this buffer. + let ts = info.timestamp(); + let dev_lat = ts.playback.duration_since(&ts.callback).map(|d| d.as_secs_f64()).unwrap_or(0.0); + latency.store((dev_lat * rate) as u64 + frames as u64, Ordering::Relaxed); + }, + |e| eprintln!("effectcraft: audio output: {e}"), + None, + ) + .map_err(|e| e.to_string()) +} + +impl AudioDevice for CpalOut { + fn sample_rate(&self) -> u32 { + self.rate + } + + fn start(&mut self, feed: Arc) -> Result<(), String> { + self.stop(); + let (stop_tx, stop_rx) = mpsc::channel::<()>(); + let (ready_tx, ready_rx) = mpsc::channel::>(); + let latency = self.latency.clone(); + let rate = self.rate; + let thread = std::thread::Builder::new() + .name("ec-audio-out".into()) + .spawn(move || { + let host = cpal::default_host(); + let Some(dev) = host.default_output_device() else { + let _ = ready_tx.send(Err("no audio output device".into())); + return; + }; + let sup = match dev.default_output_config() { + Ok(c) => c, + Err(e) => { + let _ = ready_tx.send(Err(e.to_string())); + return; + } + }; + let mut cfg: cpal::StreamConfig = sup.config(); + cfg.sample_rate = cpal::SampleRate(rate); + let stream = match sup.sample_format() { + cpal::SampleFormat::F32 => run::(&dev, &cfg, feed, latency), + cpal::SampleFormat::I16 => run::(&dev, &cfg, feed, latency), + cpal::SampleFormat::U16 => run::(&dev, &cfg, feed, latency), + cpal::SampleFormat::I32 => run::(&dev, &cfg, feed, latency), + f => Err(format!("unsupported sample format {f:?}")), + }; + let stream = match stream.and_then(|s| s.play().map(|_| s).map_err(|e| e.to_string())) { + Ok(s) => s, + Err(e) => { + let _ = ready_tx.send(Err(e)); + return; + } + }; + let _ = ready_tx.send(Ok(())); + let _ = stop_rx.recv(); + drop(stream); + }) + .map_err(|e| e.to_string())?; + self.stop = Some(stop_tx); + self.thread = Some(thread); + ready_rx.recv().map_err(|e| e.to_string())? + } + + fn stop(&mut self) { + if let Some(tx) = self.stop.take() { + let _ = tx.send(()); + } + if let Some(t) = self.thread.take() { + let _ = t.join(); + } + } + + fn latency_frames(&self) -> u64 { + self.latency.load(Ordering::Relaxed) + } +} + +impl Drop for CpalOut { + fn drop(&mut self) { + self.stop(); + } +} diff --git a/apps/effectcraft/src/main.rs b/apps/effectcraft/src/main.rs index 9e8bcd6f..2c6ca716 100644 --- a/apps/effectcraft/src/main.rs +++ b/apps/effectcraft/src/main.rs @@ -5,6 +5,7 @@ //! `--control ` (or `EFFECTCRAFT_CONTROL_PORT`) starts a localhost JSON-lines control server; //! see `effectcraft_ui_egui::control` for the methods. +mod audio_out; mod control_server; use effectcraft_ui_egui::EffectcraftApp; @@ -69,6 +70,7 @@ fn main() -> eframe::Result { })); app.hooks.pick_open_project = Some(Box::new(|| rfd::FileDialog::new().add_filter("EffectCraft Project", &["ecproj"]).pick_file().map(|p| p.to_string_lossy().to_string()))); + app.hooks.audio_device = Some(Box::new(audio_out::open)); if let Some(port) = control_port { disable_app_nap(); let rx = control_server::start(port, cc.egui_ctx.clone()); diff --git a/crates/render/src/audio.rs b/crates/render/src/audio.rs index d7ae4778..246ff683 100644 --- a/crates/render/src/audio.rs +++ b/crates/render/src/audio.rs @@ -1,10 +1,20 @@ -//! Audio mixdown of a composition: every audible footage layer (and, recursively, precomp layer) -//! summed with its Audio Levels (dB, per channel), respecting in/out points, the Audio switch and -//! solo. Levels are evaluated once per call (callers mix in short blocks, e.g. one video frame). +//! Audio mixdown of a composition: every audible layer (footage with audio, precomps, and any +//! layer with a sound-generating effect such as Tone) summed with its Audio Levels (dB, per +//! channel), respecting in/out points, the Audio switch and solo. Layer audio effects +//! (Effect > Audio, see [`effectcraft_effects::audio_fx`]) run on each layer's audio before +//! its levels; Backwards plays the layer's [in, out) span reversed. +//! +//! Callers mix in short blocks (one video frame, or an audio device buffer). Effects and levels +//! are evaluated once per block. To make the result independent of block boundaries, layer +//! effects are run from a cleared state over a pre-roll of earlier audio and only the block's +//! tail is kept — so preview playback and export produce identical samples. +//! //! Time-stretched layers are resampled nearest-sample; reversed (negative stretch) layers are //! silent for now. -use effectcraft_project::{Comp, ItemId, ItemKind, Layer, LayerSource, Project}; +use effectcraft_effects::Params; +use effectcraft_effects::audio_fx; +use effectcraft_project::{Comp, GroupKind, ItemId, ItemKind, Layer, LayerSource, Node, Project}; use effectcraft_time::{TICKS_PER_SECOND, Tick}; use crate::{EvalCtx, ExprHost, FootageSource}; @@ -19,32 +29,49 @@ pub fn mix_comp(project: &Project, footage: &dyn FootageSource, expr: Option<&dy out } -/// Whether the comp has anything audible (footage with audio, or a precomp that has). +/// Whether the comp has anything audible (footage with audio, a sound generator, or a precomp +/// that has). pub fn comp_has_audio(project: &Project, comp: ItemId) -> bool { fn walk(project: &Project, comp: ItemId, depth: usize) -> bool { let Some(c) = project.comp(comp) else { return false }; depth < MAX_DEPTH && c.layers.iter().any(|l| { l.switches.audio - && match &l.source { - LayerSource::Footage { item } => { - matches!(project.item(*item).map(|i| &i.kind), Some(ItemKind::Footage(f)) if f.has_audio && !f.missing) - } - LayerSource::Comp { item } => walk(project, *item, depth + 1), - _ => false, - } + && (generates(l) + || match &l.source { + LayerSource::Footage { item } => { + matches!(project.item(*item).map(|i| &i.kind), Some(ItemKind::Footage(f)) if f.has_audio && !f.missing) + } + LayerSource::Comp { item } => walk(project, *item, depth + 1), + _ => false, + }) }) } walk(project, comp, 0) } -fn audible(project: &Project, l: &Layer) -> bool { +/// Enabled audio effects of a layer (in stack order) with their effect ids. +fn audio_effect_groups(l: &Layer) -> impl Iterator { + l.effects().filter(|_| l.switches.effects).into_iter().flat_map(|fx| fx.groups()).filter(|g| g.enabled).filter_map(|g| match &g.kind { + GroupKind::Effect { effect } if audio_fx::is_audio_effect(effect) => Some((effect.as_str(), g)), + _ => None, + }) +} + +/// The layer makes sound on its own (Tone). +fn generates(l: &Layer) -> bool { + audio_effect_groups(l).any(|(id, _)| audio_fx::generates_audio(id)) +} + +/// Does the layer contribute to the mix (Audio switch on and something to hear)? +pub fn audible(project: &Project, l: &Layer) -> bool { l.switches.audio - && match &l.source { - LayerSource::Footage { item } => matches!(project.item(*item).map(|i| &i.kind), Some(ItemKind::Footage(f)) if f.has_audio), - LayerSource::Comp { .. } => true, - _ => false, - } + && (generates(l) + || match &l.source { + LayerSource::Footage { item } => matches!(project.item(*item).map(|i| &i.kind), Some(ItemKind::Footage(f)) if f.has_audio), + LayerSource::Comp { .. } => true, + _ => false, + }) } fn db_to_gain(db: f64) -> f32 { @@ -61,6 +88,116 @@ fn sample_tick(i: i64, rate: u32) -> Tick { Tick(((i as i128 * TICKS_PER_SECOND as i128) / rate as i128) as i64) } +struct MixEnv<'a> { + project: &'a Project, + footage: &'a dyn FootageSource, + expr: Option<&'a dyn ExprHost>, + comp_id: ItemId, + comp: &'a Comp, + rate: u32, + depth: usize, +} + +impl MixEnv<'_> { + fn ctx(&self, t: Tick) -> EvalCtx<'_> { + let mut ctx = EvalCtx::new(self.project, self.comp_id, self.comp, t); + ctx.expr = self.expr; + ctx + } + + /// The layer's own (pre-effect) audio for comp sample indices `i0 .. i0 + n`, silent + /// outside its [in, out) span. Comp sample index `k` maps to `in + out − 1 − k` when + /// `reversed`. + fn layer_raw(&self, l: &Layer, i0: i64, n: usize, reversed: bool) -> Vec { + let rate = self.rate; + let mut out = vec![0.0f32; n * 2]; + let (in_i, out_i) = (units_ceil(l.in_point, rate), units_ceil(l.out_point, rate)); + if reversed { + // Mirror the clipped range, fetch it forwards, reverse. + let a = i0.max(in_i); + let b = (i0 + n as i64).min(out_i); + if a >= b { + return out; + } + let (ma, mb) = (in_i + out_i - b, in_i + out_i - a); + let fwd = self.layer_raw(l, ma, (mb - ma) as usize, false); + for k in a..b { + let j = (in_i + out_i - 1 - k - ma) as usize; + let o = (k - i0) as usize; + out[o * 2] = fwd[j * 2]; + out[o * 2 + 1] = fwd[j * 2 + 1]; + } + return out; + } + let a = (in_i - i0).clamp(0, n as i64) as usize; + let b = (out_i - i0).clamp(0, n as i64) as usize; + if a >= b { + return out; + } + let m = b - a; + let speed = 100.0 / l.stretch; + let src_n = ((m as f64 * speed).ceil() as usize).max(1); + let lt = l.layer_time(sample_tick(i0 + a as i64, rate)); + let buf = match &l.source { + LayerSource::Footage { item } => match self.project.item(*item).map(|i| &i.kind) { + Some(ItemKind::Footage(f)) if f.has_audio => self.footage.audio(*item, f, lt, src_n, rate).unwrap_or_else(|| vec![0.0; src_n * 2]), + _ => return out, + }, + LayerSource::Comp { item } => { + if self.project.comp_contains(*item, self.comp_id) { + return out; + } + let mut sub = vec![0.0f32; src_n * 2]; + mix_into(self.project, self.footage, self.expr, *item, lt, rate, &mut sub, 1.0, 1.0, self.depth + 1); + sub + } + _ => return out, + }; + let same = (speed - 1.0).abs() < 1e-9; + for i in 0..m { + let j = if same { i } else { ((i as f64 * speed) as usize).min(src_n - 1) }; + if j * 2 + 1 >= buf.len() { + break; + } + out[(a + i) * 2] = buf[j * 2]; + out[(a + i) * 2 + 1] = buf[j * 2 + 1]; + } + out + } + + /// Effect parameters at comp time `t`. + fn params(&self, l: &Layer, g: &effectcraft_project::PropGroup, t: Tick) -> Params { + let ctx = self.ctx(t); + let mut p = Params::default(); + for c in &g.children { + if let Node::Prop(pr) = c { + p.values.insert(pr.match_id.clone(), ctx.value(l, pr)); + } + } + p + } + + /// The layer's processed audio (effects applied, levels not) for comp sample indices + /// `i0 .. i0 + n`. + fn layer_audio(&self, l: &Layer, i0: i64, n: usize) -> Vec { + let t = sample_tick(i0, self.rate); + let fx: Vec<(&str, Params)> = audio_effect_groups(l).map(|(id, g)| (id, self.params(l, g, t))).collect(); + if fx.is_empty() { + return self.layer_raw(l, i0, n, false); + } + let reversed = fx.iter().any(|(id, _)| audio_fx::reverses(id)); + let pre_s: f64 = fx.iter().map(|(id, p)| audio_fx::preroll(id, p)).sum(); + let pre = (pre_s * self.rate as f64).ceil() as usize; + let mut buf = self.layer_raw(l, i0 - pre as i64, n + pre, reversed); + let start = l.layer_time(sample_tick(i0 - pre as i64, self.rate)).seconds(); + for (id, p) in &fx { + audio_fx::process(id, p, &mut buf, self.rate, start); + } + buf.drain(..pre * 2); + buf + } +} + #[allow(clippy::too_many_arguments)] fn mix_into( project: &Project, @@ -78,6 +215,7 @@ fn mix_into( if depth >= MAX_DEPTH || rate == 0 { return; } + let env = MixEnv { project, footage, expr, comp_id, comp, rate, depth }; let frames = out.len() / 2; let any_solo = comp.layers.iter().any(|l| l.switches.solo && audible(project, l)); let s0 = start.to_units_floor(rate as i64); @@ -92,50 +230,64 @@ fn mix_into( continue; } let t_a = sample_tick(s0 + a as i64, rate); - let (ll, lr) = levels(project, comp_id, comp, l, t_a, expr); + let (ll, lr) = levels(&env, l, t_a); let (ll, lr) = (ll * gl, lr * gr); if ll == 0.0 && lr == 0.0 { continue; } - let n = b - a; - let speed = 100.0 / l.stretch; - let src_n = ((n as f64 * speed).ceil() as usize).max(1); - let lt = l.layer_time(t_a); - let buf = match &l.source { - LayerSource::Footage { item } => match project.item(*item).map(|i| &i.kind) { - Some(ItemKind::Footage(f)) => footage.audio(*item, f, lt, src_n, rate).unwrap_or_else(|| vec![0.0; src_n * 2]), - _ => continue, - }, - LayerSource::Comp { item } => { - if project.comp_contains(*item, comp_id) { - continue; - } - let mut sub = vec![0.0f32; src_n * 2]; - mix_into(project, footage, expr, *item, lt, rate, &mut sub, 1.0, 1.0, depth + 1); - sub - } - _ => continue, - }; - let same = (speed - 1.0).abs() < 1e-9; - for i in 0..n { - let j = if same { i } else { ((i as f64 * speed) as usize).min(src_n - 1) }; - if j * 2 + 1 >= buf.len() { - break; - } - out[(a + i) * 2] += buf[j * 2] * ll; - out[(a + i) * 2 + 1] += buf[j * 2 + 1] * lr; + let buf = env.layer_audio(l, s0 + a as i64, b - a); + for (i, f) in buf.chunks_exact(2).enumerate() { + out[(a + i) * 2] += f[0] * ll; + out[(a + i) * 2 + 1] += f[1] * lr; } } } -fn levels(project: &Project, comp_id: ItemId, comp: &Comp, l: &Layer, t: Tick, expr: Option<&dyn ExprHost>) -> (f32, f32) { +fn levels(env: &MixEnv, l: &Layer, t: Tick) -> (f32, f32) { let Some(g) = l.props.sub("audio") else { return (1.0, 1.0) }; - let mut ctx = EvalCtx::new(project, comp_id, comp, t); - ctx.expr = expr; - let [a, b] = ctx.v2(l, g, "levels", [0.0, 0.0]); + let [a, b] = env.ctx(t).v2(l, g, "levels", [0.0, 0.0]); (db_to_gain(a), db_to_gain(b)) } +/// A min/max peak summary of interleaved stereo audio for waveform drawing: one entry per +/// `bin` sample frames, `[min L, max L, min R, max R]`. +pub fn peak_summary(samples: &[f32], bin: usize) -> Vec<[f32; 4]> { + let bin = bin.max(1); + samples + .chunks(bin * 2) + .map(|c| { + let mut s = [f32::MAX, f32::MIN, f32::MAX, f32::MIN]; + for f in c.chunks_exact(2) { + s[0] = s[0].min(f[0]); + s[1] = s[1].max(f[0]); + s[2] = s[2].min(f[1]); + s[3] = s[3].max(f[1]); + } + if s[0] > s[1] { [0.0; 4] } else { s } + }) + .collect() +} + +/// Peak summary of a footage item's audio from source time 0 over `duration`, `bins_per_sec` +/// entries per second (rendered at a low internal rate, fetched in one-second chunks). +pub fn footage_peaks(footage: &dyn FootageSource, item: ItemId, f: &effectcraft_project::Footage, duration: Tick, bins_per_sec: u32) -> Vec<[f32; 4]> { + let bins_per_sec = bins_per_sec.max(1); + // Enough samples per bin to catch peaks of audible content, without decoding at full rate. + let bin = (64usize).max(4000 / bins_per_sec as usize); + let rate = bin as u32 * bins_per_sec; + let chunk = bin * bins_per_sec as usize; + let total = units_ceil(duration, rate).max(0) as usize; + let mut out = Vec::with_capacity(total / bin + 1); + let mut i = 0usize; + while i < total { + let n = chunk.min(total - i); + let buf = footage.audio(item, f, sample_tick(i as i64, rate), n, rate).unwrap_or_else(|| vec![0.0; n * 2]); + out.extend(peak_summary(&buf, bin)); + i += n; + } + out +} + #[cfg(test)] mod tests { use std::sync::Arc; diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 1299f427..62f9ed44 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -522,4 +522,6 @@ pub fn kurbo_path(sp: &effectcraft_keyframe::ShapePath) -> kurbo::BezPath { effectcraft_path::to_kurbo(sp) } #[cfg(test)] +mod tests_audio_fx; +#[cfg(test)] mod tests_time; diff --git a/crates/render/src/tests_audio_fx.rs b/crates/render/src/tests_audio_fx.rs new file mode 100644 index 00000000..173ffc88 --- /dev/null +++ b/crates/render/src/tests_audio_fx.rs @@ -0,0 +1,204 @@ +//! Audio effects through the mixdown (the path preview playback and export share). + +use std::sync::Arc; + +use effectcraft_keyframe::Value; +use effectcraft_project::build::{self, Ids}; +use effectcraft_project::{Comp, Footage, FootageKind, ItemId, ItemKind, LayerSource, Project, Solid}; +use effectcraft_time::{FrameRate, Tick}; + +use crate::audio::{comp_has_audio, footage_peaks, mix_comp, peak_summary}; +use crate::{FootageSource, Image}; + +const SR: u32 = 8000; + +/// Synthetic footage audio as a function of source time (seconds) → (L, R). +struct Signal(fn(f64) -> (f32, f32)); +impl FootageSource for Signal { + fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option> { + None + } + fn audio(&self, _: ItemId, _: &Footage, start: Tick, frames: usize, rate: u32) -> Option> { + let t0 = start.seconds(); + Some( + (0..frames) + .flat_map(|i| { + let t = t0 + i as f64 / rate as f64; + let (l, r) = if (0.0..2.0).contains(&t) { (self.0)(t) } else { (0.0, 0.0) }; + [l, r] + }) + .collect(), + ) + } +} + +fn audio_footage() -> Footage { + Footage { + path: "x.wav".into(), + kind: FootageKind::Audio, + width: 0, + height: 0, + pixel_aspect: 1.0, + frame_rate: FrameRate::new(25, 1), + native_rate: None, + duration: Tick::from_seconds_f64(2.0), + has_video: false, + has_audio: true, + alpha: Default::default(), + premul_color: [0.0; 3], + loop_count: 1, + codec: String::new(), + missing: false, + sequence: vec![], + } +} + +/// A 2 s comp with one audio layer (in 0, out 1 s) carrying effect `id` with `vals`. +fn comp_with(id: Option<&str>, vals: &[(&str, Value)], solid: bool) -> (Project, ItemId) { + let mut p = Project::default(); + let comp = Comp::new(64, 64, FrameRate::new(25, 1), Tick::from_seconds_f64(2.0)); + let src = if solid { + let sid = p.add_item("S", effectcraft_color::Label::Red, None, ItemKind::Solid(Solid { color: [1.0; 3], width: 64, height: 64, pixel_aspect: 1.0 })); + LayerSource::Solid { item: sid } + } else { + LayerSource::Footage { item: p.add_item("x.wav", effectcraft_color::Label::SeaFoam, None, ItemKind::Footage(audio_footage())) } + }; + let mut l = build::layer(&mut p, &comp, "a", src, (64, 64), None); + l.out_point = Tick::from_seconds_f64(1.0); + if let Some(id) = id { + let spec = effectcraft_effects::find(id).unwrap(); + let mut next = p.next_id; + let mut g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), spec.name, [64.0, 64.0]); + p.next_id = next; + for (k, v) in vals { + g.prop_mut(k).unwrap_or_else(|| panic!("{k}")).value = v.clone(); + } + l.props.sub_mut("effects").unwrap().children.push(g.into()); + } + let mut comp = comp; + comp.layers = vec![l]; + let cid = p.add_item("C", effectcraft_color::Label::Sandstone, None, ItemKind::Comp(comp.into())); + (p, cid) +} + +fn mix(p: &Project, src: &dyn FootageSource, cid: ItemId, from: f64, frames: usize) -> Vec { + mix_comp(p, src, None, cid, Tick::from_seconds_f64(from), frames, SR) +} + +fn ramp(t: f64) -> (f32, f32) { + (t as f32, -(t as f32)) +} + +fn rms(m: &[f32]) -> f32 { + (m.iter().map(|v| v * v).sum::() / m.len() as f32).sqrt() +} + +#[test] +fn backwards_reverses_the_layer_span() { + let (p, cid) = comp_with(Some("ec.audio.backwards"), &[], false); + let m = mix(&p, &Signal(ramp), cid, 0.0, SR as usize); + // Comp time t plays source time ≈ 1 − t. + for i in [0usize, 2000, 4000, 7999] { + let t = i as f64 / SR as f64; + assert!((m[i * 2] as f64 - (1.0 - t)).abs() < 2.0 / SR as f64, "i={i}: {}", m[i * 2]); + } + let (p, cid) = comp_with(None, &[], false); + let m = mix(&p, &Signal(ramp), cid, 0.0, SR as usize); + assert!((m[4000 * 2] - 0.5).abs() < 1e-3); +} + +#[test] +fn high_low_pass_attenuates() { + fn hi(t: f64) -> (f32, f32) { + let v = (2.0 * std::f64::consts::PI * 3000.0 * t).sin() as f32; + (v, v) + } + let vals = [("filterOptions", Value::Enum(1)), ("cutoffFrequency", Value::Scalar(200.0))]; + let (p, cid) = comp_with(Some("ec.audio.highlowpass"), &vals, false); + let r = rms(&mix(&p, &Signal(hi), cid, 0.5, 800)); + assert!(r < 0.02, "{r}"); + let (p, cid) = comp_with(None, &[], false); + let r = rms(&mix(&p, &Signal(hi), cid, 0.5, 800)); + assert!(r > 0.6, "{r}"); +} + +#[test] +fn delay_offsets_across_blocks() { + fn click(t: f64) -> (f32, f32) { + if (t * SR as f64).round() as i64 == 2000 { (1.0, 1.0) } else { (0.0, 0.0) } + } + let vals = [("delayTime", Value::Scalar(100.0)), ("feedback", Value::Scalar(0.0))]; + let (p, cid) = comp_with(Some("ec.audio.delay"), &vals, false); + // The block starts after the click (0.25 s): the echo at 0.35 s still comes from pre-roll. + let m = mix(&p, &Signal(click), cid, 0.3, 800); + let i = 2800 - 2400; + assert!((m[i * 2] - 0.375).abs() < 1e-5, "{}", m[i * 2]); + assert_eq!(m.iter().filter(|v| v.abs() > 1e-6).count(), 2); +} + +#[test] +fn stereo_mixer_pans() { + fn left(_: f64) -> (f32, f32) { + (0.5, 0.0) + } + let (p, cid) = comp_with(Some("ec.audio.stereomixer"), &[("leftPan", Value::Scalar(100.0))], false); + let m = mix(&p, &Signal(left), cid, 0.2, 10); + assert!(m.chunks(2).all(|f| f[0].abs() < 1e-6 && (f[1] - 0.5).abs() < 1e-6), "{:?}", &m[..4]); +} + +#[test] +fn tone_makes_a_solid_audible_at_its_frequency() { + let mut vals: Vec<(&str, Value)> = vec![("frequency1", Value::Scalar(500.0))]; + for k in ["frequency2", "frequency3", "frequency4", "frequency5"] { + vals.push((k, Value::Scalar(0.0))); + } + let (p, cid) = comp_with(Some("ec.audio.tone"), &vals, true); + assert!(comp_has_audio(&p, cid)); + let m = mix(&p, &Signal(ramp), cid, 0.0, SR as usize); + let left: Vec = m.iter().step_by(2).copied().collect(); + let ups = left.windows(2).filter(|w| w[0] <= 0.0 && w[1] > 0.0).count(); + assert!((ups as i64 - 500).abs() <= 1, "{ups}"); + // The out point (1 s) silences it. + let after = mix(&p, &Signal(ramp), cid, 1.0, 100); + assert!(after.iter().all(|v| *v == 0.0)); + let (p, cid) = comp_with(None, &[], true); + assert!(!comp_has_audio(&p, cid)); +} + +/// Mixing in video-frame blocks gives the same samples as one long block (pre-roll makes +/// effects stateless across calls), so preview and export agree. +#[test] +fn blocks_match_one_long_mix() { + fn noise(t: f64) -> (f32, f32) { + let x = ((t * 12345.678).sin() * 43758.5453).fract() as f32; + (x, -x) + } + for id in ["ec.audio.reverb", "ec.audio.basstreble", "ec.audio.flangechorus", "ec.audio.modulator", "ec.audio.delay", "ec.audio.parametriceq"] { + let vals: Vec<(&str, Value)> = match id { + "ec.audio.basstreble" => vec![("bass", Value::Scalar(50.0))], + "ec.audio.parametriceq" => vec![("band1BoostCut", Value::Scalar(9.0))], + _ => vec![], + }; + let (p, cid) = comp_with(Some(id), &vals, false); + let long = mix(&p, &Signal(noise), cid, 0.2, 1600); + let mut blocks = vec![]; + for k in 0..5 { + blocks.extend(mix(&p, &Signal(noise), cid, 0.2 + k as f64 * 0.04, 320)); + } + let worst = long.iter().zip(&blocks).map(|(a, b)| (a - b).abs()).fold(0.0f32, f32::max); + assert!(worst < 2e-3, "{id}: {worst}"); + } +} + +#[test] +fn peak_summary_bins() { + let s: Vec = (0..10).flat_map(|i| [i as f32 * 0.1, -(i as f32) * 0.1]).collect(); + let p = peak_summary(&s, 4); + assert_eq!(p.len(), 3); + assert!(p[0][0].abs() < 1e-6 && (p[0][1] - 0.3).abs() < 1e-6); + assert!((p[1][2] + 0.7).abs() < 1e-6 && (p[1][3] + 0.4).abs() < 1e-6); + let f = audio_footage(); + let peaks = footage_peaks(&Signal(ramp), ItemId(1), &f, Tick::from_seconds_f64(2.0), 100); + assert_eq!(peaks.len(), 200); + assert!((peaks[150][1] - 1.51).abs() < 0.01, "{:?}", peaks[150]); +} diff --git a/crates/ui-egui/src/audio.rs b/crates/ui-egui/src/audio.rs new file mode 100644 index 00000000..9c38a7c2 --- /dev/null +++ b/crates/ui-egui/src/audio.rs @@ -0,0 +1,345 @@ +//! Real-time audio preview. +//! +//! The frontend owns no audio API: the native app supplies an [`AudioDevice`] (cpal) through +//! [`crate::Hooks::audio_device`]. During preview playback an [`AudioPlayback`] runs a feeder +//! thread that mixes the composition ahead of the device with the same mixdown export uses +//! (`effectcraft_render::audio::mix_comp`: Audio switch, solo, Audio Levels, audio effects) and +//! pushes it into an [`AudioFeed`]; the device callback pulls from the feed. +//! +//! **A/V sync:** while audio plays, the audio clock drives playback. The clock is the number of +//! sample frames the device has actually consumed (underruns do not advance it) minus the +//! device's output latency, mapped through the work-area loop ([`clock_sample`]); the viewer +//! shows the comp frame at that sample ([`frame_of_sample`]), dropping video frames rather +//! than drifting. +//! +//! The feed also tracks per-channel peaks for the Audio panel's VU meters ([`Meter`]). + +use std::collections::VecDeque; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; + +use effectcraft_engine::project::ItemId; +use effectcraft_engine::time::{FrameRate, Tick}; + +use crate::frames::RenderSource; + +/// An audio output (implemented by the native app with cpal). +pub trait AudioDevice: Send { + /// Device sample rate in Hz. + fn sample_rate(&self) -> u32; + /// Start pulling interleaved stereo frames from `feed` (see [`AudioFeed::pull`]). + fn start(&mut self, feed: Arc) -> Result<(), String>; + fn stop(&mut self); + /// Output latency in sample frames (device buffering), for A/V sync. + fn latency_frames(&self) -> u64 { + 0 + } +} + +/// Opens the default output device (`None` when there is none). +pub type AudioDeviceFactory = Box Option>>; + +/// Lock-protected sample queue between the feeder thread and the device callback. +#[derive(Default)] +pub struct AudioFeed { + queue: Mutex>, + /// Sample frames handed to the device (real audio only: underruns do not count). + consumed: AtomicU64, + /// Peak |sample| per channel since the last [`AudioFeed::take_peaks`] (f32 bits). + peak: [AtomicU32; 2], + /// The feeder reached the end of a non-looping span. + finished: AtomicBool, +} + +impl AudioFeed { + /// Fill `out` (interleaved stereo) from the queue; silence where the queue runs dry. + pub fn pull(&self, out: &mut [f32]) { + let mut real = 0usize; + let mut pk = [0.0f32; 2]; + if let Ok(mut q) = self.queue.lock() { + let n = q.len().min(out.len()) & !1; + for (i, (o, s)) in out.iter_mut().zip(q.drain(..n)).enumerate() { + *o = s; + pk[i & 1] = pk[i & 1].max(s.abs()); + } + real = n; + } + out[real..].iter_mut().for_each(|o| *o = 0.0); + self.consumed.fetch_add((real / 2) as u64, Ordering::Relaxed); + for (c, v) in pk.into_iter().enumerate() { + let _ = self.peak[c].fetch_update(Ordering::Relaxed, Ordering::Relaxed, |old| (v > f32::from_bits(old)).then_some(v.to_bits())); + } + } + + pub fn push(&self, samples: &[f32]) { + if let Ok(mut q) = self.queue.lock() { + q.extend(samples.iter().copied()); + } + } + + /// Sample frames waiting in the queue. + pub fn queued_frames(&self) -> usize { + self.queue.lock().map(|q| q.len() / 2).unwrap_or(0) + } + + pub fn consumed(&self) -> u64 { + self.consumed.load(Ordering::Relaxed) + } + + /// Peak |sample| per channel since the last call (resets them). + pub fn take_peaks(&self) -> [f32; 2] { + [f32::from_bits(self.peak[0].swap(0, Ordering::Relaxed)), f32::from_bits(self.peak[1].swap(0, Ordering::Relaxed))] + } + + pub fn finished(&self) -> bool { + self.finished.load(Ordering::Relaxed) && self.queued_frames() == 0 + } +} + +/// What a preview plays, in comp sample indices at the device rate. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PlaySpan { + pub start: i64, + /// Work area [wa, wb). + pub wa: i64, + pub wb: i64, + pub looping: bool, +} + +impl PlaySpan { + /// Span for comp times: start, work area [wa, wb). + pub fn new(start: Tick, wa: Tick, wb: Tick, looping: bool, rate: u32) -> PlaySpan { + let r = rate as i64; + PlaySpan { start: start.to_units_floor(r), wa: wa.to_units_floor(r), wb: wb.to_units_floor(r).max(wa.to_units_floor(r) + 1), looping } + } +} + +/// The comp sample index audible now: `consumed` frames played by the device minus its +/// `latency`, from the span start, wrapped into the work area when looping (clamped at its end +/// otherwise). +pub fn clock_sample(span: &PlaySpan, consumed: u64, latency: u64) -> i64 { + let s = span.start + consumed.saturating_sub(latency) as i64; + if s < span.wb { + return s; + } + if span.looping { span.wa + (s - span.wb).rem_euclid((span.wb - span.wa).max(1)) } else { span.wb - 1 } +} + +/// Comp time of sample index `s` at `rate` Hz. +pub fn sample_time(s: i64, rate: u32) -> Tick { + Tick::from_units(s, rate.max(1) as i64) +} + +/// The comp frame showing sample `s` (the frame whose interval contains it). +pub fn frame_of_sample(s: i64, rate: u32, fps: FrameRate) -> i64 { + fps.frame_at(sample_time(s, rate)) +} + +/// Block size the feeder mixes at a time. +const BLOCK: usize = 1024; + +/// A running audio preview: device + feeder thread. +pub struct AudioPlayback { + pub feed: Arc, + device: Box, + stop: Arc, + thread: Option>, + pub rate: u32, + pub span: PlaySpan, +} + +impl AudioPlayback { + /// Start mixing `comp` over `span` into `device`. + pub fn start( + mut device: Box, + src: RenderSource, + comp: ItemId, + start: Tick, + wa: Tick, + wb: Tick, + looping: bool, + ) -> Result { + let rate = device.sample_rate().max(8000); + let span = PlaySpan::new(start, wa, wb, looping, rate); + let feed = Arc::new(AudioFeed::default()); + let stop = Arc::new(AtomicBool::new(false)); + // Prime ~50 ms so the device does not start on an empty queue. + let mut cursor = span.start; + let prime = (rate as usize / 20).div_ceil(BLOCK); + for _ in 0..prime { + cursor = feed_block(&src, comp, &feed, &span, cursor, rate); + } + let thread = { + let (feed, stop) = (feed.clone(), stop.clone()); + std::thread::Builder::new() + .name("ec-audio-feed".into()) + .spawn(move || { + // Keep ~200 ms queued ahead of the device. + let target = rate as usize / 5; + while !stop.load(Ordering::Relaxed) { + if feed.queued_frames() < target && !feed.finished.load(Ordering::Relaxed) { + cursor = feed_block(&src, comp, &feed, &span, cursor, rate); + } else { + std::thread::sleep(std::time::Duration::from_millis(4)); + } + } + }) + .map_err(|e| e.to_string())? + }; + device.start(feed.clone())?; + Ok(AudioPlayback { feed, device, stop, thread: Some(thread), rate, span }) + } + + /// The comp sample index audible now. + pub fn clock(&self) -> i64 { + clock_sample(&self.span, self.feed.consumed(), self.device.latency_frames()) + } + + pub fn finished(&self) -> bool { + self.feed.finished() + } + + pub fn stop(&mut self) { + self.device.stop(); + self.stop.store(true, Ordering::Relaxed); + if let Some(t) = self.thread.take() { + let _ = t.join(); + } + } +} + +impl Drop for AudioPlayback { + fn drop(&mut self) { + self.stop(); + } +} + +/// Mix one block at `cursor` into the feed; returns the next cursor (wrapping or finishing at +/// the work-area end). +fn feed_block(src: &RenderSource, comp: ItemId, feed: &AudioFeed, span: &PlaySpan, cursor: i64, rate: u32) -> i64 { + let mut cursor = cursor; + if cursor >= span.wb { + if !span.looping { + feed.finished.store(true, Ordering::Relaxed); + return cursor; + } + cursor = span.wa; + } + let n = ((span.wb - cursor) as usize).min(BLOCK); + let buf = effectcraft_engine::render::audio::mix_comp(&src.project, src.footage.as_ref(), src.expr.as_deref(), comp, sample_time(cursor, rate), n, rate); + feed.push(&buf); + cursor + n as i64 +} + +/// VU meter state for the Audio panel: smoothed level with a falling decay and peak hold, in +/// dBFS per channel. +#[derive(Clone, Copy, Debug)] +pub struct Meter { + pub level_db: [f32; 2], + pub peak_db: [f32; 2], + peak_at: [f64; 2], + /// Clip indicators (latched until clicked / playback restarts). + pub clipped: [bool; 2], + last: f64, +} + +impl Default for Meter { + fn default() -> Self { + Meter { level_db: [METER_FLOOR; 2], peak_db: [METER_FLOOR; 2], peak_at: [0.0; 2], clipped: [false; 2], last: 0.0 } + } +} + +/// Bottom of the meter scale (dB). +pub const METER_FLOOR: f32 = -48.0; +/// How long the peak marker holds (s) and how fast levels fall (dB/s). +const PEAK_HOLD: f64 = 1.5; +const FALL_DB_PER_S: f32 = 30.0; + +pub fn to_db(v: f32) -> f32 { + if v <= 1e-6 { METER_FLOOR } else { (20.0 * v.log10()).max(METER_FLOOR) } +} + +impl Meter { + /// Feed the peaks measured since the last update at wall time `now` (seconds). + pub fn update(&mut self, peaks: [f32; 2], now: f64) { + let dt = if self.last > 0.0 { (now - self.last).clamp(0.0, 0.5) as f32 } else { 0.0 }; + self.last = now; + for c in 0..2 { + let db = to_db(peaks[c]); + self.level_db[c] = db.max(self.level_db[c] - FALL_DB_PER_S * dt); + if db >= self.peak_db[c] || now - self.peak_at[c] > PEAK_HOLD { + self.peak_db[c] = db.max(self.level_db[c]); + self.peak_at[c] = now; + } + if peaks[c] >= 1.0 { + self.clipped[c] = true; + } + } + } + + /// Anything to animate (levels still falling)? + pub fn active(&self) -> bool { + self.level_db.iter().chain(self.peak_db.iter()).any(|d| *d > METER_FLOOR) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn clock_follows_consumed_frames_and_loops() { + let span = PlaySpan { start: 4800, wa: 0, wb: 48000, looping: true }; + assert_eq!(clock_sample(&span, 0, 0), 4800); + // Latency holds the clock back until the device has played through its buffer. + assert_eq!(clock_sample(&span, 512, 1024), 4800); + assert_eq!(clock_sample(&span, 2048, 1024), 4800 + 1024); + // Wrap at the work-area end. + assert_eq!(clock_sample(&span, 48000 - 4800 + 100, 0), 100); + let once = PlaySpan { looping: false, ..span }; + assert_eq!(clock_sample(&once, 100_000, 0), 47999); + } + + #[test] + fn frame_mapping_matches_comp_frames() { + let fps = FrameRate::new(30, 1); + // 48 kHz at 30 fps: 1600 samples per frame. + assert_eq!(frame_of_sample(0, 48000, fps), 0); + assert_eq!(frame_of_sample(1599, 48000, fps), 0); + assert_eq!(frame_of_sample(1600, 48000, fps), 1); + assert_eq!(frame_of_sample(48000 * 2, 48000, fps), 60); + let span = PlaySpan::new(Tick::from_seconds_f64(1.0), Tick::ZERO, Tick::from_seconds_f64(2.0), true, 48000); + assert_eq!((span.start, span.wa, span.wb), (48000, 0, 96000)); + // 29.97: frame boundaries are fractional in samples. + let ntsc = FrameRate::new(30000, 1001); + assert_eq!(frame_of_sample(1601, 48000, ntsc), 0); + assert_eq!(frame_of_sample(1602, 48000, ntsc), 1); + } + + #[test] + fn feed_counts_real_frames_and_peaks() { + let f = AudioFeed::default(); + f.push(&[0.5, -0.25, 0.1, 0.9]); + let mut out = [1.0f32; 8]; + f.pull(&mut out); + assert_eq!(out, [0.5, -0.25, 0.1, 0.9, 0.0, 0.0, 0.0, 0.0]); + assert_eq!(f.consumed(), 2); + assert_eq!(f.take_peaks(), [0.5, 0.9]); + assert_eq!(f.take_peaks(), [0.0, 0.0]); + } + + #[test] + fn meter_decays_and_holds_peaks() { + let mut m = Meter::default(); + m.update([1.0, 0.5], 1.0); + assert!((m.level_db[0] - 0.0).abs() < 1e-4 && (m.level_db[1] + 6.0206).abs() < 1e-3); + assert!(m.clipped[0] && !m.clipped[1]); + m.update([0.0, 0.0], 1.1); + assert!((m.level_db[0] + 3.0).abs() < 1e-3, "{}", m.level_db[0]); + assert_eq!(m.peak_db[0], 0.0, "peak holds"); + // After the hold time the peak marker follows the falling level. + m.update([0.0, 0.0], 3.0); + assert!(m.peak_db[0] < -10.0, "{}", m.peak_db[0]); + } +} diff --git a/crates/ui-egui/src/control.rs b/crates/ui-egui/src/control.rs index 5aa3c9a8..3ca1725b 100644 --- a/crates/ui-egui/src/control.rs +++ b/crates/ui-egui/src/control.rs @@ -304,9 +304,12 @@ pub fn handle(app: &mut EffectcraftApp, ctx: &egui::Context, req: &ControlReques match s("action").unwrap_or("toggle") { "play" => app.play(now), "stop" => app.stop(), + "status" => {} _ => app.toggle_play(now), } - ok(json!({"playing": app.playback.playing, "time": app.session.time().seconds()})) + ok( + json!({"playing": app.playback.playing, "time": app.session.time().seconds(), "audio": app.audio.is_some(), "levelsDb": app.meter.level_db, "peaksDb": app.meter.peak_db}), + ) } "ui.screenshot" => { let crop = s("panel").and_then(PanelKind::from_name).and_then(|pk| app.auto.find(&format!("panel.{}", pk.id())).map(|e| e.rect)); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 60613605..743e79c0 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -5,6 +5,7 @@ //! Everything goes through `effectcraft_engine::Session::execute` (or `menus::invoke` for //! frontend-only commands) so every gesture is also available to agents. +pub mod audio; pub mod automation; pub mod control; pub mod dock; @@ -54,6 +55,8 @@ pub struct Hooks { pub pick_files: Option Vec>>, pub pick_save: Option Option>>, pub pick_open_project: Option Option>>, + /// Opens the audio output for preview playback (the desktop app uses cpal). + pub audio_device: Option, } #[derive(Default)] @@ -101,6 +104,14 @@ pub struct EffectcraftApp { pub(crate) pointer_comp: Option<[f32; 2]>, pub(crate) dialog_state: panels::dialogs::DialogState, pub integrated_titlebar: bool, + /// Audio preview while playing (audio clock drives playback). + pub audio: Option, + /// Audio panel VU meters. + pub meter: audio::Meter, + /// Waveform peak summaries per footage item (see `panels::waveform`). + pub(crate) waveforms: panels::waveform::Cache, + /// Last reveal shortcut and when (double-press shortcuts such as LL). + pub(crate) last_reveal: Option<(String, f64)>, } impl EffectcraftApp { @@ -133,6 +144,10 @@ impl EffectcraftApp { pointer_comp: None, dialog_state: Default::default(), integrated_titlebar: false, + audio: None, + meter: Default::default(), + waveforms: Default::default(), + last_reveal: None, } } @@ -244,10 +259,32 @@ impl EffectcraftApp { waiting: false, fps: c.frame_rate.as_f64(), }; + self.start_audio(t); + } + + /// Start audio preview from comp time `t` when the Preview panel includes audio, the comp + /// has something audible and an output device opens. + fn start_audio(&mut self, t: Tick) { + self.audio = None; + self.meter.clipped = [false; 2]; + if !self.ui.preview_audio { + return; + } + let Some(cid) = self.session.active_comp_id() else { return }; + let Some(c) = self.session.project.comp(cid).cloned() else { return }; + if !effectcraft_engine::render::audio::comp_has_audio(&self.session.project, cid) { + return; + } + let Some(dev) = self.hooks.audio_device.as_ref().and_then(|f| f()) else { return }; + match audio::AudioPlayback::start(dev, self.render_source(), cid, t, c.work_area.0, c.work_area.1, self.ui.preview_loop) { + Ok(a) => self.audio = Some(a), + Err(e) => log::warn!("audio preview: {e}"), + } } pub fn stop(&mut self) { self.playback.playing = false; + self.audio = None; } pub fn toggle_play(&mut self, now: f64) { @@ -291,9 +328,32 @@ impl EffectcraftApp { queued += 1; } } + if let Some(a) = &self.audio { + self.meter.update(a.feed.take_peaks(), now); + } else if self.meter.active() { + self.meter.update([0.0; 2], now); + ctx.request_repaint(); + } if !self.playback.playing { return; } + // Audio clock drives playback: show the frame under the audible sample (frames that + // are not rendered yet are dropped, never delayed). + if let Some(a) = &self.audio { + if a.finished() { + self.session.set_time(fr.tick_of(wb)); + self.stop(); + return; + } + let target = audio::frame_of_sample(a.clock(), a.rate, fr).clamp(wa, wb); + self.playback.waiting = !self.frames.is_cached(&self.frame_key(cid, target, scale)); + if target != cur { + self.session.set_time(fr.tick_of(target)); + self.playback.shown += 1; + } + ctx.request_repaint(); + return; + } let elapsed = now - self.playback.start_wall; let mut target = self.playback.start_frame + (elapsed * fr.as_f64()).floor() as i64; if target > wb { diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index efc9efbf..0f5ec7f6 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -64,6 +64,9 @@ pub const UI_COMMANDS: &[UiCommand] = &[ uic!("timeline.reveal.effects", "Reveal Effects", [], Some("E")), uic!("timeline.reveal.masks", "Reveal Masks", [], Some("M")), uic!("timeline.reveal.feather", "Reveal Mask Feather", [], Some("F")), + uic!("timeline.reveal.levels", "Reveal Audio Levels (press twice: Waveform)", [], Some("L")), + uic!("timeline.reveal.waveform", "Reveal Audio Waveform", [], None), + uic!("playback.muteAudio", "Mute Audio", ["Composition", "Preview"], None), uic!("timeline.collapseAll", "Collapse All", [], Some("Cmd+`")), uic!("window.workspace.default", "Default", ["Window", "Workspace"], Some("Shift+F10")), uic!("window.workspace.standard", "Standard", ["Window", "Workspace"], Some("Shift+F11")), @@ -102,7 +105,13 @@ pub fn panel_command_id(p: PanelKind) -> String { format!("window.panel.{}", p.id()) } -fn reveal(app: &mut EffectcraftApp, kind: &str) { +fn reveal(app: &mut EffectcraftApp, kind: &str, now: f64) { + // AE's double-press shortcuts: L then L again quickly reveals the Waveform (LL). + let kind = match app.last_reveal.take() { + Some((k, t)) if k == "levels" && kind == "levels" && now - t < 0.6 => "waveform", + _ => kind, + }; + app.last_reveal = Some((kind.to_string(), now)); let add = app.ui.timeline.reveal.contains(&kind.to_string()); if add && app.ui.timeline.reveal.len() == 1 { app.ui.timeline.reveal.clear(); @@ -169,7 +178,7 @@ pub fn invoke(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, params: V return Ok(json!({"tool": tool})); } if let Some(k) = id.strip_prefix("timeline.reveal.") { - reveal(app, k); + reveal(app, k, now); return Ok(Value::Null); } if let Some(r) = id.strip_prefix("view.res.") { @@ -191,6 +200,13 @@ pub fn invoke(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, params: V app.stop(); return Ok(Value::Null); } + "playback.muteAudio" => { + app.ui.preview_audio = !app.ui.preview_audio; + if !app.ui.preview_audio { + app.audio = None; + } + return Ok(json!({"muted": !app.ui.preview_audio})); + } "view.zoomIn" | "view.zoomOut" => { let cur = app.ui.viewer.zoom.unwrap_or_else(|| crate::panels::viewer::last_fit(ctx)); const STEPS: [f32; 16] = [0.015, 0.03, 0.0625, 0.125, 0.25, 0.333, 0.5, 0.66, 1.0, 1.5, 2.0, 3.0, 4.0, 8.0, 16.0, 32.0]; diff --git a/crates/ui-egui/src/panels/misc.rs b/crates/ui-egui/src/panels/misc.rs index 99edc106..8f487a5e 100644 --- a/crates/ui-egui/src/panels/misc.rs +++ b/crates/ui-egui/src/panels/misc.rs @@ -1,6 +1,6 @@ //! Smaller panels: Preview, Audio, History, Markers, Wiggler, the Home screen and placeholders. -use egui::{Align2, Color32, Rect, Sense, pos2, vec2}; +use egui::{Align2, Color32, Rect, Sense, Stroke, pos2, vec2}; use serde_json::json; use crate::dock::PanelKind; @@ -51,6 +51,12 @@ pub fn preview(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } app.auto.add("preview.loop", lr, "Loop"); p.text(pos2(lr.max.x + 6.0, yy), Align2::LEFT_CENTER, "Loop", Tokens::ui(12.0), t.text); + let ar = Rect::from_min_size(pos2(rect.min.x + 90.0, yy - 8.0), vec2(16.0, 16.0)); + if widgets::checkbox(ui, ar, app.ui.preview_audio, &t, egui::Id::new("pv-audio")).clicked() { + let _ = crate::menus::invoke(app, &ctx, "playback.muteAudio", json!({})); + } + app.auto.add("preview.audio", ar, "Include Audio"); + p.text(pos2(ar.max.x + 6.0, yy), Align2::LEFT_CENTER, "Include Audio", Tokens::ui(12.0), t.text); yy += 24.0; row(&p, yy, "Range", "Work Area Extended By Current Time"); yy += 22.0; @@ -74,19 +80,92 @@ pub fn preview(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } } +/// Audio panel: L/R VU meters (dBFS, 0 to -48) with peak hold and clip indicators, fed by the +/// audio preview; the selected layer's Audio Levels on the right. pub fn audio(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + use crate::audio::METER_FLOOR; let t = app.tokens; let p = ui.painter().with_clip_rect(rect); - // Level meters (silent until audio playback lands). - for (i, x) in [rect.min.x + 20.0, rect.min.x + 34.0].into_iter().enumerate() { - let r = Rect::from_min_max(pos2(x, rect.min.y + 14.0), pos2(x + 10.0, rect.max.y - 20.0)); - p.rect_filled(r, 1.0, t.field_bg); - let _ = i; + let top = rect.min.y + 26.0; + let bottom = rect.max.y - 22.0; + if bottom - top < 40.0 { + return; } - for (k, db) in [0, -6, -12, -18, -24, -36, -48].iter().enumerate() { - let y = rect.min.y + 14.0 + (rect.height() - 34.0) * (-*db as f32 / 48.0); - p.text(pos2(rect.min.x + 52.0, y), Align2::LEFT_CENTER, format!("{db} dB"), Tokens::ui(10.0), t.text_faint); - let _ = k; + let y_of = |db: f32| top + (bottom - top) * (db.clamp(METER_FLOOR, 0.0) / METER_FLOOR); + let m = app.meter; + let green = Color32::from_rgb(0x3c, 0xc8, 0x5a); + let yellow = Color32::from_rgb(0xe6, 0xc8, 0x3c); + let red = Color32::from_rgb(0xe6, 0x46, 0x3c); + let seg_col = |db: f32| { + if db > -3.0 { + red + } else if db > -12.0 { + yellow + } else { + green + } + }; + for c in 0..2 { + let x = rect.min.x + 18.0 + c as f32 * 16.0; + let bar = Rect::from_min_max(pos2(x, top), pos2(x + 12.0, bottom)); + p.rect_filled(bar, 1.0, t.field_bg); + // Lit segments, 1.5 dB each, coloured by their level. + let lvl = m.level_db[c]; + let mut db = METER_FLOOR; + while db < lvl.min(0.0) { + let hi = (db + 1.5).min(lvl); + p.rect_filled(Rect::from_min_max(pos2(bar.min.x + 1.0, y_of(hi)), pos2(bar.max.x - 1.0, y_of(db) - 0.5)), 0.0, seg_col(db)); + db += 1.5; + } + if m.peak_db[c] > METER_FLOOR { + let py = y_of(m.peak_db[c]); + p.line_segment([pos2(bar.min.x, py), pos2(bar.max.x, py)], Stroke::new(2.0, seg_col(m.peak_db[c]))); + } + // Clip indicator (click to reset). + let clip = Rect::from_min_max(pos2(bar.min.x, top - 14.0), pos2(bar.max.x, top - 4.0)); + p.rect_filled(clip, 1.0, if m.clipped[c] { red } else { t.field_bg }); + let id = if c == 0 { "audio.clipLeft" } else { "audio.clipRight" }; + if ui.interact(clip, egui::Id::new(id), Sense::click()).clicked() { + app.meter.clipped[c] = false; + } + app.auto.add(id, clip, if c == 0 { "Left clip indicator" } else { "Right clip indicator" }); + app.auto.add(if c == 0 { "audio.meterLeft" } else { "audio.meterRight" }, bar, &format!("{:.1} dB", m.level_db[c])); + p.text(pos2(bar.center().x, bottom + 9.0), Align2::CENTER_CENTER, if c == 0 { "L" } else { "R" }, Tokens::ui(10.0), t.text_dim); + } + let sx = rect.min.x + 54.0; + for db in [0, -6, -12, -18, -24, -30, -36, -42, -48] { + let y = y_of(db as f32); + p.line_segment([pos2(sx - 4.0, y), pos2(sx - 1.0, y)], Stroke::new(1.0, t.text_faint)); + p.text(pos2(sx + 2.0, y), Align2::LEFT_CENTER, format!("{db:.1}"), Tokens::ui(10.0), t.text_faint); + } + p.text(pos2(sx + 30.0, top - 9.0), Align2::LEFT_CENTER, "dB", Tokens::ui(10.0), t.text_dim); + // Selected layer's Audio Levels. + let lx = rect.min.x + 120.0; + if rect.max.x - lx < 80.0 { + return; + } + let sel = app.session.state.selected_layers.first().copied(); + let levels = sel.and_then(|id| { + let c = app.session.active_comp()?; + let l = c.layer(id)?; + let pr = l.props.sub("audio")?.get("levels")?; + Some((l.name.clone(), pr.value.as_vec2())) + }); + match levels { + Some((name, [a, b])) => { + p.text(pos2(lx, top - 9.0), Align2::LEFT_CENTER, &name, Tokens::ui(11.0), t.text_dim); + for (i, (k, v)) in [("Left", a), ("Right", b)].into_iter().enumerate() { + let y = top + 12.0 + i as f32 * 20.0; + p.text(pos2(lx, y), Align2::LEFT_CENTER, k, Tokens::ui(12.0), t.text_dim); + p.text(pos2(lx + 44.0, y), Align2::LEFT_CENTER, format!("{v:+.1} dB"), Tokens::ui(12.0), t.hot_text); + } + } + None => { + p.text(pos2(lx, top + 12.0), Align2::LEFT_CENTER, "No audio layer selected", Tokens::ui(11.0), t.text_faint); + } + } + if app.meter.active() || app.audio.is_some() { + ui.ctx().request_repaint(); } } diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 8cd26cb5..a7d66376 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -15,6 +15,7 @@ pub mod render_queue; pub mod text_panels; pub mod timeline; pub mod viewer; +pub mod waveform; use effectcraft_engine::Session; use effectcraft_engine::project::{Comp, TimeDisplayStyle}; diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index a7ff2338..1bb42fce 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -36,12 +36,20 @@ enum RowKind { uid: u64, lines: usize, }, + /// Audio > Waveform of a footage layer with audio (drawn from the item's peak summary). + Waveform { + item: u64, + }, } +/// Synthetic group uid for a layer's Audio > Waveform twirl (never a real property uid). +const WAVE_BIT: u64 = 1 << 60; + /// Row height (expression editors grow with their text). fn row_height(row: &Row, rh: f32) -> f32 { match row.kind { RowKind::Expr { lines, .. } => rh * lines.clamp(1, 8) as f32 + 6.0, + RowKind::Waveform { .. } => rh * 3.0, _ => rh, } } @@ -227,9 +235,15 @@ fn build_rows(app: &EffectcraftApp, comp: &Comp) -> Vec { "opacity" => ("transform", vec!["opacity"]), "anchor" => ("transform", vec!["anchor"]), "feather" => ("masks", vec!["feather"]), + "levels" => ("audio", vec!["levels"]), _ => ("", vec![]), }; match kind.as_str() { + "waveform" => { + if let Some(item) = super::waveform::audio_item(&app.session.project, l) { + rows.push(Row { layer: l.id, depth: 1, kind: RowKind::Waveform { item: item.0 } }); + } + } "effects" => { if let Some(fx) = l.effects() { for g in fx.groups() { @@ -294,6 +308,21 @@ fn build_rows(app: &EffectcraftApp, comp: &Comp) -> Vec { }); if open { push_group(&mut rows, l, g, 2, &tl.open_groups); + // Audio > Waveform > Waveform (footage with audio). + if g.match_id == "audio" + && let Some(item) = super::waveform::audio_item(&app.session.project, l) + { + let wuid = WAVE_BIT | g.uid; + let wopen = tl.open_groups.contains(&wuid); + rows.push(Row { + layer: l.id, + depth: 2, + kind: RowKind::Group { uid: wuid, name: "Waveform".into(), open: wopen, has_children: true, fx: None }, + }); + if wopen { + rows.push(Row { layer: l.id, depth: 3, kind: RowKind::Waveform { item: item.0 } }); + } + } } } Node::Prop(p) if prop_visible(p, l) => rows.push(Row { layer: l.id, depth: 1, kind: RowKind::Prop { uid: p.uid } }), @@ -952,13 +981,33 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { lp.text(pos2(ir.max.x + 4.0, cy), Align2::LEFT_CENTER, "Inverted", Tokens::ui(11.0), t.text_dim); } let gr = ui.interact(Rect::from_min_max(pos2(indent + 8.0, r.min.y), pos2(cw.switches, r.max.y)), egui::Id::new(("grow", uid)), Sense::click()); - if gr.clicked() { + if gr.clicked() && uid & WAVE_BIT == 0 { actions.push(("prop.select".into(), json!({"layer": layer.id.0, "prop": uid, "selectKeys": false}))); } if gr.double_clicked() { ui_actions.push(UiAct::ToggleGroup(*uid)); } } + RowKind::Waveform { item } => { + lp.rect_filled(left, 0.0, if ri % 2 == 0 { t.row } else { t.row_alt }); + let indent = cw.name + 6.0 + 14.0 * row.depth as f32; + lp.text(pos2(indent + 10.0, cy), Align2::LEFT_CENTER, "Waveform", Tokens::ui(12.0), t.text); + let wr = Rect::from_min_max(pos2(graph_x0, r.min.y), r.max); + gp.rect_filled(wr, 0.0, t.tl_bg); + app.auto.add(&format!("timeline.layer.{}.waveform", layer.id.0), wr, "Waveform"); + match super::waveform::summary(app, &ctx, effectcraft_engine::project::ItemId(*item)) { + Some(s) => super::waveform::draw(&gp, wr.shrink2(vec2(0.0, 2.0)), &tm, layer, &s, Color32::from_rgb(0x5f, 0xc8, 0x8a)), + None => { + gp.text( + pos2(tm.x(layer.in_point.seconds()).max(wr.min.x) + 6.0, wr.center().y), + Align2::LEFT_CENTER, + "Building waveform…", + Tokens::ui(11.0), + t.text_faint, + ); + } + } + } RowKind::Expr { uid, lines } => { let Some(prop) = layer.props.find(*uid) else { continue }; let Some(ex) = prop.expr.clone() else { continue }; diff --git a/crates/ui-egui/src/panels/waveform.rs b/crates/ui-egui/src/panels/waveform.rs new file mode 100644 index 00000000..f9bf00db --- /dev/null +++ b/crates/ui-egui/src/panels/waveform.rs @@ -0,0 +1,95 @@ +//! Audio waveforms for the Timeline's Audio > Waveform property (revealed with `LL`). +//! +//! Each footage item's audio is summarised once into min/max peaks +//! (`effectcraft_render::audio::footage_peaks`, [`BINS_PER_SEC`] bins per second) on a +//! background thread; the timeline draws from that summary at any zoom. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use effectcraft_engine::project::{ItemId, ItemKind, Layer, LayerSource, Project}; +use effectcraft_engine::time::Tick; +use egui::{Color32, Painter, Rect, Stroke, pos2}; + +use super::timeline::TMap; +use crate::EffectcraftApp; + +/// Peak-summary resolution. +pub const BINS_PER_SEC: u32 = 200; + +/// A footage item's peak summary: `[min L, max L, min R, max R]` per bin. +pub struct Summary { + pub bins_per_sec: u32, + pub peaks: Vec<[f32; 4]>, +} + +/// Summaries by footage item (`None` while computing). +#[derive(Default)] +pub struct Cache { + map: Arc>>>>, +} + +/// The footage item whose audio a layer plays, when it has audio. +pub fn audio_item(project: &Project, l: &Layer) -> Option { + let LayerSource::Footage { item } = &l.source else { return None }; + match &project.item(*item)?.kind { + ItemKind::Footage(f) if f.has_audio && !f.missing => Some(*item), + _ => None, + } +} + +/// The summary for `item`, starting its computation on first request. +pub fn summary(app: &EffectcraftApp, ctx: &egui::Context, item: ItemId) -> Option> { + let mut m = app.waveforms.map.lock().ok()?; + if let Some(s) = m.get(&item.0) { + return s.clone(); + } + let ItemKind::Footage(f) = &app.session.project.item(item)?.kind else { return None }; + m.insert(item.0, None); + let (f, footage, map, ctx) = (f.clone(), app.session.footage.clone(), app.waveforms.map.clone(), ctx.clone()); + std::thread::Builder::new() + .name("ec-waveform".into()) + .spawn(move || { + let peaks = effectcraft_engine::render::audio::footage_peaks(footage.as_ref(), item, &f, f.duration, BINS_PER_SEC); + if let Ok(mut m) = map.lock() { + m.insert(item.0, Some(Arc::new(Summary { bins_per_sec: BINS_PER_SEC, peaks }))); + } + ctx.request_repaint(); + }) + .ok()?; + None +} + +/// Draw `layer`'s waveform in `rect` (left channel above right), columns mapped through the +/// timeline's time map and the layer's time mapping, limited to its [in, out) span. +pub(crate) fn draw(p: &Painter, rect: Rect, tm: &TMap, layer: &Layer, s: &Summary, color: Color32) { + let mid = [rect.min.y + rect.height() * 0.25, rect.min.y + rect.height() * 0.75]; + let half = rect.height() * 0.25 - 1.0; + let x0 = tm.x(layer.in_point.seconds()).max(rect.min.x).floor() as i32; + let x1 = tm.x(layer.out_point.seconds()).min(rect.max.x).ceil() as i32; + let n = s.peaks.len() as i64; + for c in 0..2 { + p.line_segment([pos2(x0 as f32, mid[c]), pos2(x1 as f32, mid[c])], Stroke::new(1.0, color.gamma_multiply(0.5))); + } + for x in x0..x1 { + let (ta, tb) = (tm.t(x as f32), tm.t(x as f32 + 1.0)); + let src = |t: f64| layer.layer_time(Tick::from_seconds_f64(t)).seconds() * s.bins_per_sec as f64; + let (ba, bb) = (src(ta).floor() as i64, src(tb).ceil() as i64); + let (ba, bb) = (ba.min(bb).clamp(0, n), ba.max(bb).max(ba + 1).clamp(0, n)); + if ba >= bb { + continue; + } + let mut agg = [0.0f32; 4]; + for b in &s.peaks[ba as usize..bb as usize] { + agg[0] = agg[0].min(b[0]); + agg[1] = agg[1].max(b[1]); + agg[2] = agg[2].min(b[2]); + agg[3] = agg[3].max(b[3]); + } + for c in 0..2 { + let (lo, hi) = (agg[c * 2].clamp(-1.0, 1.0), agg[c * 2 + 1].clamp(-1.0, 1.0)); + let (ya, yb) = (mid[c] - hi * half, mid[c] - lo * half); + p.line_segment([pos2(x as f32 + 0.5, ya), pos2(x as f32 + 0.5, yb.max(ya + 1.0))], Stroke::new(1.0, color)); + } + } +} diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index b06b0e38..3b238da1 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -338,6 +338,9 @@ pub struct UiState { /// Preview panel options. pub preview_loop: bool, pub preview_cache_first: bool, + /// Preview panel "Include Audio" (Mute Audio off). + #[serde(default = "yes")] + pub preview_audio: bool, pub start_screen: bool, } @@ -369,6 +372,7 @@ impl Default for UiState { tool_creates_shape: true, preview_loop: true, preview_cache_first: false, + preview_audio: true, start_screen: false, } } diff --git a/docs/architecture.md b/docs/architecture.md index 4c8646b3..bf057ef9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -71,6 +71,16 @@ its evaluated inputs, excluding the transform. Static and transform-only layers editing one layer re-renders only that layer. Effects that read the clock directly are declared in `effects::TIME_DEPENDENT`, and a test checks every registered effect against that list. +**Time effects** (Echo, Posterize Time, Timewarp…) read the layer at other times through +`EffectHost::self_at`: the renderer renders the layer's source and masks (plus, optionally, the +effects before it) at that layer time via `Renderer::layer_input`, cached under separate *input* +keys, with a nesting-depth guard. + +**Audio** is mixed by `render::audio::mix_comp` in blocks: Audio switch, solo, Audio Levels and +the Effect > Audio effects (`effects::audio_fx`, applied per layer with a pre-roll so blocks are +independent). Export and preview playback share it; the desktop app plays it through cpal and the +audio clock drives preview playback (`ui-egui::audio`). + Half, Third and Quarter resolution render proportionally fewer pixels end to end. ## 5. Expressions diff --git a/docs/control-protocol.md b/docs/control-protocol.md index 2f2f25d1..f7c8b964 100644 --- a/docs/control-protocol.md +++ b/docs/control-protocol.md @@ -43,7 +43,7 @@ Useful query commands for `engine.execute` (they aren't journaled or undoable): | `ui.elements` | `{prefix?}` | `[{id, label, rect: [x, y, w, h]}]`: the widgets drawn in the last frame, in points. | | `ui.set` | `{tool?, workspace?, theme?, focused?, viewer?, timeline?, menuBar?, home?}` | `viewer: {zoom (null = fit), res: Full\|Half\|Third\|Quarter\|Auto, pan: [x,y], grid, rulers, safeMargins, transparencyGrid}`; `timeline: {pps (null = fit), start, graphEditor, showModes, openLayers: [ids], openGroups: [uids]}`. | | `ui.panel.show` / `ui.panel.close` | `{panel}` | Panels: `Project`, `Composition`, `Timeline`, `EffectControls`, `EffectsPresets`, `Info`, `Preview`, `Character`, `Paragraph`, `Align` and others. | -| `ui.playback` | `{action: play\|stop\|toggle}` | `{playing, time}` | +| `ui.playback` | `{action: play\|stop\|toggle\|status}` | `{playing, time, audio, levelsDb, peaksDb}` (audio: an audio preview is running; L/R meter levels and held peaks in dBFS) | | `ui.resize` | `{width, height}` | Resizes the window (points). | | `ui.focus` | `{}` | Activates the window and takes keyboard focus. Agents normally avoid this, because the app is started without stealing focus. | | `app.quit` | `{}` | Closes the app. |