From 1c0da40173a9f484f2107b9eb64366af551d5afb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 1 Oct 2026 23:42:11 -0400 Subject: [PATCH 1/2] M9.5: time effects (Echo, Posterize Time, Time Difference, Time Displacement, Timewarp, CC Force Motion Blur, CC Wide Time, Pixel Motion Blur) EffectHost gains self_at (the layer's source + masks + first N effects at another layer time) and layer_at (another layer at another comp time); EffectEnv carries the effect's stack index. The renderer serves these via Renderer::layer_input, cached in the layer cache under input keys (footage included, keyed by source time) with a depth guard against recursion. Time effects are TIME_DEPENDENT so processed-layer keys fold in layer time; layers with time effects and expressions are not cached. Also registers the Effect > Audio specs (DSP in effects::audio_fx; the mixdown integration follows in M11.2). Audio effects are skipped by the video effect pass. Co-Authored-By: Claude Opus 5.5 --- crates/effects/src/audio_fx.rs | 682 ++++++++++++++++++++++++++++++++ crates/effects/src/lib.rs | 27 ++ crates/effects/src/time_fx.rs | 590 +++++++++++++++++++++++++++ crates/render/src/cache.rs | 42 ++ crates/render/src/lib.rs | 84 +++- crates/render/src/tests_time.rs | 105 +++++ 6 files changed, 1523 insertions(+), 7 deletions(-) create mode 100644 crates/effects/src/audio_fx.rs create mode 100644 crates/effects/src/time_fx.rs create mode 100644 crates/render/src/tests_time.rs diff --git a/crates/effects/src/audio_fx.rs b/crates/effects/src/audio_fx.rs new file mode 100644 index 00000000..855fd10e --- /dev/null +++ b/crates/effects/src/audio_fx.rs @@ -0,0 +1,682 @@ +//! Audio effects (Effect > Audio): Backwards, Bass & Treble, Delay, Flange & Chorus, +//! High-Low Pass, Modulator, Parametric EQ, Reverb, Stereo Mixer and Tone. +//! +//! They are ordinary effect instances on a layer (so their parameters animate and show in +//! Effect Controls), pass video through unchanged, and are applied to the layer's audio by the +//! composition mixdown (`effectcraft_render::audio`), which preview playback and export share. +//! +//! The mixdown renders in short blocks. To keep the result independent of block boundaries +//! every effect is stateless across calls: the mixer feeds [`preroll`] seconds of the layer's +//! audio before the block, runs the chain from a cleared state and keeps the tail. +//! +//! DSP is textbook: second-order sections use the public RBJ "Audio EQ Cookbook" formulas; +//! Reverb is a Schroeder/Moorer network (parallel low-pass feedback combs into series +//! all-passes, as in the published Freeverb description); chorus/flange/vibrato are modulated +//! fractional delay lines. + +use std::f64::consts::PI; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; + +use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; + +fn spec(id: &'static str, name: &'static str, params: Vec) -> EffectSpec { + EffectSpec { id, name, category: "Audio", params, render: passthrough, gpu: false, float: true } +} + +fn passthrough(_: &EffectCtx, b: Buf) -> Buf { + b +} + +fn pct(default: f64) -> (Value, ParamUi) { + (num(default), slider(0.0, 100.0, 0.0, 100.0, 2)) +} + +fn pp(id: &'static str, name: &'static str, v: (Value, ParamUi)) -> crate::ParamSpec { + p(id, name, v.0, v.1) +} + +pub fn specs() -> Vec { + const BANDS: [[&str; 8]; 3] = [ + ["band1Enable", "Band 1 Enable", "band1Frequency", "Band 1 Frequency", "band1Bandwidth", "Band 1 Bandwidth", "band1BoostCut", "Band 1 Boost/Cut"], + ["band2Enable", "Band 2 Enable", "band2Frequency", "Band 2 Frequency", "band2Bandwidth", "Band 2 Bandwidth", "band2BoostCut", "Band 2 Boost/Cut"], + ["band3Enable", "Band 3 Enable", "band3Frequency", "Band 3 Frequency", "band3Bandwidth", "Band 3 Bandwidth", "band3BoostCut", "Band 3 Boost/Cut"], + ]; + let mut eq = vec![]; + for (b, (f, on)) in BANDS.iter().zip([(200.0, true), (1000.0, false), (5000.0, false)]) { + eq.push(p(b[0], b[1], Value::Bool(on), ParamUi::Checkbox)); + eq.push(p(b[2], b[3], num(f), slider(20.0, 20000.0, 20.0, 20000.0, 1))); + eq.push(p(b[4], b[5], num(30.0), slider(1.0, 100.0, 1.0, 100.0, 1))); + eq.push(p(b[6], b[7], num(0.0), slider(-24.0, 24.0, -24.0, 24.0, 2))); + } + vec![ + spec("ec.audio.backwards", "Backwards", vec![p("swapChannels", "Swap Channels", Value::Bool(false), ParamUi::Checkbox)]), + spec( + "ec.audio.basstreble", + "Bass & Treble", + vec![ + p("bass", "Bass", num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 2)), + p("treble", "Treble", num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 2)), + ], + ), + spec( + "ec.audio.delay", + "Delay", + vec![ + p("delayTime", "Delay Time (milliseconds)", num(500.0), slider(1.0, 10000.0, 1.0, 2000.0, 2)), + pp("delayAmount", "Delay Amount", pct(50.0)), + pp("feedback", "Feedback", pct(50.0)), + pp("dryOut", "Dry Out", pct(75.0)), + pp("wetOut", "Wet Out", pct(75.0)), + ], + ), + spec( + "ec.audio.flangechorus", + "Flange & Chorus", + vec![ + p("voiceSeparationTime", "Voice Separation Time (ms)", num(3.0), slider(0.0, 100.0, 0.0, 50.0, 2)), + p("voices", "Voices", num(1.0), slider(1.0, 8.0, 1.0, 8.0, 0)), + p("modulationRate", "Modulation Rate", num(0.4), slider(0.0, 10.0, 0.0, 5.0, 2)), + pp("modulationDepth", "Modulation Depth", pct(50.0)), + p("voicePhaseChange", "Voice Phase Change", num(0.0), ParamUi::Angle), + p("invertPhase", "Invert Phase", Value::Bool(false), ParamUi::Checkbox), + p("stereoVoices", "Stereo Voices", Value::Bool(false), ParamUi::Checkbox), + pp("dryOut", "Dry Out", pct(50.0)), + pp("wetOut", "Wet Out", pct(50.0)), + ], + ), + spec( + "ec.audio.highlowpass", + "High-Low Pass", + vec![ + p("filterOptions", "Filter Options", Value::Enum(0), popup(&["High Pass", "Low Pass"])), + p("cutoffFrequency", "Cutoff Frequency", num(2000.0), slider(20.0, 20000.0, 20.0, 20000.0, 1)), + pp("dryOut", "Dry Out", pct(0.0)), + pp("wetOut", "Wet Out", pct(100.0)), + ], + ), + spec( + "ec.audio.modulator", + "Modulator", + vec![ + p("modulationType", "Modulation Type", Value::Enum(0), popup(&["Sine", "Triangle"])), + p("modulationRate", "Modulation Rate", num(10.0), slider(0.0, 100.0, 0.0, 20.0, 2)), + pp("modulationDepth", "Modulation Depth", pct(5.0)), + pp("amplitudeModulation", "Amplitude Modulation", pct(0.0)), + ], + ), + spec("ec.audio.parametriceq", "Parametric EQ", eq), + spec( + "ec.audio.reverb", + "Reverb", + vec![ + p("reverbTime", "Reverb Time (milliseconds)", num(100.0), slider(1.0, 5000.0, 1.0, 1000.0, 2)), + pp("diffusion", "Diffusion", pct(75.0)), + pp("decay", "Decay", pct(25.0)), + pp("brightness", "Brightness", pct(10.0)), + pp("dryOut", "Dry Out", pct(90.0)), + pp("wetOut", "Wet Out", pct(10.0)), + ], + ), + spec( + "ec.audio.stereomixer", + "Stereo Mixer", + vec![ + p("leftLevel", "Left Level", num(100.0), slider(0.0, 400.0, 0.0, 200.0, 2)), + p("rightLevel", "Right Level", num(100.0), slider(0.0, 400.0, 0.0, 200.0, 2)), + p("leftPan", "Left Pan", num(-100.0), slider(-100.0, 100.0, -100.0, 100.0, 2)), + p("rightPan", "Right Pan", num(100.0), slider(-100.0, 100.0, -100.0, 100.0, 2)), + p("invertPhase", "Invert Phase", Value::Bool(false), ParamUi::Checkbox), + ], + ), + spec( + "ec.audio.tone", + "Tone", + vec![ + p("waveformOptions", "Waveform Options", Value::Enum(0), popup(&["Sine", "Triangle", "Saw", "Square"])), + p("frequency1", "Frequency 1", num(440.0), slider(0.0, 22000.0, 0.0, 2000.0, 2)), + p("frequency2", "Frequency 2", num(493.88), slider(0.0, 22000.0, 0.0, 2000.0, 2)), + p("frequency3", "Frequency 3", num(587.33), slider(0.0, 22000.0, 0.0, 2000.0, 2)), + p("frequency4", "Frequency 4", num(659.26), slider(0.0, 22000.0, 0.0, 2000.0, 2)), + p("frequency5", "Frequency 5", num(783.99), slider(0.0, 22000.0, 0.0, 2000.0, 2)), + pp("level", "Level", pct(20.0)), + ], + ), + ] +} + +/// Is `id` an audio effect (applied by the mixdown rather than the compositor)? +pub fn is_audio_effect(id: &str) -> bool { + id.starts_with("ec.audio.") +} + +/// Does the effect produce sound on its own (so a layer without audio becomes audible)? +pub fn generates_audio(id: &str) -> bool { + id == "ec.audio.tone" +} + +/// Does the effect play the layer's audio backwards (a time mapping the mixer applies)? +pub fn reverses(id: &str) -> bool { + id == "ec.audio.backwards" +} + +/// Seconds of input history the effect needs before a block to reach the state it would have +/// had when processing continuously. +pub fn preroll(id: &str, params: &Params) -> f64 { + match id { + "ec.audio.basstreble" | "ec.audio.highlowpass" | "ec.audio.parametriceq" => 0.05, + "ec.audio.flangechorus" | "ec.audio.modulator" => 0.06, + "ec.audio.delay" => { + let d = params.f("delayTime").max(1.0) / 1000.0; + let fb = (params.f("feedback") / 100.0).clamp(0.0, 0.99); + let taps = if fb > 1e-3 { (1e-3f64.ln() / fb.ln()).ceil() + 1.0 } else { 1.0 }; + (d * taps).min(10.0) + } + "ec.audio.reverb" => { + let (fb, _, _, pre) = reverb_params(params); + // T60 of the longest comb (~37 ms) with loop gain fb, plus the pre-delay. + let t60 = if fb > 1e-3 { -3.0 * 0.0367 / fb.log10() } else { 0.05 }; + (t60 + pre).min(4.0) + } + _ => 0.0, + } +} + +/// Run audio effect `id` over interleaved stereo `buf` (L R L R …) at `rate` Hz, from a cleared +/// state. `start` is the layer time (seconds) of the first sample (oscillator phase). +pub fn process(id: &str, params: &Params, buf: &mut [f32], rate: u32, start: f64) { + if rate == 0 || buf.len() < 2 { + return; + } + let sr = rate as f64; + match id { + "ec.audio.backwards" => { + if params.b("swapChannels") { + buf.chunks_exact_mut(2).for_each(|f| f.swap(0, 1)); + } + } + "ec.audio.basstreble" => { + let bass = params.f("bass") / 100.0 * 12.0; + let treble = params.f("treble") / 100.0 * 12.0; + if bass != 0.0 { + run_biquad(buf, Biquad::low_shelf(sr, 200.0, bass)); + } + if treble != 0.0 { + run_biquad(buf, Biquad::high_shelf(sr, 4000.0, treble)); + } + } + "ec.audio.highlowpass" => { + let fc = params.f("cutoffFrequency").clamp(1.0, sr * 0.49); + let q = std::f64::consts::FRAC_1_SQRT_2; + let bq = if params.e("filterOptions") == 1 { Biquad::low_pass(sr, fc, q) } else { Biquad::high_pass(sr, fc, q) }; + let dry = (params.f("dryOut") / 100.0) as f32; + let wet = (params.f("wetOut") / 100.0) as f32; + let orig = buf.to_vec(); + run_biquad(buf, bq); + mix_dry_wet(buf, &orig, dry, wet); + } + "ec.audio.parametriceq" => { + for n in 1..=3 { + if !params.b(&format!("band{n}Enable")) { + continue; + } + let g = params.f(&format!("band{n}BoostCut")); + if g == 0.0 { + continue; + } + let f = params.f(&format!("band{n}Frequency")).clamp(1.0, sr * 0.49); + let bw = (params.f(&format!("band{n}Bandwidth")) / 100.0 * 3.0).max(0.05); + run_biquad(buf, Biquad::peaking(sr, f, bw, g)); + } + } + "ec.audio.delay" => delay(params, buf, sr), + "ec.audio.flangechorus" => flange_chorus(params, buf, sr, start), + "ec.audio.modulator" => modulator(params, buf, sr, start), + "ec.audio.reverb" => reverb(params, buf, sr), + "ec.audio.stereomixer" => stereo_mixer(params, buf), + "ec.audio.tone" => tone(params, buf, sr, start), + _ => {} + } +} + +fn mix_dry_wet(buf: &mut [f32], dry_src: &[f32], dry: f32, wet: f32) { + for (o, d) in buf.iter_mut().zip(dry_src) { + *o = *o * wet + *d * dry; + } +} + +// ---------------------------------------------------------------- biquads (RBJ cookbook) + +/// Normalised second-order section coefficients (a0 = 1). +#[derive(Clone, Copy, Debug)] +pub struct Biquad { + pub b: [f64; 3], + pub a: [f64; 2], +} + +impl Biquad { + fn norm(b0: f64, b1: f64, b2: f64, a0: f64, a1: f64, a2: f64) -> Biquad { + Biquad { b: [b0 / a0, b1 / a0, b2 / a0], a: [a1 / a0, a2 / a0] } + } + pub fn low_pass(sr: f64, f0: f64, q: f64) -> Biquad { + let w = 2.0 * PI * f0 / sr; + let (s, c) = w.sin_cos(); + let al = s / (2.0 * q); + Self::norm((1.0 - c) / 2.0, 1.0 - c, (1.0 - c) / 2.0, 1.0 + al, -2.0 * c, 1.0 - al) + } + pub fn high_pass(sr: f64, f0: f64, q: f64) -> Biquad { + let w = 2.0 * PI * f0 / sr; + let (s, c) = w.sin_cos(); + let al = s / (2.0 * q); + Self::norm((1.0 + c) / 2.0, -(1.0 + c), (1.0 + c) / 2.0, 1.0 + al, -2.0 * c, 1.0 - al) + } + /// Peaking EQ with bandwidth `bw` in octaves and gain in dB. + pub fn peaking(sr: f64, f0: f64, bw: f64, db: f64) -> Biquad { + let a = 10f64.powf(db / 40.0); + let w = 2.0 * PI * f0 / sr; + let (s, c) = w.sin_cos(); + let al = s * (2f64.ln() / 2.0 * bw * w / s).sinh(); + Self::norm(1.0 + al * a, -2.0 * c, 1.0 - al * a, 1.0 + al / a, -2.0 * c, 1.0 - al / a) + } + /// Shelf with slope S = 1. + pub fn low_shelf(sr: f64, f0: f64, db: f64) -> Biquad { + let a = 10f64.powf(db / 40.0); + let w = 2.0 * PI * f0 / sr; + let (s, c) = w.sin_cos(); + let al = s / 2.0 * std::f64::consts::SQRT_2; + let sq = 2.0 * a.sqrt() * al; + Self::norm( + a * ((a + 1.0) - (a - 1.0) * c + sq), + 2.0 * a * ((a - 1.0) - (a + 1.0) * c), + a * ((a + 1.0) - (a - 1.0) * c - sq), + (a + 1.0) + (a - 1.0) * c + sq, + -2.0 * ((a - 1.0) + (a + 1.0) * c), + (a + 1.0) + (a - 1.0) * c - sq, + ) + } + pub fn high_shelf(sr: f64, f0: f64, db: f64) -> Biquad { + let a = 10f64.powf(db / 40.0); + let w = 2.0 * PI * f0 / sr; + let (s, c) = w.sin_cos(); + let al = s / 2.0 * std::f64::consts::SQRT_2; + let sq = 2.0 * a.sqrt() * al; + Self::norm( + a * ((a + 1.0) + (a - 1.0) * c + sq), + -2.0 * a * ((a - 1.0) + (a + 1.0) * c), + a * ((a + 1.0) + (a - 1.0) * c - sq), + (a + 1.0) - (a - 1.0) * c + sq, + 2.0 * ((a - 1.0) - (a + 1.0) * c), + (a + 1.0) - (a - 1.0) * c - sq, + ) + } +} + +/// Direct form I over both channels of an interleaved buffer. +fn run_biquad(buf: &mut [f32], q: Biquad) { + for ch in 0..2 { + let (mut x1, mut x2, mut y1, mut y2) = (0.0f64, 0.0, 0.0, 0.0); + for i in (ch..buf.len()).step_by(2) { + let x = buf[i] as f64; + let y = q.b[0] * x + q.b[1] * x1 + q.b[2] * x2 - q.a[0] * y1 - q.a[1] * y2; + x2 = x1; + x1 = x; + y2 = y1; + y1 = y; + buf[i] = y as f32; + } + } +} + +// ---------------------------------------------------------------- delay lines + +/// Linear-interpolated read `d` samples (fractional) behind position `i` of channel `ch`. +fn read_frac(src: &[f32], i: usize, ch: usize, d: f64) -> f32 { + let pos = i as f64 - d; + if pos < 0.0 { + return 0.0; + } + let p0 = pos.floor() as usize; + let f = (pos - p0 as f64) as f32; + let a = src[p0 * 2 + ch]; + let b = if p0 < i { src[(p0 + 1) * 2 + ch] } else { a }; + a + (b - a) * f +} + +fn delay(params: &Params, buf: &mut [f32], sr: f64) { + let d = ((params.f("delayTime").max(0.0) / 1000.0 * sr).round() as usize).max(1); + let amount = (params.f("delayAmount") / 100.0) as f32; + let fb = (params.f("feedback") / 100.0).clamp(0.0, 0.99) as f32; + let dry = (params.f("dryOut") / 100.0) as f32; + let wet = (params.f("wetOut") / 100.0) as f32; + let n = buf.len() / 2; + // Echo line: e[i] = x[i-d] + fb * e[i-d]; out = dry*x + wet*amount*e. + let mut e = vec![0.0f32; buf.len()]; + for i in d..n { + for ch in 0..2 { + e[i * 2 + ch] = buf[(i - d) * 2 + ch] + fb * e[(i - d) * 2 + ch]; + } + } + for (o, v) in buf.iter_mut().zip(&e) { + *o = *o * dry + v * wet * amount; + } +} + +fn lfo(kind: u32, phase: f64) -> f64 { + let ph = phase.rem_euclid(1.0); + match kind { + 1 => 1.0 - 4.0 * (ph - 0.5).abs(), // triangle, −1..1 + _ => (2.0 * PI * ph).sin(), + } +} + +fn flange_chorus(params: &Params, buf: &mut [f32], sr: f64, start: f64) { + let sep = params.f("voiceSeparationTime").max(0.0) / 1000.0 * sr; + let voices = params.f("voices").round().clamp(1.0, 8.0) as usize; + let rate = params.f("modulationRate").max(0.0); + let depth = (params.f("modulationDepth") / 100.0).clamp(0.0, 1.0); + let vphase = params.f("voicePhaseChange") / 360.0; + let inv = if params.b("invertPhase") { -1.0 } else { 1.0 }; + let stereo = params.b("stereoVoices"); + let dry = (params.f("dryOut") / 100.0) as f32; + let wet = (params.f("wetOut") / 100.0) as f32; + let src = buf.to_vec(); + let n = buf.len() / 2; + for i in 0..n { + let t = start + i as f64 / sr; + let mut acc = [0.0f32; 2]; + let mut cnt = [0usize; 2]; + for v in 0..voices { + let base = sep * (v + 1) as f64; + let d = (base * (1.0 + depth * lfo(0, rate * t + vphase * v as f64))).max(0.0); + for ch in 0..2 { + if stereo && voices > 1 && v % 2 != ch { + continue; + } + acc[ch] += read_frac(&src, i, ch, d); + cnt[ch] += 1; + } + } + for ch in 0..2 { + let w = if cnt[ch] > 0 { acc[ch] / cnt[ch] as f32 } else { 0.0 }; + buf[i * 2 + ch] = src[i * 2 + ch] * dry + inv * w * wet; + } + } +} + +fn modulator(params: &Params, buf: &mut [f32], sr: f64, start: f64) { + let kind = params.e("modulationType"); + let rate = params.f("modulationRate").max(0.0); + // Depth: vibrato as a modulated delay of up to 5 ms; amplitude: tremolo. + let depth = (params.f("modulationDepth") / 100.0).clamp(0.0, 1.0) * 0.005 * sr; + let am = (params.f("amplitudeModulation") / 100.0).clamp(0.0, 1.0); + let src = buf.to_vec(); + let n = buf.len() / 2; + for i in 0..n { + let t = start + i as f64 / sr; + let l = lfo(kind, rate * t); + let d = depth * (1.0 + l) * 0.5; + let g = (1.0 - am * (1.0 - l) * 0.5) as f32; + for ch in 0..2 { + let x = if depth > 0.0 { read_frac(&src, i, ch, d) } else { src[i * 2 + ch] }; + buf[i * 2 + ch] = x * g; + } + } +} + +/// (comb feedback, damping, all-pass gain, pre-delay seconds) +fn reverb_params(params: &Params) -> (f64, f64, f64, f64) { + let decay = (params.f("decay") / 100.0).clamp(0.0, 1.0); + let fb = 0.7 + decay * 0.28; + let damp = (1.0 - params.f("brightness") / 100.0).clamp(0.0, 1.0) * 0.6; + let ap = 0.3 + 0.4 * (params.f("diffusion") / 100.0).clamp(0.0, 1.0); + let pre = params.f("reverbTime").max(0.0) / 1000.0; + (fb, damp, ap, pre) +} + +fn reverb(params: &Params, buf: &mut [f32], sr: f64) { + const COMBS: [usize; 8] = [1116, 1188, 1277, 1356, 1422, 1491, 1557, 1617]; + const ALLPASS: [usize; 4] = [556, 441, 341, 225]; + const SPREAD: usize = 23; + let (fb, damp, apg, pre) = reverb_params(params); + let dry = (params.f("dryOut") / 100.0) as f32; + let wet = (params.f("wetOut") / 100.0) as f32; + let k = sr / 44100.0; + let pre_n = (pre * sr).round() as usize; + let n = buf.len() / 2; + let src = buf.to_vec(); + for ch in 0..2 { + let spread = if ch == 1 { SPREAD } else { 0 }; + let mut combs: Vec<(Vec, usize, f64)> = COMBS.iter().map(|&l| (vec![0.0; (((l + spread) as f64 * k) as usize).max(1)], 0, 0.0)).collect(); + let mut aps: Vec<(Vec, usize)> = ALLPASS.iter().map(|&l| (vec![0.0; (((l + spread) as f64 * k) as usize).max(1)], 0)).collect(); + for i in 0..n { + let x = if i >= pre_n { (src[(i - pre_n) * 2] as f64 + src[(i - pre_n) * 2 + 1] as f64) * 0.5 * 0.03 } else { 0.0 }; + let mut out = 0.0; + for (line, pos, filt) in combs.iter_mut() { + let y = line[*pos]; + *filt = y * (1.0 - damp) + *filt * damp; + line[*pos] = x + *filt * fb; + *pos = (*pos + 1) % line.len(); + out += y; + } + for (line, pos) in aps.iter_mut() { + let b = line[*pos]; + let y = -out + b; + line[*pos] = out + b * apg; + *pos = (*pos + 1) % line.len(); + out = y; + } + buf[i * 2 + ch] = src[i * 2 + ch] * dry + out as f32 * wet * 3.0; + } + } +} + +fn stereo_mixer(params: &Params, buf: &mut [f32]) { + let ll = (params.f("leftLevel") / 100.0) as f32; + let rl = (params.f("rightLevel") / 100.0) as f32; + // Linear pan law: −100 = all left, 0 = both at full, +100 = all right. + let pan = |p: f64| -> (f32, f32) { + let p = (p / 100.0).clamp(-1.0, 1.0); + ((1.0 - p).min(1.0) as f32, (1.0 + p).min(1.0) as f32) + }; + let (l_to_l, l_to_r) = pan(params.f("leftPan")); + let (r_to_l, r_to_r) = pan(params.f("rightPan")); + let inv = if params.b("invertPhase") { -1.0 } else { 1.0 }; + for f in buf.chunks_exact_mut(2) { + let (l, r) = (f[0] * ll, f[1] * rl); + f[0] = (l * l_to_l + r * r_to_l) * inv; + f[1] = (l * l_to_r + r * r_to_r) * inv; + } +} + +fn tone(params: &Params, buf: &mut [f32], sr: f64, start: f64) { + let kind = params.e("waveformOptions"); + let level = (params.f("level") / 100.0) as f32; + let freqs: Vec = (1..=5).map(|i| params.f(&format!("frequency{i}"))).filter(|f| *f > 0.0).collect(); + let n = buf.len() / 2; + for i in 0..n { + let t = start + i as f64 / sr; + let mut v = 0.0f64; + for f in &freqs { + let ph = (f * t).rem_euclid(1.0); + v += match kind { + 1 => lfo(1, ph - 0.25), + 2 => 2.0 * ph - 1.0, + 3 => { + if ph < 0.5 { + 1.0 + } else { + -1.0 + } + } + _ => (2.0 * PI * ph).sin(), + }; + } + let v = v as f32 * level; + buf[i * 2] += v; + buf[i * 2 + 1] += v; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn defaults(id: &str) -> Params { + let s = crate::find(id).unwrap(); + Params { values: s.params.iter().map(|p| (p.id.to_string(), p.default.clone())).collect() } + } + + fn sine(freq: f64, sr: f64, n: usize) -> Vec { + (0..n) + .flat_map(|i| { + let v = (2.0 * PI * freq * i as f64 / sr).sin() as f32; + [v, v] + }) + .collect() + } + + fn rms(b: &[f32], from: usize) -> f32 { + let s: f32 = b[from * 2..].iter().map(|v| v * v).sum(); + (s / (b.len() - from * 2) as f32).sqrt() + } + + #[test] + fn registered_as_audio() { + for id in [ + "ec.audio.backwards", + "ec.audio.basstreble", + "ec.audio.delay", + "ec.audio.flangechorus", + "ec.audio.highlowpass", + "ec.audio.modulator", + "ec.audio.parametriceq", + "ec.audio.reverb", + "ec.audio.stereomixer", + "ec.audio.tone", + ] { + let s = crate::find(id).unwrap_or_else(|| panic!("{id}")); + assert_eq!(s.category, "Audio"); + assert!(is_audio_effect(id)); + } + } + + #[test] + fn high_low_pass_attenuates() { + let sr = 48000.0; + let mut p = defaults("ec.audio.highlowpass"); + p.values.insert("filterOptions".into(), Value::Enum(1)); + p.values.insert("cutoffFrequency".into(), num(500.0)); + let mut hi = sine(8000.0, sr, 4800); + process("ec.audio.highlowpass", &p, &mut hi, 48000, 0.0); + let mut lo = sine(100.0, sr, 4800); + process("ec.audio.highlowpass", &p, &mut lo, 48000, 0.0); + assert!(rms(&hi, 1000) < 0.01, "{}", rms(&hi, 1000)); + assert!(rms(&lo, 1000) > 0.6, "{}", rms(&lo, 1000)); + // High pass does the opposite. + p.values.insert("filterOptions".into(), Value::Enum(0)); + let mut lo = sine(50.0, sr, 9600); + process("ec.audio.highlowpass", &p, &mut lo, 48000, 0.0); + assert!(rms(&lo, 2000) < 0.02); + } + + #[test] + fn delay_offsets_impulse() { + let mut p = defaults("ec.audio.delay"); + p.values.insert("delayTime".into(), num(10.0)); + p.values.insert("feedback".into(), num(0.0)); + let mut b = vec![0.0f32; 2000]; + b[0] = 1.0; + b[1] = 1.0; + process("ec.audio.delay", &p, &mut b, 10000, 0.0); + // dry 75 %, then the echo at 100 samples: wet 75 % × amount 50 %. + assert!((b[0] - 0.75).abs() < 1e-6); + assert!((b[200] - 0.375).abs() < 1e-6 && (b[201] - 0.375).abs() < 1e-6); + assert!(b[2..200].iter().all(|v| *v == 0.0)); + assert!(preroll("ec.audio.delay", &p) >= 0.01); + } + + #[test] + fn stereo_mixer_pans() { + let mut p = defaults("ec.audio.stereomixer"); + let mut b = vec![1.0, 0.0, 1.0, 0.0]; + process("ec.audio.stereomixer", &p, &mut b, 48000, 0.0); + assert_eq!(b, vec![1.0, 0.0, 1.0, 0.0], "defaults are identity"); + p.values.insert("leftPan".into(), num(100.0)); + let mut b = vec![1.0, 0.0]; + process("ec.audio.stereomixer", &p, &mut b, 48000, 0.0); + assert_eq!(b, vec![0.0, 1.0]); + p.values.insert("leftPan".into(), num(0.0)); + p.values.insert("leftLevel".into(), num(50.0)); + let mut b = vec![1.0, 0.0]; + process("ec.audio.stereomixer", &p, &mut b, 48000, 0.0); + assert_eq!(b, vec![0.5, 0.5]); + } + + #[test] + fn tone_generates_frequency() { + let mut p = defaults("ec.audio.tone"); + p.values.insert("frequency1".into(), num(1000.0)); + for i in 2..=5 { + p.values.insert(format!("frequency{i}"), num(0.0)); + } + let sr = 48000; + let mut b = vec![0.0f32; 48000 * 2]; + process("ec.audio.tone", &p, &mut b, sr, 0.0); + let left: Vec = b.iter().step_by(2).copied().collect(); + let crossings = left.windows(2).filter(|w| w[0] <= 0.0 && w[1] > 0.0).count(); + assert!((crossings as i64 - 1000).abs() <= 1, "{crossings}"); + let peak = left.iter().fold(0.0f32, |m, v| m.max(v.abs())); + assert!((peak - 0.2).abs() < 1e-3, "{peak}"); + } + + #[test] + fn bass_treble_eq_reverb_modulator_flange() { + let sr = 48000.0; + // Bass boost raises 60 Hz, treble cut lowers 10 kHz. + let mut p = defaults("ec.audio.basstreble"); + p.values.insert("bass".into(), num(100.0)); + p.values.insert("treble".into(), num(-100.0)); + let mut lo = sine(60.0, sr, 9600); + process("ec.audio.basstreble", &p, &mut lo, 48000, 0.0); + assert!(rms(&lo, 4000) > 0.707 * 3.0, "{}", rms(&lo, 4000)); + let mut hi = sine(10000.0, sr, 9600); + process("ec.audio.basstreble", &p, &mut hi, 48000, 0.0); + assert!(rms(&hi, 4000) < 0.707 * 0.4, "{}", rms(&hi, 4000)); + // Parametric EQ boost at its centre. + let mut p = defaults("ec.audio.parametriceq"); + p.values.insert("band1BoostCut".into(), num(12.0)); + p.values.insert("band1Frequency".into(), num(1000.0)); + let mut s = sine(1000.0, sr, 9600); + process("ec.audio.parametriceq", &p, &mut s, 48000, 0.0); + assert!((rms(&s, 4000) * std::f32::consts::SQRT_2 - 3.98).abs() < 0.2, "{}", rms(&s, 4000) * std::f32::consts::SQRT_2); + // Reverb leaves a tail after an impulse. + let p = defaults("ec.audio.reverb"); + let mut b = vec![0.0f32; 48000]; + b[0] = 1.0; + b[1] = 1.0; + process("ec.audio.reverb", &p, &mut b, 48000, 0.0); + assert!(b[2 * 12000..].iter().any(|v| v.abs() > 1e-5)); + assert!(b.iter().all(|v| v.is_finite() && v.abs() < 2.0)); + // Modulator with amplitude modulation dips the level; flange keeps it bounded. + let mut p = defaults("ec.audio.modulator"); + p.values.insert("amplitudeModulation".into(), num(100.0)); + let mut b = vec![1.0f32; 9600]; + process("ec.audio.modulator", &p, &mut b, 48000, 0.0); + assert!(b.iter().any(|v| *v < 0.05) && b.iter().any(|v| *v > 0.95)); + let p = defaults("ec.audio.flangechorus"); + let mut b = sine(440.0, sr, 4800); + process("ec.audio.flangechorus", &p, &mut b, 48000, 0.0); + assert!(b.iter().all(|v| v.abs() <= 1.0 + 1e-4)); + } + + #[test] + fn backwards_swaps_channels() { + let mut p = defaults("ec.audio.backwards"); + p.values.insert("swapChannels".into(), Value::Bool(true)); + let mut b = vec![1.0, 2.0]; + process("ec.audio.backwards", &p, &mut b, 48000, 0.0); + assert_eq!(b, vec![2.0, 1.0]); + assert!(reverses("ec.audio.backwards")); + } +} diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 6b8b7a97..923a24e4 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -6,6 +6,7 @@ //! [`instantiate`]), so parameters animate, take expressions and are addressable like any other //! property. +pub mod audio_fx; mod blur2; mod channel; mod channel2; @@ -33,6 +34,7 @@ mod sim2; mod stylize2; mod stylize3; mod textfx; +mod time_fx; mod transition; mod transition2; pub mod util; @@ -161,6 +163,18 @@ pub trait EffectHost: Sync { /// `frames` stereo sample frames (interleaved L R …) of layer `id`'s audio starting at comp /// time `start` seconds, at `rate` Hz. `None` when the layer has no audio. fn audio(&self, id: u64, start: f64, frames: usize, rate: u32) -> Option>; + /// The effect's own layer at another **layer time** `layer_time` (seconds): its source with + /// masks applied, followed by the first `effects` effects of its stack (0 = none, which is + /// what After Effects' Time effects see). `None` when unavailable (no host, recursion + /// limit, adjustment layers). Time effects use this to read neighbouring frames. + fn self_at(&self, _layer_time: f64, _effects: usize) -> Option { + None + } + /// Layer `id` of the same composition at comp time `comp_time` (seconds), like + /// [`EffectHost::layer`] but at another time. + fn layer_at(&self, _id: u64, _comp_time: f64, _masks_and_effects: bool) -> Option { + None + } } /// Extra context the renderer may supply (all optional; `Default` is "nothing known"). @@ -173,6 +187,8 @@ pub struct EffectEnv<'a> { pub comp_time: f64, /// Composition frame rate (0 when unknown). pub frame_rate: f64, + /// Index of the running effect in its layer's stack (for [`EffectHost::self_at`]). + pub effect_index: usize, } /// What an effect gets to render with. @@ -268,6 +284,8 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(noise2::specs()); v.extend(color3::specs()); v.extend(obsolete::specs()); + v.extend(time_fx::specs()); + v.extend(audio_fx::specs()); v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); v }) @@ -319,6 +337,15 @@ pub const TIME_DEPENDENT: &[&str] = &[ "ec.obsolete.lightning", "ec.text.timecode", "ec.text.numbers", + // Time effects read neighbouring frames of the layer. + "ec.time.echo", + "ec.time.posterizetime", + "ec.time.timedifference", + "ec.time.timedisplacement", + "ec.time.timewarp", + "ec.time.ccforcemotionblur", + "ec.time.ccwidetime", + "ec.time.pixelmotionblur", ]; /// See [`TIME_DEPENDENT`]. diff --git a/crates/effects/src/time_fx.rs b/crates/effects/src/time_fx.rs new file mode 100644 index 00000000..ee0f3e3c --- /dev/null +++ b/crates/effects/src/time_fx.rs @@ -0,0 +1,590 @@ +//! Time effects (Effect > Time): Echo, Posterize Time, Time Difference, Time Displacement, +//! Timewarp, CC Force Motion Blur, CC Wide Time and Pixel Motion Blur. +//! +//! These read the layer at *other* layer times through [`EffectHost::self_at`] with zero +//! preceding effects: like After Effects' Time effects they see the layer's source with its +//! masks and ignore effects applied before them (precompose to include those). Without a host +//! (tests, thumbnails) they pass their input through. +//! +//! Pixel-motion based methods (Timewarp's Pixel Motion, Pixel Motion Blur) are approximated by +//! frame mixing / sub-frame sampling; there is no optical-flow estimation yet. +//! +//! [`EffectHost::self_at`]: crate::EffectHost::self_at + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::{Image, Px}; +use rayon::prelude::*; + +use crate::util::{fit_layer, unpremul}; +use crate::{Buf, EffectCtx, EffectSpec, num, p, popup, slider}; + +fn spec(id: &'static str, name: &'static str, params: Vec, render: crate::RenderFn) -> EffectSpec { + EffectSpec { id, name, category: "Time", params, render, gpu: false, float: true } +} + +pub fn specs() -> Vec { + vec![ + spec( + "ec.time.echo", + "Echo", + vec![ + p("echoTime", "Echo Time (seconds)", num(-1.0 / 30.0), slider(-30.0, 30.0, -5.0, 5.0, 3)), + p("numberOfEchoes", "Number Of Echoes", num(1.0), slider(0.0, 255.0, 0.0, 30.0, 0)), + p("startingIntensity", "Starting Intensity", num(1.0), slider(0.0, 1.0, 0.0, 1.0, 2)), + p("decay", "Decay", num(1.0), slider(0.0, 1.0, 0.0, 1.0, 2)), + p( + "echoOperator", + "Echo Operator", + Value::Enum(0), + popup(&["Add", "Maximum", "Minimum", "Screen", "Composite In Back", "Composite In Front", "Blend"]), + ), + ], + echo, + ), + spec("ec.time.posterizetime", "Posterize Time", vec![p("frameRate", "Frame Rate", num(12.0), slider(0.1, 99.0, 1.0, 60.0, 1))], posterize_time), + spec( + "ec.time.timedifference", + "Time Difference", + vec![ + p("targetLayer", "Target", Value::Layer(None), ParamUi::Layer), + p("timeOffset", "Time Offset (sec)", num(0.0), slider(-30.0, 30.0, -5.0, 5.0, 3)), + p("contrast", "Contrast", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("absoluteDifference", "Absolute Difference", Value::Bool(true), ParamUi::Checkbox), + p( + "alphaChannel", + "Alpha Channel", + Value::Enum(0), + popup(&[ + "Original", + "Target", + "Blend", + "Max", + "Full On", + "Lightness of Result", + "Max of Result", + "Alpha Difference", + "Alpha Difference Only", + ]), + ), + ], + time_difference, + ), + spec( + "ec.time.timedisplacement", + "Time Displacement", + vec![ + p("displacementMapLayer", "Time Displacement Layer", Value::Layer(None), ParamUi::Layer), + p("maxDisplacementTime", "Max Displacement Time [sec]", num(1.0), slider(-30.0, 30.0, -5.0, 5.0, 2)), + p("timeResolution", "Time Resolution [fps]", num(60.0), slider(1.0, 999.0, 1.0, 120.0, 1)), + p("stretchMap", "If Layer Sizes Differ: Stretch Map to Fit", Value::Bool(true), ParamUi::Checkbox), + ], + time_displacement, + ), + spec( + "ec.time.timewarp", + "Timewarp", + vec![ + p("method", "Method", Value::Enum(2), popup(&["Whole Frames", "Frame Mix", "Pixel Motion"])), + p("adjustTimeBy", "Adjust Time By", Value::Enum(0), popup(&["Speed", "Source Frame"])), + p("speed", "Speed", num(50.0), slider(-10000.0, 10000.0, 0.0, 200.0, 1)), + p("sourceFrame", "Source Frame", num(0.0), slider(-100000.0, 100000.0, 0.0, 300.0, 1)), + p("vectorDetail", "Vector Detail", num(20.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("enableMotionBlur", "Enable Motion Blur", Value::Bool(false), ParamUi::Checkbox), + p("shutterControl", "Shutter Control", Value::Enum(0), popup(&["Automatic", "Manual"])), + p("shutterAngle", "Shutter Angle", num(180.0), slider(0.0, 720.0, 0.0, 360.0, 1)), + p("shutterSamples", "Shutter Samples", num(5.0), slider(1.0, 64.0, 1.0, 32.0, 0)), + ], + timewarp, + ), + spec( + "ec.time.ccforcemotionblur", + "CC Force Motion Blur", + vec![ + p("motionBlurLevels", "Motion Blur Levels", num(8.0), slider(2.0, 64.0, 2.0, 32.0, 0)), + p("overrideShutterAngle", "Override Shutter Angle", Value::Bool(true), ParamUi::Checkbox), + p("shutterAngle", "Shutter Angle", num(180.0), slider(0.0, 3600.0, 0.0, 360.0, 1)), + p("nativeMotionBlur", "Native Motion Blur", Value::Enum(0), popup(&["Off", "On"])), + ], + force_motion_blur, + ), + spec( + "ec.time.ccwidetime", + "CC Wide Time", + vec![ + p("forwardSteps", "Forward Steps", num(3.0), slider(0.0, 45.0, 0.0, 15.0, 0)), + p("backwardSteps", "Backward Steps", num(3.0), slider(0.0, 45.0, 0.0, 15.0, 0)), + p("nativeMotionBlur", "Native Motion Blur", Value::Enum(0), popup(&["Off", "On"])), + ], + wide_time, + ), + spec( + "ec.time.pixelmotionblur", + "Pixel Motion Blur", + vec![ + p("shutterControl", "Shutter Control", Value::Enum(0), popup(&["Automatic", "Manual"])), + p("shutterAngle", "Shutter Angle", num(180.0), slider(0.0, 720.0, 0.0, 360.0, 1)), + p("shutterSamples", "Shutter Samples", num(16.0), slider(1.0, 64.0, 1.0, 32.0, 0)), + p("vectorDetail", "Vector Detail", num(20.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + ], + pixel_motion_blur, + ), + ] +} + +// ---------------------------------------------------------------- frame access + +/// Frames of the layer at several layer times, resampled onto one common pixel grid. +struct Frames { + /// Template buffer (geometry of the output: offset/scale/size). + grid: Buf, + imgs: Vec, +} + +/// Fetch the layer (source + masks) at layer times `times`. `None` without a host. +fn fetch(ctx: &EffectCtx, times: &[f64]) -> Option { + let host = ctx.env.host?; + // Dedupe times (sub-microsecond differences are the same frame). + let mut uniq: Vec = times.iter().map(|t| (t * 1e6).round() as i64).collect(); + uniq.sort_unstable(); + uniq.dedup(); + let bufs: Vec<(i64, Option)> = uniq.iter().map(|&k| (k, host.self_at(k as f64 / 1e6, 0))).collect(); + let first = bufs.iter().find_map(|(_, b)| b.as_ref())?; + let scale = first.scale; + // Union of every frame's extent in layer pixels (offset = where layer (0,0) sits). + let (mut x0, mut y0, mut x1, mut y1) = (f64::MAX, f64::MAX, f64::MIN, f64::MIN); + for b in bufs.iter().filter_map(|(_, b)| b.as_ref()) { + let k = scale / b.scale.max(1e-9); + x0 = x0.min(-b.offset[0] * k); + y0 = y0.min(-b.offset[1] * k); + x1 = x1.max((b.img.width as f64 - b.offset[0]) * k); + y1 = y1.max((b.img.height as f64 - b.offset[1]) * k); + } + let (x0, y0) = (x0.floor(), y0.floor()); + let w = ((x1.ceil() - x0) as u32).clamp(1, 16384); + let h = ((y1.ceil() - y0) as u32).clamp(1, 16384); + let grid = Buf { img: Image::new(w, h), offset: [-x0, -y0], scale }; + let place = |b: &Buf| -> Image { + if (b.scale - scale).abs() < 1e-9 && b.img.width == w && b.img.height == h && (b.offset[0] + x0).abs() < 1e-6 && (b.offset[1] + y0).abs() < 1e-6 { + return b.img.clone(); + } + let k = b.scale / scale; + let same_scale = (k - 1.0).abs() < 1e-9; + let dx = b.offset[0] + x0; + let dy = b.offset[1] + y0; + let mut out = Image::new(w, h); + let ow = w as usize; + out.data.par_chunks_mut(ow).enumerate().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + *px = if same_scale && dx.fract() == 0.0 && dy.fract() == 0.0 { + b.img.get(x as i64 + dx as i64, y as i64 + dy as i64) + } else { + let sx = (x as f64 + 0.5) * k + dx - 0.5; + let sy = (y as f64 + 0.5) * k + dy - 0.5; + b.img.sample_bilinear(sx + 0.5, sy + 0.5) + }; + } + }); + out + }; + let placed: Vec<(i64, Image)> = bufs.iter().map(|(k, b)| (*k, b.as_ref().map(place).unwrap_or_else(|| Image::new(w, h)))).collect(); + let imgs = times + .iter() + .map(|t| { + let k = (t * 1e6).round() as i64; + placed.iter().find(|(q, _)| *q == k).map(|(_, i)| i.clone()).unwrap_or_else(|| Image::new(w, h)) + }) + .collect(); + Some(Frames { grid, imgs }) +} + +fn with_img(grid: Buf, img: Image) -> Buf { + Buf { img, ..grid } +} + +/// Weighted average of frames (weights sum to 1 for a plain average). +fn average(frames: &[Image], weights: &[f32]) -> Image { + let w = frames[0].width; + let h = frames[0].height; + let mut out = Image::new(w, h); + out.data.par_iter_mut().enumerate().for_each(|(i, o)| { + let mut acc = [0.0f32; 4]; + for (f, k) in frames.iter().zip(weights) { + let p = f.data[i]; + for c in 0..4 { + acc[c] += p[c] * k; + } + } + *o = acc; + }); + out +} + +fn mean_of(ctx: &EffectCtx, b: Buf, times: &[f64]) -> Buf { + if times.is_empty() { + return b; + } + let Some(fr) = fetch(ctx, times) else { return b }; + let k = 1.0 / times.len() as f32; + let img = average(&fr.imgs, &vec![k; times.len()]); + with_img(fr.grid, img) +} + +// ---------------------------------------------------------------- Echo + +fn echo(ctx: &EffectCtx, b: Buf) -> Buf { + let n = ctx.params.f("numberOfEchoes").round().clamp(0.0, 255.0) as usize; + let dt = ctx.params.f("echoTime"); + let start = ctx.params.f("startingIntensity").clamp(0.0, 1.0) as f32; + let decay = ctx.params.f("decay").clamp(0.0, 1.0) as f32; + let op = ctx.params.e("echoOperator"); + let times: Vec = (0..=n).map(|i| ctx.time + i as f64 * dt).collect(); + let Some(fr) = fetch(ctx, ×) else { return b }; + let weights: Vec = (0..=n).map(|i| start * decay.powi(i as i32)).collect(); + let count = (n + 1) as f32; + let w = fr.grid.img.width; + let h = fr.grid.img.height; + let mut out = Image::new(w, h); + let imgs = &fr.imgs; + out.data.par_iter_mut().enumerate().for_each(|(i, o)| { + let mut acc: Px = match op { + 2 => [f32::MAX; 4], + _ => [0.0; 4], + }; + for (f, &k) in imgs.iter().zip(&weights) { + let p = f.data[i]; + let q = [p[0] * k, p[1] * k, p[2] * k, p[3] * k]; + match op { + // Maximum / Minimum + 1 => (0..4).for_each(|c| acc[c] = acc[c].max(q[c])), + 2 => (0..4).for_each(|c| acc[c] = acc[c].min(q[c])), + // Screen + 3 => (0..4).for_each(|c| acc[c] = acc[c] + q[c] - acc[c] * q[c]), + // Composite In Back: each later echo goes behind what is there. + 4 => { + let ia = 1.0 - acc[3]; + (0..4).for_each(|c| acc[c] += q[c] * ia); + } + // Composite In Front: each later echo goes on top. + 5 => { + let ia = 1.0 - q[3]; + (0..4).for_each(|c| acc[c] = q[c] + acc[c] * ia); + } + // Blend (average) and Add + _ => (0..4).for_each(|c| acc[c] += q[c]), + } + } + if op == 6 { + acc.iter_mut().for_each(|c| *c /= count); + } + if op == 2 && acc[0] == f32::MAX { + acc = [0.0; 4]; + } + acc[3] = acc[3].clamp(0.0, 1.0); + for c in 0..3 { + acc[c] = acc[c].max(0.0); + } + *o = acc; + }); + with_img(fr.grid, out) +} + +// ---------------------------------------------------------------- Posterize Time + +/// The layer time Posterize Time holds at `t` for `rate` frames per second. +pub fn posterized_time(t: f64, rate: f64) -> f64 { + if rate <= 0.0 { + return t; + } + ((t * rate) + 1e-6).floor() / rate +} + +fn posterize_time(ctx: &EffectCtx, b: Buf) -> Buf { + let tt = posterized_time(ctx.time, ctx.params.f("frameRate")); + if ctx.env.host.is_none() { + return b; + } + match fetch(ctx, &[tt]) { + Some(mut fr) => { + let img = fr.imgs.pop().unwrap_or_default(); + with_img(fr.grid, img) + } + None => b, + } +} + +// ---------------------------------------------------------------- Time Difference + +fn time_difference(ctx: &EffectCtx, b: Buf) -> Buf { + let Some(host) = ctx.env.host else { return b }; + let off = ctx.params.f("timeOffset"); + let gain = 1.0 + ctx.params.f("contrast") as f32 / 25.0; + let abs = ctx.params.b("absoluteDifference"); + let amode = ctx.params.e("alphaChannel"); + let target_id = ctx.params.get("targetLayer").and_then(Value::as_layer); + let (cur, target) = match target_id.and_then(|id| host.layer_at(id, ctx.env.comp_time + off, false)) { + Some(other) => { + let Some(mut fr) = fetch(ctx, &[ctx.time]) else { return b }; + let cur = fr.imgs.pop().unwrap_or_default(); + let tgt = fit_layer(ctx, &with_img(fr.grid.clone(), cur.clone()), &other, true); + (with_img(fr.grid, cur), tgt) + } + None => { + let Some(mut fr) = fetch(ctx, &[ctx.time, ctx.time + off]) else { return b }; + let tgt = fr.imgs.pop().unwrap_or_default(); + let cur = fr.imgs.pop().unwrap_or_default(); + (with_img(fr.grid, cur), tgt) + } + }; + let mut out = cur.img.clone(); + out.data.par_iter_mut().zip(target.data.par_iter()).for_each(|(o, t)| { + let (c, ca) = unpremul(*o); + let (d, da) = unpremul(*t); + let mut rgb = [0.0f32; 3]; + for i in 0..3 { + let diff = c[i] - d[i]; + rgb[i] = if abs { (diff.abs() * gain).clamp(0.0, 1.0) } else { (0.5 + diff * 0.5 * gain).clamp(0.0, 1.0) }; + } + let light = (rgb[0].max(rgb[1]).max(rgb[2]) + rgb[0].min(rgb[1]).min(rgb[2])) * 0.5; + let a = match amode { + 1 => da, + 2 => (ca + da) * 0.5, + 3 => ca.max(da), + 4 => 1.0, + 5 => light, + 6 => rgb[0].max(rgb[1]).max(rgb[2]), + 7 => (ca - da).abs() * gain, + 8 => { + let a = ((ca - da).abs() * gain).clamp(0.0, 1.0); + rgb = [1.0; 3]; + a + } + _ => ca, + } + .clamp(0.0, 1.0); + *o = [rgb[0] * a, rgb[1] * a, rgb[2] * a, a]; + }); + Buf { img: out, ..cur } +} + +// ---------------------------------------------------------------- Time Displacement + +fn time_displacement(ctx: &EffectCtx, b: Buf) -> Buf { + if ctx.env.host.is_none() { + return b; + } + let max = ctx.params.f("maxDisplacementTime"); + let res = ctx.params.f("timeResolution").clamp(1.0, 999.0); + let stretch = ctx.params.b("stretchMap"); + let Some(mut cur) = fetch(ctx, &[ctx.time]) else { return b }; + let cur_img = cur.imgs.pop().unwrap_or_default(); + let grid = with_img(cur.grid, cur_img); + let map = match ctx.layer_param("displacementMapLayer", true) { + Some(o) => fit_layer(ctx, &grid, &o, stretch), + None => grid.img.clone(), + }; + // Per pixel: luminance 0..1 → offset −max..+max, quantised to the time resolution. Keep at + // most 64 distinct sample times (coarser steps for huge ranges). + let span = 2.0 * max.abs() * res; + let step = if span > 63.0 { 2.0 * max.abs() / 63.0 } else { 1.0 / res }; + let idx: Vec = map + .data + .par_iter() + .map(|p| { + let (c, a) = unpremul(*p); + let l = if a > 0.0 { effectcraft_color::luminance(c[0], c[1], c[2]) } else { 0.5 }; + let off = (l as f64 - 0.5) * 2.0 * max; + (off / step).round() as i32 + }) + .collect(); + let mut keys: Vec = idx.clone(); + keys.sort_unstable(); + keys.dedup(); + if keys == [0] { + return grid; + } + let times: Vec = keys.iter().map(|k| ctx.time + *k as f64 * step).collect(); + let Some(fr) = fetch(ctx, ×) else { return grid }; + // Frames come back on the union grid; map the current grid into it. + let dx = (fr.grid.offset[0] - grid.offset[0]).round() as i64; + let dy = (fr.grid.offset[1] - grid.offset[1]).round() as i64; + let w = grid.img.width as usize; + let mut out = grid.img.clone(); + out.data.par_chunks_mut(w).enumerate().for_each(|(y, row)| { + for (x, o) in row.iter_mut().enumerate() { + let k = idx[y * w + x]; + let fi = keys.binary_search(&k).unwrap_or(0); + *o = fr.imgs[fi].get(x as i64 + dx, y as i64 + dy); + } + }); + with_img(grid, out) +} + +// ---------------------------------------------------------------- Timewarp + +/// Source layer time Timewarp shows at layer time `t`. +pub fn timewarp_source_time(t: f64, by_frame: bool, speed: f64, source_frame: f64, fps: f64) -> f64 { + if by_frame { source_frame / fps } else { t * speed / 100.0 } +} + +fn timewarp(ctx: &EffectCtx, b: Buf) -> Buf { + if ctx.env.host.is_none() { + return b; + } + let fps = ctx.fps(); + let by_frame = ctx.params.e("adjustTimeBy") == 1; + let speed = ctx.params.f("speed"); + let src = timewarp_source_time(ctx.time, by_frame, speed, ctx.params.f("sourceFrame"), fps); + let method = ctx.params.e("method"); + // Sample times (with weights) for one instant of source time. + let instant = |s: f64| -> Vec<(f64, f32)> { + let f = s * fps; + let f0 = (f + 1e-6).floor(); + let frac = (f - f0) as f32; + if method == 0 || frac < 1e-4 { vec![(f0 / fps, 1.0)] } else { vec![(f0 / fps, 1.0 - frac), ((f0 + 1.0) / fps, frac)] } + }; + let mut samples: Vec<(f64, f32)> = Vec::new(); + if ctx.params.b("enableMotionBlur") { + let angle = if ctx.params.e("shutterControl") == 1 { ctx.params.f("shutterAngle") } else { 180.0 }; + let n = ctx.params.f("shutterSamples").round().clamp(1.0, 64.0) as usize; + let rate = if by_frame { 1.0 } else { speed / 100.0 }; + let span = angle / 360.0 * rate / fps; + for i in 0..n { + let s = src + if n > 1 { span * i as f64 / (n - 1) as f64 } else { 0.0 }; + samples.extend(instant(s).into_iter().map(|(t, w)| (t, w / n as f32))); + } + } else { + samples = instant(src); + } + let times: Vec = samples.iter().map(|s| s.0).collect(); + let weights: Vec = samples.iter().map(|s| s.1).collect(); + let Some(fr) = fetch(ctx, ×) else { return b }; + let img = average(&fr.imgs, &weights); + with_img(fr.grid, img) +} + +// ---------------------------------------------------------------- CC Force Motion Blur / CC Wide Time / Pixel Motion Blur + +fn force_motion_blur(ctx: &EffectCtx, b: Buf) -> Buf { + let n = ctx.params.f("motionBlurLevels").round().clamp(2.0, 64.0) as usize; + let angle = if ctx.params.b("overrideShutterAngle") { ctx.params.f("shutterAngle") } else { 180.0 }; + let span = angle.clamp(0.0, 3600.0) / 360.0 / ctx.fps(); + let times: Vec = (0..n).map(|i| ctx.time + span * i as f64 / n as f64).collect(); + mean_of(ctx, b, ×) +} + +fn wide_time(ctx: &EffectCtx, b: Buf) -> Buf { + let fwd = ctx.params.f("forwardSteps").round().clamp(0.0, 45.0) as i64; + let back = ctx.params.f("backwardSteps").round().clamp(0.0, 45.0) as i64; + let fd = 1.0 / ctx.fps(); + let times: Vec = (-back..=fwd).map(|k| ctx.time + k as f64 * fd).collect(); + mean_of(ctx, b, ×) +} + +fn pixel_motion_blur(ctx: &EffectCtx, b: Buf) -> Buf { + let angle = if ctx.params.e("shutterControl") == 1 { ctx.params.f("shutterAngle") } else { 180.0 }; + let n = ctx.params.f("shutterSamples").round().clamp(1.0, 64.0) as usize; + let span = angle.clamp(0.0, 720.0) / 360.0 / ctx.fps(); + let times: Vec = (0..n).map(|i| ctx.time - span * 0.5 + if n > 1 { span * i as f64 / (n - 1) as f64 } else { span * 0.5 }).collect(); + mean_of(ctx, b, ×) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, EffectHost, LayerPixels, run_fx}; + + /// A layer whose every pixel at layer time `t` is grey `t` (clamped), alpha 1, on a 6×4 grid. + struct Clock { + static_layer: bool, + } + impl EffectHost for Clock { + fn layer(&self, _: u64, _: bool) -> Option { + None + } + fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option> { + None + } + fn self_at(&self, t: f64, effects: usize) -> Option { + assert_eq!(effects, 0); + let v = if self.static_layer { 0.4 } else { t as f32 }; + Some(Buf { img: Image::filled(6, 4, [v, v, v, 1.0]), offset: [0.0; 2], scale: 1.0 }) + } + } + + fn run(id: &str, vals: &[(&str, Value)], t: f64, host: &Clock) -> Buf { + let env = EffectEnv { host: Some(host), frame_rate: 10.0, comp_time: t, ..Default::default() }; + run_fx(id, vals, Image::filled(6, 4, [0.9, 0.0, 0.0, 1.0]), t, env) + } + + #[test] + fn posterize_time_holds_frames() { + let h = Clock { static_layer: false }; + for (t, held) in [(0.0, 0.0), (0.1, 0.0), (0.24, 0.0), (0.25, 0.25), (0.49, 0.25), (0.5, 0.5)] { + let out = run("ec.time.posterizetime", &[("frameRate", num(4.0))], t, &h); + assert!((out.img.data[0][0] - held).abs() < 1e-5, "t={t}: {}", out.img.data[0][0]); + } + } + + #[test] + fn echo_blends_known_frames_with_decay() { + let h = Clock { static_layer: false }; + let vals = [("echoTime", num(-0.1)), ("numberOfEchoes", num(2.0)), ("startingIntensity", num(1.0)), ("decay", num(0.5))]; + let out = run("ec.time.echo", &vals, 0.8, &h); + // Add: f(0.8) + 0.5 f(0.7) + 0.25 f(0.6) + let want = 0.8 + 0.5 * 0.7 + 0.25 * 0.6; + assert!((out.img.data[0][0] - want as f32).abs() < 1e-4, "{}", out.img.data[0][0]); + assert_eq!(out.img.data[0][3], 1.0); + // Maximum picks the brightest weighted frame; Blend averages. + let mut v = vals.to_vec(); + v.push(("echoOperator", Value::Enum(1))); + let out = run("ec.time.echo", &v, 0.8, &h); + assert!((out.img.data[0][0] - 0.8).abs() < 1e-5); + v.pop(); + v.push(("echoOperator", Value::Enum(6))); + let out = run("ec.time.echo", &v, 0.8, &h); + assert!((out.img.data[0][0] - want as f32 / 3.0).abs() < 1e-4); + } + + #[test] + fn time_difference_of_static_layer_is_black() { + let h = Clock { static_layer: true }; + let out = run("ec.time.timedifference", &[("timeOffset", num(-0.5))], 1.0, &h); + assert!(out.img.data.iter().all(|p| p[0] == 0.0 && p[1] == 0.0 && p[2] == 0.0 && p[3] == 1.0)); + // A changing layer is not black: |1.0 - 0.5| = 0.5. + let h = Clock { static_layer: false }; + let out = run("ec.time.timedifference", &[("timeOffset", num(-0.5))], 1.0, &h); + assert!((out.img.data[0][0] - 0.5).abs() < 1e-5, "{}", out.img.data[0][0]); + } + + #[test] + fn timewarp_and_wide_time() { + let h = Clock { static_layer: false }; + // Speed 50 % at t = 1 s shows source 0.5 s (frame-aligned at 10 fps). + let out = run("ec.time.timewarp", &[("method", Value::Enum(0))], 1.0, &h); + assert!((out.img.data[0][0] - 0.5).abs() < 1e-5); + // Frame Mix between 0.5 and 0.6 at source 0.55. + let out = run("ec.time.timewarp", &[("method", Value::Enum(1))], 1.1, &h); + assert!((out.img.data[0][0] - 0.55).abs() < 1e-4, "{}", out.img.data[0][0]); + // Source Frame mode: frame 3 at 10 fps. + let out = run("ec.time.timewarp", &[("method", Value::Enum(0)), ("adjustTimeBy", Value::Enum(1)), ("sourceFrame", num(3.0))], 2.0, &h); + assert!((out.img.data[0][0] - 0.3).abs() < 1e-5); + // Wide Time is symmetric around t: mean of t-0.3..t+0.3 = t. + let out = run("ec.time.ccwidetime", &[], 1.0, &h); + assert!((out.img.data[0][0] - 1.0).abs() < 1e-4); + // Pixel Motion Blur is centred too. + let out = run("ec.time.pixelmotionblur", &[], 1.0, &h); + assert!((out.img.data[0][0] - 1.0).abs() < 1e-4); + // Force Motion Blur looks forward over the shutter. + let out = run("ec.time.ccforcemotionblur", &[], 1.0, &h); + assert!(out.img.data[0][0] > 1.0); + } + + #[test] + fn time_displacement_uses_luma_as_offset() { + // Static map from a bright self frame: luminance 1.0 → +max seconds. + let h = Clock { static_layer: true }; + let out = run("ec.time.timedisplacement", &[], 0.0, &h); + assert!((out.img.data[0][0] - 0.4).abs() < 1e-5); + } +} diff --git a/crates/render/src/cache.rs b/crates/render/src/cache.rs index 78ff39b5..0f80927b 100644 --- a/crates/render/src/cache.rs +++ b/crates/render/src/cache.rs @@ -202,6 +202,41 @@ fn hash_values(h: &mut KeyHasher, ctx: &EvalCtx, layer: &Layer, g: &PropGroup) { /// time, or `None` when the layer must not be cached (footage, precomps, cameras, adjustment /// layers…). pub fn layer_key(ctx: &EvalCtx, layer: &Layer, scale: f64, draft: bool) -> Option { + // Time effects see property values at other times. Keyframes are part of the hashed + // structure (and the layer time is folded in), but expressions may read other layers at + // those times, which the key cannot see: such layers are not cached. + if reads_other_times(layer) && layer.props.children.iter().any(|c| matches!(c, Node::Group(g) if g.match_id != "transform" && has_expression(g))) { + return None; + } + key_with(ctx, layer, scale, draft, false) +} + +/// Cache key for a layer's *input* at the context time: source → masks → its first `effects` +/// effects (see `Renderer::layer_input`). Unlike [`layer_key`], footage layers are cached here +/// (keyed by item and source time), since Time effects read many neighbouring frames. +pub fn input_key(ctx: &EvalCtx, layer: &Layer, scale: f64, draft: bool, effects: usize) -> Option { + let base = key_with(ctx, layer, scale, draft, true)?; + let mut h = KeyHasher(base ^ 0x5bd1_e995_7a3c_11d3); + effects.hash(&mut h); + Some(h.finish()) +} + +/// Does the layer run an effect that reads the layer at other times (Echo, Timewarp…)? +fn reads_other_times(layer: &Layer) -> bool { + layer.switches.effects + && layer.effects().is_some_and(|fx| { + fx.groups().any(|g| g.enabled && matches!(&g.kind, effectcraft_project::GroupKind::Effect { effect } if effect.starts_with("ec.time."))) + }) +} + +fn has_expression(g: &PropGroup) -> bool { + g.children.iter().any(|c| match c { + Node::Prop(p) => p.has_expression(), + Node::Group(sub) => has_expression(sub), + }) +} + +fn key_with(ctx: &EvalCtx, layer: &Layer, scale: f64, draft: bool, footage: bool) -> Option { if layer.switches.adjustment { return None; } @@ -213,6 +248,13 @@ pub fn layer_key(ctx: &EvalCtx, layer: &Layer, scale: f64, draft: bool) -> Optio hash_debug(&mut h, s); } LayerSource::Text | LayerSource::Shape => {} + LayerSource::Footage { item } if footage => { + let it = ctx.project.item(*item)?; + let ItemKind::Footage(f) = &it.kind else { return None }; + hash_debug(&mut h, f); + item.hash(&mut h); + ctx.source_time(layer).0.hash(&mut h); + } _ => return None, } ctx.comp_id.hash(&mut h); diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 02f61df6..1299f427 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -43,12 +43,17 @@ struct FxHost<'r, 'a, 'c> { r: &'r Renderer<'a>, ctx: &'c EvalCtx<'a>, layer: &'c Layer, + /// Index of the effect being rendered (bounds `self_at` to effects before it). + index: std::sync::atomic::AtomicUsize, } +/// Nesting limit for effects that render layers (other layers, other times). +const MAX_FX_DEPTH: usize = 8; + impl EffectHost for FxHost<'_, '_, '_> { fn layer(&self, id: u64, masks_and_effects: bool) -> Option { let other = self.ctx.layer(effectcraft_project::LayerId(id))?; - if other.id == self.layer.id || self.r.depth > 8 { + if other.id == self.layer.id || self.r.depth > MAX_FX_DEPTH { return None; } let sub = Renderer { depth: self.r.depth + 1, ..*self.r }; @@ -69,6 +74,29 @@ impl EffectHost for FxHost<'_, '_, '_> { let st = other.layer_time(Tick::from_seconds_f64(start)); self.r.footage.audio(*item, f, st, frames, rate) } + + fn self_at(&self, layer_time: f64, effects: usize) -> Option { + if self.r.depth > MAX_FX_DEPTH || self.layer.switches.adjustment { + return None; + } + let n = effects.min(self.index.load(std::sync::atomic::Ordering::Relaxed)); + let t = self.layer.comp_time(Tick::from_seconds_f64(layer_time)); + let sub = Renderer { depth: self.r.depth + 1, ..*self.r }; + sub.layer_input(&self.ctx.at(t), self.layer, n).map(|b| (*b).clone()) + } + + fn layer_at(&self, id: u64, comp_time: f64, masks_and_effects: bool) -> Option { + let other = self.ctx.layer(effectcraft_project::LayerId(id))?; + if other.id == self.layer.id || self.r.depth > MAX_FX_DEPTH { + return None; + } + let ctx = self.ctx.at(Tick::from_seconds_f64(comp_time)); + let sub = Renderer { depth: self.r.depth + 1, ..*self.r }; + let buf = if masks_and_effects { (*sub.layer_buf(&ctx, other)?).clone() } else { (*sub.layer_input(&ctx, other, 0)?).clone() }; + let size = source_size(self.r.project, other); + let size = if size.0 == 0 { [self.ctx.comp.width as f64, self.ctx.comp.height as f64] } else { [size.0 as f64, size.1 as f64] }; + Some(LayerPixels { buf, size }) + } } /// No footage available (renders footage layers as transparent). @@ -186,10 +214,19 @@ impl<'a> Renderer<'a> { } fn apply_effects(&self, ctx: &EvalCtx, layer: &Layer, buf: Buf, adjustment: bool) -> Buf { - self.apply_effects_timed(ctx, layer, buf, adjustment, None) + self.apply_effects_timed(ctx, layer, buf, adjustment, usize::MAX, None) } - fn apply_effects_timed(&self, ctx: &EvalCtx, layer: &Layer, mut buf: Buf, adjustment: bool, mut timing: Option<&mut Vec<(String, f64)>>) -> Buf { + /// Run the first `limit` effects of the layer's stack (by position; disabled ones count). + fn apply_effects_timed( + &self, + ctx: &EvalCtx, + layer: &Layer, + mut buf: Buf, + adjustment: bool, + limit: usize, + mut timing: Option<&mut Vec<(String, f64)>>, + ) -> Buf { if !layer.switches.effects { return buf; } @@ -198,15 +235,24 @@ impl<'a> Renderer<'a> { let size = source_size(self.project, layer); let layer_size = if size.0 == 0 { [ctx.comp.width as f64, ctx.comp.height as f64] } else { [size.0 as f64, size.1 as f64] }; let mask_shapes = masks::shapes(ctx, layer); - let host = FxHost { r: self, ctx, layer }; - let env = EffectEnv { masks: &mask_shapes, host: Some(&host), comp_time: ctx.time.seconds(), frame_rate: ctx.comp.frame_rate.as_f64() }; - for g in fx.groups() { + let host = FxHost { r: self, ctx, layer, index: Default::default() }; + let env = + EffectEnv { masks: &mask_shapes, host: Some(&host), comp_time: ctx.time.seconds(), frame_rate: ctx.comp.frame_rate.as_f64(), effect_index: 0 }; + for (i, g) in fx.groups().enumerate() { + if i >= limit { + break; + } if !g.enabled { continue; } let GroupKind::Effect { effect } = &g.kind else { continue }; let Some(spec) = effectcraft_effects::find(effect) else { continue }; + if effectcraft_effects::audio_fx::is_audio_effect(spec.id) { + continue; + } let params = self.effect_params(ctx, layer, g); + host.index.store(i, std::sync::atomic::Ordering::Relaxed); + let env = EffectEnv { effect_index: i, ..env }; let ectx = EffectCtx { params: ¶ms, time: lt.seconds(), layer_size, seed: g.uid as u32, adjustment, env }; let t0 = std::time::Instant::now(); buf = effectcraft_effects::apply(spec, &ectx, buf); @@ -280,7 +326,29 @@ impl<'a> Renderer<'a> { } let mut buf = self.source(ctx, layer)?; masks::apply(ctx, layer, &mut buf); - let buf = Arc::new(self.apply_effects_timed(ctx, layer, buf, false, timing.map(|t| &mut t.effects))); + let buf = Arc::new(self.apply_effects_timed(ctx, layer, buf, false, usize::MAX, timing.map(|t| &mut t.effects))); + if let (Some(c), Some(k)) = (self.cache, key) { + c.insert(k, buf.clone()); + } + Some(buf) + } + + /// A layer's input at the context time: source → masks → its first `effects` effects + /// (what Time effects read at neighbouring times). Served from / stored in the layer cache + /// under its own key, so scrubbing reuses frames rendered for earlier output frames. + pub fn layer_input(&self, ctx: &EvalCtx, layer: &Layer, effects: usize) -> Option> { + let key = self.cache.and_then(|_| cache::input_key(ctx, layer, self.opts.scale, self.opts.draft, effects)); + if let (Some(c), Some(k)) = (self.cache, key) + && let Some(b) = c.get(k) + { + return Some(b); + } + let mut buf = self.source(ctx, layer)?; + masks::apply(ctx, layer, &mut buf); + if effects > 0 { + buf = self.apply_effects_timed(ctx, layer, buf, false, effects, None); + } + let buf = Arc::new(buf); if let (Some(c), Some(k)) = (self.cache, key) { c.insert(k, buf.clone()); } @@ -453,3 +521,5 @@ pub fn content_bounds(ctx: &EvalCtx, layer: &effectcraft_project::Layer) -> Opti pub fn kurbo_path(sp: &effectcraft_keyframe::ShapePath) -> kurbo::BezPath { effectcraft_path::to_kurbo(sp) } +#[cfg(test)] +mod tests_time; diff --git a/crates/render/src/tests_time.rs b/crates/render/src/tests_time.rs new file mode 100644 index 00000000..b2833fb0 --- /dev/null +++ b/crates/render/src/tests_time.rs @@ -0,0 +1,105 @@ +//! Time effects through the renderer: neighbouring frames via `EffectHost::self_at`, and the +//! layer cache staying correct while scrubbing. + +use effectcraft_color::Label; +use effectcraft_keyframe::{Keyframe, ShapePath, Value}; +use effectcraft_project::build::{self, Ids}; +use effectcraft_project::{Comp, ItemId, ItemKind, LayerSource, MaskMode, Project, Solid}; +use effectcraft_time::{FrameRate, Tick}; + +fn render(p: &Project, cid: ItemId, t: f64, cache: Option<&crate::LayerCache>) -> crate::Image { + let mut r = crate::Renderer::new(p, &crate::NoFootage, crate::RenderOpts::default()); + r.cache = cache; + r.comp_frame(cid, Tick::from_seconds_f64(t)) +} + +/// A white 200×100 solid whose full-frame mask fades in (opacity 0 → 100 over 0..1 s), with +/// effect `fx` (and parameter overrides) applied. +fn scene(fx: Option<(&str, &[(&str, Value)])>) -> (Project, ItemId) { + let mut p = Project::default(); + let comp = Comp::new(200, 100, FrameRate::FPS_30, Tick::from_seconds_f64(2.0)); + let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.clone().into())); + let sid = p.add_item("Solid", Label::Red, None, ItemKind::Solid(Solid { color: [1.0, 1.0, 1.0], width: 200, height: 100, pixel_aspect: 1.0 })); + let mut l = build::layer(&mut p, &comp, "Solid", LayerSource::Solid { item: sid }, (200, 100), None); + let mut next = p.next_id; + let mut m = build::mask(&mut Ids(&mut next), "Mask 1", ShapePath::rect([100.0, 50.0], 400.0, 200.0), MaskMode::Add, [255, 255, 0]); + if let Some(o) = m.prop_mut("opacity") { + o.keys = vec![Keyframe::new(Tick::ZERO, Value::Scalar(0.0)), Keyframe::new(Tick::from_seconds_f64(1.0), Value::Scalar(100.0))]; + } + l.props.sub_mut("masks").unwrap().children.push(m.into()); + if let Some((id, vals)) = fx { + let spec = effectcraft_effects::find(id).unwrap(); + let mut g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), spec.name, [200.0, 100.0]); + for (k, v) in vals { + g.prop_mut(k).unwrap().value = v.clone(); + } + l.props.sub_mut("effects").unwrap().children.push(g.into()); + } + p.next_id = next; + p.comp_mut(cid).unwrap().layers.push(l); + (p, cid) +} + +fn alpha(img: &crate::Image) -> f32 { + img.get(100, 50)[3] +} + +#[test] +fn posterize_time_holds_frames() { + let (p, cid) = scene(Some(("ec.time.posterizetime", &[("frameRate", Value::Scalar(4.0))]))); + for (t, want) in [(0.3, 0.25), (0.45, 0.25), (0.5, 0.5), (0.74, 0.5), (0.8, 0.75)] { + let a = alpha(&render(&p, cid, t, None)); + assert!((a - want).abs() < 0.01, "t={t}: {a}"); + } + let (plain, cid) = scene(None); + assert!((alpha(&render(&plain, cid, 0.3, None)) - 0.3).abs() < 0.01); +} + +#[test] +fn echo_blends_neighbouring_frames() { + let vals = [("echoTime", Value::Scalar(-0.1)), ("numberOfEchoes", Value::Scalar(2.0)), ("decay", Value::Scalar(0.5))]; + let (p, cid) = scene(Some(("ec.time.echo", &vals))); + let a = alpha(&render(&p, cid, 0.3, None)); + let want = 0.3 + 0.5 * 0.2 + 0.25 * 0.1; + assert!((a - want).abs() < 0.01, "{a} vs {want}"); +} + +#[test] +fn time_difference_of_static_layer_is_black() { + let mut p = Project::default(); + let comp = Comp::new(64, 32, FrameRate::FPS_30, Tick::from_seconds_f64(2.0)); + let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.clone().into())); + let sid = p.add_item("Solid", Label::Red, None, ItemKind::Solid(Solid { color: [0.8, 0.4, 0.2], width: 64, height: 32, pixel_aspect: 1.0 })); + let mut l = build::layer(&mut p, &comp, "Solid", LayerSource::Solid { item: sid }, (64, 32), None); + let spec = effectcraft_effects::find("ec.time.timedifference").unwrap(); + let mut next = p.next_id; + let mut g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), spec.name, [64.0, 32.0]); + g.prop_mut("timeOffset").unwrap().value = Value::Scalar(-0.5); + p.next_id = next; + l.props.sub_mut("effects").unwrap().children.push(g.into()); + p.comp_mut(cid).unwrap().layers.push(l); + let img = render(&p, cid, 1.0, None); + assert!(img.data.iter().all(|px| px[0] == 0.0 && px[1] == 0.0 && px[2] == 0.0 && px[3] == 1.0), "{:?}", img.get(10, 10)); +} + +/// Scrubbing back and forth with the layer cache gives exactly the uncached frames (Echo's +/// output depends on neighbouring frames, so its key must include the layer time), and an edit +/// to the animation is picked up. +#[test] +fn cache_correct_while_scrubbing() { + let vals = [("echoTime", Value::Scalar(-0.2)), ("numberOfEchoes", Value::Scalar(3.0)), ("decay", Value::Scalar(0.6)), ("echoOperator", Value::Enum(6))]; + let (mut p, cid) = scene(Some(("ec.time.echo", &vals))); + let cache = crate::LayerCache::default(); + for t in [0.3, 0.5, 0.3, 1.2, 0.8, 0.5, 0.3, 1.5] { + let a = render(&p, cid, t, Some(&cache)); + let b = render(&p, cid, t, None); + assert_eq!(a.data, b.data, "t={t}"); + } + let st = cache.stats(); + assert!(st.hits > 0, "{st:?}"); + // Re-key the fade (0 → 100 over 0..0.5 s): cached frames must not be served. + p.comp_mut(cid).unwrap().layers[0].props.prop_mut("masks/#1/opacity").unwrap().keys[1].time = Tick::from_seconds_f64(0.5); + for t in [0.3, 0.5, 0.8] { + assert_eq!(render(&p, cid, t, Some(&cache)).data, render(&p, cid, t, None).data, "edited t={t}"); + } +} From ddb33cdfd0581076d95deb8d0c2e98fd7467bc58 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 00:16:34 -0400 Subject: [PATCH 2/2] M11.2: audio playback + waveforms + audio effects - Audio effects (Backwards, Bass & Treble, Delay, Flange & Chorus, High-Low Pass, Modulator, Parametric EQ, Reverb, Stereo Mixer, Tone) applied per layer in the comp mixdown, so export and preview share them. RBJ-cookbook biquads, Schroeder/Moorer reverb, modulated delay lines; stateless per block via pre-roll. Tone makes any layer audible. - Real-time audio preview: ui-egui AudioFeed/AudioPlayback (feeder thread mixing ahead) behind an AudioDevice hook; the desktop app implements it with cpal (L6). Audio clock drives playback (consumed frames minus device latency, looped through the work area). Preview panel Include Audio / playback.muteAudio. - Audio panel VU meters with peak hold and clip indicators; selected layer's Audio Levels. - Timeline Audio > Waveform for footage with audio (L reveals Audio Levels, LL the Waveform), drawn from a cached per-item peak summary. - ui.playback reports audio state and meter levels. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 267 +++++++++++++++++++- Cargo.toml | 1 + apps/effectcraft/Cargo.toml | 1 + apps/effectcraft/src/audio_out.rs | 139 +++++++++++ apps/effectcraft/src/main.rs | 2 + crates/render/src/audio.rs | 252 +++++++++++++++---- crates/render/src/lib.rs | 2 + crates/render/src/tests_audio_fx.rs | 204 +++++++++++++++ crates/ui-egui/src/audio.rs | 345 ++++++++++++++++++++++++++ crates/ui-egui/src/control.rs | 5 +- crates/ui-egui/src/lib.rs | 60 +++++ crates/ui-egui/src/menus.rs | 20 +- crates/ui-egui/src/panels/misc.rs | 99 +++++++- crates/ui-egui/src/panels/mod.rs | 1 + crates/ui-egui/src/panels/timeline.rs | 51 +++- crates/ui-egui/src/panels/waveform.rs | 95 +++++++ crates/ui-egui/src/state.rs | 4 + docs/architecture.md | 10 + docs/control-protocol.md | 2 +- 19 files changed, 1483 insertions(+), 77 deletions(-) create mode 100644 apps/effectcraft/src/audio_out.rs create mode 100644 crates/render/src/tests_audio_fx.rs create mode 100644 crates/ui-egui/src/audio.rs create mode 100644 crates/ui-egui/src/panels/waveform.rs diff --git a/Cargo.lock b/Cargo.lock index 9d3c76d3..32b3a154 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,8 +97,8 @@ dependencies = [ "accesskit_consumer 0.35.0", "hashbrown 0.16.1", "static_assertions", - "windows", - "windows-core", + "windows 0.62.2", + "windows-core 0.62.2", ] [[package]] @@ -149,6 +149,28 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "alsa" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43" +dependencies = [ + "alsa-sys", + "bitflags 2.13.2", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "android-activity" version = "0.6.1" @@ -158,7 +180,7 @@ dependencies = [ "android-properties", "bitflags 2.13.2", "cc", - "jni", + "jni 0.22.4", "libc", "log", "ndk", @@ -737,6 +759,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "1.0.5" @@ -869,12 +897,52 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" +[[package]] +name = "coreaudio-rs" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aae284fbaf7d27aa0e292f7677dfbe26503b0d555026f702940805a630eac17" +dependencies = [ + "bitflags 1.3.2", + "libc", + "objc2-audio-toolbox", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", +] + [[package]] name = "cow-utils" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "417bef24afe1460300965a25ff4a24b8b45ad011948302ec221e8a0a81eb2c79" +[[package]] +name = "cpal" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbd307f43cc2a697e2d1f8bc7a1d824b5269e052209e28883e5bc04d095aaa3f" +dependencies = [ + "alsa", + "coreaudio-rs", + "dasp_sample", + "jni 0.21.1", + "js-sys", + "libc", + "mach2", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "objc2-audio-toolbox", + "objc2-core-audio", + "objc2-core-audio-types", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows 0.54.0", +] + [[package]] name = "cpu-time" version = "1.0.0" @@ -954,6 +1022,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "deranged" version = "0.5.8" @@ -1053,6 +1127,7 @@ dependencies = [ name = "effectcraft" version = "0.1.0" dependencies = [ + "cpal", "effectcraft-engine", "effectcraft-host", "effectcraft-ui-egui", @@ -2126,7 +2201,7 @@ dependencies = [ "log", "presser", "thiserror 2.0.21", - "windows", + "windows 0.62.2", ] [[package]] @@ -2354,6 +2429,22 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + [[package]] name = "jni" version = "0.22.4" @@ -2577,6 +2668,15 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "mach2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" +dependencies = [ + "libc", +] + [[package]] name = "memchr" version = "2.8.3" @@ -2738,6 +2838,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "num-integer" version = "0.1.47" @@ -2843,6 +2954,21 @@ dependencies = [ "objc2-foundation 0.3.2", ] +[[package]] +name = "objc2-audio-toolbox" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6948501a91121d6399b79abaa33a8aa4ea7857fe019f341b8c23ad6e81b79b08" +dependencies = [ + "bitflags 2.13.2", + "libc", + "objc2 0.6.4", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-cloud-kit" version = "0.2.2" @@ -2867,6 +2993,28 @@ dependencies = [ "objc2-foundation 0.2.2", ] +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2 0.6.4", + "objc2-core-audio-types", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags 2.13.2", + "objc2 0.6.4", +] + [[package]] name = "objc2-core-data" version = "0.2.2" @@ -5067,9 +5215,9 @@ dependencies = [ "web-sys", "wgpu-naga-bridge", "wgpu-types", - "windows", - "windows-core", - "windows-result", + "windows 0.62.2", + "windows-core 0.62.2", + "windows-result 0.4.1", ] [[package]] @@ -5129,6 +5277,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" +dependencies = [ + "windows-core 0.54.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows" version = "0.62.2" @@ -5136,7 +5294,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" dependencies = [ "windows-collections", - "windows-core", + "windows-core 0.62.2", "windows-future", "windows-numerics", ] @@ -5147,7 +5305,17 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" dependencies = [ - "windows-core", + "windows-core 0.62.2", +] + +[[package]] +name = "windows-core" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" +dependencies = [ + "windows-result 0.1.2", + "windows-targets 0.52.6", ] [[package]] @@ -5159,7 +5327,7 @@ dependencies = [ "windows-implement", "windows-interface", "windows-link", - "windows-result", + "windows-result 0.4.1", "windows-strings", ] @@ -5169,7 +5337,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" dependencies = [ - "windows-core", + "windows-core 0.62.2", "windows-link", "windows-threading", ] @@ -5208,10 +5376,19 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" dependencies = [ - "windows-core", + "windows-core 0.62.2", "windows-link", ] +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -5230,6 +5407,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -5266,6 +5452,21 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -5308,6 +5509,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -5320,6 +5527,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -5332,6 +5545,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -5356,6 +5575,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -5368,6 +5593,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -5380,6 +5611,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -5392,6 +5629,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" diff --git a/Cargo.toml b/Cargo.toml index bd1d4b20..ab4eef12 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ egui = "0.36" eframe = { version = "0.36", default-features = false } egui_kittest = "0.36" rfd = "0.17" +cpal = "0.16" # Local co-development against a FilmCraft checkout next to this repo: uncomment. # [patch."https://github.com/storytold/filmcraft"] diff --git a/apps/effectcraft/Cargo.toml b/apps/effectcraft/Cargo.toml index 3240c78d..5a2165aa 100644 --- a/apps/effectcraft/Cargo.toml +++ b/apps/effectcraft/Cargo.toml @@ -17,6 +17,7 @@ egui = { workspace = true } eframe = { workspace = true, features = ["default_fonts", "wgpu", "accesskit", "wayland", "x11"] } serde_json = { workspace = true } rfd = { workspace = true } +cpal = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] # Same winit as eframe: macOS event-loop options (no app activation when driven by an agent). diff --git a/apps/effectcraft/src/audio_out.rs b/apps/effectcraft/src/audio_out.rs new file mode 100644 index 00000000..bef9ff2e --- /dev/null +++ b/apps/effectcraft/src/audio_out.rs @@ -0,0 +1,139 @@ +//! Audio preview output through cpal (the default output device). +//! +//! cpal streams are not `Send` on every platform, so each preview runs its stream on a small +//! dedicated thread that owns it until stopped. The callback pulls interleaved stereo from the +//! frontend's [`AudioFeed`] and maps it onto the device's channels (mono: L+R average; extra +//! channels: silence), converting to the device's sample format. + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::mpsc; +use std::thread::JoinHandle; + +use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; +use cpal::{FromSample, SizedSample}; +use effectcraft_ui_egui::audio::{AudioDevice, AudioFeed}; + +pub struct CpalOut { + rate: u32, + latency: Arc, + stop: Option>, + thread: Option>, +} + +/// Open the default output device (`None` without one). +pub fn open() -> Option> { + let host = cpal::default_host(); + let dev = host.default_output_device()?; + let cfg = dev.default_output_config().ok()?; + Some(Box::new(CpalOut { rate: cfg.sample_rate().0, latency: Arc::new(AtomicU64::new(0)), stop: None, thread: None })) +} + +fn run>( + dev: &cpal::Device, + cfg: &cpal::StreamConfig, + feed: Arc, + latency: Arc, +) -> Result { + let ch = cfg.channels as usize; + let rate = cfg.sample_rate.0 as f64; + let mut stereo: Vec = Vec::new(); + dev.build_output_stream( + cfg, + move |out: &mut [T], info: &cpal::OutputCallbackInfo| { + let frames = out.len() / ch.max(1); + stereo.resize(frames * 2, 0.0); + feed.pull(&mut stereo); + for (f, s) in out.chunks_mut(ch.max(1)).zip(stereo.chunks_exact(2)) { + for (c, o) in f.iter_mut().enumerate() { + let v = match (ch, c) { + (1, _) => (s[0] + s[1]) * 0.5, + (_, 0) => s[0], + (_, 1) => s[1], + _ => 0.0, + }; + *o = T::from_sample(v.clamp(-1.0, 1.0)); + } + } + // Frames between "handed over" and "audible": device latency plus this buffer. + let ts = info.timestamp(); + let dev_lat = ts.playback.duration_since(&ts.callback).map(|d| d.as_secs_f64()).unwrap_or(0.0); + latency.store((dev_lat * rate) as u64 + frames as u64, Ordering::Relaxed); + }, + |e| eprintln!("effectcraft: audio output: {e}"), + None, + ) + .map_err(|e| e.to_string()) +} + +impl AudioDevice for CpalOut { + fn sample_rate(&self) -> u32 { + self.rate + } + + fn start(&mut self, feed: Arc) -> Result<(), String> { + self.stop(); + let (stop_tx, stop_rx) = mpsc::channel::<()>(); + let (ready_tx, ready_rx) = mpsc::channel::>(); + let latency = self.latency.clone(); + let rate = self.rate; + let thread = std::thread::Builder::new() + .name("ec-audio-out".into()) + .spawn(move || { + let host = cpal::default_host(); + let Some(dev) = host.default_output_device() else { + let _ = ready_tx.send(Err("no audio output device".into())); + return; + }; + let sup = match dev.default_output_config() { + Ok(c) => c, + Err(e) => { + let _ = ready_tx.send(Err(e.to_string())); + return; + } + }; + let mut cfg: cpal::StreamConfig = sup.config(); + cfg.sample_rate = cpal::SampleRate(rate); + let stream = match sup.sample_format() { + cpal::SampleFormat::F32 => run::(&dev, &cfg, feed, latency), + cpal::SampleFormat::I16 => run::(&dev, &cfg, feed, latency), + cpal::SampleFormat::U16 => run::(&dev, &cfg, feed, latency), + cpal::SampleFormat::I32 => run::(&dev, &cfg, feed, latency), + f => Err(format!("unsupported sample format {f:?}")), + }; + let stream = match stream.and_then(|s| s.play().map(|_| s).map_err(|e| e.to_string())) { + Ok(s) => s, + Err(e) => { + let _ = ready_tx.send(Err(e)); + return; + } + }; + let _ = ready_tx.send(Ok(())); + let _ = stop_rx.recv(); + drop(stream); + }) + .map_err(|e| e.to_string())?; + self.stop = Some(stop_tx); + self.thread = Some(thread); + ready_rx.recv().map_err(|e| e.to_string())? + } + + fn stop(&mut self) { + if let Some(tx) = self.stop.take() { + let _ = tx.send(()); + } + if let Some(t) = self.thread.take() { + let _ = t.join(); + } + } + + fn latency_frames(&self) -> u64 { + self.latency.load(Ordering::Relaxed) + } +} + +impl Drop for CpalOut { + fn drop(&mut self) { + self.stop(); + } +} diff --git a/apps/effectcraft/src/main.rs b/apps/effectcraft/src/main.rs index 9e8bcd6f..2c6ca716 100644 --- a/apps/effectcraft/src/main.rs +++ b/apps/effectcraft/src/main.rs @@ -5,6 +5,7 @@ //! `--control ` (or `EFFECTCRAFT_CONTROL_PORT`) starts a localhost JSON-lines control server; //! see `effectcraft_ui_egui::control` for the methods. +mod audio_out; mod control_server; use effectcraft_ui_egui::EffectcraftApp; @@ -69,6 +70,7 @@ fn main() -> eframe::Result { })); app.hooks.pick_open_project = Some(Box::new(|| rfd::FileDialog::new().add_filter("EffectCraft Project", &["ecproj"]).pick_file().map(|p| p.to_string_lossy().to_string()))); + app.hooks.audio_device = Some(Box::new(audio_out::open)); if let Some(port) = control_port { disable_app_nap(); let rx = control_server::start(port, cc.egui_ctx.clone()); diff --git a/crates/render/src/audio.rs b/crates/render/src/audio.rs index d7ae4778..246ff683 100644 --- a/crates/render/src/audio.rs +++ b/crates/render/src/audio.rs @@ -1,10 +1,20 @@ -//! Audio mixdown of a composition: every audible footage layer (and, recursively, precomp layer) -//! summed with its Audio Levels (dB, per channel), respecting in/out points, the Audio switch and -//! solo. Levels are evaluated once per call (callers mix in short blocks, e.g. one video frame). +//! Audio mixdown of a composition: every audible layer (footage with audio, precomps, and any +//! layer with a sound-generating effect such as Tone) summed with its Audio Levels (dB, per +//! channel), respecting in/out points, the Audio switch and solo. Layer audio effects +//! (Effect > Audio, see [`effectcraft_effects::audio_fx`]) run on each layer's audio before +//! its levels; Backwards plays the layer's [in, out) span reversed. +//! +//! Callers mix in short blocks (one video frame, or an audio device buffer). Effects and levels +//! are evaluated once per block. To make the result independent of block boundaries, layer +//! effects are run from a cleared state over a pre-roll of earlier audio and only the block's +//! tail is kept — so preview playback and export produce identical samples. +//! //! Time-stretched layers are resampled nearest-sample; reversed (negative stretch) layers are //! silent for now. -use effectcraft_project::{Comp, ItemId, ItemKind, Layer, LayerSource, Project}; +use effectcraft_effects::Params; +use effectcraft_effects::audio_fx; +use effectcraft_project::{Comp, GroupKind, ItemId, ItemKind, Layer, LayerSource, Node, Project}; use effectcraft_time::{TICKS_PER_SECOND, Tick}; use crate::{EvalCtx, ExprHost, FootageSource}; @@ -19,32 +29,49 @@ pub fn mix_comp(project: &Project, footage: &dyn FootageSource, expr: Option<&dy out } -/// Whether the comp has anything audible (footage with audio, or a precomp that has). +/// Whether the comp has anything audible (footage with audio, a sound generator, or a precomp +/// that has). pub fn comp_has_audio(project: &Project, comp: ItemId) -> bool { fn walk(project: &Project, comp: ItemId, depth: usize) -> bool { let Some(c) = project.comp(comp) else { return false }; depth < MAX_DEPTH && c.layers.iter().any(|l| { l.switches.audio - && match &l.source { - LayerSource::Footage { item } => { - matches!(project.item(*item).map(|i| &i.kind), Some(ItemKind::Footage(f)) if f.has_audio && !f.missing) - } - LayerSource::Comp { item } => walk(project, *item, depth + 1), - _ => false, - } + && (generates(l) + || match &l.source { + LayerSource::Footage { item } => { + matches!(project.item(*item).map(|i| &i.kind), Some(ItemKind::Footage(f)) if f.has_audio && !f.missing) + } + LayerSource::Comp { item } => walk(project, *item, depth + 1), + _ => false, + }) }) } walk(project, comp, 0) } -fn audible(project: &Project, l: &Layer) -> bool { +/// Enabled audio effects of a layer (in stack order) with their effect ids. +fn audio_effect_groups(l: &Layer) -> impl Iterator { + l.effects().filter(|_| l.switches.effects).into_iter().flat_map(|fx| fx.groups()).filter(|g| g.enabled).filter_map(|g| match &g.kind { + GroupKind::Effect { effect } if audio_fx::is_audio_effect(effect) => Some((effect.as_str(), g)), + _ => None, + }) +} + +/// The layer makes sound on its own (Tone). +fn generates(l: &Layer) -> bool { + audio_effect_groups(l).any(|(id, _)| audio_fx::generates_audio(id)) +} + +/// Does the layer contribute to the mix (Audio switch on and something to hear)? +pub fn audible(project: &Project, l: &Layer) -> bool { l.switches.audio - && match &l.source { - LayerSource::Footage { item } => matches!(project.item(*item).map(|i| &i.kind), Some(ItemKind::Footage(f)) if f.has_audio), - LayerSource::Comp { .. } => true, - _ => false, - } + && (generates(l) + || match &l.source { + LayerSource::Footage { item } => matches!(project.item(*item).map(|i| &i.kind), Some(ItemKind::Footage(f)) if f.has_audio), + LayerSource::Comp { .. } => true, + _ => false, + }) } fn db_to_gain(db: f64) -> f32 { @@ -61,6 +88,116 @@ fn sample_tick(i: i64, rate: u32) -> Tick { Tick(((i as i128 * TICKS_PER_SECOND as i128) / rate as i128) as i64) } +struct MixEnv<'a> { + project: &'a Project, + footage: &'a dyn FootageSource, + expr: Option<&'a dyn ExprHost>, + comp_id: ItemId, + comp: &'a Comp, + rate: u32, + depth: usize, +} + +impl MixEnv<'_> { + fn ctx(&self, t: Tick) -> EvalCtx<'_> { + let mut ctx = EvalCtx::new(self.project, self.comp_id, self.comp, t); + ctx.expr = self.expr; + ctx + } + + /// The layer's own (pre-effect) audio for comp sample indices `i0 .. i0 + n`, silent + /// outside its [in, out) span. Comp sample index `k` maps to `in + out − 1 − k` when + /// `reversed`. + fn layer_raw(&self, l: &Layer, i0: i64, n: usize, reversed: bool) -> Vec { + let rate = self.rate; + let mut out = vec![0.0f32; n * 2]; + let (in_i, out_i) = (units_ceil(l.in_point, rate), units_ceil(l.out_point, rate)); + if reversed { + // Mirror the clipped range, fetch it forwards, reverse. + let a = i0.max(in_i); + let b = (i0 + n as i64).min(out_i); + if a >= b { + return out; + } + let (ma, mb) = (in_i + out_i - b, in_i + out_i - a); + let fwd = self.layer_raw(l, ma, (mb - ma) as usize, false); + for k in a..b { + let j = (in_i + out_i - 1 - k - ma) as usize; + let o = (k - i0) as usize; + out[o * 2] = fwd[j * 2]; + out[o * 2 + 1] = fwd[j * 2 + 1]; + } + return out; + } + let a = (in_i - i0).clamp(0, n as i64) as usize; + let b = (out_i - i0).clamp(0, n as i64) as usize; + if a >= b { + return out; + } + let m = b - a; + let speed = 100.0 / l.stretch; + let src_n = ((m as f64 * speed).ceil() as usize).max(1); + let lt = l.layer_time(sample_tick(i0 + a as i64, rate)); + let buf = match &l.source { + LayerSource::Footage { item } => match self.project.item(*item).map(|i| &i.kind) { + Some(ItemKind::Footage(f)) if f.has_audio => self.footage.audio(*item, f, lt, src_n, rate).unwrap_or_else(|| vec![0.0; src_n * 2]), + _ => return out, + }, + LayerSource::Comp { item } => { + if self.project.comp_contains(*item, self.comp_id) { + return out; + } + let mut sub = vec![0.0f32; src_n * 2]; + mix_into(self.project, self.footage, self.expr, *item, lt, rate, &mut sub, 1.0, 1.0, self.depth + 1); + sub + } + _ => return out, + }; + let same = (speed - 1.0).abs() < 1e-9; + for i in 0..m { + let j = if same { i } else { ((i as f64 * speed) as usize).min(src_n - 1) }; + if j * 2 + 1 >= buf.len() { + break; + } + out[(a + i) * 2] = buf[j * 2]; + out[(a + i) * 2 + 1] = buf[j * 2 + 1]; + } + out + } + + /// Effect parameters at comp time `t`. + fn params(&self, l: &Layer, g: &effectcraft_project::PropGroup, t: Tick) -> Params { + let ctx = self.ctx(t); + let mut p = Params::default(); + for c in &g.children { + if let Node::Prop(pr) = c { + p.values.insert(pr.match_id.clone(), ctx.value(l, pr)); + } + } + p + } + + /// The layer's processed audio (effects applied, levels not) for comp sample indices + /// `i0 .. i0 + n`. + fn layer_audio(&self, l: &Layer, i0: i64, n: usize) -> Vec { + let t = sample_tick(i0, self.rate); + let fx: Vec<(&str, Params)> = audio_effect_groups(l).map(|(id, g)| (id, self.params(l, g, t))).collect(); + if fx.is_empty() { + return self.layer_raw(l, i0, n, false); + } + let reversed = fx.iter().any(|(id, _)| audio_fx::reverses(id)); + let pre_s: f64 = fx.iter().map(|(id, p)| audio_fx::preroll(id, p)).sum(); + let pre = (pre_s * self.rate as f64).ceil() as usize; + let mut buf = self.layer_raw(l, i0 - pre as i64, n + pre, reversed); + let start = l.layer_time(sample_tick(i0 - pre as i64, self.rate)).seconds(); + for (id, p) in &fx { + audio_fx::process(id, p, &mut buf, self.rate, start); + } + buf.drain(..pre * 2); + buf + } +} + #[allow(clippy::too_many_arguments)] fn mix_into( project: &Project, @@ -78,6 +215,7 @@ fn mix_into( if depth >= MAX_DEPTH || rate == 0 { return; } + let env = MixEnv { project, footage, expr, comp_id, comp, rate, depth }; let frames = out.len() / 2; let any_solo = comp.layers.iter().any(|l| l.switches.solo && audible(project, l)); let s0 = start.to_units_floor(rate as i64); @@ -92,50 +230,64 @@ fn mix_into( continue; } let t_a = sample_tick(s0 + a as i64, rate); - let (ll, lr) = levels(project, comp_id, comp, l, t_a, expr); + let (ll, lr) = levels(&env, l, t_a); let (ll, lr) = (ll * gl, lr * gr); if ll == 0.0 && lr == 0.0 { continue; } - let n = b - a; - let speed = 100.0 / l.stretch; - let src_n = ((n as f64 * speed).ceil() as usize).max(1); - let lt = l.layer_time(t_a); - let buf = match &l.source { - LayerSource::Footage { item } => match project.item(*item).map(|i| &i.kind) { - Some(ItemKind::Footage(f)) => footage.audio(*item, f, lt, src_n, rate).unwrap_or_else(|| vec![0.0; src_n * 2]), - _ => continue, - }, - LayerSource::Comp { item } => { - if project.comp_contains(*item, comp_id) { - continue; - } - let mut sub = vec![0.0f32; src_n * 2]; - mix_into(project, footage, expr, *item, lt, rate, &mut sub, 1.0, 1.0, depth + 1); - sub - } - _ => continue, - }; - let same = (speed - 1.0).abs() < 1e-9; - for i in 0..n { - let j = if same { i } else { ((i as f64 * speed) as usize).min(src_n - 1) }; - if j * 2 + 1 >= buf.len() { - break; - } - out[(a + i) * 2] += buf[j * 2] * ll; - out[(a + i) * 2 + 1] += buf[j * 2 + 1] * lr; + let buf = env.layer_audio(l, s0 + a as i64, b - a); + for (i, f) in buf.chunks_exact(2).enumerate() { + out[(a + i) * 2] += f[0] * ll; + out[(a + i) * 2 + 1] += f[1] * lr; } } } -fn levels(project: &Project, comp_id: ItemId, comp: &Comp, l: &Layer, t: Tick, expr: Option<&dyn ExprHost>) -> (f32, f32) { +fn levels(env: &MixEnv, l: &Layer, t: Tick) -> (f32, f32) { let Some(g) = l.props.sub("audio") else { return (1.0, 1.0) }; - let mut ctx = EvalCtx::new(project, comp_id, comp, t); - ctx.expr = expr; - let [a, b] = ctx.v2(l, g, "levels", [0.0, 0.0]); + let [a, b] = env.ctx(t).v2(l, g, "levels", [0.0, 0.0]); (db_to_gain(a), db_to_gain(b)) } +/// A min/max peak summary of interleaved stereo audio for waveform drawing: one entry per +/// `bin` sample frames, `[min L, max L, min R, max R]`. +pub fn peak_summary(samples: &[f32], bin: usize) -> Vec<[f32; 4]> { + let bin = bin.max(1); + samples + .chunks(bin * 2) + .map(|c| { + let mut s = [f32::MAX, f32::MIN, f32::MAX, f32::MIN]; + for f in c.chunks_exact(2) { + s[0] = s[0].min(f[0]); + s[1] = s[1].max(f[0]); + s[2] = s[2].min(f[1]); + s[3] = s[3].max(f[1]); + } + if s[0] > s[1] { [0.0; 4] } else { s } + }) + .collect() +} + +/// Peak summary of a footage item's audio from source time 0 over `duration`, `bins_per_sec` +/// entries per second (rendered at a low internal rate, fetched in one-second chunks). +pub fn footage_peaks(footage: &dyn FootageSource, item: ItemId, f: &effectcraft_project::Footage, duration: Tick, bins_per_sec: u32) -> Vec<[f32; 4]> { + let bins_per_sec = bins_per_sec.max(1); + // Enough samples per bin to catch peaks of audible content, without decoding at full rate. + let bin = (64usize).max(4000 / bins_per_sec as usize); + let rate = bin as u32 * bins_per_sec; + let chunk = bin * bins_per_sec as usize; + let total = units_ceil(duration, rate).max(0) as usize; + let mut out = Vec::with_capacity(total / bin + 1); + let mut i = 0usize; + while i < total { + let n = chunk.min(total - i); + let buf = footage.audio(item, f, sample_tick(i as i64, rate), n, rate).unwrap_or_else(|| vec![0.0; n * 2]); + out.extend(peak_summary(&buf, bin)); + i += n; + } + out +} + #[cfg(test)] mod tests { use std::sync::Arc; diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 1299f427..62f9ed44 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -522,4 +522,6 @@ pub fn kurbo_path(sp: &effectcraft_keyframe::ShapePath) -> kurbo::BezPath { effectcraft_path::to_kurbo(sp) } #[cfg(test)] +mod tests_audio_fx; +#[cfg(test)] mod tests_time; diff --git a/crates/render/src/tests_audio_fx.rs b/crates/render/src/tests_audio_fx.rs new file mode 100644 index 00000000..173ffc88 --- /dev/null +++ b/crates/render/src/tests_audio_fx.rs @@ -0,0 +1,204 @@ +//! Audio effects through the mixdown (the path preview playback and export share). + +use std::sync::Arc; + +use effectcraft_keyframe::Value; +use effectcraft_project::build::{self, Ids}; +use effectcraft_project::{Comp, Footage, FootageKind, ItemId, ItemKind, LayerSource, Project, Solid}; +use effectcraft_time::{FrameRate, Tick}; + +use crate::audio::{comp_has_audio, footage_peaks, mix_comp, peak_summary}; +use crate::{FootageSource, Image}; + +const SR: u32 = 8000; + +/// Synthetic footage audio as a function of source time (seconds) → (L, R). +struct Signal(fn(f64) -> (f32, f32)); +impl FootageSource for Signal { + fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option> { + None + } + fn audio(&self, _: ItemId, _: &Footage, start: Tick, frames: usize, rate: u32) -> Option> { + let t0 = start.seconds(); + Some( + (0..frames) + .flat_map(|i| { + let t = t0 + i as f64 / rate as f64; + let (l, r) = if (0.0..2.0).contains(&t) { (self.0)(t) } else { (0.0, 0.0) }; + [l, r] + }) + .collect(), + ) + } +} + +fn audio_footage() -> Footage { + Footage { + path: "x.wav".into(), + kind: FootageKind::Audio, + width: 0, + height: 0, + pixel_aspect: 1.0, + frame_rate: FrameRate::new(25, 1), + native_rate: None, + duration: Tick::from_seconds_f64(2.0), + has_video: false, + has_audio: true, + alpha: Default::default(), + premul_color: [0.0; 3], + loop_count: 1, + codec: String::new(), + missing: false, + sequence: vec![], + } +} + +/// A 2 s comp with one audio layer (in 0, out 1 s) carrying effect `id` with `vals`. +fn comp_with(id: Option<&str>, vals: &[(&str, Value)], solid: bool) -> (Project, ItemId) { + let mut p = Project::default(); + let comp = Comp::new(64, 64, FrameRate::new(25, 1), Tick::from_seconds_f64(2.0)); + let src = if solid { + let sid = p.add_item("S", effectcraft_color::Label::Red, None, ItemKind::Solid(Solid { color: [1.0; 3], width: 64, height: 64, pixel_aspect: 1.0 })); + LayerSource::Solid { item: sid } + } else { + LayerSource::Footage { item: p.add_item("x.wav", effectcraft_color::Label::SeaFoam, None, ItemKind::Footage(audio_footage())) } + }; + let mut l = build::layer(&mut p, &comp, "a", src, (64, 64), None); + l.out_point = Tick::from_seconds_f64(1.0); + if let Some(id) = id { + let spec = effectcraft_effects::find(id).unwrap(); + let mut next = p.next_id; + let mut g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), spec.name, [64.0, 64.0]); + p.next_id = next; + for (k, v) in vals { + g.prop_mut(k).unwrap_or_else(|| panic!("{k}")).value = v.clone(); + } + l.props.sub_mut("effects").unwrap().children.push(g.into()); + } + let mut comp = comp; + comp.layers = vec![l]; + let cid = p.add_item("C", effectcraft_color::Label::Sandstone, None, ItemKind::Comp(comp.into())); + (p, cid) +} + +fn mix(p: &Project, src: &dyn FootageSource, cid: ItemId, from: f64, frames: usize) -> Vec { + mix_comp(p, src, None, cid, Tick::from_seconds_f64(from), frames, SR) +} + +fn ramp(t: f64) -> (f32, f32) { + (t as f32, -(t as f32)) +} + +fn rms(m: &[f32]) -> f32 { + (m.iter().map(|v| v * v).sum::() / m.len() as f32).sqrt() +} + +#[test] +fn backwards_reverses_the_layer_span() { + let (p, cid) = comp_with(Some("ec.audio.backwards"), &[], false); + let m = mix(&p, &Signal(ramp), cid, 0.0, SR as usize); + // Comp time t plays source time ≈ 1 − t. + for i in [0usize, 2000, 4000, 7999] { + let t = i as f64 / SR as f64; + assert!((m[i * 2] as f64 - (1.0 - t)).abs() < 2.0 / SR as f64, "i={i}: {}", m[i * 2]); + } + let (p, cid) = comp_with(None, &[], false); + let m = mix(&p, &Signal(ramp), cid, 0.0, SR as usize); + assert!((m[4000 * 2] - 0.5).abs() < 1e-3); +} + +#[test] +fn high_low_pass_attenuates() { + fn hi(t: f64) -> (f32, f32) { + let v = (2.0 * std::f64::consts::PI * 3000.0 * t).sin() as f32; + (v, v) + } + let vals = [("filterOptions", Value::Enum(1)), ("cutoffFrequency", Value::Scalar(200.0))]; + let (p, cid) = comp_with(Some("ec.audio.highlowpass"), &vals, false); + let r = rms(&mix(&p, &Signal(hi), cid, 0.5, 800)); + assert!(r < 0.02, "{r}"); + let (p, cid) = comp_with(None, &[], false); + let r = rms(&mix(&p, &Signal(hi), cid, 0.5, 800)); + assert!(r > 0.6, "{r}"); +} + +#[test] +fn delay_offsets_across_blocks() { + fn click(t: f64) -> (f32, f32) { + if (t * SR as f64).round() as i64 == 2000 { (1.0, 1.0) } else { (0.0, 0.0) } + } + let vals = [("delayTime", Value::Scalar(100.0)), ("feedback", Value::Scalar(0.0))]; + let (p, cid) = comp_with(Some("ec.audio.delay"), &vals, false); + // The block starts after the click (0.25 s): the echo at 0.35 s still comes from pre-roll. + let m = mix(&p, &Signal(click), cid, 0.3, 800); + let i = 2800 - 2400; + assert!((m[i * 2] - 0.375).abs() < 1e-5, "{}", m[i * 2]); + assert_eq!(m.iter().filter(|v| v.abs() > 1e-6).count(), 2); +} + +#[test] +fn stereo_mixer_pans() { + fn left(_: f64) -> (f32, f32) { + (0.5, 0.0) + } + let (p, cid) = comp_with(Some("ec.audio.stereomixer"), &[("leftPan", Value::Scalar(100.0))], false); + let m = mix(&p, &Signal(left), cid, 0.2, 10); + assert!(m.chunks(2).all(|f| f[0].abs() < 1e-6 && (f[1] - 0.5).abs() < 1e-6), "{:?}", &m[..4]); +} + +#[test] +fn tone_makes_a_solid_audible_at_its_frequency() { + let mut vals: Vec<(&str, Value)> = vec![("frequency1", Value::Scalar(500.0))]; + for k in ["frequency2", "frequency3", "frequency4", "frequency5"] { + vals.push((k, Value::Scalar(0.0))); + } + let (p, cid) = comp_with(Some("ec.audio.tone"), &vals, true); + assert!(comp_has_audio(&p, cid)); + let m = mix(&p, &Signal(ramp), cid, 0.0, SR as usize); + let left: Vec = m.iter().step_by(2).copied().collect(); + let ups = left.windows(2).filter(|w| w[0] <= 0.0 && w[1] > 0.0).count(); + assert!((ups as i64 - 500).abs() <= 1, "{ups}"); + // The out point (1 s) silences it. + let after = mix(&p, &Signal(ramp), cid, 1.0, 100); + assert!(after.iter().all(|v| *v == 0.0)); + let (p, cid) = comp_with(None, &[], true); + assert!(!comp_has_audio(&p, cid)); +} + +/// Mixing in video-frame blocks gives the same samples as one long block (pre-roll makes +/// effects stateless across calls), so preview and export agree. +#[test] +fn blocks_match_one_long_mix() { + fn noise(t: f64) -> (f32, f32) { + let x = ((t * 12345.678).sin() * 43758.5453).fract() as f32; + (x, -x) + } + for id in ["ec.audio.reverb", "ec.audio.basstreble", "ec.audio.flangechorus", "ec.audio.modulator", "ec.audio.delay", "ec.audio.parametriceq"] { + let vals: Vec<(&str, Value)> = match id { + "ec.audio.basstreble" => vec![("bass", Value::Scalar(50.0))], + "ec.audio.parametriceq" => vec![("band1BoostCut", Value::Scalar(9.0))], + _ => vec![], + }; + let (p, cid) = comp_with(Some(id), &vals, false); + let long = mix(&p, &Signal(noise), cid, 0.2, 1600); + let mut blocks = vec![]; + for k in 0..5 { + blocks.extend(mix(&p, &Signal(noise), cid, 0.2 + k as f64 * 0.04, 320)); + } + let worst = long.iter().zip(&blocks).map(|(a, b)| (a - b).abs()).fold(0.0f32, f32::max); + assert!(worst < 2e-3, "{id}: {worst}"); + } +} + +#[test] +fn peak_summary_bins() { + let s: Vec = (0..10).flat_map(|i| [i as f32 * 0.1, -(i as f32) * 0.1]).collect(); + let p = peak_summary(&s, 4); + assert_eq!(p.len(), 3); + assert!(p[0][0].abs() < 1e-6 && (p[0][1] - 0.3).abs() < 1e-6); + assert!((p[1][2] + 0.7).abs() < 1e-6 && (p[1][3] + 0.4).abs() < 1e-6); + let f = audio_footage(); + let peaks = footage_peaks(&Signal(ramp), ItemId(1), &f, Tick::from_seconds_f64(2.0), 100); + assert_eq!(peaks.len(), 200); + assert!((peaks[150][1] - 1.51).abs() < 0.01, "{:?}", peaks[150]); +} diff --git a/crates/ui-egui/src/audio.rs b/crates/ui-egui/src/audio.rs new file mode 100644 index 00000000..9c38a7c2 --- /dev/null +++ b/crates/ui-egui/src/audio.rs @@ -0,0 +1,345 @@ +//! Real-time audio preview. +//! +//! The frontend owns no audio API: the native app supplies an [`AudioDevice`] (cpal) through +//! [`crate::Hooks::audio_device`]. During preview playback an [`AudioPlayback`] runs a feeder +//! thread that mixes the composition ahead of the device with the same mixdown export uses +//! (`effectcraft_render::audio::mix_comp`: Audio switch, solo, Audio Levels, audio effects) and +//! pushes it into an [`AudioFeed`]; the device callback pulls from the feed. +//! +//! **A/V sync:** while audio plays, the audio clock drives playback. The clock is the number of +//! sample frames the device has actually consumed (underruns do not advance it) minus the +//! device's output latency, mapped through the work-area loop ([`clock_sample`]); the viewer +//! shows the comp frame at that sample ([`frame_of_sample`]), dropping video frames rather +//! than drifting. +//! +//! The feed also tracks per-channel peaks for the Audio panel's VU meters ([`Meter`]). + +use std::collections::VecDeque; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; + +use effectcraft_engine::project::ItemId; +use effectcraft_engine::time::{FrameRate, Tick}; + +use crate::frames::RenderSource; + +/// An audio output (implemented by the native app with cpal). +pub trait AudioDevice: Send { + /// Device sample rate in Hz. + fn sample_rate(&self) -> u32; + /// Start pulling interleaved stereo frames from `feed` (see [`AudioFeed::pull`]). + fn start(&mut self, feed: Arc) -> Result<(), String>; + fn stop(&mut self); + /// Output latency in sample frames (device buffering), for A/V sync. + fn latency_frames(&self) -> u64 { + 0 + } +} + +/// Opens the default output device (`None` when there is none). +pub type AudioDeviceFactory = Box Option>>; + +/// Lock-protected sample queue between the feeder thread and the device callback. +#[derive(Default)] +pub struct AudioFeed { + queue: Mutex>, + /// Sample frames handed to the device (real audio only: underruns do not count). + consumed: AtomicU64, + /// Peak |sample| per channel since the last [`AudioFeed::take_peaks`] (f32 bits). + peak: [AtomicU32; 2], + /// The feeder reached the end of a non-looping span. + finished: AtomicBool, +} + +impl AudioFeed { + /// Fill `out` (interleaved stereo) from the queue; silence where the queue runs dry. + pub fn pull(&self, out: &mut [f32]) { + let mut real = 0usize; + let mut pk = [0.0f32; 2]; + if let Ok(mut q) = self.queue.lock() { + let n = q.len().min(out.len()) & !1; + for (i, (o, s)) in out.iter_mut().zip(q.drain(..n)).enumerate() { + *o = s; + pk[i & 1] = pk[i & 1].max(s.abs()); + } + real = n; + } + out[real..].iter_mut().for_each(|o| *o = 0.0); + self.consumed.fetch_add((real / 2) as u64, Ordering::Relaxed); + for (c, v) in pk.into_iter().enumerate() { + let _ = self.peak[c].fetch_update(Ordering::Relaxed, Ordering::Relaxed, |old| (v > f32::from_bits(old)).then_some(v.to_bits())); + } + } + + pub fn push(&self, samples: &[f32]) { + if let Ok(mut q) = self.queue.lock() { + q.extend(samples.iter().copied()); + } + } + + /// Sample frames waiting in the queue. + pub fn queued_frames(&self) -> usize { + self.queue.lock().map(|q| q.len() / 2).unwrap_or(0) + } + + pub fn consumed(&self) -> u64 { + self.consumed.load(Ordering::Relaxed) + } + + /// Peak |sample| per channel since the last call (resets them). + pub fn take_peaks(&self) -> [f32; 2] { + [f32::from_bits(self.peak[0].swap(0, Ordering::Relaxed)), f32::from_bits(self.peak[1].swap(0, Ordering::Relaxed))] + } + + pub fn finished(&self) -> bool { + self.finished.load(Ordering::Relaxed) && self.queued_frames() == 0 + } +} + +/// What a preview plays, in comp sample indices at the device rate. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PlaySpan { + pub start: i64, + /// Work area [wa, wb). + pub wa: i64, + pub wb: i64, + pub looping: bool, +} + +impl PlaySpan { + /// Span for comp times: start, work area [wa, wb). + pub fn new(start: Tick, wa: Tick, wb: Tick, looping: bool, rate: u32) -> PlaySpan { + let r = rate as i64; + PlaySpan { start: start.to_units_floor(r), wa: wa.to_units_floor(r), wb: wb.to_units_floor(r).max(wa.to_units_floor(r) + 1), looping } + } +} + +/// The comp sample index audible now: `consumed` frames played by the device minus its +/// `latency`, from the span start, wrapped into the work area when looping (clamped at its end +/// otherwise). +pub fn clock_sample(span: &PlaySpan, consumed: u64, latency: u64) -> i64 { + let s = span.start + consumed.saturating_sub(latency) as i64; + if s < span.wb { + return s; + } + if span.looping { span.wa + (s - span.wb).rem_euclid((span.wb - span.wa).max(1)) } else { span.wb - 1 } +} + +/// Comp time of sample index `s` at `rate` Hz. +pub fn sample_time(s: i64, rate: u32) -> Tick { + Tick::from_units(s, rate.max(1) as i64) +} + +/// The comp frame showing sample `s` (the frame whose interval contains it). +pub fn frame_of_sample(s: i64, rate: u32, fps: FrameRate) -> i64 { + fps.frame_at(sample_time(s, rate)) +} + +/// Block size the feeder mixes at a time. +const BLOCK: usize = 1024; + +/// A running audio preview: device + feeder thread. +pub struct AudioPlayback { + pub feed: Arc, + device: Box, + stop: Arc, + thread: Option>, + pub rate: u32, + pub span: PlaySpan, +} + +impl AudioPlayback { + /// Start mixing `comp` over `span` into `device`. + pub fn start( + mut device: Box, + src: RenderSource, + comp: ItemId, + start: Tick, + wa: Tick, + wb: Tick, + looping: bool, + ) -> Result { + let rate = device.sample_rate().max(8000); + let span = PlaySpan::new(start, wa, wb, looping, rate); + let feed = Arc::new(AudioFeed::default()); + let stop = Arc::new(AtomicBool::new(false)); + // Prime ~50 ms so the device does not start on an empty queue. + let mut cursor = span.start; + let prime = (rate as usize / 20).div_ceil(BLOCK); + for _ in 0..prime { + cursor = feed_block(&src, comp, &feed, &span, cursor, rate); + } + let thread = { + let (feed, stop) = (feed.clone(), stop.clone()); + std::thread::Builder::new() + .name("ec-audio-feed".into()) + .spawn(move || { + // Keep ~200 ms queued ahead of the device. + let target = rate as usize / 5; + while !stop.load(Ordering::Relaxed) { + if feed.queued_frames() < target && !feed.finished.load(Ordering::Relaxed) { + cursor = feed_block(&src, comp, &feed, &span, cursor, rate); + } else { + std::thread::sleep(std::time::Duration::from_millis(4)); + } + } + }) + .map_err(|e| e.to_string())? + }; + device.start(feed.clone())?; + Ok(AudioPlayback { feed, device, stop, thread: Some(thread), rate, span }) + } + + /// The comp sample index audible now. + pub fn clock(&self) -> i64 { + clock_sample(&self.span, self.feed.consumed(), self.device.latency_frames()) + } + + pub fn finished(&self) -> bool { + self.feed.finished() + } + + pub fn stop(&mut self) { + self.device.stop(); + self.stop.store(true, Ordering::Relaxed); + if let Some(t) = self.thread.take() { + let _ = t.join(); + } + } +} + +impl Drop for AudioPlayback { + fn drop(&mut self) { + self.stop(); + } +} + +/// Mix one block at `cursor` into the feed; returns the next cursor (wrapping or finishing at +/// the work-area end). +fn feed_block(src: &RenderSource, comp: ItemId, feed: &AudioFeed, span: &PlaySpan, cursor: i64, rate: u32) -> i64 { + let mut cursor = cursor; + if cursor >= span.wb { + if !span.looping { + feed.finished.store(true, Ordering::Relaxed); + return cursor; + } + cursor = span.wa; + } + let n = ((span.wb - cursor) as usize).min(BLOCK); + let buf = effectcraft_engine::render::audio::mix_comp(&src.project, src.footage.as_ref(), src.expr.as_deref(), comp, sample_time(cursor, rate), n, rate); + feed.push(&buf); + cursor + n as i64 +} + +/// VU meter state for the Audio panel: smoothed level with a falling decay and peak hold, in +/// dBFS per channel. +#[derive(Clone, Copy, Debug)] +pub struct Meter { + pub level_db: [f32; 2], + pub peak_db: [f32; 2], + peak_at: [f64; 2], + /// Clip indicators (latched until clicked / playback restarts). + pub clipped: [bool; 2], + last: f64, +} + +impl Default for Meter { + fn default() -> Self { + Meter { level_db: [METER_FLOOR; 2], peak_db: [METER_FLOOR; 2], peak_at: [0.0; 2], clipped: [false; 2], last: 0.0 } + } +} + +/// Bottom of the meter scale (dB). +pub const METER_FLOOR: f32 = -48.0; +/// How long the peak marker holds (s) and how fast levels fall (dB/s). +const PEAK_HOLD: f64 = 1.5; +const FALL_DB_PER_S: f32 = 30.0; + +pub fn to_db(v: f32) -> f32 { + if v <= 1e-6 { METER_FLOOR } else { (20.0 * v.log10()).max(METER_FLOOR) } +} + +impl Meter { + /// Feed the peaks measured since the last update at wall time `now` (seconds). + pub fn update(&mut self, peaks: [f32; 2], now: f64) { + let dt = if self.last > 0.0 { (now - self.last).clamp(0.0, 0.5) as f32 } else { 0.0 }; + self.last = now; + for c in 0..2 { + let db = to_db(peaks[c]); + self.level_db[c] = db.max(self.level_db[c] - FALL_DB_PER_S * dt); + if db >= self.peak_db[c] || now - self.peak_at[c] > PEAK_HOLD { + self.peak_db[c] = db.max(self.level_db[c]); + self.peak_at[c] = now; + } + if peaks[c] >= 1.0 { + self.clipped[c] = true; + } + } + } + + /// Anything to animate (levels still falling)? + pub fn active(&self) -> bool { + self.level_db.iter().chain(self.peak_db.iter()).any(|d| *d > METER_FLOOR) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn clock_follows_consumed_frames_and_loops() { + let span = PlaySpan { start: 4800, wa: 0, wb: 48000, looping: true }; + assert_eq!(clock_sample(&span, 0, 0), 4800); + // Latency holds the clock back until the device has played through its buffer. + assert_eq!(clock_sample(&span, 512, 1024), 4800); + assert_eq!(clock_sample(&span, 2048, 1024), 4800 + 1024); + // Wrap at the work-area end. + assert_eq!(clock_sample(&span, 48000 - 4800 + 100, 0), 100); + let once = PlaySpan { looping: false, ..span }; + assert_eq!(clock_sample(&once, 100_000, 0), 47999); + } + + #[test] + fn frame_mapping_matches_comp_frames() { + let fps = FrameRate::new(30, 1); + // 48 kHz at 30 fps: 1600 samples per frame. + assert_eq!(frame_of_sample(0, 48000, fps), 0); + assert_eq!(frame_of_sample(1599, 48000, fps), 0); + assert_eq!(frame_of_sample(1600, 48000, fps), 1); + assert_eq!(frame_of_sample(48000 * 2, 48000, fps), 60); + let span = PlaySpan::new(Tick::from_seconds_f64(1.0), Tick::ZERO, Tick::from_seconds_f64(2.0), true, 48000); + assert_eq!((span.start, span.wa, span.wb), (48000, 0, 96000)); + // 29.97: frame boundaries are fractional in samples. + let ntsc = FrameRate::new(30000, 1001); + assert_eq!(frame_of_sample(1601, 48000, ntsc), 0); + assert_eq!(frame_of_sample(1602, 48000, ntsc), 1); + } + + #[test] + fn feed_counts_real_frames_and_peaks() { + let f = AudioFeed::default(); + f.push(&[0.5, -0.25, 0.1, 0.9]); + let mut out = [1.0f32; 8]; + f.pull(&mut out); + assert_eq!(out, [0.5, -0.25, 0.1, 0.9, 0.0, 0.0, 0.0, 0.0]); + assert_eq!(f.consumed(), 2); + assert_eq!(f.take_peaks(), [0.5, 0.9]); + assert_eq!(f.take_peaks(), [0.0, 0.0]); + } + + #[test] + fn meter_decays_and_holds_peaks() { + let mut m = Meter::default(); + m.update([1.0, 0.5], 1.0); + assert!((m.level_db[0] - 0.0).abs() < 1e-4 && (m.level_db[1] + 6.0206).abs() < 1e-3); + assert!(m.clipped[0] && !m.clipped[1]); + m.update([0.0, 0.0], 1.1); + assert!((m.level_db[0] + 3.0).abs() < 1e-3, "{}", m.level_db[0]); + assert_eq!(m.peak_db[0], 0.0, "peak holds"); + // After the hold time the peak marker follows the falling level. + m.update([0.0, 0.0], 3.0); + assert!(m.peak_db[0] < -10.0, "{}", m.peak_db[0]); + } +} diff --git a/crates/ui-egui/src/control.rs b/crates/ui-egui/src/control.rs index 5aa3c9a8..3ca1725b 100644 --- a/crates/ui-egui/src/control.rs +++ b/crates/ui-egui/src/control.rs @@ -304,9 +304,12 @@ pub fn handle(app: &mut EffectcraftApp, ctx: &egui::Context, req: &ControlReques match s("action").unwrap_or("toggle") { "play" => app.play(now), "stop" => app.stop(), + "status" => {} _ => app.toggle_play(now), } - ok(json!({"playing": app.playback.playing, "time": app.session.time().seconds()})) + ok( + json!({"playing": app.playback.playing, "time": app.session.time().seconds(), "audio": app.audio.is_some(), "levelsDb": app.meter.level_db, "peaksDb": app.meter.peak_db}), + ) } "ui.screenshot" => { let crop = s("panel").and_then(PanelKind::from_name).and_then(|pk| app.auto.find(&format!("panel.{}", pk.id())).map(|e| e.rect)); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 60613605..743e79c0 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -5,6 +5,7 @@ //! Everything goes through `effectcraft_engine::Session::execute` (or `menus::invoke` for //! frontend-only commands) so every gesture is also available to agents. +pub mod audio; pub mod automation; pub mod control; pub mod dock; @@ -54,6 +55,8 @@ pub struct Hooks { pub pick_files: Option Vec>>, pub pick_save: Option Option>>, pub pick_open_project: Option Option>>, + /// Opens the audio output for preview playback (the desktop app uses cpal). + pub audio_device: Option, } #[derive(Default)] @@ -101,6 +104,14 @@ pub struct EffectcraftApp { pub(crate) pointer_comp: Option<[f32; 2]>, pub(crate) dialog_state: panels::dialogs::DialogState, pub integrated_titlebar: bool, + /// Audio preview while playing (audio clock drives playback). + pub audio: Option, + /// Audio panel VU meters. + pub meter: audio::Meter, + /// Waveform peak summaries per footage item (see `panels::waveform`). + pub(crate) waveforms: panels::waveform::Cache, + /// Last reveal shortcut and when (double-press shortcuts such as LL). + pub(crate) last_reveal: Option<(String, f64)>, } impl EffectcraftApp { @@ -133,6 +144,10 @@ impl EffectcraftApp { pointer_comp: None, dialog_state: Default::default(), integrated_titlebar: false, + audio: None, + meter: Default::default(), + waveforms: Default::default(), + last_reveal: None, } } @@ -244,10 +259,32 @@ impl EffectcraftApp { waiting: false, fps: c.frame_rate.as_f64(), }; + self.start_audio(t); + } + + /// Start audio preview from comp time `t` when the Preview panel includes audio, the comp + /// has something audible and an output device opens. + fn start_audio(&mut self, t: Tick) { + self.audio = None; + self.meter.clipped = [false; 2]; + if !self.ui.preview_audio { + return; + } + let Some(cid) = self.session.active_comp_id() else { return }; + let Some(c) = self.session.project.comp(cid).cloned() else { return }; + if !effectcraft_engine::render::audio::comp_has_audio(&self.session.project, cid) { + return; + } + let Some(dev) = self.hooks.audio_device.as_ref().and_then(|f| f()) else { return }; + match audio::AudioPlayback::start(dev, self.render_source(), cid, t, c.work_area.0, c.work_area.1, self.ui.preview_loop) { + Ok(a) => self.audio = Some(a), + Err(e) => log::warn!("audio preview: {e}"), + } } pub fn stop(&mut self) { self.playback.playing = false; + self.audio = None; } pub fn toggle_play(&mut self, now: f64) { @@ -291,9 +328,32 @@ impl EffectcraftApp { queued += 1; } } + if let Some(a) = &self.audio { + self.meter.update(a.feed.take_peaks(), now); + } else if self.meter.active() { + self.meter.update([0.0; 2], now); + ctx.request_repaint(); + } if !self.playback.playing { return; } + // Audio clock drives playback: show the frame under the audible sample (frames that + // are not rendered yet are dropped, never delayed). + if let Some(a) = &self.audio { + if a.finished() { + self.session.set_time(fr.tick_of(wb)); + self.stop(); + return; + } + let target = audio::frame_of_sample(a.clock(), a.rate, fr).clamp(wa, wb); + self.playback.waiting = !self.frames.is_cached(&self.frame_key(cid, target, scale)); + if target != cur { + self.session.set_time(fr.tick_of(target)); + self.playback.shown += 1; + } + ctx.request_repaint(); + return; + } let elapsed = now - self.playback.start_wall; let mut target = self.playback.start_frame + (elapsed * fr.as_f64()).floor() as i64; if target > wb { diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index efc9efbf..0f5ec7f6 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -64,6 +64,9 @@ pub const UI_COMMANDS: &[UiCommand] = &[ uic!("timeline.reveal.effects", "Reveal Effects", [], Some("E")), uic!("timeline.reveal.masks", "Reveal Masks", [], Some("M")), uic!("timeline.reveal.feather", "Reveal Mask Feather", [], Some("F")), + uic!("timeline.reveal.levels", "Reveal Audio Levels (press twice: Waveform)", [], Some("L")), + uic!("timeline.reveal.waveform", "Reveal Audio Waveform", [], None), + uic!("playback.muteAudio", "Mute Audio", ["Composition", "Preview"], None), uic!("timeline.collapseAll", "Collapse All", [], Some("Cmd+`")), uic!("window.workspace.default", "Default", ["Window", "Workspace"], Some("Shift+F10")), uic!("window.workspace.standard", "Standard", ["Window", "Workspace"], Some("Shift+F11")), @@ -102,7 +105,13 @@ pub fn panel_command_id(p: PanelKind) -> String { format!("window.panel.{}", p.id()) } -fn reveal(app: &mut EffectcraftApp, kind: &str) { +fn reveal(app: &mut EffectcraftApp, kind: &str, now: f64) { + // AE's double-press shortcuts: L then L again quickly reveals the Waveform (LL). + let kind = match app.last_reveal.take() { + Some((k, t)) if k == "levels" && kind == "levels" && now - t < 0.6 => "waveform", + _ => kind, + }; + app.last_reveal = Some((kind.to_string(), now)); let add = app.ui.timeline.reveal.contains(&kind.to_string()); if add && app.ui.timeline.reveal.len() == 1 { app.ui.timeline.reveal.clear(); @@ -169,7 +178,7 @@ pub fn invoke(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, params: V return Ok(json!({"tool": tool})); } if let Some(k) = id.strip_prefix("timeline.reveal.") { - reveal(app, k); + reveal(app, k, now); return Ok(Value::Null); } if let Some(r) = id.strip_prefix("view.res.") { @@ -191,6 +200,13 @@ pub fn invoke(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, params: V app.stop(); return Ok(Value::Null); } + "playback.muteAudio" => { + app.ui.preview_audio = !app.ui.preview_audio; + if !app.ui.preview_audio { + app.audio = None; + } + return Ok(json!({"muted": !app.ui.preview_audio})); + } "view.zoomIn" | "view.zoomOut" => { let cur = app.ui.viewer.zoom.unwrap_or_else(|| crate::panels::viewer::last_fit(ctx)); const STEPS: [f32; 16] = [0.015, 0.03, 0.0625, 0.125, 0.25, 0.333, 0.5, 0.66, 1.0, 1.5, 2.0, 3.0, 4.0, 8.0, 16.0, 32.0]; diff --git a/crates/ui-egui/src/panels/misc.rs b/crates/ui-egui/src/panels/misc.rs index 99edc106..8f487a5e 100644 --- a/crates/ui-egui/src/panels/misc.rs +++ b/crates/ui-egui/src/panels/misc.rs @@ -1,6 +1,6 @@ //! Smaller panels: Preview, Audio, History, Markers, Wiggler, the Home screen and placeholders. -use egui::{Align2, Color32, Rect, Sense, pos2, vec2}; +use egui::{Align2, Color32, Rect, Sense, Stroke, pos2, vec2}; use serde_json::json; use crate::dock::PanelKind; @@ -51,6 +51,12 @@ pub fn preview(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } app.auto.add("preview.loop", lr, "Loop"); p.text(pos2(lr.max.x + 6.0, yy), Align2::LEFT_CENTER, "Loop", Tokens::ui(12.0), t.text); + let ar = Rect::from_min_size(pos2(rect.min.x + 90.0, yy - 8.0), vec2(16.0, 16.0)); + if widgets::checkbox(ui, ar, app.ui.preview_audio, &t, egui::Id::new("pv-audio")).clicked() { + let _ = crate::menus::invoke(app, &ctx, "playback.muteAudio", json!({})); + } + app.auto.add("preview.audio", ar, "Include Audio"); + p.text(pos2(ar.max.x + 6.0, yy), Align2::LEFT_CENTER, "Include Audio", Tokens::ui(12.0), t.text); yy += 24.0; row(&p, yy, "Range", "Work Area Extended By Current Time"); yy += 22.0; @@ -74,19 +80,92 @@ pub fn preview(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } } +/// Audio panel: L/R VU meters (dBFS, 0 to -48) with peak hold and clip indicators, fed by the +/// audio preview; the selected layer's Audio Levels on the right. pub fn audio(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + use crate::audio::METER_FLOOR; let t = app.tokens; let p = ui.painter().with_clip_rect(rect); - // Level meters (silent until audio playback lands). - for (i, x) in [rect.min.x + 20.0, rect.min.x + 34.0].into_iter().enumerate() { - let r = Rect::from_min_max(pos2(x, rect.min.y + 14.0), pos2(x + 10.0, rect.max.y - 20.0)); - p.rect_filled(r, 1.0, t.field_bg); - let _ = i; + let top = rect.min.y + 26.0; + let bottom = rect.max.y - 22.0; + if bottom - top < 40.0 { + return; } - for (k, db) in [0, -6, -12, -18, -24, -36, -48].iter().enumerate() { - let y = rect.min.y + 14.0 + (rect.height() - 34.0) * (-*db as f32 / 48.0); - p.text(pos2(rect.min.x + 52.0, y), Align2::LEFT_CENTER, format!("{db} dB"), Tokens::ui(10.0), t.text_faint); - let _ = k; + let y_of = |db: f32| top + (bottom - top) * (db.clamp(METER_FLOOR, 0.0) / METER_FLOOR); + let m = app.meter; + let green = Color32::from_rgb(0x3c, 0xc8, 0x5a); + let yellow = Color32::from_rgb(0xe6, 0xc8, 0x3c); + let red = Color32::from_rgb(0xe6, 0x46, 0x3c); + let seg_col = |db: f32| { + if db > -3.0 { + red + } else if db > -12.0 { + yellow + } else { + green + } + }; + for c in 0..2 { + let x = rect.min.x + 18.0 + c as f32 * 16.0; + let bar = Rect::from_min_max(pos2(x, top), pos2(x + 12.0, bottom)); + p.rect_filled(bar, 1.0, t.field_bg); + // Lit segments, 1.5 dB each, coloured by their level. + let lvl = m.level_db[c]; + let mut db = METER_FLOOR; + while db < lvl.min(0.0) { + let hi = (db + 1.5).min(lvl); + p.rect_filled(Rect::from_min_max(pos2(bar.min.x + 1.0, y_of(hi)), pos2(bar.max.x - 1.0, y_of(db) - 0.5)), 0.0, seg_col(db)); + db += 1.5; + } + if m.peak_db[c] > METER_FLOOR { + let py = y_of(m.peak_db[c]); + p.line_segment([pos2(bar.min.x, py), pos2(bar.max.x, py)], Stroke::new(2.0, seg_col(m.peak_db[c]))); + } + // Clip indicator (click to reset). + let clip = Rect::from_min_max(pos2(bar.min.x, top - 14.0), pos2(bar.max.x, top - 4.0)); + p.rect_filled(clip, 1.0, if m.clipped[c] { red } else { t.field_bg }); + let id = if c == 0 { "audio.clipLeft" } else { "audio.clipRight" }; + if ui.interact(clip, egui::Id::new(id), Sense::click()).clicked() { + app.meter.clipped[c] = false; + } + app.auto.add(id, clip, if c == 0 { "Left clip indicator" } else { "Right clip indicator" }); + app.auto.add(if c == 0 { "audio.meterLeft" } else { "audio.meterRight" }, bar, &format!("{:.1} dB", m.level_db[c])); + p.text(pos2(bar.center().x, bottom + 9.0), Align2::CENTER_CENTER, if c == 0 { "L" } else { "R" }, Tokens::ui(10.0), t.text_dim); + } + let sx = rect.min.x + 54.0; + for db in [0, -6, -12, -18, -24, -30, -36, -42, -48] { + let y = y_of(db as f32); + p.line_segment([pos2(sx - 4.0, y), pos2(sx - 1.0, y)], Stroke::new(1.0, t.text_faint)); + p.text(pos2(sx + 2.0, y), Align2::LEFT_CENTER, format!("{db:.1}"), Tokens::ui(10.0), t.text_faint); + } + p.text(pos2(sx + 30.0, top - 9.0), Align2::LEFT_CENTER, "dB", Tokens::ui(10.0), t.text_dim); + // Selected layer's Audio Levels. + let lx = rect.min.x + 120.0; + if rect.max.x - lx < 80.0 { + return; + } + let sel = app.session.state.selected_layers.first().copied(); + let levels = sel.and_then(|id| { + let c = app.session.active_comp()?; + let l = c.layer(id)?; + let pr = l.props.sub("audio")?.get("levels")?; + Some((l.name.clone(), pr.value.as_vec2())) + }); + match levels { + Some((name, [a, b])) => { + p.text(pos2(lx, top - 9.0), Align2::LEFT_CENTER, &name, Tokens::ui(11.0), t.text_dim); + for (i, (k, v)) in [("Left", a), ("Right", b)].into_iter().enumerate() { + let y = top + 12.0 + i as f32 * 20.0; + p.text(pos2(lx, y), Align2::LEFT_CENTER, k, Tokens::ui(12.0), t.text_dim); + p.text(pos2(lx + 44.0, y), Align2::LEFT_CENTER, format!("{v:+.1} dB"), Tokens::ui(12.0), t.hot_text); + } + } + None => { + p.text(pos2(lx, top + 12.0), Align2::LEFT_CENTER, "No audio layer selected", Tokens::ui(11.0), t.text_faint); + } + } + if app.meter.active() || app.audio.is_some() { + ui.ctx().request_repaint(); } } diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 8cd26cb5..a7d66376 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -15,6 +15,7 @@ pub mod render_queue; pub mod text_panels; pub mod timeline; pub mod viewer; +pub mod waveform; use effectcraft_engine::Session; use effectcraft_engine::project::{Comp, TimeDisplayStyle}; diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index a7ff2338..1bb42fce 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -36,12 +36,20 @@ enum RowKind { uid: u64, lines: usize, }, + /// Audio > Waveform of a footage layer with audio (drawn from the item's peak summary). + Waveform { + item: u64, + }, } +/// Synthetic group uid for a layer's Audio > Waveform twirl (never a real property uid). +const WAVE_BIT: u64 = 1 << 60; + /// Row height (expression editors grow with their text). fn row_height(row: &Row, rh: f32) -> f32 { match row.kind { RowKind::Expr { lines, .. } => rh * lines.clamp(1, 8) as f32 + 6.0, + RowKind::Waveform { .. } => rh * 3.0, _ => rh, } } @@ -227,9 +235,15 @@ fn build_rows(app: &EffectcraftApp, comp: &Comp) -> Vec { "opacity" => ("transform", vec!["opacity"]), "anchor" => ("transform", vec!["anchor"]), "feather" => ("masks", vec!["feather"]), + "levels" => ("audio", vec!["levels"]), _ => ("", vec![]), }; match kind.as_str() { + "waveform" => { + if let Some(item) = super::waveform::audio_item(&app.session.project, l) { + rows.push(Row { layer: l.id, depth: 1, kind: RowKind::Waveform { item: item.0 } }); + } + } "effects" => { if let Some(fx) = l.effects() { for g in fx.groups() { @@ -294,6 +308,21 @@ fn build_rows(app: &EffectcraftApp, comp: &Comp) -> Vec { }); if open { push_group(&mut rows, l, g, 2, &tl.open_groups); + // Audio > Waveform > Waveform (footage with audio). + if g.match_id == "audio" + && let Some(item) = super::waveform::audio_item(&app.session.project, l) + { + let wuid = WAVE_BIT | g.uid; + let wopen = tl.open_groups.contains(&wuid); + rows.push(Row { + layer: l.id, + depth: 2, + kind: RowKind::Group { uid: wuid, name: "Waveform".into(), open: wopen, has_children: true, fx: None }, + }); + if wopen { + rows.push(Row { layer: l.id, depth: 3, kind: RowKind::Waveform { item: item.0 } }); + } + } } } Node::Prop(p) if prop_visible(p, l) => rows.push(Row { layer: l.id, depth: 1, kind: RowKind::Prop { uid: p.uid } }), @@ -952,13 +981,33 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { lp.text(pos2(ir.max.x + 4.0, cy), Align2::LEFT_CENTER, "Inverted", Tokens::ui(11.0), t.text_dim); } let gr = ui.interact(Rect::from_min_max(pos2(indent + 8.0, r.min.y), pos2(cw.switches, r.max.y)), egui::Id::new(("grow", uid)), Sense::click()); - if gr.clicked() { + if gr.clicked() && uid & WAVE_BIT == 0 { actions.push(("prop.select".into(), json!({"layer": layer.id.0, "prop": uid, "selectKeys": false}))); } if gr.double_clicked() { ui_actions.push(UiAct::ToggleGroup(*uid)); } } + RowKind::Waveform { item } => { + lp.rect_filled(left, 0.0, if ri % 2 == 0 { t.row } else { t.row_alt }); + let indent = cw.name + 6.0 + 14.0 * row.depth as f32; + lp.text(pos2(indent + 10.0, cy), Align2::LEFT_CENTER, "Waveform", Tokens::ui(12.0), t.text); + let wr = Rect::from_min_max(pos2(graph_x0, r.min.y), r.max); + gp.rect_filled(wr, 0.0, t.tl_bg); + app.auto.add(&format!("timeline.layer.{}.waveform", layer.id.0), wr, "Waveform"); + match super::waveform::summary(app, &ctx, effectcraft_engine::project::ItemId(*item)) { + Some(s) => super::waveform::draw(&gp, wr.shrink2(vec2(0.0, 2.0)), &tm, layer, &s, Color32::from_rgb(0x5f, 0xc8, 0x8a)), + None => { + gp.text( + pos2(tm.x(layer.in_point.seconds()).max(wr.min.x) + 6.0, wr.center().y), + Align2::LEFT_CENTER, + "Building waveform…", + Tokens::ui(11.0), + t.text_faint, + ); + } + } + } RowKind::Expr { uid, lines } => { let Some(prop) = layer.props.find(*uid) else { continue }; let Some(ex) = prop.expr.clone() else { continue }; diff --git a/crates/ui-egui/src/panels/waveform.rs b/crates/ui-egui/src/panels/waveform.rs new file mode 100644 index 00000000..f9bf00db --- /dev/null +++ b/crates/ui-egui/src/panels/waveform.rs @@ -0,0 +1,95 @@ +//! Audio waveforms for the Timeline's Audio > Waveform property (revealed with `LL`). +//! +//! Each footage item's audio is summarised once into min/max peaks +//! (`effectcraft_render::audio::footage_peaks`, [`BINS_PER_SEC`] bins per second) on a +//! background thread; the timeline draws from that summary at any zoom. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use effectcraft_engine::project::{ItemId, ItemKind, Layer, LayerSource, Project}; +use effectcraft_engine::time::Tick; +use egui::{Color32, Painter, Rect, Stroke, pos2}; + +use super::timeline::TMap; +use crate::EffectcraftApp; + +/// Peak-summary resolution. +pub const BINS_PER_SEC: u32 = 200; + +/// A footage item's peak summary: `[min L, max L, min R, max R]` per bin. +pub struct Summary { + pub bins_per_sec: u32, + pub peaks: Vec<[f32; 4]>, +} + +/// Summaries by footage item (`None` while computing). +#[derive(Default)] +pub struct Cache { + map: Arc>>>>, +} + +/// The footage item whose audio a layer plays, when it has audio. +pub fn audio_item(project: &Project, l: &Layer) -> Option { + let LayerSource::Footage { item } = &l.source else { return None }; + match &project.item(*item)?.kind { + ItemKind::Footage(f) if f.has_audio && !f.missing => Some(*item), + _ => None, + } +} + +/// The summary for `item`, starting its computation on first request. +pub fn summary(app: &EffectcraftApp, ctx: &egui::Context, item: ItemId) -> Option> { + let mut m = app.waveforms.map.lock().ok()?; + if let Some(s) = m.get(&item.0) { + return s.clone(); + } + let ItemKind::Footage(f) = &app.session.project.item(item)?.kind else { return None }; + m.insert(item.0, None); + let (f, footage, map, ctx) = (f.clone(), app.session.footage.clone(), app.waveforms.map.clone(), ctx.clone()); + std::thread::Builder::new() + .name("ec-waveform".into()) + .spawn(move || { + let peaks = effectcraft_engine::render::audio::footage_peaks(footage.as_ref(), item, &f, f.duration, BINS_PER_SEC); + if let Ok(mut m) = map.lock() { + m.insert(item.0, Some(Arc::new(Summary { bins_per_sec: BINS_PER_SEC, peaks }))); + } + ctx.request_repaint(); + }) + .ok()?; + None +} + +/// Draw `layer`'s waveform in `rect` (left channel above right), columns mapped through the +/// timeline's time map and the layer's time mapping, limited to its [in, out) span. +pub(crate) fn draw(p: &Painter, rect: Rect, tm: &TMap, layer: &Layer, s: &Summary, color: Color32) { + let mid = [rect.min.y + rect.height() * 0.25, rect.min.y + rect.height() * 0.75]; + let half = rect.height() * 0.25 - 1.0; + let x0 = tm.x(layer.in_point.seconds()).max(rect.min.x).floor() as i32; + let x1 = tm.x(layer.out_point.seconds()).min(rect.max.x).ceil() as i32; + let n = s.peaks.len() as i64; + for c in 0..2 { + p.line_segment([pos2(x0 as f32, mid[c]), pos2(x1 as f32, mid[c])], Stroke::new(1.0, color.gamma_multiply(0.5))); + } + for x in x0..x1 { + let (ta, tb) = (tm.t(x as f32), tm.t(x as f32 + 1.0)); + let src = |t: f64| layer.layer_time(Tick::from_seconds_f64(t)).seconds() * s.bins_per_sec as f64; + let (ba, bb) = (src(ta).floor() as i64, src(tb).ceil() as i64); + let (ba, bb) = (ba.min(bb).clamp(0, n), ba.max(bb).max(ba + 1).clamp(0, n)); + if ba >= bb { + continue; + } + let mut agg = [0.0f32; 4]; + for b in &s.peaks[ba as usize..bb as usize] { + agg[0] = agg[0].min(b[0]); + agg[1] = agg[1].max(b[1]); + agg[2] = agg[2].min(b[2]); + agg[3] = agg[3].max(b[3]); + } + for c in 0..2 { + let (lo, hi) = (agg[c * 2].clamp(-1.0, 1.0), agg[c * 2 + 1].clamp(-1.0, 1.0)); + let (ya, yb) = (mid[c] - hi * half, mid[c] - lo * half); + p.line_segment([pos2(x as f32 + 0.5, ya), pos2(x as f32 + 0.5, yb.max(ya + 1.0))], Stroke::new(1.0, color)); + } + } +} diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index b06b0e38..3b238da1 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -338,6 +338,9 @@ pub struct UiState { /// Preview panel options. pub preview_loop: bool, pub preview_cache_first: bool, + /// Preview panel "Include Audio" (Mute Audio off). + #[serde(default = "yes")] + pub preview_audio: bool, pub start_screen: bool, } @@ -369,6 +372,7 @@ impl Default for UiState { tool_creates_shape: true, preview_loop: true, preview_cache_first: false, + preview_audio: true, start_screen: false, } } diff --git a/docs/architecture.md b/docs/architecture.md index 4c8646b3..bf057ef9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -71,6 +71,16 @@ its evaluated inputs, excluding the transform. Static and transform-only layers editing one layer re-renders only that layer. Effects that read the clock directly are declared in `effects::TIME_DEPENDENT`, and a test checks every registered effect against that list. +**Time effects** (Echo, Posterize Time, Timewarp…) read the layer at other times through +`EffectHost::self_at`: the renderer renders the layer's source and masks (plus, optionally, the +effects before it) at that layer time via `Renderer::layer_input`, cached under separate *input* +keys, with a nesting-depth guard. + +**Audio** is mixed by `render::audio::mix_comp` in blocks: Audio switch, solo, Audio Levels and +the Effect > Audio effects (`effects::audio_fx`, applied per layer with a pre-roll so blocks are +independent). Export and preview playback share it; the desktop app plays it through cpal and the +audio clock drives preview playback (`ui-egui::audio`). + Half, Third and Quarter resolution render proportionally fewer pixels end to end. ## 5. Expressions diff --git a/docs/control-protocol.md b/docs/control-protocol.md index 2f2f25d1..f7c8b964 100644 --- a/docs/control-protocol.md +++ b/docs/control-protocol.md @@ -43,7 +43,7 @@ Useful query commands for `engine.execute` (they aren't journaled or undoable): | `ui.elements` | `{prefix?}` | `[{id, label, rect: [x, y, w, h]}]`: the widgets drawn in the last frame, in points. | | `ui.set` | `{tool?, workspace?, theme?, focused?, viewer?, timeline?, menuBar?, home?}` | `viewer: {zoom (null = fit), res: Full\|Half\|Third\|Quarter\|Auto, pan: [x,y], grid, rulers, safeMargins, transparencyGrid}`; `timeline: {pps (null = fit), start, graphEditor, showModes, openLayers: [ids], openGroups: [uids]}`. | | `ui.panel.show` / `ui.panel.close` | `{panel}` | Panels: `Project`, `Composition`, `Timeline`, `EffectControls`, `EffectsPresets`, `Info`, `Preview`, `Character`, `Paragraph`, `Align` and others. | -| `ui.playback` | `{action: play\|stop\|toggle}` | `{playing, time}` | +| `ui.playback` | `{action: play\|stop\|toggle\|status}` | `{playing, time, audio, levelsDb, peaksDb}` (audio: an audio preview is running; L/R meter levels and held peaks in dBFS) | | `ui.resize` | `{width, height}` | Resizes the window (points). | | `ui.focus` | `{}` | Activates the window and takes keyboard focus. Agents normally avoid this, because the app is started without stealing focus. | | `app.quit` | `{}` | Closes the app. |