diff --git a/Cargo.lock b/Cargo.lock index 4eacb0c9..7f1466da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1546,6 +1546,7 @@ dependencies = [ name = "filmcraft-render" version = "0.1.0" dependencies = [ + "filmcraft-audio-dsp", "filmcraft-color", "filmcraft-frame", "filmcraft-geom", diff --git a/crates/render/Cargo.toml b/crates/render/Cargo.toml index 9893ec6d..4b5594b2 100644 --- a/crates/render/Cargo.toml +++ b/crates/render/Cargo.toml @@ -16,6 +16,7 @@ filmcraft-color = { workspace = true } filmcraft-frame = { workspace = true } filmcraft-media = { workspace = true } filmcraft-project = { workspace = true } +filmcraft-audio-dsp = { workspace = true } rayon = { workspace = true } [dev-dependencies] diff --git a/crates/render/src/audio.rs b/crates/render/src/audio.rs index b4029492..01cb5643 100644 --- a/crates/render/src/audio.rs +++ b/crates/render/src/audio.rs @@ -107,6 +107,43 @@ fn mix_item(project: &Project, item: &TrackItem, start: i64, frames: usize, sr: return; } let n = (a1 - a0) as usize; + let read = |x0: i64, len: usize| raw_stereo(item, src.as_ref(), x0, len, sr); + let buf = if crate::audio_fx::has_effects(item) { crate::audio_fx::process(item, a0, n, sr, &read) } else { read(a0, n) }; + // gains: clip gain × Volume (keyframed, per 64-sample block) × channel volume × panner + let clip_gain = db_to_gain(item.gain_db); + let vol = item.effect("volume").filter(|e| e.enabled && !e.param("bypass").and_then(|p| p.value.as_bool()).unwrap_or(false)); + let chv = item.effect("channel_volume").filter(|e| e.enabled && !e.param("bypass").and_then(|p| p.value.as_bool()).unwrap_or(false)); + let pan = item.effect("panner").filter(|e| e.enabled); + let off = (a0 - start) as usize; + for blk in (0..n).step_by(64) { + let t_tl = Tick::from_units(a0 + blk as i64, sr as i64); + let mt = item.source_time_at(t_tl); + let v = vol.map(|e| db_to_gain(e.f64_at("level", mt))).unwrap_or(1.0); + let (cl, cr) = chv.map(|e| (db_to_gain(e.f64_at("left", mt)), db_to_gain(e.f64_at("right", mt)))).unwrap_or((1.0, 1.0)); + let (pl, pr) = pan.map(|e| pan_gains((e.f64_at("balance", mt) / 100.0) as f32)).unwrap_or((1.0, 1.0)); + let g = clip_gain * v * extra; + let end = (blk + 64).min(n); + for i in blk..end { + let (l, r) = (buf[0][i], buf[1][i]); + out.channels[0][off + i] += l * g * cl * pl; + out.channels[1][off + i] += r * g * cr * pr; + } + } +} + +/// The clip's audio (after speed/reverse) for timeline samples `[x0, x0 + len)` as stereo, with +/// silence outside the clip. Mono sources are duplicated to both channels. +fn raw_stereo(item: &TrackItem, src: &dyn filmcraft_media::MediaSource, x0: i64, len: usize, sr: u32) -> [Vec; 2] { + let mut out = [vec![0.0f32; len], vec![0.0f32; len]]; + let item_s0 = item.start.to_units_floor(sr as i64); + let item_s1 = item.end().to_units_floor(sr as i64); + let a0 = x0.max(item_s0); + let a1 = (x0 + len as i64).min(item_s1); + if a1 <= a0 { + return out; + } + let n = (a1 - a0) as usize; + let off = (a0 - x0) as usize; let speed = item.speed.abs().max(1e-6); let src_start_ticks = item.source_time_at(Tick::from_units(a0, sr as i64)); let buf = if (speed - 1.0).abs() < 1e-9 && !item.reverse { @@ -130,29 +167,14 @@ fn mix_item(project: &Project, item: &TrackItem, start: i64, frames: usize, sr: b }) }; - let Some(buf) = buf else { return }; - // gains: clip gain × Volume (keyframed, per 64-sample block) × channel volume × panner - let clip_gain = db_to_gain(item.gain_db); - let vol = item.effect("volume").filter(|e| e.enabled && !e.param("bypass").and_then(|p| p.value.as_bool()).unwrap_or(false)); - let chv = item.effect("channel_volume").filter(|e| e.enabled && !e.param("bypass").and_then(|p| p.value.as_bool()).unwrap_or(false)); - let pan = item.effect("panner").filter(|e| e.enabled); - let off = (a0 - start) as usize; - let stereo_src = buf.channel_count() >= 2; - for blk in (0..n).step_by(64) { - let t_tl = Tick::from_units(a0 + blk as i64, sr as i64); - let mt = item.source_time_at(t_tl); - let v = vol.map(|e| db_to_gain(e.f64_at("level", mt))).unwrap_or(1.0); - let (cl, cr) = chv.map(|e| (db_to_gain(e.f64_at("left", mt)), db_to_gain(e.f64_at("right", mt)))).unwrap_or((1.0, 1.0)); - let (pl, pr) = pan.map(|e| pan_gains((e.f64_at("balance", mt) / 100.0) as f32)).unwrap_or((1.0, 1.0)); - let g = clip_gain * v * extra; - let end = (blk + 64).min(n); - for i in blk..end { - let l = buf.channels[0][i]; - let r = if stereo_src { buf.channels[1][i] } else { l }; - out.channels[0][off + i] += l * g * cl * pl; - out.channels[1][off + i] += r * g * cr * pr; - } + let Some(buf) = buf else { return out }; + let right = if buf.channel_count() >= 2 { 1 } else { 0 }; + for (c, dst) in out.iter_mut().enumerate() { + let ch = &buf.channels[if c == 0 { 0 } else { right }]; + let m = n.min(ch.len()); + dst[off..off + m].copy_from_slice(&ch[..m]); } + out } /// Peak (min, max) pairs per bucket of `bucket` samples for channel `ch` — waveform display. diff --git a/crates/render/src/audio_fx.rs b/crates/render/src/audio_fx.rs new file mode 100644 index 00000000..a6a10e8c --- /dev/null +++ b/crates/render/src/audio_fx.rs @@ -0,0 +1,221 @@ +//! Clip audio effects: project effect instances → `filmcraft-audio-dsp` processors. +//! +//! The mixer is pull-based (any sample range, any order), but most audio effects are stateful +//! (filters, dynamics, delay/reverb tails). A small cache keeps processed effect chains per clip +//! keyed by the next timeline sample they expect: sequential consumers (playback, export, meters) +//! continue their chain; any other request starts a fresh chain with enough pre-roll for the +//! chain's tails to build up. Latency (limiter look-ahead, STFT effects) is compensated by feeding +//! the chain ahead of its output. + +use std::collections::HashMap; +use std::sync::{Mutex, OnceLock}; + +use filmcraft_audio_dsp::AudioEffect; +use filmcraft_project::{EffectInstance, TrackItem}; +use filmcraft_time::Tick; + +/// How a project effect maps onto a DSP effect. +struct Mapping { + dsp: &'static str, + /// Pre-roll (seconds) for a fresh chain; closures get the effect instance at the start time. + preroll: fn(&EffectInstance, Tick) -> f64, + /// Set DSP parameters from the (keyframed) project parameters at media time `mt`. + apply: fn(&mut dyn AudioEffect, &EffectInstance, Tick), +} + +fn ignore(_: bool) {} + +fn f(e: &EffectInstance, id: &str, mt: Tick) -> f32 { + e.f64_at(id, mt) as f32 +} + +fn mapping(id: &str) -> Option { + Some(match id { + "amplify" => Mapping { dsp: "amplify", preroll: |_, _| 0.0, apply: |d, e, t| ignore(d.set_param("gain", f(e, "gain", t))) }, + "dynamics" => Mapping { + dsp: "compressor", + preroll: |e, t| (f(e, "release", t) as f64 / 1000.0 * 5.0).clamp(0.05, 3.0), + apply: |d, e, t| { + d.set_param("threshold", f(e, "threshold", t)); + d.set_param("ratio", f(e, "ratio", t)); + d.set_param("attack", f(e, "attack", t)); + d.set_param("release", f(e, "release", t)); + }, + }, + "hard_limiter" => Mapping { + dsp: "limiter", + preroll: |e, t| (f(e, "release", t) as f64 / 1000.0 * 5.0).clamp(0.05, 2.0), + apply: |d, e, t| { + d.set_param("ceiling", f(e, "max", t)); + d.set_param("input_gain", f(e, "boost", t).max(0.0)); + d.set_param("release", f(e, "release", t)); + }, + }, + "delay" => Mapping { + dsp: "delay", + preroll: |e, t| { + // enough echoes to decay by ~80 dB + let time = f(e, "delay", t).max(0.001) as f64; + let fb = (f(e, "feedback", t) as f64 / 100.0).clamp(0.0, 0.95); + let repeats = if fb > 0.001 { (-4.0 / fb.log10()).clamp(1.0, 60.0) } else { 1.0 }; + (time * repeats).min(8.0) + }, + apply: |d, e, t| { + d.set_param("time", f(e, "delay", t) * 1000.0); + d.set_param("feedback", f(e, "feedback", t)); + d.set_param("mix", f(e, "mix", t)); + }, + }, + "parametric_eq" => Mapping { + dsp: "simple_eq", + preroll: |_, _| 0.05, + apply: |d, e, t| { + d.set_param("low_freq", f(e, "low_freq", t)); + d.set_param("low", f(e, "low_gain", t)); + d.set_param("mid_freq", f(e, "mid_freq", t)); + d.set_param("mid", f(e, "mid_gain", t)); + d.set_param("mid_q", f(e, "mid_q", t)); + d.set_param("high_freq", f(e, "high_freq", t)); + d.set_param("high", f(e, "high_gain", t)); + }, + }, + // Single-band filters via band 1 of the parametric EQ (types: see FILTER_TYPE_NAMES). + "highpass" => Mapping { dsp: "parametric_eq", preroll: |_, _| 0.05, apply: |d, e, t| band(d, 4.0, f(e, "cutoff", t), 0.707) }, + "lowpass" => Mapping { dsp: "parametric_eq", preroll: |_, _| 0.05, apply: |d, e, t| band(d, 3.0, f(e, "cutoff", t), 0.707) }, + "bandpass" => Mapping { dsp: "parametric_eq", preroll: |_, _| 0.05, apply: |d, e, t| band(d, 6.0, f(e, "center", t), f(e, "q", t)) }, + "denoise" => Mapping { dsp: "denoise", preroll: |_, _| 1.0, apply: |d, e, t| ignore(d.set_param("amount", f(e, "amount", t))) }, + "dehummer" => Mapping { + dsp: "dehum", + preroll: |_, _| 0.2, + apply: |d, e, t| ignore(d.set_param("frequency", e.param("freq").and_then(|p| p.value_at(t).as_f64()).unwrap_or(1.0) as f32)), + }, + "studio_reverb" => Mapping { + dsp: "reverb", + preroll: |e, t| 0.3 * 30f64.powf(f(e, "decay", t) as f64 / 100.0).min(8.0), + apply: |d, e, t| { + d.set_param("size", f(e, "room", t)); + d.set_param("decay", (0.3 * 30f32.powf(f(e, "decay", t) / 100.0)).clamp(0.1, 20.0)); + d.set_param("damping", f(e, "damping", t)); + let (dry, wet) = (f(e, "dry", t).max(0.0), f(e, "wet", t).max(0.0)); + d.set_param("mix", if dry + wet > 0.0 { wet / (dry + wet) * 100.0 } else { 0.0 }); + }, + }, + "invert_a" => Mapping { dsp: "invert", preroll: |_, _| 0.0, apply: |_, _, _| {} }, + "pitch_shifter" => Mapping { + dsp: "pitch_shifter", + preroll: |_, _| 0.1, + apply: |d, e, t| ignore(d.set_param("semitones", (f(e, "semitones", t) + f(e, "cents", t) / 100.0).clamp(-12.0, 12.0))), + }, + _ => return None, + }) +} + +fn band(d: &mut dyn AudioEffect, kind: f32, freq: f32, q: f32) { + d.set_param("b1.on", 1.0); + d.set_param("b1.type", kind); + d.set_param("b1.freq", freq); + d.set_param("b1.q", q); +} + +/// The item's enabled, DSP-backed audio effects in rack order. +fn active(item: &TrackItem) -> Vec<(&EffectInstance, Mapping)> { + item.effects.iter().filter(|e| e.enabled && !e.def().is_some_and(|d| d.intrinsic)).filter_map(|e| mapping(&e.effect).map(|m| (e, m))).collect() +} + +/// Whether the item has audio effects that change its signal. +pub fn has_effects(item: &TrackItem) -> bool { + !active(item).is_empty() +} + +struct Chain { + fx: Vec>, + latency: usize, + /// Next timeline sample this chain will output. + next_out: i64, +} + +type Key = (u64, u64, u32); + +fn cache() -> &'static Mutex>> { + static C: OnceLock>>> = OnceLock::new(); + C.get_or_init(Default::default) +} + +fn structure_hash(item: &TrackItem) -> u64 { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + for (e, m) in active(item) { + e.effect.hash(&mut h); + m.dsp.hash(&mut h); + } + h.finish() +} + +const BLOCK: usize = 256; + +/// Run `input` (stereo, `input[c].len()` samples starting at timeline sample `in0`) through the +/// chain, setting keyframed parameters per block. +fn run(chain: &mut Chain, item: &TrackItem, in0: i64, input: &mut [Vec; 2], sr: u32) { + let act = active(item); + let n = input[0].len(); + let mut i = 0; + while i < n { + let end = (i + BLOCK).min(n); + let mt = item.source_time_at(Tick::from_units(in0 + i as i64, sr as i64)); + for ((e, m), d) in act.iter().zip(chain.fx.iter_mut()) { + (m.apply)(d.as_mut(), e, mt); + } + let [l, r] = input; + let mut chans: [&mut [f32]; 2] = [&mut l[i..end], &mut r[i..end]]; + for d in chain.fx.iter_mut() { + d.process(&mut chans); + } + i = end; + } +} + +/// Process the item's audio for timeline samples `[a0, a0 + n)`. `read(x0, len)` returns the clip's +/// raw stereo audio for timeline samples `[x0, x0 + len)` (silence outside the clip). +pub fn process(item: &TrackItem, a0: i64, n: usize, sr: u32, read: &dyn Fn(i64, usize) -> [Vec; 2]) -> [Vec; 2] { + let key = (item.id.0, structure_hash(item), sr); + let taken = { + let mut c = cache().lock().unwrap_or_else(|e| e.into_inner()); + c.get_mut(&key).and_then(|v| v.iter().position(|ch| ch.next_out == a0).map(|i| v.swap_remove(i))) + }; + let (mut chain, mut input, skip) = match taken { + Some(chain) => { + let l = chain.latency as i64; + (chain, read(a0 + l, n), 0usize) + } + None => { + let act = active(item); + let mt = item.source_time_at(Tick::from_units(a0, sr as i64)); + let pre_s = act.iter().map(|(e, m)| (m.preroll)(e, mt)).fold(0.0, f64::max); + let pre = (pre_s * sr as f64).ceil() as usize; + let mut fx: Vec> = Vec::new(); + for (_, m) in &act { + if let Some(d) = filmcraft_audio_dsp::create_effect(m.dsp, sr as f32, 2) { + fx.push(d); + } + } + let latency = fx.iter().map(|d| d.latency()).sum::(); + let chain = Chain { fx, latency, next_out: a0 - pre as i64 }; + let x0 = a0 - pre as i64; + (chain, read(x0 + latency as i64, pre + n), pre) + } + }; + let in0 = chain.next_out + chain.latency as i64; + run(&mut chain, item, in0, &mut input, sr); + chain.next_out = a0 + n as i64; + let out = [input[0][skip..].to_vec(), input[1][skip..].to_vec()]; + let mut c = cache().lock().unwrap_or_else(|e| e.into_inner()); + let v = c.entry(key).or_default(); + v.push(chain); + if v.len() > 4 { + v.remove(0); + } + if c.len() > 256 { + c.clear(); + } + out +} diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 4f6de63e..aca70ee0 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -8,6 +8,7 @@ //! The same code renders monitors, thumbnails and exports, and is the oracle for the GPU path. pub mod audio; +pub mod audio_fx; pub mod blend; pub mod effects; pub mod image; diff --git a/crates/render/src/tests.rs b/crates/render/src/tests.rs index e0e71880..4200575a 100644 --- a/crates/render/src/tests.rs +++ b/crates/render/src/tests.rs @@ -193,3 +193,77 @@ fn plan_matches_reference_renderer() { assert!(max < 0.02, "frame {f}: max diff {max}"); } } + +/// A project with a 4 s bars-and-tone clip (1 kHz at −20 dBFS) on A1, with extra audio effects. +fn tone_with(effects: &[(&str, &[(&str, f64)])]) -> (Project, ItemId, SourceMap) { + let mut p = Project::new("fx"); + let g = GeneratorSource::new(Generator::BarsAndTone, 64, 36, FrameRate::FPS_24, Tick(10 * TICKS_PER_SECOND)); + let info = g.info().clone(); + let id = p.add_item( + "bars", + Label::Iris, + ItemKind::Media(MediaClip { + media: MediaRef::Generator(Generator::BarsAndTone), + info, + interpret: Default::default(), + mark_in: None, + mark_out: None, + markers: vec![], + offline: false, + proxy: None, + }), + None, + ); + let mut map = SourceMap::default(); + map.0.insert(id, Arc::new(g)); + let seq = p.new_sequence("s", SequenceSettings::default(), 1, 1, None); + let r = FrameRate::FPS_23_976; + let mut ti = p.make_track_item(id, TrackKind::Audio, Tick::ZERO, TimeRange::new(Tick::ZERO, Tick(4 * TICKS_PER_SECOND)), r).unwrap(); + for (fx, params) in effects { + let mut e = filmcraft_project::find_effect(fx).expect("effect").instance(); + for (k, v) in *params { + e.param_mut(k).expect("param").value = ParamValue::Float(*v); + } + ti.effects.push(e); + } + p.sequence_mut(seq).unwrap().audio_tracks[0].items.push(ti); + (p, seq, map) +} + +fn mix(p: &Project, seq: ItemId, map: &SourceMap, start: i64, n: usize) -> Vec { + audio::mix_sequence(p, p.sequence(seq).unwrap(), start, n, map).channels[0].clone() +} + +#[test] +fn audio_fx_limiter_holds_ceiling() { + // +30 dB boost via Amplify (tone at +10 dBFS), then a −6 dB hard limiter. + let (p, seq, map) = tone_with(&[("amplify", &[("gain", 30.0)]), ("hard_limiter", &[("max", -6.0)])]); + let out = mix(&p, seq, &map, 24_000, 48_000); + let peak = out.iter().fold(0f32, |m, s| m.max(s.abs())); + let ceiling = 10f32.powf(-6.0 / 20.0); + assert!(peak <= ceiling * 1.01 && peak > ceiling * 0.7, "peak {peak} vs ceiling {ceiling}"); +} + +#[test] +fn audio_fx_chain_is_continuous_and_random_access_matches() { + let (p, seq, map) = tone_with(&[("delay", &[("delay", 0.05), ("feedback", 50.0), ("mix", 50.0)])]); + let start = 48_000; + // one long call (fresh chain with pre-roll) + let whole = mix(&p, seq, &map, start, 9_600); + // the same range in consecutive blocks from a different position (continues a cached chain) + let (p2, seq2, map2) = tone_with(&[("delay", &[("delay", 0.05), ("feedback", 50.0), ("mix", 50.0)])]); + let mut parts = Vec::new(); + let mut s = start - 4_800; + let _ = mix(&p2, seq2, &map2, s, 4_800); + s += 4_800; + while s < start + 9_600 { + parts.extend(mix(&p2, seq2, &map2, s, 1_200)); + s += 1_200; + } + let max = whole.iter().zip(&parts).map(|(a, b)| (a - b).abs()).fold(0f32, f32::max); + assert!(max < 1e-4, "sequential blocks differ from one call by {max}"); + // the delay changes the signal (wet echoes mixed in) + let dry = mix(&tone_with(&[]).0, tone_with(&[]).1, &tone_with(&[]).2, start, 9_600); + let diff = whole.iter().zip(&dry).map(|(a, b)| (a - b).abs()).fold(0f32, f32::max); + assert!(diff > 1e-3, "delay had no effect"); +}