M7.3: clip audio effects run on filmcraft-audio-dsp (Amplify, Dynamics, Hard Limiter, Delay, Parametric EQ, High/Low/Bandpass, DeNoise, DeHummer, Studio Reverb, Invert, Pitch Shifter) with keyframed params, latency compensation and a continuation cache with pre-roll for random access

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Brandon Thomas 2026-09-30 11:40:58 -04:00
parent de7e84307d
commit dc9e3933a8
6 changed files with 342 additions and 22 deletions

1
Cargo.lock generated
View File

@ -1546,6 +1546,7 @@ dependencies = [
name = "filmcraft-render"
version = "0.1.0"
dependencies = [
"filmcraft-audio-dsp",
"filmcraft-color",
"filmcraft-frame",
"filmcraft-geom",

View File

@ -16,6 +16,7 @@ filmcraft-color = { workspace = true }
filmcraft-frame = { workspace = true }
filmcraft-media = { workspace = true }
filmcraft-project = { workspace = true }
filmcraft-audio-dsp = { workspace = true }
rayon = { workspace = true }
[dev-dependencies]

View File

@ -107,6 +107,43 @@ fn mix_item(project: &Project, item: &TrackItem, start: i64, frames: usize, sr:
return;
}
let n = (a1 - a0) as usize;
let read = |x0: i64, len: usize| raw_stereo(item, src.as_ref(), x0, len, sr);
let buf = if crate::audio_fx::has_effects(item) { crate::audio_fx::process(item, a0, n, sr, &read) } else { read(a0, n) };
// gains: clip gain × Volume (keyframed, per 64-sample block) × channel volume × panner
let clip_gain = db_to_gain(item.gain_db);
let vol = item.effect("volume").filter(|e| e.enabled && !e.param("bypass").and_then(|p| p.value.as_bool()).unwrap_or(false));
let chv = item.effect("channel_volume").filter(|e| e.enabled && !e.param("bypass").and_then(|p| p.value.as_bool()).unwrap_or(false));
let pan = item.effect("panner").filter(|e| e.enabled);
let off = (a0 - start) as usize;
for blk in (0..n).step_by(64) {
let t_tl = Tick::from_units(a0 + blk as i64, sr as i64);
let mt = item.source_time_at(t_tl);
let v = vol.map(|e| db_to_gain(e.f64_at("level", mt))).unwrap_or(1.0);
let (cl, cr) = chv.map(|e| (db_to_gain(e.f64_at("left", mt)), db_to_gain(e.f64_at("right", mt)))).unwrap_or((1.0, 1.0));
let (pl, pr) = pan.map(|e| pan_gains((e.f64_at("balance", mt) / 100.0) as f32)).unwrap_or((1.0, 1.0));
let g = clip_gain * v * extra;
let end = (blk + 64).min(n);
for i in blk..end {
let (l, r) = (buf[0][i], buf[1][i]);
out.channels[0][off + i] += l * g * cl * pl;
out.channels[1][off + i] += r * g * cr * pr;
}
}
}
/// The clip's audio (after speed/reverse) for timeline samples `[x0, x0 + len)` as stereo, with
/// silence outside the clip. Mono sources are duplicated to both channels.
fn raw_stereo(item: &TrackItem, src: &dyn filmcraft_media::MediaSource, x0: i64, len: usize, sr: u32) -> [Vec<f32>; 2] {
let mut out = [vec![0.0f32; len], vec![0.0f32; len]];
let item_s0 = item.start.to_units_floor(sr as i64);
let item_s1 = item.end().to_units_floor(sr as i64);
let a0 = x0.max(item_s0);
let a1 = (x0 + len as i64).min(item_s1);
if a1 <= a0 {
return out;
}
let n = (a1 - a0) as usize;
let off = (a0 - x0) as usize;
let speed = item.speed.abs().max(1e-6);
let src_start_ticks = item.source_time_at(Tick::from_units(a0, sr as i64));
let buf = if (speed - 1.0).abs() < 1e-9 && !item.reverse {
@ -130,29 +167,14 @@ fn mix_item(project: &Project, item: &TrackItem, start: i64, frames: usize, sr:
b
})
};
let Some(buf) = buf else { return };
// gains: clip gain × Volume (keyframed, per 64-sample block) × channel volume × panner
let clip_gain = db_to_gain(item.gain_db);
let vol = item.effect("volume").filter(|e| e.enabled && !e.param("bypass").and_then(|p| p.value.as_bool()).unwrap_or(false));
let chv = item.effect("channel_volume").filter(|e| e.enabled && !e.param("bypass").and_then(|p| p.value.as_bool()).unwrap_or(false));
let pan = item.effect("panner").filter(|e| e.enabled);
let off = (a0 - start) as usize;
let stereo_src = buf.channel_count() >= 2;
for blk in (0..n).step_by(64) {
let t_tl = Tick::from_units(a0 + blk as i64, sr as i64);
let mt = item.source_time_at(t_tl);
let v = vol.map(|e| db_to_gain(e.f64_at("level", mt))).unwrap_or(1.0);
let (cl, cr) = chv.map(|e| (db_to_gain(e.f64_at("left", mt)), db_to_gain(e.f64_at("right", mt)))).unwrap_or((1.0, 1.0));
let (pl, pr) = pan.map(|e| pan_gains((e.f64_at("balance", mt) / 100.0) as f32)).unwrap_or((1.0, 1.0));
let g = clip_gain * v * extra;
let end = (blk + 64).min(n);
for i in blk..end {
let l = buf.channels[0][i];
let r = if stereo_src { buf.channels[1][i] } else { l };
out.channels[0][off + i] += l * g * cl * pl;
out.channels[1][off + i] += r * g * cr * pr;
}
let Some(buf) = buf else { return out };
let right = if buf.channel_count() >= 2 { 1 } else { 0 };
for (c, dst) in out.iter_mut().enumerate() {
let ch = &buf.channels[if c == 0 { 0 } else { right }];
let m = n.min(ch.len());
dst[off..off + m].copy_from_slice(&ch[..m]);
}
out
}
/// Peak (min, max) pairs per bucket of `bucket` samples for channel `ch` — waveform display.

View File

@ -0,0 +1,221 @@
//! Clip audio effects: project effect instances → `filmcraft-audio-dsp` processors.
//!
//! The mixer is pull-based (any sample range, any order), but most audio effects are stateful
//! (filters, dynamics, delay/reverb tails). A small cache keeps processed effect chains per clip
//! keyed by the next timeline sample they expect: sequential consumers (playback, export, meters)
//! continue their chain; any other request starts a fresh chain with enough pre-roll for the
//! chain's tails to build up. Latency (limiter look-ahead, STFT effects) is compensated by feeding
//! the chain ahead of its output.
use std::collections::HashMap;
use std::sync::{Mutex, OnceLock};
use filmcraft_audio_dsp::AudioEffect;
use filmcraft_project::{EffectInstance, TrackItem};
use filmcraft_time::Tick;
/// How a project effect maps onto a DSP effect.
struct Mapping {
dsp: &'static str,
/// Pre-roll (seconds) for a fresh chain; closures get the effect instance at the start time.
preroll: fn(&EffectInstance, Tick) -> f64,
/// Set DSP parameters from the (keyframed) project parameters at media time `mt`.
apply: fn(&mut dyn AudioEffect, &EffectInstance, Tick),
}
fn ignore(_: bool) {}
fn f(e: &EffectInstance, id: &str, mt: Tick) -> f32 {
e.f64_at(id, mt) as f32
}
fn mapping(id: &str) -> Option<Mapping> {
Some(match id {
"amplify" => Mapping { dsp: "amplify", preroll: |_, _| 0.0, apply: |d, e, t| ignore(d.set_param("gain", f(e, "gain", t))) },
"dynamics" => Mapping {
dsp: "compressor",
preroll: |e, t| (f(e, "release", t) as f64 / 1000.0 * 5.0).clamp(0.05, 3.0),
apply: |d, e, t| {
d.set_param("threshold", f(e, "threshold", t));
d.set_param("ratio", f(e, "ratio", t));
d.set_param("attack", f(e, "attack", t));
d.set_param("release", f(e, "release", t));
},
},
"hard_limiter" => Mapping {
dsp: "limiter",
preroll: |e, t| (f(e, "release", t) as f64 / 1000.0 * 5.0).clamp(0.05, 2.0),
apply: |d, e, t| {
d.set_param("ceiling", f(e, "max", t));
d.set_param("input_gain", f(e, "boost", t).max(0.0));
d.set_param("release", f(e, "release", t));
},
},
"delay" => Mapping {
dsp: "delay",
preroll: |e, t| {
// enough echoes to decay by ~80 dB
let time = f(e, "delay", t).max(0.001) as f64;
let fb = (f(e, "feedback", t) as f64 / 100.0).clamp(0.0, 0.95);
let repeats = if fb > 0.001 { (-4.0 / fb.log10()).clamp(1.0, 60.0) } else { 1.0 };
(time * repeats).min(8.0)
},
apply: |d, e, t| {
d.set_param("time", f(e, "delay", t) * 1000.0);
d.set_param("feedback", f(e, "feedback", t));
d.set_param("mix", f(e, "mix", t));
},
},
"parametric_eq" => Mapping {
dsp: "simple_eq",
preroll: |_, _| 0.05,
apply: |d, e, t| {
d.set_param("low_freq", f(e, "low_freq", t));
d.set_param("low", f(e, "low_gain", t));
d.set_param("mid_freq", f(e, "mid_freq", t));
d.set_param("mid", f(e, "mid_gain", t));
d.set_param("mid_q", f(e, "mid_q", t));
d.set_param("high_freq", f(e, "high_freq", t));
d.set_param("high", f(e, "high_gain", t));
},
},
// Single-band filters via band 1 of the parametric EQ (types: see FILTER_TYPE_NAMES).
"highpass" => Mapping { dsp: "parametric_eq", preroll: |_, _| 0.05, apply: |d, e, t| band(d, 4.0, f(e, "cutoff", t), 0.707) },
"lowpass" => Mapping { dsp: "parametric_eq", preroll: |_, _| 0.05, apply: |d, e, t| band(d, 3.0, f(e, "cutoff", t), 0.707) },
"bandpass" => Mapping { dsp: "parametric_eq", preroll: |_, _| 0.05, apply: |d, e, t| band(d, 6.0, f(e, "center", t), f(e, "q", t)) },
"denoise" => Mapping { dsp: "denoise", preroll: |_, _| 1.0, apply: |d, e, t| ignore(d.set_param("amount", f(e, "amount", t))) },
"dehummer" => Mapping {
dsp: "dehum",
preroll: |_, _| 0.2,
apply: |d, e, t| ignore(d.set_param("frequency", e.param("freq").and_then(|p| p.value_at(t).as_f64()).unwrap_or(1.0) as f32)),
},
"studio_reverb" => Mapping {
dsp: "reverb",
preroll: |e, t| 0.3 * 30f64.powf(f(e, "decay", t) as f64 / 100.0).min(8.0),
apply: |d, e, t| {
d.set_param("size", f(e, "room", t));
d.set_param("decay", (0.3 * 30f32.powf(f(e, "decay", t) / 100.0)).clamp(0.1, 20.0));
d.set_param("damping", f(e, "damping", t));
let (dry, wet) = (f(e, "dry", t).max(0.0), f(e, "wet", t).max(0.0));
d.set_param("mix", if dry + wet > 0.0 { wet / (dry + wet) * 100.0 } else { 0.0 });
},
},
"invert_a" => Mapping { dsp: "invert", preroll: |_, _| 0.0, apply: |_, _, _| {} },
"pitch_shifter" => Mapping {
dsp: "pitch_shifter",
preroll: |_, _| 0.1,
apply: |d, e, t| ignore(d.set_param("semitones", (f(e, "semitones", t) + f(e, "cents", t) / 100.0).clamp(-12.0, 12.0))),
},
_ => return None,
})
}
fn band(d: &mut dyn AudioEffect, kind: f32, freq: f32, q: f32) {
d.set_param("b1.on", 1.0);
d.set_param("b1.type", kind);
d.set_param("b1.freq", freq);
d.set_param("b1.q", q);
}
/// The item's enabled, DSP-backed audio effects in rack order.
fn active(item: &TrackItem) -> Vec<(&EffectInstance, Mapping)> {
item.effects.iter().filter(|e| e.enabled && !e.def().is_some_and(|d| d.intrinsic)).filter_map(|e| mapping(&e.effect).map(|m| (e, m))).collect()
}
/// Whether the item has audio effects that change its signal.
pub fn has_effects(item: &TrackItem) -> bool {
!active(item).is_empty()
}
struct Chain {
fx: Vec<Box<dyn AudioEffect>>,
latency: usize,
/// Next timeline sample this chain will output.
next_out: i64,
}
type Key = (u64, u64, u32);
fn cache() -> &'static Mutex<HashMap<Key, Vec<Chain>>> {
static C: OnceLock<Mutex<HashMap<Key, Vec<Chain>>>> = OnceLock::new();
C.get_or_init(Default::default)
}
fn structure_hash(item: &TrackItem) -> u64 {
use std::hash::{Hash, Hasher};
let mut h = std::collections::hash_map::DefaultHasher::new();
for (e, m) in active(item) {
e.effect.hash(&mut h);
m.dsp.hash(&mut h);
}
h.finish()
}
const BLOCK: usize = 256;
/// Run `input` (stereo, `input[c].len()` samples starting at timeline sample `in0`) through the
/// chain, setting keyframed parameters per block.
fn run(chain: &mut Chain, item: &TrackItem, in0: i64, input: &mut [Vec<f32>; 2], sr: u32) {
let act = active(item);
let n = input[0].len();
let mut i = 0;
while i < n {
let end = (i + BLOCK).min(n);
let mt = item.source_time_at(Tick::from_units(in0 + i as i64, sr as i64));
for ((e, m), d) in act.iter().zip(chain.fx.iter_mut()) {
(m.apply)(d.as_mut(), e, mt);
}
let [l, r] = input;
let mut chans: [&mut [f32]; 2] = [&mut l[i..end], &mut r[i..end]];
for d in chain.fx.iter_mut() {
d.process(&mut chans);
}
i = end;
}
}
/// Process the item's audio for timeline samples `[a0, a0 + n)`. `read(x0, len)` returns the clip's
/// raw stereo audio for timeline samples `[x0, x0 + len)` (silence outside the clip).
pub fn process(item: &TrackItem, a0: i64, n: usize, sr: u32, read: &dyn Fn(i64, usize) -> [Vec<f32>; 2]) -> [Vec<f32>; 2] {
let key = (item.id.0, structure_hash(item), sr);
let taken = {
let mut c = cache().lock().unwrap_or_else(|e| e.into_inner());
c.get_mut(&key).and_then(|v| v.iter().position(|ch| ch.next_out == a0).map(|i| v.swap_remove(i)))
};
let (mut chain, mut input, skip) = match taken {
Some(chain) => {
let l = chain.latency as i64;
(chain, read(a0 + l, n), 0usize)
}
None => {
let act = active(item);
let mt = item.source_time_at(Tick::from_units(a0, sr as i64));
let pre_s = act.iter().map(|(e, m)| (m.preroll)(e, mt)).fold(0.0, f64::max);
let pre = (pre_s * sr as f64).ceil() as usize;
let mut fx: Vec<Box<dyn AudioEffect>> = Vec::new();
for (_, m) in &act {
if let Some(d) = filmcraft_audio_dsp::create_effect(m.dsp, sr as f32, 2) {
fx.push(d);
}
}
let latency = fx.iter().map(|d| d.latency()).sum::<usize>();
let chain = Chain { fx, latency, next_out: a0 - pre as i64 };
let x0 = a0 - pre as i64;
(chain, read(x0 + latency as i64, pre + n), pre)
}
};
let in0 = chain.next_out + chain.latency as i64;
run(&mut chain, item, in0, &mut input, sr);
chain.next_out = a0 + n as i64;
let out = [input[0][skip..].to_vec(), input[1][skip..].to_vec()];
let mut c = cache().lock().unwrap_or_else(|e| e.into_inner());
let v = c.entry(key).or_default();
v.push(chain);
if v.len() > 4 {
v.remove(0);
}
if c.len() > 256 {
c.clear();
}
out
}

View File

@ -8,6 +8,7 @@
//! The same code renders monitors, thumbnails and exports, and is the oracle for the GPU path.
pub mod audio;
pub mod audio_fx;
pub mod blend;
pub mod effects;
pub mod image;

View File

@ -193,3 +193,77 @@ fn plan_matches_reference_renderer() {
assert!(max < 0.02, "frame {f}: max diff {max}");
}
}
/// A project with a 4 s bars-and-tone clip (1 kHz at −20 dBFS) on A1, with extra audio effects.
fn tone_with(effects: &[(&str, &[(&str, f64)])]) -> (Project, ItemId, SourceMap) {
let mut p = Project::new("fx");
let g = GeneratorSource::new(Generator::BarsAndTone, 64, 36, FrameRate::FPS_24, Tick(10 * TICKS_PER_SECOND));
let info = g.info().clone();
let id = p.add_item(
"bars",
Label::Iris,
ItemKind::Media(MediaClip {
media: MediaRef::Generator(Generator::BarsAndTone),
info,
interpret: Default::default(),
mark_in: None,
mark_out: None,
markers: vec![],
offline: false,
proxy: None,
}),
None,
);
let mut map = SourceMap::default();
map.0.insert(id, Arc::new(g));
let seq = p.new_sequence("s", SequenceSettings::default(), 1, 1, None);
let r = FrameRate::FPS_23_976;
let mut ti = p.make_track_item(id, TrackKind::Audio, Tick::ZERO, TimeRange::new(Tick::ZERO, Tick(4 * TICKS_PER_SECOND)), r).unwrap();
for (fx, params) in effects {
let mut e = filmcraft_project::find_effect(fx).expect("effect").instance();
for (k, v) in *params {
e.param_mut(k).expect("param").value = ParamValue::Float(*v);
}
ti.effects.push(e);
}
p.sequence_mut(seq).unwrap().audio_tracks[0].items.push(ti);
(p, seq, map)
}
fn mix(p: &Project, seq: ItemId, map: &SourceMap, start: i64, n: usize) -> Vec<f32> {
audio::mix_sequence(p, p.sequence(seq).unwrap(), start, n, map).channels[0].clone()
}
#[test]
fn audio_fx_limiter_holds_ceiling() {
// +30 dB boost via Amplify (tone at +10 dBFS), then a −6 dB hard limiter.
let (p, seq, map) = tone_with(&[("amplify", &[("gain", 30.0)]), ("hard_limiter", &[("max", -6.0)])]);
let out = mix(&p, seq, &map, 24_000, 48_000);
let peak = out.iter().fold(0f32, |m, s| m.max(s.abs()));
let ceiling = 10f32.powf(-6.0 / 20.0);
assert!(peak <= ceiling * 1.01 && peak > ceiling * 0.7, "peak {peak} vs ceiling {ceiling}");
}
#[test]
fn audio_fx_chain_is_continuous_and_random_access_matches() {
let (p, seq, map) = tone_with(&[("delay", &[("delay", 0.05), ("feedback", 50.0), ("mix", 50.0)])]);
let start = 48_000;
// one long call (fresh chain with pre-roll)
let whole = mix(&p, seq, &map, start, 9_600);
// the same range in consecutive blocks from a different position (continues a cached chain)
let (p2, seq2, map2) = tone_with(&[("delay", &[("delay", 0.05), ("feedback", 50.0), ("mix", 50.0)])]);
let mut parts = Vec::new();
let mut s = start - 4_800;
let _ = mix(&p2, seq2, &map2, s, 4_800);
s += 4_800;
while s < start + 9_600 {
parts.extend(mix(&p2, seq2, &map2, s, 1_200));
s += 1_200;
}
let max = whole.iter().zip(&parts).map(|(a, b)| (a - b).abs()).fold(0f32, f32::max);
assert!(max < 1e-4, "sequential blocks differ from one call by {max}");
// the delay changes the signal (wet echoes mixed in)
let dry = mix(&tone_with(&[]).0, tone_with(&[]).1, &tone_with(&[]).2, start, 9_600);
let diff = whole.iter().zip(&dry).map(|(a, b)| (a - b).abs()).fold(0f32, f32::max);
assert!(diff > 1e-3, "delay had no effect");
}