From cd30edbed7269e38354502223f0875a0856caa0a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 1 Oct 2026 02:35:14 -0400 Subject: [PATCH 1/5] =?UTF-8?q?M7.4:=20Essential=20Sound=20DSP=20=E2=80=94?= =?UTF-8?q?=20DeEsser,=20DeReverb,=20Enhance=20Speech=20chain,=20Stereo=20?= =?UTF-8?q?Width,=20ducking=20analysis?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/audio-dsp/README.md | 40 +- crates/audio-dsp/src/ducking.rs | 229 +++++++ crates/audio-dsp/src/effects/essential.rs | 764 ++++++++++++++++++++++ crates/audio-dsp/src/effects/mod.rs | 6 + crates/audio-dsp/src/lib.rs | 1 + 5 files changed, 1039 insertions(+), 1 deletion(-) create mode 100644 crates/audio-dsp/src/ducking.rs create mode 100644 crates/audio-dsp/src/effects/essential.rs diff --git a/crates/audio-dsp/README.md b/crates/audio-dsp/README.md index e1ae13a6..83928438 100644 --- a/crates/audio-dsp/README.md +++ b/crates/audio-dsp/README.md @@ -3,7 +3,8 @@ Real-time audio DSP for FilmCraft. Layer **L1**, **no dependencies**, compiles for `wasm32-unknown-unknown`, no `unsafe`. Clean-room: written from the public standards (ITU-R BS.1770-4, EBU R128 / Tech 3341 / Tech 3342) and textbook DSP (RBJ cookbook biquads, -Householder FDN, phase vocoder). +Householder FDN, phase vocoder, spectral subtraction with the statistical late-reverberation +model). ## Conventions @@ -50,6 +51,43 @@ id, name, range, default, `Unit`, log-scale hint and choice labels so UIs can be | `dehum` | 50/60 Hz notch comb with up to 10 harmonics | | `denoise` | STFT spectral gating, minimum-statistics noise floor (latency = FFT size) | | `pitch_shifter` | ±12 semitones, phase vocoder with peak shifting + phase locking (latency = FFT size) | +| `deesser` | high-pass sidechain at `frequency` (2–12 kHz) detects sibilance relative to the broadband level (`threshold`); a dynamic high shelf cuts the band by up to `reduction` dB (1 ms / 60 ms, stereo-linked); exact identity when idle | +| `dereverb` | STFT late-reverb suppression: λ_r(t) = e^{−2Δ·T_d}·λ_x(t − T_d) (Δ = 3 ln10 / `rt60`, T_d ≈ 50 ms), over-subtraction 1 + `amount`, gain floor down to −18 dB, smoothed gains (latency = FFT size) | +| `speech_enhance` | Enhance Speech DSP chain: 80 Hz HPF, de-mud cut (250 / 350 Hz −3 dB), presence (2.5 / 4 kHz +4 dB) and air (+2 dB @ 10 kHz) boosts by `tone`, downward expander below −50 dBFS (3:1, ≤ 18 dB), 3:1 soft-knee compressor above −24 dBFS, +3 dB make-up, `mix` | +| `stereo_width` | mid/side width 0–200 % (100 % = identity, 0 % = mono) | + +Essential Sound builds on these (Repair: `denoise`, high-pass via `parametric_eq`, `dehum`, +`deesser`, `dereverb`; Clarity: `compressor`, `simple_eq`, `speech_enhance`; Creative: `reverb`, +`stereo_width`). + +**Enhance Speech decision.** No machine-learning model is bundled: we found no openly licensed +speech-enhancement model that is also practical to ship here (pure Rust, wasm, asset rules). +DeepFilterNet (MIT/Apache-2.0, with a Rust implementation on `tract`) is the candidate for a future +optional integration behind a trait outside this dependency-free crate, once its weights' licence and +size are vetted as an asset. Until then `speech_enhance` is the DSP chain above. + +Measured on synthetic signals (`effects::essential::tests`, 48 kHz): + +| Effect | Signal | Result | +|---|---|---| +| `deesser` | 150 Hz harmonic voice + 5–10 kHz noise bursts | sibilance −5.1 dB (8 dB max) / −9.8 dB (16 dB max) during bursts; voice band ±0.01 dB; non-sibilant passages ±0.001 dB | +| `dereverb` (100 %) | harmonic syllables (100 ms / 400 ms gaps) + Polack-model late reverb, RT60 0.8 s | tail-to-burst energy −5.0 → −10.1 dB (5.0 dB better); burst energy −1.7 dB | +| `speech_enhance` | voice + fricative noise syllables over −60 dBFS noise | presence/mud tilt +5.5 dB; noise between syllables −8.7 dB; overall level +1.0 dB | +| Dialogue chain | `denoise` + `parametric_eq` (2 HP bands) + `dehum` + `deesser` + `dereverb` + `compressor` + `simple_eq` + `speech_enhance` + `reverb`, 60 s stereo | 1.36 s = **44× realtime** on one core (release, Apple Silicon; `cargo test --release -p filmcraft-audio-dsp dialogue_chain_realtime -- --ignored --nocapture`) | + +## Ducking analysis (`ducking`) + +Pure functions for Essential Sound auto-ducking: + +- `envelope_db(channels, sample_rate, hop_s, window_s)`: centred-window RMS level per hop (dBFS, + channel-averaged power, floor −120). +- `activity(env_db, hop_s, threshold_db, min_on_s, bridge_s)`: regions above the threshold; gaps + shorter than `bridge_s` are merged first, then regions shorter than `min_on_s` are dropped. +- `sensitivity_threshold_db(s)`: Sensitivity 0–10 → −20 − 4·s dBFS. +- `duck_keyframes(regions, clip_start, clip_end, base_db, reduce_db, fade_s)`: (time, dB) volume + keyframes — ramp down over `fade_s` before each region, hold, ramp up after; overlapping ramps + merge; clamped to the clip (a ramp cut by an edge gets the ramp's value at the edge); strictly + increasing times. ## Known limits diff --git a/crates/audio-dsp/src/ducking.rs b/crates/audio-dsp/src/ducking.rs new file mode 100644 index 00000000..1ae11f56 --- /dev/null +++ b/crates/audio-dsp/src/ducking.rs @@ -0,0 +1,229 @@ +//! Offline analysis for Essential Sound auto-ducking: level envelope of the trigger audio +//! (dialogue, SFX…), activity detection with blip removal and gap bridging, and the volume +//! keyframes that duck a music clip under the detected regions. +//! +//! Pure functions on plain slices; times are seconds, levels dBFS / dB. + +/// Level floor (dBFS) for silence. +pub const FLOOR_DB: f32 = -120.0; + +/// RMS level (dBFS) per hop: value `k` is the RMS of the window of `window_s` seconds centred on +/// `k · hop_s` (clipped at the signal edges), with the power averaged over channels. The result +/// has `ceil(len / hop)` entries; silence reads [`FLOOR_DB`]. +pub fn envelope_db(channels: &[&[f32]], sample_rate: f32, hop_s: f32, window_s: f32) -> Vec { + let len = channels.iter().map(|c| c.len()).min().unwrap_or(0); + if len == 0 || channels.is_empty() { + return Vec::new(); + } + let hop = ((hop_s * sample_rate).round() as usize).max(1); + let half = ((window_s * sample_rate / 2.0).round() as usize).max(1); + // prefix sums of the channel-averaged power + let nch = channels.len() as f64; + let mut pre = Vec::with_capacity(len + 1); + pre.push(0.0f64); + let mut acc = 0.0f64; + for i in 0..len { + let p: f64 = channels.iter().map(|c| (c[i] as f64) * (c[i] as f64)).sum::() / nch; + acc += p; + pre.push(acc); + } + (0..len.div_ceil(hop)) + .map(|k| { + let c = k * hop; + let (a, b) = (c.saturating_sub(half), (c + half).min(len)); + let ms = (pre[b] - pre[a]).max(0.0) / (b - a).max(1) as f64; + if ms <= 1e-12 { FLOOR_DB } else { ((10.0 * ms.log10()) as f32).max(FLOOR_DB) } + }) + .collect() +} + +/// Regions `(start_s, end_s)` where the envelope exceeds `threshold_db`. Gaps shorter than +/// `bridge_s` are merged first, then regions shorter than `min_on_s` are dropped. A frame `k` +/// covers `[k·hop, (k+1)·hop)`. +pub fn activity(env_db: &[f32], hop_s: f32, threshold_db: f32, min_on_s: f32, bridge_s: f32) -> Vec<(f64, f64)> { + // the f32 hop rounded to whole microseconds, so 0.01 means exactly 0.01 s + let hop = (hop_s as f64 * 1e6).round() / 1e6; + let mut raw: Vec<(f64, f64)> = Vec::new(); + let mut start: Option = None; + for (k, &v) in env_db.iter().enumerate() { + match (v > threshold_db, start) { + (true, None) => start = Some(k), + (false, Some(s)) => { + raw.push((s as f64 * hop, k as f64 * hop)); + start = None; + } + _ => {} + } + } + if let Some(s) = start { + raw.push((s as f64 * hop, env_db.len() as f64 * hop)); + } + let mut merged: Vec<(f64, f64)> = Vec::new(); + for r in raw { + match merged.last_mut() { + Some(last) if r.0 - last.1 < bridge_s as f64 - 1e-9 => last.1 = r.1, + _ => merged.push(r), + } + } + merged.retain(|(a, b)| b - a >= min_on_s as f64 - 1e-9); + merged +} + +/// Essential Sound ducking "Sensitivity" (0…10) → detection threshold in dBFS: +/// `−20 − 4·s` (5 → −40 dBFS; higher sensitivity reacts to quieter triggers). +pub fn sensitivity_threshold_db(sensitivity: f32) -> f32 { + -20.0 - 4.0 * sensitivity.clamp(0.0, 10.0) +} + +/// Volume keyframes `(time_s, level_db)` that duck by `reduce_db` under each region: ramp from +/// `base_db` at `a − fade` down to `base_db − reduce_db` at `a`, hold until `b`, ramp back up to +/// `base_db` at `b + fade`. Regions whose ramps would overlap are merged (the level stays +/// ducked between them). Everything is clamped to `[clip_start_s, clip_end_s]`; a ramp cut by a +/// clip edge gets a keyframe at the edge with the ramp's value there (fully ducked when the region +/// itself reaches the edge). Times are strictly increasing; no regions → no keyframes. `fade_s` is +/// at least 1 ms so a ramp never collapses into two keyframes at the same time. +pub fn duck_keyframes(regions: &[(f64, f64)], clip_start_s: f64, clip_end_s: f64, base_db: f64, reduce_db: f64, fade_s: f64) -> Vec<(f64, f64)> { + let fade = fade_s.max(1e-3); + let ducked = base_db - reduce_db.abs(); + let mut sorted: Vec<(f64, f64)> = regions.iter().copied().filter(|(a, b)| b > a && *b > clip_start_s && *a < clip_end_s).collect(); + sorted.sort_by(|x, y| x.0.total_cmp(&y.0)); + // merge regions whose ramps touch or overlap + let mut groups: Vec<(f64, f64)> = Vec::new(); + for r in sorted { + match groups.last_mut() { + Some(g) if r.0 - fade <= g.1 + fade => g.1 = g.1.max(r.1), + _ => groups.push(r), + } + } + let mut out: Vec<(f64, f64)> = Vec::new(); + let mut push = |t: f64, v: f64| { + if t < clip_start_s || t > clip_end_s { + return; + } + match out.last_mut() { + Some(last) if t <= last.0 + 1e-9 => { + last.1 = v; + } + _ => out.push((t, v)), + } + }; + // value of the down-ramp ending at `a` / up-ramp starting at `b`, at time t + let down = |a: f64, t: f64| base_db + (ducked - base_db) * ((t - (a - fade)) / fade).clamp(0.0, 1.0); + let up = |b: f64, t: f64| ducked + (base_db - ducked) * ((t - b) / fade).clamp(0.0, 1.0); + for (a, b) in groups { + let ramp_start = a - fade; + if ramp_start >= clip_start_s { + push(ramp_start, base_db); + } else { + push(clip_start_s, down(a, clip_start_s)); + } + push(a.max(clip_start_s), ducked); + if b < clip_end_s { + push(b, ducked); + if b + fade <= clip_end_s { + push(b + fade, base_db); + } else { + push(clip_end_s, up(b, clip_end_s)); + } + } else { + push(clip_end_s, ducked); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn envelope_levels() { + let sr = 1000.0; + // 1 s silence, 1 s of a ±0.1 square wave (−20 dBFS RMS) + let x: Vec = (0..2000) + .map(|i| { + if i < 1000 { + 0.0 + } else if i % 2 == 0 { + 0.1 + } else { + -0.1 + } + }) + .collect(); + let e = envelope_db(&[&x, &x], sr, 0.01, 0.02); + assert_eq!(e.len(), 200); + assert_eq!(e[10], FLOOR_DB); + assert!((e[150] + 20.0).abs() < 0.01, "{}", e[150]); + // the window straddling the edge is in between + assert!(e[100] < -20.0 && e[100] > -30.0); + assert!(envelope_db(&[], sr, 0.01, 0.02).is_empty()); + } + + #[test] + fn activity_bridges_then_drops_blips() { + let hop = 0.01; + let mut env = vec![-80.0f32; 300]; + // speech 0.50–1.00 and 1.10–1.50 (gap 0.1 s → bridged), blip 2.00–2.03 (dropped) + for (a, b) in [(50, 100), (110, 150), (200, 203)] { + env[a..b].iter_mut().for_each(|v| *v = -20.0); + } + let r = activity(&env, hop, -40.0, 0.1, 0.2); + assert_eq!(r.len(), 1); + assert!((r[0].0 - 0.5).abs() < 1e-6 && (r[0].1 - 1.5).abs() < 1e-6, "{r:?}"); + // without bridging both remain, the blip still goes + let r = activity(&env, hop, -40.0, 0.1, 0.0); + assert_eq!(r.len(), 2); + // a region running to the end is closed at the end + let mut env2 = vec![-80.0f32; 10]; + env2[5..].iter_mut().for_each(|v| *v = 0.0); + assert_eq!(activity(&env2, 0.1, -40.0, 0.0, 0.0), vec![(0.5, 1.0)]); + } + + #[test] + fn sensitivity_mapping() { + assert_eq!(sensitivity_threshold_db(5.0), -40.0); + assert_eq!(sensitivity_threshold_db(0.0), -20.0); + assert_eq!(sensitivity_threshold_db(10.0), -60.0); + assert_eq!(sensitivity_threshold_db(99.0), -60.0); + } + + fn strictly_increasing(k: &[(f64, f64)]) -> bool { + k.windows(2).all(|w| w[1].0 > w[0].0) + } + + #[test] + fn duck_keyframes_basic_and_merge() { + let k = duck_keyframes(&[(2.0, 4.0)], 0.0, 10.0, 0.0, 15.0, 0.5); + assert_eq!(k, vec![(1.5, 0.0), (2.0, -15.0), (4.0, -15.0), (4.5, 0.0)]); + // ramps overlap (4.5 > 4.8 − 0.5 = 4.3) → stay ducked + let k = duck_keyframes(&[(2.0, 4.0), (4.8, 6.0)], 0.0, 10.0, -3.0, 12.0, 0.5); + assert_eq!(k, vec![(1.5, -3.0), (2.0, -15.0), (6.0, -15.0), (6.5, -3.0)]); + // separate regions + let k = duck_keyframes(&[(5.0, 6.0), (1.0, 2.0)], 0.0, 10.0, 0.0, 10.0, 0.25); + assert_eq!(k.len(), 8); + assert!(strictly_increasing(&k)); + assert!(duck_keyframes(&[], 0.0, 10.0, 0.0, 10.0, 0.5).is_empty()); + } + + #[test] + fn duck_keyframes_clamped_to_clip() { + // region starts before the clip: fully ducked at the start edge + let k = duck_keyframes(&[(-1.0, 2.0)], 0.0, 10.0, 0.0, 10.0, 0.5); + assert_eq!(k, vec![(0.0, -10.0), (2.0, -10.0), (2.5, 0.0)]); + // ramp partially before the clip: ramp value at the edge + let k = duck_keyframes(&[(0.25, 2.0)], 0.0, 10.0, 0.0, 10.0, 0.5); + assert_eq!(k[0], (0.0, -5.0)); + assert_eq!(k[1], (0.25, -10.0)); + // region runs past the end: ducked at the end edge + let k = duck_keyframes(&[(8.0, 12.0)], 0.0, 10.0, 0.0, 10.0, 0.5); + assert_eq!(k, vec![(7.5, 0.0), (8.0, -10.0), (10.0, -10.0)]); + // up-ramp cut by the end + let k = duck_keyframes(&[(8.0, 9.75)], 0.0, 10.0, 0.0, 10.0, 0.5); + assert_eq!(*k.last().unwrap(), (10.0, -5.0)); + // outside the clip: nothing; zero fade still strictly increasing + assert!(duck_keyframes(&[(11.0, 12.0)], 0.0, 10.0, 0.0, 10.0, 0.5).is_empty()); + let k = duck_keyframes(&[(2.0, 3.0)], 0.0, 10.0, 0.0, 10.0, 0.0); + assert!(strictly_increasing(&k) && k.len() == 4); + } +} diff --git a/crates/audio-dsp/src/effects/essential.rs b/crates/audio-dsp/src/effects/essential.rs new file mode 100644 index 00000000..4c46902c --- /dev/null +++ b/crates/audio-dsp/src/effects/essential.rs @@ -0,0 +1,764 @@ +//! Essential Sound processors: DeEsser (split-band), DeReverb (spectral late-reverberation +//! suppression), Enhance Speech (a DSP voice-enhancement chain) and Stereo Width (mid/side). +//! +//! All written from textbook descriptions: +//! * de-esser: complementary split (band = 2nd-order high-pass, rest = x − band) with a relative +//! band-vs-broadband detector driving the band gain; +//! * de-reverb: the statistical late-reverberation model (exponentially decaying diffuse tail, +//! Polack): the late-reverb PSD at frame t is predicted from the reverberant PSD T_d earlier, +//! λ_r(t) = e^{−2Δ·T_d}·λ_x(t − T_d), Δ = 3·ln10 / RT60, and removed by spectral subtraction +//! with over-subtraction and a gain floor (Lebart et al. 2001; Habets 2007); +//! * enhance speech: high-pass, de-mud cut, presence and air boosts (RBJ biquads), then one gain +//! computer combining a downward expander (noise between words) and a soft-knee compressor. + +use super::stft::{Spectral, StftChannel, fft_size}; +use crate::biquad::{Biquad, Coeffs, FilterType}; +use crate::{AudioEffect, ParamSpec, ParamValues, Smoothed, Unit, block_len, db_to_gain, flush, param_plumbing}; + +/// One-pole smoothing coefficient for a time constant in ms. +fn coef(ms: f64, sr: f64) -> f64 { + if ms <= 0.0 { 0.0 } else { (-1.0 / (ms * 0.001 * sr)).exp() } +} + +#[inline] +fn pow_db(p: f64) -> f64 { + 10.0 * (p + 1e-20).log10() +} + +// --------------------------------------------------------------------------------------------- + +/// De-esser: a 2nd-order high-pass sidechain at `frequency` detects sibilance (the band level +/// exceeds the broadband level by more than `threshold`); the band above `frequency` is then cut +/// by a dynamic RBJ high shelf whose gain follows the gain reduction (attack ~1 ms, release +/// ~60 ms, stereo-linked). At 0 dB reduction the shelf is an exact identity, so non-sibilant +/// passages pass unchanged. (A subtractive split `x − HP(x)` is not used: the high-pass phase +/// shift around the crossover makes `x − (1 − g)·HP(x)` boost there instead of cutting.) +pub struct DeEsser { + pv: ParamValues, + sr: f64, + freq: Smoothed, + coeffs: Coeffs, + hp: Vec, + shelf: Vec, + shelf_coeffs: Coeffs, + shelf_gr: f64, + env_band: f64, + env_full: f64, + /// Smoothed gain reduction (dB, ≤ 0). + gr: f64, + det: f64, + att: f64, + rel: f64, + threshold: f64, + reduction: f64, + gr_meter: f32, +} + +impl DeEsser { + pub const PARAMS: &'static [ParamSpec] = &[ + ParamSpec::log("frequency", "Frequency", 2000.0, 12000.0, 6000.0, Unit::Hertz), + ParamSpec::new("threshold", "Threshold", -40.0, 0.0, -12.0, Unit::Decibels), + ParamSpec::new("reduction", "Max Reduction", 0.0, 24.0, 8.0, Unit::Decibels), + ]; + + pub fn new(sample_rate: f32, channels: usize) -> Self { + let sr = sample_rate as f64; + let mut s = DeEsser { + pv: ParamValues::new(Self::PARAMS), + sr, + freq: Smoothed::with_ms(6000.0, sample_rate, 20.0), + coeffs: Coeffs::IDENTITY, + hp: vec![Biquad::new(); channels.max(1)], + shelf: vec![Biquad::new(); channels.max(1)], + shelf_coeffs: Coeffs::IDENTITY, + shelf_gr: 0.0, + env_band: 0.0, + env_full: 0.0, + gr: 0.0, + det: coef(2.0, sr), + att: coef(1.0, sr), + rel: coef(60.0, sr), + threshold: -12.0, + reduction: 8.0, + gr_meter: 0.0, + }; + s.apply_params(true); + s + } + + fn design(&mut self) { + self.coeffs = Coeffs::design(FilterType::HighPass, self.sr, self.freq.value() as f64, std::f64::consts::FRAC_1_SQRT_2, 0.0); + self.design_shelf(); + } + + fn design_shelf(&mut self) { + self.shelf_coeffs = if self.gr == 0.0 { + Coeffs::IDENTITY + } else { + Coeffs::design(FilterType::HighShelf, self.sr, self.freq.value() as f64, std::f64::consts::FRAC_1_SQRT_2, self.gr) + }; + self.shelf_gr = self.gr; + } + + fn apply_params(&mut self, snap: bool) { + self.freq.set(self.pv.v("frequency")); + self.threshold = self.pv.v("threshold") as f64; + self.reduction = self.pv.v("reduction") as f64; + if snap { + self.freq.snap(); + self.design(); + } + } + + /// Current band gain reduction in dB (≤ 0), for metering. + pub fn gain_reduction_db(&self) -> f32 { + self.gr_meter + } +} + +impl AudioEffect for DeEsser { + param_plumbing!("deesser"); + fn reset(&mut self) { + self.freq.snap(); + self.design(); + self.hp.iter_mut().for_each(Biquad::reset); + self.shelf.iter_mut().for_each(Biquad::reset); + self.env_band = 0.0; + self.env_full = 0.0; + self.gr = 0.0; + self.gr_meter = 0.0; + self.design_shelf(); + } + fn process(&mut self, channels: &mut [&mut [f32]]) { + let n = block_len(channels); + let nch = channels.len().min(self.hp.len()); + for i in 0..n { + if self.freq.is_smoothing() { + self.freq.tick(); + self.design(); + } + let (mut pb, mut pf) = (0.0f64, 0.0f64); + for ch in 0..nch { + let x = channels[ch][i] as f64; + let b = self.hp[ch].tick(&self.coeffs, x); + pb += b * b; + pf += x * x; + } + self.env_band = flush(self.det * self.env_band + (1.0 - self.det) * pb); + self.env_full = flush(self.det * self.env_full + (1.0 - self.det) * pf); + let bdb = pow_db(self.env_band); + let excess = bdb - pow_db(self.env_full) - self.threshold; + let target = if bdb > -70.0 && excess > 0.0 { -(excess * 2.0).min(self.reduction) } else { 0.0 }; + let c = if target < self.gr { self.att } else { self.rel }; + self.gr = flush(c * self.gr + (1.0 - c) * target); + if target == 0.0 && self.gr > -1e-4 { + self.gr = 0.0; + } + if (self.gr - self.shelf_gr).abs() > 0.05 || (self.gr == 0.0) != (self.shelf_gr == 0.0) { + self.design_shelf(); + } + for ch in 0..nch { + let x = channels[ch][i] as f64; + channels[ch][i] = self.shelf[ch].tick(&self.shelf_coeffs, x) as f32; + } + } + self.gr_meter = self.gr as f32; + } +} + +// --------------------------------------------------------------------------------------------- + +/// Late-reverberation delay T_d (s): reflections later than this count as reverb. +const DEREVERB_TD: f64 = 0.05; +/// Time constant (s) of the PSD smoothing. +const DEREVERB_PSD_TAU: f64 = 0.02; + +#[derive(Clone, Debug)] +struct ReverbState { + psd: Vec, + /// Ring of past smoothed PSDs, `delay_frames` deep. + hist: Vec, + pos: usize, + gains: Vec, + frames: usize, +} + +impl ReverbState { + fn new(bins: usize, delay: usize) -> Self { + ReverbState { psd: vec![0.0; bins], hist: vec![0.0; bins * delay], pos: 0, gains: vec![1.0; bins], frames: 0 } + } + fn reset(&mut self) { + self.psd.iter_mut().for_each(|v| *v = 0.0); + self.hist.iter_mut().for_each(|v| *v = 0.0); + self.gains.iter_mut().for_each(|v| *v = 1.0); + self.pos = 0; + self.frames = 0; + } +} + +/// Spectral de-reverberation: suppresses the late reverberant tail predicted by the exponential +/// decay model for the assumed `rt60`. `amount` sets the over-subtraction and the gain floor +/// (0 % = identity apart from the STFT delay, 100 % ≈ −18 dB floor). +pub struct DeReverb { + pv: ParamValues, + sr: f32, + n: usize, + hop: usize, + delay: usize, + spec: Spectral, + stft: Vec, + state: Vec, + smoothed_gains: Vec, + /// e^{−2Δ·T_d} (power decay over the delay). + decay: f32, + beta: f32, + floor: f32, + alpha: f32, +} + +impl DeReverb { + pub const PARAMS: &'static [ParamSpec] = + &[ParamSpec::new("amount", "Amount", 0.0, 100.0, 50.0, Unit::Percent), ParamSpec::log("rt60", "Decay (RT60)", 0.1, 5.0, 0.8, Unit::Seconds)]; + + pub fn new(sample_rate: f32, channels: usize) -> Self { + let n = fft_size(sample_rate); + let hop = n / 4; + let bins = n / 2 + 1; + let delay = ((DEREVERB_TD * sample_rate as f64 / hop as f64).round() as usize).max(1); + let ch = channels.max(1); + let mut s = DeReverb { + pv: ParamValues::new(Self::PARAMS), + sr: sample_rate, + n, + hop, + delay, + spec: Spectral::new(n, true), + stft: vec![StftChannel::new(n); ch], + state: vec![ReverbState::new(bins, delay); ch], + smoothed_gains: vec![1.0; bins], + decay: 1.0, + beta: 1.0, + floor: 1.0, + alpha: (-(hop as f64) / (sample_rate as f64 * DEREVERB_PSD_TAU)).exp() as f32, + }; + s.apply_params(true); + s + } + + fn apply_params(&mut self, _snap: bool) { + let amount = self.pv.v("amount") / 100.0; + let rt60 = self.pv.v("rt60") as f64; + let delta = 3.0 * std::f64::consts::LN_10 / rt60; + let td = self.delay as f64 * self.hop as f64 / self.sr as f64; + self.decay = (-2.0 * delta * td).exp() as f32; + self.beta = 1.0 + amount; + self.floor = db_to_gain(-18.0 * amount); + } +} + +impl AudioEffect for DeReverb { + param_plumbing!("dereverb"); + fn latency(&self) -> usize { + self.n + } + fn reset(&mut self) { + self.stft.iter_mut().for_each(StftChannel::reset); + self.state.iter_mut().for_each(ReverbState::reset); + } + fn process(&mut self, channels: &mut [&mut [f32]]) { + let len = block_len(channels); + let nch = channels.len().min(self.stft.len()); + let (n, hop, delay) = (self.n, self.hop, self.delay); + let (decay, beta, floor, alpha) = (self.decay, self.beta, self.floor, self.alpha); + let scale = 1.0 / (n as f32 * 2.0); + let bins = n / 2 + 1; + for ch in 0..nch { + let st = &mut self.state[ch]; + let spec = &mut self.spec; + let sg = &mut self.smoothed_gains; + let mut frame = |input: &[f32], out: &mut [f32]| { + spec.analyse(input); + let warm = st.frames >= delay; + let base = st.pos * bins; + for k in 0..bins { + let p = spec.re[k] * spec.re[k] + spec.im[k] * spec.im[k]; + let psd = alpha * st.psd[k] + (1.0 - alpha) * p; + st.psd[k] = if psd < 1e-30 { 0.0 } else { psd }; + // Late reverb predicted from the PSD `delay` frames ago (the ring slot we are + // about to overwrite holds exactly that frame). + let late = if warm { decay * st.hist[base + k] } else { 0.0 }; + st.hist[base + k] = st.psd[k]; + let target = if st.psd[k] > 0.0 { (1.0 - beta * late / st.psd[k]).max(floor) } else { 1.0 }; + let g = &mut st.gains[k]; + // fast recovery on onsets, slower suppression (less musical noise) + let c = if target > *g { 0.8 } else { 0.5 }; + *g += (target - *g) * c; + } + st.pos = (st.pos + 1) % delay; + st.frames = st.frames.saturating_add(1); + for k in 0..bins { + let a = st.gains[k.saturating_sub(1)]; + let c = st.gains[(k + 1).min(bins - 1)]; + sg[k] = 0.25 * a + 0.5 * st.gains[k] + 0.25 * c; + } + for k in 0..bins { + spec.re[k] *= sg[k]; + spec.im[k] *= sg[k]; + } + spec.synthesise_add(out, scale); + }; + for v in channels[ch][..len].iter_mut() { + *v = self.stft[ch].tick(*v, hop, &mut frame); + } + } + } +} + +// --------------------------------------------------------------------------------------------- + +const ENH_STAGES: usize = 4; +/// Expander: below this level (dBFS) the signal is pushed down (noise between words). +const ENH_EXP_THRESHOLD: f64 = -50.0; +const ENH_EXP_RATIO: f64 = 3.0; +const ENH_EXP_RANGE: f64 = 18.0; +/// Compressor above this level (dBFS). +const ENH_COMP_THRESHOLD: f64 = -24.0; +const ENH_COMP_RATIO: f64 = 3.0; +const ENH_COMP_KNEE: f64 = 6.0; +/// Make-up gain: puts conversational speech (≈ −20 dBFS RMS) back near its input loudness. +const ENH_MAKEUP: f64 = 3.0; + +/// Enhance Speech: a DSP voice-enhancement chain (no machine learning) — 80 Hz high-pass, +/// de-mud cut, presence and air boosts, then a combined downward expander + soft-knee +/// compressor with make-up gain. `tone` picks the cut/boost frequencies for lower or higher +/// voices; `mix` blends with the dry signal. +pub struct SpeechEnhance { + pv: ParamValues, + sr: f64, + coeffs: [Coeffs; ENH_STAGES], + filters: Vec<[Biquad; ENH_STAGES]>, + mix: Smoothed, + /// Detector mean square. + ms: f64, + /// Smoothed expander gain (dB, ≤ 0): opens fast, closes slowly. + env_exp: f64, + /// Smoothed compressor gain (dB, ≤ 0): fast attack, slow release. + env_comp: f64, + det: f64, + fast: f64, + exp_close: f64, + comp_rel: f64, +} + +impl SpeechEnhance { + pub const PARAMS: &'static [ParamSpec] = + &[ParamSpec::new("mix", "Mix", 0.0, 100.0, 100.0, Unit::Percent), ParamSpec::choice("tone", "Tone", &["Low Tone", "High Tone"], 0)]; + + pub fn new(sample_rate: f32, channels: usize) -> Self { + let sr = sample_rate as f64; + let mut s = SpeechEnhance { + pv: ParamValues::new(Self::PARAMS), + sr, + coeffs: [Coeffs::IDENTITY; ENH_STAGES], + filters: vec![[Biquad::new(); ENH_STAGES]; channels.max(1)], + mix: Smoothed::with_ms(1.0, sample_rate, 20.0), + ms: 0.0, + env_exp: 0.0, + env_comp: 0.0, + det: coef(5.0, sr), + fast: coef(3.0, sr), + exp_close: coef(30.0, sr), + comp_rel: coef(120.0, sr), + }; + s.apply_params(true); + s + } + + fn apply_params(&mut self, snap: bool) { + let high = self.pv.idx("tone") == 1; + let (mud, presence) = if high { (350.0, 4000.0) } else { (250.0, 2500.0) }; + let sr = self.sr; + self.coeffs = [ + Coeffs::design(FilterType::HighPass, sr, 80.0, std::f64::consts::FRAC_1_SQRT_2, 0.0), + Coeffs::design(FilterType::Peaking, sr, mud, 1.0, -3.0), + Coeffs::design(FilterType::Peaking, sr, presence, 1.0, 4.0), + Coeffs::design(FilterType::HighShelf, sr, 10000.0, std::f64::consts::FRAC_1_SQRT_2, 2.0), + ]; + self.mix.set(self.pv.v("mix") / 100.0); + if snap { + self.mix.snap(); + } + } + + /// Static downward-expander gain (dB) for a detector level (dBFS). + fn expander_db(level: f64) -> f64 { + if level < ENH_EXP_THRESHOLD { ((level - ENH_EXP_THRESHOLD) * (ENH_EXP_RATIO - 1.0)).max(-ENH_EXP_RANGE) } else { 0.0 } + } + + /// Static soft-knee compressor gain (dB) for a detector level (dBFS). + fn compressor_db(level: f64) -> f64 { + let d = level - ENH_COMP_THRESHOLD; + if 2.0 * d.abs() <= ENH_COMP_KNEE { + (1.0 / ENH_COMP_RATIO - 1.0) * (d + ENH_COMP_KNEE / 2.0).powi(2) / (2.0 * ENH_COMP_KNEE) + } else if d > 0.0 { + d / ENH_COMP_RATIO - d + } else { + 0.0 + } + } +} + +impl AudioEffect for SpeechEnhance { + param_plumbing!("speech_enhance"); + fn reset(&mut self) { + self.filters.iter_mut().for_each(|f| f.iter_mut().for_each(Biquad::reset)); + self.mix.snap(); + self.ms = 0.0; + self.env_exp = 0.0; + self.env_comp = 0.0; + } + fn process(&mut self, channels: &mut [&mut [f32]]) { + let n = block_len(channels); + let nch = channels.len().min(self.filters.len()).min(8); + let mut wet = [0.0f64; 8]; + for i in 0..n { + let mut sq = 0.0f64; + for ch in 0..nch { + let mut x = channels[ch][i] as f64; + for (st, c) in self.filters[ch].iter_mut().zip(self.coeffs.iter()) { + x = st.tick(c, x); + } + wet[ch] = x; + sq = sq.max(x * x); + } + self.ms = flush(self.det * self.ms + (1.0 - self.det) * sq); + let level = pow_db(self.ms); + let te = Self::expander_db(level); + let c = if te > self.env_exp { self.fast } else { self.exp_close }; + self.env_exp = c * self.env_exp + (1.0 - c) * te; + let tc = Self::compressor_db(level); + let c = if tc < self.env_comp { self.fast } else { self.comp_rel }; + self.env_comp = c * self.env_comp + (1.0 - c) * tc; + if self.env_exp.abs() < 1e-9 { + self.env_exp = 0.0; + } + if self.env_comp.abs() < 1e-9 { + self.env_comp = 0.0; + } + let g = 10f64.powf((self.env_exp + self.env_comp + ENH_MAKEUP) / 20.0); + let m = self.mix.tick() as f64; + for (ch, w) in wet.iter().enumerate().take(nch) { + let dry = channels[ch][i] as f64; + channels[ch][i] = (dry + m * (w * g - dry)) as f32; + } + } + } +} + +// --------------------------------------------------------------------------------------------- + +/// Stereo width by mid/side scaling: 0 % = mono, 100 % = unchanged, 200 % = side doubled. +/// Only the first two channels are processed; mono input passes through. +pub struct StereoWidth { + pv: ParamValues, + width: Smoothed, +} + +impl StereoWidth { + pub const PARAMS: &'static [ParamSpec] = &[ParamSpec::new("width", "Width", 0.0, 200.0, 100.0, Unit::Percent)]; + + pub fn new(sample_rate: f32, _channels: usize) -> Self { + let mut s = StereoWidth { pv: ParamValues::new(Self::PARAMS), width: Smoothed::with_ms(1.0, sample_rate, 20.0) }; + s.apply_params(true); + s + } + fn apply_params(&mut self, snap: bool) { + self.width.set(self.pv.v("width") / 100.0); + if snap { + self.width.snap(); + } + } +} + +impl AudioEffect for StereoWidth { + param_plumbing!("stereo_width"); + fn reset(&mut self) { + self.width.snap(); + } + fn process(&mut self, channels: &mut [&mut [f32]]) { + let n = block_len(channels); + if channels.len() < 2 { + // keep the smoother in step with time + for _ in 0..n { + self.width.tick(); + } + return; + } + let (l, rest) = channels.split_at_mut(1); + let (l, r) = (&mut l[0], &mut rest[0]); + for i in 0..n { + let w = self.width.tick(); + let m = 0.5 * (l[i] + r[i]); + let s = 0.5 * (l[i] - r[i]) * w; + l[i] = m + s; + r[i] = m - s; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testutil::*; + + const SR: f32 = 48000.0; + + fn energy(x: &[f32]) -> f64 { + x.iter().map(|&v| (v as f64) * (v as f64)).sum::() + } + + /// Band-limited energy via a 4th-order band-pass (two cascaded RBJ band-passes at the + /// geometric centre) — good enough for relative before/after comparisons. + fn band_energy(x: &[f32], lo: f64, hi: f64) -> f64 { + let fc = (lo * hi).sqrt(); + let q = fc / (hi - lo); + let c = Coeffs::design(FilterType::BandPass, SR as f64, fc, q, 0.0); + let (mut a, mut b) = (Biquad::new(), Biquad::new()); + x.iter().map(|&v| b.tick(&c, a.tick(&c, v as f64))).map(|v| v * v).sum::() + } + + fn harmonic_voice(n: usize, f0: f64, amp: f64) -> Vec { + let mut v = vec![0.0f32; n]; + let mut k = 1; + while f0 * k as f64 <= 1500.0 { + let s = sine(f0 * k as f64, amp / k as f64, SR as f64, n, k as f64 * 0.7); + for (o, x) in v.iter_mut().zip(&s) { + *o += x; + } + k += 1; + } + v + } + + fn bandpassed_noise(seed: u64, n: usize, lo: f64, hi: f64, amp: f32) -> Vec { + let mut r = Rng::new(seed); + let fc = (lo * hi).sqrt(); + let c = Coeffs::design(FilterType::BandPass, SR as f64, fc, fc / (hi - lo), 0.0); + let (mut a, mut b) = (Biquad::new(), Biquad::new()); + (0..n).map(|_| (b.tick(&c, a.tick(&c, (r.uniform() * amp) as f64)) * 2.0) as f32).collect() + } + + #[test] + fn deesser_zero_reduction_is_identity() { + let mut d = DeEsser::new(SR, 1); + d.set_param("reduction", 0.0); + d.reset(); + let mut r = Rng::new(3); + let x: Vec = (0..20000).map(|_| r.uniform() * 0.5).collect(); + let mut y = x.clone(); + d.process(&mut [&mut y]); + for (a, b) in x.iter().zip(&y) { + assert!((a - b).abs() < 1e-6); + } + } + + #[test] + fn deesser_reduces_sibilance_keeps_voice() { + let n = SR as usize * 4; + let voice = harmonic_voice(n, 150.0, 0.15); + let sib = bandpassed_noise(11, n, 5000.0, 10000.0, 0.6); + // sibilant bursts: 150 ms every 500 ms (from 250 ms) + let burst = |i: usize| { + let p = i % (SR as usize / 2); + p >= SR as usize / 4 && p < SR as usize / 4 + (0.15 * SR) as usize + }; + let x: Vec = (0..n).map(|i| voice[i] + if burst(i) { sib[i] } else { 0.0 }).collect(); + for (reduction, need) in [(8.0f32, 5.0f64), (16.0, 6.0)] { + let mut d = DeEsser::new(SR, 1); + d.set_param("reduction", reduction); + d.reset(); + let mut y = x.clone(); + d.process(&mut [&mut y]); + let pick = |v: &[f32], on: bool| -> Vec { v.iter().enumerate().skip(SR as usize).filter(|(i, _)| burst(*i) == on).map(|(_, s)| *s).collect() }; + let (xb, yb) = (pick(&x, true), pick(&y, true)); + let sib_drop = 10.0 * (band_energy(&yb, 5000.0, 10000.0) / band_energy(&xb, 5000.0, 10000.0)).log10(); + let low_change = 10.0 * (band_energy(&y, 150.0, 1500.0) / band_energy(&x, 150.0, 1500.0)).log10(); + let (xq, yq) = (pick(&x, false), pick(&y, false)); + let quiet_change = 10.0 * (energy(&yq) / energy(&xq)).log10(); + println!("deesser reduction {reduction} dB: sibilance {sib_drop:.2} dB, voice band {low_change:.3} dB, non-sibilant passages {quiet_change:.3} dB"); + assert!(sib_drop < -need, "sibilance only {sib_drop} dB"); + assert!(low_change.abs() < 1.0, "voice band changed {low_change} dB"); + assert!(quiet_change.abs() < 0.5, "non-sibilant passages changed {quiet_change} dB"); + } + } + + #[test] + fn dereverb_suppresses_tail() { + let sr = SR as f64; + let n = SR as usize * 6; + let period = (0.5 * sr) as usize; + let on = (0.1 * sr) as usize; + let dry_src = harmonic_voice(n, 180.0, 0.25); + let dry: Vec = (0..n).map(|i| if i % period < on { dry_src[i] } else { 0.0 }).collect(); + // Statistical late reverb (Polack): white noise shaped by the energy envelope of the dry + // signal convolved with an exponential decay — a one-pole filter on the energy. + let rt60 = 0.8; + let a = (-6.0 * std::f64::consts::LN_10 / (rt60 * sr)).exp(); // power decay per sample + let mut e = 0.0f64; + let mut rng = Rng::new(77); + let wet_gain = 1.0 - a; // tail energy ≈ direct energy (DRR ≈ 0 dB) + let x: Vec = dry + .iter() + .map(|&d| { + e = a * e + wet_gain * (d as f64) * (d as f64); + d + (rng.uniform() as f64 * 3f64.sqrt() * e.sqrt()) as f32 + }) + .collect(); + let mut fx = DeReverb::new(SR, 1); + fx.set_param("amount", 100.0); + fx.reset(); + let lat = fx.latency(); + let mut y = x.clone(); + fx.process(&mut [&mut y]); + let y = &y[lat..]; + let start = SR as usize; // after warm-up + let split = |v: &[f32]| { + let (mut b, mut g) = (0.0, 0.0); + for (i, s) in v.iter().enumerate().take(n - lat).skip(start) { + let p = (*s as f64) * (*s as f64); + if i % period < on { b += p } else { g += p } + } + (b, g) + }; + let (xb, xg) = split(&x); + let (yb, yg) = split(y); + let before = 10.0 * (xg / xb).log10(); + let after = 10.0 * (yg / yb).log10(); + let burst_loss = 10.0 * (yb / xb).log10(); + println!("dereverb: tail/burst {before:.2} dB → {after:.2} dB (improvement {:.2} dB), burst energy {burst_loss:.2} dB", before - after); + assert!(before - after >= 4.0, "tail improvement only {} dB", before - after); + assert!(burst_loss > -3.0, "burst energy lost {burst_loss} dB"); + } + + #[test] + fn dereverb_zero_amount_is_delayed_identity() { + let mut d = DeReverb::new(SR, 1); + d.set_param("amount", 0.0); + let lat = d.latency(); + let mut rng = Rng::new(9); + let x: Vec = (0..20000).map(|_| rng.uniform() * 0.5).collect(); + let mut y = x.clone(); + d.process(&mut [&mut y]); + for i in lat..x.len() { + assert!((y[i] - x[i - lat]).abs() < 1e-4); + } + } + + #[test] + fn speech_enhance_presence_noise_and_level() { + let sr = SR as usize; + let n = sr * 4; + let voice = harmonic_voice(n, 140.0, 0.08); + let fric = bandpassed_noise(5, n, 1500.0, 6000.0, 0.05); + let mut rng = Rng::new(42); + // syllables: 200 ms on, 200 ms off; −60 dBFS-ish background noise throughout + let syl = |i: usize| (i / (sr / 5)).is_multiple_of(2); + let x: Vec = (0..n).map(|i| rng.uniform() * 0.0017 + if syl(i) { voice[i] + fric[i] } else { 0.0 }).collect(); + let mut fx = SpeechEnhance::new(SR, 1); + let mut y = x.clone(); + fx.process(&mut [&mut y]); + let pick = |v: &[f32], on: bool| -> Vec { + v.iter() + .enumerate() + .skip(sr) + .filter(|(i, _)| { + let p = i % (2 * sr / 5); + // skip 50 ms after each edge + syl(*i) == on && p % (sr / 5) > sr / 20 + }) + .map(|(_, s)| *s) + .collect() + }; + let (xs, ys) = (pick(&x, true), pick(&y, true)); + let tilt_in = 10.0 * (band_energy(&xs, 2000.0, 5000.0) / band_energy(&xs, 200.0, 400.0)).log10(); + let tilt_out = 10.0 * (band_energy(&ys, 2000.0, 5000.0) / band_energy(&ys, 200.0, 400.0)).log10(); + let noise = 10.0 * (energy(&pick(&y, false)) / energy(&pick(&x, false))).log10(); + let level = 10.0 * (energy(&y[sr..]) / energy(&x[sr..])).log10(); + println!( + "enhance speech: presence/mud tilt {tilt_in:.2} → {tilt_out:.2} dB (+{:.2}), noise between syllables {noise:.2} dB, level {level:.2} dB", + tilt_out - tilt_in + ); + assert!(tilt_out - tilt_in > 3.0); + assert!(noise <= -3.0, "noise floor only {noise} dB"); + assert!(level.abs() <= 3.0, "level changed {level} dB"); + // mix 0 is identity + let mut fx = SpeechEnhance::new(SR, 1); + fx.set_param("mix", 0.0); + fx.reset(); + let mut z = x.clone(); + fx.process(&mut [&mut z]); + assert!(z.iter().zip(&x).all(|(a, b)| (a - b).abs() < 1e-6)); + } + + #[test] + fn stereo_width_laws() { + let mut rng = Rng::new(1); + let l0: Vec = (0..4000).map(|_| rng.uniform() * 0.5).collect(); + let r0: Vec = (0..4000).map(|_| rng.uniform() * 0.5).collect(); + let run = |w: f32| { + let mut fx = StereoWidth::new(SR, 2); + fx.set_param("width", w); + fx.reset(); + let (mut l, mut r) = (l0.clone(), r0.clone()); + fx.process(&mut [&mut l, &mut r]); + (l, r) + }; + let (l, r) = run(0.0); + assert!(l.iter().zip(&r).all(|(a, b)| (a - b).abs() < 1e-7)); + let (l, r) = run(100.0); + assert!(l.iter().zip(&l0).chain(r.iter().zip(&r0)).all(|(a, b)| (a - b).abs() < 1e-6)); + let (l, r) = run(200.0); + for i in 0..l.len() { + let (m, s) = (0.5 * (l0[i] + r0[i]), 0.5 * (l0[i] - r0[i])); + assert!((0.5 * (l[i] + r[i]) - m).abs() < 1e-6); + assert!((0.5 * (l[i] - r[i]) - 2.0 * s).abs() < 1e-6); + } + // mono passes through + let mut fx = StereoWidth::new(SR, 2); + fx.set_param("width", 0.0); + let mut m = l0.clone(); + fx.process(&mut [&mut m]); + assert_eq!(m, l0); + } + + /// The Essential Sound "Dialogue" chain on 60 s of stereo 48 kHz. + /// `cargo test --release -p filmcraft-audio-dsp dialogue_chain_realtime -- --ignored --nocapture` + #[test] + #[ignore] + fn dialogue_chain_realtime() { + let sr = SR as usize; + let n = sr * 60; + let mut rng = Rng::new(3); + let voice = harmonic_voice(n, 140.0, 0.1); + let mut l: Vec = (0..n).map(|i| voice[i] + rng.uniform() * 0.01).collect(); + let mut r: Vec = l.iter().map(|v| v * 0.9 + rng.uniform() * 0.01).collect(); + let mut chain: Vec> = + ["denoise", "parametric_eq", "dehum", "deesser", "dereverb", "compressor", "simple_eq", "speech_enhance", "reverb"] + .iter() + .map(|id| crate::create_effect(id, SR, 2).unwrap()) + .collect(); + chain[1].set_param("b1.on", 1.0); + chain[1].set_param("b2.on", 1.0); + chain[1].set_param("b2.type", 4.0); + chain[1].set_param("b2.freq", 80.0); + let t0 = std::time::Instant::now(); + for (lb, rb) in l.chunks_mut(1024).zip(r.chunks_mut(1024)) { + for fx in chain.iter_mut() { + fx.process(&mut [&mut *lb, &mut *rb]); + } + } + let secs = t0.elapsed().as_secs_f64(); + println!("dialogue chain: 60 s stereo in {secs:.3} s = {:.1}× realtime (one core)", 60.0 / secs); + assert!(l.iter().chain(&r).all(|v| v.is_finite())); + } +} diff --git a/crates/audio-dsp/src/effects/mod.rs b/crates/audio-dsp/src/effects/mod.rs index 1be4f57b..fd11ffc9 100644 --- a/crates/audio-dsp/src/effects/mod.rs +++ b/crates/audio-dsp/src/effects/mod.rs @@ -3,6 +3,7 @@ pub mod channel; pub mod dynamics; pub mod eq; +pub mod essential; pub mod pitch; pub mod restoration; mod stft; @@ -11,6 +12,7 @@ pub mod time; pub use channel::{Amplify, Balance, ChannelVolume, Invert, Route, Routing}; pub use dynamics::{Compressor, Gate, Limiter}; pub use eq::{ParametricEq, SimpleEq}; +pub use essential::{DeEsser, DeReverb, SpeechEnhance, StereoWidth}; pub use pitch::PitchShifter; pub use restoration::{DeHum, DeNoise}; pub use time::{Delay, Reverb}; @@ -92,6 +94,10 @@ static REGISTRY: &[EffectInfo] = &[ entry!("dehum", "DeHum", Restoration, DeHum), entry!("denoise", "DeNoise", Restoration, DeNoise), entry!("pitch_shifter", "Pitch Shifter", Pitch, PitchShifter), + entry!("deesser", "DeEsser", Restoration, DeEsser), + entry!("dereverb", "DeReverb", Restoration, DeReverb), + entry!("speech_enhance", "Enhance Speech", Filter, SpeechEnhance), + entry!("stereo_width", "Stereo Width", Channel, StereoWidth), ]; /// All registered effects. diff --git a/crates/audio-dsp/src/lib.rs b/crates/audio-dsp/src/lib.rs index 15edf5b0..9908cf6b 100644 --- a/crates/audio-dsp/src/lib.rs +++ b/crates/audio-dsp/src/lib.rs @@ -15,6 +15,7 @@ //! portable to wasm). pub mod biquad; +pub mod ducking; pub mod effects; pub mod fft; pub mod loudness; From 460b018d2af5b0d4f634f560448d588997040759 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 1 Oct 2026 08:14:17 -0400 Subject: [PATCH 2/5] README: VP9, titles and captions, dynamic trimming, shortcuts editor, auto-save/recovery, render previews, audio mixer with automation; ~45% parity; architecture table and docs links current Co-Authored-By: Claude Opus 5.5 --- README.md | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 96df345d..6a707158 100644 --- a/README.md +++ b/README.md @@ -39,9 +39,11 @@ FilmCraft is a non-linear editor for people who know Premiere: the same panels, **A timeline you already know.** Source and Program monitors, bins with thumbnails, a multi-track timeline with patch and target buttons, sync locks and linked selection. The Editing, Assembly, Color, Effects and Audio workspaces are all there, and every panel docks wherever you want it. - **Three-point editing.** Mark In and Out in the Source monitor, then insert (`,`) or overwrite (`.`) onto the patched tracks. Lift (`;`) and extract (`'`) take ranges back out. -- **Every trim.** Ripple, roll, slip, slide, rate stretch and razor tools. Trim mode selects edit points as ripple, roll or trim, nudges them a frame at a time (`⌥←` `⌥→`, ×5 with `⇧`), toggles the trim type with `⌃T` and extends them to the playhead with `E`. `Q` and `W` ripple-trim to the playhead. +- **Every trim.** Ripple, roll, slip, slide, rate stretch and razor tools. Trim mode selects edit points as ripple, roll or trim, nudges them a frame at a time (`⌥←` `⌥→`, ×5 with `⇧`), toggles the trim type with `⌃T` and extends them to the playhead with `E`. `Q` and `W` ripple-trim to the playhead. The **Trim Monitor** shows both sides of the edit, and **dynamic trimming** trims live while it plays: `L` forward, `J` back, `K` to stop and commit as one undo step. - **Exact time.** Every edit is computed on integer ticks: 254,016,000,000 per second, which divides evenly by every common frame rate and sample rate. 23.976, 29.97 drop-frame and 59.94 are exact, not approximate. - **The details pros rely on.** Markers with colours, names and durations; add edit (`⌘K`) on one or all tracks; nesting; copy, paste and paste insert; ripple delete and close gap; snapping; unlimited undo with a History panel. +- **Your keys.** A Keyboard Shortcuts editor (`⌥⌘K`) with a drawn keyboard, panel-specific shortcuts, conflict warnings and presets for FilmCraft, Premiere Pro, Final Cut Pro and Avid key layouts. +- **Never lose work.** Saves are atomic, auto-save keeps a rolling set of versions, and a crash-recovery journal written about a second after each edit brings back unsaved changes after a crash or power cut.

Timeline detail: Night of the Living Dead shots with dissolves, a Carnival of Souls insert on V2, the Chopin score as a waveform on A2, markers and loudness meters @@ -96,7 +98,8 @@ FilmCraft is a non-linear editor for people who know Premiere: the same panels, - **Loudness meters to broadcast standards.** Momentary, short-term and integrated loudness and true peak to ITU-R BS.1770 and EBU R128, alongside the peak meters. Our integrated reading matches ffmpeg's `ebur128` filter to the tenth of a LU. - **Clip effects on our own DSP library:** parametric EQ, high-pass, low-pass and band-pass filters, dynamics, a true-peak limiter, delay, reverb, DeNoise, DeHummer, invert and pitch shift. Parameters can be keyframed. Effects stay continuous across scrubbing, playback and export, with latency compensated. -- **Mixing:** clip gain, volume and channel volume with keyframes; track volume, pan, mute, solo and sync lock; constant-power crossfades; waveforms drawn on every clip. +- **Audio Track Mixer and Audio Clip Mixer:** faders, pan, mute, solo, five insert slots per track (pre- or post-fader), sends, submixes and a Mix track, all sample-accurate and latency-compensated. 24 tracks with three effects each mix about six times faster than real time on one core. +- **Automation like a console:** Read, Latch, Touch and Write modes recorded live from the mixer during playback, shown and edited as track keyframes on the timeline. Plus the Audio Gain dialog (set, adjust, normalize) and constant-power crossfades. - **Audio-clock playback.** The sound card is the master clock, so picture follows sound and never the other way round.
@@ -111,6 +114,7 @@ No FFmpeg inside. The video codecs, AAC, Opus and the containers are our own Rus | **HEVC / H.265** | ✓ | | Main and Main 10, bit-exact on 41 streams (tiles, WPP, PCM, long-term references), about 225 fps at 1080p | | **Apple ProRes** | ✓ | ✓ | Decodes 422 Proxy to 4444 XQ; export writes 422 HQ | | **AAC-LC** | ✓ | ✓ | | +| **VP9** | ✓ | | Profiles 0–3, 8/10/12-bit, 4:2:0 to 4:4:4, tiles and superframes; bit-exact on 50+ streams; in WebM/Matroska and MP4 | | **Opus** | ✓ | | SILK, CELT, hybrid and multistream surround; range-exact on every RFC 8251 conformance vector; in WebM/Matroska and MP4 | | **MJPEG, PCM** | ✓ | ✓ | | | **MP3, FLAC, ALAC, Vorbis** | ✓ | | Via the [symphonia](https://github.com/pdeljanov/Symphonia) crate (MPL-2.0) for now, to be replaced by our own | @@ -122,6 +126,14 @@ VP9, AV1, DNxHR and MXF are next ([roadmap](ROADMAP.md)).
+## Titles and captions + +- **A real text engine:** OpenType shaping with kerning and ligatures, bidirectional text, line breaking, tracking and leading, drawn in linear light and sharp at any scale or rotation. +- **Type, Shape and Pen tools** on the Program monitor: click to type, edit with a caret and selection, drag out rectangles, ellipses and paths. Graphic clips hold text and shape layers with fill, strokes, background and shadow, all keyframable, and edited in the Properties panel. +- **Captions:** caption tracks in Subtitle, CEA-608, CEA-708 and Teletext formats; import and export SRT, WebVTT and SCC with frame-exact timing; edit captions in the Text panel and burn them into exports. + +
+ ## Export

@@ -131,6 +143,7 @@ VP9, AV1, DNxHR and MXF are next ([roadmap](ROADMAP.md)). - **H.264 MP4 with AAC, using our own encoders.** A 6-second 960×540 render takes 1.3 seconds and decodes cleanly in ffmpeg with error concealment switched off. - **Also:** Apple ProRes 422 HQ and Motion JPEG in QuickTime, PNG sequences, animated GIF and WAV. - **Background jobs** with progress and cancel, so you keep editing while it renders. +- **Render previews:** the render bar marks segments green, yellow or red; rendered previews are cached by content, so an edit only invalidates what it touches and undo brings the green back.
@@ -193,11 +206,13 @@ The control protocol is documented in [docs/control-protocol.md](docs/control-pr | [docs/testing.md](docs/testing.md) | Unit, property and ffmpeg-oracle tests, accuracy criteria, benchmarks | | [docs/agents.md](docs/agents.md) | Driving FilmCraft over MCP and the control channel; how agents develop it | | [docs/control-protocol.md](docs/control-protocol.md) | Control-channel and MCP reference | +| [docs/project-files.md](docs/project-files.md) | `.fcproj` format, schema migrations, auto-save and crash recovery | +| [docs/graphics.md](docs/graphics.md) · [docs/captions.md](docs/captions.md) | Text engine, graphic clips and tools; caption tracks and formats | | [ROADMAP.md](ROADMAP.md) | Milestones and estimates | ## Status -FilmCraft is young and moving fast: roughly 30% of the way to Premiere Pro parity, with editing, colour, keyframes, codecs and export working today. [ROADMAP.md](ROADMAP.md) tracks every milestone with estimates. +FilmCraft is young and moving fast: roughly 45% of the way to Premiere Pro parity, with editing, trimming, colour, keyframes, titles, captions, mixing, codecs and export working today. [ROADMAP.md](ROADMAP.md) tracks every milestone with estimates. ## Architecture @@ -205,12 +220,13 @@ A layered Cargo workspace: | Layer | Crates | |---|---| -| Codecs and containers | `h264`, `h264enc`, `hevc`, `prores`, `aac`, `isobmff`, `matroska`, `bitstream` | -| Foundations | `time`, `geom`, `color`, `frame`, `media`, `project`, `audio-dsp` | -| Editing and interchange | `edit`, `codecs`, `interchange` | +| Codecs and containers | `h264`, `h264enc`, `hevc`, `vp9`, `prores`, `aac`, `opus`, `isobmff`, `matroska`, `bitstream` | +| Foundations | `time`, `geom`, `color`, `frame`, `media`, `text`, `project`, `audio-dsp` | +| Editing and interchange | `edit`, `codecs`, `captions`, `format` (project files), `interchange` | | Rendering and output | `render`, `gpu`, `export` | | Engine | `engine`: command registry, session, undo, jobs | | Front ends | `ui-egui`, `automation` (MCP), `apps/filmcraft`, `apps/filmcraft-cli` | +| Test support | `testkit` (ffmpeg oracles, fixtures), `golden` (golden-image tests) | Nothing below the front ends depends on a UI toolkit or OS API. `cargo xtask ci` checks formatting, lints, tests, the layering rules, asset attribution and the wasm build. From 6a0a3add7f838849767fa64d2f4f291721ccfdbb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 1 Oct 2026 08:28:38 -0400 Subject: [PATCH 3/5] =?UTF-8?q?M7.4:=20Essential=20Sound=20model=20and=20c?= =?UTF-8?q?ommands=20=E2=80=94=20audio=20types,=20per-type=20settings=20as?= =?UTF-8?q?=20clip=20effects,=20presets,=20loudness=20auto-match,=20duckin?= =?UTF-8?q?g=20keyframes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - project::essential: Dialogue/Music/SFX/Ambience settings (Loudness, Repair, Clarity, Creative, Ducking, Pan, Clip Volume, Mute), our own EQ/reverb/type presets, and apply(): settings become ordinary clip effects (EffectInstance::essential) in canonical order; only changed parameters are written, so keyframes survive; auto-match gain → clip gain, Clip Volume → Volume offset, Pan → Panner - effect defs: DeEsser, DeReverb (params), Enhance Speech, Stereo Width; render mappings for them, and DeNoise amount / DeHummer gain now reach the DSP; Mute; render::audio::clip_signal - engine essentialSound.* (inspect, setType, clearType, set, applyPreset, savePreset, deletePreset, autoMatch, generateDucking), undoable; per-type loudness targets and user presets in preferences - tests: auto-match within 0.001 LU of target (−23 / −16, with and without a chain), ducking keyframes within 20 ms on a synthetic dialogue+music project and −15.00 dB in the mix, repair metrics (hum −39 dB, rumble −22 dB, noise floor −13 dB, sibilance −19 dB, reverb tail −10 dB), render chunking bit-identical and WAV export = render; full Dialogue chain 22× realtime (release) Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + crates/edit/src/tests.rs | 1 + crates/engine/Cargo.toml | 1 + crates/engine/src/autosave.rs | 41 + crates/engine/src/commands.rs | 1 + crates/engine/src/essential_sound.rs | 469 +++++++++++ crates/engine/src/essential_sound_tests.rs | 467 +++++++++++ crates/engine/src/lib.rs | 3 + crates/interchange/src/common.rs | 2 + crates/project/src/effect.rs | 24 +- crates/project/src/essential.rs | 885 +++++++++++++++++++++ crates/project/src/lib.rs | 6 + crates/render/src/audio.rs | 39 +- crates/render/src/audio_fx.rs | 35 +- 14 files changed, 1970 insertions(+), 5 deletions(-) create mode 100644 crates/engine/src/essential_sound.rs create mode 100644 crates/engine/src/essential_sound_tests.rs create mode 100644 crates/project/src/essential.rs diff --git a/Cargo.lock b/Cargo.lock index 72dbdba5..b17228e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1429,6 +1429,7 @@ dependencies = [ name = "filmcraft-engine" version = "0.1.0" dependencies = [ + "filmcraft-audio-dsp", "filmcraft-captions", "filmcraft-codecs", "filmcraft-color", diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index b9d41d4d..9ca16a77 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -54,6 +54,7 @@ impl Fx { gain_db: 0.0, frame_hold: None, scale_to_frame: false, + essential: None, } } fn put(&mut self, track: TrackId, start: i64, dur: i64, src_in: i64) -> ClipId { diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 287c1a32..49c5ea37 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -20,6 +20,7 @@ filmcraft-project = { workspace = true } filmcraft-edit = { workspace = true } filmcraft-render = { workspace = true } filmcraft-text = { workspace = true } +filmcraft-audio-dsp = { workspace = true } filmcraft-export = { workspace = true } filmcraft-format = { workspace = true } filmcraft-interchange = { workspace = true } diff --git a/crates/engine/src/autosave.rs b/crates/engine/src/autosave.rs index e66b7c2d..64c154b8 100644 --- a/crates/engine/src/autosave.rs +++ b/crates/engine/src/autosave.rs @@ -104,6 +104,21 @@ pub struct Preferences { pub audio: AudioPrefs, pub playback: PlaybackPrefs, pub trim: TrimPrefs, + pub essential_sound: EssentialSoundPrefs, +} + +/// Essential Sound: user presets saved from the panel (built-in presets live in +/// `filmcraft_project::essential`). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct EssentialSoundPrefs { + pub user_presets: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct UserSoundPreset { + pub name: String, + pub settings: filmcraft_project::EssentialSound, } /// Preferences ▸ Audio (the mixer-automation part) and the Track Mixer panel-menu toggle. @@ -122,6 +137,11 @@ pub struct AudioPrefs { pub minimum_time_ms: u32, /// Track Mixer ▸ "Switch to Touch after Write". pub switch_to_touch_after_write: bool, + /// Essential Sound Loudness Auto-Match targets (integrated LUFS) per audio type. + pub dialogue_target_lufs: f64, + pub music_target_lufs: f64, + pub sfx_target_lufs: f64, + pub ambience_target_lufs: f64, } impl Default for AudioPrefs { @@ -133,15 +153,36 @@ impl Default for AudioPrefs { minimum_time_interval_thinning: false, minimum_time_ms: 20, switch_to_touch_after_write: true, + dialogue_target_lufs: -23.0, + music_target_lufs: -25.0, + sfx_target_lufs: -21.0, + ambience_target_lufs: -30.0, } } } impl AudioPrefs { + /// Auto-Match target for an audio type. + pub fn loudness_target(&self, t: filmcraft_project::AudioType) -> f64 { + match t { + filmcraft_project::AudioType::Dialogue => self.dialogue_target_lufs, + filmcraft_project::AudioType::Music => self.music_target_lufs, + filmcraft_project::AudioType::Sfx => self.sfx_target_lufs, + filmcraft_project::AudioType::Ambience => self.ambience_target_lufs, + } + } fn clamp(&mut self) { self.automatch_time = if self.automatch_time.is_finite() { self.automatch_time.clamp(0.0, 30.0) } else { 1.0 }; self.large_volume_adjustment = if self.large_volume_adjustment.is_finite() { self.large_volume_adjustment.clamp(0.0, 96.0) } else { 6.0 }; self.minimum_time_ms = self.minimum_time_ms.clamp(1, 10_000); + for (v, d) in [ + (&mut self.dialogue_target_lufs, -23.0), + (&mut self.music_target_lufs, -25.0), + (&mut self.sfx_target_lufs, -21.0), + (&mut self.ambience_target_lufs, -30.0), + ] { + *v = if v.is_finite() { v.clamp(-60.0, 0.0) } else { d }; + } } } diff --git a/crates/engine/src/commands.rs b/crates/engine/src/commands.rs index 7f3ae7e0..978ed43f 100644 --- a/crates/engine/src/commands.rs +++ b/crates/engine/src/commands.rs @@ -1849,6 +1849,7 @@ fn build() -> Vec { ]; v.extend(crate::captions::commands()); v.extend(crate::mixer::commands()); + v.extend(crate::essential_sound::commands()); v.extend(crate::graphics::commands()); v.extend(crate::shortcuts::commands()); // Labels as individual commands (Edit ▸ Label ▸ ) diff --git a/crates/engine/src/essential_sound.rs b/crates/engine/src/essential_sound.rs new file mode 100644 index 00000000..cd05fca8 --- /dev/null +++ b/crates/engine/src/essential_sound.rs @@ -0,0 +1,469 @@ +//! Essential Sound commands (`essentialSound.*`). +//! +//! Clips get an audio type (Dialogue / Music / SFX / Ambience) and per-type settings +//! ([`filmcraft_project::essential`]). Every change goes through one undoable edit that updates the +//! settings and reconciles the clip's essential effects, clip gain, Volume and Panner +//! ([`filmcraft_project::essential::apply`]), so what the panel does is ordinary clip data that the +//! mixer, playback and export render without special cases. +//! +//! | command | does | +//! |---|---| +//! | `essentialSound.inspect` | types, settings and essential effects of the selected audio clips; presets; targets | +//! | `essentialSound.setType` / `clearType` | assign / clear the audio type (clearing removes its effects and gain) | +//! | `essentialSound.set` | one setting by dotted key (`repair.noise.amount`), or several (`values`) | +//! | `essentialSound.applyPreset` / `savePreset` / `deletePreset` | built-in and user presets | +//! | `essentialSound.autoMatch` | measure integrated loudness (BS.1770) and set the match gain to the target | +//! | `essentialSound.generateDucking` | Volume keyframes that duck Music / Ambience under the trigger types | + +use serde_json::{Value, json}; + +use filmcraft_audio_dsp::LoudnessMeter; +use filmcraft_project::essential::{self as es, AudioType, EssentialSound, Preset, Section}; +use filmcraft_project::{ClipId, ParamValue, Project, TrackItem}; +use filmcraft_render::SourceProvider; +use filmcraft_time::Tick; + +use crate::autosave::UserSoundPreset; +use crate::commands::{CommandSpec, always, bad, bool_p, f64_p, has_seq, str_p, with_links}; +use crate::{EngineError, Result, Session}; + +type Run = fn(&mut Session, &Value) -> Result; +type Enabled = fn(&Session) -> std::result::Result<(), String>; + +/// The audio clips a command acts on: `clips` (or `clip`), else the selection, with linked partners. +pub fn targets(s: &Session, p: &Value) -> Vec { + let sel: Vec = match p.get("clips").and_then(Value::as_array) { + Some(a) => a.iter().filter_map(|v| v.as_u64().map(ClipId)).collect(), + None => match p.get("clip").and_then(Value::as_u64) { + Some(c) => vec![ClipId(c)], + None => s.state.selection.clone(), + }, + }; + let Some(seq) = s.active_sequence() else { return Vec::new() }; + let all = with_links(s, &sel); + seq.audio_tracks.iter().flat_map(|t| t.items.iter()).filter(|i| all.contains(&i.id)).map(|i| i.id).collect() +} + +fn mt_of(it: &TrackItem, t: Tick) -> Tick { + it.source_time_at(t.clamp(it.start, (it.end() - Tick(1)).max(it.start))) +} + +/// Change the settings of `clips` with `f` and reconcile their effects, as one undo step (or merged +/// into the previous step with the same `merge` key while a control is dragged). +fn update( + s: &mut Session, + cmd: &str, + label: &str, + merge: Option, + clips: &[ClipId], + f: impl Fn(&TrackItem, &mut Option) -> std::result::Result<(), String>, +) -> Result { + if clips.is_empty() { + return Err(bad(cmd, "select one or more audio clips")); + } + let seq_id = s.state.active_sequence.ok_or(EngineError::NoSequence)?; + let ph = s.playhead(); + let body = |p: &mut Project, _: &mut crate::EditorState| -> Result { + let seq = p.sequence_mut(seq_id).ok_or(EngineError::NoSequence)?; + let mut n = 0; + for it in seq.audio_tracks.iter_mut().flat_map(|t| t.items.iter_mut()).filter(|i| clips.contains(&i.id)) { + let old = it.essential.clone(); + let mut next = old.clone(); + f(it, &mut next).map_err(|e| bad(cmd, e))?; + if next != old { + it.essential = next; + let mt = mt_of(it, ph); + es::apply(it, old.as_ref(), mt); + n += 1; + } + } + Ok(n) + }; + match merge { + Some(k) => s.edit_merged(label, &k, body), + None => s.edit(label, body), + } +} + +/// Built-in presets followed by the user's. +pub fn presets(s: &Session) -> Vec { + let mut v = es::builtin_presets(); + v.extend(s.prefs.essential_sound.user_presets.iter().map(|u| Preset { + name: u.name.clone(), + kind: u.settings.kind, + settings: u.settings.clone(), + builtin: false, + })); + v +} + +fn type_p(p: &Value, cmd: &str) -> Result> { + match str_p(p, "type") { + None => Ok(None), + Some(t) => AudioType::parse(t).map(Some).ok_or_else(|| bad(cmd, format!("unknown audio type `{t}` (dialogue, music, sfx, ambience)"))), + } +} + +fn clip_json(s: &Session, it: &TrackItem, track: &str) -> Value { + let t = s.playhead(); + let level = it.effect("volume").map(|e| e.f64_at("level", mt_of(it, t))).unwrap_or(0.0); + json!({ + "clip": it.id.0, + "name": it.name, + "track": track, + "type": it.essential.as_ref().map(|e| e.kind.label()), + "settings": it.essential.as_ref().map(EssentialSound::to_value), + "gainDb": it.gain_db, + "volumeDb": level, + "volumeAnimated": it.effect("volume").and_then(|e| e.param("level")).is_some_and(|p| p.is_animated()), + "effects": it.effects.iter().enumerate().filter(|(_, e)| e.essential).map(|(i, e)| json!({"index": i, "effect": e.effect, "enabled": e.enabled})).collect::>(), + }) +} + +fn inspect(s: &mut Session, p: &Value) -> Result { + let clips = targets(s, p); + let mut out = Vec::new(); + let mut kinds = Vec::new(); + if let Some(seq) = s.active_sequence() { + for (k, t) in seq.audio_tracks.iter().enumerate() { + for it in t.items.iter().filter(|i| clips.contains(&i.id)) { + out.push(clip_json(s, it, &format!("A{}", k + 1))); + kinds.push(it.essential.as_ref().map(|e| e.kind)); + } + } + } + kinds.dedup(); + let kind = match kinds.as_slice() { + [Some(k)] => json!(k.label()), + [None] | [] => Value::Null, + _ => json!("mixed"), + }; + let ps = presets(s); + let by_type = |t: AudioType| ps.iter().filter(|p| p.kind == t).map(|p| json!({"name": p.name, "builtin": p.builtin})).collect::>(); + Ok(json!({ + "clips": out, + "type": kind, + "presets": AudioType::ALL.iter().map(|t| (t.label().to_string(), json!(by_type(*t)))).collect::>(), + "sections": AudioType::ALL.iter().map(|t| (t.label().to_string(), json!(t.sections().iter().map(|s| s.label()).collect::>()))).collect::>(), + "targets": AudioType::ALL.iter().map(|t| (t.label().to_string(), json!(s.prefs.audio.loudness_target(*t)))).collect::>(), + "eqPresets": es::EQ_PRESETS.iter().map(|p| p.name).collect::>(), + "reverbPresets": es::REVERB_PRESETS.iter().map(|p| p.name).collect::>(), + })) +} + +fn set_type(s: &mut Session, p: &Value) -> Result { + let t = type_p(p, "essentialSound.setType")?.ok_or_else(|| bad("essentialSound.setType", "need `type`"))?; + let clips = targets(s, p); + let n = update(s, "essentialSound.setType", &format!("Set Audio Type to {}", t.label()), None, &clips, |_, e| { + if e.as_ref().is_none_or(|x| x.kind != t) { + *e = Some(EssentialSound::new(t)); + } + Ok(()) + })?; + Ok(json!({"clips": n, "type": t.label()})) +} + +fn clear_type(s: &mut Session, p: &Value) -> Result { + let clips = targets(s, p); + let n = update(s, "essentialSound.clearType", "Clear Audio Type", None, &clips, |_, e| { + *e = None; + Ok(()) + })?; + Ok(json!({"clips": n})) +} + +fn set(s: &mut Session, p: &Value) -> Result { + let mut kv: Vec<(String, Value)> = Vec::new(); + if let Some(k) = str_p(p, "key") { + kv.push((k.to_string(), p.get("value").cloned().ok_or_else(|| bad("essentialSound.set", "need `value`"))?)); + } + if let Some(m) = p.get("values").and_then(Value::as_object) { + kv.extend(m.iter().map(|(k, v)| (k.clone(), v.clone()))); + } + if kv.is_empty() { + return Err(bad("essentialSound.set", "need `key` + `value` or `values`")); + } + let clips = targets(s, p); + if bool_p(p, "begin").unwrap_or(false) { + s.history.merge_key = None; + } + let keys: Vec<&str> = kv.iter().map(|(k, _)| k.as_str()).collect(); + let merge = format!("essentialSound:{clips:?}:{}", keys.join(",")); + let n = update(s, "essentialSound.set", "Essential Sound", Some(merge), &clips, |it, e| { + let st = e.as_mut().ok_or_else(|| format!("clip `{}` has no audio type; set one first", it.name))?; + for (k, v) in &kv { + st.set(k, v)?; + } + Ok(()) + })?; + Ok(json!({"clips": n})) +} + +fn apply_preset(s: &mut Session, p: &Value) -> Result { + let cmd = "essentialSound.applyPreset"; + let name = str_p(p, "preset").ok_or_else(|| bad(cmd, "need `preset`"))?.to_string(); + let forced = type_p(p, cmd)?; + let ps = presets(s); + let clips = targets(s, p); + let n = update(s, cmd, &format!("Apply Sound Preset {name}"), None, &clips, |it, e| { + let kind = forced.or(e.as_ref().map(|x| x.kind)).ok_or_else(|| format!("clip `{}` has no audio type; pass `type`", it.name))?; + let pr = + ps.iter().rev().find(|x| x.kind == kind && x.name.eq_ignore_ascii_case(&name)).ok_or_else(|| format!("no {} preset `{name}`", kind.label()))?; + *e = Some(es::with_preset(e.as_ref(), pr)); + Ok(()) + })?; + Ok(json!({"clips": n, "preset": name})) +} + +fn save_preset(s: &mut Session, p: &Value) -> Result { + let cmd = "essentialSound.savePreset"; + let name = str_p(p, "name").map(str::trim).filter(|n| !n.is_empty()).ok_or_else(|| bad(cmd, "need `name`"))?.to_string(); + let clips = targets(s, p); + let seq = s.active_sequence().ok_or(EngineError::NoSequence)?; + let mut st = + clips.iter().find_map(|c| seq.find_item(*c).and_then(|(_, i)| i.essential.clone())).ok_or_else(|| bad(cmd, "select a clip with an audio type"))?; + if es::builtin_presets().iter().any(|b| b.kind == st.kind && b.name.eq_ignore_ascii_case(&name)) { + return Err(bad(cmd, format!("`{name}` is a built-in preset name"))); + } + st.preset = name.clone(); + st.loudness = es::Loudness { enabled: st.loudness.enabled, ..Default::default() }; + st.volume = Default::default(); + st.mute = false; + let kind = st.kind; + let mut prefs = s.prefs.clone(); + prefs.essential_sound.user_presets.retain(|u| !(u.settings.kind == kind && u.name.eq_ignore_ascii_case(&name))); + prefs.essential_sound.user_presets.push(UserSoundPreset { name: name.clone(), settings: st }); + s.set_prefs(prefs).map_err(|e| EngineError::Other(format!("saving preferences: {e}")))?; + // the clips now show the saved preset + let _ = update(s, cmd, "Save Sound Preset", None, &clips, |_, e| { + if let Some(x) = e.as_mut().filter(|x| x.kind == kind) { + x.preset = name.clone(); + } + Ok(()) + }); + Ok(json!({"name": name, "type": kind.label()})) +} + +fn delete_preset(s: &mut Session, p: &Value) -> Result { + let cmd = "essentialSound.deletePreset"; + let name = str_p(p, "name").or_else(|| str_p(p, "preset")).ok_or_else(|| bad(cmd, "need `name`"))?.to_string(); + let kind = type_p(p, cmd)?; + let mut prefs = s.prefs.clone(); + let before = prefs.essential_sound.user_presets.len(); + prefs.essential_sound.user_presets.retain(|u| !(u.name.eq_ignore_ascii_case(&name) && kind.is_none_or(|k| k == u.settings.kind))); + if prefs.essential_sound.user_presets.len() == before { + return Err(bad(cmd, format!("no user preset `{name}` (built-in presets cannot be deleted)"))); + } + s.set_prefs(prefs).map_err(|e| EngineError::Other(format!("saving preferences: {e}")))?; + Ok(json!({"deleted": name})) +} + +// ------------------------------------------------------------------------------------- loudness + +/// Integrated loudness (LUFS) of a clip's own signal (clip gain + clip effects) over its whole +/// timeline range, or `None` when it is silent / has no audio. +pub fn clip_loudness(item: &TrackItem, sr: u32, sources: &dyn SourceProvider) -> Option { + let a0 = item.start.to_units_floor(sr as i64); + let a1 = item.end().to_units_floor(sr as i64); + let mut meter = LoudnessMeter::new(sr as f64, 2); + let mut pos = a0; + while pos < a1 { + let n = (a1 - pos).min(sr as i64) as usize; + let [l, r] = filmcraft_render::audio::clip_signal(item, pos, n, sr, sources)?; + meter.process(&[&l, &r]); + pos += n as i64; + } + Some(meter.integrated()).filter(|v| v.is_finite()) +} + +fn auto_match(s: &mut Session, p: &Value) -> Result { + let cmd = "essentialSound.autoMatch"; + let clips = targets(s, p); + let target_p = f64_p(p, "target"); + let seq = s.active_sequence().ok_or(EngineError::NoSequence)?; + let sr = seq.settings.sample_rate.max(1); + let provider = s.media.provider(s.project.clone(), s.services.clone()); + // measure first (read-only), then write every match in one undo step + let mut matches: Vec<(ClipId, f64, f64, f64)> = Vec::new(); // clip, measured (without match gain), target, new match gain + let mut report = Vec::new(); + for it in seq.audio_tracks.iter().flat_map(|t| t.items.iter()).filter(|i| clips.contains(&i.id)) { + let Some(st) = it.essential.as_ref() else { + report.push(json!({"clip": it.id.0, "skipped": "no audio type"})); + continue; + }; + let target = target_p.unwrap_or_else(|| s.prefs.audio.loudness_target(st.kind)).clamp(-60.0, 0.0); + let Some(l) = clip_loudness(it, sr, &provider) else { + report.push(json!({"clip": it.id.0, "skipped": "silent"})); + continue; + }; + // the clip gain (including any earlier match gain) applies linearly after the effects + let current = st.loudness_gain_db(); + let measured = l - current; + let gain = filmcraft_audio_dsp::normalize_gain_db(measured, target); + matches.push((it.id, measured, target, gain)); + report.push(json!({"clip": it.id.0, "measuredLufs": measured, "targetLufs": target, "gainDb": gain})); + } + if matches.is_empty() { + return Err(bad(cmd, "no clips with an audio type and audible audio in the selection")); + } + let ids: Vec = matches.iter().map(|m| m.0).collect(); + update(s, cmd, "Auto-Match Loudness", None, &ids, |it, e| { + if let (Some(st), Some(m)) = (e.as_mut(), matches.iter().find(|m| m.0 == it.id)) { + st.loudness.enabled = true; + st.loudness.measured_lufs = Some(m.1); + st.loudness.target_lufs = Some(m.2); + st.loudness.gain_db = m.3; + } + Ok(()) + })?; + Ok(json!({"clips": report})) +} + +// ------------------------------------------------------------------------------------- ducking + +/// Analysis hop and smoothing window of the ducking detector. +const DUCK_HOP_S: f64 = 0.01; +const DUCK_WINDOW_HOPS: usize = 5; +/// Ignore trigger blips shorter than this; bridge pauses between words shorter than this. +const DUCK_MIN_ON_S: f32 = 0.1; +const DUCK_BRIDGE_S: f32 = 0.25; + +/// Level envelope (dBFS, 10 ms hops, 50 ms centred window) of the summed trigger clips over +/// timeline samples `[a0, a1)`. +fn trigger_envelope(triggers: &[&TrackItem], a0: i64, a1: i64, sr: u32, sources: &dyn SourceProvider) -> Vec { + let hop = ((sr as f64 * DUCK_HOP_S).round() as usize).max(1); + let hops = ((a1 - a0).max(0) as usize).div_ceil(hop); + let mut power = vec![0.0f64; hops]; + let chunk = hop * 100; + let mut pos = a0; + while pos < a1 { + let n = ((a1 - pos) as usize).min(chunk); + let mut sum = [vec![0.0f32; n], vec![0.0f32; n]]; + for t in triggers { + if let Some(b) = filmcraft_render::audio::clip_signal(t, pos, n, sr, sources) { + for c in 0..2 { + for (d, x) in sum[c].iter_mut().zip(&b[c]) { + *d += x; + } + } + } + } + let base = ((pos - a0) as usize) / hop; + for (k, i) in (0..n).step_by(hop).enumerate() { + let e = (i + hop).min(n); + let pw: f64 = (i..e).map(|j| 0.5 * ((sum[0][j] as f64).powi(2) + (sum[1][j] as f64).powi(2))).sum::() / (e - i) as f64; + if let Some(p) = power.get_mut(base + k) { + *p = pw; + } + } + pos += n as i64; + } + let h = DUCK_WINDOW_HOPS / 2; + (0..hops) + .map(|k| { + let (lo, hi) = (k.saturating_sub(h), (k + h + 1).min(hops)); + let m = power[lo..hi].iter().sum::() / (hi - lo) as f64; + (10.0 * m.max(1e-12).log10()).max(-120.0) as f32 + }) + .collect() +} + +fn generate_ducking(s: &mut Session, p: &Value) -> Result { + let cmd = "essentialSound.generateDucking"; + let clips = targets(s, p); + let seq = s.active_sequence().ok_or(EngineError::NoSequence)?; + let sr = seq.settings.sample_rate.max(1); + let provider = s.media.provider(s.project.clone(), s.services.clone()); + let all: Vec<&TrackItem> = seq.audio_tracks.iter().flat_map(|t| t.items.iter()).collect(); + let mut plans: Vec<(ClipId, Vec<(Tick, f64)>, f64)> = Vec::new(); + let mut report = Vec::new(); + for it in all.iter().copied().filter(|i| clips.contains(&i.id)) { + let Some(st) = it.essential.as_ref().filter(|e| e.kind.has(Section::Ducking)) else { continue }; + let d = &st.ducking; + let triggers: Vec<&TrackItem> = all + .iter() + .copied() + .filter(|o| o.id != it.id && o.enabled && o.range().overlaps(&it.range())) + .filter(|o| match o.essential.as_ref() { + Some(e) => !e.mute && d.against.contains(&e.kind), + None => d.against_untyped, + }) + .collect(); + let (a0, a1) = (it.start.to_units_floor(sr as i64), it.end().to_units_floor(sr as i64)); + let env = trigger_envelope(&triggers, a0, a1, sr, &provider); + let thr = filmcraft_audio_dsp::ducking::sensitivity_threshold_db(d.sensitivity as f32); + let regions = filmcraft_audio_dsp::ducking::activity(&env, DUCK_HOP_S as f32, thr, DUCK_MIN_ON_S, DUCK_BRIDGE_S); + let start_s = a0 as f64 / sr as f64; + let end_s = a1 as f64 / sr as f64; + let abs: Vec<(f64, f64)> = regions.iter().map(|(x, y)| (start_s + x, start_s + y)).collect(); + let lvl = it.effect("volume").and_then(|e| e.param("level")); + let base = match lvl { + Some(p) if p.is_animated() => p.keyframes.iter().filter_map(|k| k.value.as_f64()).fold(f64::NEG_INFINITY, f64::max), + Some(p) => p.value.as_f64().unwrap_or(0.0), + None => 0.0, + }; + let kfs = filmcraft_audio_dsp::ducking::duck_keyframes(&abs, start_s, end_s, base, d.reduce_db, d.fade_s); + let ticks: Vec<(Tick, f64)> = kfs.iter().map(|(t, v)| (Tick::from_seconds_f64(*t).clamp(it.start, it.end()), *v)).collect(); + report.push(json!({ + "clip": it.id.0, + "triggers": triggers.iter().map(|t| t.id.0).collect::>(), + "regions": abs.iter().map(|(a, b)| json!([a, b])).collect::>(), + "keyframes": ticks.iter().map(|(t, v)| json!({"time": t.0, "seconds": t.seconds(), "levelDb": v})).collect::>(), + })); + plans.push((it.id, ticks, base)); + } + if plans.is_empty() { + return Err(bad(cmd, "select Music or Ambience clips (ducking applies to those types)")); + } + let seq_id = s.state.active_sequence.ok_or(EngineError::NoSequence)?; + s.edit("Generate Ducking Keyframes", |pr, _| { + let seq = pr.sequence_mut(seq_id).ok_or(EngineError::NoSequence)?; + for (c, ticks, base) in &plans { + let Some((_, it)) = seq.find_item_mut(*c) else { continue }; + if let Some(e) = it.essential.as_mut() { + e.ducking.enabled = true; + } + let at: Vec<(Tick, f64)> = ticks.iter().map(|(t, v)| (mt_of(it, *t), *v)).collect(); + let Some(par) = it.effect_mut("volume").and_then(|e| e.param_mut("level")) else { continue }; + par.keyframes.clear(); + par.value = ParamValue::Float(*base); + for (mt, v) in at { + par.put_keyframe(mt, ParamValue::Float(v)); + } + } + Ok(()) + })?; + Ok(json!({"clips": report})) +} + +fn spec(id: &'static str, label: &'static str, params: &'static str, enabled: Enabled, run: Run, journal: bool) -> CommandSpec { + CommandSpec { id, label, menu: &[], shortcut: None, params, enabled, run, journal } +} + +pub fn commands() -> Vec { + vec![ + spec("essentialSound.inspect", "Inspect Essential Sound", r#"{"clips":[id]?}"#, always, inspect, false), + spec("essentialSound.setType", "Set Audio Type", r#"{"clips":[id]?,"type":"dialogue|music|sfx|ambience"}"#, has_seq, set_type, true), + spec("essentialSound.clearType", "Clear Audio Type", r#"{"clips":[id]?}"#, has_seq, clear_type, true), + spec( + "essentialSound.set", + "Essential Sound Setting", + r#"{"clips":[id]?,"key":"repair.noise.on|repair.noise.amount|repair.humHz|clarity.eqPreset|creative.reverbPreset|ducking.reduceDb|volume.levelDb|mute|…","value":any,"values":{key:value}?,"begin":bool?}"#, + has_seq, + set, + true, + ), + spec( + "essentialSound.applyPreset", + "Apply Sound Preset", + r#"{"clips":[id]?,"preset":str,"type":"dialogue|music|sfx|ambience"?}"#, + has_seq, + apply_preset, + true, + ), + spec("essentialSound.savePreset", "Save Sound Preset", r#"{"clips":[id]?,"name":str}"#, has_seq, save_preset, true), + spec("essentialSound.deletePreset", "Delete Sound Preset", r#"{"name":str,"type":"dialogue|music|sfx|ambience"?}"#, always, delete_preset, true), + spec("essentialSound.autoMatch", "Auto-Match Loudness", r#"{"clips":[id]?,"target":lufs?}"#, has_seq, auto_match, true), + spec("essentialSound.generateDucking", "Generate Ducking Keyframes", r#"{"clips":[id]?}"#, has_seq, generate_ducking, true), + ] +} diff --git a/crates/engine/src/essential_sound_tests.rs b/crates/engine/src/essential_sound_tests.rs new file mode 100644 index 00000000..2474972a --- /dev/null +++ b/crates/engine/src/essential_sound_tests.rs @@ -0,0 +1,467 @@ +//! Essential Sound: types, settings → clip effects, presets, loudness auto-match, ducking, repair +//! metrics through the render path, render-vs-export identity. + +use std::sync::Arc; + +use filmcraft_audio_dsp::LoudnessMeter; +use filmcraft_project::{AudioType, ClipId, TrackItem}; +use filmcraft_time::Tick; +use serde_json::{Value, json}; + +use super::*; + +const SR: u32 = 48_000; + +/// Deterministic xorshift noise in [-1, 1). +struct Rng(u64); +impl Rng { + fn next(&mut self) -> f32 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + ((self.0 >> 40) as f32 / (1u64 << 24) as f32) * 2.0 - 1.0 + } +} + +/// A speech-like signal: a voiced source (gliding f0 ≈ 110–160 Hz, harmonics falling 6 dB/oct up +/// to 4 kHz) under a syllable envelope (~4.5 syllables/s), only where `active(t)`. +fn speech(secs: f64, amp: f32, active: impl Fn(f64) -> bool) -> Vec { + let n = (secs * SR as f64) as usize; + let mut phase = 0.0f64; + (0..n) + .map(|i| { + let t = i as f64 / SR as f64; + if !active(t) { + return 0.0; + } + let f0 = 135.0 + 25.0 * (2.0 * std::f64::consts::PI * 0.7 * t).sin(); + phase += f0 / SR as f64; + let mut v = 0.0; + let mut k = 1; + while k as f64 * f0 < 4000.0 { + v += (2.0 * std::f64::consts::PI * phase * k as f64).sin() / k as f64; + k += 1; + } + let syl = 0.6 - 0.4 * (2.0 * std::f64::consts::PI * 4.5 * t).cos(); + (v * syl * 0.5) as f32 * amp + }) + .collect() +} + +fn tone(secs: f64, f: f64, amp: f32) -> Vec { + (0..(secs * SR as f64) as usize).map(|i| (amp as f64 * (2.0 * std::f64::consts::PI * f * i as f64 / SR as f64).sin()) as f32).collect() +} + +fn add(a: &[f32], b: &[f32]) -> Vec { + a.iter().zip(b).map(|(x, y)| x + y).collect() +} + +/// A session with one sequence and each signal on its own audio track (A1, A2…) at `start` seconds. +fn session_with(tracks: &[(Vec, f64)]) -> (Session, Vec) { + let mut s = Session::default(); + s.execute("file.newSequence", json!({"name": "es", "audio": tracks.len().max(2), "video": 1})).unwrap(); + let mut clips = Vec::new(); + for (k, (x, start)) in tracks.iter().enumerate() { + let inter: Vec = x.iter().flat_map(|v| [*v, *v]).collect(); + let bytes: Arc<[u8]> = crate::previews::write_wav_f32(&inter, SR).into(); + let item = crate::commands::import_bytes(&mut s, &format!("/sig{k}.wav"), bytes, None).unwrap(); + let r = s.execute("timeline.place", json!({"item": item.0, "audioTrack": format!("A{}", k + 1), "seconds": start})).unwrap(); + clips.push(ClipId(r["clips"][0].as_u64().unwrap())); + } + s.execute("edit.deselectAll", json!({})).unwrap(); + (s, clips) +} + +fn item(s: &Session, c: ClipId) -> TrackItem { + s.active_sequence().unwrap().find_item(c).unwrap().1.clone() +} + +fn es_ids(s: &Session, c: ClipId) -> Vec { + item(s, c).effects.iter().filter(|e| e.essential).map(|e| e.effect.clone()).collect() +} + +/// The full sequence mix for `[a, a + n)` samples, in requests of `chunk` samples. +fn mix(s: &Session, a: i64, n: usize, chunk: usize) -> [Vec; 2] { + let provider = s.media.provider(s.project.clone(), s.services.clone()); + let seq = s.active_sequence().unwrap(); + let mut out = [Vec::new(), Vec::new()]; + let mut pos = a; + while pos < a + n as i64 { + let m = chunk.min((a + n as i64 - pos) as usize); + let b = filmcraft_render::audio::mix_sequence(&s.project, seq, pos, m, &provider); + out[0].extend_from_slice(&b.channels[0]); + out[1].extend_from_slice(&b.channels[1]); + pos += m as i64; + } + out +} + +fn lufs(x: &[Vec; 2]) -> f64 { + let mut m = LoudnessMeter::new(SR as f64, 2); + m.process(&[&x[0], &x[1]]); + m.integrated() +} + +/// The clip's own processed signal (clip gain + effects) over its whole range. +fn clip_out(s: &Session, c: ClipId) -> [Vec; 2] { + let it = item(s, c); + let provider = s.media.provider(s.project.clone(), s.services.clone()); + let a0 = it.start.to_units_floor(SR as i64); + let n = (it.end().to_units_floor(SR as i64) - a0) as usize; + let mut out = [Vec::new(), Vec::new()]; + let mut pos = a0; + while pos < a0 + n as i64 { + let m = 4800.min((a0 + n as i64 - pos) as usize); + let b = filmcraft_render::audio::clip_signal(&it, pos, m, SR, &provider).unwrap(); + out[0].extend_from_slice(&b[0]); + out[1].extend_from_slice(&b[1]); + pos += m as i64; + } + out +} + +fn db(x: f64) -> f64 { + 20.0 * x.max(1e-12).log10() +} + +fn rms(x: &[f32]) -> f64 { + (x.iter().map(|v| (*v as f64).powi(2)).sum::() / x.len().max(1) as f64).sqrt() +} + +/// Amplitude of the `f` component (single-bin DFT). +fn tone_amp(x: &[f32], f: f64) -> f64 { + let (mut re, mut im) = (0.0f64, 0.0f64); + for (i, v) in x.iter().enumerate() { + let w = 2.0 * std::f64::consts::PI * f * i as f64 / SR as f64; + re += *v as f64 * w.cos(); + im += *v as f64 * w.sin(); + } + 2.0 * (re * re + im * im).sqrt() / x.len() as f64 +} + +fn secs(a: f64, b: f64) -> std::ops::Range { + (a * SR as f64) as usize..(b * SR as f64) as usize +} + +#[test] +fn commands_are_registered() { + for id in [ + "essentialSound.inspect", + "essentialSound.setType", + "essentialSound.clearType", + "essentialSound.set", + "essentialSound.applyPreset", + "essentialSound.savePreset", + "essentialSound.deletePreset", + "essentialSound.autoMatch", + "essentialSound.generateDucking", + ] { + let c = commands::find(id).unwrap_or_else(|| panic!("{id} missing")); + assert!(!c.label.is_empty() && !c.params.is_empty()); + } + assert!(!Session::default().is_enabled("essentialSound.setType"), "needs a sequence"); +} + +#[test] +fn type_settings_effects_undo_redo_and_clear() { + let (mut s, c) = session_with(&[(speech(2.0, 0.3, |_| true), 0.0)]); + let c = c[0]; + assert!(s.execute("essentialSound.set", json!({"clips": [c.0], "key": "repair.noise.on", "value": true})).is_err(), "untyped"); + s.execute("timeline.select", json!({"clips": [c.0]})).unwrap(); + s.execute("essentialSound.setType", json!({"type": "dialogue"})).unwrap(); + assert_eq!(item(&s, c).essential.as_ref().unwrap().kind, AudioType::Dialogue); + s.execute( + "essentialSound.set", + json!({"values": {"repair.noise.on": true, "repair.noise.amount": 7.0, "clarity.eq.on": true, "clarity.eqPreset": "Telephone"}}), + ) + .unwrap(); + assert_eq!(es_ids(&s, c), ["denoise", "parametric_eq"]); + let it = item(&s, c); + assert_eq!(it.effect("denoise").unwrap().f64_at("amount", Tick::ZERO), 70.0); + assert_eq!(it.effect("parametric_eq").unwrap().f64_at("low_gain", Tick::ZERO), -9.0, "Telephone at amount 5 of 10"); + // a drag (same key, no begin) is one undo step + for v in [6.0, 5.0, 4.0] { + s.execute("essentialSound.set", json!({"key": "repair.noise.amount", "value": v})).unwrap(); + } + assert_eq!(item(&s, c).effect("denoise").unwrap().f64_at("amount", Tick::ZERO), 40.0); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(item(&s, c).effect("denoise").unwrap().f64_at("amount", Tick::ZERO), 70.0); + s.execute("edit.redo", json!({})).unwrap(); + // inspect + let v = s.execute("essentialSound.inspect", json!({})).unwrap(); + assert_eq!(v["type"], "Dialogue"); + assert_eq!(v["clips"][0]["settings"]["repair"]["noise"]["amount"], 4.0); + assert_eq!(v["clips"][0]["effects"].as_array().unwrap().len(), 2); + assert!(v["presets"]["Dialogue"].as_array().unwrap().len() > 3); + assert_eq!(v["targets"]["Dialogue"], -23.0); + // type change resets, clear removes everything + s.execute("essentialSound.setType", json!({"type": "music"})).unwrap(); + assert!(es_ids(&s, c).is_empty()); + s.execute("essentialSound.clearType", json!({})).unwrap(); + assert!(item(&s, c).essential.is_none()); + s.execute("edit.undo", json!({})).unwrap(); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(es_ids(&s, c), ["denoise", "parametric_eq"]); + // bad input + assert!(s.execute("essentialSound.set", json!({"key": "repair.nope", "value": 1})).is_err()); + assert!(s.execute("essentialSound.setType", json!({"type": "podcast"})).is_err()); +} + +#[test] +fn presets_apply_save_and_delete() { + let (mut s, c) = session_with(&[(speech(2.0, 0.3, |_| true), 0.0)]); + let c = c[0]; + s.execute("essentialSound.applyPreset", json!({"clips": [c.0], "type": "dialogue", "preset": "Podcast Voice"})).unwrap(); + assert_eq!(es_ids(&s, c), ["highpass", "deesser", "dynamics", "parametric_eq", "speech_enhance"]); + assert_eq!(item(&s, c).essential.unwrap().preset, "Podcast Voice"); + s.execute("essentialSound.set", json!({"clips": [c.0], "key": "creative.reverb.on", "value": true})).unwrap(); + assert!(item(&s, c).essential.unwrap().preset.is_empty()); + s.execute("essentialSound.savePreset", json!({"clips": [c.0], "name": "My Voice"})).unwrap(); + assert_eq!(item(&s, c).essential.unwrap().preset, "My Voice"); + assert!(s.execute("essentialSound.savePreset", json!({"clips": [c.0], "name": "(Default)"})).is_err()); + s.execute("essentialSound.applyPreset", json!({"clips": [c.0], "preset": "(Default)"})).unwrap(); + assert!(es_ids(&s, c).is_empty()); + s.execute("essentialSound.applyPreset", json!({"clips": [c.0], "preset": "My Voice"})).unwrap(); + assert!(es_ids(&s, c).contains(&"studio_reverb".to_string())); + let v = s.execute("essentialSound.inspect", json!({"clips": [c.0]})).unwrap(); + assert!(v["presets"]["Dialogue"].as_array().unwrap().iter().any(|p| p["name"] == "My Voice" && p["builtin"] == false)); + s.execute("essentialSound.deletePreset", json!({"name": "My Voice"})).unwrap(); + assert!(s.execute("essentialSound.deletePreset", json!({"name": "Podcast Voice"})).is_err()); + assert!(s.execute("essentialSound.applyPreset", json!({"clips": [c.0], "preset": "My Voice"})).is_err()); +} + +#[test] +fn auto_match_hits_the_target_loudness() { + // quiet and loud speech, one with a repair/clarity chain, plus a preference target + let quiet = speech(12.0, 0.05, |t| (t % 3.0) < 2.2); + let loud = speech(12.0, 0.9, |t| (t % 2.0) < 1.5); + for (sig, chain, target) in [(quiet.clone(), false, None), (loud, false, None), (quiet, true, Some(-16.0))] { + let (mut s, c) = session_with(&[(sig, 0.0)]); + let c = c[0]; + s.execute("essentialSound.setType", json!({"clips": [c.0], "type": "dialogue"})).unwrap(); + if chain { + s.execute( + "essentialSound.set", + json!({"clips": [c.0], "values": {"repair.rumble.on": true, "repair.noise.on": true, "clarity.dynamics.on": true, "clarity.eq.on": true}}), + ) + .unwrap(); + } + if let Some(t) = target { + s.execute("prefs.set", json!({"key": "audio.dialogueTargetLufs", "value": t})).unwrap(); + } + let want = target.unwrap_or(-23.0); + let r = s.execute("essentialSound.autoMatch", json!({"clips": [c.0]})).unwrap(); + let got = lufs(&mix(&s, 0, 12 * SR as usize, 48_000)); + println!("auto-match: measured {:.2} LUFS → mix {got:.3} LUFS (target {want}), chain {chain}", r["clips"][0]["measuredLufs"].as_f64().unwrap()); + assert!((got - want).abs() <= 0.5, "mix {got} LUFS, target {want}"); + let st = item(&s, c).essential.unwrap(); + assert_eq!(st.loudness.target_lufs, Some(want)); + assert!((item(&s, c).gain_db - st.loudness.gain_db).abs() < 1e-9, "match gain is clip gain"); + // matching again is stable; switching Loudness off removes the gain + s.execute("essentialSound.autoMatch", json!({"clips": [c.0]})).unwrap(); + assert!((lufs(&mix(&s, 0, 12 * SR as usize, 48_000)) - want).abs() <= 0.5); + s.execute("essentialSound.set", json!({"clips": [c.0], "key": "loudness.enabled", "value": false})).unwrap(); + assert!(item(&s, c).gain_db.abs() < 1e-9); + } +} + +#[test] +fn ducking_generates_volume_keyframes_under_dialogue() { + // dialogue in [2, 4] and [6, 7.5] s; music for 10 s + let dlg = speech(10.0, 0.4, |t| (2.0..4.0).contains(&t) || (6.0..7.5).contains(&t)); + let music = add(&tone(10.0, 220.0, 0.2), &tone(10.0, 330.0, 0.1)); + let (mut s, c) = session_with(&[(dlg, 0.0), (music, 0.0)]); + s.execute("essentialSound.setType", json!({"clips": [c[0].0], "type": "dialogue"})).unwrap(); + s.execute("essentialSound.setType", json!({"clips": [c[1].0], "type": "music"})).unwrap(); + s.execute("essentialSound.set", json!({"clips": [c[1].0], "values": {"ducking.enabled": true, "ducking.reduceDb": -15.0, "ducking.fadeS": 0.8}})).unwrap(); + let r = s.execute("essentialSound.generateDucking", json!({"clips": [c[1].0]})).unwrap(); + let kf = r["clips"][0]["keyframes"].as_array().unwrap(); + let got: Vec<(f64, f64)> = kf.iter().map(|k| (k["seconds"].as_f64().unwrap(), k["levelDb"].as_f64().unwrap())).collect(); + println!("ducking keyframes: {got:?}"); + let want = [(1.2, 0.0), (2.0, -15.0), (4.0, -15.0), (4.8, 0.0), (5.2, 0.0), (6.0, -15.0), (7.5, -15.0), (8.3, 0.0)]; + assert_eq!(got.len(), want.len(), "{got:?}"); + for ((t, v), (wt, wv)) in got.iter().zip(want) { + assert!((t - wt).abs() <= 0.06, "keyframe at {t} s, expected {wt} s"); + assert!((v - wv).abs() < 1e-9); + } + // the keyframes are on the music clip's Volume level, and the mix follows them + let it = item(&s, c[1]); + let lvl = it.effect("volume").unwrap().param("level").unwrap(); + assert_eq!(lvl.keyframes.len(), 8); + assert_eq!(lvl.f64_at(it.source_time_at(Tick::from_seconds_f64(3.0))), -15.0); + assert_eq!(lvl.f64_at(it.source_time_at(Tick::from_seconds_f64(5.0))), 0.0); + // listen to the music alone + s.execute("timeline.select", json!({"clips": [c[0].0]})).unwrap(); + s.execute("clip.enable", json!({"clips": [c[0].0]})).unwrap(); + let m = mix(&s, 0, 10 * SR as usize, 48_000); + let during = tone_amp(&m[0][secs(2.5, 3.5)], 220.0); + let between = tone_amp(&m[0][secs(0.2, 1.0)], 220.0); + println!("music 220 Hz: {:.2} dB ducked vs open", db(during / between)); + assert!((db(during / between) + 15.0).abs() < 0.3); + // regenerate after changing the depth: replaces the keyframes, no stacking + s.execute("clip.enable", json!({"clips": [c[0].0]})).unwrap(); + s.execute("essentialSound.set", json!({"clips": [c[1].0], "key": "ducking.reduceDb", "value": -6.0})).unwrap(); + s.execute("essentialSound.generateDucking", json!({"clips": [c[1].0]})).unwrap(); + let lvl = item(&s, c[1]).effects.iter().find(|e| e.effect == "volume").unwrap().params["level"].clone(); + assert_eq!(lvl.keyframes.len(), 8); + assert!(lvl.keyframes.iter().all(|k| matches!(k.value.as_f64(), Some(v) if v == 0.0 || v == -6.0))); + // undo restores the previous keyframes + s.execute("edit.undo", json!({})).unwrap(); + let lvl = item(&s, c[1]).effects.iter().find(|e| e.effect == "volume").unwrap().params["level"].clone(); + assert!(lvl.keyframes.iter().any(|k| k.value.as_f64() == Some(-15.0))); + // nothing to duck against → no keyframes + assert!(s.execute("essentialSound.generateDucking", json!({"clips": [c[0].0]})).is_err(), "dialogue does not duck"); +} + +/// Run `sig` through one Dialogue repair setting and return (input, output) over 1…len−0.5 s. +fn repair(sig: Vec, values: Value) -> (Vec, Vec) { + let (mut s, c) = session_with(&[(sig.clone(), 0.0)]); + let c = c[0]; + s.execute("essentialSound.setType", json!({"clips": [c.0], "type": "dialogue"})).unwrap(); + s.execute("essentialSound.set", json!({"clips": [c.0], "values": values})).unwrap(); + let out = clip_out(&s, c); + (sig, out[0].clone()) +} + +#[test] +fn repair_stages_improve_their_metrics() { + let n_s = 6.0; + let voice = speech(n_s, 0.3, |t| (t % 1.0) < 0.6); + // DeHum: 60 Hz hum + harmonics + let hum = add(&add(&tone(n_s, 60.0, 0.05), &tone(n_s, 180.0, 0.03)), &voice); + let (x, y) = repair(hum, json!({"repair.dehum.on": true, "repair.dehum.amount": 5.0, "repair.humHz": 60})); + let r = secs(2.0, 5.5); + let (h0, h1) = (tone_amp(&x[r.clone()], 60.0), tone_amp(&y[r.clone()], 60.0)); + println!("DeHum: 60 Hz {:.1} dB", db(h1 / h0)); + assert!(db(h1 / h0) < -20.0); + // Reduce Rumble: 25 Hz rumble + let rum = add(&tone(n_s, 25.0, 0.1), &voice); + let (x, y) = repair(rum, json!({"repair.rumble.on": true, "repair.rumble.amount": 5.0})); + let (r0, r1) = (tone_amp(&x[r.clone()], 25.0), tone_amp(&y[r.clone()], 25.0)); + println!("Reduce Rumble: 25 Hz {:.1} dB", db(r1 / r0)); + assert!(db(r1 / r0) < -15.0); + // Reduce Noise: white noise floor, measured in the pauses (latency-aligned by the render path) + let mut g = Rng(0x1234_5678); + let noise: Vec = (0..voice.len()).map(|_| g.next() * 0.01).collect(); + let (x, y) = repair(add(&voice, &noise), json!({"repair.noise.on": true, "repair.noise.amount": 6.0})); + let pause = secs(4.7, 4.95); + let nr = db(rms(&y[pause.clone()]) / rms(&x[pause.clone()])); + let speech_change = db(rms(&y[secs(4.1, 4.5)]) / rms(&x[secs(4.1, 4.5)])); + println!("Reduce Noise: floor {nr:.1} dB, speech {speech_change:.2} dB"); + assert!(nr < -6.0); + assert!(speech_change.abs() < 2.0); +} + +#[test] +fn deess_and_dereverb_improve_their_metrics() { + let n_s = 6.0; + let voice = speech(n_s, 0.3, |t| (t % 1.0) < 0.6); + // sibilants: 5–10 kHz noise bursts (2nd-difference of white noise ≈ high-passed) at the + // start of every syllable group + let mut g = Rng(99); + let mut prev = (0.0f32, 0.0f32); + let sib: Vec = (0..voice.len()) + .map(|i| { + let w = g.next(); + let hp = w - 2.0 * prev.0 + prev.1; + prev = (w, prev.0); + let t = i as f64 / SR as f64; + if (t % 1.0) < 0.12 { hp * 0.12 } else { 0.0 } + }) + .collect(); + let (x, y) = repair(add(&voice, &sib), json!({"repair.deess.on": true, "repair.deess.amount": 8.0})); + let hi = |v: &[f32]| { + // energy above ~5 kHz: first difference twice (a crude high-pass), RMS + let d: Vec = v.windows(3).map(|w| w[2] - 2.0 * w[1] + w[0]).collect(); + rms(&d) + }; + let burst = secs(4.0, 4.1); + let red = db(hi(&y[burst.clone()]) / hi(&x[burst.clone()])); + let body = db(tone_amp(&y[secs(4.2, 4.55)], 135.0).max(1e-9) / tone_amp(&x[secs(4.2, 4.55)], 135.0).max(1e-9)); + println!("DeEss: sibilance {red:.1} dB, voice body {body:.2} dB"); + assert!(red < -4.0, "sibilance only {red} dB"); + // Reduce Reverb: dry bursts convolved with an exponentially decaying noise tail (RT60 0.8 s) + let dry = speech(n_s, 0.3, |t| (t % 0.5) < 0.1); + let mut g = Rng(7); + let ir_len = (0.8 * SR as f64) as usize; + let ir: Vec = (0..ir_len).map(|i| if i == 0 { 1.0 } else { g.next() * 0.08 * (-6.9 * i as f64 / ir_len as f64).exp() as f32 }).collect(); + // sparse convolution (decimated IR taps are enough for a diffuse tail) + let mut wet = dry.clone(); + for (k, h) in ir.iter().enumerate().skip(1).step_by(7) { + for i in k..dry.len() { + wet[i] += dry[i - k] * h * 7.0f32.sqrt(); + } + } + let (x, y) = repair(wet, json!({"repair.reverb.on": true, "repair.reverb.amount": 10.0})); + let ratio = |v: &[f32]| { + // tail energy (gaps 0.2–0.45 s after each burst) relative to burst energy, over 3…5.5 s + let (mut b, mut t) = (0.0, 0.0); + for k in 6..11 { + let s0 = k as f64 * 0.5; + b += rms(&v[secs(s0 + 0.01, s0 + 0.09)]).powi(2); + t += rms(&v[secs(s0 + 0.2, s0 + 0.45)]).powi(2); + } + 10.0 * (t / b).log10() + }; + let (rx, ry) = (ratio(&x), ratio(&y)); + println!("Reduce Reverb: tail/burst {rx:.1} dB → {ry:.1} dB"); + assert!(ry < rx - 3.0); +} + +#[test] +fn rendered_mix_is_identical_however_requests_are_cut_and_matches_export() { + let mut g = Rng(42); + let voice: Vec = speech(4.0, 0.3, |t| (t % 1.0) < 0.6).iter().map(|v| v + g.next() * 0.005).collect(); + let (mut s, c) = session_with(&[(voice, 0.5)]); + s.execute("essentialSound.applyPreset", json!({"clips": [c[0].0], "type": "dialogue", "preset": "Podcast Voice"})).unwrap(); + s.execute( + "essentialSound.set", + json!({"clips": [c[0].0], "values": {"repair.noise.on": true, "repair.reverb.on": true, "repair.dehum.on": true, "creative.reverb.on": true}}), + ) + .unwrap(); + s.execute("essentialSound.autoMatch", json!({"clips": [c[0].0]})).unwrap(); + let n = 5 * SR as usize; + let whole = mix(&s, 0, n, n); + for chunk in [512usize, 1000, 4096 + 7] { + let cut = mix(&s, 0, n, chunk); + let d = whole[0].iter().chain(&whole[1]).zip(cut[0].iter().chain(&cut[1])).map(|(a, b)| (a - b).abs()).fold(0.0f32, f32::max); + assert!(d <= 1e-6, "chunk {chunk}: max diff {d}"); + } + assert!(rms(&whole[0]) > 0.01); + // export (WAV, 16-bit) equals the mix + let dir = std::env::temp_dir().join(format!("fc-es-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("es.wav"); + s.execute("file.exportMedia", json!({"path": path.to_string_lossy(), "format": "wav", "wait": true})).unwrap(); + let bytes: Arc<[u8]> = std::fs::read(&path).unwrap().into(); + let src = filmcraft_media::open_bytes("es.wav", bytes, &[]).unwrap(); + let back = src.audio(0, n, SR).unwrap(); + let d = (0..n).map(|i| (back.channels[0][i] - whole[0][i]).abs().max((back.channels[1][i] - whole[1][i]).abs())).fold(0.0f32, f32::max); + println!("export vs render: max diff {d:e}"); + assert!(d <= 2.0 / 32768.0, "export differs from render by {d}"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// `cargo test --release -p filmcraft-engine essential_sound_tests::perf -- --ignored --nocapture` +#[test] +#[ignore] +fn perf_full_dialogue_chain_realtime_factor() { + let secs_len = 60.0; + let mut g = Rng(5); + let voice: Vec = speech(secs_len, 0.3, |t| (t % 1.0) < 0.6).iter().map(|v| v + g.next() * 0.005).collect(); + let (mut s, c) = session_with(&[(voice, 0.0)]); + s.execute("essentialSound.setType", json!({"clips": [c[0].0], "type": "dialogue"})).unwrap(); + s.execute( + "essentialSound.set", + json!({"clips": [c[0].0], "values": { + "repair.noise.on": true, "repair.rumble.on": true, "repair.dehum.on": true, "repair.deess.on": true, "repair.reverb.on": true, + "clarity.dynamics.on": true, "clarity.eq.on": true, "clarity.enhance.on": true, "creative.reverb.on": true}}), + ) + .unwrap(); + assert_eq!(es_ids(&s, c[0]).len(), 9); + let t0 = std::time::Instant::now(); + let out = clip_out(&s, c[0]); + let el = t0.elapsed().as_secs_f64(); + assert!(rms(&out[0]) > 0.0); + println!("full Dialogue chain (9 effects), 60 s stereo 48 kHz: {el:.2} s → {:.1}× realtime", secs_len / el); +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index e9a9f278..54ae949c 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -12,6 +12,7 @@ pub mod autosave; pub mod captions; pub mod commands; pub mod demo; +pub mod essential_sound; pub mod graphics; pub mod interchange; pub mod media_pool; @@ -555,6 +556,8 @@ pub fn media_duration(p: &Project, _pool: &MediaPool, id: ItemId) -> Option params: Default::default(), masks: vec![], post_fader: false, + essential: false, }) } diff --git a/crates/project/src/effect.rs b/crates/project/src/effect.rs index 9d0378bd..5e93f788 100644 --- a/crates/project/src/effect.rs +++ b/crates/project/src/effect.rs @@ -86,6 +86,7 @@ impl EffectDef { params: self.params.iter().map(|p| (p.id.to_string(), Param::new(p.default.clone()))).collect(), masks: Vec::new(), post_fader: false, + essential: false, } } } @@ -117,6 +118,9 @@ pub struct EffectInstance { /// Track/Mix inserts only: process after the fader instead of before it. #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub post_fader: bool, + /// Managed by the Essential Sound panel (see [`crate::essential`]). + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub essential: bool, } impl EffectInstance { @@ -252,6 +256,7 @@ const A_FILTER: &[&str] = &["Audio Effects", "Filter and EQ"]; const A_NOISE: &[&str] = &["Audio Effects", "Noise Reduction/Restoration"]; const A_REVERB: &[&str] = &["Audio Effects", "Reverb"]; const A_SPECIAL: &[&str] = &["Audio Effects", "Special"]; +const A_STEREO: &[&str] = &["Audio Effects", "Stereo Imagery"]; const A_TIME: &[&str] = &["Audio Effects", "Time and Pitch"]; const A_TRANS: &[&str] = &["Audio Transitions", "Crossfade"]; @@ -882,7 +887,24 @@ fn build_effects() -> Vec { audio("denoise", "DeNoise", A_NOISE, vec![f("amount", "Amount", 40.0, 0.0, 100.0, "%")]), audio("dehummer", "DeHummer", A_NOISE, vec![ch("freq", "Frequency", &["50 Hz", "60 Hz"], 1), f("gain", "Gain", -40.0, -80.0, 0.0, "dB")]), audio("declicker", "Automatic Click Remover", A_NOISE, vec![]), - audio("dereverb", "DeReverb", A_NOISE, vec![]), + audio( + "dereverb", + "DeReverb", + A_NOISE, + vec![f("amount", "Amount", 50.0, 0.0, 100.0, "%"), fs("rt60", "Decay Time (RT60)", 0.8, (0.1, 5.0), (0.1, 3.0), "s", 2)], + ), + audio( + "deesser", + "DeEsser", + A_AMP, + vec![ + fs("frequency", "Frequency", 6000.0, (2000.0, 12000.0), (2000.0, 12000.0), "Hz", 0), + fs("threshold", "Threshold", -12.0, (-40.0, 0.0), (-40.0, 0.0), "dB", 1), + fs("reduction", "Maximum Reduction", 8.0, (0.0, 24.0), (0.0, 24.0), "dB", 1), + ], + ), + audio("speech_enhance", "Enhance Speech", A_NOISE, vec![f("mix", "Mix", 100.0, 0.0, 100.0, "%"), ch("tone", "Voice", &["Low Tone", "High Tone"], 0)]), + audio("stereo_width", "Stereo Width", A_STEREO, vec![fs("width", "Width", 100.0, (0.0, 200.0), (0.0, 200.0), "%", 0)]), audio( "studio_reverb", "Studio Reverb", diff --git a/crates/project/src/essential.rs b/crates/project/src/essential.rs new file mode 100644 index 00000000..d3a3d821 --- /dev/null +++ b/crates/project/src/essential.rs @@ -0,0 +1,885 @@ +//! Essential Sound: clip audio types (Dialogue / Music / SFX / Ambience), their section settings +//! and presets, and how those settings become ordinary clip effects. +//! +//! The panel never processes audio itself. Like Premiere, it drives effects on the clip "under the +//! hood": Reduce Rumble is a Highpass, Reduce Noise a DeNoise, DeHum a DeHummer, DeEss a DeEsser, +//! Reduce Reverb a DeReverb, Dynamics a Dynamics Processing, EQ a Parametric Equalizer, Enhance +//! Speech the Enhance Speech chain, Stereo Width the Stereo Width effect and Reverb a Studio +//! Reverb. These instances carry [`EffectInstance::essential`], appear in Effect Controls (where +//! their parameters can be keyframed) and render through the normal clip-effect path, so the +//! mixer, playback and export need nothing special. Loudness auto-match writes clip gain, Clip +//! Volume the intrinsic Volume level, Pan the Panner, and Ducking writes Volume keyframes. +//! +//! [`apply`] reconciles a clip's essential effects with its settings after every change: missing +//! effects are inserted in canonical order, effects of slots that were switched off are removed, +//! and only parameters whose derived value changed are written (at the playhead, so keyframes the +//! user added in Effect Controls survive unrelated slider moves). + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::effect::{EffectInstance, find_effect}; +use crate::keyframe::ParamValue; +use crate::{Param, TrackItem}; +use filmcraft_time::Tick; + +/// Clip audio type. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum AudioType { + Dialogue, + Music, + #[serde(rename = "SFX")] + Sfx, + Ambience, +} + +impl AudioType { + pub const ALL: [AudioType; 4] = [AudioType::Dialogue, AudioType::Music, AudioType::Sfx, AudioType::Ambience]; + pub fn label(self) -> &'static str { + match self { + AudioType::Dialogue => "Dialogue", + AudioType::Music => "Music", + AudioType::Sfx => "SFX", + AudioType::Ambience => "Ambience", + } + } + /// Command-parameter id (`dialogue`, `music`, `sfx`, `ambience`). + pub fn id(self) -> &'static str { + match self { + AudioType::Dialogue => "dialogue", + AudioType::Music => "music", + AudioType::Sfx => "sfx", + AudioType::Ambience => "ambience", + } + } + pub fn parse(s: &str) -> Option { + AudioType::ALL.into_iter().find(|t| t.id().eq_ignore_ascii_case(s) || t.label().eq_ignore_ascii_case(s)) + } + /// The sections the panel shows for this type (Premiere's layout). + pub fn sections(self) -> &'static [Section] { + match self { + AudioType::Dialogue => &[Section::Loudness, Section::Repair, Section::Clarity, Section::Creative], + AudioType::Music => &[Section::Loudness, Section::Duration, Section::Ducking], + AudioType::Sfx => &[Section::Loudness, Section::Creative, Section::Pan], + AudioType::Ambience => &[Section::Loudness, Section::Creative, Section::Ducking], + } + } + pub fn has(self, s: Section) -> bool { + self.sections().contains(&s) + } +} + +/// A collapsible panel section. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum Section { + Loudness, + Repair, + Clarity, + Creative, + Ducking, + Duration, + Pan, +} + +impl Section { + pub fn label(self) -> &'static str { + match self { + Section::Loudness => "Loudness", + Section::Repair => "Repair", + Section::Clarity => "Clarity", + Section::Creative => "Creative", + Section::Ducking => "Ducking", + Section::Duration => "Duration", + Section::Pan => "Pan", + } + } + /// Settings key of the section's on/off switch (`None` for Duration, which has none). + pub fn key(self) -> Option<&'static str> { + Some(match self { + Section::Loudness => "loudness.enabled", + Section::Repair => "repair.enabled", + Section::Clarity => "clarity.enabled", + Section::Creative => "creative.enabled", + Section::Ducking => "ducking.enabled", + Section::Pan => "pan.enabled", + Section::Duration => return None, + }) + } +} + +/// A checkbox + 0…10 slider pair. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +pub struct Slot { + pub on: bool, + pub amount: f64, +} + +impl Slot { + const fn off(amount: f64) -> Self { + Slot { on: false, amount } + } + const fn at(amount: f64) -> Self { + Slot { on: true, amount } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct Loudness { + pub enabled: bool, + /// Integrated loudness measured by the last Auto-Match (LUFS, before the match gain). + pub measured_lufs: Option, + /// Target used by the last Auto-Match. + pub target_lufs: Option, + /// Gain the match applied (dB); part of the clip gain while the section is on. + pub gain_db: f64, +} + +impl Default for Loudness { + fn default() -> Self { + Loudness { enabled: true, measured_lufs: None, target_lufs: None, gain_db: 0.0 } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct Repair { + pub enabled: bool, + pub noise: Slot, + pub rumble: Slot, + pub dehum: Slot, + /// Mains frequency for DeHum: 50 or 60 Hz. + pub hum_hz: u32, + pub deess: Slot, + pub reverb: Slot, +} + +impl Default for Repair { + fn default() -> Self { + Repair { + enabled: true, + noise: Slot::off(5.0), + rumble: Slot::off(5.0), + dehum: Slot::off(5.0), + hum_hz: 60, + deess: Slot::off(5.0), + reverb: Slot::off(5.0), + } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct Clarity { + pub enabled: bool, + pub dynamics: Slot, + pub eq: Slot, + pub eq_preset: String, + /// Enhance Speech; `amount` is the mix (0…10). + pub enhance: Slot, + /// 0 = low tone (lower voices), 1 = high tone. + pub enhance_tone: u32, +} + +impl Default for Clarity { + fn default() -> Self { + Clarity { enabled: true, dynamics: Slot::off(5.0), eq: Slot::off(5.0), eq_preset: EQ_PRESETS[0].name.into(), enhance: Slot::off(10.0), enhance_tone: 0 } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct Creative { + pub enabled: bool, + pub reverb: Slot, + pub reverb_preset: String, + /// Stereo Width (Ambience): 0…10 → 100…200 %. + pub width: Slot, +} + +impl Default for Creative { + fn default() -> Self { + Creative { enabled: true, reverb: Slot::off(5.0), reverb_preset: REVERB_PRESETS[0].name.into(), width: Slot::off(5.0) } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct Ducking { + pub enabled: bool, + /// Duck against clips of these types. + pub against: Vec, + /// Also duck against clips without an audio type. + pub against_untyped: bool, + /// 0…10: how quiet a trigger may be and still count. + pub sensitivity: f64, + /// Reduction in dB (negative). + pub reduce_db: f64, + /// Fade length (s). + pub fade_s: f64, +} + +impl Default for Ducking { + fn default() -> Self { + Ducking { enabled: false, against: vec![AudioType::Dialogue], against_untyped: false, sensitivity: 6.0, reduce_db: -15.0, fade_s: 0.8 } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct Pan { + pub enabled: bool, + /// −100 (left) … +100 (right). + pub value: f64, +} + +/// Clip Volume (footer): an offset in dB on the clip's Volume level while `on`. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct ClipVolume { + pub on: bool, + pub level_db: f64, +} + +/// A clip's Essential Sound settings. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct EssentialSound { + pub kind: AudioType, + /// Last applied preset ("(Default)" or a preset name); cleared when a setting is changed by hand. + #[serde(default)] + pub preset: String, + #[serde(default)] + pub loudness: Loudness, + #[serde(default)] + pub repair: Repair, + #[serde(default)] + pub clarity: Clarity, + #[serde(default)] + pub creative: Creative, + #[serde(default)] + pub ducking: Ducking, + #[serde(default)] + pub pan: Pan, + #[serde(default)] + pub volume: ClipVolume, + #[serde(default)] + pub mute: bool, +} + +pub const DEFAULT_PRESET: &str = "(Default)"; + +impl EssentialSound { + /// The type's default settings. + pub fn new(kind: AudioType) -> Self { + let mut ducking = Ducking::default(); + if kind == AudioType::Ambience { + ducking.reduce_db = -10.0; + } + EssentialSound { + kind, + preset: DEFAULT_PRESET.into(), + loudness: Loudness::default(), + repair: Repair::default(), + clarity: Clarity::default(), + creative: Creative::default(), + ducking, + pan: Pan::default(), + volume: ClipVolume::default(), + mute: false, + } + } + + /// The settings as JSON (keys as in [`EssentialSound::set`]). + pub fn to_value(&self) -> Value { + serde_json::to_value(self).unwrap_or(Value::Null) + } + + /// A setting by dotted key (`repair.noise.amount`, `clarity.eqPreset`…; snake_case is + /// accepted too). + pub fn get(&self, key: &str) -> Option { + let v = self.to_value(); + let mut cur = &v; + for part in key.split('.') { + cur = cur.get(snake(part).as_str())?; + } + Some(cur.clone()) + } + + /// Set one setting by dotted key. Values are validated and clamped; the type cannot be changed + /// here (use a type assignment, which resets the settings). + pub fn set(&mut self, key: &str, value: &Value) -> Result<(), String> { + let parts: Vec = key.split('.').map(snake).collect(); + if parts.first().map(String::as_str) == Some("kind") { + return Err("the audio type is changed with essentialSound.setType".into()); + } + let mut v = self.to_value(); + { + let mut cur = &mut v; + for part in &parts { + cur = cur.get_mut(part.as_str()).ok_or_else(|| format!("unknown Essential Sound setting `{key}`"))?; + } + let ok = match (&*cur, value) { + (Value::Bool(_), Value::Bool(_)) | (Value::String(_), Value::String(_)) | (Value::Array(_), Value::Array(_)) => true, + (Value::Number(_), Value::Number(_)) | (Value::Null, Value::Number(_)) => true, + (Value::Null, Value::Null) => true, + (Value::Number(_), Value::Null) => false, + _ => false, + }; + if !ok { + return Err(format!("`{key}`: expected {}", type_name(cur))); + } + *cur = value.clone(); + } + let mut next: EssentialSound = serde_json::from_value(v).map_err(|e| format!("`{key}`: {e}"))?; + next.clamp(); + if parts.first().map(String::as_str) == Some("clarity") + && parts.get(1).map(String::as_str) == Some("eq_preset") + && eq_preset(&next.clarity.eq_preset).is_none() + { + return Err(format!("unknown EQ preset `{}`", next.clarity.eq_preset)); + } + if parts.first().map(String::as_str) == Some("creative") + && parts.get(1).map(String::as_str) == Some("reverb_preset") + && reverb_preset(&next.creative.reverb_preset).is_none() + { + return Err(format!("unknown reverb preset `{}`", next.creative.reverb_preset)); + } + next.kind = self.kind; + if !matches!(parts.first().map(String::as_str), Some("preset" | "volume" | "mute")) { + next.preset.clear(); + } + *self = next; + Ok(()) + } + + fn clamp(&mut self) { + let c = |v: &mut f64, lo: f64, hi: f64, def: f64| *v = if v.is_finite() { v.clamp(lo, hi) } else { def }; + for s in [ + &mut self.repair.noise, + &mut self.repair.rumble, + &mut self.repair.dehum, + &mut self.repair.deess, + &mut self.repair.reverb, + &mut self.clarity.dynamics, + &mut self.clarity.eq, + &mut self.clarity.enhance, + &mut self.creative.reverb, + &mut self.creative.width, + ] { + c(&mut s.amount, 0.0, 10.0, 5.0); + } + self.repair.hum_hz = if self.repair.hum_hz < 55 { 50 } else { 60 }; + self.clarity.enhance_tone = self.clarity.enhance_tone.min(1); + c(&mut self.ducking.sensitivity, 0.0, 10.0, 6.0); + c(&mut self.ducking.reduce_db, -40.0, 0.0, -15.0); + c(&mut self.ducking.fade_s, 0.0, 5.0, 0.8); + self.ducking.against.dedup(); + c(&mut self.pan.value, -100.0, 100.0, 0.0); + c(&mut self.volume.level_db, -96.0, 15.0, 0.0); + c(&mut self.loudness.gain_db, -96.0, 96.0, 0.0); + } + + /// Clip gain this state contributes (the Auto-Match gain while Loudness is on). + pub fn loudness_gain_db(&self) -> f64 { + if self.loudness.enabled { self.loudness.gain_db } else { 0.0 } + } + /// Offset this state adds to the Volume level. + pub fn volume_offset_db(&self) -> f64 { + if self.volume.on { self.volume.level_db } else { 0.0 } + } + /// Panner balance this state sets (`None` when Pan is not in use). + pub fn pan_value(&self) -> Option { + self.kind.has(Section::Pan).then_some(if self.pan.enabled { self.pan.value } else { 0.0 }) + } + + /// The clip effects these settings call for, in processing order, with their parameter values. + pub fn effects(&self) -> Vec { + let mut out = Vec::new(); + let k = self.kind; + let mut push = |id: &str, section_on: bool, params: &[(&str, ParamValue)]| { + if let Some(d) = find_effect(id) { + let mut e = d.instance(); + e.essential = true; + e.enabled = section_on; + for (pid, v) in params { + if let Some(p) = e.params.get_mut(*pid) { + p.value = v.clone(); + } + } + out.push(e); + } + }; + let f = ParamValue::Float; + if k.has(Section::Repair) { + let r = &self.repair; + if r.rumble.on { + push("highpass", r.enabled, &[("cutoff", f(rumble_cutoff(r.rumble.amount)))]); + } + if r.noise.on { + push("denoise", r.enabled, &[("amount", f(r.noise.amount * 10.0))]); + } + if r.dehum.on { + push("dehummer", r.enabled, &[("freq", ParamValue::Choice(u32::from(r.hum_hz == 60))), ("gain", f(-r.dehum.amount * 8.0))]); + } + if r.deess.on { + let a = r.deess.amount; + push("deesser", r.enabled, &[("threshold", f(-6.0 - a * 1.2)), ("reduction", f(a * 2.4))]); + } + if r.reverb.on { + push("dereverb", r.enabled, &[("amount", f(r.reverb.amount * 10.0))]); + } + } + if k.has(Section::Clarity) { + let c = &self.clarity; + if c.dynamics.on { + let a = c.dynamics.amount; + push("dynamics", c.enabled, &[("threshold", f(-12.0 - a * 2.4)), ("ratio", f(1.5 + a * 0.45)), ("attack", f(5.0)), ("release", f(120.0))]); + } + if c.eq.on { + let p = eq_preset(&c.eq_preset).unwrap_or(&EQ_PRESETS[0]); + let s = c.eq.amount / 10.0; + push( + "parametric_eq", + c.enabled, + &[ + ("low_freq", f(p.low.0)), + ("low_gain", f(p.low.1 * s)), + ("mid_freq", f(p.mid.0)), + ("mid_gain", f(p.mid.1 * s)), + ("mid_q", f(p.mid_q)), + ("high_freq", f(p.high.0)), + ("high_gain", f(p.high.1 * s)), + ], + ); + } + if c.enhance.on { + push("speech_enhance", c.enabled, &[("mix", f(c.enhance.amount * 10.0)), ("tone", ParamValue::Choice(c.enhance_tone))]); + } + } + if k.has(Section::Creative) { + let c = &self.creative; + if c.width.on && k == AudioType::Ambience { + push("stereo_width", c.enabled, &[("width", f(100.0 + c.width.amount * 10.0))]); + } + if c.reverb.on { + let p = reverb_preset(&c.reverb_preset).unwrap_or(&REVERB_PRESETS[0]); + push( + "studio_reverb", + c.enabled, + &[("room", f(p.room)), ("decay", f(p.decay)), ("damping", f(p.damping)), ("dry", f(100.0)), ("wet", f(c.reverb.amount * 8.0))], + ); + } + } + out + } +} + +fn snake(s: &str) -> String { + let mut o = String::with_capacity(s.len() + 4); + for c in s.chars() { + if c.is_ascii_uppercase() { + o.push('_'); + o.push(c.to_ascii_lowercase()); + } else { + o.push(c); + } + } + o +} + +fn type_name(v: &Value) -> &'static str { + match v { + Value::Bool(_) => "a boolean", + Value::Number(_) | Value::Null => "a number", + Value::String(_) => "a string", + Value::Array(_) => "a list", + Value::Object(_) => "a setting inside this group", + } +} + +/// Reduce Rumble amount (0…10) → highpass cutoff (30…150 Hz). +pub fn rumble_cutoff(amount: f64) -> f64 { + 30.0 + amount * 12.0 +} + +/// Canonical processing order of the effects Essential Sound manages. +pub const EFFECT_ORDER: &[&str] = + &["highpass", "denoise", "dehummer", "deesser", "dereverb", "dynamics", "parametric_eq", "speech_enhance", "stereo_width", "studio_reverb"]; + +fn rank(id: &str) -> usize { + EFFECT_ORDER.iter().position(|e| *e == id).unwrap_or(EFFECT_ORDER.len()) +} + +/// Reconcile `item`'s essential effects, clip gain, Volume and Panner with `item.essential`, +/// given the settings before the change (`old`; `None` = nothing was applied yet). Parameters +/// whose derived value did not change are left alone; changed ones are written at media time `mt` +/// (as a keyframe when the parameter is animated). +pub fn apply(item: &mut TrackItem, old: Option<&EssentialSound>, mt: Tick) { + let new = item.essential.clone(); + let want = new.as_ref().map(EssentialSound::effects).unwrap_or_default(); + let had = old.map(EssentialSound::effects).unwrap_or_default(); + // remove essential effects no longer wanted + item.effects.retain(|e| !e.essential || want.iter().any(|w| w.effect == e.effect)); + for w in &want { + match item.effects.iter().position(|e| e.essential && e.effect == w.effect) { + Some(i) => { + let prev = had.iter().find(|h| h.effect == w.effect); + let e = &mut item.effects[i]; + e.enabled = w.enabled; + for (pid, p) in &w.params { + let changed = prev.and_then(|h| h.params.get(pid)).is_none_or(|hp| hp.value != p.value); + if changed && let Some(dst) = e.params.get_mut(pid) { + dst.set_at(mt, p.value.clone()); + } + } + } + None => { + // after the last essential effect of a lower rank, else before the first other + // standard effect + let r = rank(&w.effect); + let first_std = item.effects.iter().position(|e| !e.def().is_some_and(|d| d.intrinsic)).unwrap_or(item.effects.len()); + let pos = item.effects.iter().rposition(|e| e.essential && rank(&e.effect) < r).map(|i| i + 1).unwrap_or(first_std); + item.effects.insert(pos, w.clone()); + } + } + } + // clip gain (Auto-Match), Volume offset, Pan + let g_old = old.map(EssentialSound::loudness_gain_db).unwrap_or(0.0); + let g_new = new.as_ref().map(EssentialSound::loudness_gain_db).unwrap_or(0.0); + if (g_new - g_old).abs() > 1e-12 { + item.gain_db = (item.gain_db + g_new - g_old).clamp(-96.0, 96.0); + } + let v_old = old.map(EssentialSound::volume_offset_db).unwrap_or(0.0); + let v_new = new.as_ref().map(EssentialSound::volume_offset_db).unwrap_or(0.0); + if (v_new - v_old).abs() > 1e-12 + && let Some(p) = item.effect_mut("volume").and_then(|e| e.param_mut("level")) + { + offset_param(p, v_new - v_old); + } + let p_old = old.and_then(EssentialSound::pan_value); + let p_new = new.as_ref().and_then(EssentialSound::pan_value); + if p_new != p_old + && let Some(p) = item.effect_mut("panner").and_then(|e| e.param_mut("balance")) + { + p.set_at(mt, ParamValue::Float(p_new.unwrap_or(0.0))); + } +} + +/// Add `db` to a level parameter: the static value, or every keyframe (keeps ducking shapes). +pub fn offset_param(p: &mut Param, db: f64) { + let add = |v: &mut ParamValue| { + if let ParamValue::Float(x) = v { + *x = (*x + db).clamp(-287.5, 15.0); + } + }; + add(&mut p.value); + for k in &mut p.keyframes { + add(&mut k.value); + } +} + +// ------------------------------------------------------------------------------------- presets + +/// An EQ preset for Clarity ▸ EQ: (frequency Hz, gain dB at full amount) per band. Values are our +/// own. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct EqPreset { + pub name: &'static str, + pub low: (f64, f64), + pub mid: (f64, f64), + pub mid_q: f64, + pub high: (f64, f64), +} + +pub const EQ_PRESETS: &[EqPreset] = &[ + EqPreset { name: "Balanced Voice", low: (120.0, -3.0), mid: (3000.0, 3.0), mid_q: 1.0, high: (10000.0, 2.0) }, + EqPreset { name: "Presence Lift", low: (100.0, 0.0), mid: (4000.0, 5.0), mid_q: 0.8, high: (12000.0, 1.0) }, + EqPreset { name: "Warm Voice", low: (200.0, 3.0), mid: (3500.0, 1.5), mid_q: 1.0, high: (9000.0, -2.0) }, + EqPreset { name: "Clear Muddiness", low: (100.0, -2.0), mid: (300.0, -6.0), mid_q: 1.4, high: (10000.0, 1.0) }, + EqPreset { name: "Soften Harshness", low: (100.0, 0.0), mid: (3200.0, -5.0), mid_q: 1.2, high: (9000.0, -2.0) }, + EqPreset { name: "Bright Air", low: (100.0, 0.0), mid: (3000.0, 1.0), mid_q: 1.0, high: (8000.0, 6.0) }, + EqPreset { name: "Radio Voice", low: (150.0, -10.0), mid: (2500.0, 6.0), mid_q: 1.0, high: (6000.0, -8.0) }, + EqPreset { name: "Telephone", low: (400.0, -18.0), mid: (1500.0, 6.0), mid_q: 0.7, high: (3500.0, -18.0) }, +]; + +pub fn eq_preset(name: &str) -> Option<&'static EqPreset> { + EQ_PRESETS.iter().find(|p| p.name.eq_ignore_ascii_case(name)) +} + +/// A reverb preset for Creative ▸ Reverb (Studio Reverb room / decay / damping, %). Our own values. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct ReverbPreset { + pub name: &'static str, + pub room: f64, + pub decay: f64, + pub damping: f64, +} + +pub const REVERB_PRESETS: &[ReverbPreset] = &[ + ReverbPreset { name: "Small Room", room: 25.0, decay: 25.0, damping: 60.0 }, + ReverbPreset { name: "Warm Room", room: 40.0, decay: 35.0, damping: 80.0 }, + ReverbPreset { name: "Medium Room", room: 50.0, decay: 45.0, damping: 50.0 }, + ReverbPreset { name: "Bright Studio", room: 45.0, decay: 40.0, damping: 15.0 }, + ReverbPreset { name: "Large Hall", room: 85.0, decay: 70.0, damping: 40.0 }, + ReverbPreset { name: "Stone Church", room: 100.0, decay: 92.0, damping: 30.0 }, + ReverbPreset { name: "Open Air", room: 90.0, decay: 15.0, damping: 70.0 }, +]; + +pub fn reverb_preset(name: &str) -> Option<&'static ReverbPreset> { + REVERB_PRESETS.iter().find(|p| p.name.eq_ignore_ascii_case(name)) +} + +/// A named Essential Sound preset for one audio type: settings applied on top of the type's +/// defaults. +#[derive(Clone, Debug, PartialEq)] +pub struct Preset { + pub name: String, + pub kind: AudioType, + pub settings: EssentialSound, + /// Built in (cannot be deleted). + pub builtin: bool, +} + +/// The built-in presets (our own names and values), "(Default)" first for each type. +pub fn builtin_presets() -> Vec { + use AudioType::*; + let mk = |kind: AudioType, name: &str, f: &dyn Fn(&mut EssentialSound)| { + let mut s = EssentialSound::new(kind); + f(&mut s); + s.preset = name.to_string(); + Preset { name: name.to_string(), kind, settings: s, builtin: true } + }; + vec![ + mk(Dialogue, DEFAULT_PRESET, &|_| {}), + mk(Dialogue, "Balanced Voice", &|s| { + s.repair.rumble = Slot::at(4.0); + s.clarity.dynamics = Slot::at(4.0); + s.clarity.eq = Slot::at(5.0); + s.clarity.eq_preset = "Balanced Voice".into(); + }), + mk(Dialogue, "Clean Up Noisy Dialogue", &|s| { + s.repair.noise = Slot::at(6.0); + s.repair.rumble = Slot::at(5.0); + s.repair.dehum = Slot::at(5.0); + s.clarity.dynamics = Slot::at(3.0); + }), + mk(Dialogue, "Podcast Voice", &|s| { + s.repair.rumble = Slot::at(5.0); + s.repair.deess = Slot::at(4.0); + s.clarity.dynamics = Slot::at(6.0); + s.clarity.eq = Slot::at(6.0); + s.clarity.eq_preset = "Presence Lift".into(); + s.clarity.enhance = Slot::at(7.0); + }), + mk(Dialogue, "Dry Up a Roomy Voice", &|s| { + s.repair.reverb = Slot::at(6.0); + s.repair.rumble = Slot::at(4.0); + s.clarity.eq = Slot::at(4.0); + s.clarity.eq_preset = "Clear Muddiness".into(); + }), + mk(Dialogue, "Phone Call", &|s| { + s.clarity.eq = Slot::at(10.0); + s.clarity.eq_preset = "Telephone".into(); + s.clarity.dynamics = Slot::at(5.0); + }), + mk(Dialogue, "Radio Announcer", &|s| { + s.clarity.eq = Slot::at(8.0); + s.clarity.eq_preset = "Radio Voice".into(); + s.clarity.dynamics = Slot::at(8.0); + s.repair.deess = Slot::at(5.0); + }), + mk(Dialogue, "Voice in a Hall", &|s| { + s.creative.reverb = Slot::at(5.0); + s.creative.reverb_preset = "Large Hall".into(); + }), + mk(Music, DEFAULT_PRESET, &|_| {}), + mk(Music, "Duck Under Dialogue", &|s| { + s.ducking.enabled = true; + }), + mk(Music, "Duck Deep Under Dialogue", &|s| { + s.ducking.enabled = true; + s.ducking.reduce_db = -24.0; + s.ducking.fade_s = 1.2; + }), + mk(Music, "Duck Under Voice and Effects", &|s| { + s.ducking.enabled = true; + s.ducking.against = vec![Dialogue, Sfx]; + s.ducking.reduce_db = -12.0; + }), + mk(Sfx, DEFAULT_PRESET, &|_| {}), + mk(Sfx, "From the Left", &|s| { + s.pan.enabled = true; + s.pan.value = -60.0; + }), + mk(Sfx, "From the Right", &|s| { + s.pan.enabled = true; + s.pan.value = 60.0; + }), + mk(Sfx, "In a Room", &|s| { + s.creative.reverb = Slot::at(4.0); + s.creative.reverb_preset = "Small Room".into(); + }), + mk(Sfx, "Far Away", &|s| { + s.creative.reverb = Slot::at(8.0); + s.creative.reverb_preset = "Open Air".into(); + }), + mk(Ambience, DEFAULT_PRESET, &|_| {}), + mk(Ambience, "Wide Outdoor", &|s| { + s.creative.width = Slot::at(7.0); + }), + mk(Ambience, "Interior Room Tone", &|s| { + s.creative.reverb = Slot::at(3.0); + s.creative.reverb_preset = "Warm Room".into(); + }), + mk(Ambience, "Duck Under Dialogue", &|s| { + s.ducking.enabled = true; + }), + ] +} + +/// Apply a preset's settings to a clip's state, keeping what a preset does not own (the +/// measured loudness and its match gain, Clip Volume, Mute). +pub fn with_preset(current: Option<&EssentialSound>, preset: &Preset) -> EssentialSound { + let mut s = preset.settings.clone(); + s.preset = preset.name.clone(); + if let Some(c) = current.filter(|c| c.kind == preset.kind) { + s.loudness.measured_lufs = c.loudness.measured_lufs; + s.loudness.target_lufs = c.loudness.target_lufs; + s.loudness.gain_db = c.loudness.gain_db; + s.volume = c.volume.clone(); + s.mute = c.mute; + } + s +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::effect::intrinsic_audio; + use serde_json::json; + + fn clip() -> TrackItem { + TrackItem { + id: crate::ClipId(1), + item: crate::ItemId(2), + name: "d".into(), + label: crate::Label::Iris, + start: Tick::ZERO, + duration: Tick(1000), + source_in: Tick::ZERO, + speed: 1.0, + reverse: false, + enabled: true, + link: None, + group: None, + effects: intrinsic_audio(), + markers: vec![], + gain_db: 0.0, + frame_hold: None, + scale_to_frame: false, + essential: None, + } + } + + fn ids(it: &TrackItem) -> Vec { + it.effects.iter().map(|e| e.effect.clone()).collect() + } + + #[test] + fn set_get_and_validation() { + let mut s = EssentialSound::new(AudioType::Dialogue); + s.set("repair.noise.on", &json!(true)).unwrap(); + s.set("repair.noise.amount", &json!(42.0)).unwrap(); + assert_eq!(s.repair.noise, Slot { on: true, amount: 10.0 }); + s.set("clarity.eqPreset", &json!("Telephone")).unwrap(); + assert_eq!(s.get("clarity.eqPreset"), Some(json!("Telephone"))); + assert!(s.set("clarity.eqPreset", &json!("Nope")).is_err()); + assert!(s.set("repair.noise.on", &json!(3)).is_err()); + assert!(s.set("nope", &json!(3)).is_err()); + assert!(s.set("kind", &json!("Music")).is_err()); + assert!(s.preset.is_empty(), "a manual change leaves the preset"); + s.set("ducking.against", &json!(["Dialogue", "SFX"])).unwrap(); + assert_eq!(s.ducking.against, vec![AudioType::Dialogue, AudioType::Sfx]); + } + + #[test] + fn apply_inserts_updates_and_removes_effects_in_order() { + let mut it = clip(); + it.effects.push(find_effect("delay").unwrap().instance()); + let mut s = EssentialSound::new(AudioType::Dialogue); + s.repair.deess = Slot::at(5.0); + s.creative.reverb = Slot::at(5.0); + s.repair.rumble = Slot::at(5.0); + it.essential = Some(s.clone()); + apply(&mut it, None, Tick::ZERO); + assert_eq!(ids(&it), ["volume", "channel_volume", "panner", "highpass", "deesser", "studio_reverb", "delay"]); + // keyframe a parameter by hand, then change an unrelated slider: keyframes survive + let k = it.effects.iter_mut().find(|e| e.effect == "deesser").unwrap(); + k.params.get_mut("reduction").unwrap().put_keyframe(Tick(10), ParamValue::Float(3.0)); + let old = s.clone(); + s.repair.rumble.amount = 10.0; + s.repair.noise = Slot::at(2.0); + it.essential = Some(s.clone()); + apply(&mut it, Some(&old), Tick::ZERO); + assert_eq!(ids(&it), ["volume", "channel_volume", "panner", "highpass", "denoise", "deesser", "studio_reverb", "delay"]); + assert_eq!(it.effect("highpass").unwrap().f64_at("cutoff", Tick::ZERO), 150.0); + assert!(it.effect("deesser").unwrap().param("reduction").unwrap().is_animated()); + // section off bypasses, slot off removes + let old = s.clone(); + s.repair.enabled = false; + s.creative.reverb.on = false; + it.essential = Some(s.clone()); + apply(&mut it, Some(&old), Tick::ZERO); + assert!(!it.effect("highpass").unwrap().enabled); + assert!(it.effect("studio_reverb").is_none()); + // clearing removes every essential effect but keeps the user's + let old = s.clone(); + it.essential = None; + apply(&mut it, Some(&old), Tick::ZERO); + assert_eq!(ids(&it), ["volume", "channel_volume", "panner", "delay"]); + } + + #[test] + fn gain_volume_and_pan_are_deltas() { + let mut it = clip(); + it.gain_db = 2.0; + let mut s = EssentialSound::new(AudioType::Sfx); + s.loudness.gain_db = -5.0; + s.volume = ClipVolume { on: true, level_db: -3.0 }; + s.pan = Pan { enabled: true, value: 40.0 }; + it.essential = Some(s.clone()); + apply(&mut it, None, Tick::ZERO); + assert_eq!(it.gain_db, -3.0); + assert_eq!(it.effect("volume").unwrap().f64_at("level", Tick::ZERO), -3.0); + assert_eq!(it.effect("panner").unwrap().f64_at("balance", Tick::ZERO), 40.0); + let old = s.clone(); + it.essential = None; + apply(&mut it, Some(&old), Tick::ZERO); + assert_eq!(it.gain_db, 2.0); + assert_eq!(it.effect("volume").unwrap().f64_at("level", Tick::ZERO), 0.0); + assert_eq!(it.effect("panner").unwrap().f64_at("balance", Tick::ZERO), 0.0); + } + + #[test] + fn presets_are_valid_and_unique() { + let ps = builtin_presets(); + for t in AudioType::ALL { + let mine: Vec<_> = ps.iter().filter(|p| p.kind == t).collect(); + assert_eq!(mine[0].name, DEFAULT_PRESET); + let mut names: Vec<_> = mine.iter().map(|p| p.name.as_str()).collect(); + names.dedup(); + assert_eq!(names.len(), mine.len()); + } + for p in &ps { + assert!(eq_preset(&p.settings.clarity.eq_preset).is_some()); + assert!(reverb_preset(&p.settings.creative.reverb_preset).is_some()); + for e in p.settings.effects() { + assert!(e.def().is_some(), "{}: unknown effect {}", p.name, e.effect); + } + } + // serde round trip + let s = &ps[3].settings; + let back: EssentialSound = serde_json::from_value(s.to_value()).unwrap(); + assert_eq!(&back, s); + } +} diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index b502794f..de149c64 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -9,6 +9,7 @@ pub mod caption; pub mod effect; +pub mod essential; pub mod graphic; pub mod keyframe; pub mod mixer; @@ -22,6 +23,7 @@ use serde::{Deserialize, Serialize}; pub use caption::{Caption, CaptionAlign, CaptionAnchor, CaptionFormat, CaptionStyle, CaptionTrack, plain_text}; pub use effect::{EffectDef, EffectInstance, EffectKind, ParamDef, ParamKind, effect_defs, find_effect}; +pub use essential::{AudioType, EssentialSound}; pub use keyframe::{Interpolation, Keyframe, Param, ParamValue}; pub use mixer::{AutomationMode, InputMap, MixerStrip, TrackSend}; @@ -431,6 +433,9 @@ pub struct TrackItem { /// Scale to frame size (Set to Frame Size / Scale to Frame Size). #[serde(default)] pub scale_to_frame: bool, + /// Essential Sound audio type and settings (audio clips). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub essential: Option, } impl TrackItem { @@ -899,6 +904,7 @@ impl Project { gain_db: 0.0, frame_hold: None, scale_to_frame: false, + essential: None, }) } diff --git a/crates/render/src/audio.rs b/crates/render/src/audio.rs index 1f051593..81ab684e 100644 --- a/crates/render/src/audio.rs +++ b/crates/render/src/audio.rs @@ -93,9 +93,11 @@ fn mix_item(project: &Project, item: &TrackItem, start: i64, frames: usize, sr: if a1 <= a0 { return; } + if item.essential.as_ref().is_some_and(|e| e.mute) { + return; + } let n = (a1 - a0) as usize; - let read = |x0: i64, len: usize| raw_stereo(item, src.as_ref(), x0, len, sr); - let buf = if crate::audio_fx::has_effects(item) { crate::audio_fx::process(item, a0, n, sr, &read) } else { read(a0, n) }; + let buf = effected(item, src.as_ref(), a0, n, sr); // gains: clip gain × Volume (keyframed, per 64-sample block) × channel volume × panner let clip_gain = db_to_gain(item.gain_db); let vol = item.effect("volume").filter(|e| e.enabled && !e.param("bypass").and_then(|p| p.value.as_bool()).unwrap_or(false)); @@ -122,6 +124,39 @@ fn mix_item(project: &Project, item: &TrackItem, start: i64, frames: usize, sr: } } +/// The clip's audio after speed/reverse and clip effects (before clip gain, Volume and Panner) for +/// timeline samples `[a0, a0 + n)`. +fn effected(item: &TrackItem, src: &dyn filmcraft_media::MediaSource, a0: i64, n: usize, sr: u32) -> [Vec; 2] { + let read = |x0: i64, len: usize| raw_stereo(item, src, x0, len, sr); + if crate::audio_fx::has_effects(item) { crate::audio_fx::process(item, a0, n, sr, &read) } else { read(a0, n) } +} + +/// One clip's own signal for timeline samples `[start, start + frames)` (sequence rate): clip gain +/// and clip effects, without Volume / Channel Volume / Panner, transitions or Mute (silence outside +/// the clip). Loudness Auto-Match and ducking analyse this. `None` when the clip has no audio +/// source (nested sequences are not analysed). +pub fn clip_signal(item: &TrackItem, start: i64, frames: usize, sr: u32, sources: &dyn SourceProvider) -> Option<[Vec; 2]> { + let src = sources.source(item.item)?; + if !src.info().has_audio() { + return None; + } + let mut out = [vec![0.0f32; frames], vec![0.0f32; frames]]; + let a0 = start.max(item.start.to_units_floor(sr as i64)); + let a1 = (start + frames as i64).min(item.end().to_units_floor(sr as i64)); + if a1 > a0 { + let n = (a1 - a0) as usize; + let off = (a0 - start) as usize; + let g = db_to_gain(item.gain_db); + let buf = effected(item, src.as_ref(), a0, n, sr); + for (dst, b) in out.iter_mut().zip(&buf) { + for (d, x) in dst[off..off + n].iter_mut().zip(b) { + *d = x * g; + } + } + } + Some(out) +} + /// The clip's audio (after speed/reverse) for timeline samples `[x0, x0 + len)` as stereo, with /// silence outside the clip. Mono sources are duplicated to both channels. fn raw_stereo(item: &TrackItem, src: &dyn filmcraft_media::MediaSource, x0: i64, len: usize, sr: u32) -> [Vec; 2] { diff --git a/crates/render/src/audio_fx.rs b/crates/render/src/audio_fx.rs index 28f335b4..79174fbf 100644 --- a/crates/render/src/audio_fx.rs +++ b/crates/render/src/audio_fx.rs @@ -83,12 +83,43 @@ pub(crate) fn mapping(id: &str) -> Option { "highpass" => Mapping { dsp: "parametric_eq", preroll: |_, _| 0.05, apply: |d, e, t| band(d, 4.0, f(e, "cutoff", t), 0.707) }, "lowpass" => Mapping { dsp: "parametric_eq", preroll: |_, _| 0.05, apply: |d, e, t| band(d, 3.0, f(e, "cutoff", t), 0.707) }, "bandpass" => Mapping { dsp: "parametric_eq", preroll: |_, _| 0.05, apply: |d, e, t| band(d, 6.0, f(e, "center", t), f(e, "q", t)) }, - "denoise" => Mapping { dsp: "denoise", preroll: |_, _| 1.0, apply: |d, e, t| ignore(d.set_param("amount", f(e, "amount", t))) }, + // Amount 0…100 % → 0…40 dB of reduction. + "denoise" => Mapping { dsp: "denoise", preroll: |_, _| 1.0, apply: |d, e, t| ignore(d.set_param("reduction", f(e, "amount", t) * 0.4)) }, + // Gain is the notch depth: the wet/dry amount that leaves 10^(gain/20) of the hum. "dehummer" => Mapping { dsp: "dehum", preroll: |_, _| 0.2, - apply: |d, e, t| ignore(d.set_param("frequency", e.param("freq").and_then(|p| p.value_at(t).as_f64()).unwrap_or(1.0) as f32)), + apply: |d, e, t| { + d.set_param("frequency", e.param("freq").and_then(|p| p.value_at(t).as_f64()).unwrap_or(1.0) as f32); + d.set_param("amount", (1.0 - 10f32.powf(f(e, "gain", t).min(0.0) / 20.0)) * 100.0); + }, }, + "deesser" => Mapping { + dsp: "deesser", + preroll: |_, _| 0.1, + apply: |d, e, t| { + d.set_param("frequency", f(e, "frequency", t)); + d.set_param("threshold", f(e, "threshold", t)); + d.set_param("reduction", f(e, "reduction", t)); + }, + }, + "dereverb" => Mapping { + dsp: "dereverb", + preroll: |e, t| (f(e, "rt60", t) as f64 + 0.3).clamp(0.5, 3.0), + apply: |d, e, t| { + d.set_param("amount", f(e, "amount", t)); + d.set_param("rt60", f(e, "rt60", t)); + }, + }, + "speech_enhance" => Mapping { + dsp: "speech_enhance", + preroll: |_, _| 0.3, + apply: |d, e, t| { + d.set_param("mix", f(e, "mix", t)); + d.set_param("tone", e.param("tone").and_then(|p| p.value_at(t).as_f64()).unwrap_or(0.0) as f32); + }, + }, + "stereo_width" => Mapping { dsp: "stereo_width", preroll: |_, _| 0.0, apply: |d, e, t| ignore(d.set_param("width", f(e, "width", t))) }, "studio_reverb" => Mapping { dsp: "reverb", preroll: |e, t| 0.3 * 30f64.powf(f(e, "decay", t) as f64 / 100.0).min(8.0), From 995621ff671af7e34800d405a68aa84eeb40c047 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 1 Oct 2026 08:40:20 -0400 Subject: [PATCH 4/5] =?UTF-8?q?M7.4:=20Essential=20Sound=20panel=20?= =?UTF-8?q?=E2=80=94=20Browse/Edit=20tabs,=20type=20buttons,=20collapsible?= =?UTF-8?q?=20sections=20with=20switches=20and=20sliders,=20presets,=20Cli?= =?UTF-8?q?p=20Volume/Mute=20footer;=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - crates/ui-egui/src/panels/essential_sound.rs: per-type sections (Loudness with Auto-Match, Repair, Clarity, Creative, Ducking with Generate Keyframes, Duration note, Pan), preset dropdown with save/delete, code-drawn type glyphs, automation ids for every control; every control runs an essentialSound.* command and a slider drag is one undo step - headless kittest test (essential_sound_ui.rs) with optional wgpu snapshots - docs: architecture §5.2, control-protocol ids, testing, ROADMAP Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 4 +- crates/ui-egui/src/panels/essential_sound.rs | 654 +++++++++++++++++++ crates/ui-egui/src/panels/mod.rs | 2 + crates/ui-egui/src/state.rs | 4 + crates/ui-egui/tests/essential_sound_ui.rs | 255 ++++++++ docs/architecture.md | 37 ++ docs/control-protocol.md | 14 + docs/testing.md | 12 + 8 files changed, 981 insertions(+), 1 deletion(-) create mode 100644 crates/ui-egui/src/panels/essential_sound.rs create mode 100644 crates/ui-egui/tests/essential_sound_ui.rs diff --git a/ROADMAP.md b/ROADMAP.md index 07f80214..33c66757 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -35,7 +35,7 @@ Status: ✅ done · 🟡 in progress · ⬜ not started. Estimates are remaining | M4 | Playback | 🟡 | Audio-clock master, prefetch, J/K/L, dropped-frame stats, playback resolution | Render previews, render bar, 4K/8K tuning | 8–12 | | M5 | Effects, keyframes, GPU | 🟡 | ~60 CPU effects, 30 transitions, keyframes + value/velocity graphs, wgpu compositor | Full ~150-effect catalogue, WGSL parity for all, masks + tracking, adjustment layers, presets, Warp Stabilizer, Morph Cut | 30–45 | | M6 | Export | ✅ | Own H.264 encoder (High/Main/Baseline, B-frames, VBR/CBR/2-pass) → MP4 + own AAC; ProRes, MJPEG, PNG, GIF, WAV; background jobs | Preset library, queue UI, smart render | 6–8 | -| M7 | Audio | 🟡 | Mixer graph (tracks → submixes → Mix, pre/post-fader inserts and sends, latency-compensated, sample-accurate, ~6× realtime for 24 tracks × 3 effects on one core), Audio Track Mixer + Audio Clip Mixer panels, track automation (Off/Read/Latch/Touch/Write, recorded live while playing, thinned to keyframes, timeline lanes with pen editing), solo/solo-safe, channel mapping basics, peak + BS.1770 loudness meters (match ffmpeg), DSP crate with 12 clip/track effects, Audio Gain (set/adjust/normalize), Constant Power / Constant Gain / Exponential Fade | Essential Sound, 5.1 panner and multichannel buses, voice-over record, effect editor windows (EQ curve), remaining effects (multiband, convolution reverb), clip-mixer automation recording | 6–9 | +| M7 | Audio | 🟡 | Mixer graph (tracks → submixes → Mix, pre/post-fader inserts and sends, latency-compensated, sample-accurate, ~6× realtime for 24 tracks × 3 effects on one core), Audio Track Mixer + Audio Clip Mixer panels, track automation (Off/Read/Latch/Touch/Write, recorded live while playing, thinned to keyframes, timeline lanes with pen editing), solo/solo-safe, channel mapping basics, peak + BS.1770 loudness meters (match ffmpeg), DSP crate with 16 clip/track effects, Audio Gain (set/adjust/normalize), Constant Power / Constant Gain / Exponential Fade, Essential Sound (types, Loudness auto-match, Repair incl. DeEss/DeReverb, Clarity, Creative, Ducking keyframes, presets; full Dialogue chain 22× realtime) | Music duration remix, ML speech enhancement, 5.1 panner and multichannel buses, voice-over record, effect editor windows (EQ curve), remaining effects (multiband, convolution reverb), clip-mixer automation recording | 6–9 | | M8 | Colour | 🟡 | Lumetri: basic, creative + looks, RGB & hue curves, wheels, HSL secondary, vignette; basic scopes | LUT import UI, colour match, colour management + HDR (PQ/HLG, log, tone mapping) | 8–12 | | M9 | More codecs | 🟡 | ProRes decode+encode, AAC decode+encode, HEVC Main/Main 10 decoder (bit-exact on 41 fixtures, ~225 fps 1080p), VP9 decoder (profiles 0–3, 8/10/12-bit, bit-exact on 50+ fixtures; WebM/MKV `V_VP9` and MP4 `vp09` import with key-frame-checked seeking), Matroska/WebM import (H.264/HEVC/VP9/ProRes/MJPEG + AAC/Opus/FLAC/MP3/Vorbis/PCM), Opus decoder (SILK/CELT/hybrid, 5.1/7.1 multistream; all RFC 8251 vectors range-exact; WebM/MKV/MP4) | VP9 frame threading, Ogg Opus files, AV1 (rav1d), DNxHR, MXF, hardware decode | 12–20 | | M10 | Graphics & captions | 🟡 | Caption tracks (Subtitle/CEA-608/708/Teletext formats, track style), SRT/WebVTT/SCC import+export (frame-exact, property-tested), caption editing (add/split/merge/trim/move, sync-locked insert/extract), Text panel Captions tab, burn-in in Program monitor and export; text engine (`crates/text`: bundled + system fonts, harfrust shaping, bidi, line breaking, paragraph layout, glyph cache; 3-line 1080p title ≈ 0.15 ms warm); graphic clips with text + shape layers (fill, 2 strokes, background, shadow, keyframable transform), Type tool with on-monitor editing, shape/pen tools, Properties/Essential Graphics editor, align/distribute | Responsive design pins, rolls/crawls, per-character styles, motion graphics templates, MCC/STL/TTML, 608/708 embedding, speech-to-text | 12–18 | @@ -48,6 +48,8 @@ Status: ✅ done · 🟡 in progress · ⬜ not started. Estimates are remaining ## Log +- **2026-10-01 (M7.4):** Essential Sound panel: Dialogue/Music/SFX/Ambience types, Loudness auto-match (BS.1770, exact to the target), Repair (noise, rumble, hum, new DeEsser and spectral DeReverb), Clarity (dynamics, EQ presets, Enhance Speech DSP chain), Creative reverb / stereo width, ducking that writes Volume keyframes, presets; all `essentialSound.*` commands, effects visible and keyframable in Effect Controls. + - **2026-10-01 (later):** text engine (shaping, bidi, line breaking) + Type/Shape/Pen tools + graphic clips + graphics panel; Trim Monitor + dynamic trimming; Keyboard Shortcuts editor; audio mixer with automation. Fixed: MP4 muxer wrote unreadable all-empty sample tables; system font scan race. - **2026-10-01:** Audio mixing (M7.2/M7.5/M7.6 basics): mixer graph with submixes, sends, inserts and latency compensation; Audio Track / Clip Mixer panels; Latch/Touch/Write automation recorded live; timeline track keyframes; Audio Gain dialog. diff --git a/crates/ui-egui/src/panels/essential_sound.rs b/crates/ui-egui/src/panels/essential_sound.rs new file mode 100644 index 00000000..2a64b572 --- /dev/null +++ b/crates/ui-egui/src/panels/essential_sound.rs @@ -0,0 +1,654 @@ +//! Essential Sound panel: Browse / Edit tabs, audio-type assignment, per-type collapsible sections +//! (Loudness, Repair, Clarity, Creative, Ducking, Duration, Pan) with switch + slider rows, presets, +//! and the Clip Volume / Mute footer. Every control runs an `essentialSound.*` command (slider drags +//! are one undo step), so the panel, MCP and the control channel share one code path. +//! +//! Automation ids: `essentialSound.tab.`, `essentialSound.type.`, `essentialSound.clearType`, `essentialSound.preset` (+ `.save`, `.delete`, +//! `.name`, `.ok`), `essentialSound.section.` (twirl) and `.toggle` (switch), setting rows by +//! key (`essentialSound.repair.noise.on`, `essentialSound.repair.noise.amount`, +//! `essentialSound.clarity.eqPreset`…), `essentialSound.autoMatch`, +//! `essentialSound.ducking.against.`, `essentialSound.generateDucking`, +//! `essentialSound.volume.on` / `.levelDb`, `essentialSound.mute`, and in Browse +//! `essentialSound.browse..`. + +use egui::{Align2, Color32, Painter, Rect, Sense, Stroke, StrokeKind, pos2, vec2}; +use filmcraft_project::TrackItem; +use filmcraft_project::essential::{self as es, AudioType, EssentialSound, Section, Slot}; +use serde_json::{Value, json}; + +use crate::FilmcraftApp; +use crate::icons::{self, Icon}; +use crate::theme::Tokens; + +const PAD: f32 = 14.0; +const FOOTER_H: f32 = 86.0; + +/// A command to run after layout. +type Actions = Vec<(String, Value)>; + +struct Ctx<'a> { + t: Tokens, + auto: &'a mut crate::automation::Registry, + actions: Actions, + clips: Vec, +} + +impl Ctx<'_> { + fn run(&mut self, cmd: &str, mut p: Value) { + if p.get("clips").is_none() && !cmd.ends_with("deletePreset") { + p["clips"] = json!(self.clips); + } + self.actions.push((cmd.to_string(), p)); + } + fn set(&mut self, key: &str, v: Value, begin: bool) { + self.run("essentialSound.set", json!({"key": key, "value": v, "begin": begin})); + } +} + +pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let clip_ids = filmcraft_engine::essential_sound::targets(&app.session, &Value::Null); + let items: Vec = + app.session.active_sequence().map(|seq| clip_ids.iter().filter_map(|c| seq.find_item(*c).map(|(_, i)| i.clone())).collect()).unwrap_or_default(); + let presets = filmcraft_engine::essential_sound::presets(&app.session); + let tab = if app.ui.essential_sound_tab == "Browse" { "Browse" } else { "Edit" }; + let mut cx = Ctx { t, auto: &mut app.auto, actions: Vec::new(), clips: clip_ids.iter().map(|c| c.0).collect() }; + let painter = ui.painter().clone(); + painter.rect_filled(rect, 0.0, t.panel_bg); + // ---- Browse | Edit + let mut x = rect.min.x + PAD; + let ty = rect.min.y + 16.0; + let mut new_tab = None; + for name in ["Browse", "Edit"] { + let g = painter.layout_no_wrap(name.to_string(), Tokens::ui(12.5), t.text); + let r = Rect::from_min_size(pos2(x, ty - 9.0), vec2(g.size().x, 20.0)); + let resp = ui.interact(r, egui::Id::new(("es-tab", name)), Sense::click()); + let col = if name == tab || resp.hovered() { t.text } else { t.text_dim }; + painter.text(pos2(x, ty), Align2::LEFT_CENTER, name, Tokens::ui(12.5), col); + if name == tab { + painter.line_segment([pos2(x, ty + 10.0), pos2(x + g.size().x, ty + 10.0)], Stroke::new(1.5, t.text)); + } + cx.auto.add(&format!("essentialSound.tab.{name}"), r, name); + if resp.clicked() { + new_tab = Some(name.to_string()); + } + x += g.size().x + 22.0; + } + let body = Rect::from_min_max(pos2(rect.min.x, rect.min.y + 32.0), rect.max); + if tab == "Browse" { + browse(ui, &mut cx, body, &presets, &items); + } else { + edit(ui, &mut cx, body, &items, &presets, app.ui.collapsed_fx.clone(), app.ui.expanded_fx.clone()); + } + if let Some(n) = new_tab { + app.ui.essential_sound_tab = n; + } + // section twirls are UI state; everything else is a command + for (cmd, p) in std::mem::take(&mut cx.actions) { + if cmd == "ui.twirl" { + let key = p["key"].as_str().unwrap_or_default().to_string(); + if p["open"].as_bool().unwrap_or(false) { + app.ui.collapsed_fx.retain(|k| *k != key); + app.ui.expanded_fx.push(key); + } else { + app.ui.expanded_fx.retain(|k| *k != key); + app.ui.collapsed_fx.push(key); + } + continue; + } + if let Err(e) = app.session.execute(&cmd, p) { + app.ui.status = e.to_string(); + } + } +} + +// ------------------------------------------------------------------------------------- browse + +fn browse(ui: &mut egui::Ui, cx: &mut Ctx, rect: Rect, presets: &[es::Preset], items: &[TrackItem]) { + let t = cx.t; + let mut child = ui.new_child(egui::UiBuilder::new().max_rect(rect.shrink2(vec2(PAD, 4.0))).id_salt("es-browse")); + egui::ScrollArea::vertical().auto_shrink([false, false]).show(&mut child, |ui| { + ui.set_max_width(ui.available_width() - 10.0); + if items.is_empty() { + ui.label(egui::RichText::new("Select audio clips, then click a preset to apply it.").color(t.text_faint).size(11.5)); + } + for kind in AudioType::ALL { + ui.add_space(8.0); + let (r, _) = ui.allocate_exact_size(vec2(ui.available_width(), 22.0), Sense::hover()); + type_icon(ui.painter(), Rect::from_center_size(pos2(r.min.x + 8.0, r.center().y), vec2(16.0, 16.0)), kind, t.text); + ui.painter().text(pos2(r.min.x + 24.0, r.center().y), Align2::LEFT_CENTER, kind.label(), Tokens::semibold(12.5), t.text); + for p in presets.iter().filter(|p| p.kind == kind) { + let (r, resp) = ui.allocate_exact_size(vec2(ui.available_width(), 22.0), Sense::click()); + if resp.hovered() { + ui.painter().rect_filled(r, 2.0, t.hover); + } + let name = if p.builtin { p.name.clone() } else { format!("{} (user)", p.name) }; + ui.painter().text( + pos2(r.min.x + 24.0, r.center().y), + Align2::LEFT_CENTER, + name, + Tokens::ui(12.0), + if items.is_empty() { t.text_dim } else { t.text }, + ); + cx.auto.add(&format!("essentialSound.browse.{}.{}", kind.label(), p.name), r, &p.name); + if resp.clicked() && !items.is_empty() { + cx.run("essentialSound.applyPreset", json!({"type": kind.id(), "preset": p.name})); + } + } + } + }); +} + +// ------------------------------------------------------------------------------------- edit + +fn edit(ui: &mut egui::Ui, cx: &mut Ctx, rect: Rect, items: &[TrackItem], presets: &[es::Preset], collapsed: Vec, expanded: Vec) { + let t = cx.t; + if items.is_empty() { + crate::dock::placeholder(ui, rect, &t, "Select audio clips to edit them here"); + return; + } + let mut kinds: Vec> = items.iter().map(|i| i.essential.as_ref().map(|e| e.kind)).collect(); + kinds.dedup(); + let title = if items.len() > 1 { "Multiple Clips Selected".to_string() } else { items[0].name.clone() }; + let body = Rect::from_min_max(rect.min, pos2(rect.max.x, rect.max.y - FOOTER_H)); + let mut child = + ui.new_child(egui::UiBuilder::new().max_rect(Rect::from_min_max(body.min + vec2(PAD, 2.0), body.max - vec2(PAD - 2.0, 2.0))).id_salt("es-edit")); + let typed = items.iter().find_map(|i| i.essential.clone()); + egui::ScrollArea::vertical().auto_shrink([false, false]).show(&mut child, |ui| { + ui.set_max_width(ui.available_width() - 10.0); + ui.label(egui::RichText::new(&title).font(Tokens::semibold(14.0)).color(t.text)); + ui.add_space(6.0); + match (&kinds[..], &typed) { + ([Some(_)], Some(st)) => typed_body(ui, cx, st, items, presets, &collapsed, &expanded), + _ => { + if kinds.len() > 1 { + ui.label( + egui::RichText::new("The selected clips have different audio types. Assign one type to edit them together.") + .color(t.text_dim) + .size(11.5), + ); + } else { + ui.label(egui::RichText::new("Assign an audio type to the selection:").color(t.text_dim).size(12.0)); + } + ui.add_space(6.0); + for kind in AudioType::ALL { + let (r, resp) = ui.allocate_exact_size(vec2(ui.available_width(), 46.0), Sense::click()); + let p = ui.painter(); + p.rect_filled(r.shrink(2.0), 4.0, if resp.hovered() { t.hover } else { t.field_bg }); + p.rect_stroke(r.shrink(2.0), 4.0, Stroke::new(1.0, t.field_border), StrokeKind::Inside); + type_icon(p, Rect::from_center_size(pos2(r.min.x + 26.0, r.center().y), vec2(22.0, 22.0)), kind, t.text); + p.text(pos2(r.min.x + 50.0, r.center().y), Align2::LEFT_CENTER, kind.label(), Tokens::semibold(13.0), t.text); + cx.auto.add(&format!("essentialSound.type.{}", kind.label()), r, kind.label()); + if resp.clicked() { + cx.run("essentialSound.setType", json!({"type": kind.id()})); + } + } + } + } + ui.add_space(24.0); + }); + footer(ui, cx, Rect::from_min_max(pos2(rect.min.x, rect.max.y - FOOTER_H), rect.max), typed.as_ref()); +} + +#[allow(clippy::too_many_arguments)] +fn typed_body(ui: &mut egui::Ui, cx: &mut Ctx, st: &EssentialSound, items: &[TrackItem], presets: &[es::Preset], collapsed: &[String], expanded: &[String]) { + let t = cx.t; + let kind = st.kind; + // type badge + Clear Audio Type + let (r, _) = ui.allocate_exact_size(vec2(ui.available_width(), 30.0), Sense::hover()); + type_icon(ui.painter(), Rect::from_center_size(pos2(r.min.x + 10.0, r.center().y), vec2(18.0, 18.0)), kind, t.text); + ui.painter().text(pos2(r.min.x + 28.0, r.center().y), Align2::LEFT_CENTER, kind.label(), Tokens::ui(12.5), t.text); + let br = Rect::from_min_max(pos2(r.max.x - 122.0, r.min.y + 3.0), pos2(r.max.x, r.max.y - 3.0)); + if button(ui, cx, br, "Clear Audio Type", "essentialSound.clearType") { + cx.run("essentialSound.clearType", json!({})); + } + ui.add_space(6.0); + // Preset: [▾] save delete + ui.label(egui::RichText::new("Preset:").color(t.text_dim).size(12.0)); + let mine: Vec<&es::Preset> = presets.iter().filter(|p| p.kind == kind).collect(); + let current = if st.preset.is_empty() { "(Custom)".to_string() } else { st.preset.clone() }; + let mut picked = None; + let draft_id = egui::Id::new("es-preset-draft"); + let mut draft: Option = ui.data(|d| d.get_temp(draft_id)); + ui.horizontal(|ui| { + let w = ui.available_width() - 56.0; + let cb = egui::ComboBox::from_id_salt("es-preset").selected_text(¤t).width(w).show_ui(ui, |ui| { + for p in &mine { + if ui.selectable_label(p.name == st.preset, &p.name).clicked() { + picked = Some(p.name.clone()); + } + } + }); + cx.auto.add("essentialSound.preset", cb.response.rect, "Preset"); + let (sr, sresp) = ui.allocate_exact_size(vec2(22.0, 22.0), Sense::click()); + save_icon(ui.painter(), sr.shrink(3.0), if sresp.hovered() { t.text } else { t.text_dim }); + cx.auto.add("essentialSound.preset.save", sr, "Save Preset"); + if sresp.clicked() { + draft = Some(String::new()); + } + let deletable = mine.iter().any(|p| !p.builtin && p.name == st.preset); + let (dr, dresp) = ui.allocate_exact_size(vec2(22.0, 22.0), Sense::click()); + icons::paint(ui.painter(), dr.shrink(3.0), Icon::Trash, if deletable { t.text_dim } else { t.text_faint }); + cx.auto.add("essentialSound.preset.delete", dr, "Delete Preset"); + if dresp.clicked() && deletable { + cx.run("essentialSound.deletePreset", json!({"name": st.preset, "type": kind.id()})); + } + }); + if let Some(name) = picked { + cx.run("essentialSound.applyPreset", json!({"preset": name})); + } + if let Some(mut d) = draft.take() { + let mut done = false; + ui.horizontal(|ui| { + let r = ui.add(egui::TextEdit::singleline(&mut d).hint_text("Preset name").desired_width(ui.available_width() - 70.0)); + cx.auto.add("essentialSound.preset.name", r.rect, "Preset name"); + let ok = ui.button("Save"); + cx.auto.add("essentialSound.preset.ok", ok.rect, "Save"); + if (ok.clicked() || (r.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)))) && !d.trim().is_empty() { + cx.run("essentialSound.savePreset", json!({"name": d.trim()})); + done = true; + } + if ui.input(|i| i.key_pressed(egui::Key::Escape)) { + done = true; + } + }); + ui.data_mut(|m| { + if done { + m.remove::(draft_id); + } else { + m.insert_temp(draft_id, d); + } + }); + } else { + ui.data_mut(|m| m.remove::(draft_id)); + } + ui.add_space(10.0); + for sec in kind.sections() { + let key = format!("essentialSound:{}", sec.label()); + let default_open = !matches!(sec, Section::Loudness | Section::Pan | Section::Duration); + let open = if default_open { !collapsed.contains(&key) } else { expanded.contains(&key) }; + let enabled = sec.key().and_then(|k| st.get(k)).and_then(|v| v.as_bool()); + if section_header(ui, cx, *sec, open, enabled, &key) { + ui.add_space(4.0); + let on = enabled.unwrap_or(true); + match sec { + Section::Loudness => loudness(ui, cx, st, items), + Section::Repair => { + slot(ui, cx, "Reduce Noise", "repair.noise", st.repair.noise, on); + slot(ui, cx, "Reduce Rumble", "repair.rumble", st.repair.rumble, on); + slot(ui, cx, "DeHum", "repair.dehum", st.repair.dehum, on); + choice_row(ui, cx, "repair.humHz", &["50 Hz", "60 Hz"], usize::from(st.repair.hum_hz == 60), on && st.repair.dehum.on, |i| { + json!(if i == 1 { 60 } else { 50 }) + }); + slot(ui, cx, "DeEss", "repair.deess", st.repair.deess, on); + slot(ui, cx, "Reduce Reverb", "repair.reverb", st.repair.reverb, on); + } + Section::Clarity => { + slot(ui, cx, "Dynamics", "clarity.dynamics", st.clarity.dynamics, on); + switch_row(ui, cx, "EQ", "clarity.eq.on", st.clarity.eq.on, on); + let names: Vec<&str> = es::EQ_PRESETS.iter().map(|p| p.name).collect(); + combo(ui, cx, "clarity.eqPreset", &names, &st.clarity.eq_preset, on && st.clarity.eq.on); + slider(ui, cx, "Amount", "clarity.eq.amount", st.clarity.eq.amount, (0.0, 10.0), 1, "", on && st.clarity.eq.on); + switch_row(ui, cx, "Enhance Speech", "clarity.enhance.on", st.clarity.enhance.on, on); + choice_row(ui, cx, "clarity.enhanceTone", &["Low Tone", "High Tone"], st.clarity.enhance_tone as usize, on && st.clarity.enhance.on, |i| { + json!(i) + }); + slider(ui, cx, "Mix", "clarity.enhance.amount", st.clarity.enhance.amount, (0.0, 10.0), 1, "", on && st.clarity.enhance.on); + } + Section::Creative => { + switch_row(ui, cx, "Reverb", "creative.reverb.on", st.creative.reverb.on, on); + ui.label(egui::RichText::new("Preset:").color(t.text_dim).size(12.0)); + let names: Vec<&str> = es::REVERB_PRESETS.iter().map(|p| p.name).collect(); + combo(ui, cx, "creative.reverbPreset", &names, &st.creative.reverb_preset, on && st.creative.reverb.on); + slider(ui, cx, "Amount", "creative.reverb.amount", st.creative.reverb.amount, (0.0, 10.0), 1, "", on && st.creative.reverb.on); + if kind == AudioType::Ambience { + slot(ui, cx, "Stereo Width", "creative.width", st.creative.width, on); + } + } + Section::Ducking => ducking(ui, cx, st, on), + Section::Duration => { + ui.label( + egui::RichText::new("Remixing music to a target duration is not available yet. Trim the clip and add a crossfade instead.") + .color(t.text_faint) + .size(11.5), + ); + } + Section::Pan => slider(ui, cx, "Pan", "pan.value", st.pan.value, (-100.0, 100.0), 1, "", on), + } + ui.add_space(8.0); + } + } + let r = ui.allocate_exact_size(vec2(ui.available_width(), 1.0), Sense::hover()).0; + ui.painter().line_segment([r.left_top(), r.right_top()], Stroke::new(1.0, t.separator)); +} + +fn loudness(ui: &mut egui::Ui, cx: &mut Ctx, st: &EssentialSound, items: &[TrackItem]) { + let t = cx.t; + let (r, _) = ui.allocate_exact_size(vec2(ui.available_width(), 28.0), Sense::hover()); + let br = Rect::from_min_size(r.min + vec2(0.0, 2.0), vec2(104.0, 24.0)); + if button(ui, cx, br, "Auto-Match", "essentialSound.autoMatch") { + cx.run("essentialSound.autoMatch", json!({})); + } + let text = match (st.loudness.measured_lufs, st.loudness.target_lufs) { + (Some(m), Some(tg)) => format!("{m:.1} → {tg:.1} LUFS ({:+.1} dB)", st.loudness.gain_db), + _ => "Not matched yet".to_string(), + }; + ui.painter().text(pos2(br.max.x + 10.0, r.center().y), Align2::LEFT_CENTER, text, Tokens::ui(11.5), t.text_dim); + if items.len() > 1 { + ui.label(egui::RichText::new(format!("Matches each of the {} clips on its own.", items.len())).color(t.text_faint).size(11.0)); + } +} + +fn ducking(ui: &mut egui::Ui, cx: &mut Ctx, st: &EssentialSound, on: bool) { + let t = cx.t; + ui.label(egui::RichText::new("Duck against:").color(t.text_dim).size(12.0)); + let (r, _) = ui.allocate_exact_size(vec2(ui.available_width(), 30.0), Sense::hover()); + let mut x = r.min.x; + for kind in AudioType::ALL { + let br = Rect::from_min_size(pos2(x, r.min.y + 2.0), vec2(30.0, 26.0)); + let sel = st.ducking.against.contains(&kind); + let resp = ui.interact(br, egui::Id::new(("es-duck", kind.id())), Sense::click()); + ui.painter().rect_filled( + br, + 4.0, + if sel { + Color32::from_gray(0x4b) + } else if resp.hovered() { + t.hover + } else { + Color32::TRANSPARENT + }, + ); + type_icon(ui.painter(), br.shrink(6.0), kind, if sel { t.text } else { t.text_faint }); + cx.auto.add(&format!("essentialSound.ducking.against.{}", kind.label()), br, kind.label()); + if resp.clicked() && on { + let mut v = st.ducking.against.clone(); + if sel { + v.retain(|k| *k != kind); + } else { + v.push(kind); + } + cx.set("ducking.against", json!(v), true); + } + x += 34.0; + } + let br = Rect::from_min_size(pos2(x + 4.0, r.min.y + 2.0), vec2(84.0, 26.0)); + let resp = ui.interact(br, egui::Id::new("es-duck-untyped"), Sense::click()); + let sel = st.ducking.against_untyped; + ui.painter().rect_filled( + br, + 4.0, + if sel { + Color32::from_gray(0x4b) + } else if resp.hovered() { + t.hover + } else { + Color32::TRANSPARENT + }, + ); + ui.painter().text(br.center(), Align2::CENTER_CENTER, "Untagged", Tokens::ui(11.5), if sel { t.text } else { t.text_faint }); + cx.auto.add("essentialSound.ducking.against.Untagged", br, "Untagged"); + if resp.clicked() && on { + cx.set("ducking.againstUntyped", json!(!sel), true); + } + slider(ui, cx, "Sensitivity", "ducking.sensitivity", st.ducking.sensitivity, (0.0, 10.0), 1, "", on); + slider(ui, cx, "Reduce By", "ducking.reduceDb", st.ducking.reduce_db, (-40.0, 0.0), 1, " dB", on); + slider(ui, cx, "Fades", "ducking.fadeS", st.ducking.fade_s * 1000.0, (0.0, 5000.0), 0, " ms", on); + ui.add_space(4.0); + let (r, _) = ui.allocate_exact_size(vec2(ui.available_width(), 30.0), Sense::hover()); + let br = Rect::from_min_size(r.min + vec2(0.0, 2.0), vec2(150.0, 24.0)); + if button(ui, cx, br, "Generate Keyframes", "essentialSound.generateDucking") && on { + cx.run("essentialSound.generateDucking", json!({})); + } +} + +fn footer(ui: &mut egui::Ui, cx: &mut Ctx, rect: Rect, st: Option<&EssentialSound>) { + let t = cx.t; + let p = ui.painter(); + p.line_segment([pos2(rect.min.x + PAD, rect.min.y + 4.0), pos2(rect.max.x - PAD, rect.min.y + 4.0)], Stroke::new(1.0, t.separator)); + let Some(st) = st else { return }; + // Clip Volume: switch, label, blue value, slider + let row = Rect::from_min_max(pos2(rect.min.x + PAD, rect.min.y + 12.0), pos2(rect.max.x - PAD, rect.min.y + 34.0)); + if switch(ui, cx, Rect::from_min_size(row.min + vec2(0.0, 4.0), vec2(28.0, 14.0)), st.volume.on, "essentialSound.volume.on", "Clip Volume") { + cx.set("volume.on", json!(!st.volume.on), true); + } + ui.painter().text(pos2(row.min.x + 38.0, row.center().y), Align2::LEFT_CENTER, "Clip Volume", Tokens::ui(12.5), t.text); + let vr = Rect::from_min_max(pos2(row.max.x - 70.0, row.min.y), row.max); + ui.painter().text(pos2(vr.max.x - 18.0, vr.center().y), Align2::RIGHT_CENTER, format!("{:.1}", st.volume.level_db), Tokens::ui(12.0), t.hot_text); + ui.painter().text(pos2(vr.max.x, vr.center().y), Align2::RIGHT_CENTER, "dB", Tokens::ui(12.0), t.text_dim); + let track = Rect::from_min_max(pos2(row.min.x + 38.0, row.max.y + 6.0), pos2(row.max.x, row.max.y + 18.0)); + slider_track(ui, cx, track, "volume.levelDb", st.volume.level_db, (-24.0, 15.0), st.volume.on); + let mr = Rect::from_min_size(pos2(rect.min.x + PAD, rect.max.y - 24.0), vec2(28.0, 14.0)); + if switch(ui, cx, mr, st.mute, "essentialSound.mute", "Mute") { + cx.set("mute", json!(!st.mute), true); + } + ui.painter().text(pos2(mr.max.x + 10.0, mr.center().y), Align2::LEFT_CENTER, "Mute", Tokens::ui(12.5), t.text); +} + +// ------------------------------------------------------------------------------------- widgets + +/// Section header: twirl, bold title, switch at the right. Returns whether it is open. +fn section_header(ui: &mut egui::Ui, cx: &mut Ctx, sec: Section, open: bool, enabled: Option, key: &str) -> bool { + let t = cx.t; + let (r, resp) = ui.allocate_exact_size(vec2(ui.available_width(), 44.0), Sense::click()); + ui.painter().line_segment([r.left_top(), r.right_top()], Stroke::new(1.0, t.separator)); + icons::paint( + ui.painter(), + Rect::from_center_size(pos2(r.min.x + 10.0, r.center().y), vec2(11.0, 11.0)), + if open { Icon::ChevronDown } else { Icon::ChevronRight }, + t.text, + ); + ui.painter().text(pos2(r.min.x + 30.0, r.center().y), Align2::LEFT_CENTER, sec.label(), Tokens::semibold(13.0), t.text); + cx.auto.add(&format!("essentialSound.section.{}", sec.label()), r, sec.label()); + if let (Some(on), Some(k)) = (enabled, sec.key()) { + let sw = Rect::from_center_size(pos2(r.max.x - 16.0, r.center().y), vec2(30.0, 15.0)); + if switch(ui, cx, sw, on, &format!("essentialSound.section.{}.toggle", sec.label()), sec.label()) { + cx.set(k, json!(!on), true); + return open; + } + } + if resp.clicked() { + cx.actions.push(("ui.twirl".into(), json!({"key": key, "open": !open}))); + } + open +} + +/// A pill switch. Returns clicked. +fn switch(ui: &mut egui::Ui, cx: &mut Ctx, r: Rect, on: bool, id: &str, label: &str) -> bool { + let resp = ui.interact(r.expand(3.0), egui::Id::new(("es-sw", id)), Sense::click()); + let p = ui.painter(); + let rad = r.height() / 2.0; + if on { + p.rect_filled(r, rad, Color32::from_gray(0xd4)); + p.circle_filled(pos2(r.max.x - rad, r.center().y), rad - 3.0, Color32::from_gray(0x1d)); + } else { + p.rect_stroke(r, rad, Stroke::new(1.5, Color32::from_gray(if resp.hovered() { 0xb0 } else { 0x8a })), StrokeKind::Inside); + p.circle_stroke(pos2(r.min.x + rad, r.center().y), rad - 3.0, Stroke::new(1.5, Color32::from_gray(0xd4))); + } + cx.auto.add(id, r, label); + resp.clicked() +} + +fn switch_row(ui: &mut egui::Ui, cx: &mut Ctx, label: &str, key: &str, on: bool, active: bool) { + let t = cx.t; + let (r, _) = ui.allocate_exact_size(vec2(ui.available_width(), 30.0), Sense::hover()); + let sw = Rect::from_min_size(pos2(r.min.x, r.center().y - 7.5), vec2(30.0, 15.0)); + if switch(ui, cx, sw, on, &format!("essentialSound.{key}"), label) && active { + cx.set(key, json!(!on), true); + } + ui.painter().text(pos2(sw.max.x + 12.0, r.center().y), Align2::LEFT_CENTER, label, Tokens::ui(12.5), if active { t.text } else { t.text_faint }); +} + +/// Switch + label row and its 0…10 amount slider. +fn slot(ui: &mut egui::Ui, cx: &mut Ctx, label: &str, key: &str, s: Slot, active: bool) { + switch_row(ui, cx, label, &format!("{key}.on"), s.on, active); + slider(ui, cx, "", &format!("{key}.amount"), s.amount, (0.0, 10.0), 1, "", active && s.on); +} + +/// Label + value line, then a track with a ring knob. +#[allow(clippy::too_many_arguments)] +fn slider(ui: &mut egui::Ui, cx: &mut Ctx, label: &str, key: &str, v: f64, range: (f64, f64), decimals: usize, suffix: &str, active: bool) { + let t = cx.t; + let h = if label.is_empty() { 26.0 } else { 46.0 }; + let (r, _) = ui.allocate_exact_size(vec2(ui.available_width(), h), Sense::hover()); + let col = if active { t.text_dim } else { t.text_faint }; + let ty = if label.is_empty() { + let vx = r.max.x; + ui.painter().text(pos2(vx, r.center().y), Align2::RIGHT_CENTER, format!("{v:.decimals$}{suffix}"), Tokens::ui(11.5), col); + r.center().y + } else { + ui.painter().text(pos2(r.min.x, r.min.y + 10.0), Align2::LEFT_CENTER, label, Tokens::ui(12.5), if active { t.text } else { t.text_faint }); + ui.painter().text(pos2(r.max.x, r.min.y + 10.0), Align2::RIGHT_CENTER, format!("{v:.decimals$}{suffix}"), Tokens::ui(11.5), col); + r.min.y + 32.0 + }; + let right = if label.is_empty() { r.max.x - 40.0 } else { r.max.x }; + let track = Rect::from_min_max(pos2(r.min.x + if label.is_empty() { 42.0 } else { 0.0 }, ty - 6.0), pos2(right, ty + 6.0)); + slider_track(ui, cx, track, key, v, range, active); +} + +/// The bare slider: 1 pt track, hollow ring knob. Sends `essentialSound.set` while dragged. +fn slider_track(ui: &mut egui::Ui, cx: &mut Ctx, track: Rect, key: &str, v: f64, (lo, hi): (f64, f64), active: bool) { + let t = cx.t; + let y = track.center().y; + let f = ((v - lo) / (hi - lo)).clamp(0.0, 1.0) as f32; + let kx = track.min.x + f * track.width(); + let resp = ui.interact(track.expand2(vec2(6.0, 4.0)), egui::Id::new(("es-slider", key)), Sense::click_and_drag()); + let line = if active { Color32::from_gray(0x8a) } else { Color32::from_gray(0x4a) }; + ui.painter().line_segment([pos2(track.min.x, y), pos2(kx - 7.0, y)], Stroke::new(1.5, line)); + ui.painter().line_segment([pos2(kx + 7.0, y), pos2(track.max.x, y)], Stroke::new(1.5, line)); + ui.painter().circle_filled(pos2(kx, y), 6.0, t.panel_bg); + ui.painter().circle_stroke( + pos2(kx, y), + 6.0, + Stroke::new( + 1.6, + if resp.dragged() { + t.hot_text + } else if active { + Color32::from_gray(0xd4) + } else { + Color32::from_gray(0x6a) + }, + ), + ); + cx.auto.add(&format!("essentialSound.{key}"), track, key); + if !active { + return; + } + if (resp.dragged() || resp.clicked()) + && let Some(p) = resp.interact_pointer_pos() + { + let nf = ((p.x - track.min.x) / track.width()).clamp(0.0, 1.0) as f64; + let mut nv = lo + nf * (hi - lo); + nv = (nv * 10.0).round() / 10.0; + if (nv - v).abs() > 1e-9 { + // Fades are edited in ms and stored in seconds. + let send = if key == "ducking.fadeS" { (nv / 1000.0 * 100.0).round() / 100.0 } else { nv }; + cx.set(key, json!(send), resp.drag_started() || resp.clicked()); + } + } +} + +fn combo(ui: &mut egui::Ui, cx: &mut Ctx, key: &str, names: &[&str], current: &str, active: bool) { + let mut picked = None; + ui.add_enabled_ui(active, |ui| { + let cb = egui::ComboBox::from_id_salt(("es-combo", key)).selected_text(current).width(ui.available_width() - 4.0).show_ui(ui, |ui| { + for n in names { + if ui.selectable_label(*n == current, *n).clicked() { + picked = Some(n.to_string()); + } + } + }); + cx.auto.add(&format!("essentialSound.{key}"), cb.response.rect, key); + }); + if let Some(n) = picked { + cx.set(key, json!(n), true); + } +} + +/// Segmented choice (50 Hz | 60 Hz, Low Tone | High Tone). +fn choice_row(ui: &mut egui::Ui, cx: &mut Ctx, key: &str, opts: &[&str], sel: usize, active: bool, value: impl Fn(usize) -> Value) { + let t = cx.t; + let (r, _) = ui.allocate_exact_size(vec2(ui.available_width(), 28.0), Sense::hover()); + let w = 84.0; + for (i, o) in opts.iter().enumerate() { + let br = Rect::from_min_size(pos2(r.min.x + 42.0 + i as f32 * (w + 4.0), r.min.y + 3.0), vec2(w, 22.0)); + let resp = ui.interact(br, egui::Id::new(("es-choice", key, i)), Sense::click()); + let on = i == sel; + ui.painter().rect_filled( + br, + 4.0, + if on { + Color32::from_gray(0x4b) + } else if resp.hovered() && active { + t.hover + } else { + t.field_bg + }, + ); + ui.painter().text(br.center(), Align2::CENTER_CENTER, *o, Tokens::ui(11.5), if active { t.text } else { t.text_faint }); + cx.auto.add(&format!("essentialSound.{key}.{i}"), br, o); + if resp.clicked() && active && !on { + cx.set(key, value(i), true); + } + } +} + +fn button(ui: &mut egui::Ui, cx: &mut Ctx, r: Rect, text: &str, id: &str) -> bool { + let t = cx.t; + let resp = ui.interact(r, egui::Id::new(("es-btn", id)), Sense::click()); + ui.painter().rect_filled(r, 4.0, if resp.hovered() { Color32::from_gray(0x2a) } else { t.field_bg }); + ui.painter().rect_stroke(r, 4.0, Stroke::new(1.0, Color32::from_gray(0x4b)), StrokeKind::Inside); + ui.painter().text(r.center(), Align2::CENTER_CENTER, text, Tokens::ui(12.0), t.text); + cx.auto.add(id, r, text); + resp.clicked() +} + +// ------------------------------------------------------------------------------------- icons + +/// Audio-type glyphs, drawn from scratch: speech bubble, eighth note, burst, waves. +pub fn type_icon(p: &Painter, r: Rect, kind: AudioType, col: Color32) { + let s = Stroke::new(1.4, col); + let at = |x: f32, y: f32| pos2(r.min.x + x * r.width(), r.min.y + y * r.height()); + match kind { + AudioType::Dialogue => { + let b = Rect::from_min_max(at(0.08, 0.12), at(0.92, 0.68)); + p.rect_stroke(b, r.width() * 0.16, s, StrokeKind::Middle); + p.add(egui::Shape::line(vec![at(0.3, 0.68), at(0.24, 0.9), at(0.48, 0.68)], s)); + } + AudioType::Music => { + p.circle_filled(at(0.32, 0.78), r.width() * 0.15, col); + p.line_segment([at(0.45, 0.78), at(0.45, 0.1)], s); + p.add(egui::Shape::line(vec![at(0.45, 0.1), at(0.78, 0.3), at(0.78, 0.48)], s)); + } + AudioType::Sfx => { + let c = r.center(); + for k in 0..12 { + let a = k as f32 * std::f32::consts::TAU / 12.0; + let len = if k % 2 == 0 { 0.5 } else { 0.3 } * r.width(); + p.line_segment([c, c + vec2(a.cos(), a.sin()) * len], s); + } + } + AudioType::Ambience => { + for (k, y) in [0.3f32, 0.55, 0.8].iter().enumerate() { + let pts: Vec<_> = (0..=12) + .map(|i| { + let x = i as f32 / 12.0; + at(0.05 + x * 0.9, *y + 0.07 * (x * std::f32::consts::TAU + k as f32).sin()) + }) + .collect(); + p.add(egui::Shape::line(pts, s)); + } + } + } +} + +/// Save preset: a downward arrow into a tray. +fn save_icon(p: &Painter, r: Rect, col: Color32) { + let s = Stroke::new(1.4, col); + let at = |x: f32, y: f32| pos2(r.min.x + x * r.width(), r.min.y + y * r.height()); + p.line_segment([at(0.5, 0.05), at(0.5, 0.62)], s); + p.add(egui::Shape::line(vec![at(0.28, 0.42), at(0.5, 0.64), at(0.72, 0.42)], s)); + p.add(egui::Shape::line(vec![at(0.08, 0.6), at(0.08, 0.92), at(0.92, 0.92), at(0.92, 0.6)], s)); +} diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index ccb6ec9b..4c0df9bd 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -4,6 +4,7 @@ pub mod dialogs; pub mod effect_controls; pub mod effects; +pub mod essential_sound; pub mod export_mode; pub mod file_dialogs; pub mod graphics; @@ -52,6 +53,7 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, p: PanelKind, rect: Rect) PanelKind::AudioClipMixer => mixer::clip_mixer(app, ui, rect), PanelKind::LumetriScopes => misc::scopes(app, ui, rect), PanelKind::Text => text::show(app, ui, rect), + PanelKind::EssentialSound => essential_sound::show(app, ui, rect), other => crate::dock::placeholder(ui, rect, &app.tokens, &format!("{} — coming in a later milestone", other.title())), } } diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index d90aeaad..784563e8 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -264,6 +264,9 @@ pub struct UiState { /// Transient status line shown in the footer. pub status: String, /// Export mode: chosen format id and output path. + /// Essential Sound sub-tab: "Edit" or "Browse". + #[serde(default)] + pub essential_sound_tab: String, #[serde(default)] pub export_format: String, #[serde(default)] @@ -346,6 +349,7 @@ impl Default for UiState { dark: true, show_scopes: false, status: String::new(), + essential_sound_tab: "Edit".into(), export_format: "h264".into(), export_path: String::new(), export_burn_captions: false, diff --git a/crates/ui-egui/tests/essential_sound_ui.rs b/crates/ui-egui/tests/essential_sound_ui.rs new file mode 100644 index 00000000..d412deec --- /dev/null +++ b/crates/ui-egui/tests/essential_sound_ui.rs @@ -0,0 +1,255 @@ +//! Headless UI test of the Essential Sound panel: the real `FilmcraftApp` under `egui_kittest`, +//! driven by automation id and by frame-by-frame pointer drags. +//! +//! Set `FILMCRAFT_UI_SNAPSHOT_DIR=

` to also render the window offscreen with wgpu and write +//! `essential-sound-*.png` there; without it no GPU is needed. + +use std::sync::mpsc::{Sender, channel}; + +use egui_kittest::Harness; +use filmcraft_engine::Session; +use filmcraft_ui_egui::FilmcraftApp; +use filmcraft_ui_egui::control::ControlRequest; +use serde_json::{Value, json}; + +struct Driver { + harness: Harness<'static, FilmcraftApp>, + tx: Sender, + snapshots: Option, +} + +impl Driver { + fn demo() -> Self { + let mut session = Session::default(); + session.execute("file.openDemoProject", json!({})).expect("demo project"); + let (tx, rx) = channel(); + let app = FilmcraftApp::new(session).with_control(rx); + let snapshots = std::env::var_os("FILMCRAFT_UI_SNAPSHOT_DIR").map(std::path::PathBuf::from); + let mut b = Harness::builder().with_size(egui::vec2(1600.0, 980.0)).with_max_steps(10_000); + if snapshots.is_some() { + b = b.wgpu(); + } + let harness = b.build_eframe(move |_cc| app); + let mut d = Driver { harness, tx, snapshots }; + d.frames(4); + d + } + + fn frames(&mut self, n: usize) { + for _ in 0..n { + let ctx = self.harness.ctx.clone(); + let mut raw = std::mem::take(self.harness.input_mut()); + eframe::App::raw_input_hook(self.harness.state_mut(), &ctx, &mut raw); + *self.harness.input_mut() = raw; + self.harness.step(); + } + } + + fn call(&mut self, method: &str, params: Value) -> Value { + let (req, reply) = ControlRequest::new(method, params.clone()); + self.tx.send(req).unwrap(); + for _ in 0..600 { + self.frames(1); + if let Ok(v) = reply.try_recv() { + return v; + } + } + panic!("no reply to {method} {params}"); + } + + fn ok(&mut self, method: &str, params: Value) -> Value { + let v = self.call(method, params.clone()); + assert_eq!(v["ok"], json!(true), "{method} {params} failed: {v}"); + v["result"].clone() + } + + fn exec(&mut self, command: &str, params: Value) -> Value { + self.ok("engine.execute", json!({"command": command, "params": params})) + } + + fn click(&mut self, id: &str) { + self.ok("ui.click", json!({"id": id})); + self.frames(2); + } + + fn element(&mut self, id: &str) -> Option<[f32; 4]> { + let v = self.ok("ui.elements", json!({"prefix": id})); + v.as_array()?.iter().find(|e| e["id"] == id).map(|e| { + let r = &e["rect"]; + [r[0].as_f64().unwrap() as f32, r[1].as_f64().unwrap() as f32, r[2].as_f64().unwrap() as f32, r[3].as_f64().unwrap() as f32] + }) + } + + fn ids(&mut self, prefix: &str) -> Vec { + let v = self.ok("ui.elements", json!({"prefix": prefix})); + v.as_array().unwrap().iter().filter_map(|e| e["id"].as_str().map(str::to_string)).collect() + } + + /// Press on `id` at fraction `fx` of its width, move to fraction `tx` over several frames, release. + fn drag_x(&mut self, id: &str, fx: f32, tx: f32, steps: usize) { + let r = self.element(id).unwrap_or_else(|| panic!("no element {id}")); + let mut p = egui::pos2(r[0] + r[2] * fx, r[1] + r[3] / 2.0); + let push = |d: &mut Self, e: egui::Event| d.harness.input_mut().events.push(e); + push(self, egui::Event::PointerMoved(p)); + self.frames(1); + push(self, egui::Event::PointerButton { pos: p, button: egui::PointerButton::Primary, pressed: true, modifiers: Default::default() }); + self.frames(1); + for _ in 0..steps { + p.x += r[2] * (tx - fx) / steps as f32; + push(self, egui::Event::PointerMoved(p)); + self.frames(1); + } + push(self, egui::Event::PointerButton { pos: p, button: egui::PointerButton::Primary, pressed: false, modifiers: Default::default() }); + self.frames(2); + } + + /// Offscreen render of the window (or the bounding box of the elements under `prefix`). + fn snapshot(&mut self, name: &str, prefix: Option<&str>) { + let Some(dir) = self.snapshots.clone() else { return }; + let crop = prefix.map(|p| { + let v = self.ok("ui.elements", json!({"prefix": p})); + let mut bb = [f32::MAX, f32::MAX, f32::MIN, f32::MIN]; + for e in v.as_array().unwrap() { + let r = &e["rect"]; + let (x, y, w, h) = (r[0].as_f64().unwrap() as f32, r[1].as_f64().unwrap() as f32, r[2].as_f64().unwrap() as f32, r[3].as_f64().unwrap() as f32); + bb = [bb[0].min(x), bb[1].min(y), bb[2].max(x + w), bb[3].max(y + h)]; + } + bb + }); + self.frames(2); + let img = match self.harness.render() { + Ok(i) => i, + Err(e) => { + eprintln!("snapshot {name} skipped: {e}"); + return; + } + }; + let ppp = img.width() as f32 / 1600.0; + let img = match crop { + Some(bb) => { + let x0 = ((bb[0] - 24.0) * ppp).max(0.0) as u32; + let y0 = ((bb[1] - 30.0) * ppp).max(0.0) as u32; + let x1 = (((bb[2] + 16.0) * ppp) as u32).min(img.width()); + let y1 = (((bb[3] + 46.0) * ppp) as u32).min(img.height()); + image::imageops::crop_imm(&img, x0, y0, x1 - x0, y1 - y0).to_image() + } + None => img, + }; + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join(format!("{name}.png")); + img.save(&path).unwrap(); + eprintln!("snapshot: {}", path.display()); + } +} + +fn es(d: &mut Driver, clip: u64) -> Value { + d.exec("essentialSound.inspect", json!({"clips": [clip]}))["clips"][0].clone() +} + +#[test] +fn essential_sound_panel_by_id_and_by_drag() { + let mut d = Driver::demo(); + d.ok("ui.set", json!({"workspace": "Audio"})); + d.frames(3); + let seq = d.exec("sequence.inspect", json!({})); + let a1 = seq["audio"][0]["items"][0]["clip"].as_u64().unwrap(); + let music = seq["audio"][1]["items"][0]["clip"].as_u64().unwrap(); + // select a clip → the type buttons + d.exec("timeline.select", json!({"clips": [a1]})); + d.frames(3); + let ids = d.ids("essentialSound."); + for id in [ + "essentialSound.tab.Edit", + "essentialSound.tab.Browse", + "essentialSound.type.Dialogue", + "essentialSound.type.Music", + "essentialSound.type.SFX", + "essentialSound.type.Ambience", + ] { + assert!(ids.iter().any(|i| i == id), "{id} missing from {ids:?}"); + } + d.snapshot("essential-sound-untyped", Some("essentialSound.")); + d.click("essentialSound.type.Dialogue"); + assert_eq!(es(&mut d, a1)["type"], "Dialogue"); + d.frames(2); + let ids = d.ids("essentialSound."); + for id in [ + "essentialSound.clearType", + "essentialSound.preset", + "essentialSound.preset.save", + "essentialSound.section.Loudness", + "essentialSound.section.Repair.toggle", + "essentialSound.section.Clarity", + "essentialSound.section.Creative", + "essentialSound.repair.noise.on", + "essentialSound.repair.noise.amount", + "essentialSound.repair.humHz.1", + "essentialSound.clarity.eqPreset", + "essentialSound.creative.reverbPreset", + "essentialSound.volume.on", + "essentialSound.volume.levelDb", + "essentialSound.mute", + ] { + assert!(ids.iter().any(|i| i == id), "{id} missing from {ids:?}"); + } + // switch a repair slot on, drag its slider: one undo step + d.click("essentialSound.repair.noise.on"); + let c = es(&mut d, a1); + assert_eq!(c["settings"]["repair"]["noise"]["on"], true); + assert!(c["effects"].as_array().unwrap().iter().any(|e| e["effect"] == "denoise")); + let undo0 = d.exec("history.list", json!({}))["undo"].as_array().unwrap().len(); + d.drag_x("essentialSound.repair.noise.amount", 0.5, 0.9, 6); + let amt = es(&mut d, a1)["settings"]["repair"]["noise"]["amount"].as_f64().unwrap(); + assert!(amt > 8.0, "slider dragged to {amt}"); + assert_eq!(d.exec("history.list", json!({}))["undo"].as_array().unwrap().len(), undo0 + 1, "a drag is one undo step"); + // the section switch bypasses the section's effects + d.click("essentialSound.section.Repair.toggle"); + let c = es(&mut d, a1); + assert_eq!(c["settings"]["repair"]["enabled"], false); + assert!(c["effects"].as_array().unwrap().iter().all(|e| e["enabled"] == false)); + d.click("essentialSound.section.Repair.toggle"); + // Loudness twirl + Auto-Match + d.click("essentialSound.section.Loudness"); + d.click("essentialSound.autoMatch"); + let c = es(&mut d, a1); + assert!(c["settings"]["loudness"]["target_lufs"].is_number(), "auto-matched: {c}"); + // a preset + d.exec("essentialSound.applyPreset", json!({"preset": "Podcast Voice"})); + d.frames(3); + d.snapshot("essential-sound-dialogue", Some("essentialSound.")); + // footer: clip volume and mute + d.click("essentialSound.volume.on"); + d.click("essentialSound.mute"); + let c = es(&mut d, a1); + assert_eq!(c["settings"]["volume"]["on"], true); + assert_eq!(c["settings"]["mute"], true); + d.click("essentialSound.mute"); + // music: Ducking section and Generate Keyframes + d.exec("timeline.select", json!({"clips": [music]})); + d.frames(2); + d.click("essentialSound.type.Music"); + d.frames(2); + let ids = d.ids("essentialSound."); + for id in [ + "essentialSound.section.Ducking.toggle", + "essentialSound.ducking.against.Dialogue", + "essentialSound.ducking.sensitivity", + "essentialSound.generateDucking", + ] { + assert!(ids.iter().any(|i| i == id), "{id} missing from {ids:?}"); + } + d.click("essentialSound.section.Ducking.toggle"); + d.click("essentialSound.generateDucking"); + assert_eq!(es(&mut d, music)["settings"]["ducking"]["enabled"], true); + d.snapshot("essential-sound-music", Some("essentialSound.")); + // Browse: presets per type; a click applies + d.click("essentialSound.tab.Browse"); + d.frames(2); + let ids = d.ids("essentialSound.browse."); + assert!(ids.iter().any(|i| i == "essentialSound.browse.Music.Duck Deep Under Dialogue"), "{ids:?}"); + d.click("essentialSound.browse.Music.Duck Deep Under Dialogue"); + assert_eq!(es(&mut d, music)["settings"]["preset"], "Duck Deep Under Dialogue"); + d.snapshot("essential-sound-browse", Some("essentialSound.")); + d.click("essentialSound.tab.Edit"); + d.snapshot("essential-sound-audio-workspace", None); +} diff --git a/docs/architecture.md b/docs/architecture.md index b53241a1..fc5b9649 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -259,6 +259,43 @@ Mix: bus sum → pre-fader inserts → fader → meter → post-fader inserts mix (Track Mixer and Audio Meters read them) and the newest project snapshot, which the audio callback uses, so edits made during playback are heard. +### 5.2 Essential Sound + +```text +clip.essential (type + settings) ──apply()──► clip effects marked `essential`, clip gain, Volume, Panner +essentialSound.autoMatch BS.1770 integrated loudness of render::audio::clip_signal → match gain (clip gain) +essentialSound.generateDucking trigger clips' summed level (10 ms hops) → activity regions → Volume keyframes +``` + +- **Model** (`project::essential`). A clip's `essential: Option` holds its audio type + (Dialogue / Music / SFX / Ambience) and per-type sections: Loudness (match gain, measured and target + LUFS), Repair (Reduce Noise, Reduce Rumble, DeHum 50/60 Hz, DeEss, Reduce Reverb), Clarity + (Dynamics, EQ preset + amount, Enhance Speech), Creative (Reverb preset + amount, Stereo Width for + Ambience), Ducking (against types, sensitivity, reduce by, fades), Pan, Clip Volume and Mute. +- **Effects under the hood.** `essential::apply(item, old, t)` turns the settings into ordinary clip + effects (Highpass, DeNoise, DeHummer, DeEsser, DeReverb, Dynamics Processing, Parametric Equalizer, + Enhance Speech, Stereo Width, Studio Reverb) flagged `EffectInstance::essential`, in that order, ahead + of the user's own effects. Only parameters whose derived value changed are written (at the playhead), + so keyframes added in Effect Controls survive. A section switch bypasses its effects, a slot switch + removes its effect, clearing the type removes them all. Auto-match gain, Clip Volume (an offset on + the Volume level or on all its keyframes) and Pan are applied as deltas, so clearing restores the + clip. Because these are normal effects, playback, the mixer and export need nothing special. +- **Loudness.** The clip signal (clip gain + effects, before Volume) is measured with the BS.1770 + meter; the gain is linear after the effects, so one measurement hits the target exactly. Targets are + preferences (`audio.dialogueTargetLufs` −23, `musicTargetLufs` −25, `sfxTargetLufs` −21, + `ambienceTargetLufs` −30). +- **Ducking.** Sensitivity 0…10 maps to a threshold −20 − 4·s dBFS on the summed trigger signal (50 ms + window); regions shorter than 100 ms are dropped and pauses under 250 ms bridged; each region gets a + fade-down before it and a fade-up after it (`audio_dsp::ducking::duck_keyframes`), written as Volume + level keyframes that replace earlier ones. +- **Presets** (our own names and values) per type; user presets are saved in preferences + (`essentialSound.userPresets`). Music remixing to a duration is out of scope (the Duration section + says so). +- **Enhance Speech** is a DSP chain (high-pass, de-mud, presence and air EQ, expander, compressor), not a + model: no speech-enhancement model with an open licence that we could ship and verify is bundled. + DeepFilterNet (MIT/Apache-2.0, Rust inference via tract) is the candidate for a future optional + integration behind a trait. + ## 6. Export jobs (`filmcraft-export`) ```text diff --git a/docs/control-protocol.md b/docs/control-protocol.md index 065034d2..431fbe4f 100644 --- a/docs/control-protocol.md +++ b/docs/control-protocol.md @@ -45,6 +45,20 @@ record|soloSafe|output|input|fx.|send.|meter|name>` (popup entries below t `clipMixer.A1.`, `timeline.track.A1.keyframes[.|.clip]`, `timeline.track.A1.lane[.kf.]`, `audioGain.`. +Essential Sound: `essentialSound.inspect`, `essentialSound.setType {type: dialogue|music|sfx|ambience}`, +`essentialSound.clearType`, `essentialSound.set {key, value}` or `{values: {key: value}}` (keys such as +`repair.noise.on`, `repair.noise.amount` (0–10), `repair.humHz`, `clarity.eqPreset`, `clarity.enhanceTone`, +`creative.reverbPreset`, `ducking.against`, `ducking.reduceDb`, `ducking.fadeS`, `pan.value`, +`volume.levelDb`, `mute`, section switches `
.enabled`), `essentialSound.applyPreset {preset, type?}`, +`essentialSound.savePreset {name}`, `essentialSound.deletePreset {name}`, `essentialSound.autoMatch {target?}`, +`essentialSound.generateDucking`; all take `clips` (default: the selected audio clips). UI ids: +`essentialSound.tab.`, `essentialSound.type.`, +`essentialSound.clearType`, `essentialSound.preset[.save|.delete|.name|.ok]`, +`essentialSound.section.[.toggle]`, a row per setting key (`essentialSound.repair.noise.on`, +`essentialSound.repair.noise.amount`, `essentialSound.repair.humHz.`), `essentialSound.autoMatch`, +`essentialSound.ducking.against.`, `essentialSound.generateDucking`, `essentialSound.volume.on`, +`essentialSound.volume.levelDb`, `essentialSound.mute`, `essentialSound.browse..`. + Project files, auto-save, crash recovery and preferences commands (`file.recover`, `prefs.set`, …) and their automation ids are listed in [project-files.md](project-files.md). diff --git a/docs/testing.md b/docs/testing.md index a9fdb454..b57547a9 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -133,6 +133,18 @@ keyframes and the Audio Gain dialog by automation id and with multi-frame pointe over several frames, release). With `FILMCRAFT_UI_SNAPSHOT_DIR=` it also renders the window offscreen through wgpu (`Harness::render`) and writes `mixer-*.png`; this works without a visible window (for example on a locked screen, where `ui.screenshot` cannot capture). +`crates/ui-egui/tests/essential_sound_ui.rs` does the same for the Essential Sound panel (type buttons, +switches, a slider drag as one undo step, section bypass, Auto-Match, ducking, Browse presets; +`essential-sound-*.png`). + +Essential Sound engine tests (`crates/engine/src/essential_sound_tests.rs`) build projects from +generated speech-like and tonal WAVs: Auto-Match lands within ±0.5 LU of the target (measured: 0.000 LU, +with and without a repair/clarity chain); ducking on a dialogue + music project gives keyframes within +60 ms of the expected times (measured 20 ms) and −15.00 dB in the mix; each repair stage improves its +metric through the render path (hum −39 dB, rumble −22 dB, noise floor −13 dB, sibilance −19 dB, +reverb tail −10 dB); the mix is bit-identical however requests are cut and the WAV export equals it. +`perf_full_dialogue_chain_realtime_factor` (ignored; run with `--release`) prints the realtime factor of +all nine Dialogue effects on one clip (22× on one core). Not covered headless: `ui.screenshot` (needs a real viewport), the wgpu monitor path (the harness runs the CPU texture path), audio output, and wall-clock playback advance (kittest frames do not From 59629eae0f629e593aef9b7236a9bcfd99693852 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 1 Oct 2026 08:42:02 -0400 Subject: [PATCH 5/5] M7.4: project schema v4 for Essential Sound (older builds refuse instead of silently dropping the settings on save) Co-Authored-By: Claude Opus 5.5 --- crates/format/src/lib.rs | 11 +++++++++-- docs/project-files.md | 10 ++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/crates/format/src/lib.rs b/crates/format/src/lib.rs index d81da80e..208ec6c9 100644 --- a/crates/format/src/lib.rs +++ b/crates/format/src/lib.rs @@ -22,6 +22,8 @@ //! - **v3** (M10.2): graphic clips (`ItemKind::Graphic`, text/shape layers). The shape of older //! data is unchanged (no-op step); the bump makes builds without graphics refuse such files //! cleanly instead of failing to parse them. +//! - **v4** (M7.4): Essential Sound (clip audio types and settings, `EffectInstance::essential`). +//! No-op step; older builds would otherwise drop these fields silently when saving. pub mod atomic; pub mod autosave; @@ -40,7 +42,7 @@ pub type Migration = fn(Value) -> Result; /// `MIGRATIONS[i]` upgrades schema `i + 1` to `i + 2`. Append one function per schema bump; never /// edit a shipped one. -pub const MIGRATIONS: &[Migration] = &[v1_to_v2, v2_to_v3]; +pub const MIGRATIONS: &[Migration] = &[v1_to_v2, v2_to_v3, v3_to_v4]; /// The schema version this build writes (and the newest it reads). pub const SCHEMA_VERSION: u32 = MIGRATIONS.len() as u32 + 1; @@ -219,13 +221,18 @@ fn v2_to_v3(doc: Value) -> Result { Ok(doc) } +/// v3 → v4: Essential Sound fields added; existing data needs no change. +fn v3_to_v4(doc: Value) -> Result { + Ok(doc) +} + #[cfg(test)] mod tests { use super::*; #[test] fn schema_version_matches_table() { - assert_eq!(SCHEMA_VERSION, 3); + assert_eq!(SCHEMA_VERSION, 4); } #[test] diff --git a/docs/project-files.md b/docs/project-files.md index 9b6dfc6a..4da57a12 100644 --- a/docs/project-files.md +++ b/docs/project-files.md @@ -8,7 +8,7 @@ writing them is `crates/format` (`filmcraft-format`); the engine's `file.*` comm ```json { "format": "filmcraft.project", - "schema_version": 3, + "schema_version": 4, "generator": "FilmCraft 0.1.0", "project": { "name": "…", "settings": { … }, "root": { … }, "items": { … }, "next_id": 48 } } @@ -27,6 +27,7 @@ writing them is `crates/format` (`filmcraft-format`); the engine's `file.*` comm | 1 | M0 | the bare `Project` object with a `"version": 1` field | | 2 | M11.1 | the envelope above; the project's own `version` field is gone | | 3 | M10.2 | graphic clips (`ItemKind::Graphic` with text/shape layers); a no-op step for older data | +| 4 | M7.4 | Essential Sound: clip audio types and settings; no-op step | ### Migrations @@ -43,7 +44,8 @@ document; after the last one the result is deserialized into the current model. Fields added with `#[serde(default)]` don't need a migration: missing values take their defaults. But when new data would make the file unreadable to an older build (a new enum variant such as `ItemKind::Graphic`, a new required shape), bump the schema with a no-op step so older builds refuse -the file with the message below instead of failing with a parse error. +the file with the message below instead of failing with a parse error. Do the same when an older +build would load the file but silently drop new fields on its next save. When you open an older file FilmCraft says so ("Upgraded project from schema v1 to v2"). The file on disk is unchanged until you save. The **first save over it** keeps the original next to it as @@ -53,8 +55,8 @@ on disk is unchanged until you save. The **first save over it** keeps the origin A file with a `schema_version` newer than the build supports is **refused**, not half-read: -> this project was saved by a newer version of FilmCraft (project schema v4); this build reads up to -> v3. Update FilmCraft to open it. +> this project was saved by a newer version of FilmCraft (project schema v5); this build reads up to +> v4. Update FilmCraft to open it. Reading it partially and saving it back would silently drop whatever the newer version added.