diff --git a/apps/filmcraft/src/audio.rs b/apps/filmcraft/src/audio.rs index 4e54897c..914f10f8 100644 --- a/apps/filmcraft/src/audio.rs +++ b/apps/filmcraft/src/audio.rs @@ -12,6 +12,7 @@ pub struct CpalOut { stream: Option, played: Arc, rate: u32, + channels: u16, hw: AudioHardwarePrefs, /// Sequence sample rate for "Attempt to force hardware to document sample rate". document_rate: Option, @@ -44,7 +45,14 @@ impl CpalOut { let host = cpal::default_host(); let dev = host.default_output_device()?; let cfg = dev.default_output_config().ok()?; - Some(Self { stream: None, played: Arc::new(AtomicU64::new(0)), rate: cfg.sample_rate().0, hw: AudioHardwarePrefs::default(), document_rate: None }) + Some(Self { + stream: None, + played: Arc::new(AtomicU64::new(0)), + rate: cfg.sample_rate().0, + channels: cfg.channels(), + hw: AudioHardwarePrefs::default(), + document_rate: None, + }) } /// The stream configuration the settings ask for, falling back to the device default. @@ -77,6 +85,7 @@ impl AudioOut for CpalOut { config.buffer_size = cpal::BufferSize::Fixed(self.hw.buffer_size); } self.rate = cfg.sample_rate().0; + self.channels = cfg.channels(); self.played.store(0, Ordering::SeqCst); let played = self.played.clone(); let err = |e| eprintln!("filmcraft: audio stream error: {e}"); @@ -103,6 +112,9 @@ impl AudioOut for CpalOut { fn sample_rate(&self) -> u32 { self.rate } + fn channels(&self) -> usize { + self.channels as usize + } fn played_frames(&self) -> Option { self.stream.as_ref().map(|_| self.played.load(Ordering::SeqCst)) } @@ -122,6 +134,7 @@ impl AudioOut for CpalOut { && let Ok(cfg) = self.config(&dev) { self.rate = cfg.sample_rate().0; + self.channels = cfg.channels(); } } } diff --git a/crates/aac/src/decoder.rs b/crates/aac/src/decoder.rs index b112eb37..14dbbf4e 100644 --- a/crates/aac/src/decoder.rs +++ b/crates/aac/src/decoder.rs @@ -19,6 +19,10 @@ const ID_PCE: u32 = 5; const ID_FIL: u32 = 6; const ID_END: u32 = 7; +/// `extension_type` of a fill element carrying SBR data (ISO/IEC 14496-3 §4.5.2.8.2, Table 4.121). +const EXT_SBR_DATA: u32 = 13; +const EXT_SBR_DATA_CRC: u32 = 14; + /// One decoded `individual_channel_stream` before stereo processing and synthesis. struct Ics { info: IcsInfo, @@ -45,6 +49,8 @@ pub struct Decoder { offsets: Vec, channels: Vec, noise_state: u32, + /// A fill element carried SBR data (implicitly signalled HE-AAC). + sbr_seen: bool, } impl Decoder { @@ -67,7 +73,7 @@ impl Decoder { if layout.is_empty() && asc.channel_config != 0 { return Err(Error::InvalidConfig("unsupported channel configuration")); } - let mut d = Decoder { asc, layout: Vec::new(), offsets: Vec::new(), channels: Vec::new(), noise_state: 0x2545_F491 }; + let mut d = Decoder { asc, layout: Vec::new(), offsets: Vec::new(), channels: Vec::new(), noise_state: 0x2545_F491, sbr_seen: false }; d.set_layout(layout); Ok(d) } @@ -90,6 +96,12 @@ impl Decoder { pub fn sample_rate(&self) -> u32 { self.asc.sample_rate } + /// Whether the stream is HE-AAC: SBR signalled in the `AudioSpecificConfig` (explicit) or SBR + /// data seen in a fill element of a decoded access unit (implicit). Only the AAC-LC core is + /// decoded, at [`Self::sample_rate`]; the SBR high band is not reconstructed. + pub fn sbr(&self) -> bool { + self.asc.sbr || self.sbr_seen + } pub fn channels(&self) -> usize { self.channels.len() } @@ -174,7 +186,11 @@ impl Decoder { count += br.read_bits(8)? as usize; count -= 1; } - br.skip(count * 8)?; + if count > 0 { + let ext = br.read_bits(4)?; + self.sbr_seen |= matches!(ext, EXT_SBR_DATA | EXT_SBR_DATA_CRC); + br.skip(count * 8 - 4)?; + } } _ => break, // ID_END } @@ -542,3 +558,27 @@ fn synth(state: &mut ChannelState, info: &IcsInfo, spec: &[f32]) -> Vec { state.prev_shape = cur; out } + +#[cfg(test)] +mod tests { + use super::*; + + /// Implicitly signalled HE-AAC (an AAC-LC config) is recognised from the SBR fill element. + #[test] + fn sbr_fill_element_marks_implicit_he_aac() { + // AAC-LC, 22.05 kHz, stereo + let mut dec = Decoder::new(&[0x13, 0x90]).unwrap(); + assert!(!dec.sbr()); + // ID_FIL, count 1, extension_type EXT_SBR_DATA (+ 4 bits of payload), ID_END + let _ = dec.decode(&[0b1100_0011, 0b1010_0001, 0b1100_0000]); + assert!(dec.sbr()); + // other fill payloads (EXT_FILL) don't + let mut dec = Decoder::new(&[0x13, 0x90]).unwrap(); + let _ = dec.decode(&[0b1100_0010, 0b0000_0001, 0b1100_0000]); + assert!(!dec.sbr()); + // explicit signalling: AOT 5, core 22.05 kHz, stereo, extension 44.1 kHz, core AOT 2 + let dec = Decoder::new(&[0x2B, 0x92, 0x08, 0x00]).unwrap(); + assert!(dec.sbr()); + assert_eq!(dec.sample_rate(), 22_050); + } +} diff --git a/crates/audio-dsp/src/lib.rs b/crates/audio-dsp/src/lib.rs index 739abeca..977e5a97 100644 --- a/crates/audio-dsp/src/lib.rs +++ b/crates/audio-dsp/src/lib.rs @@ -7,6 +7,7 @@ //! engine and UI can build controls generically. //! * [`biquad`] — RBJ-cookbook biquads (TDF-II) with analytic magnitude response. //! * [`sync`] — offset between two recordings of one event (GCC-PHAT, sample-accurate). +//! * [`resample`] — streaming, block-invariant rate conversion for playback on a device at another rate. //! * [`channels`] — channel layouts, ITU-R BS.775 up/downmix, 5.1 mixdown types, the 5.1 panner. //! //! Conventions: audio is **planar f32** (`&mut [&mut [f32]]`, one slice per channel, all the same @@ -27,6 +28,7 @@ pub mod fft; pub mod loudness; pub mod oversample; pub mod remix; +pub mod resample; mod smooth; pub mod sync; diff --git a/crates/audio-dsp/src/resample.rs b/crates/audio-dsp/src/resample.rs new file mode 100644 index 00000000..01dfee55 --- /dev/null +++ b/crates/audio-dsp/src/resample.rs @@ -0,0 +1,170 @@ +//! Streaming sample-rate conversion for playback, when the output device runs at another rate +//! than the sequence (a 44.1 kHz device playing a 48 kHz sequence). +//! +//! Every output sample's source position comes from its absolute output frame, as an exact +//! rational (`frame × from / to`), and is linearly interpolated, so the output is identical however +//! the stream is cut into blocks: no phase jump at block boundaries. Source frames are pulled in +//! contiguous, non-overlapping spans, so whatever produces them (the mixer, with its effect state) +//! sees one continuous stream. Output lags the source by one source sample (≈21 µs at 48 kHz), +//! which is what lets each block interpolate without reading ahead. + +/// Converts a planar stream from `from` Hz to `to` Hz, block by block. +#[derive(Debug, Clone)] +pub struct StreamResampler { + from: u32, + to: u32, + /// Source frames `[held_start, held_start + held[c].len())` kept from the last pull. + held_start: i64, + held: Vec>, + /// The output frame the next block is expected to start at (contiguous playback). + next_out: Option, +} + +impl StreamResampler { + /// Rates of zero are treated as 1 Hz rather than dividing by zero. + pub fn new(from: u32, to: u32) -> Self { + Self { from: from.max(1), to: to.max(1), held_start: 0, held: Vec::new(), next_out: None } + } + + pub fn rates(&self) -> (u32, u32) { + (self.from, self.to) + } + + /// The source frame whose sample lands at (or just before) output frame `out`. + fn source_index(&self, out: i64) -> (i64, f32) { + let p = out as i128 * self.from as i128; + let to = self.to as i128; + (p.div_euclid(to) as i64, (p.rem_euclid(to) as f64 / to as f64) as f32) + } + + /// Output frames `[out, out + n)`, planar. `pull(start, frames)` returns source frames + /// `[start, start + frames)`, planar (missing samples read as silence). A block that does not + /// continue the previous one (a seek) starts the stream afresh. + pub fn process(&mut self, out: i64, n: usize, mut pull: impl FnMut(i64, usize) -> Vec>) -> Vec> { + if n == 0 { + return vec![Vec::new(); self.held.len().max(1)]; + } + let (first, _) = self.source_index(out); + let (last, _) = self.source_index(out.saturating_add(n as i64 - 1)); + // interpolation reads source frames first - 1 ..= last + if self.next_out != Some(out) || self.held.is_empty() { + self.held_start = first - 1; + self.held.clear(); + } + let held_end = self.held_start + self.held.first().map_or(0, Vec::len) as i64; + if last + 1 > held_end { + let want = (last + 1 - held_end) as usize; + let got = pull(held_end, want); + if self.held.is_empty() { + self.held = vec![Vec::new(); got.len().max(1)]; + } + for (c, h) in self.held.iter_mut().enumerate() { + let src = got.get(c).map(Vec::as_slice).unwrap_or_default(); + h.extend((0..want).map(|i| src.get(i).copied().unwrap_or(0.0))); + } + } + let mut dst = vec![vec![0.0f32; n]; self.held.len()]; + for (i, frame) in (out..).take(n).enumerate() { + let (s, frac) = self.source_index(frame); + // held starts at first - 1, so the earlier frame is always held + let k = usize::try_from(s - self.held_start).unwrap_or(0); + for (d, h) in dst.iter_mut().zip(&self.held) { + let a = k.checked_sub(1).and_then(|j| h.get(j)).copied().unwrap_or(0.0); + let b = h.get(k).copied().unwrap_or(0.0); + if let Some(x) = d.get_mut(i) { + *x = a + (b - a) * frac; + } + } + } + // keep what the next block can still read: from its first frame - 1 on + let (next_first, _) = self.source_index(out.saturating_add(n as i64)); + let drop = (next_first - 1 - self.held_start).clamp(0, self.held.first().map_or(0, Vec::len) as i64) as usize; + for h in &mut self.held { + h.drain(..drop); + } + self.held_start += drop as i64; + self.next_out = Some(out.saturating_add(n as i64)); + dst + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A sine at `hz`, sampled at `rate`, from absolute frame `start`; logs every pull. + fn sine(hz: f64, rate: u32, pulls: &mut Vec<(i64, usize)>) -> impl FnMut(i64, usize) -> Vec> + '_ { + move |start, frames| { + pulls.push((start, frames)); + let ch: Vec = (0..frames).map(|i| (2.0 * std::f64::consts::PI * hz * (start + i as i64) as f64 / rate as f64).sin() as f32).collect(); + vec![ch.clone(), ch] + } + } + + fn convert(from: u32, to: u32, blocks: &[usize], total: usize) -> (Vec, Vec<(i64, usize)>) { + let mut r = StreamResampler::new(from, to); + let mut pulls = Vec::new(); + let mut out = Vec::new(); + let mut pos = 0i64; + let mut k = 0; + while (pos as usize) < total { + let n = blocks[k % blocks.len()].min(total - pos as usize); + let b = r.process(pos, n, sine(1000.0, from, &mut pulls)); + assert_eq!(b.len(), 2); + assert!(b.iter().all(|c| c.len() == n)); + out.extend_from_slice(&b[0]); + pos += n as i64; + k += 1; + } + (out, pulls) + } + + #[test] + fn output_is_the_same_however_the_stream_is_cut() { + for (from, to) in [(48_000, 44_100), (44_100, 48_000), (48_000, 96_000), (96_000, 48_000)] { + let (whole, _) = convert(from, to, &[20_000], 20_000); + for blocks in [&[512][..], &[333], &[1, 7, 512, 64]] { + let (cut, _) = convert(from, to, blocks, 20_000); + assert_eq!(cut, whole, "{from}→{to} in blocks {blocks:?}"); + } + } + } + + #[test] + fn a_tone_converts_to_the_same_tone() { + // 1 kHz at 48 kHz played on a 44.1 kHz device: within linear interpolation's error of the + // ideal tone, one source sample late + let (from, to) = (48_000u32, 44_100u32); + let (out, _) = convert(from, to, &[512], 20_000); + for (o, x) in out.iter().enumerate().skip(2) { + let t = o as f64 / to as f64 - 1.0 / from as f64; + let want = (2.0 * std::f64::consts::PI * 1000.0 * t).sin() as f32; + assert!((x - want).abs() < 3e-3, "frame {o}: {x} vs {want}"); + } + } + + #[test] + fn the_source_is_pulled_as_one_continuous_stream() { + for (from, to) in [(48_000, 44_100), (44_100, 48_000)] { + let (_, pulls) = convert(from, to, &[512, 333], 20_000); + for w in pulls.windows(2) { + assert_eq!(w[0].0 + w[0].1 as i64, w[1].0, "{from}→{to}: {:?} then {:?}", w[0], w[1]); + } + } + } + + #[test] + fn a_seek_starts_afresh_and_hostile_input_stays_finite() { + let mut r = StreamResampler::new(48_000, 44_100); + let mut pulls = Vec::new(); + let _ = r.process(0, 512, sine(1000.0, 48_000, &mut pulls)); + let _ = r.process(1_000_000, 512, sine(1000.0, 48_000, &mut pulls)); + assert_eq!(pulls.last().map(|p| p.0), Some(1_000_000 * 48_000 / 44_100 - 1)); + // a mixer returning too few channels and samples, and zero rates + let mut r = StreamResampler::new(0, 0); + let b = r.process(5, 64, |_, _| vec![vec![1.0; 3]]); + assert_eq!(b.len(), 1); + assert!(b[0].iter().all(|x| x.is_finite())); + assert!(r.process(0, 0, |_, _| Vec::new()).iter().all(Vec::is_empty)); + } +} diff --git a/crates/codecs/src/audio.rs b/crates/codecs/src/audio.rs index 7954473d..4942f6a6 100644 --- a/crates/codecs/src/audio.rs +++ b/crates/codecs/src/audio.rs @@ -15,8 +15,8 @@ use symphonia::core::probe::Hint; use crate::{CodecError, Result}; enum Inner { - /// Our own AAC-LC decoder. - Aac { dec: Box, asc: Vec }, + /// Our own AAC-LC decoder. `up` interpolates HE-AAC's core to the output rate (see [`Upsample2x`]). + Aac { dec: Box, asc: Vec, up: Option> }, /// AAC whose configuration arrives with the first frame (ADTS / LATM). AacPending, /// Our own Opus decoder (always 48 kHz output; pre-skip is left to container timestamps). @@ -82,11 +82,13 @@ impl PacketDecoder { let dec = symphonia::default::get_codecs().make(&p, &DecoderOptions::default()).map_err(|e| CodecError::Unsupported(e.to_string()))?; Ok(Self { inner: Inner::Symphonia(dec), channels: 0 }) } - /// AAC: our decoder for AAC-LC; symphonia for other object types (HE-AAC…). + /// AAC: our decoder for AAC-LC and the AAC-LC core of HE-AAC (v1/v2); symphonia for other + /// object types. `sample_rate` is the stream's output rate: when it is twice the core rate + /// (HE-AAC, see [`aac_output_rate`]) the decoded core is upsampled to it. pub fn aac(asc: &[u8], sample_rate: u32) -> Result { - let lc = asc.first().map(|b| b >> 3) == Some(2); - if lc && let Ok(dec) = filmcraft_aac::Decoder::new(asc) { - return Ok(Self { inner: Inner::Aac { dec: Box::new(dec), asc: asc.to_vec() }, channels: 0 }); + if let Ok(dec) = filmcraft_aac::Decoder::new(asc) { + let up = (dec.sample_rate().checked_mul(2) == Some(sample_rate)).then(|| Box::new(Upsample2x::new(dec.channels()))); + return Ok(Self { inner: Inner::Aac { dec: Box::new(dec), asc: asc.to_vec(), up }, channels: 0 }); } Self::new(CODEC_TYPE_AAC, sample_rate, Some(asc.to_vec())) } @@ -119,7 +121,7 @@ impl PacketDecoder { return Ok(()); } let dec = filmcraft_aac::Decoder::new(asc).map_err(|e| CodecError::Unsupported(format!("AAC: {e}")))?; - self.inner = Inner::Aac { dec: Box::new(dec), asc: asc.to_vec() }; + self.inner = Inner::Aac { dec: Box::new(dec), asc: asc.to_vec(), up: None }; Ok(()) } /// AC-3 (ATSC A/52). @@ -133,7 +135,7 @@ impl PacketDecoder { pub fn for_isobmff(c: &filmcraft_isobmff::CodecConfig, rate: u32) -> Result { use filmcraft_isobmff::CodecConfig as C; match c { - C::Aac(a) => Self::aac(&a.asc, if a.sample_rate > 0 { a.sample_rate } else { rate }), + C::Aac(a) => Self::aac(&a.asc, if rate > 0 { rate } else { a.sample_rate }), C::Mp3 => Self::new(CODEC_TYPE_MP3, rate, None), C::Ac3 { .. } => Self::ac3(), C::Alac { cookie } => Self::new(CODEC_TYPE_ALAC, rate, Some(cookie.clone())), @@ -145,8 +147,11 @@ impl PacketDecoder { /// Decode one packet into planar channels. pub fn decode(&mut self, data: &[u8], ts: u64) -> Result>> { let dec = match &mut self.inner { - Inner::Aac { dec, .. } => { - let out = dec.decode(data).map_err(|e| CodecError::Decode(e.to_string()))?; + Inner::Aac { dec, up, .. } => { + let mut out = dec.decode(data).map_err(|e| CodecError::Decode(e.to_string()))?; + if let Some(up) = up { + out = up.process(out); + } self.channels = out.len(); return Ok(out); } @@ -202,10 +207,13 @@ impl PacketDecoder { } pub fn reset(&mut self) { match &mut self.inner { - Inner::Aac { dec, asc } => { + Inner::Aac { dec, asc, up } => { if let Ok(d) = filmcraft_aac::Decoder::new(asc) { **dec = d; } + if let Some(up) = up { + up.reset(); + } } Inner::Opus { dec, .. } => dec.reset(), Inner::Symphonia(d) => d.reset(), @@ -215,6 +223,94 @@ impl PacketDecoder { } } +/// The output rate of an AAC stream: twice the core rate for HE-AAC (ISO/IEC 14496-3 §1.6.5). +/// Explicit signalling names the rate in the `AudioSpecificConfig`; with implicit signalling (an +/// AAC-LC config at 24 kHz or less) SBR is only visible as fill-element data in the access units, +/// so the first ones are decoded to look for it. `None` if `asc` is not one our decoder reads. +pub fn aac_output_rate<'a>(asc: &[u8], first_units: impl IntoIterator) -> Option { + let mut dec = filmcraft_aac::Decoder::new(asc).ok()?; + let core = dec.sample_rate(); + if let Some(ext) = dec.config().extension_sample_rate { + return Some(ext); + } + if core <= 24_000 { + for au in first_units { + let _ = dec.decode(au); + if dec.sbr() { + return Some(core * 2); + } + } + } + Some(core) +} + +/// Taps on each side of the interpolation point in [`Upsample2x`]. +const UPSAMPLE_HALF: usize = 12; + +/// 2× upsampler for HE-AAC. Our decoder reconstructs the AAC-LC core at half the output rate and +/// not yet the SBR high band, so the core is interpolated to the output rate: timing and pitch are +/// right and the band above the core's Nyquist stays empty. Even outputs are the input samples, +/// odd ones a Blackman-windowed sinc half-band interpolation over 2 × [`UPSAMPLE_HALF`] inputs. +/// The look-ahead delays the output by `UPSAMPLE_HALF` input samples (0.5 ms at 24 kHz); the +/// state carries across packets, so a seek primed with the preceding packet gives the same +/// samples as continuous decoding. +#[derive(Clone)] +pub struct Upsample2x { + taps: [f32; 2 * UPSAMPLE_HALF], + /// The last `2 × UPSAMPLE_HALF` input samples per channel. + hist: Vec>, +} + +impl Upsample2x { + pub fn new(channels: usize) -> Self { + let m = UPSAMPLE_HALF as f64; + let mut taps = [0f32; 2 * UPSAMPLE_HALF]; + let mut sum = 0.0; + for (k, t) in taps.iter_mut().enumerate() { + // distance from the interpolation point (halfway between two inputs) + let x = k as f64 - m + 0.5; + let sinc = (std::f64::consts::PI * x).sin() / (std::f64::consts::PI * x); + let w = 0.42 + 0.5 * (std::f64::consts::PI * x / m).cos() + 0.08 * (2.0 * std::f64::consts::PI * x / m).cos(); + *t = (sinc * w) as f32; + sum += sinc * w; + } + for t in &mut taps { + *t = (*t as f64 / sum) as f32; + } + Self { taps, hist: vec![vec![0.0; 2 * UPSAMPLE_HALF]; channels] } + } + + pub fn reset(&mut self) { + for h in &mut self.hist { + h.fill(0.0); + } + } + + pub fn process(&mut self, input: Vec>) -> Vec> { + if self.hist.len() < input.len() { + self.hist.resize(input.len(), vec![0.0; 2 * UPSAMPLE_HALF]); + } + let m = UPSAMPLE_HALF; + input + .into_iter() + .zip(&mut self.hist) + .map(|(x, hist)| { + let mut buf = std::mem::take(hist); + buf.extend_from_slice(&x); + let mut out = Vec::with_capacity(x.len() * 2); + // output pair i is centred on buf[i + m]: the sample itself, then the point halfway + // to the next one from buf[i + 1 ..= i + 2m] + for w in buf.windows(2 * m + 1).take(x.len()) { + out.push(w[m]); + out.push(w[1..].iter().zip(&self.taps).map(|(a, b)| a * b).sum()); + } + *hist = buf.split_off(buf.len() - 2 * m); + out + }) + .collect() + } +} + /// Whether AC-3 audio decodes (our ATSC A/52 decoder). pub const AC3_DECODER: bool = true; @@ -603,3 +699,43 @@ pub fn opener(name: &str, bytes: Arc<[u8]>) -> Option = (0..4096).map(|i| (2.0 * std::f32::consts::PI * f * i as f32).sin()).collect(); + let mut up = Upsample2x::new(1); + let mut y = Vec::new(); + for chunk in x.chunks(1024) { + y.extend(up.process(vec![chunk.to_vec()]).remove(0)); + } + assert_eq!(y.len(), 8192); + // output 2k + 1 is input k − UPSAMPLE_HALF + ½ (the look-ahead delay), within 0.1 % + let d = UPSAMPLE_HALF as f32; + for k in 100..4000 { + let want = (2.0 * std::f32::consts::PI * f * (k as f32 - d + 0.5)).sin(); + assert!((y[2 * k + 1] - want).abs() < 1e-3, "{k}: {} vs {want}", y[2 * k + 1]); + assert_eq!(y[2 * k], x[k - UPSAMPLE_HALF]); + } + } +} diff --git a/crates/codecs/src/mp4.rs b/crates/codecs/src/mp4.rs index 1ac8daca..d2149761 100644 --- a/crates/codecs/src/mp4.rs +++ b/crates/codecs/src/mp4.rs @@ -18,6 +18,9 @@ use crate::gop::{GopCache, VideoSamples}; use crate::video::VideoDecoder; use crate::{CodecError, make_video_decoder}; +/// Most bytes read from one AAC access unit when probing for HE-AAC at open (8 channels × 6144 bits). +const MAX_AAC_PROBE_UNIT: usize = 8 * 6144 / 8; + struct AudioState { decoder: Option, /// Decoded packets by sample index. @@ -198,7 +201,21 @@ impl Mp4Source { let ap = entry.audio.clone().unwrap_or_default(); let (rate, ch, bits) = match &entry.codec { CodecConfig::Aac(a) => ( - if a.sample_rate > 0 { a.sample_rate } else { ap.sample_rate as u32 }, + { + // HE-AAC plays at twice the core rate the AudioSpecificConfig starts with + // an AAC access unit is at most 6144 bits per channel: a hostile `stsz` can't make us read more + let units = t + .samples + .iter() + .take(8) + .filter_map(|x| filmcraft_media::reader::read_range(&*bytes.0, x.offset, (x.size as usize).min(MAX_AAC_PROBE_UNIT)).ok()); + let units: Vec> = units.collect(); + match crate::audio::aac_output_rate(&a.asc, units.iter().map(Vec::as_slice)) { + Some(r) => r, + None if a.sample_rate > 0 => a.sample_rate, + None => ap.sample_rate as u32, + } + }, if a.channel_config > 0 { a.channel_config as u32 } else { ap.channels }, None, ), @@ -783,3 +800,40 @@ mod tests { } } } + +#[cfg(test)] +mod he_aac_tests { + use filmcraft_isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; + use filmcraft_media::MediaSource; + + /// #108: an HE-AAC MP4 (explicit SBR signalling, core 22.05 kHz, output 44.1 kHz, timescale + /// 44.1 kHz) built here from our AAC-LC encoder: the stream has no SBR data, which only the + /// high band would need. It was reported at the 22.05 kHz core rate, so sequences made from + /// such a clip were created at 22.05 kHz. + #[test] + fn he_aac_plays_at_the_output_rate() { + let core = 22_050u32; + let n = core as usize * 2; + let tone: Vec = (0..n).map(|i| 0.25 * (2.0 * std::f32::consts::PI * 1000.0 * i as f32 / core as f32).sin()).collect(); + let mut enc = filmcraft_aac::Encoder::new(filmcraft_aac::EncoderConfig::cbr(core, 2, 64_000)).unwrap(); + let mut aus = enc.encode(&[&tone, &tone]); + aus.extend(enc.flush()); + // AOT 5, core 22.05 kHz, stereo, extension 44.1 kHz, core AOT 2 + let asc = vec![0x2B, 0x92, 0x08, 0x00]; + let mut mux = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mp4)).unwrap(); + let t = mux.add_track(TrackConfig::new(SampleEntry::aac(asc, 2, 44_100), 44_100)).unwrap(); + for au in &aus { + mux.write_sample(t, WriteSample { data: au, duration: 2048, composition_offset: 0, is_sync: true }).unwrap(); + } + let b = mux.finish().unwrap().into_inner(); + let src = super::Mp4Source::open("he.mp4", b.into()).unwrap(); + assert_eq!(src.info().audio.as_ref().unwrap().sample_rate, 44_100); + // one second from 0.5 s at 44.1 kHz: the 1 kHz tone at its level (not silence, not shifted) + let buf = src.audio(22_050, 44_100, 44_100).unwrap(); + let x = &buf.channels[0]; + let peak = x.iter().fold(0f32, |m, v| m.max(v.abs())); + assert!((0.2..0.3).contains(&peak), "peak {peak}"); + let crossings = x.windows(2).filter(|w| w[0] < 0.0 && w[1] >= 0.0).count(); + assert!((995..=1005).contains(&crossings), "{crossings} Hz"); + } +} diff --git a/crates/engine/src/commands.rs b/crates/engine/src/commands.rs index adbf45b5..3525f5fe 100644 --- a/crates/engine/src/commands.rs +++ b/crates/engine/src/commands.rs @@ -1374,22 +1374,15 @@ fn build() -> Vec { s.state.linked_selection = bool_p(p, "on").unwrap_or(!s.state.linked_selection); Ok(json!({"linkedSelection": s.state.linked_selection})) }), - cmd!("sequence.addTracks", "Add Tracks…", ["Sequence"], None, r#"{"video":n=1,"audio":n=0}"#, has_seq, |s, p| { - let nv = u64_p(p, "video").unwrap_or(1) as usize; - let na = u64_p(p, "audio").unwrap_or(0) as usize; - s.edit_sequence("Add Tracks", |q, ctx, _| { - for _ in 0..nv { - let n = q.video_tracks.len() + 1; - q.video_tracks.push(filmcraft_project::Track::new(TrackId(ctx.alloc()), TrackKind::Video, format!("Video {n}"))); - } - for _ in 0..na { - let n = q.audio_tracks.len() + 1; - q.audio_tracks.push(filmcraft_project::Track::new(TrackId(ctx.alloc()), TrackKind::Audio, format!("Audio {n}"))); - } - Ok(()) - })?; - Ok(Value::Null) - }), + cmd!( + "sequence.addTracks", + "Add Tracks…", + ["Sequence"], + None, + r#"{"video":n=1,"audio":n=0,"submix":n=0,"videoAfter":"first"|"V2"|n?,"audioAfter":"first"|"A2"|n?,"submixAfter":"first"|"S1"|n?,"audioType":"standard|5.1|adaptive|mono"?,"submixType":"stereo|5.1|adaptive|mono"?}"#, + has_seq, + crate::sequence_tools::add_tracks + ), cmd!("sequence.deleteTrack", "Delete Track", [], None, r#"{"track":"V3"|id}"#, has_seq, |s, p| { let tr = track_p(s, p, "track", "sequence.deleteTrack")?.ok_or_else(|| bad("sequence.deleteTrack", "need `track`"))?; s.edit_sequence("Delete Track", |q, _, _| { diff --git a/crates/engine/src/sequence_tools.rs b/crates/engine/src/sequence_tools.rs index 86059131..d753b95e 100644 --- a/crates/engine/src/sequence_tools.rs +++ b/crates/engine/src/sequence_tools.rs @@ -27,7 +27,7 @@ //! takes its own range and lands offset by the difference of the In points (J- and L-cuts). use filmcraft_edit as edit; -use filmcraft_project::{ClipId, ItemId, Label, Marker, MarkerId, MarkerKind, SplitMarks, TrackId, TrackItem, TrackKind}; +use filmcraft_project::{AudioChannels, ClipId, ItemId, Label, Marker, MarkerId, MarkerKind, SplitMarks, TrackId, TrackItem, TrackKind}; use filmcraft_time::{Tick, TimeRange}; use serde_json::{Value, json}; @@ -448,6 +448,114 @@ fn make_subsequence(s: &mut Session, p: &Value) -> Result { /// Sequence ▸ Delete Tracks…: delete all empty video / audio tracks (`"empty"`) or one track per /// kind. A sequence always keeps at least one video and one audio track. +/// The most tracks one `sequence.addTracks` adds of a kind, and the most a sequence may then +/// have of it. (Amounts come from dialogs, scripts and the control channel: never trusted.) +const MAX_ADDED_TRACKS: u64 = 99; +const MAX_TRACKS: usize = 999; + +/// Where `sequence.addTracks` puts new tracks among the `len` tracks of their kind: the number of +/// tracks before them. `key` holds `"first"` (Before First Track), a track of that kind by name +/// (`"V2"`: after Video 2; `letter` is V, A or S) or a number (after that many tracks); without it +/// they go after the last track. +fn placement(p: &Value, key: &str, letter: char, len: usize) -> Result { + let cmd = "sequence.addTracks"; + let after = match p.get(key) { + None | Some(Value::Null) => return Ok(len), + Some(v) => match (v.as_u64(), v.as_str()) { + (Some(n), _) => usize::try_from(n).ok(), + (None, Some(x)) if matches!(x.to_ascii_lowercase().as_str(), "first" | "before" | "beforefirst") => Some(0), + (None, Some(x)) if matches!(x.to_ascii_lowercase().as_str(), "last" | "end") => Some(len), + (None, Some(x)) => { + let mut rest = x.chars(); + rest.next().filter(|c| c.eq_ignore_ascii_case(&letter)).and_then(|_| rest.as_str().parse::().ok()) + } + (None, None) => None, + }, + }; + after.filter(|n| *n <= len).ok_or_else(|| bad(cmd, format!("`{key}` must be \"first\", a track ({letter}1–{letter}{len}) or how many tracks come before"))) +} + +/// Give the tracks that still carry a default name (` `) the number of their +/// place, as Premiere numbers its tracks; a track the user named keeps its name. +fn renumber(tracks: &mut [filmcraft_project::Track], prefix: &str) { + for (i, t) in tracks.iter_mut().enumerate() { + let default = t.name.strip_prefix(prefix).and_then(|r| r.strip_prefix(' ')).is_some_and(|n| !n.is_empty() && n.bytes().all(|b| b.is_ascii_digit())); + if default { + t.name = format!("{prefix} {}", i + 1); + } + } +} + +/// Sequence ▸ Add Tracks…: `video` / `audio` / `submix` tracks, each kind at its own place +/// (`videoAfter`, `audioAfter`, `submixAfter`; after the last track when not given), audio tracks +/// of `audioType` (Standard unless given) and submix tracks of `submixType` (Stereo). One undo +/// step. Tracks are numbered by their place, so default names after the new tracks move up. +pub(crate) fn add_tracks(s: &mut Session, p: &Value) -> Result { + use crate::commands::u64_p; + let cmd = "sequence.addTracks"; + let q = s.active_sequence().ok_or(EngineError::NoSequence)?; + let amount = |key: &str, default: u64, have: usize| -> Result { + let n = match p.get(key) { + None | Some(Value::Null) => default, + Some(_) => u64_p(p, key).ok_or_else(|| bad(cmd, format!("`{key}` must be a number of tracks")))?, + }; + if n > MAX_ADDED_TRACKS { + return Err(bad(cmd, format!("`{key}`: at most {MAX_ADDED_TRACKS} tracks at a time"))); + } + if have.saturating_add(n as usize) > MAX_TRACKS { + return Err(bad(cmd, format!("`{key}`: a sequence has at most {MAX_TRACKS} tracks of a kind"))); + } + Ok(n as usize) + }; + let (nv, na, ns) = (amount("video", 1, q.video_tracks.len())?, amount("audio", 0, q.audio_tracks.len())?, amount("submix", 0, q.submix_tracks.len())?); + let (at_v, at_a, at_s) = ( + placement(p, "videoAfter", 'V', q.video_tracks.len())?, + placement(p, "audioAfter", 'A', q.audio_tracks.len())?, + placement(p, "submixAfter", 'S', q.submix_tracks.len())?, + ); + let channels = |key: &str, default: AudioChannels| match p.get(key).and_then(Value::as_str) { + None => Ok(default), + Some(x) => crate::mixer::channels_from(x).ok_or_else(|| bad(cmd, format!("`{key}`: unknown track type {x:?} (standard, stereo, 5.1, adaptive, mono)"))), + }; + let (audio_type, submix_type) = (channels("audioType", AudioChannels::Stereo)?, channels("submixType", AudioChannels::Stereo)?); + if nv + na + ns == 0 { + return Err(bad(cmd, "no tracks to add")); + } + let added = s.edit_sequence("Add Tracks", |q, ctx, _| { + let mut new = |kind: TrackKind, prefix: &str, channels: Option| { + let mut t = filmcraft_project::Track::new(TrackId(ctx.alloc()), kind, format!("{prefix} 0")); + if let Some(c) = channels { + t.channels = c; + } + t + }; + let mut ids: [Vec; 3] = Default::default(); + for (k, (tracks, n, at, kind, prefix, ch)) in [ + (&mut q.video_tracks, nv, at_v, TrackKind::Video, "Video", None), + (&mut q.audio_tracks, na, at_a, TrackKind::Audio, "Audio", Some(audio_type)), + (&mut q.submix_tracks, ns, at_s, TrackKind::Audio, "Submix", Some(submix_type)), + ] + .into_iter() + .enumerate() + { + let at = at.min(tracks.len()); + for i in 0..n { + let t = new(kind, prefix, ch); + if let Some(slot) = ids.get_mut(k) { + slot.push(t.id.0); + } + tracks.insert(at + i, t); + } + if n > 0 { + renumber(tracks, prefix); + } + } + Ok(ids) + })?; + let [video, audio, submix] = added; + Ok(json!({"video": video, "audio": audio, "submix": submix})) +} + fn delete_tracks(s: &mut Session, p: &Value) -> Result { let seq_id = s.state.active_sequence.ok_or(EngineError::NoSequence)?; let q = s.active_sequence().ok_or(EngineError::NoSequence)?; diff --git a/crates/engine/src/sequence_tools_tests.rs b/crates/engine/src/sequence_tools_tests.rs index c69016a4..d9df881f 100644 --- a/crates/engine/src/sequence_tools_tests.rs +++ b/crates/engine/src/sequence_tools_tests.rs @@ -1,7 +1,7 @@ //! Sequence / Markers menu commands (`sequence_tools`). use super::*; -use filmcraft_project::{Label, MarkerKind}; +use filmcraft_project::{AudioChannels, Label, MarkerKind, TrackKind}; use serde_json::json; fn demo() -> Session { @@ -189,6 +189,108 @@ fn make_subsequence_from_in_out_and_from_selection() { assert!(s.project.items.values().all(|i| i.name != "Shot 3")); } +/// Sequence ▸ Add Tracks…: where the new tracks go, what they are, and what the amounts may be. +/// Premiere Pro 26.5.2: Placement is "Before First Track" or after one of the tracks (after the +/// last by default); a track added after Video 1 is the new Video 2, and what was on Video 2 is +/// then on Video 3; audio tracks are Standard, 5.1, Adaptive or Mono, submix tracks Stereo, 5.1, +/// Adaptive or Mono. +#[test] +fn add_tracks_places_and_types_them() { + let mut s = demo(); + let names = |s: &Session, kind: TrackKind| s.active_sequence().unwrap().tracks(kind).iter().map(|t| t.name.clone()).collect::>(); + let before = s.active_sequence().unwrap().clone(); + assert_eq!(names(&s, TrackKind::Video), ["Video 1", "Video 2", "Video 3"]); + let on_v2 = before.video_tracks[1].items[0].id; + + // without a place: after the last track (and one video track when nothing is said, as before) + let r = s.execute("sequence.addTracks", json!({})).unwrap(); + assert_eq!((r["video"].as_array().unwrap().len(), r["audio"].as_array().unwrap().len(), r["submix"].as_array().unwrap().len()), (1, 0, 0)); + assert_eq!(names(&s, TrackKind::Video), ["Video 1", "Video 2", "Video 3", "Video 4"]); + assert_eq!(s.active_sequence().unwrap().video_tracks[3].id.0, r["video"][0].as_u64().unwrap()); + s.execute("edit.undo", json!({})).unwrap(); + + // after Video 1: two new empty tracks there, the tracks above move up and are numbered again + s.execute("sequence.addTracks", json!({"video": 2, "videoAfter": "V1"})).unwrap(); + let q = s.active_sequence().unwrap(); + assert_eq!(names(&s, TrackKind::Video), ["Video 1", "Video 2", "Video 3", "Video 4", "Video 5"]); + assert!(q.video_tracks[1].items.is_empty() && q.video_tracks[2].items.is_empty()); + assert_eq!(q.video_tracks[3].id, before.video_tracks[1].id); + assert_eq!(q.video_tracks[3].items[0].id, on_v2, "what was on Video 2 is on Video 4"); + assert_eq!(q.audio_tracks.len(), before.audio_tracks.len(), "no audio track was asked for"); + // one undo step + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(*s.active_sequence().unwrap(), before); + + // before the first track; a name the user gave is kept, default names follow their place + s.edit_sequence("Rename Track", |q, _, _| { + q.audio_tracks[1].name = "Music".into(); + Ok(()) + }) + .unwrap(); + s.execute("sequence.addTracks", json!({"video": 0, "audio": 1, "audioAfter": "first", "audioType": "mono"})).unwrap(); + assert_eq!(names(&s, TrackKind::Audio), ["Audio 1", "Audio 2", "Music", "Audio 4"]); + let q = s.active_sequence().unwrap(); + assert_eq!(q.audio_tracks[0].channels, AudioChannels::Mono); + assert_eq!(q.audio_tracks[1].id, before.audio_tracks[0].id); + // a number counts the tracks before the new ones; "Standard" is a stereo track + for (ty, want) in [("standard", AudioChannels::Stereo), ("5.1", AudioChannels::Surround51), ("adaptive", AudioChannels::Adaptive)] { + let r = s.execute("sequence.addTracks", json!({"video": 0, "audio": 1, "audioAfter": 2, "audioType": ty})).unwrap(); + let q = s.active_sequence().unwrap(); + assert_eq!((q.audio_tracks[2].id.0, q.audio_tracks[2].channels), (r["audio"][0].as_u64().unwrap(), want), "{ty}"); + } + + // submix tracks: stereo unless said, placed among the submix tracks + assert!(before.submix_tracks.is_empty()); + s.execute("sequence.addTracks", json!({"video": 0, "submix": 2})).unwrap(); + s.execute("sequence.addTracks", json!({"video": 0, "submix": 1, "submixAfter": "S1", "submixType": "5.1"})).unwrap(); + let q = s.active_sequence().unwrap(); + assert_eq!( + q.submix_tracks.iter().map(|t| (t.name.as_str(), t.channels)).collect::>(), + [("Submix 1", AudioChannels::Stereo), ("Submix 2", AudioChannels::Surround51), ("Submix 3", AudioChannels::Stereo)] + ); + // all three kinds at once are one undo step + let before = s.active_sequence().unwrap().clone(); + let r = s.execute("sequence.addTracks", json!({"video": 1, "audio": 1, "submix": 1})).unwrap(); + assert_eq!((r["video"].as_array().unwrap().len(), r["audio"].as_array().unwrap().len(), r["submix"].as_array().unwrap().len()), (1, 1, 1)); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(*s.active_sequence().unwrap(), before); +} + +/// Amounts and places come from a dialog, a script or the control channel: a wrong one is an +/// error that changes nothing, and no amount can make the sequence grow without bound. +#[test] +fn add_tracks_refuses_what_it_cannot_do() { + let mut s = demo(); + let before = s.active_sequence().unwrap().clone(); + for bad in [ + json!({"video": 0}), + json!({"video": 100}), + json!({"video": u64::MAX}), + json!({"video": 0, "audio": 1_000_000}), + json!({"video": "many"}), + json!({"video": 1, "videoAfter": "V9"}), + json!({"video": 1, "videoAfter": "A1"}), + json!({"video": 1, "videoAfter": 4}), + json!({"video": 1, "videoAfter": "top"}), + json!({"video": 0, "audio": 1, "audioAfter": "V1"}), + json!({"video": 0, "audio": 1, "audioType": "quad"}), + json!({"video": 0, "submix": 1, "submixAfter": "S1"}), + json!({"video": 0, "submix": 1, "submixType": "standard-ish"}), + ] { + let r = s.execute("sequence.addTracks", bad.clone()); + assert!(r.is_err(), "{bad} was accepted: {r:?}"); + assert_eq!(*s.active_sequence().unwrap(), before, "{bad} changed the sequence"); + } + // 99 at a time, and a sequence stops at 999 tracks of a kind + for _ in 0..10 { + s.execute("sequence.addTracks", json!({"video": 99})).unwrap(); + } + assert_eq!(s.active_sequence().unwrap().video_tracks.len(), 3 + 990); + assert!(s.execute("sequence.addTracks", json!({"video": 7})).is_err()); + s.execute("sequence.addTracks", json!({"video": 6})).unwrap(); + assert_eq!(s.active_sequence().unwrap().video_tracks.len(), 999); +} + #[test] fn delete_tracks_empty_and_specific() { let mut s = demo(); diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index d07bd349..d50e1f30 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -628,7 +628,10 @@ impl AudioEncoder for AacEncoder { } fn aac_factory(_format: Format, sample_rate: u32, channels: u32, s: &ExportSettings) -> Option>> { - let bps = s.audio.bitrate_kbps.clamp(32, 512) * 1000; + // AAC caps a frame at 6144 bits per channel (ISO/IEC 14496-3 §4.5.3.2): 6 bits per sample, so + // 264.6 kbps for stereo at 22.05 kHz; a higher setting is capped instead of refused + let max = (6 * sample_rate as u64 * channels.max(1) as u64).min(u32::MAX as u64) as u32; + let bps = (s.audio.bitrate_kbps.clamp(32, 512) * 1000).min(max); Some( filmcraft_aac::Encoder::new(filmcraft_aac::EncoderConfig::cbr(sample_rate, channels as usize, bps)) .map(|enc| Box::new(AacEncoder { enc, rate: sample_rate, channels }) as Box) diff --git a/crates/export/src/settings_tests.rs b/crates/export/src/settings_tests.rs index 9904d3c3..4a7562fe 100644 --- a/crates/export/src/settings_tests.rs +++ b/crates/export/src/settings_tests.rs @@ -398,3 +398,15 @@ fn builtin_presets_are_valid_and_unique() { let back: ExportPreset = serde_json::from_value(json).unwrap(); assert_eq!(back.settings.adaptive_bitrate, Some(0.2)); } + +/// #108: the default 320 kbps is more than AAC can carry in stereo at 22.05 kHz (6 bits per sample +/// and channel). The export failed with "bitrate out of range"; it is now capped to the limit. +#[test] +fn aac_bitrate_is_capped_for_low_sample_rates() { + let mut s = ExportSettings::default(); + s.audio.bitrate_kbps = 320; + for (rate, ch) in [(22_050, 2), (16_000, 2), (8_000, 6), (48_000, 2)] { + let enc = aac_factory(Format::H264, rate, ch, &s).expect("AAC is always available"); + assert!(enc.is_ok(), "{rate} Hz × {ch}: {:?}", enc.err()); + } +} diff --git a/crates/ui-egui/examples/bench_audio.rs b/crates/ui-egui/examples/bench_audio.rs new file mode 100644 index 00000000..5bf1fbfd --- /dev/null +++ b/crates/ui-egui/examples/bench_audio.rs @@ -0,0 +1,205 @@ +//! Headless audio playback bench: plays a project's sequences through a simulated 48 kHz stereo +//! device with a 512-frame buffer (the app's defaults) and counts dropouts, the way the device +//! would hear them. Nothing is played out loud. +//! +//! ```sh +//! FILMCRAFT_DATA_DIR=/tmp/fc-bench cargo run --release -p filmcraft-ui-egui --example bench_audio -- \ +//! "Project.fcproj" --seconds 20 [--sequence "One hero" …] [--mode direct|ahead|both] [--json out.json] [--wav dir] +//! ``` +//! +//! `direct` mixes inside the device callback (how playback worked before sound was mixed ahead); +//! `ahead` plays through [`play_ahead`] as the app does now. A callback that returns after its +//! buffer was due is a dropout, and the device cycles it overran are lost. For `ahead` the ring's +//! own underruns (silence played because the mixer fell behind) count too. `--wav` writes what a +//! device would have played for each run, to listen to. +//! +//! The simulated device thread runs at normal priority (CoreAudio's own I/O thread runs higher), +//! so absolute counts are pessimistic for both modes; compare the modes on the same machine load. + +use std::sync::Arc; +use std::sync::atomic::Ordering; +use std::time::{Duration, Instant}; + +use filmcraft_engine::Session; +use filmcraft_project::ItemId; +use filmcraft_ui_egui::play_ahead::{self, AudioStats}; +use serde_json::{Value, json}; + +const SR: u32 = 48_000; +const PERIOD: usize = 512; +const CH: usize = 2; + +#[derive(Default)] +struct Run { + callbacks: u64, + /// Callbacks that returned after their buffer was due. + late: u64, + /// Callbacks whose own work took longer than one buffer: a dropout on any device. + overruns: u64, + /// Device cycles lost to late callbacks. + lost_cycles: u64, + max_callback_ms: f64, + /// `ahead` only: callbacks the ring could not fill. + ring_underruns: u64, + min_lead_ms: Option, + /// What a device whose I/O thread always wakes on time plays (interleaved stereo): a callback + /// that overran loses its buffer and the slots it overran to silence. + heard: Vec, +} + +impl Run { + /// What a device whose I/O thread always wakes on time would hear: overrunning callbacks plus + /// buffers the ring could not fill. (`late` also counts this bench's normal-priority device + /// thread waking late under load, which CoreAudio's real-time thread does not.) + fn dropouts(&self) -> u64 { + self.overruns + self.ring_underruns + } +} + +/// Calls `fill` once per device period for `seconds` of wall time, as a device does. +fn device(fill: &mut dyn FnMut(&mut [f32], usize), seconds: f64) -> Run { + let period = PERIOD as f64 / SR as f64; + let total = (seconds / period) as u64; + let mut buf = vec![0.0f32; PERIOD * CH]; + let mut r = Run::default(); + let t0 = Instant::now(); + let mut k = 0u64; + while k < total { + let due = Duration::from_secs_f64(period * k as f64); + if let Some(wait) = due.checked_sub(t0.elapsed()) { + std::thread::sleep(wait); + } + let c0 = Instant::now(); + fill(&mut buf, CH); + r.callbacks += 1; + let took = c0.elapsed().as_secs_f64(); + r.max_callback_ms = r.max_callback_ms.max(took * 1e3); + if took > period { + r.overruns += 1; + let lost = (took / period).ceil() as usize; + r.heard.resize(r.heard.len() + lost * PERIOD * CH, 0.0); + } else { + r.heard.extend_from_slice(&buf); + } + let done = t0.elapsed().as_secs_f64(); + let deadline = period * (k + 1) as f64; + if done > deadline { + r.late += 1; + let next = (done / period).ceil() as u64; + r.lost_cycles += next.saturating_sub(k + 1); + k = next; + } else { + k += 1; + } + } + r +} + +/// 32-bit float stereo WAV at the device rate. +fn write_wav(path: &std::path::Path, samples: &[f32]) -> std::io::Result<()> { + let data = (samples.len() * 4) as u32; + let mut b = Vec::with_capacity(44 + data as usize); + b.extend_from_slice(b"RIFF"); + b.extend_from_slice(&(36 + data).to_le_bytes()); + b.extend_from_slice(b"WAVEfmt "); + b.extend_from_slice(&16u32.to_le_bytes()); + b.extend_from_slice(&3u16.to_le_bytes()); // IEEE float + b.extend_from_slice(&(CH as u16).to_le_bytes()); + b.extend_from_slice(&SR.to_le_bytes()); + b.extend_from_slice(&(SR * CH as u32 * 4).to_le_bytes()); + b.extend_from_slice(&(CH as u16 * 4).to_le_bytes()); + b.extend_from_slice(&32u16.to_le_bytes()); + b.extend_from_slice(b"data"); + b.extend_from_slice(&data.to_le_bytes()); + for x in samples { + b.extend_from_slice(&x.to_le_bytes()); + } + std::fs::write(path, b) +} + +fn play(s: &Session, seq: ItemId, mode: &str, seconds: f64) -> Run { + let mut mix = filmcraft_ui_egui::playback_mix(s, seq, SR); + match mode { + "direct" => { + let mut cursor = 0i64; + let mut fill = move |buf: &mut [f32], ch: usize| { + mix(cursor, buf, ch); + cursor += (buf.len() / ch) as i64; + }; + device(&mut fill, seconds) + } + _ => { + let stats = Arc::new(AudioStats::default()); + let mut fill = play_ahead::spawn(mix, 0, SR, CH, stats.clone()); + let mut r = device(&mut *fill, seconds); + r.ring_underruns = stats.underruns.load(Ordering::Relaxed); + let min = stats.min_lead_frames.load(Ordering::Relaxed); + r.min_lead_ms = (min != u64::MAX).then(|| min as f64 * 1e3 / SR as f64); + r + } + } +} + +fn main() { + filmcraft_platform::register(); + let mut args = std::env::args().skip(1); + let (mut project, mut seconds, mut only, mut mode, mut json_out, mut wav_dir) = (None, 20.0, Vec::new(), "both".to_string(), None, None); + while let Some(a) = args.next() { + match a.as_str() { + "--seconds" => seconds = args.next().and_then(|v| v.parse().ok()).unwrap_or(seconds), + "--sequence" => only.extend(args.next()), + "--mode" => mode = args.next().unwrap_or(mode), + "--json" => json_out = args.next(), + "--wav" => wav_dir = args.next().map(std::path::PathBuf::from), + p => project = Some(p.to_string()), + } + } + let Some(project) = project else { + eprintln!("usage: bench_audio [--seconds N] [--sequence NAME] [--mode direct|ahead|both] [--json PATH] [--wav DIR]"); + std::process::exit(2); + }; + let mut s = Session::default(); + if let Err(e) = s.execute("file.open", json!({ "path": project })) { + eprintln!("{project}: {e}"); + std::process::exit(1); + } + let mut seqs: Vec<(ItemId, String)> = s.project.sequences().map(|i| (i.id, i.name.clone())).filter(|(_, n)| only.is_empty() || only.contains(n)).collect(); + seqs.sort_by(|a, b| a.1.cmp(&b.1)); + let modes: Vec<&str> = if mode == "both" { vec!["direct", "ahead"] } else { vec![mode.as_str()] }; + println!( + "{:<48} {:<7} {:>9} {:>8} {:>9} {:>9} {:>8} {:>11} {:>9}", + "sequence", "mode", "callbacks", "dropouts", "overruns", "underruns", "late", "max cb (ms)", "min lead" + ); + let mut out = Vec::::new(); + for (id, name) in &seqs { + for m in &modes { + let r = play(&s, *id, m, seconds); + if let Some(d) = &wav_dir + && let Err(e) = write_wav(&d.join(format!("{name} - {m}.wav")), &r.heard) + { + eprintln!("{}: {e}", d.display()); + } + let lost_ms = r.lost_cycles as f64 * PERIOD as f64 * 1e3 / SR as f64; + let lead = r.min_lead_ms.map_or("-".to_string(), |v| format!("{v:.0} ms")); + println!( + "{:<48} {:<7} {:>9} {:>8} {:>9} {:>9} {:>8} {:>11.1} {:>9}", + name, + m, + r.callbacks, + r.dropouts(), + r.overruns, + r.ring_underruns, + r.late, + r.max_callback_ms, + lead + ); + out.push(json!({"sequence": name, "mode": m, "seconds": seconds, "callbacks": r.callbacks, "dropouts": r.dropouts(), "overruns": r.overruns, "late": r.late, + "lostMs": lost_ms, "ringUnderruns": r.ring_underruns, "maxCallbackMs": r.max_callback_ms, "minLeadMs": r.min_lead_ms})); + } + } + if let Some(p) = json_out + && let Err(e) = std::fs::write(&p, serde_json::to_string_pretty(&out).unwrap_or_default()) + { + eprintln!("{p}: {e}"); + } +} diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 98dd0463..cb923eba 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -19,6 +19,8 @@ pub mod links; pub mod menus; pub mod panels; pub mod perf; +#[cfg(not(target_arch = "wasm32"))] +pub mod play_ahead; pub mod state; pub mod theme; pub mod widgets; @@ -45,6 +47,10 @@ pub trait AudioOut { fn stop(&mut self); /// Device sample rate. fn sample_rate(&self) -> u32; + /// Device output channels (what `fill` will be called with). + fn channels(&self) -> usize { + 2 + } /// Frames played since `start` (the playback master clock), if the device reports it. fn played_frames(&self) -> Option; /// Hosts and devices that can be chosen in Settings ▸ Audio Hardware. @@ -106,6 +112,8 @@ pub enum Dialog { AudioGain, /// Sequence ▸ Delete Tracks…. DeleteTracks, + /// Sequence ▸ Add Tracks…. + AddTracks, } #[derive(Default)] @@ -118,6 +126,9 @@ pub struct Playback { anchor_tick: Tick, /// Audio frames played at anchor (when the audio clock drives). pub audio_clock: bool, + /// Audio underruns for the current (or last) play (desktop: sound is mixed ahead). + #[cfg(not(target_arch = "wasm32"))] + pub audio_stats: std::sync::Arc, /// Shown / dropped frame accounting for the current (or last) play. pub meter: frames::PlaybackMeter, /// Waiting for the first frames before starting the clock: when the wait began (egui time, @@ -592,57 +603,24 @@ impl FilmcraftApp { return; } let Some(seq_id) = self.session.state.active_sequence else { return }; - let project = self.session.project.clone(); - let provider = self.session.media.provider(project.clone(), self.session.services.clone()); - let previews = self.session.previews.clone(); let start_tick = self.session.playhead(); - // Settings ▸ Audio Hardware ▸ Output Mapping - let map = [self.session.prefs.audio_hardware.map_left, self.session.prefs.audio_hardware.map_right]; - // Preferences ▸ Audio ▸ 5.1 Mixdown Type: how a 5.1 Mix plays on a stereo device - let mixdown = filmcraft_audio_dsp::channels::Mixdown::from_id(&self.session.prefs.audio.mixdown_type).unwrap_or_default(); let Some(a) = self.audio.as_mut() else { return }; let sr = a.sample_rate(); - let mut cursor = start_tick.to_units_floor(sr as i64); - let cues = panels::voiceover::cues(&self.session, sr); - previews.live.publish_project(project.clone()); - let fill = Box::new(move |buf: &mut [f32], ch: usize| { - // the newest project snapshot: mixer moves and other edits are heard while playing - let project = previews.live.project().filter(|p| p.sequence(seq_id).is_some()).unwrap_or_else(|| project.clone()); - let Some(seq) = project.sequence(seq_id) else { return }; - let n = buf.len() / ch.max(1); - // Mix at the sequence rate; convert when the device rate differs (nearest sample). - let seq_sr = seq.settings.sample_rate; - // a 5.1 Mix plays as six channels (L, R, C, LFE, Ls, Rs) on a device with at least six - use filmcraft_audio_dsp::channels::Layout; - let layout = if ch >= 6 && seq.settings.audio_master == filmcraft_project::AudioChannels::Surround51 { Layout::Surround51 } else { Layout::Stereo }; - let mix = if seq_sr == sr { - previews.mix_layout(&project, seq_id, cursor, n, &provider, layout, mixdown) - } else { - let s0 = (cursor as i128 * seq_sr as i128 / sr as i128) as i64; - let m = n * seq_sr as usize / sr as usize + 2; - let b = previews.mix_layout(&project, seq_id, s0, m, &provider, layout, mixdown); - let mut out = filmcraft_frame::AudioBuffer::silence(sr, b.channels.len(), n); - for (o, c) in out.channels.iter_mut().zip(&b.channels) { - for (i, x) in o.iter_mut().enumerate() { - let j = (i * seq_sr as usize / sr as usize).min(m - 1); - *x = c[j]; - } - } - out - }; - if layout == Layout::Surround51 { - buf.fill(0.0); - for (i, frame) in buf.chunks_mut(ch).enumerate() { - for (c, x) in frame.iter_mut().take(6).enumerate() { - *x = mix.channels[c][i]; - } - } - } else { - filmcraft_engine::settings::map_output(&mix.channels[0], &mix.channels[1.min(mix.channels.len() - 1)], buf, ch, map); - } - panels::voiceover::mix_cues(buf, ch, cursor, &cues); - cursor += n as i64; - }); + let cursor = start_tick.to_units_floor(sr as i64); + let mix = playback_mix(&self.session, seq_id, sr); + // the old stream stops before the new mixer resets the underrun counters + a.stop(); + // Desktop: mix ahead on a thread so the device callback never waits on decoding. + #[cfg(not(target_arch = "wasm32"))] + let fill = play_ahead::spawn(mix, cursor, sr, a.channels(), self.playback.audio_stats.clone()); + #[cfg(target_arch = "wasm32")] + let fill = { + let (mut mix, mut cursor) = (mix, cursor); + Box::new(move |buf: &mut [f32], ch: usize| { + mix(cursor, buf, ch); + cursor += (buf.len() / ch.max(1)) as i64; + }) + }; match a.start(fill) { Ok(_) => self.playback.audio_clock = true, Err(e) => { @@ -1352,6 +1330,62 @@ impl eframe::App for FilmcraftApp { } } +/// The program mix that playback plays: `mix(device_frame, interleaved, channels)` renders the +/// active sequence at device rate `sr` from `device_frame`, with the live project snapshot (edits +/// made while playing are heard), Output Mapping, the 5.1 mixdown and voice-over cues. +pub fn playback_mix(session: &Session, seq_id: filmcraft_project::ItemId, sr: u32) -> impl FnMut(i64, &mut [f32], usize) + Send + 'static { + let project = session.project.clone(); + let provider = session.media.provider(project.clone(), session.services.clone()); + let previews = session.previews.clone(); + // Settings ▸ Audio Hardware ▸ Output Mapping + let map = [session.prefs.audio_hardware.map_left, session.prefs.audio_hardware.map_right]; + // Preferences ▸ Audio ▸ 5.1 Mixdown Type: how a 5.1 Mix plays on a stereo device + let mixdown = filmcraft_audio_dsp::channels::Mixdown::from_id(&session.prefs.audio.mixdown_type).unwrap_or_default(); + let cues = panels::voiceover::cues(session, sr); + previews.live.publish_project(project.clone()); + let mut resampler = None; + move |cursor: i64, buf: &mut [f32], ch: usize| { + // the newest project snapshot: mixer moves and other edits are heard while playing + let project = previews.live.project().filter(|p| p.sequence(seq_id).is_some()).unwrap_or_else(|| project.clone()); + let Some(seq) = project.sequence(seq_id) else { + buf.fill(0.0); + return; + }; + let n = buf.len() / ch.max(1); + let seq_sr = seq.settings.sample_rate; + // a 5.1 Mix plays as six channels (L, R, C, LFE, Ls, Rs) on a device with at least six + use filmcraft_audio_dsp::channels::Layout; + let layout = if ch >= 6 && seq.settings.audio_master == filmcraft_project::AudioChannels::Surround51 { Layout::Surround51 } else { Layout::Stereo }; + let mix = if seq_sr == sr { + previews.mix_layout(&project, seq_id, cursor, n, &provider, layout, mixdown) + } else { + // Mix at the sequence rate as one continuous stream, interpolated to the device rate + // (started afresh when the rate or the channel layout changes while playing). + let key = (seq_sr, layout); + if resampler.as_ref().is_none_or(|(k, _)| *k != key) { + resampler = Some((key, filmcraft_audio_dsp::resample::StreamResampler::new(seq_sr, sr))); + } + let Some((_, r)) = resampler.as_mut() else { + buf.fill(0.0); + return; + }; + let channels = r.process(cursor, n, |s0, m| previews.mix_layout(&project, seq_id, s0, m, &provider, layout, mixdown).channels); + filmcraft_frame::AudioBuffer { sample_rate: sr, channels } + }; + if layout == Layout::Surround51 { + buf.fill(0.0); + for (i, frame) in buf.chunks_mut(ch).enumerate() { + for (c, x) in frame.iter_mut().take(6).enumerate() { + *x = mix.channels[c][i]; + } + } + } else { + filmcraft_engine::settings::map_output(&mix.channels[0], &mix.channels[1.min(mix.channels.len() - 1)], buf, ch, map); + } + panels::voiceover::mix_cues(buf, ch, cursor, &cues); + } +} + #[cfg(test)] mod gpu_fallback_tests { use filmcraft_render::plan::{FramePlan, PlanLayer}; diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index dff937cf..620aaea4 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -327,6 +327,21 @@ pub fn invoke(app: &mut FilmcraftApp, ctx: &egui::Context, id: &str, params: Val filmcraft_engine::find_command(id).map_or(Ok(()), |c| (c.enabled)(&app.session))?; return crate::panels::remix::open(app, ctx); } + // Add Tracks… from the menu opens the dialog (new tracks after the last ones, as Premiere + // offers); with params it adds the tracks directly. + "sequence.addTracks" if params.as_object().is_none_or(|m| m.is_empty()) => { + filmcraft_engine::find_command(id).map_or(Ok(()), |c| (c.enabled)(&app.session))?; + let seq = app.session.active_sequence(); + let count = |f: fn(&filmcraft_engine::project::Sequence) -> usize| seq.map_or(0, f); + app.ui.add_tracks = crate::state::AddTracksDraft { + video_after: count(|q| q.video_tracks.len()), + audio_after: count(|q| q.audio_tracks.len()), + submix_after: count(|q| q.submix_tracks.len()), + ..Default::default() + }; + app.dialog = Some(crate::Dialog::AddTracks); + return Ok(json!({"dialog": "addTracks"})); + } "sequence.deleteTracks" if params.as_object().is_none_or(|m| m.is_empty()) => { filmcraft_engine::find_command(id).map_or(Ok(()), |c| (c.enabled)(&app.session))?; app.ui.delete_tracks = Default::default(); diff --git a/crates/ui-egui/src/panels/dialogs.rs b/crates/ui-egui/src/panels/dialogs.rs index 3032b53d..f3390701 100644 --- a/crates/ui-egui/src/panels/dialogs.rs +++ b/crates/ui-egui/src/panels/dialogs.rs @@ -58,6 +58,12 @@ pub fn show(app: &mut FilmcraftApp, ctx: &egui::Context) { } return; } + Dialog::AddTracks => { + if !add_tracks(app, ctx) { + app.dialog = None; + } + return; + } Dialog::NewSequence | Dialog::Preferences | Dialog::Recovery | Dialog::RevertConfirm => {} } if !open { @@ -146,6 +152,134 @@ fn audio_gain(app: &mut FilmcraftApp, ctx: &egui::Context) -> bool { keep } +/// Sequence ▸ Add Tracks…, laid out as Premiere Pro's dialog: "Add video tracks" (Amount, +/// Placement), "Add audio tracks" (Amount, Placement, Track type: Standard, 5.1, Adaptive, Mono) +/// and "Add audio submix tracks" (Amount, Placement, Track type: Stereo, 5.1, Adaptive, Mono). +/// Placement is "Before First Track" or after one of the tracks; the submix placement is off +/// while the sequence has no submix track. Returns whether the dialog stays open. +/// +/// Automation ids, with `` = `video`, `audio` or `submix`: `addTracks..amount`, +/// `addTracks..placement` and, while its list is open, `addTracks..placement.option.` +/// (n tracks before the new ones; 0 = Before First Track); `addTracks..type` and +/// `addTracks..type.option.`; `addTracks.ok`, +/// `addTracks.cancel`. +fn add_tracks(app: &mut FilmcraftApp, ctx: &egui::Context) -> bool { + let Some(seq) = app.session.active_sequence() else { return false }; + let names = |tracks: &[filmcraft_engine::project::Track]| tracks.iter().map(|t| t.name.clone()).collect::>(); + let (vnames, anames, snames) = (names(&seq.video_tracks), names(&seq.audio_tracks), names(&seq.submix_tracks)); + let mut d = app.ui.add_tracks.clone(); + let mut keep = true; + let mut apply = false; + let mut elems: Vec<(String, egui::Rect, String)> = Vec::new(); + const AUDIO_TYPES: [(&str, &str); 4] = [("standard", "Standard"), ("5.1", "5.1"), ("adaptive", "Adaptive"), ("mono", "Mono")]; + const SUBMIX_TYPES: [(&str, &str); 4] = [("stereo", "Stereo"), ("5.1", "5.1"), ("adaptive", "Adaptive"), ("mono", "Mono")]; + egui::Window::new("Add Tracks").collapsible(false).resizable(false).default_width(320.0).anchor(egui::Align2::CENTER_CENTER, [0.0, 0.0]).show(ctx, |ui| { + // everything spans the width the window has: no empty band beside the fields + ui.set_width(ui.available_width()); + // the label column is as wide as its widest label; the lists take the rest of the row, + // up to the group's right edge (as in Premiere's dialog) + let font = egui::TextStyle::Body.resolve(ui.style()); + let label_w = ["Amount", "Placement", "Track type"] + .iter() + .map(|l| ui.painter().layout_no_wrap(l.to_string(), font.clone(), egui::Color32::WHITE).size().x) + .fold(0.0, f32::max); + let row_h = ui.spacing().interact_size.y; + let label = |ui: &mut egui::Ui, text: &str| { + ui.allocate_ui_with_layout(egui::vec2(label_w, row_h), egui::Layout::right_to_left(egui::Align::Center), |ui| ui.label(text)); + }; + let groups: [(&str, &str, &mut u32, &mut usize, &[String], Option<(&mut String, &[(&str, &str); 4])>); 3] = [ + ("video", "Add video tracks", &mut d.video, &mut d.video_after, &vnames, None), + ("audio", "Add audio tracks", &mut d.audio, &mut d.audio_after, &anames, Some((&mut d.audio_type, &AUDIO_TYPES))), + ("submix", "Add audio submix tracks", &mut d.submix, &mut d.submix_after, &snames, Some((&mut d.submix_type, &SUBMIX_TYPES))), + ]; + for (kind, title, amount, after, tracks, track_type) in groups { + ui.add_space(4.0); + ui.label(egui::RichText::new(title).strong()); + ui.group(|ui| { + ui.set_width(ui.available_width()); + ui.horizontal(|ui| { + label(ui, "Amount"); + let r = ui.add(egui::DragValue::new(amount).range(0..=99).speed(0.1)); + elems.push((format!("addTracks.{kind}.amount"), r.rect, amount.to_string())); + }); + ui.horizontal(|ui| { + label(ui, "Placement"); + *after = (*after).min(tracks.len()); + let place = |n: usize| match n.checked_sub(1).and_then(|i| tracks.get(i)) { + Some(name) => format!("After {name}"), + None => "Before First Track".to_string(), + }; + let shown = place(*after); + let list_w = ui.available_width(); + // nowhere to choose from while the sequence has no track of the kind + ui.add_enabled_ui(!tracks.is_empty(), |ui| { + let r = egui::ComboBox::from_id_salt(("add-tracks-placement", kind)).selected_text(&shown).width(list_w).show_ui(ui, |ui| { + for n in 0..=tracks.len() { + let label = place(n); + let o = ui.selectable_value(after, n, &label); + elems.push((format!("addTracks.{kind}.placement.option.{n}"), o.rect, label)); + } + }); + elems.push((format!("addTracks.{kind}.placement"), r.response.rect, shown.clone())); + }); + }); + if let Some((chosen, types)) = track_type { + ui.horizontal(|ui| { + label(ui, "Track type"); + let shown = types.iter().find(|(id, _)| *id == chosen.as_str()).map_or(types[0].1, |(_, label)| label).to_string(); + let list_w = ui.available_width(); + let r = egui::ComboBox::from_id_salt(("add-tracks-type", kind)).selected_text(&shown).width(list_w).show_ui(ui, |ui| { + for (id, label) in types { + let o = ui.selectable_value(chosen, id.to_string(), *label); + elems.push((format!("addTracks.{kind}.type.option.{id}"), o.rect, label.to_string())); + } + }); + elems.push((format!("addTracks.{kind}.type"), r.response.rect, shown)); + }); + } + }); + } + ui.add_space(10.0); + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + let o = ui.add(egui::Button::new(egui::RichText::new("OK").color(egui::Color32::WHITE)).fill(app.tokens.accent)); + elems.push(("addTracks.ok".into(), o.rect, "OK".into())); + if o.clicked() { + apply = true; + } + let c = ui.button("Cancel"); + elems.push(("addTracks.cancel".into(), c.rect, "Cancel".into())); + if c.clicked() { + keep = false; + } + }); + }); + for (id, r, l) in elems { + app.auto.add(&id, r, &l); + } + if ctx.input(|i| i.key_pressed(egui::Key::Escape)) { + keep = false; + } + if ctx.input(|i| i.key_pressed(egui::Key::Enter)) && !ctx.egui_wants_keyboard_input() { + apply = true; + } + if apply { + // nothing asked for: OK closes the dialog and changes nothing + if d.video > 0 || d.audio > 0 || d.submix > 0 { + let params = serde_json::json!({ + "video": d.video, "videoAfter": d.video_after, + "audio": d.audio, "audioAfter": d.audio_after, "audioType": d.audio_type, + "submix": d.submix, "submixAfter": d.submix_after, "submixType": d.submix_type, + }); + if let Err(e) = app.session.execute("sequence.addTracks", params) { + app.ui.status = e.to_string(); + } + } + keep = false; + } + app.ui.add_tracks = d; + keep +} + /// Sequence ▸ Delete Tracks…: "Delete Video Tracks" / "Delete Audio Tracks" checkboxes, each with a /// track choice (All Empty Tracks or one track). Returns whether the dialog stays open. /// diff --git a/crates/ui-egui/src/perf.rs b/crates/ui-egui/src/perf.rs index 6164faa0..79df3c52 100644 --- a/crates/ui-egui/src/perf.rs +++ b/crates/ui-egui/src/perf.rs @@ -20,6 +20,11 @@ pub fn stats(app: &FilmcraftApp) -> Value { "resolution": app.ui.program.res.label(), "draftDecode": app.session.prefs.playback.draft_decode, }); + #[cfg(not(target_arch = "wasm32"))] + { + let sr = app.audio.as_ref().map_or(0, |a| a.sample_rate()); + v["playback"]["audio"] = app.playback.audio_stats.to_json(sr); + } let mut frames = app.frames.stats().to_json(); frames["workers"] = json!(app.frames.workers()); frames["queued"] = json!(app.frames.queue_len()); diff --git a/crates/ui-egui/src/play_ahead.rs b/crates/ui-egui/src/play_ahead.rs new file mode 100644 index 00000000..b3243cbd --- /dev/null +++ b/crates/ui-egui/src/play_ahead.rs @@ -0,0 +1,193 @@ +//! Audio played ahead: playback mixes on its own thread into a ring buffer that stays +//! [`LEAD_S`] ahead of the device, and the device callback only copies samples out of it. +//! +//! Mixing (decoding AAC, effects, opening a clip's file at a cut) used to run inside the device +//! callback, which has one I/O buffer (~10 ms) to return. On a busy Mac any stall there missed the +//! device's deadline, heard as static and dropouts that were different on every play. Now a stall +//! only drains part of the lead; the callback never decodes, allocates or takes a lock. +//! +//! When the ring does run dry (an underrun), the callback plays silence for the missing frames and +//! counts them, and the mixer skips the same span, so sound stays locked to the picture. +//! +//! Desktop only: the web build has no threads and keeps mixing in its callback. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::time::Duration; + +/// How far ahead of the device the mixer keeps the ring (s). Edits made while playing are heard +/// this much later. +pub const LEAD_S: f64 = 0.2; +/// Frames mixed per block on the mixing thread. +pub const BLOCK_FRAMES: usize = 1024; +/// Ceiling on mixed channels (large interfaces and aggregate devices stay well under it). +const MAX_CHANNELS: usize = 64; + +/// Underrun counters for the current (or last) play, readable from any thread (`perf.stats`). +#[derive(Debug, Default)] +pub struct AudioStats { + /// Device callbacks served. + pub callbacks: AtomicU64, + /// Callbacks the ring could not fill completely. + pub underruns: AtomicU64, + /// Frames played as silence because the ring was empty. + pub missing_frames: AtomicU64, + /// Fewest frames left in the ring after a callback, once it first filled to the lead + /// (`u64::MAX` until then). + pub min_lead_frames: AtomicU64, +} + +impl AudioStats { + pub fn reset(&self) { + self.callbacks.store(0, Ordering::Relaxed); + self.underruns.store(0, Ordering::Relaxed); + self.missing_frames.store(0, Ordering::Relaxed); + self.min_lead_frames.store(u64::MAX, Ordering::Relaxed); + } + + pub fn to_json(&self, sample_rate: u32) -> serde_json::Value { + let min = self.min_lead_frames.load(Ordering::Relaxed); + serde_json::json!({ + "callbacks": self.callbacks.load(Ordering::Relaxed), + "underruns": self.underruns.load(Ordering::Relaxed), + "missingFrames": self.missing_frames.load(Ordering::Relaxed), + "minLeadMs": if min == u64::MAX || sample_rate == 0 { serde_json::Value::Null } else { serde_json::json!(min as f64 * 1e3 / sample_rate as f64) }, + }) + } +} + +/// Single-producer, single-consumer ring of interleaved f32 samples (stored as bits, so it needs +/// no `unsafe`). Indices count samples written / read since the start and only grow. +struct Ring { + slots: Vec, + write: AtomicU64, + read: AtomicU64, +} + +impl Ring { + fn new(samples: usize) -> Self { + Self { slots: (0..samples.max(1)).map(|_| AtomicU32::new(0)).collect(), write: AtomicU64::new(0), read: AtomicU64::new(0) } + } + + fn len(&self) -> usize { + self.write.load(Ordering::Acquire).saturating_sub(self.read.load(Ordering::Acquire)) as usize + } + + /// Producer: append as many samples as fit; returns how many were written. + fn push(&self, src: &[f32]) -> usize { + let cap = self.slots.len(); + let w = self.write.load(Ordering::Relaxed); + let free = cap.saturating_sub(w.saturating_sub(self.read.load(Ordering::Acquire)) as usize); + let n = src.len().min(free); + for (i, x) in src.iter().take(n).enumerate() { + if let Some(s) = self.slots.get(((w + i as u64) % cap as u64) as usize) { + s.store(x.to_bits(), Ordering::Relaxed); + } + } + self.write.store(w + n as u64, Ordering::Release); + n + } + + /// Consumer: fill `dst` (`dev_ch` interleaved channels) with whole frames of `ch` channels from + /// the ring: the first `min(ch, dev_ch)` channels copied, any further device channels zeroed. + /// Returns the frames copied; the rest of `dst` is left for the caller. + fn pop(&self, dst: &mut [f32], dev_ch: usize, ch: usize) -> usize { + let cap = self.slots.len(); + let r = self.read.load(Ordering::Relaxed); + let avail = self.write.load(Ordering::Acquire).saturating_sub(r) as usize / ch.max(1); + let n = (dst.len() / dev_ch.max(1)).min(avail); + for (f, frame) in dst.chunks_mut(dev_ch.max(1)).take(n).enumerate() { + for (c, x) in frame.iter_mut().enumerate() { + *x = if c < ch { + self.slots.get(((r + (f * ch + c) as u64) % cap as u64) as usize).map_or(0.0, |s| f32::from_bits(s.load(Ordering::Relaxed))) + } else { + 0.0 + }; + } + } + self.read.store(r + (n * ch) as u64, Ordering::Release); + n + } +} + +/// Stops the mixing thread when the device callback that owns it is dropped (stream stopped). +struct StopOnDrop(Arc); + +impl Drop for StopOnDrop { + fn drop(&mut self) { + self.0.store(true, Ordering::Release); + } +} + +/// Mixes `mix(cursor, buf, channels)` (interleaved frames from device frame `cursor`) on its own +/// thread, from `start`, and returns the device callback that plays it (`fill(buf, channels)`). +/// The first block is mixed before this returns, so playback starts with sound. The thread runs +/// until the returned callback is dropped. +pub fn spawn(mut mix: M, start: i64, sample_rate: u32, channels: usize, stats: Arc) -> Box +where + M: FnMut(i64, &mut [f32], usize) + Send + 'static, +{ + stats.reset(); + let ch = channels.clamp(1, MAX_CHANNELS); + let lead = ((sample_rate as f64 * LEAD_S) as usize).max(BLOCK_FRAMES * 2); + let ring = Arc::new(Ring::new(lead * ch)); + let mut block = vec![0.0; BLOCK_FRAMES * ch]; + mix(start, &mut block, ch); + ring.push(&block); + // Frames the callback played as silence that the mixer has not skipped yet. + let skip = Arc::new(AtomicU64::new(0)); + let stop = Arc::new(AtomicBool::new(false)); + { + let (ring, skip, stop) = (ring.clone(), skip.clone(), stop.clone()); + let body = move || { + let mut cursor = start + BLOCK_FRAMES as i64; + while !stop.load(Ordering::Acquire) { + if ring.len() + block.len() > lead * ch { + std::thread::sleep(Duration::from_millis(2)); + continue; + } + cursor += skip.swap(0, Ordering::AcqRel) as i64; + block.fill(0.0); + mix(cursor, &mut block, ch); + cursor += BLOCK_FRAMES as i64; + // silence played while this block was mixing: its first frames are already late + let late = skip.swap(0, Ordering::AcqRel) as usize; + let drop = late.min(BLOCK_FRAMES); + cursor += (late - drop) as i64; + ring.push(block.get(drop * ch..).unwrap_or_default()); + } + }; + let spawned = std::thread::Builder::new().name("filmcraft-audio-mix".into()).spawn(move || { + if std::panic::catch_unwind(std::panic::AssertUnwindSafe(body)).is_err() { + log::error!("audio mixing stopped after a panic; playback continues silent"); + } + }); + if let Err(e) = spawned { + log::error!("audio mixing thread: {e}"); + } + } + let guard = StopOnDrop(stop); + let mut primed = false; + Box::new(move |buf: &mut [f32], dev_ch: usize| { + let _keep = &guard; + stats.callbacks.fetch_add(1, Ordering::Relaxed); + // a device wider or narrower than the mix (it changed since `channels` was read) still + // plays the channels both have + let dev_ch = dev_ch.max(1); + let have = ring.pop(buf, dev_ch, ch); + if let Some(rest) = buf.get_mut(have * dev_ch..) { + rest.fill(0.0); + } + let n = buf.len() / dev_ch; + if have < n { + stats.underruns.fetch_add(1, Ordering::Relaxed); + stats.missing_frames.fetch_add((n - have) as u64, Ordering::Relaxed); + skip.fetch_add((n - have) as u64, Ordering::AcqRel); + } + let left = ring.len(); + primed |= left + 2 * BLOCK_FRAMES * ch > lead * ch; + if primed { + stats.min_lead_frames.fetch_min((left / ch) as u64, Ordering::Relaxed); + } + }) +} diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index 04c29090..12f810e2 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -422,6 +422,9 @@ pub struct UiState { /// Audio Gain dialog draft (mode, dB values). #[serde(default)] pub audio_gain: AudioGainDraft, + /// Add Tracks dialog draft. + #[serde(default)] + pub add_tracks: AddTracksDraft, /// Delete Tracks dialog draft. #[serde(default)] pub delete_tracks: DeleteTracksDraft, @@ -701,6 +704,30 @@ pub struct AudioGainDraft { pub all_peaks_db: f64, } +/// The Add Tracks dialog (Sequence ▸ Add Tracks…): for video, audio and audio submix tracks, how +/// many to add and where (`*_after`: how many tracks of the kind come before the new ones; 0 is +/// "Before First Track"), and the type of the audio and submix tracks (`standard` / `stereo`, +/// `5.1`, `adaptive`, `mono`). Premiere's defaults: one video track and one audio track after the +/// last ones, no submix track. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct AddTracksDraft { + pub video: u32, + pub video_after: usize, + pub audio: u32, + pub audio_after: usize, + pub audio_type: String, + pub submix: u32, + pub submix_after: usize, + pub submix_type: String, +} + +impl Default for AddTracksDraft { + fn default() -> Self { + Self { video: 1, video_after: 0, audio: 1, audio_after: 0, audio_type: "standard".into(), submix: 0, submix_after: 0, submix_type: "stereo".into() } + } +} + /// The Delete Tracks dialog (Sequence ▸ Delete Tracks…): per kind, whether to delete and which /// track (`"empty"` = All Empty Tracks, or a track name such as `"V2"`). #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -770,6 +797,7 @@ impl Default for UiState { mixer_meter_input_only: false, settings: None, audio_gain: AudioGainDraft::default(), + add_tracks: AddTracksDraft::default(), delete_tracks: DeleteTracksDraft::default(), gfx_edit: None, pen_points: vec![], diff --git a/crates/ui-egui/tests/play_ahead.rs b/crates/ui-egui/tests/play_ahead.rs new file mode 100644 index 00000000..f14a04d9 --- /dev/null +++ b/crates/ui-egui/tests/play_ahead.rs @@ -0,0 +1,128 @@ +//! Audio mixed ahead (`play_ahead`): driven by a wall-clock device. Its own test binary, because one +//! test panics a mixer on purpose and the crash hook's tests read the process-wide last panic. + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +use filmcraft_ui_egui::play_ahead::{AudioStats, BLOCK_FRAMES, LEAD_S, spawn}; + +const SR: u32 = 48_000; +const PERIOD: usize = 512; + +/// A mixer whose every sample is its own device frame number, so the output shows which frame +/// played where. +fn frame_numbers(delay: Duration, calls: Arc) -> impl FnMut(i64, &mut [f32], usize) + Send + 'static { + move |cursor, buf, ch| { + calls.fetch_add(1, Ordering::Relaxed); + std::thread::sleep(delay); + for (i, f) in buf.chunks_mut(ch).enumerate() { + f.fill((cursor + i as i64) as f32); + } + } +} + +/// Plays `callbacks` device periods, one every period on the wall clock as a device does, and +/// returns the interleaved output. +fn play(fill: &mut Box, ch: usize, callbacks: usize) -> Vec { + let period = Duration::from_secs_f64(PERIOD as f64 / SR as f64); + let t0 = Instant::now(); + let mut out = Vec::new(); + for k in 0..callbacks { + let mut buf = vec![f32::NAN; PERIOD * ch]; + fill(&mut buf, ch); + out.extend_from_slice(&buf); + if let Some(wait) = (period * (k as u32 + 1)).checked_sub(t0.elapsed()) { + std::thread::sleep(wait); + } + } + out +} + +#[test] +fn plays_every_frame_in_order_from_the_start() { + let stats = Arc::new(AudioStats::default()); + let mut fill = spawn(frame_numbers(Duration::ZERO, Arc::default()), 1000, SR, 2, stats.clone()); + std::thread::sleep(Duration::from_millis(30)); + let out = play(&mut fill, 2, 60); + for (p, f) in out.chunks(2).enumerate() { + assert_eq!(f, [(1000 + p) as f32; 2], "frame {p}"); + } + assert_eq!(stats.underruns.load(Ordering::Relaxed), 0); + assert_eq!(stats.callbacks.load(Ordering::Relaxed), 60); +} + +#[test] +fn a_mixer_slower_than_the_device_plays_silence_and_stays_in_time() { + // each 1024-frame block takes 40 ms to mix, about twice its 21 ms of sound + let stats = Arc::new(AudioStats::default()); + let mut fill = spawn(frame_numbers(Duration::from_millis(40), Arc::default()), 1, SR, 2, stats.clone()); + let out = play(&mut fill, 2, 120); + assert!(stats.underruns.load(Ordering::Relaxed) > 0); + assert!(stats.missing_frames.load(Ordering::Relaxed) > 0); + let mut heard = 0; + for (p, f) in out.chunks(2).enumerate() { + assert!(f.iter().all(|x| x.is_finite()), "frame {p} left unwritten"); + if f[0] != 0.0 { + // every frame that plays is the one due at that moment: no drift after an underrun + assert_eq!(f[0], (1 + p) as f32, "frame {p}"); + heard += 1; + } + } + assert!(heard > PERIOD, "some sound still plays"); +} + +#[test] +fn a_mixer_that_panics_leaves_silence_not_a_crash() { + let stats = Arc::new(AudioStats::default()); + let mut first = true; + let mix = move |cursor: i64, buf: &mut [f32], ch: usize| { + if !first { + panic!("mixer bug"); + } + first = false; + frame_numbers(Duration::ZERO, Arc::default())(cursor, buf, ch); + }; + let mut fill = spawn(mix, 1, SR, 2, stats.clone()); + let out = play(&mut fill, 2, 20); + assert!(out.iter().all(|x| x.is_finite())); + assert_eq!(out.get(..2), Some(&[1.0, 1.0][..]), "the block mixed before the panic plays"); + assert!(stats.underruns.load(Ordering::Relaxed) > 0); +} + +#[test] +fn a_device_with_other_channels_plays_the_channels_both_have() { + for dev_ch in [1, 6, 10, 16] { + let stats = Arc::new(AudioStats::default()); + let mut fill = spawn(frame_numbers(Duration::ZERO, Arc::default()), 1, SR, 2, stats.clone()); + // let the mixer get ahead first, as a device would before its first callback + std::thread::sleep(Duration::from_millis(30)); + let out = play(&mut fill, dev_ch, 4); + for (p, f) in out.chunks(dev_ch).enumerate() { + let want: Vec = (0..dev_ch).map(|c| if c < 2 { (1 + p) as f32 } else { 0.0 }).collect(); + assert_eq!(f, &want[..], "{dev_ch} channels, frame {p}"); + } + assert_eq!(stats.underruns.load(Ordering::Relaxed), 0, "{dev_ch} channels"); + } +} + +#[test] +fn stopping_the_stream_stops_the_mixer() { + let calls = Arc::new(AtomicU64::new(0)); + let fill = spawn(frame_numbers(Duration::ZERO, calls.clone()), 1, SR, 2, Arc::default()); + std::thread::sleep(Duration::from_millis(50)); + drop(fill); + std::thread::sleep(Duration::from_millis(20)); + let after_stop = calls.load(Ordering::Relaxed); + std::thread::sleep(Duration::from_millis(50)); + assert_eq!(calls.load(Ordering::Relaxed), after_stop); +} + +#[test] +fn the_ring_keeps_no_more_than_the_lead() { + let calls = Arc::new(AtomicU64::new(0)); + let _fill = spawn(frame_numbers(Duration::ZERO, calls.clone()), 1, SR, 2, Arc::default()); + std::thread::sleep(Duration::from_millis(100)); + let lead_blocks = (SR as f64 * LEAD_S / BLOCK_FRAMES as f64).ceil() as u64; + assert!(calls.load(Ordering::Relaxed) <= lead_blocks + 1, "mixed {} blocks", calls.load(Ordering::Relaxed)); +} diff --git a/crates/ui-egui/tests/sequence_ui.rs b/crates/ui-egui/tests/sequence_ui.rs index 3e498217..ad1b300e 100644 --- a/crates/ui-egui/tests/sequence_ui.rs +++ b/crates/ui-egui/tests/sequence_ui.rs @@ -1,4 +1,4 @@ -//! Headless UI tests of the Sequence / Markers menu additions: the Delete Tracks dialog, the +//! Headless UI tests of the Sequence / Markers menu additions: the Add Tracks and Delete Tracks dialogs, the //! through-edit marks on the timeline, the Markers panel colour filter and the Shift+; gap key. //! The real `FilmcraftApp` under `egui_kittest`, driven over the control channel by automation id. @@ -64,12 +64,161 @@ impl Driver { self.frames(2); } + fn app(&mut self) -> &mut FilmcraftApp { + self.harness.state_mut() + } + + /// (video, audio, submix) track names of the active sequence, lowest first. + fn tracks(&mut self) -> [Vec; 3] { + let q = self.app().session.active_sequence().unwrap(); + [&q.video_tracks, &q.audio_tracks, &q.submix_tracks].map(|t| t.iter().map(|t| t.name.clone()).collect()) + } + + fn label(&mut self, id: &str) -> String { + let v = self.ok("ui.elements", json!({"prefix": id})); + let e = v.as_array().unwrap().iter().find(|e| e["id"] == json!(id)).unwrap_or_else(|| panic!("no element {id}")).clone(); + e["label"].as_str().unwrap_or_default().to_string() + } + fn ids(&mut self, prefix: &str) -> Vec { let v = self.ok("ui.elements", json!({"prefix": prefix})); v.as_array().unwrap().iter().filter_map(|e| e["id"].as_str().map(str::to_string)).collect() } } +/// Sequence ▸ Add Tracks… opens Premiere Pro's dialog instead of adding a video track at once: +/// amounts, a placement and (for audio and submix tracks) a track type for each kind, with +/// Premiere's defaults (one video and one audio track after the last ones, no submix track). +#[test] +fn add_tracks_dialog_offers_amount_placement_and_type() { + let mut d = Driver::demo(); + let before = d.tracks(); + assert_eq!( + before, + [vec!["Video 1", "Video 2", "Video 3"], vec!["Audio 1", "Audio 2", "Audio 3"], vec![]] + .map(|v: Vec<&str>| v.into_iter().map(String::from).collect::>()) + ); + let r = d.ok("ui.menu.invoke", json!({"id": "sequence.addTracks"})); + assert_eq!(r["dialog"], "addTracks", "{r}"); + d.frames(3); + assert_eq!(d.tracks(), before, "opening the dialog adds nothing"); + let ids = d.ids("addTracks."); + for id in [ + "addTracks.video.amount", + "addTracks.video.placement", + "addTracks.audio.amount", + "addTracks.audio.placement", + "addTracks.audio.type", + "addTracks.submix.amount", + "addTracks.submix.placement", + "addTracks.submix.type", + "addTracks.ok", + "addTracks.cancel", + ] { + assert!(ids.iter().any(|i| i == id), "{id} missing: {ids:?}"); + } + assert!(!ids.iter().any(|i| i == "addTracks.video.type"), "video tracks have no type"); + // the defaults, as shown + for (id, shown) in [ + ("addTracks.video.amount", "1"), + ("addTracks.video.placement", "After Video 3"), + ("addTracks.audio.amount", "1"), + ("addTracks.audio.placement", "After Audio 3"), + ("addTracks.audio.type", "Standard"), + ("addTracks.submix.amount", "0"), + ("addTracks.submix.placement", "Before First Track"), + ("addTracks.submix.type", "Stereo"), + ] { + assert_eq!(d.label(id), shown, "{id}"); + } + // OK with the defaults: one video and one audio track after the last ones + d.click("addTracks.ok"); + assert!(d.ids("addTracks.").is_empty(), "dialog closed"); + let [v, a, s] = d.tracks(); + assert_eq!((v.len(), a.len(), s.len()), (4, 4, 0)); + assert_eq!((v[3].as_str(), a[3].as_str()), ("Video 4", "Audio 4")); + assert_eq!(d.app().session.active_sequence().unwrap().audio_tracks[3].channels, filmcraft_engine::project::AudioChannels::Stereo); + // one undo step for the whole dialog + d.exec("edit.undo", json!({})); + assert_eq!(d.tracks(), before); +} + +#[test] +fn add_tracks_dialog_places_and_types_the_new_tracks() { + let mut d = Driver::demo(); + let on_v2 = d.app().session.active_sequence().unwrap().video_tracks[1].items[0].id; + d.ok("ui.menu.invoke", json!({"id": "sequence.addTracks"})); + d.frames(3); + // Placement lists "Before First Track" and every track; pick "After Video 1" + d.click("addTracks.video.placement"); + let options = d.ids("addTracks.video.placement.option."); + assert_eq!(options.len(), 4, "{options:?}"); + assert_eq!( + (d.label("addTracks.video.placement.option.0"), d.label("addTracks.video.placement.option.1")), + ("Before First Track".into(), "After Video 1".into()) + ); + d.click("addTracks.video.placement.option.1"); + assert_eq!(d.label("addTracks.video.placement"), "After Video 1"); + // audio: a mono track before the first one + d.click("addTracks.audio.placement"); + d.click("addTracks.audio.placement.option.0"); + d.click("addTracks.audio.type"); + let types = d.ids("addTracks.audio.type.option."); + assert_eq!(types.len(), 4, "Standard, 5.1, Adaptive, Mono: {types:?}"); + d.click("addTracks.audio.type.option.mono"); + // submix: 5.1 (the types are Stereo, 5.1, Adaptive, Mono) + d.click("addTracks.submix.type"); + assert!( + d.ids("addTracks.submix.type.option.").iter().any(|i| i.ends_with(".stereo")) + && !d.ids("addTracks.submix.type.option.").iter().any(|i| i.ends_with(".standard")) + ); + d.click("addTracks.submix.type.option.5.1"); + // amounts (number fields): two video tracks, one audio, one submix + d.app().ui.add_tracks.video = 2; + d.app().ui.add_tracks.submix = 1; + d.frames(2); + assert_eq!((d.label("addTracks.video.amount"), d.label("addTracks.submix.amount")), ("2".into(), "1".into())); + d.click("addTracks.ok"); + assert!(d.ids("addTracks.").is_empty(), "dialog closed"); + let [v, a, s] = d.tracks(); + assert_eq!(v, ["Video 1", "Video 2", "Video 3", "Video 4", "Video 5"]); + assert_eq!(a, ["Audio 1", "Audio 2", "Audio 3", "Audio 4"]); + assert_eq!(s, ["Submix 1"]); + let q = d.app().session.active_sequence().unwrap().clone(); + assert!(q.video_tracks[1].items.is_empty() && q.video_tracks[2].items.is_empty(), "the new tracks are after Video 1"); + assert_eq!(q.video_tracks[3].items[0].id, on_v2, "what was on Video 2 moved up to Video 4"); + use filmcraft_engine::project::AudioChannels; + assert_eq!((q.audio_tracks[0].channels, q.audio_tracks[0].items.len()), (AudioChannels::Mono, 0)); + assert_eq!(q.submix_tracks[0].channels, AudioChannels::Surround51); + + // with a submix track in the sequence its placement can be chosen too + d.ok("ui.menu.invoke", json!({"id": "sequence.addTracks"})); + d.frames(3); + assert_eq!(d.label("addTracks.submix.placement"), "After Submix 1"); + assert_eq!( + (d.label("addTracks.video.amount"), d.label("addTracks.audio.type")), + ("1".into(), "Standard".into()), + "the dialog starts from the defaults again" + ); + // Cancel and Escape add nothing; neither does OK with every amount at 0 + d.click("addTracks.cancel"); + assert!(d.ids("addTracks.").is_empty()); + d.ok("ui.menu.invoke", json!({"id": "sequence.addTracks"})); + d.frames(3); + d.ok("ui.key", json!({"key": "Escape"})); + d.frames(3); + assert!(d.ids("addTracks.").is_empty()); + d.ok("ui.menu.invoke", json!({"id": "sequence.addTracks"})); + d.frames(3); + d.app().ui.add_tracks.video = 0; + d.app().ui.add_tracks.audio = 0; + d.frames(2); + d.click("addTracks.ok"); + assert!(d.ids("addTracks.").is_empty()); + assert_eq!(d.tracks(), [v, a, s]); + assert!(d.app().ui.status.is_empty(), "{}", d.app().ui.status); +} + #[test] fn delete_tracks_dialog_deletes_empty_tracks() { let mut d = Driver::demo(); diff --git a/docs/control-protocol.md b/docs/control-protocol.md index 402319d3..6419e154 100644 --- a/docs/control-protocol.md +++ b/docs/control-protocol.md @@ -39,7 +39,7 @@ Methods (handlers in `crates/ui-egui/src/control.rs`): | `ui.playback` | `{action: play|stop|toggle, speed?}` | | | `ui.screenshot` | `{path?, panel?}` | PNG of the window or one panel; fails after 10 s when no frame is presented (window hidden, display asleep) | | `ui.resize`, `ui.focus`, `app.quit` | | `ui.focus` is the only request that activates the app and takes keyboard focus | -| `perf.stats` | – | performance counters (also the command `perf.stats`, so `engine.execute` and MCP `command_run` reach it; headless sessions return the engine part). `decode`: GOP-cache requests / hits / `cacheHitRate`, decoder seeks, samples decoded and skipped while catching up, `draftFrames` (decoded in draft mode), `h264Threads` (frame-threading workers per H.264 decoder), `liveDecoders` (sources holding a decoder now), `cacheMB` / `cacheBudgetMB` (decoded frames all sources hold, and the shared budget), `planePoolMB` / `planesReused` (idle plane buffers of evicted frames, and planes decoded into a recycled one), decoder ms (total and per sample), `framesDecoded`, `hardware` (Settings ▸ Playback ▸ Hardware decoding: `enabled`, pictures from hardware decoders `frames` vs `softwareFrames`, `sessions`, `declined` streams, mid-stream `fallbacks`); `playback`: playing, shown / dropped frames and `dropRate` of the current or last play, resolution, `draftDecode` (Settings ▸ Playback ▸ Draft decoding); `frames`: frame-worker jobs, cancelled, `requestHitRate`, `decodeMs` / `renderMs` (mean, p50, p95 of the last 256 jobs), queue, render-cost estimate, cache use; `ui`: fps and frame ms; `process`: CPU seconds; `media`, `jobs`. Counters are cumulative: diff two readings to measure an interval ([performance.md](performance.md)) | +| `perf.stats` | – | performance counters (also the command `perf.stats`, so `engine.execute` and MCP `command_run` reach it; headless sessions return the engine part). `decode`: GOP-cache requests / hits / `cacheHitRate`, decoder seeks, samples decoded and skipped while catching up, `draftFrames` (decoded in draft mode), `h264Threads` (frame-threading workers per H.264 decoder), `liveDecoders` (sources holding a decoder now), `cacheMB` / `cacheBudgetMB` (decoded frames all sources hold, and the shared budget), `planePoolMB` / `planesReused` (idle plane buffers of evicted frames, and planes decoded into a recycled one), decoder ms (total and per sample), `framesDecoded`, `hardware` (Settings ▸ Playback ▸ Hardware decoding: `enabled`, pictures from hardware decoders `frames` vs `softwareFrames`, `sessions`, `declined` streams, mid-stream `fallbacks`); `playback`: playing, shown / dropped frames and `dropRate` of the current or last play, resolution, `draftDecode` (Settings ▸ Playback ▸ Draft decoding), `audio` (desktop: sound is mixed 200 ms ahead of the device; device `callbacks`, `underruns` and `missingFrames` played as silence because the mixer fell behind, `minLeadMs` the least sound left buffered); `frames`: frame-worker jobs, cancelled, `requestHitRate`, `decodeMs` / `renderMs` (mean, p50, p95 of the last 256 jobs), queue, render-cost estimate, cache use; `ui`: fps and frame ms; `process`: CPU seconds; `media`, `jobs`. Counters are cumulative: diff two readings to measure an interval ([performance.md](performance.md)) | **Selection or explicit targets.** A command that acts on the selection is "not available right now" when nothing is selected. When `params` name the targets under a key the command documents (`clips` / diff --git a/docs/testing.md b/docs/testing.md index 4f52ed21..83dd97fe 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -203,7 +203,11 @@ brute-force barycentric reference, `.cube`/`.3dl` round trips), `filmcraft-gpu` Not covered headless: `ui.screenshot` (needs a real viewport), the wgpu monitor path (the harness runs the CPU texture path), audio output, and wall-clock playback advance (kittest frames do not -advance real time). Use `Driver` for new UI regressions: `d.exec(command, params)`, +advance real time). Playing sound is covered without a device: `play_ahead` tests drive the +mixed-ahead ring with a wall-clock device (order, underruns stay in time, mixer panic, stop), and +`cargo run --release -p filmcraft-ui-egui --example bench_audio -- ` plays a real +project's sequences through a simulated 48 kHz device and counts dropouts, mixing in the callback +(`direct`, as before) against mixing ahead (`ahead`). Use `Driver` for new UI regressions: `d.exec(command, params)`, `d.ok(method, params)`, `d.frames(n)`. ### Interactive checks