From f146d7ba9fedb75666c99f03b3bd8e3911eeb393 Mon Sep 17 00:00:00 2001 From: Idiotic Date: Thu, 8 Oct 2026 22:33:35 -0400 Subject: [PATCH] feat(media,engine): implement multi-stream container audio support and schema v13 - Update MediaInfo to store all discovered audio streams in audio_streams - Retain independent stream metadata and decoder state for MP4/MOV and Matroska/WebM - Update audio_placement_specs to allocate successive tracks via ensure_audio_tracks - Isolate clip rendering to read assigned audio stream rather than replaying stream 0 - Bump project schema to v13 with backwards compatibility for legacy v12 projects Resolves #312 Fixes #200 Fixes #315 --- apps/filmcraft-web/src/webcodecs.rs | 3 + crates/codecs/src/apv.rs | 2 +- crates/codecs/src/audio.rs | 4 +- crates/codecs/src/mkv.rs | 166 ++++++++++------ crates/codecs/src/mp4.rs | 210 ++++++++++++--------- crates/codecs/src/mpeg.rs | 10 +- crates/codecs/src/mxf.rs | 4 +- crates/codecs/src/ogg.rs | 2 +- crates/codecs/src/tests.rs | 4 +- crates/codecs/tests/mpeg_oracle.rs | 22 +-- crates/codecs/tests/mxf_oracle.rs | 14 +- crates/codecs/tests/ogg_oracle.rs | 6 +- crates/edit/src/tests.rs | 1 + crates/edit/src/through.rs | 1 + crates/edit/src/transcript/tests.rs | 1 + crates/engine/src/audio_placement_tests.rs | 66 ++++++- crates/engine/src/clip_ops.rs | 31 +-- crates/engine/src/clip_ops_tests.rs | 6 +- crates/engine/src/commands.rs | 28 ++- crates/engine/src/media_browser.rs | 4 +- crates/engine/src/media_browser_tests.rs | 2 +- crates/engine/src/media_pool.rs | 11 +- crates/engine/src/panels.rs | 2 +- crates/engine/src/project_manager.rs | 2 +- crates/engine/src/project_panel.rs | 4 +- crates/engine/src/project_tools.rs | 4 +- crates/engine/src/proxies.rs | 4 +- crates/engine/src/relink.rs | 2 +- crates/export/src/surround_tests.rs | 4 +- crates/format/Cargo.toml | 2 +- crates/format/src/lib.rs | 27 ++- crates/format/tests/fixtures.rs | 25 +++ crates/interchange/src/aaf/tests.rs | 2 +- crates/interchange/src/ale.rs | 9 +- crates/interchange/src/common.rs | 7 +- crates/interchange/src/comp.rs | 2 +- crates/interchange/src/essence.rs | 2 +- crates/interchange/src/fcp7.rs | 2 +- crates/interchange/src/fcpxml.rs | 6 +- crates/interchange/src/omf/tests.rs | 2 +- crates/interchange/tests/aaf_omf.rs | 4 +- crates/interchange/tests/support/mod.rs | 5 +- crates/media/src/generators.rs | 2 +- crates/media/src/lib.rs | 38 +++- crates/media/src/reader.rs | 2 +- crates/media/src/sequence.rs | 2 +- crates/media/src/still.rs | 2 +- crates/media/src/wav.rs | 6 +- crates/project/src/essential.rs | 1 + crates/project/src/find.rs | 2 +- crates/project/src/lib.rs | 8 + crates/render/src/audio.rs | 4 +- crates/render/src/mixer_tests.rs | 68 ++++++- crates/ui-egui/src/frames.rs | 3 + crates/ui-egui/src/panels/clip_dialogs.rs | 2 +- crates/ui-egui/src/panels/misc.rs | 2 +- crates/ui-egui/src/panels/timecode.rs | 2 +- docs/project-files.md | 3 +- 58 files changed, 604 insertions(+), 258 deletions(-) diff --git a/apps/filmcraft-web/src/webcodecs.rs b/apps/filmcraft-web/src/webcodecs.rs index 612a782b..addd4cfc 100644 --- a/apps/filmcraft-web/src/webcodecs.rs +++ b/apps/filmcraft-web/src/webcodecs.rs @@ -532,4 +532,7 @@ impl MediaSource for WcSource { fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> Result { self.inner.audio(start, frames, sample_rate) } + fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> Result { + self.inner.audio_stream(stream, start, frames, sample_rate) + } } diff --git a/crates/codecs/src/apv.rs b/crates/codecs/src/apv.rs index 0001e2a3..c7068ccb 100644 --- a/crates/codecs/src/apv.rs +++ b/crates/codecs/src/apv.rs @@ -114,7 +114,7 @@ impl ApvSource { bitrate, hdr: None, }), - audio: None, + audio_streams: Vec::new(), container: "APV".into(), start_timecode: None, file_size: Some(len), diff --git a/crates/codecs/src/audio.rs b/crates/codecs/src/audio.rs index 4942f6a6..c7d3762e 100644 --- a/crates/codecs/src/audio.rs +++ b/crates/codecs/src/audio.rs @@ -666,12 +666,12 @@ impl AudioFileSource { kind: MediaKind::AudioOnly, duration: Tick::from_units(frames as i64, rate as i64), video: None, - audio: Some(AudioStreamInfo { + audio_streams: vec![AudioStreamInfo { sample_rate: rate, channels: samples.len() as u32, codec: codec_name, bits_per_sample: track.codec_params.bits_per_sample, - }), + }], container: ext.to_uppercase(), start_timecode: None, file_size: Some(bytes.len() as u64), diff --git a/crates/codecs/src/mkv.rs b/crates/codecs/src/mkv.rs index ee23eede..b3980161 100644 --- a/crates/codecs/src/mkv.rs +++ b/crates/codecs/src/mkv.rs @@ -35,20 +35,27 @@ struct AudioState { last_decoded: Option, } +/// One playable audio track with its own decoder state and timing tables. +struct MkvAudio { + /// Index into `MkvFile::tracks`. + track: usize, + state: Mutex, + /// Packet start positions in source sample frames. + starts: Vec, + /// Decoder pre-roll after a seek, in source sample frames (0: prime with one packet). + preroll: i64, +} + pub struct MkvSource { info: MediaInfo, bytes: crate::Src, file: MkvFile, vtrack: Option, - atrack: Option, /// Video codec as an ISO-BMFF sample entry (for the decoder factories). ventry: Option, video: GopCache, - audio: Mutex, - /// Audio packet start positions in source sample frames. - audio_starts: Vec, - /// Decoder pre-roll after a seek, in source sample frames (0: prime with one packet). - audio_preroll: i64, + /// The playable audio tracks in file order: `info.audio_streams[k]` describes `audios[k]`. + audios: Vec, } /// Sample-exact Opus packet start positions (48 kHz frames, pre-skip removed). @@ -236,8 +243,16 @@ impl MkvSource { let bytes = crate::Src(reader); let file = filmcraft_matroska::open(&bytes).map_err(|e| CodecError::Container(e.to_string()))?; let vtrack = file.tracks.iter().position(|t| t.kind == TrackKind::Video && !t.samples.is_empty()); - let atrack = file.tracks.iter().position(|t| t.kind == TrackKind::Audio && !t.samples.is_empty()); - if vtrack.is_none() && atrack.is_none() { + // every audio track is a stream (the track count comes from the file: capped) + let atracks: Vec = file + .tracks + .iter() + .enumerate() + .filter(|(_, t)| t.kind == TrackKind::Audio && !t.samples.is_empty()) + .map(|(i, _)| i) + .take(filmcraft_media::MAX_AUDIO_STREAMS) + .collect(); + if vtrack.is_none() && atracks.is_empty() { return Err(CodecError::Unsupported("no playable tracks".into())); } let mut explicit_color = None; @@ -279,21 +294,24 @@ impl MkvSource { hdr: v.colour.as_ref().and_then(hdr_metadata), } }); - let audio = atrack.map(|i| { - let t = &file.tracks[i]; - let a = t.audio.clone().unwrap_or_default(); - let mut rate = a.output_sampling_frequency.unwrap_or(a.sampling_frequency).round().max(1.0) as u32; - let mut channels = (a.channels as u32).max(1); - if let Some(h) = opus_head(&t.codec) { - rate = crate::audio::OPUS_RATE; - channels = h.channels as u32; - } - AudioStreamInfo { sample_rate: rate, channels, codec: codec_label(&t.codec), bits_per_sample: a.bit_depth.map(|b| b as u32) } - }); + let audio_streams: Vec = atracks + .iter() + .filter_map(|&i| file.tracks.get(i)) + .map(|t| { + let a = t.audio.clone().unwrap_or_default(); + let mut rate = a.output_sampling_frequency.unwrap_or(a.sampling_frequency).round().max(1.0) as u32; + let mut channels = (a.channels as u32).max(1); + if let Some(h) = opus_head(&t.codec) { + rate = crate::audio::OPUS_RATE; + channels = h.channels as u32; + } + AudioStreamInfo { sample_rate: rate, channels, codec: codec_label(&t.codec), bits_per_sample: a.bit_depth.map(|b| b as u32) } + }) + .collect(); let duration = match file.duration_ns() { Some(ns) => Tick::from_rational(ns as i64, 1, 1_000_000_000), None => vtrack - .or(atrack) + .or(atracks.first().copied()) .map(|i| { let t = &file.tracks[i]; let end = t.samples.iter().map(|s| s.pts + s.duration as i64).max().unwrap_or(0); @@ -306,48 +324,36 @@ impl MkvSource { kind: if video.is_some() { MediaKind::Movie } else { MediaKind::AudioOnly }, duration, video, - audio, + audio_streams, container: if file.is_webm() { "WebM".into() } else { "Matroska".into() }, start_timecode: None, file_size: Some(bytes.0.len()), }; - let audio_starts = atrack - .map(|i| { - let t = &file.tracks[i]; - let rate = info.audio.as_ref().map_or(48_000, |a| a.sample_rate) as i64; + // `info.audio_streams[k]` was made from `atracks[k]`, so the two stay index-aligned + let audios: Vec = atracks + .iter() + .zip(&info.audio_streams) + .filter_map(|(&i, ainfo)| { + let t = file.tracks.get(i)?; + let rate = ainfo.sample_rate as i64; let (n, d) = tb(t); let at = |pts: i64| (pts as i128 * n as i128 * rate as i128 / d as i128) as i64; - match opus_head(&t.codec) { + let starts = match opus_head(&t.codec) { Some(h) => opus_starts(&file, &bytes, i, &h, rate), None => t.samples.iter().map(|s| at(s.pts)).collect(), - } - }) - .unwrap_or_default(); - let audio_preroll = atrack - .map(|i| { - let t = &file.tracks[i]; - let rate = info.audio.as_ref().map_or(48_000, |a| a.sample_rate) as i64; - match t.codec { + }; + let preroll = match t.codec { Codec::Opus { .. } => { let container = (t.seek_pre_roll_ns as u128 * rate as u128).div_ceil(1_000_000_000) as i64; container.max(crate::audio::OPUS_PRE_ROLL as i64 * rate / 48_000) } _ => 0, - } + }; + let state = Mutex::new(AudioState { decoder: None, packets: HashMap::new(), order: Vec::new(), last_decoded: None }); + Some(MkvAudio { track: i, state, starts, preroll }) }) - .unwrap_or(0); - Ok(Self { - info, - bytes, - file, - vtrack, - atrack, - ventry, - video: GopCache::new(explicit_color).with_rotation(rotation), - audio: Mutex::new(AudioState { decoder: None, packets: HashMap::new(), order: Vec::new(), last_decoded: None }), - audio_starts, - audio_preroll, - }) + .collect(); + Ok(Self { info, bytes, file, vtrack, ventry, video: GopCache::new(explicit_color).with_rotation(rotation), audios }) } fn read(&self, track: usize, i: usize) -> crate::Result> { @@ -373,13 +379,14 @@ impl MkvSource { } } - fn audio_packet(&self, st: &mut AudioState, i: usize) -> crate::Result>>> { + /// Decoded packet `i` of audio stream `a` (`ainfo` is its `AudioStreamInfo`; `st` its decoder state). + fn audio_packet(&self, a: &MkvAudio, ainfo: &AudioStreamInfo, st: &mut AudioState, i: usize) -> crate::Result>>> { if let Some(p) = st.packets.get(&i) { return Ok(p.clone()); } - let ti = self.atrack.ok_or_else(|| CodecError::Unsupported("no audio track".into()))?; - let track = &self.file.tracks[ti]; - let ainfo = self.info.audio.as_ref().ok_or_else(|| CodecError::Unsupported("no audio track".into()))?; + let ti = a.track; + let track = self.file.tracks.get(ti).ok_or_else(|| CodecError::Unsupported("no audio track".into()))?; + let pk_start = a.starts.get(i).copied().ok_or_else(|| CodecError::Container("audio packet out of range".into()))?; let data = self.read(ti, i)?; let decoded = match &track.codec { Codec::Pcm { float, big_endian, bits } => { @@ -402,8 +409,8 @@ impl MkvSource { if st.last_decoded.is_none_or(|l| l + 1 != i) { let d = st.decoder.as_mut().ok_or_else(|| CodecError::Decode("no audio decoder".into()))?; d.reset(); - let from = if self.audio_preroll > 0 { - self.audio_starts.partition_point(|&x| x <= self.audio_starts[i] - self.audio_preroll).saturating_sub(1) + let from = if a.preroll > 0 { + a.starts.partition_point(|&x| x <= pk_start.saturating_sub(a.preroll)).saturating_sub(1) } else { i.saturating_sub(1) }; @@ -413,7 +420,7 @@ impl MkvSource { } } } - let r = st.decoder.as_mut().ok_or_else(|| CodecError::Decode("no audio decoder".into()))?.decode(&data, self.audio_starts[i].max(0) as u64); + let r = st.decoder.as_mut().ok_or_else(|| CodecError::Decode("no audio decoder".into()))?.decode(&data, pk_start.max(0) as u64); st.last_decoded = Some(i); r.unwrap_or_default() } @@ -473,23 +480,27 @@ impl MediaSource for MkvSource { } fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> Result { - let ti = self.atrack.ok_or(MediaError::NoStream("audio"))?; - let n_packets = self.file.tracks[ti].samples.len(); - let ainfo = self.info.audio.as_ref().ok_or(MediaError::NoStream("audio"))?; + self.audio_stream(0, start, frames, sample_rate) + } + + fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> Result { + let a = self.audios.get(stream).ok_or(MediaError::NoStream("audio"))?; + let n_packets = self.file.tracks.get(a.track).ok_or(MediaError::NoStream("audio"))?.samples.len(); + let ainfo = self.info.audio_streams.get(stream).ok_or(MediaError::NoStream("audio"))?; let src_rate = ainfo.sample_rate; let ch = ainfo.channels.max(1) as usize; let ratio = src_rate as f64 / sample_rate as f64; let s0 = (start as f64 * ratio).floor() as i64; let need = (frames as f64 * ratio).ceil() as i64 + 2; let mut src: Vec> = vec![vec![0.0; need.max(0) as usize]; ch]; - let mut st = self.audio.lock().unwrap_or_else(|e| e.into_inner()); - let mut i = self.audio_starts.partition_point(|&x| x <= s0).saturating_sub(1); + let mut st = a.state.lock().unwrap_or_else(|e| e.into_inner()); + let mut i = a.starts.partition_point(|&x| x <= s0).saturating_sub(1); while i < n_packets { - let pk_start = self.audio_starts[i]; + let Some(&pk_start) = a.starts.get(i) else { break }; if pk_start >= s0 + need { break; } - let pk = self.audio_packet(&mut st, i)?; + let pk = self.audio_packet(a, ainfo, &mut st, i)?; for (c, dst) in src.iter_mut().enumerate() { let Some(chan) = pk.get(c.min(pk.len().saturating_sub(1))) else { continue }; for (k, v) in chan.iter().enumerate() { @@ -532,3 +543,34 @@ pub fn reader_opener(name: &str, head: &[u8], reader: &filmcraft_media::SharedRe } Some(MkvSource::open_reader(name, reader.clone()).map(|s| Arc::new(s) as SharedSource).map_err(Into::into)) } + +#[cfg(test)] +mod multi_audio_tests { + use filmcraft_matroska::{MkvWriter, MuxOptions, TrackKind, TrackSpec}; + use filmcraft_media::MediaSource; + + #[test] + fn every_audio_track_is_inspected_and_decoded_independently() { + let tracks: Vec = [(48_000.0, "first"), (24_000.0, "second")] + .into_iter() + .map(|(rate, name)| { + let mut t = TrackSpec::new(TrackKind::Audio, "A_PCM/INT/LIT"); + t.audio = Some((rate, 1, Some(16))); + t.name = Some(name.into()); + t + }) + .collect(); + let mut writer = MkvWriter::new(std::io::Cursor::new(Vec::new()), tracks, MuxOptions::default()).unwrap(); + for (stream, value) in [(0, 8192i16), (1, -16384)] { + let data: Vec = (0..100).flat_map(|_| value.to_le_bytes()).collect(); + writer.write_frame(stream, 0, true, &data, Some(4_000_000)).unwrap(); + } + let source = super::MkvSource::open("two.mkv", writer.finish().unwrap().into_inner().into()).unwrap(); + assert_eq!(source.info().audio_streams.iter().map(|a| a.sample_rate).collect::>(), vec![48_000, 24_000]); + let primary = source.audio(0, 50, 48_000).unwrap(); + let secondary = source.audio_stream(1, 0, 50, 48_000).unwrap(); + assert!((primary.channels[0][20] - 0.25).abs() < 0.001); + assert!((secondary.channels[0][20] + 0.5).abs() < 0.001); + assert!(source.audio_stream(2, 0, 50, 48_000).is_err()); + } +} diff --git a/crates/codecs/src/mp4.rs b/crates/codecs/src/mp4.rs index 2fedf3be..65eff772 100644 --- a/crates/codecs/src/mp4.rs +++ b/crates/codecs/src/mp4.rs @@ -29,20 +29,27 @@ struct AudioState { last_decoded: Option, } +/// One playable audio track with its own decoder state and timing tables. +struct Mp4Audio { + /// Index into `Mp4File::tracks`. + track: usize, + state: Mutex, + /// Cumulative sample start frames for the track (for packet lookup). + starts: Vec, + /// Presentation offset of the track in its timescale (edit list, or Opus pre-skip). + offset: i64, + /// Decoder pre-roll after a seek, in track timescale units (0: prime with one packet). + preroll: i64, +} + pub struct Mp4Source { info: MediaInfo, bytes: crate::Src, file: Mp4File, vtrack: Option, - atrack: Option, video: GopCache, - audio: Mutex, - /// Cumulative sample start frames for the audio track (for packet lookup). - audio_starts: Vec, - /// Presentation offset of the audio track in its timescale (edit list, or Opus pre-skip). - audio_offset: i64, - /// Decoder pre-roll after a seek, in audio track timescale units (0: prime with one packet). - audio_preroll: i64, + /// The playable audio tracks in file order: `info.audio_streams[k]` describes `audios[k]`. + audios: Vec, } pub fn sniff(b: &[u8]) -> bool { @@ -120,8 +127,10 @@ impl Mp4Source { // files can lack them: indexing `entries[0]` or dividing by the timescale used to panic). let playable = |t: &&filmcraft_isobmff::Track, kind| t.kind == kind && !t.samples.is_empty() && !t.entries.is_empty() && t.timescale > 0; let vtrack = file.tracks.iter().position(|t| playable(&t, TrackKind::Video)); - let atrack = file.tracks.iter().position(|t| playable(&t, TrackKind::Audio)); - if vtrack.is_none() && atrack.is_none() { + // every audio track is a stream (the track count comes from the file: capped) + let atracks: Vec = + file.tracks.iter().enumerate().filter(|(_, t)| playable(t, TrackKind::Audio)).map(|(i, _)| i).take(filmcraft_media::MAX_AUDIO_STREAMS).collect(); + if vtrack.is_none() && atracks.is_empty() { return Err(CodecError::Unsupported("no playable tracks".into())); } let mut color = ColorInfo::REC709; @@ -195,37 +204,11 @@ impl Mp4Source { } info }); - let audio = atrack.map(|i| { - let t = &file.tracks[i]; - let entry = &t.entries[0]; - let ap = entry.audio.clone().unwrap_or_default(); - let (rate, ch, bits) = match &entry.codec { - CodecConfig::Aac(a) => ( - { - // HE-AAC plays at twice the core rate the AudioSpecificConfig starts with - // an AAC access unit is at most 6144 bits per channel: a hostile `stsz` can't make us read more - let units = t - .samples - .iter() - .take(8) - .filter_map(|x| filmcraft_media::reader::read_range(&*bytes.0, x.offset, (x.size as usize).min(MAX_AAC_PROBE_UNIT)).ok()); - let units: Vec> = units.collect(); - match crate::audio::aac_output_rate(&a.asc, units.iter().map(Vec::as_slice)) { - Some(r) => r, - None if a.sample_rate > 0 => a.sample_rate, - None => ap.sample_rate as u32, - } - }, - if a.channel_config > 0 { a.channel_config as u32 } else { ap.channels }, - None, - ), - CodecConfig::Pcm(p) => (p.sample_rate as u32, p.channels, Some(p.bits as u32)), - CodecConfig::Opus(o) => (crate::audio::OPUS_RATE, (o.output_channels as u32).max(1), None), - _ => (if ap.sample_rate > 0.0 { ap.sample_rate as u32 } else { t.timescale }, ap.channels.max(1), None), - }; - AudioStreamInfo { sample_rate: rate.max(1), channels: ch.max(1), codec: codec_label(&entry.codec), bits_per_sample: bits } - }); - let duration = vtrack.or(atrack).and_then(|i| file.tracks.get(i)).map(|t| presentation_duration(&file, t)).unwrap_or_default(); + // one entry per playable audio track (`playable` guarantees a sample description); a track + // that somehow has none is dropped so `audio_streams` and `audios` stay index-aligned + let atracks: Vec = atracks.into_iter().filter(|&i| file.tracks.get(i).is_some_and(|t| !t.entries.is_empty())).collect(); + let audio_streams: Vec = atracks.iter().filter_map(|&i| file.tracks.get(i)).filter_map(|t| audio_stream_info(t, &bytes)).collect(); + let duration = vtrack.or(atracks.first().copied()).and_then(|i| file.tracks.get(i)).map(|t| presentation_duration(&file, t)).unwrap_or_default(); let start_timecode = file.tracks.iter().find_map(|t| match t.codec() { Some(CodecConfig::Timecode(tc)) => tc.start_frame.map(|f| f as i64), _ => None, @@ -235,50 +218,38 @@ impl Mp4Source { kind: if video.is_some() { MediaKind::Movie } else { MediaKind::AudioOnly }, duration, video, - audio, + audio_streams, container: if file.is_quicktime { "QuickTime".into() } else { "MPEG-4".into() }, start_timecode, file_size: Some(bytes.0.len()), }; - let audio_starts = atrack - .map(|i| { - let t = &file.tracks[i]; + let audios: Vec = atracks + .iter() + .filter_map(|&i| { + let t = file.tracks.get(i)?; let mut acc = 0i64; - t.samples + let starts = t + .samples .iter() .map(|s| { let st = acc; - acc += s.duration as i64; + acc = acc.saturating_add(s.duration as i64); st }) - .collect() - }) - .unwrap_or_default(); - let (audio_offset, audio_preroll) = atrack - .map(|i| { - let t = &file.tracks[i]; + .collect(); let ts = t.timescale.max(1) as i64; - match &t.entries[0].codec { - CodecConfig::Opus(o) => { + let (offset, preroll) = match t.entries.first().map(|e| &e.codec) { + Some(CodecConfig::Opus(o)) => { let off = if t.edit_offset != 0 { t.edit_offset } else { -(o.pre_skip as i64) * ts / 48_000 }; (off, (crate::audio::OPUS_PRE_ROLL as u64 * ts as u64).div_ceil(48_000) as i64) } _ => (t.edit_offset, 0), - } + }; + let state = Mutex::new(AudioState { decoder: None, packets: HashMap::new(), order: Vec::new(), last_decoded: None }); + Some(Mp4Audio { track: i, state, starts, offset, preroll }) }) - .unwrap_or((0, 0)); - Ok(Self { - info, - bytes, - file, - vtrack, - atrack, - video: GopCache::new(explicit_color).with_rotation(rotation), - audio: Mutex::new(AudioState { decoder: None, packets: HashMap::new(), order: Vec::new(), last_decoded: None }), - audio_starts, - audio_offset, - audio_preroll, - }) + .collect(); + Ok(Self { info, bytes, file, vtrack, video: GopCache::new(explicit_color).with_rotation(rotation), audios }) } fn read(&self, track: usize, i: usize) -> crate::Result> { @@ -294,27 +265,30 @@ impl Mp4Source { self.video.frame_late(&Mp4Video { src: self, track: ti }, t.to_rational_round(1, ts), late) } - fn audio_packet(&self, st: &mut AudioState, i: usize) -> crate::Result>>> { + /// Decoded packet `i` of audio stream `a` (`ainfo` is its `AudioStreamInfo`; `st` its decoder state). + fn audio_packet(&self, a: &Mp4Audio, ainfo: &AudioStreamInfo, st: &mut AudioState, i: usize) -> crate::Result>>> { if let Some(p) = st.packets.get(&i) { return Ok(p.clone()); } - let ti = self.atrack.ok_or_else(|| CodecError::Unsupported("no audio track".into()))?; - let track = &self.file.tracks[ti]; + let ti = a.track; + let track = self.file.tracks.get(ti).ok_or_else(|| CodecError::Unsupported("no audio track".into()))?; let entry = track.entries.first().ok_or_else(|| CodecError::Container("audio track without a sample description".into()))?; + let sample = track.samples.get(i).ok_or_else(|| CodecError::Container("audio packet out of range".into()))?; + let pk_start = a.starts.get(i).copied().unwrap_or(0); let data = self.read(ti, i)?; let decoded = match &entry.codec { CodecConfig::Pcm(p) => decode_pcm(&data, p), c => { if st.decoder.is_none() { - st.decoder = Some(PacketDecoder::for_isobmff(c, self.info.audio.as_ref().map_or(48_000, |a| a.sample_rate))?); + st.decoder = Some(PacketDecoder::for_isobmff(c, ainfo.sample_rate)?); } // Non-sequential access: reset and prime with the preceding packets (codec pre-roll: // one packet, or `OPUS_PRE_ROLL` worth for Opus). if st.last_decoded.is_none_or(|l| l + 1 != i) { let d = st.decoder.as_mut().ok_or_else(|| CodecError::Decode("no audio decoder".into()))?; d.reset(); - let from = if self.audio_preroll > 0 { - self.audio_starts.partition_point(|&x| x <= self.audio_starts[i] - self.audio_preroll).saturating_sub(1) + let from = if a.preroll > 0 { + a.starts.partition_point(|&x| x <= pk_start.saturating_sub(a.preroll)).saturating_sub(1) } else { i.saturating_sub(1) }; @@ -324,11 +298,11 @@ impl Mp4Source { } } } - let r = st.decoder.as_mut().ok_or_else(|| CodecError::Decode("no audio decoder".into()))?.decode(&data, self.audio_starts[i].max(0) as u64); + let r = st.decoder.as_mut().ok_or_else(|| CodecError::Decode("no audio decoder".into()))?.decode(&data, pk_start.max(0) as u64); st.last_decoded = Some(i); match r { Ok(v) => v, - Err(_) => vec![vec![0.0; track.samples[i].duration as usize]; self.info.audio.as_ref().map_or(2, |a| a.channels as usize)], + Err(_) => vec![vec![0.0; sample.duration as usize]; ainfo.channels.max(1) as usize], } } }; @@ -381,30 +355,36 @@ impl MediaSource for Mp4Source { } fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> Result { - let ti = self.atrack.ok_or(MediaError::NoStream("audio"))?; - let track = &self.file.tracks[ti]; - let ainfo = self.info.audio.as_ref().ok_or(MediaError::NoStream("audio"))?; + self.audio_stream(0, start, frames, sample_rate) + } + + fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> Result { + let a = self.audios.get(stream).ok_or(MediaError::NoStream("audio"))?; + let ti = a.track; + let track = self.file.tracks.get(ti).ok_or(MediaError::NoStream("audio"))?; + let ainfo = self.info.audio_streams.get(stream).ok_or(MediaError::NoStream("audio"))?; let src_rate = ainfo.sample_rate; let ch = ainfo.channels.max(1) as usize; // Map the requested window to source samples (edit list offset applied: presentation = pts + edit_offset). let ratio = src_rate as f64 / sample_rate as f64; let p0 = (start as f64 * ratio).floor() as i64; // (in i128: a hostile edit list can make the offset large enough to overflow i64) - let offset = i128::from(self.audio_offset) * i128::from(src_rate) / i128::from(track.timescale.max(1)); + let offset = i128::from(a.offset) * i128::from(src_rate) / i128::from(track.timescale.max(1)); let s0 = i64::try_from(i128::from(p0) - offset).unwrap_or(if offset > 0 { i64::MIN } else { i64::MAX }); let need = (frames as f64 * ratio).ceil() as i64 + 2; // Samples-per-unit: the track timescale is usually the sample rate for audio. let unit = src_rate as f64 / track.timescale.max(1) as f64; let mut src: Vec> = vec![vec![0.0; need.max(0) as usize]; ch]; - let mut st = self.audio.lock().unwrap_or_else(|e| e.into_inner()); - let first = self.audio_starts.partition_point(|&x| (x as f64 * unit) as i64 <= s0.max(0)).saturating_sub(1); + let mut st = a.state.lock().unwrap_or_else(|e| e.into_inner()); + let first = a.starts.partition_point(|&x| (x as f64 * unit) as i64 <= s0.max(0)).saturating_sub(1); let mut i = first; while i < track.samples.len() { - let pk_start = (self.audio_starts[i] as f64 * unit) as i64; + let Some(&start_units) = a.starts.get(i) else { break }; + let pk_start = (start_units as f64 * unit) as i64; if pk_start >= s0.saturating_add(need) { break; } - let pk = self.audio_packet(&mut st, i)?; + let pk = self.audio_packet(a, ainfo, &mut st, i)?; for (c, dst) in src.iter_mut().enumerate() { let Some(chan) = pk.get(c.min(pk.len().saturating_sub(1))) else { continue }; for (k, v) in chan.iter().enumerate() { @@ -443,6 +423,37 @@ impl MediaSource for Mp4Source { } } +/// What the file says about one audio track (`None`: the track has no sample description). +fn audio_stream_info(t: &filmcraft_isobmff::Track, bytes: &crate::Src) -> Option { + let entry = t.entries.first()?; + let ap = entry.audio.clone().unwrap_or_default(); + let (rate, ch, bits) = match &entry.codec { + CodecConfig::Aac(a) => ( + { + // HE-AAC plays at twice the core rate the AudioSpecificConfig starts with + // an AAC access unit is at most 6144 bits per channel: a hostile `stsz` can't make us read more + let units = t + .samples + .iter() + .take(8) + .filter_map(|x| filmcraft_media::reader::read_range(&*bytes.0, x.offset, (x.size as usize).min(MAX_AAC_PROBE_UNIT)).ok()); + let units: Vec> = units.collect(); + match crate::audio::aac_output_rate(&a.asc, units.iter().map(Vec::as_slice)) { + Some(r) => r, + None if a.sample_rate > 0 => a.sample_rate, + None => ap.sample_rate as u32, + } + }, + if a.channel_config > 0 { a.channel_config as u32 } else { ap.channels }, + None, + ), + CodecConfig::Pcm(p) => (p.sample_rate as u32, p.channels, Some(p.bits as u32)), + CodecConfig::Opus(o) => (crate::audio::OPUS_RATE, (o.output_channels as u32).max(1), None), + _ => (if ap.sample_rate > 0.0 { ap.sample_rate as u32 } else { t.timescale }, ap.channels.max(1), None), + }; + Some(AudioStreamInfo { sample_rate: rate.max(1), channels: ch.max(1), codec: codec_label(&entry.codec), bits_per_sample: bits }) +} + fn codec_label(c: &CodecConfig) -> String { match c { CodecConfig::Avc(a) => format!( @@ -803,6 +814,33 @@ mod tests { } } +#[cfg(test)] +mod multi_audio_tests { + use filmcraft_isobmff::{Brand, Mp4Writer, PcmConfig, SampleEntry, TrackConfig, WriteSample, WriterOptions}; + use filmcraft_media::MediaSource; + + /// Two mono PCM tracks with different rates and samples. Stream indices follow file order; + /// decoder caches, sample timing and resampling must not cross streams. + #[test] + fn every_audio_track_is_inspected_and_decoded_independently() { + let mut mux = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mov)).unwrap(); + for (rate, value) in [(48_000u32, 8192i16), (24_000, -16384)] { + let entry = SampleEntry::pcm(PcmConfig { bits: 16, float: false, big_endian: false, signed: true, channels: 1, sample_rate: rate as f64 }); + let track = mux.add_track(TrackConfig::new(entry, rate)).unwrap(); + let data: Vec = (0..rate / 10).flat_map(|_| value.to_le_bytes()).collect(); + mux.write_sample(track, WriteSample { data: &data, duration: rate / 10, composition_offset: 0, is_sync: true }).unwrap(); + } + let bytes = mux.finish().unwrap().into_inner(); + let source = super::Mp4Source::open("two.mov", bytes.into()).unwrap(); + assert_eq!(source.info().audio_streams.iter().map(|a| a.sample_rate).collect::>(), vec![48_000, 24_000]); + let primary = source.audio(0, 100, 48_000).unwrap(); + let secondary = source.audio_stream(1, 0, 100, 48_000).unwrap(); + assert!((primary.channels[0][50] - 0.25).abs() < 0.001); + assert!((secondary.channels[0][50] + 0.5).abs() < 0.001); + assert!(source.audio_stream(2, 0, 100, 48_000).is_err()); + } +} + #[cfg(test)] mod he_aac_tests { use filmcraft_isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions}; @@ -829,7 +867,7 @@ mod he_aac_tests { } let b = mux.finish().unwrap().into_inner(); let src = super::Mp4Source::open("he.mp4", b.into()).unwrap(); - assert_eq!(src.info().audio.as_ref().unwrap().sample_rate, 44_100); + assert_eq!(src.info().audio().unwrap().sample_rate, 44_100); // one second from 0.5 s at 44.1 kHz: the 1 kHz tone at its level (not silence, not shifted) let buf = src.audio(22_050, 44_100, 44_100).unwrap(); let x = &buf.channels[0]; diff --git a/crates/codecs/src/mpeg.rs b/crates/codecs/src/mpeg.rs index c2d0ba67..ef2a6999 100644 --- a/crates/codecs/src/mpeg.rs +++ b/crates/codecs/src/mpeg.rs @@ -211,7 +211,7 @@ impl MpegSource { kind: MediaKind::Movie, duration: Tick::ZERO, video: None, - audio: None, + audio_streams: Vec::new(), container: file.format.name().to_string(), start_timecode: None, file_size: Some(bytes.0.len()), @@ -316,7 +316,7 @@ impl MpegSource { kind: MediaKind::Movie, duration: Tick::ZERO, video: None, - audio: None, + audio_streams: Vec::new(), container: "MPEG video elementary stream".into(), start_timecode: None, file_size: Some(bytes.0.len()), @@ -485,7 +485,8 @@ impl MpegSource { /// Mark the audio stream as not decodable (it still shows in the media info). fn audio_unsupported(&mut self, codec_name: &str, why: String) -> Option { self.unsupported_audio = Some(why); - self.info.audio = Some(AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: format!("{codec_name} (unsupported)"), bits_per_sample: None }); + self.info.audio_streams = + vec![AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: format!("{codec_name} (unsupported)"), bits_per_sample: None }]; None } @@ -599,7 +600,8 @@ impl MpegSource { self.info.kind = MediaKind::AudioOnly; } } - self.info.audio = Some(AudioStreamInfo { sample_rate: rate.max(1), channels: channels.max(1) as u32, codec: codec_name, bits_per_sample: bits }); + self.info.audio_streams = + vec![AudioStreamInfo { sample_rate: rate.max(1), channels: channels.max(1) as u32, codec: codec_name, bits_per_sample: bits }]; Some(AudioTrack { stream: ai, codec, diff --git a/crates/codecs/src/mxf.rs b/crates/codecs/src/mxf.rs index 5980906b..1b230a8d 100644 --- a/crates/codecs/src/mxf.rs +++ b/crates/codecs/src/mxf.rs @@ -361,7 +361,7 @@ impl MxfSource { kind: if video.is_some() { MediaKind::Movie } else { MediaKind::AudioOnly }, duration, video, - audio, + audio_streams: audio.into_iter().collect(), container: format!("MXF {}", file.operational_pattern.name()), start_timecode: None, file_size: Some(bytes.0.len()), @@ -457,7 +457,7 @@ impl MediaSource for MxfSource { } fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> Result { - let ainfo = self.info.audio.as_ref().ok_or(MediaError::NoStream("audio"))?; + let ainfo = self.info.audio().ok_or(MediaError::NoStream("audio"))?; let src_rate = ainfo.sample_rate; let ch = ainfo.channels.max(1) as usize; let mut out = AudioBuffer::silence(sample_rate, ch, frames); diff --git a/crates/codecs/src/ogg.rs b/crates/codecs/src/ogg.rs index 18092667..1e729837 100644 --- a/crates/codecs/src/ogg.rs +++ b/crates/codecs/src/ogg.rs @@ -110,7 +110,7 @@ impl OggSource { kind: MediaKind::AudioOnly, duration: Tick::from_units(frames, rate as i64), video: None, - audio: Some(AudioStreamInfo { sample_rate: rate, channels: channels.max(1), codec: codec.into(), bits_per_sample: None }), + audio_streams: vec![AudioStreamInfo { sample_rate: rate, channels: channels.max(1), codec: codec.into(), bits_per_sample: None }], container: "Ogg".into(), start_timecode: None, file_size: Some(bytes.0.len()), diff --git a/crates/codecs/src/tests.rs b/crates/codecs/src/tests.rs index 819a41f9..4753785a 100644 --- a/crates/codecs/src/tests.rs +++ b/crates/codecs/src/tests.rs @@ -70,7 +70,7 @@ fn mjpeg_mov_with_pcm() { fn aac_mp4_decodes() { let Some(b) = fixture("tone_aac.mp4", &["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:d=3", "-c:a", "aac", "-b:a", "128k"]) else { return }; let src = crate::open_bytes("tone_aac.mp4", b).unwrap(); - assert!(src.info().audio.as_ref().unwrap().codec.contains("AAC")); + assert!(src.info().audio().unwrap().codec.contains("AAC")); // read in the middle (random access) and sequentially let a = src.audio(48_000, 4800, 48_000).unwrap(); let p = a.peaks()[0]; @@ -510,7 +510,7 @@ fn check_opus(name: &str, channels: usize, expr: &str, extra: &[&str], container let src = crate::open_bytes(name, b).expect("open"); let info = src.info().clone(); assert_eq!(info.container, container); - let a = info.audio.as_ref().expect("audio"); + let a = info.audio().expect("audio"); assert_eq!((a.codec.as_str(), a.sample_rate, a.channels as usize), ("Opus", 48_000, channels)); let d = info.duration.seconds(); assert!((d - 3.0).abs() < 0.03, "{name}: duration {d}"); diff --git a/crates/codecs/tests/mpeg_oracle.rs b/crates/codecs/tests/mpeg_oracle.rs index 5fdbc0bf..e9a78fb4 100644 --- a/crates/codecs/tests/mpeg_oracle.rs +++ b/crates/codecs/tests/mpeg_oracle.rs @@ -245,7 +245,7 @@ fn check_video(ff: &Path, file: &Path, tol: u16, seeks: usize) -> (u16, f64) { /// Our decoded audio from the first audio frame against ffmpeg's decode. Returns the max error. fn check_audio(ff: &Path, file: &Path, tol: f32) -> f32 { let src = open(file); - let a = src.info().audio.clone().unwrap(); + let a = src.info().audio().cloned().unwrap(); let (start, len) = src.audio_extent().unwrap(); let want = ffmpeg_audio_f32(ff, file, &[]); let ch = a.channels as usize; @@ -314,7 +314,7 @@ fn transport_stream_mpeg2_and_mp2() { let (worst, psnr) = check_video(&ff, &f, 4, 12); assert!(psnr >= 58.0, "PSNR {psnr}"); println!("ts_mpeg2_mp2: video max diff {worst}, PSNR {psnr:.2} dB"); - let a = src.info().audio.clone().unwrap(); + let a = src.info().audio().cloned().unwrap(); assert_eq!((a.codec.as_str(), a.sample_rate, a.channels), ("MPEG Audio", 48_000, 2)); let e = check_audio(&ff, &f, 2e-4); println!("ts_mpeg2_mp2: MP2 audio max error {e:.2e}"); @@ -337,7 +337,7 @@ fn bdav_h264_with_lpcm_sample_exact() { let f = make(&ff, "m2ts_h264_lpcm.m2ts"); let src = open(&f); assert_eq!(src.info().container, "MPEG-2 TS (BDAV/AVCHD)"); - let a = src.info().audio.clone().unwrap(); + let a = src.info().audio().cloned().unwrap(); assert_eq!((a.codec.as_str(), a.channels, a.bits_per_sample), ("LPCM (Blu-ray)", 2, Some(24))); check_video(&ff, &f, 0, 10); assert_eq!(check_audio(&ff, &f, 0.0), 0.0); @@ -349,7 +349,7 @@ fn avchd_ac3_audio() { let f = make(&ff, "m2ts_h264_ac3.mts"); let src = open(&f); check_video(&ff, &f, 0, 6); - let a = src.info().audio.clone().unwrap(); + let a = src.info().audio().cloned().unwrap(); if filmcraft_codecs::audio::AC3_DECODER { assert_eq!((a.codec.as_str(), a.channels), ("AC-3", 2)); // AC-3 zero-bit mantissas carry decoder-specific dither (A/52 §7.3.4) @@ -370,7 +370,7 @@ fn transport_stream_hevc_and_latm() { let f = make(&ff, "ts_hevc_latm.ts"); let src = open(&f); assert_eq!(src.info().video.as_ref().unwrap().codec, "HEVC"); - assert_eq!(src.info().audio.as_ref().unwrap().codec, "AAC (LATM)"); + assert_eq!(src.info().audio().unwrap().codec, "AAC (LATM)"); check_video(&ff, &f, 0, 8); let e = check_audio(&ff, &f, 1e-4); println!("ts_hevc_latm: AAC (LATM) max error {e:.2e}"); @@ -389,17 +389,17 @@ fn program_streams() { // DVD LPCM: sample-exact let f = make(&ff, "ps_mpeg2_lpcm.vob"); let src = open(&f); - assert_eq!(src.info().audio.as_ref().unwrap().codec, "LPCM (DVD)"); - assert_eq!(src.info().audio.as_ref().unwrap().bits_per_sample, Some(24)); + assert_eq!(src.info().audio().unwrap().codec, "LPCM (DVD)"); + assert_eq!(src.info().audio().unwrap().bits_per_sample, Some(24)); assert_eq!(check_audio(&ff, &f, 0.0), 0.0); check_video(&ff, &f, 4, 4); let f = make(&ff, "ps_mpeg2_lpcm16.vob"); - assert_eq!(open(&f).info().audio.as_ref().unwrap().bits_per_sample, Some(16)); + assert_eq!(open(&f).info().audio().unwrap().bits_per_sample, Some(16)); assert_eq!(check_audio(&ff, &f, 0.0), 0.0); // AC-3 in private stream 1 let f = make(&ff, "ps_mpeg2_ac3.vob"); let src = open(&f); - assert_eq!(src.info().audio.as_ref().unwrap().codec, "AC-3"); + assert_eq!(src.info().audio().unwrap().codec, "AC-3"); let e = check_audio(&ff, &f, 5e-3); println!("ps_mpeg2_ac3: AC-3 max error {e:.2e}"); // MPEG-1 system stream @@ -457,7 +457,7 @@ fn standalone_mp2_file() { let ff = filmcraft_testkit::require_ffmpeg!(); let f = make(&ff, "audio.mp2"); let src = filmcraft_codecs::open_bytes("audio.mp2", bytes(&f)).unwrap(); - let a = src.info().audio.clone().unwrap(); + let a = src.info().audio().cloned().unwrap(); assert_eq!((a.sample_rate, a.channels), (48_000, 2)); let want = ffmpeg_audio_f32(&ff, &f, &[]); let n = want.len() / 2; @@ -483,7 +483,7 @@ fn truncated_and_corrupt_files_never_panic() { let _ = s.video_frame(FrameRequest::full(v.frame_rate.tick_of(i))); } } - if let Some(a) = &info.audio { + if let Some(a) = info.audio() { let _ = s.audio(rng.below(48_000) as i64, 2048, a.sample_rate); } }; diff --git a/crates/codecs/tests/mxf_oracle.rs b/crates/codecs/tests/mxf_oracle.rs index 874b6e9a..5a965b81 100644 --- a/crates/codecs/tests/mxf_oracle.rs +++ b/crates/codecs/tests/mxf_oracle.rs @@ -179,7 +179,7 @@ fn check_video(ff: &Path, file: &Path, pix_fmt: &str, chroma: (usize, usize), bp /// Our audio, as interleaved f32, equals ffmpeg's decode exactly. fn check_audio_exact(ff: &Path, file: &Path, extra: &[&str]) { let src = open(file); - let a = src.info().audio.clone().unwrap(); + let a = src.info().audio().cloned().unwrap(); let ch = a.channels as usize; let want = ffmpeg_audio_f32(ff, file, extra); let n = want.len() / ch; @@ -236,7 +236,7 @@ fn h264_2997_drop_frame_timecode_and_24_bit_audio() { assert!(tc.drop_frame); assert_eq!(tc.format(), "01:00:00;00"); assert_eq!(info.start_timecode, Some(107_892)); - assert_eq!(info.audio.as_ref().unwrap().bits_per_sample, Some(24)); + assert_eq!(info.audio().unwrap().bits_per_sample, Some(24)); check_video(&ff, &f, "yuv420p", (2, 2), 1, 0, 10); check_audio_exact(&ff, &f, &[]); } @@ -299,14 +299,14 @@ fn op_atom_video_and_audio_files() { let v = make(&ff, "mxf_atom_dnxhr.mxf"); let src = open(&v); assert_eq!(src.info().container, "MXF OP-Atom"); - assert!(src.info().audio.is_none()); + assert!(!src.info().has_audio()); assert_eq!(src.file().tracks[0].wrapping, filmcraft_mxf::Wrapping::Clip); check_video(&ff, &v, "yuv422p", (2, 1), 1, 2, 10); let a = make(&ff, "mxf_atom_pcm.mxf"); let src = open(&a); assert_eq!(src.info().kind, MediaKind::AudioOnly); assert_eq!(src.info().container, "MXF OP-Atom"); - let ai = src.info().audio.clone().unwrap(); + let ai = src.info().audio().cloned().unwrap(); assert_eq!((ai.sample_rate, ai.channels, ai.bits_per_sample), (48_000, 1, Some(24))); assert_eq!(src.info().duration, filmcraft_time::Tick::from_units(48_000, 48_000)); check_audio_exact(&ff, &a, &[]); @@ -317,7 +317,7 @@ fn d10_aes3_elements_sample_exact() { let ff = filmcraft_testkit::require_ffmpeg!(); let f = make(&ff, "mxf_d10.mxf"); let src = open(&f); - let a = src.info().audio.clone().unwrap(); + let a = src.info().audio().cloned().unwrap(); assert_eq!(a.codec, "AES3 PCM"); assert!(a.channels >= 4, "{}", a.channels); let v = src.info().video.as_ref().unwrap(); @@ -347,7 +347,7 @@ fn truncated_and_corrupt_files_never_panic() { let _ = s.video_frame(FrameRequest::full(t)); } } - if let Some(a) = &info.audio { + if let Some(a) = info.audio() { let _ = s.audio(rng.below(96_000) as i64, 4000, a.sample_rate); } } @@ -364,7 +364,7 @@ fn truncated_and_corrupt_files_never_panic() { let _ = s.video_frame(FrameRequest::full(s.info().frame_rate().tick_of(i))); } } - if s.info().audio.is_some() { + if s.info().has_audio() { let _ = s.audio(0, 9600, 48_000); } } diff --git a/crates/codecs/tests/ogg_oracle.rs b/crates/codecs/tests/ogg_oracle.rs index 043715a1..93940775 100644 --- a/crates/codecs/tests/ogg_oracle.rs +++ b/crates/codecs/tests/ogg_oracle.rs @@ -88,7 +88,7 @@ fn snr_db(reference: &[f32], ours: &[f32]) -> f64 { /// (our continuous decode interleaved, ffmpeg's decode interleaved, channels, rate) fn decode_both(ff: &Path, file: &Path, decoder: &[&str]) -> (OggSource, Vec, Vec, usize, u32) { let src = OggSource::open(file.file_name().unwrap().to_str().unwrap(), bytes(file)).unwrap(); - let a = src.info().audio.clone().unwrap(); + let a = src.info().audio().cloned().unwrap(); let ch = a.channels as usize; let mut args: Vec<&str> = decoder.to_vec(); args.extend_from_slice(&["-i", file.to_str().unwrap(), "-map", "0:a:0", "-f", "f32le", "-"]); @@ -134,7 +134,7 @@ fn opus_celt_music_matches_libopus() { assert_eq!(src.info().container, "Ogg"); assert_eq!(src.info().kind, MediaKind::AudioOnly); assert_eq!((rate, ch), (48_000, 2)); - assert_eq!(src.info().audio.as_ref().unwrap().codec, "Opus"); + assert_eq!(src.info().audio().unwrap().codec, "Opus"); let snr = snr_db(&want, &ours); eprintln!("{}: {} samples, SNR {snr:.1} dB vs libopus", f.display(), want.len() / ch); assert!(snr >= 50.0, "SNR {snr:.1} dB"); @@ -173,7 +173,7 @@ fn vorbis_matches_ffmpeg() { let ff = filmcraft_testkit::require_ffmpeg!(); let f = make(&ff, "ogg_vorbis.ogg"); let (src, ours, want, ch, rate) = decode_both(&ff, &f, &[]); - assert_eq!(src.info().audio.as_ref().unwrap().codec, "Vorbis"); + assert_eq!(src.info().audio().unwrap().codec, "Vorbis"); assert_eq!((rate, ch), (44_100, 2)); let snr = snr_db(&want, &ours); eprintln!("{}: SNR {snr:.1} dB vs ffmpeg's Vorbis decoder", f.display()); diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index 4f49f1fa..558e1ebc 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -60,6 +60,7 @@ impl Fx { hold_filters: false, field_options: None, source_channels: Vec::new(), + audio_stream: 0, graphic: None, } } diff --git a/crates/edit/src/through.rs b/crates/edit/src/through.rs index 0905149a..c8267535 100644 --- a/crates/edit/src/through.rs +++ b/crates/edit/src/through.rs @@ -180,6 +180,7 @@ mod tests { hold_filters: false, field_options: None, source_channels: Vec::new(), + audio_stream: 0, graphic: None, } } diff --git a/crates/edit/src/transcript/tests.rs b/crates/edit/src/transcript/tests.rs index 2fd8ed17..afd8c52b 100644 --- a/crates/edit/src/transcript/tests.rs +++ b/crates/edit/src/transcript/tests.rs @@ -57,6 +57,7 @@ fn clip(id: u64, start: f64, dur: f64, src_in: f64) -> TrackItem { hold_filters: false, field_options: None, source_channels: Vec::new(), + audio_stream: 0, graphic: None, } } diff --git a/crates/engine/src/audio_placement_tests.rs b/crates/engine/src/audio_placement_tests.rs index c8b6cc21..541324ae 100644 --- a/crates/engine/src/audio_placement_tests.rs +++ b/crates/engine/src/audio_placement_tests.rs @@ -1,6 +1,8 @@ //! Multi-clip audio placement: `audio_placement_specs` and dynamic audio track allocation. +use filmcraft_media::MediaSource; use filmcraft_project::{AudioChannels, ItemId, TrackKind}; +use filmcraft_time::{FrameRate, Tick, TimeRange}; use serde_json::json; @@ -19,13 +21,13 @@ fn active(s: &Session) -> ItemId { #[test] fn specs_always_place_one_clip_on_the_destination() { - assert_eq!(audio_placement_specs(&[]), vec![AudioPlacementSpec { track_offset: 0, source_channels: None }]); - assert_eq!(audio_placement_specs(&[vec![0, 1]]).len(), 1); + assert_eq!(audio_placement_specs(&[], 1), vec![AudioPlacementSpec { track_offset: 0, audio_stream: 0, source_channels: None }]); + assert_eq!(audio_placement_specs(&[vec![0, 1]], 1).len(), 1); } #[test] fn specs_put_further_clips_on_the_tracks_below() { - let specs = audio_placement_specs(&[vec![0], vec![1], vec![2, 3]]); + let specs = audio_placement_specs(&[vec![0], vec![1], vec![2, 3]], 1); let got: Vec<(usize, Option>)> = specs.into_iter().map(|s| (s.track_offset, s.source_channels)).collect(); assert_eq!(got, vec![(0, None), (1, Some(vec![1])), (2, Some(vec![2, 3]))]); } @@ -33,7 +35,63 @@ fn specs_put_further_clips_on_the_tracks_below() { #[test] fn specs_are_capped_for_hostile_channel_maps() { let clips = vec![vec![0u16]; 100_000]; - assert!(audio_placement_specs(&clips).len() <= crate::sequence_tools::MAX_TRACKS); + assert!(audio_placement_specs(&clips, usize::MAX).len() <= crate::sequence_tools::MAX_TRACKS); +} + +#[test] +fn specs_append_container_streams_after_channel_map_clips() { + let specs = audio_placement_specs(&[vec![1], vec![0]], 3); + assert_eq!( + specs.iter().map(|s| (s.track_offset, s.audio_stream, s.source_channels.clone())).collect::>(), + vec![(0, 0, None), (1, 0, Some(vec![0])), (2, 1, None), (3, 2, None)] + ); + assert_eq!(audio_placement_specs(&[], 0).len(), 1); +} + +#[test] +fn placing_media_with_two_streams_expands_tracks_and_keeps_stream_indices() { + let mut s = Session::default(); + let mut p = (*s.project).clone(); + let rate = FrameRate::FPS_24; + let seq = p.new_sequence("Two streams", Default::default(), 1, 1, None); + let source = filmcraft_media::generators::GeneratorSource::new(filmcraft_media::Generator::BarsAndTone, 32, 32, rate, rate.tick_of(24)); + let mut info = source.info().clone(); + info.audio_streams.push(info.audio_streams[0].clone()); + let item = p.add_item( + "two streams", + filmcraft_project::Label::Iris, + filmcraft_project::ItemKind::Media(filmcraft_project::MediaClip { + media: filmcraft_project::MediaRef::Generator(filmcraft_media::Generator::BarsAndTone), + info, + interpret: Default::default(), + mark_in: None, + mark_out: None, + markers: vec![], + offline: false, + proxy: None, + identity: None, + }), + None, + ); + let (vdest, adest) = { + let q = p.sequence(seq).unwrap(); + (q.video_tracks[0].id, q.audio_tracks[0].id) + }; + s.project = std::sync::Arc::new(p); + s.state.active_sequence = Some(seq); + let ids = + crate::commands::place_item(&mut s, item, TimeRange::new(Tick::ZERO, rate.tick_of(24)), Tick::ZERO, Some(vdest), Some(adest), false, "test", None) + .unwrap(); + assert_eq!(ids.len(), 3); + let q = s.project.sequence(seq).unwrap(); + assert_eq!(q.audio_tracks.len(), 2); + let (a, b) = (&q.audio_tracks[0].items[0], &q.audio_tracks[1].items[0]); + assert_eq!((a.audio_stream, b.audio_stream), (0, 1)); + assert_eq!(a.link, b.link); + assert!(a.link.is_some()); + assert_ne!(a.id, b.id); + assert_eq!(a.source_in, b.source_in); + assert_eq!(a.start, b.start); } #[test] diff --git a/crates/engine/src/clip_ops.rs b/crates/engine/src/clip_ops.rs index a1910bfb..e15dd353 100644 --- a/crates/engine/src/clip_ops.rs +++ b/crates/engine/src/clip_ops.rs @@ -636,8 +636,12 @@ fn sequence_from_clip(s: &mut Session, p: &Value) -> Result { settings.height = *height; settings.frame_rate = *rate; } - let channels = media_root(&s.project, first).and_then(|(_, m, _)| m.info.audio.as_ref().map(|a| a.channels)).unwrap_or(2); - let n_audio = media_root(&s.project, first).and_then(|(_, m, _)| m.interpret.audio_channels.as_ref().map(|a| a.clips.len())).unwrap_or(1).max(1); + let channels = media_root(&s.project, first).and_then(|(_, m, _)| m.info.audio().map(|a| a.channels)).unwrap_or(2); + let n_audio = media_root(&s.project, first) + .map(|(_, m, _)| { + crate::commands::audio_placement_specs(m.interpret.audio_channels.as_ref().map_or(&[], |a| a.clips.as_slice()), m.info.audio_streams.len()).len() + }) + .unwrap_or(1); let n0 = s.history.undo.len(); let name = pi.name.clone(); let bin = s.project.root.parent_of(first).filter(|b| *b != s.project.root.id); @@ -761,12 +765,15 @@ fn offline_file(s: &mut Session, p: &Value) -> Result { bitrate: None, hdr: None, }), - audio: has_a.then(|| filmcraft_media::AudioStreamInfo { - sample_rate: u64_p(p, "sampleRate").unwrap_or(48_000) as u32, - channels: u64_p(p, "channels").unwrap_or(2).max(1) as u32, - codec: String::new(), - bits_per_sample: None, - }), + audio_streams: has_a + .then(|| filmcraft_media::AudioStreamInfo { + sample_rate: u64_p(p, "sampleRate").unwrap_or(48_000) as u32, + channels: u64_p(p, "channels").unwrap_or(2).max(1) as u32, + codec: String::new(), + bits_per_sample: None, + }) + .into_iter() + .collect(), container: String::new(), start_timecode: start_tc, file_size: None, @@ -913,7 +920,7 @@ fn consolidate_duplicates(s: &mut Session, _: &Value) -> Result { for it in s.project.items.values() { if let ItemKind::Media(m) = &it.kind { // same file (or generator), interpretation and streams - let shape = (m.info.video.is_some(), m.info.audio.is_some(), m.info.duration); + let shape = (m.info.video.is_some(), m.info.has_audio(), m.info.duration); let key = serde_json::to_string(&(&m.media, &m.interpret, m.offline, shape)).unwrap_or_default(); groups.entry(key).or_default().push(it.id); } @@ -1249,7 +1256,7 @@ fn audio_channels(s: &mut Session, p: &Value) -> Result { let mut maps = Vec::new(); for i in &items { let Some(m) = s.project.item(*i).and_then(|it| it.as_media()) else { continue }; - let n = m.info.audio.as_ref().map_or(2, |a| a.channels) as u16; + let n = m.info.audio().map_or(2, |a| a.channels) as u16; let fmt = format.unwrap_or_else(|| m.interpret.audio_channels.as_ref().map_or(AudioChannels::Stereo, |a| a.format)); let map = match &custom { Some(c) => { @@ -1554,7 +1561,7 @@ fn breakout_to_mono(s: &mut Session, p: &Value) -> Result { for i in &items { let Some(it) = pr.item(*i).cloned() else { continue }; let Some(m) = it.as_media() else { continue }; - let n = m.info.audio.as_ref().map_or(1, |a| a.channels).max(1) as u16; + let n = m.info.audio().map_or(1, |a| a.channels).max(1) as u16; let bin = pr.root.parent_of(*i).filter(|b| *b != pr.root.id); for c in 0..n { let suffix = match (n, c) { @@ -1590,7 +1597,7 @@ fn extract_audio(s: &mut Session, p: &Value) -> Result { let mut out = Vec::new(); for item in items { let (root, m, sub) = media_root(&s.project, item).ok_or_else(|| bad("clip.extractAudio", "not a media clip"))?; - let a = m.info.audio.clone().ok_or_else(|| bad("clip.extractAudio", "the clip has no audio"))?; + let a = m.info.audio().cloned().ok_or_else(|| bad("clip.extractAudio", "the clip has no audio"))?; let range = sub.unwrap_or(TimeRange::new(Tick::ZERO, m.info.duration)); let dir = match str_p(p, "dir") { Some(d) => d.to_string(), diff --git a/crates/engine/src/clip_ops_tests.rs b/crates/engine/src/clip_ops_tests.rs index 1df15fc5..9fc46de9 100644 --- a/crates/engine/src/clip_ops_tests.rs +++ b/crates/engine/src/clip_ops_tests.rs @@ -261,7 +261,7 @@ fn new_sequence_from_clip_bin_from_selection_and_offline_file() { .unwrap(); let it = s.project.item(ItemId(r["item"].as_u64().unwrap())).unwrap(); let m = it.as_media().unwrap(); - assert!(m.offline && m.info.audio.is_none() && m.info.video.is_some()); + assert!(m.offline && !m.info.has_audio() && m.info.video.is_some()); assert_eq!(m.info.duration, s.sequence_rate().snap_nearest(Tick(4 * TICKS_PER_SECOND))); assert_eq!(m.info.start_timecode, Some(3600 * 24)); assert_eq!(it.metadata["Tape Name"], "A001"); @@ -372,7 +372,7 @@ fn modify_audio_channels_and_breakout_to_mono() { s.project = std::sync::Arc::new({ let mut p = (*s.project).clone(); if let Some(m) = p.item_mut(item_named(&s, "Bars and Tone")).and_then(|i| i.as_media_mut()) { - m.info.audio = None; + m.info.audio_streams.clear(); } p }); @@ -557,7 +557,7 @@ fn extract_audio_writes_a_wav_and_imports_it() { let item = s.project.item(ItemId(e["item"].as_u64().unwrap())).unwrap(); let m = item.as_media().unwrap(); assert!(m.info.video.is_none()); - assert_eq!(m.info.audio.as_ref().unwrap().channels, 2); + assert_eq!(m.info.audio().unwrap().channels, 2); let src_dur = s.project.item(forest).unwrap().duration(); assert!((m.info.duration - src_dur).0.abs() < TICKS_PER_SECOND / 100, "same length"); assert_eq!(s.project.root.parent_of(item.id), s.project.root.parent_of(forest)); diff --git a/crates/engine/src/commands.rs b/crates/engine/src/commands.rs index 0c4c9631..72333075 100644 --- a/crates/engine/src/commands.rs +++ b/crates/engine/src/commands.rs @@ -358,7 +358,7 @@ pub(crate) fn default_seq_settings_for(info: &filmcraft_media::MediaInfo) -> Seq st.frame_rate = v.frame_rate; st.preset = format!("{}x{} {}", v.width, v.height, v.frame_rate.label()); } - if let Some(a) = &info.audio { + if let Some(a) = info.audio() { st.sample_rate = a.sample_rate.max(8000); } st @@ -490,6 +490,7 @@ fn set_matte_color(s: &mut Session, p: &Value) -> Result { #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct AudioPlacementSpec { pub track_offset: usize, + pub audio_stream: usize, pub source_channels: Option>, } @@ -497,10 +498,22 @@ pub(crate) struct AudioPlacementSpec { /// per clip): always at least one (on the destination track), then one per further entry on the /// tracks below. The count is capped at [`crate::sequence_tools::MAX_TRACKS`] (the map comes from /// project files and scripts: never trusted). -pub(crate) fn audio_placement_specs(clips: &[Vec]) -> Vec { - let mut specs = vec![AudioPlacementSpec { track_offset: 0, source_channels: None }]; +pub(crate) fn audio_placement_specs(clips: &[Vec], streams: usize) -> Vec { + let mut specs = vec![AudioPlacementSpec { track_offset: 0, audio_stream: 0, source_channels: None }]; let extra = clips.iter().skip(1).take(crate::sequence_tools::MAX_TRACKS.saturating_sub(1)); - specs.extend(extra.enumerate().map(|(k, chans)| AudioPlacementSpec { track_offset: k.saturating_add(1), source_channels: Some(chans.clone()) })); + specs.extend(extra.enumerate().map(|(k, chans)| AudioPlacementSpec { + track_offset: k.saturating_add(1), + audio_stream: 0, + source_channels: Some(chans.clone()), + })); + // Interpret Footage's channel map describes stream 0; further container streams keep their + // own channel layouts and get one clip each, after any channel-map clips. + for stream in 1..streams.min(filmcraft_media::MAX_AUDIO_STREAMS) { + if specs.len() >= crate::sequence_tools::MAX_TRACKS { + break; + } + specs.push(AudioPlacementSpec { track_offset: specs.len(), audio_stream: stream, source_channels: None }); + } specs } @@ -592,7 +605,8 @@ pub(crate) fn place_item( // Modify ▸ Audio Channels with several audio clips: one per clip, on the tracks below let clips: Vec> = p.item(item).and_then(|i| i.as_media()).and_then(|m| m.interpret.audio_channels.as_ref()).map(|m| m.clips.clone()).unwrap_or_default(); - let specs = audio_placement_specs(&clips); + let streams = p.resolve_media(item).map_or(1, |(_, m, _)| m.info.audio_streams.len()); + let specs = audio_placement_specs(&clips, streams); let first = p.sequence(seq_id).and_then(|q| q.audio_tracks.iter().position(|t| t.id == adest)).unwrap_or(0); // too few audio tracks below the destination: grow the sequence from the destination track let tracks = ensure_audio_tracks(p, seq_id, first, specs.len(), label)?; @@ -606,9 +620,13 @@ pub(crate) fn place_item( }, }; let mut b = a.clone(); + b.audio_stream = spec.audio_stream; if spec.track_offset > 0 { b.id = ClipId(p.alloc_id()); } + if spec.audio_stream > 0 { + b.source_channels.clear(); + } if let Some(chans) = spec.source_channels { b.source_channels = chans; } diff --git a/crates/engine/src/media_browser.rs b/crates/engine/src/media_browser.rs index 60cc6c24..53062260 100644 --- a/crates/engine/src/media_browser.rs +++ b/crates/engine/src/media_browser.rs @@ -516,9 +516,9 @@ pub fn column_text(e: &Entry, column: &str, probe: Option<&MediaInfo>) -> String }) .unwrap_or_default(), "Video Info" => probe.and_then(|i| i.video.as_ref()).map(|v| format!("{} x {}", v.width, v.height)).unwrap_or_default(), - "Audio Info" => probe.and_then(|i| i.audio.as_ref()).map(|a| format!("{} Hz - {} ch", a.sample_rate, a.channels)).unwrap_or_default(), + "Audio Info" => probe.and_then(|i| i.audio()).map(|a| format!("{} Hz - {} ch", a.sample_rate, a.channels)).unwrap_or_default(), "Video Codec" => probe.and_then(|i| i.video.as_ref()).map(|v| v.codec.clone()).unwrap_or_default(), - "Audio Codec" => probe.and_then(|i| i.audio.as_ref()).map(|a| a.codec.clone()).unwrap_or_default(), + "Audio Codec" => probe.and_then(|i| i.audio()).map(|a| a.codec.clone()).unwrap_or_default(), _ => String::new(), } } diff --git a/crates/engine/src/media_browser_tests.rs b/crates/engine/src/media_browser_tests.rs index df956ea0..972220b3 100644 --- a/crates/engine/src/media_browser_tests.rs +++ b/crates/engine/src/media_browser_tests.rs @@ -269,7 +269,7 @@ fn probing_large_unstreamable_files_does_not_read_them_whole() { crate::media_test_util::make_movie(&mov, filmcraft_media::DemoScene::OceanSunset, 64, 36, 24); let mut s = Session::default(); assert!(media_browser::probe(&mut s, &avi.to_string_lossy()).is_none()); - assert_eq!(media_browser::probe(&mut s, &wav.to_string_lossy()).unwrap().audio.unwrap().channels, 2); + assert_eq!(media_browser::probe(&mut s, &wav.to_string_lossy()).unwrap().audio().unwrap().channels, 2); assert!(media_browser::probe(&mut s, &mov.to_string_lossy()).unwrap().video.is_some()); // importing is unchanged: it still opens files through the full path let r = s.execute("file.import", json!({"paths": [wav.to_string_lossy()]})).unwrap(); diff --git a/crates/engine/src/media_pool.rs b/crates/engine/src/media_pool.rs index cf468bd2..125a9988 100644 --- a/crates/engine/src/media_pool.rs +++ b/crates/engine/src/media_pool.rs @@ -279,11 +279,15 @@ impl MediaPool { return Some(s.clone()); } match self.open_file(path, services) { - Ok(s) => { + Ok(s) if s.info().audio_streams.len() >= m.info.audio_streams.len() => { let src: SharedSource = Arc::new(ProxySource { proxy: s, info: m.info.clone() }); self.proxies.write().unwrap_or_else(|e| e.into_inner()).insert(item, (key, src.clone())); Some(src) } + Ok(_) => { + log::warn!("proxy has fewer audio streams than the original: {path}; using full-resolution media"); + None + } Err(e) => { // a missing proxy falls back to the full-resolution media log::warn!("proxy offline: {path}: {e}"); @@ -357,7 +361,7 @@ impl MediaSource for SlateSource { Ok(Arc::new((*f).clone().with_pts(req.time))) } fn audio(&self, _start: i64, frames: usize, sample_rate: u32) -> filmcraft_media::Result { - Ok(AudioBuffer::silence(sample_rate, self.info.audio.as_ref().map_or(2, |a| a.channels as usize), frames)) + Ok(AudioBuffer::silence(sample_rate, self.info.audio().map_or(2, |a| a.channels as usize), frames)) } } @@ -397,6 +401,9 @@ impl MediaSource for ProxySource { fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> filmcraft_media::Result { self.proxy.audio(start, frames, sample_rate) } + fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> filmcraft_media::Result { + self.proxy.audio_stream(stream, start, frames, sample_rate) + } } #[cfg(test)] diff --git a/crates/engine/src/panels.rs b/crates/engine/src/panels.rs index 0c1b994f..45da2e21 100644 --- a/crates/engine/src/panels.rs +++ b/crates/engine/src/panels.rs @@ -224,7 +224,7 @@ pub fn metadata_fields(p: &Project, id: ItemId) -> Vec { let cs = media.and_then(|m| m.interpret.color_space).unwrap_or(detected); add("Clip", "Color Space", cs.label().to_string(), false); } - if let Some(a) = media.and_then(|m| m.info.audio.as_ref()) { + if let Some(a) = media.and_then(|m| m.info.audio()) { add("Clip", "Audio Info", format!("{} Hz - {} ch", a.sample_rate, a.channels), false); add("Clip", "Audio Codec", a.codec.clone(), false); } diff --git a/crates/engine/src/project_manager.rs b/crates/engine/src/project_manager.rs index 756c3b4e..2e5f7501 100644 --- a/crates/engine/src/project_manager.rs +++ b/crates/engine/src/project_manager.rs @@ -311,7 +311,7 @@ fn execute( fn estimate_bytes(m: &filmcraft_project::MediaClip, dur: Tick, pr: &Preset) -> u64 { let secs = dur.seconds().max(0.0); - let audio = m.info.audio.as_ref().map_or(0.0, |a| a.sample_rate as f64 * 2.0 * 2.0 * secs); + let audio = m.info.audio().map_or(0.0, |a| a.sample_rate as f64 * 2.0 * 2.0 * secs); let Some(v) = &m.info.video else { return audio as u64 }; let fps = v.frame_rate.num as f64 / v.frame_rate.den.max(1) as f64; let px = (v.width * v.height) as f64 * (pr.scale * pr.scale) as f64; diff --git a/crates/engine/src/project_panel.rs b/crates/engine/src/project_panel.rs index ea18581d..84659296 100644 --- a/crates/engine/src/project_panel.rs +++ b/crates/engine/src/project_panel.rs @@ -384,7 +384,7 @@ pub fn cell(p: &Project, it: &ProjectItem, column: &str) -> (String, SortKey) { "Video Codec" => text(media.and_then(|m| m.info.video.as_ref()).map(|v| v.codec.clone()).unwrap_or_default()), "Audio Info" => text( media - .and_then(|m| m.info.audio.as_ref()) + .and_then(|m| m.info.audio()) .map(|a| { format!( "{} Hz - {}", @@ -400,7 +400,7 @@ pub fn cell(p: &Project, it: &ProjectItem, column: &str) -> (String, SortKey) { }) .unwrap_or_default(), ), - "Audio Codec" => text(media.and_then(|m| m.info.audio.as_ref()).map(|a| a.codec.clone()).unwrap_or_default()), + "Audio Codec" => text(media.and_then(|m| m.info.audio()).map(|a| a.codec.clone()).unwrap_or_default()), "Color Space" => text(match (media.and_then(|m| m.info.video.as_ref().map(|v| (m, v))), &it.kind) { (Some((m, v)), _) => m.interpret.color_space.unwrap_or_else(|| filmcraft_color::ColorSpace::from_info(&v.color)).label().to_string(), (None, ItemKind::Sequence(q)) => q.settings.color.working.label().to_string(), diff --git a/crates/engine/src/project_tools.rs b/crates/engine/src/project_tools.rs index 4bda2d66..0dce9fdc 100644 --- a/crates/engine/src/project_tools.rs +++ b/crates/engine/src/project_tools.rs @@ -805,7 +805,7 @@ pub fn info_json(name: &str, path: Option<&str>, info: &filmcraft_media::MediaIn "bitrate": vs.bitrate, "color": format!("{:?} / {:?} / {:?}", vs.color.primaries, vs.color.transfer, vs.color.matrix), }); } - if let Some(a) = &info.audio { + if let Some(a) = info.audio() { v["audio"] = json!({"codec": a.codec, "sampleRate": a.sample_rate, "channels": a.channels, "bitsPerSample": a.bits_per_sample}); } v @@ -1002,7 +1002,7 @@ fn edit_offline(s: &mut Session, p: &Value) -> Result { fn source_settings(s: &mut Session, p: &Value) -> Result { let id = targets(s, p).into_iter().find(|i| media_of(&s.project, *i).is_some()).ok_or_else(|| EngineError::Other("select a clip".into()))?; let (_, m) = media_of(&s.project, id).ok_or_else(|| EngineError::Other("select a clip".into()))?; - let codec = m.info.video.as_ref().map(|v| v.codec.clone()).or_else(|| m.info.audio.as_ref().map(|a| a.codec.clone())).unwrap_or_default(); + let codec = m.info.video.as_ref().map(|v| v.codec.clone()).or_else(|| m.info.audio().map(|a| a.codec.clone())).unwrap_or_default(); Ok(json!({ "item": id.0, "codec": codec, diff --git a/crates/engine/src/proxies.rs b/crates/engine/src/proxies.rs index 55d131ee..045c9789 100644 --- a/crates/engine/src/proxies.rs +++ b/crates/engine/src/proxies.rs @@ -105,14 +105,14 @@ pub fn transcode( st.height = v.height; } st.frame_rate = rate; - st.sample_rate = info.audio.as_ref().map_or(48_000, |a| a.sample_rate.max(8000)); + st.sample_rate = info.audio().map_or(48_000, |a| a.sample_rate.max(8000)); let mut p = Project::new("transcode"); let mut c = clip.clone(); c.offline = false; c.proxy = None; let item = p.add_item(name, filmcraft_project::Label::Iris, ItemKind::Media(c), None); let has_v = info.video.is_some(); - let has_a = info.audio.is_some(); + let has_a = info.has_audio(); let seq = p.new_sequence("transcode", st, usize::from(has_v), usize::from(has_a), None); let range = range.unwrap_or(TimeRange { start: Tick::ZERO, duration: info.duration }); let mut placed = Vec::new(); diff --git a/crates/engine/src/relink.rs b/crates/engine/src/relink.rs index 7adf81c6..9f5c23e6 100644 --- a/crates/engine/src/relink.rs +++ b/crates/engine/src/relink.rs @@ -263,7 +263,7 @@ pub fn check_candidate(s: &Session, item: ItemId, path: &str, o: &MatchOptions) if info.video.is_some() != m.info.video.is_some() { c.problems.push("video stream presence differs".into()); } - if let (Some(a), Some(b)) = (&info.audio, &m.info.audio) + if let (Some(a), Some(b)) = (info.audio(), m.info.audio()) && a.channels != b.channels { c.problems.push(format!("audio channels differ ({} vs {})", a.channels, b.channels)); diff --git a/crates/export/src/surround_tests.rs b/crates/export/src/surround_tests.rs index bb7068ec..890e9131 100644 --- a/crates/export/src/surround_tests.rs +++ b/crates/export/src/surround_tests.rs @@ -46,7 +46,7 @@ fn surround() -> (Arc, ItemId, SourceMap) { kind: MediaKind::AudioOnly, duration: Tick(2 * TICKS_PER_SECOND), video: None, - audio: Some(AudioStreamInfo { sample_rate: SR, channels: 6, codec: "test".into(), bits_per_sample: Some(32) }), + audio_streams: vec![AudioStreamInfo { sample_rate: SR, channels: 6, codec: "test".into(), bits_per_sample: Some(32) }], container: "test".into(), start_timecode: None, file_size: None, @@ -185,7 +185,7 @@ fn aac_51_in_mp4_keeps_every_channel_in_place() { // our own demuxer sees an AAC track with channel configuration 6 let bytes: Arc<[u8]> = std::fs::read(&path).unwrap().into(); let src = filmcraft_codecs::open_bytes("s51.mp4", bytes).unwrap(); - assert_eq!(src.info().audio.as_ref().unwrap().channels, 6); + assert_eq!(src.info().audio().unwrap().channels, 6); if let Some((n, layout)) = probe(&path) { assert_eq!(n, 6); assert!(is_51(&layout), "{layout}"); diff --git a/crates/format/Cargo.toml b/crates/format/Cargo.toml index cb8140ce..5e48f640 100644 --- a/crates/format/Cargo.toml +++ b/crates/format/Cargo.toml @@ -11,10 +11,10 @@ workspace = true [dependencies] filmcraft-project = { workspace = true } +filmcraft-media = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } [dev-dependencies] filmcraft-time = { workspace = true } -filmcraft-media = { workspace = true } diff --git a/crates/format/src/lib.rs b/crates/format/src/lib.rs index f71fdce7..d4bd2d17 100644 --- a/crates/format/src/lib.rs +++ b/crates/format/src/lib.rs @@ -33,6 +33,8 @@ //! - **v12** (M10.7): graphics templates, rolls / crawls, responsive design (pins, intro / outro), //! per-character text styles (`TrackItem::graphic`, `EffectInstance::layer`) and source graphics //! (`Project::source_graphics`). No-op step; older builds would drop these fields when saving. +//! - **v13**: all container audio streams, and per-timeline-clip stream selection. Old single-stream +//! `audio` objects read as stream zero; older builds must refuse new multi-stream projects. #![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] @@ -53,7 +55,8 @@ pub type Migration = fn(Value) -> Result; /// `MIGRATIONS[i]` upgrades schema `i + 1` to `i + 2`. Append one function per schema bump; never /// edit a shipped one. -pub const MIGRATIONS: &[Migration] = &[v1_to_v2, v2_to_v3, v3_to_v4, v4_to_v5, v5_to_v6, v6_to_v7, v7_to_v8, v8_to_v9, v9_to_v10, v10_to_v11, v11_to_v12]; +pub const MIGRATIONS: &[Migration] = + &[v1_to_v2, v2_to_v3, v3_to_v4, v4_to_v5, v5_to_v6, v6_to_v7, v7_to_v8, v8_to_v9, v9_to_v10, v10_to_v11, v11_to_v12, v12_to_v13]; /// The schema version this build writes (and the newest it reads). pub const SCHEMA_VERSION: u32 = MIGRATIONS.len() as u32 + 1; @@ -209,6 +212,20 @@ pub fn decode(bytes: &[u8]) -> Result { /// Refuse only values that would overflow or exhaust memory later (`Sequence::check_bounds`); /// structural rules such as overlaps are left to the editor, so projects that open today still open. fn validate_loaded(loaded: Loaded) -> Result { + for item in loaded.project.items.values() { + if let filmcraft_project::ItemKind::Media(m) = &item.kind + && m.info.audio_streams.len() > filmcraft_media::MAX_AUDIO_STREAMS + { + return Err(FormatError::Corrupt(format!("media `{}`: too many audio streams", item.name))); + } + if let filmcraft_project::ItemKind::Sequence(sequence) = &item.kind { + for track in &sequence.audio_tracks { + if track.items.iter().any(|clip| clip.audio_stream >= filmcraft_media::MAX_AUDIO_STREAMS) { + return Err(FormatError::Corrupt(format!("sequence `{}`: audio stream index out of range", item.name))); + } + } + } + } for item in loaded.project.sequences() { if let filmcraft_project::ItemKind::Sequence(sequence) = &item.kind { sequence.check_bounds().map_err(|e| FormatError::Corrupt(format!("sequence `{}`: {e}", item.name)))?; @@ -315,13 +332,19 @@ fn v11_to_v12(doc: Value) -> Result { Ok(doc) } +/// v12 → v13: container audio streams and per-clip stream selection. `MediaInfo` reads the old +/// `audio` field as stream zero; `TrackItem::audio_stream` defaults to zero. +fn v12_to_v13(doc: Value) -> Result { + Ok(doc) +} + #[cfg(test)] mod tests { use super::*; #[test] fn schema_version_matches_table() { - assert_eq!(SCHEMA_VERSION, 12); + assert_eq!(SCHEMA_VERSION, 13); } #[test] diff --git a/crates/format/tests/fixtures.rs b/crates/format/tests/fixtures.rs index 1026e8c7..644a60e5 100644 --- a/crates/format/tests/fixtures.rs +++ b/crates/format/tests/fixtures.rs @@ -52,6 +52,31 @@ fn v1_edit_loads_with_defaults_for_later_fields() { seq.check().unwrap(); } +#[test] +fn legacy_audio_field_becomes_primary_stream_and_clip_defaults_to_zero() { + let p = decode(&fixture("v1-edit.fcproj")).unwrap().project; + let media = p.item(ItemId(5)).unwrap().as_media().unwrap(); + assert_eq!(media.info.audio_streams.len(), 1); + assert!(media.info.has_audio()); + let clip = &p.sequence(ItemId(21)).unwrap().audio_tracks[0].items[0]; + assert_eq!(clip.audio_stream, 0); + let again = decode(&encode(&p, false)).unwrap(); + assert_eq!(again.project.item(ItemId(5)).unwrap().as_media().unwrap().info.audio_streams, media.info.audio_streams); +} + +#[test] +fn hostile_audio_stream_counts_and_indices_are_rejected() { + let mut p = decode(&fixture("v1-edit.fcproj")).unwrap().project; + let seq = ItemId(21); + p.sequence_mut(seq).unwrap().audio_tracks[0].items[0].audio_stream = filmcraft_media::MAX_AUDIO_STREAMS; + assert!(matches!(decode(&encode(&p, false)), Err(FormatError::Corrupt(_)))); + p.sequence_mut(seq).unwrap().audio_tracks[0].items[0].audio_stream = 0; + let media = p.item_mut(ItemId(5)).unwrap().as_media_mut().unwrap(); + let stream = media.info.audio_streams[0].clone(); + media.info.audio_streams.resize(filmcraft_media::MAX_AUDIO_STREAMS + 1, stream); + assert!(matches!(decode(&encode(&p, false)), Err(FormatError::Corrupt(_)))); +} + /// Schema 8 (before M3.10): no time interpolation / Hold Filters / Field Options / source channels /// on clips, no audio channel map, subclips without Restrict Trims. #[test] diff --git a/crates/interchange/src/aaf/tests.rs b/crates/interchange/src/aaf/tests.rs index 4cfbf62e..c050b55c 100644 --- a/crates/interchange/src/aaf/tests.rs +++ b/crates/interchange/src/aaf/tests.rs @@ -28,7 +28,7 @@ fn project() -> (Project, ItemId) { bitrate: None, hdr: None, }), - audio: Some(filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: String::new(), bits_per_sample: Some(24) }), + audio_streams: vec![filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: String::new(), bits_per_sample: Some(24) }], container: String::new(), start_timecode: Some(86_400), file_size: None, diff --git a/crates/interchange/src/ale.rs b/crates/interchange/src/ale.rs index b833d00c..55bb1784 100644 --- a/crates/interchange/src/ale.rs +++ b/crates/interchange/src/ale.rs @@ -87,7 +87,7 @@ pub fn export_items(p: &Project, items: &[ItemId]) -> AleDoc { let rows_src: Vec = items.iter().copied().filter(|i| media_of(*i).is_some()).collect(); let first_video = rows_src.iter().find_map(|i| media_of(*i).and_then(|(m, ..)| m.info.video.as_ref().map(|v| (v.frame_rate, v.height)))); let (rate, height) = first_video.unwrap_or((FrameRate::FPS_24, 1080)); - let sr = rows_src.iter().find_map(|i| media_of(*i).and_then(|(m, ..)| m.info.audio.as_ref().map(|a| a.sample_rate))).unwrap_or(48_000); + let sr = rows_src.iter().find_map(|i| media_of(*i).and_then(|(m, ..)| m.info.audio().map(|a| a.sample_rate))).unwrap_or(48_000); let heading = vec![ ("FIELD_DELIM".to_string(), "TABS".to_string()), ("VIDEO_FORMAT".to_string(), video_format(height).to_string()), @@ -103,7 +103,7 @@ pub fn export_items(p: &Project, items: &[ItemId]) -> AleDoc { if m.info.video.is_some() { tracks.push('V'); } - for c in 1..=m.info.audio.as_ref().map_or(0, |a| a.channels.min(16)) { + for c in 1..=m.info.audio().map_or(0, |a| a.channels.min(16)) { tracks.push_str(&format!("A{c}")); } // media start timecode (frames at the media's own rate) + the subclip offset @@ -223,7 +223,10 @@ mod tests { bitrate: None, hdr: None, }), - audio: (audio > 0).then(|| AudioStreamInfo { sample_rate: 48_000, channels: audio, codec: "aac".into(), bits_per_sample: None }), + audio_streams: (audio > 0) + .then(|| AudioStreamInfo { sample_rate: 48_000, channels: audio, codec: "aac".into(), bits_per_sample: None }) + .into_iter() + .collect(), container: "mp4".into(), start_timecode: tc, file_size: None, diff --git a/crates/interchange/src/common.rs b/crates/interchange/src/common.rs index 1988ffa7..5527e06e 100644 --- a/crates/interchange/src/common.rs +++ b/crates/interchange/src/common.rs @@ -251,7 +251,11 @@ pub(crate) fn media_info(name: &str, spec: &MediaSpec) -> MediaInfo { bitrate: None, hdr: None, }), - audio: spec.audio.map(|(sr, ch)| AudioStreamInfo { sample_rate: sr, channels: ch, codec: String::new(), bits_per_sample: None }), + audio_streams: spec + .audio + .map(|(sr, ch)| AudioStreamInfo { sample_rate: sr, channels: ch, codec: String::new(), bits_per_sample: None }) + .into_iter() + .collect(), container: String::new(), start_timecode: spec.start_tc, file_size: None, @@ -443,6 +447,7 @@ impl Builder { hold_filters: false, field_options: None, source_channels: Vec::new(), + audio_stream: 0, graphic: None, } } diff --git a/crates/interchange/src/comp.rs b/crates/interchange/src/comp.rs index 839e4342..bf9b2fba 100644 --- a/crates/interchange/src/comp.rs +++ b/crates/interchange/src/comp.rs @@ -443,7 +443,7 @@ impl Exporter<'_> { return Some(i); } let video = m.info.video.as_ref(); - let audio = m.info.audio.as_ref(); + let audio = m.info.audio(); let still = m.info.kind == MediaKind::Still; match ckind { CKind::Picture if video.is_none() && !still => return None, diff --git a/crates/interchange/src/essence.rs b/crates/interchange/src/essence.rs index c0c20eb0..45a56878 100644 --- a/crates/interchange/src/essence.rs +++ b/crates/interchange/src/essence.rs @@ -163,7 +163,7 @@ fn collect_needs( let (mut start, mut end) = ((lo - opts.handles).max(Tick::ZERO), hi + opts.handles); match project.item(item).map(|i| &i.kind) { Some(ItemKind::Media(m)) => { - let Some(a) = m.info.audio.as_ref() else { continue }; + let Some(a) = m.info.audio() else { continue }; let path = match &m.media { MediaRef::File { path } => Some(path.clone()), MediaRef::Generator(_) => None, diff --git a/crates/interchange/src/fcp7.rs b/crates/interchange/src/fcp7.rs index 047b013f..a24f86a4 100644 --- a/crates/interchange/src/fcp7.rs +++ b/crates/interchange/src/fcp7.rs @@ -1055,7 +1055,7 @@ impl Exp<'_, '_> { self.w.close(); self.w.close(); } - if let Some(a) = &m.info.audio { + if let Some(a) = m.info.audio() { self.w.open("audio", &[]); self.w.open("samplecharacteristics", &[]); self.w.text("depth", a.bits_per_sample.unwrap_or(16)); diff --git a/crates/interchange/src/fcpxml.rs b/crates/interchange/src/fcpxml.rs index 623afb08..faf6a77c 100644 --- a/crates/interchange/src/fcpxml.rs +++ b/crates/interchange/src/fcpxml.rs @@ -730,12 +730,12 @@ impl Exp<'_, '_> { fmt_time(start, mrate), fmt_time(m.info.duration, mrate), m.info.video.is_some() as u8, - m.info.audio.is_some() as u8 + m.info.has_audio() as u8 ); if let Some(f) = fmt { a.push_str(&format!(" format=\"{f}\"")); } - if let Some(au) = &m.info.audio { + if let Some(au) = m.info.audio() { a.push_str(&format!(" audioSources=\"1\" audioChannels=\"{}\" audioRate=\"{}\"", au.channels, au.sample_rate)); } let path = item_path(self.p, item).unwrap_or(""); @@ -986,7 +986,7 @@ impl Exp<'_, '_> { }; let retimed = it.speed != 1.0 || it.reverse || it.frame_hold.is_some(); let local = origin + it.source_in; - let has_audio = item_media(self.p, base).is_some_and(|m| m.info.audio.is_some()) || (is_seq && audio.is_some()); + let has_audio = item_media(self.p, base).is_some_and(|m| m.info.has_audio()) || (is_seq && audio.is_some()); let has_video = item_media(self.p, base).is_none_or(|m| m.info.video.is_some()); let src_enable = match (kind, audio.is_some()) { (TrackKind::Video, true) => "", diff --git a/crates/interchange/src/omf/tests.rs b/crates/interchange/src/omf/tests.rs index fb47e58f..b291d2f9 100644 --- a/crates/interchange/src/omf/tests.rs +++ b/crates/interchange/src/omf/tests.rs @@ -17,7 +17,7 @@ fn project() -> (Project, ItemId) { kind: filmcraft_media::MediaKind::AudioOnly, duration: FrameRate::FPS_25.tick_of(2500), video: None, - audio: Some(filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: String::new(), bits_per_sample: Some(16) }), + audio_streams: vec![filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: String::new(), bits_per_sample: Some(16) }], container: String::new(), start_timecode: None, file_size: None, diff --git a/crates/interchange/tests/aaf_omf.rs b/crates/interchange/tests/aaf_omf.rs index 6b6ac868..3e16492f 100644 --- a/crates/interchange/tests/aaf_omf.rs +++ b/crates/interchange/tests/aaf_omf.rs @@ -132,7 +132,7 @@ fn aaf_round_trip_preserves_the_edit() { .unwrap(); let ma = a.as_media().unwrap(); assert_eq!(ma.info.start_timecode, Some(rate.timecode_base() * 3600 + 12)); - assert!(ma.info.video.is_some() && ma.info.audio.is_some()); + assert!(ma.info.video.is_some() && ma.info.has_audio()); assert_eq!(ma.markers.len(), 1); assert_eq!((ma.markers[0].name.as_str(), ma.markers[0].comment.as_str()), ("slate", "take 3")); let mus = imp.project.items.values().find(|i| i.name == "music.wav").unwrap(); @@ -373,7 +373,7 @@ fn omf_separate_files_and_breakout() { let k = media_key(&imp.project, c.item); assert!(k.starts_with("/omf/media/") && k.ends_with(".aif"), "{k}"); let m = imp.project.item(c.item).unwrap().as_media().unwrap(); - assert_eq!(m.info.audio.as_ref().map(|a| (a.channels, a.sample_rate)), Some((1, 48_000))); + assert_eq!(m.info.audio().map(|a| (a.channels, a.sample_rate)), Some((1, 48_000))); } } } diff --git a/crates/interchange/tests/support/mod.rs b/crates/interchange/tests/support/mod.rs index 52bd5266..4c9835c8 100644 --- a/crates/interchange/tests/support/mod.rs +++ b/crates/interchange/tests/support/mod.rs @@ -25,7 +25,10 @@ pub fn media_info(name: &str, video: bool, audio: bool, rate: FrameRate, secs: i bitrate: None, hdr: None, }), - audio: audio.then(|| filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "aac".into(), bits_per_sample: None }), + audio_streams: audio + .then(|| filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "aac".into(), bits_per_sample: None }) + .into_iter() + .collect(), container: "mp4".into(), start_timecode: None, file_size: None, diff --git a/crates/media/src/generators.rs b/crates/media/src/generators.rs index bbe9efbe..97d42d1d 100644 --- a/crates/media/src/generators.rs +++ b/crates/media/src/generators.rs @@ -108,7 +108,7 @@ impl GeneratorSource { kind: if matches!(generator, Generator::Demo(_)) { MediaKind::Movie } else { MediaKind::Synthetic }, duration, video, - audio, + audio_streams: audio.into_iter().collect(), container: if matches!(generator, Generator::Demo(_)) { "Synthetic MPEG-4".into() } else { "Synthetic".into() }, start_timecode: None, file_size: None, diff --git a/crates/media/src/lib.rs b/crates/media/src/lib.rs index 3dc3cdd5..6b5ff1b1 100644 --- a/crates/media/src/lib.rs +++ b/crates/media/src/lib.rs @@ -77,6 +77,9 @@ pub struct VideoStreamInfo { pub hdr: Option, } +/// Most audio streams a source reports (the count comes from the file: never trusted). +pub const MAX_AUDIO_STREAMS: usize = 64; + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct AudioStreamInfo { pub sample_rate: u32, @@ -92,13 +95,32 @@ pub struct MediaInfo { /// Media duration. Stills report a default duration (the still-image default preference). pub duration: Tick, pub video: Option, - pub audio: Option, + /// Every audio stream the container carries, in file order. Stream 0 is the one + /// [`MediaSource::audio`] reads; the others are read with [`MediaSource::audio_stream`]. + /// Project files written before multi-stream support hold a single `audio` object (or + /// `null`): both still load. + #[serde(default, alias = "audio", deserialize_with = "deserialize_audio_streams")] + pub audio_streams: Vec, pub container: String, /// Timecode of the first frame (in frames at `video.frame_rate`), if the file carries one. pub start_timecode: Option, pub file_size: Option, } +/// Reads `audio_streams` from a list, or from the single optional stream older project files wrote. +fn deserialize_audio_streams<'de, D: serde::Deserializer<'de>>(d: D) -> std::result::Result, D::Error> { + #[derive(Deserialize)] + #[serde(untagged)] + enum Streams { + Many(Vec), + One(Option), + } + Ok(match Streams::deserialize(d)? { + Streams::Many(v) => v, + Streams::One(o) => o.into_iter().collect(), + }) +} + impl MediaInfo { pub fn frame_rate(&self) -> FrameRate { self.video.as_ref().map(|v| v.frame_rate).unwrap_or_default() @@ -107,7 +129,11 @@ impl MediaInfo { self.video.is_some() } pub fn has_audio(&self) -> bool { - self.audio.is_some() + !self.audio_streams.is_empty() + } + /// The primary audio stream (stream 0), the one [`MediaSource::audio`] reads. + pub fn audio(&self) -> Option<&AudioStreamInfo> { + self.audio_streams.first() } } @@ -132,6 +158,12 @@ pub trait MediaSource: Send + Sync { fn video_frame(&self, req: FrameRequest) -> Result>; /// `frames` audio frames starting at sample index `start` (at `sample_rate`, media time). fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> Result; + /// Like [`audio`](Self::audio) for audio stream `stream` (an index into + /// [`MediaInfo::audio_streams`]). Sources with a single audio stream need not override this: + /// stream 0 is `audio`, any other index is [`MediaError::NoStream`]. + fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> Result { + if stream == 0 { self.audio(start, frames, sample_rate) } else { Err(MediaError::NoStream("audio")) } + } } pub type SharedSource = Arc; @@ -195,6 +227,6 @@ impl MediaSource for OfflineSource { Ok(Arc::new(VideoFrame::rgba8(w, h, px).with_pts(req.time))) } fn audio(&self, _start: i64, frames: usize, sample_rate: u32) -> Result { - Ok(AudioBuffer::silence(sample_rate, self.info.audio.as_ref().map_or(2, |a| a.channels as usize), frames)) + Ok(AudioBuffer::silence(sample_rate, self.info.audio().map_or(2, |a| a.channels as usize), frames)) } } diff --git a/crates/media/src/reader.rs b/crates/media/src/reader.rs index d4f87ab8..2aeb47bf 100644 --- a/crates/media/src/reader.rs +++ b/crates/media/src/reader.rs @@ -189,7 +189,7 @@ mod tests { let small = wav.len() as u64; let r = Arc::new(Counting(MemReader(Arc::from(wav.clone())), Default::default())); let src = open_reader_within("a.wav", r.clone(), &[], &[], small).unwrap(); - assert_eq!(src.info().audio.as_ref().unwrap().channels, 2); + assert_eq!(src.info().audio().unwrap().channels, 2); // the same file over the limit: refused after reading the sniffing head only let r = Arc::new(Counting(MemReader(Arc::from(wav.clone())), Default::default())); let e = open_reader_within("a.wav", r.clone(), &[], &[], small - 1).err().unwrap(); diff --git a/crates/media/src/sequence.rs b/crates/media/src/sequence.rs index 281867ce..9521cba4 100644 --- a/crates/media/src/sequence.rs +++ b/crates/media/src/sequence.rs @@ -147,7 +147,7 @@ impl ImageSequenceSource { bitrate: None, hdr: None, }), - audio: None, + audio_streams: Vec::new(), container: "Image Sequence".into(), start_timecode: None, file_size: None, diff --git a/crates/media/src/still.rs b/crates/media/src/still.rs index 199395a6..4a1cb53a 100644 --- a/crates/media/src/still.rs +++ b/crates/media/src/still.rs @@ -52,7 +52,7 @@ impl StillSource { bitrate: None, hdr: None, }), - audio: None, + audio_streams: Vec::new(), container: "Image".into(), start_timecode: None, file_size: Some(file_size), diff --git a/crates/media/src/wav.rs b/crates/media/src/wav.rs index a7d9acb0..6b63bf4a 100644 --- a/crates/media/src/wav.rs +++ b/crates/media/src/wav.rs @@ -84,12 +84,12 @@ impl WavSource { kind: MediaKind::AudioOnly, duration: Tick::from_units(frames as i64, rate as i64), video: None, - audio: Some(AudioStreamInfo { + audio_streams: vec![AudioStreamInfo { sample_rate: rate, channels: ch as u32, codec: if float { "PCM float".into() } else { "PCM".into() }, bits_per_sample: Some(bits as u32), - }), + }], container: if time_reference.is_some() { "Broadcast WAV".into() } else { "WAV".into() }, start_timecode: None, file_size: Some(b.len() as u64), @@ -201,7 +201,7 @@ mod tests { let s: Vec = (0..200).map(|i| ((i as f32) * 0.1).sin() * 0.5).collect(); let bytes = write_wav16(&s, 2, 48000); let src = WavSource::parse("t.wav", bytes.into()).unwrap(); - assert_eq!(src.info().audio.as_ref().unwrap().channels, 2); + assert_eq!(src.info().audio().unwrap().channels, 2); let a = src.audio(0, 100, 48000).unwrap(); assert!((a.channels[0][3] - s[6]).abs() < 1e-4); assert!((a.channels[1][3] - s[7]).abs() < 1e-4); diff --git a/crates/project/src/essential.rs b/crates/project/src/essential.rs index e23132e7..cd7bf31b 100644 --- a/crates/project/src/essential.rs +++ b/crates/project/src/essential.rs @@ -783,6 +783,7 @@ mod tests { hold_filters: false, field_options: None, source_channels: Vec::new(), + audio_stream: 0, graphic: None, } } diff --git a/crates/project/src/find.rs b/crates/project/src/find.rs index e9584685..a25d3d18 100644 --- a/crates/project/src/find.rs +++ b/crates/project/src/find.rs @@ -133,7 +133,7 @@ pub fn column_text(p: &Project, it: &ProjectItem, column: &str) -> String { .and_then(|m| m.info.video.as_ref()) .map(|v| format!("{} x {} ({:.4}) {}", v.width, v.height, v.par.0 as f64 / v.par.1.max(1) as f64, v.codec)) .unwrap_or_default(), - "audio info" => media.and_then(|m| m.info.audio.as_ref()).map(|a| format!("{} Hz - {} ch {}", a.sample_rate, a.channels, a.codec)).unwrap_or_default(), + "audio info" => media.and_then(|m| m.info.audio()).map(|a| format!("{} Hz - {} ch {}", a.sample_rate, a.channels, a.codec)).unwrap_or_default(), "file path" | "path" => match media.map(|m| &m.media) { Some(MediaRef::File { path }) => path.clone(), _ => String::new(), diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index eec6c21f..40c2eb9b 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -493,6 +493,10 @@ pub enum AudioChannels { } /// A clip instance on a track. +fn is_zero_stream(stream: &usize) -> bool { + *stream == 0 +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct TrackItem { pub id: ClipId, @@ -547,6 +551,9 @@ pub struct TrackItem { /// left/right). Empty = the first two channels (mono sources on both sides). #[serde(default, skip_serializing_if = "Vec::is_empty")] pub source_channels: Vec, + /// Audio stream within the media container (0 = the primary stream). + #[serde(default, skip_serializing_if = "is_zero_stream")] + pub audio_stream: usize, /// Graphic clips: roll / crawl, responsive time and the template the graphic came from /// ([`GraphicMeta`]). Schema v12. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -1526,6 +1533,7 @@ impl Project { hold_filters: false, field_options: None, source_channels, + audio_stream: 0, graphic: source_graphic.and_then(|sg| sg.meta.map(Box::new)), }) } diff --git a/crates/render/src/audio.rs b/crates/render/src/audio.rs index 45438fee..f0588818 100644 --- a/crates/render/src/audio.rs +++ b/crates/render/src/audio.rs @@ -208,7 +208,7 @@ fn mix_item( if !src.info().has_audio() { return; } - effected(item, &|m0, len| src.audio(m0, len, sr).ok(), a0, n, sr, w) + effected(item, &|m0, len| src.audio_stream(item.audio_stream, m0, len, sr).ok(), a0, n, sr, w) }; // gains: clip gain × Volume (keyframed, per 64-sample block) × channel volume × panner let clip_gain = db_to_gain(item.gain_db); @@ -328,7 +328,7 @@ pub fn clip_signal(item: &TrackItem, start: i64, frames: usize, sr: u32, sources let n = (a1 - a0) as usize; let off = (a0 - start) as usize; let g = db_to_gain(item.gain_db); - let buf = effected(item, &|m0, len| src.audio(m0, len, sr).ok(), a0, n, sr, 2); + let buf = effected(item, &|m0, len| src.audio_stream(item.audio_stream, m0, len, sr).ok(), a0, n, sr, 2); for (dst, b) in out.iter_mut().zip(&buf) { for (d, x) in dst[off..off + n].iter_mut().zip(b) { *d = x * g; diff --git a/crates/render/src/mixer_tests.rs b/crates/render/src/mixer_tests.rs index 0eea28c3..2eca462d 100644 --- a/crates/render/src/mixer_tests.rs +++ b/crates/render/src/mixer_tests.rs @@ -58,7 +58,7 @@ impl Rig { kind: MediaKind::AudioOnly, duration: Tick(10 * TICKS_PER_SECOND), video: None, - audio: Some(AudioStreamInfo { sample_rate: 48_000, channels: ch as u32, codec: "test".into(), bits_per_sample: Some(32) }), + audio_streams: vec![AudioStreamInfo { sample_rate: 48_000, channels: ch as u32, codec: "test".into(), bits_per_sample: Some(32) }], container: "test".into(), start_timecode: None, file_size: None, @@ -128,6 +128,70 @@ fn close(a: f32, b: f32, tol: f32) -> bool { (a - b).abs() <= tol } +/// The renderer must request the stream selected by each clip, rather than reading stream zero +/// twice when a container item has clips on two audio tracks. +#[test] +fn audio_clips_from_one_container_mix_distinct_streams() { + struct TwoStreams(MediaInfo); + impl MediaSource for TwoStreams { + fn info(&self) -> &MediaInfo { + &self.0 + } + fn video_frame(&self, _: FrameRequest) -> filmcraft_media::Result> { + Err(MediaError::NoStream("video")) + } + fn audio(&self, start: i64, frames: usize, sr: u32) -> filmcraft_media::Result { + self.audio_stream(0, start, frames, sr) + } + fn audio_stream(&self, stream: usize, _start: i64, frames: usize, sr: u32) -> filmcraft_media::Result { + let v = match stream { + 0 => 0.25, + 1 => -0.125, + _ => return Err(MediaError::NoStream("audio")), + }; + Ok(AudioBuffer { sample_rate: sr, channels: vec![vec![v; frames], vec![v; frames]] }) + } + } + let mut p = Project::new("streams"); + let seq = p.new_sequence("mix", SequenceSettings::default(), 0, 2, None); + let mut info = MediaInfo { + name: "two.mov".into(), + kind: MediaKind::AudioOnly, + duration: Tick(TICKS_PER_SECOND), + video: None, + audio_streams: vec![AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "PCM".into(), bits_per_sample: Some(16) }], + container: "MOV".into(), + start_timecode: None, + file_size: None, + }; + info.audio_streams.push(info.audio_streams[0].clone()); + let id = p.add_item( + "two.mov", + Label::Iris, + ItemKind::Media(MediaClip { + media: MediaRef::File { path: "two.mov".into() }, + info: info.clone(), + interpret: Default::default(), + mark_in: None, + mark_out: None, + markers: vec![], + offline: false, + proxy: None, + identity: None, + }), + None, + ); + let mut map = SourceMap::default(); + map.0.insert(id, Arc::new(TwoStreams(info))); + for stream in 0..2 { + let mut clip = p.make_track_item(id, TrackKind::Audio, Tick::ZERO, TimeRange::new(Tick::ZERO, Tick(TICKS_PER_SECOND)), FrameRate::FPS_24).unwrap(); + clip.audio_stream = stream; + p.sequence_mut(seq).unwrap().audio_tracks[stream].items.push(clip); + } + let buffer = crate::audio::mix_sequence(&p, p.sequence(seq).unwrap(), 0, 32, &map); + assert!(buffer.channels.iter().all(|c| c.iter().all(|&v| close(v, 0.125, 1e-6)))); +} + #[test] fn fader_gain_is_sample_exact() { let mut r = Rig::new(&[dc]); @@ -666,7 +730,7 @@ fn nested_tone(inner_rate: u32) -> (Project, ItemId, SourceMap, Arc) { kind: MediaKind::AudioOnly, duration: ten.duration, video: None, - audio: Some(AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "test".into(), bits_per_sample: Some(32) }), + audio_streams: vec![AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "test".into(), bits_per_sample: Some(32) }], container: "test".into(), start_timecode: None, file_size: None, diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index 6e0ef1ef..90423c3a 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -234,6 +234,9 @@ impl filmcraft_media::MediaSource for TimedSource { fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> filmcraft_media::Result { self.0.audio(start, frames, sample_rate) } + fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> filmcraft_media::Result { + self.0.audio_stream(stream, start, frames, sample_rate) + } } /// The job's source provider: the pool, with source fetches timed (the time feeds diff --git a/crates/ui-egui/src/panels/clip_dialogs.rs b/crates/ui-egui/src/panels/clip_dialogs.rs index a205e1ce..0c04724a 100644 --- a/crates/ui-egui/src/panels/clip_dialogs.rs +++ b/crates/ui-egui/src/panels/clip_dialogs.rs @@ -155,7 +155,7 @@ fn defaults(app: &FilmcraftApp, id: &str) -> (Value, Value) { let item = s.state.project_selection.iter().find_map(|i| s.project.item(*i).and_then(|it| it.as_media()).filter(|m| m.info.has_audio())); match item { Some(m) => { - let n = m.info.audio.as_ref().map_or(2, |a| a.channels) as u16; + let n = m.info.audio().map_or(2, |a| a.channels) as u16; let map = m.interpret.audio_channels.clone().unwrap_or_else(|| AudioChannelMap::for_format(AudioChannels::Stereo, n)); (json!({"format": format_name(map.format), "clips": map.clips}), json!({"channels": n, "items": true})) } diff --git a/crates/ui-egui/src/panels/misc.rs b/crates/ui-egui/src/panels/misc.rs index d9f55b67..040aa590 100644 --- a/crates/ui-egui/src/panels/misc.rs +++ b/crates/ui-egui/src/panels/misc.rs @@ -151,7 +151,7 @@ pub fn info(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) { line(ui, "Video:", format!("{} fps, {} x {} ({:.4})", v.frame_rate.label(), v.width, v.height, v.par.0 as f32 / v.par.1 as f32)); line(ui, "Codec:", v.codec.clone()); } - if let Some(a) = &m.info.audio { + if let Some(a) = m.info.audio() { line(ui, "Audio:", format!("{} Hz - {} ch - {}", a.sample_rate, a.channels, a.codec)); } } diff --git a/crates/ui-egui/src/panels/timecode.rs b/crates/ui-egui/src/panels/timecode.rs index 02d0c3e4..e1dbdf07 100644 --- a/crates/ui-egui/src/panels/timecode.rs +++ b/crates/ui-egui/src/panels/timecode.rs @@ -44,7 +44,7 @@ pub fn row_text(app: &FilmcraftApp, row: &TimecodeRow) -> (String, String) { ItemKind::Sequence(q) => (q.mark_in, q.mark_out), _ => (None, None), }; - let sr = it.as_media().and_then(|m| m.info.audio.as_ref()).map(|a| a.sample_rate as i64).unwrap_or(48000); + let sr = it.as_media().and_then(|m| m.info.audio()).map(|a| a.sample_rate as i64).unwrap_or(48000); View { name: it.name.clone(), rate: it.frame_rate(), diff --git a/docs/project-files.md b/docs/project-files.md index 68980cea..b5f9beec 100644 --- a/docs/project-files.md +++ b/docs/project-files.md @@ -8,7 +8,7 @@ writing them is `crates/format` (`filmcraft-format`); the engine's `file.*` comm ```json { "format": "filmcraft.project", - "schema_version": 12, + "schema_version": 13, "generator": "FilmCraft 0.1.0", "project": { "name": "…", "settings": { … }, "root": { … }, "items": { … }, "next_id": 48 }, "view": { "open_sequences": [21, 50], "active_sequence": 50, "sequences": { "21": { "pps": 40.0, "scroll": 0.0, … } } } @@ -52,6 +52,7 @@ writing them is `crates/format` (`filmcraft-format`); the engine's `file.*` comm | 10 | M3.10 | clip time interpolation, Hold Filters, Field Options, audio source channels, Modify ▸ Audio Channels map, subclip Restrict Trims; no-op step | | 11 | M3.11 | search bins (`project.search_bins`), Flash Cue markers, Project Settings safe areas, capture format and scratch disks; no-op step | | 12 | M10.7 | graphics design data: `TrackItem::graphic` (roll / crawl, responsive time, template link), `EffectInstance::layer` (layer uid, per-character styles, responsive pins), `project.source_graphics`; no-op step | +| 13 | Multi-stream audio | `MediaInfo::audio_streams` and `TrackItem::audio_stream`; older `audio` objects load as stream zero, missing clip indices default to zero; no-op step | ### Migrations