shifathosn
/
filmcraft
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(media,engine): implement multi-stream container audio support and schema v13 

- Update MediaInfo to store all discovered audio streams in audio_streams

- Retain independent stream metadata and decoder state for MP4/MOV and Matroska/WebM

- Update audio_placement_specs to allocate successive tracks via ensure_audio_tracks

- Isolate clip rendering to read assigned audio stream rather than replaying stream 0

- Bump project schema to v13 with backwards compatibility for legacy v12 projects

Resolves #312

Fixes #200

Fixes #315
This commit is contained in:
Idiotic 2026-10-08 22:33:35 -04:00
parent 78ed3edd84
commit f146d7ba9f
58 changed files with 604 additions and 258 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/filmcraft-web/src/webcodecs.rs
View File

@ -532,4 +532,7 @@ impl MediaSource for WcSource {
fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer, MediaError> {
self.inner.audio(start, frames, sample_rate)
}
fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer, MediaError> {
self.inner.audio_stream(stream, start, frames, sample_rate)
}
}

2
crates/codecs/src/apv.rs
View File

@ -114,7 +114,7 @@ impl ApvSource {
bitrate,
hdr: None,
}),
audio: None,
audio_streams: Vec::new(),
container: "APV".into(),
start_timecode: None,
file_size: Some(len),

4
crates/codecs/src/audio.rs
View File

@ -666,12 +666,12 @@ impl AudioFileSource {
kind: MediaKind::AudioOnly,
duration: Tick::from_units(frames as i64, rate as i64),
video: None,
audio: Some(AudioStreamInfo {
audio_streams: vec![AudioStreamInfo {
sample_rate: rate,
channels: samples.len() as u32,
codec: codec_name,
bits_per_sample: track.codec_params.bits_per_sample,
}),
}],
container: ext.to_uppercase(),
start_timecode: None,
file_size: Some(bytes.len() as u64),

166
crates/codecs/src/mkv.rs
View File

@ -35,20 +35,27 @@ struct AudioState {
last_decoded: Option<usize>,
}
/// One playable audio track with its own decoder state and timing tables.
struct MkvAudio {
/// Index into `MkvFile::tracks`.
track: usize,
state: Mutex<AudioState>,
/// Packet start positions in source sample frames.
starts: Vec<i64>,
/// Decoder pre-roll after a seek, in source sample frames (0: prime with one packet).
preroll: i64,
}
pub struct MkvSource {
info: MediaInfo,
bytes: crate::Src,
file: MkvFile,
vtrack: Option<usize>,
atrack: Option<usize>,
/// Video codec as an ISO-BMFF sample entry (for the decoder factories).
ventry: Option<SampleEntry>,
video: GopCache,
audio: Mutex<AudioState>,
/// Audio packet start positions in source sample frames.
audio_starts: Vec<i64>,
/// Decoder pre-roll after a seek, in source sample frames (0: prime with one packet).
audio_preroll: i64,
/// The playable audio tracks in file order: `info.audio_streams[k]` describes `audios[k]`.
audios: Vec<MkvAudio>,
}
/// Sample-exact Opus packet start positions (48 kHz frames, pre-skip removed).
@ -236,8 +243,16 @@ impl MkvSource {
let bytes = crate::Src(reader);
let file = filmcraft_matroska::open(&bytes).map_err(|e| CodecError::Container(e.to_string()))?;
let vtrack = file.tracks.iter().position(|t| t.kind == TrackKind::Video && !t.samples.is_empty());
let atrack = file.tracks.iter().position(|t| t.kind == TrackKind::Audio && !t.samples.is_empty());
if vtrack.is_none() && atrack.is_none() {
// every audio track is a stream (the track count comes from the file: capped)
let atracks: Vec<usize> = file
.tracks
.iter()
.enumerate()
.filter(|(_, t)| t.kind == TrackKind::Audio && !t.samples.is_empty())
.map(|(i, _)| i)
.take(filmcraft_media::MAX_AUDIO_STREAMS)
.collect();
if vtrack.is_none() && atracks.is_empty() {
return Err(CodecError::Unsupported("no playable tracks".into()));
}
let mut explicit_color = None;
@ -279,21 +294,24 @@ impl MkvSource {
hdr: v.colour.as_ref().and_then(hdr_metadata),
}
});
let audio = atrack.map(|i| {
let t = &file.tracks[i];
let a = t.audio.clone().unwrap_or_default();
let mut rate = a.output_sampling_frequency.unwrap_or(a.sampling_frequency).round().max(1.0) as u32;
let mut channels = (a.channels as u32).max(1);
if let Some(h) = opus_head(&t.codec) {
rate = crate::audio::OPUS_RATE;
channels = h.channels as u32;
}
AudioStreamInfo { sample_rate: rate, channels, codec: codec_label(&t.codec), bits_per_sample: a.bit_depth.map(|b| b as u32) }
});
let audio_streams: Vec<AudioStreamInfo> = atracks
.iter()
.filter_map(|&i| file.tracks.get(i))
.map(|t| {
let a = t.audio.clone().unwrap_or_default();
let mut rate = a.output_sampling_frequency.unwrap_or(a.sampling_frequency).round().max(1.0) as u32;
let mut channels = (a.channels as u32).max(1);
if let Some(h) = opus_head(&t.codec) {
rate = crate::audio::OPUS_RATE;
channels = h.channels as u32;
}
AudioStreamInfo { sample_rate: rate, channels, codec: codec_label(&t.codec), bits_per_sample: a.bit_depth.map(|b| b as u32) }
})
.collect();
let duration = match file.duration_ns() {
Some(ns) => Tick::from_rational(ns as i64, 1, 1_000_000_000),
None => vtrack
.or(atrack)
.or(atracks.first().copied())
.map(|i| {
let t = &file.tracks[i];
let end = t.samples.iter().map(|s| s.pts + s.duration as i64).max().unwrap_or(0);
@ -306,48 +324,36 @@ impl MkvSource {
kind: if video.is_some() { MediaKind::Movie } else { MediaKind::AudioOnly },
duration,
video,
audio,
audio_streams,
container: if file.is_webm() { "WebM".into() } else { "Matroska".into() },
start_timecode: None,
file_size: Some(bytes.0.len()),
};
let audio_starts = atrack
.map(|i| {
let t = &file.tracks[i];
let rate = info.audio.as_ref().map_or(48_000, |a| a.sample_rate) as i64;
// `info.audio_streams[k]` was made from `atracks[k]`, so the two stay index-aligned
let audios: Vec<MkvAudio> = atracks
.iter()
.zip(&info.audio_streams)
.filter_map(|(&i, ainfo)| {
let t = file.tracks.get(i)?;
let rate = ainfo.sample_rate as i64;
let (n, d) = tb(t);
let at = |pts: i64| (pts as i128 * n as i128 * rate as i128 / d as i128) as i64;
match opus_head(&t.codec) {
let starts = match opus_head(&t.codec) {
Some(h) => opus_starts(&file, &bytes, i, &h, rate),
None => t.samples.iter().map(|s| at(s.pts)).collect(),
}
})
.unwrap_or_default();
let audio_preroll = atrack
.map(|i| {
let t = &file.tracks[i];
let rate = info.audio.as_ref().map_or(48_000, |a| a.sample_rate) as i64;
match t.codec {
};
let preroll = match t.codec {
Codec::Opus { .. } => {
let container = (t.seek_pre_roll_ns as u128 * rate as u128).div_ceil(1_000_000_000) as i64;
container.max(crate::audio::OPUS_PRE_ROLL as i64 * rate / 48_000)
}
_ => 0,
}
};
let state = Mutex::new(AudioState { decoder: None, packets: HashMap::new(), order: Vec::new(), last_decoded: None });
Some(MkvAudio { track: i, state, starts, preroll })
})
.unwrap_or(0);
Ok(Self {
info,
bytes,
file,
vtrack,
atrack,
ventry,
video: GopCache::new(explicit_color).with_rotation(rotation),
audio: Mutex::new(AudioState { decoder: None, packets: HashMap::new(), order: Vec::new(), last_decoded: None }),
audio_starts,
audio_preroll,
})
.collect();
Ok(Self { info, bytes, file, vtrack, ventry, video: GopCache::new(explicit_color).with_rotation(rotation), audios })
}
fn read(&self, track: usize, i: usize) -> crate::Result<Vec<u8>> {
@ -373,13 +379,14 @@ impl MkvSource {
}
}
fn audio_packet(&self, st: &mut AudioState, i: usize) -> crate::Result<Arc<Vec<Vec<f32>>>> {
/// Decoded packet `i` of audio stream `a` (`ainfo` is its `AudioStreamInfo`; `st` its decoder state).
fn audio_packet(&self, a: &MkvAudio, ainfo: &AudioStreamInfo, st: &mut AudioState, i: usize) -> crate::Result<Arc<Vec<Vec<f32>>>> {
if let Some(p) = st.packets.get(&i) {
return Ok(p.clone());
}
let ti = self.atrack.ok_or_else(|| CodecError::Unsupported("no audio track".into()))?;
let track = &self.file.tracks[ti];
let ainfo = self.info.audio.as_ref().ok_or_else(|| CodecError::Unsupported("no audio track".into()))?;
let ti = a.track;
let track = self.file.tracks.get(ti).ok_or_else(|| CodecError::Unsupported("no audio track".into()))?;
let pk_start = a.starts.get(i).copied().ok_or_else(|| CodecError::Container("audio packet out of range".into()))?;
let data = self.read(ti, i)?;
let decoded = match &track.codec {
Codec::Pcm { float, big_endian, bits } => {
@ -402,8 +409,8 @@ impl MkvSource {
if st.last_decoded.is_none_or(|l| l + 1 != i) {
let d = st.decoder.as_mut().ok_or_else(|| CodecError::Decode("no audio decoder".into()))?;
d.reset();
let from = if self.audio_preroll > 0 {
self.audio_starts.partition_point(|&x| x <= self.audio_starts[i] - self.audio_preroll).saturating_sub(1)
let from = if a.preroll > 0 {
a.starts.partition_point(|&x| x <= pk_start.saturating_sub(a.preroll)).saturating_sub(1)
} else {
i.saturating_sub(1)
};
@ -413,7 +420,7 @@ impl MkvSource {
}
}
}
let r = st.decoder.as_mut().ok_or_else(|| CodecError::Decode("no audio decoder".into()))?.decode(&data, self.audio_starts[i].max(0) as u64);
let r = st.decoder.as_mut().ok_or_else(|| CodecError::Decode("no audio decoder".into()))?.decode(&data, pk_start.max(0) as u64);
st.last_decoded = Some(i);
r.unwrap_or_default()
}
@ -473,23 +480,27 @@ impl MediaSource for MkvSource {
}
fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer, MediaError> {
let ti = self.atrack.ok_or(MediaError::NoStream("audio"))?;
let n_packets = self.file.tracks[ti].samples.len();
let ainfo = self.info.audio.as_ref().ok_or(MediaError::NoStream("audio"))?;
self.audio_stream(0, start, frames, sample_rate)
}
fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer, MediaError> {
let a = self.audios.get(stream).ok_or(MediaError::NoStream("audio"))?;
let n_packets = self.file.tracks.get(a.track).ok_or(MediaError::NoStream("audio"))?.samples.len();
let ainfo = self.info.audio_streams.get(stream).ok_or(MediaError::NoStream("audio"))?;
let src_rate = ainfo.sample_rate;
let ch = ainfo.channels.max(1) as usize;
let ratio = src_rate as f64 / sample_rate as f64;
let s0 = (start as f64 * ratio).floor() as i64;
let need = (frames as f64 * ratio).ceil() as i64 + 2;
let mut src: Vec<Vec<f32>> = vec![vec![0.0; need.max(0) as usize]; ch];
let mut st = self.audio.lock().unwrap_or_else(|e| e.into_inner());
let mut i = self.audio_starts.partition_point(|&x| x <= s0).saturating_sub(1);
let mut st = a.state.lock().unwrap_or_else(|e| e.into_inner());
let mut i = a.starts.partition_point(|&x| x <= s0).saturating_sub(1);
while i < n_packets {
let pk_start = self.audio_starts[i];
let Some(&pk_start) = a.starts.get(i) else { break };
if pk_start >= s0 + need {
break;
}
let pk = self.audio_packet(&mut st, i)?;
let pk = self.audio_packet(a, ainfo, &mut st, i)?;
for (c, dst) in src.iter_mut().enumerate() {
let Some(chan) = pk.get(c.min(pk.len().saturating_sub(1))) else { continue };
for (k, v) in chan.iter().enumerate() {
@ -532,3 +543,34 @@ pub fn reader_opener(name: &str, head: &[u8], reader: &filmcraft_media::SharedRe
}
Some(MkvSource::open_reader(name, reader.clone()).map(|s| Arc::new(s) as SharedSource).map_err(Into::into))
}
#[cfg(test)]
mod multi_audio_tests {
use filmcraft_matroska::{MkvWriter, MuxOptions, TrackKind, TrackSpec};
use filmcraft_media::MediaSource;
#[test]
fn every_audio_track_is_inspected_and_decoded_independently() {
let tracks: Vec<TrackSpec> = [(48_000.0, "first"), (24_000.0, "second")]
.into_iter()
.map(|(rate, name)| {
let mut t = TrackSpec::new(TrackKind::Audio, "A_PCM/INT/LIT");
t.audio = Some((rate, 1, Some(16)));
t.name = Some(name.into());
t
})
.collect();
let mut writer = MkvWriter::new(std::io::Cursor::new(Vec::new()), tracks, MuxOptions::default()).unwrap();
for (stream, value) in [(0, 8192i16), (1, -16384)] {
let data: Vec<u8> = (0..100).flat_map(|_| value.to_le_bytes()).collect();
writer.write_frame(stream, 0, true, &data, Some(4_000_000)).unwrap();
}
let source = super::MkvSource::open("two.mkv", writer.finish().unwrap().into_inner().into()).unwrap();
assert_eq!(source.info().audio_streams.iter().map(|a| a.sample_rate).collect::<Vec<_>>(), vec![48_000, 24_000]);
let primary = source.audio(0, 50, 48_000).unwrap();
let secondary = source.audio_stream(1, 0, 50, 48_000).unwrap();
assert!((primary.channels[0][20] - 0.25).abs() < 0.001);
assert!((secondary.channels[0][20] + 0.5).abs() < 0.001);
assert!(source.audio_stream(2, 0, 50, 48_000).is_err());
}
}

210
crates/codecs/src/mp4.rs
View File

@ -29,20 +29,27 @@ struct AudioState {
last_decoded: Option<usize>,
}
/// One playable audio track with its own decoder state and timing tables.
struct Mp4Audio {
/// Index into `Mp4File::tracks`.
track: usize,
state: Mutex<AudioState>,
/// Cumulative sample start frames for the track (for packet lookup).
starts: Vec<i64>,
/// Presentation offset of the track in its timescale (edit list, or Opus pre-skip).
offset: i64,
/// Decoder pre-roll after a seek, in track timescale units (0: prime with one packet).
preroll: i64,
}
pub struct Mp4Source {
info: MediaInfo,
bytes: crate::Src,
file: Mp4File,
vtrack: Option<usize>,
atrack: Option<usize>,
video: GopCache,
audio: Mutex<AudioState>,
/// Cumulative sample start frames for the audio track (for packet lookup).
audio_starts: Vec<i64>,
/// Presentation offset of the audio track in its timescale (edit list, or Opus pre-skip).
audio_offset: i64,
/// Decoder pre-roll after a seek, in audio track timescale units (0: prime with one packet).
audio_preroll: i64,
/// The playable audio tracks in file order: `info.audio_streams[k]` describes `audios[k]`.
audios: Vec<Mp4Audio>,
}
pub fn sniff(b: &[u8]) -> bool {
@ -120,8 +127,10 @@ impl Mp4Source {
// files can lack them: indexing `entries[0]` or dividing by the timescale used to panic).
let playable = |t: &&filmcraft_isobmff::Track, kind| t.kind == kind && !t.samples.is_empty() && !t.entries.is_empty() && t.timescale > 0;
let vtrack = file.tracks.iter().position(|t| playable(&t, TrackKind::Video));
let atrack = file.tracks.iter().position(|t| playable(&t, TrackKind::Audio));
if vtrack.is_none() && atrack.is_none() {
// every audio track is a stream (the track count comes from the file: capped)
let atracks: Vec<usize> =
file.tracks.iter().enumerate().filter(|(_, t)| playable(t, TrackKind::Audio)).map(|(i, _)| i).take(filmcraft_media::MAX_AUDIO_STREAMS).collect();
if vtrack.is_none() && atracks.is_empty() {
return Err(CodecError::Unsupported("no playable tracks".into()));
}
let mut color = ColorInfo::REC709;
@ -195,37 +204,11 @@ impl Mp4Source {
}
info
});
let audio = atrack.map(|i| {
let t = &file.tracks[i];
let entry = &t.entries[0];
let ap = entry.audio.clone().unwrap_or_default();
let (rate, ch, bits) = match &entry.codec {
CodecConfig::Aac(a) => (
{
// HE-AAC plays at twice the core rate the AudioSpecificConfig starts with
// an AAC access unit is at most 6144 bits per channel: a hostile `stsz` can't make us read more
let units = t
.samples
.iter()
.take(8)
.filter_map(|x| filmcraft_media::reader::read_range(&*bytes.0, x.offset, (x.size as usize).min(MAX_AAC_PROBE_UNIT)).ok());
let units: Vec<Vec<u8>> = units.collect();
match crate::audio::aac_output_rate(&a.asc, units.iter().map(Vec::as_slice)) {
Some(r) => r,
None if a.sample_rate > 0 => a.sample_rate,
None => ap.sample_rate as u32,
}
},
if a.channel_config > 0 { a.channel_config as u32 } else { ap.channels },
None,
),
CodecConfig::Pcm(p) => (p.sample_rate as u32, p.channels, Some(p.bits as u32)),
CodecConfig::Opus(o) => (crate::audio::OPUS_RATE, (o.output_channels as u32).max(1), None),
_ => (if ap.sample_rate > 0.0 { ap.sample_rate as u32 } else { t.timescale }, ap.channels.max(1), None),
};
AudioStreamInfo { sample_rate: rate.max(1), channels: ch.max(1), codec: codec_label(&entry.codec), bits_per_sample: bits }
});
let duration = vtrack.or(atrack).and_then(|i| file.tracks.get(i)).map(|t| presentation_duration(&file, t)).unwrap_or_default();
// one entry per playable audio track (`playable` guarantees a sample description); a track
// that somehow has none is dropped so `audio_streams` and `audios` stay index-aligned
let atracks: Vec<usize> = atracks.into_iter().filter(|&i| file.tracks.get(i).is_some_and(|t| !t.entries.is_empty())).collect();
let audio_streams: Vec<AudioStreamInfo> = atracks.iter().filter_map(|&i| file.tracks.get(i)).filter_map(|t| audio_stream_info(t, &bytes)).collect();
let duration = vtrack.or(atracks.first().copied()).and_then(|i| file.tracks.get(i)).map(|t| presentation_duration(&file, t)).unwrap_or_default();
let start_timecode = file.tracks.iter().find_map(|t| match t.codec() {
Some(CodecConfig::Timecode(tc)) => tc.start_frame.map(|f| f as i64),
_ => None,
@ -235,50 +218,38 @@ impl Mp4Source {
kind: if video.is_some() { MediaKind::Movie } else { MediaKind::AudioOnly },
duration,
video,
audio,
audio_streams,
container: if file.is_quicktime { "QuickTime".into() } else { "MPEG-4".into() },
start_timecode,
file_size: Some(bytes.0.len()),
};
let audio_starts = atrack
.map(|i| {
let t = &file.tracks[i];
let audios: Vec<Mp4Audio> = atracks
.iter()
.filter_map(|&i| {
let t = file.tracks.get(i)?;
let mut acc = 0i64;
t.samples
let starts = t
.samples
.iter()
.map(|s| {
let st = acc;
acc += s.duration as i64;
acc = acc.saturating_add(s.duration as i64);
st
})
.collect()
})
.unwrap_or_default();
let (audio_offset, audio_preroll) = atrack
.map(|i| {
let t = &file.tracks[i];
.collect();
let ts = t.timescale.max(1) as i64;
match &t.entries[0].codec {
CodecConfig::Opus(o) => {
let (offset, preroll) = match t.entries.first().map(|e| &e.codec) {
Some(CodecConfig::Opus(o)) => {
let off = if t.edit_offset != 0 { t.edit_offset } else { -(o.pre_skip as i64) * ts / 48_000 };
(off, (crate::audio::OPUS_PRE_ROLL as u64 * ts as u64).div_ceil(48_000) as i64)
}
_ => (t.edit_offset, 0),
}
};
let state = Mutex::new(AudioState { decoder: None, packets: HashMap::new(), order: Vec::new(), last_decoded: None });
Some(Mp4Audio { track: i, state, starts, offset, preroll })
})
.unwrap_or((0, 0));
Ok(Self {
info,
bytes,
file,
vtrack,
atrack,
video: GopCache::new(explicit_color).with_rotation(rotation),
audio: Mutex::new(AudioState { decoder: None, packets: HashMap::new(), order: Vec::new(), last_decoded: None }),
audio_starts,
audio_offset,
audio_preroll,
})
.collect();
Ok(Self { info, bytes, file, vtrack, video: GopCache::new(explicit_color).with_rotation(rotation), audios })
}
fn read(&self, track: usize, i: usize) -> crate::Result<Vec<u8>> {
@ -294,27 +265,30 @@ impl Mp4Source {
self.video.frame_late(&Mp4Video { src: self, track: ti }, t.to_rational_round(1, ts), late)
}
fn audio_packet(&self, st: &mut AudioState, i: usize) -> crate::Result<Arc<Vec<Vec<f32>>>> {
/// Decoded packet `i` of audio stream `a` (`ainfo` is its `AudioStreamInfo`; `st` its decoder state).
fn audio_packet(&self, a: &Mp4Audio, ainfo: &AudioStreamInfo, st: &mut AudioState, i: usize) -> crate::Result<Arc<Vec<Vec<f32>>>> {
if let Some(p) = st.packets.get(&i) {
return Ok(p.clone());
}
let ti = self.atrack.ok_or_else(|| CodecError::Unsupported("no audio track".into()))?;
let track = &self.file.tracks[ti];
let ti = a.track;
let track = self.file.tracks.get(ti).ok_or_else(|| CodecError::Unsupported("no audio track".into()))?;
let entry = track.entries.first().ok_or_else(|| CodecError::Container("audio track without a sample description".into()))?;
let sample = track.samples.get(i).ok_or_else(|| CodecError::Container("audio packet out of range".into()))?;
let pk_start = a.starts.get(i).copied().unwrap_or(0);
let data = self.read(ti, i)?;
let decoded = match &entry.codec {
CodecConfig::Pcm(p) => decode_pcm(&data, p),
c => {
if st.decoder.is_none() {
st.decoder = Some(PacketDecoder::for_isobmff(c, self.info.audio.as_ref().map_or(48_000, |a| a.sample_rate))?);
st.decoder = Some(PacketDecoder::for_isobmff(c, ainfo.sample_rate)?);
}
// Non-sequential access: reset and prime with the preceding packets (codec pre-roll:
// one packet, or `OPUS_PRE_ROLL` worth for Opus).
if st.last_decoded.is_none_or(|l| l + 1 != i) {
let d = st.decoder.as_mut().ok_or_else(|| CodecError::Decode("no audio decoder".into()))?;
d.reset();
let from = if self.audio_preroll > 0 {
self.audio_starts.partition_point(|&x| x <= self.audio_starts[i] - self.audio_preroll).saturating_sub(1)
let from = if a.preroll > 0 {
a.starts.partition_point(|&x| x <= pk_start.saturating_sub(a.preroll)).saturating_sub(1)
} else {
i.saturating_sub(1)
};
@ -324,11 +298,11 @@ impl Mp4Source {
}
}
}
let r = st.decoder.as_mut().ok_or_else(|| CodecError::Decode("no audio decoder".into()))?.decode(&data, self.audio_starts[i].max(0) as u64);
let r = st.decoder.as_mut().ok_or_else(|| CodecError::Decode("no audio decoder".into()))?.decode(&data, pk_start.max(0) as u64);
st.last_decoded = Some(i);
match r {
Ok(v) => v,
Err(_) => vec![vec![0.0; track.samples[i].duration as usize]; self.info.audio.as_ref().map_or(2, |a| a.channels as usize)],
Err(_) => vec![vec![0.0; sample.duration as usize]; ainfo.channels.max(1) as usize],
}
}
};
@ -381,30 +355,36 @@ impl MediaSource for Mp4Source {
}
fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer, MediaError> {
let ti = self.atrack.ok_or(MediaError::NoStream("audio"))?;
let track = &self.file.tracks[ti];
let ainfo = self.info.audio.as_ref().ok_or(MediaError::NoStream("audio"))?;
self.audio_stream(0, start, frames, sample_rate)
}
fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer, MediaError> {
let a = self.audios.get(stream).ok_or(MediaError::NoStream("audio"))?;
let ti = a.track;
let track = self.file.tracks.get(ti).ok_or(MediaError::NoStream("audio"))?;
let ainfo = self.info.audio_streams.get(stream).ok_or(MediaError::NoStream("audio"))?;
let src_rate = ainfo.sample_rate;
let ch = ainfo.channels.max(1) as usize;
// Map the requested window to source samples (edit list offset applied: presentation = pts + edit_offset).
let ratio = src_rate as f64 / sample_rate as f64;
let p0 = (start as f64 * ratio).floor() as i64;
// (in i128: a hostile edit list can make the offset large enough to overflow i64)
let offset = i128::from(self.audio_offset) * i128::from(src_rate) / i128::from(track.timescale.max(1));
let offset = i128::from(a.offset) * i128::from(src_rate) / i128::from(track.timescale.max(1));
let s0 = i64::try_from(i128::from(p0) - offset).unwrap_or(if offset > 0 { i64::MIN } else { i64::MAX });
let need = (frames as f64 * ratio).ceil() as i64 + 2;
// Samples-per-unit: the track timescale is usually the sample rate for audio.
let unit = src_rate as f64 / track.timescale.max(1) as f64;
let mut src: Vec<Vec<f32>> = vec![vec![0.0; need.max(0) as usize]; ch];
let mut st = self.audio.lock().unwrap_or_else(|e| e.into_inner());
let first = self.audio_starts.partition_point(|&x| (x as f64 * unit) as i64 <= s0.max(0)).saturating_sub(1);
let mut st = a.state.lock().unwrap_or_else(|e| e.into_inner());
let first = a.starts.partition_point(|&x| (x as f64 * unit) as i64 <= s0.max(0)).saturating_sub(1);
let mut i = first;
while i < track.samples.len() {
let pk_start = (self.audio_starts[i] as f64 * unit) as i64;
let Some(&start_units) = a.starts.get(i) else { break };
let pk_start = (start_units as f64 * unit) as i64;
if pk_start >= s0.saturating_add(need) {
break;
}
let pk = self.audio_packet(&mut st, i)?;
let pk = self.audio_packet(a, ainfo, &mut st, i)?;
for (c, dst) in src.iter_mut().enumerate() {
let Some(chan) = pk.get(c.min(pk.len().saturating_sub(1))) else { continue };
for (k, v) in chan.iter().enumerate() {
@ -443,6 +423,37 @@ impl MediaSource for Mp4Source {
}
}
/// What the file says about one audio track (`None`: the track has no sample description).
fn audio_stream_info(t: &filmcraft_isobmff::Track, bytes: &crate::Src) -> Option<AudioStreamInfo> {
let entry = t.entries.first()?;
let ap = entry.audio.clone().unwrap_or_default();
let (rate, ch, bits) = match &entry.codec {
CodecConfig::Aac(a) => (
{
// HE-AAC plays at twice the core rate the AudioSpecificConfig starts with
// an AAC access unit is at most 6144 bits per channel: a hostile `stsz` can't make us read more
let units = t
.samples
.iter()
.take(8)
.filter_map(|x| filmcraft_media::reader::read_range(&*bytes.0, x.offset, (x.size as usize).min(MAX_AAC_PROBE_UNIT)).ok());
let units: Vec<Vec<u8>> = units.collect();
match crate::audio::aac_output_rate(&a.asc, units.iter().map(Vec::as_slice)) {
Some(r) => r,
None if a.sample_rate > 0 => a.sample_rate,
None => ap.sample_rate as u32,
}
},
if a.channel_config > 0 { a.channel_config as u32 } else { ap.channels },
None,
),
CodecConfig::Pcm(p) => (p.sample_rate as u32, p.channels, Some(p.bits as u32)),
CodecConfig::Opus(o) => (crate::audio::OPUS_RATE, (o.output_channels as u32).max(1), None),
_ => (if ap.sample_rate > 0.0 { ap.sample_rate as u32 } else { t.timescale }, ap.channels.max(1), None),
};
Some(AudioStreamInfo { sample_rate: rate.max(1), channels: ch.max(1), codec: codec_label(&entry.codec), bits_per_sample: bits })
}
fn codec_label(c: &CodecConfig) -> String {
match c {
CodecConfig::Avc(a) => format!(
@ -803,6 +814,33 @@ mod tests {
}
}
#[cfg(test)]
mod multi_audio_tests {
use filmcraft_isobmff::{Brand, Mp4Writer, PcmConfig, SampleEntry, TrackConfig, WriteSample, WriterOptions};
use filmcraft_media::MediaSource;
/// Two mono PCM tracks with different rates and samples. Stream indices follow file order;
/// decoder caches, sample timing and resampling must not cross streams.
#[test]
fn every_audio_track_is_inspected_and_decoded_independently() {
let mut mux = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mov)).unwrap();
for (rate, value) in [(48_000u32, 8192i16), (24_000, -16384)] {
let entry = SampleEntry::pcm(PcmConfig { bits: 16, float: false, big_endian: false, signed: true, channels: 1, sample_rate: rate as f64 });
let track = mux.add_track(TrackConfig::new(entry, rate)).unwrap();
let data: Vec<u8> = (0..rate / 10).flat_map(|_| value.to_le_bytes()).collect();
mux.write_sample(track, WriteSample { data: &data, duration: rate / 10, composition_offset: 0, is_sync: true }).unwrap();
}
let bytes = mux.finish().unwrap().into_inner();
let source = super::Mp4Source::open("two.mov", bytes.into()).unwrap();
assert_eq!(source.info().audio_streams.iter().map(|a| a.sample_rate).collect::<Vec<_>>(), vec![48_000, 24_000]);
let primary = source.audio(0, 100, 48_000).unwrap();
let secondary = source.audio_stream(1, 0, 100, 48_000).unwrap();
assert!((primary.channels[0][50] - 0.25).abs() < 0.001);
assert!((secondary.channels[0][50] + 0.5).abs() < 0.001);
assert!(source.audio_stream(2, 0, 100, 48_000).is_err());
}
}
#[cfg(test)]
mod he_aac_tests {
use filmcraft_isobmff::{Brand, Mp4Writer, SampleEntry, TrackConfig, WriteSample, WriterOptions};
@ -829,7 +867,7 @@ mod he_aac_tests {
}
let b = mux.finish().unwrap().into_inner();
let src = super::Mp4Source::open("he.mp4", b.into()).unwrap();
assert_eq!(src.info().audio.as_ref().unwrap().sample_rate, 44_100);
assert_eq!(src.info().audio().unwrap().sample_rate, 44_100);
// one second from 0.5 s at 44.1 kHz: the 1 kHz tone at its level (not silence, not shifted)
let buf = src.audio(22_050, 44_100, 44_100).unwrap();
let x = &buf.channels[0];

10
crates/codecs/src/mpeg.rs
View File

@ -211,7 +211,7 @@ impl MpegSource {
kind: MediaKind::Movie,
duration: Tick::ZERO,
video: None,
audio: None,
audio_streams: Vec::new(),
container: file.format.name().to_string(),
start_timecode: None,
file_size: Some(bytes.0.len()),
@ -316,7 +316,7 @@ impl MpegSource {
kind: MediaKind::Movie,
duration: Tick::ZERO,
video: None,
audio: None,
audio_streams: Vec::new(),
container: "MPEG video elementary stream".into(),
start_timecode: None,
file_size: Some(bytes.0.len()),
@ -485,7 +485,8 @@ impl MpegSource {
/// Mark the audio stream as not decodable (it still shows in the media info).
fn audio_unsupported(&mut self, codec_name: &str, why: String) -> Option<AudioTrack> {
self.unsupported_audio = Some(why);
self.info.audio = Some(AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: format!("{codec_name} (unsupported)"), bits_per_sample: None });
self.info.audio_streams =
vec![AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: format!("{codec_name} (unsupported)"), bits_per_sample: None }];
None
}
@ -599,7 +600,8 @@ impl MpegSource {
self.info.kind = MediaKind::AudioOnly;
}
}
self.info.audio = Some(AudioStreamInfo { sample_rate: rate.max(1), channels: channels.max(1) as u32, codec: codec_name, bits_per_sample: bits });
self.info.audio_streams =
vec![AudioStreamInfo { sample_rate: rate.max(1), channels: channels.max(1) as u32, codec: codec_name, bits_per_sample: bits }];
Some(AudioTrack {
stream: ai,
codec,

4
crates/codecs/src/mxf.rs
View File

@ -361,7 +361,7 @@ impl MxfSource {
kind: if video.is_some() { MediaKind::Movie } else { MediaKind::AudioOnly },
duration,
video,
audio,
audio_streams: audio.into_iter().collect(),
container: format!("MXF {}", file.operational_pattern.name()),
start_timecode: None,
file_size: Some(bytes.0.len()),
@ -457,7 +457,7 @@ impl MediaSource for MxfSource {
}
fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer, MediaError> {
let ainfo = self.info.audio.as_ref().ok_or(MediaError::NoStream("audio"))?;
let ainfo = self.info.audio().ok_or(MediaError::NoStream("audio"))?;
let src_rate = ainfo.sample_rate;
let ch = ainfo.channels.max(1) as usize;
let mut out = AudioBuffer::silence(sample_rate, ch, frames);

2
crates/codecs/src/ogg.rs
View File

@ -110,7 +110,7 @@ impl OggSource {
kind: MediaKind::AudioOnly,
duration: Tick::from_units(frames, rate as i64),
video: None,
audio: Some(AudioStreamInfo { sample_rate: rate, channels: channels.max(1), codec: codec.into(), bits_per_sample: None }),
audio_streams: vec![AudioStreamInfo { sample_rate: rate, channels: channels.max(1), codec: codec.into(), bits_per_sample: None }],
container: "Ogg".into(),
start_timecode: None,
file_size: Some(bytes.0.len()),

4
crates/codecs/src/tests.rs
View File

@ -70,7 +70,7 @@ fn mjpeg_mov_with_pcm() {
fn aac_mp4_decodes() {
let Some(b) = fixture("tone_aac.mp4", &["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:d=3", "-c:a", "aac", "-b:a", "128k"]) else { return };
let src = crate::open_bytes("tone_aac.mp4", b).unwrap();
assert!(src.info().audio.as_ref().unwrap().codec.contains("AAC"));
assert!(src.info().audio().unwrap().codec.contains("AAC"));
// read in the middle (random access) and sequentially
let a = src.audio(48_000, 4800, 48_000).unwrap();
let p = a.peaks()[0];
@ -510,7 +510,7 @@ fn check_opus(name: &str, channels: usize, expr: &str, extra: &[&str], container
let src = crate::open_bytes(name, b).expect("open");
let info = src.info().clone();
assert_eq!(info.container, container);
let a = info.audio.as_ref().expect("audio");
let a = info.audio().expect("audio");
assert_eq!((a.codec.as_str(), a.sample_rate, a.channels as usize), ("Opus", 48_000, channels));
let d = info.duration.seconds();
assert!((d - 3.0).abs() < 0.03, "{name}: duration {d}");

22
crates/codecs/tests/mpeg_oracle.rs
View File

@ -245,7 +245,7 @@ fn check_video(ff: &Path, file: &Path, tol: u16, seeks: usize) -> (u16, f64) {
/// Our decoded audio from the first audio frame against ffmpeg's decode. Returns the max error.
fn check_audio(ff: &Path, file: &Path, tol: f32) -> f32 {
let src = open(file);
let a = src.info().audio.clone().unwrap();
let a = src.info().audio().cloned().unwrap();
let (start, len) = src.audio_extent().unwrap();
let want = ffmpeg_audio_f32(ff, file, &[]);
let ch = a.channels as usize;
@ -314,7 +314,7 @@ fn transport_stream_mpeg2_and_mp2() {
let (worst, psnr) = check_video(&ff, &f, 4, 12);
assert!(psnr >= 58.0, "PSNR {psnr}");
println!("ts_mpeg2_mp2: video max diff {worst}, PSNR {psnr:.2} dB");
let a = src.info().audio.clone().unwrap();
let a = src.info().audio().cloned().unwrap();
assert_eq!((a.codec.as_str(), a.sample_rate, a.channels), ("MPEG Audio", 48_000, 2));
let e = check_audio(&ff, &f, 2e-4);
println!("ts_mpeg2_mp2: MP2 audio max error {e:.2e}");
@ -337,7 +337,7 @@ fn bdav_h264_with_lpcm_sample_exact() {
let f = make(&ff, "m2ts_h264_lpcm.m2ts");
let src = open(&f);
assert_eq!(src.info().container, "MPEG-2 TS (BDAV/AVCHD)");
let a = src.info().audio.clone().unwrap();
let a = src.info().audio().cloned().unwrap();
assert_eq!((a.codec.as_str(), a.channels, a.bits_per_sample), ("LPCM (Blu-ray)", 2, Some(24)));
check_video(&ff, &f, 0, 10);
assert_eq!(check_audio(&ff, &f, 0.0), 0.0);
@ -349,7 +349,7 @@ fn avchd_ac3_audio() {
let f = make(&ff, "m2ts_h264_ac3.mts");
let src = open(&f);
check_video(&ff, &f, 0, 6);
let a = src.info().audio.clone().unwrap();
let a = src.info().audio().cloned().unwrap();
if filmcraft_codecs::audio::AC3_DECODER {
assert_eq!((a.codec.as_str(), a.channels), ("AC-3", 2));
// AC-3 zero-bit mantissas carry decoder-specific dither (A/52 §7.3.4)
@ -370,7 +370,7 @@ fn transport_stream_hevc_and_latm() {
let f = make(&ff, "ts_hevc_latm.ts");
let src = open(&f);
assert_eq!(src.info().video.as_ref().unwrap().codec, "HEVC");
assert_eq!(src.info().audio.as_ref().unwrap().codec, "AAC (LATM)");
assert_eq!(src.info().audio().unwrap().codec, "AAC (LATM)");
check_video(&ff, &f, 0, 8);
let e = check_audio(&ff, &f, 1e-4);
println!("ts_hevc_latm: AAC (LATM) max error {e:.2e}");
@ -389,17 +389,17 @@ fn program_streams() {
// DVD LPCM: sample-exact
let f = make(&ff, "ps_mpeg2_lpcm.vob");
let src = open(&f);
assert_eq!(src.info().audio.as_ref().unwrap().codec, "LPCM (DVD)");
assert_eq!(src.info().audio.as_ref().unwrap().bits_per_sample, Some(24));
assert_eq!(src.info().audio().unwrap().codec, "LPCM (DVD)");
assert_eq!(src.info().audio().unwrap().bits_per_sample, Some(24));
assert_eq!(check_audio(&ff, &f, 0.0), 0.0);
check_video(&ff, &f, 4, 4);
let f = make(&ff, "ps_mpeg2_lpcm16.vob");
assert_eq!(open(&f).info().audio.as_ref().unwrap().bits_per_sample, Some(16));
assert_eq!(open(&f).info().audio().unwrap().bits_per_sample, Some(16));
assert_eq!(check_audio(&ff, &f, 0.0), 0.0);
// AC-3 in private stream 1
let f = make(&ff, "ps_mpeg2_ac3.vob");
let src = open(&f);
assert_eq!(src.info().audio.as_ref().unwrap().codec, "AC-3");
assert_eq!(src.info().audio().unwrap().codec, "AC-3");
let e = check_audio(&ff, &f, 5e-3);
println!("ps_mpeg2_ac3: AC-3 max error {e:.2e}");
// MPEG-1 system stream
@ -457,7 +457,7 @@ fn standalone_mp2_file() {
let ff = filmcraft_testkit::require_ffmpeg!();
let f = make(&ff, "audio.mp2");
let src = filmcraft_codecs::open_bytes("audio.mp2", bytes(&f)).unwrap();
let a = src.info().audio.clone().unwrap();
let a = src.info().audio().cloned().unwrap();
assert_eq!((a.sample_rate, a.channels), (48_000, 2));
let want = ffmpeg_audio_f32(&ff, &f, &[]);
let n = want.len() / 2;
@ -483,7 +483,7 @@ fn truncated_and_corrupt_files_never_panic() {
let _ = s.video_frame(FrameRequest::full(v.frame_rate.tick_of(i)));
}
}
if let Some(a) = &info.audio {
if let Some(a) = info.audio() {
let _ = s.audio(rng.below(48_000) as i64, 2048, a.sample_rate);
}
};

14
crates/codecs/tests/mxf_oracle.rs
View File

@ -179,7 +179,7 @@ fn check_video(ff: &Path, file: &Path, pix_fmt: &str, chroma: (usize, usize), bp
/// Our audio, as interleaved f32, equals ffmpeg's decode exactly.
fn check_audio_exact(ff: &Path, file: &Path, extra: &[&str]) {
let src = open(file);
let a = src.info().audio.clone().unwrap();
let a = src.info().audio().cloned().unwrap();
let ch = a.channels as usize;
let want = ffmpeg_audio_f32(ff, file, extra);
let n = want.len() / ch;
@ -236,7 +236,7 @@ fn h264_2997_drop_frame_timecode_and_24_bit_audio() {
assert!(tc.drop_frame);
assert_eq!(tc.format(), "01:00:00;00");
assert_eq!(info.start_timecode, Some(107_892));
assert_eq!(info.audio.as_ref().unwrap().bits_per_sample, Some(24));
assert_eq!(info.audio().unwrap().bits_per_sample, Some(24));
check_video(&ff, &f, "yuv420p", (2, 2), 1, 0, 10);
check_audio_exact(&ff, &f, &[]);
}
@ -299,14 +299,14 @@ fn op_atom_video_and_audio_files() {
let v = make(&ff, "mxf_atom_dnxhr.mxf");
let src = open(&v);
assert_eq!(src.info().container, "MXF OP-Atom");
assert!(src.info().audio.is_none());
assert!(!src.info().has_audio());
assert_eq!(src.file().tracks[0].wrapping, filmcraft_mxf::Wrapping::Clip);
check_video(&ff, &v, "yuv422p", (2, 1), 1, 2, 10);
let a = make(&ff, "mxf_atom_pcm.mxf");
let src = open(&a);
assert_eq!(src.info().kind, MediaKind::AudioOnly);
assert_eq!(src.info().container, "MXF OP-Atom");
let ai = src.info().audio.clone().unwrap();
let ai = src.info().audio().cloned().unwrap();
assert_eq!((ai.sample_rate, ai.channels, ai.bits_per_sample), (48_000, 1, Some(24)));
assert_eq!(src.info().duration, filmcraft_time::Tick::from_units(48_000, 48_000));
check_audio_exact(&ff, &a, &[]);
@ -317,7 +317,7 @@ fn d10_aes3_elements_sample_exact() {
let ff = filmcraft_testkit::require_ffmpeg!();
let f = make(&ff, "mxf_d10.mxf");
let src = open(&f);
let a = src.info().audio.clone().unwrap();
let a = src.info().audio().cloned().unwrap();
assert_eq!(a.codec, "AES3 PCM");
assert!(a.channels >= 4, "{}", a.channels);
let v = src.info().video.as_ref().unwrap();
@ -347,7 +347,7 @@ fn truncated_and_corrupt_files_never_panic() {
let _ = s.video_frame(FrameRequest::full(t));
}
}
if let Some(a) = &info.audio {
if let Some(a) = info.audio() {
let _ = s.audio(rng.below(96_000) as i64, 4000, a.sample_rate);
}
}
@ -364,7 +364,7 @@ fn truncated_and_corrupt_files_never_panic() {
let _ = s.video_frame(FrameRequest::full(s.info().frame_rate().tick_of(i)));
}
}
if s.info().audio.is_some() {
if s.info().has_audio() {
let _ = s.audio(0, 9600, 48_000);
}
}

6
crates/codecs/tests/ogg_oracle.rs
View File

@ -88,7 +88,7 @@ fn snr_db(reference: &[f32], ours: &[f32]) -> f64 {
/// (our continuous decode interleaved, ffmpeg's decode interleaved, channels, rate)
fn decode_both(ff: &Path, file: &Path, decoder: &[&str]) -> (OggSource, Vec<f32>, Vec<f32>, usize, u32) {
let src = OggSource::open(file.file_name().unwrap().to_str().unwrap(), bytes(file)).unwrap();
let a = src.info().audio.clone().unwrap();
let a = src.info().audio().cloned().unwrap();
let ch = a.channels as usize;
let mut args: Vec<&str> = decoder.to_vec();
args.extend_from_slice(&["-i", file.to_str().unwrap(), "-map", "0:a:0", "-f", "f32le", "-"]);
@ -134,7 +134,7 @@ fn opus_celt_music_matches_libopus() {
assert_eq!(src.info().container, "Ogg");
assert_eq!(src.info().kind, MediaKind::AudioOnly);
assert_eq!((rate, ch), (48_000, 2));
assert_eq!(src.info().audio.as_ref().unwrap().codec, "Opus");
assert_eq!(src.info().audio().unwrap().codec, "Opus");
let snr = snr_db(&want, &ours);
eprintln!("{}: {} samples, SNR {snr:.1} dB vs libopus", f.display(), want.len() / ch);
assert!(snr >= 50.0, "SNR {snr:.1} dB");
@ -173,7 +173,7 @@ fn vorbis_matches_ffmpeg() {
let ff = filmcraft_testkit::require_ffmpeg!();
let f = make(&ff, "ogg_vorbis.ogg");
let (src, ours, want, ch, rate) = decode_both(&ff, &f, &[]);
assert_eq!(src.info().audio.as_ref().unwrap().codec, "Vorbis");
assert_eq!(src.info().audio().unwrap().codec, "Vorbis");
assert_eq!((rate, ch), (44_100, 2));
let snr = snr_db(&want, &ours);
eprintln!("{}: SNR {snr:.1} dB vs ffmpeg's Vorbis decoder", f.display());

1
crates/edit/src/tests.rs
View File

@ -60,6 +60,7 @@ impl Fx {
hold_filters: false,
field_options: None,
source_channels: Vec::new(),
audio_stream: 0,
graphic: None,
}
}

1
crates/edit/src/through.rs
View File

@ -180,6 +180,7 @@ mod tests {
hold_filters: false,
field_options: None,
source_channels: Vec::new(),
audio_stream: 0,
graphic: None,
}
}

1
crates/edit/src/transcript/tests.rs
View File

@ -57,6 +57,7 @@ fn clip(id: u64, start: f64, dur: f64, src_in: f64) -> TrackItem {
hold_filters: false,
field_options: None,
source_channels: Vec::new(),
audio_stream: 0,
graphic: None,
}
}

66
crates/engine/src/audio_placement_tests.rs
View File

@ -1,6 +1,8 @@
//! Multi-clip audio placement: `audio_placement_specs` and dynamic audio track allocation.
use filmcraft_media::MediaSource;
use filmcraft_project::{AudioChannels, ItemId, TrackKind};
use filmcraft_time::{FrameRate, Tick, TimeRange};
use serde_json::json;
@ -19,13 +21,13 @@ fn active(s: &Session) -> ItemId {
#[test]
fn specs_always_place_one_clip_on_the_destination() {
assert_eq!(audio_placement_specs(&[]), vec![AudioPlacementSpec { track_offset: 0, source_channels: None }]);
assert_eq!(audio_placement_specs(&[vec![0, 1]]).len(), 1);
assert_eq!(audio_placement_specs(&[], 1), vec![AudioPlacementSpec { track_offset: 0, audio_stream: 0, source_channels: None }]);
assert_eq!(audio_placement_specs(&[vec![0, 1]], 1).len(), 1);
}
#[test]
fn specs_put_further_clips_on_the_tracks_below() {
let specs = audio_placement_specs(&[vec![0], vec![1], vec![2, 3]]);
let specs = audio_placement_specs(&[vec![0], vec![1], vec![2, 3]], 1);
let got: Vec<(usize, Option<Vec<u16>>)> = specs.into_iter().map(|s| (s.track_offset, s.source_channels)).collect();
assert_eq!(got, vec![(0, None), (1, Some(vec![1])), (2, Some(vec![2, 3]))]);
}
@ -33,7 +35,63 @@ fn specs_put_further_clips_on_the_tracks_below() {
#[test]
fn specs_are_capped_for_hostile_channel_maps() {
let clips = vec![vec![0u16]; 100_000];
assert!(audio_placement_specs(&clips).len() <= crate::sequence_tools::MAX_TRACKS);
assert!(audio_placement_specs(&clips, usize::MAX).len() <= crate::sequence_tools::MAX_TRACKS);
}
#[test]
fn specs_append_container_streams_after_channel_map_clips() {
let specs = audio_placement_specs(&[vec![1], vec![0]], 3);
assert_eq!(
specs.iter().map(|s| (s.track_offset, s.audio_stream, s.source_channels.clone())).collect::<Vec<_>>(),
vec![(0, 0, None), (1, 0, Some(vec![0])), (2, 1, None), (3, 2, None)]
);
assert_eq!(audio_placement_specs(&[], 0).len(), 1);
}
#[test]
fn placing_media_with_two_streams_expands_tracks_and_keeps_stream_indices() {
let mut s = Session::default();
let mut p = (*s.project).clone();
let rate = FrameRate::FPS_24;
let seq = p.new_sequence("Two streams", Default::default(), 1, 1, None);
let source = filmcraft_media::generators::GeneratorSource::new(filmcraft_media::Generator::BarsAndTone, 32, 32, rate, rate.tick_of(24));
let mut info = source.info().clone();
info.audio_streams.push(info.audio_streams[0].clone());
let item = p.add_item(
"two streams",
filmcraft_project::Label::Iris,
filmcraft_project::ItemKind::Media(filmcraft_project::MediaClip {
media: filmcraft_project::MediaRef::Generator(filmcraft_media::Generator::BarsAndTone),
info,
interpret: Default::default(),
mark_in: None,
mark_out: None,
markers: vec![],
offline: false,
proxy: None,
identity: None,
}),
None,
);
let (vdest, adest) = {
let q = p.sequence(seq).unwrap();
(q.video_tracks[0].id, q.audio_tracks[0].id)
};
s.project = std::sync::Arc::new(p);
s.state.active_sequence = Some(seq);
let ids =
crate::commands::place_item(&mut s, item, TimeRange::new(Tick::ZERO, rate.tick_of(24)), Tick::ZERO, Some(vdest), Some(adest), false, "test", None)
.unwrap();
assert_eq!(ids.len(), 3);
let q = s.project.sequence(seq).unwrap();
assert_eq!(q.audio_tracks.len(), 2);
let (a, b) = (&q.audio_tracks[0].items[0], &q.audio_tracks[1].items[0]);
assert_eq!((a.audio_stream, b.audio_stream), (0, 1));
assert_eq!(a.link, b.link);
assert!(a.link.is_some());
assert_ne!(a.id, b.id);
assert_eq!(a.source_in, b.source_in);
assert_eq!(a.start, b.start);
}
#[test]

31
crates/engine/src/clip_ops.rs
View File

@ -636,8 +636,12 @@ fn sequence_from_clip(s: &mut Session, p: &Value) -> Result<Value> {
settings.height = *height;
settings.frame_rate = *rate;
}
let channels = media_root(&s.project, first).and_then(|(_, m, _)| m.info.audio.as_ref().map(|a| a.channels)).unwrap_or(2);
let n_audio = media_root(&s.project, first).and_then(|(_, m, _)| m.interpret.audio_channels.as_ref().map(|a| a.clips.len())).unwrap_or(1).max(1);
let channels = media_root(&s.project, first).and_then(|(_, m, _)| m.info.audio().map(|a| a.channels)).unwrap_or(2);
let n_audio = media_root(&s.project, first)
.map(|(_, m, _)| {
crate::commands::audio_placement_specs(m.interpret.audio_channels.as_ref().map_or(&[], |a| a.clips.as_slice()), m.info.audio_streams.len()).len()
})
.unwrap_or(1);
let n0 = s.history.undo.len();
let name = pi.name.clone();
let bin = s.project.root.parent_of(first).filter(|b| *b != s.project.root.id);
@ -761,12 +765,15 @@ fn offline_file(s: &mut Session, p: &Value) -> Result<Value> {
bitrate: None,
hdr: None,
}),
audio: has_a.then(|| filmcraft_media::AudioStreamInfo {
sample_rate: u64_p(p, "sampleRate").unwrap_or(48_000) as u32,
channels: u64_p(p, "channels").unwrap_or(2).max(1) as u32,
codec: String::new(),
bits_per_sample: None,
}),
audio_streams: has_a
.then(|| filmcraft_media::AudioStreamInfo {
sample_rate: u64_p(p, "sampleRate").unwrap_or(48_000) as u32,
channels: u64_p(p, "channels").unwrap_or(2).max(1) as u32,
codec: String::new(),
bits_per_sample: None,
})
.into_iter()
.collect(),
container: String::new(),
start_timecode: start_tc,
file_size: None,
@ -913,7 +920,7 @@ fn consolidate_duplicates(s: &mut Session, _: &Value) -> Result<Value> {
for it in s.project.items.values() {
if let ItemKind::Media(m) = &it.kind {
// same file (or generator), interpretation and streams
let shape = (m.info.video.is_some(), m.info.audio.is_some(), m.info.duration);
let shape = (m.info.video.is_some(), m.info.has_audio(), m.info.duration);
let key = serde_json::to_string(&(&m.media, &m.interpret, m.offline, shape)).unwrap_or_default();
groups.entry(key).or_default().push(it.id);
}
@ -1249,7 +1256,7 @@ fn audio_channels(s: &mut Session, p: &Value) -> Result<Value> {
let mut maps = Vec::new();
for i in &items {
let Some(m) = s.project.item(*i).and_then(|it| it.as_media()) else { continue };
let n = m.info.audio.as_ref().map_or(2, |a| a.channels) as u16;
let n = m.info.audio().map_or(2, |a| a.channels) as u16;
let fmt = format.unwrap_or_else(|| m.interpret.audio_channels.as_ref().map_or(AudioChannels::Stereo, |a| a.format));
let map = match &custom {
Some(c) => {
@ -1554,7 +1561,7 @@ fn breakout_to_mono(s: &mut Session, p: &Value) -> Result<Value> {
for i in &items {
let Some(it) = pr.item(*i).cloned() else { continue };
let Some(m) = it.as_media() else { continue };
let n = m.info.audio.as_ref().map_or(1, |a| a.channels).max(1) as u16;
let n = m.info.audio().map_or(1, |a| a.channels).max(1) as u16;
let bin = pr.root.parent_of(*i).filter(|b| *b != pr.root.id);
for c in 0..n {
let suffix = match (n, c) {
@ -1590,7 +1597,7 @@ fn extract_audio(s: &mut Session, p: &Value) -> Result<Value> {
let mut out = Vec::new();
for item in items {
let (root, m, sub) = media_root(&s.project, item).ok_or_else(|| bad("clip.extractAudio", "not a media clip"))?;
let a = m.info.audio.clone().ok_or_else(|| bad("clip.extractAudio", "the clip has no audio"))?;
let a = m.info.audio().cloned().ok_or_else(|| bad("clip.extractAudio", "the clip has no audio"))?;
let range = sub.unwrap_or(TimeRange::new(Tick::ZERO, m.info.duration));
let dir = match str_p(p, "dir") {
Some(d) => d.to_string(),

6
crates/engine/src/clip_ops_tests.rs
View File

@ -261,7 +261,7 @@ fn new_sequence_from_clip_bin_from_selection_and_offline_file() {
.unwrap();
let it = s.project.item(ItemId(r["item"].as_u64().unwrap())).unwrap();
let m = it.as_media().unwrap();
assert!(m.offline && m.info.audio.is_none() && m.info.video.is_some());
assert!(m.offline && !m.info.has_audio() && m.info.video.is_some());
assert_eq!(m.info.duration, s.sequence_rate().snap_nearest(Tick(4 * TICKS_PER_SECOND)));
assert_eq!(m.info.start_timecode, Some(3600 * 24));
assert_eq!(it.metadata["Tape Name"], "A001");
@ -372,7 +372,7 @@ fn modify_audio_channels_and_breakout_to_mono() {
s.project = std::sync::Arc::new({
let mut p = (*s.project).clone();
if let Some(m) = p.item_mut(item_named(&s, "Bars and Tone")).and_then(|i| i.as_media_mut()) {
m.info.audio = None;
m.info.audio_streams.clear();
}
p
});
@ -557,7 +557,7 @@ fn extract_audio_writes_a_wav_and_imports_it() {
let item = s.project.item(ItemId(e["item"].as_u64().unwrap())).unwrap();
let m = item.as_media().unwrap();
assert!(m.info.video.is_none());
assert_eq!(m.info.audio.as_ref().unwrap().channels, 2);
assert_eq!(m.info.audio().unwrap().channels, 2);
let src_dur = s.project.item(forest).unwrap().duration();
assert!((m.info.duration - src_dur).0.abs() < TICKS_PER_SECOND / 100, "same length");
assert_eq!(s.project.root.parent_of(item.id), s.project.root.parent_of(forest));

28
crates/engine/src/commands.rs
View File

@ -358,7 +358,7 @@ pub(crate) fn default_seq_settings_for(info: &filmcraft_media::MediaInfo) -> Seq
st.frame_rate = v.frame_rate;
st.preset = format!("{}x{} {}", v.width, v.height, v.frame_rate.label());
}
if let Some(a) = &info.audio {
if let Some(a) = info.audio() {
st.sample_rate = a.sample_rate.max(8000);
}
st
@ -490,6 +490,7 @@ fn set_matte_color(s: &mut Session, p: &Value) -> Result<Value> {
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct AudioPlacementSpec {
pub track_offset: usize,
pub audio_stream: usize,
pub source_channels: Option<Vec<u16>>,
}
@ -497,10 +498,22 @@ pub(crate) struct AudioPlacementSpec {
/// per clip): always at least one (on the destination track), then one per further entry on the
/// tracks below. The count is capped at [`crate::sequence_tools::MAX_TRACKS`] (the map comes from
/// project files and scripts: never trusted).
pub(crate) fn audio_placement_specs(clips: &[Vec<u16>]) -> Vec<AudioPlacementSpec> {
let mut specs = vec![AudioPlacementSpec { track_offset: 0, source_channels: None }];
pub(crate) fn audio_placement_specs(clips: &[Vec<u16>], streams: usize) -> Vec<AudioPlacementSpec> {
let mut specs = vec![AudioPlacementSpec { track_offset: 0, audio_stream: 0, source_channels: None }];
let extra = clips.iter().skip(1).take(crate::sequence_tools::MAX_TRACKS.saturating_sub(1));
specs.extend(extra.enumerate().map(|(k, chans)| AudioPlacementSpec { track_offset: k.saturating_add(1), source_channels: Some(chans.clone()) }));
specs.extend(extra.enumerate().map(|(k, chans)| AudioPlacementSpec {
track_offset: k.saturating_add(1),
audio_stream: 0,
source_channels: Some(chans.clone()),
}));
// Interpret Footage's channel map describes stream 0; further container streams keep their
// own channel layouts and get one clip each, after any channel-map clips.
for stream in 1..streams.min(filmcraft_media::MAX_AUDIO_STREAMS) {
if specs.len() >= crate::sequence_tools::MAX_TRACKS {
break;
}
specs.push(AudioPlacementSpec { track_offset: specs.len(), audio_stream: stream, source_channels: None });
}
specs
}
@ -592,7 +605,8 @@ pub(crate) fn place_item(
// Modify ▸ Audio Channels with several audio clips: one per clip, on the tracks below
let clips: Vec<Vec<u16>> =
p.item(item).and_then(|i| i.as_media()).and_then(|m| m.interpret.audio_channels.as_ref()).map(|m| m.clips.clone()).unwrap_or_default();
let specs = audio_placement_specs(&clips);
let streams = p.resolve_media(item).map_or(1, |(_, m, _)| m.info.audio_streams.len());
let specs = audio_placement_specs(&clips, streams);
let first = p.sequence(seq_id).and_then(|q| q.audio_tracks.iter().position(|t| t.id == adest)).unwrap_or(0);
// too few audio tracks below the destination: grow the sequence from the destination track
let tracks = ensure_audio_tracks(p, seq_id, first, specs.len(), label)?;
@ -606,9 +620,13 @@ pub(crate) fn place_item(
},
};
let mut b = a.clone();
b.audio_stream = spec.audio_stream;
if spec.track_offset > 0 {
b.id = ClipId(p.alloc_id());
}
if spec.audio_stream > 0 {
b.source_channels.clear();
}
if let Some(chans) = spec.source_channels {
b.source_channels = chans;
}

4
crates/engine/src/media_browser.rs
View File

@ -516,9 +516,9 @@ pub fn column_text(e: &Entry, column: &str, probe: Option<&MediaInfo>) -> String
})
.unwrap_or_default(),
"Video Info" => probe.and_then(|i| i.video.as_ref()).map(|v| format!("{} x {}", v.width, v.height)).unwrap_or_default(),
"Audio Info" => probe.and_then(|i| i.audio.as_ref()).map(|a| format!("{} Hz - {} ch", a.sample_rate, a.channels)).unwrap_or_default(),
"Audio Info" => probe.and_then(|i| i.audio()).map(|a| format!("{} Hz - {} ch", a.sample_rate, a.channels)).unwrap_or_default(),
"Video Codec" => probe.and_then(|i| i.video.as_ref()).map(|v| v.codec.clone()).unwrap_or_default(),
"Audio Codec" => probe.and_then(|i| i.audio.as_ref()).map(|a| a.codec.clone()).unwrap_or_default(),
"Audio Codec" => probe.and_then(|i| i.audio()).map(|a| a.codec.clone()).unwrap_or_default(),
_ => String::new(),
}
}

2
crates/engine/src/media_browser_tests.rs
View File

@ -269,7 +269,7 @@ fn probing_large_unstreamable_files_does_not_read_them_whole() {
crate::media_test_util::make_movie(&mov, filmcraft_media::DemoScene::OceanSunset, 64, 36, 24);
let mut s = Session::default();
assert!(media_browser::probe(&mut s, &avi.to_string_lossy()).is_none());
assert_eq!(media_browser::probe(&mut s, &wav.to_string_lossy()).unwrap().audio.unwrap().channels, 2);
assert_eq!(media_browser::probe(&mut s, &wav.to_string_lossy()).unwrap().audio().unwrap().channels, 2);
assert!(media_browser::probe(&mut s, &mov.to_string_lossy()).unwrap().video.is_some());
// importing is unchanged: it still opens files through the full path
let r = s.execute("file.import", json!({"paths": [wav.to_string_lossy()]})).unwrap();

11
crates/engine/src/media_pool.rs
View File

@ -279,11 +279,15 @@ impl MediaPool {
return Some(s.clone());
}
match self.open_file(path, services) {
Ok(s) => {
Ok(s) if s.info().audio_streams.len() >= m.info.audio_streams.len() => {
let src: SharedSource = Arc::new(ProxySource { proxy: s, info: m.info.clone() });
self.proxies.write().unwrap_or_else(|e| e.into_inner()).insert(item, (key, src.clone()));
Some(src)
}
Ok(_) => {
log::warn!("proxy has fewer audio streams than the original: {path}; using full-resolution media");
None
}
Err(e) => {
// a missing proxy falls back to the full-resolution media
log::warn!("proxy offline: {path}: {e}");
@ -357,7 +361,7 @@ impl MediaSource for SlateSource {
Ok(Arc::new((*f).clone().with_pts(req.time)))
}
fn audio(&self, _start: i64, frames: usize, sample_rate: u32) -> filmcraft_media::Result<AudioBuffer> {
Ok(AudioBuffer::silence(sample_rate, self.info.audio.as_ref().map_or(2, |a| a.channels as usize), frames))
Ok(AudioBuffer::silence(sample_rate, self.info.audio().map_or(2, |a| a.channels as usize), frames))
}
}
@ -397,6 +401,9 @@ impl MediaSource for ProxySource {
fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> filmcraft_media::Result<AudioBuffer> {
self.proxy.audio(start, frames, sample_rate)
}
fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> filmcraft_media::Result<AudioBuffer> {
self.proxy.audio_stream(stream, start, frames, sample_rate)
}
}
#[cfg(test)]

2
crates/engine/src/panels.rs
View File

@ -224,7 +224,7 @@ pub fn metadata_fields(p: &Project, id: ItemId) -> Vec<Field> {
let cs = media.and_then(|m| m.interpret.color_space).unwrap_or(detected);
add("Clip", "Color Space", cs.label().to_string(), false);
}
if let Some(a) = media.and_then(|m| m.info.audio.as_ref()) {
if let Some(a) = media.and_then(|m| m.info.audio()) {
add("Clip", "Audio Info", format!("{} Hz - {} ch", a.sample_rate, a.channels), false);
add("Clip", "Audio Codec", a.codec.clone(), false);
}

2
crates/engine/src/project_manager.rs
View File

@ -311,7 +311,7 @@ fn execute(
fn estimate_bytes(m: &filmcraft_project::MediaClip, dur: Tick, pr: &Preset) -> u64 {
let secs = dur.seconds().max(0.0);
let audio = m.info.audio.as_ref().map_or(0.0, |a| a.sample_rate as f64 * 2.0 * 2.0 * secs);
let audio = m.info.audio().map_or(0.0, |a| a.sample_rate as f64 * 2.0 * 2.0 * secs);
let Some(v) = &m.info.video else { return audio as u64 };
let fps = v.frame_rate.num as f64 / v.frame_rate.den.max(1) as f64;
let px = (v.width * v.height) as f64 * (pr.scale * pr.scale) as f64;

4
crates/engine/src/project_panel.rs
View File

@ -384,7 +384,7 @@ pub fn cell(p: &Project, it: &ProjectItem, column: &str) -> (String, SortKey) {
"Video Codec" => text(media.and_then(|m| m.info.video.as_ref()).map(|v| v.codec.clone()).unwrap_or_default()),
"Audio Info" => text(
media
.and_then(|m| m.info.audio.as_ref())
.and_then(|m| m.info.audio())
.map(|a| {
format!(
"{} Hz - {}",
@ -400,7 +400,7 @@ pub fn cell(p: &Project, it: &ProjectItem, column: &str) -> (String, SortKey) {
})
.unwrap_or_default(),
),
"Audio Codec" => text(media.and_then(|m| m.info.audio.as_ref()).map(|a| a.codec.clone()).unwrap_or_default()),
"Audio Codec" => text(media.and_then(|m| m.info.audio()).map(|a| a.codec.clone()).unwrap_or_default()),
"Color Space" => text(match (media.and_then(|m| m.info.video.as_ref().map(|v| (m, v))), &it.kind) {
(Some((m, v)), _) => m.interpret.color_space.unwrap_or_else(|| filmcraft_color::ColorSpace::from_info(&v.color)).label().to_string(),
(None, ItemKind::Sequence(q)) => q.settings.color.working.label().to_string(),

4
crates/engine/src/project_tools.rs
View File

@ -805,7 +805,7 @@ pub fn info_json(name: &str, path: Option<&str>, info: &filmcraft_media::MediaIn
"bitrate": vs.bitrate, "color": format!("{:?} / {:?} / {:?}", vs.color.primaries, vs.color.transfer, vs.color.matrix),
});
}
if let Some(a) = &info.audio {
if let Some(a) = info.audio() {
v["audio"] = json!({"codec": a.codec, "sampleRate": a.sample_rate, "channels": a.channels, "bitsPerSample": a.bits_per_sample});
}
v
@ -1002,7 +1002,7 @@ fn edit_offline(s: &mut Session, p: &Value) -> Result<Value> {
fn source_settings(s: &mut Session, p: &Value) -> Result<Value> {
let id = targets(s, p).into_iter().find(|i| media_of(&s.project, *i).is_some()).ok_or_else(|| EngineError::Other("select a clip".into()))?;
let (_, m) = media_of(&s.project, id).ok_or_else(|| EngineError::Other("select a clip".into()))?;
let codec = m.info.video.as_ref().map(|v| v.codec.clone()).or_else(|| m.info.audio.as_ref().map(|a| a.codec.clone())).unwrap_or_default();
let codec = m.info.video.as_ref().map(|v| v.codec.clone()).or_else(|| m.info.audio().map(|a| a.codec.clone())).unwrap_or_default();
Ok(json!({
"item": id.0,
"codec": codec,

4
crates/engine/src/proxies.rs
View File

@ -105,14 +105,14 @@ pub fn transcode(
st.height = v.height;
}
st.frame_rate = rate;
st.sample_rate = info.audio.as_ref().map_or(48_000, |a| a.sample_rate.max(8000));
st.sample_rate = info.audio().map_or(48_000, |a| a.sample_rate.max(8000));
let mut p = Project::new("transcode");
let mut c = clip.clone();
c.offline = false;
c.proxy = None;
let item = p.add_item(name, filmcraft_project::Label::Iris, ItemKind::Media(c), None);
let has_v = info.video.is_some();
let has_a = info.audio.is_some();
let has_a = info.has_audio();
let seq = p.new_sequence("transcode", st, usize::from(has_v), usize::from(has_a), None);
let range = range.unwrap_or(TimeRange { start: Tick::ZERO, duration: info.duration });
let mut placed = Vec::new();

2
crates/engine/src/relink.rs
View File

@ -263,7 +263,7 @@ pub fn check_candidate(s: &Session, item: ItemId, path: &str, o: &MatchOptions)
if info.video.is_some() != m.info.video.is_some() {
c.problems.push("video stream presence differs".into());
}
if let (Some(a), Some(b)) = (&info.audio, &m.info.audio)
if let (Some(a), Some(b)) = (info.audio(), m.info.audio())
&& a.channels != b.channels
{
c.problems.push(format!("audio channels differ ({} vs {})", a.channels, b.channels));

4
crates/export/src/surround_tests.rs
View File

@ -46,7 +46,7 @@ fn surround() -> (Arc<Project>, ItemId, SourceMap) {
kind: MediaKind::AudioOnly,
duration: Tick(2 * TICKS_PER_SECOND),
video: None,
audio: Some(AudioStreamInfo { sample_rate: SR, channels: 6, codec: "test".into(), bits_per_sample: Some(32) }),
audio_streams: vec![AudioStreamInfo { sample_rate: SR, channels: 6, codec: "test".into(), bits_per_sample: Some(32) }],
container: "test".into(),
start_timecode: None,
file_size: None,
@ -185,7 +185,7 @@ fn aac_51_in_mp4_keeps_every_channel_in_place() {
// our own demuxer sees an AAC track with channel configuration 6
let bytes: Arc<[u8]> = std::fs::read(&path).unwrap().into();
let src = filmcraft_codecs::open_bytes("s51.mp4", bytes).unwrap();
assert_eq!(src.info().audio.as_ref().unwrap().channels, 6);
assert_eq!(src.info().audio().unwrap().channels, 6);
if let Some((n, layout)) = probe(&path) {
assert_eq!(n, 6);
assert!(is_51(&layout), "{layout}");

2
crates/format/Cargo.toml
View File

@ -11,10 +11,10 @@ workspace = true
[dependencies]
filmcraft-project = { workspace = true }
filmcraft-media = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
[dev-dependencies]
filmcraft-time = { workspace = true }
filmcraft-media = { workspace = true }

27
crates/format/src/lib.rs
View File

@ -33,6 +33,8 @@
//! - **v12** (M10.7): graphics templates, rolls / crawls, responsive design (pins, intro / outro),
//! per-character text styles (`TrackItem::graphic`, `EffectInstance::layer`) and source graphics
//! (`Project::source_graphics`). No-op step; older builds would drop these fields when saving.
//! - **v13**: all container audio streams, and per-timeline-clip stream selection. Old single-stream
//! `audio` objects read as stream zero; older builds must refuse new multi-stream projects.
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
@ -53,7 +55,8 @@ pub type Migration = fn(Value) -> Result<Value, String>;
/// `MIGRATIONS[i]` upgrades schema `i + 1` to `i + 2`. Append one function per schema bump; never
/// edit a shipped one.
pub const MIGRATIONS: &[Migration] = &[v1_to_v2, v2_to_v3, v3_to_v4, v4_to_v5, v5_to_v6, v6_to_v7, v7_to_v8, v8_to_v9, v9_to_v10, v10_to_v11, v11_to_v12];
pub const MIGRATIONS: &[Migration] =
&[v1_to_v2, v2_to_v3, v3_to_v4, v4_to_v5, v5_to_v6, v6_to_v7, v7_to_v8, v8_to_v9, v9_to_v10, v10_to_v11, v11_to_v12, v12_to_v13];
/// The schema version this build writes (and the newest it reads).
pub const SCHEMA_VERSION: u32 = MIGRATIONS.len() as u32 + 1;
@ -209,6 +212,20 @@ pub fn decode(bytes: &[u8]) -> Result<Loaded, FormatError> {
/// Refuse only values that would overflow or exhaust memory later (`Sequence::check_bounds`);
/// structural rules such as overlaps are left to the editor, so projects that open today still open.
fn validate_loaded(loaded: Loaded) -> Result<Loaded, FormatError> {
for item in loaded.project.items.values() {
if let filmcraft_project::ItemKind::Media(m) = &item.kind
&& m.info.audio_streams.len() > filmcraft_media::MAX_AUDIO_STREAMS
{
return Err(FormatError::Corrupt(format!("media `{}`: too many audio streams", item.name)));
}
if let filmcraft_project::ItemKind::Sequence(sequence) = &item.kind {
for track in &sequence.audio_tracks {
if track.items.iter().any(|clip| clip.audio_stream >= filmcraft_media::MAX_AUDIO_STREAMS) {
return Err(FormatError::Corrupt(format!("sequence `{}`: audio stream index out of range", item.name)));
}
}
}
}
for item in loaded.project.sequences() {
if let filmcraft_project::ItemKind::Sequence(sequence) = &item.kind {
sequence.check_bounds().map_err(|e| FormatError::Corrupt(format!("sequence `{}`: {e}", item.name)))?;
@ -315,13 +332,19 @@ fn v11_to_v12(doc: Value) -> Result<Value, String> {
Ok(doc)
}
/// v12 → v13: container audio streams and per-clip stream selection. `MediaInfo` reads the old
/// `audio` field as stream zero; `TrackItem::audio_stream` defaults to zero.
fn v12_to_v13(doc: Value) -> Result<Value, String> {
Ok(doc)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn schema_version_matches_table() {
assert_eq!(SCHEMA_VERSION, 12);
assert_eq!(SCHEMA_VERSION, 13);
}
#[test]

25
crates/format/tests/fixtures.rs
View File

@ -52,6 +52,31 @@ fn v1_edit_loads_with_defaults_for_later_fields() {
seq.check().unwrap();
}
#[test]
fn legacy_audio_field_becomes_primary_stream_and_clip_defaults_to_zero() {
let p = decode(&fixture("v1-edit.fcproj")).unwrap().project;
let media = p.item(ItemId(5)).unwrap().as_media().unwrap();
assert_eq!(media.info.audio_streams.len(), 1);
assert!(media.info.has_audio());
let clip = &p.sequence(ItemId(21)).unwrap().audio_tracks[0].items[0];
assert_eq!(clip.audio_stream, 0);
let again = decode(&encode(&p, false)).unwrap();
assert_eq!(again.project.item(ItemId(5)).unwrap().as_media().unwrap().info.audio_streams, media.info.audio_streams);
}
#[test]
fn hostile_audio_stream_counts_and_indices_are_rejected() {
let mut p = decode(&fixture("v1-edit.fcproj")).unwrap().project;
let seq = ItemId(21);
p.sequence_mut(seq).unwrap().audio_tracks[0].items[0].audio_stream = filmcraft_media::MAX_AUDIO_STREAMS;
assert!(matches!(decode(&encode(&p, false)), Err(FormatError::Corrupt(_))));
p.sequence_mut(seq).unwrap().audio_tracks[0].items[0].audio_stream = 0;
let media = p.item_mut(ItemId(5)).unwrap().as_media_mut().unwrap();
let stream = media.info.audio_streams[0].clone();
media.info.audio_streams.resize(filmcraft_media::MAX_AUDIO_STREAMS + 1, stream);
assert!(matches!(decode(&encode(&p, false)), Err(FormatError::Corrupt(_))));
}
/// Schema 8 (before M3.10): no time interpolation / Hold Filters / Field Options / source channels
/// on clips, no audio channel map, subclips without Restrict Trims.
#[test]

2
crates/interchange/src/aaf/tests.rs
View File

@ -28,7 +28,7 @@ fn project() -> (Project, ItemId) {
bitrate: None,
hdr: None,
}),
audio: Some(filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: String::new(), bits_per_sample: Some(24) }),
audio_streams: vec![filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: String::new(), bits_per_sample: Some(24) }],
container: String::new(),
start_timecode: Some(86_400),
file_size: None,

9
crates/interchange/src/ale.rs
View File

@ -87,7 +87,7 @@ pub fn export_items(p: &Project, items: &[ItemId]) -> AleDoc {
let rows_src: Vec<ItemId> = items.iter().copied().filter(|i| media_of(*i).is_some()).collect();
let first_video = rows_src.iter().find_map(|i| media_of(*i).and_then(|(m, ..)| m.info.video.as_ref().map(|v| (v.frame_rate, v.height))));
let (rate, height) = first_video.unwrap_or((FrameRate::FPS_24, 1080));
let sr = rows_src.iter().find_map(|i| media_of(*i).and_then(|(m, ..)| m.info.audio.as_ref().map(|a| a.sample_rate))).unwrap_or(48_000);
let sr = rows_src.iter().find_map(|i| media_of(*i).and_then(|(m, ..)| m.info.audio().map(|a| a.sample_rate))).unwrap_or(48_000);
let heading = vec![
("FIELD_DELIM".to_string(), "TABS".to_string()),
("VIDEO_FORMAT".to_string(), video_format(height).to_string()),
@ -103,7 +103,7 @@ pub fn export_items(p: &Project, items: &[ItemId]) -> AleDoc {
if m.info.video.is_some() {
tracks.push('V');
}
for c in 1..=m.info.audio.as_ref().map_or(0, |a| a.channels.min(16)) {
for c in 1..=m.info.audio().map_or(0, |a| a.channels.min(16)) {
tracks.push_str(&format!("A{c}"));
}
// media start timecode (frames at the media's own rate) + the subclip offset
@ -223,7 +223,10 @@ mod tests {
bitrate: None,
hdr: None,
}),
audio: (audio > 0).then(|| AudioStreamInfo { sample_rate: 48_000, channels: audio, codec: "aac".into(), bits_per_sample: None }),
audio_streams: (audio > 0)
.then(|| AudioStreamInfo { sample_rate: 48_000, channels: audio, codec: "aac".into(), bits_per_sample: None })
.into_iter()
.collect(),
container: "mp4".into(),
start_timecode: tc,
file_size: None,

7
crates/interchange/src/common.rs
View File

@ -251,7 +251,11 @@ pub(crate) fn media_info(name: &str, spec: &MediaSpec) -> MediaInfo {
bitrate: None,
hdr: None,
}),
audio: spec.audio.map(|(sr, ch)| AudioStreamInfo { sample_rate: sr, channels: ch, codec: String::new(), bits_per_sample: None }),
audio_streams: spec
.audio
.map(|(sr, ch)| AudioStreamInfo { sample_rate: sr, channels: ch, codec: String::new(), bits_per_sample: None })
.into_iter()
.collect(),
container: String::new(),
start_timecode: spec.start_tc,
file_size: None,
@ -443,6 +447,7 @@ impl Builder {
hold_filters: false,
field_options: None,
source_channels: Vec::new(),
audio_stream: 0,
graphic: None,
}
}

2
crates/interchange/src/comp.rs
View File

@ -443,7 +443,7 @@ impl Exporter<'_> {
return Some(i);
}
let video = m.info.video.as_ref();
let audio = m.info.audio.as_ref();
let audio = m.info.audio();
let still = m.info.kind == MediaKind::Still;
match ckind {
CKind::Picture if video.is_none() && !still => return None,

2
crates/interchange/src/essence.rs
View File

@ -163,7 +163,7 @@ fn collect_needs(
let (mut start, mut end) = ((lo - opts.handles).max(Tick::ZERO), hi + opts.handles);
match project.item(item).map(|i| &i.kind) {
Some(ItemKind::Media(m)) => {
let Some(a) = m.info.audio.as_ref() else { continue };
let Some(a) = m.info.audio() else { continue };
let path = match &m.media {
MediaRef::File { path } => Some(path.clone()),
MediaRef::Generator(_) => None,

2
crates/interchange/src/fcp7.rs
View File

@ -1055,7 +1055,7 @@ impl Exp<'_, '_> {
self.w.close();
self.w.close();
}
if let Some(a) = &m.info.audio {
if let Some(a) = m.info.audio() {
self.w.open("audio", &[]);
self.w.open("samplecharacteristics", &[]);
self.w.text("depth", a.bits_per_sample.unwrap_or(16));

6
crates/interchange/src/fcpxml.rs
View File

@ -730,12 +730,12 @@ impl Exp<'_, '_> {
fmt_time(start, mrate),
fmt_time(m.info.duration, mrate),
m.info.video.is_some() as u8,
m.info.audio.is_some() as u8
m.info.has_audio() as u8
);
if let Some(f) = fmt {
a.push_str(&format!(" format=\"{f}\""));
}
if let Some(au) = &m.info.audio {
if let Some(au) = m.info.audio() {
a.push_str(&format!(" audioSources=\"1\" audioChannels=\"{}\" audioRate=\"{}\"", au.channels, au.sample_rate));
}
let path = item_path(self.p, item).unwrap_or("");
@ -986,7 +986,7 @@ impl Exp<'_, '_> {
};
let retimed = it.speed != 1.0 || it.reverse || it.frame_hold.is_some();
let local = origin + it.source_in;
let has_audio = item_media(self.p, base).is_some_and(|m| m.info.audio.is_some()) || (is_seq && audio.is_some());
let has_audio = item_media(self.p, base).is_some_and(|m| m.info.has_audio()) || (is_seq && audio.is_some());
let has_video = item_media(self.p, base).is_none_or(|m| m.info.video.is_some());
let src_enable = match (kind, audio.is_some()) {
(TrackKind::Video, true) => "",

2
crates/interchange/src/omf/tests.rs
View File

@ -17,7 +17,7 @@ fn project() -> (Project, ItemId) {
kind: filmcraft_media::MediaKind::AudioOnly,
duration: FrameRate::FPS_25.tick_of(2500),
video: None,
audio: Some(filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: String::new(), bits_per_sample: Some(16) }),
audio_streams: vec![filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: String::new(), bits_per_sample: Some(16) }],
container: String::new(),
start_timecode: None,
file_size: None,

4
crates/interchange/tests/aaf_omf.rs
View File

@ -132,7 +132,7 @@ fn aaf_round_trip_preserves_the_edit() {
.unwrap();
let ma = a.as_media().unwrap();
assert_eq!(ma.info.start_timecode, Some(rate.timecode_base() * 3600 + 12));
assert!(ma.info.video.is_some() && ma.info.audio.is_some());
assert!(ma.info.video.is_some() && ma.info.has_audio());
assert_eq!(ma.markers.len(), 1);
assert_eq!((ma.markers[0].name.as_str(), ma.markers[0].comment.as_str()), ("slate", "take 3"));
let mus = imp.project.items.values().find(|i| i.name == "music.wav").unwrap();
@ -373,7 +373,7 @@ fn omf_separate_files_and_breakout() {
let k = media_key(&imp.project, c.item);
assert!(k.starts_with("/omf/media/") && k.ends_with(".aif"), "{k}");
let m = imp.project.item(c.item).unwrap().as_media().unwrap();
assert_eq!(m.info.audio.as_ref().map(|a| (a.channels, a.sample_rate)), Some((1, 48_000)));
assert_eq!(m.info.audio().map(|a| (a.channels, a.sample_rate)), Some((1, 48_000)));
}
}
}

5
crates/interchange/tests/support/mod.rs
View File

@ -25,7 +25,10 @@ pub fn media_info(name: &str, video: bool, audio: bool, rate: FrameRate, secs: i
bitrate: None,
hdr: None,
}),
audio: audio.then(|| filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "aac".into(), bits_per_sample: None }),
audio_streams: audio
.then(|| filmcraft_media::AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "aac".into(), bits_per_sample: None })
.into_iter()
.collect(),
container: "mp4".into(),
start_timecode: None,
file_size: None,

2
crates/media/src/generators.rs
View File

@ -108,7 +108,7 @@ impl GeneratorSource {
kind: if matches!(generator, Generator::Demo(_)) { MediaKind::Movie } else { MediaKind::Synthetic },
duration,
video,
audio,
audio_streams: audio.into_iter().collect(),
container: if matches!(generator, Generator::Demo(_)) { "Synthetic MPEG-4".into() } else { "Synthetic".into() },
start_timecode: None,
file_size: None,

38
crates/media/src/lib.rs
View File

@ -77,6 +77,9 @@ pub struct VideoStreamInfo {
pub hdr: Option<filmcraft_color::HdrMetadata>,
}
/// Most audio streams a source reports (the count comes from the file: never trusted).
pub const MAX_AUDIO_STREAMS: usize = 64;
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct AudioStreamInfo {
pub sample_rate: u32,
@ -92,13 +95,32 @@ pub struct MediaInfo {
/// Media duration. Stills report a default duration (the still-image default preference).
pub duration: Tick,
pub video: Option<VideoStreamInfo>,
pub audio: Option<AudioStreamInfo>,
/// Every audio stream the container carries, in file order. Stream 0 is the one
/// [`MediaSource::audio`] reads; the others are read with [`MediaSource::audio_stream`].
/// Project files written before multi-stream support hold a single `audio` object (or
/// `null`): both still load.
#[serde(default, alias = "audio", deserialize_with = "deserialize_audio_streams")]
pub audio_streams: Vec<AudioStreamInfo>,
pub container: String,
/// Timecode of the first frame (in frames at `video.frame_rate`), if the file carries one.
pub start_timecode: Option<i64>,
pub file_size: Option<u64>,
}
/// Reads `audio_streams` from a list, or from the single optional stream older project files wrote.
fn deserialize_audio_streams<'de, D: serde::Deserializer<'de>>(d: D) -> std::result::Result<Vec<AudioStreamInfo>, D::Error> {
#[derive(Deserialize)]
#[serde(untagged)]
enum Streams {
Many(Vec<AudioStreamInfo>),
One(Option<AudioStreamInfo>),
}
Ok(match Streams::deserialize(d)? {
Streams::Many(v) => v,
Streams::One(o) => o.into_iter().collect(),
})
}
impl MediaInfo {
pub fn frame_rate(&self) -> FrameRate {
self.video.as_ref().map(|v| v.frame_rate).unwrap_or_default()
@ -107,7 +129,11 @@ impl MediaInfo {
self.video.is_some()
}
pub fn has_audio(&self) -> bool {
self.audio.is_some()
!self.audio_streams.is_empty()
}
/// The primary audio stream (stream 0), the one [`MediaSource::audio`] reads.
pub fn audio(&self) -> Option<&AudioStreamInfo> {
self.audio_streams.first()
}
}
@ -132,6 +158,12 @@ pub trait MediaSource: Send + Sync {
fn video_frame(&self, req: FrameRequest) -> Result<Arc<VideoFrame>>;
/// `frames` audio frames starting at sample index `start` (at `sample_rate`, media time).
fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer>;
/// Like [`audio`](Self::audio) for audio stream `stream` (an index into
/// [`MediaInfo::audio_streams`]). Sources with a single audio stream need not override this:
/// stream 0 is `audio`, any other index is [`MediaError::NoStream`].
fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer> {
if stream == 0 { self.audio(start, frames, sample_rate) } else { Err(MediaError::NoStream("audio")) }
}
}
pub type SharedSource = Arc<dyn MediaSource>;
@ -195,6 +227,6 @@ impl MediaSource for OfflineSource {
Ok(Arc::new(VideoFrame::rgba8(w, h, px).with_pts(req.time)))
}
fn audio(&self, _start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer> {
Ok(AudioBuffer::silence(sample_rate, self.info.audio.as_ref().map_or(2, |a| a.channels as usize), frames))
Ok(AudioBuffer::silence(sample_rate, self.info.audio().map_or(2, |a| a.channels as usize), frames))
}
}

2
crates/media/src/reader.rs
View File

@ -189,7 +189,7 @@ mod tests {
let small = wav.len() as u64;
let r = Arc::new(Counting(MemReader(Arc::from(wav.clone())), Default::default()));
let src = open_reader_within("a.wav", r.clone(), &[], &[], small).unwrap();
assert_eq!(src.info().audio.as_ref().unwrap().channels, 2);
assert_eq!(src.info().audio().unwrap().channels, 2);
// the same file over the limit: refused after reading the sniffing head only
let r = Arc::new(Counting(MemReader(Arc::from(wav.clone())), Default::default()));
let e = open_reader_within("a.wav", r.clone(), &[], &[], small - 1).err().unwrap();

2
crates/media/src/sequence.rs
View File

@ -147,7 +147,7 @@ impl ImageSequenceSource {
bitrate: None,
hdr: None,
}),
audio: None,
audio_streams: Vec::new(),
container: "Image Sequence".into(),
start_timecode: None,
file_size: None,

2
crates/media/src/still.rs
View File

@ -52,7 +52,7 @@ impl StillSource {
bitrate: None,
hdr: None,
}),
audio: None,
audio_streams: Vec::new(),
container: "Image".into(),
start_timecode: None,
file_size: Some(file_size),

6
crates/media/src/wav.rs
View File

@ -84,12 +84,12 @@ impl WavSource {
kind: MediaKind::AudioOnly,
duration: Tick::from_units(frames as i64, rate as i64),
video: None,
audio: Some(AudioStreamInfo {
audio_streams: vec![AudioStreamInfo {
sample_rate: rate,
channels: ch as u32,
codec: if float { "PCM float".into() } else { "PCM".into() },
bits_per_sample: Some(bits as u32),
}),
}],
container: if time_reference.is_some() { "Broadcast WAV".into() } else { "WAV".into() },
start_timecode: None,
file_size: Some(b.len() as u64),
@ -201,7 +201,7 @@ mod tests {
let s: Vec<f32> = (0..200).map(|i| ((i as f32) * 0.1).sin() * 0.5).collect();
let bytes = write_wav16(&s, 2, 48000);
let src = WavSource::parse("t.wav", bytes.into()).unwrap();
assert_eq!(src.info().audio.as_ref().unwrap().channels, 2);
assert_eq!(src.info().audio().unwrap().channels, 2);
let a = src.audio(0, 100, 48000).unwrap();
assert!((a.channels[0][3] - s[6]).abs() < 1e-4);
assert!((a.channels[1][3] - s[7]).abs() < 1e-4);

1
crates/project/src/essential.rs
View File

@ -783,6 +783,7 @@ mod tests {
hold_filters: false,
field_options: None,
source_channels: Vec::new(),
audio_stream: 0,
graphic: None,
}
}

2
crates/project/src/find.rs
View File

@ -133,7 +133,7 @@ pub fn column_text(p: &Project, it: &ProjectItem, column: &str) -> String {
.and_then(|m| m.info.video.as_ref())
.map(|v| format!("{} x {} ({:.4}) {}", v.width, v.height, v.par.0 as f64 / v.par.1.max(1) as f64, v.codec))
.unwrap_or_default(),
"audio info" => media.and_then(|m| m.info.audio.as_ref()).map(|a| format!("{} Hz - {} ch {}", a.sample_rate, a.channels, a.codec)).unwrap_or_default(),
"audio info" => media.and_then(|m| m.info.audio()).map(|a| format!("{} Hz - {} ch {}", a.sample_rate, a.channels, a.codec)).unwrap_or_default(),
"file path" | "path" => match media.map(|m| &m.media) {
Some(MediaRef::File { path }) => path.clone(),
_ => String::new(),

8
crates/project/src/lib.rs
View File

@ -493,6 +493,10 @@ pub enum AudioChannels {
}
/// A clip instance on a track.
fn is_zero_stream(stream: &usize) -> bool {
*stream == 0
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct TrackItem {
pub id: ClipId,
@ -547,6 +551,9 @@ pub struct TrackItem {
/// left/right). Empty = the first two channels (mono sources on both sides).
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub source_channels: Vec<u16>,
/// Audio stream within the media container (0 = the primary stream).
#[serde(default, skip_serializing_if = "is_zero_stream")]
pub audio_stream: usize,
/// Graphic clips: roll / crawl, responsive time and the template the graphic came from
/// ([`GraphicMeta`]). Schema v12.
#[serde(default, skip_serializing_if = "Option::is_none")]
@ -1526,6 +1533,7 @@ impl Project {
hold_filters: false,
field_options: None,
source_channels,
audio_stream: 0,
graphic: source_graphic.and_then(|sg| sg.meta.map(Box::new)),
})
}

4
crates/render/src/audio.rs
View File

@ -208,7 +208,7 @@ fn mix_item(
if !src.info().has_audio() {
return;
}
effected(item, &|m0, len| src.audio(m0, len, sr).ok(), a0, n, sr, w)
effected(item, &|m0, len| src.audio_stream(item.audio_stream, m0, len, sr).ok(), a0, n, sr, w)
};
// gains: clip gain × Volume (keyframed, per 64-sample block) × channel volume × panner
let clip_gain = db_to_gain(item.gain_db);
@ -328,7 +328,7 @@ pub fn clip_signal(item: &TrackItem, start: i64, frames: usize, sr: u32, sources
let n = (a1 - a0) as usize;
let off = (a0 - start) as usize;
let g = db_to_gain(item.gain_db);
let buf = effected(item, &|m0, len| src.audio(m0, len, sr).ok(), a0, n, sr, 2);
let buf = effected(item, &|m0, len| src.audio_stream(item.audio_stream, m0, len, sr).ok(), a0, n, sr, 2);
for (dst, b) in out.iter_mut().zip(&buf) {
for (d, x) in dst[off..off + n].iter_mut().zip(b) {
*d = x * g;

68
crates/render/src/mixer_tests.rs
View File

@ -58,7 +58,7 @@ impl Rig {
kind: MediaKind::AudioOnly,
duration: Tick(10 * TICKS_PER_SECOND),
video: None,
audio: Some(AudioStreamInfo { sample_rate: 48_000, channels: ch as u32, codec: "test".into(), bits_per_sample: Some(32) }),
audio_streams: vec![AudioStreamInfo { sample_rate: 48_000, channels: ch as u32, codec: "test".into(), bits_per_sample: Some(32) }],
container: "test".into(),
start_timecode: None,
file_size: None,
@ -128,6 +128,70 @@ fn close(a: f32, b: f32, tol: f32) -> bool {
(a - b).abs() <= tol
}
/// The renderer must request the stream selected by each clip, rather than reading stream zero
/// twice when a container item has clips on two audio tracks.
#[test]
fn audio_clips_from_one_container_mix_distinct_streams() {
struct TwoStreams(MediaInfo);
impl MediaSource for TwoStreams {
fn info(&self) -> &MediaInfo {
&self.0
}
fn video_frame(&self, _: FrameRequest) -> filmcraft_media::Result<Arc<VideoFrame>> {
Err(MediaError::NoStream("video"))
}
fn audio(&self, start: i64, frames: usize, sr: u32) -> filmcraft_media::Result<AudioBuffer> {
self.audio_stream(0, start, frames, sr)
}
fn audio_stream(&self, stream: usize, _start: i64, frames: usize, sr: u32) -> filmcraft_media::Result<AudioBuffer> {
let v = match stream {
0 => 0.25,
1 => -0.125,
_ => return Err(MediaError::NoStream("audio")),
};
Ok(AudioBuffer { sample_rate: sr, channels: vec![vec![v; frames], vec![v; frames]] })
}
}
let mut p = Project::new("streams");
let seq = p.new_sequence("mix", SequenceSettings::default(), 0, 2, None);
let mut info = MediaInfo {
name: "two.mov".into(),
kind: MediaKind::AudioOnly,
duration: Tick(TICKS_PER_SECOND),
video: None,
audio_streams: vec![AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "PCM".into(), bits_per_sample: Some(16) }],
container: "MOV".into(),
start_timecode: None,
file_size: None,
};
info.audio_streams.push(info.audio_streams[0].clone());
let id = p.add_item(
"two.mov",
Label::Iris,
ItemKind::Media(MediaClip {
media: MediaRef::File { path: "two.mov".into() },
info: info.clone(),
interpret: Default::default(),
mark_in: None,
mark_out: None,
markers: vec![],
offline: false,
proxy: None,
identity: None,
}),
None,
);
let mut map = SourceMap::default();
map.0.insert(id, Arc::new(TwoStreams(info)));
for stream in 0..2 {
let mut clip = p.make_track_item(id, TrackKind::Audio, Tick::ZERO, TimeRange::new(Tick::ZERO, Tick(TICKS_PER_SECOND)), FrameRate::FPS_24).unwrap();
clip.audio_stream = stream;
p.sequence_mut(seq).unwrap().audio_tracks[stream].items.push(clip);
}
let buffer = crate::audio::mix_sequence(&p, p.sequence(seq).unwrap(), 0, 32, &map);
assert!(buffer.channels.iter().all(|c| c.iter().all(|&v| close(v, 0.125, 1e-6))));
}
#[test]
fn fader_gain_is_sample_exact() {
let mut r = Rig::new(&[dc]);
@ -666,7 +730,7 @@ fn nested_tone(inner_rate: u32) -> (Project, ItemId, SourceMap, Arc<Tone>) {
kind: MediaKind::AudioOnly,
duration: ten.duration,
video: None,
audio: Some(AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "test".into(), bits_per_sample: Some(32) }),
audio_streams: vec![AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: "test".into(), bits_per_sample: Some(32) }],
container: "test".into(),
start_timecode: None,
file_size: None,

3
crates/ui-egui/src/frames.rs
View File

@ -234,6 +234,9 @@ impl filmcraft_media::MediaSource for TimedSource {
fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> filmcraft_media::Result<filmcraft_frame::AudioBuffer> {
self.0.audio(start, frames, sample_rate)
}
fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> filmcraft_media::Result<filmcraft_frame::AudioBuffer> {
self.0.audio_stream(stream, start, frames, sample_rate)
}
}
/// The job's source provider: the pool, with source fetches timed (the time feeds

2
crates/ui-egui/src/panels/clip_dialogs.rs
View File

@ -155,7 +155,7 @@ fn defaults(app: &FilmcraftApp, id: &str) -> (Value, Value) {
let item = s.state.project_selection.iter().find_map(|i| s.project.item(*i).and_then(|it| it.as_media()).filter(|m| m.info.has_audio()));
match item {
Some(m) => {
let n = m.info.audio.as_ref().map_or(2, |a| a.channels) as u16;
let n = m.info.audio().map_or(2, |a| a.channels) as u16;
let map = m.interpret.audio_channels.clone().unwrap_or_else(|| AudioChannelMap::for_format(AudioChannels::Stereo, n));
(json!({"format": format_name(map.format), "clips": map.clips}), json!({"channels": n, "items": true}))
}

2
crates/ui-egui/src/panels/misc.rs
View File

@ -151,7 +151,7 @@ pub fn info(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) {
line(ui, "Video:", format!("{} fps, {} x {} ({:.4})", v.frame_rate.label(), v.width, v.height, v.par.0 as f32 / v.par.1 as f32));
line(ui, "Codec:", v.codec.clone());
}
if let Some(a) = &m.info.audio {
if let Some(a) = m.info.audio() {
line(ui, "Audio:", format!("{} Hz - {} ch - {}", a.sample_rate, a.channels, a.codec));
}
}

2
crates/ui-egui/src/panels/timecode.rs
View File

@ -44,7 +44,7 @@ pub fn row_text(app: &FilmcraftApp, row: &TimecodeRow) -> (String, String) {
ItemKind::Sequence(q) => (q.mark_in, q.mark_out),
_ => (None, None),
};
let sr = it.as_media().and_then(|m| m.info.audio.as_ref()).map(|a| a.sample_rate as i64).unwrap_or(48000);
let sr = it.as_media().and_then(|m| m.info.audio()).map(|a| a.sample_rate as i64).unwrap_or(48000);
View {
name: it.name.clone(),
rate: it.frame_rate(),

3
docs/project-files.md
View File

@ -8,7 +8,7 @@ writing them is `crates/format` (`filmcraft-format`); the engine's `file.*` comm
```json
{
"format": "filmcraft.project",
"schema_version": 12,
"schema_version": 13,
"generator": "FilmCraft 0.1.0",
"project": { "name": "…", "settings": { … }, "root": { … }, "items": { … }, "next_id": 48 },
"view": { "open_sequences": [21, 50], "active_sequence": 50, "sequences": { "21": { "pps": 40.0, "scroll": 0.0, … } } }
@ -52,6 +52,7 @@ writing them is `crates/format` (`filmcraft-format`); the engine's `file.*` comm
| 10 | M3.10 | clip time interpolation, Hold Filters, Field Options, audio source channels, Modify ▸ Audio Channels map, subclip Restrict Trims; no-op step |
| 11 | M3.11 | search bins (`project.search_bins`), Flash Cue markers, Project Settings safe areas, capture format and scratch disks; no-op step |
| 12 | M10.7 | graphics design data: `TrackItem::graphic` (roll / crawl, responsive time, template link), `EffectInstance::layer` (layer uid, per-character styles, responsive pins), `project.source_graphics`; no-op step |
| 13 | Multi-stream audio | `MediaInfo::audio_streams` and `TrackItem::audio_stream`; older `audio` objects load as stream zero, missing clip indices default to zero; no-op step |
### Migrations