#823: import every MPEG audio stream, not only the first PID
Transport and program streams listed a single audio stream, so file.import and New Sequence From Clip kept one clip. Probe every audio PID (capped), decode each independently, and place linked clips per stream. MP4/MKV already did this; MXF same-rate PCM is still interleaved as one stream.
This commit is contained in:
Vikram Bhaskaran
parent
4e2d0bc2d3
commit
f733230b9f
|
|
@ -1890,6 +1890,7 @@ dependencies = [
|
|||
"filmcraft-frame",
|
||||
"filmcraft-geom",
|
||||
"filmcraft-interchange",
|
||||
"filmcraft-isobmff",
|
||||
"filmcraft-media",
|
||||
"filmcraft-mxf",
|
||||
"filmcraft-project",
|
||||
|
|
|
|||
|
|
@ -7,8 +7,9 @@
|
|||
//! first random-access picture; display order comes from the PTS (or, without timestamps, the
|
||||
//! GOP and temporal reference). Leading pictures of an open GOP decode from the previous GOP.
|
||||
//! - Audio: MPEG audio layers I-III (symphonia), AAC in ADTS or LATM (our decoder), AC-3 (our
|
||||
//! decoder), Blu-ray / AVCHD and DVD LPCM. Audio frames are placed on the timeline from their
|
||||
//! PTS relative to the first video frame, sample-contiguously (re-anchored only at gaps).
|
||||
//! decoder), Blu-ray / AVCHD and DVD LPCM. Every audio PID is a stream (capped). Audio frames
|
||||
//! are placed on the timeline from their PTS relative to the first video frame,
|
||||
//! sample-contiguously (re-anchored only at gaps).
|
||||
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
|
@ -104,7 +105,8 @@ pub struct MpegSource {
|
|||
es: Option<(Arc<[u8]>, Vec<std::ops::Range<usize>>)>,
|
||||
video: Option<VideoTrack>,
|
||||
gop: GopCache,
|
||||
audio: Option<AudioTrack>,
|
||||
/// Playable audio tracks in file order: `info.audio_streams[k]` describes `audios[k]`.
|
||||
audios: Vec<AudioTrack>,
|
||||
/// Why the video or audio stream cannot be decoded.
|
||||
unsupported_video: Option<String>,
|
||||
unsupported_audio: Option<String>,
|
||||
|
|
@ -201,8 +203,8 @@ impl MpegSource {
|
|||
log::warn!("{name}: {w}");
|
||||
}
|
||||
let vstream = file.find(Kind::Video, |c| vkind(c).is_some()).or_else(|| file.find(Kind::Video, |_| true));
|
||||
let astream = file.find(Kind::Audio, audio_supported).or_else(|| file.find(Kind::Audio, |_| true));
|
||||
if vstream.is_none() && astream.is_none() {
|
||||
let has_audio = file.streams.iter().any(|s| s.kind() == Kind::Audio && !s.units.is_empty());
|
||||
if vstream.is_none() && !has_audio {
|
||||
return Err(CodecError::Unsupported(format!("{}: no video or audio streams", file.format.name())));
|
||||
}
|
||||
let mut src = MpegSource {
|
||||
|
|
@ -221,7 +223,7 @@ impl MpegSource {
|
|||
es: None,
|
||||
video: None,
|
||||
gop: GopCache::new(None),
|
||||
audio: None,
|
||||
audios: Vec::new(),
|
||||
unsupported_video: None,
|
||||
unsupported_audio: None,
|
||||
};
|
||||
|
|
@ -256,7 +258,7 @@ impl MpegSource {
|
|||
src.video = Some(track);
|
||||
src.describe_video(rate_from_pts)?;
|
||||
if let Some(file) = src.file.take() {
|
||||
src.setup_audio(file, astream, origin);
|
||||
src.setup_audio(file, origin);
|
||||
}
|
||||
}
|
||||
None => {
|
||||
|
|
@ -273,12 +275,12 @@ impl MpegSource {
|
|||
bitrate: None,
|
||||
hdr: None,
|
||||
});
|
||||
src.setup_audio(file, astream, None);
|
||||
src.setup_audio(file, None);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
src.info.kind = MediaKind::AudioOnly;
|
||||
src.setup_audio(file, astream, None);
|
||||
src.setup_audio(file, None);
|
||||
}
|
||||
Ok(src)
|
||||
}
|
||||
|
|
@ -326,7 +328,7 @@ impl MpegSource {
|
|||
es: Some((all, aus)),
|
||||
video: Some(track),
|
||||
gop: GopCache::new(None),
|
||||
audio: None,
|
||||
audios: Vec::new(),
|
||||
unsupported_video: None,
|
||||
unsupported_audio: None,
|
||||
};
|
||||
|
|
@ -347,15 +349,15 @@ impl MpegSource {
|
|||
/// Timeline position (samples at the audio stream's rate) of the first audio frame, and the
|
||||
/// number of samples in the audio stream.
|
||||
pub fn audio_extent(&self) -> Option<(i64, i64)> {
|
||||
let a = self.audio.as_ref()?;
|
||||
let a = self.audios.first()?;
|
||||
let first = *a.starts.first()?;
|
||||
let end = a.starts.last()? + *a.lens.last()? as i64;
|
||||
Some((first, end - first))
|
||||
}
|
||||
|
||||
/// The demuxed stream indices of the presented video and audio.
|
||||
/// The demuxed stream indices of the presented video and first audio.
|
||||
pub fn stream_indices(&self) -> (Option<usize>, Option<usize>) {
|
||||
(self.video.as_ref().filter(|_| self.es.is_none()).map(|v| v.stream), self.audio.as_ref().map(|a| a.stream))
|
||||
(self.video.as_ref().filter(|_| self.es.is_none()).map(|v| v.stream), self.audios.first().map(|a| a.stream))
|
||||
}
|
||||
|
||||
/// Presentation timestamps (90 kHz) of the presented video frames in display order. Frames
|
||||
|
|
@ -475,15 +477,31 @@ impl MpegSource {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Choose and index the audio stream; `origin`: PTS of the first video frame.
|
||||
fn setup_audio(&mut self, file: File, astream: Option<usize>, origin: Option<i64>) {
|
||||
let track = astream.and_then(|ai| self.audio_track(&file, ai, origin));
|
||||
/// Index every audio stream the container carries (capped); `origin`: PTS of the first video frame.
|
||||
fn setup_audio(&mut self, file: File, origin: Option<i64>) {
|
||||
let indices: Vec<usize> = file
|
||||
.streams
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, s)| s.kind() == Kind::Audio && !s.units.is_empty())
|
||||
.map(|(i, _)| i)
|
||||
.take(filmcraft_media::MAX_AUDIO_STREAMS)
|
||||
.collect();
|
||||
let mut tracks = Vec::new();
|
||||
for ai in indices {
|
||||
if let Some(t) = self.audio_track(&file, ai, origin) {
|
||||
tracks.push(t);
|
||||
}
|
||||
}
|
||||
self.file = Some(file);
|
||||
self.audio = track;
|
||||
self.audios = tracks;
|
||||
}
|
||||
|
||||
/// Mark the audio stream as not decodable (it still shows in the media info).
|
||||
/// Mark the audio stream as not decodable (it still shows in the media info when nothing plays).
|
||||
fn audio_unsupported(&mut self, codec_name: &str, why: String) -> Option<AudioTrack> {
|
||||
if !self.info.audio_streams.is_empty() {
|
||||
return None;
|
||||
}
|
||||
self.unsupported_audio = Some(why);
|
||||
self.info.audio_streams =
|
||||
vec![AudioStreamInfo { sample_rate: 48_000, channels: 2, codec: format!("{codec_name} (unsupported)"), bits_per_sample: None }];
|
||||
|
|
@ -594,14 +612,16 @@ impl MpegSource {
|
|||
next = Some(s + n as i64);
|
||||
}
|
||||
let total = next.unwrap_or(0).max(0);
|
||||
if self.info.video.is_none() || self.unsupported_video.is_some() {
|
||||
if self.unsupported_audio.take().is_some() {
|
||||
self.info.audio_streams.clear();
|
||||
}
|
||||
if self.info.audio_streams.is_empty() && (self.info.video.is_none() || self.unsupported_video.is_some()) {
|
||||
self.info.duration = Tick::from_units(total, rate.max(1) as i64);
|
||||
if self.info.video.is_none() {
|
||||
self.info.kind = MediaKind::AudioOnly;
|
||||
}
|
||||
}
|
||||
self.info.audio_streams =
|
||||
vec![AudioStreamInfo { sample_rate: rate.max(1), channels: channels.max(1) as u32, codec: codec_name, bits_per_sample: bits }];
|
||||
self.info.audio_streams.push(AudioStreamInfo { sample_rate: rate.max(1), channels: channels.max(1) as u32, codec: codec_name, bits_per_sample: bits });
|
||||
Some(AudioTrack {
|
||||
stream: ai,
|
||||
codec,
|
||||
|
|
@ -783,10 +803,16 @@ impl MediaSource for MpegSource {
|
|||
}
|
||||
|
||||
fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer, MediaError> {
|
||||
if let Some(why) = &self.unsupported_audio {
|
||||
self.audio_stream(0, start, frames, sample_rate)
|
||||
}
|
||||
|
||||
fn audio_stream(&self, stream: usize, start: i64, frames: usize, sample_rate: u32) -> Result<AudioBuffer, MediaError> {
|
||||
if self.audios.is_empty()
|
||||
&& let Some(why) = &self.unsupported_audio
|
||||
{
|
||||
return Err(MediaError::Unsupported(why.clone()));
|
||||
}
|
||||
let a = self.audio.as_ref().ok_or(MediaError::NoStream("audio"))?;
|
||||
let a = self.audios.get(stream).ok_or(MediaError::NoStream("audio"))?;
|
||||
let ch = a.channels;
|
||||
let ratio = a.rate as f64 / sample_rate as f64;
|
||||
let exact = a.rate == sample_rate;
|
||||
|
|
@ -852,6 +878,7 @@ pub fn reader_opener(name: &str, head: &[u8], reader: &filmcraft_media::SharedRe
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use filmcraft_media::MediaSource;
|
||||
|
||||
/// A mutated transport stream whose PTS span gave ~0 fps panicked on open with "attempt to
|
||||
/// divide by zero" (`FrameRate::tick_of`).
|
||||
|
|
@ -868,6 +895,118 @@ mod tests {
|
|||
assert_eq!(stream_rate(None, Some(50.0)), FrameRate::from_f64(50.0));
|
||||
}
|
||||
|
||||
/// MPEG-1 layer II frame, 48 kHz 192 kb/s (576 bytes).
|
||||
fn mp2_frame(k: u8) -> Vec<u8> {
|
||||
let mut v = vec![0xFF, 0xFD, 0xA4, 0x00];
|
||||
v.extend((0..572).map(|i| (i as u8) ^ k));
|
||||
v
|
||||
}
|
||||
|
||||
fn ts_bytes(prefix: u8, v: i64) -> [u8; 5] {
|
||||
let v = v & ((1 << 33) - 1);
|
||||
[(prefix << 4) | (((v >> 30) & 7) as u8) << 1 | 1, (v >> 22) as u8, (((v >> 15) & 0x7F) as u8) << 1 | 1, (v >> 7) as u8, ((v & 0x7F) as u8) << 1 | 1]
|
||||
}
|
||||
|
||||
fn pes(stream_id: u8, pts: Option<i64>, payload: &[u8]) -> Vec<u8> {
|
||||
let mut h = Vec::new();
|
||||
if let Some(p) = pts {
|
||||
h.extend(ts_bytes(2, p));
|
||||
}
|
||||
let flags = if pts.is_some() { 0x80 } else { 0 };
|
||||
let len = 3 + h.len() + payload.len();
|
||||
let mut v = vec![0, 0, 1, stream_id, (len >> 8) as u8, len as u8, 0x80, flags, h.len() as u8];
|
||||
v.extend(h);
|
||||
v.extend_from_slice(payload);
|
||||
v
|
||||
}
|
||||
|
||||
/// Tiny TS muxer: PAT, PMT and PES packets in 188-byte packets.
|
||||
struct TsMux {
|
||||
out: Vec<u8>,
|
||||
cc: std::collections::HashMap<u16, u8>,
|
||||
}
|
||||
|
||||
impl TsMux {
|
||||
fn packet(&mut self, pid: u16, pusi: bool, payload: &[u8]) -> usize {
|
||||
let cc = self.cc.entry(pid).or_insert(0);
|
||||
let mut p = vec![0x47, (pusi as u8) << 6 | (pid >> 8) as u8, pid as u8];
|
||||
let n = payload.len().min(184);
|
||||
if n < 184 {
|
||||
let al = 183 - n;
|
||||
p.push(0x30 | *cc);
|
||||
p.push(al as u8);
|
||||
if al > 0 {
|
||||
p.push(0);
|
||||
p.extend(std::iter::repeat_n(0xFF, al - 1));
|
||||
}
|
||||
} else {
|
||||
p.push(0x10 | *cc);
|
||||
}
|
||||
*cc = (*cc + 1) & 15;
|
||||
p.extend_from_slice(&payload[..n]);
|
||||
self.out.extend(&p);
|
||||
n
|
||||
}
|
||||
fn section(&mut self, pid: u16, body: &[u8]) {
|
||||
let mut s = body.to_vec();
|
||||
s.extend(crc32_mpeg(&s).to_be_bytes());
|
||||
let mut payload = vec![0];
|
||||
payload.extend(s);
|
||||
self.packet(pid, true, &payload);
|
||||
}
|
||||
fn psi(&mut self, streams: &[(u8, u16)]) {
|
||||
let pat = [0x00, 0xB0, 13, 0, 1, 0xC1, 0, 0, 0, 1, 0xE1, 0x00];
|
||||
self.section(0, &pat);
|
||||
let mut pmt = vec![0x02, 0xB0, 0, 0, 1, 0xC1, 0, 0, 0xE1, 0x00, 0xF0, 0];
|
||||
for &(st, pid) in streams {
|
||||
pmt.extend([st, 0xE0 | (pid >> 8) as u8, pid as u8, 0xF0, 0]);
|
||||
}
|
||||
let len = pmt.len() - 3 + 4;
|
||||
pmt[1] = 0xB0 | (len >> 8) as u8;
|
||||
pmt[2] = len as u8;
|
||||
self.section(0x100, &pmt);
|
||||
}
|
||||
fn pes(&mut self, pid: u16, data: &[u8]) {
|
||||
let mut rest = data;
|
||||
let mut first = true;
|
||||
while !rest.is_empty() {
|
||||
let n = self.packet(pid, first, rest);
|
||||
rest = &rest[n..];
|
||||
first = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn crc32_mpeg(data: &[u8]) -> u32 {
|
||||
let mut crc = 0xFFFF_FFFFu32;
|
||||
for &b in data {
|
||||
crc ^= (b as u32) << 24;
|
||||
for _ in 0..8 {
|
||||
crc = if crc & 0x8000_0000 != 0 { (crc << 1) ^ 0x04C1_1DB7 } else { crc << 1 };
|
||||
}
|
||||
}
|
||||
crc
|
||||
}
|
||||
|
||||
/// Dual MPEG audio in one transport stream: import used to keep only the first PID.
|
||||
#[test]
|
||||
fn every_audio_pid_is_a_stream() {
|
||||
let mut mux = TsMux { out: Vec::new(), cc: Default::default() };
|
||||
mux.psi(&[(0x03, 0x1100), (0x03, 0x1101)]);
|
||||
for k in 0..4u16 {
|
||||
mux.pes(0x1100, &pes(0xC0, Some(90_000 + 2160 * k as i64), &mp2_frame(k as u8)));
|
||||
mux.pes(0x1101, &pes(0xC1, Some(90_000 + 2160 * k as i64), &mp2_frame(k as u8 + 16)));
|
||||
}
|
||||
let source = MpegSource::open("dual.ts", mux.out.into()).unwrap();
|
||||
assert_eq!(source.info().audio_streams.len(), 2, "{:?}", source.info().audio_streams);
|
||||
assert_eq!(source.info().audio_streams.iter().map(|a| a.sample_rate).collect::<Vec<_>>(), vec![48_000, 48_000]);
|
||||
let a0 = source.audio(0, 64, 48_000).unwrap();
|
||||
let a1 = source.audio_stream(1, 0, 64, 48_000).unwrap();
|
||||
assert_eq!(a0.channels.len(), 2);
|
||||
assert_eq!(a1.channels.len(), 2);
|
||||
assert!(source.audio_stream(2, 0, 64, 48_000).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn presentation_drops_leading_pictures_and_orders_by_key() {
|
||||
// decode order I(2) B(0) B(1) P(5) B(3) B(4): the leading B pictures are not presented
|
||||
|
|
|
|||
|
|
@ -46,4 +46,5 @@ web-time = "1"
|
|||
|
||||
[dev-dependencies]
|
||||
filmcraft-testkit = { workspace = true }
|
||||
filmcraft-isobmff = { workspace = true }
|
||||
image = { version = "0.25", default-features = false, features = ["png"] }
|
||||
|
|
|
|||
|
|
@ -168,6 +168,32 @@ fn missing_audio_tracks_are_cloned_from_the_destination() {
|
|||
assert_eq!(again, ids);
|
||||
}
|
||||
|
||||
/// #823: a container with two PCM tracks imports both streams, and overwrite places linked clips.
|
||||
#[test]
|
||||
fn import_keeps_every_audio_stream_and_places_linked_clips() {
|
||||
use filmcraft_isobmff::{Brand, Mp4Writer, PcmConfig, SampleEntry, TrackConfig, WriteSample, WriterOptions};
|
||||
let mut mux = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mov)).unwrap();
|
||||
for (rate, value) in [(48_000u32, 8192i16), (24_000, -16384)] {
|
||||
let entry = SampleEntry::pcm(PcmConfig { bits: 16, float: false, big_endian: false, signed: true, channels: 1, sample_rate: rate as f64 });
|
||||
let track = mux.add_track(TrackConfig::new(entry, rate)).unwrap();
|
||||
let data: Vec<u8> = (0..rate / 10).flat_map(|_| value.to_le_bytes()).collect();
|
||||
mux.write_sample(track, WriteSample { data: &data, duration: rate / 10, composition_offset: 0, is_sync: true }).unwrap();
|
||||
}
|
||||
let bytes = mux.finish().unwrap().into_inner();
|
||||
let mut s = Session::default();
|
||||
let item = crate::commands::import_bytes(&mut s, "/two-audio.mov", bytes.into(), None).unwrap();
|
||||
let streams = s.project.item(item).unwrap().as_media().unwrap().info.audio_streams.len();
|
||||
assert_eq!(streams, 2);
|
||||
s.execute("file.newSequence", json!({"name": "S", "fromItem": item.0, "video": 1, "audio": 1})).unwrap();
|
||||
let seq = s.state.active_sequence.unwrap();
|
||||
let q = s.project.sequence(seq).unwrap();
|
||||
assert_eq!(q.audio_tracks.len(), 2);
|
||||
let (a, b) = (&q.audio_tracks[0].items[0], &q.audio_tracks[1].items[0]);
|
||||
assert_eq!((a.audio_stream, b.audio_stream), (0, 1));
|
||||
assert_eq!(a.link, b.link);
|
||||
assert!(a.link.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_track_counts_are_an_error_not_a_crash() {
|
||||
let s = demo();
|
||||
|
|
|
|||
Loading…
Reference in New Issue