diff --git a/crates/codecs/src/audio.rs b/crates/codecs/src/audio.rs index 63351636..daba8f4c 100644 --- a/crates/codecs/src/audio.rs +++ b/crates/codecs/src/audio.rs @@ -20,6 +20,9 @@ enum Inner { Aac { dec: Box, asc: Vec, up: Option> }, /// AAC whose configuration arrives with the first frame (ADTS / LATM). AacPending, + /// MPEG audio at this rate whose layer arrives with the first frame (MP4 / QuickTime name the + /// codec, not the layer, and symphonia's decoder reads only the layer it was made for). + MpegPending(u32), /// Our own Opus decoder (always 48 kHz output; pre-skip is left to container timestamps). /// `order` maps output channel → decoded channel (Vorbis → WAV/SMPTE order for surround). Opus { dec: Box, order: Option<&'static [usize]> }, @@ -133,6 +136,12 @@ impl PacketDecoder { }; Self::new(codec, sample_rate, None) } + /// MPEG-1/2 audio of a layer the frame headers name (see [`mpeg_audio_layer`]): MP4's + /// `esds` object types 0x69 / 0x6B and QuickTime's `.mp3` / `.mp2` cover layers I–III alike + /// (ffmpeg stores MP2 as `mp4a` + 0x6B). + pub fn lazy_mpeg_audio(sample_rate: u32) -> Self { + Self { inner: Inner::MpegPending(sample_rate), channels: 0 } + } /// AAC configured by [`Self::ensure_aac`] from in-band headers (ADTS, LATM). pub fn lazy_aac() -> Self { Self { inner: Inner::AacPending, channels: 0 } @@ -160,7 +169,7 @@ impl PacketDecoder { use filmcraft_isobmff::CodecConfig as C; match c { C::Aac(a) => Self::aac(&a.asc, if rate > 0 { rate } else { a.sample_rate }), - C::Mp3 => Self::new(CODEC_TYPE_MP3, rate, None), + C::Mp3 => Ok(Self::lazy_mpeg_audio(rate)), C::Ac3 { .. } | C::Eac3 { .. } => Self::ac3(), C::Alac { cookie } => Self::new(CODEC_TYPE_ALAC, rate, Some(cookie.clone())), C::Flac(_) => Self::new(CODEC_TYPE_FLAC, rate, None), @@ -170,6 +179,10 @@ impl PacketDecoder { } /// Decode one packet into planar channels. pub fn decode(&mut self, data: &[u8], ts: u64) -> Result>> { + if let Inner::MpegPending(rate) = self.inner { + let layer = mpeg_audio_layer(data).ok_or_else(|| CodecError::Decode("MPEG audio: no frame header".into()))?; + self.inner = Self::mpeg_audio(layer, rate)?.inner; + } let dec = match &mut self.inner { Inner::Aac { dec, up, .. } => { let mut out = dec.decode(data).map_err(|e| CodecError::Decode(e.to_string()))?; @@ -192,6 +205,7 @@ impl PacketDecoder { } Inner::Symphonia(d) => d, Inner::AacPending => return Err(CodecError::Decode("AAC: no configuration yet".into())), + Inner::MpegPending(_) => return Err(CodecError::Decode("MPEG audio: no frame header".into())), Inner::Ac3(d) => { // a packet may hold several syncframes (MP4 / Matroska); E-AC-3 dependent // substreams and further programs decode to no channels and are skipped @@ -245,11 +259,21 @@ impl PacketDecoder { Inner::Opus { dec, .. } => dec.reset(), Inner::Symphonia(d) => d.reset(), Inner::Ac3(d) => d.reset(), - Inner::AacPending => {} + Inner::AacPending | Inner::MpegPending(_) => {} } } } +/// The layer (1, 2 or 3) named by the MPEG-1/2 audio frame header that starts `frame`: the 12-bit +/// sync word (11 bits for MPEG-2.5), then `ID` and the 2-bit `layer` field, 3 for layer I down to 1 +/// for layer III (ISO/IEC 11172-3 §2.4.2.3). `None` without a sync word or for the reserved layer 0. +pub fn mpeg_audio_layer(frame: &[u8]) -> Option { + match *frame { + [0xFF, b1, ..] if b1 & 0xE0 == 0xE0 && (b1 >> 1) & 3 != 0 => Some(4 - ((b1 >> 1) & 3)), + _ => None, + } +} + /// The output rate of an AAC stream: twice the core rate for HE-AAC (ISO/IEC 14496-3 §1.6.5). /// Explicit signalling names the rate in the `AudioSpecificConfig`; with implicit signalling (an /// AAC-LC config at 24 kHz or less) SBR is only visible as fill-element data in the access units, @@ -1000,3 +1024,49 @@ mod packet_time_tests { assert_eq!(fixed_packet_samples(FixedFrames::Eac3, &[0x0B, 0x77], 48_000), None); } } + +#[cfg(test)] +mod mpeg_audio_tests { + use super::*; + + /// A silent mono 48 kHz MPEG-1 frame of `layer` (2: 64 kbps, 192 bytes; 3: 56 kbps, 168 + /// bytes): header, then all-zero bit allocation / side information. + fn silent_frame(layer: u8) -> Vec { + let (b1, len) = if layer == 2 { (0xFD, 192) } else { (0xFB, 168) }; + let mut f = vec![0xFF, b1, 0x44, 0xC0]; + f.resize(len, 0); + f + } + + #[test] + fn layer_comes_from_the_frame_header() { + assert_eq!(mpeg_audio_layer(&[0xFF, 0xFF, 0x10, 0xC0]), Some(1)); + assert_eq!(mpeg_audio_layer(&silent_frame(2)), Some(2)); + assert_eq!(mpeg_audio_layer(&silent_frame(3)), Some(3)); + // MPEG-2 layer III, MPEG-2.5 layer II + assert_eq!(mpeg_audio_layer(&[0xFF, 0xF3, 0x90, 0x44]), Some(3)); + assert_eq!(mpeg_audio_layer(&[0xFF, 0xE5, 0x90, 0x44]), Some(2)); + // reserved layer, no sync word, too short + for bad in [&[0xFF, 0xF9, 0x44, 0xC0][..], &[0xFF, 0x1D], &[0x00, 0xFD], &[0xFF], &[]] { + assert_eq!(mpeg_audio_layer(bad), None, "{bad:?}"); + } + } + + /// #800: ffmpeg stores MP2 in MP4 as `mp4a` + object type 0x6B, which names no layer. Handing + /// it to the layer III decoder rejected every frame and the track played as silence. + #[test] + fn mp4_mpeg_audio_decodes_the_layer_its_frames_name() { + for layer in [2u8, 3] { + let mut d = PacketDecoder::for_isobmff(&filmcraft_isobmff::CodecConfig::Mp3, 48_000).unwrap(); + // a packet without a frame header is an error, not a crash, and configures nothing + assert!(d.decode(&[0u8; 16], 0).is_err()); + d.reset(); + for k in 0..3u64 { + let out = d.decode(&silent_frame(layer), k * 1152).unwrap_or_else(|e| panic!("layer {layer}: {e}")); + assert_eq!(out.len(), 1, "layer {layer}"); + assert_eq!(out[0].len(), 1152, "layer {layer}"); + assert!(out[0].iter().all(|&s| s == 0.0), "layer {layer}"); + } + } + } +} diff --git a/crates/codecs/src/mp4.rs b/crates/codecs/src/mp4.rs index 4a3a2c53..c7ea9d06 100644 --- a/crates/codecs/src/mp4.rs +++ b/crates/codecs/src/mp4.rs @@ -547,7 +547,18 @@ fn audio_stream_info(t: &filmcraft_isobmff::Track, bytes: &crate::Src) -> Option CodecConfig::Opus(o) => (crate::audio::OPUS_RATE, (o.output_channels as u32).max(1), None), _ => (if ap.sample_rate > 0.0 { ap.sample_rate as u32 } else { t.timescale }, ap.channels.max(1), None), }; - Some(AudioStreamInfo { sample_rate: rate.max(1), channels: ch.max(1), codec: codec_label(&entry.codec), bits_per_sample: bits }) + let codec = match &entry.codec { + // these entries name no layer (ffmpeg stores MP2 as `mp4a` + 0x6B): the first frame does + CodecConfig::Mp3 => { + let head = t.samples.first().and_then(|s| filmcraft_media::reader::read_range(&*bytes.0, s.offset, (s.size as usize).min(4)).ok()); + match head.as_deref().and_then(crate::audio::mpeg_audio_layer) { + Some(1 | 2) => "MPEG Audio".into(), + _ => codec_label(&entry.codec), + } + } + c => codec_label(c), + }; + Some(AudioStreamInfo { sample_rate: rate.max(1), channels: ch.max(1), codec, bits_per_sample: bits }) } fn codec_label(c: &CodecConfig) -> String { diff --git a/crates/codecs/tests/mp4_oracle.rs b/crates/codecs/tests/mp4_oracle.rs index 012424d7..cbb71ba0 100644 --- a/crates/codecs/tests/mp4_oracle.rs +++ b/crates/codecs/tests/mp4_oracle.rs @@ -215,3 +215,35 @@ fn open_gop_leading_pictures_decode_alone() { } } } + +/// #800: MP2 in MP4 (`mp4a` + object type 0x6B, which names no layer) and in MOV (QuickTime +/// `.mp2`) decodes sample-exactly as ffmpeg does: same length, the encoder delay (481 samples) +/// trimmed by the edit list, and the same samples from the first one on, also after a seek. It +/// used to go to the layer III decoder (MP4: silence) or not decode at all (MOV). +#[test] +fn mp2_in_mp4_and_mov_matches_ffmpeg() { + let ff = filmcraft_testkit::require_ffmpeg!(); + for (name, rate) in [("mp2_stereo_224k_48k.mp4", 48_000u32), ("mp2_stereo_224k_44k.mov", 44_100)] { + let input = format!("anoisesrc=d=1:c=pink:r={rate}:a=0.3"); + let Some(f) = fixture(&ff, name, &["-f", "lavfi", "-i", &input, "-ac", "2", "-c:a", "mp2", "-b:a", "224k"]) else { + eprintln!("SKIPPED ({name}): this ffmpeg can't encode it"); + continue; + }; + let src = filmcraft_codecs::open_bytes(name, bytes(&f)).unwrap(); + let a = src.info().audio().cloned().unwrap(); + assert_eq!((a.sample_rate, a.channels, a.codec.as_str()), (rate, 2, "MPEG Audio"), "{name}"); + let want = ffmpeg_audio_f32(&ff, &f, &[]); + let n = want.len() / 2; + assert!(n > rate as usize * 9 / 10, "{name}: ffmpeg decoded {n} samples"); + let worst = |from: usize, len: usize| { + let got = src.audio(from as i64, len, rate).unwrap(); + (0..len).map(|k| (0..2).map(|c| (got.channels[c][k] - want[2 * (from + k) + c]).abs()).fold(0f32, f32::max)).fold(0f32, f32::max) + }; + let level = want.iter().fold(0f32, |m, v| m.max(v.abs())); + assert!(level > 0.1, "{name}: the source is silent ({level})"); + // a seek into the middle first, then the whole track from the start + let (mid, all) = (worst(n / 2 + 77, 3000), worst(0, n)); + println!("{name}: {n} samples, max error mid {mid:.2e}, whole {all:.2e}"); + assert!(mid <= 1e-4 && all <= 1e-4, "{name}: differs from ffmpeg by {mid} (mid) / {all} (whole)"); + } +} diff --git a/crates/isobmff/README.md b/crates/isobmff/README.md index d74ae54a..47c31bb4 100644 --- a/crates/isobmff/README.md +++ b/crates/isobmff/README.md @@ -60,6 +60,7 @@ Boxes with 64-bit sizes and with size 0 (runs to the end of the file) are suppor | `jpeg mjpa mjpb` | `Jpeg { fourcc }` | | `AVdn AVdh` | `Dnx { fourcc }` | | `mp4a` + `esds` (in `wave` too) | `Aac(AacConfig)`: ASC bytes + decoded object type/rate/channels; `Mp3` for OTI 0x69/0x6B | +| `.mp3`/`.mp2` | `Mp3`: MPEG-1/2 audio of any layer (each frame header names it) | | PCM fourccs | `Pcm(PcmConfig)`: bits, float, endianness (`enda`, lpcm flags, `pcmC`), channels, rate | | `alac` | `Alac { cookie }` | | `Opus` + `dOps` | `Opus(OpusConfig)` | diff --git a/crates/isobmff/src/codec.rs b/crates/isobmff/src/codec.rs index b9afd53e..6d2ff79b 100644 --- a/crates/isobmff/src/codec.rs +++ b/crates/isobmff/src/codec.rs @@ -181,7 +181,8 @@ pub enum CodecConfig { /// Advanced Professional Video (`apv1` + `apvC`). Apv(ApvConfig), Aac(AacConfig), - /// MPEG-1/2 audio layer III (esds object type 0x69/0x6B, or QuickTime `.mp3`). + /// MPEG-1/2 audio (esds object type 0x69/0x6B, or QuickTime `.mp3` / `.mp2`). Usually layer + /// III, but these name no layer: ffmpeg stores MP2 as `mp4a` + 0x6B. Each frame header has it. Mp3, Pcm(PcmConfig), /// ALAC magic cookie (ALACSpecificConfig, 24 bytes, optionally followed by a channel layout). @@ -883,6 +884,7 @@ fn is_audio_fourcc(f: &[u8; 4]) -> bool { | b"ac-3" | b"ec-3" | b".mp3" + | b".mp2" | b"enca" | b"ulaw" | b"alaw" @@ -1028,7 +1030,7 @@ pub(crate) fn parse_sample_entry(format: FourCc, payload: &[u8], handler: FourCc Some(p) => parse_esds(p)?, None => unknown(), }, - b".mp3" => CodecConfig::Mp3, + b".mp3" | b".mp2" => CodecConfig::Mp3, b"alac" => match get(b"alac") { Some(p) if p.len() >= 4 => CodecConfig::Alac { cookie: p[4..].to_vec() }, _ => unknown(), @@ -1643,6 +1645,20 @@ mod tests { assert_eq!(roundtrip(&e, true, b"soun"), e); } + /// QuickTime's `.mp2` entry (what ffmpeg writes for MP2 in MOV) is MPEG audio, as `.mp3` is, + /// even under a handler that does not say "sound" (#800). + #[test] + fn quicktime_mpeg_audio_entries() { + for fourcc in [*b".mp3", *b".mp2"] { + let e = SampleEntry::audio(FourCc(fourcc), CodecConfig::Mp3, 2, 48000.0, 16); + for handler in [b"soun", b"\0\0\0\0"] { + let back = roundtrip(&e, true, handler); + assert_eq!(back.codec, CodecConfig::Mp3, "{fourcc:?} {handler:?}"); + assert_eq!(back.audio.map(|a| (a.channels, a.sample_rate)), Some((2, 48000.0)), "{fourcc:?} {handler:?}"); + } + } + } + #[test] fn pcm_entries() { for (bits, float, be, qt) in [ diff --git a/docs/codec-parity.md b/docs/codec-parity.md index ba0da95b..8fe1ff6b 100644 --- a/docs/codec-parity.md +++ b/docs/codec-parity.md @@ -42,7 +42,7 @@ contents). Percent is ready-for-real-work for that codec: depth, profiles, speed | AAC (LC, HE, LATM) | decode + encode | own decoder (LC, HE core), LATM | own LC encoder | loudness oracle vs ffmpeg | 90% | 3–5 h | | AC-3 | decode (licensed) | own decoder (ATSC A/52) | none | conformance | 80% | — | | E-AC-3 (Dolby Digital Plus) | decode | own decoder (ATSC A/52 Annex E, #647): independent substream, 1–6 blocks, spectral extension; **no** adaptive hybrid transform, enhanced coupling or 7.1 dependent substreams (plays the 5.1 core) | none | SNR vs ffmpeg 65–91 dB (raw, MP4, MOV, Matroska, TS) | 75% | 10–20 h | -| MP3 / MP2 | decode + MP3 encode | decode (symphonia, MPL-2.0; own MP2) | **none** | — | 60% | 10–20 h (MP3 encoder) | +| MP3 / MP2 | decode + MP3 encode | decode (symphonia, MPL-2.0; own MP2) in raw, MP4, MOV (#800), Matroska, TS, PS; **not** MP2 mono above 192 kbit/s or stereo below 64 kbit/s (ffmpeg's mono default; symphonia rejects them) | **none** | MP2 sample-exact vs ffmpeg (raw, MP4, MOV, TS, PS) | 60% | 10–20 h (MP3 encoder) | | Opus | — (not a Premiere import format) | own decoder, RFC 8251 range-exact | none | conformance vectors | ahead | — | | FLAC, ALAC, Vorbis | partial | decode (symphonia); FLAC in MP4 silent (#603) | none | — | 70% | 3–6 h | | WMA | decode on Windows | none | none | — | 0% | 10–20 h | diff --git a/docs/gaps.md b/docs/gaps.md index 56eddd32..4e14ba75 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -155,9 +155,13 @@ platforms** G4, G9, G17; **performance** G16; **ecosystem** G11; **measurement** ## G10. Container and codec gaps on import - **Have:** E-AC-3 (Dolby Digital Plus) in MP4, MOV, Matroska and MPEG-TS (#647): independent - substream, 1–6 blocks per syncframe, spectral extension. -- **Missing:** E-AC-3 adaptive hybrid transform, enhanced coupling and the extra channels of 7.1 - dependent substreams (played as the 5.1 core), AVI (#598; listed as an extension but no + substream, 1–6 blocks per syncframe, spectral extension. MP2 (MPEG audio layer II) in MP4 and + MOV, sample-exact against ffmpeg (#800). +- **Missing:** MP2 at bit rates ISO/IEC 11172-3 disallows: mono above 192 kbit/s (ffmpeg's default + for mono MP2 is 384 kbit/s) and stereo below 64 kbit/s; the bootstrap decoder (symphonia, also + 0.6) rejects every such frame, so the track plays silent: needs our own layer I/II decoder. + E-AC-3 adaptive hybrid transform, enhanced coupling and the extra channels of 7.1 dependent + substreams (played as the 5.1 core), AVI (#598; listed as an extension but no demuxer), WMV / ASF, DV / DVCPRO / DV100, MPEG-4 Part 2, FLAC in MP4 (#603), camera RAW (R3D incl. R3D NE, ARRIRAW, Sony RAW / X-OCN, Canon RAW, BRAW, ProRes RAW; #345, #383), JPEG 2000 / JPEG XS MXF, Cinema DNG; stills: PSD, EXR, DPX, Targa, HEIF / HEIC, Radiance HDR, AI / EPS;