From f761a299a7334f16aea04c2916859f6a33f2927c Mon Sep 17 00:00:00 2001 From: heojunfo Date: Wed, 7 Oct 2026 10:44:38 +0900 Subject: [PATCH] codecs: MP4/MOV pixel format and alpha come from hvcC, avcC and the ProRes frame header (#103) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The probe guessed the pixel format from the codec profile alone: HEVC reported nothing, H.264 High 4:2:2 had no bit depth, and ProRes 4444 claimed alpha because of its fourcc. - HEVC: chroma format and bit depth from hvcC. - H.264: from the avcC High-profile extension (ISO/IEC 14496-15 §5.3.3), else from the first SPS; unknown (empty) if neither can be read. - ProRes: the first frame header gives the chroma format and alpha (alpha_channel_type), as the first DNx frame header already does. Checked against ffprobe on ffmpeg-made HEVC 4:2:0/4:2:2 8/10-bit, H.264 4:2:0/4:2:2 8/10-bit and ProRes 422 HQ / 4444 with and without alpha. Unit tests cover hvcC, avcC (extension and SPS fallback) and ProRes 4444 with and without alpha. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/codecs/src/mp4.rs | 130 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 122 insertions(+), 8 deletions(-) diff --git a/crates/codecs/src/mp4.rs b/crates/codecs/src/mp4.rs index 9820fbc8..3c7a58a3 100644 --- a/crates/codecs/src/mp4.rs +++ b/crates/codecs/src/mp4.rs @@ -148,7 +148,7 @@ impl Mp4Source { codec: codec_label(&entry.codec), pixel_format: pixfmt_label(&entry.codec), color, - has_alpha: matches!(&entry.codec, CodecConfig::ProRes { fourcc } if fourcc.0 == *b"ap4h" || fourcc.0 == *b"ap4x"), + has_alpha: false, bitrate, hdr: entry.video.as_ref().and_then(|v| hdr_metadata(v.mastering_display.as_ref(), v.content_light)), }; @@ -170,6 +170,19 @@ impl Mp4Source { info.has_alpha = h.alpha; } } + if matches!(entry.codec, CodecConfig::ProRes { .. }) { + // a 4444 fourcc doesn't mean the frames carry alpha (alpha_channel_type does): read the first frame header + let s0 = &t.samples[0]; + if let Some(h) = filmcraft_media::reader::read_range(&*bytes.0, s0.offset, s0.size as usize).ok().and_then(|d| filmcraft_prores::probe(&d).ok()) + { + info.has_alpha = h.alpha != filmcraft_prores::AlphaType::None; + let sub = match h.chroma { + filmcraft_prores::ChromaFormat::Yuv422 => "4:2:2 10-bit", + filmcraft_prores::ChromaFormat::Yuv444 => "4:4:4 12-bit", + }; + info.pixel_format = format!("{} {sub}", if info.has_alpha { "YUVA" } else { "YUV" }); + } + } info }); let audio = atrack.map(|i| { @@ -453,14 +466,37 @@ fn codec_label(c: &CodecConfig) -> String { } } +/// "YUV 4:2:2 10-bit" from an H.264 / HEVC `chroma_format_idc` and bit depth. +fn yuv_label(chroma_format_idc: u32, bits: u32) -> String { + let sub = match chroma_format_idc { + 0 => "4:0:0", + 1 => "4:2:0", + 2 => "4:2:2", + _ => "4:4:4", + }; + format!("YUV {sub} {bits}-bit") +} + +/// `chroma_format_idc` and luma bit depth of an H.264 track: from the `avcC` High-profile extension +/// (ISO/IEC 14496-15 §5.3.3), else from its first SPS. The profile alone can't say (High 4:2:2 is +/// 8 or 10-bit, High 10 can be 8-bit). +fn avc_format(a: &filmcraft_isobmff::AvcConfig) -> Option<(u32, u32)> { + // reserved bits set: '111111' chroma_format, '11111' bit_depth_luma_minus8 (else not an extension) + if !matches!(a.profile, 66 | 77 | 88) + && let &[chroma, depth, ..] = a.ext.as_slice() + && chroma & 0xFC == 0xFC + && depth & 0xF8 == 0xF8 + { + return Some(((chroma & 3) as u32, (depth & 7) as u32 + 8)); + } + let sps = filmcraft_h264::params::Sps::parse(&filmcraft_bitstream::unescape_rbsp(a.sps.first()?.get(1..)?)).ok()?; + Some((sps.chroma_format_idc, sps.bit_depth_luma)) +} + fn pixfmt_label(c: &CodecConfig) -> String { match c { - CodecConfig::Avc(a) => match a.profile { - 110 => "YUV 4:2:0 10-bit".into(), - 122 => "YUV 4:2:2".into(), - 244 => "YUV 4:4:4".into(), - _ => "YUV 4:2:0 8-bit".into(), - }, + CodecConfig::Avc(a) => avc_format(a).map(|(chroma, bits)| yuv_label(chroma, bits)).unwrap_or_default(), + CodecConfig::Hevc(h) => yuv_label(h.chroma_format_idc as u32, h.bit_depth_luma as u32), CodecConfig::Vp9(c) => { let sub = match c.chroma_subsampling { 2 => "4:2:2", @@ -469,7 +505,8 @@ fn pixfmt_label(c: &CodecConfig) -> String { }; format!("YUV {sub} {}-bit", c.bit_depth.max(8)) } - CodecConfig::ProRes { fourcc } if fourcc.0[2] == b'4' => "YUVA 4:4:4 12-bit".into(), + // 4444 until the first frame header says whether it codes alpha (`open_reader`) + CodecConfig::ProRes { fourcc } if fourcc.0[2] == b'4' => "YUV 4:4:4 12-bit".into(), CodecConfig::ProRes { .. } => "YUV 4:2:2 10-bit".into(), CodecConfig::Jpeg { .. } => "YUV 4:2:x 8-bit".into(), _ => String::new(), @@ -603,4 +640,81 @@ mod tests { let s = Mp4Source::open("cw2.mov", rotated_mov([0, 2 * ONE, 0, -2 * ONE, 0, 0, 64 * ONE, 0, W])).expect("open"); assert_eq!(s.info().video.as_ref().map(|v| (v.width, v.height)), Some((32, 64))); } + + #[test] + fn hevc_pixel_format_comes_from_hvcc() { + for (chroma, bits, want) in [(1, 8, "YUV 4:2:0 8-bit"), (1, 10, "YUV 4:2:0 10-bit"), (2, 8, "YUV 4:2:2 8-bit"), (2, 10, "YUV 4:2:2 10-bit")] { + let c = filmcraft_isobmff::HevcConfig { general_profile_idc: 1, chroma_format_idc: chroma, bit_depth_luma: bits, ..Default::default() }; + assert_eq!(pixfmt_label(&CodecConfig::Hevc(c)), want); + } + } + + /// A High-profile SPS NAL unit (ITU-T H.264 §7.3.2.1.1) with the given chroma format and bit depth, 16×16. + fn high_sps(profile: u8, chroma: u32, bits: u32) -> Vec { + let mut w = filmcraft_bitstream::BitWriter::new(); + w.write_bits(0x67, 8); + w.write_bits(profile as u32, 8); + w.write_bits(0, 8); // constraint flags + w.write_bits(30, 8); // level + w.write_ue(0); // seq_parameter_set_id + w.write_ue(chroma); + if chroma == 3 { + w.write_bit(false); // separate_colour_plane_flag + } + w.write_ue(bits - 8); // luma + w.write_ue(bits - 8); // chroma + w.write_bits(0, 2); // no transform bypass, no scaling matrices + w.write_ue(0); // log2_max_frame_num_minus4 + w.write_ue(2); // pic_order_cnt_type + w.write_ue(1); // max_num_ref_frames + w.write_bit(false); // gaps_in_frame_num_allowed + w.write_ue(0); // pic_width_in_mbs_minus1 + w.write_ue(0); // pic_height_in_map_units_minus1 + w.write_bits(0b110, 3); // frame_mbs_only, direct_8x8_inference, no cropping + w.write_bit(false); // no VUI + w.rbsp_trailing(); + w.finish() + } + + #[test] + fn avc_pixel_format_comes_from_avcc_not_the_profile() { + let avc = + |profile: u8, sps: Vec, ext: Vec| CodecConfig::Avc(filmcraft_isobmff::AvcConfig { profile, sps: vec![sps], ext, ..Default::default() }); + // High 4:2:2 at 10 and 8 bits, from the avcC extension (chroma_format 2, bit_depth_luma_minus8) + assert_eq!(pixfmt_label(&avc(122, high_sps(122, 2, 10), vec![0xFE, 0xFA, 0xFA, 0])), "YUV 4:2:2 10-bit"); + assert_eq!(pixfmt_label(&avc(122, high_sps(122, 2, 8), vec![0xFE, 0xF8, 0xF8, 0])), "YUV 4:2:2 8-bit"); + // no extension (or not a valid one): the SPS says + assert_eq!(pixfmt_label(&avc(122, high_sps(122, 2, 10), Vec::new())), "YUV 4:2:2 10-bit"); + assert_eq!(pixfmt_label(&avc(110, high_sps(110, 1, 10), vec![0, 0, 0, 0])), "YUV 4:2:0 10-bit"); + assert_eq!(pixfmt_label(&avc(244, high_sps(244, 3, 10), Vec::new())), "YUV 4:4:4 10-bit"); + // an unreadable SPS and no extension: unknown, not a guess + assert_eq!(pixfmt_label(&avc(122, vec![0x67, 122], Vec::new())), ""); + } + + /// A one-frame 32×16 ProRes MOV with the given profile, chroma format and fourcc. + fn prores_mov(profile: filmcraft_prores::Profile, fourcc: &[u8; 4], chroma: filmcraft_prores::ChromaFormat, alpha: bool) -> Arc<[u8]> { + let (w, h) = (32u32, 16u32); + let fr = filmcraft_prores::Frame::new(w, h, chroma, if chroma == filmcraft_prores::ChromaFormat::Yuv444 { 12 } else { 10 }, alpha); + let data = filmcraft_prores::Encoder::new(profile, w, h).encode(&fr).expect("encode"); + let mut mux = Mp4Writer::new(std::io::Cursor::new(Vec::new()), WriterOptions::new(Brand::Mov)).expect("writer"); + let t = mux.add_track(TrackConfig::new(SampleEntry::prores(FourCc(*fourcc), w as u16, h as u16), 25)).expect("track"); + mux.write_sample(t, WriteSample { data: &data, duration: 1, composition_offset: 0, is_sync: true }).expect("sample"); + mux.finish().expect("finish").into_inner().into() + } + + /// `ap4h` / `ap4x` don't imply alpha: the frame header's alpha_channel_type does. + #[test] + fn prores_alpha_comes_from_the_frame_header() { + use filmcraft_prores::{ChromaFormat, Profile}; + for (profile, fourcc, chroma, alpha, want) in [ + (Profile::P4444, b"ap4h", ChromaFormat::Yuv444, false, "YUV 4:4:4 12-bit"), + (Profile::P4444, b"ap4h", ChromaFormat::Yuv444, true, "YUVA 4:4:4 12-bit"), + (Profile::P4444Xq, b"ap4x", ChromaFormat::Yuv444, false, "YUV 4:4:4 12-bit"), + (Profile::Hq, b"apch", ChromaFormat::Yuv422, false, "YUV 4:2:2 10-bit"), + ] { + let s = Mp4Source::open("p.mov", prores_mov(profile, fourcc, chroma, alpha)).expect("open"); + let v = s.info().video.as_ref().expect("video"); + assert_eq!((v.pixel_format.as_str(), v.has_alpha), (want, alpha), "{}", String::from_utf8_lossy(fourcc)); + } + } }