250 lines
12 KiB
Rust
250 lines
12 KiB
Rust
//! MP4 audio against ffmpeg: ffmpeg writes an AAC track (with the edit list that skips the encoder
|
||
//! priming), and reads at another sample rate must land on the same source samples as the native
|
||
//! decode, however the requests are cut.
|
||
|
||
mod common;
|
||
|
||
use common::*;
|
||
|
||
/// A source whose rate differs from the sequence's is resampled per request. An AAC track in MP4
|
||
/// carries an edit list that skips the encoder priming (ffmpeg: 1024 samples), and every request
|
||
/// must still land on the same source samples: reading at 44.1 kHz in video-frame-sized pieces (as
|
||
/// an export does) equals the 48 kHz decode interpolated at the same times.
|
||
#[test]
|
||
fn aac_mp4_resampled_reads_follow_the_edit_list() {
|
||
let ff = filmcraft_testkit::require_ffmpeg!();
|
||
let Some(f) = fixture(&ff, "tone_aac_48k.mp4", &["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:d=3", "-c:a", "aac", "-b:a", "128k"]) else {
|
||
panic!("could not write the AAC fixture")
|
||
};
|
||
let src = filmcraft_codecs::open_bytes("tone_aac_48k.mp4", bytes(&f)).unwrap();
|
||
let native = src.audio(0, 2 * 48_000, 48_000).unwrap().channels[0].clone();
|
||
let ratio = 48_000.0 / 44_100.0;
|
||
// 30 fps at 44.1 kHz, starting mid-file and off the 48 kHz grid
|
||
let (mut start, piece) = (22_051i64, 1470usize);
|
||
let (mut worst, mut peak) = (0f32, 0f32);
|
||
for _ in 0..20 {
|
||
let got = src.audio(start, piece, 44_100).unwrap();
|
||
for (k, &v) in got.channels[0].iter().enumerate() {
|
||
let pos = (start + k as i64) as f64 * ratio;
|
||
let i0 = pos.floor() as usize;
|
||
let want = native[i0] + (native[i0 + 1] - native[i0]) * (pos - i0 as f64) as f32;
|
||
worst = worst.max((v - want).abs());
|
||
peak = peak.max(v.abs());
|
||
}
|
||
start += piece as i64;
|
||
}
|
||
let level = native.iter().fold(0f32, |m, v| m.max(v.abs()));
|
||
assert!(peak <= level * 1.01, "resampled peak {peak} above the source's {level}");
|
||
assert!(worst < 1e-4, "resampled read differs from the source by {worst}");
|
||
}
|
||
|
||
/// ffmpeg's MP4 muxer skips the B-frame reorder delay with an edit list, so the media (`mdhd`) runs
|
||
/// one frame past what plays. The clip must be as long as ffmpeg decodes it: 60 frames, not 61.
|
||
#[test]
|
||
fn bframe_mp4_duration_is_the_decoded_frame_count() {
|
||
let ff = filmcraft_testkit::require_ffmpeg!();
|
||
let args = ["-f", "lavfi", "-i", "testsrc2=size=160x96:rate=30:duration=2", "-c:v", "libx264", "-bf", "2", "-pix_fmt", "yuv420p"];
|
||
let Some(f) = fixture(&ff, "bframes_x264.mp4", &args) else {
|
||
eprintln!("SKIPPED: ffmpeg without libx264");
|
||
return;
|
||
};
|
||
let n = ffmpeg_frames(&ff, &f, "yuv420p").len() / (160 * 96 * 3 / 2);
|
||
assert_eq!(n, 60);
|
||
let src = filmcraft_codecs::open_bytes("bframes_x264.mp4", bytes(&f)).unwrap();
|
||
assert_eq!(src.info().duration, filmcraft_time::FrameRate::FPS_30.tick_of(n as i64));
|
||
}
|
||
|
||
/// `mp4` (moov after mdat, as ffmpeg writes it) with a clean aperture (`clap`) of `w`×`h` moved by
|
||
/// (`dx`, `dy`) from the centre, inserted at the end of the first video sample entry.
|
||
fn with_clap(mp4: &[u8], (w, h, dx, dy): (u32, u32, i32, i32)) -> Vec<u8> {
|
||
let be = |b: &[u8], at: usize| u32::from_be_bytes([b[at], b[at + 1], b[at + 2], b[at + 3]]) as usize;
|
||
// the chain moov > trak > mdia > minf > stbl > stsd > avc1 (stsd has 8 bytes before its entries)
|
||
let mut chain = Vec::new();
|
||
let (mut off, mut end) = (0, mp4.len());
|
||
for (want, skip) in [(b"moov", 8), (b"trak", 8), (b"mdia", 8), (b"minf", 8), (b"stbl", 8), (b"stsd", 16), (b"avc1", 0)] {
|
||
loop {
|
||
assert!(off + 8 <= end, "no {}", String::from_utf8_lossy(want));
|
||
let size = be(mp4, off);
|
||
if &mp4[off + 4..off + 8] == want {
|
||
chain.push(off);
|
||
end = off + size;
|
||
off += skip;
|
||
break;
|
||
}
|
||
off += size;
|
||
}
|
||
}
|
||
let entry = *chain.last().unwrap();
|
||
assert!(mp4.windows(4).position(|x| x == b"mdat").unwrap() < chain[0], "moov must follow mdat");
|
||
let mut clap = Vec::new();
|
||
for v in [40u32, u32::from_be_bytes(*b"clap"), w, 1, h, 1, dx as u32, 1, dy as u32, 1] {
|
||
clap.extend_from_slice(&v.to_be_bytes());
|
||
}
|
||
let mut out = mp4.to_vec();
|
||
let at = entry + be(mp4, entry);
|
||
out.splice(at..at, clap);
|
||
for &b in &chain {
|
||
let size = (be(&out, b) + 40) as u32;
|
||
out[b..b + 4].copy_from_slice(&size.to_be_bytes());
|
||
}
|
||
out
|
||
}
|
||
|
||
/// Cropped pictures are the size and the samples ffmpeg decodes (#288): H.264 frame cropping (SPS)
|
||
/// and an HEVC conformance window while the sample entry gives the uncropped size, and a clean
|
||
/// aperture (`clap`, as iPhones write), centred, off-centre, and with a 90° display rotation
|
||
/// applied after it.
|
||
#[test]
|
||
fn cropped_pictures_match_ffmpeg() {
|
||
let ff = filmcraft_testkit::require_ffmpeg!();
|
||
let src = ["-f", "lavfi", "-i", "testsrc2=size=320x240:rate=30:duration=1,noise=alls=10:allf=t", "-c:v", "libx264", "-bf", "2", "-pix_fmt", "yuv420p"];
|
||
let Some(plain) = fixture(&ff, "crop_plain_x264.mp4", &src) else {
|
||
eprintln!("SKIPPED: ffmpeg without libx264");
|
||
return;
|
||
};
|
||
let sps =
|
||
fixture(&ff, "crop_sps_x264.mp4", &[&src[..], &["-bsf:v", "h264_metadata=crop_left=16:crop_right=16:crop_top=8:crop_bottom=8"]].concat()).unwrap();
|
||
// the display matrix added without re-encoding (an encode would turn the pixels instead)
|
||
let rotated = fixture(&ff, "crop_rotated_x264.mp4", &["-display_rotation", "90", "-i", plain.to_str().unwrap(), "-c", "copy"]).unwrap();
|
||
let dir = dir();
|
||
let mut cases = vec![("SPS crop", sps, (288, 224))];
|
||
// HEVC: the conformance window
|
||
let hevc = [
|
||
"-f",
|
||
"lavfi",
|
||
"-i",
|
||
"testsrc2=size=320x240:rate=30:duration=1,noise=alls=10:allf=t",
|
||
"-c:v",
|
||
"libx265",
|
||
"-x265-params",
|
||
"log-level=error:bframes=3",
|
||
"-tag:v",
|
||
"hvc1",
|
||
"-pix_fmt",
|
||
"yuv420p",
|
||
];
|
||
match fixture(&ff, "crop_conformance_x265.mp4", &[&hevc[..], &["-bsf:v", "hevc_metadata=crop_left=16:crop_right=16:crop_top=8:crop_bottom=8"]].concat()) {
|
||
Some(p) => cases.push(("HEVC conformance window", p, (288, 224))),
|
||
None => eprintln!("SKIPPED (HEVC case): ffmpeg without libx265"),
|
||
}
|
||
for (name, base, clap, size) in [
|
||
("centred clap", &plain, (280, 200, 0, 0), (280, 200)),
|
||
("off-centre clap", &plain, (200, 150, -30, 22), (200, 150)),
|
||
("clap then rotation", &rotated, (280, 200, 0, 0), (200, 280)),
|
||
] {
|
||
let path = dir.join(format!("crop_{}.mp4", name.replace(' ', "_")));
|
||
std::fs::write(&path, with_clap(&bytes(base), clap)).unwrap();
|
||
cases.push((name, path, size));
|
||
}
|
||
for (name, path, (w, h)) in cases {
|
||
let src = filmcraft_codecs::open_bytes(path.file_name().unwrap().to_str().unwrap(), bytes(&path)).unwrap();
|
||
let v = src.info().video.clone().unwrap();
|
||
assert_eq!((v.width, v.height), (w, h), "{name}: reported size");
|
||
let (cw, ch) = (w.div_ceil(2) as usize, h.div_ceil(2) as usize);
|
||
// `-flags unaligned`: ffmpeg otherwise skips a left crop that would misalign its frame
|
||
// pointers (on AVX-512 machines it needs 64-byte alignment) and returns wider frames
|
||
let raw = ffmpeg_out(
|
||
&ff,
|
||
&["-flags", "unaligned", "-i", path.to_str().unwrap(), "-map", "0:v:0", "-fps_mode", "passthrough", "-f", "rawvideo", "-pix_fmt", "yuv420p", "-"],
|
||
);
|
||
let frame_bytes = (w * h) as usize + 2 * cw * ch;
|
||
if name.contains("clap") && !raw.len().is_multiple_of(frame_bytes) {
|
||
// ffmpeg before 7.1 ignores `clap` and decodes the whole picture
|
||
eprintln!("SKIPPED ({name}): this ffmpeg does not apply the clean aperture");
|
||
continue;
|
||
}
|
||
assert_eq!(raw.len() % frame_bytes, 0, "{name}: ffmpeg's frames are {w}x{h}");
|
||
let n = raw.len() / frame_bytes;
|
||
assert_eq!(n, 30, "{name}");
|
||
for (i, want) in raw.chunks(frame_bytes).enumerate() {
|
||
let f = src.video_frame(filmcraft_media::FrameRequest::full(v.frame_rate.tick_of(i as i64))).unwrap();
|
||
assert_eq!((f.width, f.height), (w, h), "{name}: frame {i} size");
|
||
assert!(planes(&f) == raw_planes(want, w as usize, h as usize, cw, ch, 1), "{name}: frame {i} differs from ffmpeg's");
|
||
}
|
||
}
|
||
}
|
||
|
||
/// An open GOP (x265 `open-gop`, like iPhone HEVC): each CRA after the first is followed in the
|
||
/// file by RASL pictures shown before it, predicted from the GOP before. Asked for first (a clip
|
||
/// trimmed to start on one, the playhead parked on one), such a picture came out black: decoding
|
||
/// started at the CRA after it, which leaves its RASL pictures out. Every frame, asked of a fresh
|
||
/// source and in an order that visits each CRA before its leading pictures, is ffmpeg's, from MP4
|
||
/// and from Matroska.
|
||
#[test]
|
||
fn open_gop_leading_pictures_decode_alone() {
|
||
let ff = filmcraft_testkit::require_ffmpeg!();
|
||
let args = [
|
||
"-f",
|
||
"lavfi",
|
||
"-i",
|
||
"testsrc2=size=160x96:rate=30:duration=2,noise=alls=10:allf=t",
|
||
"-c:v",
|
||
"libx265",
|
||
"-x265-params",
|
||
"log-level=error:keyint=12:min-keyint=12:scenecut=0:open-gop=1:bframes=3",
|
||
"-tag:v",
|
||
"hvc1",
|
||
"-pix_fmt",
|
||
"yuv420p",
|
||
];
|
||
let Some(mp4) = fixture(&ff, "open_gop_x265.mp4", &args) else {
|
||
eprintln!("SKIPPED: ffmpeg without libx265");
|
||
return;
|
||
};
|
||
let mkv = fixture(&ff, "open_gop_x265.mkv", &["-i", mp4.to_str().unwrap(), "-c", "copy"]).unwrap();
|
||
let raw = ffmpeg_frames(&ff, &mp4, "yuv420p");
|
||
let frame_bytes = 160 * 96 * 3 / 2;
|
||
let want: Vec<_> = raw.chunks(frame_bytes).map(|f| raw_planes(f, 160, 96, 80, 48, 1)).collect();
|
||
assert_eq!(want.len(), 60);
|
||
for path in [&mp4, &mkv] {
|
||
let name = path.file_name().unwrap().to_str().unwrap();
|
||
let open = || filmcraft_codecs::open_bytes(name, bytes(path)).unwrap();
|
||
let rate = open().info().frame_rate();
|
||
let frame = |src: &filmcraft_media::SharedSource, i: usize| {
|
||
let f = src.video_frame(filmcraft_media::FrameRequest::full(rate.tick_of(i as i64))).unwrap_or_else(|e| panic!("{name}: frame {i}: {e}"));
|
||
assert!(planes(&f) == want[i], "{name}: frame {i} differs from ffmpeg's");
|
||
};
|
||
for i in 0..want.len() {
|
||
frame(&open(), i);
|
||
}
|
||
let src = open();
|
||
for cra in [12, 24, 36, 48] {
|
||
for i in [cra, cra - 1, cra - 2, cra - 3] {
|
||
frame(&src, i);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/// #800: MP2 in MP4 (`mp4a` + object type 0x6B, which names no layer) and in MOV (QuickTime
|
||
/// `.mp2`) decodes sample-exactly as ffmpeg does: same length, the encoder delay (481 samples)
|
||
/// trimmed by the edit list, and the same samples from the first one on, also after a seek. It
|
||
/// used to go to the layer III decoder (MP4: silence) or not decode at all (MOV).
|
||
#[test]
|
||
fn mp2_in_mp4_and_mov_matches_ffmpeg() {
|
||
let ff = filmcraft_testkit::require_ffmpeg!();
|
||
for (name, rate) in [("mp2_stereo_224k_48k.mp4", 48_000u32), ("mp2_stereo_224k_44k.mov", 44_100)] {
|
||
let input = format!("anoisesrc=d=1:c=pink:r={rate}:a=0.3");
|
||
let Some(f) = fixture(&ff, name, &["-f", "lavfi", "-i", &input, "-ac", "2", "-c:a", "mp2", "-b:a", "224k"]) else {
|
||
eprintln!("SKIPPED ({name}): this ffmpeg can't encode it");
|
||
continue;
|
||
};
|
||
let src = filmcraft_codecs::open_bytes(name, bytes(&f)).unwrap();
|
||
let a = src.info().audio().cloned().unwrap();
|
||
assert_eq!((a.sample_rate, a.channels, a.codec.as_str()), (rate, 2, "MPEG Audio"), "{name}");
|
||
let want = ffmpeg_audio_f32(&ff, &f, &[]);
|
||
let n = want.len() / 2;
|
||
assert!(n > rate as usize * 9 / 10, "{name}: ffmpeg decoded {n} samples");
|
||
let worst = |from: usize, len: usize| {
|
||
let got = src.audio(from as i64, len, rate).unwrap();
|
||
(0..len).map(|k| (0..2).map(|c| (got.channels[c][k] - want[2 * (from + k) + c]).abs()).fold(0f32, f32::max)).fold(0f32, f32::max)
|
||
};
|
||
let level = want.iter().fold(0f32, |m, v| m.max(v.abs()));
|
||
assert!(level > 0.1, "{name}: the source is silent ({level})");
|
||
// a seek into the middle first, then the whole track from the start
|
||
let (mid, all) = (worst(n / 2 + 77, 3000), worst(0, n));
|
||
println!("{name}: {n} samples, max error mid {mid:.2e}, whole {all:.2e}");
|
||
assert!(mid <= 1e-4 && all <= 1e-4, "{name}: differs from ffmpeg by {mid} (mid) / {all} (whole)");
|
||
}
|
||
}
|