Merge pull request #42 from punkhop/fix/mp4-audio-thumps

codecs: resampled MP4 audio reads apply the edit list once

crates/codecs/src/mp4.rs

@@ -359,7 +359,8 @@ impl MediaSource for Mp4Source {
         let ch = ainfo.channels.max(1) as usize;
         // Map the requested window to source samples (edit list offset applied: presentation = pts + edit_offset).
         let ratio = src_rate as f64 / sample_rate as f64;
-        let s0 = (start as f64 * ratio).floor() as i64 - self.audio_offset * src_rate as i64 / track.timescale.max(1) as i64;
+        let p0 = (start as f64 * ratio).floor() as i64;
+        let s0 = p0 - self.audio_offset * src_rate as i64 / track.timescale.max(1) as i64;
         let need = (frames as f64 * ratio).ceil() as i64 + 2;
         // Samples-per-unit: the track timescale is usually the sample rate for audio.
         let unit = src_rate as f64 / track.timescale.max(1) as f64;
@@ -393,8 +394,9 @@ impl MediaSource for Mp4Source {
             }
             return Ok(out);
         }
-        // resample the local window
-        let frac = s0 as f64 - start as f64 * ratio;
+        // Resample the local window. Its first sample is presentation sample `p0`; the edit list
+        // offset is already in `s0` and must not shift the read position a second time.
+        let frac = p0 as f64 - start as f64 * ratio;
         let mut out = AudioBuffer::silence(sample_rate, ch, frames);
         for k in 0..frames {
             let pos = k as f64 * ratio - frac;

crates/codecs/tests/mp4_oracle.rs

@@ -0,0 +1,39 @@
+//! MP4 audio against ffmpeg: ffmpeg writes an AAC track (with the edit list that skips the encoder
+//! priming), and reads at another sample rate must land on the same source samples as the native
+//! decode, however the requests are cut.
+
+mod common;
+
+use common::*;
+
+/// A source whose rate differs from the sequence's is resampled per request. An AAC track in MP4
+/// carries an edit list that skips the encoder priming (ffmpeg: 1024 samples), and every request
+/// must still land on the same source samples: reading at 44.1 kHz in video-frame-sized pieces (as
+/// an export does) equals the 48 kHz decode interpolated at the same times.
+#[test]
+fn aac_mp4_resampled_reads_follow_the_edit_list() {
+    let ff = filmcraft_testkit::require_ffmpeg!();
+    let Some(f) = fixture(&ff, "tone_aac_48k.mp4", &["-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:d=3", "-c:a", "aac", "-b:a", "128k"]) else {
+        panic!("could not write the AAC fixture")
+    };
+    let src = filmcraft_codecs::open_bytes("tone_aac_48k.mp4", bytes(&f)).unwrap();
+    let native = src.audio(0, 2 * 48_000, 48_000).unwrap().channels[0].clone();
+    let ratio = 48_000.0 / 44_100.0;
+    // 30 fps at 44.1 kHz, starting mid-file and off the 48 kHz grid
+    let (mut start, piece) = (22_051i64, 1470usize);
+    let (mut worst, mut peak) = (0f32, 0f32);
+    for _ in 0..20 {
+        let got = src.audio(start, piece, 44_100).unwrap();
+        for (k, &v) in got.channels[0].iter().enumerate() {
+            let pos = (start + k as i64) as f64 * ratio;
+            let i0 = pos.floor() as usize;
+            let want = native[i0] + (native[i0 + 1] - native[i0]) * (pos - i0 as f64) as f32;
+            worst = worst.max((v - want).abs());
+            peak = peak.max(v.abs());
+        }
+        start += piece as i64;
+    }
+    let level = native.iter().fold(0f32, |m, v| m.max(v.abs()));
+    assert!(peak <= level * 1.01, "resampled peak {peak} above the source's {level}");
+    assert!(worst < 1e-4, "resampled read differs from the source by {worst}");
+}

docs/testing.md

@@ -90,6 +90,7 @@ are never linked or shipped ([AGENTS.md](../AGENTS.md) §2).
 | MPEG TS / PS (via `codecs/tests/mpeg_oracle.rs`) | ffmpeg-written TS (MPEG-2 + MP2, H.264 + AAC, HEVC + LATM AAC), BDAV `.m2ts` / `.mts` (H.264 + Blu-ray LPCM / AC-3), VOB (MPEG-2 + MP2 / DVD LPCM 16 and 24-bit / AC-3), MPEG-1 system streams, `.m2v`; MPEG-2 in MOV (XDCAM `xd5c`), MP4 (`mp4v`), MKV | frame count and every frame's **PTS exact** (ffprobe); H.264 / HEVC **bit-exact**, MPEG-2 within the `mpeg2v` criterion; random seeks = sequential decode; LPCM **sample-exact**; AAC ≤ 2.1·10⁻⁷, MP2 (symphonia) ≤ 3.2·10⁻⁵, AC-3 ≤ 5·10⁻³ (decoder-specific dither; measured 2.3·10⁻³); random-access audio = sequential |
 | `ac3` | ffmpeg AC-3 encodes (2/0 with coupling and rematrixing, 1/0 32 kHz, 3/2+LFE 448 kb/s, noise) decoded by ffmpeg | SNR per channel ≥ 40-60 dB on tonal content (measured 66-94 dB), ≥ 18 dB on noise (27.7 dB, the same as between two of our own dither seeds) |
 | `ogg` (via `codecs/tests/ogg_oracle.rs`) | ffmpeg (libopus, Vorbis) files decoded by libopus / ffmpeg | length **exact** (pre-skip, end trimming); SNR ≥ 50 dB CELT (measured 77.7), ≥ 10 dB SILK, Vorbis 139 dB; random seeks within 6·10⁻⁶ of the continuous decode |
+| MP4 audio at another rate (`codecs/tests/mp4_oracle.rs`) | ffmpeg-written AAC in MP4 (48 kHz, edit list skipping the priming) | reads resampled to 44.1 kHz in frame-sized requests equal the native decode interpolated at the same times (difference < 10⁻⁴, peak ≤ the source level) |
 | image sequences / BWF (`codecs/tests/media_oracle.rs`) | ffmpeg-written PNG/TIFF/BMP/JPEG stills read by ffmpeg's image2 demuxer; ffmpeg BWF (`-write_bext`) | PNG/TIFF/BMP **exact**, JPEG ±12 (measured 3); missing frames hold the previous one; BWF TimeReference = ffprobe's, start timecode, samples exact |
 | `export` | our own demuxer/decoder reads the file back; ffprobe counts frames when present | expected size, duration, colour, audio level; exact frame count |
 | `export` settings and presets (M6.5) | `export/src/settings_tests.rs`, engine `export_tests.rs`: every built-in preset exports a slice of the demo sequence; ffprobe reads codec, profile, size, rate, audio format and data rate; our decoders read pixels back | codec / profile / container / size / `r_frame_rate` / sample format exact; H.264 bitrate ≤ VBR max × 1.1 and ≥ ½ target over 3 s; ProRes ≤ 1.5 × nominal; DNxHR within 25 % of nominal; loudness normalization within **0.5 LU** of the target (WAV and AAC), true peak ≤ ceiling + 0.1 dB; image sequences numbered `<name>000…`; limiter, overlays, metadata (`title`/`copyright` tags), MOV multiplexer, two-pass/CBR progress |