shifathosn
/
filmcraft
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

codecs: MP4 frames stamped just after the frame grid show on their own frame (fixes #1282) 

Phone footage stamps its frames up to a millisecond after the frame grid
(a first frame 34.1 ms long at 30 fps). The frame shown at a time is the
last one starting at or before it, so a sequence frame asked at its exact
start got the picture before: the clip ran one frame behind and showed
frame 0 twice. The request is already rounded to the track timescale
(#73, as WebM in #83), but at 90 kHz a millisecond is 90 units.

When opening an MP4, the video samples of a file that is a constant rate
gone slightly off its grid are put back on it: the grid starts at the
first picture and steps by the nominal frame duration, and the file is
snapped only when every picture is less than a quarter of a frame from
its own grid point and no two pictures share a grid point. Anything
further off is variable frame rate and keeps all its times; a picture one
unit off is left where it is, since that can't be told from timescale
rounding. Only the video sample times move: the audio, the first picture
and the clip duration are unchanged.
This commit is contained in:
Marco Arnulfo 2026-10-12 16:36:10 +02:00
parent 0541d4d182
commit 3f61c036ff
2 changed files with 162 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
crates/codecs/src/mp4.rs
View File

@ -347,6 +347,7 @@ impl Mp4Source {
Some(Mp4Audio { track: i, state, starts, offset, preroll })
})
.collect();
snap_to_frame_grid(&mut file, vtrack);
rebase_unedited_video(&mut file, vtrack, &mut audios);
Ok(Self { info, bytes, file, vtrack, video: GopCache::new(explicit_color).with_rotation(rotation).with_crop(aperture), audios })
}
@ -666,6 +667,66 @@ pub fn opener(name: &str, bytes: Arc<[u8]>) -> Option<Result<SharedSource, Media
Some(Mp4Source::open(name, bytes).map(|s| Arc::new(s) as SharedSource).map_err(Into::into))
}
/// Put the video samples of a constant-rate file whose pictures sit just off its frame grid back
/// on the grid (#1282). Phone footage stamps frames up to a millisecond after where they belong (a
/// first frame 34.1 ms long at 30 fps), and the frame shown at a time is the last one that starts at
/// or before it: a sequence frame asked at the exact grid time got the picture before.
///
/// The grid starts at the first picture and steps by the nominal frame duration. A file is snapped
/// only when every picture is less than a quarter of a frame from its own grid point and no two land
/// on the same one: a frame that far off was placed there (variable frame rate), so a file with one
/// such picture keeps all its times. A picture at most one unit off is left where it is: that can't
/// be told from the timescale's rounding. Snapping keeps the samples' presentation order. Only the video samples move: the first picture, which the audio and
/// `rebase_unedited_video` go by, is the grid's origin, and the clip's duration comes from the
/// track header.
fn snap_to_frame_grid(file: &mut Mp4File, vtrack: Option<usize>) {
let Some(t) = vtrack.and_then(|i| file.tracks.get_mut(i)) else { return };
let Some(first) = t.samples.iter().map(|s| s.pts).min() else { return };
// The nominal duration: the median sample duration, averaged with the durations one unit from
// it. Those are the same duration rounded to the timescale (1501 and 1502 at 59.94 fps over
// 90 kHz): the median alone drifts half a unit a frame, enough to snap a short clip onto the
// wrong points.
let mut durs: Vec<u32> = t.samples.iter().map(|s| s.duration).collect();
durs.sort_unstable();
let Some(&median) = durs.get(durs.len() / 2) else { return };
let near: Vec<f64> = durs.iter().filter(|&&d| d.abs_diff(median) <= 1).map(|&d| f64::from(d)).collect();
let mean = near.iter().sum::<f64>() / near.len().max(1) as f64;
if mean <= 0.0 {
return;
}
let rate = FrameRate::from_f64(f64::from(t.timescale) / mean);
// one frame lasts `step_num / step_den` track units
let (step_num, step_den) = (i128::from(t.timescale) * i128::from(rate.den), i128::from(rate.num));
if step_num <= 0 || step_den <= 0 {
return;
}
// a sample's grid index and its new time
let snap = |pts: i64| -> Option<(i128, i64)> {
let off = i128::from(pts) - i128::from(first);
// the nearest grid index, then its grid point (both rounded half up)
let n = (2 * off * step_den + step_num).div_euclid(2 * step_num);
let at = i128::from(first) + (2 * n * step_num + step_den).div_euclid(2 * step_den);
let off_grid = (i128::from(pts) - at).abs();
if off_grid * 4 * step_den >= step_num {
return None;
}
Some((n, if off_grid <= 1 { pts } else { i64::try_from(at).ok()? }))
};
let Some(snapped) = t.samples.iter().map(|s| snap(s.pts)).collect::<Option<Vec<_>>>() else { return };
// Two pictures by one grid point: not a constant rate. Checked on the index, not the new time:
// one picture kept a unit after the point and another moved onto it would be distinct times in
// the wrong order.
let mut points: Vec<i128> = snapped.iter().map(|&(n, _)| n).collect();
points.sort_unstable();
points.dedup();
if points.len() != snapped.len() {
return;
}
for (s, (_, p)) in t.samples.iter_mut().zip(snapped) {
s.pts = p;
}
}
/// Start the file at its first picture when its video track has no edit list (#714). Without an
/// edit list the media timeline is the presentation, so a B-frame stream's first picture sits at its
/// composition delay (two frames on an x264 MP4 written without `elst`, and on fragmented MP4s):

101
crates/codecs/src/rounded_pts_tests.rs
View File

@ -134,3 +134,104 @@ fn webm_with_1ms_timestamps_shows_every_frame_once() {
fn mp4_with_a_1ms_timescale_shows_every_frame_once() {
check("mov", mp4_1ms);
}
/// An MP4 at a 90 kHz timescale whose frames after the first start `late` units after the frame
/// grid, as phone footage does (issue #1282): the first sample lasts `late` units longer.
fn mp4_90k_late(rate: FrameRate, late: i64) -> Arc<[u8]> {
let mut w = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mov)).unwrap();
let t = w.add_track(TrackConfig::new(SampleEntry::jpeg((BITS * BLOCK) as u16, BLOCK as u16), 90_000)).unwrap();
let start = |k: usize| {
let grid = (k as i128 * 90_000 * rate.den as i128 + rate.num as i128 / 2) / rate.num as i128;
grid as i64 + if k > 0 { late } else { 0 }
};
for k in 0..frame_count(rate) {
let duration = (start(k + 1) - start(k)) as u32;
w.write_sample(t, WriteSample { data: &picture(k), duration, composition_offset: 0, is_sync: true }).unwrap();
}
w.finish().unwrap().into_inner().into()
}
fn check_90k_late(late: i64) {
let mut failures = Vec::new();
for rate in RATES {
let src = crate::open_bytes("synthetic.mov", mp4_90k_late(rate, late)).unwrap();
for p in problems(src.as_ref(), rate) {
failures.push(format!("{rate}, frames {late}/90000 s late: {p}"));
}
}
assert!(failures.is_empty(), "mov:\n{}", failures.join("\n"));
}
#[test]
fn mp4_with_frames_1ms_after_the_grid_shows_every_frame_once() {
check_90k_late(90);
}
#[test]
fn mp4_with_frames_on_the_grid_shows_every_frame_once() {
check_90k_late(0);
}
/// An MP4 at 90 kHz with these sample durations.
fn mp4_90k(durations: &[u32]) -> Arc<[u8]> {
let mut w = Mp4Writer::new(Cursor::new(Vec::new()), WriterOptions::new(Brand::Mov)).unwrap();
let t = w.add_track(TrackConfig::new(SampleEntry::jpeg((BITS * BLOCK) as u16, BLOCK as u16), 90_000)).unwrap();
for (k, &duration) in durations.iter().enumerate() {
w.write_sample(t, WriteSample { data: &picture(k), duration, composition_offset: 0, is_sync: true }).unwrap();
}
w.finish().unwrap().into_inner().into()
}
/// Files that are not a constant rate gone slightly off its grid keep their own times: whatever is
/// asked (each frame's start, the unit before it, every 30 fps grid point), the frame shown is the
/// last one starting at or before it.
#[test]
fn mp4_variable_rate_and_on_grid_frame_times_stay_put() {
let cases: [(&str, Vec<u32>); 5] = [
// 1/30 s and 1/20 s in turn
("alternating", (0..60).map(|k| if k % 2 == 0 { 3000 } else { 4500 }).collect()),
// 30 fps, then every frame 0.4 frame later (one frame lasts 1.4 frames)
("shifted", (0..80).map(|k| if k == 40 { 4200 } else { 3000 }).collect()),
// 30 fps with a frame shown for a fifth of a frame: it and the next are both near one grid point
(
"stutter",
(0..80)
.map(|k| match k {
40 => 600,
41 => 5400,
_ => 3000,
})
.collect(),
),
// 30 fps with two frames by one grid point, one a unit after it and one five units after
(
"one unit and five after one point",
(0..80)
.map(|k| match k {
39 => 3001,
40 => 4,
41 => 2995,
_ => 3000,
})
.collect(),
),
// 59.94 fps on its grid, stamps truncated to the timescale (1501, 1502, …)
("59.94 truncated", (0..120).map(|k: u64| ((k + 1) * 1_501_500 / 1000 - k * 1_501_500 / 1000) as u32).collect()),
];
let mut failures = Vec::new();
for (name, durations) in cases {
let src = crate::open_bytes("synthetic.mov", mp4_90k(&durations)).unwrap();
let starts: Vec<i64> = durations.iter().scan(0i64, |t, &d| Some(std::mem::replace(t, *t + i64::from(d)))).collect();
let end = *starts.last().unwrap();
let mut asked: Vec<i64> = starts.iter().flat_map(|&s| [s, s - 1]).filter(|&s| s >= 0).collect();
asked.extend((0..).map(|k| k * 3000).take_while(|&t| t <= end));
for t in asked {
let want = starts.iter().rposition(|&s| s <= t).unwrap();
let got = index_of(&src.video_frame(FrameRequest::full(Tick::from_rational(t, 1, 90_000))).unwrap());
if got != want {
failures.push(format!("{name}: at {t}/90000 s wanted frame {want}, got {got}"));
}
}
}
assert!(failures.is_empty(), "{}", failures[..failures.len().min(8)].join("\n"));
}