shifathosn
/
filmcraft
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #19 from storybored379/pr/deterministic-export 

Export: same file on every machine (fixed mux interleave, H.264 slices by frame size)
This commit is contained in:
Brandon Thomas 2026-10-06 07:27:44 -04:00 committed by GitHub
parent 1f7e4f802e a55d20c454
commit 5012fbd598
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 133 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
crates/export/src/determinism_tests.rs Normal file
View File

@ -0,0 +1,80 @@
//! An export depends only on the project and the settings: the same file on every machine, however
//! many cores render it and however the work is cut into steps.
use super::*;
use crate::tests::project;
fn tmp(name: &str) -> String {
let d = std::env::temp_dir().join(format!("fc-export-det-{}", std::process::id()));
std::fs::create_dir_all(&d).unwrap();
d.join(name).to_string_lossy().to_string()
}
/// Export on a machine with `threads` cores: `export` renders one frame per core of the pool it
/// runs in.
fn export_with_cores(threads: usize, settings: &ExportSettings) -> Vec<u8> {
let (p, seq, m) = project();
let pool = rayon::ThreadPoolBuilder::new().num_threads(threads).build().unwrap();
pool.install(|| export(&p, seq, settings, &m, &Progress::default())).unwrap();
std::fs::read(&settings.path).unwrap()
}
/// Export a step at a time with `batch` frames per step (the web app uses 1).
fn export_stepped(batch: i64, settings: &ExportSettings) -> Vec<u8> {
let (p, seq, m) = project();
let prog = Progress::default();
let mut ex = Exporter::new(p, seq, settings, &prog).unwrap();
ex.set_batch(batch);
while !matches!(ex.step(&m, &prog).unwrap(), Step::Done(_)) {}
std::fs::read(&settings.path).unwrap()
}
#[test]
fn same_file_whatever_the_core_count() {
// 24 frames: one full interleave group and a partial one
for (format, ext) in [(Format::ProRes, "mov"), (Format::H264, "mp4"), (Format::Mjpeg, "mov")] {
let settings = |name: &str| ExportSettings { format, path: tmp(&format!("{name}.{ext}")), ..Default::default() };
let reference = export_with_cores(2, &settings(&format!("{}-2", format.id())));
for threads in [3, 5, 16] {
let other = export_with_cores(threads, &settings(&format!("{}-{threads}", format.id())));
assert!(reference == other, "{}: {threads} cores give a different file than 2 cores", format.label());
}
for batch in [1, 7] {
let other = export_stepped(batch, &settings(&format!("{}-b{batch}", format.id())));
assert!(reference == other, "{}: {batch} frame(s) per step give a different file", format.label());
}
}
}
/// Slice NAL units (types 1 and 5) in a length-prefixed (4-byte) H.264 sample.
fn slices_in(sample: &[u8]) -> usize {
let (mut n, mut i) = (0, 0);
while i + 4 <= sample.len() {
let len = u32::from_be_bytes([sample[i], sample[i + 1], sample[i + 2], sample[i + 3]]) as usize;
if let Some(&h) = sample.get(i + 4)
&& matches!(h & 0x1f, 1 | 5)
{
n += 1;
}
i += 4 + len;
}
n
}
#[test]
fn h264_slices_follow_the_frame_size_not_the_cores() {
// 4096 lines = 256 macroblock rows: one slice per four rows on every machine, also on machines
// with fewer than 64 cores (the encoder's default caps the slices at the core count)
let path = tmp("tall.mp4");
let s = ExportSettings { format: Format::H264, path: path.clone(), frame_size: Some((64, 4096)), include_audio: false, ..Default::default() };
let (p, seq, m) = project();
export(&p, seq, &s, &m, &Progress::default()).unwrap();
let bytes = std::fs::read(&path).unwrap();
let file = filmcraft_isobmff::open(bytes.as_slice()).unwrap();
let vt = file.track_of_kind(filmcraft_isobmff::TrackKind::Video).unwrap();
assert_eq!(file.tracks[vt].samples.len(), 24);
for i in 0..file.tracks[vt].samples.len() {
let sample = file.read_sample(bytes.as_slice(), vt, i).unwrap();
assert_eq!(slices_in(&sample), 64, "picture {i}");
}
}

50
crates/export/src/job.rs
View File

@ -11,6 +11,10 @@
//! When a source's bytes are not available yet (asynchronous web reads mark
//! [`filmcraft_media::pending`]), the batch is rendered again on the next call: nothing is encoded
//! from frames or audio with missing media.
//!
//! The file does not depend on how many frames a step renders (one per core natively, one on the
//! web): packets go to the muxer in fixed groups of [`INTERLEAVE`] output frames, each followed by
//! the audio up to its end, so the same project and settings give the same bytes on any machine.
use std::sync::Arc;
use std::sync::atomic::Ordering;
@ -30,6 +34,11 @@ use crate::{
audio_factories, export_range, frame_span, video_factories,
};
/// Output frames per interleaved group: the video packets of these frames, then the audio up to
/// their end, form one chunk of each track. Fixed (not the batch size, which follows the core
/// count) so the file layout is the same on every machine.
pub const INTERLEAVE: i64 = 16;
/// What one [`Exporter::step`] did.
#[derive(Debug)]
pub enum Step {
@ -58,7 +67,9 @@ pub struct Exporter {
measured: bool,
/// The first pass of a two-pass encode is running.
first_pass: bool,
/// Created after the first batch (encoders may finalise their codec config then).
/// Encoded video packets not written yet (the current [`INTERLEAVE`] group).
queued: Vec<EncodedPacket>,
/// Created after the first group was encoded (encoders may finalise their codec config then).
mux: Option<Mp4Writer<Out>>,
/// MXF exports write through this instead of `mux`.
mxf: Option<MxfMux>,
@ -140,6 +151,7 @@ impl Exporter {
audio,
measured,
first_pass: two_pass,
queued: Vec::new(),
mux: None,
mxf: None,
seq,
@ -295,22 +307,33 @@ impl Exporter {
self.next = end;
return Ok(Step::Progress);
}
let mixed = match self.audio.as_ref().map(|a| a.sr) {
Some(sr) if (self.mux.is_none() && self.mxf.is_none()) || self.at.is_some() || self.mxf.is_some() => {
let until = self.sample_at_frame(end, sr);
self.audio.as_mut().and_then(|a| a.pull(until, sources))
// the audio of every interleave group that ends in this batch (before encoding, so a
// pending source can still make the batch run again)
let cuts: Vec<i64> = (f + 1..=end).filter(|&g| (g - self.f0) % INTERLEAVE == 0).collect();
let mut mixed = Vec::with_capacity(cuts.len());
if let Some(sr) = self.audio.as_ref().map(|a| a.sr) {
for &g in &cuts {
let until = self.sample_at_frame(g, sr);
mixed.push(self.audio.as_mut().and_then(|a| a.pull(until, sources)));
}
_ => None,
};
}
if filmcraft_media::pending::take() {
return Ok(Step::Pending);
}
let packets = self.encode(&frames, f)?;
if self.mux.is_none() && self.mxf.is_none() {
self.open_mux()?;
let mut mixed = mixed.into_iter();
for (k, frame) in frames.iter().enumerate() {
let fi = f + k as i64;
let packets = self.encode(std::slice::from_ref(frame), fi)?;
self.queued.extend(packets);
if cuts.contains(&(fi + 1)) {
if self.mux.is_none() && self.mxf.is_none() {
self.open_mux()?;
}
let packets = std::mem::take(&mut self.queued);
self.write_video(packets)?;
self.write_audio(mixed.next().flatten())?;
}
}
self.write_video(packets)?;
self.write_audio(mixed)?;
progress.done.fetch_add((end - f) as u64, Ordering::Relaxed);
self.next = end;
return Ok(Step::Progress);
@ -332,7 +355,8 @@ impl Exporter {
if self.mux.is_none() && self.mxf.is_none() {
self.open_mux()?;
}
let tail = self.venc.flush()?;
let mut tail = std::mem::take(&mut self.queued);
tail.extend(self.venc.flush()?);
self.write_video(tail)?;
self.write_audio(mixed)?;
if let (Some(at), Some(a)) = (self.at, self.aenc.as_mut()) {

11
crates/export/src/lib.rs
View File

@ -916,6 +916,13 @@ pub fn rgbf_to_yuv420_8(rgb: &[f32], w: usize, h: usize, kr: f32, kb: f32, y: &m
});
}
/// Slices per H.264 picture: one per four macroblock rows. The encoder's default follows the core
/// count, which would make the stream (and every decoded picture) depend on the machine; this is
/// what it picks on a machine with enough cores.
pub(crate) fn h264_slices(height: u32) -> usize {
(height.div_ceil(16) as usize).div_ceil(4).max(1)
}
fn h264_factory(format: Format, w: u32, h: u32, rate: FrameRate, s: &ExportSettings) -> Option<Result<Box<dyn VideoEncoder>>> {
if format != Format::H264 {
return None;
@ -925,6 +932,7 @@ fn h264_factory(format: Format, w: u32, h: u32, rate: FrameRate, s: &ExportSetti
cfg.format = if s.format.is_mxf() { filmcraft_h264enc::PacketFormat::AnnexB } else { filmcraft_h264enc::PacketFormat::LengthPrefixed };
cfg.aud = s.format.is_mxf();
cfg.keyint = s.keyframe_distance.filter(|k| *k > 0).unwrap_or_else(|| (rate.num as f64 / rate.den as f64 * 2.0).round().max(1.0) as u32);
cfg.slices = h264_slices(h);
let kbps = s.bitrate_kbps.max(100);
let max = s.max_bitrate_kbps.filter(|m| *m >= kbps).unwrap_or(kbps * 3 / 2);
cfg.rate = match s.bitrate_mode {
@ -1129,3 +1137,6 @@ mod surround_tests;
#[cfg(test)]
mod mxf_tests;
#[cfg(test)]
mod determinism_tests;

5
docs/architecture.md
View File

@ -601,6 +601,11 @@ file.exportMedia {path, preset?, settings?, format?, range?, …} export.qui
| `png`, `tiff`, `bmp` | `image` | numbered stills `<name>000.<ext>`, `<name>001.<ext>` … |
| `gif`, `wav`, `aiff` | built in / `image` | GIF / RIFF WAVE (`WAVE_FORMAT_EXTENSIBLE` for 5.1) / AIFF (16- or 24-bit PCM) |
- **Reproducible.** The same project and settings give the same file on every machine. Frames render
in batches of one per core, but the muxer gets them in fixed groups of 16 output frames
(`INTERLEAVE` in `crates/export/src/job.rs`), each followed by its audio, and H.264 uses one slice
per four macroblock rows instead of one per core. MXF files are the exception: their UMIDs and
modification date come from the clock, as SMPTE ST 330 / ST 377 expect.
- **Settings.** `ExportSettings` (serde, camelCase, every field optional) holds Video (frame size or
Match Source, frame rate, Scale to Fit / Fill / Stretch, pixel aspect, field order — progressive
only, profile / level, bitrate encoding, target / maximum / adaptive bitrate, keyframe distance,