From c9b9bf3aae5c78764dba264f74fd69de59715311 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 11 Oct 2026 00:26:39 +0200 Subject: [PATCH 1/4] #800: MP2 audio in MP4 and MOV decodes instead of playing silent Problem: MP4 and QuickTime name MPEG audio without its layer (ffmpeg stores MP2 as `mp4a` + esds object type 0x6B), and we always made the layer III decoder for it. symphonia's decoder reads only the layer it was made for, so every MP2 frame was rejected and the track played as silence; MOV's `.mp2` sample entry was not recognised at all. Fix: the MPEG audio decoder for MP4 / MOV is made on the first packet, for the layer its frame header names (ISO/IEC 11172-3 2.4.2.3); `.mp2` maps to the MPEG audio codec like `.mp3`; the track is labelled "MPEG Audio" for layers I/II, as Matroska's is. Decoded tracks now match ffmpeg sample-exactly (edit list trims the 481-sample encoder delay). Co-Authored-By: Claude Opus 5.5 --- crates/codecs/src/audio.rs | 74 ++++++++++++++++++++++++++++++- crates/codecs/src/mp4.rs | 13 +++++- crates/codecs/tests/mp4_oracle.rs | 32 +++++++++++++ crates/isobmff/README.md | 1 + crates/isobmff/src/codec.rs | 20 ++++++++- docs/codec-parity.md | 2 +- docs/gaps.md | 10 +++-- 7 files changed, 143 insertions(+), 9 deletions(-) diff --git a/crates/codecs/src/audio.rs b/crates/codecs/src/audio.rs index 63351636..daba8f4c 100644 --- a/crates/codecs/src/audio.rs +++ b/crates/codecs/src/audio.rs @@ -20,6 +20,9 @@ enum Inner { Aac { dec: Box, asc: Vec, up: Option> }, /// AAC whose configuration arrives with the first frame (ADTS / LATM). AacPending, + /// MPEG audio at this rate whose layer arrives with the first frame (MP4 / QuickTime name the + /// codec, not the layer, and symphonia's decoder reads only the layer it was made for). + MpegPending(u32), /// Our own Opus decoder (always 48 kHz output; pre-skip is left to container timestamps). /// `order` maps output channel → decoded channel (Vorbis → WAV/SMPTE order for surround). Opus { dec: Box, order: Option<&'static [usize]> }, @@ -133,6 +136,12 @@ impl PacketDecoder { }; Self::new(codec, sample_rate, None) } + /// MPEG-1/2 audio of a layer the frame headers name (see [`mpeg_audio_layer`]): MP4's + /// `esds` object types 0x69 / 0x6B and QuickTime's `.mp3` / `.mp2` cover layers I–III alike + /// (ffmpeg stores MP2 as `mp4a` + 0x6B). + pub fn lazy_mpeg_audio(sample_rate: u32) -> Self { + Self { inner: Inner::MpegPending(sample_rate), channels: 0 } + } /// AAC configured by [`Self::ensure_aac`] from in-band headers (ADTS, LATM). pub fn lazy_aac() -> Self { Self { inner: Inner::AacPending, channels: 0 } @@ -160,7 +169,7 @@ impl PacketDecoder { use filmcraft_isobmff::CodecConfig as C; match c { C::Aac(a) => Self::aac(&a.asc, if rate > 0 { rate } else { a.sample_rate }), - C::Mp3 => Self::new(CODEC_TYPE_MP3, rate, None), + C::Mp3 => Ok(Self::lazy_mpeg_audio(rate)), C::Ac3 { .. } | C::Eac3 { .. } => Self::ac3(), C::Alac { cookie } => Self::new(CODEC_TYPE_ALAC, rate, Some(cookie.clone())), C::Flac(_) => Self::new(CODEC_TYPE_FLAC, rate, None), @@ -170,6 +179,10 @@ impl PacketDecoder { } /// Decode one packet into planar channels. pub fn decode(&mut self, data: &[u8], ts: u64) -> Result>> { + if let Inner::MpegPending(rate) = self.inner { + let layer = mpeg_audio_layer(data).ok_or_else(|| CodecError::Decode("MPEG audio: no frame header".into()))?; + self.inner = Self::mpeg_audio(layer, rate)?.inner; + } let dec = match &mut self.inner { Inner::Aac { dec, up, .. } => { let mut out = dec.decode(data).map_err(|e| CodecError::Decode(e.to_string()))?; @@ -192,6 +205,7 @@ impl PacketDecoder { } Inner::Symphonia(d) => d, Inner::AacPending => return Err(CodecError::Decode("AAC: no configuration yet".into())), + Inner::MpegPending(_) => return Err(CodecError::Decode("MPEG audio: no frame header".into())), Inner::Ac3(d) => { // a packet may hold several syncframes (MP4 / Matroska); E-AC-3 dependent // substreams and further programs decode to no channels and are skipped @@ -245,11 +259,21 @@ impl PacketDecoder { Inner::Opus { dec, .. } => dec.reset(), Inner::Symphonia(d) => d.reset(), Inner::Ac3(d) => d.reset(), - Inner::AacPending => {} + Inner::AacPending | Inner::MpegPending(_) => {} } } } +/// The layer (1, 2 or 3) named by the MPEG-1/2 audio frame header that starts `frame`: the 12-bit +/// sync word (11 bits for MPEG-2.5), then `ID` and the 2-bit `layer` field, 3 for layer I down to 1 +/// for layer III (ISO/IEC 11172-3 §2.4.2.3). `None` without a sync word or for the reserved layer 0. +pub fn mpeg_audio_layer(frame: &[u8]) -> Option { + match *frame { + [0xFF, b1, ..] if b1 & 0xE0 == 0xE0 && (b1 >> 1) & 3 != 0 => Some(4 - ((b1 >> 1) & 3)), + _ => None, + } +} + /// The output rate of an AAC stream: twice the core rate for HE-AAC (ISO/IEC 14496-3 §1.6.5). /// Explicit signalling names the rate in the `AudioSpecificConfig`; with implicit signalling (an /// AAC-LC config at 24 kHz or less) SBR is only visible as fill-element data in the access units, @@ -1000,3 +1024,49 @@ mod packet_time_tests { assert_eq!(fixed_packet_samples(FixedFrames::Eac3, &[0x0B, 0x77], 48_000), None); } } + +#[cfg(test)] +mod mpeg_audio_tests { + use super::*; + + /// A silent mono 48 kHz MPEG-1 frame of `layer` (2: 64 kbps, 192 bytes; 3: 56 kbps, 168 + /// bytes): header, then all-zero bit allocation / side information. + fn silent_frame(layer: u8) -> Vec { + let (b1, len) = if layer == 2 { (0xFD, 192) } else { (0xFB, 168) }; + let mut f = vec![0xFF, b1, 0x44, 0xC0]; + f.resize(len, 0); + f + } + + #[test] + fn layer_comes_from_the_frame_header() { + assert_eq!(mpeg_audio_layer(&[0xFF, 0xFF, 0x10, 0xC0]), Some(1)); + assert_eq!(mpeg_audio_layer(&silent_frame(2)), Some(2)); + assert_eq!(mpeg_audio_layer(&silent_frame(3)), Some(3)); + // MPEG-2 layer III, MPEG-2.5 layer II + assert_eq!(mpeg_audio_layer(&[0xFF, 0xF3, 0x90, 0x44]), Some(3)); + assert_eq!(mpeg_audio_layer(&[0xFF, 0xE5, 0x90, 0x44]), Some(2)); + // reserved layer, no sync word, too short + for bad in [&[0xFF, 0xF9, 0x44, 0xC0][..], &[0xFF, 0x1D], &[0x00, 0xFD], &[0xFF], &[]] { + assert_eq!(mpeg_audio_layer(bad), None, "{bad:?}"); + } + } + + /// #800: ffmpeg stores MP2 in MP4 as `mp4a` + object type 0x6B, which names no layer. Handing + /// it to the layer III decoder rejected every frame and the track played as silence. + #[test] + fn mp4_mpeg_audio_decodes_the_layer_its_frames_name() { + for layer in [2u8, 3] { + let mut d = PacketDecoder::for_isobmff(&filmcraft_isobmff::CodecConfig::Mp3, 48_000).unwrap(); + // a packet without a frame header is an error, not a crash, and configures nothing + assert!(d.decode(&[0u8; 16], 0).is_err()); + d.reset(); + for k in 0..3u64 { + let out = d.decode(&silent_frame(layer), k * 1152).unwrap_or_else(|e| panic!("layer {layer}: {e}")); + assert_eq!(out.len(), 1, "layer {layer}"); + assert_eq!(out[0].len(), 1152, "layer {layer}"); + assert!(out[0].iter().all(|&s| s == 0.0), "layer {layer}"); + } + } + } +} diff --git a/crates/codecs/src/mp4.rs b/crates/codecs/src/mp4.rs index 4a3a2c53..c7ea9d06 100644 --- a/crates/codecs/src/mp4.rs +++ b/crates/codecs/src/mp4.rs @@ -547,7 +547,18 @@ fn audio_stream_info(t: &filmcraft_isobmff::Track, bytes: &crate::Src) -> Option CodecConfig::Opus(o) => (crate::audio::OPUS_RATE, (o.output_channels as u32).max(1), None), _ => (if ap.sample_rate > 0.0 { ap.sample_rate as u32 } else { t.timescale }, ap.channels.max(1), None), }; - Some(AudioStreamInfo { sample_rate: rate.max(1), channels: ch.max(1), codec: codec_label(&entry.codec), bits_per_sample: bits }) + let codec = match &entry.codec { + // these entries name no layer (ffmpeg stores MP2 as `mp4a` + 0x6B): the first frame does + CodecConfig::Mp3 => { + let head = t.samples.first().and_then(|s| filmcraft_media::reader::read_range(&*bytes.0, s.offset, (s.size as usize).min(4)).ok()); + match head.as_deref().and_then(crate::audio::mpeg_audio_layer) { + Some(1 | 2) => "MPEG Audio".into(), + _ => codec_label(&entry.codec), + } + } + c => codec_label(c), + }; + Some(AudioStreamInfo { sample_rate: rate.max(1), channels: ch.max(1), codec, bits_per_sample: bits }) } fn codec_label(c: &CodecConfig) -> String { diff --git a/crates/codecs/tests/mp4_oracle.rs b/crates/codecs/tests/mp4_oracle.rs index 012424d7..cbb71ba0 100644 --- a/crates/codecs/tests/mp4_oracle.rs +++ b/crates/codecs/tests/mp4_oracle.rs @@ -215,3 +215,35 @@ fn open_gop_leading_pictures_decode_alone() { } } } + +/// #800: MP2 in MP4 (`mp4a` + object type 0x6B, which names no layer) and in MOV (QuickTime +/// `.mp2`) decodes sample-exactly as ffmpeg does: same length, the encoder delay (481 samples) +/// trimmed by the edit list, and the same samples from the first one on, also after a seek. It +/// used to go to the layer III decoder (MP4: silence) or not decode at all (MOV). +#[test] +fn mp2_in_mp4_and_mov_matches_ffmpeg() { + let ff = filmcraft_testkit::require_ffmpeg!(); + for (name, rate) in [("mp2_stereo_224k_48k.mp4", 48_000u32), ("mp2_stereo_224k_44k.mov", 44_100)] { + let input = format!("anoisesrc=d=1:c=pink:r={rate}:a=0.3"); + let Some(f) = fixture(&ff, name, &["-f", "lavfi", "-i", &input, "-ac", "2", "-c:a", "mp2", "-b:a", "224k"]) else { + eprintln!("SKIPPED ({name}): this ffmpeg can't encode it"); + continue; + }; + let src = filmcraft_codecs::open_bytes(name, bytes(&f)).unwrap(); + let a = src.info().audio().cloned().unwrap(); + assert_eq!((a.sample_rate, a.channels, a.codec.as_str()), (rate, 2, "MPEG Audio"), "{name}"); + let want = ffmpeg_audio_f32(&ff, &f, &[]); + let n = want.len() / 2; + assert!(n > rate as usize * 9 / 10, "{name}: ffmpeg decoded {n} samples"); + let worst = |from: usize, len: usize| { + let got = src.audio(from as i64, len, rate).unwrap(); + (0..len).map(|k| (0..2).map(|c| (got.channels[c][k] - want[2 * (from + k) + c]).abs()).fold(0f32, f32::max)).fold(0f32, f32::max) + }; + let level = want.iter().fold(0f32, |m, v| m.max(v.abs())); + assert!(level > 0.1, "{name}: the source is silent ({level})"); + // a seek into the middle first, then the whole track from the start + let (mid, all) = (worst(n / 2 + 77, 3000), worst(0, n)); + println!("{name}: {n} samples, max error mid {mid:.2e}, whole {all:.2e}"); + assert!(mid <= 1e-4 && all <= 1e-4, "{name}: differs from ffmpeg by {mid} (mid) / {all} (whole)"); + } +} diff --git a/crates/isobmff/README.md b/crates/isobmff/README.md index d74ae54a..47c31bb4 100644 --- a/crates/isobmff/README.md +++ b/crates/isobmff/README.md @@ -60,6 +60,7 @@ Boxes with 64-bit sizes and with size 0 (runs to the end of the file) are suppor | `jpeg mjpa mjpb` | `Jpeg { fourcc }` | | `AVdn AVdh` | `Dnx { fourcc }` | | `mp4a` + `esds` (in `wave` too) | `Aac(AacConfig)`: ASC bytes + decoded object type/rate/channels; `Mp3` for OTI 0x69/0x6B | +| `.mp3`/`.mp2` | `Mp3`: MPEG-1/2 audio of any layer (each frame header names it) | | PCM fourccs | `Pcm(PcmConfig)`: bits, float, endianness (`enda`, lpcm flags, `pcmC`), channels, rate | | `alac` | `Alac { cookie }` | | `Opus` + `dOps` | `Opus(OpusConfig)` | diff --git a/crates/isobmff/src/codec.rs b/crates/isobmff/src/codec.rs index b9afd53e..6d2ff79b 100644 --- a/crates/isobmff/src/codec.rs +++ b/crates/isobmff/src/codec.rs @@ -181,7 +181,8 @@ pub enum CodecConfig { /// Advanced Professional Video (`apv1` + `apvC`). Apv(ApvConfig), Aac(AacConfig), - /// MPEG-1/2 audio layer III (esds object type 0x69/0x6B, or QuickTime `.mp3`). + /// MPEG-1/2 audio (esds object type 0x69/0x6B, or QuickTime `.mp3` / `.mp2`). Usually layer + /// III, but these name no layer: ffmpeg stores MP2 as `mp4a` + 0x6B. Each frame header has it. Mp3, Pcm(PcmConfig), /// ALAC magic cookie (ALACSpecificConfig, 24 bytes, optionally followed by a channel layout). @@ -883,6 +884,7 @@ fn is_audio_fourcc(f: &[u8; 4]) -> bool { | b"ac-3" | b"ec-3" | b".mp3" + | b".mp2" | b"enca" | b"ulaw" | b"alaw" @@ -1028,7 +1030,7 @@ pub(crate) fn parse_sample_entry(format: FourCc, payload: &[u8], handler: FourCc Some(p) => parse_esds(p)?, None => unknown(), }, - b".mp3" => CodecConfig::Mp3, + b".mp3" | b".mp2" => CodecConfig::Mp3, b"alac" => match get(b"alac") { Some(p) if p.len() >= 4 => CodecConfig::Alac { cookie: p[4..].to_vec() }, _ => unknown(), @@ -1643,6 +1645,20 @@ mod tests { assert_eq!(roundtrip(&e, true, b"soun"), e); } + /// QuickTime's `.mp2` entry (what ffmpeg writes for MP2 in MOV) is MPEG audio, as `.mp3` is, + /// even under a handler that does not say "sound" (#800). + #[test] + fn quicktime_mpeg_audio_entries() { + for fourcc in [*b".mp3", *b".mp2"] { + let e = SampleEntry::audio(FourCc(fourcc), CodecConfig::Mp3, 2, 48000.0, 16); + for handler in [b"soun", b"\0\0\0\0"] { + let back = roundtrip(&e, true, handler); + assert_eq!(back.codec, CodecConfig::Mp3, "{fourcc:?} {handler:?}"); + assert_eq!(back.audio.map(|a| (a.channels, a.sample_rate)), Some((2, 48000.0)), "{fourcc:?} {handler:?}"); + } + } + } + #[test] fn pcm_entries() { for (bits, float, be, qt) in [ diff --git a/docs/codec-parity.md b/docs/codec-parity.md index ba0da95b..8fe1ff6b 100644 --- a/docs/codec-parity.md +++ b/docs/codec-parity.md @@ -42,7 +42,7 @@ contents). Percent is ready-for-real-work for that codec: depth, profiles, speed | AAC (LC, HE, LATM) | decode + encode | own decoder (LC, HE core), LATM | own LC encoder | loudness oracle vs ffmpeg | 90% | 3–5 h | | AC-3 | decode (licensed) | own decoder (ATSC A/52) | none | conformance | 80% | — | | E-AC-3 (Dolby Digital Plus) | decode | own decoder (ATSC A/52 Annex E, #647): independent substream, 1–6 blocks, spectral extension; **no** adaptive hybrid transform, enhanced coupling or 7.1 dependent substreams (plays the 5.1 core) | none | SNR vs ffmpeg 65–91 dB (raw, MP4, MOV, Matroska, TS) | 75% | 10–20 h | -| MP3 / MP2 | decode + MP3 encode | decode (symphonia, MPL-2.0; own MP2) | **none** | — | 60% | 10–20 h (MP3 encoder) | +| MP3 / MP2 | decode + MP3 encode | decode (symphonia, MPL-2.0; own MP2) in raw, MP4, MOV (#800), Matroska, TS, PS; **not** MP2 mono above 192 kbit/s or stereo below 64 kbit/s (ffmpeg's mono default; symphonia rejects them) | **none** | MP2 sample-exact vs ffmpeg (raw, MP4, MOV, TS, PS) | 60% | 10–20 h (MP3 encoder) | | Opus | — (not a Premiere import format) | own decoder, RFC 8251 range-exact | none | conformance vectors | ahead | — | | FLAC, ALAC, Vorbis | partial | decode (symphonia); FLAC in MP4 silent (#603) | none | — | 70% | 3–6 h | | WMA | decode on Windows | none | none | — | 0% | 10–20 h | diff --git a/docs/gaps.md b/docs/gaps.md index 56eddd32..4e14ba75 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -155,9 +155,13 @@ platforms** G4, G9, G17; **performance** G16; **ecosystem** G11; **measurement** ## G10. Container and codec gaps on import - **Have:** E-AC-3 (Dolby Digital Plus) in MP4, MOV, Matroska and MPEG-TS (#647): independent - substream, 1–6 blocks per syncframe, spectral extension. -- **Missing:** E-AC-3 adaptive hybrid transform, enhanced coupling and the extra channels of 7.1 - dependent substreams (played as the 5.1 core), AVI (#598; listed as an extension but no + substream, 1–6 blocks per syncframe, spectral extension. MP2 (MPEG audio layer II) in MP4 and + MOV, sample-exact against ffmpeg (#800). +- **Missing:** MP2 at bit rates ISO/IEC 11172-3 disallows: mono above 192 kbit/s (ffmpeg's default + for mono MP2 is 384 kbit/s) and stereo below 64 kbit/s; the bootstrap decoder (symphonia, also + 0.6) rejects every such frame, so the track plays silent: needs our own layer I/II decoder. + E-AC-3 adaptive hybrid transform, enhanced coupling and the extra channels of 7.1 dependent + substreams (played as the 5.1 core), AVI (#598; listed as an extension but no demuxer), WMV / ASF, DV / DVCPRO / DV100, MPEG-4 Part 2, FLAC in MP4 (#603), camera RAW (R3D incl. R3D NE, ARRIRAW, Sony RAW / X-OCN, Canon RAW, BRAW, ProRes RAW; #345, #383), JPEG 2000 / JPEG XS MXF, Cinema DNG; stills: PSD, EXR, DPX, Targa, HEIF / HEIC, Radiance HDR, AI / EPS; From 50ce8b090cb207748a7dd36bbf148a6509f84f87 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 11 Oct 2026 00:32:08 +0200 Subject: [PATCH 2/4] #805: Morph Cut morphs along optical flow instead of cross-dissolving Problem: Morph Cut rendered as a plain Cross Dissolve, so a jump cut in a talking-head shot showed the subject twice at half opacity during the transition, and the GPU plan mixed it as a dissolve too. Fix: Morph Cut warps both frames along the dense optical flow between them (the Lucas-Kanade flow of #780, crates/render/src/flow.rs) and blends, so the subject moves from its place in the outgoing shot to its place in the incoming one; where the frames do not correspond it stays the cross dissolve. Frames inside a Morph Cut are planned whole on the CPU (no longer a GPU-mixable "simple" transition) and cost as a complex transition in the preview estimate. Morph Cut goldens re-blessed. No face tracking yet. Co-Authored-By: Claude Opus 5.5 --- crates/render/src/plan.rs | 2 +- crates/render/src/preview.rs | 2 +- crates/render/src/tests.rs | 26 ++++++++++ crates/render/src/transitions/dissolve.rs | 5 ++ crates/render/src/transitions/goldens.txt | 6 +-- crates/render/src/transitions/legacy.rs | 3 +- crates/render/src/transitions/mod.rs | 4 +- crates/render/src/transitions/tests.rs | 58 +++++++++++++++++++++++ docs/gaps.md | 9 ++-- docs/target-app-parity.md | 2 +- 10 files changed, 103 insertions(+), 14 deletions(-) diff --git a/crates/render/src/plan.rs b/crates/render/src/plan.rs index 92b333cd..4dd0d4d0 100644 --- a/crates/render/src/plan.rs +++ b/crates/render/src/plan.rs @@ -159,7 +159,7 @@ fn cpu_frame(img: crate::Image) -> Arc { } fn simple_transition(id: &str) -> bool { - matches!(id, "cross_dissolve" | "dip_to_black" | "dip_to_white" | "morph_cut") + matches!(id, "cross_dissolve" | "dip_to_black" | "dip_to_white") } /// The standard effects of a media clip if the GPU can draw it (any blend mode, no opacity masks, diff --git a/crates/render/src/preview.rs b/crates/render/src/preview.rs index f113e904..8079502b 100644 --- a/crates/render/src/preview.rs +++ b/crates/render/src/preview.rs @@ -487,7 +487,7 @@ fn estimate_cost(project: &Project, seq: &Sequence, layers: &[Layer]) -> f64 { total += match l { Layer::Item(_, it) => layer_cost(project, it, px, false), Layer::Transition(_, x, a, b) => { - let simple = matches!(x.effect.effect.as_str(), "cross_dissolve" | "dip_to_black" | "dip_to_white" | "non_additive_dissolve" | "morph_cut"); + let simple = matches!(x.effect.effect.as_str(), "cross_dissolve" | "dip_to_black" | "dip_to_white" | "non_additive_dissolve"); let sides: f64 = [a, b].iter().filter_map(|s| s.map(|it| layer_cost(project, it, px, !simple))).sum(); sides + if simple { 1.0 } else { 12.0 * px } } diff --git a/crates/render/src/tests.rs b/crates/render/src/tests.rs index 31f5edfc..31eed64c 100644 --- a/crates/render/src/tests.rs +++ b/crates/render/src/tests.rs @@ -250,6 +250,32 @@ fn push_plan_preserves_reference_track_order_and_reverse() { } } +#[test] +fn morph_cut_frames_render_on_the_cpu_with_the_morph() { + // the GPU compositor can only cross-fade layers: a Morph Cut frame is planned whole on the + // CPU, so preview and export show the optical-flow morph, not a dissolve + let (mut p, red, ocean, seq, map) = setup(); + let a = place(&mut p, seq, 0, red, 0, 24); + let b = place(&mut p, seq, 0, ocean, 24, 24); + let rate = FrameRate::FPS_24; + p.sequence_mut(seq).unwrap().video_tracks[0].transitions.push(filmcraft_project::Transition { + id: filmcraft_project::TransitionId(998), + effect: filmcraft_project::find_effect("morph_cut").unwrap().instance(), + start: rate.tick_of(18), + duration: rate.tick_of(12), + from: Some(a), + to: Some(b), + align: Default::default(), + reverse: false, + }); + let opts = RenderOptions { scale: 0.5, ..Default::default() }; + assert!(plan::is_cpu_frame(&p, seq, rate.tick_of(24))); + assert!(!plan::is_cpu_frame(&p, seq, rate.tick_of(10)), "outside the transition the frame stays on the GPU"); + let plan = plan::plan_frame(&p, seq, rate.tick_of(24), opts, &map); + assert!(matches!(plan, plan::FramePlan::Image(_))); + assert_eq!(plan::execute_cpu(&plan).px, render_sequence(&p, seq, rate.tick_of(24), opts, &map).px); +} + /// A project with a 4 s bars-and-tone clip (1 kHz at −20 dBFS) on A1, with extra audio effects. fn tone_with(effects: &[(&str, &[(&str, f64)])]) -> (Project, ItemId, SourceMap) { let mut p = Project::new("fx"); diff --git a/crates/render/src/transitions/dissolve.rs b/crates/render/src/transitions/dissolve.rs index 208a74ee..14756eef 100644 --- a/crates/render/src/transitions/dissolve.rs +++ b/crates/render/src/transitions/dissolve.rs @@ -54,6 +54,11 @@ pub(super) fn apply(id: &str, t: &Tx) -> Option { } }) } + // a jump cut smoothed by morphing: both frames are warped along the dense optical flow + // between them and blended, so the subject moves from its place in the outgoing shot to + // its place in the incoming one instead of showing twice; where the frames do not + // correspond (no consistent motion) it is the cross dissolve + "morph_cut" => crate::flow::interpolate(t.a, t.b, p), "luma_fade" => { let soft = t.frac("softness").clamp(0.005, 1.0) * 0.5; let from_b = t.choice("source") == 1; diff --git a/crates/render/src/transitions/goldens.txt b/crates/render/src/transitions/goldens.txt index 91f56bba..82692a17 100644 --- a/crates/render/src/transitions/goldens.txt +++ b/crates/render/src/transitions/goldens.txt @@ -49,9 +49,9 @@ film_dissolve 0.75 1933591e335936415928335919426e1e426e9fa16e28426e1950821e50824 luma_fade 0.25 5c2320892320b62320e223205c3620893620b63620e236205c4920894920b64920e249205c5c20895c20b65c20e05c22 luma_fade 0.50 472835462f4c4547621a38702540681d438199cb8f0e46951352a80e53b43b81b60d53b60d60d40d60d60d60d60d60d6 luma_fade 0.75 0d39760d3976224f760d39760d46960d46969ad3960d46960d53b60d53b63b81b60d53b60d60d60d60d60d60d60d60d6 -morph_cut 0.25 4929356a2935912e35ad2935493a3d6a3a3daf5d3dad3a3d494c456a4c45975745ad4c45495d4d6a5d4d8b5d4dad5d4d -morph_cut 0.50 352e4b4b2e4b6c394b782e4b353e5b4b3e5ba8855b783e5b354e6b4b4e6b78656b784e6b355e7b4b5e7b615e7b785e7b -morph_cut 0.75 2134602c34604744604234602142782c4278a1ac784242782150902c50905a7390425090215fa82c5fa8375fa8425fa8 +morph_cut 0.25 472a36582c40892d3a563c75483b43683a3e9f5a44684671434c51604d528a685b7f5271465d556c5c44925c3fb9593d +morph_cut 0.50 372e473e31536a364a3f3b763e3d51503e559e7d5b444780324e71484e6f6972803e57a6375e77505d6e6b5c6b9c5557 +morph_cut 0.75 263357273563493f5c2a3a752f406737416e99a27729478d21508e344f88537b9b2e53a5295e98355d9b3560aa495da6 mosaic_transition 0.25 7f3633973633cb3633f336337f3633973633cb3633f336337f3633973633cb3633f3363359650d70650da4650dcd650d mosaic_transition 0.50 0d4a9f0d4a9fcc3b20cc3b200d4a9f0d4a9fcc3b20cc3b200d4a9f0d4a9fcc3b20cc3b2036566f36566f774e77774e77 mosaic_transition 0.75 0d46963d7696a6df964680960d46963d7696a6df964680960d46963d7696a6df964680960d66e50d66e50d66e50d66e5 diff --git a/crates/render/src/transitions/legacy.rs b/crates/render/src/transitions/legacy.rs index 7af9bb24..1cf8ddfd 100644 --- a/crates/render/src/transitions/legacy.rs +++ b/crates/render/src/transitions/legacy.rs @@ -56,7 +56,6 @@ pub(super) fn apply(e: &EffectInstance, a: &Image, b: &Image, p: f32) -> Option< id, "cross_dissolve" | "cross_dissolve_legacy" - | "morph_cut" | "film_dissolve" | "film_dissolve_legacy" | "additive_dissolve" @@ -93,7 +92,7 @@ fn legacy(id: &str, e: &EffectInstance, a: &Image, b: &Image, p: f32) -> Image { let aspect = (a.w as f32) / (a.h as f32).max(1.0); let pd = p as f64; match id { - "cross_dissolve" | "morph_cut" => masked(a, b, |_, _| p), + "cross_dissolve" => masked(a, b, |_, _| p), "film_dissolve" => { // blend in a gamma-2.2-like space for a filmic, less-dippy dissolve let mut out = Image::new(a.w, a.h); diff --git a/crates/render/src/transitions/mod.rs b/crates/render/src/transitions/mod.rs index ceb515f2..27c26922 100644 --- a/crates/render/src/transitions/mod.rs +++ b/crates/render/src/transitions/mod.rs @@ -18,8 +18,8 @@ //! //! Immersive (VR) transitions are flat approximations on the equirectangular frame: sampling wraps //! horizontally and the iris uses great-circle distance, but there is no sphere re-projection. -//! Morph Cut is a cross dissolve (no face tracking), and the Smart Tools / Text transitions are -//! procedural stand-ins (no content analysis, no typed text). +//! Morph Cut morphs with dense optical flow ([`crate::flow`]) but has no face tracking, and the +//! Smart Tools / Text transitions are procedural stand-ins (no content analysis, no typed text). use filmcraft_project::{EffectInstance, ParamValue}; use filmcraft_time::Tick; diff --git a/crates/render/src/transitions/tests.rs b/crates/render/src/transitions/tests.rs index b209a782..a096c55f 100644 --- a/crates/render/src/transitions/tests.rs +++ b/crates/render/src/transitions/tests.rs @@ -248,6 +248,64 @@ fn modern_and_legacy_ids_are_distinct_renders() { assert!(mean_abs(&m, &want) < 1e-6); } +/// A jump cut (256×144): a static background with a 64×72 subject whose left edge is at `x0`; +/// flat colours, or a smooth texture on both (the subject's own texture moves with it). +fn jump_cut_frame(x0: f32, textured: bool) -> Image { + let tex = |u: f32, v: f32, k: f32| { + 0.35 + 0.15 * (u * 0.21 * k).sin() * (v * 0.17).cos() + 0.1 * ((u + v) * 0.11 * k).sin() + 0.05 * (u * 0.53 - v * 0.31 * k).cos() + }; + paint(256, 144, |x, y| { + let inside = (x0..x0 + 64.0).contains(&x) && (36.0..108.0).contains(&y); + let l = match (textured, inside) { + (false, true) => 0.95, + (false, false) => 0.3, + (true, true) => 0.4 + tex(x - x0, y, 0.6), + (true, false) => 0.6 * tex(x, y, 1.0), + }; + [l, l * 0.9, l * 0.8, 1.0] + }) +} + +#[test] +fn morph_cut_moves_the_subject_instead_of_double_exposing() { + // the subject jumps 12 px between the shots; at the midpoint Morph Cut shows it once, 6 px + // along, where a cross dissolve shows two half-bright copies + let (a, b) = (jump_cut_frame(96.0, false), jump_cut_frame(108.0, false)); + let want = jump_cut_frame(102.0, false); + let morph = apply(&filmcraft_project::find_effect("morph_cut").unwrap().instance(), &a, &b, 0.5); + let dissolve = apply(&filmcraft_project::find_effect("cross_dissolve").unwrap().instance(), &a, &b, 0.5); + // mean red error down a column through the subject's rows + let column_err = |img: &Image, x: usize| (40..104).map(|y| (img.get(x, y)[0] - want.get(x, y)[0]).abs()).sum::() / 64.0; + // uncovered background, both edges, the middle, background still to be covered + for x in [98usize, 103, 134, 165, 170] { + let (m, d) = (column_err(&morph, x), column_err(&dissolve, x)); + assert!(m < 0.12, "x={x}: morph off by {m} (dissolve {d})"); + } + for x in [98usize, 170] { + assert!(column_err(&dissolve, x) > 0.3, "x={x}: the dissolve ghosts here"); + } + let (e_morph, e_dissolve) = (mean_abs(&morph, &want), mean_abs(&dissolve, &want)); + assert!(e_morph * 10.0 < e_dissolve, "morph {e_morph} vs dissolve {e_dissolve}"); + // the background away from the subject is left alone + for (x, y) in [(8, 8), (40, 70), (220, 130)] { + assert!(morph.get(x, y).iter().zip(a.get(x, y)).all(|(p, q)| (p - q).abs() < 1e-4), "({x}, {y})"); + } +} + +#[test] +fn morph_cut_tracks_textured_subjects_and_follows_the_progress() { + // detail on the subject and behind it: the morph lands much closer than the dissolve, at + // the midpoint and a quarter of the way through + for (p, from, to, at) in [(0.5f32, 96.0f32, 108.0f32, 102.0f32), (0.25, 96.0, 112.0, 100.0)] { + let (a, b) = (jump_cut_frame(from, true), jump_cut_frame(to, true)); + let want = jump_cut_frame(at, true); + let morph = apply(&filmcraft_project::find_effect("morph_cut").unwrap().instance(), &a, &b, p); + let dissolve = a.clone().lerp(&b, p); + let (e_morph, e_dissolve) = (mean_abs(&morph, &want), mean_abs(&dissolve, &want)); + assert!(e_morph * 2.0 < e_dissolve, "p={p}: morph {e_morph} vs dissolve {e_dissolve}"); + } +} + // --------------------------------------------------------------------------------------------- // Golden fingerprints diff --git a/docs/gaps.md b/docs/gaps.md index ab25b8af..628bd5a2 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -227,11 +227,12 @@ platforms** G4, G9, G17; **performance** G16; **ecosystem** G11; **measurement** ## G18. Effects depth: approximations and missing behaviour -- **Missing:** Warp Stabilizer is 2-D (Premiere's is 3-D subspace), Morph Cut approximate, Lumetri - HDR-aware maths, multicam paging beyond 16 angles, redesigned 26.0 mask tools (rounded corners, - constrained lines). +- **Missing:** Warp Stabilizer is 2-D (Premiere's is 3-D subspace), Morph Cut has no face tracking + (and occlusion edges are soft), Lumetri HDR-aware maths, multicam paging beyond 16 angles, + redesigned 26.0 mask tools (rounded corners, constrained lines). - **Have:** Optical Flow time interpolation synthesises motion-compensated in-between frames (dense - coarse-to-fine Lucas–Kanade, CPU; #780); Transform shutter angle motion blur (#415); Ultra Key + coarse-to-fine Lucas–Kanade, CPU; #780); Morph Cut morphs the subject along that optical flow + instead of cross-dissolving (CPU; #805); Transform shutter angle motion blur (#415); Ultra Key Setting applied as a parameter preset (#458). - **Estimate:** 60–110 h. diff --git a/docs/target-app-parity.md b/docs/target-app-parity.md index 7703c9a7..e74879f6 100644 --- a/docs/target-app-parity.md +++ b/docs/target-app-parity.md @@ -106,7 +106,7 @@ agent-hours to close the area to Premiere, sequential. | Area | Weight | Breadth | Ready | Hours | Evidence | Detail | |---|---|---|---|---|---|---| | Editing, timeline, trimming, multicam | 20% | ~90% | ~70% | 80–140 | Edit algebra, trim modes, Trim Monitor, multicam with audio sync, Premiere default keyboard (143 commands), Source playback and range dragging (10-08). Open: ripple-delete of a gap by column (#668), Backspace on gaps (#648), clip stretching (#653), deselect on empty space (#683), new tracks by dragging media (#483), Escape cancelling drags (#580), time remapping (#510), effects on a whole track (#623) | [ui-parity.md](ui-parity.md) | -| Effects and transitions | 12% | 100% (measured) | ~60% | 80–140 | Every effect exists; Warp Stabilizer is 2-D, Morph Cut and Auto Reframe approximate; Optical Flow time interpolation is motion compensated on the CPU (#780). 34 effect ids run on the GPU against ~82 of Premiere's ~92 marked Accelerated. No Object Mask (26.0, AI). | [hardware-parity.md](hardware-parity.md) | +| Effects and transitions | 12% | 100% (measured) | ~60% | 80–140 | Every effect exists; Warp Stabilizer is 2-D, Auto Reframe approximate, Morph Cut morphs with optical flow but without face tracking (#805); Optical Flow time interpolation is motion compensated on the CPU (#780). 34 effect ids run on the GPU against ~82 of Premiere's ~92 marked Accelerated. No Object Mask (26.0, AI). | [hardware-parity.md](hardware-parity.md) | | Media import and codecs | 12% | ~75% | ~60% | 150–250 | Own bit-exact decoders for H.264, HEVC, VP9, AV1, ProRes, DNx, APV, MPEG-2, AAC, AC-3, Opus. But H.264 only 8-bit 4:2:0 and HEVC no 4:2:2 in software (most 10-bit cameras, #626); no camera RAW (R3D, ARRIRAW, BRAW, Sony RAW, Canon RAW, ProRes RAW, #345, #383); no E-AC-3 (#647), AVI (#598), WMV, DV, XAVC / P2 metadata; Sony start timecode (#460); WebM black (#432) | [codec-parity.md](codec-parity.md), [file-format-parity.md](file-format-parity.md) | | Audio | 10% | ~88% | ~65% | 50–90 | Mixer graph, automation, all 53 effects, Essential Sound, 5.1, voice-over, Remix, multi-stream container audio (#361). Device and playback reports: crackle / cut-outs (#336, #410, #602, #605, #617); Enhance Speech is DSP, not a model; ALSA fallback fixed (#254) | | | Panels and UI fidelity | 10% | ~88% | ~60% | 120–200 | All main panels. Missing: on-screen transform handles in the Program monitor (#639), Effect Controls keyframe zoom / divider / row lines (#640–#645), docking rearrangement (#493), multi-monitor windows (#466), filmstrip thumbnails (#614), marquee in Project panel (#578) | [ui-parity.md](ui-parity.md) | From fb5863bc8a14d33edf7fcef0bd5ac6f41cc76cb1 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 11 Oct 2026 00:34:28 +0200 Subject: [PATCH 3/4] #806: export OpenEXR image sequences Problem: Premiere offers OpenEXR as an image-sequence export format; FilmCraft could not write EXR at all (docs/gaps.md G13). Fix: a new export format (id exr, "OpenEXR") writing numbered .exr stills, with a preset, the Include Alpha Channel option, the file.exportMedia format list and es/ja/zh-cn strings. The writer in crates/export/src/raster.rs follows the public OpenEXR file layout: version 2 single-part scan-line file, half-float B/G/R(/A) channels, no compression, one scan line per chunk, Rec.709 chromaticities; values are scene-linear (the pipeline's transfer undone) with premultiplied alpha, as EXR defines. No new dependency. ZIP / PIZ / DWA compression stays open. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/commands.rs | 2 +- crates/engine/src/export_tests.rs | 3 +- crates/export/src/lib.rs | 34 ++++- crates/export/src/presets.rs | 6 + crates/export/src/raster.rs | 227 +++++++++++++++++++++++++++- crates/export/src/settings.rs | 9 +- crates/export/src/settings_tests.rs | 45 +++++- crates/ui-egui/src/i18n/es.tsv | 3 + crates/ui-egui/src/i18n/ja.tsv | 3 + crates/ui-egui/src/i18n/zh-cn.tsv | 3 + docs/architecture.md | 2 +- docs/file-format-parity.md | 2 +- docs/gaps.md | 5 +- 13 files changed, 326 insertions(+), 18 deletions(-) diff --git a/crates/engine/src/commands.rs b/crates/engine/src/commands.rs index e3b162fd..72a32cb8 100644 --- a/crates/engine/src/commands.rs +++ b/crates/engine/src/commands.rs @@ -1285,7 +1285,7 @@ fn build() -> Vec { "Media…", ["File", "Export"], None, - r#"{"path":str,"preset":str?,"settings":ExportSettings?,"format":"h264|hevc|prores|dnxhr|apv|mjpeg|mxf-op1a|mxf-opatom|png|tiff|bmp|jpg|tga|dpx|gif|wav|aiff"?,"width":u32?,"height":u32?,"fps":f64?,"bitrateKbps":u32?,"bitrateMode":"cbr|vbr1Pass|vbr2Pass|crf"?,"crf":0..51=23,"hardwareEncoding":"off|auto"?,"scale":f32=1,"audio":bool=true,"quality":0..100,"burnCaptions":bool=false,"alpha":bool=false,"captionSidecar":"srt|vtt"?,"loudnessLufs":f64?,"proresProfile":"proxy|lt|standard|hq|4444|4444xq"?,"dnxProfile":"lb|sq|hq|hqx"?,"apvProfile":"422-10|422-12|444-10|444-12"?,"mxfVideoCodec":"dnxhr|proRes|h264"?,"sequence":id?,"range":"entire|inOut|workArea|custom"?,"startSeconds":f64?,"endSeconds":f64?,"wait":bool=false}"#, + r#"{"path":str,"preset":str?,"settings":ExportSettings?,"format":"h264|hevc|prores|dnxhr|apv|mjpeg|mxf-op1a|mxf-opatom|png|tiff|bmp|jpg|tga|dpx|exr|gif|wav|aiff"?,"width":u32?,"height":u32?,"fps":f64?,"bitrateKbps":u32?,"bitrateMode":"cbr|vbr1Pass|vbr2Pass|crf"?,"crf":0..51=23,"hardwareEncoding":"off|auto"?,"scale":f32=1,"audio":bool=true,"quality":0..100,"burnCaptions":bool=false,"alpha":bool=false,"captionSidecar":"srt|vtt"?,"loudnessLufs":f64?,"proresProfile":"proxy|lt|standard|hq|4444|4444xq"?,"dnxProfile":"lb|sq|hq|hqx"?,"apvProfile":"422-10|422-12|444-10|444-12"?,"mxfVideoCodec":"dnxhr|proRes|h264"?,"sequence":id?,"range":"entire|inOut|workArea|custom"?,"startSeconds":f64?,"endSeconds":f64?,"wait":bool=false}"#, has_seq, crate::export_tools::export_media ), diff --git a/crates/engine/src/export_tests.rs b/crates/engine/src/export_tests.rs index 6a8c5d71..b6d65d10 100644 --- a/crates/engine/src/export_tests.rs +++ b/crates/engine/src/export_tests.rs @@ -137,7 +137,7 @@ fn every_builtin_preset_exports_and_ffprobe_confirms() { Some(a) => (a[0].as_u64().unwrap(), a[1].as_u64().unwrap()), None => (1920, 1080), }; - if matches!(fmt.as_str(), "png" | "tiff" | "bmp" | "jpg" | "tga" | "dpx") { + if matches!(fmt.as_str(), "png" | "tiff" | "bmp" | "jpg" | "tga" | "dpx" | "exr") { let files = files_in(&sub); let ext = Path::new(&out).extension().unwrap().to_string_lossy().to_string(); assert_eq!(frames, 7, "frames 23..=29 at 23.976 fps"); @@ -151,6 +151,7 @@ fn every_builtin_preset_exports_and_ffprobe_confirms() { "jpg" => "mjpeg", "tga" => "targa", "dpx" => "dpx", + "exr" => "exr", _ => "bmp", }; assert_eq!(v["codec_name"], codec, "{name}"); diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 279384f2..c868e91c 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -6,7 +6,7 @@ //! //! Video encoders implement [`VideoEncoder`]; codec crates register theirs with //! [`register_encoder`] (H.264, ProRes …). Built in: Motion-JPEG (MOV), PNG / TIFF / BMP / JPEG / -//! Targa / DPX sequences, GIF, WAV and AIFF. +//! Targa / DPX / OpenEXR sequences, GIF, WAV and AIFF. //! //! [`ExportSettings`] carries every Export-mode setting (frame size, rate, bitrate encoding, audio //! format, multiplexer, captions, effects, metadata) as serde data; [`presets`] defines the @@ -106,6 +106,9 @@ pub enum Format { /// Numbered DPX stills (SMPTE ST 268, 10-bit RGB, BT.709). #[serde(rename = "dpx", alias = "DpxSequence")] DpxSequence, + /// Numbered OpenEXR stills (half-float scene-linear RGB, RGBA with [`ExportSettings::alpha`]). + #[serde(rename = "exr", alias = "ExrSequence")] + ExrSequence, #[serde(rename = "gif", alias = "Gif")] Gif, #[serde(rename = "wav", alias = "Wav")] @@ -140,6 +143,7 @@ impl Format { "jpg" | "jpgsequence" | "jpegsequence" => Format::JpegSequence, "tga" | "targa" | "tgasequence" | "targasequence" => Format::TgaSequence, "dpx" | "dpxsequence" => Format::DpxSequence, + "exr" | "openexr" | "exrsequence" | "openexrsequence" => Format::ExrSequence, "gif" | "animatedgif" => Format::Gif, "wav" | "waveform" | "waveformaudio" => Format::Wav, "aif" | "aiff" | "aifc" => Format::Aiff, @@ -162,6 +166,7 @@ impl Format { Format::JpegSequence => "jpg", Format::TgaSequence => "tga", Format::DpxSequence => "dpx", + Format::ExrSequence => "exr", Format::Gif => "gif", Format::Wav => "wav", Format::Aiff => "aiff", @@ -179,6 +184,7 @@ impl Format { Format::JpegSequence => "jpg", Format::TgaSequence => "tga", Format::DpxSequence => "dpx", + Format::ExrSequence => "exr", Format::Gif => "gif", Format::Wav => "wav", Format::Aiff => "aif", @@ -200,6 +206,7 @@ impl Format { Format::JpegSequence => "JPEG", Format::TgaSequence => "Targa", Format::DpxSequence => "DPX", + Format::ExrSequence => "OpenEXR", Format::Gif => "Animated GIF", Format::Wav => "Waveform Audio", Format::Aiff => "AIFF", @@ -209,7 +216,16 @@ impl Format { } /// Numbered stills, one per frame ([`image_sequence_path`]). pub fn is_image_sequence(self) -> bool { - matches!(self, Format::PngSequence | Format::TiffSequence | Format::BmpSequence | Format::JpegSequence | Format::TgaSequence | Format::DpxSequence) + matches!( + self, + Format::PngSequence + | Format::TiffSequence + | Format::BmpSequence + | Format::JpegSequence + | Format::TgaSequence + | Format::DpxSequence + | Format::ExrSequence + ) } /// An MXF container format. pub fn is_mxf(self) -> bool { @@ -232,7 +248,7 @@ impl Format { pub fn is_mp4_with(self, mux: Multiplexer) -> bool { self == Format::Av1 || (self.is_h26x() && mux == Multiplexer::Mp4) } - pub const ALL: [Format; 18] = [ + pub const ALL: [Format; 19] = [ Format::H264, Format::Hevc, Format::Av1, @@ -246,6 +262,7 @@ impl Format { Format::JpegSequence, Format::TgaSequence, Format::DpxSequence, + Format::ExrSequence, Format::Gif, Format::Wav, Format::Aiff, @@ -302,9 +319,10 @@ pub struct ExportSettings { /// sequence's HDR space and signal it (VUI / `colr` / `mdcv` / `clli` / SEI). #[serde(default)] pub sdr: bool, - /// Keep the alpha channel in PNG, TIFF and Targa sequences and ProRes 4444 / 4444 XQ QuickTime movies - /// (straight alpha, Premiere's "Include Alpha Channel"). Off, every frame is flattened over - /// black (#160). Other formats ignore it. + /// Keep the alpha channel in PNG, TIFF, Targa and OpenEXR sequences and ProRes 4444 / 4444 XQ + /// QuickTime movies (straight alpha, Premiere's "Include Alpha Channel"; OpenEXR stores it + /// premultiplied, as the format defines). Off, every frame is flattened over black (#160). + /// Other formats ignore it. #[serde(default)] pub alpha: bool, /// Output frame size (None = Match Source: the sequence size times `scale`). @@ -1659,7 +1677,8 @@ pub fn encode_png(rgba: Vec, w: u32, h: u32) -> Result> { } /// Encode one still of an image sequence (also Export Frame). PNG always carries the alpha -/// channel it is given; `alpha` keeps it in TIFF and Targa too, which are otherwise written as RGB. +/// channel it is given; `alpha` keeps it in TIFF, Targa and OpenEXR too, which are otherwise written +/// as RGB. /// JPEG uses the default export quality ([`encode_still_with_quality`] picks another). pub fn encode_still(format: Format, rgba: Vec, w: u32, h: u32, alpha: bool) -> Result> { encode_still_with_quality(format, rgba, w, h, alpha, ExportSettings::default().quality) @@ -1683,6 +1702,7 @@ pub fn encode_still_with_quality(format: Format, rgba: Vec, w: u32, h: u32, } Format::TgaSequence => return raster::tga(&rgba, w, h, alpha), Format::DpxSequence => return raster::dpx(&rgba, w, h), + Format::ExrSequence => return raster::exr(&rgba, w, h, alpha), Format::PngSequence => { image::ImageEncoder::write_image(image::codecs::png::PngEncoder::new(&mut out), &rgba, w, h, image::ExtendedColorType::Rgba8).map_err(enc)? } diff --git a/crates/export/src/presets.rs b/crates/export/src/presets.rs index 2a021d49..3846ce9a 100644 --- a/crates/export/src/presets.rs +++ b/crates/export/src/presets.rs @@ -160,6 +160,12 @@ pub fn builtin_presets() -> Vec { ExportSettings { format: Format::TgaSequence, ..Default::default() }, ), p("DPX Sequence", "Image Sequence", "Numbered DPX stills (10-bit RGB)", ExportSettings { format: Format::DpxSequence, ..Default::default() }), + p( + "OpenEXR Sequence", + "Image Sequence", + "Numbered OpenEXR stills (half float, linear, uncompressed)", + ExportSettings { format: Format::ExrSequence, ..Default::default() }, + ), p( "Animated GIF 640×360", "Animated GIF", diff --git a/crates/export/src/raster.rs b/crates/export/src/raster.rs index 59641319..74ea4081 100644 --- a/crates/export/src/raster.rs +++ b/crates/export/src/raster.rs @@ -1,9 +1,12 @@ -//! Uncompressed still formats written directly from RGBA8 frames: Targa and DPX. +//! Uncompressed still formats written directly from RGBA8 frames: Targa, DPX and OpenEXR. //! //! Targa follows the Truevision TGA File Format Specification 2.0 (image type 2, uncompressed //! true-colour, bottom-left origin, the layout every reader accepts). DPX follows SMPTE ST 268 //! (version 2.0 header): one RGB image element, 10 bits per component packed into 32-bit //! big-endian words (method A, two padding bits at the bottom), BT.709 transfer and colorimetry. +//! OpenEXR follows the public OpenEXR file layout (version 2, single-part scan-line file): half-float +//! B, G, R (and A) channels, no compression, one scan line per chunk, scene-linear Rec.709 values +//! with premultiplied alpha, as EXR readers expect. use crate::{ExportError, Result}; /// `w * h * 4` when `rgba` holds exactly that many bytes. @@ -129,6 +132,120 @@ pub(crate) fn dpx(rgba: &[u8], w: u32, h: u32) -> Result> { Ok(out) } +/// IEEE 754 binary16 bits of `v`, rounded to nearest even (overflow → infinity, NaN → quiet NaN). +fn half(v: f32) -> u16 { + let b = v.to_bits(); + let sign = ((b >> 16) & 0x8000) as u16; + let exp = ((b >> 23) & 0xFF) as i32; + let man = b & 0x7F_FFFF; + if exp == 0xFF { + return sign | 0x7C00 | if man != 0 { 0x200 } else { 0 }; + } + // rounds `m >> shift` to nearest, ties to even (`shift` is 1..=24) + let round = |m: u32, shift: u32| { + let (q, rem, mid) = (m >> shift, m & ((1 << shift) - 1), 1 << (shift - 1)); + if rem > mid || (rem == mid && q & 1 == 1) { q + 1 } else { q } + }; + let e = exp - 127 + 15; + if e >= 0x1F { + sign | 0x7C00 + } else if e <= 0 { + if e < -10 { + return sign; + } + // subnormal half: the 24-bit significand scaled by 2^(e - 14); rounding up may carry + // into the smallest normal, which is the right encoding + sign | round(man | 0x80_0000, (14 - e) as u32) as u16 + } else { + // a carry out of the mantissa bumps the exponent (up to infinity), as it should + sign | round(((e as u32) << 23) | man, 13) as u16 + } +} + +/// An OpenEXR file: half-float scene-linear Rec.709 RGB, or RGBA with `alpha` (premultiplied, as +/// EXR defines alpha; the frame's straight alpha is multiplied in after linearisation). +pub(crate) fn exr(rgba: &[u8], w: u32, h: u32, alpha: bool) -> Result> { + let n = checked_len(rgba, w, h)?; + let (Ok(x_max), Ok(y_max)) = (i32::try_from(w - 1), i32::try_from(h - 1)) else { + return Err(ExportError::Unsupported(format!("an OpenEXR frame of {w}×{h} is too large"))); + }; + // channels in the alphabetical order the file layout requires + let names: &[u8] = if alpha { b"ABGR" } else { b"BGR" }; + let line = (w as usize).checked_mul(names.len() * 2).ok_or_else(|| ExportError::Encode("frame size".into()))?; + let line_size = i32::try_from(line).map_err(|_| ExportError::Unsupported(format!("an OpenEXR frame of {w}×{h} is too large")))?; + + let mut hd = Vec::with_capacity(512); + hd.extend_from_slice(&[0x76, 0x2F, 0x31, 0x01]); // magic number 20000630 + hd.extend_from_slice(&[2, 0, 0, 0]); // version 2, single-part scan-line file, short names + let attr = |hd: &mut Vec, name: &str, ty: &str, value: &[u8]| { + for s in [name, ty] { + hd.extend_from_slice(s.as_bytes()); + hd.push(0); + } + hd.extend_from_slice(&(value.len() as i32).to_le_bytes()); + hd.extend_from_slice(value); + }; + let le = |v: &[f32]| v.iter().flat_map(|f| f.to_le_bytes()).collect::>(); + let mut channels = Vec::new(); + for &c in names { + channels.extend_from_slice(&[c, 0]); + channels.extend_from_slice(&1_i32.to_le_bytes()); // pixel type HALF + channels.extend_from_slice(&[0, 0, 0, 0]); // pLinear 0, reserved + channels.extend_from_slice(&1_i32.to_le_bytes()); // x sampling + channels.extend_from_slice(&1_i32.to_le_bytes()); // y sampling + } + channels.push(0); + let window: Vec = [0, 0, x_max, y_max].iter().flat_map(|v| v.to_le_bytes()).collect(); + attr(&mut hd, "channels", "chlist", &channels); + attr(&mut hd, "chromaticities", "chromaticities", &le(&[0.64, 0.33, 0.30, 0.60, 0.15, 0.06, 0.3127, 0.3290])); + attr(&mut hd, "compression", "compression", &[0]); // NO_COMPRESSION + attr(&mut hd, "dataWindow", "box2i", &window); + attr(&mut hd, "displayWindow", "box2i", &window); + attr(&mut hd, "lineOrder", "lineOrder", &[0]); // INCREASING_Y + attr(&mut hd, "pixelAspectRatio", "float", &le(&[1.0])); + attr(&mut hd, "screenWindowCenter", "v2f", &le(&[0.0, 0.0])); + attr(&mut hd, "screenWindowWidth", "float", &le(&[1.0])); + hd.push(0); // end of header + + // offset table (one chunk per scan line), then the chunks: y, data size, pixel data + let chunk = line + 8; + let table = (h as usize).checked_mul(8); + let body = (h as usize).checked_mul(chunk); + let size = table.zip(body).and_then(|(t, b)| hd.len().checked_add(t)?.checked_add(b)).ok_or_else(|| ExportError::Encode("frame size".into()))?; + let mut out = Vec::with_capacity(size); + out.extend_from_slice(&hd); + let first = (hd.len() + h as usize * 8) as u64; + for y in 0..h as u64 { + out.extend_from_slice(&(first + y * chunk as u64).to_le_bytes()); + } + let lin = filmcraft_color::srgb_u8_to_linear_table(); + let opaque: [u16; 256] = std::array::from_fn(|i| half(lin[i])); + let row = w as usize * 4; + let mut px = vec![0_u16; w as usize * names.len()]; + for (y, src) in rgba[..n].chunks_exact(row).enumerate() { + // planar per line: every pixel of the first channel, then the next channel … + let wn = w as usize; + for (x, &[r, g, b, a]) in src.as_chunks::<4>().0.iter().enumerate() { + let rgb = [b, g, r]; + if alpha { + let k = f32::from(a) / 255.0; + px[x] = half(k); + for (c, &v) in rgb.iter().enumerate() { + px[(c + 1) * wn + x] = if a == 255 { opaque[v as usize] } else { half(lin[v as usize] * k) }; + } + } else { + for (c, &v) in rgb.iter().enumerate() { + px[c * wn + x] = opaque[v as usize]; + } + } + } + out.extend_from_slice(&(y as i32).to_le_bytes()); + out.extend_from_slice(&line_size.to_le_bytes()); + out.extend(px.iter().flat_map(|v| v.to_le_bytes())); + } + Ok(out) +} + #[cfg(test)] mod tests { use super::*; @@ -167,6 +284,111 @@ mod tests { assert_eq!(&f[18..26], &[255, 0, 0, 128, 0, 0, 255, 255]); } + #[test] + fn half_floats_round_to_nearest_even() { + for (v, h) in [ + (0.0, 0x0000), + (-0.0, 0x8000), + (1.0, 0x3C00), + (0.5, 0x3800), + (-2.0, 0xC000), + (65504.0, 0x7BFF), + (65520.0, 0x7C00), // halfway past the largest half: ties to even = infinity + (1.0e9, 0x7C00), // overflow + (6.103_515_6e-5, 0x0400), // smallest normal + (5.960_464_5e-8, 0x0001), // smallest subnormal, 2^-24 + (2.980_232_2e-8, 0x0000), // 2^-25: halfway to zero, ties to even + (1.0e-10, 0x0000), + (1.000_488_3, 0x3C00), // 1 + 2^-11: halfway, ties to even (down) + (1.001_464_8, 0x3C02), // 1 + 3·2^-11: halfway, ties to even (up) + (f32::INFINITY, 0x7C00), + ] { + assert_eq!(half(v), h, "{v}"); + } + assert_eq!(half(f32::NAN) & 0x7E00, 0x7E00); + } + + fn le32(b: &[u8], at: usize) -> u32 { + u32::from_le_bytes(b[at..at + 4].try_into().unwrap()) + } + + /// Where `needle` (an attribute name, NUL-terminated) starts in the header. + fn find(b: &[u8], needle: &[u8]) -> usize { + b.windows(needle.len()).position(|w| w == needle).unwrap() + } + + #[test] + fn exr_layout() { + // 2×1: opaque white, then (255, 0, 0) at alpha 0.5 + let px = [255, 255, 255, 255, 255, 0, 0, 128]; + for alpha in [false, true] { + let f = exr(&px, 2, 1, alpha).unwrap(); + let nch = if alpha { 4 } else { 3 }; + assert_eq!(&f[..8], &[0x76, 0x2F, 0x31, 0x01, 2, 0, 0, 0]); + let ch = find(&f, b"channels\0chlist\0") + 16; + assert_eq!(le32(&f, ch) as usize, nch * 18 + 1); + let names: Vec = (0..nch).map(|i| f[ch + 4 + i * 18]).collect(); + assert_eq!(names, if alpha { b"ABGR".to_vec() } else { b"BGR".to_vec() }); + let dw = find(&f, b"dataWindow\0box2i\0") + 17; + assert_eq!((le32(&f, dw), le32(&f, dw + 4), le32(&f, dw + 8), le32(&f, dw + 12)), (16, 0, 0, 1)); + let cmp = find(&f, b"compression\0compression\0") + 24; + assert_eq!((le32(&f, cmp), f[cmp + 4]), (1, 0)); + // header ends with a NUL; one offset (scan line 0) points at the only chunk + let end = f.len() - (8 + 2 * nch * 2) - 8; + assert_eq!(f[end - 1], 0); + let off = u64::from_le_bytes(f[end..end + 8].try_into().unwrap()) as usize; + assert_eq!(off, end + 8); + assert_eq!((le32(&f, off), le32(&f, off + 4) as usize), (0, 2 * nch * 2)); + let hv = |c: usize, x: usize| u16::from_le_bytes(f[off + 8 + (c * 2 + x) * 2..][..2].try_into().unwrap()); + let (b, g, r) = if alpha { (1, 2, 3) } else { (0, 1, 2) }; + // white is 1.0 everywhere; red is linear 1.0 · alpha (premultiplied) or 1.0 flat + assert_eq!((hv(b, 0), hv(g, 0), hv(r, 0)), (0x3C00, 0x3C00, 0x3C00)); + assert_eq!((hv(b, 1), hv(g, 1)), (0, 0)); + if alpha { + assert_eq!((hv(0, 0), hv(0, 1)), (0x3C00, half(128.0 / 255.0))); + assert_eq!(hv(r, 1), half(128.0 / 255.0)); + } else { + assert_eq!(hv(r, 1), 0x3C00); + } + } + } + + /// ffmpeg (external test oracle) decodes our OpenEXR files to the values we meant: linear + /// light, premultiplied alpha, rows top to bottom. + #[test] + fn exr_decodes_in_ffmpeg() { + let Some(ffmpeg) = filmcraft_testkit::ffmpeg_or_skip("OpenEXR stills") else { return }; + let (w, h) = (5_u32, 3_u32); + let px: Vec = (0..w * h).flat_map(|i| [(i * 17) as u8, (255 - i * 13) as u8, (i * 40 % 256) as u8, [255, 128, 0, 64][i as usize % 4]]).collect(); + let dir = filmcraft_testkit::fixtures::workspace_root().join("target").join("export-tests").join(format!("exr-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let lin = filmcraft_color::srgb_u8_to_linear_table(); + for alpha in [false, true] { + let path = dir.join(format!("f{alpha}.exr")); + std::fs::write(&path, exr(&px, w, h, alpha).unwrap()).unwrap(); + let out = std::process::Command::new(&ffmpeg) + .args(["-v", "error", "-i"]) + .arg(&path) + .args(["-f", "rawvideo", "-pix_fmt", "gbrapf32le", "-"]) + .output() + .unwrap(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + let n = (w * h) as usize; + assert_eq!(out.stdout.len(), n * 4 * 4); + let plane = |p: usize, i: usize| f32::from_le_bytes(out.stdout[(p * n + i) * 4..][..4].try_into().unwrap()); + for (i, &[r, g, b, a]) in px.as_chunks::<4>().0.iter().enumerate() { + let k = if alpha { f32::from(a) / 255.0 } else { 1.0 }; + // gbrap planes: G, B, R, A + let want = [lin[g as usize] * k, lin[b as usize] * k, lin[r as usize] * k, k]; + for (p, want) in want.into_iter().enumerate() { + let got = plane(p, i); + assert!((got - want).abs() <= 1e-3 + want * 1e-3, "alpha {alpha}, pixel {i}, plane {p}: {got} vs {want}"); + } + } + } + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn hostile_sizes_are_errors() { assert!(dpx(&[0; 7], 2, 1).is_err()); @@ -175,5 +397,8 @@ mod tests { assert!(tga(&[0; 4], u32::MAX, u32::MAX, false).is_err()); assert!(rgb(&[0; 3], 1, 1).is_err()); assert!(dpx(&[0; 4], u32::MAX, u32::MAX).is_err()); + assert!(exr(&[0; 4], u32::MAX, u32::MAX, true).is_err()); + assert!(exr(&[0; 7], 2, 1, false).is_err()); + assert!(exr(&[], 0, 1, false).is_err()); } } diff --git a/crates/export/src/settings.rs b/crates/export/src/settings.rs index 61ee9dbd..162c958a 100644 --- a/crates/export/src/settings.rs +++ b/crates/export/src/settings.rs @@ -378,11 +378,11 @@ impl ExportSettings { if self.format.is_mxf() { self.mxf_video_codec.encoder_format() } else { self.format } } - /// Whether the output can carry an alpha channel ([`ExportSettings::alpha`]): PNG, TIFF and - /// Targa sequences and ProRes 4444 / 4444 XQ QuickTime movies. + /// Whether the output can carry an alpha channel ([`ExportSettings::alpha`]): PNG, TIFF, + /// Targa and OpenEXR sequences and ProRes 4444 / 4444 XQ QuickTime movies. pub fn supports_alpha(&self) -> bool { match self.format { - Format::PngSequence | Format::TiffSequence | Format::TgaSequence => true, + Format::PngSequence | Format::TiffSequence | Format::TgaSequence | Format::ExrSequence => true, Format::ProRes => crate::prores_profile(&self.prores_profile).chroma() == filmcraft_prores::ChromaFormat::Yuv444, _ => false, } @@ -488,6 +488,8 @@ impl ExportSettings { Format::TgaSequence => px * if self.keeps_alpha() { 4.0 } else { 3.0 } * 8.0 * fps, // 10-bit RGB in one 32-bit word per pixel Format::DpxSequence => px * 4.0 * 8.0 * fps, + // two bytes per half-float channel + Format::ExrSequence => px * if self.keeps_alpha() { 8.0 } else { 6.0 } * 8.0 * fps, Format::Gif => px * 0.6 * 8.0 * fps, Format::Wav | Format::Aiff | Format::MxfOp1a | Format::MxfOpAtom => 0.0, }; @@ -571,6 +573,7 @@ impl ExportSettings { Format::Apv => v += &format!(", {}", crate::apv_profile(&self.apv_profile).name()), Format::Mjpeg | Format::JpegSequence => v += &format!(", quality {}", self.quality), Format::DpxSequence => v += ", 10-bit RGB", + Format::ExrSequence => v += ", 16-bit half float, linear", _ => {} } v diff --git a/crates/export/src/settings_tests.rs b/crates/export/src/settings_tests.rs index 8cfb3170..5381a194 100644 --- a/crates/export/src/settings_tests.rs +++ b/crates/export/src/settings_tests.rs @@ -160,6 +160,49 @@ fn jpeg_targa_and_dpx_sequences() { assert!(jpeg_sizes[0] > jpeg_sizes[1], "quality 95 must be larger than quality 20: {jpeg_sizes:?}"); } +/// OpenEXR sequences (#806): numbered like the others, half-float linear light, RGBA with +/// premultiplied alpha when asked; ffprobe / ffmpeg (external test oracles) read every still. +#[test] +fn openexr_sequences() { + let (p, seq, m) = matte([0.0, 0.0, 1.0, 0.5], 64, 36, None); + for alpha in [false, true] { + let dir = Scratch::new(&format!("seq-exr-{alpha}")); + let mut s = ExportSettings { format: Format::ExrSequence, path: dir.path("Shot.exr"), alpha, ..Default::default() }; + s.range = Some(TimeRange::new(Tick::ZERO, FrameRate::FPS_24.tick_of(3))); + assert_eq!(Format::from_name("OpenEXR"), Some(Format::ExrSequence)); + assert!(s.is_image_sequence() && s.supports_alpha() && s.keeps_alpha() == alpha && !s.has_audio()); + let r = export(&p, seq, &s, &m, &Progress::default()).unwrap(); + assert_eq!(r.frames, 3); + let want: Vec = (0..3).map(|i| format!("Shot{i:03}.exr")).collect(); + assert_eq!(files_in(&dir.0), want); + let first = dir.0.join(&want[0]); + let bytes = std::fs::read(&first).unwrap(); + assert_eq!(&bytes[..4], &[0x76, 0x2F, 0x31, 0x01]); + let Some(ffmpeg) = filmcraft_testkit::ffmpeg_or_skip("OpenEXR sequences") else { continue }; + let out = std::process::Command::new(ffmpeg) + .args(["-v", "error", "-i"]) + .arg(&first) + .args(["-f", "rawvideo", "-pix_fmt", "gbrapf32le", "-"]) + .output() + .unwrap(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + let n = 64 * 36; + assert_eq!(out.stdout.len(), n * 16); + let plane = |p: usize| -> Vec { out.stdout[p * n * 4..(p + 1) * n * 4].as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect() }; + let (g, b, r, a) = (plane(0), plane(1), plane(2), plane(3)); + for i in 0..n { + assert!(g[i].abs() < 1e-3 && r[i].abs() < 1e-3, "pixel {i}: no red or green"); + if alpha { + // pure blue (linear 1.0) at 50 % opacity, premultiplied + assert!((a[i] - 0.5).abs() < 0.01 && (b[i] - a[i]).abs() < 2e-3, "pixel {i}: {} / {}", b[i], a[i]); + } else { + // flattened over black, then linearised: well below the encoded half level + assert!(a[i] == 1.0 && b[i] > 0.1 && b[i] < 0.5, "pixel {i}: {} / {}", b[i], a[i]); + } + } + } +} + /// `alpha` keeps straight alpha in PNG and TIFF sequences; off (the default) flattens over black (#160). #[test] fn image_sequences_keep_alpha_when_asked() { @@ -658,7 +701,7 @@ fn h265_is_a_format_that_needs_a_registered_encoder() { } assert_eq!(Format::from_name(Format::Hevc.id()), Some(Format::Hevc)); assert_eq!((Format::Hevc.extension(), Format::Hevc.label()), ("mp4", "H.265 (HEVC)")); - assert!(Format::ALL.contains(&Format::Hevc) && Format::ALL.len() == 18); + assert!(Format::ALL.contains(&Format::Hevc) && Format::ALL.len() == 19); let s: ExportSettings = serde_json::from_value(serde_json::json!({"format": "hevc"})).unwrap(); assert_eq!(s.format, Format::Hevc); assert_eq!(serde_json::to_value(&s).unwrap()["format"], "hevc"); diff --git a/crates/ui-egui/src/i18n/es.tsv b/crates/ui-egui/src/i18n/es.tsv index 51d1cd03..e8fd5a32 100644 --- a/crates/ui-egui/src/i18n/es.tsv +++ b/crates/ui-egui/src/i18n/es.tsv @@ -3182,6 +3182,9 @@ Numbered JPEG stills (quality 90) Imágenes fijas JPEG numeradas (calidad 90) Numbered Targa stills (24-bit, uncompressed) Imágenes fijas Targa numeradas (24 bits, sin comprimir) Numbered DPX stills (10-bit RGB) Imágenes fijas DPX numeradas (RGB de 10 bits) + OpenEXR OpenEXR + OpenEXR Sequence Secuencia OpenEXR + Numbered OpenEXR stills (half float, linear, uncompressed) Imágenes fijas OpenEXR numeradas (coma flotante de 16 bits, lineal, sin comprimir) Numbered PNG stills at the sequence's frame size Imágenes fijas PNG numeradas al tamaño de fotograma de la secuencia Numbered TIFF stills (RGB, 8-bit) Imágenes fijas TIFF numeradas (RGB, 8 bits) Opacity 0 → 100 % over the first second Opacidad de 0 a 100 % durante el primer segundo diff --git a/crates/ui-egui/src/i18n/ja.tsv b/crates/ui-egui/src/i18n/ja.tsv index 79c7d025..defaea9a 100644 --- a/crates/ui-egui/src/i18n/ja.tsv +++ b/crates/ui-egui/src/i18n/ja.tsv @@ -3171,6 +3171,9 @@ Numbered JPEG stills (quality 90) 連番 JPEG 静止画 (品質 90) Numbered Targa stills (24-bit, uncompressed) 連番 Targa 静止画 (24-bit、非圧縮) Numbered DPX stills (10-bit RGB) 連番 DPX 静止画 (10-bit RGB) + OpenEXR OpenEXR + OpenEXR Sequence OpenEXR 連番画像 + Numbered OpenEXR stills (half float, linear, uncompressed) 連番 OpenEXR 静止画 (16-bit 浮動小数点、リニア、非圧縮) Numbered PNG stills at the sequence's frame size シーケンスと同じフレームサイズの連番 PNG 静止画 Numbered TIFF stills (RGB, 8-bit) 連番 TIFF 静止画 (RGB、8-bit) Opacity 0 → 100 % over the first second 最初の 1 秒で不透明度を 0 → 100 % に変化 diff --git a/crates/ui-egui/src/i18n/zh-cn.tsv b/crates/ui-egui/src/i18n/zh-cn.tsv index 16472ef0..21b84306 100644 --- a/crates/ui-egui/src/i18n/zh-cn.tsv +++ b/crates/ui-egui/src/i18n/zh-cn.tsv @@ -3165,6 +3165,9 @@ Numbered JPEG stills (quality 90) 带编号的 JPEG 静帧（质量 90） Numbered Targa stills (24-bit, uncompressed) 带编号的 Targa 静帧（24 位，未压缩） Numbered DPX stills (10-bit RGB) 带编号的 DPX 静帧（10 位 RGB） + OpenEXR OpenEXR + OpenEXR Sequence OpenEXR 序列 + Numbered OpenEXR stills (half float, linear, uncompressed) 带编号的 OpenEXR 静帧（16 位半精度浮点，线性，未压缩） Numbered PNG stills at the sequence's frame size 带编号的 PNG 静帧（采用序列帧大小） Numbered TIFF stills (RGB, 8-bit) 带编号的 TIFF 静帧（RGB，8 位） Opacity 0 → 100 % over the first second 在最初一秒内不透明度从 0 → 100% diff --git a/docs/architecture.md b/docs/architecture.md index ed1486af..1f83f353 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -667,7 +667,7 @@ file.exportMedia {path, preset?, settings?, format?, range?, …} export.qui | `mxf-op1a` | DNxHR (default), ProRes or H.264 (Annex B, long GOP), `mxfVideoCodec`; PCM 16/24-bit | MXF OP1a (`filmcraft-mxf` writer: frame-wrapped, index with temporal offsets, start timecode) | | `mxf-opatom` | as `mxf-op1a` | Avid-style MXF OP-Atom: clip-wrapped picture at `path`, one mono PCM file per channel (`_A1.mxf` …, `Report::extra_files`) | | `png`, `tiff`, `bmp`, `jpg` | `image` | numbered stills `000.`, `001.` … | -| `tga`, `dpx` | built in (`raster.rs`: uncompressed Targa 24/32-bit; DPX 10-bit RGB, SMPTE ST 268) | numbered stills, as above | +| `tga`, `dpx`, `exr` | built in (`raster.rs`: uncompressed Targa 24/32-bit; DPX 10-bit RGB, SMPTE ST 268; OpenEXR half-float linear RGB(A), uncompressed scan lines) | numbered stills, as above | | `gif`, `wav`, `aiff` | built in / `image` | GIF / RIFF WAVE (`WAVE_FORMAT_EXTENSIBLE` for 5.1) / AIFF (16- or 24-bit PCM) | - **Reproducible.** The same project and settings give the same file on every machine. Frames render diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 58c45cb6..b62d7853 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -55,7 +55,7 @@ only). "Tested" names the evidence. | PNG, JPEG, TIFF, BMP, GIF | read + write | yes | PNG / TIFF / BMP / JPEG sequences, GIF | 90% | | WebP | — | yes | no | ahead | | **PSD** (layers as sequence) | read (layered) | no | — | 0% | -| **OpenEXR** | read + write | no | no | 0% | +| **OpenEXR** | read + write | no | sequences (half float, linear RGB / premultiplied RGBA, uncompressed; #806) | 35% | | **DPX** | read + write | no | sequences (10-bit RGB, BT.709; #779) | 40% | | Targa | read + write | no | sequences (24-bit, 32-bit with alpha; #779) | 40% | | HEIF / HEIC | read | no | — | 0% | diff --git a/docs/gaps.md b/docs/gaps.md index ab25b8af..164ede9b 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -187,10 +187,11 @@ platforms** G4, G9, G17; **performance** G16; **ecosystem** G11; **measurement** ## G13. Delivery formats - **Missing:** AV1 export (software or hardware), MPEG-2 / DVD / Blu-ray, DCP, JPEG 2000 MXF, - AS-10 / AS-11, XDCAM HD MXF, P2, OpenEXR sequences, MP3 and AAC-only audio, + AS-10 / AS-11, XDCAM HD MXF, P2, compressed (ZIP / PIZ / DWA) OpenEXR, MP3 and AAC-only audio, ProRes 4444 / XQ in the export UI (#342; the encoder has the profiles), interlaced encoding, smart render, HEVC software encoder (HEVC exists only where a hardware encoder does). -- **Have:** PNG, TIFF, BMP, JPEG, Targa and DPX (10-bit) image sequences (#779). +- **Have:** PNG, TIFF, BMP, JPEG, Targa and DPX (10-bit) image sequences (#779); OpenEXR + sequences (half float, linear, uncompressed; #806). - **Estimate:** 60–110 h. Doc: [file-format-parity.md](file-format-parity.md). ## G14. Localization: 8 of the 12 key languages missing From f114b5f10c73795e0bd57dbc2220174d293f7542 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 10 Oct 2026 19:01:15 -0400 Subject: [PATCH 4/4] Revert #783 (Linux H.265 through NVENC): intermittent heap corruption in the CLI With NVENC registered on Linux, `filmcraft-cli --test cli` aborts in 2 of 45 runs on an RTX 3090 (`malloc(): mismatching next->prev_size`, `malloc(): unaligned fastbin chunk detected`) right after an export finishes; 0 of 30 runs fail with this reverted. Co-Authored-By: Claude Opus 5.5 --- README.md | 6 +-- crates/export/src/settings.rs | 2 +- crates/platform/README.md | 24 +++++------ crates/platform/src/lib.rs | 42 ++++++++----------- crates/platform/src/nvenc/export.rs | 5 ++- crates/platform/src/nvenc/mod.rs | 4 +- crates/platform/tests/nvenc_hevc.rs | 2 +- crates/platform/tests/nvenc_hevc_export.rs | 2 +- .../platform/tests/nvenc_hevc_hdr_export.rs | 2 +- crates/platform/tests/nvenc_hevc_main10.rs | 2 +- crates/platform/tests/nvenc_hevc_probe.rs | 2 +- crates/platform/tests/nvenc_hevc_warm.rs | 2 +- crates/platform/tests/nvenc_rgba_input.rs | 4 +- docs/codec-parity.md | 2 +- docs/target-app-parity.md | 4 +- docs/testing.md | 2 +- 16 files changed, 51 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 81968754..e74820ab 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ No FFmpeg inside. The video codecs, AAC, Opus and the containers are our own Rus | **Matroska / WebM** | ✓ | | Lacing, Cues, header stripping, HDR colour metadata | | **Stills** | ✓ | ✓ | Import PNG, JPEG, GIF, WebP, TIFF and BMP; export PNG sequences and animated GIF | -Also imported: MPEG-2 / MPEG-1 video, AC-3, MP2, MXF (OP1a / OP-Atom), MPEG transport and program streams (AVCHD, broadcast, DVD), Ogg and image sequences. Not yet: AV1 export, camera RAW, E-AC-3. Hardware decoding works on macOS (VideoToolbox), Windows (Media Foundation) and, for H.264 and HEVC, Linux (VA-API); H.264 export can use NVIDIA's encoder on Windows and Linux (opt-in), and H.265 (HEVC Main, 8-bit) export works through it (NVENC on Windows and Linux, VideoToolbox on macOS) ([#30](https://github.com/storytold/filmcraft/issues/30)). +Also imported: MPEG-2 / MPEG-1 video, AC-3, MP2, MXF (OP1a / OP-Atom), MPEG transport and program streams (AVCHD, broadcast, DVD), Ogg and image sequences. Not yet: AV1 export, camera RAW, E-AC-3. Hardware decoding works on macOS (VideoToolbox), Windows (Media Foundation) and, for H.264 and HEVC, Linux (VA-API); H.264 export can use NVIDIA's encoder on Windows and Linux (opt-in), and H.265 (HEVC Main, 8-bit) export works through it (NVENC on Windows, VideoToolbox on macOS) ([#30](https://github.com/storytold/filmcraft/issues/30)).
@@ -289,10 +289,10 @@ FilmCraft is in **alpha**. We track two numbers ([docs/target-app-parity.md](doc The biggest gaps today: -- **Speed on big footage.** Hardware decoding works on macOS and Windows, and for H.264 and HEVC on Linux (VA-API, or NVDEC with NVIDIA's driver; VP9 and AV1 decode in software there). Blend modes and the most common effects run on the GPU, but Lumetri, keys and export rendering still run on the CPU; H.264 encoding can use the hardware encoder on macOS and on Windows and Linux with an NVIDIA GPU (opt-in, Export ▸ Hardware encoding), and H.265 export is hardware-only on all three (HDR sequences export Main 10 PQ / HLG with NVENC; VideoToolbox writes 8-bit SDR) ([#30](https://github.com/storytold/filmcraft/issues/30)). +- **Speed on big footage.** Hardware decoding works on macOS and Windows, and for H.264 and HEVC on Linux (VA-API, or NVDEC with NVIDIA's driver; VP9 and AV1 decode in software there). Blend modes and the most common effects run on the GPU, but Lumetri, keys and export rendering still run on the CPU; H.264 encoding can use the hardware encoder on macOS and on Windows and Linux with an NVIDIA GPU (opt-in, Export ▸ Hardware encoding), and H.265 export is hardware-only on both (HDR sequences export Main 10 PQ / HLG with NVENC; VideoToolbox writes 8-bit SDR) ([#30](https://github.com/storytold/filmcraft/issues/30)). - **Premiere projects.** `.prproj` files can't be opened yet; bring sequences over as FCP7 XML, AAF or OTIO. - **No plugins.** No VST3 / Audio Units or OpenFX hosting. -- **Delivery codecs.** H.264 is our only software delivery-codec export; H.265 exports only through a hardware encoder (NVIDIA on Windows and Linux: Main 8-bit SDR and Main 10 HDR PQ / HLG; VideoToolbox on macOS: 8-bit Main, SDR); no AV1 export yet. +- **Delivery codecs.** H.264 is our only software delivery-codec export; H.265 exports only through a hardware encoder (NVIDIA on Windows: Main 8-bit SDR and Main 10 HDR PQ / HLG; VideoToolbox on macOS: 8-bit Main, SDR); no AV1 export yet. - **Real-world media and platforms.** Our decoders are bit-exact on conformance streams, but camera and phone files in the wild are less tested. Windows and Linux get far less testing than macOS. - **AI features.** Few so far; speech to text is optional and off by default. diff --git a/crates/export/src/settings.rs b/crates/export/src/settings.rs index 162c958a..ed0a928b 100644 --- a/crates/export/src/settings.rs +++ b/crates/export/src/settings.rs @@ -62,7 +62,7 @@ impl H264Profile { } /// Export ▸ Hardware encoding: whether H.264 may be encoded by the system's hardware video encoder -/// (VideoToolbox on macOS, NVENC on NVIDIA GPUs on Windows and Linux) instead of FilmCraft's own encoder. Off +/// (VideoToolbox on macOS, NVENC on NVIDIA GPUs on Windows) instead of FilmCraft's own encoder. Off /// by default: hardware encoders make different streams, and exports are otherwise byte-identical /// from run to run and machine to machine. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/platform/README.md b/crates/platform/README.md index cefdfa0d..b78c5773 100644 --- a/crates/platform/README.md +++ b/crates/platform/README.md @@ -61,7 +61,7 @@ let availability = filmcraft_platform::register(); // Available("VideoToolbox") - After a `flush` (which drains the MFT) the MFT only restarts at an IDR picture; the GOP cache always seeks after a flush, and a caller that continues from the middle of a GOP gets an error that `HybridDecoder` answers by replaying the run in software. -- **Windows and 64-bit Linux: NVIDIA NVENC H.264 and H.265 (HEVC) encoding** (`nvenc/`), 8-bit SDR 4:2:0 for Export. The driver's +- **Windows and 64-bit Linux: NVIDIA NVENC H.264 encoding, and H.265 (HEVC) on Windows** (`nvenc/`), 8-bit SDR 4:2:0 for Export. The driver's `nvEncodeAPI64.dll` (Windows) or `libnvidia-encode.so.1` (Linux), API 12.1, is loaded at run time, so machines without NVIDIA still start. Windows uses a Direct3D 11 device; Linux retains a primary CUDA context through the runtime-loaded `libcuda.so.1`, released after the encoder is destroyed. @@ -76,8 +76,8 @@ let availability = filmcraft_platform::register(); // Available("VideoToolbox") It declines two-pass VBR, HDR, MXF, interlaced output, sizes outside NVENC's limits and systems without an NVIDIA GPU or driver, and the software encoder runs instead. A failure during an export ends it with an error, since a hardware stream cannot be finished in software. The same session, - ring and input path encode H.265, see [Hardware H.265 (HEVC) encoding - (NVENC)](#hardware-h265-hevc-encoding-nvenc). + ring and input path encode H.265, see [Hardware H.265 (HEVC) encoding (Windows, + NVENC)](#hardware-h265-hevc-encoding-windows-nvenc). - **Linux: NVDEC, H.264 (`avcC`) and HEVC (`hvcC`)** on NVIDIA's proprietary driver (`nvdec/`): H.264 8-bit and HEVC 8- / 10-bit, 4:2:0 progressive. `libnvcuvid.so.1` and `libcuda.so.1` are loaded at run time; without them (or without a device) nothing is registered. NVIDIA's own parser takes the @@ -188,7 +188,7 @@ The sample entry's `hvcC` record is the one VideoToolbox wrote for the stream (r format description's sample description extension atoms, with the VPS / SPS / PPS), so profile, level and flags are the encoder's own. If it is missing the export stops with that reason. -## Hardware H.265 (HEVC) encoding (NVENC, Windows and Linux) +## Hardware H.265 (HEVC) encoding (Windows, NVENC) One codec enum (`nvenc::Codec`, chosen by `Profile::HevcMain` the way `VtProfile::HevcMain` does it for VideoToolbox) parameterises the NVENC session: the codec and profile GUIDs, the capability @@ -295,13 +295,13 @@ level 4 High tier, which many hardware decoders refuse; it is now level 4.1 Main | `tests/hardware_encode.rs` (macOS) | what the hardware path takes and declines; round trip through our software decoder (every picture, in order, luma PSNR above 30 dB, keyframes no further apart than asked, no composition offsets); an export through `filmcraft_export` that decodes in our decoder and in ffmpeg / ffprobe (profile, size, frame count, BT.709); the built-in encoder still exporting everything hardware declines; exact output size at sizes that are not multiples of 16; hostile configurations (zero, huge, odd sizes, frame rates, bitrates, keyframe intervals, wrong planes) give errors and never panic; encoders dropped at any point do not crash or hang. The same for **HEVC**: Main profile, 8-bit 4:2:0, `hvc1` entry with VPS / SPS / PPS and 4-byte lengths, MP4 and QuickTime, AAC audio, two-pass refused, the format list agreeing with the probe, ffprobe reading `codec_name=hevc`, `profile=Main`, `codec_tag_string=hvc1`, `pix_fmt=yuv420p`, BT.709 | | `tests/nvenc.rs` (Windows / Linux, NVIDIA) | H.264 from NVENC (1280×720, 6 Mbps, 72 frames) decodes with our decoder at worst 46.9 dB luma PSNR; IDR at 0, 24 and 48; dts / pts right | | `tests/nvenc_export.rs` (Windows / Linux; fallback also tested without NVIDIA) | Export with hardware encoding against the software encoder through the export pipeline: the two decoded files at worst 54.8 dB luma PSNR; ffmpeg decodes the file without errors; declined cases go to the software encoder; the counters | -| `tests/nvenc_hevc.rs` (Windows / Linux, NVIDIA with HEVC) | HEVC from NVENC (1280×720, 6 Mbps, 72 frames, keyframe every 24) decodes with our HEVC decoder at worst 48.9 dB luma / 51.0 dB chroma PSNR; IDR at 0, 24, 48; dts strictly increasing, pts a permutation, `sps_max_num_reorder_pics` within the dts shift; no parameter sets in the samples; the `hvcC`'s profile / tier / level bytes are the SPS's own; VUI BT.709 limited and timing `(1, 24)`; hostile configurations, sizes, planes and encoders dropped mid-stream give errors, never panics; keyframes every 1–2 pictures without B-frames | -| `tests/nvenc_hevc_export.rs` (Windows / Linux, NVIDIA with HEVC) | H.265 export through the real pipeline, MP4 and QuickTime, toggle Auto and Off: counters (72 frames, 1 session, 0 declined), our decoder against the software H.264 export at worst 54.8 dB luma PSNR, ffprobe `codec_name=hevc`, `profile=Main`, `codec_tag_string=hvc1`, `pix_fmt=yuv420p`, BT.709 limited, 72 frames, 24/1, keyframes at 0 and 48, start 0, `ffmpeg -xerror` clean; with AAC audio; 1920×1080 and 642×362 cropped back from the coded size; every decline (HDR, analysis pass, two-pass, interlaced, non-square pixels, MXF, sizes over 65535, odd sizes, sizes outside the GPU's limits) an error naming NVENC, counted once; H.264 with hardware encoding Off never touches NVENC | -| `tests/nvenc_rgba_input.rs` (Windows / Linux, NVIDIA) | RGBA input, HEVC Main and H.264 High, 1280×720: solid red, green, blue, white, black and grey decode (our decoder) to the exact BT.709 limited codes of `rgba_to_yuv420_8`, within 2; a moving picture at worst 47.7 dB luma / 50.7 dB chroma PSNR against that conversion; Main 10 has no RGBA input; short pictures and planar / RGBA pictures for the other kind of encoder are errors | -| `tests/nvenc_hevc_probe.rs` (Windows / Linux) | the HEVC probe in a fresh process: its cost, the cached answer, `available(Hevc)` following it after `register()` (twice) | -| `tests/nvenc_hevc_warm.rs` (Windows / Linux) | `register()` answers the HEVC and the Main 10 questions on a thread of its own, before anyone asks; the answers are kept and `hdr_available(Hevc)` follows the Main 10 probe | -| `tests/nvenc_hevc_main10.rs` (Windows / Linux, NVIDIA with 10-bit HEVC) | HEVC Main 10 in process, PQ and HLG: a float picture (grey ramp to peak white, a moving box, saturated BT.2020 bars) is converted with `rgbf_to_yuv420_10`, encoded and decoded with our HEVC decoder (`Yuv16`, 10 bits, codes below 1024): worst 71.1 dB luma / 70.6 dB chroma PSNR on the 10-bit scale, 803 distinct luma levels on a ramp row, peak code 940 survives; IDR at 0 and 24; the VUI is BT.2020 / 16 or 18 / BT.2020 NCL / limited; the IDR samples carry two prefix SEI NAL units whose messages are byte-for-byte the expected 137 / 144 payloads (HLG: none; other pictures: none); Main and Main 10 refuse each other's pictures; hostile sizes, bitrates, signals, huge or empty SEI payloads, encoders dropped with SEI in flight | -| `tests/nvenc_hevc_hdr_export.rs` (Windows / Linux, NVIDIA with 10-bit HEVC) | HDR sequences through the real export job (PQ MP4, HLG MP4, PQ QuickTime): 24 frames, 1 session, 0 declined; ffprobe `hevc` / `Main 10` / `hvc1` / `yuv420p10le`, `bt2020` / `smpte2084` or `arib-std-b67` / `bt2020nc` / `tv`; PQ first-frame side data Mastering display (max 10000000/10000, min 1/10000) and Content light level, HLG none; `ffmpeg -xerror` clean; ffmpeg's yuv420p10le picture equals our decoder's, code for code; our importer reads PQ / HLG, BT.2020, 10-bit, and the `colr` / `mdcv` / `clli` boxes; luma PSNR against a ProRes HDR export of the same sequence 62.6 dB (PQ) / 61.8 dB (HLG); `settings.sdr` and SDR sequences stay Main 8-bit BT.709 with no HDR boxes; the factory rejects mismatched or wrong-length pictures and clamps NaN / infinity | +| `tests/nvenc_hevc.rs` (Windows, NVIDIA with HEVC) | HEVC from NVENC (1280×720, 6 Mbps, 72 frames, keyframe every 24) decodes with our HEVC decoder at worst 48.9 dB luma / 51.0 dB chroma PSNR; IDR at 0, 24, 48; dts strictly increasing, pts a permutation, `sps_max_num_reorder_pics` within the dts shift; no parameter sets in the samples; the `hvcC`'s profile / tier / level bytes are the SPS's own; VUI BT.709 limited and timing `(1, 24)`; hostile configurations, sizes, planes and encoders dropped mid-stream give errors, never panics; keyframes every 1–2 pictures without B-frames | +| `tests/nvenc_hevc_export.rs` (Windows, NVIDIA with HEVC) | H.265 export through the real pipeline, MP4 and QuickTime, toggle Auto and Off: counters (72 frames, 1 session, 0 declined), our decoder against the software H.264 export at worst 54.8 dB luma PSNR, ffprobe `codec_name=hevc`, `profile=Main`, `codec_tag_string=hvc1`, `pix_fmt=yuv420p`, BT.709 limited, 72 frames, 24/1, keyframes at 0 and 48, start 0, `ffmpeg -xerror` clean; with AAC audio; 1920×1080 and 642×362 cropped back from the coded size; every decline (HDR, analysis pass, two-pass, interlaced, non-square pixels, MXF, sizes over 65535, odd sizes, sizes outside the GPU's limits) an error naming NVENC, counted once; H.264 with hardware encoding Off never touches NVENC | +| `tests/nvenc_rgba_input.rs` (Windows, NVIDIA) | RGBA input, HEVC Main and H.264 High, 1280×720: solid red, green, blue, white, black and grey decode (our decoder) to the exact BT.709 limited codes of `rgba_to_yuv420_8`, within 2; a moving picture at worst 47.7 dB luma / 50.7 dB chroma PSNR against that conversion; Main 10 has no RGBA input; short pictures and planar / RGBA pictures for the other kind of encoder are errors | +| `tests/nvenc_hevc_probe.rs` (Windows) | the HEVC probe in a fresh process: its cost, the cached answer, `available(Hevc)` following it after `register()` (twice) | +| `tests/nvenc_hevc_warm.rs` (Windows) | `register()` answers the HEVC and the Main 10 questions on a thread of its own, before anyone asks; the answers are kept and `hdr_available(Hevc)` follows the Main 10 probe | +| `tests/nvenc_hevc_main10.rs` (Windows, NVIDIA with 10-bit HEVC) | HEVC Main 10 in process, PQ and HLG: a float picture (grey ramp to peak white, a moving box, saturated BT.2020 bars) is converted with `rgbf_to_yuv420_10`, encoded and decoded with our HEVC decoder (`Yuv16`, 10 bits, codes below 1024): worst 71.1 dB luma / 70.6 dB chroma PSNR on the 10-bit scale, 803 distinct luma levels on a ramp row, peak code 940 survives; IDR at 0 and 24; the VUI is BT.2020 / 16 or 18 / BT.2020 NCL / limited; the IDR samples carry two prefix SEI NAL units whose messages are byte-for-byte the expected 137 / 144 payloads (HLG: none; other pictures: none); Main and Main 10 refuse each other's pictures; hostile sizes, bitrates, signals, huge or empty SEI payloads, encoders dropped with SEI in flight | +| `tests/nvenc_hevc_hdr_export.rs` (Windows, NVIDIA with 10-bit HEVC) | HDR sequences through the real export job (PQ MP4, HLG MP4, PQ QuickTime): 24 frames, 1 session, 0 declined; ffprobe `hevc` / `Main 10` / `hvc1` / `yuv420p10le`, `bt2020` / `smpte2084` or `arib-std-b67` / `bt2020nc` / `tv`; PQ first-frame side data Mastering display (max 10000000/10000, min 1/10000) and Content light level, HLG none; `ffmpeg -xerror` clean; ffmpeg's yuv420p10le picture equals our decoder's, code for code; our importer reads PQ / HLG, BT.2020, 10-bit, and the `colr` / `mdcv` / `clli` boxes; luma PSNR against a ProRes HDR export of the same sequence 62.6 dB (PQ) / 61.8 dB (HLG); `settings.sdr` and SDR sequences stay Main 8-bit BT.709 with no HDR boxes; the factory rejects mismatched or wrong-length pictures and clamps NaN / infinity | | `src/nvenc/abi_tests.rs` (Windows / Linux) | FFI structs' sizes, alignments, field offsets, constants, GUIDs and the bit-field masks of the `flags` words (H.264, HEVC, and the HEVC picture parameters with the SEI payload array) against a C compiler's view of NVIDIA's `nvEncodeAPI.h` (12.1) | | `src/nvenc/{mod,hevc,export}.rs` unit tests (Windows) | HEVC NAL types, stripping and parameter-set splitting (including `IDR_N_LP`, whose header byte reads as a PPS in H.264, and one-byte or truncated NAL units); the `hvcC` built from real VPS / SPS / PPS (Main and Main 10: accepted as what was asked for, refused as the other, other depths refused), every truncation and bit flip of the SPS, wrong profile, size and NAL types; HEVC levels; P010 layout (shift, little-endian, interleaving, pitch in bytes, narrow pitches and short buffers); the 10-bit capability decline; the SEI messages | | `crates/export/src/hdr_tests.rs` | `rgbf_to_yuv420_10`: exact codes for black, white, grey and the BT.2020 primaries and secondaries, NaN / infinity / out-of-range, odd sizes, wrong lengths and overflowing sizes; the HDR probe registry; which exports are HDR (HEVC SDR without a probe, HDR with one, `settings.sdr` forcing SDR); the SEI of the software H.264 encoder equals `ColorSignal::static_metadata` | @@ -344,7 +344,7 @@ Zero-copy upload of decoded pictures into wgpu textures (`CVPixelBuffer`s on mac textures on Windows); B-frames in the VideoToolbox encoders and 10-bit / HDR HEVC (Main 10) in VideoToolbox; VP9 and AV1 through VA-API (Linux), and VA-API encoding; field-coded H.264; 4:4:4 and 4:2:2 encoding with NVENC, HDR H.264 with NVENC (HDR H.264 stays with the software encoder), HDR10 MaxCLL / MaxFALL measured from -the pictures (they are written as 0, unknown), AV1 with NVENC, H.265 with the other Windows and Linux vendors, and encoders +the pictures (they are written as 0, unknown), AV1 with NVENC, H.265 with the other Windows vendors and on Linux, and encoders from other vendors on Windows (through Media Foundation); VP9 / AV1 4:4:4 and 12-bit on Windows. ### Linux NVENC sources and limits diff --git a/crates/platform/src/lib.rs b/crates/platform/src/lib.rs index e42db755..8744da0e 100644 --- a/crates/platform/src/lib.rs +++ b/crates/platform/src/lib.rs @@ -8,7 +8,7 @@ //! Linux with neither) registration does nothing and reports [`Availability::Unavailable`]. It also registers a //! hardware H.264 encoder factory (`filmcraft_export::register_encoder`) that only acts when an //! export asks for it (`ExportSettings::hardware_encoding` = `Auto`), see [`hardware_encode`]; on -//! macOS and with NVENC ([`nvenc`], Windows and Linux) it also makes the H.265 export format available, and with NVENC +//! macOS and on Windows (NVENC, [`nvenc`]) it also makes the H.265 export format available, and on Windows //! (Main 10) lets HDR sequences export as HDR H.265. //! //! Hardware decoding never makes a file undecodable: @@ -68,6 +68,8 @@ pub enum Availability { /// harmless). Streams they do not take, and every stream while hardware decoding is Off, keep /// using FilmCraft's own decoders. pub fn register() -> Availability { + #[cfg(all(target_os = "linux", target_pointer_width = "64"))] + filmcraft_export::register_encoder(nvenc::export::factory); #[cfg(target_os = "macos")] { filmcraft_codecs::register_video_decoder(videotoolbox_factory); @@ -79,9 +81,23 @@ pub fn register() -> Availability { filmcraft_codecs::hw::set_hw_backend("VideoToolbox"); Availability::Available("VideoToolbox") } + // Export ▸ H.265: NVENC is FilmCraft's only HEVC encoder on Windows, so the format is available + // when a small HEVC session opens. That takes about half a second, so it is asked on a thread of + // its own now rather than by the first draw of the format list. #[cfg(target_os = "windows")] { - register_nvenc(); + filmcraft_export::register_format_probe(filmcraft_export::Format::Hevc, nvenc::hevc_available); + // HDR sequences export as HEVC Main 10 (PQ / HLG) only where NVENC has a 10-bit encoder; elsewhere + // (and on macOS, whose VideoToolbox path is 8-bit) they are tone-mapped to SDR, as before + filmcraft_export::register_hdr_probe(filmcraft_export::Format::Hevc, nvenc::hevc_hdr_available); + nvenc::warm_hevc_probe(); + } + #[cfg(target_os = "windows")] + { + static ENCODERS: std::sync::Once = std::sync::Once::new(); + // Export ▸ Hardware encoding (NVENC H.264): in front of the software encoder, taking an + // export only when asked for and when NVENC can do it + ENCODERS.call_once(|| filmcraft_export::register_encoder(nvenc::export::factory)); filmcraft_codecs::register_video_decoder(media_foundation_factory); filmcraft_codecs::hw::set_hw_backend("Media Foundation"); Availability::Available("Media Foundation") @@ -114,11 +130,6 @@ pub fn register() -> Availability { } Err(why) => log::info!("no NVDEC hardware decoding: {why}"), } - // After the VA-API probe, never beside it: libva-nvidia-driver calls `cuInit` from its - // library constructor (under the loader lock) while a first `cuInit` on another thread - // loads libraries under CUDA's own lock, and the two wait for each other forever. - #[cfg(target_pointer_width = "64")] - register_nvenc(); available } #[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] @@ -127,23 +138,6 @@ pub fn register() -> Availability { } } -/// NVENC: the Export encoder factory, and what makes the H.265 format (and HDR H.265) available. -#[cfg(any(target_os = "windows", all(target_os = "linux", target_pointer_width = "64")))] -fn register_nvenc() { - // Export ▸ H.265: NVENC is FilmCraft's only HEVC encoder on Windows and Linux, so the format is available - // when a small HEVC session opens. That takes about half a second, so it is asked on a thread of - // its own now rather than by the first draw of the format list. - filmcraft_export::register_format_probe(filmcraft_export::Format::Hevc, nvenc::hevc_available); - // HDR sequences export as HEVC Main 10 (PQ / HLG) only where NVENC has a 10-bit encoder; elsewhere - // (and on macOS, whose VideoToolbox path is 8-bit) they are tone-mapped to SDR, as before - filmcraft_export::register_hdr_probe(filmcraft_export::Format::Hevc, nvenc::hevc_hdr_available); - nvenc::warm_hevc_probe(); - static ENCODERS: std::sync::Once = std::sync::Once::new(); - // Export ▸ Hardware encoding (NVENC H.264): in front of the software encoder, taking an - // export only when asked for and when NVENC can do it - ENCODERS.call_once(|| filmcraft_export::register_encoder(nvenc::export::factory)); -} - /// Whether [`register`] has put a hardware decoder factory in front of our decoders. pub fn registered() -> bool { #[cfg(target_os = "macos")] diff --git a/crates/platform/src/nvenc/export.rs b/crates/platform/src/nvenc/export.rs index a796742f..4344cb52 100644 --- a/crates/platform/src/nvenc/export.rs +++ b/crates/platform/src/nvenc/export.rs @@ -211,7 +211,8 @@ fn config(format: Format, w: u32, h: u32, rate: FrameRate, s: &ExportSettings) - /// The Export encoder factory (see the module documentation). pub fn factory(format: Format, w: u32, h: u32, rate: FrameRate, s: &ExportSettings) -> Option>> { if format == Format::Hevc { - return hevc_factory(w, h, rate, s); + // H.265 through NVENC is Windows-only for now; on Linux this backend takes H.264 only + return if cfg!(target_os = "windows") { hevc_factory(w, h, rate, s) } else { None }; } factory_with(format, w, h, rate, s, |cfg| open(cfg, &Signal::default())) } @@ -260,7 +261,7 @@ fn factory_with( } } -/// The H.265 side of [`factory`]: FilmCraft's only HEVC encoder on Windows and Linux, so the Hardware encoding toggle +/// The H.265 side of [`factory`]: FilmCraft's only HEVC encoder on Windows, so the Hardware encoding toggle /// does not apply and a request NVENC cannot take is an error, not a fall-through. fn hevc_factory(w: u32, h: u32, rate: FrameRate, s: &ExportSettings) -> Option>> { let declined = |why: &str| { diff --git a/crates/platform/src/nvenc/mod.rs b/crates/platform/src/nvenc/mod.rs index d15928e1..4fe2357f 100644 --- a/crates/platform/src/nvenc/mod.rs +++ b/crates/platform/src/nvenc/mod.rs @@ -1,5 +1,5 @@ -//! NVIDIA NVENC hardware encoding: H.264 and H.265 (HEVC) Main / Main 10 (Windows and -//! 64-bit Linux). +//! NVIDIA NVENC hardware encoding: H.264 (Windows and 64-bit Linux) and H.265 (HEVC) Main / Main 10 +//! (Windows). //! //! The encoder runs on the GPU's NVENC engine through the driver's `nvEncodeAPI64.dll` on Windows //! (on a Direct3D 11 device) or `libnvidia-encode.so.1` on Linux (on the CUDA driver's primary diff --git a/crates/platform/tests/nvenc_hevc.rs b/crates/platform/tests/nvenc_hevc.rs index d5f1ca9d..acd2bee9 100644 --- a/crates/platform/tests/nvenc_hevc.rs +++ b/crates/platform/tests/nvenc_hevc.rs @@ -2,7 +2,7 @@ //! with our own HEVC decoder to pictures close to the source, keyframes and timestamps are right, the //! samples carry no parameter sets and the `hvcC` describes the SPS it was built from. Skips //! without an NVIDIA GPU / driver with HEVC encoding. -#![cfg(any(target_os = "windows", all(target_os = "linux", target_pointer_width = "64")))] +#![cfg(target_os = "windows")] use filmcraft_bitstream::unescape_rbsp; use filmcraft_hevc::params::Sps; diff --git a/crates/platform/tests/nvenc_hevc_export.rs b/crates/platform/tests/nvenc_hevc_export.rs index 1b449188..95f074bb 100644 --- a/crates/platform/tests/nvenc_hevc_export.rs +++ b/crates/platform/tests/nvenc_hevc_export.rs @@ -4,7 +4,7 @@ //! counters; and what NVENC declines (an error that says why, counted, never a crash). Its own test //! binary: the encoder registry and the counters are process-wide. Skips without an NVIDIA GPU with //! an HEVC encoder. -#![cfg(any(target_os = "windows", all(target_os = "linux", target_pointer_width = "64")))] +#![cfg(target_os = "windows")] use std::path::Path; use std::process::Command; diff --git a/crates/platform/tests/nvenc_hevc_hdr_export.rs b/crates/platform/tests/nvenc_hevc_hdr_export.rs index 157e536e..8acbc1ca 100644 --- a/crates/platform/tests/nvenc_hevc_hdr_export.rs +++ b/crates/platform/tests/nvenc_hevc_hdr_export.rs @@ -5,7 +5,7 @@ //! our importer reads back as PQ / HLG, 10-bit; whose pixels follow a ProRes HDR export of the same //! sequence. SDR stays Main 8-bit BT.709, `settings.sdr` forces it. Its own test binary: the encoder //! registry and the counters are process-wide. Skips (SKIPPED) only without a 10-bit HEVC encoder. -#![cfg(any(target_os = "windows", all(target_os = "linux", target_pointer_width = "64")))] +#![cfg(target_os = "windows")] use std::path::Path; use std::process::Command; diff --git a/crates/platform/tests/nvenc_hevc_main10.rs b/crates/platform/tests/nvenc_hevc_main10.rs index 74a8e280..76d0c9cc 100644 --- a/crates/platform/tests/nvenc_hevc_main10.rs +++ b/crates/platform/tests/nvenc_hevc_main10.rs @@ -3,7 +3,7 @@ //! pictures that come out of our own HEVC decoder are 10-bit and close to them (smooth ramps keep far more //! than 256 levels, bright code values survive), the VUI and the HDR10 SEI messages are right, and //! hostile inputs are errors. Skips (with SKIPPED) only on a machine without a 10-bit HEVC encoder. -#![cfg(any(target_os = "windows", all(target_os = "linux", target_pointer_width = "64")))] +#![cfg(target_os = "windows")] use filmcraft_bitstream::unescape_rbsp; use filmcraft_export::{ColorSignal, rgbf_to_yuv420_10}; diff --git a/crates/platform/tests/nvenc_hevc_probe.rs b/crates/platform/tests/nvenc_hevc_probe.rs index 28f28f34..e2a41cd7 100644 --- a/crates/platform/tests/nvenc_hevc_probe.rs +++ b/crates/platform/tests/nvenc_hevc_probe.rs @@ -1,7 +1,7 @@ //! The HEVC availability probe of the NVENC backend: it opens one small encoder session, once. Its own //! test binary so that "first call" means a fresh process (driver load, Direct3D 11 device, NVENC //! session and encoder initialisation all included). -#![cfg(any(target_os = "windows", all(target_os = "linux", target_pointer_width = "64")))] +#![cfg(target_os = "windows")] use std::time::{Duration, Instant}; diff --git a/crates/platform/tests/nvenc_hevc_warm.rs b/crates/platform/tests/nvenc_hevc_warm.rs index 1a616328..adc0ebb2 100644 --- a/crates/platform/tests/nvenc_hevc_warm.rs +++ b/crates/platform/tests/nvenc_hevc_warm.rs @@ -1,7 +1,7 @@ //! `register` asks the NVENC HEVC probe on a thread of its own, so the first draw of Export's format //! list does not open an encoder session on the UI thread. Its own test binary: anything else asking //! `hevc_available` first would make the wait meaningless. -#![cfg(any(target_os = "windows", all(target_os = "linux", target_pointer_width = "64")))] +#![cfg(target_os = "windows")] use std::time::{Duration, Instant}; diff --git a/crates/platform/tests/nvenc_rgba_input.rs b/crates/platform/tests/nvenc_rgba_input.rs index 62b8b9bd..4009f3e6 100644 --- a/crates/platform/tests/nvenc_rgba_input.rs +++ b/crates/platform/tests/nvenc_rgba_input.rs @@ -1,7 +1,7 @@ -//! NVENC with packed RGBA input (Windows and Linux, NVIDIA GPU): the GPU's RGB → 4:2:0 conversion matches the +//! NVENC with packed RGBA input (Windows, NVIDIA GPU): the GPU's RGB → 4:2:0 conversion matches the //! export's own BT.709 limited-range conversion (`filmcraft_export::rgba_to_yuv420_8`), for HEVC Main //! and H.264. Skips without an NVIDIA GPU / driver. -#![cfg(any(target_os = "windows", all(target_os = "linux", target_pointer_width = "64")))] +#![cfg(target_os = "windows")] use filmcraft_isobmff::{AvcConfig, SampleEntry}; use filmcraft_platform::nvenc::{Codec, Config, Nvenc, Packet, Profile, Signal, available, hevc_available}; diff --git a/docs/codec-parity.md b/docs/codec-parity.md index 8fe1ff6b..8aaa7655 100644 --- a/docs/codec-parity.md +++ b/docs/codec-parity.md @@ -16,7 +16,7 @@ contents). Percent is ready-for-real-work for that codec: depth, profiles, speed | Codec | Premiere | Our decode | Our encode | Fidelity and tests | % | Est. | |---|---|---|---|---|---|---| | H.264 / AVC | decode all common profiles incl. High 10, 4:2:2 (XAVC S-I, AVC-Intra); encode 8/10-bit | own decoder, **8-bit 4:2:0 only** (Baseline / Main / High, CABAC / CAVLC); no High 10 / 4:2:2 / 4:4:4, interlaced field / MBAFF, lossless | own encoder (High / Main / Baseline, B-frames, CBR / VBR / 2-pass); 8-bit 4:2:0 only; hardware VideoToolbox / NVENC | decode bit-exact on 37+ conformance streams, 500–600 fps 1080p; encode ffprobe-checked, byte-identical across machines | 65% | 20–35 h | -| HEVC / H.265 | decode Main, Main 10, 4:2:2 10 (XAVC HS); encode 8/10-bit, HDR | own decoder Main / Main 10 / Main Still Picture, bit-exact on 41 fixtures, ~225 fps 1080p; **no 4:2:2 / RExt** | **hardware only** (VideoToolbox 8-bit; NVENC 8-bit and Main 10 HDR on Windows and Linux); no software encoder | NVENC round trip 48.9 dB (8-bit), 71.1 dB (10-bit); ffprobe | 60% | 30–60 h | +| HEVC / H.265 | decode Main, Main 10, 4:2:2 10 (XAVC HS); encode 8/10-bit, HDR | own decoder Main / Main 10 / Main Still Picture, bit-exact on 41 fixtures, ~225 fps 1080p; **no 4:2:2 / RExt** | **hardware only** (VideoToolbox 8-bit; NVENC 8-bit and Main 10 HDR on Windows); no software encoder | NVENC round trip 48.9 dB (8-bit), 71.1 dB (10-bit); ffprobe | 60% | 30–60 h | | ProRes 422 family / 4444 / XQ | decode + encode all | own decode + encode, all six profiles, alpha | yes (export UI exposes 422 flavours; 4444 / XQ request #342) | ±1 LSB, 318 fps | 85% | 3–6 h | | ProRes RAW | decode | none (#345) | — | — | 0% | 30–50 h (spec availability uncertain) | | DNxHD / DNxHR | decode + encode, MXF / MOV | own decode (all SMPTE ST 2019-1 CIDs, 8/10/12-bit, 4:2:2 / 4:4:4, interlaced, alpha) and encode LB / SQ / HQ / HQX / 444 | yes, MOV + MXF | within IDCT precision of ffmpeg on 22 fixtures | 90% | 2–4 h | diff --git a/docs/target-app-parity.md b/docs/target-app-parity.md index e74879f6..13bb67a4 100644 --- a/docs/target-app-parity.md +++ b/docs/target-app-parity.md @@ -139,8 +139,8 @@ Evidence for the cross-cutting rows: - **Hardware ~45%.** Hardware decode exists on all three desktop OSs (VideoToolbox; Media Foundation / D3D11 incl. VP9 and AV1; VA-API and NVDEC for H.264 / HEVC on Linux). Hardware - encode: VideoToolbox H.264 / HEVC 8-bit; NVENC H.264 and HEVC incl. Main 10 HDR (Windows, - Linux). Missing against Premiere: Intel Quick Sync and AMD encoders, 10-bit HEVC on macOS, + encode: VideoToolbox H.264 / HEVC 8-bit; NVENC H.264 (Windows, Linux) and HEVC incl. Main 10 HDR + (Windows). Missing against Premiere: Intel Quick Sync and AMD encoders, 10-bit HEVC on macOS, ProRes hardware, zero-copy upload, ~48 of Premiere's accelerated effects, external video I/O, control surfaces. Detail: [hardware-parity.md](hardware-parity.md). - **Performance ~55%.** 4K H.264 / HEVC play with no dropped frames with hardware decoding (M4 diff --git a/docs/testing.md b/docs/testing.md index a35f48b4..856dad73 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -77,7 +77,7 @@ are never linked or shipped ([AGENTS.md](../AGENTS.md) §2). The NVENC tests (`crates/platform/tests/nvenc.rs` and `nvenc_export.rs` for H.264, `nvenc_rgba_input.rs` for the GPU's RGB → 4:2:0 conversion, `nvenc_hevc.rs`, `nvenc_hevc_export.rs`, `nvenc_hevc_probe.rs` and `nvenc_hevc_warm.rs` for H.265, `nvenc_hevc_main10.rs` and -`nvenc_hevc_hdr_export.rs` for Main 10 HDR; Windows and Linux) skip, printing `SKIPPED`, without an NVIDIA GPU +`nvenc_hevc_hdr_export.rs` for Main 10 HDR; Windows only) skip, printing `SKIPPED`, without an NVIDIA GPU with NVENC (H.265: when the HEVC probe says there is no HEVC encoder; Main 10: when the Main 10 probe says there is no 10-bit encoder; on a machine that has one, a configuration NVENC refuses fails the test). The FFI layout tests in `crates/platform/src/nvenc/abi_tests.rs` were generated from a C program built with MSVC