diff --git a/crates/engine/src/commands.rs b/crates/engine/src/commands.rs index eb966d67..5f46ecc6 100644 --- a/crates/engine/src/commands.rs +++ b/crates/engine/src/commands.rs @@ -1545,7 +1545,7 @@ fn build() -> Vec { "Media…", ["File", "Export"], None, - r#"{"path":str,"preset":str?,"settings":ExportSettings?,"format":"h264|hevc|prores|dnxhr|apv|mjpeg|mxf-op1a|mxf-opatom|png|tiff|bmp|jpg|tga|dpx|exr|gif|wav|aiff|aac|flac"?,"audioCodec":"auto|aac|pcm|flac"?,"flacLevel":0..8=5,"width":u32?,"height":u32?,"fps":f64?,"bitrateKbps":u32?,"bitrateMode":"cbr|vbr1Pass|vbr2Pass|crf"?,"crf":0..51=23,"hardwareEncoding":"off|auto"?,"scale":f32=1,"audio":bool=true,"quality":0..100,"burnCaptions":bool=false,"alpha":bool=false,"captionSidecar":"srt|vtt"?,"loudnessLufs":f64?,"proresProfile":"proxy|lt|standard|hq|4444|4444xq"?,"dnxProfile":"lb|sq|hq|hqx"?,"apvProfile":"422-10|422-12|444-10|444-12"?,"mxfVideoCodec":"dnxhr|proRes|h264"?,"sequence":id?,"range":"entire|inOut|workArea|custom"?,"startSeconds":f64?,"endSeconds":f64?,"wait":bool=false}"#, + r#"{"path":str,"preset":str?,"settings":ExportSettings?,"format":"h264|hevc|prores|dnxhr|apv|mjpeg|mxf-op1a|mxf-opatom|png|tiff|bmp|jpg|tga|dpx|exr|gif|wav|aiff|aac|flac"?,"audioCodec":"auto|aac|pcm|flac"?,"flacLevel":0..8=5,"width":u32?,"height":u32?,"fps":f64?,"bitrateKbps":u32?,"bitrateMode":"cbr|vbr1Pass|vbr2Pass|crf"?,"crf":0..51=23,"hardwareEncoding":"off|auto"?,"scale":f32=1,"audio":bool=true,"quality":0..100,"burnCaptions":bool=false,"alpha":bool=false,"exrCompression":"none|rle|zips|zip"?,"captionSidecar":"srt|vtt"?,"loudnessLufs":f64?,"proresProfile":"proxy|lt|standard|hq|4444|4444xq"?,"dnxProfile":"lb|sq|hq|hqx"?,"apvProfile":"422-10|422-12|444-10|444-12"?,"mxfVideoCodec":"dnxhr|proRes|h264"?,"sequence":id?,"range":"entire|inOut|workArea|custom"?,"startSeconds":f64?,"endSeconds":f64?,"wait":bool=false}"#, has_seq, crate::export_tools::export_media ), diff --git a/crates/engine/src/export_tests.rs b/crates/engine/src/export_tests.rs index 3e74d375..d5f1883e 100644 --- a/crates/engine/src/export_tests.rs +++ b/crates/engine/src/export_tests.rs @@ -551,6 +551,21 @@ fn crf_param_picks_constant_quality() { assert!(e.contains("crf"), "{e}"); } +#[test] +fn exr_compression_param_is_parsed_and_defaults_to_none() { + use filmcraft_export::ExrCompression::{None as Plain, Rle, Zip, Zips}; + let s = demo(); + let setting = |p: Value| crate::export_tools::settings_from_params(&s, &p, "file.exportMedia").map(|(_, st)| st.exr_compression); + assert_eq!(setting(json!({"path": "x.exr", "format": "exr"})).unwrap(), Plain); + for (name, want) in [("none", Plain), ("rle", Rle), ("zips", Zips), ("zip", Zip)] { + assert_eq!(setting(json!({"path": "x.exr", "format": "exr", "exrCompression": name})).unwrap(), want); + } + let e = setting(json!({"path": "x.exr", "format": "exr", "exrCompression": "piz"})).unwrap_err().to_string(); + assert!(e.contains("none | rle | zips | zip"), "{e}"); + let old: filmcraft_export::ExportSettings = serde_json::from_value(json!({"format": "exr"})).unwrap(); + assert_eq!(old.exr_compression, Plain); +} + #[test] fn queue_exports_several_sequences_and_ranges() { let mut s = demo(); diff --git a/crates/engine/src/export_tools.rs b/crates/engine/src/export_tools.rs index 044c6329..1d316aa5 100644 --- a/crates/engine/src/export_tools.rs +++ b/crates/engine/src/export_tools.rs @@ -301,6 +301,9 @@ pub fn settings_from_params(s: &Session, p: &Value, cmd: &str) -> Result<(Option if let Some(v) = bool_p(p, "alpha") { settings.alpha = v; } + if let Some(v) = str_p(p, "exrCompression") { + settings.exr_compression = serde_json::from_value(json!(v)).map_err(|_| bad(cmd, "exrCompression: none | rle | zips | zip"))?; + } match (crate::commands::checked_u32_p(p, "width", cmd)?, crate::commands::checked_u32_p(p, "height", cmd)?) { (Some(w), Some(h)) => settings.frame_size = Some((w, h)), (None, None) => {} diff --git a/crates/export/src/deflate.rs b/crates/export/src/deflate.rs new file mode 100644 index 00000000..41d82620 --- /dev/null +++ b/crates/export/src/deflate.rs @@ -0,0 +1,609 @@ +//! A small DEFLATE encoder (RFC 1951) in a zlib wrapper (RFC 1950), written from the RFCs for the +//! ZIP-compressed OpenEXR chunks. +//! +//! Greedy LZ77 over a 32 KB window (hash chains of bounded length), then each block of tokens is +//! written as whichever of stored, fixed-Huffman or dynamic-Huffman is smallest. Huffman code +//! lengths are limited (15 bits, 7 for the code-length code) by halving the symbol frequencies +//! until the tree fits. Every code has at least two symbols, so all codes are complete. The +//! output favours simplicity and correctness over ratio; worst-case expansion is the stored-block +//! overhead (5 bytes per 64 KB). + +use std::cmp::Reverse; +use std::collections::BinaryHeap; + +const WINDOW: usize = 32 * 1024; +const MIN_MATCH: usize = 3; +const MAX_MATCH: usize = 258; +/// Hash chain candidates tried per position. +const MAX_CHAIN: usize = 48; +/// Tokens per block (a new set of Huffman codes after this many). +const BLOCK_TOKENS: usize = 1 << 15; +const MAX_STORED: usize = 65_535; + +const LEN_BASE: [u16; 29] = [3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258]; +const LEN_EXTRA: [u8; 29] = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0]; +const DIST_BASE: [u16; 30] = + [1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577]; +const DIST_EXTRA: [u8; 30] = [0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13]; +/// Order in which the code-length code's lengths are sent. +const CL_ORDER: [usize; 19] = [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]; + +/// `data` compressed as a zlib stream (CMF 0x78: deflate, 32 KB window). +pub(crate) fn zlib(data: &[u8]) -> Vec { + let mut w = Bits { out: Vec::with_capacity(data.len() / 2 + 64), acc: 0, n: 0 }; + w.out.extend_from_slice(&[0x78, 0x9C]); // (0x78 << 8 | 0x9C) % 31 == 0 + deflate(data, &mut w); + w.align(); + w.out.extend_from_slice(&adler32(data).to_be_bytes()); + w.out +} + +/// Adler-32 (RFC 1950 §9); 5552 bytes is the longest run that cannot overflow the sums. +fn adler32(data: &[u8]) -> u32 { + let (mut a, mut b) = (1_u32, 0_u32); + for c in data.chunks(5552) { + for &x in c { + a += u32::from(x); + b += a; + } + a %= 65_521; + b %= 65_521; + } + (b << 16) | a +} + +/// LSB-first bit writer. +struct Bits { + out: Vec, + acc: u64, + n: u32, +} + +impl Bits { + /// The low `n` bits of `v` (n ≤ 32). + fn put(&mut self, v: u32, n: u32) { + let mask = (1_u64 << n) - 1; + self.acc |= (u64::from(v) & mask) << self.n; + self.n += n; + while self.n >= 8 { + self.out.push(self.acc as u8); + self.acc >>= 8; + self.n -= 8; + } + } + + fn align(&mut self) { + if self.n > 0 { + self.out.push(self.acc as u8); + } + self.acc = 0; + self.n = 0; + } +} + +#[derive(Clone, Copy)] +enum Tok { + Lit(u8), + Match { len: u16, dist: u16 }, +} + +/// Index into [`LEN_BASE`] of a match length (3..=258). +fn len_code(len: u16) -> usize { + LEN_BASE.partition_point(|&b| b <= len).saturating_sub(1) +} + +/// Index into [`DIST_BASE`] of a distance (1..=32768). +fn dist_code(dist: u16) -> usize { + DIST_BASE.partition_point(|&b| b <= dist).saturating_sub(1) +} + +/// LZ77 match finder: hash heads and chains over the window. +struct Matcher<'a> { + data: &'a [u8], + head: Vec, + prev: Vec, + bits: u32, +} + +impl<'a> Matcher<'a> { + fn new(data: &'a [u8]) -> Self { + // tables sized to the input, so short chunks (single scan lines) stay cheap + let span = data.len().clamp(256, WINDOW).next_power_of_two(); + let bits = span.trailing_zeros().clamp(8, 15); + Matcher { data, head: vec![0; 1 << bits], prev: vec![0; span], bits } + } + + fn hash(&self, p: usize) -> Option { + let b = self.data.get(p..p + MIN_MATCH)?; + let v = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]); + Some((v.wrapping_mul(0x9E37_79B1) >> (32 - self.bits)) as usize) + } + + /// Record position `p` (stored as `p + 1`; 0 = empty). + fn insert(&mut self, p: usize) { + let Some(h) = self.hash(p) else { return }; + let mask = self.prev.len() - 1; + self.prev[p & mask] = self.head[h]; + self.head[h] = p as u32 + 1; + } + + /// The longest earlier match at `p`: (length, distance), length 0 if none. + fn longest(&self, p: usize) -> (usize, usize) { + let Some(h) = self.hash(p) else { return (0, 0) }; + let max = (self.data.len() - p).min(MAX_MATCH); + let here = &self.data[p..p + max]; + let mask = self.prev.len() - 1; + let (mut best, mut best_dist) = (0, 0); + let mut cand = self.head[h] as usize; + for _ in 0..MAX_CHAIN { + let Some(c) = cand.checked_sub(1) else { break }; + if c >= p || p - c > WINDOW { + break; + } + let l = self.data[c..].iter().zip(here).take_while(|(a, b)| a == b).count(); + if l > best { + (best, best_dist) = (l, p - c); + if l == max { + break; + } + } + let next = self.prev[c & mask] as usize; + // chains only go back; anything else is a stale slot + if next == 0 || next > c { + break; + } + cand = next; + } + (best, best_dist) + } +} + +/// The deflate blocks of `data`, the last one final. +fn deflate(data: &[u8], w: &mut Bits) { + if data.is_empty() { + // a final fixed-Huffman block holding only end-of-block (code 0000000) + w.put(1, 1); + w.put(1, 2); + w.put(0, 7); + return; + } + // positions are kept in u32; longer inputs (never an OpenEXR chunk) go out stored + let mut m = (data.len() < u32::MAX as usize).then(|| Matcher::new(data)); + let mut toks = Vec::with_capacity(BLOCK_TOKENS.min(data.len()) + 1); + let (mut start, mut i) = (0, 0); + while i < data.len() { + let (len, dist) = m.as_ref().map_or((0, 0), |m| m.longest(i)); + let step = if len >= MIN_MATCH { + toks.push(Tok::Match { len: len as u16, dist: dist as u16 }); + len + } else { + toks.push(Tok::Lit(data[i])); + 1 + }; + if let Some(m) = m.as_mut() { + for p in i..i + step { + m.insert(p); + } + } + i += step; + if toks.len() >= BLOCK_TOKENS || i == data.len() { + block(w, &toks, &data[start..i], i == data.len()); + toks.clear(); + start = i; + } + } +} + +/// Bits the tokens take with the given code lengths (EOB included). +fn data_bits(lf: &[u32], df: &[u32], ll: &[u8], dl: &[u8]) -> u64 { + let mut n = 0_u64; + for (s, (&f, &l)) in lf.iter().zip(ll).enumerate() { + let extra = s.checked_sub(257).and_then(|k| LEN_EXTRA.get(k)).copied().unwrap_or(0); + n += u64::from(f) * u64::from(l + extra); + } + for (d, (&f, &l)) in df.iter().zip(dl).enumerate() { + n += u64::from(f) * u64::from(l + DIST_EXTRA.get(d).copied().unwrap_or(0)); + } + n +} + +/// One block (split into several stored blocks when stored is smallest and `raw` is long). +fn block(w: &mut Bits, toks: &[Tok], raw: &[u8], last: bool) { + let mut lf = [0_u32; 286]; + let mut df = [0_u32; 30]; + for t in toks { + match *t { + Tok::Lit(b) => lf[b as usize] += 1, + Tok::Match { len, dist } => { + lf[257 + len_code(len)] += 1; + df[dist_code(dist)] += 1; + } + } + } + lf[256] += 1; + + // dynamic codes and their header + let ll = lengths(&lf, 15); + let dl = lengths(&df, 15); + let hlit = ll.iter().rposition(|&l| l > 0).map_or(257, |p| (p + 1).max(257)); + let hdist = dl.iter().rposition(|&l| l > 0).map_or(1, |p| p + 1); + let seq = rle_lengths(&[&ll[..hlit], &dl[..hdist]].concat()); + let mut cf = [0_u32; 19]; + for &(s, _) in &seq { + cf[s as usize] += 1; + } + let cl = lengths(&cf, 7); + let hclen = CL_ORDER.iter().rposition(|&s| cl[s] > 0).map_or(4, |p| (p + 1).max(4)); + let seq_bits: u64 = seq.iter().map(|&(s, _)| u64::from(cl[s as usize]) + [2_u64, 3, 7].get((s as usize).wrapping_sub(16)).copied().unwrap_or(0)).sum(); + let dyn_bits = 3 + 14 + 3 * hclen as u64 + seq_bits + data_bits(&lf, &df, &ll, &dl); + + let (fll, fdl) = fixed_lengths(); + let fixed_bits = 3 + data_bits(&lf, &df, &fll, &fdl); + // per stored piece: header, at most 7 padding bits, LEN and NLEN + let pieces = raw.len().div_ceil(MAX_STORED).max(1) as u64; + let stored_bits = pieces * (3 + 7 + 32) + 8 * raw.len() as u64; + + if stored_bits <= fixed_bits.min(dyn_bits) { + let n = raw.chunks(MAX_STORED).count(); + for (k, piece) in raw.chunks(MAX_STORED).enumerate() { + w.put(u32::from(last && k + 1 == n), 1); + w.put(0, 2); + w.align(); + let len = piece.len() as u16; + w.out.extend_from_slice(&len.to_le_bytes()); + w.out.extend_from_slice(&(!len).to_le_bytes()); + w.out.extend_from_slice(piece); + } + return; + } + w.put(u32::from(last), 1); + if fixed_bits <= dyn_bits { + w.put(1, 2); + write_tokens(w, toks, &fll, &fdl); + return; + } + w.put(2, 2); + w.put((hlit - 257) as u32, 5); + w.put((hdist - 1) as u32, 5); + w.put((hclen - 4) as u32, 4); + for &s in &CL_ORDER[..hclen] { + w.put(u32::from(cl[s]), 3); + } + let cc = codes(&cl); + for &(s, x) in &seq { + w.put(u32::from(cc[s as usize]), u32::from(cl[s as usize])); + match s { + 16 => w.put(u32::from(x), 2), + 17 => w.put(u32::from(x), 3), + 18 => w.put(u32::from(x), 7), + _ => {} + } + } + write_tokens(w, toks, &ll, &dl); +} + +/// The fixed Huffman code lengths (RFC 1951 §3.2.6). +fn fixed_lengths() -> ([u8; 288], [u8; 30]) { + let mut ll = [8_u8; 288]; + ll[144..256].fill(9); + ll[256..280].fill(7); + (ll, [5; 30]) +} + +fn write_tokens(w: &mut Bits, toks: &[Tok], ll: &[u8], dl: &[u8]) { + let lc = codes(ll); + let dc = codes(dl); + let sym = |w: &mut Bits, s: usize| w.put(u32::from(lc[s]), u32::from(ll[s])); + for t in toks { + match *t { + Tok::Lit(b) => sym(w, b as usize), + Tok::Match { len, dist } => { + let k = len_code(len); + sym(w, 257 + k); + w.put(u32::from(len - LEN_BASE[k]), u32::from(LEN_EXTRA[k])); + let d = dist_code(dist); + w.put(u32::from(dc[d]), u32::from(dl[d])); + w.put(u32::from(dist - DIST_BASE[d]), u32::from(DIST_EXTRA[d])); + } + } + } + sym(w, 256); +} + +/// Canonical Huffman codes for `lens`, bit-reversed for the LSB-first writer. +fn codes(lens: &[u8]) -> Vec { + let mut count = [0_u32; 16]; + for &l in lens { + if (1..16).contains(&l) { + count[l as usize] += 1; + } + } + let mut next = [0_u32; 16]; + let mut code = 0; + for bits in 1..16 { + code = (code + count[bits - 1]) << 1; + next[bits] = code; + } + lens.iter() + .map(|&l| { + if !(1..16).contains(&l) { + return 0; + } + let c = next[l as usize]; + next[l as usize] += 1; + (c.reverse_bits() >> (32 - u32::from(l))) as u16 + }) + .collect() +} + +/// Code lengths for `freq`, none longer than `limit`; at least two symbols always get a code. +fn lengths(freq: &[u32], limit: u8) -> Vec { + let mut f: Vec = freq.iter().map(|&x| u64::from(x)).collect(); + let mut used = f.iter().filter(|&&x| x > 0).count(); + for x in f.iter_mut() { + if used >= 2 { + break; + } + if *x == 0 { + *x = 1; + used += 1; + } + } + // halving converges: once every used frequency is 1 the tree is balanced (depth ≤ 9 here) + for _ in 0..66 { + let l = huffman(&f); + if l.iter().all(|&d| d <= limit) { + return l; + } + for x in f.iter_mut().filter(|x| **x > 0) { + *x = (*x >> 1).max(1); + } + } + huffman(&f) +} + +/// Huffman tree depths of the symbols with non-zero frequency (0 for the rest). +fn huffman(f: &[u64]) -> Vec { + let n = f.len(); + let mut heap: BinaryHeap> = f.iter().enumerate().filter(|(_, w)| **w > 0).map(|(i, &w)| Reverse((w, i))).collect(); + let mut parent = vec![usize::MAX; 2 * n]; + let mut next = n; + while heap.len() > 1 { + let (Some(Reverse((wa, a))), Some(Reverse((wb, b)))) = (heap.pop(), heap.pop()) else { break }; + parent[a] = next; + parent[b] = next; + heap.push(Reverse((wa + wb, next))); + next += 1; + } + // parents always come after their children, so walk down from the root + let mut depth = vec![0_u32; next]; + for i in (0..next).rev() { + if let Some(&p) = parent.get(i).filter(|&&p| p < next) { + depth[i] = depth[p].saturating_add(1); + } + } + depth[..n].iter().map(|&d| d.min(u32::from(u8::MAX)) as u8).collect() +} + +/// Code lengths run-length coded with the code-length alphabet: (symbol, extra-bits value). +fn rle_lengths(lens: &[u8]) -> Vec<(u8, u8)> { + let mut out = Vec::new(); + let mut i = 0; + while i < lens.len() { + let l = lens[i]; + let run = lens[i..].iter().take_while(|&&x| x == l).count(); + let mut r = run; + if l == 0 { + while r >= 11 { + let k = r.min(138); + out.push((18, (k - 11) as u8)); + r -= k; + } + if r >= 3 { + out.push((17, (r - 3) as u8)); + r = 0; + } + } else { + out.push((l, 0)); + r -= 1; + while r >= 3 { + let k = r.min(6); + out.push((16, (k - 3) as u8)); + r -= k; + } + } + out.extend(std::iter::repeat_n((l, 0), r)); + i += run; + } + out +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + /// A minimal inflater (RFC 1951), only to check the encoder; returns None on any error. + pub(crate) fn inflate(z: &[u8]) -> Option> { + let (cmf, flg) = (*z.first()?, *z.get(1)?); + if cmf & 0x0F != 8 || (u16::from(cmf) << 8 | u16::from(flg)) % 31 != 0 || flg & 0x20 != 0 { + return None; + } + let mut pos = 16_usize; // bit position + let bit = |pos: &mut usize| -> Option { + let b = (*z.get(*pos / 8)? >> (*pos % 8)) & 1; + *pos += 1; + Some(u32::from(b)) + }; + let bits = |pos: &mut usize, n: u32| -> Option { + let mut v = 0; + for k in 0..n { + v |= bit(pos)? << k; + } + Some(v) + }; + // decode one symbol with canonical lengths (slow but simple) + let decode = |pos: &mut usize, lens: &[u8]| -> Option { + let (mut code, mut first, mut index) = (0_i32, 0_i32, 0_i32); + let mut count = [0_i32; 16]; + for &l in lens { + count[l as usize] += 1; + } + let mut sorted: Vec = (0..lens.len()).filter(|&s| lens[s] > 0).collect(); + sorted.sort_by_key(|&s| (lens[s], s)); + for len in 1..16 { + code |= bit(pos)? as i32; + let c = count[len]; + if code - c < first { + return sorted.get((index + code - first) as usize).copied(); + } + index += c; + first = (first + c) << 1; + code <<= 1; + } + None + }; + let mut out = Vec::new(); + loop { + let last = bit(&mut pos)?; + match bits(&mut pos, 2)? { + 0 => { + pos = pos.div_ceil(8) * 8; + let at = pos / 8; + let len = u16::from_le_bytes([*z.get(at)?, *z.get(at + 1)?]); + let nlen = u16::from_le_bytes([*z.get(at + 2)?, *z.get(at + 3)?]); + if len != !nlen { + return None; + } + out.extend_from_slice(z.get(at + 4..at + 4 + len as usize)?); + pos = (at + 4 + len as usize) * 8; + } + t @ (1 | 2) => { + let (ll, dl) = if t == 1 { + let (a, b) = fixed_lengths(); + (a.to_vec(), b.to_vec()) + } else { + let hlit = bits(&mut pos, 5)? as usize + 257; + let hdist = bits(&mut pos, 5)? as usize + 1; + let hclen = bits(&mut pos, 4)? as usize + 4; + let mut cl = [0_u8; 19]; + for &s in &CL_ORDER[..hclen] { + cl[s] = bits(&mut pos, 3)? as u8; + } + let mut lens = Vec::new(); + while lens.len() < hlit + hdist { + match decode(&mut pos, &cl)? { + s @ 0..=15 => lens.push(s as u8), + 16 => { + let p = *lens.last()?; + lens.extend(std::iter::repeat_n(p, 3 + bits(&mut pos, 2)? as usize)); + } + 17 => lens.extend(std::iter::repeat_n(0, 3 + bits(&mut pos, 3)? as usize)), + _ => lens.extend(std::iter::repeat_n(0, 11 + bits(&mut pos, 7)? as usize)), + } + } + if lens.len() != hlit + hdist { + return None; + } + (lens[..hlit].to_vec(), lens[hlit..].to_vec()) + }; + loop { + let s = decode(&mut pos, &ll)?; + if s < 256 { + out.push(s as u8); + } else if s == 256 { + break; + } else { + let k = s - 257; + let len = *LEN_BASE.get(k)? as usize + bits(&mut pos, u32::from(LEN_EXTRA[k]))? as usize; + let d = decode(&mut pos, &dl)?; + let dist = *DIST_BASE.get(d)? as usize + bits(&mut pos, u32::from(DIST_EXTRA[d]))? as usize; + let from = out.len().checked_sub(dist)?; + for j in 0..len { + out.push(out[from + j]); + } + } + } + } + _ => return None, + } + if last == 1 { + break; + } + } + let at = pos.div_ceil(8); + let sum = u32::from_be_bytes(z.get(at..at + 4)?.try_into().ok()?); + (sum == adler32(&out) && at + 4 == z.len()).then_some(out) + } + + /// A deterministic pseudo-random byte stream. + fn noise(n: usize, seed: u32) -> Vec { + let mut s = seed | 1; + (0..n) + .map(|_| { + s ^= s << 13; + s ^= s >> 17; + s ^= s << 5; + s as u8 + }) + .collect() + } + + #[test] + fn round_trips() { + let mut cases: Vec> = vec![vec![], vec![7], vec![1, 2], vec![0; 1], vec![0; 300_000], b"abcabcabcabcabcabd".to_vec()]; + cases.push(noise(100_000, 9)); // incompressible: stored blocks + cases.push(noise(70_000, 3).iter().map(|b| b % 4).collect()); // few symbols: dynamic + cases.push((0..200_000_u32).map(|i| ((i / 7) % 251) as u8).collect()); // long repeats + // text-like: a small vocabulary repeated at varying distances + let words: [&[u8]; 6] = [b"red ", b"green ", b"blue ", b"alpha ", b"half ", b"linear "]; + cases.push(noise(40_000, 5).iter().flat_map(|b| words[*b as usize % 6].iter().copied()).collect()); + // skewed frequencies: Huffman lengths beyond 15 before limiting + let mut fib = Vec::new(); + let (mut a, mut b) = (1_usize, 1_usize); + for s in 0..25_u8 { + fib.extend(std::iter::repeat_n(s.wrapping_mul(37), a.min(200_000))); + (a, b) = (b, a + b); + } + let r = noise(fib.len() * 4, 11); + for i in (1..fib.len()).rev() { + let j = u32::from_le_bytes(r[i * 4..i * 4 + 4].try_into().unwrap()) as usize % (i + 1); + fib.swap(i, j); + } + cases.push(fib); + for (k, data) in cases.iter().enumerate() { + let z = zlib(data); + assert_eq!(inflate(&z).as_deref(), Some(data.as_slice()), "case {k}"); + // worst case: stored overhead only + assert!(z.len() <= data.len() + (data.len().div_ceil(BLOCK_TOKENS) + 1) * 5 + 6, "case {k}: {} > {}", z.len(), data.len()); + } + // repetitive data actually compresses + assert!(zlib(&cases[4]).len() < 2_000); + assert!(zlib(&cases[8]).len() < 20_000); + } + + #[test] + fn length_limits_hold() { + // Fibonacci frequencies give the deepest possible Huffman tree + let mut f = vec![0_u32; 286]; + let (mut a, mut b) = (1_u32, 1_u32); + for x in f.iter_mut().take(30) { + *x = a; + (a, b) = (b, a.saturating_add(b)); + } + let l = lengths(&f, 15); + assert!(l.iter().all(|&d| d <= 15)); + // a complete code: Kraft sum exactly 1 + let kraft: f64 = l.iter().filter(|&&d| d > 0).map(|&d| 0.5_f64.powi(i32::from(d))).sum(); + assert!((kraft - 1.0).abs() < 1e-12); + // one used symbol still gets a two-symbol code + let l = lengths(&[0, 5, 0], 7); + assert_eq!(l.iter().filter(|&&d| d == 1).count(), 2); + } + + #[test] + fn adler_matches_rfc_example() { + // the classic test vector for "Wikipedia" + assert_eq!(adler32(b"Wikipedia"), 0x11E6_0398); + assert_eq!(adler32(&[]), 1); + } +} diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 17858c27..66905c99 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -17,6 +17,7 @@ use std::io::Write; mod aac_out; mod audio_out; +mod deflate; mod job; mod mxf_out; mod pace; @@ -347,6 +348,9 @@ pub struct ExportSettings { /// Other formats ignore it. #[serde(default)] pub alpha: bool, + /// Compression of OpenEXR sequences (uncompressed unless set); other formats ignore it. + #[serde(default)] + pub exr_compression: ExrCompression, /// Output frame size (None = Match Source: the sequence size times `scale`). pub frame_size: Option<(u32, u32)>, /// Output frame rate (None = Match Source). @@ -560,6 +564,7 @@ impl Default for ExportSettings { mxf_video_codec: MxfVideoCodec::default(), sdr: false, alpha: false, + exr_compression: ExrCompression::None, frame_size: None, frame_rate: None, scaling: Scaling::default(), @@ -1785,6 +1790,11 @@ pub fn encode_still(format: Format, rgba: Vec, w: u32, h: u32, alpha: bool) /// [`encode_still`] with the JPEG quality (1–100; ignored by the lossless formats). pub fn encode_still_with_quality(format: Format, rgba: Vec, w: u32, h: u32, alpha: bool, quality: u8) -> Result> { + encode_still_with(format, rgba, w, h, alpha, quality, ExrCompression::None) +} + +/// [`encode_still_with_quality`] with the OpenEXR compression (ignored by the other formats). +pub fn encode_still_with(format: Format, rgba: Vec, w: u32, h: u32, alpha: bool, quality: u8, exr: ExrCompression) -> Result> { let enc = |e: image::ImageError| ExportError::Encode(e.to_string()); let mut out = std::io::Cursor::new(Vec::new()); match format { @@ -1801,7 +1811,7 @@ pub fn encode_still_with_quality(format: Format, rgba: Vec, w: u32, h: u32, } Format::TgaSequence => return raster::tga(&rgba, w, h, alpha), Format::DpxSequence => return raster::dpx(&rgba, w, h), - Format::ExrSequence => return raster::exr(&rgba, w, h, alpha), + Format::ExrSequence => return raster::exr(&rgba, w, h, alpha, exr), Format::PngSequence => { image::ImageEncoder::write_image(image::codecs::png::PngEncoder::new(&mut out), &rgba, w, h, image::ExtendedColorType::Rgba8).map_err(enc)? } @@ -1939,7 +1949,15 @@ pub fn export(project: &Arc, seq: ItemId, settings: &ExportSettings, so let written: Vec> = (f..end) .into_par_iter() .map(|fi| { - let data = encode_still_with_quality(settings.format, pipe.frame(fi, sources).0, w, h, settings.alpha, settings.quality)?; + let data = encode_still_with( + settings.format, + pipe.frame(fi, sources).0, + w, + h, + settings.alpha, + settings.quality, + settings.exr_compression, + )?; write_output(settings, &pcm::image_sequence_path(&settings.path, (fi - f0) as u64, count), data) }) .collect(); diff --git a/crates/export/src/raster.rs b/crates/export/src/raster.rs index 74ea4081..e3026ceb 100644 --- a/crates/export/src/raster.rs +++ b/crates/export/src/raster.rs @@ -5,9 +5,11 @@ //! (version 2.0 header): one RGB image element, 10 bits per component packed into 32-bit //! big-endian words (method A, two padding bits at the bottom), BT.709 transfer and colorimetry. //! OpenEXR follows the public OpenEXR file layout (version 2, single-part scan-line file): half-float -//! B, G, R (and A) channels, no compression, one scan line per chunk, scene-linear Rec.709 values -//! with premultiplied alpha, as EXR readers expect. -use crate::{ExportError, Result}; +//! B, G, R (and A) channels, scene-linear Rec.709 values with premultiplied alpha, as EXR readers +//! expect; uncompressed, RLE, ZIPS (zlib per scan line) or ZIP (zlib per 16 scan lines). The +//! compressed modes first split each chunk's bytes into even and odd halves and delta-code them +//! (the format's predictor); a chunk that would not shrink is stored raw, as the format allows. +use crate::{ExportError, ExrCompression, Result}; /// `w * h * 4` when `rgba` holds exactly that many bytes. fn checked_len(rgba: &[u8], w: u32, h: u32) -> Result { @@ -164,15 +166,20 @@ fn half(v: f32) -> u16 { /// An OpenEXR file: half-float scene-linear Rec.709 RGB, or RGBA with `alpha` (premultiplied, as /// EXR defines alpha; the frame's straight alpha is multiplied in after linearisation). -pub(crate) fn exr(rgba: &[u8], w: u32, h: u32, alpha: bool) -> Result> { +pub(crate) fn exr(rgba: &[u8], w: u32, h: u32, alpha: bool, compression: ExrCompression) -> Result> { let n = checked_len(rgba, w, h)?; + let too_large = || ExportError::Unsupported(format!("an OpenEXR frame of {w}×{h} is too large")); let (Ok(x_max), Ok(y_max)) = (i32::try_from(w - 1), i32::try_from(h - 1)) else { - return Err(ExportError::Unsupported(format!("an OpenEXR frame of {w}×{h} is too large"))); + return Err(too_large()); }; // channels in the alphabetical order the file layout requires let names: &[u8] = if alpha { b"ABGR" } else { b"BGR" }; let line = (w as usize).checked_mul(names.len() * 2).ok_or_else(|| ExportError::Encode("frame size".into()))?; - let line_size = i32::try_from(line).map_err(|_| ExportError::Unsupported(format!("an OpenEXR frame of {w}×{h} is too large")))?; + // a chunk's byte count is a signed 32-bit field + let lines = compression.lines(); + if line.checked_mul(lines).and_then(|c| i32::try_from(c).ok()).is_none() { + return Err(too_large()); + } let mut hd = Vec::with_capacity(512); hd.extend_from_slice(&[0x76, 0x2F, 0x31, 0x01]); // magic number 20000630 @@ -198,7 +205,7 @@ pub(crate) fn exr(rgba: &[u8], w: u32, h: u32, alpha: bool) -> Result> { let window: Vec = [0, 0, x_max, y_max].iter().flat_map(|v| v.to_le_bytes()).collect(); attr(&mut hd, "channels", "chlist", &channels); attr(&mut hd, "chromaticities", "chromaticities", &le(&[0.64, 0.33, 0.30, 0.60, 0.15, 0.06, 0.3127, 0.3290])); - attr(&mut hd, "compression", "compression", &[0]); // NO_COMPRESSION + attr(&mut hd, "compression", "compression", &[compression.code()]); attr(&mut hd, "dataWindow", "box2i", &window); attr(&mut hd, "displayWindow", "box2i", &window); attr(&mut hd, "lineOrder", "lineOrder", &[0]); // INCREASING_Y @@ -207,45 +214,102 @@ pub(crate) fn exr(rgba: &[u8], w: u32, h: u32, alpha: bool) -> Result> { attr(&mut hd, "screenWindowWidth", "float", &le(&[1.0])); hd.push(0); // end of header - // offset table (one chunk per scan line), then the chunks: y, data size, pixel data - let chunk = line + 8; - let table = (h as usize).checked_mul(8); - let body = (h as usize).checked_mul(chunk); + // offset table (one entry per chunk), then the chunks: first y, data size, data + let chunks = (h as usize).div_ceil(lines); + let table = chunks.checked_mul(8); + let body = (h as usize).checked_mul(line + 8); let size = table.zip(body).and_then(|(t, b)| hd.len().checked_add(t)?.checked_add(b)).ok_or_else(|| ExportError::Encode("frame size".into()))?; + // the uncompressed size: compressed chunks only shrink it let mut out = Vec::with_capacity(size); out.extend_from_slice(&hd); - let first = (hd.len() + h as usize * 8) as u64; - for y in 0..h as u64 { - out.extend_from_slice(&(first + y * chunk as u64).to_le_bytes()); - } + let table_at = out.len(); + out.resize(table_at + chunks * 8, 0); let lin = filmcraft_color::srgb_u8_to_linear_table(); let opaque: [u16; 256] = std::array::from_fn(|i| half(lin[i])); let row = w as usize * 4; - let mut px = vec![0_u16; w as usize * names.len()]; - for (y, src) in rgba[..n].chunks_exact(row).enumerate() { - // planar per line: every pixel of the first channel, then the next channel … - let wn = w as usize; - for (x, &[r, g, b, a]) in src.as_chunks::<4>().0.iter().enumerate() { - let rgb = [b, g, r]; - if alpha { - let k = f32::from(a) / 255.0; - px[x] = half(k); - for (c, &v) in rgb.iter().enumerate() { - px[(c + 1) * wn + x] = if a == 255 { opaque[v as usize] } else { half(lin[v as usize] * k) }; - } - } else { - for (c, &v) in rgb.iter().enumerate() { - px[c * wn + x] = opaque[v as usize]; + let wn = w as usize; + let mut px = vec![0_u16; wn * names.len()]; + let mut raw = Vec::with_capacity(line * lines); + for (k, block) in rgba[..n].chunks(row * lines).enumerate() { + raw.clear(); + for src in block.chunks_exact(row) { + // planar per line: every pixel of the first channel, then the next channel … + for (x, &[r, g, b, a]) in src.as_chunks::<4>().0.iter().enumerate() { + let rgb = [b, g, r]; + if alpha { + let k = f32::from(a) / 255.0; + px[x] = half(k); + for (c, &v) in rgb.iter().enumerate() { + px[(c + 1) * wn + x] = if a == 255 { opaque[v as usize] } else { half(lin[v as usize] * k) }; + } + } else { + for (c, &v) in rgb.iter().enumerate() { + px[c * wn + x] = opaque[v as usize]; + } } } + raw.extend(px.iter().flat_map(|v| v.to_le_bytes())); } - out.extend_from_slice(&(y as i32).to_le_bytes()); - out.extend_from_slice(&line_size.to_le_bytes()); - out.extend(px.iter().flat_map(|v| v.to_le_bytes())); + let packed = match compression { + ExrCompression::None => None, + ExrCompression::Rle => Some(rle(&predict(&raw))), + ExrCompression::Zips | ExrCompression::Zip => Some(crate::deflate::zlib(&predict(&raw))), + }; + // a chunk that does not shrink is stored as it is (readers compare the sizes) + let data = packed.as_deref().filter(|p| p.len() < raw.len()).unwrap_or(&raw); + let offset = (out.len() as u64).to_le_bytes(); + out[table_at + k * 8..table_at + k * 8 + 8].copy_from_slice(&offset); + out.extend_from_slice(&((k * lines) as i32).to_le_bytes()); + out.extend_from_slice(&(data.len() as i32).to_le_bytes()); + out.extend_from_slice(data); } Ok(out) } +/// The OpenEXR RLE / ZIP predictor: bytes split into even and odd halves, then each byte replaced +/// by its difference from the previous one, plus 128. +fn predict(raw: &[u8]) -> Vec { + let half = raw.len().div_ceil(2); + let mut t = vec![0_u8; raw.len()]; + for (i, &b) in raw.iter().enumerate() { + t[if i % 2 == 0 { i / 2 } else { half + i / 2 }] = b; + } + let mut p = t.first().copied().unwrap_or(0); + for v in t.iter_mut().skip(1) { + let d = v.wrapping_sub(p).wrapping_add(128); + p = *v; + *v = d; + } + t +} + +/// OpenEXR run-length coding: a signed count byte, then either `count + 1` copies of one byte +/// (count ≥ 0, runs of 3 to 128) or `-count` literal bytes (1 to 127). +fn rle(src: &[u8]) -> Vec { + let mut out = Vec::with_capacity(src.len() + src.len() / 127 + 1); + let mut i = 0; + while i < src.len() { + let b = src[i]; + let run = src[i..].iter().take(128).take_while(|&&x| x == b).count(); + if run >= 3 { + out.extend_from_slice(&[(run - 1) as u8, b]); + i += run; + continue; + } + // literals up to the next run of three (or 127 bytes) + let s = i; + while i < src.len() && i - s < 127 { + if src.get(i..i + 3).is_some_and(|t| t[0] == t[1] && t[1] == t[2]) { + break; + } + i += 1; + } + out.push(((i - s) as u8).wrapping_neg()); + out.extend_from_slice(&src[s..i]); + } + out +} + #[cfg(test)] mod tests { use super::*; @@ -322,7 +386,7 @@ mod tests { // 2×1: opaque white, then (255, 0, 0) at alpha 0.5 let px = [255, 255, 255, 255, 255, 0, 0, 128]; for alpha in [false, true] { - let f = exr(&px, 2, 1, alpha).unwrap(); + let f = exr(&px, 2, 1, alpha, ExrCompression::None).unwrap(); let nch = if alpha { 4 } else { 3 }; assert_eq!(&f[..8], &[0x76, 0x2F, 0x31, 0x01, 2, 0, 0, 0]); let ch = find(&f, b"channels\0chlist\0") + 16; @@ -353,36 +417,134 @@ mod tests { } } + /// A 40×35 test picture: gradients, a flat band and a noisy corner, with varying alpha (35 + /// rows: the last 16-line ZIP chunk is short). + fn picture() -> (Vec, u32, u32) { + let (w, h) = (40_u32, 35_u32); + let px = (0..h) + .flat_map(|y| (0..w).map(move |x| (x, y))) + .flat_map(|(x, y)| { + let noise = (x * 7919 + y * 104_729) % 251; + let r = if y < 12 { + (x * 6) as u8 + } else if y < 20 { + 200 + } else { + noise as u8 + }; + [r, (y * 7) as u8, ((x + y) * 3) as u8, [255, 128, 0, 64][(x / 10) as usize % 4]] + }) + .collect(); + (px, w, h) + } + + /// The chunks of an OpenEXR file we wrote: (first y, data). + fn chunks(f: &[u8], count: usize) -> Vec<(u32, &[u8])> { + let end = find(f, b"screenWindowWidth\0float\0") + 24 + 4 + 4 + 1; + (0..count) + .map(|k| { + let off = u64::from_le_bytes(f[end + k * 8..end + k * 8 + 8].try_into().unwrap()) as usize; + let n = le32(f, off + 4) as usize; + (le32(f, off), &f[off + 8..off + 8 + n]) + }) + .collect() + } + + /// Undo [`predict`]. + fn unpredict(t: &[u8]) -> Vec { + let mut t = t.to_vec(); + for i in 1..t.len() { + t[i] = t[i - 1].wrapping_add(t[i]).wrapping_sub(128); + } + let half = t.len().div_ceil(2); + (0..t.len()).map(|i| t[if i % 2 == 0 { i / 2 } else { half + i / 2 }]).collect() + } + + fn unrle(mut s: &[u8]) -> Vec { + let mut out = Vec::new(); + while let [c, rest @ ..] = s { + let c = *c as i8; + if c < 0 { + out.extend_from_slice(&rest[..c.unsigned_abs() as usize]); + s = &rest[c.unsigned_abs() as usize..]; + } else { + out.extend(std::iter::repeat_n(rest[0], c as usize + 1)); + s = &rest[1..]; + } + } + out + } + + #[test] + fn rle_codes_runs_and_literals() { + for src in [vec![], vec![5], vec![1, 1], vec![9; 300], (0..=255).collect::>(), [vec![1, 2], vec![3; 3], vec![4, 4, 5]].concat()] { + assert_eq!(unrle(&rle(&src)), src); + } + assert_eq!(rle(&[7; 130]), vec![127, 7, 0xFE, 7, 7]); + assert_eq!(rle(&[1, 2, 2, 2]), vec![0xFF, 1, 2, 2]); + } + + /// Every compression stores the same pixels as the uncompressed file, in chunks of the right + /// height, and the compressed files are smaller. + #[test] + fn compressed_exr_holds_the_same_pixels() { + let (px, w, h) = picture(); + for alpha in [false, true] { + let plain = exr(&px, w, h, alpha, ExrCompression::None).unwrap(); + let lines: Vec<&[u8]> = chunks(&plain, h as usize).into_iter().map(|(_, d)| d).collect(); + for c in [ExrCompression::Rle, ExrCompression::Zips, ExrCompression::Zip] { + let f = exr(&px, w, h, alpha, c).unwrap(); + let cmp = find(&f, b"compression\0compression\0") + 24; + assert_eq!(f[cmp + 4], c.code()); + assert!(f.len() < plain.len() * 3 / 4, "{c:?}: {} vs {}", f.len(), plain.len()); + let n = (h as usize).div_ceil(c.lines()); + for (k, (y, data)) in chunks(&f, n).into_iter().enumerate() { + assert_eq!(y as usize, k * c.lines()); + let want: Vec = lines[k * c.lines()..((k + 1) * c.lines()).min(h as usize)].concat(); + let got = if data.len() == want.len() { + data.to_vec() + } else if c == ExrCompression::Rle { + unpredict(&unrle(data)) + } else { + unpredict(&crate::deflate::tests::inflate(data).unwrap()) + }; + assert_eq!(got, want, "{c:?} chunk {k}"); + } + } + } + } + /// ffmpeg (external test oracle) decodes our OpenEXR files to the values we meant: linear - /// light, premultiplied alpha, rows top to bottom. + /// light, premultiplied alpha, rows top to bottom; in every compression. #[test] fn exr_decodes_in_ffmpeg() { let Some(ffmpeg) = filmcraft_testkit::ffmpeg_or_skip("OpenEXR stills") else { return }; - let (w, h) = (5_u32, 3_u32); - let px: Vec = (0..w * h).flat_map(|i| [(i * 17) as u8, (255 - i * 13) as u8, (i * 40 % 256) as u8, [255, 128, 0, 64][i as usize % 4]]).collect(); + let (px, w, h) = picture(); let dir = filmcraft_testkit::fixtures::workspace_root().join("target").join("export-tests").join(format!("exr-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); let lin = filmcraft_color::srgb_u8_to_linear_table(); - for alpha in [false, true] { - let path = dir.join(format!("f{alpha}.exr")); - std::fs::write(&path, exr(&px, w, h, alpha).unwrap()).unwrap(); - let out = std::process::Command::new(&ffmpeg) - .args(["-v", "error", "-i"]) - .arg(&path) - .args(["-f", "rawvideo", "-pix_fmt", "gbrapf32le", "-"]) - .output() - .unwrap(); - assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); - let n = (w * h) as usize; - assert_eq!(out.stdout.len(), n * 4 * 4); - let plane = |p: usize, i: usize| f32::from_le_bytes(out.stdout[(p * n + i) * 4..][..4].try_into().unwrap()); - for (i, &[r, g, b, a]) in px.as_chunks::<4>().0.iter().enumerate() { - let k = if alpha { f32::from(a) / 255.0 } else { 1.0 }; - // gbrap planes: G, B, R, A - let want = [lin[g as usize] * k, lin[b as usize] * k, lin[r as usize] * k, k]; - for (p, want) in want.into_iter().enumerate() { - let got = plane(p, i); - assert!((got - want).abs() <= 1e-3 + want * 1e-3, "alpha {alpha}, pixel {i}, plane {p}: {got} vs {want}"); + for c in ExrCompression::ALL { + for alpha in [false, true] { + let path = dir.join(format!("f{alpha}-{c:?}.exr")); + std::fs::write(&path, exr(&px, w, h, alpha, c).unwrap()).unwrap(); + let out = std::process::Command::new(&ffmpeg) + .args(["-v", "error", "-i"]) + .arg(&path) + .args(["-f", "rawvideo", "-pix_fmt", "gbrapf32le", "-"]) + .output() + .unwrap(); + assert!(out.status.success(), "{c:?}: {}", String::from_utf8_lossy(&out.stderr)); + let n = (w * h) as usize; + assert_eq!(out.stdout.len(), n * 4 * 4, "{c:?}"); + let plane = |p: usize, i: usize| f32::from_le_bytes(out.stdout[(p * n + i) * 4..][..4].try_into().unwrap()); + for (i, &[r, g, b, a]) in px.as_chunks::<4>().0.iter().enumerate() { + let k = if alpha { f32::from(a) / 255.0 } else { 1.0 }; + // gbrap planes: G, B, R, A + let want = [lin[g as usize] * k, lin[b as usize] * k, lin[r as usize] * k, k]; + for (p, want) in want.into_iter().enumerate() { + let got = plane(p, i); + assert!((got - want).abs() <= 1e-3 + want * 1e-3, "{c:?}, alpha {alpha}, pixel {i}, plane {p}: {got} vs {want}"); + } } } } @@ -397,8 +559,10 @@ mod tests { assert!(tga(&[0; 4], u32::MAX, u32::MAX, false).is_err()); assert!(rgb(&[0; 3], 1, 1).is_err()); assert!(dpx(&[0; 4], u32::MAX, u32::MAX).is_err()); - assert!(exr(&[0; 4], u32::MAX, u32::MAX, true).is_err()); - assert!(exr(&[0; 7], 2, 1, false).is_err()); - assert!(exr(&[], 0, 1, false).is_err()); + for c in ExrCompression::ALL { + assert!(exr(&[0; 4], u32::MAX, u32::MAX, true, c).is_err()); + assert!(exr(&[0; 7], 2, 1, false, c).is_err()); + assert!(exr(&[], 0, 1, false, c).is_err()); + } } } diff --git a/crates/export/src/settings.rs b/crates/export/src/settings.rs index 70831e9c..a59f481e 100644 --- a/crates/export/src/settings.rs +++ b/crates/export/src/settings.rs @@ -160,6 +160,46 @@ impl MxfVideoCodec { } } +/// Compression of OpenEXR stills (the file's `compression` attribute). Every mode is lossless. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ExrCompression { + /// Uncompressed. + #[default] + None, + /// Run-length coded bytes (RLE_COMPRESSION), one scan line per chunk. + Rle, + /// zlib (deflate) per scan line (ZIPS_COMPRESSION). + Zips, + /// zlib (deflate) per block of 16 scan lines (ZIP_COMPRESSION). + Zip, +} + +impl ExrCompression { + pub const ALL: [ExrCompression; 4] = [ExrCompression::None, ExrCompression::Rle, ExrCompression::Zips, ExrCompression::Zip]; + pub fn label(self) -> &'static str { + match self { + ExrCompression::None => "None", + ExrCompression::Rle => "RLE", + ExrCompression::Zips => "ZIP (1 scan line)", + ExrCompression::Zip => "ZIP (16 scan lines)", + } + } + /// The value of the OpenEXR `compression` attribute. + pub(crate) fn code(self) -> u8 { + match self { + ExrCompression::None => 0, + ExrCompression::Rle => 1, + ExrCompression::Zips => 2, + ExrCompression::Zip => 3, + } + } + /// Scan lines per chunk. + pub(crate) fn lines(self) -> usize { + if self == ExrCompression::Zip { 16 } else { 1 } + } +} + /// Audio codec. `Auto` = AAC in MP4 and AAC Audio, PCM in QuickTime / WAV / AIFF, FLAC in FLAC. /// MP4 and QuickTime can also carry FLAC; MXF is always PCM. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -499,8 +539,11 @@ impl ExportSettings { Format::TgaSequence => px * if self.keeps_alpha() { 4.0 } else { 3.0 } * 8.0 * fps, // 10-bit RGB in one 32-bit word per pixel Format::DpxSequence => px * 4.0 * 8.0 * fps, - // two bytes per half-float channel - Format::ExrSequence => px * if self.keeps_alpha() { 8.0 } else { 6.0 } * 8.0 * fps, + // two bytes per half-float channel; lossless compression roughly halves it + Format::ExrSequence => { + let k = if self.exr_compression == ExrCompression::None { 1.0 } else { 0.5 }; + px * if self.keeps_alpha() { 8.0 } else { 6.0 } * 8.0 * fps * k + } Format::Gif => px * 0.6 * 8.0 * fps, Format::Wav | Format::Aiff | Format::Flac | Format::MxfOp1a | Format::MxfOpAtom => 0.0, }; @@ -587,7 +630,13 @@ impl ExportSettings { Format::Apv => v += &format!(", {}", crate::apv_profile(&self.apv_profile).name()), Format::Mjpeg | Format::JpegSequence => v += &format!(", quality {}", self.quality), Format::DpxSequence => v += ", 10-bit RGB", - Format::ExrSequence => v += ", 16-bit half float, linear", + Format::ExrSequence => { + v += ", 16-bit half float, linear"; + match self.exr_compression { + ExrCompression::None => v += ", uncompressed", + c => v += &format!(", {} compression", c.label()), + } + } _ => {} } v diff --git a/crates/export/src/settings_tests.rs b/crates/export/src/settings_tests.rs index 1def9d1e..b2ada981 100644 --- a/crates/export/src/settings_tests.rs +++ b/crates/export/src/settings_tests.rs @@ -203,6 +203,41 @@ fn openexr_sequences() { } } +/// Compressed OpenEXR sequences (#874): the export setting reaches every still, which shrinks +/// and (ffmpeg as the external oracle) decodes to the same pixels as the uncompressed file. +#[test] +fn openexr_sequences_compress() { + let (p, seq, m) = matte([0.2, 0.6, 1.0, 0.5], 64, 36, None); + let mut files = Vec::new(); + for c in ExrCompression::ALL { + let dir = Scratch::new(&format!("seq-exr-{c:?}")); + let mut s = ExportSettings { format: Format::ExrSequence, path: dir.path("c.exr"), alpha: true, exr_compression: c, ..Default::default() }; + s.range = Some(TimeRange::new(Tick::ZERO, FrameRate::FPS_24.tick_of(1))); + export(&p, seq, &s, &m, &Progress::default()).unwrap(); + let path = dir.0.join("c000.exr"); + let bytes = std::fs::read(&path).unwrap(); + let at = bytes.windows(24).position(|w| w == b"compression\0compression\0").unwrap() + 28; + assert_eq!(bytes[at], [0, 1, 2, 3][files.len()], "{c:?}"); + files.push((c, path, bytes.len(), dir)); + } + let plain = files[0].2; + for (c, _, len, _) in &files[1..] { + assert!(*len < plain / 2, "{c:?}: {len} bytes vs {plain} uncompressed"); + } + let Some(ffmpeg) = filmcraft_testkit::ffmpeg_or_skip("compressed OpenEXR sequences") else { return }; + let decode = |path: &Path| { + let out = + std::process::Command::new(&ffmpeg).args(["-v", "error", "-i"]).arg(path).args(["-f", "rawvideo", "-pix_fmt", "gbrapf32le", "-"]).output().unwrap(); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + out.stdout + }; + let want = decode(&files[0].1); + assert_eq!(want.len(), 64 * 36 * 16); + for (c, path, _, _) in &files[1..] { + assert!(decode(path) == want, "{c:?} decodes differently"); + } +} + /// `alpha` keeps straight alpha in PNG and TIFF sequences; off (the default) flattens over black (#160). #[test] fn image_sequences_keep_alpha_when_asked() { diff --git a/crates/ui-egui/src/i18n/es.tsv b/crates/ui-egui/src/i18n/es.tsv index c2e1529e..6119ccc9 100644 --- a/crates/ui-egui/src/i18n/es.tsv +++ b/crates/ui-egui/src/i18n/es.tsv @@ -3570,6 +3570,9 @@ Edit Project Notes Editar notas del proyecto Notes for this project: to-dos, instructions, frame numbers… Saved with the project. Notas de este proyecto: tareas pendientes, instrucciones, números de fotograma… Se guardan con el proyecto. Use Effect Shutter Angle Usar el ángulo de obturación del efecto + Compression Compresión + ZIP (1 scan line) ZIP (1 línea de exploración) + ZIP (16 scan lines) ZIP (16 líneas de exploración) Safe graphics mode: {mode} Modo gráfico seguro: {mode} An earlier start failed with the default graphics. Updating the graphics driver usually fixes this. Un inicio anterior falló con los gráficos predeterminados. Actualizar el controlador gráfico suele solucionarlo. Use Default Graphics Usar gráficos predeterminados diff --git a/crates/ui-egui/src/i18n/fr.tsv b/crates/ui-egui/src/i18n/fr.tsv index ef315bda..edcba88a 100644 --- a/crates/ui-egui/src/i18n/fr.tsv +++ b/crates/ui-egui/src/i18n/fr.tsv @@ -3546,6 +3546,9 @@ Notes for this project: to-dos, instructions, frame numbers… Saved with the project. Notes pour ce projet : tâches, instructions, numéros d'image… Enregistrées avec le projet. Use Effect Shutter Angle Utiliser l'angle d'obturation de l'effet Settings Préférences + Compression Compression + ZIP (1 scan line) ZIP (1 ligne de balayage) + ZIP (16 scan lines) ZIP (16 lignes de balayage) Safe graphics mode: {mode} Mode graphique sans échec : {mode} An earlier start failed with the default graphics. Updating the graphics driver usually fixes this. Un démarrage précédent a échoué avec les graphismes par défaut. Mettre à jour le pilote graphique résout généralement le problème. Use Default Graphics Utiliser les graphismes par défaut diff --git a/crates/ui-egui/src/i18n/ja.tsv b/crates/ui-egui/src/i18n/ja.tsv index 450c44fb..d142b44b 100644 --- a/crates/ui-egui/src/i18n/ja.tsv +++ b/crates/ui-egui/src/i18n/ja.tsv @@ -3552,6 +3552,9 @@ Edit Project Notes プロジェクトメモを編集 Notes for this project: to-dos, instructions, frame numbers… Saved with the project. このプロジェクトのメモ: やること、指示、フレーム番号など… プロジェクトと一緒に保存されます。 GPU acceleration is unavailable; preview renders on the CPU GPU アクセラレーションを利用できません。プレビューは CPU でレンダリングされます + Compression 圧縮 + ZIP (1 scan line) ZIP（1 スキャンライン） + ZIP (16 scan lines) ZIP（16 スキャンライン） Caption Tracks キャプショントラック Delete Caption Track キャプショントラックを削除 diff --git a/crates/ui-egui/src/i18n/zh-cn.tsv b/crates/ui-egui/src/i18n/zh-cn.tsv index 75df54ee..15bc785f 100644 --- a/crates/ui-egui/src/i18n/zh-cn.tsv +++ b/crates/ui-egui/src/i18n/zh-cn.tsv @@ -3548,6 +3548,9 @@ Delete Caption Track 删除字幕轨道 Delete Caption Tracks 删除多个字幕轨道 Delete Empty Caption Tracks 删除空字幕轨道 + Compression 压缩 + ZIP (1 scan line) ZIP（1 条扫描线） + ZIP (16 scan lines) ZIP（16 条扫描线） Safe graphics mode: {mode} 安全图形模式：{mode} An earlier start failed with the default graphics. Updating the graphics driver usually fixes this. 上次启动时默认图形模式失败。更新显卡驱动程序通常可以解决此问题。 Use Default Graphics 使用默认图形 diff --git a/crates/ui-egui/src/panels/export_mode.rs b/crates/ui-egui/src/panels/export_mode.rs index aed80470..0716f7ad 100644 --- a/crates/ui-egui/src/panels/export_mode.rs +++ b/crates/ui-egui/src/panels/export_mode.rs @@ -23,8 +23,8 @@ use egui::{Align2, Color32, Rect, Sense, pos2, vec2}; use filmcraft_engine::export::presets::{DEFAULT_PRESET, preset_key}; use filmcraft_engine::export::{ - AudioCodec, BitrateMode, ExportSettings, FieldOrder, Format, GpuRendering, H264Profile, HardwareEncoding, Multiplexer, MxfVideoCodec, Placement, Scaling, - TextOverlay, builtin_presets, format_bytes, + AudioCodec, BitrateMode, ExportSettings, ExrCompression, FieldOrder, Format, GpuRendering, H264Profile, HardwareEncoding, Multiplexer, MxfVideoCodec, + Placement, Scaling, TextOverlay, builtin_presets, format_bytes, }; use filmcraft_engine::time::{FrameRate, Tick}; use serde::{Deserialize, Serialize}; @@ -810,6 +810,21 @@ fn video_section(ui: &mut egui::Ui, reg: &mut Reg, s: &mut ExportSettings, t: &T } }); } + Format::ExrSequence => { + row(ui, t, tl!("Compression"), |ui| { + // every mode is lossless; RLE and the ZIP variants shrink the files + let label = |c: ExrCompression| match c { + ExrCompression::None => tl!("None"), + ExrCompression::Rle => "RLE", + ExrCompression::Zips => tl!("ZIP (1 scan line)"), + ExrCompression::Zip => tl!("ZIP (16 scan lines)"), + }; + let labels: Vec<&str> = ExrCompression::ALL.iter().map(|&c| label(c)).collect(); + if let Some(i) = combo(ui, reg, "export.video.exrCompression", label(s.exr_compression), &opts(&labels), 220.0) { + s.exr_compression = ExrCompression::ALL[i]; + } + }); + } Format::Mjpeg | Format::JpegSequence => { row(ui, t, tl!("Quality"), |ui| { let mut q = s.quality as f64; diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index b7a64326..44dd1192 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -55,7 +55,7 @@ only). "Tested" names the evidence. | PNG, JPEG, TIFF, BMP, GIF | read + write | yes | PNG / TIFF / BMP / JPEG sequences, GIF | 90% | | WebP | — | yes | no | ahead | | **PSD** (layers as sequence) | read (layered) | no | — | 0% | -| **OpenEXR** | read + write | no | sequences (half float, linear RGB / premultiplied RGBA, uncompressed; #806) | 35% | +| **OpenEXR** | read + write | no | sequences (half float, linear RGB / premultiplied RGBA; uncompressed, RLE, ZIPS or ZIP; #806, #874) | 35% | | **DPX** | read + write | yes: luma / RGB / RGBA, 8–16-bit, filled 10/12-bit, either byte order (#821); packed 10/12-bit, Y'CbCr 4:2:2 / 4:4:4 with or without alpha (#841); no RLE | sequences (10-bit RGB, BT.709; #779) | 90% | | Targa | read + write | yes: every image type, RLE, colour maps, 15–32-bit (#821) | sequences (24-bit, 32-bit with alpha; #779) | 95% | | HEIF / HEIC | read | no | — | 0% | diff --git a/docs/gaps.md b/docs/gaps.md index 840536c1..137c8d26 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -216,11 +216,12 @@ platforms** G4, G9, G17; **performance** G16; **ecosystem** G11; **measurement** ## G13. Delivery formats - **Missing:** AV1 export (software or hardware), MPEG-2 / DVD / Blu-ray, DCP, JPEG 2000 MXF, - AS-10 / AS-11, XDCAM HD MXF, P2, compressed (ZIP / PIZ / DWA) OpenEXR, MP3 audio, interlaced - encoding, smart render, HEVC software encoder (HEVC exists only where a hardware encoder does). + AS-10 / AS-11, XDCAM HD MXF, P2, PIZ / PXR24 / B44 / DWA OpenEXR compression, MP3 audio, + interlaced encoding, smart render, HEVC software encoder (HEVC exists only where a hardware + encoder does). - **Have:** PNG, TIFF, BMP, JPEG, Targa and DPX (10-bit) image sequences (#779); OpenEXR - sequences (half float, linear, uncompressed; #806); AAC Audio (audio-only `.m4a`, or ADTS - `.aac`; #814); ProRes 4444 / 4444 XQ in the export UI (#342). + sequences (half float, linear; uncompressed, RLE, ZIPS or ZIP; #806, #874); AAC Audio + (audio-only `.m4a`, or ADTS `.aac`; #814); ProRes 4444 / 4444 XQ in the export UI (#342). - **Estimate:** 60–110 h. Doc: [file-format-parity.md](file-format-parity.md). ## G14. Localization: 8 of the 12 key languages missing