From a95fd5c829699ec1bcf1d12a4d4e2b17690e5215 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:54:50 -0400 Subject: [PATCH 1/7] M4.9: H.264 vectorised deblocking edge filters and branch-free CABAC bins Deblocking filters all lines of an edge at once with per-lane masks (bit-exact with the per-line filter, checked on random edges); CABAC decode_decision / bypass select instead of branching. 4K single-thread CPU -19 %, output identical to ffmpeg. Co-Authored-By: Claude Opus 5.5 --- crates/h264/src/cabac.rs | 45 +++--- crates/h264/src/deblock.rs | 310 +++++++++++++++++++++++++++++++------ 2 files changed, 284 insertions(+), 71 deletions(-) diff --git a/crates/h264/src/cabac.rs b/crates/h264/src/cabac.rs index 42e32bce..b6b9a44a 100644 --- a/crates/h264/src/cabac.rs +++ b/crates/h264/src/cabac.rs @@ -145,35 +145,31 @@ impl<'a> Cabac<'a> { #[inline(always)] fn renorm(&mut self) { - if self.range < 256 { - let shift = self.range.leading_zeros() - 23; - self.range <<= shift; - if self.bits < shift { - self.refill(); - } - self.bits -= shift; + // codIRange < 512 always, so the shift is 0 when no renormalisation is needed: no branch on + // the (unpredictable) range, only on the rare refill. + let shift = self.range.leading_zeros() - 23; + self.range <<= shift; + if self.bits < shift { + self.refill(); } + self.bits -= shift; } + /// 9.3.3.2.1, branch-free: the MPS / LPS outcome of a context-coded bin is close to random + /// at high bit rates, so it selects values instead of jumping. #[inline(always)] pub fn decode_decision(&mut self, ctx_idx: usize) -> u32 { let s = self.ctx[ctx_idx] as usize; let q = ((self.range >> 6) & 3) as usize; let lps = RANGE_TAB_LPS[s >> 1][q] as u32; - self.range -= lps; - let scaled = (self.range as u64) << self.bits; - let bin; - if self.value >= scaled { - bin = 1 - (s & 1) as u32; - self.value -= scaled; - self.range = lps; - self.ctx[ctx_idx] = NEXT_STATE[s][1]; - } else { - bin = (s & 1) as u32; - self.ctx[ctx_idx] = NEXT_STATE[s][0]; - } + let mps_range = self.range - lps; + let scaled = (mps_range as u64) << self.bits; + let is_lps = self.value >= scaled; + self.value -= scaled & (is_lps as u64).wrapping_neg(); + self.range = if is_lps { lps } else { mps_range }; + self.ctx[ctx_idx] = NEXT_STATE[s][is_lps as usize]; self.renorm(); - bin + (s & 1) as u32 ^ is_lps as u32 } #[inline(always)] @@ -183,12 +179,9 @@ impl<'a> Cabac<'a> { } self.bits -= 1; let scaled = (self.range as u64) << self.bits; - if self.value >= scaled { - self.value -= scaled; - 1 - } else { - 0 - } + let one = self.value >= scaled; + self.value -= scaled & (one as u64).wrapping_neg(); + one as u32 } pub fn decode_terminate(&mut self) -> u32 { diff --git a/crates/h264/src/deblock.rs b/crates/h264/src/deblock.rs index 015bbc5f..310daf21 100644 --- a/crates/h264/src/deblock.rs +++ b/crates/h264/src/deblock.rs @@ -72,13 +72,14 @@ fn motion_bs(p: &Motion, q: &Motion) -> u8 { } } -#[inline(always)] +#[cfg(test)] fn clip3(lo: i32, hi: i32, v: i32) -> i32 { v.clamp(lo, hi) } -/// Filter one set of eight samples p3 p2 p1 p0 | q0 q1 q2 q3 across an edge (8.7.2.3 / 8.7.2.4). -#[inline(always)] +/// Filter one set of eight samples p3 p2 p1 p0 | q0 q1 q2 q3 across an edge (8.7.2.3 / 8.7.2.4): +/// the per-line reference for [`filter_lanes`]. +#[cfg(test)] fn filter8(v: &mut [u8; 8], bs: u8, alpha: i32, beta: i32, tc0: i32, chroma: bool) { let p0 = v[3] as i32; let q0 = v[4] as i32; @@ -143,51 +144,130 @@ struct EdgeParams { beta: i32, } -/// Filter a vertical edge whose q0 column is at `x`, for `n` lines starting at row `y`. +/// Filter one edge for `N` lines at once (lane `i` = line `i` across the edge). `s[k][i]` holds +/// sample p3 p2 p1 p0 q0 q1 q2 q3 (k = 0..8) of line `i`. Branch-free per lane, so the loops +/// vectorise; bit-exact with [`filter8`] applied to every line (8.7.2.3 / 8.7.2.4). #[inline(always)] -fn filter_vertical(pix: &mut [u8], stride: usize, x: usize, y: usize, n: usize, ep: &EdgeParams, chroma: bool) { - let per = n / 4; - for i in 0..n { - let k = i / per; - let b = ep.bs[k]; - if b == 0 { - continue; +fn filter_lanes(s: &mut [[i16; N]; 8], ep: &EdgeParams, chroma: bool) { + let per = N / 4; + let (alpha, beta) = (ep.alpha as i16, ep.beta as i16); + let mut bs_on = [false; N]; + let mut tc0 = [0i16; N]; + for k in 0..4 { + for i in k * per..(k + 1) * per { + bs_on[i] = ep.bs[k] != 0; + tc0[i] = ep.tc0[k] as i16; + } + } + let [p3, p2, p1, p0, q0, q1, q2, q3] = *s; + // Masks are 0 / -1 per lane, selects are bitwise, and clipping to per-lane bounds uses + // min / max (`clamp` would assert its bounds per lane), so every loop below vectorises. + let lt = |x: i16, y: i16| -((x < y) as i16); + let sel = |m: i16, a: i16, b: i16| (a & m) | (b & !m); + let mut on = [0i16; N]; + let mut any = 0i16; + for i in 0..N { + on[i] = -(bs_on[i] as i16) & lt((p0[i] - q0[i]).abs(), alpha) & lt((p1[i] - p0[i]).abs(), beta) & lt((q1[i] - q0[i]).abs(), beta); + any |= on[i]; + } + if any == 0 { + return; + } + // bS 4 is only ever assigned to all four segments of an edge + let strong_edge = ep.bs[0] >= 4; + if chroma { + for i in 0..N { + let (np0, nq0) = if strong_edge { + ((2 * p1[i] + p0[i] + q1[i] + 2) >> 2, (2 * q1[i] + q0[i] + p1[i] + 2) >> 2) + } else { + let tc = tc0[i] + 1; + let delta = ((((q0[i] - p0[i]) << 2) + (p1[i] - q1[i]) + 4) >> 3).max(-tc).min(tc); + ((p0[i] + delta).clamp(0, 255), (q0[i] - delta).clamp(0, 255)) + }; + s[3][i] = sel(on[i], np0, p0[i]); + s[4][i] = sel(on[i], nq0, q0[i]); + } + return; + } + if strong_edge { + let lim = (alpha >> 2) + 2; + for i in 0..N { + let (a, b, c, d, e, f, g, h) = (p3[i], p2[i], p1[i], p0[i], q0[i], q1[i], q2[i], q3[i]); + let strong = lt((d - e).abs(), lim); + let sp = on[i] & lt((b - d).abs(), beta) & strong; + let sq = on[i] & lt((g - e).abs(), beta) & strong; + let wp0 = (2 * c + d + f + 2) >> 2; + let wq0 = (2 * f + e + c + 2) >> 2; + let sp0 = (b + 2 * c + 2 * d + 2 * e + f + 4) >> 3; + let sq0 = (c + 2 * d + 2 * e + 2 * f + g + 4) >> 3; + s[3][i] = sel(sp, sp0, sel(on[i], wp0, d)); + s[2][i] = sel(sp, (b + c + d + e + 2) >> 2, c); + s[1][i] = sel(sp, (2 * a + 3 * b + c + d + e + 4) >> 3, b); + s[4][i] = sel(sq, sq0, sel(on[i], wq0, e)); + s[5][i] = sel(sq, (d + e + f + g + 2) >> 2, f); + s[6][i] = sel(sq, (2 * h + 3 * g + f + e + d + 4) >> 3, g); + } + } else { + for i in 0..N { + let ap = lt((p2[i] - p0[i]).abs(), beta); + let aq = lt((q2[i] - q0[i]).abs(), beta); + let t0 = tc0[i]; + let tc = t0 - ap - aq; + let delta = ((((q0[i] - p0[i]) << 2) + (p1[i] - q1[i]) + 4) >> 3).max(-tc).min(tc); + let avg = (p0[i] + q0[i] + 1) >> 1; + let np1 = p1[i] + ((p2[i] + avg - (p1[i] << 1)) >> 1).max(-t0).min(t0); + let nq1 = q1[i] + ((q2[i] + avg - (q1[i] << 1)) >> 1).max(-t0).min(t0); + s[3][i] = sel(on[i], (p0[i] + delta).clamp(0, 255), p0[i]); + s[4][i] = sel(on[i], (q0[i] - delta).clamp(0, 255), q0[i]); + s[2][i] = sel(on[i] & ap, np1, p1[i]); + s[5][i] = sel(on[i] & aq, nq1, q1[i]); } - let o = (y + i) * stride + x - 4; - let v: &mut [u8; 8] = (&mut pix[o..o + 8]).try_into().unwrap(); - filter8(v, b, ep.alpha, ep.beta, ep.tc0[k], chroma); } } -/// Filter a horizontal edge whose q0 row is `y`, for `n` columns starting at `x`. +/// Filter a vertical edge whose q0 column is at `x`, for `N` lines starting at row `y`. #[inline(always)] -fn filter_horizontal(pix: &mut [u8], stride: usize, x: usize, y: usize, n: usize, ep: &EdgeParams, chroma: bool) { - let base = (y - 4) * stride + x; - let rest = &mut pix[base..]; - let (r0, rest) = rest.split_at_mut(stride); - let (r1, rest) = rest.split_at_mut(stride); - let (r2, rest) = rest.split_at_mut(stride); - let (r3, rest) = rest.split_at_mut(stride); - let (r4, rest) = rest.split_at_mut(stride); - let (r5, rest) = rest.split_at_mut(stride); - let (r6, rest) = rest.split_at_mut(stride); - let r7 = &mut rest[..n]; - let (r0, r1, r2, r3, r4, r5, r6) = (&mut r0[..n], &mut r1[..n], &mut r2[..n], &mut r3[..n], &mut r4[..n], &mut r5[..n], &mut r6[..n]); - let per = n / 4; - for c in 0..n { - let k = c / per; - let b = ep.bs[k]; - if b == 0 { - continue; +fn filter_vertical(pix: &mut [u8], stride: usize, x: usize, y: usize, ep: &EdgeParams, chroma: bool) { + // Each line's eight samples as one little-endian word: byte k = sample k (p3 .. q3). + let mut w = [0u64; N]; + for (i, w) in w.iter_mut().enumerate() { + let o = (y + i) * stride + x - 4; + *w = u64::from_le_bytes(pix[o..o + 8].try_into().unwrap()); + } + let mut s = [[0i16; N]; 8]; + for (k, row) in s.iter_mut().enumerate() { + for i in 0..N { + row[i] = ((w[i] >> (8 * k)) & 0xff) as i16; + } + } + filter_lanes(&mut s, ep, chroma); + for (i, w) in w.iter_mut().enumerate() { + *w = (0..8).fold(0u64, |acc, k| acc | ((s[k][i] as u8 as u64) << (8 * k))); + } + for (i, w) in w.iter().enumerate() { + let o = (y + i) * stride + x - 4; + pix[o..o + 8].copy_from_slice(&w.to_le_bytes()); + } +} + +/// Filter a horizontal edge whose q0 row is `y`, for `N` columns starting at `x`. +#[inline(always)] +fn filter_horizontal(pix: &mut [u8], stride: usize, x: usize, y: usize, ep: &EdgeParams, chroma: bool) { + let base = (y - 4) * stride + x; + let mut s = [[0i16; N]; 8]; + for (k, row) in s.iter_mut().enumerate() { + let src: &[u8; N] = pix[base + k * stride..base + k * stride + N].try_into().unwrap(); + for i in 0..N { + row[i] = src[i] as i16; + } + } + filter_lanes(&mut s, ep, chroma); + let (k0, k1) = if chroma { (3, 5) } else { (1, 7) }; + for (k, row) in s.iter().enumerate().take(k1).skip(k0) { + let dst: &mut [u8; N] = (&mut pix[base + k * stride..base + k * stride + N]).try_into().unwrap(); + for i in 0..N { + dst[i] = row[i] as u8; } - let mut v = [r0[c], r1[c], r2[c], r3[c], r4[c], r5[c], r6[c], r7[c]]; - filter8(&mut v, b, ep.alpha, ep.beta, ep.tc0[k], chroma); - r1[c] = v[1]; - r2[c] = v[2]; - r3[c] = v[3]; - r4[c] = v[4]; - r5[c] = v[5]; - r6[c] = v[6]; } } @@ -288,9 +368,9 @@ pub fn deblock_mb(pic: &mut PicState, addr: usize, mb_w: usize) { let qpp = if e == 0 { p_qp[dir].map(|p| p.0).unwrap_or(q_qp) } else { q_qp }; let ep = params(qpp, q_qp, bs[dir][e]); if dir == 0 { - filter_vertical(&mut pic.planes.y, width, mx * 16 + e * 4, my * 16, 16, &ep, false); + filter_vertical::<16>(&mut pic.planes.y, width, mx * 16 + e * 4, my * 16, &ep, false); } else { - filter_horizontal(&mut pic.planes.y, width, mx * 16, my * 16 + e * 4, 16, &ep, false); + filter_horizontal::<16>(&mut pic.planes.y, width, mx * 16, my * 16 + e * 4, &ep, false); } } } @@ -306,11 +386,151 @@ pub fn deblock_mb(pic: &mut PicState, addr: usize, mb_w: usize) { let ep = params(qpp, q_qpc[c] as i32, bs[dir][e]); let plane = if c == 0 { &mut pic.planes.cb } else { &mut pic.planes.cr }; if dir == 0 { - filter_vertical(plane, cwidth, mx * 8 + ce * 4, my * 8, 8, &ep, true); + filter_vertical::<8>(plane, cwidth, mx * 8 + ce * 4, my * 8, &ep, true); } else { - filter_horizontal(plane, cwidth, mx * 8, my * 8 + ce * 4, 8, &ep, true); + filter_horizontal::<8>(plane, cwidth, mx * 8, my * 8 + ce * 4, &ep, true); } } } } } + +#[cfg(test)] +mod tests { + use super::*; + + /// The vectorised edge filter equals the per-line reference [`filter8`] on random edges + /// (small differences across the edge so that every filter branch is taken). + #[test] + fn lane_filter_matches_per_line_reference() { + let mut rng = 0x9e37_79b9_7f4a_7c15u64; + let mut next = move || { + rng ^= rng << 13; + rng ^= rng >> 7; + rng ^= rng << 17; + rng + }; + for iter in 0..20_000 { + let chroma = iter % 3 == 0; + let strong = iter % 4 == 1; + let bs: [u8; 4] = if strong { [4; 4] } else { std::array::from_fn(|_| (next() % 4) as u8) }; + let index_a = (next() % 52) as usize; + let index_b = (next() % 52) as usize; + let tc = |b: u8| if (1..4).contains(&b) { TC0[index_a][b as usize - 1] as i32 } else { 0 }; + let ep = EdgeParams { bs, tc0: bs.map(tc), alpha: ALPHA[index_a] as i32, beta: BETA[index_b] as i32 }; + let base = (next() % 256) as i32; + let spread = 1 + (next() % 24) as i32; + let mut lines = [[0u8; 8]; 16]; + for l in lines.iter_mut() { + for v in l.iter_mut() { + *v = (base + (next() % (2 * spread as u64 + 1)) as i32 - spread).clamp(0, 255) as u8; + } + } + let n = if chroma { 8 } else { 16 }; + let mut want = lines; + for (i, l) in want.iter_mut().enumerate().take(n) { + let k = i / (n / 4); + if bs[k] != 0 { + filter8(l, bs[k], ep.alpha, ep.beta, ep.tc0[k], chroma); + } + } + let got: Vec<[u8; 8]> = if chroma { + let mut s = [[0i16; 8]; 8]; + for i in 0..8 { + for k in 0..8 { + s[k][i] = lines[i][k] as i16; + } + } + filter_lanes(&mut s, &ep, true); + (0..8).map(|i| std::array::from_fn(|k| s[k][i] as u8)).collect() + } else { + let mut s = [[0i16; 16]; 8]; + for i in 0..16 { + for k in 0..8 { + s[k][i] = lines[i][k] as i16; + } + } + filter_lanes(&mut s, &ep, false); + (0..16).map(|i| std::array::from_fn(|k| s[k][i] as u8)).collect() + }; + assert_eq!(&got[..], &want[..n], "iteration {iter}: bs {bs:?} indexA {index_a} indexB {index_b}"); + } + } +} + +#[cfg(test)] +mod speed { + use super::*; + + /// Rough timing of the edge filters (`cargo test --release -p filmcraft-h264 --lib edge_filter_speed -- --ignored --nocapture`). + #[test] + #[ignore] + fn edge_filter_speed() { + let (w, h) = (1024usize, 256usize); + let mut pix: Vec = (0..w * h).map(|i| (128 + ((i * 7919) % 13) as i32 - 6) as u8).collect(); + let eps = [ + EdgeParams { bs: [4; 4], tc0: [0; 4], alpha: 40, beta: 9 }, + EdgeParams { bs: [3; 4], tc0: [2; 4], alpha: 40, beta: 9 }, + EdgeParams { bs: [2, 1, 0, 2], tc0: [1, 1, 0, 1], alpha: 40, beta: 9 }, + ]; + let t0 = std::time::Instant::now(); + let mut n = 0; + for _ in 0..20 { + for my in 1..h / 16 { + for mx in 1..w / 16 { + for (j, ep) in eps.iter().enumerate() { + filter_horizontal::<16>(&mut pix, w, mx * 16, my * 16 + j * 4, ep, false); + n += 1; + } + } + } + } + let dt = t0.elapsed(); + let tv = std::time::Instant::now(); + for _ in 0..20 { + for my in 1..h / 16 { + for mx in 1..w / 16 { + for (j, ep) in eps.iter().enumerate() { + filter_vertical::<16>(&mut pix, w, mx * 16 + j * 4, my * 16, ep, false); + } + } + } + } + println!("vertical: {:.1} ns/edge", tv.elapsed().as_nanos() as f64 / n as f64); + let t1 = std::time::Instant::now(); + for _ in 0..20 { + for my in 1..h / 16 { + for mx in 1..w / 16 { + for ep in eps.iter() { + for (vert, x, y) in [(true, mx * 16, my * 16), (false, mx * 16, my * 16)] { + for i in 0..16 { + let b = ep.bs[i / 4]; + if b == 0 { + continue; + } + let mut v = [0u8; 8]; + for k in 0..8 { + v[k] = if vert { pix[(y + i) * w + x - 4 + k] } else { pix[(y - 4 + k) * w + x + i] }; + } + filter8(&mut v, b, ep.alpha, ep.beta, ep.tc0[i / 4], false); + for k in 1..7 { + if vert { + pix[(y + i) * w + x - 4 + k] = v[k]; + } else { + pix[(y - 4 + k) * w + x + i] = v[k]; + } + } + } + } + } + } + } + } + println!("scalar reference (both): {:.1} ns/edge", t1.elapsed().as_nanos() as f64 / (2 * n) as f64); + println!( + "horizontal: {n} luma edges in {dt:?}: {:.1} ns/edge (checksum {})", + dt.as_nanos() as f64 / n as f64, + pix.iter().map(|&v| v as u64).sum::() + ); + } +} From b3a09fab6480ccbca712dbac2f2d90bbb736541b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 23:06:54 -0400 Subject: [PATCH 2/7] M4.9: H.264 draft mode (no deblocking of non-reference pictures, off by default) Decoder::set_draft: non-reference pictures skip the deblocking filter and are flagged Picture::draft; all other pictures stay bit-exact (conformance test, single- and frame-threaded). Conformance fixtures now also decode with 3 threads. Decoder::threads / default_threads for perf.stats. Co-Authored-By: Claude Opus 5.5 --- crates/h264/examples/h264dec.rs | 4 ++- crates/h264/src/cabac.rs | 5 +++- crates/h264/src/decoder.rs | 51 +++++++++++++++++++++++++++----- crates/h264/src/dpb.rs | 2 ++ crates/h264/src/lib.rs | 6 +++- crates/h264/src/slicedec.rs | 10 +++++-- crates/h264/tests/common/mod.rs | 15 ++++++++-- crates/h264/tests/conformance.rs | 42 ++++++++++++++++++++++++++ 8 files changed, 121 insertions(+), 14 deletions(-) diff --git a/crates/h264/examples/h264dec.rs b/crates/h264/examples/h264dec.rs index 58f8dfa6..1fe10ba5 100644 --- a/crates/h264/examples/h264dec.rs +++ b/crates/h264/examples/h264dec.rs @@ -1,5 +1,6 @@ //! Decode an Annex-B H.264 file to raw yuv420p: `h264dec in.h264 [out.yuv]`. -//! Prints timing information to stderr. +//! Prints timing information to stderr. `H264_BENCH_ITERS=n` (with `H264_THREADS=t`, and +//! `H264_DRAFT=1` for draft mode) turns it into a benchmark. use filmcraft_h264::Decoder; use std::io::Write; @@ -18,6 +19,7 @@ fn main() { let threads = std::env::var("H264_THREADS").ok().and_then(|v| v.parse::().ok()); for _ in 0..n { let mut dec = threads.map(Decoder::with_threads).unwrap_or_default(); + dec.set_draft(std::env::var_os("H264_DRAFT").is_some()); let t0 = std::time::Instant::now(); frames = dec.decode(&data, 0).expect("decode").len() + dec.flush().len(); best = best.min(t0.elapsed().as_secs_f64()); diff --git a/crates/h264/src/cabac.rs b/crates/h264/src/cabac.rs index b6b9a44a..184406ae 100644 --- a/crates/h264/src/cabac.rs +++ b/crates/h264/src/cabac.rs @@ -159,7 +159,10 @@ impl<'a> Cabac<'a> { /// at high bit rates, so it selects values instead of jumping. #[inline(always)] pub fn decode_decision(&mut self, ctx_idx: usize) -> u32 { - let s = self.ctx[ctx_idx] as usize; + // Every ctxIdx is < 1024 and every state < 128: the masks only let the compiler drop the + // bounds checks from the per-bin path. + let ctx_idx = ctx_idx & (NUM_CTX - 1); + let s = (self.ctx[ctx_idx] & 127) as usize; let q = ((self.range >> 6) & 3) as usize; let lps = RANGE_TAB_LPS[s >> 1][q] as u32; let mps_range = self.range - lps; diff --git a/crates/h264/src/decoder.rs b/crates/h264/src/decoder.rs index 35bd2957..97826b20 100644 --- a/crates/h264/src/decoder.rs +++ b/crates/h264/src/decoder.rs @@ -63,6 +63,8 @@ pub struct DecodeStats { pub frame_num_gaps: u64, pub weighted_slices: u64, pub temporal_direct_slices: u64, + /// Non-reference pictures decoded in draft mode (deblocking skipped). + pub draft_pictures: u64, } /// State shared with decoding jobs. @@ -93,6 +95,9 @@ pub struct Decoder { pool: Option, in_flight: VecDeque, max_in_flight: usize, + threads: usize, + /// Draft mode: skip deblocking of non-reference pictures ([`Decoder::set_draft`]). + draft: bool, } impl Default for Decoder { @@ -122,7 +127,9 @@ fn is_new_picture(prev: &SliceHeader, sh: &SliceHeader, sps: &Sps) -> bool { false } -fn default_threads() -> usize { +/// The worker threads [`Decoder::new`] uses: the available cores (at most 16) with the `threads` +/// feature on native targets, 1 otherwise. +pub fn default_threads() -> usize { #[cfg(all(feature = "threads", not(target_arch = "wasm32")))] { std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1).min(16) @@ -148,6 +155,10 @@ impl Decoder { } else { None }; + #[cfg(feature = "threads")] + let workers = if pool.is_some() { threads } else { 1 }; + #[cfg(not(feature = "threads"))] + let workers = 1; Decoder { spss: vec![None; 32], ppss: vec![None; 256], @@ -165,9 +176,30 @@ impl Decoder { pool, in_flight: VecDeque::new(), max_in_flight: threads.max(1) + 2, + threads: workers, + draft: false, } } + /// Worker threads decoding pictures (1: on the calling thread). + pub fn threads(&self) -> usize { + self.threads + } + + /// Draft mode (off by default) for reduced-resolution playback: pictures submitted from now + /// on that are non-reference pictures (`nal_ref_idc` 0) skip the deblocking filter (8.7). + /// Nothing predicts from a non-reference picture (and deblocking never changes the motion + /// data used for direct prediction), so every other picture stays bit-exact; the draft + /// pictures themselves are approximate and flagged [`Picture::draft`]. Never use it for + /// export or for a frame shown while paused. + pub fn set_draft(&mut self, on: bool) { + self.draft = on; + } + + pub fn draft(&self) -> bool { + self.draft + } + /// Configure from an `avcC` (AVCDecoderConfigurationRecord) box payload. pub fn from_avcc(avcc: &[u8]) -> Result { let mut d = Decoder::new(); @@ -450,19 +482,20 @@ impl Decoder { fn submit_pending(&mut self) { let Some(p) = self.pending.take() else { return }; let PendingPic { frame, sps, first, poc, pts, key, slices } = p; - self.dispatch(frame.clone(), slices); + let draft = self.draft && first.nal_ref_idc == 0; + self.dispatch(frame.clone(), slices, draft); self.poc_state.update(&first, &poc); if first.nal_ref_idc != 0 { self.prev_ref_frame_num = if first.has_mmco5() { 0 } else { first.frame_num }; } - let meta = Arc::new(output_meta(&sps, pts, key)); + let meta = Arc::new(OutputMeta { draft, ..output_meta(&sps, pts, key) }); let mut outs = Vec::new(); let fpoc = frame.poc; self.dpb.store_picture(&first, frame, fpoc, sps.max_frame_num(), sps.max_num_ref_frames as usize, meta, &mut outs); self.emit(outs); } - fn dispatch(&mut self, frame: FrameRef, slices: Vec) { + fn dispatch(&mut self, frame: FrameRef, slices: Vec, draft: bool) { let shared = self.shared.clone(); #[cfg(feature = "threads")] if let Some(pool) = &self.pool { @@ -473,10 +506,10 @@ impl Decoder { } } self.in_flight.push_back(frame.clone()); - pool.spawn_fifo(move || run_job(frame, slices, &shared)); + pool.spawn_fifo(move || run_job(frame, slices, &shared, draft)); return; } - run_job(frame, slices, &shared); + run_job(frame, slices, &shared, draft); } fn emit(&mut self, outs: Vec) { @@ -485,7 +518,7 @@ impl Decoder { } /// Decode all slices of one picture and publish its rows. -fn run_job(frame: FrameRef, slices: Vec, shared: &Shared) { +fn run_job(frame: FrameRef, slices: Vec, shared: &Shared, draft: bool) { let reuse = shared.pool.lock().ok().and_then(|mut p| p.pop()); let mut pic = match reuse { Some(mut p) if p.mb_w == frame.mb_w && p.mb_h == frame.mb_h() => { @@ -494,6 +527,7 @@ fn run_job(frame: FrameRef, slices: Vec, shared: &Shared) { } _ => PicState::new(frame.clone()), }; + pic.skip_deblock = draft; let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { let mut first_err = None; for s in &slices { @@ -521,6 +555,7 @@ fn run_job(frame: FrameRef, slices: Vec, shared: &Shared) { }; if let Ok(mut s) = shared.stats.lock() { s.pictures += 1; + s.draft_pictures += draft as u64; for st in &pic.mbs { match st.kind { MbKind::I4x4 => s.mb_i4x4 += 1, @@ -570,6 +605,7 @@ fn output_meta(sps: &Sps, pts: i64, key: bool) -> OutputMeta { transfer_characteristics: vui.transfer_characteristics, matrix_coefficients: vui.matrix_coefficients, sar: vui.sar, + draft: false, } } @@ -598,5 +634,6 @@ fn make_picture(o: &Output) -> Picture { matrix: o.meta.matrix_coefficients, }, sar: o.meta.sar, + draft: o.meta.draft, } } diff --git a/crates/h264/src/dpb.rs b/crates/h264/src/dpb.rs index 2603c77b..691c62b1 100644 --- a/crates/h264/src/dpb.rs +++ b/crates/h264/src/dpb.rs @@ -24,6 +24,8 @@ pub struct OutputMeta { pub transfer_characteristics: u8, pub matrix_coefficients: u8, pub sar: (u16, u16), + /// Decoded without deblocking (draft mode, non-reference picture). + pub draft: bool, } pub struct DpbEntry { diff --git a/crates/h264/src/lib.rs b/crates/h264/src/lib.rs index 8094d234..9ad3a123 100644 --- a/crates/h264/src/lib.rs +++ b/crates/h264/src/lib.rs @@ -41,7 +41,7 @@ mod synth_tests; mod tables; mod transform; -pub use decoder::{DecodeStats, Decoder}; +pub use decoder::{DecodeStats, Decoder, default_threads}; pub use error::{Error, Result}; /// Colour description from the VUI (ITU-T H.273 code points). @@ -77,4 +77,8 @@ pub struct Picture { pub color: ColorInfo, /// Sample aspect ratio (0, 0 when unspecified). pub sar: (u16, u16), + /// Decoded in draft mode ([`Decoder::set_draft`]): a non-reference picture whose deblocking + /// filter was skipped. Its samples are approximate (not the conforming output); no other + /// picture is affected. + pub draft: bool, } diff --git a/crates/h264/src/slicedec.rs b/crates/h264/src/slicedec.rs index 75f192b4..ac82b54d 100644 --- a/crates/h264/src/slicedec.rs +++ b/crates/h264/src/slicedec.rs @@ -40,6 +40,8 @@ pub struct PicState { rows_recon: usize, rows_deblocked: usize, rows_published: usize, + /// Draft mode for a non-reference picture: rows are published without deblocking. + pub skip_deblock: bool, } impl PicState { @@ -58,6 +60,7 @@ impl PicState { rows_recon: 0, rows_deblocked: 0, rows_published: 0, + skip_deblock: false, } } @@ -76,6 +79,7 @@ impl PicState { self.rows_recon = 0; self.rows_deblocked = 0; self.rows_published = 0; + self.skip_deblock = false; } /// Record a decoded macroblock and deblock / publish rows that became final. @@ -100,8 +104,10 @@ impl PicState { let deblock_limit = if finished { self.mb_h } else { self.rows_recon.saturating_sub(1) }; while self.rows_deblocked < deblock_limit { let r = self.rows_deblocked; - for addr in r * self.mb_w..(r + 1) * self.mb_w { - deblock::deblock_mb(self, addr, self.mb_w); + if !self.skip_deblock { + for addr in r * self.mb_w..(r + 1) * self.mb_w { + deblock::deblock_mb(self, addr, self.mb_w); + } } self.rows_deblocked += 1; } diff --git a/crates/h264/tests/common/mod.rs b/crates/h264/tests/common/mod.rs index 78466ff5..ab75ea00 100644 --- a/crates/h264/tests/common/mod.rs +++ b/crates/h264/tests/common/mod.rs @@ -422,8 +422,15 @@ pub fn decode_file(path: &Path) -> Result, DecodeFailure> { /// Decode with `threads` worker threads (0 = decoder default). pub fn decode_file_threads(path: &Path, threads: usize) -> Result, DecodeFailure> { + decode_file_opts(path, threads, false) +} + +/// [`decode_file_threads`] with draft mode ([`Decoder::set_draft`]) on or off. +pub fn decode_file_opts(path: &Path, threads: usize, draft: bool) -> Result, DecodeFailure> { let data = std::fs::read(path).unwrap(); let mut dec = if threads == 0 { Decoder::new() } else { Decoder::with_threads(threads) }; + assert!(!dec.draft(), "draft mode is off by default"); + dec.set_draft(draft); let mut out = Vec::new(); for (i, au) in split_access_units(&data).into_iter().enumerate() { match dec.decode(au, i as i64) { @@ -494,8 +501,9 @@ pub fn check_fixture(name: &str) -> Result { let f = fixture(name); let Some((h264, yuv)) = ensure(f) else { return Ok(false) }; let reference = std::fs::read(&yuv).unwrap(); - // single-threaded and frame-threaded decoding must both be bit-exact - for threads in [1, 0] { + // single-threaded and frame-threaded decoding (an odd thread count and every core) must all + // be bit-exact + for threads in [1, 3, 0] { let pics = match decode_file_threads(&h264, threads) { Ok(p) => p, Err((au, e, partial)) => { @@ -505,6 +513,9 @@ pub fn check_fixture(name: &str) -> Result { } }; compare(&pics, &reference, f.width as usize, f.height as usize).map_err(|e| format!("{name} (threads {threads}): {e}"))?; + if pics.iter().any(|p| p.draft) { + return Err(format!("{name} (threads {threads}): draft picture without draft mode")); + } check_pts(&pics).map_err(|e| format!("{name} (threads {threads}): {e}"))?; } Ok(true) diff --git a/crates/h264/tests/conformance.rs b/crates/h264/tests/conformance.rs index a2654c9c..154bb8e9 100644 --- a/crates/h264/tests/conformance.rs +++ b/crates/h264/tests/conformance.rs @@ -56,6 +56,48 @@ fixture_tests!( vt_high_1080p, ); +/// Draft mode (reduced-resolution playback) skips deblocking of non-reference pictures only: +/// every other picture stays bit-exact with ffmpeg, single- and frame-threaded, the skipped ones +/// are flagged, and they are the only ones that differ. +#[test] +fn draft_mode_changes_only_flagged_non_reference_pictures() { + let mut checked = 0; + for name in ["main_cabac_b", "bpyramid", "cavlc_b", "direct_temporal", "weightb"] { + let f = common::fixture(name); + let Some((h264, yuv)) = common::ensure(f) else { + eprintln!("skipped {name}"); + continue; + }; + let reference = std::fs::read(&yuv).unwrap(); + let (w, h) = (f.width as usize, f.height as usize); + let fsize = w * h + 2 * w.div_ceil(2) * h.div_ceil(2); + for threads in [1, 0] { + let pics = common::decode_file_opts(&h264, threads, true).unwrap_or_else(|(au, e, _)| panic!("{name}: error at {au}: {e}")); + common::check_pts(&pics).unwrap_or_else(|e| panic!("{name}: {e}")); + assert_eq!(pics.len() * fsize, reference.len(), "{name}: picture count"); + let (mut draft, mut changed) = (0, 0); + for (i, p) in pics.iter().enumerate() { + let r = &reference[i * fsize..(i + 1) * fsize]; + let same = p.y == r[..w * h] && [&p.u[..], &p.v[..]].concat() == r[w * h..]; + if p.draft { + draft += 1; + changed += !same as usize; + } else { + assert!(same, "{name} (threads {threads}): picture {i} (not draft) differs from the reference"); + } + } + assert!(draft > 0, "{name}: has non-reference pictures"); + if name != "no_deblock" { + assert!(changed > 0, "{name}: skipping deblocking changes some draft picture"); + } + checked += 1; + } + } + if checked == 0 { + eprintln!("skipped: no fixtures"); + } +} + /// Print which coding tools each fixture exercises (run with `-- --ignored --nocapture`). #[test] #[ignore] From 1f4546d54986a75f1a85846809e38afe8e52b585 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 23:33:03 -0400 Subject: [PATCH 3/7] M4.9: draft decoding for reduced-resolution playback (opt-in preference) Settings > Playback > Draft decoding (off by default): the Program monitor asks for draft frames only while playing at 1/2 resolution or lower (FrameKey::draft, filmcraft_media::cancel::with_draft). The GOP cache runs the decoder in draft mode for those requests and serves draft frames to draft requests only, so paused frames, renders and exports always get exact pictures (codecs test: exports after draft playback match ffmpeg). Draft plans hand the GPU box-decimated YUV planes at the drawn size. perf.stats reports draft frames, H.264 threads and the preference; the bench gains 2160p quarter and draft playback cases. Co-Authored-By: Claude Opus 5.5 --- crates/codecs/src/gop.rs | 120 ++++++++++++++++++---- crates/codecs/src/video.rs | 40 ++++++-- crates/codecs/tests/h264_draft.rs | 68 ++++++++++++ crates/engine/src/autosave.rs | 5 + crates/engine/src/perf.rs | 4 +- crates/engine/src/settings.rs | 1 + crates/frame/src/lib.rs | 96 +++++++++++++++++ crates/media/src/cancel.rs | 31 ++++++ crates/render/src/lib.rs | 2 +- crates/render/src/plan.rs | 9 ++ crates/render/src/tests.rs | 51 +++++++++ crates/ui-egui/examples/bench/main.rs | 1 + crates/ui-egui/examples/bench/playback.rs | 17 +-- crates/ui-egui/examples/bench/sections.rs | 26 +++-- crates/ui-egui/src/frames.rs | 57 +++++++--- crates/ui-egui/src/lib.rs | 2 +- crates/ui-egui/src/panels/export_mode.rs | 2 +- crates/ui-egui/src/panels/monitor.rs | 6 +- crates/ui-egui/src/panels/multicam.rs | 2 +- crates/ui-egui/src/panels/reference.rs | 2 +- crates/ui-egui/src/panels/scopes.rs | 2 +- crates/ui-egui/src/panels/trim_monitor.rs | 2 +- crates/ui-egui/src/perf.rs | 1 + 23 files changed, 471 insertions(+), 76 deletions(-) create mode 100644 crates/codecs/tests/h264_draft.rs diff --git a/crates/codecs/src/gop.rs b/crates/codecs/src/gop.rs index 9acdbde3..52c29ecb 100644 --- a/crates/codecs/src/gop.rs +++ b/crates/codecs/src/gop.rs @@ -31,6 +31,8 @@ pub struct GopStats { pub decode_ns: u64, /// Non-reference samples left out while catching up (frames already late). pub skipped: u64, + /// Frames decoded in draft mode (reduced-resolution playback, [`VideoDecoder::set_draft`]). + pub draft: u64, } impl GopStats { @@ -53,6 +55,7 @@ static DECODED: AtomicU64 = AtomicU64::new(0); static EVICTED: AtomicU64 = AtomicU64::new(0); static DECODE_NS: AtomicU64 = AtomicU64::new(0); static SKIPPED: AtomicU64 = AtomicU64::new(0); +static DRAFT: AtomicU64 = AtomicU64::new(0); /// Run a decoder call, adding its wall time to the process-wide counter. fn timed(f: impl FnOnce() -> R) -> R { @@ -72,6 +75,7 @@ pub fn gop_stats() -> GopStats { evicted: EVICTED.load(Ordering::Relaxed), decode_ns: DECODE_NS.load(Ordering::Relaxed), skipped: SKIPPED.load(Ordering::Relaxed), + draft: DRAFT.load(Ordering::Relaxed), } } @@ -86,6 +90,7 @@ impl std::ops::Sub for GopStats { evicted: self.evicted - o.evicted, decode_ns: self.decode_ns - o.decode_ns, skipped: self.skipped - o.skipped, + draft: self.draft - o.draft, } } } @@ -116,9 +121,23 @@ struct State { out_max: i64, /// Decoded frames by presentation pts (bounded). frames: BTreeMap>, + /// The cached frames that were decoded in draft mode: served to draft requests only. + drafts: std::collections::BTreeSet, bytes: usize, } +impl State { + /// The cached frame at `pts`, unless it is a draft frame and the request wants the exact one. + fn cached(&self, pts: i64, draft_ok: bool) -> Option<&Arc> { + self.frames.get(&pts).filter(|_| draft_ok || !self.drafts.contains(&pts)) + } + + /// The nearest usable frame at or before `pts` (robust to decoder pts quirks). + fn at_or_before(&self, pts: i64, draft_ok: bool) -> Option> { + self.cached(pts, draft_ok).or_else(|| self.frames.range(..=pts).rev().find(|(p, _)| draft_ok || !self.drafts.contains(p)).map(|(_, f)| f)).cloned() + } +} + thread_local! { /// Caches whose lock this thread holds while decoding (see [`GopCache::frame`]). static DECODING: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; @@ -177,6 +196,7 @@ impl GopCache { start: 0, out_max: i64::MIN, frames: BTreeMap::new(), + drafts: Default::default(), bytes: 0, }), explicit_color, @@ -193,8 +213,14 @@ impl GopCache { } } - fn store(&self, st: &mut State, pts: i64, mut f: VideoFrame) { + fn store(&self, st: &mut State, pts: i64, mut f: VideoFrame, draft: bool) { self.apply_color(&mut f); + if draft { + st.drafts.insert(pts); + DRAFT.fetch_add(1, Ordering::Relaxed); + } else { + st.drafts.remove(&pts); + } let budget = self.budget.max(MIN_FRAMES * f.byte_size()); st.bytes += f.byte_size(); if let Some(old) = st.frames.insert(pts, Arc::new(f)) { @@ -206,6 +232,7 @@ impl GopCache { let last = *st.frames.keys().next_back().expect("non-empty"); let victim = if pts - first > last - pts { first } else { last }; if let Some(v) = st.frames.remove(&victim) { + st.drafts.remove(&victim); st.bytes -= v.byte_size(); EVICTED.fetch_add(1, Ordering::Relaxed); } @@ -216,7 +243,7 @@ impl GopCache { fn store_output(&self, st: &mut State, out: Vec) { for d in out { st.out_max = st.out_max.max(d.pts); - self.store(st, d.pts, d.frame); + self.store(st, d.pts, d.frame, d.draft); } } @@ -234,6 +261,10 @@ impl GopCache { /// late. Non-reference samples of late frames are left out on the way to the wanted frame /// (no other picture depends on them, so the wanted frame decodes exactly as it would /// otherwise; a later request for a skipped frame re-seeks). + /// + /// Inside [`filmcraft_media::cancel::with_draft`]`(true, …)` the decoder runs in draft mode + /// ([`VideoDecoder::set_draft`]); its draft frames are cached for draft requests only, and an + /// exact request for one re-decodes it. pub fn frame_late(&self, s: &dyn VideoSamples, target: i64, late_before: Option) -> crate::Result> { let n = s.count(); let i = s.sample_at(target.max(0)).or_else(|| (n > 0).then(|| n - 1)).ok_or_else(|| CodecError::Decode("empty track".into()))?; @@ -242,9 +273,10 @@ impl GopCache { if DecodingGuard::active(me) { return self.private_frame(s, i, want_pts, n); } + let draft = filmcraft_media::cancel::draft(); let mut st = self.state.lock().unwrap_or_else(|e| e.into_inner()); let _decoding = DecodingGuard::enter(me); - if let Some(f) = st.frames.get(&want_pts) { + if let Some(f) = st.cached(want_pts, draft) { HITS.fetch_add(1, Ordering::Relaxed); return Ok(f.clone()); } @@ -293,6 +325,7 @@ impl GopCache { } let limit = (i.max(st.next) + 64).min(n); let late = late_before.unwrap_or(i64::MIN).min(want_pts); + st.decoder.as_mut().expect("decoder").set_draft(draft); while st.next < limit { if filmcraft_media::cancel::cancelled() { // The decoder state stays consistent (`next`, `out_max`): a later request continues. @@ -309,21 +342,17 @@ impl GopCache { st.next += 1; DECODED.fetch_add(1, Ordering::Relaxed); self.store_output(&mut st, out); - if st.frames.contains_key(&want_pts) { + if st.cached(want_pts, draft).is_some() { break; } } - if !st.frames.contains_key(&want_pts) { + if st.cached(want_pts, draft).is_none() { let out = timed(|| st.decoder.as_mut().expect("decoder").flush()); self.store_output(&mut st, out); st.next = usize::MAX; } // nearest decoded frame at or before the wanted pts (robust to decoder pts quirks) - st.frames - .get(&want_pts) - .cloned() - .or_else(|| st.frames.range(..=want_pts).next_back().map(|(_, f)| f.clone())) - .ok_or_else(|| CodecError::Decode("frame not produced".into())) + st.at_or_before(want_pts, draft).ok_or_else(|| CodecError::Decode("frame not produced".into())) } /// A nested request (see [`Self::frame`]): decode from the sync sample with a fresh decoder, @@ -407,13 +436,9 @@ impl GopCache { st.spare.push(dec); } for d in res? { - self.store(&mut st, d.pts, d.frame); + self.store(&mut st, d.pts, d.frame, d.draft); } - st.frames - .get(&want_pts) - .cloned() - .or_else(|| st.frames.range(..=want_pts).next_back().map(|(_, f)| f.clone())) - .ok_or_else(|| CodecError::Decode("frame not produced".into())) + st.at_or_before(want_pts, true).ok_or_else(|| CodecError::Decode("frame not produced".into())) } } @@ -438,20 +463,24 @@ mod tests { struct Dec { delay: usize, intra: bool, - held: VecDeque, + /// (pts, decoded as a draft picture) + held: VecDeque<(i64, bool)>, + draft: bool, resets: Arc, decodes: Arc, } - fn picture(pts: i64) -> DecodedFrame { + /// A picture whose pixel encodes its index (and, in blue, whether it is a draft picture). + fn picture((pts, draft): (i64, bool)) -> DecodedFrame { let i = (pts / 1000) as u32; - DecodedFrame { pts, frame: VideoFrame::rgba8(1, 1, vec![i as u8, (i >> 8) as u8, 0, 255]) } + DecodedFrame { pts, frame: VideoFrame::rgba8(1, 1, vec![i as u8, (i >> 8) as u8, draft as u8, 255]), draft } } impl VideoDecoder for Dec { - fn decode(&mut self, _sample: &[u8], pts: i64) -> crate::Result> { + fn decode(&mut self, sample: &[u8], pts: i64) -> crate::Result> { self.decodes.fetch_add(1, Ordering::Relaxed); - self.held.push_back(pts); + // like H.264 draft mode: only non-reference (odd) pictures are approximate + self.held.push_back((pts, self.draft && sample[0] % 2 == 1)); let mut out = Vec::new(); while self.held.len() > self.delay { out.push(picture(self.held.pop_front().expect("held"))); @@ -465,6 +494,9 @@ mod tests { self.resets.fetch_add(1, Ordering::Relaxed); self.held.clear(); } + fn set_draft(&mut self, on: bool) { + self.draft = on; + } fn name(&self) -> &str { "test" } @@ -495,7 +527,14 @@ mod tests { Ok(vec![i as u8]) } fn make_decoder(&self) -> crate::Result> { - Ok(Box::new(Dec { delay: self.delay, intra: self.intra, held: VecDeque::new(), resets: self.resets.clone(), decodes: self.decodes.clone() })) + Ok(Box::new(Dec { + delay: self.delay, + intra: self.intra, + held: VecDeque::new(), + draft: false, + resets: self.resets.clone(), + decodes: self.decodes.clone(), + })) } } @@ -592,6 +631,43 @@ mod tests { assert_eq!(s3.resets.load(Ordering::Relaxed), 1); } + fn is_draft(f: &VideoFrame) -> bool { + match &f.data { + filmcraft_frame::PixelData::Rgba8(d) => d[2] == 1, + _ => unreachable!(), + } + } + + #[test] + fn draft_frames_are_served_to_draft_requests_only() { + let s = samples(300, 250, 3, false); + let c = GopCache::new(None); + let before = gop_stats().draft; + // reduced-resolution playback: odd (non-reference) pictures come out as drafts + for i in 0..12usize { + let f = filmcraft_media::cancel::with_draft(true, || c.frame(&s, i as i64 * 1000)).expect("frame"); + assert_eq!(index_of(&f), i); + assert_eq!(is_draft(&f), i % 2 == 1, "frame {i}"); + } + assert!(gop_stats().draft - before >= 6); + assert_eq!(s.resets.load(Ordering::Relaxed), 1); + // paused / export (no draft hint): an even frame is exact and cached + let f = c.frame(&s, 4_000).expect("frame"); + assert!(!is_draft(&f) && index_of(&f) == 4); + assert_eq!(s.resets.load(Ordering::Relaxed), 1, "exact frames are served from the cache"); + // a draft frame is decoded again, exactly + let f = c.frame(&s, 5_000).expect("frame"); + assert!(!is_draft(&f) && index_of(&f) == 5); + assert_eq!(s.resets.load(Ordering::Relaxed), 2, "re-decoded from the sync sample"); + // and a draft request takes the exact frame now cached + let f = filmcraft_media::cancel::with_draft(true, || c.frame(&s, 5_000)).expect("frame"); + assert!(!is_draft(&f)); + // without the hint nothing is ever a draft + let s2 = samples(300, 250, 3, false); + let c2 = GopCache::new(None); + assert!((0..12).all(|i| !is_draft(&c2.frame(&s2, i * 1000).expect("frame")))); + } + #[test] fn cancelled_request_stops_decoding_and_keeps_state() { let s = samples(300, 250, 0, false); diff --git a/crates/codecs/src/video.rs b/crates/codecs/src/video.rs index be39c6ad..20cfe49b 100644 --- a/crates/codecs/src/video.rs +++ b/crates/codecs/src/video.rs @@ -9,6 +9,9 @@ use crate::{CodecError, Result}; pub struct DecodedFrame { pub pts: i64, pub frame: VideoFrame, + /// Decoded in draft mode ([`VideoDecoder::set_draft`]): approximate, for reduced-resolution + /// playback only. + pub draft: bool, } /// A stateful video decoder. Samples are fed in decode order; pictures come out in @@ -37,6 +40,11 @@ pub trait VideoDecoder: Send { fn is_disposable(&self, _sample: &[u8]) -> bool { false } + /// Draft decoding for reduced-resolution playback (off by default; see + /// [`filmcraft_media::cancel::with_draft`]): samples fed from now on may be decoded with + /// shortcuts that only change pictures nothing references, which come out flagged + /// [`DecodedFrame::draft`]. Decoders without such a mode ignore it. + fn set_draft(&mut self, _on: bool) {} } /// NAL unit headers of a length-prefixed (avcC / hvcC) sample: the first two bytes of each unit. @@ -132,7 +140,7 @@ impl VideoDecoder for MjpegDecoder { let img = image::load_from_memory_with_format(sample, image::ImageFormat::Jpeg).map_err(|e| CodecError::Decode(e.to_string()))?; let rgba = img.to_rgba8(); let (w, h) = rgba.dimensions(); - Ok(vec![DecodedFrame { pts, frame: VideoFrame::rgba8(w, h, rgba.into_raw()) }]) + Ok(vec![DecodedFrame { pts, frame: VideoFrame::rgba8(w, h, rgba.into_raw()), draft: false }]) } fn flush(&mut self) -> Vec { Vec::new() @@ -155,20 +163,22 @@ pub struct H264Decoder { avcc: Vec, dec: filmcraft_h264::Decoder, length_size: usize, + draft: bool, } impl H264Decoder { pub fn new(avcc: Vec) -> Result { let dec = filmcraft_h264::Decoder::from_avcc(&avcc).map_err(|e| CodecError::Decode(e.to_string()))?; let length_size = avcc.get(4).map_or(4, |b| (b & 3) as usize + 1); - Ok(Self { avcc, dec, length_size }) + Ok(Self { avcc, dec, length_size, draft: false }) } /// A decoder for Annex B byte-stream samples (start codes, in-band parameter sets: MXF, TS). pub fn annexb() -> Self { - Self { avcc: Vec::new(), dec: filmcraft_h264::Decoder::new(), length_size: 0 } + Self { avcc: Vec::new(), dec: filmcraft_h264::Decoder::new(), length_size: 0, draft: false } } fn convert(p: filmcraft_h264::Picture) -> DecodedFrame { use std::sync::Arc; + let draft = p.draft; let (w, h) = (p.width as usize, p.height as usize); let (cw, ch) = (p.chroma_width as usize, p.chroma_height as usize); let y = tight_plane(p.y, p.y_stride, w, h); @@ -193,7 +203,7 @@ impl H264Decoder { par, pts: filmcraft_time::Tick::ZERO, }; - DecodedFrame { pts: p.pts, frame } + DecodedFrame { pts: p.pts, frame, draft } } } @@ -211,10 +221,15 @@ impl VideoDecoder for H264Decoder { } else if let Ok(d) = filmcraft_h264::Decoder::from_avcc(&self.avcc) { self.dec = d; } + self.dec.set_draft(self.draft); } fn name(&self) -> &str { "FilmCraft H.264" } + fn set_draft(&mut self, on: bool) { + self.draft = on; + self.dec.set_draft(on); + } fn is_disposable(&self, sample: &[u8]) -> bool { if self.length_size == 0 { return h264_disposable_annexb(sample); @@ -227,6 +242,11 @@ impl VideoDecoder for H264Decoder { } } +/// Worker threads each H.264 decoder uses (frame threading; 1 on wasm). +pub fn h264_threads() -> usize { + filmcraft_h264::default_threads() +} + pub fn h264_factory(e: &SampleEntry) -> Option>> { match &e.codec { CodecConfig::Avc(a) => Some(H264Decoder::new(a.to_bytes()).map(|d| Box::new(d) as Box)), @@ -287,7 +307,7 @@ impl HevcDecoder { } let par = if p.sar.0 > 0 && p.sar.1 > 0 { (p.sar.0 as u32, p.sar.1 as u32) } else { (1, 1) }; let frame = VideoFrame { width: p.width, height: p.height, data, color, par, pts: filmcraft_time::Tick::ZERO }; - DecodedFrame { pts: p.pts, frame } + DecodedFrame { pts: p.pts, frame, draft: false } } } @@ -388,7 +408,7 @@ impl Vp9Decoder { } let mut frame = VideoFrame::rgba8(p.width, p.height, rgba); frame.color = filmcraft_color::ColorInfo { range: Range::Full, transfer: filmcraft_color::Transfer::Srgb, ..color }; - return DecodedFrame { pts, frame }; + return DecodedFrame { pts, frame, draft: false }; } // 4:4:0 has no frame format of its own: chroma rows are repeated to 4:4:4. let (chroma, rows_440) = match (p.subsampling_x, p.subsampling_y) { @@ -425,7 +445,7 @@ impl Vp9Decoder { // render_size (the intended display size) is not applied: the container's display // dimensions / pixel aspect describe the presentation. let par = (1, 1); - DecodedFrame { pts, frame: VideoFrame { width: p.width, height: p.height, data, color, par, pts: filmcraft_time::Tick::ZERO } } + DecodedFrame { pts, frame: VideoFrame { width: p.width, height: p.height, data, color, par, pts: filmcraft_time::Tick::ZERO }, draft: false } } } @@ -509,7 +529,7 @@ impl Av1Decoder { } else { filmcraft_frame::PixelData::Yuv16 { planes: [Arc::new(y), Arc::new(u), Arc::new(v)], chroma, bits: p.bit_depth as u32, alpha: None } }; - DecodedFrame { pts, frame: VideoFrame { width: p.width, height: p.height, data, color, par: (1, 1), pts: filmcraft_time::Tick::ZERO } } + DecodedFrame { pts, frame: VideoFrame { width: p.width, height: p.height, data, color, par: (1, 1), pts: filmcraft_time::Tick::ZERO }, draft: false } } } @@ -591,7 +611,7 @@ impl VideoDecoder for ProResDecoder { par: (1, 1), pts: filmcraft_time::Tick::ZERO, }; - Ok(vec![DecodedFrame { pts, frame }]) + Ok(vec![DecodedFrame { pts, frame, draft: false }]) } fn flush(&mut self) -> Vec { Vec::new() @@ -663,7 +683,7 @@ pub fn dnx_to_video_frame(f: filmcraft_dnx::Frame) -> VideoFrame { impl VideoDecoder for DnxDecoder { fn decode(&mut self, sample: &[u8], pts: i64) -> Result> { let f = filmcraft_dnx::decode_frame(sample).map_err(|e| CodecError::Decode(e.to_string()))?; - Ok(vec![DecodedFrame { pts, frame: dnx_to_video_frame(f) }]) + Ok(vec![DecodedFrame { pts, frame: dnx_to_video_frame(f), draft: false }]) } fn flush(&mut self) -> Vec { Vec::new() diff --git a/crates/codecs/tests/h264_draft.rs b/crates/codecs/tests/h264_draft.rs new file mode 100644 index 00000000..81e8c3bb --- /dev/null +++ b/crates/codecs/tests/h264_draft.rs @@ -0,0 +1,68 @@ +//! H.264 draft decoding (opt-in, reduced-resolution playback) through the media stack: it is off +//! by default, its approximate frames only ever reach draft requests, and decoding for an export +//! after draft playback (same source, same GOP cache) still matches ffmpeg bit for bit. + +mod common; + +use common::*; +use filmcraft_media::FrameRequest; +use filmcraft_media::cancel::with_draft; + +#[test] +fn draft_playback_never_reaches_exports() { + let ff = filmcraft_testkit::require_ffmpeg!(); + // B-pyramid: references and non-reference B pictures (the ones draft mode leaves unfiltered) + let Some(file) = fixture( + &ff, + "draft_h264.mp4", + &[ + "-f", + "lavfi", + "-i", + "testsrc2=s=320x240:r=24:d=2,noise=alls=10:allf=t", + "-c:v", + "libx264", + "-preset", + "fast", + "-x264-params", + "bframes=3:b-pyramid=normal:keyint=48", + "-pix_fmt", + "yuv420p", + ], + ) else { + return; + }; + let raw = ffmpeg_frames(&ff, &file, "yuv420p"); + let (w, h) = (320usize, 240usize); + let fsize = w * h * 3 / 2; + let n = raw.len() / fsize; + assert!(n >= 40, "{n} frames"); + let reference = |i: usize| raw_planes(&raw[i * fsize..(i + 1) * fsize], w, h, w / 2, h / 2, 1); + let src = filmcraft_codecs::open_bytes("draft_h264.mp4", bytes(&file)).expect("open"); + let rate = src.info().frame_rate(); + + // Default: nothing is a draft, every frame exact. + let g0 = filmcraft_codecs::gop_stats(); + for i in 0..n { + let f = src.video_frame(FrameRequest { time: rate.tick_of(i as i64), scale: 0.5 }).expect("frame"); + assert_eq!(max_diff(&planes(&f), &reference(i)), 0, "frame {i} without draft mode"); + } + assert_eq!(filmcraft_codecs::gop_stats().draft, g0.draft, "draft decoding is off by default"); + + // Draft playback at 1/2 resolution on a fresh source: non-reference pictures are drafts. + let src = filmcraft_codecs::open_bytes("draft_h264.mp4", bytes(&file)).expect("open"); + let g1 = filmcraft_codecs::gop_stats(); + let mut approximate = 0; + for i in 0..n { + let f = with_draft(true, || src.video_frame(FrameRequest { time: rate.tick_of(i as i64), scale: 0.5 })).expect("frame"); + approximate += (max_diff(&planes(&f), &reference(i)) > 0) as usize; + } + let drafts = filmcraft_codecs::gop_stats().draft - g1.draft; + assert!(drafts > 0 && approximate > 0 && approximate as u64 <= drafts, "{drafts} draft frames, {approximate} differ"); + + // Then an export from the same source (no draft hint): every frame is exact again. + for i in 0..n { + let f = src.video_frame(FrameRequest::full(rate.tick_of(i as i64))).expect("frame"); + assert_eq!(max_diff(&planes(&f), &reference(i)), 0, "export frame {i} after draft playback"); + } +} diff --git a/crates/engine/src/autosave.rs b/crates/engine/src/autosave.rs index 211bf7b9..3e870b36 100644 --- a/crates/engine/src/autosave.rs +++ b/crates/engine/src/autosave.rs @@ -78,6 +78,10 @@ pub struct PlaybackPrefs { pub pause_encoder_queue: bool, pub enable_transmit: bool, pub disable_video_in_background: bool, + /// Draft decoding while playing at 1/2 resolution or lower: H.264 non-reference pictures + /// skip deblocking (faster; those frames are approximate). Never used for exports, renders or + /// a paused frame. Off by default. + pub draft_decode: bool, } impl Default for PlaybackPrefs { @@ -89,6 +93,7 @@ impl Default for PlaybackPrefs { pause_encoder_queue: true, enable_transmit: false, disable_video_in_background: true, + draft_decode: false, } } } diff --git a/crates/engine/src/perf.rs b/crates/engine/src/perf.rs index de656bc3..c9eafbf9 100644 --- a/crates/engine/src/perf.rs +++ b/crates/engine/src/perf.rs @@ -17,6 +17,8 @@ pub fn decode_json() -> Value { "seeks": g.seeks, "samplesDecoded": g.decoded, "samplesSkipped": g.skipped, + "draftFrames": g.draft, + "h264Threads": filmcraft_codecs::video::h264_threads(), "evicted": g.evicted, "decodeMs": g.decode_ns as f64 / 1e6, "decodeMsPerSample": g.decode_ms_per_sample(), @@ -46,7 +48,7 @@ mod tests { s.execute("file.openDemoProject", json!({})).unwrap(); let undo = s.history.undo.len(); let v = s.execute("perf.stats", json!({})).unwrap(); - for k in ["requests", "cacheHitRate", "seeks", "samplesDecoded", "samplesSkipped", "decodeMs", "decodeMsPerSample"] { + for k in ["requests", "cacheHitRate", "seeks", "samplesDecoded", "samplesSkipped", "draftFrames", "h264Threads", "decodeMs", "decodeMsPerSample"] { assert!(v["decode"][k].is_number(), "decode.{k} in {v}"); } assert!(v["media"]["openSources"].is_number()); diff --git a/crates/engine/src/settings.rs b/crates/engine/src/settings.rs index 4f6d0eb3..57586d77 100644 --- a/crates/engine/src/settings.rs +++ b/crates/engine/src/settings.rs @@ -951,6 +951,7 @@ static CATEGORIES: &[Category] = &[ b("playback.pauseEncoderQueue", "Pause Media Encoder queue during playback", false), b("playback.enableTransmit", "Enable Mercury Transmit", false), b("playback.disableVideoInBackground", "Disable video output when in the background", false), + b("playback.draftDecode", "Draft decoding at reduced playback resolution (H.264: faster, some frames less filtered)", true), ], }, Category { diff --git a/crates/frame/src/lib.rs b/crates/frame/src/lib.rs index f7bfbe30..6a667048 100644 --- a/crates/frame/src/lib.rs +++ b/crates/frame/src/lib.rs @@ -98,6 +98,51 @@ impl VideoFrame { } } + /// A planar Y'CbCr frame reduced by `n` (2, 4, 8…) in each direction: every sample (luma, + /// chroma at its own resolution, alpha) is the rounded mean of an `n`×`n` block, chroma format + /// and bit depth unchanged. Reduced-resolution playback hands the GPU this instead of the full + /// picture (a quarter / sixteenth of the upload). None for RGBA frames or `n` < 2. + pub fn box_decimated(&self, n: usize) -> Option { + if n < 2 { + return None; + } + let (w, h) = (self.width as usize, self.height as usize); + let (ow, oh) = ((w / n).max(1), (h / n).max(1)); + let data = match &self.data { + PixelData::Yuv8 { planes, chroma, alpha } => { + let (sx, sy) = chroma.shifts(); + let (cw, ch) = (w.div_ceil(1 << sx), h.div_ceil(1 << sy)); + let (ocw, och) = (ow.div_ceil(1 << sx), oh.div_ceil(1 << sy)); + PixelData::Yuv8 { + planes: [ + Arc::new(box_plane(&planes[0], w, h, ow, oh, n)), + Arc::new(box_plane(&planes[1], cw, ch, ocw, och, n)), + Arc::new(box_plane(&planes[2], cw, ch, ocw, och, n)), + ], + chroma: *chroma, + alpha: alpha.as_ref().map(|a| Arc::new(box_plane(a, w, h, ow, oh, n))), + } + } + PixelData::Yuv16 { planes, chroma, bits, alpha } => { + let (sx, sy) = chroma.shifts(); + let (cw, ch) = (w.div_ceil(1 << sx), h.div_ceil(1 << sy)); + let (ocw, och) = (ow.div_ceil(1 << sx), oh.div_ceil(1 << sy)); + PixelData::Yuv16 { + planes: [ + Arc::new(box_plane(&planes[0], w, h, ow, oh, n)), + Arc::new(box_plane(&planes[1], cw, ch, ocw, och, n)), + Arc::new(box_plane(&planes[2], cw, ch, ocw, och, n)), + ], + chroma: *chroma, + bits: *bits, + alpha: alpha.as_ref().map(|a| Arc::new(box_plane(a, w, h, ow, oh, n))), + } + } + PixelData::Rgba8(_) | PixelData::RgbaF32(_) => return None, + }; + Some(VideoFrame { width: ow as u32, height: oh as u32, data, color: self.color, par: self.par, pts: self.pts }) + } + /// Convert to premultiplied linear RGBA f32 (the compositor's working format). pub fn to_linear_f32(&self) -> Vec { self.to_linear_f32_decimated(1).2 @@ -376,8 +421,59 @@ pub fn default_matrix(width: u32, height: u32) -> Matrix { if width <= 1024 && height <= 576 { Matrix::Bt601 } else { Matrix::Bt709 } } +/// `n`×`n` box mean of a `w`×`h` plane into `ow`×`oh` (blocks clamped at the right / bottom). +fn box_plane + TryFrom + Send + Sync + Default>(src: &[T], w: usize, h: usize, ow: usize, oh: usize, n: usize) -> Vec { + let mut out = vec![T::default(); ow * oh]; + if w == 0 || h == 0 || src.len() < w * h { + return out; + } + out.par_chunks_mut(ow).enumerate().for_each(|(oy, row)| { + let mut acc = vec![0u32; ow]; + let mut cnt = vec![0u32; ow]; + for y in (oy * n..(oy + 1) * n).filter(|&y| y < h) { + let line = &src[y * w..y * w + w]; + for ((a, c), chunk) in acc.iter_mut().zip(cnt.iter_mut()).zip(line.chunks(n)) { + *a += chunk.iter().map(|&v| v.into()).sum::(); + *c += chunk.len() as u32; + } + } + for ((o, a), c) in row.iter_mut().zip(&acc).zip(&cnt) { + let c = (*c).max(1); + *o = T::try_from((a + c / 2) / c).unwrap_or_default(); + } + }); + out +} + #[cfg(test)] mod tests { + + #[test] + fn box_decimation_averages_blocks_per_plane() { + // 6x4 4:2:0: luma ramp, chroma 3x2 + let y: Vec = (0..24).map(|i| (i * 10) as u8).collect(); + let u = vec![10, 20, 30, 40, 50, 60]; + let v = vec![200; 6]; + let f = VideoFrame { + width: 6, + height: 4, + data: PixelData::Yuv8 { planes: [Arc::new(y), Arc::new(u), Arc::new(v)], chroma: Chroma::C420, alpha: None }, + color: ColorInfo::REC709, + par: (1, 1), + pts: Tick::ZERO, + }; + let d = f.box_decimated(2).expect("yuv"); + assert_eq!((d.width, d.height), (3, 2)); + let PixelData::Yuv8 { planes, .. } = &d.data else { panic!() }; + // luma block (0,0): 0, 10, 60, 70 -> 35 + assert_eq!(planes[0][0], 35); + assert_eq!(planes[0].len(), 6); + // chroma 3x2 -> 2x1: blocks {10,20,40,50} = 30 and the clamped edge block {30,60} = 45 + assert_eq!(&planes[1][..], &[30, 45]); + assert_eq!(&planes[2][..], &[200, 200]); + assert!(f.box_decimated(1).is_none()); + assert!(VideoFrame::rgba8(2, 2, vec![0; 16]).box_decimated(2).is_none()); + } use super::*; #[test] diff --git a/crates/media/src/cancel.rs b/crates/media/src/cancel.rs index b6622dad..7848a845 100644 --- a/crates/media/src/cancel.rs +++ b/crates/media/src/cancel.rs @@ -9,6 +9,10 @@ //! The same way, [`with_catch_up`] tells sources which frames before the requested one are late //! (playback has passed them): a decoder that has to decode forward to the frame may skip //! pictures that only produce late frames and that nothing else references. +//! +//! [`with_draft`] marks reduced-resolution playback (opt-in): decoders may take spec-safe +//! shortcuts that only change pictures nothing references (H.264: no deblocking of non-reference +//! pictures). Never set for exports, renders or a frame shown while paused. use std::cell::{Cell, RefCell}; use std::sync::Arc; @@ -19,6 +23,22 @@ use filmcraft_time::Tick; thread_local! { static CURRENT: RefCell>> = const { RefCell::new(None) }; static CATCH_UP: Cell> = const { Cell::new(None) }; + static DRAFT: Cell = const { Cell::new(false) }; +} + +/// Run `f` with the draft-decoding hint `on` (restoring the previous one after). Frames decoded +/// in draft mode are only handed to requests made in draft mode. +pub fn with_draft(on: bool, f: impl FnOnce() -> R) -> R { + let prev = DRAFT.with(|c| c.replace(on)); + let r = f(); + DRAFT.with(|c| c.set(prev)); + r +} + +/// Whether the work running on this thread accepts draft-quality decoding (off unless inside +/// [`with_draft`]`(true, …)`). +pub fn draft() -> bool { + DRAFT.with(Cell::get) } /// Run `f` with the catch-up hint `margin` (restoring the previous one after): frames shown more @@ -66,6 +86,17 @@ mod tests { assert!(!cancelled()); } + #[test] + fn draft_is_off_by_default_and_scoped() { + assert!(!draft()); + with_draft(true, || { + assert!(draft()); + with_draft(false, || assert!(!draft())); + assert!(draft()); + }); + assert!(!draft()); + } + #[test] fn catch_up_is_scoped_to_the_closure() { assert_eq!(catch_up(), None); diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 6f117a69..5c6a35e2 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -441,7 +441,7 @@ pub fn interpolation_blend(item: &TrackItem, t: Tick, src_rate: filmcraft_time:: } /// Largest power-of-two box decimation that keeps at least `target_w` pixels of width. -fn decimation(have_w: f32, target_w: f32) -> usize { +pub(crate) fn decimation(have_w: f32, target_w: f32) -> usize { let mut n = 1usize; while n < 16 && have_w / (n as f32 * 2.0) >= target_w.max(1.0) { n *= 2; diff --git a/crates/render/src/plan.rs b/crates/render/src/plan.rs index 76e74ab8..9f5202dd 100644 --- a/crates/render/src/plan.rs +++ b/crates/render/src/plan.rs @@ -187,6 +187,15 @@ fn push_item( // log / HDR / wide-gamut media is converted on the CPU (below), and so are blended // in-between frames (Frame Blending / Optical Flow on speed-changed clips) if !crate::colorman::needs_management(&seq.settings.color, cs, &frame) && crate::interpolation_blend(item, t, src.info().frame_rate()).is_none() { + // Draft playback at reduced resolution: hand over planes box-filtered to the size + // drawn instead of the full picture (a quarter at 1/2, a sixteenth at 1/4 of the + // upload and sampling). The mean is taken over Y'CbCr codes, not linear light as + // the shader's supersampling does, so this stays limited to the opt-in draft mode. + let n = if filmcraft_media::cancel::draft() { crate::decimation(frame.width as f32, size.0 as f32 * want) } else { 1 }; + let frame = match frame.box_decimated(n) { + Some(small) => Arc::new(small), + None => frame, + }; let px_scale = frame.width as f64 / size.0.max(1) as f64; let m = Affine::scale(opts.scale as f64, opts.scale as f64).then_apply(&motion).then_apply(&Affine::scale(1.0 / px_scale, 1.0 / px_scale)); out.push(PlanLayer { frame, matrix: m, opacity: op * extra_opacity }); diff --git a/crates/render/src/tests.rs b/crates/render/src/tests.rs index 36ada7ea..4489447d 100644 --- a/crates/render/src/tests.rs +++ b/crates/render/src/tests.rs @@ -299,3 +299,54 @@ fn offline_slate_is_deterministic_and_marked() { ::image::save_buffer(p, &a, 640, 360, ::image::ExtendedColorType::Rgba8).unwrap(); } } + +/// A 320×180 4:2:0 source with smooth gradients (what decoders hand the renderer). +struct YuvSource(GeneratorSource); + +impl MediaSource for YuvSource { + fn info(&self) -> &filmcraft_media::MediaInfo { + self.0.info() + } + fn video_frame(&self, _req: FrameRequest) -> filmcraft_media::Result> { + let (w, h) = (320usize, 180usize); + let y = (0..w * h).map(|i| (16 + (i % w) * 200 / w + (i / w) * 20 / h) as u8).collect(); + let u = (0..w * h / 4).map(|i| (90 + (i % (w / 2)) * 60 / (w / 2)) as u8).collect(); + let v = (0..w * h / 4).map(|i| (100 + (i / (w / 2)) * 50 / (h / 2)) as u8).collect(); + Ok(Arc::new(filmcraft_frame::VideoFrame { + width: w as u32, + height: h as u32, + data: filmcraft_frame::PixelData::Yuv8 { planes: [Arc::new(y), Arc::new(u), Arc::new(v)], chroma: filmcraft_frame::Chroma::C420, alpha: None }, + color: filmcraft_color::ColorInfo::REC709, + par: (1, 1), + pts: Tick::ZERO, + })) + } + fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> filmcraft_media::Result { + self.0.audio(start, frames, sample_rate) + } +} + +#[test] +fn draft_reduced_resolution_plans_carry_decimated_yuv() { + let (mut p, red, _ocean, seq, mut map) = setup(); + let matte = GeneratorSource::new(Generator::ColorMatte { color: [0.0, 0.0, 0.0, 1.0] }, 320, 180, FrameRate::FPS_24, Tick(10 * TICKS_PER_SECOND)); + map.0.insert(red, Arc::new(YuvSource(matte)) as SharedSource); + place(&mut p, seq, 0, red, 0, 48); + let t = FrameRate::FPS_24.tick_of(3); + for draft in [false, true] { + for (scale, small_w) in [(1.0f32, 320u32), (0.5, 160), (0.25, 80)] { + let opts = RenderOptions { scale, ..Default::default() }; + let plan = filmcraft_media::cancel::with_draft(draft, || plan::plan_frame(&p, seq, t, opts, &map)); + let plan::FramePlan::Layers { layers, .. } = &plan else { panic!("GPU-drawable") }; + // exact playback hands over the decoded picture; draft playback planes at the drawn size + assert_eq!(layers[0].frame.width, if draft { small_w } else { 320 }, "scale {scale}, draft {draft}"); + assert!(matches!(layers[0].frame.data, filmcraft_frame::PixelData::Yuv8 { .. })); + let got = plan::execute_cpu(&plan); + let reference = render_sequence(&p, seq, t, opts, &map); + assert_eq!((got.w, got.h), (reference.w, reference.h)); + let max = got.px.iter().zip(&reference.px).map(|(a, b)| (a - b).abs()).fold(0f32, f32::max); + // the Y'CbCr-code mean differs from a linear-light mean by a little on gradients + assert!(max < if draft { 0.03 } else { 1e-5 }, "scale {scale}, draft {draft}: max diff {max}"); + } + } +} diff --git a/crates/ui-egui/examples/bench/main.rs b/crates/ui-egui/examples/bench/main.rs index 597472ea..89dd64d2 100644 --- a/crates/ui-egui/examples/bench/main.rs +++ b/crates/ui-egui/examples/bench/main.rs @@ -258,6 +258,7 @@ pub fn section_markdown(v: &Value) -> String { ("ui_p95_ms", "UI p95 ms"), ("seeks", "seeks"), ("skipped", "skipped"), + ("draft_frames", "draft"), ("load", "load"), ], 1, diff --git a/crates/ui-egui/examples/bench/playback.rs b/crates/ui-egui/examples/bench/playback.rs index e412332a..22710a7b 100644 --- a/crates/ui-egui/examples/bench/playback.rs +++ b/crates/ui-egui/examples/bench/playback.rs @@ -229,6 +229,9 @@ pub struct Bench { pub server: FrameServer, pub refresh_hz: f64, pub seconds: f64, + /// Play with draft decoding (Settings ▸ Playback ▸ Draft decoding), as the monitor does at + /// 1/2 resolution or lower. + pub draft: bool, } impl Bench { @@ -253,7 +256,7 @@ impl Bench { let last_frame = rate.frame_at(q.duration()) - 1; let t_pre = Instant::now(); loop { - let key = FrameKey { target, frame: first, size, revision: s.revision }; + let key = FrameKey { target, frame: first, size, revision: s.revision, draft: self.draft }; self.server.schedule_playback(key, rate, scale, &project, 1.0, true); let exact = display.refresh(&self.server, key); meter.refresh(first, exact); @@ -275,7 +278,7 @@ impl Bench { } frame = rate.frame_at(t); due_at.entry(frame).or_insert(now); - let key = FrameKey { target, frame, size, revision: s.revision }; + let key = FrameKey { target, frame, size, revision: s.revision, draft: self.draft }; self.server.schedule_playback(key, rate, scale, &project, 1.0, false); let exact = display.refresh(&self.server, key); meter.refresh(frame, exact); @@ -368,7 +371,7 @@ impl Bench { rng ^= rng >> 7; rng ^= rng << 17; let frame = (rng % frames as u64) as i64; - let key = FrameKey { target, frame, size, revision: s.revision }; + let key = FrameKey { target, frame, size, revision: s.revision, draft: false }; let start = Instant::now(); rep.due += 1; loop { @@ -454,7 +457,7 @@ impl Bench { }; for _ in 0..seeks { let frame = (next() % frames as u64) as i64; - let key = FrameKey { target, frame, size, revision: s.revision }; + let key = FrameKey { target, frame, size, revision: s.revision, draft: false }; match wait(display, key, Instant::now()) { Some(l) => rep.seek_ms.push(l), None => rep.timeouts += 1, @@ -463,7 +466,7 @@ impl Bench { for _ in 0..drags { let from = (next() % (frames - drag_frames).max(1) as u64) as i64; for f in from..from + drag_frames { - let key = FrameKey { target, frame: f.min(frames - 1), size, revision: s.revision }; + let key = FrameKey { target, frame: f.min(frames - 1), size, revision: s.revision, draft: false }; self.server.request(key, rate.tick_of(key.frame), scale, &project, 0); if display.refresh(&self.server, key) { rep.drag_shown += 1; @@ -471,7 +474,7 @@ impl Bench { rep.drag_steps += 1; std::thread::sleep(period); } - let key = FrameKey { target, frame: (from + drag_frames).min(frames - 1), size, revision: s.revision }; + let key = FrameKey { target, frame: (from + drag_frames).min(frames - 1), size, revision: s.revision, draft: false }; match wait(display, key, Instant::now()) { Some(l) => rep.settle_ms.push(l), None => rep.timeouts += 1, @@ -720,7 +723,7 @@ pub fn cli_main() { // A fresh frame server per run so caches start cold, as after opening a project. let server = FrameServer::new(s.media.clone(), s.services.clone(), s.previews.clone(), args.workers); server.set_profiling(true); - let bench = Bench { server, refresh_hz: args.refresh, seconds: args.seconds }; + let bench = Bench { server, refresh_hz: args.refresh, seconds: args.seconds, draft: false }; let mut display = Display::new(args.gpu); if args.gpu && display.gpu.is_none() { eprintln!("no GPU adapter; using the CPU path"); diff --git a/crates/ui-egui/examples/bench/sections.rs b/crates/ui-egui/examples/bench/sections.rs index d25c2057..95c47f45 100644 --- a/crates/ui-egui/examples/bench/sections.rs +++ b/crates/ui-egui/examples/bench/sections.rs @@ -149,23 +149,27 @@ pub fn seek(o: &Opts) -> Vec { fn new_bench(s: &Session, seconds: f64) -> Bench { let server = FrameServer::new(s.media.clone(), s.services.clone(), s.previews.clone(), FrameServer::default_workers()); server.set_profiling(true); - Bench { server, refresh_hz: 60.0, seconds } + Bench { server, refresh_hz: 60.0, seconds, draft: false } } /// Program-monitor playback through the real scheduler (see `playback.rs`), frames shown vs /// dropped over N seconds. pub fn playback(o: &Opts) -> Vec { let seconds = if o.quick { 4.0 } else { 8.0 }; - let cases: &[(&str, &str, f32)] = &[ - ("h264-1080", "full", 1.0), - ("stack3", "full", 1.0), - ("h264-2160", "full", 1.0), - ("h264-2160", "half", 0.5), - ("hevc-2160", "full", 1.0), - ("hevc-2160", "half", 0.5), + // (scenario, resolution, scale, draft decoding) + let cases: &[(&str, &str, f32, bool)] = &[ + ("h264-1080", "full", 1.0, false), + ("stack3", "full", 1.0, false), + ("h264-2160", "full", 1.0, false), + ("h264-2160", "half", 0.5, false), + ("h264-2160", "half draft", 0.5, true), + ("h264-2160", "quarter", 0.25, false), + ("h264-2160", "quarter draft", 0.25, true), + ("hevc-2160", "full", 1.0, false), + ("hevc-2160", "half", 0.5, false), ]; let mut rows = Vec::new(); - for &(scenario, res, scale) in cases.iter().filter(|c| o.wants(&format!("{} {}", c.0, c.1))) { + for &(scenario, res, scale, draft) in cases.iter().filter(|c| o.wants(&format!("{} {}", c.0, c.1))) { let Some((s, seq)) = playback::scenario_session(scenario) else { eprintln!("playback {scenario}: skipped (fixtures unavailable)"); continue; @@ -173,7 +177,8 @@ pub fn playback(o: &Opts) -> Vec { let fps = s.project.sequence(seq).map(|q| q.settings.frame_rate.as_f64()).unwrap_or(24.0); for rep in 0..o.repeat { // A fresh frame server per run: caches start cold, as after opening a project. - let bench = new_bench(&s, seconds); + let mut bench = new_bench(&s, seconds); + bench.draft = draft; let mut display = Display::new(o.gpu); let path = if display.gpu_available() { "gpu" } else { "cpu" }; let r = bench.play(&s, seq, &mut display, scale, Tick::ZERO, &format!("{scenario} {res} #{}", rep + 1)); @@ -184,6 +189,7 @@ pub fn playback(o: &Opts) -> Vec { v["ui_p95_ms"] = json!(pct(&r.present, 0.95)); v["seeks"] = json!(r.gop.seeks); v["skipped"] = json!(r.gop.skipped); + v["draft_frames"] = json!(r.gop.draft); v["load"] = json!(r.load); v["cores_needed"] = json!(r.process_cpu / frames * fps / 1000.0); eprintln!("playback {scenario} {res}: {}/{} shown/dropped (load {})", r.shown, r.dropped, r.load); diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index 01810b37..b195cfba 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -40,6 +40,17 @@ pub struct FrameKey { /// Output width in pixels (thumbnails) or scale ×1000 (monitors). pub size: u32, pub revision: u64, + /// Draft decoding (reduced-resolution playback with Settings ▸ Playback ▸ Draft decoding): + /// sources may decode approximate non-reference pictures + /// ([`filmcraft_media::cancel::with_draft`]). Draft frames have their own keys, so a paused + /// monitor, an export or a render never shows one. + pub draft: bool, +} + +/// Whether a monitor frame is requested with draft decoding ([`FrameKey::draft`]): only while +/// playing, at 1/2 resolution or lower, with the preference on. +pub fn draft_playback(playing: bool, scale: f32, enabled: bool) -> bool { + enabled && playing && scale <= 0.5 } pub struct Rgba { @@ -597,7 +608,7 @@ impl FrameServer { /// The most recent cached frame for `target` at or before `frame` within `max_back` frames. pub fn nearest(&self, target: Target, frame: i64, size: u32, revision: u64, max_back: i64) -> Option> { for d in 0..=max_back { - if let Some(v) = self.get(&FrameKey { target, frame: frame - d, size, revision }) { + if let Some(v) = self.get(&FrameKey { target, frame: frame - d, size, revision, draft: false }) { return Some(v); } } @@ -756,23 +767,25 @@ fn run_job(sh: &Shared, job: &Job, pool: &Arc, services: &Arc, services: &Arc, pr mod tests { use super::{PlaybackMeter, playback_plan}; + #[test] + fn draft_decoding_only_for_reduced_resolution_playback() { + use super::draft_playback; + assert!(draft_playback(true, 0.5, true) && draft_playback(true, 0.25, true)); + assert!(!draft_playback(true, 1.0, true), "full resolution is never draft"); + assert!(!draft_playback(false, 0.5, true), "a paused frame is never draft"); + assert!(!draft_playback(true, 0.5, false), "off unless enabled"); + assert!(!filmcraft_engine::autosave::Preferences::default().playback.draft_decode, "off by default"); + } + #[test] fn playback_plan_skips_frames_only_when_workers_cannot_keep_up() { // decode-only / light frames: every frame from the playhead on diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index d603e23c..c240e376 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -625,7 +625,7 @@ impl FilmcraftApp { let rate = pi.frame_rate(); let frame = rate.frame_at(t); let rev = self.item_revision(item); - let key = FrameKey { target: Target::Item(item), frame, size: width, revision: rev }; + let key = FrameKey { target: Target::Item(item), frame, size: width, revision: rev, draft: false }; let name = format!("thumb-{}-{}-{}", item.0, frame, width); if let Some(img) = self.frames.get(&key) { let id = self.texture_for(ctx, &name, key, &img); diff --git a/crates/ui-egui/src/panels/export_mode.rs b/crates/ui-egui/src/panels/export_mode.rs index 5d8061be..47accba5 100644 --- a/crates/ui-egui/src/panels/export_mode.rs +++ b/crates/ui-egui/src/panels/export_mode.rs @@ -820,7 +820,7 @@ fn preview_column(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect, seq_id: ui.painter().rect_filled(pic, 0.0, Color32::BLACK); if app.ui.export.settings.has_video() { let frame = rate.frame_at(app.session.playhead()); - let key = FrameKey { target: Target::Sequence(seq_id), frame, size: 500, revision: app.session.revision }; + let key = FrameKey { target: Target::Sequence(seq_id), frame, size: 500, revision: app.session.revision, draft: false }; let project = app.session.project.clone(); app.frames.request(key, rate.tick_of(frame), 0.5, &project, 0); if let Some(img) = app.frames.get(&key) { diff --git a/crates/ui-egui/src/panels/monitor.rs b/crates/ui-egui/src/panels/monitor.rs index 12e013e0..3d264b40 100644 --- a/crates/ui-egui/src/panels/monitor.rs +++ b/crates/ui-egui/src/panels/monitor.rs @@ -154,7 +154,9 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect, which: Which) (true, Target::Sequence(s)) => Target::SequencePlan(s), (_, t) => t, }; - let key = FrameKey { target, frame, size: size_key, revision: rev }; + // Draft decoding (opt-in): only while playing at 1/2 resolution or lower. + let draft = crate::frames::draft_playback(playing, scale, app.session.prefs.playback.draft_decode); + let key = FrameKey { target, frame, size: size_key, revision: rev, draft }; let project = app.session.project.clone(); if playing { let preroll = app.playback.preroll.is_some(); @@ -190,7 +192,7 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect, which: Which) if let Some(ra) = ref_area { // the reference frame (CPU path) let rf = rate.frame_at(Tick(mv.compare_ref.unwrap_or(time.0))); - let rkey = FrameKey { target: cpu_target, frame: rf, size: size_key, revision: rev }; + let rkey = FrameKey { target: cpu_target, frame: rf, size: size_key, revision: rev, draft: false }; app.frames.request(rkey, rate.tick_of(rf), scale, &project, 1); let rpic = fit(ra, frame_size.0 as f32, frame_size.1 as f32); ui.painter().rect_filled(rpic, 0.0, t.monitor_bg); diff --git a/crates/ui-egui/src/panels/multicam.rs b/crates/ui-egui/src/panels/multicam.rs index 51a0c933..e2f7b652 100644 --- a/crates/ui-egui/src/panels/multicam.rs +++ b/crates/ui-egui/src/panels/multicam.rs @@ -285,7 +285,7 @@ pub fn grid(app: &mut FilmcraftApp, ui: &mut egui::Ui, area: Rect) { let scale = crate::panels::monitor::quantize_scale(app.ui.program.res.scale().min((cell_px / sw).max(1.0 / 32.0))); let mt = Tick(info["sourceTime"].as_i64().unwrap_or(0)); let frame = rate.frame_at(mt); - let key = FrameKey { target: Target::MulticamGrid(src), frame, size: (scale * 1000.0) as u32, revision: grid_revision(app, src) }; + let key = FrameKey { target: Target::MulticamGrid(src), frame, size: (scale * 1000.0) as u32, revision: grid_revision(app, src), draft: false }; let project = app.session.project.clone(); if app.playback.playing { app.frames.schedule_playback(key, rate, scale, &project, app.playback.speed, app.playback.preroll.is_some()); diff --git a/crates/ui-egui/src/panels/reference.rs b/crates/ui-egui/src/panels/reference.rs index 21f46543..e2e2404c 100644 --- a/crates/ui-egui/src/panels/reference.rs +++ b/crates/ui-egui/src/panels/reference.rs @@ -48,7 +48,7 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) { ui.painter().rect_filled(pic, 0.0, t.monitor_bg); let ppp = ctx.pixels_per_point(); let scale = crate::panels::monitor::quantize_scale((pic.width() * ppp / w).clamp(1.0 / 32.0, 1.0)); - let key = FrameKey { target: Target::Sequence(seq_id), frame, size: (scale * 1000.0) as u32, revision: app.session.revision }; + let key = FrameKey { target: Target::Sequence(seq_id), frame, size: (scale * 1000.0) as u32, revision: app.session.revision, draft: false }; let project = app.session.project.clone(); app.frames.request(key, rate.tick_of(frame), scale, &project, 3); let tex = match app.frames.get(&key) { diff --git a/crates/ui-egui/src/panels/scopes.rs b/crates/ui-egui/src/panels/scopes.rs index bbb0e893..c5a81519 100644 --- a/crates/ui-egui/src/panels/scopes.rs +++ b/crates/ui-egui/src/panels/scopes.rs @@ -68,7 +68,7 @@ pub fn frame_signal(app: &mut FilmcraftApp, ctx: &egui::Context, slot: &str, fra let fresh = if pq { filmcraft_engine::scopes::scope_signal(&app.session, rate.tick_of(frame), 0.125, true) } else { - let fkey = FrameKey { target: Target::Sequence(seq_id), frame, size: 250, revision: rev }; + let fkey = FrameKey { target: Target::Sequence(seq_id), frame, size: 250, revision: rev, draft: false }; let project = app.session.project.clone(); app.frames.request(fkey, rate.tick_of(frame), 0.25, &project, prio); app.frames.get(&fkey).map(|img| Signal::from_rgba8(img.w, img.h, &img.px)) diff --git a/crates/ui-egui/src/panels/trim_monitor.rs b/crates/ui-egui/src/panels/trim_monitor.rs index d813a2a5..4347bf6f 100644 --- a/crates/ui-egui/src/panels/trim_monitor.rs +++ b/crates/ui-egui/src/panels/trim_monitor.rs @@ -126,7 +126,7 @@ fn picture(app: &mut FilmcraftApp, ui: &mut egui::Ui, area: Rect, side: &Value, let scale = scale.min(app.ui.program.res.scale()); let size_key = (scale * 1000.0) as u32; let rev = app.item_revision(item); - let key = FrameKey { target: Target::Item(item), frame, size: size_key, revision: rev }; + let key = FrameKey { target: Target::Item(item), frame, size: size_key, revision: rev, draft: false }; let project = app.session.project.clone(); app.frames.request(key, rate.tick_of(frame), scale, &project, 0); let ctx = ui.ctx().clone(); diff --git a/crates/ui-egui/src/perf.rs b/crates/ui-egui/src/perf.rs index 59e0658c..6164faa0 100644 --- a/crates/ui-egui/src/perf.rs +++ b/crates/ui-egui/src/perf.rs @@ -18,6 +18,7 @@ pub fn stats(app: &FilmcraftApp) -> Value { "dropped": dropped, "dropRate": if shown + dropped == 0 { 0.0 } else { dropped as f64 / (shown + dropped) as f64 }, "resolution": app.ui.program.res.label(), + "draftDecode": app.session.prefs.playback.draft_decode, }); let mut frames = app.frames.stats().to_json(); frames["workers"] = json!(app.frames.workers()); From aa3ad1d950022337000efccc61b50d8d32ec9f21 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 23:46:52 -0400 Subject: [PATCH 4/7] M4.9: H.264 inverse transforms reconstruct across columns at once The column pass and the add/clip of idct4/idct8 work on whole rows so they vectorise (same integer steps in the spec's row-then-column order; checked against the per-column formulation on random blocks). Docs: draft decoding in architecture.md and perf.stats fields in control-protocol.md. Co-Authored-By: Claude Opus 5.5 --- crates/h264/src/transform.rs | 102 +++++++++++++++++++++++++++-------- docs/architecture.md | 10 +++- docs/control-protocol.md | 2 +- 3 files changed, 89 insertions(+), 25 deletions(-) diff --git a/crates/h264/src/transform.rs b/crates/h264/src/transform.rs index 0247f186..b7a5ad00 100644 --- a/crates/h264/src/transform.rs +++ b/crates/h264/src/transform.rs @@ -88,28 +88,34 @@ fn clip_u8(v: i32) -> u8 { } /// Inverse 4x4 transform of raster-order `d` and add to `dst` (8.5.12.2, 8.5.14). +/// +/// Rows first, then columns (the spec's order: the `>> 1` steps make it matter). The column +/// pass and the reconstruction run across all columns at once so they vectorise. pub fn idct4_add(d: &[i32; 16], dst: &mut [u8], stride: usize) { - let mut t = [0i32; 16]; - for i in 0..4 { + let mut t = [[0i32; 4]; 4]; + for (i, row) in t.iter_mut().enumerate() { let r = &d[i * 4..i * 4 + 4]; let e = r[0] + r[2]; let f = r[0] - r[2]; let g = (r[1] >> 1) - r[3]; let h = r[1] + (r[3] >> 1); - t[i * 4] = e + h; - t[i * 4 + 1] = f + g; - t[i * 4 + 2] = f - g; - t[i * 4 + 3] = e - h; + *row = [e + h, f + g, f - g, e - h]; } + let mut o = [[0i32; 4]; 4]; for j in 0..4 { - let e = t[j] + t[8 + j]; - let f = t[j] - t[8 + j]; - let g = (t[4 + j] >> 1) - t[12 + j]; - let h = t[4 + j] + (t[12 + j] >> 1); - let col = [e + h, f + g, f - g, e - h]; - for i in 0..4 { - let p = &mut dst[i * stride + j]; - *p = clip_u8(*p as i32 + ((col[i] + 32) >> 6)); + let e = t[0][j] + t[2][j]; + let f = t[0][j] - t[2][j]; + let g = (t[1][j] >> 1) - t[3][j]; + let h = t[1][j] + (t[3][j] >> 1); + o[0][j] = e + h; + o[1][j] = f + g; + o[2][j] = f - g; + o[3][j] = e - h; + } + for (i, row) in o.iter().enumerate() { + let line: &mut [u8; 4] = (&mut dst[i * stride..i * stride + 4]).try_into().unwrap(); + for (p, &v) in line.iter_mut().zip(row) { + *p = clip_u8(*p as i32 + ((v + 32) >> 6)); } } } @@ -146,18 +152,24 @@ fn idct8_1d(d: [i32; 8]) -> [i32; 8] { [b0 + b7, b2 + b5, b4 + b3, b6 + b1, b6 - b1, b4 - b3, b2 - b5, b0 - b7] } -/// Inverse 8x8 transform of raster-order `d` and add to `dst` (8.5.13.2). +/// Inverse 8x8 transform of raster-order `d` and add to `dst` (8.5.13.2): rows, then columns, +/// the column pass and reconstruction across all columns at once (they vectorise). pub fn idct8_add(d: &[i32; 64], dst: &mut [u8], stride: usize) { - let mut t = [0i32; 64]; - for i in 0..8 { - let row: [i32; 8] = d[i * 8..i * 8 + 8].try_into().unwrap(); - t[i * 8..i * 8 + 8].copy_from_slice(&idct8_1d(row)); + let mut t = [[0i32; 8]; 8]; + for (i, row) in t.iter_mut().enumerate() { + *row = idct8_1d(d[i * 8..i * 8 + 8].try_into().unwrap()); } + let mut o = [[0i32; 8]; 8]; for j in 0..8 { - let col = idct8_1d(std::array::from_fn(|i| t[i * 8 + j])); + let col = idct8_1d(std::array::from_fn(|i| t[i][j])); for i in 0..8 { - let p = &mut dst[i * stride + j]; - *p = clip_u8(*p as i32 + ((col[i] + 32) >> 6)); + o[i][j] = col[i]; + } + } + for (i, row) in o.iter().enumerate() { + let line: &mut [u8; 8] = (&mut dst[i * stride..i * stride + 8]).try_into().unwrap(); + for (p, &v) in line.iter_mut().zip(row) { + *p = clip_u8(*p as i32 + ((v + 32) >> 6)); } } } @@ -166,6 +178,52 @@ pub fn idct8_add(d: &[i32; 64], dst: &mut [u8], stride: usize) { mod tests { use super::*; + /// The straightforward per-column formulation (8.5.12.2 / 8.5.13.2) the vectorised + /// transforms must equal. + fn reference_add(d: &[i32], n: usize, dst: &mut [u8], stride: usize) { + let one = |v: &[i32]| -> Vec { + if n == 4 { + let (e, f, g, h) = (v[0] + v[2], v[0] - v[2], (v[1] >> 1) - v[3], v[1] + (v[3] >> 1)); + vec![e + h, f + g, f - g, e - h] + } else { + idct8_1d(v.try_into().unwrap()).to_vec() + } + }; + let rows: Vec> = (0..n).map(|i| one(&d[i * n..i * n + n])).collect(); + for j in 0..n { + let col = one(&(0..n).map(|i| rows[i][j]).collect::>()); + for i in 0..n { + let p = &mut dst[i * stride + j]; + *p = (*p as i32 + ((col[i] + 32) >> 6)).clamp(0, 255) as u8; + } + } + } + + #[test] + fn vectorised_transforms_match_reference() { + let mut rng = 0x1234_5678_9abc_def1u64; + let mut next = move || { + rng ^= rng << 13; + rng ^= rng >> 7; + rng ^= rng << 17; + rng + }; + for iter in 0..5000 { + let range = [64u64, 1024, 8192, 1 << 15][iter % 4]; + let d4: [i32; 16] = std::array::from_fn(|_| (next() % (2 * range + 1)) as i32 - range as i32); + let d8: [i32; 64] = std::array::from_fn(|_| (next() % (2 * range + 1)) as i32 - range as i32); + let base: Vec = (0..24 * 8).map(|_| next() as u8).collect(); + let (mut a, mut b) = (base.clone(), base.clone()); + idct4_add(&d4, &mut a[3..], 24); + reference_add(&d4, 4, &mut b[3..], 24); + assert_eq!(a, b, "4x4 iteration {iter}"); + let (mut a, mut b) = (base.clone(), base); + idct8_add(&d8, &mut a[5..], 24); + reference_add(&d8, 8, &mut b[5..], 24); + assert_eq!(a, b, "8x8 iteration {iter}"); + } + } + #[test] fn idct4_dc_only_matches_shortcut() { let mut d = [0i32; 16]; diff --git a/docs/architecture.md b/docs/architecture.md index fef13f36..97208309 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -242,8 +242,14 @@ file ──► codecs (MP4/MOV, MKV, audio) demux + decode, GOP-aware see playhead's distance, two seconds for a scrubbed-to frame), and the GOP cache skips non-reference pictures of late frames on the way (`VideoDecoder::is_disposable`, H.264 `nal_ref_idc` 0 and HEVC sub-layer non-reference pictures; the wanted frame decodes exactly as - before). `perf.stats` reports the counters. GPU plans carry their texels already - converted for upload (`filmcraft_gpu::prepare`), so the UI thread only copies them. + before). With Settings ▸ Playback ▸ Draft decoding (off by default), frames played at 1/2 + resolution or lower are requested as draft frames (`FrameKey::draft`, + `filmcraft_media::cancel::with_draft`): the H.264 decoder skips deblocking of non-reference + pictures (`VideoDecoder::set_draft`), the GOP cache serves those draft frames to draft + requests only (a paused frame, render or export decodes them again exactly), and draft plans + hand the GPU box-decimated Y'CbCr planes at the drawn size. `perf.stats` reports the counters. + GPU plans carry their texels already converted for upload (`filmcraft_gpu::prepare`), so the UI + thread only copies them. - **Audio clock.** The desktop app passes a cpal output (`apps/filmcraft/src/audio.rs`) to the UI as `AudioOut`. While playing, the samples played by the sound card drive the playhead and video follows. Without an audio device, playback falls back to the wall clock. `PlaybackMeter` counts timeline diff --git a/docs/control-protocol.md b/docs/control-protocol.md index 657f8fc1..0b30f9ea 100644 --- a/docs/control-protocol.md +++ b/docs/control-protocol.md @@ -29,7 +29,7 @@ Methods (handlers in `crates/ui-egui/src/control.rs`): | `ui.playback` | `{action: play|stop|toggle, speed?}` | | | `ui.screenshot` | `{path?, panel?}` | PNG of the window or one panel; fails after 10 s when no frame is presented (window hidden, display asleep) | | `ui.resize`, `ui.focus`, `app.quit` | | `ui.focus` is the only request that activates the app and takes keyboard focus | -| `perf.stats` | – | performance counters (also the command `perf.stats`, so `engine.execute` and MCP `command_run` reach it; headless sessions return the engine part). `decode`: GOP-cache requests / hits / `cacheHitRate`, decoder seeks, samples decoded and skipped while catching up, decoder ms (total and per sample); `playback`: playing, shown / dropped frames and `dropRate` of the current or last play, resolution; `frames`: frame-worker jobs, cancelled, `requestHitRate`, `decodeMs` / `renderMs` (mean, p50, p95 of the last 256 jobs), queue, render-cost estimate, cache use; `ui`: fps and frame ms; `process`: CPU seconds; `media`, `jobs`. Counters are cumulative: diff two readings to measure an interval ([performance.md](performance.md)) | +| `perf.stats` | – | performance counters (also the command `perf.stats`, so `engine.execute` and MCP `command_run` reach it; headless sessions return the engine part). `decode`: GOP-cache requests / hits / `cacheHitRate`, decoder seeks, samples decoded and skipped while catching up, `draftFrames` (decoded in draft mode), `h264Threads` (frame-threading workers per H.264 decoder), decoder ms (total and per sample); `playback`: playing, shown / dropped frames and `dropRate` of the current or last play, resolution, `draftDecode` (Settings ▸ Playback ▸ Draft decoding); `frames`: frame-worker jobs, cancelled, `requestHitRate`, `decodeMs` / `renderMs` (mean, p50, p95 of the last 256 jobs), queue, render-cost estimate, cache use; `ui`: fps and frame ms; `process`: CPU seconds; `media`, `jobs`. Counters are cumulative: diff two readings to measure an interval ([performance.md](performance.md)) | **Focus.** Driving the app never steals the user's keyboard. Started with `--control`, the app opens without activating itself. UI requests and screenshots that need a rendered frame bring the window From 849f26952509e0b244ff7f595243757657727084 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 23:52:31 -0400 Subject: [PATCH 5/7] M4.9: H.264 deblocking skips edges whose alpha or beta is 0 No sample can pass the filterSamplesFlag test (8.7.2.2) at those indices. Co-Authored-By: Claude Opus 5.5 --- crates/h264/src/deblock.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/h264/src/deblock.rs b/crates/h264/src/deblock.rs index 310daf21..8b54cf00 100644 --- a/crates/h264/src/deblock.rs +++ b/crates/h264/src/deblock.rs @@ -367,6 +367,11 @@ pub fn deblock_mb(pic: &mut PicState, addr: usize, mb_w: usize) { } let qpp = if e == 0 { p_qp[dir].map(|p| p.0).unwrap_or(q_qp) } else { q_qp }; let ep = params(qpp, q_qp, bs[dir][e]); + if ep.alpha == 0 || ep.beta == 0 { + // filterSamplesFlag needs |p0 - q0| < alpha and |p1 - p0| < beta (8.7.2.2): no + // sample of the edge changes (low QPs) + continue; + } if dir == 0 { filter_vertical::<16>(&mut pic.planes.y, width, mx * 16 + e * 4, my * 16, &ep, false); } else { @@ -384,6 +389,9 @@ pub fn deblock_mb(pic: &mut PicState, addr: usize, mb_w: usize) { } let qpp = if e == 0 { p_qp[dir].map(|p| p.1[c]).unwrap_or(q_qpc[c]) } else { q_qpc[c] } as i32; let ep = params(qpp, q_qpc[c] as i32, bs[dir][e]); + if ep.alpha == 0 || ep.beta == 0 { + continue; + } let plane = if c == 0 { &mut pic.planes.cb } else { &mut pic.planes.cr }; if dir == 0 { filter_vertical::<8>(plane, cwidth, mx * 8 + ce * 4, my * 8, &ep, true); From 7e991ea95b6f78e2a1f495830808aa194c4cffb8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 00:03:58 -0400 Subject: [PATCH 6/7] M4.9: performance docs, h264 README and roadmap for 4K H.264 playback Interleaved before/after bench (decode, playback incl. 1/4 and draft cases, 4K cold seeks) and decoder cycle counts in docs/performance.md; threading, draft mode and 4K numbers in crates/h264/README.md; bench cases in docs/testing.md. Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 4 +- crates/h264/README.md | 64 ++++++++++++++++++++++++---- docs/performance.md | 97 ++++++++++++++++++++++++++++++++++++++++++- docs/testing.md | 5 ++- 4 files changed, 159 insertions(+), 11 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 6f4d5811..3fcac3d5 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -22,7 +22,7 @@ matched against `filmcraft-cli commands` and the UI command table. Effects: the | Graphics and captions | 8% | text engine, shapes, captions, transcripts; no MOGRT, rolls/crawls, responsive design | ~60% | | Export | 8% | own H.264/AAC, ProRes, DNxHR, image sequences, GIF, WAV; preset library + user presets, full Export-mode settings (loudness normalization, overlays, captions), export queue, Quick Export; no AAF/OMF, MXF export | ~80% | | Preferences and project management | 5% | Settings dialog with 16 categories (most settings wired), project settings, scratch disks, search bins, templates | ~80% | -| Performance | 5% | 1080p real-time, 3×1080p; HEVC 2.5× cheaper, catch-up decoding halves 4K CPU/frame, `cargo xtask bench` + `perf.stats`; 4K/8K and AV1 not yet real-time on a loaded machine | ~55% | +| Performance | 5% | 1080p real-time, 3×1080p; 4K H.264 real-time at load ≤140 (−23% decoder cycles, opt-in draft decoding at 1/2–1/4); HEVC 2.5× cheaper, catch-up decoding, `cargo xtask bench` + `perf.stats`; 8K and AV1 not yet real-time on a loaded machine | ~60% | | **Weighted total** | | | **~77%** | ## Estimate to parity @@ -77,6 +77,8 @@ Status: ✅ done · 🟡 in progress · ⬜ not started. Estimates are remaining ## Log +- **2026-10-03:** 4K H.264 playback (M4.9): vectorised deblocking, branch-free CABAC bins and column-parallel inverse transforms cut H.264 decoding by 23 % (cycles per frame, bit-exact; 4K cold seeks −27 % CPU); opt-in draft decoding (Settings ▸ Playback) skips deblocking of non-reference pictures and uploads decimated planes while playing at 1/2–1/4, never for paused frames or exports; conformance runs 1, 3 and all threads. 4K plays 192/0 at load 100–140 (before: only at load ~90). + - **2026-10-02 (late):** export parity (M6.5): built-in preset library + user presets, full Export-mode settings (loudness normalization, image/name/timecode overlays, captions, range), export queue, Quick Export, Premiere-style image-sequence export, `filmcraft-cli export --preset`; media import (M9.14): MXF OP1a/OP-Atom (new `crates/mxf`, H.264/DNx/ProRes/PCM, timecode), Ogg Opus (new `crates/ogg`), image sequences as clips, BWF timecode. 1373 tests. - **2026-10-02 (panels):** complete Lumetri Scopes (new `filmcraft-scopes` crate: Vectorscope YUV/HLS, Histogram, Parade RGB/YUV/RGB-White, Waveform RGB/Luma/YC/YC no Chroma; wrench menu with presets, colour space, brightness, scale; < 1 ms per scope at 1080p; `scopes.read` returns the numbers); Metadata panel (editable log fields, undoable `metadata.set`), Timecode panel, Events panel (event log of failed commands, jobs, auto-save), Progress panel (jobs with Cancel), Reference Monitor (parked or ganged, picture or scopes). diff --git a/crates/h264/README.md b/crates/h264/README.md index ec989728..21e7a10f 100644 --- a/crates/h264/README.md +++ b/crates/h264/README.md @@ -34,6 +34,8 @@ oracles and fixture generators. - Output: cropped 8-bit planar 4:2:0 pictures in output order with the caller's `pts`, POC, key flag, VUI colour description (range, primaries, transfer, matrix) and sample aspect ratio. - Frame-level multithreading (see below). +- Draft mode for reduced-resolution playback (off by default): non-reference pictures skip the + deblocking filter and are flagged `Picture::draft`; every other picture stays bit-exact. ## API @@ -41,6 +43,7 @@ oracles and fixture generators. use filmcraft_h264::Decoder; let mut dec = Decoder::new(); // or Decoder::with_threads(n), Decoder::from_avcc(&avcc)? +// dec.threads(): worker threads in use; dec.set_draft(true): draft mode (playback only) for (pts, access_unit) in access_units { for pic in dec.decode(access_unit, pts)? { // pic.width / pic.height (cropped), pic.y / pic.u / pic.v, pic.y_stride / pic.uv_stride, @@ -67,6 +70,29 @@ bit-exact. Without the `threads` feature (or on wasm, or with `with_threads(1)`) With threads, `decode` returns pictures once their job has finished, so output lags input by up to roughly the number of threads. +Deblocking is row-parallel in this pipelined form: each picture's job filters macroblock row r as +soon as row r + 1 is reconstructed (intra prediction of row r + 1 reads the unfiltered bottom +line of row r) and publishes it one row later, while the jobs of other pictures run on other +threads. A picture waiting on a reference therefore waits only for that reference's rows its +motion vectors reach plus about two rows, so a chain of reference pictures (a seek from an IDR +picture) advances a few rows behind its predecessor instead of a whole picture behind it. A +separate deblocking thread per picture would shorten that lag by about one row and would need its +own copy of the reconstructed rows and macroblock state; it is not worth it while the pool has +more pictures in flight than cores. The hot loops are written to vectorise without `unsafe` or +intrinsics: the edge filters process all 16 (luma) or 8 (chroma) lines of an edge at once with +per-lane masks, the inverse transforms do their column pass across all columns, and the CABAC +engine decodes a context-coded bin without branching on its outcome. + +### Draft mode + +`Decoder::set_draft(true)` (reduced-resolution playback only; FilmCraft enables it with Settings ▸ +Playback ▸ Draft decoding while the Program monitor plays at 1/2 or 1/4 resolution) skips the +deblocking filter (8.7) of non-reference pictures (`nal_ref_idc` 0). No picture is predicted from +a non-reference picture, and deblocking never changes the motion data that direct prediction +reads, so every other picture is unchanged; the skipped pictures are flagged `Picture::draft` and +the media stack never hands them to a paused monitor, a render or an export. On the 4K fixture +below it saves ~9 % of the decoding work (the B-pyramid's non-reference half of the B pictures). + ## Tests `cargo test -p filmcraft-h264` (fixtures are generated on first use into `target/fixtures/h264/` @@ -82,10 +108,16 @@ with ffmpeg + libx264; tests print a message and skip when `/opt/homebrew/bin/ff explicit bi-prediction. The expected output is known exactly and is also checked against ffmpeg. - Robustness: avcC input with pts round trip; randomly corrupted, truncated and garbage input never panics or deadlocks (single- and multi-threaded). -- Conformance (`tests/conformance.rs`): every fixture is decoded single-threaded and frame-threaded - and compared byte-for-byte with `ffmpeg -f rawvideo -pix_fmt yuv420p`; on mismatch the first - differing frame, plane, sample position and macroblock are reported. Output pts/POC order is - checked too. +- Conformance (`tests/conformance.rs`): every fixture is decoded single-threaded, with 3 threads + and with every core, and compared byte-for-byte with `ffmpeg -f rawvideo -pix_fmt yuv420p`; on + mismatch the first differing frame, plane, sample position and macroblock are reported. Output + pts/POC order is checked too, and that no picture is flagged draft without draft mode. Draft + mode (`draft_mode_changes_only_flagged_non_reference_pictures`): on five B-pyramid / B-frame + fixtures, single- and frame-threaded, every unflagged picture is bit-exact and only flagged + pictures differ. +- Vectorised kernels against their straightforward forms on random input: the deblocking edge + filter against the per-line filter, the 4x4 / 8x8 inverse transforms against the per-column + formulation. ### Fixture matrix (all bit-exact, single- and multi-threaded) @@ -146,9 +178,26 @@ Pro (14 cores) while the machine was heavily shared (load average 40-180), so th | 1 | ~105-122 (≈ 33 Mcycles per frame) | | 14 | ~500-600 | +4K High profile (testsrc2 + grain, 3840x2160, CABAC, B-pyramid, ~170 Mbit/s; the first 48 frames +of the playback benchmark's `a2160.mp4`), CPU cycles per frame counted by the kernel +(`proc_pid_rusage`; independent of machine load, all threads summed; bit-exact with ffmpeg): + +| build | Mcycles / frame | instructions / frame | +|---|---|---| +| before M4.9 | 505 | 1232 M | +| M4.9 | **390** (−23 %) | 1195 M | +| M4.9, draft mode | **356** (−30 %) | 1068 M | + +The same on the benchmark's 1080p fixture (`dec_h264_1080.mp4`, 120 frames): 132 → 100 Mcycles +per frame (−24 %), 90 in draft mode. Single-threaded 4K time (`sample` profile) splits into CABAC +residual decoding 41 %, deblocking 16 % (25-30 % before), other macroblock-layer parsing 11 %, +inverse transforms and dequantisation 9 %, motion compensation 8 %, intra prediction 6 %, row and +picture copies 4 %. At ~4.5 GHz a 4K frame takes ~85 ms of one +core, so 23.976 fps needs about two cores' worth of frame threads. + The example `cargo run --release -p filmcraft-h264 --example h264dec -- in.h264 out.yuv` decodes a -file; `H264_BENCH_ITERS=n` (and `H264_THREADS=t`) turns it into a benchmark, `H264_STATS=1` prints -the coverage counters. +file; `H264_BENCH_ITERS=n` (and `H264_THREADS=t`, `H264_DRAFT=1`) turns it into a benchmark, +`H264_STATS=1` prints the coverage counters. ## Known gaps @@ -159,7 +208,8 @@ the coverage counters. - FMO (slice groups), SP/SI slices, data partitioning (Extended profile), MVC/SVC NAL units. - Error concealment is minimal: undecodable slices leave the prediction/previous content; missing references are replaced by other references. -- Single-threaded speed has no SIMD-specific code yet (the hot loops are written to auto-vectorise). +- No SIMD intrinsics (the hot loops are written to auto-vectorise); CABAC residual decoding is + inherently serial and now dominates. ### Extending to High 10 and 4:2:2 diff --git a/docs/performance.md b/docs/performance.md index 3e2063ee..6a4fc465 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -9,11 +9,104 @@ and `.md` (options and sections: [testing.md](testing.md) §5). Agents read live **75–200** during every run below. Wall-clock columns (fps, shown/dropped frames, seek latency) swing by 2–5× between back-to-back runs at that load and are only comparable within one run. CPU time per frame, samples decoded and skipped, and peak RSS are much less sensitive, and are what -the before/after comparison relies on. Baseline = commit `457aca1` (the harness on unchanged -code); after = M4.8. Runs alternated base / after (2 rounds each, 2026-10-02). +the before/after comparison relies on. Each milestone below alternated base / after runs (2 rounds +each); the most load-independent figure is the decoder's own cycle count (`proc_pid_rusage` +instructions and cycles of a process, all threads summed). + +## Results (M4.9: 4K H.264, before → after) + +Baseline = commit `3d14099` (M3.12, unchanged decoder); after = M4.9. Interleaved runs on +2026-10-02/03, load average 92–178 (the playback rows say which load each run saw). + +### Decoder work per frame (load-independent) + +`h264dec` example, one decoder, CPU cycles and instructions counted by the kernel (all threads), +output bit-exact with ffmpeg: + +| stream | Mcycles / frame before | after | after, draft mode | +|---|---|---|---| +| 2160p (`a2160.mp4`, 170 Mbit/s, first 48 frames) | 505 | **390** (−23 %) | **356** (−30 %) | +| 1080p (`dec_h264_1080.mp4`, 120 frames) | 132 | **100** (−24 %) | **90** (−32 %) | + +The cycle count is the same with 1, 4 or 14 frame threads (no spinning or contention overhead): +at ~4.5 GHz a 4K frame is ~85 ms of one core, so 23.976 fps needs about two cores. + +### Decode (every frame through the media stack) + +| codec | size | CPU ms/frame before (2 rounds) | after (2 rounds) | +|---|---|---|---| +| H.264 | 1080p | 35.8, 39.5 | **33.0, 29.4** | +| H.264 | 2160p | 138.9, 152.1 | **125.7, 112.3** | + +(A run at load ~190 before any change gave 41.6 / 160.2 ms; an intermediate build 32.2 / 118.3.) + +### Cold seeks on the 4K fixture (12 random targets, GOP 250, fresh source per seek) + +| mode | CPU ms per seek before | after | wall p50 ms before → after | +|---|---|---|---| +| full decode from the keyframe | 7529, 7418 | **5531, 5268** (−27 %) | 2757, 1848 → 924, 739 | +| keep 2 s (scrub) | 7488, 7373 | **5442, 5397** | 2408, 1995 → 872, 795 | +| late (playback catch-up) | 5597, 5487 | **4079, 3949** | 2014, 1248 → 735, 526 | + +Samples decoded per seek are identical (~48, ~34 in catch-up): the same pictures, decoded with +less work. The cold seek is a chain of reference pictures, so it shows the per-picture cost +directly. + +### Program-monitor playback, 2160p (8 s, GPU path) + +| case | before: shown/dropped (load) | after: shown/dropped (load) | CPU ms/frame before → after | +|---|---|---|---| +| Full | 192/0 (92), 9/183 (167) | 113/79 (171), **192/0** (139) | 159 (no skips), 133 (17 skipped) → 131, 130 | +| 1/2 | 12/180 (95), 5/187 (169) | 173/19 (172), **192/0** (124) | 143, 127 → 133, 132 | +| 1/2 draft | – | 183/9 (175), **192/0** (119) | – → 144, 134 | +| 1/4 | – | 186/6 (177), **192/0** (111) | – → 134, 125 | +| 1/4 draft | – | **192/0** (178), 190/2 (101) | – → 132, 131 | +| 1080p Full | 192/0, 192/0 | 192/0, 192/0 | 41, 48 → 39, 36 | + +Before, the 4K clip kept up in one run (Full at load 92; 1/2 dropped 94 % at load 95) and dropped +95–97 % at load ~168. After, every 4K case plays 192/0 at load 100–140; at load 171–178 Full drops +41 %, 1/2 10 % (5 % with draft decoding), 1/4 3 % (none with draft decoding). CPU per +displayed frame (whole process: decode, frame workers, compositing) drops ~15 % where no frames +were skipped; draft decoding at 1/2 and 1/4 is within run-to-run noise in this column (its saving +is the ~9 % of decoder cycles above, plus a quarter / sixteenth of the texture upload), while the +frame jobs wait for the decoder less (decode ms/job 212 → 67 at 1/2, 98 → 44 at 1/4 in the busy +round). On an idle machine the 4K clip needs about two cores. + +## What M4.9 changed + +1. **Deblocking** (25–30 % of 4K decode time before): the edge filter runs on all 16 luma / 8 + chroma lines of an edge at once with per-lane masks and min/max clipping (no `clamp`, whose + bounds assert blocked vectorisation), so it compiles to NEON; edges whose alpha or beta index + is 0 are skipped. Bit-exact with the per-line filter (random-edge unit test) and ffmpeg. +2. **CABAC**: `decode_decision` / `decode_bypass` select instead of branching on the bin value + (the MPS/LPS outcome is close to random at high bit rates), renormalisation has no branch on + the range, and context / state indexing has no bounds checks. +3. **Inverse transforms**: the 4x4 / 8x8 column pass and the add/clip run across all columns. +4. **Frame threading** was already in place (M2): pictures decode concurrently and wait per + macroblock row for the reference rows their motion vectors reach, with deblocking pipelined row + by row inside each picture's job. The conformance suite now decodes every fixture with 1, 3 and + all threads (bit-exact each time). A separate deblocking thread was not added: a picture's rows + are published about two rows after they are reconstructed, so a reference chain advances a few + rows behind its predecessor and the pool already has more pictures in flight than cores + (crates/h264/README.md, "Threading model"). +5. **Draft decoding** (Settings ▸ Playback ▸ Draft decoding, off by default; `perf.stats` + `playback.draftDecode`, `decode.draftFrames`, `decode.h264Threads`): while the Program monitor + plays at 1/2 or 1/4, H.264 non-reference pictures skip deblocking (no other picture can change), + and draft plans hand the GPU box-decimated Y'CbCr planes at the drawn size. Draft frames carry + their own frame-cache keys and the GOP cache serves them to draft requests only, so pausing, + rendering and exporting always decode exact pictures (tested through the media stack against + ffmpeg after draft playback). The CPU renderer already converted reduced-resolution frames + straight to the decimated size (`to_linear_f32_decimated`). + +Not done: CABAC residual decoding (now 41 % of the work, ~19 cycles per bin) is inherently serial; +the remaining deblocking cost is mostly the vertical-edge transposes; motion compensation and the +per-row / per-picture copies (4 %) are unchanged. ## Results (M4.8, before → after) +Baseline = commit `457aca1` (the harness on unchanged code); after = M4.8. Runs alternated base / +after (2 rounds each, 2026-10-02). + ### Decode (every frame through the media stack: container, GOP cache, decoder, conversion) CPU ms per frame (both rounds; lower is better). fps is the best of the run at that moment's load. diff --git a/docs/testing.md b/docs/testing.md index 710684a1..09f3a6f3 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -231,7 +231,10 @@ wgpu render ms per frame), `export` (H.264 + AAC and ProRes) and `project` (save Options: `--sections decode,scrub`, `--only ` (fixture or scenario), `--repeat N`, `--quick`, `--cpu`, `--label NAME` (output `target/bench/bench-