From c8edb9daaf2f6a4dbc1c30f9dd82df86801aefd2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 07:38:29 -0400 Subject: [PATCH] M4.10: HEVC vectorised MC, SAO and output, batched bypass bins, draft mode Interpolation keeps its reference windows in reusable scratch (no 2 x 10 KB zeroing per block) and filters with taps outer / outputs inner over fixed block widths; SAO edge and band offsets select the offset per lane instead of a table lookup; uni / bi output clips with min / max; runs of bypass bins decode with one division; context-coded bins are branch-free. Draft mode skips deblocking and SAO of sub-layer non-reference pictures of the top sub-layer. Bit-exact with 1, 3 and all threads; draft output differs only on flagged pictures. Co-Authored-By: Claude Opus 5.5 --- crates/codecs/src/video.rs | 5 +- crates/hevc/examples/hevcdec.rs | 2 + crates/hevc/src/cabac.rs | 71 ++++++----- crates/hevc/src/decoder.rs | 55 ++++++-- crates/hevc/src/dpb.rs | 2 + crates/hevc/src/filter.rs | 73 +++++++++-- crates/hevc/src/inter.rs | 210 ++++++++++++++++--------------- crates/hevc/src/lib.rs | 3 + crates/hevc/src/slicedec.rs | 20 ++- crates/hevc/src/synth_tests.rs | 2 +- crates/hevc/tests/common/mod.rs | 12 +- crates/hevc/tests/conformance.rs | 40 ++++++ 12 files changed, 344 insertions(+), 151 deletions(-) diff --git a/crates/codecs/src/video.rs b/crates/codecs/src/video.rs index 636e0cfc..a9cf13df 100644 --- a/crates/codecs/src/video.rs +++ b/crates/codecs/src/video.rs @@ -307,7 +307,7 @@ impl HevcDecoder { } let par = if p.sar.0 > 0 && p.sar.1 > 0 { (p.sar.0 as u32, p.sar.1 as u32) } else { (1, 1) }; let frame = VideoFrame { width: p.width, height: p.height, data, color, par, pts: filmcraft_time::Tick::ZERO }; - DecodedFrame { pts: p.pts, frame, draft: false } + DecodedFrame { pts: p.pts, frame, draft: p.draft } } } @@ -330,6 +330,9 @@ impl VideoDecoder for HevcDecoder { fn is_disposable(&self, sample: &[u8]) -> bool { hevc_disposable(sample, self.length_size, self.highest_tid) } + fn set_draft(&mut self, on: bool) { + self.dec.set_draft(on); + } } pub fn hevc_factory(e: &SampleEntry) -> Option>> { diff --git a/crates/hevc/examples/hevcdec.rs b/crates/hevc/examples/hevcdec.rs index 5c4338d2..d66b107b 100644 --- a/crates/hevc/examples/hevcdec.rs +++ b/crates/hevc/examples/hevcdec.rs @@ -17,6 +17,8 @@ fn main() { let mut pics = Vec::new(); for _ in 0..loops { let mut dec = if threads > 0 { Decoder::with_threads(threads) } else { Decoder::new() }; + // HEVC_DRAFT=1: draft mode (non-reference pictures skip deblocking and SAO). + dec.set_draft(std::env::var_os("HEVC_DRAFT").is_some()); pics = dec.decode(&data, 0).expect("decode"); pics.extend(dec.flush()); } diff --git a/crates/hevc/src/cabac.rs b/crates/hevc/src/cabac.rs index e45fcc35..100fc2fe 100644 --- a/crates/hevc/src/cabac.rs +++ b/crates/hevc/src/cabac.rs @@ -60,7 +60,9 @@ pub(crate) static NEXT_STATE: [[u8; 2]; 128] = { t }; -pub type Contexts = [u8; NUM_CTX]; +/// Context states, padded to a power of two so that the per-bin index needs no bounds check. +pub type Contexts = [u8; CTX_SLOTS]; +const CTX_SLOTS: usize = NUM_CTX.next_power_of_two(); /// Initialise all context variables for `slice_qp` and `init_type` (9.3.2.2). Each context is stored as /// `pStateIdx << 1 | valMps`. @@ -92,7 +94,7 @@ impl<'a> Cabac<'a> { /// Create an engine over `data` starting at byte `pos`, with contexts initialised for `slice_qp` /// and `init_type`. pub fn new(data: &'a [u8], pos: usize, slice_qp: i32, init_type: usize) -> Result { - let mut c = Cabac { data, next_byte: pos, range: 0, value: 0, bits: 0, ctx: [0; NUM_CTX] }; + let mut c = Cabac { data, next_byte: pos, range: 0, value: 0, bits: 0, ctx: [0; CTX_SLOTS] }; init_contexts(&mut c.ctx, slice_qp, init_type); c.init_engine()?; Ok(c) @@ -146,35 +148,32 @@ impl<'a> Cabac<'a> { #[inline(always)] fn renorm(&mut self) { - if self.range < 256 { - let shift = self.range.leading_zeros() - 23; - self.range <<= shift; - if self.bits < shift { - self.refill(); - } - self.bits -= shift; + // ivlCurrRange < 512 always, so the shift is 0 when no renormalisation is needed: no + // branch on the (unpredictable) range, only on the rare refill. + let shift = self.range.leading_zeros() - 23; + self.range <<= shift; + if self.bits < shift { + self.refill(); } + self.bits -= shift; } + /// 9.3.4.3.2, branch-free: the MPS / LPS outcome of a context-coded bin is close to random + /// at high bit rates, so values are selected instead of jumping. #[inline(always)] pub fn decode_decision(&mut self, ctx_idx: usize) -> u32 { - let s = self.ctx[ctx_idx] as usize; + let ctx_idx = ctx_idx & (CTX_SLOTS - 1); + let s = (self.ctx[ctx_idx] & 127) as usize; let q = ((self.range >> 6) & 3) as usize; let lps = RANGE_TAB_LPS[s >> 1][q] as u32; - self.range -= lps; - let scaled = (self.range as u64) << self.bits; - let bin; - if self.value >= scaled { - bin = 1 - (s & 1) as u32; - self.value -= scaled; - self.range = lps; - self.ctx[ctx_idx] = NEXT_STATE[s][1]; - } else { - bin = (s & 1) as u32; - self.ctx[ctx_idx] = NEXT_STATE[s][0]; - } + let mps_range = self.range - lps; + let scaled = (mps_range as u64) << self.bits; + let is_lps = self.value >= scaled; + self.value -= scaled & (is_lps as u64).wrapping_neg(); + self.range = if is_lps { lps } else { mps_range }; + self.ctx[ctx_idx] = NEXT_STATE[s][is_lps as usize]; self.renorm(); - bin + (s & 1) as u32 ^ is_lps as u32 } #[inline(always)] @@ -184,12 +183,26 @@ impl<'a> Cabac<'a> { } self.bits -= 1; let scaled = (self.range as u64) << self.bits; - if self.value >= scaled { - self.value -= scaled; - 1 - } else { - 0 + let one = self.value >= scaled; + self.value -= scaled & (one as u64).wrapping_neg(); + one as u32 + } + + /// `n` (<= 16) bypass bins at once, most significant first. The bins of a run of bypass + /// decodes are the binary digits of floor(ivlOffset / ivlCurrRange) in the bit-scaled + /// representation (no renormalisation happens between them), so one division replaces `n` + /// compare-subtract steps. + #[inline(always)] + pub fn decode_bypass_bits(&mut self, n: u32) -> u32 { + debug_assert!(n <= 16); + if self.bits < n { + self.refill(); } + let s = self.bits - n; + let r = ((self.value >> s) / self.range as u64) as u32; + self.value -= ((r as u64) * self.range as u64) << s; + self.bits = s; + r } pub fn decode_terminate(&mut self) -> u32 { @@ -296,7 +309,7 @@ mod tests { #[test] fn context_init_formula() { - let mut ctx = [0u8; NUM_CTX]; + let mut ctx: Contexts = [0u8; CTX_SLOTS]; // initValue 154 is the "equiprobable" value: m = 0, n = 64 -> pStateIdx 0, valMps 1 init_contexts(&mut ctx, 30, 0); assert_eq!(ctx[crate::spec_tables::CU_TRANSQUANT_BYPASS], 1); diff --git a/crates/hevc/src/decoder.rs b/crates/hevc/src/decoder.rs index 37d71fb3..ee851f5b 100644 --- a/crates/hevc/src/decoder.rs +++ b/crates/hevc/src/decoder.rs @@ -56,6 +56,8 @@ pub struct DecodeStats { pub transform_skip_pictures: u64, pub long_term_slices: u64, pub missing_refs: u64, + /// Pictures decoded in draft mode without in-loop filters ([`Decoder::set_draft`]). + pub draft_pictures: u64, } #[derive(Default)] @@ -86,6 +88,7 @@ pub struct Decoder { pool: Option, in_flight: VecDeque, max_in_flight: usize, + draft: bool, } impl Default for Decoder { @@ -139,6 +142,7 @@ impl Decoder { pool, in_flight: VecDeque::new(), max_in_flight: threads.max(1) + 2, + draft: false, } } @@ -237,6 +241,14 @@ impl Decoder { /// First error reported by a decoding job since the last call (also returned by /// [`Decoder::decode`]); useful after [`Decoder::flush`]. + /// Draft mode for reduced-resolution playback (off by default): sub-layer non-reference + /// pictures of the highest temporal sub-layer (pictures nothing predicts from) skip + /// deblocking and SAO and come out flagged [`Picture::draft`]; every other picture is + /// unchanged. + pub fn set_draft(&mut self, on: bool) { + self.draft = on; + } + pub fn take_error(&mut self) -> Option { self.shared.error.lock().ok().and_then(|mut e| e.take()) } @@ -422,7 +434,13 @@ impl Decoder { self.next_id += 1; let frame = Arc::new(Frame::new(id, poc, w, h, sps.log2_ctb, sps.bit_depth_luma, sps.bit_depth_chroma)); let output = sh.pic_output && !(hdr.is_rasl() && self.skip_rasl); - let meta = Arc::new(output_meta(sps, pts, hdr.is_irap())); + // Draft mode: a sub-layer non-reference picture of the highest sub-layer is never used + // for prediction, so its in-loop filters can be left out without changing anything else. + let draft = self.draft && hdr.is_sub_layer_non_ref() && hdr.temporal_id as u32 >= sps.max_sub_layers_minus1; + if draft { + self.stat(|s| s.draft_pictures += 1); + } + let meta = Arc::new(output_meta(sps, pts, hdr.is_irap(), draft)); self.pending = Some(PendingPic { frame, sps: sps.clone(), @@ -452,14 +470,24 @@ impl Decoder { let Some(p) = self.pending.take() else { return }; let PendingPic { frame, sps, pps, layout, first_sh, poc, output, meta, slices, .. } = p; let scaling = self.layout_cache.iter().find(|(pp, ss, _, _)| Arc::ptr_eq(pp, &pps) && Arc::ptr_eq(ss, &sps)).and_then(|(_, _, _, s)| s.clone()); - self.dispatch(frame.clone(), sps, pps, layout, scaling, slices); + self.dispatch(frame.clone(), sps, pps, layout, scaling, slices, meta.draft); self.poc_state.update(&first_sh, poc); let mut outs = Vec::new(); self.dpb.insert(frame, poc, output, meta, &mut outs); self.out_queue.extend(outs); } - fn dispatch(&mut self, frame: FrameRef, sps: Arc, pps: Arc, layout: Arc, scaling: Option>>>, slices: Vec) { + #[allow(clippy::too_many_arguments)] + fn dispatch( + &mut self, + frame: FrameRef, + sps: Arc, + pps: Arc, + layout: Arc, + scaling: Option>>>, + slices: Vec, + draft: bool, + ) { let shared = self.shared.clone(); #[cfg(feature = "threads")] if let Some(pool) = &self.pool { @@ -470,16 +498,27 @@ impl Decoder { } } self.in_flight.push_back(frame.clone()); - pool.spawn_fifo(move || run_job(frame, sps, pps, layout, scaling, slices, &shared)); + pool.spawn_fifo(move || run_job(frame, sps, pps, layout, scaling, slices, draft, &shared)); return; } - run_job(frame, sps, pps, layout, scaling, slices, &shared); + run_job(frame, sps, pps, layout, scaling, slices, draft, &shared); } } /// Decode all slices of one picture and publish its rows. -fn run_job(frame: FrameRef, sps: Arc, pps: Arc, layout: Arc, scaling: Option>>>, slices: Vec, shared: &Shared) { +#[allow(clippy::too_many_arguments)] +fn run_job( + frame: FrameRef, + sps: Arc, + pps: Arc, + layout: Arc, + scaling: Option>>>, + slices: Vec, + draft: bool, + shared: &Shared, +) { let mut pic = PicState::new(frame.clone(), sps, pps, layout, scaling); + pic.draft = draft; let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { let mut first_err = None; for s in &slices { @@ -521,7 +560,7 @@ fn run_job(frame: FrameRef, sps: Arc, pps: Arc, layout: Arc, s } } -fn output_meta(sps: &Sps, pts: i64, key: bool) -> OutputMeta { +fn output_meta(sps: &Sps, pts: i64, key: bool, draft: bool) -> OutputMeta { let vui = sps.vui.clone().unwrap_or_default(); OutputMeta { pts, @@ -533,6 +572,7 @@ fn output_meta(sps: &Sps, pts: i64, key: bool) -> OutputMeta { matrix_coefficients: vui.matrix_coefficients, sar: vui.sar, bit_depth: sps.bit_depth_luma, + draft, } } @@ -564,5 +604,6 @@ fn make_picture(o: &Output) -> Picture { matrix: o.meta.matrix_coefficients, }, sar: o.meta.sar, + draft: o.meta.draft, } } diff --git a/crates/hevc/src/dpb.rs b/crates/hevc/src/dpb.rs index 40448ecf..6bb4e177 100644 --- a/crates/hevc/src/dpb.rs +++ b/crates/hevc/src/dpb.rs @@ -18,6 +18,8 @@ pub struct OutputMeta { pub matrix_coefficients: u8, pub sar: (u16, u16), pub bit_depth: u32, + /// Decoded in draft mode (no in-loop filters). + pub draft: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/crates/hevc/src/filter.rs b/crates/hevc/src/filter.rs index 95c326c2..9f4546ce 100644 --- a/crates/hevc/src/filter.rs +++ b/crates/hevc/src/filter.rs @@ -129,7 +129,7 @@ fn edge_params(pic: &PicState, px: usize, py: usize, qx: usize, qy: usize) -> Op /// Deblock CTB row `r`: vertical edges of the row, then horizontal edges whose q side is in the row. fn deblock_row(pic: &mut PicState, r: usize) { - if !pic.deblocking_enabled_anywhere { + if !pic.deblocking_enabled_anywhere || pic.draft { return; } let ctb = 1usize << pic.log2_ctb; @@ -319,7 +319,7 @@ fn sao_row(pic: &PicState, r: usize) -> [Vec; 3] { let y0 = r * cs; let y1 = (y0 + cs).min(h); out[c] = pic.planes[c][y0 * w..y1 * w].to_vec(); - for rx in 0..pic.wctb { + for rx in (0..pic.wctb).filter(|_| !pic.draft) { let rs = r * pic.wctb + rx; let si = pic.ctb_slice[rs]; if si == u32::MAX { @@ -377,8 +377,13 @@ fn sao_ctb( table[(k + p.band_pos[c] as usize) & 31] = offs[k]; } for y in y0..y1 { + if !any_nofilter { + let band = (p.band_pos[c] as i32, offs); + sao_band_run(&pic.planes[c][y * w + x0..y * w + x1], &mut dst[(y - y0) * w + x0..(y - y0) * w + x1], shift, band, max); + continue; + } for x in x0..x1 { - if any_nofilter && nofilter(pic, x, y) { + if nofilter(pic, x, y) { continue; } let v = pic.planes[c][y * w + x] as i32; @@ -473,12 +478,66 @@ fn sao_ctb( let na = &plane[ya * w + xa as usize..][..n]; let nb = &plane[yb * w + xb as usize..][..n]; let out = &mut dst[(y - y0) * w + x0 + mx..][..n]; - for i in 0..n { - let v = cur[i] as i32; - let sum = (v - na[i] as i32).signum() + (v - nb[i] as i32).signum(); - out[i] = (v + by_sum[(2 + sum) as usize]).clamp(0, max) as u16; + sao_edge_run(cur, na, nb, out, &by_sum, max); + } +} + +/// Lanes of the SAO loops (16-bit arithmetic: samples are at most 12 bits, offsets 7 bits). +const SAO_LANES: usize = 16; + +/// Edge offset of a run of samples whose two neighbours (`na`, `nb`) are all usable: per lane +/// the sum of the two neighbour signs selects the offset (no table lookup), so the loop +/// vectorises; bit-exact with the per-sample form. +#[inline(always)] +fn sao_edge_run(cur: &[u16], na: &[u16], nb: &[u16], out: &mut [u16], by_sum: &[i32; 5], max: i32) { + let o = by_sum.map(|v| v as i16); + let max = max as i16; + let lane = |v: i16, a: i16, b: i16| -> u16 { + let s = ((v > a) as i16 - (v < a) as i16) + ((v > b) as i16 - (v < b) as i16); + let m = |k: i16| -((s == k) as i16); + let off = (o[0] & m(-2)) | (o[1] & m(-1)) | (o[3] & m(1)) | (o[4] & m(2)); + (v + off).max(0).min(max) as u16 + }; + let n = cur.len(); + let full = n - n % SAO_LANES; + for i in (0..full).step_by(SAO_LANES) { + let c: &[u16; SAO_LANES] = cur[i..i + SAO_LANES].try_into().expect("lanes"); + let a: &[u16; SAO_LANES] = na[i..i + SAO_LANES].try_into().expect("lanes"); + let b: &[u16; SAO_LANES] = nb[i..i + SAO_LANES].try_into().expect("lanes"); + let d: &mut [u16; SAO_LANES] = (&mut out[i..i + SAO_LANES]).try_into().expect("lanes"); + for l in 0..SAO_LANES { + d[l] = lane(c[l] as i16, a[l] as i16, b[l] as i16); } } + for i in full..n { + out[i] = lane(cur[i] as i16, na[i] as i16, nb[i] as i16); + } +} + +/// Band offset of a run of samples: the band index relative to `band_pos` selects one of the +/// four offsets per lane. +#[inline(always)] +fn sao_band_run(src: &[u16], out: &mut [u16], shift: u32, (band_pos, offs): (i32, [i32; 4]), max: i32) { + let o = offs.map(|v| v as i16); + let (bp, max) = (band_pos as i16, max as i16); + let lane = |v: i16| -> u16 { + let k = ((v >> shift) - bp) & 31; + let m = |j: i16| -((k == j) as i16); + let off = (o[0] & m(0)) | (o[1] & m(1)) | (o[2] & m(2)) | (o[3] & m(3)); + (v + off).max(0).min(max) as u16 + }; + let n = src.len(); + let full = n - n % SAO_LANES; + for i in (0..full).step_by(SAO_LANES) { + let c: &[u16; SAO_LANES] = src[i..i + SAO_LANES].try_into().expect("lanes"); + let d: &mut [u16; SAO_LANES] = (&mut out[i..i + SAO_LANES]).try_into().expect("lanes"); + for l in 0..SAO_LANES { + d[l] = lane(c[l] as i16); + } + } + for i in full..n { + out[i] = lane(src[i] as i16); + } } /// Publish CTB row `r` (final samples + compressed motion) into the frame. diff --git a/crates/hevc/src/inter.rs b/crates/hevc/src/inter.rs index 3bcb768e..fc1e9aac 100644 --- a/crates/hevc/src/inter.rs +++ b/crates/hevc/src/inter.rs @@ -6,114 +6,126 @@ use crate::spec_tables::{CHROMA_FILTER, LUMA_FILTER}; /// Max prediction block size + filter margin. const WIN: usize = 64 + 7; +/// Reference windows and first-pass rows, reused between blocks (no per-block zeroing). +pub struct McScratch { + win: Vec, + tmp: Vec, +} + +impl Default for McScratch { + fn default() -> Self { + McScratch { win: vec![0; WIN * WIN], tmp: vec![0; WIN * WIN] } + } +} + +/// `out[j] = (sum_k f[k] * src[j + k * step]) >> shift` for `W` outputs: taps outer, outputs +/// inner, so each tap is one vector multiply-add across the row (exact integer sums). +#[inline(always)] +fn filt_w(src: &[i16], step: usize, f: &[i8; T], shift: u32, out: &mut [i16]) { + let mut acc = [0i32; W]; + for k in 0..T { + let c = f[k] as i32; + let p: &[i16; W] = src[k * step..k * step + W].try_into().expect("taps"); + for j in 0..W { + acc[j] += c * p[j] as i32; + } + } + let o: &mut [i16; W] = (&mut out[..W]).try_into().expect("row"); + for j in 0..W { + o[j] = (acc[j] >> shift) as i16; + } +} + +/// [`filt_w`] for a run of `w` outputs (prediction block widths are fixed sizes). +#[inline(always)] +fn filt(src: &[i16], step: usize, f: &[i8; T], shift: u32, out: &mut [i16], w: usize) { + match w { + 2 => filt_w::(src, step, f, shift, out), + 4 => filt_w::(src, step, f, shift, out), + 6 => filt_w::(src, step, f, shift, out), + 8 => filt_w::(src, step, f, shift, out), + 12 => filt_w::(src, step, f, shift, out), + 16 => filt_w::(src, step, f, shift, out), + 24 => filt_w::(src, step, f, shift, out), + 32 => filt_w::(src, step, f, shift, out), + 48 => filt_w::(src, step, f, shift, out), + 64 => filt_w::(src, step, f, shift, out), + _ => { + for (j, o) in out[..w].iter_mut().enumerate() { + let s: i32 = (0..T).map(|k| f[k] as i32 * src[j + k * step] as i32).sum(); + *o = (s >> shift) as i16; + } + } + } +} + +/// Separable sub-sample interpolation of a w x h block from `win` (stride WIN, the block's +/// top-left sample at (m, m) for a T-tap filter with m = T / 2 - 1). +#[allow(clippy::too_many_arguments)] +fn interpolate( + win: &[i16], + tmp: &mut [i16], + fx: usize, + fy: usize, + hf: &[i8; T], + vf: &[i8; T], + w: usize, + h: usize, + shift1: u32, + shift3: u32, + out: &mut [i16], +) { + let m = T / 2 - 1; + if fx == 0 && fy == 0 { + for r in 0..h { + for (o, &s) in out[r * w..r * w + w].iter_mut().zip(&win[(r + m) * WIN + m..]) { + *o = s << shift3; + } + } + } else if fy == 0 { + for r in 0..h { + filt(&win[(r + m) * WIN..], 1, hf, shift1, &mut out[r * w..], w); + } + } else if fx == 0 { + for r in 0..h { + filt(&win[r * WIN + m..], WIN, vf, shift1, &mut out[r * w..], w); + } + } else { + for r in 0..h + T - 1 { + filt(&win[r * WIN..], 1, hf, shift1, &mut tmp[r * WIN..], w); + } + for r in 0..h { + filt(&tmp[r * WIN..], WIN, vf, 6, &mut out[r * w..], w); + } + } +} + /// Luma prediction samples (14-bit intermediate precision) of a w x h block at (x, y) displaced by the /// quarter-sample motion vector `mv`, into `out` (stride w). -pub fn mc_luma(f: &Frame, x: i32, y: i32, mv: [i16; 2], w: usize, h: usize, out: &mut [i16]) { +#[allow(clippy::too_many_arguments)] +pub fn mc_luma(f: &Frame, x: i32, y: i32, mv: [i16; 2], w: usize, h: usize, out: &mut [i16], sc: &mut McScratch) { let bd = f.bit_depth; let (fx, fy) = ((mv[0] & 3) as usize, (mv[1] & 3) as usize); let xi = x + (mv[0] as i32 >> 2); let yi = y + (mv[1] as i32 >> 2); let shift1 = bd.min(12) - 8; let shift3 = 14 - bd; - let mut win = [0i16; WIN * WIN]; - if fx == 0 && fy == 0 { - f.luma_window(xi, yi, w, h, &mut win, WIN); - for r in 0..h { - for c in 0..w { - out[r * w + c] = win[r * WIN + c] << shift3; - } - } - return; - } - f.luma_window(xi - 3, yi - 3, w + 7, h + 7, &mut win, WIN); - let hf = &LUMA_FILTER[fx]; - let vf = &LUMA_FILTER[fy]; - if fy == 0 { - for r in 0..h { - let row = &win[(r + 3) * WIN..]; - for c in 0..w { - let s: i32 = (0..8).map(|k| hf[k] as i32 * row[c + k] as i32).sum(); - out[r * w + c] = (s >> shift1) as i16; - } - } - } else if fx == 0 { - for r in 0..h { - for c in 0..w { - let s: i32 = (0..8).map(|k| vf[k] as i32 * win[(r + k) * WIN + c + 3] as i32).sum(); - out[r * w + c] = (s >> shift1) as i16; - } - } - } else { - let mut tmp = [0i16; WIN * WIN]; - for r in 0..h + 7 { - let row = &win[r * WIN..]; - for c in 0..w { - let s: i32 = (0..8).map(|k| hf[k] as i32 * row[c + k] as i32).sum(); - tmp[r * WIN + c] = (s >> shift1) as i16; - } - } - for r in 0..h { - for c in 0..w { - let s: i32 = (0..8).map(|k| vf[k] as i32 * tmp[(r + k) * WIN + c] as i32).sum(); - out[r * w + c] = (s >> 6) as i16; - } - } - } + f.luma_window(xi - 3, yi - 3, w + 7, h + 7, &mut sc.win, WIN); + interpolate::<8>(&sc.win, &mut sc.tmp, fx, fy, &LUMA_FILTER[fx], &LUMA_FILTER[fy], w, h, shift1, shift3, out); } /// Chroma prediction samples of a w x h chroma block at chroma position (x, y) with a 1/8-sample /// vector (4:2:0). -pub fn mc_chroma(f: &Frame, c: usize, x: i32, y: i32, mv: [i16; 2], w: usize, h: usize, out: &mut [i16]) { +#[allow(clippy::too_many_arguments)] +pub fn mc_chroma(f: &Frame, c: usize, x: i32, y: i32, mv: [i16; 2], w: usize, h: usize, out: &mut [i16], sc: &mut McScratch) { let bd = f.bit_depth_c; let (fx, fy) = ((mv[0] & 7) as usize, (mv[1] & 7) as usize); let xi = x + (mv[0] as i32 >> 3); let yi = y + (mv[1] as i32 >> 3); let shift1 = bd.min(12) - 8; let shift3 = 14 - bd; - let mut win = [0i16; WIN * WIN]; - if fx == 0 && fy == 0 { - f.chroma_window(c, xi, yi, w, h, &mut win, WIN); - for r in 0..h { - for cc in 0..w { - out[r * w + cc] = win[r * WIN + cc] << shift3; - } - } - return; - } - f.chroma_window(c, xi - 1, yi - 1, w + 3, h + 3, &mut win, WIN); - let hf = &CHROMA_FILTER[fx]; - let vf = &CHROMA_FILTER[fy]; - if fy == 0 { - for r in 0..h { - let row = &win[(r + 1) * WIN..]; - for cc in 0..w { - let s: i32 = (0..4).map(|k| hf[k] as i32 * row[cc + k] as i32).sum(); - out[r * w + cc] = (s >> shift1) as i16; - } - } - } else if fx == 0 { - for r in 0..h { - for cc in 0..w { - let s: i32 = (0..4).map(|k| vf[k] as i32 * win[(r + k) * WIN + cc + 1] as i32).sum(); - out[r * w + cc] = (s >> shift1) as i16; - } - } - } else { - let mut tmp = [0i16; WIN * WIN]; - for r in 0..h + 3 { - let row = &win[r * WIN..]; - for cc in 0..w { - let s: i32 = (0..4).map(|k| hf[k] as i32 * row[cc + k] as i32).sum(); - tmp[r * WIN + cc] = (s >> shift1) as i16; - } - } - for r in 0..h { - for cc in 0..w { - let s: i32 = (0..4).map(|k| vf[k] as i32 * tmp[(r + k) * WIN + cc] as i32).sum(); - out[r * w + cc] = (s >> 6) as i16; - } - } - } + f.chroma_window(c, xi - 1, yi - 1, w + 3, h + 3, &mut sc.win, WIN); + interpolate::<4>(&sc.win, &mut sc.tmp, fx, fy, &CHROMA_FILTER[fx], &CHROMA_FILTER[fy], w, h, shift1, shift3, out); } /// Default weighted prediction, one list (8-262). @@ -121,9 +133,10 @@ pub fn put_uni(p: &[i16], w: usize, h: usize, bd: u32, dst: &mut [u16], ds: usiz let shift = 14 - bd; let off = if shift > 0 { 1 << (shift - 1) } else { 0 }; let max = (1i32 << bd) - 1; - for r in 0..h { - for c in 0..w { - dst[r * ds + c] = ((p[r * w + c] as i32 + off) >> shift).clamp(0, max) as u16; + // Rows as slices and min / max clipping (`clamp` asserts its bounds) so the loop vectorises. + for (d, s) in dst.chunks_mut(ds).zip(p[..w * h].chunks(w)) { + for (d, &s) in d[..w].iter_mut().zip(s) { + *d = ((s as i32 + off) >> shift).max(0).min(max) as u16; } } } @@ -133,10 +146,9 @@ pub fn put_bi(p0: &[i16], p1: &[i16], w: usize, h: usize, bd: u32, dst: &mut [u1 let shift = 15 - bd; let off = 1 << (shift - 1); let max = (1i32 << bd) - 1; - for r in 0..h { - for c in 0..w { - let i = r * w + c; - dst[r * ds + c] = ((p0[i] as i32 + p1[i] as i32 + off) >> shift).clamp(0, max) as u16; + for ((d, a), b) in dst.chunks_mut(ds).zip(p0[..w * h].chunks(w)).zip(p1[..w * h].chunks(w)) { + for ((d, &a), &b) in d[..w].iter_mut().zip(a).zip(b) { + *d = ((a as i32 + b as i32 + off) >> shift).max(0).min(max) as u16; } } } @@ -148,7 +160,7 @@ pub fn put_weighted_uni(p: &[i16], w: usize, h: usize, bd: u32, log2wd: u32, wt: for c in 0..w { let v = p[r * w + c] as i32 * wt; let v = if log2wd >= 1 { ((v + (1 << (log2wd - 1))) >> log2wd) + o } else { v + o }; - dst[r * ds + c] = v.clamp(0, max) as u16; + dst[r * ds + c] = v.max(0).min(max) as u16; } } } @@ -160,7 +172,7 @@ pub fn put_weighted_bi(p0: &[i16], p1: &[i16], w: usize, h: usize, bd: u32, log2 for c in 0..w { let i = r * w + c; let v = (p0[i] as i32 * w0 + p1[i] as i32 * w1 + ((o0 + o1 + 1) << log2wd)) >> (log2wd + 1); - dst[r * ds + c] = v.clamp(0, max) as u16; + dst[r * ds + c] = v.max(0).min(max) as u16; } } } diff --git a/crates/hevc/src/lib.rs b/crates/hevc/src/lib.rs index 3cc3f711..975e0c4f 100644 --- a/crates/hevc/src/lib.rs +++ b/crates/hevc/src/lib.rs @@ -125,4 +125,7 @@ pub struct Picture { pub color: ColorInfo, /// Sample aspect ratio (0, 0 when unspecified). pub sar: (u16, u16), + /// Decoded in draft mode without deblocking / SAO ([`Decoder::set_draft`]): approximate, + /// for reduced-resolution playback only. Never set without draft mode. + pub draft: bool, } diff --git a/crates/hevc/src/slicedec.rs b/crates/hevc/src/slicedec.rs index 354cf9d8..4cd9deb8 100644 --- a/crates/hevc/src/slicedec.rs +++ b/crates/hevc/src/slicedec.rs @@ -132,6 +132,8 @@ pub struct PicState { /// Scaling factors [sizeId][matrixId] when scaling lists are enabled. pub scaling: Option>>>, pub deblocking_enabled_anywhere: bool, + /// Draft mode: no deblocking or SAO. + pub draft: bool, } impl PicState { @@ -176,6 +178,7 @@ impl PicState { ds_ctx: None, scaling, deblocking_enabled_anywhere: false, + draft: false, } } @@ -259,6 +262,7 @@ pub struct SliceDecoder<'a> { pub coeffs: Vec, pub pred0: Vec, pub pred1: Vec, + pub mc: crate::inter::McScratch, pub refs_buf: Refs, /// Collocated picture for TMVP. pub col: Option, @@ -279,11 +283,14 @@ impl<'a> SliceDecoder<'a> { } #[inline] fn bypass_bits(&mut self, n: u32) -> u32 { - let mut v = 0; - for _ in 0..n { - v = (v << 1) | self.c.decode_bypass(); + let mut v = 0u32; + let mut left = n; + while left > 16 { + v = (v << 16) | self.c.decode_bypass_bits(16); + left -= 16; } - v + // (bits beyond 32 shift out, as the bin-by-bin loop did) + v.checked_shl(left).unwrap_or(0) | self.c.decode_bypass_bits(left) } /// Decode the slice segment `job` into `pic`. @@ -376,6 +383,7 @@ impl<'a> SliceDecoder<'a> { coeffs: vec![0; 32 * 32], pred0: vec![0; 64 * 64], pred1: vec![0; 64 * 64], + mc: Default::default(), refs_buf: Refs::default(), col, no_backward_pred, @@ -1449,9 +1457,9 @@ impl<'a> SliceDecoder<'a> { let frame = &self.refs[l][f.ref_idx[l] as usize].frame; let buf = if l == 0 || !bi { &mut self.pred0 } else { &mut self.pred1 }; if c == 0 { - inter::mc_luma(frame, x, y, f.mv[l], bw, bh, buf); + inter::mc_luma(frame, x, y, f.mv[l], bw, bh, buf, &mut self.mc); } else { - inter::mc_chroma(frame, c - 1, x, y, f.mv[l], bw, bh, buf); + inter::mc_chroma(frame, c - 1, x, y, f.mv[l], bw, bh, buf, &mut self.mc); } } let stride = if c == 0 { self.pic.width } else { self.pic.cwidth }; diff --git a/crates/hevc/src/synth_tests.rs b/crates/hevc/src/synth_tests.rs index 44b8a828..aee0419f 100644 --- a/crates/hevc/src/synth_tests.rs +++ b/crates/hevc/src/synth_tests.rs @@ -28,7 +28,7 @@ struct Enc { impl Enc { fn new(w: BitWriter, qp: i32, init_type: usize) -> Self { - let mut ctx = [0u8; NUM_CTX]; + let mut ctx: Contexts = [0u8; NUM_CTX.next_power_of_two()]; init_contexts(&mut ctx, qp, init_type); Enc { w, low: 0, range: 510, outstanding: 0, first: true, ctx } } diff --git a/crates/hevc/tests/common/mod.rs b/crates/hevc/tests/common/mod.rs index 56d36829..a8dcb8da 100644 --- a/crates/hevc/tests/common/mod.rs +++ b/crates/hevc/tests/common/mod.rs @@ -203,8 +203,14 @@ pub type DecodeFailure = (usize, filmcraft_hevc::Error, Vec); /// Decode with `threads` worker threads (0 = decoder default), pts = access unit index. pub fn decode_file_threads(path: &Path, threads: usize) -> Result, DecodeFailure> { + decode_file_opts(path, threads, false) +} + +/// [`decode_file_threads`] with draft mode on or off. +pub fn decode_file_opts(path: &Path, threads: usize, draft: bool) -> Result, DecodeFailure> { let data = std::fs::read(path).unwrap(); let mut dec = if threads == 0 { Decoder::new() } else { Decoder::with_threads(threads) }; + dec.set_draft(draft); let mut out = Vec::new(); for (i, au) in split_access_units(&data).into_iter().enumerate() { match dec.decode(au, i as i64) { @@ -278,7 +284,8 @@ pub fn check_fixture(name: &str) -> Result { let f = fixture(name); let Some((hevc, yuv)) = ensure(f) else { return Ok(false) }; let reference = std::fs::read(&yuv).unwrap(); - for threads in [1, 0] { + // Single-threaded, frame threads with a small pool, and every core. + for threads in [1, 3, 0] { let pics = match decode_file_threads(&hevc, threads) { Ok(p) => p, Err((au, e, partial)) => { @@ -288,6 +295,9 @@ pub fn check_fixture(name: &str) -> Result { }; compare(&pics, &reference, f.width as usize, f.height as usize, f.bit_depth).map_err(|e| format!("{name} (threads {threads}): {e}"))?; check_pts(&pics).map_err(|e| format!("{name} (threads {threads}): {e}"))?; + if pics.iter().any(|p| p.draft) { + return Err(format!("{name} (threads {threads}): picture flagged draft without draft mode")); + } } Ok(true) } diff --git a/crates/hevc/tests/conformance.rs b/crates/hevc/tests/conformance.rs index a6109ddf..4fa8b8cb 100644 --- a/crates/hevc/tests/conformance.rs +++ b/crates/hevc/tests/conformance.rs @@ -62,6 +62,46 @@ fixture_tests!( vt_main10, ); +/// Draft mode: sub-layer non-reference pictures skip deblocking and SAO and are flagged; every +/// other picture stays bit-exact, single-threaded and frame-threaded. +#[test] +fn draft_mode_changes_only_flagged_non_reference_pictures() { + let mut checked = 0; + for name in ["bframes", "medium", "slow", "weightp_main10"] { + let f = common::fixture(name); + let Some((hevc, yuv)) = common::ensure(f) else { + eprintln!("skipped {name}"); + continue; + }; + let reference = std::fs::read(&yuv).unwrap(); + let (w, h) = (f.width as usize, f.height as usize); + let bps = if f.bit_depth > 8 { 2 } else { 1 }; + let fsize = (w * h + 2 * w.div_ceil(2) * h.div_ceil(2)) * bps; + for threads in [1, 3] { + let pics = common::decode_file_opts(&hevc, threads, true).unwrap_or_else(|(au, e, _)| panic!("{name}: error at {au}: {e}")); + common::check_pts(&pics).unwrap_or_else(|e| panic!("{name}: {e}")); + assert_eq!(pics.len() * fsize, reference.len(), "{name}: picture count"); + let (mut draft, mut changed) = (0, 0); + for (i, p) in pics.iter().enumerate() { + let bytes: Vec = [&p.y, &p.u, &p.v].iter().flat_map(|pl| pl.to_le_bytes()).collect(); + let same = bytes == reference[i * fsize..(i + 1) * fsize]; + if p.draft { + draft += 1; + changed += !same as usize; + } else { + assert!(same, "{name} (threads {threads}): picture {i} (not draft) differs from the reference"); + } + } + assert!(draft > 0, "{name}: has non-reference pictures"); + assert!(changed > 0, "{name}: skipping the in-loop filters changes some draft picture"); + checked += 1; + } + } + if checked == 0 { + eprintln!("skipped: no fixtures"); + } +} + /// Print which coding tools each fixture exercises (run with `-- --ignored --nocapture`). #[test] #[ignore]