From 74b151cedc42baed560e253da62dafe24ace7409 Mon Sep 17 00:00:00 2001 From: daijro Date: Fri, 9 Oct 2026 11:34:28 -0500 Subject: [PATCH] Footage: reduced-resolution renders get movie frames converted at their size Cause: at Half, Third and Quarter resolution (the viewer's Auto resolution in most windows) every movie frame was converted from Y'CbCr to f32 RGBA at full size and then resampled down. In the browser (wasm, one thread) that was about 8 ms converting a 1080p frame, 6 ms resampling it and 2 ms zeroing a 33 MB buffer, per layer and frame, for a frame shown at a quarter or a ninth of its pixels; the footage frame cache held the full-size frames. Fix: FootageSource::frame_at_size asks for a frame shown at a given size (default: the full frame, as before). The renderer asks for it where it resampled footage (below 3/4 scale; not for proxies, fields or Interpret Footage's pixel options, which need the full frame). MediaPool makes opaque 4:2:0 movie frames of the footage's own size there: it box-filters them to half size in Y'CbCr before converting (each 2x2 block of luma, and the [1 6 1]/8 neighbourhood of its chroma sample, which is the mean of the upsampled chroma its pixels get), resamples the rest of the way and caches the result per size. Conversion is linear until colours clip, so this is the RGB box filter resample starts with, except in blocks where some colours clip. Other frames convert at full size and resample as before. In the browser the conversion and resampling take about 4 ms instead of 16. Docs (architecture). Tests: - convert::scaled_conversion_matches_resampling_the_converted_frame: half, third, quarter and odd sizes of an unclipped 4:2:0 frame are within 1e-5 of resampling the converted frame; larger sizes, odd frame sizes and 4:4:4 convert at full size. - tests_footage_size::reduced_renders_take_footage_resampled_by_the_source: a source that resamples frames itself renders the same frames as one that doesn't, at 8 and 32 bpc and every resolution, and is asked only below 3/4 scale, and not with Invert Alpha. - footage::h264_frames_at_reduced_sizes (ffmpeg fixture): 1080p H.264 testsrc2 frames, whose saturated colours clip, at 960x540, 640x360, 480x270 and 323x182 are 58-62 dB from the resampled full frames, and are cached per size. - convert's ignored 1080p bench also times the conversion at a third. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/media/src/convert.rs | 107 ++++++++++++++++++++++++ crates/media/src/pool.rs | 71 +++++++++++----- crates/media/tests/footage.rs | 20 +++++ crates/render/src/lib.rs | 47 ++++++++--- crates/render/src/tests_footage_size.rs | 83 ++++++++++++++++++ docs/architecture.md | 6 +- 6 files changed, 298 insertions(+), 36 deletions(-) create mode 100644 crates/render/src/tests_footage_size.rs diff --git a/crates/media/src/convert.rs b/crates/media/src/convert.rs index 114873e7..d9ef9323 100644 --- a/crates/media/src/convert.rs +++ b/crates/media/src/convert.rs @@ -104,6 +104,67 @@ pub(crate) fn frame_to_image_in(f: &VideoFrame, op: AlphaOp, mut buf: Vec) - img } +/// A FilmCraft frame for a render that shows it at `w` × `h`, at most half its size each way: +/// what `resample(&frame_to_image(f, op), w, h)` makes, converting a quarter of the pixels. +/// Opaque 4:2:0 Y'CbCr is box-filtered to half size before conversion (each 2×2 block of luma, +/// and the [1 6 1]/8 neighbourhood of its chroma sample: the mean of the upsampled chroma its +/// pixels get), then resampled the rest of the way. Conversion is linear until it clips, so +/// this is the RGB box filter `resample` starts with, except in blocks where some colours clip. +/// `None` for other frames. +pub(crate) fn frame_to_image_scaled(f: &VideoFrame, w: u32, h: u32) -> Option { + let (fw, fh) = (f.width as usize, f.height as usize); + if w == 0 || h == 0 || fw % 2 != 0 || fh % 2 != 0 || (w as usize).saturating_mul(2) > fw || (h as usize).saturating_mul(2) > fh { + return None; + } + let half = match &f.data { + PixelData::Yuv8 { planes, chroma: Chroma::C420, alpha: None } => { + half_420(fw, fh, [&planes[0][..], &planes[1][..], &planes[2][..]], &Yuv::new(f.color.matrix, f.color.range, 8))? + } + PixelData::Yuv16 { planes, chroma: Chroma::C420, bits, alpha: None } => { + half_420(fw, fh, [&planes[0][..], &planes[1][..], &planes[2][..]], &Yuv::new(f.color.matrix, f.color.range, *bits))? + } + _ => return None, + }; + Some(if (half.width, half.height) == (w, h) { half } else { effectcraft_raster::resample(&half, w, h) }) +} + +/// An opaque 4:2:0 frame of even size `w` × `h` converted at half that size (see +/// [`frame_to_image_scaled`]); `None` when the planes are too short. +fn half_420(w: usize, h: usize, [luma, cb, cr]: [&[T]; 3], k: &Yuv) -> Option { + /// Row `r` of a plane `n` samples wide (empty past its end). + fn row_of(p: &[T], r: usize, n: usize) -> &[T] { + p.get(r.saturating_mul(n)..(r + 1).saturating_mul(n)).unwrap_or_default() + } + let (cw, ch) = (w / 2, h / 2); + if luma.len() < w.checked_mul(h)? || cb.len() < cw * ch || cr.len() < cw * ch { + return None; + } + let mut img = Image::new(cw as u32, ch as u32); + img.data.par_chunks_mut(cw).enumerate().for_each_init( + // Chroma rows with their edge samples repeated on both ends, for the horizontal [1 6 1]. + || (vec![0f32; cw + 2], vec![0f32; cw + 2]), + |(uc, vc), (cy, row)| { + // The chroma rows above and below weigh 1/8, as `centred` upsampling averages out. + let rows = [cy.saturating_sub(1), cy, (cy + 1).min(ch - 1)]; + for (dst, plane) in [(&mut *uc, cb), (&mut *vc, cr)] { + let [above, at, below] = rows.map(|r| row_of(plane, r, cw)); + for (d, ((a, b), c)) in dst[1..=cw].iter_mut().zip(above.iter().zip(at).zip(below)) { + *d = ((a.f() + c.f()) * 0.125 + b.f() * 0.75 - k.c_off) * k.c_scale; + } + (dst[0], dst[cw + 1]) = (dst[1], dst[cw]); + } + let (top, bottom) = (row_of(luma, 2 * cy, w), row_of(luma, 2 * cy + 1, w)); + let pixels = top.chunks_exact(2).zip(bottom.chunks_exact(2)).zip(uc.windows(3).zip(vc.windows(3))); + for (o, ((a, b), (u, v))) in row.iter_mut().zip(pixels) { + let (u, v) = ((u[0] + u[2]) * 0.125 + u[1] * 0.75, (v[0] + v[2]) * 0.125 + v[1] * 0.75); + let yy = ((a[0].f() + a[1].f() + b[0].f() + b[1].f()) * 0.25 - k.y_off) * k.y_scale; + *o = [(yy + k.cr_r * v).clamp(0.0, 1.0), (yy - k.cr_g * v - k.cb_g * u).clamp(0.0, 1.0), (yy + k.cb_b * u).clamp(0.0, 1.0), 1.0]; + } + }, + ); + Some(img) +} + /// 0..255 → 0..1. static U8: [f32; 256] = { let mut t = [0f32; 256]; @@ -338,6 +399,47 @@ mod tests { assert!((img.data[0][0] - 110.0 / 219.0).abs() < 1e-5); } + /// A smooth opaque 4:2:0 frame whose colours never clip. + fn smooth_420(w: usize, h: usize) -> VideoFrame { + let plane = |w: usize, h: usize, base: f32, amp: f32, fx: f32, fy: f32| -> Arc> { + Arc::new((0..w * h).map(|i| (base + amp * ((i % w) as f32 * fx + (i / w) as f32 * fy).sin()) as u8).collect()) + }; + VideoFrame { + width: w as u32, + height: h as u32, + data: PixelData::Yuv8 { + planes: [plane(w, h, 125.0, 45.0, 0.31, 0.17), plane(w / 2, h / 2, 128.0, 10.0, 0.4, -0.3), plane(w / 2, h / 2, 128.0, 10.0, -0.2, 0.5)], + chroma: Chroma::C420, + alpha: None, + }, + color: filmcraft_color::ColorInfo::REC709, + par: (1, 1), + pts: filmcraft_time::Tick::ZERO, + } + } + + #[test] + fn scaled_conversion_matches_resampling_the_converted_frame() { + let f = smooth_420(96, 64); + let full = frame_to_image(&f, AlphaOp::new(AlphaMode::Straight, [0.0; 3])); + for (w, h) in [(48, 32), (32, 21), (24, 16), (13, 9)] { + let scaled = frame_to_image_scaled(&f, w, h).expect("4:2:0, at most half size"); + let want = effectcraft_raster::resample(&full, w, h); + assert_eq!((scaled.width, scaled.height), (w, h)); + let worst = scaled.data.iter().zip(&want.data).flat_map(|(a, b)| (0..4).map(move |c| (a[c] - b[c]).abs())).fold(0.0f32, f32::max); + assert!(worst < 1e-5, "{w}×{h}: {worst}"); + } + // More than half the size, odd sizes, other chroma layouts: convert the full frame. + assert!(frame_to_image_scaled(&f, 49, 32).is_none()); + assert!(frame_to_image_scaled(&smooth_420(95, 64), 40, 30).is_none()); + assert!(frame_to_image_scaled(&yuv_frame(100, 128, 128, filmcraft_color::ColorInfo::REC709), 2, 1).is_some()); + let mut f444 = smooth_420(96, 64); + if let PixelData::Yuv8 { chroma, .. } = &mut f444.data { + *chroma = Chroma::C444; + } + assert!(frame_to_image_scaled(&f444, 48, 32).is_none()); + } + #[test] fn bt709_red() { // 8-bit limited BT.709 red: Y 63, Cb 102, Cr 240 @@ -402,6 +504,11 @@ mod tests { keep.push(frame_to_image(&f, op)); } eprintln!("convert into fresh memory {:?}", t.elapsed() / 20); + let t = std::time::Instant::now(); + for _ in 0..20 { + std::hint::black_box(frame_to_image_scaled(&f, 640, 360)); + } + eprintln!("convert at a third {:?}", t.elapsed() / 20); } } } diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 3df9ec45..f10389fd 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -10,7 +10,7 @@ use effectcraft_render::FootageSource; use effectcraft_time::{FrameRate, TICKS_PER_SECOND, Tick}; use filmcraft_media::{FrameRequest, SharedSource}; -use crate::convert::{AlphaOp, dynamic_to_image, frame_to_image_in}; +use crate::convert::{AlphaOp, dynamic_to_image, frame_to_image_in, frame_to_image_scaled}; use crate::{MediaError, Result}; /// Default frame-cache budget: 1 GiB of decoded frames. @@ -41,6 +41,8 @@ struct Key { /// pixels. interp: u8, matte: [u32; 3], + /// A movie frame made at this size for a reduced-resolution render ((0, 0): as decoded). + size: (u32, u32), } struct Entry { @@ -322,7 +324,21 @@ impl MediaPool { /// Decode (or fetch from cache) the frame of `footage` at source time `t`. pub fn frame_at(&self, footage: &Footage, t: Tick) -> Result> { - Inner::frame_at(&self.inner, footage, t, true) + Inner::frame_at(&self.inner, footage, t, true, None) + } + + /// [`FootageSource::frame`] / [`FootageSource::frame_at_size`]: `None` (logged) on errors. + fn frame_sized(&self, _item: ItemId, footage: &Footage, t: Tick, size: Option<(u32, u32)>) -> Option> { + if footage.missing { + return None; + } + match Inner::frame_at(&self.inner, footage, t, true, size) { + Ok(img) => Some(img), + Err(e) => { + log::warn!("media: frame of {} at {:?}: {e}", footage.path, t); + None + } + } } /// Settings ▸ Disk ▸ Conformed Audio Folder: write each footage file's decoded audio there @@ -469,7 +485,7 @@ impl MediaPool { /// A thumbnail of `footage` (its first frame) fitting in `max_side` × `max_side`, aspect kept. pub fn thumbnail(&self, footage: &Footage, max_side: u32) -> Result { - let img = Inner::frame_at(&self.inner, footage, Tick::ZERO, false)?; + let img = Inner::frame_at(&self.inner, footage, Tick::ZERO, false, None)?; let (w, h) = (img.width.max(1), img.height.max(1)); let s = (max_side.max(1) as f64 / w.max(h) as f64).min(1.0); if s >= 1.0 { @@ -561,7 +577,7 @@ impl Inner { }; let interp = alpha | (u8::from(footage.preserve_rgb) << 2); let matte = if footage.alpha == AlphaMode::Premultiplied { footage.premul_color.map(f32::to_bits) } else { [0; 3] }; - let key = |path: &str, frame| Key { path: path.into(), frame, interp, matte }; + let key = |path: &str, frame| Key { path: path.into(), frame, interp, matte, size: (0, 0) }; match footage.kind { FootageKind::Sequence if !footage.sequence.is_empty() => { let i = Self::frame_index(footage.frame_rate, t, footage.sequence_frames(), footage.loop_count); @@ -591,21 +607,27 @@ impl Inner { } } - fn frame_at(self: &Arc, footage: &Footage, t: Tick, playback: bool) -> Result> { + /// The frame of `footage` at `t`; a movie's resampled to `size` when it can be made at that + /// size directly (see [`FootageSource::frame_at_size`]). + fn frame_at(self: &Arc, footage: &Footage, t: Tick, playback: bool, size: Option<(u32, u32)>) -> Result> { if !footage.has_video { return Err(MediaError::Unsupported(format!("{}: no video", footage.path))); } - let loc = Self::locate(footage, t); + let mut loc = Self::locate(footage, t); let movie = loc.media_t.is_some(); + if movie && let Some(size) = size { + loc.key.size = size; + } let mut c = lock(&self.cache); // sequential playback of a movie: read the next frame ahead if playback && movie { let prev = c.last.insert(loc.key.path.clone(), loc.key.frame); if prev == Some(loc.key.frame - 1) && self.read_ahead.load(Ordering::Relaxed) { let next_t = t + footage.frame_rate.frame_duration(); - let next = Self::locate(footage, next_t); + let mut next = Self::locate(footage, next_t); + next.key.size = loc.key.size; if next.key != loc.key && !c.lru.map.contains_key(&next.key) && !c.inflight.contains(&next.key) { - self.prefetch(footage, next_t); + self.prefetch(footage, next_t, size); } } } @@ -638,17 +660,17 @@ impl Inner { } #[cfg(not(target_arch = "wasm32"))] - fn prefetch(self: &Arc, footage: &Footage, t: Tick) { + fn prefetch(self: &Arc, footage: &Footage, t: Tick, size: Option<(u32, u32)>) { let (me, f) = (self.clone(), footage.clone()); self.prefetched.fetch_add(1, Ordering::Relaxed); // A plain thread, not a rayon job: the decode below parallelises with rayon itself. std::thread::spawn(move || { - let _ = me.frame_at(&f, t, false); + let _ = me.frame_at(&f, t, false, size); }); } #[cfg(target_arch = "wasm32")] - fn prefetch(self: &Arc, _footage: &Footage, _t: Tick) {} + fn prefetch(self: &Arc, _footage: &Footage, _t: Tick, _size: Option<(u32, u32)>) {} fn decode(&self, loc: &Loc, footage: &Footage) -> Result { let op = AlphaOp::of(footage); @@ -670,6 +692,15 @@ impl Inner { Some(mt) => { let src = self.source(path).ok_or_else(|| MediaError::Io(format!("{path}: cannot open")))?; let vf = src.video_frame(FrameRequest::full(filmcraft_time::Tick(mt.0))).map_err(MediaError::from)?; + // A reduced-resolution render's frame, made at its size (a frame of the footage's + // own size only, as the renderer resamples only those). + let (w, h) = loc.key.size; + if w > 0 + && (vf.width, vf.height) == (footage.width, footage.height) + && let Some(img) = frame_to_image_scaled(&vf, w, h) + { + return Ok(img); + } let buf = lock(&self.cache).lru.take_spare(vf.width as usize * vf.height as usize); Ok(frame_to_image_in(&vf, op, buf)) } @@ -696,17 +727,11 @@ impl FootageSource for MediaPool { fn cache_budget(&self) -> Option { Some(self.budget()) } - fn frame(&self, _item: ItemId, footage: &Footage, t: Tick) -> Option> { - if footage.missing { - return None; - } - match self.frame_at(footage, t) { - Ok(img) => Some(img), - Err(e) => { - log::warn!("media: frame of {} at {:?}: {e}", footage.path, t); - None - } - } + fn frame(&self, item: ItemId, footage: &Footage, t: Tick) -> Option> { + self.frame_sized(item, footage, t, None) + } + fn frame_at_size(&self, item: ItemId, footage: &Footage, t: Tick, width: u32, height: u32) -> Option> { + self.frame_sized(item, footage, t, Some((width, height))) } fn vector_frame(&self, _item: ItemId, footage: &Footage, scale: f64) -> Option> { @@ -834,7 +859,7 @@ mod tests { fn lru_respects_budget_and_recycles() { let mut c = Lru::default(); let img = || Arc::new(Image::new(16, 16)); // 4 KiB + overhead - let k = |i| Key { path: "a".into(), frame: i, interp: 0, matte: [0; 3] }; + let k = |i| Key { path: "a".into(), frame: i, interp: 0, matte: [0; 3], size: (0, 0) }; let budget = 3 * (4096 + 64); for i in 0..10 { c.insert(k(i), img(), budget); diff --git a/crates/media/tests/footage.rs b/crates/media/tests/footage.rs index ed113c35..8d49f087 100644 --- a/crates/media/tests/footage.rs +++ b/crates/media/tests/footage.rs @@ -236,6 +236,26 @@ fn h264_mp4_probe_and_frames() { assert!(Arc::ptr_eq(&a, &b)); } +/// A reduced-resolution render's movie frames are converted at its size: they match resampling +/// the full frames (testsrc2's saturated colours clip, where the two differ most), and are cached +/// per size. +#[test] +fn h264_frames_at_reduced_sizes() { + let Some(p) = h264_mp4() else { return }; + let f = probe(&p).expect("probe"); + let pool = MediaPool::new(); + for (n, (w, h)) in [(0i64, (960, 540)), (1, (640, 360)), (2, (480, 270)), (30, (323, 182))] { + let t = t_of(f.frame_rate, n); + let sized = pool.frame_at_size(ItemId(1), &f, t, w, h).expect("frame"); + let full = pool.frame(ItemId(1), &f, t).expect("frame"); + assert_eq!((sized.width, sized.height), (w, h)); + let db = psnr(&sized, &effectcraft_raster::resample(&full, w, h)); + eprintln!("h264 frame {n} at {w}×{h}: {db:.2} dB against the resampled full frame"); + assert!(db > 55.0, "frame {n} at {w}×{h}: {db:.2} dB"); + assert!(Arc::ptr_eq(&sized, &pool.frame_at_size(ItemId(1), &f, t, w, h).expect("frame")), "cached"); + } +} + #[test] fn h264_audio_samples() { let Some(p) = h264_mp4() else { return }; diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 591213e6..324e2a5f 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -40,6 +40,13 @@ use rayon::prelude::*; pub trait FootageSource: Send + Sync { /// The frame of `item` at source time `t`, straight from the file (any size). fn frame(&self, item: ItemId, footage: &Footage, t: Tick) -> Option>; + /// [`FootageSource::frame`] for a render that shows it at `width` × `height`, at most half its + /// size: either that frame or, when the source can make it more cheaply than the whole frame + /// (movies, see `effectcraft-media`), the frame already resampled to that size as + /// [`effectcraft_raster::resample`] would. + fn frame_at_size(&self, item: ItemId, footage: &Footage, t: Tick, _width: u32, _height: u32) -> Option> { + self.frame(item, footage, t) + } /// `frames` interleaved stereo sample frames of `item`'s audio from source time `t` at /// `rate` Hz (`None` when unavailable). fn audio(&self, _item: ItemId, _footage: &Footage, _t: Tick, _frames: usize, _rate: u32) -> Option> { @@ -1104,8 +1111,10 @@ impl<'a> Renderer<'a> { } let lt = ctx.source_time(layer); // The proxy is decoded instead, at its own size, and fills the footage's frame. - let pf = self.proxy_for(*item).unwrap_or(f); - let img = self.footage_frame(ctx, layer, *item, pf, lt)?; + let proxy = self.proxy_for(*item); + let pf = proxy.unwrap_or(f); + let size = self.reduced_size(f.width, f.height).filter(|_| proxy.is_none()); + let img = self.footage_frame(ctx, layer, *item, pf, lt, size)?; Some(self.footage_buf(&img, pf, f.width, f.height)) } LayerSource::Text => Some(self.authored(text::render(ctx, layer, self.raster_scale(ctx, layer)))), @@ -1127,12 +1136,11 @@ impl<'a> Renderer<'a> { let s = self.opts.scale; let k = if img.width > 0 { w.max(1) as f64 / img.width as f64 } else { 1.0 }; // Keep native pixels when downsampling is small; resample otherwise. - let mut buf = if s < 0.75 && k <= 1.0 + 1e-9 { - let rw = ((w as f64 * s).round() as u32).max(1); - let rh = ((h as f64 * s).round() as u32).max(1); - Buf { img: effectcraft_raster::resample(img, rw, rh), offset: [0.0; 2], scale: s } - } else { - Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 / k } + let mut buf = match self.reduced_size(w, h) { + // The source resampled it already (`FootageSource::frame_at_size`). + Some(size) if (img.width, img.height) == size => Buf { img: img.clone(), offset: [0.0; 2], scale: s }, + Some((rw, rh)) if k <= 1.0 + 1e-9 => Buf { img: effectcraft_raster::resample(img, rw, rh), offset: [0.0; 2], scale: s }, + _ => Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 / k }, }; if let Some(c) = self.pipe.media_in(f) { color::convert(&mut buf.img, &c); @@ -1140,6 +1148,13 @@ impl<'a> Renderer<'a> { buf } + /// The size of `w` × `h` layer pixels in this render when it is reduced enough that footage + /// is resampled to it (below 3/4 scale). + fn reduced_size(&self, w: u32, h: u32) -> Option<(u32, u32)> { + let s = self.opts.scale; + (s < 0.75).then(|| (((w as f64 * s).round() as u32).max(1), ((h as f64 * s).round() as u32).max(1))) + } + /// Authored colours into the working space (linear working spaces). fn authored(&self, mut buf: Buf) -> Buf { if let Some(c) = self.pipe.authored_in() { @@ -1151,7 +1166,8 @@ impl<'a> Renderer<'a> { /// A footage frame at source time `t`, frame-blended between the two nearest source frames /// when the layer's Frame Blending switch (and the comp's Enable Frame Blending) is on and /// `t` falls between frames (footage rate ≠ comp rate, time stretch, time remapping). - fn footage_frame(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick) -> Option> { + /// `size`: a plain frame may come resampled to it ([`FootageSource::frame_at_size`]). + fn footage_frame(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick, size: Option<(u32, u32)>) -> Option> { // Interpret Footage ▸ Separate Fields: each field is a frame at twice the rate. if f.fields != effectcraft_project::FieldOrder::Off && matches!(f.kind, FootageKind::Video | FootageKind::Sequence) { let field_rate = FrameRate::new(f.frame_rate.num * 2, f.frame_rate.den); @@ -1162,11 +1178,13 @@ impl<'a> Renderer<'a> { let parity = if (i % 2 == 0) == dominant_upper { 0 } else { 1 }; return Some(Arc::new(interpret_pixels(&field_frame(&img, parity), f))); } - let img = self.footage_frame_raw(ctx, layer, item, f, t)?; + // Interpretation is per pixel but not linear: it runs on the frame before resampling. + let size = size.filter(|_| !(f.invert_alpha || f.linear_light_applies())); + let img = self.footage_frame_raw(ctx, layer, item, f, t, size)?; if f.invert_alpha || f.linear_light_applies() { Some(Arc::new(interpret_pixels(&img, f))) } else { Some(img) } } - fn footage_frame_raw(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick) -> Option> { + fn footage_frame_raw(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick, size: Option<(u32, u32)>) -> Option> { let mode = frame_blend_mode(ctx, layer); if mode != FrameBlend::Off && matches!(f.kind, FootageKind::Video | FootageKind::Sequence) { let (i, w) = frame_position(t, f.frame_rate); @@ -1194,7 +1212,10 @@ impl<'a> Renderer<'a> { } } } - self.footage.frame(item, f, t) + match size { + Some((w, h)) => self.footage.frame_at_size(item, f, t, w, h), + None => self.footage.frame(item, f, t), + } } /// Source → masks, clamped/quantised to the project depth. @@ -2152,6 +2173,8 @@ mod tests_collapse; #[cfg(test)] mod tests_color; #[cfg(test)] +mod tests_footage_size; +#[cfg(test)] mod tests_frame_blend; #[cfg(test)] mod tests_mask_blur; diff --git a/crates/render/src/tests_footage_size.rs b/crates/render/src/tests_footage_size.rs new file mode 100644 index 00000000..88df726e --- /dev/null +++ b/crates/render/src/tests_footage_size.rs @@ -0,0 +1,83 @@ +//! Footage at a reduced resolution: a source that resamples frames itself +//! ([`FootageSource::frame_at_size`], movies in `effectcraft-media`) renders what resampling the +//! full frame renders. + +use std::sync::{Arc, Mutex}; + +use rayon::prelude::*; + +use effectcraft_color::Label; +use effectcraft_project::build; +use effectcraft_project::{AlphaMode, BitDepth, Comp, Footage, FootageKind, ItemId, ItemKind, LayerSource, Project}; +use effectcraft_time::{FrameRate, Tick}; + +use crate::{FootageSource, Image, RenderOpts, Renderer}; + +/// Footage with a textured frame; `sizes` (when set) resamples it itself and records the sizes +/// it was asked for. +struct Clip { + sizes: Option>>, +} + +impl FootageSource for Clip { + fn frame(&self, _: ItemId, f: &Footage, _: Tick) -> Option> { + let mut img = Image::new(f.width, f.height); + img.rows_mut().for_each(|(y, row)| { + for (x, p) in row.iter_mut().enumerate() { + let s = (0.5 + 0.3 * (x as f32 * 0.37 + y as f32 * 0.11).sin()) * 0.9; + *p = [s, 1.0 - s, s * 0.5, 1.0]; + } + }); + Some(Arc::new(img)) + } + fn frame_at_size(&self, item: ItemId, f: &Footage, t: Tick, w: u32, h: u32) -> Option> { + let Some(sizes) = &self.sizes else { return self.frame(item, f, t) }; + sizes.lock().unwrap().push((w, h)); + Some(Arc::new(effectcraft_raster::resample(&*self.frame(item, f, t)?, w, h))) + } +} + +fn project(depth: BitDepth, invert_alpha: bool) -> (Project, ItemId) { + let mut p = Project::default(); + p.settings.bit_depth = depth; + let comp = Comp::new(96, 54, FrameRate::FPS_30, Tick::from_seconds_f64(1.0)); + let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.clone().into())); + let f = Footage { + path: "clip.mp4".into(), + kind: FootageKind::Video, + width: 96, + height: 54, + pixel_aspect: 1.0, + frame_rate: FrameRate::FPS_30, + duration: Tick::from_seconds_f64(1.0), + has_video: true, + alpha: AlphaMode::Ignore, + loop_count: 1, + invert_alpha, + ..Default::default() + }; + let fid = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(f)); + let l = build::layer(&mut p, &comp, "clip", LayerSource::Footage { item: fid }, (96, 54), None); + p.comp_mut(cid).unwrap().layers.push(l); + (p, cid) +} + +#[test] +fn reduced_renders_take_footage_resampled_by_the_source() { + for depth in [BitDepth::Bpc8, BitDepth::Bpc32] { + let (p, cid) = project(depth, false); + for (scale, asked) in [(1.0, None), (0.75, None), (0.5, Some((48, 27))), (1.0 / 3.0, Some((32, 18))), (0.25, Some((24, 14)))] { + let opts = RenderOpts { scale, ..Default::default() }; + let plain = Renderer::new(&p, &Clip { sizes: None }, opts).comp_frame(cid, Tick::ZERO); + let sized = Clip { sizes: Some(Mutex::default()) }; + let frame = Renderer::new(&p, &sized, opts).comp_frame(cid, Tick::ZERO); + assert_eq!(frame, plain, "{depth:?} at {scale}"); + assert_eq!(sized.sizes.unwrap().into_inner().unwrap(), asked.into_iter().collect::>(), "{depth:?} at {scale}"); + } + } + // Interpreted pixels (Invert Alpha) are interpreted at full size, then resampled. + let (p, cid) = project(BitDepth::Bpc8, true); + let sized = Clip { sizes: Some(Mutex::default()) }; + Renderer::new(&p, &sized, RenderOpts { scale: 0.5, ..Default::default() }).comp_frame(cid, Tick::ZERO); + assert!(sized.sizes.unwrap().into_inner().unwrap().is_empty()); +} diff --git a/docs/architecture.md b/docs/architecture.md index 33e38215..6a284b8e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -230,7 +230,11 @@ Nulls / Nulls Follow Points take pins as well as paths, and `puppet.follow` give Position expression that trails the leader by a delay. The renderer flattens nested effect groups into `Params` keys (`effects::flatten_params`). Commands: `paint.*`, `puppet.*`. -Half, Third and Quarter resolution render proportionally fewer pixels end to end. +Half, Third and Quarter resolution render proportionally fewer pixels end to end. Footage +starts at the render's size where the source can make it more cheaply than the full frame +(`FootageSource::frame_at_size`): movies in opaque 4:2:0 Y'CbCr are box-filtered to half size +before conversion (a quarter of the conversions; the result differs from converting first only +in blocks where colours clip) and resampled from there, as other frames are from full size. **GPU compositor** (`crates/gpu`; Project Settings ▸ Video Rendering and Effects ▸ Mercury GPU Acceleration, the default, or Mercury Software Only; `render.backend`). The CPU renderer is the