diff --git a/crates/media/src/convert.rs b/crates/media/src/convert.rs index 114873e7..d9ef9323 100644 --- a/crates/media/src/convert.rs +++ b/crates/media/src/convert.rs @@ -104,6 +104,67 @@ pub(crate) fn frame_to_image_in(f: &VideoFrame, op: AlphaOp, mut buf: Vec) - img } +/// A FilmCraft frame for a render that shows it at `w` × `h`, at most half its size each way: +/// what `resample(&frame_to_image(f, op), w, h)` makes, converting a quarter of the pixels. +/// Opaque 4:2:0 Y'CbCr is box-filtered to half size before conversion (each 2×2 block of luma, +/// and the [1 6 1]/8 neighbourhood of its chroma sample: the mean of the upsampled chroma its +/// pixels get), then resampled the rest of the way. Conversion is linear until it clips, so +/// this is the RGB box filter `resample` starts with, except in blocks where some colours clip. +/// `None` for other frames. +pub(crate) fn frame_to_image_scaled(f: &VideoFrame, w: u32, h: u32) -> Option { + let (fw, fh) = (f.width as usize, f.height as usize); + if w == 0 || h == 0 || fw % 2 != 0 || fh % 2 != 0 || (w as usize).saturating_mul(2) > fw || (h as usize).saturating_mul(2) > fh { + return None; + } + let half = match &f.data { + PixelData::Yuv8 { planes, chroma: Chroma::C420, alpha: None } => { + half_420(fw, fh, [&planes[0][..], &planes[1][..], &planes[2][..]], &Yuv::new(f.color.matrix, f.color.range, 8))? + } + PixelData::Yuv16 { planes, chroma: Chroma::C420, bits, alpha: None } => { + half_420(fw, fh, [&planes[0][..], &planes[1][..], &planes[2][..]], &Yuv::new(f.color.matrix, f.color.range, *bits))? + } + _ => return None, + }; + Some(if (half.width, half.height) == (w, h) { half } else { effectcraft_raster::resample(&half, w, h) }) +} + +/// An opaque 4:2:0 frame of even size `w` × `h` converted at half that size (see +/// [`frame_to_image_scaled`]); `None` when the planes are too short. +fn half_420(w: usize, h: usize, [luma, cb, cr]: [&[T]; 3], k: &Yuv) -> Option { + /// Row `r` of a plane `n` samples wide (empty past its end). + fn row_of(p: &[T], r: usize, n: usize) -> &[T] { + p.get(r.saturating_mul(n)..(r + 1).saturating_mul(n)).unwrap_or_default() + } + let (cw, ch) = (w / 2, h / 2); + if luma.len() < w.checked_mul(h)? || cb.len() < cw * ch || cr.len() < cw * ch { + return None; + } + let mut img = Image::new(cw as u32, ch as u32); + img.data.par_chunks_mut(cw).enumerate().for_each_init( + // Chroma rows with their edge samples repeated on both ends, for the horizontal [1 6 1]. + || (vec![0f32; cw + 2], vec![0f32; cw + 2]), + |(uc, vc), (cy, row)| { + // The chroma rows above and below weigh 1/8, as `centred` upsampling averages out. + let rows = [cy.saturating_sub(1), cy, (cy + 1).min(ch - 1)]; + for (dst, plane) in [(&mut *uc, cb), (&mut *vc, cr)] { + let [above, at, below] = rows.map(|r| row_of(plane, r, cw)); + for (d, ((a, b), c)) in dst[1..=cw].iter_mut().zip(above.iter().zip(at).zip(below)) { + *d = ((a.f() + c.f()) * 0.125 + b.f() * 0.75 - k.c_off) * k.c_scale; + } + (dst[0], dst[cw + 1]) = (dst[1], dst[cw]); + } + let (top, bottom) = (row_of(luma, 2 * cy, w), row_of(luma, 2 * cy + 1, w)); + let pixels = top.chunks_exact(2).zip(bottom.chunks_exact(2)).zip(uc.windows(3).zip(vc.windows(3))); + for (o, ((a, b), (u, v))) in row.iter_mut().zip(pixels) { + let (u, v) = ((u[0] + u[2]) * 0.125 + u[1] * 0.75, (v[0] + v[2]) * 0.125 + v[1] * 0.75); + let yy = ((a[0].f() + a[1].f() + b[0].f() + b[1].f()) * 0.25 - k.y_off) * k.y_scale; + *o = [(yy + k.cr_r * v).clamp(0.0, 1.0), (yy - k.cr_g * v - k.cb_g * u).clamp(0.0, 1.0), (yy + k.cb_b * u).clamp(0.0, 1.0), 1.0]; + } + }, + ); + Some(img) +} + /// 0..255 → 0..1. static U8: [f32; 256] = { let mut t = [0f32; 256]; @@ -338,6 +399,47 @@ mod tests { assert!((img.data[0][0] - 110.0 / 219.0).abs() < 1e-5); } + /// A smooth opaque 4:2:0 frame whose colours never clip. + fn smooth_420(w: usize, h: usize) -> VideoFrame { + let plane = |w: usize, h: usize, base: f32, amp: f32, fx: f32, fy: f32| -> Arc> { + Arc::new((0..w * h).map(|i| (base + amp * ((i % w) as f32 * fx + (i / w) as f32 * fy).sin()) as u8).collect()) + }; + VideoFrame { + width: w as u32, + height: h as u32, + data: PixelData::Yuv8 { + planes: [plane(w, h, 125.0, 45.0, 0.31, 0.17), plane(w / 2, h / 2, 128.0, 10.0, 0.4, -0.3), plane(w / 2, h / 2, 128.0, 10.0, -0.2, 0.5)], + chroma: Chroma::C420, + alpha: None, + }, + color: filmcraft_color::ColorInfo::REC709, + par: (1, 1), + pts: filmcraft_time::Tick::ZERO, + } + } + + #[test] + fn scaled_conversion_matches_resampling_the_converted_frame() { + let f = smooth_420(96, 64); + let full = frame_to_image(&f, AlphaOp::new(AlphaMode::Straight, [0.0; 3])); + for (w, h) in [(48, 32), (32, 21), (24, 16), (13, 9)] { + let scaled = frame_to_image_scaled(&f, w, h).expect("4:2:0, at most half size"); + let want = effectcraft_raster::resample(&full, w, h); + assert_eq!((scaled.width, scaled.height), (w, h)); + let worst = scaled.data.iter().zip(&want.data).flat_map(|(a, b)| (0..4).map(move |c| (a[c] - b[c]).abs())).fold(0.0f32, f32::max); + assert!(worst < 1e-5, "{w}×{h}: {worst}"); + } + // More than half the size, odd sizes, other chroma layouts: convert the full frame. + assert!(frame_to_image_scaled(&f, 49, 32).is_none()); + assert!(frame_to_image_scaled(&smooth_420(95, 64), 40, 30).is_none()); + assert!(frame_to_image_scaled(&yuv_frame(100, 128, 128, filmcraft_color::ColorInfo::REC709), 2, 1).is_some()); + let mut f444 = smooth_420(96, 64); + if let PixelData::Yuv8 { chroma, .. } = &mut f444.data { + *chroma = Chroma::C444; + } + assert!(frame_to_image_scaled(&f444, 48, 32).is_none()); + } + #[test] fn bt709_red() { // 8-bit limited BT.709 red: Y 63, Cb 102, Cr 240 @@ -402,6 +504,11 @@ mod tests { keep.push(frame_to_image(&f, op)); } eprintln!("convert into fresh memory {:?}", t.elapsed() / 20); + let t = std::time::Instant::now(); + for _ in 0..20 { + std::hint::black_box(frame_to_image_scaled(&f, 640, 360)); + } + eprintln!("convert at a third {:?}", t.elapsed() / 20); } } } diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 3df9ec45..f10389fd 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -10,7 +10,7 @@ use effectcraft_render::FootageSource; use effectcraft_time::{FrameRate, TICKS_PER_SECOND, Tick}; use filmcraft_media::{FrameRequest, SharedSource}; -use crate::convert::{AlphaOp, dynamic_to_image, frame_to_image_in}; +use crate::convert::{AlphaOp, dynamic_to_image, frame_to_image_in, frame_to_image_scaled}; use crate::{MediaError, Result}; /// Default frame-cache budget: 1 GiB of decoded frames. @@ -41,6 +41,8 @@ struct Key { /// pixels. interp: u8, matte: [u32; 3], + /// A movie frame made at this size for a reduced-resolution render ((0, 0): as decoded). + size: (u32, u32), } struct Entry { @@ -322,7 +324,21 @@ impl MediaPool { /// Decode (or fetch from cache) the frame of `footage` at source time `t`. pub fn frame_at(&self, footage: &Footage, t: Tick) -> Result> { - Inner::frame_at(&self.inner, footage, t, true) + Inner::frame_at(&self.inner, footage, t, true, None) + } + + /// [`FootageSource::frame`] / [`FootageSource::frame_at_size`]: `None` (logged) on errors. + fn frame_sized(&self, _item: ItemId, footage: &Footage, t: Tick, size: Option<(u32, u32)>) -> Option> { + if footage.missing { + return None; + } + match Inner::frame_at(&self.inner, footage, t, true, size) { + Ok(img) => Some(img), + Err(e) => { + log::warn!("media: frame of {} at {:?}: {e}", footage.path, t); + None + } + } } /// Settings ▸ Disk ▸ Conformed Audio Folder: write each footage file's decoded audio there @@ -469,7 +485,7 @@ impl MediaPool { /// A thumbnail of `footage` (its first frame) fitting in `max_side` × `max_side`, aspect kept. pub fn thumbnail(&self, footage: &Footage, max_side: u32) -> Result { - let img = Inner::frame_at(&self.inner, footage, Tick::ZERO, false)?; + let img = Inner::frame_at(&self.inner, footage, Tick::ZERO, false, None)?; let (w, h) = (img.width.max(1), img.height.max(1)); let s = (max_side.max(1) as f64 / w.max(h) as f64).min(1.0); if s >= 1.0 { @@ -561,7 +577,7 @@ impl Inner { }; let interp = alpha | (u8::from(footage.preserve_rgb) << 2); let matte = if footage.alpha == AlphaMode::Premultiplied { footage.premul_color.map(f32::to_bits) } else { [0; 3] }; - let key = |path: &str, frame| Key { path: path.into(), frame, interp, matte }; + let key = |path: &str, frame| Key { path: path.into(), frame, interp, matte, size: (0, 0) }; match footage.kind { FootageKind::Sequence if !footage.sequence.is_empty() => { let i = Self::frame_index(footage.frame_rate, t, footage.sequence_frames(), footage.loop_count); @@ -591,21 +607,27 @@ impl Inner { } } - fn frame_at(self: &Arc, footage: &Footage, t: Tick, playback: bool) -> Result> { + /// The frame of `footage` at `t`; a movie's resampled to `size` when it can be made at that + /// size directly (see [`FootageSource::frame_at_size`]). + fn frame_at(self: &Arc, footage: &Footage, t: Tick, playback: bool, size: Option<(u32, u32)>) -> Result> { if !footage.has_video { return Err(MediaError::Unsupported(format!("{}: no video", footage.path))); } - let loc = Self::locate(footage, t); + let mut loc = Self::locate(footage, t); let movie = loc.media_t.is_some(); + if movie && let Some(size) = size { + loc.key.size = size; + } let mut c = lock(&self.cache); // sequential playback of a movie: read the next frame ahead if playback && movie { let prev = c.last.insert(loc.key.path.clone(), loc.key.frame); if prev == Some(loc.key.frame - 1) && self.read_ahead.load(Ordering::Relaxed) { let next_t = t + footage.frame_rate.frame_duration(); - let next = Self::locate(footage, next_t); + let mut next = Self::locate(footage, next_t); + next.key.size = loc.key.size; if next.key != loc.key && !c.lru.map.contains_key(&next.key) && !c.inflight.contains(&next.key) { - self.prefetch(footage, next_t); + self.prefetch(footage, next_t, size); } } } @@ -638,17 +660,17 @@ impl Inner { } #[cfg(not(target_arch = "wasm32"))] - fn prefetch(self: &Arc, footage: &Footage, t: Tick) { + fn prefetch(self: &Arc, footage: &Footage, t: Tick, size: Option<(u32, u32)>) { let (me, f) = (self.clone(), footage.clone()); self.prefetched.fetch_add(1, Ordering::Relaxed); // A plain thread, not a rayon job: the decode below parallelises with rayon itself. std::thread::spawn(move || { - let _ = me.frame_at(&f, t, false); + let _ = me.frame_at(&f, t, false, size); }); } #[cfg(target_arch = "wasm32")] - fn prefetch(self: &Arc, _footage: &Footage, _t: Tick) {} + fn prefetch(self: &Arc, _footage: &Footage, _t: Tick, _size: Option<(u32, u32)>) {} fn decode(&self, loc: &Loc, footage: &Footage) -> Result { let op = AlphaOp::of(footage); @@ -670,6 +692,15 @@ impl Inner { Some(mt) => { let src = self.source(path).ok_or_else(|| MediaError::Io(format!("{path}: cannot open")))?; let vf = src.video_frame(FrameRequest::full(filmcraft_time::Tick(mt.0))).map_err(MediaError::from)?; + // A reduced-resolution render's frame, made at its size (a frame of the footage's + // own size only, as the renderer resamples only those). + let (w, h) = loc.key.size; + if w > 0 + && (vf.width, vf.height) == (footage.width, footage.height) + && let Some(img) = frame_to_image_scaled(&vf, w, h) + { + return Ok(img); + } let buf = lock(&self.cache).lru.take_spare(vf.width as usize * vf.height as usize); Ok(frame_to_image_in(&vf, op, buf)) } @@ -696,17 +727,11 @@ impl FootageSource for MediaPool { fn cache_budget(&self) -> Option { Some(self.budget()) } - fn frame(&self, _item: ItemId, footage: &Footage, t: Tick) -> Option> { - if footage.missing { - return None; - } - match self.frame_at(footage, t) { - Ok(img) => Some(img), - Err(e) => { - log::warn!("media: frame of {} at {:?}: {e}", footage.path, t); - None - } - } + fn frame(&self, item: ItemId, footage: &Footage, t: Tick) -> Option> { + self.frame_sized(item, footage, t, None) + } + fn frame_at_size(&self, item: ItemId, footage: &Footage, t: Tick, width: u32, height: u32) -> Option> { + self.frame_sized(item, footage, t, Some((width, height))) } fn vector_frame(&self, _item: ItemId, footage: &Footage, scale: f64) -> Option> { @@ -834,7 +859,7 @@ mod tests { fn lru_respects_budget_and_recycles() { let mut c = Lru::default(); let img = || Arc::new(Image::new(16, 16)); // 4 KiB + overhead - let k = |i| Key { path: "a".into(), frame: i, interp: 0, matte: [0; 3] }; + let k = |i| Key { path: "a".into(), frame: i, interp: 0, matte: [0; 3], size: (0, 0) }; let budget = 3 * (4096 + 64); for i in 0..10 { c.insert(k(i), img(), budget); diff --git a/crates/media/tests/footage.rs b/crates/media/tests/footage.rs index ed113c35..8d49f087 100644 --- a/crates/media/tests/footage.rs +++ b/crates/media/tests/footage.rs @@ -236,6 +236,26 @@ fn h264_mp4_probe_and_frames() { assert!(Arc::ptr_eq(&a, &b)); } +/// A reduced-resolution render's movie frames are converted at its size: they match resampling +/// the full frames (testsrc2's saturated colours clip, where the two differ most), and are cached +/// per size. +#[test] +fn h264_frames_at_reduced_sizes() { + let Some(p) = h264_mp4() else { return }; + let f = probe(&p).expect("probe"); + let pool = MediaPool::new(); + for (n, (w, h)) in [(0i64, (960, 540)), (1, (640, 360)), (2, (480, 270)), (30, (323, 182))] { + let t = t_of(f.frame_rate, n); + let sized = pool.frame_at_size(ItemId(1), &f, t, w, h).expect("frame"); + let full = pool.frame(ItemId(1), &f, t).expect("frame"); + assert_eq!((sized.width, sized.height), (w, h)); + let db = psnr(&sized, &effectcraft_raster::resample(&full, w, h)); + eprintln!("h264 frame {n} at {w}×{h}: {db:.2} dB against the resampled full frame"); + assert!(db > 55.0, "frame {n} at {w}×{h}: {db:.2} dB"); + assert!(Arc::ptr_eq(&sized, &pool.frame_at_size(ItemId(1), &f, t, w, h).expect("frame")), "cached"); + } +} + #[test] fn h264_audio_samples() { let Some(p) = h264_mp4() else { return }; diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 591213e6..324e2a5f 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -40,6 +40,13 @@ use rayon::prelude::*; pub trait FootageSource: Send + Sync { /// The frame of `item` at source time `t`, straight from the file (any size). fn frame(&self, item: ItemId, footage: &Footage, t: Tick) -> Option>; + /// [`FootageSource::frame`] for a render that shows it at `width` × `height`, at most half its + /// size: either that frame or, when the source can make it more cheaply than the whole frame + /// (movies, see `effectcraft-media`), the frame already resampled to that size as + /// [`effectcraft_raster::resample`] would. + fn frame_at_size(&self, item: ItemId, footage: &Footage, t: Tick, _width: u32, _height: u32) -> Option> { + self.frame(item, footage, t) + } /// `frames` interleaved stereo sample frames of `item`'s audio from source time `t` at /// `rate` Hz (`None` when unavailable). fn audio(&self, _item: ItemId, _footage: &Footage, _t: Tick, _frames: usize, _rate: u32) -> Option> { @@ -1104,8 +1111,10 @@ impl<'a> Renderer<'a> { } let lt = ctx.source_time(layer); // The proxy is decoded instead, at its own size, and fills the footage's frame. - let pf = self.proxy_for(*item).unwrap_or(f); - let img = self.footage_frame(ctx, layer, *item, pf, lt)?; + let proxy = self.proxy_for(*item); + let pf = proxy.unwrap_or(f); + let size = self.reduced_size(f.width, f.height).filter(|_| proxy.is_none()); + let img = self.footage_frame(ctx, layer, *item, pf, lt, size)?; Some(self.footage_buf(&img, pf, f.width, f.height)) } LayerSource::Text => Some(self.authored(text::render(ctx, layer, self.raster_scale(ctx, layer)))), @@ -1127,12 +1136,11 @@ impl<'a> Renderer<'a> { let s = self.opts.scale; let k = if img.width > 0 { w.max(1) as f64 / img.width as f64 } else { 1.0 }; // Keep native pixels when downsampling is small; resample otherwise. - let mut buf = if s < 0.75 && k <= 1.0 + 1e-9 { - let rw = ((w as f64 * s).round() as u32).max(1); - let rh = ((h as f64 * s).round() as u32).max(1); - Buf { img: effectcraft_raster::resample(img, rw, rh), offset: [0.0; 2], scale: s } - } else { - Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 / k } + let mut buf = match self.reduced_size(w, h) { + // The source resampled it already (`FootageSource::frame_at_size`). + Some(size) if (img.width, img.height) == size => Buf { img: img.clone(), offset: [0.0; 2], scale: s }, + Some((rw, rh)) if k <= 1.0 + 1e-9 => Buf { img: effectcraft_raster::resample(img, rw, rh), offset: [0.0; 2], scale: s }, + _ => Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 / k }, }; if let Some(c) = self.pipe.media_in(f) { color::convert(&mut buf.img, &c); @@ -1140,6 +1148,13 @@ impl<'a> Renderer<'a> { buf } + /// The size of `w` × `h` layer pixels in this render when it is reduced enough that footage + /// is resampled to it (below 3/4 scale). + fn reduced_size(&self, w: u32, h: u32) -> Option<(u32, u32)> { + let s = self.opts.scale; + (s < 0.75).then(|| (((w as f64 * s).round() as u32).max(1), ((h as f64 * s).round() as u32).max(1))) + } + /// Authored colours into the working space (linear working spaces). fn authored(&self, mut buf: Buf) -> Buf { if let Some(c) = self.pipe.authored_in() { @@ -1151,7 +1166,8 @@ impl<'a> Renderer<'a> { /// A footage frame at source time `t`, frame-blended between the two nearest source frames /// when the layer's Frame Blending switch (and the comp's Enable Frame Blending) is on and /// `t` falls between frames (footage rate ≠ comp rate, time stretch, time remapping). - fn footage_frame(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick) -> Option> { + /// `size`: a plain frame may come resampled to it ([`FootageSource::frame_at_size`]). + fn footage_frame(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick, size: Option<(u32, u32)>) -> Option> { // Interpret Footage ▸ Separate Fields: each field is a frame at twice the rate. if f.fields != effectcraft_project::FieldOrder::Off && matches!(f.kind, FootageKind::Video | FootageKind::Sequence) { let field_rate = FrameRate::new(f.frame_rate.num * 2, f.frame_rate.den); @@ -1162,11 +1178,13 @@ impl<'a> Renderer<'a> { let parity = if (i % 2 == 0) == dominant_upper { 0 } else { 1 }; return Some(Arc::new(interpret_pixels(&field_frame(&img, parity), f))); } - let img = self.footage_frame_raw(ctx, layer, item, f, t)?; + // Interpretation is per pixel but not linear: it runs on the frame before resampling. + let size = size.filter(|_| !(f.invert_alpha || f.linear_light_applies())); + let img = self.footage_frame_raw(ctx, layer, item, f, t, size)?; if f.invert_alpha || f.linear_light_applies() { Some(Arc::new(interpret_pixels(&img, f))) } else { Some(img) } } - fn footage_frame_raw(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick) -> Option> { + fn footage_frame_raw(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick, size: Option<(u32, u32)>) -> Option> { let mode = frame_blend_mode(ctx, layer); if mode != FrameBlend::Off && matches!(f.kind, FootageKind::Video | FootageKind::Sequence) { let (i, w) = frame_position(t, f.frame_rate); @@ -1194,7 +1212,10 @@ impl<'a> Renderer<'a> { } } } - self.footage.frame(item, f, t) + match size { + Some((w, h)) => self.footage.frame_at_size(item, f, t, w, h), + None => self.footage.frame(item, f, t), + } } /// Source → masks, clamped/quantised to the project depth. @@ -2152,6 +2173,8 @@ mod tests_collapse; #[cfg(test)] mod tests_color; #[cfg(test)] +mod tests_footage_size; +#[cfg(test)] mod tests_frame_blend; #[cfg(test)] mod tests_mask_blur; diff --git a/crates/render/src/tests_footage_size.rs b/crates/render/src/tests_footage_size.rs new file mode 100644 index 00000000..88df726e --- /dev/null +++ b/crates/render/src/tests_footage_size.rs @@ -0,0 +1,83 @@ +//! Footage at a reduced resolution: a source that resamples frames itself +//! ([`FootageSource::frame_at_size`], movies in `effectcraft-media`) renders what resampling the +//! full frame renders. + +use std::sync::{Arc, Mutex}; + +use rayon::prelude::*; + +use effectcraft_color::Label; +use effectcraft_project::build; +use effectcraft_project::{AlphaMode, BitDepth, Comp, Footage, FootageKind, ItemId, ItemKind, LayerSource, Project}; +use effectcraft_time::{FrameRate, Tick}; + +use crate::{FootageSource, Image, RenderOpts, Renderer}; + +/// Footage with a textured frame; `sizes` (when set) resamples it itself and records the sizes +/// it was asked for. +struct Clip { + sizes: Option>>, +} + +impl FootageSource for Clip { + fn frame(&self, _: ItemId, f: &Footage, _: Tick) -> Option> { + let mut img = Image::new(f.width, f.height); + img.rows_mut().for_each(|(y, row)| { + for (x, p) in row.iter_mut().enumerate() { + let s = (0.5 + 0.3 * (x as f32 * 0.37 + y as f32 * 0.11).sin()) * 0.9; + *p = [s, 1.0 - s, s * 0.5, 1.0]; + } + }); + Some(Arc::new(img)) + } + fn frame_at_size(&self, item: ItemId, f: &Footage, t: Tick, w: u32, h: u32) -> Option> { + let Some(sizes) = &self.sizes else { return self.frame(item, f, t) }; + sizes.lock().unwrap().push((w, h)); + Some(Arc::new(effectcraft_raster::resample(&*self.frame(item, f, t)?, w, h))) + } +} + +fn project(depth: BitDepth, invert_alpha: bool) -> (Project, ItemId) { + let mut p = Project::default(); + p.settings.bit_depth = depth; + let comp = Comp::new(96, 54, FrameRate::FPS_30, Tick::from_seconds_f64(1.0)); + let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.clone().into())); + let f = Footage { + path: "clip.mp4".into(), + kind: FootageKind::Video, + width: 96, + height: 54, + pixel_aspect: 1.0, + frame_rate: FrameRate::FPS_30, + duration: Tick::from_seconds_f64(1.0), + has_video: true, + alpha: AlphaMode::Ignore, + loop_count: 1, + invert_alpha, + ..Default::default() + }; + let fid = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(f)); + let l = build::layer(&mut p, &comp, "clip", LayerSource::Footage { item: fid }, (96, 54), None); + p.comp_mut(cid).unwrap().layers.push(l); + (p, cid) +} + +#[test] +fn reduced_renders_take_footage_resampled_by_the_source() { + for depth in [BitDepth::Bpc8, BitDepth::Bpc32] { + let (p, cid) = project(depth, false); + for (scale, asked) in [(1.0, None), (0.75, None), (0.5, Some((48, 27))), (1.0 / 3.0, Some((32, 18))), (0.25, Some((24, 14)))] { + let opts = RenderOpts { scale, ..Default::default() }; + let plain = Renderer::new(&p, &Clip { sizes: None }, opts).comp_frame(cid, Tick::ZERO); + let sized = Clip { sizes: Some(Mutex::default()) }; + let frame = Renderer::new(&p, &sized, opts).comp_frame(cid, Tick::ZERO); + assert_eq!(frame, plain, "{depth:?} at {scale}"); + assert_eq!(sized.sizes.unwrap().into_inner().unwrap(), asked.into_iter().collect::>(), "{depth:?} at {scale}"); + } + } + // Interpreted pixels (Invert Alpha) are interpreted at full size, then resampled. + let (p, cid) = project(BitDepth::Bpc8, true); + let sized = Clip { sizes: Some(Mutex::default()) }; + Renderer::new(&p, &sized, RenderOpts { scale: 0.5, ..Default::default() }).comp_frame(cid, Tick::ZERO); + assert!(sized.sizes.unwrap().into_inner().unwrap().is_empty()); +} diff --git a/docs/architecture.md b/docs/architecture.md index 33e38215..6a284b8e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -230,7 +230,11 @@ Nulls / Nulls Follow Points take pins as well as paths, and `puppet.follow` give Position expression that trails the leader by a delay. The renderer flattens nested effect groups into `Params` keys (`effects::flatten_params`). Commands: `paint.*`, `puppet.*`. -Half, Third and Quarter resolution render proportionally fewer pixels end to end. +Half, Third and Quarter resolution render proportionally fewer pixels end to end. Footage +starts at the render's size where the source can make it more cheaply than the full frame +(`FootageSource::frame_at_size`): movies in opaque 4:2:0 Y'CbCr are box-filtered to half size +before conversion (a quarter of the conversions; the result differs from converting first only +in blocks where colours clip) and resampled from there, as other frames are from full size. **GPU compositor** (`crates/gpu`; Project Settings ▸ Video Rendering and Effects ▸ Mercury GPU Acceleration, the default, or Mercury Software Only; `render.backend`). The CPU renderer is the