GPU1: blend modes on the GPU compositor (#30)
All 27 blend modes now composite on the GPU instead of sending the whole frame to the CPU. - render::plan: PlanLayer carries its Blend; plan_frame no longer falls back to a CPU frame for non-Normal modes. Media, graphic clips, multicam angles and CPU-rendered layer images (standard effects) keep their clip's mode; inside a transition the mode is Normal, as in the CPU reference. execute_cpu composites with each layer's mode. - gpu: Normal keeps the single fixed-function pass. Dissolve also uses fixed-function "over": the CPU's 64-bit pixel hash is reproduced in WGSL on u32 pairs (pattern bit-exact). The other 25 modes copy the accumulator region under the layer's quad into a backdrop texture and composite in fs_blend with the CPU math (sRGB-encoded straight colour, W3C formulas, back to linear, same premultiplied mix and clamping). - Tests: per-mode GPU-vs-CPU parity (gpu crate: overlapping transformed layers, partial opacity/alpha, empty/partial backdrop; golden: every mode through plan_frame incl. an effect layer, blended clips in a cross dissolve), exact-value edge cases (black/white, Divide by 0, Dodge/Burn at 0 and 1, zero alpha) within 4.9e-4, Dissolve pattern exact, off-output layers, quad bounds. - Bench: new stack3-blend playback scenario (V2 Screen, V3 Overlay); CPU ms/frame 151-214 -> 86-89, Full 19/173 and 138/54 -> 191/1, 192/0. - Docs: architecture §5, gpu crate docs, testing, performance. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Brandon Thomas
parent
4f06306e45
commit
6102a3794a
|
|
@ -775,3 +775,108 @@ fn gpu_matches_cpu_on_golden_scenes() {
|
|||
}
|
||||
assert!(failures.is_empty(), "GPU differs from the CPU reference:\n{}", failures.join("\n"));
|
||||
}
|
||||
|
||||
/// Per-pixel max RGB difference (99th percentile, mean) over the pixels at least 1.5 px from every
|
||||
/// plan layer's quad outline: the CPU resampler fades a layer's edge over one more pixel than the
|
||||
/// GPU rasterises, and with blend modes those edge pixels are a visible share of a 320×180 frame.
|
||||
fn interior_diff(plan: &filmcraft_render::plan::FramePlan, a: &Rgba8, b: &Rgba8) -> (u32, f64) {
|
||||
let filmcraft_render::plan::FramePlan::Layers { width, layers, .. } = plan else { return (u32::MAX, f64::MAX) };
|
||||
let inv: Vec<_> = layers.iter().filter_map(|l| Some((l.matrix.inverse()?, l.frame.width as f64, l.frame.height as f64))).collect();
|
||||
let mut d: Vec<u32> = Vec::new();
|
||||
for (i, (pa, pb)) in a.px.chunks(4).zip(b.px.chunks(4)).enumerate() {
|
||||
let (x, y) = ((i % width) as f64 + 0.5, (i / width) as f64 + 0.5);
|
||||
let edge = inv.iter().any(|(m, fw, fh)| {
|
||||
let c = [(-1.5, -1.5), (1.5, -1.5), (-1.5, 1.5), (1.5, 1.5)].map(|(dx, dy)| {
|
||||
let p = m.apply(Vec2::new(x + dx, y + dy));
|
||||
p.x >= 0.0 && p.y >= 0.0 && p.x <= *fw && p.y <= *fh
|
||||
});
|
||||
c.iter().any(|v| *v != c[0])
|
||||
});
|
||||
if !edge {
|
||||
d.push((0..3).map(|k| (pa[k] as i32 - pb[k] as i32).unsigned_abs()).max().unwrap_or(0));
|
||||
}
|
||||
}
|
||||
d.sort_unstable();
|
||||
(d[d.len() * 99 / 100], d.iter().sum::<u32>() as f64 / d.len() as f64)
|
||||
}
|
||||
|
||||
/// Every blend mode composited by the GPU from `plan_frame` (no CPU fallback): two overlapping
|
||||
/// transformed clips at partial opacity in the mode over footage, plus a clip with a standard
|
||||
/// effect (rendered on the CPU as a layer image) in the same mode, match the CPU render with the
|
||||
/// GPU parity tolerance (p99 ≤ 6, mean < 1.5) away from layer outlines; whole-frame numbers are
|
||||
/// printed too.
|
||||
#[test]
|
||||
fn gpu_blend_modes_match_cpu_render() {
|
||||
let Some((dev, q)) = device() else {
|
||||
eprintln!("SKIPPED (gpu blend parity): no GPU adapter");
|
||||
return;
|
||||
};
|
||||
let mut c = filmcraft_gpu::GpuCompositor::new(&dev, &q);
|
||||
let mut failures = Vec::new();
|
||||
for mode in filmcraft_project::effect::BLEND_MODES {
|
||||
let mut b = Builder::new(4);
|
||||
let bg = b.demo(DemoScene::OceanSunset);
|
||||
let (dunes, aurora, bars) = (b.demo(DemoScene::Dunes), b.demo(DemoScene::Aurora), b.media(Generator::BarsAndTone));
|
||||
b.place(0, bg, 0, 48);
|
||||
let v2 = b.place(1, dunes, 0, 48);
|
||||
b.fixed(v2, "motion", &[("scale", fl(60.0)), ("rotation", fl(12.0)), ("position", pt(130.0, 80.0))]);
|
||||
b.fixed(v2, "opacity", &[("opacity", fl(70.0)), ("blend", blend(mode))]);
|
||||
let v3 = b.place(2, aurora, 0, 48);
|
||||
b.fixed(v3, "motion", &[("scale", fl(45.0)), ("rotation", fl(-20.0)), ("position", pt(210.0, 110.0))]);
|
||||
b.fixed(v3, "opacity", &[("opacity", fl(60.0)), ("blend", blend(mode))]);
|
||||
let v4 = b.place(3, bars, 0, 48);
|
||||
b.fixed(v4, "motion", &[("scale", fl(30.0)), ("position", pt(250.0, 50.0))]);
|
||||
b.fixed(v4, "opacity", &[("opacity", fl(85.0)), ("blend", blend(mode))]);
|
||||
b.effect(v4, "crop", &[("left", fl(10.0)), ("bottom", fl(20.0))]);
|
||||
let s = b.at(12);
|
||||
let t = RATE.tick_of(s.frame);
|
||||
let cpu = render_cpu(&s);
|
||||
let plan = filmcraft_render::plan::plan_frame(&s.project, s.seq, t, RenderOptions::default(), &s.sources);
|
||||
let filmcraft_render::plan::FramePlan::Layers { layers, .. } = &plan else {
|
||||
failures.push(format!("{mode}: planned as a CPU image"));
|
||||
continue;
|
||||
};
|
||||
assert_eq!(layers.len(), 4, "{mode}");
|
||||
c.composite(&plan);
|
||||
let (gw, gh, mut px) = c.read_output().expect("GPU readback");
|
||||
px.as_chunks_mut::<4>().0.iter_mut().for_each(|p| p[3] = 255);
|
||||
let gpu = Rgba8::new(gw, gh, px);
|
||||
let d = diff(&cpu, &gpu).unwrap();
|
||||
let (p99, mean) = interior_diff(&plan, &cpu, &gpu);
|
||||
eprintln!("{mode}: GPU vs CPU interior p99 {p99}, mean {mean:.3}; whole frame {d}");
|
||||
if p99 > 6 || mean >= 1.5 {
|
||||
failures.push(format!("{mode}: interior p99 {p99}, mean {mean}; {d}"));
|
||||
}
|
||||
}
|
||||
assert!(failures.is_empty(), "GPU blend modes differ from the CPU render:\n{}", failures.join("\n"));
|
||||
}
|
||||
|
||||
/// Inside a transition the CPU mixes the clips and composites the result Normal, ignoring their
|
||||
/// blend modes: the plan does the same (Screen clips cross-dissolving on V2 over V1).
|
||||
#[test]
|
||||
fn gpu_transition_of_blended_clips_matches_cpu_render() {
|
||||
let Some((dev, q)) = device() else {
|
||||
eprintln!("SKIPPED (gpu blend parity): no GPU adapter");
|
||||
return;
|
||||
};
|
||||
let mut b = Builder::new(2);
|
||||
let (bg, x, y) = (b.demo(DemoScene::OceanSunset), b.demo(DemoScene::Aurora), b.demo(DemoScene::Dunes));
|
||||
b.place(0, bg, 0, 48);
|
||||
let ca = b.place(1, x, 0, 24);
|
||||
let cb = b.place(1, y, 24, 24);
|
||||
for id in [ca, cb] {
|
||||
b.fixed(id, "opacity", &[("blend", blend("Screen"))]);
|
||||
}
|
||||
b.transition(1, "cross_dissolve", ca, cb, 24, 12);
|
||||
let s = b.at(22);
|
||||
let plan = filmcraft_render::plan::plan_frame(&s.project, s.seq, RATE.tick_of(s.frame), RenderOptions::default(), &s.sources);
|
||||
let filmcraft_render::plan::FramePlan::Layers { layers, .. } = &plan else { panic!("planned as a CPU image") };
|
||||
assert!(layers.iter().all(|l| l.blend == filmcraft_render::Blend::Normal));
|
||||
let mut c = filmcraft_gpu::GpuCompositor::new(&dev, &q);
|
||||
c.composite(&plan);
|
||||
let (gw, gh, mut px) = c.read_output().expect("GPU readback");
|
||||
px.as_chunks_mut::<4>().0.iter_mut().for_each(|p| p[3] = 255);
|
||||
let d = diff(&render_cpu(&s), &Rgba8::new(gw, gh, px)).unwrap();
|
||||
eprintln!("Screen clips in a cross dissolve: GPU vs CPU {d}");
|
||||
assert!(d.p99 <= 6 && d.mean_abs * 4.0 / 3.0 < 1.5, "{d}");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,336 @@
|
|||
//! GPU-vs-CPU parity of the blend modes (`filmcraft_render::blend::composite` is the reference).
|
||||
//!
|
||||
//! Tolerances: the composited 8-bit output uses the compositor's parity criterion (99th percentile
|
||||
//! of the per-pixel max channel difference ≤ 6, mean < 1.5; antialiased edges and half-float
|
||||
//! textures differ slightly). Exact inputs drawn 1:1 (no resampling) must agree to 2e-3 in linear
|
||||
//! premultiplied values (half-float accumulator), except Hard Mix, a step function, which may
|
||||
//! flip a pixel whose blend lands on its 0.5 threshold.
|
||||
|
||||
use super::*;
|
||||
use filmcraft_geom::{Affine, Vec2};
|
||||
use filmcraft_render::plan::execute_cpu;
|
||||
|
||||
fn device() -> Option<(wgpu::Device, wgpu::Queue)> {
|
||||
let instance = wgpu::Instance::default();
|
||||
let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())).ok()?;
|
||||
pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())).ok()
|
||||
}
|
||||
|
||||
fn yuv_frame(w: u32, h: u32) -> Arc<VideoFrame> {
|
||||
let (cw, ch) = (w.div_ceil(2), h.div_ceil(2));
|
||||
let y: Vec<u8> = (0..w * h).map(|i| (16 + ((i % w) * 219 / w)) as u8).collect();
|
||||
let u: Vec<u8> = (0..cw * ch).map(|i| (64 + (i / cw) * 128 / ch) as u8).collect();
|
||||
let v: Vec<u8> = (0..cw * ch).map(|i| (200 - (i % cw) * 100 / cw) as u8).collect();
|
||||
Arc::new(VideoFrame {
|
||||
width: w,
|
||||
height: h,
|
||||
data: PixelData::Yuv8 { planes: [Arc::new(y), Arc::new(u), Arc::new(v)], chroma: Chroma::C420, alpha: None },
|
||||
color: filmcraft_color::ColorInfo::REC709,
|
||||
par: (1, 1),
|
||||
pts: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Decode an IEEE half.
|
||||
fn f16_to_f32(h: u16) -> f32 {
|
||||
let s = if h & 0x8000 != 0 { -1.0 } else { 1.0 };
|
||||
let e = ((h >> 10) & 0x1f) as i32;
|
||||
let m = (h & 0x3ff) as f32;
|
||||
match e {
|
||||
0 => s * m * 2f32.powi(-24),
|
||||
31 => s * f32::INFINITY,
|
||||
_ => s * (1.0 + m / 1024.0) * 2f32.powi(e - 15),
|
||||
}
|
||||
}
|
||||
|
||||
/// Read the linear premultiplied accumulator back as f32 RGBA (what the CPU plan executor returns).
|
||||
fn read_accum(c: &GpuCompositor) -> Vec<f32> {
|
||||
let (tex, _, (w, h)) = c.accum.as_ref().expect("accumulator");
|
||||
let row = (w * 8).div_ceil(256) * 256;
|
||||
let buf = c.device.create_buffer(&wgpu::BufferDescriptor {
|
||||
label: Some("accum-readback"),
|
||||
size: (row * h) as u64,
|
||||
usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ,
|
||||
mapped_at_creation: false,
|
||||
});
|
||||
let mut enc = c.device.create_command_encoder(&Default::default());
|
||||
enc.copy_texture_to_buffer(
|
||||
wgpu::TexelCopyTextureInfo { texture: tex, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All },
|
||||
wgpu::TexelCopyBufferInfo { buffer: &buf, layout: wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(row), rows_per_image: Some(*h) } },
|
||||
wgpu::Extent3d { width: *w, height: *h, depth_or_array_layers: 1 },
|
||||
);
|
||||
c.queue.submit([enc.finish()]);
|
||||
let slice = buf.slice(..);
|
||||
slice.map_async(wgpu::MapMode::Read, |_| {});
|
||||
let _ = c.device.poll(wgpu::PollType::wait_indefinitely());
|
||||
let data = slice.get_mapped_range().expect("mapped");
|
||||
let mut out = Vec::with_capacity((w * h * 4) as usize);
|
||||
for y in 0..*h {
|
||||
let r = &data[(y * row) as usize..(y * row + w * 8) as usize];
|
||||
out.extend(r.as_chunks::<2>().0.iter().map(|b| f16_to_f32(u16::from_le_bytes(*b))));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Per-pixel max channel difference of two RGBA8 images over the pixels `keep` selects: (p99,
|
||||
/// mean, max).
|
||||
fn stats8(a: &[u8], b: &[u8], keep: &[bool]) -> (u32, f64, u32) {
|
||||
let mut d: Vec<u32> = a
|
||||
.chunks(4)
|
||||
.zip(b.chunks(4))
|
||||
.zip(keep)
|
||||
.filter(|(_, k)| **k)
|
||||
.map(|((a, b), _)| (0..3).map(|k| (a[k] as i32 - b[k] as i32).unsigned_abs()).max().unwrap_or(0))
|
||||
.collect();
|
||||
d.sort_unstable();
|
||||
(d[d.len() * 99 / 100], d.iter().sum::<u32>() as f64 / d.len() as f64, d[d.len() - 1])
|
||||
}
|
||||
|
||||
/// Pixels at least 1.5 px away from every layer's quad outline (the CPU resampler fades a layer's
|
||||
/// edge over a pixel beyond it, the GPU rasterises the quad: compared separately).
|
||||
fn interior(plan: &FramePlan) -> Vec<bool> {
|
||||
let FramePlan::Layers { width, height, layers } = plan else { return Vec::new() };
|
||||
let inv: Vec<(Affine, f64, f64)> = layers.iter().filter_map(|l| Some((l.matrix.inverse()?, l.frame.width as f64, l.frame.height as f64))).collect();
|
||||
let mut keep = vec![true; width * height];
|
||||
for (i, k) in keep.iter_mut().enumerate() {
|
||||
let (x, y) = ((i % width) as f64 + 0.5, (i / width) as f64 + 0.5);
|
||||
*k = inv.iter().all(|(m, fw, fh)| {
|
||||
let inside = |dx: f64, dy: f64| {
|
||||
let p = m.apply(Vec2::new(x + dx, y + dy));
|
||||
p.x >= 0.0 && p.y >= 0.0 && p.x <= *fw && p.y <= *fh
|
||||
};
|
||||
let c = [inside(-1.5, -1.5), inside(1.5, -1.5), inside(-1.5, 1.5), inside(1.5, 1.5)];
|
||||
c.iter().all(|v| *v == c[0])
|
||||
});
|
||||
}
|
||||
keep
|
||||
}
|
||||
|
||||
/// Linear premultiplied RGBA f32 test frame: a colour ramp with black and white bands; alpha 0 in
|
||||
/// the left tenth, 1 in the top third, partial below.
|
||||
fn ramp_layer(w: u32, h: u32, phase: u32) -> Arc<VideoFrame> {
|
||||
let mut px = Vec::with_capacity((w * h * 4) as usize);
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
let fx = x as f32 / (w - 1) as f32;
|
||||
let fy = y as f32 / (h - 1) as f32;
|
||||
let c = match (x + phase) / 8 % 6 {
|
||||
0 => [0.0, 0.0, 0.0],
|
||||
1 => [1.0, 1.0, 1.0],
|
||||
_ => [fx, (fy * 1.3 + phase as f32 * 0.1).fract(), ((fx + fy) * 0.7).fract()],
|
||||
};
|
||||
let a = if x < w / 10 {
|
||||
0.0
|
||||
} else if y < h / 3 {
|
||||
1.0
|
||||
} else {
|
||||
0.25 + 0.75 * fy
|
||||
};
|
||||
px.extend_from_slice(&[c[0] * a, c[1] * a, c[2] * a, a]);
|
||||
}
|
||||
}
|
||||
Arc::new(VideoFrame::rgba_f32(w, h, px))
|
||||
}
|
||||
|
||||
/// Every blend mode over a backdrop with opaque, partially transparent and empty regions, on two
|
||||
/// overlapping transformed layers (rotated, scaled, partly off the output) with partial opacity
|
||||
/// and partial alpha: the GPU matches the CPU plan executor within the compositor's tolerance,
|
||||
/// away from the layers' outlines (which are reported: the CPU fades them over an extra pixel).
|
||||
#[test]
|
||||
fn gpu_blend_modes_match_cpu() {
|
||||
let Some((dev, q)) = device() else {
|
||||
eprintln!("no GPU adapter; skipping");
|
||||
return;
|
||||
};
|
||||
let mut c = GpuCompositor::new(&dev, &q);
|
||||
let (w, h) = (320usize, 180usize);
|
||||
let pixels = filmcraft_media::generators::render(
|
||||
&filmcraft_media::Generator::Demo(filmcraft_media::DemoScene::Dunes),
|
||||
320,
|
||||
180,
|
||||
1.0,
|
||||
24,
|
||||
filmcraft_time::FrameRate::FPS_24,
|
||||
);
|
||||
let demo = Arc::new(VideoFrame::rgba8(320, 180, pixels));
|
||||
let base = ramp_layer(200, 180, 3);
|
||||
let top = ramp_layer(160, 120, 0);
|
||||
for mode in Blend::ALL {
|
||||
let plan = FramePlan::Layers {
|
||||
width: w,
|
||||
height: h,
|
||||
layers: vec![
|
||||
// opaque YUV on the left, a partially transparent ramp on the right; the top-right
|
||||
// corner stays empty (destination alpha 0)
|
||||
PlanLayer { frame: yuv_frame(640, 360), matrix: Affine::scale(0.35, 0.5), opacity: 1.0, blend: Blend::Normal },
|
||||
PlanLayer { frame: base.clone(), matrix: Affine::translate(200.0, 40.0), opacity: 0.9, blend: Blend::Normal },
|
||||
PlanLayer {
|
||||
frame: top.clone(),
|
||||
matrix: Affine::motion(Vec2::new(170.0, 95.0), Vec2::new(1.1, 1.1), 17.0, Vec2::new(80.0, 60.0)),
|
||||
opacity: 0.8,
|
||||
blend: mode,
|
||||
},
|
||||
PlanLayer {
|
||||
frame: demo.clone(),
|
||||
matrix: Affine::motion(Vec2::new(250.0, 70.0), Vec2::new(0.45, 0.45), -9.0, Vec2::new(160.0, 90.0)),
|
||||
opacity: 0.6,
|
||||
blend: mode,
|
||||
},
|
||||
],
|
||||
};
|
||||
let cpu_img = execute_cpu(&plan);
|
||||
let cpu = cpu_img.over_black_rgba8();
|
||||
c.composite(&plan);
|
||||
let (_, _, gpu) = c.read_output().expect("readback");
|
||||
let keep = interior(&plan);
|
||||
let (p99, mean, max) = stats8(&cpu, &gpu, &keep);
|
||||
let all = vec![true; keep.len()];
|
||||
let (ap99, amean, _) = stats8(&cpu, &gpu, &all);
|
||||
// linear premultiplied accumulator, alpha included
|
||||
let acc = read_accum(&c);
|
||||
let mut fd: Vec<f32> =
|
||||
cpu_img.px.chunks(4).zip(acc.chunks(4)).zip(&keep).filter(|(_, k)| **k).flat_map(|((a, b), _)| (0..4).map(|k| (a[k] - b[k]).abs())).collect();
|
||||
fd.sort_unstable_by(f32::total_cmp);
|
||||
let (fp99, fmax) = (fd[fd.len() * 99 / 100], fd[fd.len() - 1]);
|
||||
eprintln!("{mode:?}: interior 8-bit p99 {p99} mean {mean:.3} max {max}; linear p99 {fp99:.2e} max {fmax:.2e}; with edges p99 {ap99} mean {amean:.3}");
|
||||
assert!(p99 <= 6 && mean < 1.5, "{mode:?}: p99 {p99}, mean {mean}");
|
||||
}
|
||||
assert!(c.backdrop.is_some(), "blend modes use the backdrop copy");
|
||||
}
|
||||
|
||||
/// Formula edge cases on exact (half-float representable) values, drawn 1:1 so no resampling is
|
||||
/// involved: black / white / mid inputs, Divide by zero, Color Dodge and Burn at 0 and 1, zero and
|
||||
/// partial alpha on either side.
|
||||
#[test]
|
||||
fn gpu_blend_edge_cases_match_cpu() {
|
||||
let Some((dev, q)) = device() else {
|
||||
eprintln!("no GPU adapter; skipping");
|
||||
return;
|
||||
};
|
||||
let vals = [0.0f32, 1.0, 0.5, 0.25, 0.75, 0.125, 0.875, 0.0625];
|
||||
let alphas = [1.0f32, 0.5, 0.0, 0.25];
|
||||
let n = vals.len() * alphas.len();
|
||||
// backdrop varies along x, source along y: every (b, s, alpha) combination; channels are
|
||||
// shifted along the other axis so greys, primaries and mixed colours all occur
|
||||
let make = |along_x: bool| {
|
||||
let mut px = Vec::with_capacity(n * n * 4);
|
||||
for y in 0..n {
|
||||
for x in 0..n {
|
||||
let (i, o) = if along_x { (x, y) } else { (y, x) };
|
||||
let (v, a) = (i % vals.len(), alphas[i / vals.len()]);
|
||||
let j = o % 3;
|
||||
let c = [vals[v], vals[(v + j) % vals.len()], vals[(v + 2 * j) % vals.len()]];
|
||||
px.extend_from_slice(&[c[0] * a, c[1] * a, c[2] * a, a]);
|
||||
}
|
||||
}
|
||||
Arc::new(VideoFrame::rgba_f32(n as u32, n as u32, px))
|
||||
};
|
||||
let (back, src) = (make(true), make(false));
|
||||
let mut c = GpuCompositor::new(&dev, &q);
|
||||
for mode in Blend::ALL {
|
||||
for opacity in [1.0f32, 0.5] {
|
||||
let plan = FramePlan::Layers {
|
||||
width: n,
|
||||
height: n,
|
||||
layers: vec![
|
||||
PlanLayer { frame: back.clone(), matrix: Affine::IDENTITY, opacity: 1.0, blend: Blend::Normal },
|
||||
PlanLayer { frame: src.clone(), matrix: Affine::IDENTITY, opacity, blend: mode },
|
||||
],
|
||||
};
|
||||
let cpu = execute_cpu(&plan);
|
||||
c.composite(&plan);
|
||||
let gpu = read_accum(&c);
|
||||
let mut worst = (0f32, 0usize);
|
||||
let mut steps = 0;
|
||||
for (i, (a, b)) in cpu.px.iter().zip(&gpu).enumerate() {
|
||||
assert!(b.is_finite(), "{mode:?}: non-finite GPU value at {i}");
|
||||
let d = (a - b).abs();
|
||||
if d > 0.05 {
|
||||
steps += 1;
|
||||
}
|
||||
if d > worst.0 {
|
||||
worst = (d, i);
|
||||
}
|
||||
}
|
||||
eprintln!("{mode:?} @ {opacity}: max |cpu − gpu| = {:.2e} (value {} of {}), {steps} step flips", worst.0, worst.1, gpu.len());
|
||||
assert!(worst.0 < 2e-3 || mode == Blend::HardMix && steps <= 2, "{mode:?} @ {opacity}: {worst:?}, {steps}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Dissolve on the GPU reproduces the CPU's pixel pattern exactly (opaque layer at partial opacity).
|
||||
#[test]
|
||||
fn gpu_dissolve_pattern_is_exact() {
|
||||
let Some((dev, q)) = device() else {
|
||||
eprintln!("no GPU adapter; skipping");
|
||||
return;
|
||||
};
|
||||
let (w, h) = (257usize, 131usize);
|
||||
let white = Arc::new(VideoFrame::rgba_f32(1, 1, vec![1.0; 4]));
|
||||
let red = Arc::new(VideoFrame::rgba_f32(1, 1, vec![1.0, 0.0, 0.0, 1.0]));
|
||||
let mut c = GpuCompositor::new(&dev, &q);
|
||||
for op in [0.1f32, 0.5, 0.9] {
|
||||
let plan = FramePlan::Layers {
|
||||
width: w,
|
||||
height: h,
|
||||
layers: vec![
|
||||
PlanLayer { frame: red.clone(), matrix: Affine::scale(w as f64, h as f64), opacity: 1.0, blend: Blend::Normal },
|
||||
PlanLayer { frame: white.clone(), matrix: Affine::scale(w as f64, h as f64), opacity: op, blend: Blend::Dissolve },
|
||||
],
|
||||
};
|
||||
let cpu = execute_cpu(&plan).over_black_rgba8();
|
||||
c.composite(&plan);
|
||||
let (_, _, gpu) = c.read_output().expect("readback");
|
||||
let differ = cpu.chunks(4).zip(gpu.chunks(4)).filter(|(a, b)| a[..3] != b[..3]).count();
|
||||
let kept = gpu.chunks(4).filter(|p| p[1] > 128).count() as f32 / (w * h) as f32;
|
||||
eprintln!("Dissolve @ {op}: {differ} differing pixels, {kept:.3} kept");
|
||||
assert_eq!(differ, 0, "opacity {op}");
|
||||
assert!((kept - op).abs() < 0.02);
|
||||
}
|
||||
// fixed-function only: no backdrop copy
|
||||
assert!(c.backdrop.is_none());
|
||||
}
|
||||
|
||||
/// Normal-only plans keep the single fixed-function pass (no backdrop texture), and blend-mode
|
||||
/// layers entirely off the output draw nothing.
|
||||
#[test]
|
||||
fn normal_fast_path_and_blend_layers_off_output() {
|
||||
let Some((dev, q)) = device() else {
|
||||
eprintln!("no GPU adapter; skipping");
|
||||
return;
|
||||
};
|
||||
let mut c = GpuCompositor::new(&dev, &q);
|
||||
let grey = Arc::new(VideoFrame::rgba_f32(4, 4, vec![0.2; 64]));
|
||||
let layer = |m: Affine, blend: Blend| PlanLayer { frame: grey.clone(), matrix: m, opacity: 1.0, blend };
|
||||
let normal = FramePlan::Layers { width: 32, height: 16, layers: vec![layer(Affine::scale(8.0, 4.0), Blend::Normal)] };
|
||||
c.composite(&normal);
|
||||
assert!(c.backdrop.is_none());
|
||||
let (_, _, before) = c.read_output().expect("readback");
|
||||
let off = FramePlan::Layers {
|
||||
width: 32,
|
||||
height: 16,
|
||||
layers: vec![
|
||||
layer(Affine::scale(8.0, 4.0), Blend::Normal),
|
||||
layer(Affine::translate(100.0, 0.0), Blend::Multiply),
|
||||
layer(Affine::translate(-10.0, -10.0), Blend::Screen),
|
||||
],
|
||||
};
|
||||
c.composite(&off);
|
||||
let (_, _, after) = c.read_output().expect("readback");
|
||||
assert_eq!(before, after);
|
||||
// only blend-mode layers first: the accumulator is still cleared
|
||||
let only = FramePlan::Layers { width: 32, height: 16, layers: vec![layer(Affine::translate(100.0, 0.0), Blend::Multiply)] };
|
||||
c.composite(&only);
|
||||
let (_, _, empty) = c.read_output().expect("readback");
|
||||
assert!(empty.chunks(4).all(|p| p[..3] == [0, 0, 0]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quad_bounds_clamps_and_handles_non_finite() {
|
||||
let l = |m: Affine| PlanLayer { frame: Arc::new(VideoFrame::rgba_f32(10, 10, vec![0.0; 400])), matrix: m, opacity: 1.0, blend: Blend::Screen };
|
||||
assert_eq!(quad_bounds(&l(Affine::translate(5.0, 5.0)), 100, 100), Some((4, 4, 12, 12)));
|
||||
assert_eq!(quad_bounds(&l(Affine::translate(-5.0, 95.0)), 100, 100), Some((0, 94, 6, 6)));
|
||||
assert_eq!(quad_bounds(&l(Affine::translate(f64::NAN, 0.0)), 100, 100), Some((0, 0, 100, 100)));
|
||||
assert_eq!(quad_bounds(&l(Affine::scale(f64::INFINITY, 1.0)), 100, 100), Some((0, 0, 100, 100)));
|
||||
assert_eq!(quad_bounds(&l(Affine::translate(-50.0, 0.0)), 100, 100), None);
|
||||
}
|
||||
|
|
@ -2,7 +2,10 @@
|
|||
// Layers are drawn as transformed quads; the fragment shader samples the source (YUV planes or
|
||||
// RGBA) with manual bilinear filtering (textureLoad, so any float format works) and N×N
|
||||
// supersampling over the pixel footprint when minifying, converts to linear light, premultiplies
|
||||
// and scales by opacity. Blending into the Rgba16Float accumulator is premultiplied "over".
|
||||
// and scales by opacity. Normal layers blend into the Rgba16Float accumulator with fixed-function
|
||||
// premultiplied "over"; Dissolve does too (each pixel is either dropped or drawn opaque). The other
|
||||
// blend modes (`fs_blend`) read the accumulator under the layer from a copy (`backdrop`) and write
|
||||
// the composited result, with the math of `filmcraft_render::blend::composite`.
|
||||
|
||||
struct U {
|
||||
m0: vec4<f32>, // a b c d
|
||||
|
|
@ -11,12 +14,15 @@ struct U {
|
|||
p0: vec4<f32>, // opacity, kind (0 rgba8 srgb straight, 1 rgba16f premul linear, 2 yuv), taps, transfer (0 srgb, 1 linear, 2 pq, 3 hlg)
|
||||
p1: vec4<f32>, // y_off y_scale c_off c_scale (code units)
|
||||
p2: vec4<f32>, // kr kb code_scale footprint
|
||||
p3: vec4<f32>, // blend mode (index into filmcraft_render::Blend::ALL), unused ×3
|
||||
};
|
||||
|
||||
@group(0) @binding(0) var<uniform> u: U;
|
||||
@group(0) @binding(1) var tex0: texture_2d<f32>;
|
||||
@group(0) @binding(2) var tex1: texture_2d<f32>;
|
||||
@group(0) @binding(3) var tex2: texture_2d<f32>;
|
||||
// The accumulator under the layer (blend modes other than Normal / Dissolve only).
|
||||
@group(0) @binding(4) var backdrop: texture_2d<f32>;
|
||||
|
||||
struct VOut {
|
||||
@builtin(position) pos: vec4<f32>,
|
||||
|
|
@ -100,20 +106,186 @@ fn sample(p: vec2<f32>) -> vec4<f32> {
|
|||
return vec4(to_linear(vec3(R, G, B)), 1.0);
|
||||
}
|
||||
|
||||
@fragment
|
||||
fn fs(in: VOut) -> @location(0) vec4<f32> {
|
||||
// The layer's premultiplied linear colour at this pixel, opacity applied.
|
||||
fn layer_color(sp: vec2<f32>) -> vec4<f32> {
|
||||
let n = max(u32(u.p0.z), 1u);
|
||||
let fp = u.p2.w;
|
||||
var acc = vec4(0.0);
|
||||
for (var j = 0u; j < n; j++) {
|
||||
for (var i = 0u; i < n; i++) {
|
||||
let off = (vec2(f32(i), f32(j)) + 0.5) / f32(n) - 0.5;
|
||||
acc += sample(in.sp + off * fp);
|
||||
acc += sample(sp + off * fp);
|
||||
}
|
||||
}
|
||||
return acc / f32(n * n) * u.p0.x;
|
||||
}
|
||||
|
||||
// ---- Dissolve: the CPU's 64-bit pixel hash (`blend::hash2`), on 32-bit halves (lo, hi).
|
||||
|
||||
// Full 64-bit product of two u32.
|
||||
fn mul32(a: u32, b: u32) -> vec2<u32> {
|
||||
let a0 = a & 0xffffu; let a1 = a >> 16u; let b0 = b & 0xffffu; let b1 = b >> 16u;
|
||||
let p00 = a0 * b0; let p01 = a0 * b1; let p10 = a1 * b0; let p11 = a1 * b1;
|
||||
let mid = (p00 >> 16u) + (p01 & 0xffffu) + (p10 & 0xffffu);
|
||||
return vec2((mid << 16u) | (p00 & 0xffffu), p11 + (p01 >> 16u) + (p10 >> 16u) + (mid >> 16u));
|
||||
}
|
||||
|
||||
// Wrapping 64-bit multiply.
|
||||
fn mul64(a: vec2<u32>, b: vec2<u32>) -> vec2<u32> {
|
||||
let p = mul32(a.x, b.x);
|
||||
return vec2(p.x, p.y + a.x * b.y + a.y * b.x);
|
||||
}
|
||||
|
||||
// 64-bit shift right by 0 < n < 32.
|
||||
fn shr64(a: vec2<u32>, n: u32) -> vec2<u32> {
|
||||
return vec2((a.x >> n) | (a.y << (32u - n)), a.y >> n);
|
||||
}
|
||||
|
||||
fn hash2(x: u32, y: u32) -> f32 {
|
||||
var h = mul64(vec2(x, 0u), vec2(0x7F4A7C15u, 0x9E3779B9u)) ^ mul64(vec2(y, 0u), vec2(0x27D4EB4Fu, 0xC2B2AE3Du));
|
||||
h = h ^ shr64(h, 31u);
|
||||
h = mul64(h, vec2(0x1CE4E5B9u, 0xBF58476Du));
|
||||
h = h ^ shr64(h, 29u);
|
||||
return f32(h.y >> 8u) / 16777216.0;
|
||||
}
|
||||
|
||||
// Normal and Dissolve (fixed-function premultiplied "over").
|
||||
@fragment
|
||||
fn fs(in: VOut) -> @location(0) vec4<f32> {
|
||||
let c = layer_color(in.sp);
|
||||
if u32(u.p3.x) == 1u {
|
||||
// Dissolve: keep the pixel with probability alpha, then draw it opaque.
|
||||
let sa = c.a;
|
||||
if hash2(u32(in.pos.x), u32(in.pos.y)) >= sa {
|
||||
discard;
|
||||
}
|
||||
return vec4(c.rgb / sa, 1.0);
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
// ---- the other blend modes: W3C Compositing and Blending formulas on straight, sRGB-encoded
|
||||
// colour, exactly as `filmcraft_render::blend` (no recursion in WGSL, so helpers are split out).
|
||||
|
||||
fn screen(b: f32, s: f32) -> f32 { return b + s - b * s; }
|
||||
|
||||
fn color_burn(b: f32, s: f32) -> f32 {
|
||||
if b >= 1.0 { return 1.0; }
|
||||
if s <= 0.0 { return 0.0; }
|
||||
return 1.0 - min((1.0 - b) / s, 1.0);
|
||||
}
|
||||
|
||||
fn color_dodge(b: f32, s: f32) -> f32 {
|
||||
if b <= 0.0 { return 0.0; }
|
||||
if s >= 1.0 { return 1.0; }
|
||||
return min(b / (1.0 - s), 1.0);
|
||||
}
|
||||
|
||||
fn hard_light(b: f32, s: f32) -> f32 {
|
||||
if s <= 0.5 { return b * 2.0 * s; }
|
||||
return screen(b, 2.0 * s - 1.0);
|
||||
}
|
||||
|
||||
fn vivid_light(b: f32, s: f32) -> f32 {
|
||||
if s <= 0.5 { return color_burn(b, 2.0 * s); }
|
||||
return color_dodge(b, 2.0 * s - 1.0);
|
||||
}
|
||||
|
||||
fn sep(mode: u32, b: f32, s: f32) -> f32 {
|
||||
switch mode {
|
||||
case 2u: { return min(b, s); } // Darken
|
||||
case 3u: { return b * s; } // Multiply
|
||||
case 4u: { return color_burn(b, s); }
|
||||
case 5u: { return max(b + s - 1.0, 0.0); } // Linear Burn
|
||||
case 7u: { return max(b, s); } // Lighten
|
||||
case 8u: { return screen(b, s); }
|
||||
case 9u: { return color_dodge(b, s); }
|
||||
case 10u: { return min(b + s, 1.0); } // Linear Dodge (Add)
|
||||
case 12u: { return hard_light(s, b); } // Overlay
|
||||
case 13u: { // Soft Light
|
||||
if s <= 0.5 { return b - (1.0 - 2.0 * s) * b * (1.0 - b); }
|
||||
var d = sqrt(b);
|
||||
if b <= 0.25 { d = ((16.0 * b - 12.0) * b + 4.0) * b; }
|
||||
return b + (2.0 * s - 1.0) * (d - b);
|
||||
}
|
||||
case 14u: { return hard_light(b, s); }
|
||||
case 15u: { return vivid_light(b, s); }
|
||||
case 16u: { return clamp(b + 2.0 * s - 1.0, 0.0, 1.0); } // Linear Light
|
||||
case 17u: { // Pin Light
|
||||
if s <= 0.5 { return min(b, 2.0 * s); }
|
||||
return max(b, 2.0 * s - 1.0);
|
||||
}
|
||||
case 18u: { return select(0.0, 1.0, vivid_light(b, s) >= 0.5); } // Hard Mix
|
||||
case 19u: { return abs(b - s); } // Difference
|
||||
case 20u: { return b + s - 2.0 * b * s; } // Exclusion
|
||||
case 21u: { return max(b - s, 0.0); } // Subtract
|
||||
case 22u: { // Divide
|
||||
if s <= 0.0 { return 1.0; }
|
||||
return min(b / s, 1.0);
|
||||
}
|
||||
default: { return s; }
|
||||
}
|
||||
}
|
||||
|
||||
fn lum(c: vec3<f32>) -> f32 { return 0.3 * c.r + 0.59 * c.g + 0.11 * c.b; }
|
||||
|
||||
fn clip_color(c: vec3<f32>) -> vec3<f32> {
|
||||
let l = lum(c);
|
||||
let n = min(min(c.r, c.g), c.b);
|
||||
let x = max(max(c.r, c.g), c.b);
|
||||
var o = c;
|
||||
if n < 0.0 { o = l + (o - l) * l / max(l - n, 1e-6); }
|
||||
if x > 1.0 { o = l + (o - l) * (1.0 - l) / max(x - l, 1e-6); }
|
||||
return o;
|
||||
}
|
||||
|
||||
fn set_lum(c: vec3<f32>, l: f32) -> vec3<f32> { return clip_color(c + (l - lum(c))); }
|
||||
|
||||
fn sat(c: vec3<f32>) -> f32 { return max(max(c.r, c.g), c.b) - min(min(c.r, c.g), c.b); }
|
||||
|
||||
fn set_sat(c: vec3<f32>, s: f32) -> vec3<f32> {
|
||||
let mx = max(max(c.r, c.g), c.b);
|
||||
let mn = min(min(c.r, c.g), c.b);
|
||||
if mx - mn <= 1e-6 { return vec3(0.0); }
|
||||
return (c - mn) * s / (mx - mn);
|
||||
}
|
||||
|
||||
fn blend_rgb(mode: u32, b: vec3<f32>, s: vec3<f32>) -> vec3<f32> {
|
||||
switch mode {
|
||||
case 23u: { return set_lum(set_sat(s, sat(b)), lum(b)); } // Hue
|
||||
case 24u: { return set_lum(set_sat(b, sat(s)), lum(b)); } // Saturation
|
||||
case 25u: { return set_lum(s, lum(b)); } // Color
|
||||
case 26u: { return set_lum(b, lum(s)); } // Luminosity
|
||||
case 6u: { return select(b, s, lum(s) < lum(b)); } // Darker Color
|
||||
case 11u: { return select(b, s, lum(s) > lum(b)); } // Lighter Color
|
||||
default: { return vec3(sep(mode, b.r, s.r), sep(mode, b.g, s.g), sep(mode, b.b, s.b)); }
|
||||
}
|
||||
}
|
||||
|
||||
fn linear_to_srgb(v: vec3<f32>) -> vec3<f32> {
|
||||
let hi = 1.055 * pow(max(v, vec3(0.0)), vec3(1.0 / 2.4)) - 0.055;
|
||||
return select(hi, v * 12.92, v <= vec3(0.0031308));
|
||||
}
|
||||
|
||||
// Blend modes that need the destination: written without fixed-function blending.
|
||||
@fragment
|
||||
fn fs_blend(in: VOut) -> @location(0) vec4<f32> {
|
||||
let sp = layer_color(in.sp);
|
||||
let sa = sp.a;
|
||||
if sa <= 0.0 {
|
||||
discard;
|
||||
}
|
||||
let d = textureLoad(backdrop, vec2<i32>(in.pos.xy), 0);
|
||||
let da = d.a;
|
||||
if da <= 0.0 {
|
||||
return sp + d * (1.0 - sa);
|
||||
}
|
||||
let cs = linear_to_srgb(sp.rgb / sa);
|
||||
let cb = linear_to_srgb(d.rgb / da);
|
||||
let mixed = srgb_to_linear(clamp(blend_rgb(u32(u.p3.x), cb, cs), vec3(0.0), vec3(1.0)));
|
||||
return vec4(sp.rgb * (1.0 - da) + d.rgb * (1.0 - sa) + sa * da * mixed, sa + da - sa * da);
|
||||
}
|
||||
|
||||
// ---- final pass: accumulator (linear premul) over black → sRGB target
|
||||
|
||||
@group(0) @binding(0) var accum: texture_2d<f32>;
|
||||
|
|
|
|||
|
|
@ -7,6 +7,13 @@
|
|||
//! registers as a native texture. Uploads are cached by pixel-buffer identity, so a paused frame or
|
||||
//! a still costs nothing.
|
||||
//!
|
||||
//! Every blend mode of [`filmcraft_render::Blend`] runs here. Normal and Dissolve (whose per-pixel
|
||||
//! pattern is the CPU's 64-bit hash, reproduced in WGSL) use fixed-function "over" blending. The
|
||||
//! other 25 modes need the colour under the layer: before such a layer is drawn, the accumulator
|
||||
//! region under its quad is copied into a backdrop texture, and the fragment shader composites
|
||||
//! with the CPU reference's math (`filmcraft_render::blend::composite`: sRGB-encoded straight
|
||||
//! colour, W3C formulas, back to linear) and writes the result. Only those layers pay the copy.
|
||||
//!
|
||||
//! Layers that need converting before upload (linear f32 RGBA from CPU-rendered layers, 16-bit
|
||||
//! YUV such as ProRes) are converted to half floats by [`prepare`], which frame workers run off the
|
||||
//! UI thread; [`GpuCompositor::composite_prepared`] then only copies bytes into textures.
|
||||
|
|
@ -22,6 +29,7 @@ use rayon::prelude::*;
|
|||
|
||||
use filmcraft_color::{Matrix, Range, Transfer};
|
||||
use filmcraft_frame::{Chroma, PixelData, VideoFrame};
|
||||
use filmcraft_render::Blend;
|
||||
use filmcraft_render::plan::{FramePlan, PlanLayer};
|
||||
|
||||
pub mod lut;
|
||||
|
|
@ -49,10 +57,15 @@ pub struct GpuCompositor {
|
|||
device: wgpu::Device,
|
||||
queue: wgpu::Queue,
|
||||
layer_pipeline: wgpu::RenderPipeline,
|
||||
/// Blend modes that read the destination: `fs_blend`, no fixed-function blending.
|
||||
blend_pipeline: wgpu::RenderPipeline,
|
||||
final_pipeline: wgpu::RenderPipeline,
|
||||
layer_bgl: wgpu::BindGroupLayout,
|
||||
blend_bgl: wgpu::BindGroupLayout,
|
||||
final_bgl: wgpu::BindGroupLayout,
|
||||
accum: Option<(wgpu::Texture, wgpu::TextureView, (u32, u32))>,
|
||||
/// Copy of the accumulator under a blend-mode layer (allocated on first use).
|
||||
backdrop: Option<(wgpu::Texture, wgpu::TextureView, (u32, u32))>,
|
||||
output: Option<(wgpu::Texture, wgpu::TextureView, (u32, u32))>,
|
||||
uploads: HashMap<(usize, u32, u32), Uploaded>,
|
||||
clock: u64,
|
||||
|
|
@ -203,6 +216,21 @@ impl GpuCompositor {
|
|||
tex_entry(3),
|
||||
],
|
||||
});
|
||||
let blend_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
|
||||
label: Some("blend-layer"),
|
||||
entries: &[
|
||||
wgpu::BindGroupLayoutEntry {
|
||||
binding: 0,
|
||||
visibility: wgpu::ShaderStages::VERTEX_FRAGMENT,
|
||||
ty: wgpu::BindingType::Buffer { ty: wgpu::BufferBindingType::Uniform, has_dynamic_offset: false, min_binding_size: None },
|
||||
count: None,
|
||||
},
|
||||
tex_entry(1),
|
||||
tex_entry(2),
|
||||
tex_entry(3),
|
||||
tex_entry(4),
|
||||
],
|
||||
});
|
||||
let final_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
|
||||
label: Some("final"),
|
||||
entries: &[wgpu::BindGroupLayoutEntry { binding: 0, ..tex_entry(0) }],
|
||||
|
|
@ -229,6 +257,27 @@ impl GpuCompositor {
|
|||
multiview_mask: None,
|
||||
cache: None,
|
||||
});
|
||||
let bpl = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
|
||||
label: Some("blend-layer"),
|
||||
bind_group_layouts: &[Some(&blend_bgl)],
|
||||
immediate_size: 0,
|
||||
});
|
||||
let blend_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
|
||||
label: Some("blend-layer"),
|
||||
layout: Some(&bpl),
|
||||
vertex: wgpu::VertexState { module: &shader, entry_point: Some("vs"), compilation_options: Default::default(), buffers: &[] },
|
||||
primitive: wgpu::PrimitiveState::default(),
|
||||
depth_stencil: None,
|
||||
multisample: wgpu::MultisampleState::default(),
|
||||
fragment: Some(wgpu::FragmentState {
|
||||
module: &shader,
|
||||
entry_point: Some("fs_blend"),
|
||||
compilation_options: Default::default(),
|
||||
targets: &[Some(wgpu::ColorTargetState { format: ACCUM_FORMAT, blend: None, write_mask: wgpu::ColorWrites::ALL })],
|
||||
}),
|
||||
multiview_mask: None,
|
||||
cache: None,
|
||||
});
|
||||
let fpl =
|
||||
device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: Some("final"), bind_group_layouts: &[Some(&final_bgl)], immediate_size: 0 });
|
||||
let final_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
|
||||
|
|
@ -262,10 +311,13 @@ impl GpuCompositor {
|
|||
device: device.clone(),
|
||||
queue: queue.clone(),
|
||||
layer_pipeline,
|
||||
blend_pipeline,
|
||||
final_pipeline,
|
||||
layer_bgl,
|
||||
blend_bgl,
|
||||
final_bgl,
|
||||
accum: None,
|
||||
backdrop: None,
|
||||
output: None,
|
||||
uploads: HashMap::new(),
|
||||
clock: 0,
|
||||
|
|
@ -394,7 +446,7 @@ impl GpuCompositor {
|
|||
key
|
||||
}
|
||||
|
||||
fn uniforms(&self, l: &PlanLayer, key: (usize, u32, u32), out: (u32, u32)) -> [f32; 24] {
|
||||
fn uniforms(&self, l: &PlanLayer, key: (usize, u32, u32), out: (u32, u32)) -> [f32; 28] {
|
||||
let up = &self.uploads[&key];
|
||||
let f = &l.frame;
|
||||
let m = &l.matrix;
|
||||
|
|
@ -442,6 +494,10 @@ impl GpuCompositor {
|
|||
kb,
|
||||
up.code_scale,
|
||||
footprint.max(1.0),
|
||||
l.blend.index() as f32,
|
||||
0.0,
|
||||
0.0,
|
||||
0.0,
|
||||
]
|
||||
}
|
||||
|
||||
|
|
@ -467,12 +523,17 @@ impl GpuCompositor {
|
|||
},
|
||||
None => VideoFrame::rgba_f32(img.w as u32, img.h as u32, img.px.clone()),
|
||||
};
|
||||
owned = [PlanLayer { frame: Arc::new(frame), matrix: filmcraft_geom::Affine::IDENTITY, opacity: 1.0 }];
|
||||
owned = [PlanLayer { frame: Arc::new(frame), matrix: filmcraft_geom::Affine::IDENTITY, opacity: 1.0, blend: Blend::Normal }];
|
||||
(img.w as u32, img.h as u32, &owned[..])
|
||||
}
|
||||
};
|
||||
let (w, h) = (w.max(1), h.max(1));
|
||||
let accum_view = Self::target(&self.device, &mut self.accum, w, h, ACCUM_FORMAT, wgpu::TextureUsages::empty());
|
||||
let accum_view = Self::target(&self.device, &mut self.accum, w, h, ACCUM_FORMAT, wgpu::TextureUsages::COPY_SRC);
|
||||
let backdrop_view = if layers.iter().any(|l| l.blend.reads_destination()) {
|
||||
Some(Self::target(&self.device, &mut self.backdrop, w, h, ACCUM_FORMAT, wgpu::TextureUsages::COPY_DST))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let out_view = Self::target(&self.device, &mut self.output, w, h, OUTPUT_FORMAT, wgpu::TextureUsages::COPY_SRC);
|
||||
let keys: Vec<(usize, u32, u32)> = layers
|
||||
.iter()
|
||||
|
|
@ -482,6 +543,8 @@ impl GpuCompositor {
|
|||
self.upload(&l.frame, p)
|
||||
})
|
||||
.collect();
|
||||
// (bind group, None for a fixed-function layer or Some(region of the accumulator to copy
|
||||
// into the backdrop) for a layer that reads the destination; Some(None): off the output)
|
||||
let mut bind_groups = Vec::with_capacity(layers.len());
|
||||
for (l, k) in layers.iter().zip(&keys) {
|
||||
let u = self.uniforms(l, *k, (w, h));
|
||||
|
|
@ -494,17 +557,21 @@ impl GpuCompositor {
|
|||
});
|
||||
self.queue.write_buffer(&buf, 0, &bytes);
|
||||
let up = &self.uploads[k];
|
||||
let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
|
||||
label: Some("layer"),
|
||||
layout: &self.layer_bgl,
|
||||
entries: &[
|
||||
let mut entries = vec![
|
||||
wgpu::BindGroupEntry { binding: 0, resource: buf.as_entire_binding() },
|
||||
wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::TextureView(&up.views[0]) },
|
||||
wgpu::BindGroupEntry { binding: 2, resource: wgpu::BindingResource::TextureView(&up.views[1]) },
|
||||
wgpu::BindGroupEntry { binding: 3, resource: wgpu::BindingResource::TextureView(&up.views[2]) },
|
||||
],
|
||||
});
|
||||
bind_groups.push(bg);
|
||||
];
|
||||
let (layout, region) = match (&backdrop_view, l.blend.reads_destination()) {
|
||||
(Some(b), true) => {
|
||||
entries.push(wgpu::BindGroupEntry { binding: 4, resource: wgpu::BindingResource::TextureView(b) });
|
||||
(&self.blend_bgl, Some(quad_bounds(l, w, h)))
|
||||
}
|
||||
_ => (&self.layer_bgl, None),
|
||||
};
|
||||
let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("layer"), layout, entries: &entries });
|
||||
bind_groups.push((bg, region));
|
||||
}
|
||||
let final_bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
|
||||
label: Some("final"),
|
||||
|
|
@ -512,25 +579,42 @@ impl GpuCompositor {
|
|||
entries: &[wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView(&accum_view) }],
|
||||
});
|
||||
let mut enc = self.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("filmcraft-composite") });
|
||||
{
|
||||
let mut pass = enc.begin_render_pass(&wgpu::RenderPassDescriptor {
|
||||
label: Some("layers"),
|
||||
color_attachments: &[Some(wgpu::RenderPassColorAttachment {
|
||||
view: &accum_view,
|
||||
depth_slice: None,
|
||||
resolve_target: None,
|
||||
ops: wgpu::Operations { load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), store: wgpu::StoreOp::Store },
|
||||
})],
|
||||
depth_stencil_attachment: None,
|
||||
timestamp_writes: None,
|
||||
occlusion_query_set: None,
|
||||
multiview_mask: None,
|
||||
});
|
||||
// Runs of fixed-function layers draw in one pass. A layer that reads the destination
|
||||
// ends it, copies the accumulator under its quad into the backdrop and draws on its own.
|
||||
let accum_tex = self.accum.as_ref().map(|a| a.0.clone());
|
||||
let backdrop_tex = self.backdrop.as_ref().map(|b| b.0.clone());
|
||||
let mut cleared = false;
|
||||
let mut i = 0;
|
||||
while i < bind_groups.len() || !cleared {
|
||||
if let Some((bg, Some(region))) = bind_groups.get(i) {
|
||||
i += 1;
|
||||
let (Some(rect), Some(src), Some(dst)) = (region, &accum_tex, &backdrop_tex) else { continue };
|
||||
if !cleared {
|
||||
accum_pass(&mut enc, &accum_view, wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT));
|
||||
cleared = true;
|
||||
}
|
||||
let origin = wgpu::Origin3d { x: rect.0, y: rect.1, z: 0 };
|
||||
enc.copy_texture_to_texture(
|
||||
wgpu::TexelCopyTextureInfo { texture: src, mip_level: 0, origin, aspect: wgpu::TextureAspect::All },
|
||||
wgpu::TexelCopyTextureInfo { texture: dst, mip_level: 0, origin, aspect: wgpu::TextureAspect::All },
|
||||
wgpu::Extent3d { width: rect.2, height: rect.3, depth_or_array_layers: 1 },
|
||||
);
|
||||
let mut pass = accum_pass(&mut enc, &accum_view, wgpu::LoadOp::Load);
|
||||
pass.set_pipeline(&self.blend_pipeline);
|
||||
pass.set_bind_group(0, bg, &[]);
|
||||
pass.draw(0..6, 0..1);
|
||||
continue;
|
||||
}
|
||||
let end = bind_groups.iter().skip(i).position(|(_, r)| r.is_some()).map_or(bind_groups.len(), |n| i + n);
|
||||
let load = if cleared { wgpu::LoadOp::Load } else { wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT) };
|
||||
let mut pass = accum_pass(&mut enc, &accum_view, load);
|
||||
cleared = true;
|
||||
pass.set_pipeline(&self.layer_pipeline);
|
||||
for bg in &bind_groups {
|
||||
for (bg, _) in bind_groups.get(i..end).unwrap_or_default() {
|
||||
pass.set_bind_group(0, bg, &[]);
|
||||
pass.draw(0..6, 0..1);
|
||||
}
|
||||
i = end;
|
||||
}
|
||||
{
|
||||
let mut pass = enc.begin_render_pass(&wgpu::RenderPassDescriptor {
|
||||
|
|
@ -589,5 +673,44 @@ impl GpuCompositor {
|
|||
}
|
||||
}
|
||||
|
||||
/// A render pass drawing into the accumulator.
|
||||
fn accum_pass<'e>(enc: &'e mut wgpu::CommandEncoder, view: &wgpu::TextureView, load: wgpu::LoadOp<wgpu::Color>) -> wgpu::RenderPass<'e> {
|
||||
enc.begin_render_pass(&wgpu::RenderPassDescriptor {
|
||||
label: Some("layers"),
|
||||
color_attachments: &[Some(wgpu::RenderPassColorAttachment {
|
||||
view,
|
||||
depth_slice: None,
|
||||
resolve_target: None,
|
||||
ops: wgpu::Operations { load, store: wgpu::StoreOp::Store },
|
||||
})],
|
||||
depth_stencil_attachment: None,
|
||||
timestamp_writes: None,
|
||||
occlusion_query_set: None,
|
||||
multiview_mask: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// The output pixels a layer's quad can touch, as (x, y, width, height) clamped to the `w`×`h`
|
||||
/// output with a one-pixel margin; None when it touches none. A non-finite matrix covers it all.
|
||||
fn quad_bounds(l: &PlanLayer, w: u32, h: u32) -> Option<(u32, u32, u32, u32)> {
|
||||
let m = &l.matrix;
|
||||
let (fw, fh) = (l.frame.width as f64, l.frame.height as f64);
|
||||
let pts = [(0.0, 0.0), (fw, 0.0), (0.0, fh), (fw, fh)].map(|(x, y)| (m.a * x + m.c * y + m.e, m.b * x + m.d * y + m.f));
|
||||
if pts.iter().any(|p| !p.0.is_finite() || !p.1.is_finite()) {
|
||||
return Some((0, 0, w, h));
|
||||
}
|
||||
let (mut x0, mut y0, mut x1, mut y1) = (f64::INFINITY, f64::INFINITY, f64::NEG_INFINITY, f64::NEG_INFINITY);
|
||||
for (x, y) in pts {
|
||||
(x0, y0, x1, y1) = (x0.min(x), y0.min(y), x1.max(x), y1.max(y));
|
||||
}
|
||||
// in range before the cast (`as` saturates anyway; NaN is excluded above)
|
||||
let px = |v: f64, hi: u32| v.max(0.0).min(hi as f64) as u32;
|
||||
let (x0, y0) = (px(x0.floor() - 1.0, w), px(y0.floor() - 1.0, h));
|
||||
let (x1, y1) = (px(x1.ceil() + 1.0, w), px(y1.ceil() + 1.0, h));
|
||||
(x1 > x0 && y1 > y0).then(|| (x0, y0, x1 - x0, y1 - y0))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod blend_tests;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
use super::*;
|
||||
use filmcraft_geom::{Affine, Vec2};
|
||||
use filmcraft_render::Blend;
|
||||
use filmcraft_render::plan::execute_cpu;
|
||||
|
||||
fn device() -> Option<(wgpu::Device, wgpu::Queue)> {
|
||||
|
|
@ -44,8 +45,13 @@ fn gpu_matches_cpu_plan() {
|
|||
width: w,
|
||||
height: h,
|
||||
layers: vec![
|
||||
PlanLayer { frame: yuv_frame(640, 360), matrix: Affine::scale(0.5, 0.5), opacity: 1.0 },
|
||||
PlanLayer { frame: rgba, matrix: Affine::motion(Vec2::new(200.0, 100.0), Vec2::new(0.4, 0.4), 12.0, Vec2::new(160.0, 90.0)), opacity: 0.7 },
|
||||
PlanLayer { frame: yuv_frame(640, 360), matrix: Affine::scale(0.5, 0.5), opacity: 1.0, blend: Blend::Normal },
|
||||
PlanLayer {
|
||||
frame: rgba,
|
||||
matrix: Affine::motion(Vec2::new(200.0, 100.0), Vec2::new(0.4, 0.4), 12.0, Vec2::new(160.0, 90.0)),
|
||||
opacity: 0.7,
|
||||
blend: Blend::Normal,
|
||||
},
|
||||
],
|
||||
};
|
||||
let cpu = execute_cpu(&plan).over_black_rgba8();
|
||||
|
|
@ -108,8 +114,8 @@ fn prepared_upload_matches_inline_conversion() {
|
|||
width: w as usize,
|
||||
height: h as usize,
|
||||
layers: vec![
|
||||
PlanLayer { frame: yuv16, matrix: Affine::IDENTITY, opacity: 1.0 },
|
||||
PlanLayer { frame: rgbaf, matrix: Affine::scale(0.5, 0.5), opacity: 0.8 },
|
||||
PlanLayer { frame: yuv16, matrix: Affine::IDENTITY, opacity: 1.0, blend: Blend::Normal },
|
||||
PlanLayer { frame: rgbaf, matrix: Affine::scale(0.5, 0.5), opacity: 0.8, blend: Blend::Normal },
|
||||
],
|
||||
};
|
||||
let image = FramePlan::Image(filmcraft_render::Image { w: w as usize, h: h as usize, px: f32_layer });
|
||||
|
|
@ -138,7 +144,7 @@ fn upload_cache_keeps_buffers_alive() {
|
|||
let mut c = GpuCompositor::new(&dev, &q);
|
||||
let px = Arc::new(vec![200u8; 16 * 8 * 4]);
|
||||
let frame = Arc::new(VideoFrame { width: 16, height: 8, data: PixelData::Rgba8(px.clone()), ..(*yuv_frame(16, 8)).clone() });
|
||||
let plan = FramePlan::Layers { width: 16, height: 8, layers: vec![PlanLayer { frame, matrix: Affine::IDENTITY, opacity: 1.0 }] };
|
||||
let plan = FramePlan::Layers { width: 16, height: 8, layers: vec![PlanLayer { frame, matrix: Affine::IDENTITY, opacity: 1.0, blend: Blend::Normal }] };
|
||||
c.composite(&plan);
|
||||
drop(plan);
|
||||
assert!(Arc::strong_count(&px) > 1, "cached upload must own its pixel buffer");
|
||||
|
|
|
|||
|
|
@ -70,6 +70,15 @@ impl Blend {
|
|||
Blend::Color,
|
||||
Blend::Luminosity,
|
||||
];
|
||||
/// Position in [`Blend::ALL`] (= `filmcraft_project::effect::BLEND_MODES`).
|
||||
pub fn index(self) -> u32 {
|
||||
Self::ALL.iter().position(|b| *b == self).unwrap_or(0) as u32
|
||||
}
|
||||
/// Whether compositing needs the destination colour (every mode but Normal and Dissolve,
|
||||
/// which are plain "over" per pixel).
|
||||
pub fn reads_destination(self) -> bool {
|
||||
!matches!(self, Blend::Normal | Blend::Dissolve)
|
||||
}
|
||||
/// Index into `filmcraft_project::effect::BLEND_MODES`.
|
||||
pub fn from_index(i: u32) -> Blend {
|
||||
Self::ALL.get(i as usize).copied().unwrap_or(Blend::Normal)
|
||||
|
|
|
|||
|
|
@ -2,10 +2,11 @@
|
|||
//!
|
||||
//! [`plan_frame`] resolves what is visible at a time into a list of layers the GPU can draw
|
||||
//! directly: a decoded source frame (YUV planes or RGBA), the matrix from source pixels to output
|
||||
//! pixels, and an opacity. Anything the shaders don't cover yet — non-Normal blend modes,
|
||||
//! standard effects, adjustment layers, nested sequences, non-dissolve transitions — is rendered
|
||||
//! on the CPU for that layer (or the whole frame) and handed over as a pre-composited image, so
|
||||
//! the GPU path is always exact with respect to the CPU reference.
|
||||
//! pixels, an opacity and a blend mode (all 27 of [`Blend`] are composited by the GPU). Anything
|
||||
//! the shaders don't cover yet — standard effects, adjustment layers, nested sequences,
|
||||
//! non-dissolve transitions — is rendered on the CPU for that layer (or the whole frame) and
|
||||
//! handed over as a pre-composited image, so the GPU path is always exact with respect to the CPU
|
||||
//! reference.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
|
|
@ -24,6 +25,8 @@ pub struct PlanLayer {
|
|||
/// Maps frame pixels (0..w, 0..h) to output pixels.
|
||||
pub matrix: Affine,
|
||||
pub opacity: f32,
|
||||
/// How the layer combines with what is under it ([`crate::blend::composite`]).
|
||||
pub blend: Blend,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
|
|
@ -42,14 +45,12 @@ fn simple_transition(id: &str) -> bool {
|
|||
matches!(id, "cross_dissolve" | "dip_to_black" | "dip_to_white" | "morph_cut")
|
||||
}
|
||||
|
||||
/// Whether a track item can be drawn by the GPU as-is (no standard effects, Normal blend).
|
||||
fn gpu_simple(project: &Project, item: &TrackItem, mt: Tick) -> bool {
|
||||
/// Whether a track item's source can be drawn by the GPU as-is (media without standard effects or
|
||||
/// opacity masks; any blend mode).
|
||||
fn gpu_simple(project: &Project, item: &TrackItem) -> bool {
|
||||
let is_media = project.item(item.item).is_some_and(|p| matches!(p.kind, ItemKind::Media(_) | ItemKind::Subclip { .. }));
|
||||
let no_fx = !item.has_standard_effects() && !item.has_opacity_masks();
|
||||
let normal =
|
||||
item.effect("opacity").is_none_or(|e| !e.enabled || e.param("blend").is_none_or(|p| matches!(p.value, filmcraft_project::ParamValue::Choice(0))));
|
||||
let _ = mt;
|
||||
is_media && no_fx && normal
|
||||
is_media && no_fx
|
||||
}
|
||||
|
||||
/// Plan the frame at timeline `t`.
|
||||
|
|
@ -71,8 +72,7 @@ pub fn plan_frame(project: &Project, seq_id: ItemId, t: Tick, opts: RenderOption
|
|||
return FramePlan::Image(crate::render_sequence(project, seq_id, t, opts, sources));
|
||||
}
|
||||
if let Some(it) = tr.item_at(t)
|
||||
&& (project.item(it.item).is_some_and(|p| matches!(p.kind, ItemKind::AdjustmentLayer { .. }))
|
||||
|| crate::opacity_blend(it, it.source_time_at(t)).1 != Blend::Normal)
|
||||
&& project.item(it.item).is_some_and(|p| matches!(p.kind, ItemKind::AdjustmentLayer { .. }))
|
||||
{
|
||||
return FramePlan::Image(crate::render_sequence(project, seq_id, t, opts, sources));
|
||||
}
|
||||
|
|
@ -94,19 +94,20 @@ pub fn plan_frame(project: &Project, seq_id: ItemId, t: Tick, opts: RenderOption
|
|||
frame: Arc::new(VideoFrame::rgba_f32(1, 1, col.to_vec())),
|
||||
matrix: Affine::scale(w as f64, h as f64),
|
||||
opacity: 1.0,
|
||||
blend: Blend::Normal,
|
||||
});
|
||||
let (it, k) = if p < 0.5 { (a, 1.0 - p * 2.0) } else { (b, (p - 0.5) * 2.0) };
|
||||
if let Some(it) = it {
|
||||
push_item(project, seq, it, t, opts, sources, k, &mut layers);
|
||||
push_item(project, seq, it, t, opts, sources, k, Some(Blend::Normal), &mut layers);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
// cross dissolve: A at full, B over it at p (premultiplied over == linear mix when A is opaque)
|
||||
if let Some(it) = a {
|
||||
push_item(project, seq, it, t, opts, sources, 1.0 - if b.is_none() { p } else { 0.0 }, &mut layers);
|
||||
push_item(project, seq, it, t, opts, sources, 1.0 - if b.is_none() { p } else { 0.0 }, Some(Blend::Normal), &mut layers);
|
||||
}
|
||||
if let Some(it) = b {
|
||||
push_item(project, seq, it, t, opts, sources, p, &mut layers);
|
||||
push_item(project, seq, it, t, opts, sources, p, Some(Blend::Normal), &mut layers);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -116,7 +117,7 @@ pub fn plan_frame(project: &Project, seq_id: ItemId, t: Tick, opts: RenderOption
|
|||
if !item.enabled {
|
||||
continue;
|
||||
}
|
||||
push_item(project, seq, item, t, opts, sources, 1.0, &mut layers);
|
||||
push_item(project, seq, item, t, opts, sources, 1.0, None, &mut layers);
|
||||
}
|
||||
if opts.captions {
|
||||
for o in crate::caption_overlays(seq, t, w, h) {
|
||||
|
|
@ -124,12 +125,15 @@ pub fn plan_frame(project: &Project, seq_id: ItemId, t: Tick, opts: RenderOption
|
|||
frame: Arc::new(VideoFrame::rgba_f32(o.w as u32, o.h as u32, o.px)),
|
||||
matrix: Affine::translate(o.x as f64, o.y as f64),
|
||||
opacity: 1.0,
|
||||
blend: Blend::Normal,
|
||||
});
|
||||
}
|
||||
}
|
||||
FramePlan::Layers { width: w, height: h, layers }
|
||||
}
|
||||
|
||||
/// Push the layer(s) of `item`. `blend` overrides the item's own blend mode: inside a transition
|
||||
/// the CPU reference mixes the clips and composites the result Normal, ignoring their modes.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn push_item(
|
||||
project: &Project,
|
||||
|
|
@ -139,32 +143,35 @@ fn push_item(
|
|||
opts: RenderOptions,
|
||||
sources: &dyn SourceProvider,
|
||||
extra_opacity: f32,
|
||||
blend: Option<Blend>,
|
||||
out: &mut Vec<PlanLayer>,
|
||||
) {
|
||||
// frame time (`ft`) vs. effect time (`mt`): they differ inside a frame hold without Hold Filters
|
||||
let ft = item.source_time_at(t);
|
||||
let mt = item.effect_time_at(t);
|
||||
let (op, bl) = crate::opacity_blend(item, mt);
|
||||
let (op, own) = crate::opacity_blend(item, mt);
|
||||
let bl = blend.unwrap_or(own);
|
||||
// A multi-camera clip that only shows its angle (no effects, untransformed, same frame size)
|
||||
// draws the angle's clip directly: no CPU pass over the nested sequence.
|
||||
if let Some(ItemKind::Sequence(nested)) = project.item(item.item).map(|p| &p.kind)
|
||||
&& let Some(angle) = item.multicam_angle(nested)
|
||||
&& bl == Blend::Normal
|
||||
&& !(opts.effects && item.has_standard_effects())
|
||||
&& (nested.settings.width, nested.settings.height) == (seq.settings.width, seq.settings.height)
|
||||
&& near_identity(&motion_matrix(seq, item, (nested.settings.width, nested.settings.height), mt))
|
||||
&& let Some(tr) = nested.angle_video_track_index(angle).and_then(|i| nested.video_tracks.get(i))
|
||||
&& !tr.transitions.iter().any(|x| x.range().contains(ft))
|
||||
&& tr.item_at(ft).is_none_or(|i| crate::opacity_blend(i, i.effect_time_at(ft)).1 != Blend::Dissolve)
|
||||
{
|
||||
// Inside the nested sequence the angle's clip is composited onto an empty canvas, where
|
||||
// every mode but Dissolve is Normal; the multicam clip's own mode then applies to it.
|
||||
if let Some(inner) = tr.item_at(ft).filter(|i| i.enabled) {
|
||||
push_item(project, nested, inner, ft, opts, sources, extra_opacity * op, out);
|
||||
push_item(project, nested, inner, ft, opts, sources, extra_opacity * op, Some(bl), out);
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Graphic clips without standard effects: the layers are rasterised (cached) into one tight
|
||||
// image the GPU places as a layer.
|
||||
if bl == Blend::Normal
|
||||
&& !(opts.effects && item.has_standard_effects())
|
||||
if !(opts.effects && item.has_standard_effects())
|
||||
&& !item.has_opacity_masks()
|
||||
&& project.item(item.item).is_some_and(|p| matches!(p.kind, ItemKind::Graphic { .. }))
|
||||
{
|
||||
|
|
@ -172,11 +179,11 @@ fn push_item(
|
|||
let (w, h) = output_size(seq, opts.scale);
|
||||
let m = Affine::scale(opts.scale as f64, opts.scale as f64).then_apply(&motion_matrix(seq, item, size, mt));
|
||||
if let Some((img, x, y)) = crate::graphic_clip::render_graphic_tight(item, mt, size, &m, w, h) {
|
||||
out.push(PlanLayer { frame: cpu_frame(img), matrix: Affine::translate(x as f64, y as f64), opacity: op * extra_opacity });
|
||||
out.push(PlanLayer { frame: cpu_frame(img), matrix: Affine::translate(x as f64, y as f64), opacity: op * extra_opacity, blend: bl });
|
||||
}
|
||||
return;
|
||||
}
|
||||
if gpu_simple(project, item, mt) && bl == Blend::Normal {
|
||||
if gpu_simple(project, item) {
|
||||
let Some(src) = sources.source(item.item) else { return };
|
||||
let Some(size) = crate::source_size(project, item.item) else { return };
|
||||
let motion = motion_matrix(seq, item, size, mt);
|
||||
|
|
@ -198,14 +205,14 @@ fn push_item(
|
|||
};
|
||||
let px_scale = frame.width as f64 / size.0.max(1) as f64;
|
||||
let m = Affine::scale(opts.scale as f64, opts.scale as f64).then_apply(&motion).then_apply(&Affine::scale(1.0 / px_scale, 1.0 / px_scale));
|
||||
out.push(PlanLayer { frame, matrix: m, opacity: op * extra_opacity });
|
||||
out.push(PlanLayer { frame, matrix: m, opacity: op * extra_opacity, blend: bl });
|
||||
return;
|
||||
}
|
||||
}
|
||||
// CPU-rendered layer (standard effects): drawn by the GPU as a pre-rendered canvas image.
|
||||
let tc = filmcraft_time::format_time(t, seq.settings.frame_rate, seq.settings.drop_frame, filmcraft_time::TimeDisplay::Timecode, 48_000);
|
||||
if let Some((img, op2, _)) = crate::item_layer(project, seq, item, t, opts, sources, &tc) {
|
||||
out.push(PlanLayer { frame: cpu_frame(img), matrix: Affine::IDENTITY, opacity: op2 * extra_opacity });
|
||||
out.push(PlanLayer { frame: cpu_frame(img), matrix: Affine::IDENTITY, opacity: op2 * extra_opacity, blend: bl });
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -226,7 +233,7 @@ pub fn execute_cpu(plan: &FramePlan) -> crate::Image {
|
|||
} else {
|
||||
src.transformed(*width, *height, &l.matrix)
|
||||
};
|
||||
crate::blend::composite(&mut canvas, &placed, l.opacity, Blend::Normal);
|
||||
crate::blend::composite(&mut canvas, &placed, l.opacity, l.blend);
|
||||
}
|
||||
canvas
|
||||
}
|
||||
|
|
|
|||
|
|
@ -626,7 +626,7 @@ fn parse_args() -> Args {
|
|||
"--json" => a.json = it.next(),
|
||||
"--help" | "-h" => {
|
||||
println!(
|
||||
"bench_playback [--scenario h264-1080,stack3,h264-2160,hevc-2160,vp9-2160,av1-2160,demo,after-preview,seek-storm] [--res full,half] [--cpu|--gpu] [--seconds 8] [--refresh 60] [--workers N] [--repeat N] [--json out.json]"
|
||||
"bench_playback [--scenario h264-1080,stack3,stack3-blend,h264-2160,hevc-2160,vp9-2160,av1-2160,demo,after-preview,seek-storm] [--res full,half] [--cpu|--gpu] [--seconds 8] [--refresh 60] [--workers N] [--repeat N] [--json out.json]"
|
||||
);
|
||||
std::process::exit(0);
|
||||
}
|
||||
|
|
@ -634,7 +634,8 @@ fn parse_args() -> Args {
|
|||
}
|
||||
}
|
||||
if a.scenarios.is_empty() {
|
||||
a.scenarios = ["h264-1080", "stack3", "h264-2160", "hevc-2160", "demo", "after-preview", "seek-storm"].iter().map(|s| s.to_string()).collect();
|
||||
a.scenarios =
|
||||
["h264-1080", "stack3", "stack3-blend", "h264-2160", "hevc-2160", "demo", "after-preview", "seek-storm"].iter().map(|s| s.to_string()).collect();
|
||||
}
|
||||
a
|
||||
}
|
||||
|
|
@ -647,17 +648,34 @@ pub fn scenario_session(name: &str) -> Option<(Session, ItemId)> {
|
|||
let a = import(&mut s, &fixture("a1080.mp4")?);
|
||||
build_sequence(&mut s, 1920, 1080, &[(a, 100.0, None, 0.0, 100.0)], &[], 20.0)
|
||||
}
|
||||
"stack3" => {
|
||||
"stack3" | "stack3-blend" => {
|
||||
let (a, b, c) = (fixture("a1080.mp4")?, fixture("b1080.mp4")?, fixture("c1080.mp4")?);
|
||||
let (a, b, c) = (import(&mut s, &a), import(&mut s, &b), import(&mut s, &c));
|
||||
build_sequence(
|
||||
let seq = build_sequence(
|
||||
&mut s,
|
||||
1920,
|
||||
1080,
|
||||
&[(a, 100.0, None, 0.0, 100.0), (b, 60.0, Some((700.0, 420.0)), 0.0, 70.0), (c, 40.0, Some((1450.0, 760.0)), 12.0, 85.0)],
|
||||
&[],
|
||||
20.0,
|
||||
)
|
||||
);
|
||||
if name == "stack3-blend" {
|
||||
// V2 in Screen, V3 in Overlay
|
||||
s.edit("blend modes", |p, _| {
|
||||
let q = p.sequence_mut(seq).expect("seq");
|
||||
for (track, mode) in [(1, "Screen"), (2, "Overlay")] {
|
||||
let i = filmcraft_project::effect::BLEND_MODES.iter().position(|b| *b == mode).expect("blend mode") as u32;
|
||||
for it in &mut q.video_tracks[track].items {
|
||||
if let Some(b) = it.effect_mut("opacity").and_then(|o| o.params.get_mut("blend")) {
|
||||
b.value = ParamValue::Choice(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.ok()?;
|
||||
}
|
||||
seq
|
||||
}
|
||||
"h264-2160" | "seek-storm-2160" => {
|
||||
let a = import(&mut s, &fixture("a2160.mp4")?);
|
||||
|
|
|
|||
|
|
@ -160,6 +160,7 @@ pub fn playback(o: &Opts) -> Vec<Value> {
|
|||
let cases: &[(&str, &str, f32, bool)] = &[
|
||||
("h264-1080", "full", 1.0, false),
|
||||
("stack3", "full", 1.0, false),
|
||||
("stack3-blend", "full", 1.0, false),
|
||||
("h264-2160", "full", 1.0, false),
|
||||
("h264-2160", "half", 0.5, false),
|
||||
("h264-2160", "half draft", 0.5, true),
|
||||
|
|
|
|||
|
|
@ -883,12 +883,13 @@ fn plan_job(job: &Job, seq: ItemId, pool: &Arc<MediaPool>, services: &Arc<dyn Se
|
|||
// captions (they are not part of the preview hash), so captions are layered on live.
|
||||
let (w, h) = filmcraft_render::output_size(q, job.scale);
|
||||
let matrix = filmcraft_geom::Affine::scale(w as f64 / frame.width.max(1) as f64, h as f64 / frame.height.max(1) as f64);
|
||||
let mut layers = vec![filmcraft_render::plan::PlanLayer { frame, matrix, opacity: 1.0 }];
|
||||
let mut layers = vec![filmcraft_render::plan::PlanLayer { frame, matrix, opacity: 1.0, blend: filmcraft_render::Blend::Normal }];
|
||||
for o in filmcraft_render::caption_overlays(q, job.time, w, h) {
|
||||
layers.push(filmcraft_render::plan::PlanLayer {
|
||||
frame: Arc::new(filmcraft_frame::VideoFrame::rgba_f32(o.w as u32, o.h as u32, o.px)),
|
||||
matrix: filmcraft_geom::Affine::translate(o.x as f64, o.y as f64),
|
||||
opacity: 1.0,
|
||||
blend: filmcraft_render::Blend::Normal,
|
||||
});
|
||||
}
|
||||
return (filmcraft_render::plan::FramePlan::Layers { width: w, height: h, layers }, true);
|
||||
|
|
|
|||
|
|
@ -1314,7 +1314,7 @@ mod gpu_fallback_tests {
|
|||
#[test]
|
||||
fn plan_side_covers_output_and_every_layer() {
|
||||
let frame = |w: u32, h: u32| std::sync::Arc::new(filmcraft_frame::VideoFrame::rgba_f32(w, h, vec![0.0; (w * h * 4) as usize]));
|
||||
let layer = |w, h| PlanLayer { frame: frame(w, h), matrix: filmcraft_geom::Affine::IDENTITY, opacity: 1.0 };
|
||||
let layer = |w, h| PlanLayer { frame: frame(w, h), matrix: filmcraft_geom::Affine::IDENTITY, opacity: 1.0, blend: filmcraft_render::Blend::Normal };
|
||||
// a 4K source in an HD sequence still needs a 3840-wide texture on the GPU
|
||||
let p = FramePlan::Layers { width: 1920, height: 1080, layers: vec![layer(64, 64), layer(3840, 2160)] };
|
||||
assert_eq!(super::plan_side(&p), 3840);
|
||||
|
|
|
|||
|
|
@ -206,7 +206,7 @@ file ──► codecs (MP4/MOV, MKV, audio) demux + decode, GOP-aware see
|
|||
in linear-light premultiplied f32
|
||||
│
|
||||
└─ render::plan::plan_frame → gpu::GpuCompositor GPU path
|
||||
layers = decoded YUV/RGBA frames + matrix + opacity;
|
||||
layers = decoded YUV/RGBA frames + matrix + opacity + blend mode;
|
||||
anything the shaders don't cover is pre-rendered on the CPU
|
||||
▼
|
||||
ui-egui frames.rs worker pool ──► monitors (program/source), thumbnails, prefetch
|
||||
|
|
@ -229,9 +229,12 @@ file ──► codecs (MP4/MOV, MKV, audio) demux + decode, GOP-aware see
|
|||
frame of the same source. A request that finds the shared decoder busy decodes with a private
|
||||
decoder.
|
||||
- **Compositor.** `render` is the reference for monitors, thumbnails and export. `render::plan`
|
||||
turns a frame into GPU layers. Non-Normal blend modes, standard effects, adjustment layers, nested
|
||||
sequences and non-dissolve transitions are rendered on the CPU for that layer or frame and handed to
|
||||
the GPU as an image, so both paths give the same picture. Setting `FILMCRAFT_CPU_COMPOSITE=1`
|
||||
turns a frame into GPU layers, each with its opacity and blend mode. All 27 blend modes run on the
|
||||
GPU (`filmcraft-gpu`): Normal and Dissolve with fixed-function "over" blending, the others by
|
||||
copying the accumulator under the layer into a backdrop texture and compositing in the fragment
|
||||
shader with the CPU reference's formulas. Standard effects, adjustment layers, nested sequences
|
||||
and non-dissolve transitions are rendered on the CPU for that layer or frame and handed to the
|
||||
GPU as an image (a layer image keeps its clip's blend mode), so both paths give the same picture. Setting `FILMCRAFT_CPU_COMPOSITE=1`
|
||||
forces the CPU path in the desktop app.
|
||||
- **Frame scheduling.** `crates/ui-egui/src/frames.rs` runs a small pool of worker threads with
|
||||
prioritised jobs: the frame on screen first, then playback prefetch, then thumbnails. The UI never
|
||||
|
|
|
|||
|
|
@ -13,6 +13,27 @@ the before/after comparison relies on. Each milestone below alternated base / af
|
|||
each); the most load-independent figure is the decoder's own cycle count (`proc_pid_rusage`
|
||||
instructions and cycles of a process, all threads summed).
|
||||
|
||||
## Results (GPU1: blend modes on the GPU compositor, #30, before → after)
|
||||
|
||||
Before = this change with the old whole-frame CPU fallback for non-Normal blend modes put back
|
||||
in `render::plan` (same binary otherwise); after = GPU1. New `stack3-blend` scenario: `stack3`
|
||||
(three 1080p23.976 H.264 clips, V2/V3 scaled, positioned, rotated, 70–85 % opacity) with V2 in
|
||||
Screen and V3 in Overlay. `cargo xtask bench-playback --scenario stack3-blend,stack3 --res
|
||||
full,half`, GPU path, 8 s plays, two alternating rounds each on 2026-10-05 at load average
|
||||
33–63 (1-minute; 5-minute 67–81).
|
||||
|
||||
| case | shown/dropped before (2 runs) | after (2 runs) | CPU ms/frame before → after | worker svc p50 before → after |
|
||||
|---|---|---|---|---|
|
||||
| stack3-blend Full | 19/173, 138/54 | **191/1, 192/0** | 168, 214 → **86, 87** | 258, 219 → 1.2, 1.2 ms |
|
||||
| stack3-blend 1/2 | 192/0, 192/0 | 191/1, 192/0 | 158, 151 → **89, 88** | 66, 99 → 1.1, 1.0 ms |
|
||||
| stack3 Full (control, Normal) | 192/0, 192/0 | 191/1, 192/0 | 88, 88 → 91, 88 | 1.3, 1.6 → 0.9, 1.5 ms |
|
||||
| stack3 1/2 (control, Normal) | 191/1, 192/0 | 192/0, 192/0 | 82, 90 → 90, 89 | |
|
||||
|
||||
A stack with blend modes now costs what the same stack in Normal does: the frame workers only
|
||||
decode, and the UI-thread present (upload and draw, including the backdrop copies for the two
|
||||
blended layers) stays at 9–10 ms p50, as for `stack3`. Before, every frame was composited on the
|
||||
CPU (≈ 2 cores more at 23.976 fps) and Full resolution could not keep up at this load.
|
||||
|
||||
## Results (M4.10: 4K VP9, AV1 and HEVC, before → after)
|
||||
|
||||
Baseline = commit `a814ff6` (decoders unchanged since M4.8); after = M4.10. Interleaved runs on
|
||||
|
|
|
|||
|
|
@ -241,7 +241,7 @@ Options: `--sections decode,scrub`, `--only <substring>` (fixture or scenario),
|
|||
`--quick`, `--cpu`, `--label NAME` (output `target/bench/bench-<label>.{json,md}`),
|
||||
`--section NAME --json FILE` (one section in-process, e.g. under a profiler). Fixtures are made
|
||||
with ffmpeg in `target/fixtures/playback/` (or `$FILMCRAFT_FIXTURES/playback`). The suite's
|
||||
`playback` section plays `h264-1080` and `stack3` at Full, `h264-2160` at Full, 1/2 and 1/4 and with
|
||||
`playback` section plays `h264-1080`, `stack3` and `stack3-blend` at Full, `h264-2160` at Full, 1/2 and 1/4 and with
|
||||
draft decoding (Settings ▸ Playback ▸ Draft decoding) at 1/2 and 1/4, and `hevc-2160`, `vp9-2160`
|
||||
and `av1-2160` at Full, 1/2 and 1/2 draft; its `draft` column counts the frames decoded in draft
|
||||
mode.
|
||||
|
|
@ -268,6 +268,7 @@ cargo xtask bench-playback --json target/bench-playback.json # machine-readable
|
|||
|---|---|
|
||||
| `h264-1080` | one 1080p23.976 H.264 clip (testsrc2 + grain, ~45 Mbit/s, 250-frame GOP) |
|
||||
| `stack3` | three 1080p H.264 clips on V1–V3; V2/V3 scaled, positioned, rotated, 70–85 % opacity |
|
||||
| `stack3-blend` | `stack3` with V2 in Screen and V3 in Overlay (blend modes on the GPU) |
|
||||
| `h264-2160` | one 2160p23.976 H.264 clip |
|
||||
| `demo` | the built-in demo project (procedural footage, transitions, effects) |
|
||||
| `after-preview` | 1080p H.264 + Lumetri/Sharpen/Levels/Tint: a live play, Render Effects In to Out, then two plays of the green segment |
|
||||
|
|
|
|||
Loading…
Reference in New Issue