M13.13: fuse 8/16 bpc quantise into the layer composite; steady-state bench traffic

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

apps/effectcraft-cli/src/main.rs

@@ -836,6 +836,8 @@ fn bench_gpu(s: &Session, args: &Args) -> Result<(), Failure> {
                 std::hint::black_box(mk(Backend::Cpu, Some(&cache)));
             });
             let gcache = LayerCache::default();
+            // Traffic in steady state: after the frame that fills the layer cache.
+            std::hint::black_box(mk(Backend::Gpu, Some(&gcache)));
             let t0 = gpu.context().transfer_stats();
             let gpu_warm = time_ms(n, || {
                 std::hint::black_box(mk(Backend::Gpu, Some(&gcache)));

crates/gpu/src/context.rs

@@ -56,12 +56,15 @@ pub struct GpuImage {
     pub width: u32,
     pub height: u32,
     _lease: Option<Arc<Lease>>,
+    /// Already clamped and quantised to this many levels (8/16 bpc), so quantising again is a
+    /// no-op (see `ops::quantize`).
+    pub(crate) levels: Option<f32>,
 }
 
 impl GpuImage {
     /// An image on a texture of its own (not pooled).
     pub fn new(texture: wgpu::Texture, width: u32, height: u32) -> GpuImage {
-        GpuImage { texture, width, height, _lease: None }
+        GpuImage { texture, width, height, _lease: None, levels: None }
     }
 }
 
@@ -503,7 +506,7 @@ impl GpuContext {
         let was_reused = reused.is_some();
         let texture = reused.unwrap_or_else(|| self.texture(w, h));
         let lease = Arc::new(Lease { texture: texture.clone(), pool: Arc::downgrade(&self.pool) });
-        (GpuImage { texture, width: w, height: h, _lease: Some(lease) }, was_reused)
+        (GpuImage { texture, width: w, height: h, _lease: Some(lease), levels: None }, was_reused)
     }
 
     /// Uploads and readbacks so far (images; uploads of cached layers count once).

crates/gpu/src/ops.rs

@@ -44,6 +44,24 @@ pub fn warp(
     opacity: f32,
     seed: u32,
     accumulate: Option<f32>,
+) -> GpuImage {
+    warp_q(e, dst, src, m, sampling, mode, opacity, seed, accumulate, None)
+}
+
+/// [`warp`], clamping and quantising the result to `levels` (8/16 bpc: the comp after a layer,
+/// in the same pass).
+#[allow(clippy::too_many_arguments)]
+pub fn warp_q(
+    e: &mut Enc,
+    dst: &GpuImage,
+    src: &GpuImage,
+    m: &Mat3,
+    sampling: Sampling,
+    mode: BlendMode,
+    opacity: f32,
+    seed: u32,
+    accumulate: Option<f32>,
+    levels: Option<f32>,
 ) -> GpuImage {
     if src.width == 0 || src.height == 0 || opacity <= 0.0 {
         return dst.clone();
@@ -79,20 +97,23 @@ pub fn warp(
     p.f[0] = r[0];
     p.f[1] = r[1];
     p.f[2] = r[2];
-    p.f[3] = [opacity, accumulate.unwrap_or(1.0), 0.0, 0.0];
+    p.f[3] = [opacity, accumulate.unwrap_or(1.0), levels.unwrap_or(0.0), 0.0];
     p.f[4] = [x0 as f32, y0 as f32, x1 as f32, y1 as f32];
-    let out = e.scratch(dst.width, dst.height);
+    let mut out = e.scratch(dst.width, dst.height);
     e.pixels("warp_blend", &p, &img, Some(dst), &out, None);
+    out.levels = levels;
     out
 }
 
-/// `Image::blend_from`: blend `src` (same size) onto `dst`.
+/// `Image::blend_from`: blend `src` (same size) onto `dst`, clamping and quantising the result
-pub fn blend_full(e: &mut Enc, dst: &GpuImage, src: &GpuImage, mode: BlendMode, opacity: f32, seed: u32) -> GpuImage {
+/// to `levels` when set (see [`warp_q`]).
+pub fn blend_full(e: &mut Enc, dst: &GpuImage, src: &GpuImage, mode: BlendMode, opacity: f32, seed: u32, levels: Option<f32>) -> GpuImage {
     let mut p = Params::default();
     p.u[0] = [mode_id(mode), 0, seed, 0];
-    p.f[0] = [opacity, 0.0, 0.0, 0.0];
+    p.f[0] = [opacity, levels.unwrap_or(0.0), 0.0, 0.0];
-    let out = e.scratch(dst.width, dst.height);
+    let mut out = e.scratch(dst.width, dst.height);
     e.pixels("blend_full", &p, src, Some(dst), &out, None);
+    out.levels = levels;
     out
 }
 
@@ -147,10 +168,15 @@ pub fn adjust_mix(e: &mut Enc, canvas: &GpuImage, adjusted: &GpuImage, matte: &G
 
 /// Clamp and quantise to `levels` steps (8/16 bpc).
 pub fn quantize(e: &mut Enc, img: &GpuImage, levels: f32) -> GpuImage {
+    // Quantising is idempotent: an image a fused kernel already quantised stays as it is.
+    if img.levels == Some(levels) {
+        return img.clone();
+    }
     let mut p = Params::default();
     p.f[0][0] = levels;
-    let out = e.scratch(img.width, img.height);
+    let mut out = e.scratch(img.width, img.height);
     e.pixels("quantize", &p, img, None, &out, None);
+    out.levels = Some(levels);
     out
 }
 

crates/gpu/src/shaders/kernels.wgsl

@@ -33,7 +33,7 @@ fn warp_blend(@builtin(global_invocation_id) gid: vec3<u32>) {
         if ((flags & 2u) != 0u) {
             d = vec4<f32>(0.0);
         }
-        textureStore(out, vec2<i32>(x, y), d);
+        store_q(vec2<i32>(x, y), d, P.f[3].z);
         return;
     }
     let r0 = P.f[0];
@@ -52,7 +52,7 @@ fn warp_blend(@builtin(global_invocation_id) gid: vec3<u32>) {
         if (stencil && (flags & 1u) == 0u) {
             d = blend_pixel(mode, d, vec4<f32>(0.0), 0.5);
         }
-        textureStore(out, vec2<i32>(x, y), d);
+        store_q(vec2<i32>(x, y), d, P.f[3].z);
         return;
     }
     var s = sample(src, sampling, px, py);
@@ -60,11 +60,11 @@ fn warp_blend(@builtin(global_invocation_id) gid: vec3<u32>) {
         if (s.w > 0.0) {
             d = d + s * P.f[3].y;
         }
-        textureStore(out, vec2<i32>(x, y), d);
+        store_q(vec2<i32>(x, y), d, P.f[3].z);
         return;
     }
     if (s.w <= 0.0 && !stencil) {
-        textureStore(out, vec2<i32>(x, y), d);
+        store_q(vec2<i32>(x, y), d, P.f[3].z);
         return;
     }
     let op = P.f[3].x;
@@ -75,7 +75,7 @@ fn warp_blend(@builtin(global_invocation_id) gid: vec3<u32>) {
     if (mode == 1u || mode == 2u) {
         n = hash_noise(u32(x), u32(y), seed);
     }
-    textureStore(out, vec2<i32>(x, y), blend_pixel(mode, d, s, n));
+    store_q(vec2<i32>(x, y), blend_pixel(mode, d, s, n), P.f[3].z);
 }
 
 // Blend `src` (same size) onto `aux` (Image::blend_from).
@@ -91,7 +91,7 @@ fn blend_full(@builtin(global_invocation_id) gid: vec3<u32>) {
     let d = textureLoad(aux, p, 0);
     var s = textureLoad(src, p, 0);
     if (s.w <= 0.0 && !is_stencil(mode)) {
-        textureStore(out, p, d);
+        store_q(p, d, P.f[0].y);
         return;
     }
     s = s * P.f[0].x;
@@ -99,7 +99,7 @@ fn blend_full(@builtin(global_invocation_id) gid: vec3<u32>) {
     if (mode == 1u || mode == 2u) {
         n = hash_noise(gid.x, gid.y, P.u[0].z);
     }
-    textureStore(out, p, blend_pixel(mode, d, s, n));
+    store_q(p, blend_pixel(mode, d, s, n), P.f[0].y);
 }
 
 // Track matte: `src` × matte factor of `aux`. u[0].x = 0 alpha, 1 alpha inverted, 2 luma,
@@ -179,15 +179,27 @@ fn quantize(@builtin(global_invocation_id) gid: vec3<u32>) {
     if (p.x >= dims.x || p.y >= dims.y) {
         return;
     }
-    let levels = P.f[0].x;
+    textureStore(out, p, quantize_px(textureLoad(src, p, 0), P.f[0].x));
+}
+
+fn quantize_px(s: vec4<f32>, levels: f32) -> vec4<f32> {
     let inv = 1.0 / levels;
-    let s = textureLoad(src, p, 0);
     let a = round_away(clamp(s.w, 0.0, 1.0) * levels) * inv;
     var o = vec4<f32>(0.0, 0.0, 0.0, a);
     o.x = round_away(clamp(s.x, 0.0, a) * levels) * inv;
     o.y = round_away(clamp(s.y, 0.0, a) * levels) * inv;
     o.z = round_away(clamp(s.z, 0.0, a) * levels) * inv;
-    textureStore(out, p, o);
+    return o;
+}
+
+// Store a compositing result, clamped and quantised to `levels` when > 0 (8/16 bpc: the comp
+// after a layer, fused into the layer's composite).
+fn store_q(p: vec2<i32>, v: vec4<f32>, levels: f32) {
+    if (levels > 0.0) {
+        textureStore(out, p, quantize_px(v, levels));
+    } else {
+        textureStore(out, p, v);
+    }
 }
 
 // Colour conversion on straight colour (render::color::convert).

crates/gpu/src/walk.rs

@@ -158,20 +158,20 @@ fn draw_layer(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, canvas: &
     if r.track_matte(ctx, layer).is_some() || layer.preserve_transparency {
         let mut iso = e.zeros(w, h);
         for (b, m) in &st.passes {
-            iso = place(e, r, ctx, layer, b, &iso, *m, 1.0)?;
+            iso = place(e, r, ctx, layer, b, &iso, *m, 1.0, None)?;
         }
-        iso = place(e, r, ctx, layer, &st.body, &iso, BlendMode::Normal, 1.0)?;
+        iso = place(e, r, ctx, layer, &st.body, &iso, BlendMode::Normal, 1.0, None)?;
         return composite_iso(e, r, ctx, layer, iso, canvas, opacity);
     }
     let mut tmp = canvas.clone();
     if bl.knockout > 0 {
-        let k = place(e, r, ctx, layer, &st.content, &e.zeros(w, h), BlendMode::Normal, 1.0)?;
+        let k = place(e, r, ctx, layer, &st.content, &e.zeros(w, h), BlendMode::Normal, 1.0, None)?;
         tmp = ops::knockout(e, &tmp, &k);
     }
     for (b, m) in &st.passes {
-        tmp = place(e, r, ctx, layer, b, &tmp, *m, 1.0)?;
+        tmp = place(e, r, ctx, layer, b, &tmp, *m, 1.0, None)?;
     }
-    tmp = place(e, r, ctx, layer, &st.body, &tmp, layer.blend_mode, 1.0)?;
+    tmp = place(e, r, ctx, layer, &st.body, &tmp, layer.blend_mode, 1.0, None)?;
     *canvas = ops::channel_mix(e, canvas, &tmp, bl.channels, opacity);
     Some(())
 }
@@ -183,7 +183,7 @@ fn draw_adjustment(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, canv
     let opacity = r.layer_opacity(ctx, layer);
     let Some(foot) = r.adjustment_footprint(ctx, layer) else { return Some(()) };
     let (w, h) = (canvas.width, canvas.height);
-    let matte = place(e, r, ctx, layer, &foot, &e.zeros(w, h), BlendMode::Normal, 1.0)?;
+    let matte = place(e, r, ctx, layer, &foot, &e.zeros(w, h), BlendMode::Normal, 1.0, None)?;
     let pipe = r.pipe();
     // Effects run in the working space, not the linear blending space.
     let below = match pipe.from_blend() {
@@ -206,11 +206,12 @@ fn draw_adjustment(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, canv
 fn composite_layer(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, buf: &Arc<Buf>, canvas: &mut GpuImage, opacity: f32) -> Option<()> {
     let matte = r.track_matte(ctx, layer).is_some();
     if !matte && !layer.preserve_transparency {
-        *canvas = place(e, r, ctx, layer, buf, canvas, layer.blend_mode, opacity)?;
+        // The comp is quantised after the layer (8/16 bpc) in the same pass.
+        *canvas = place(e, r, ctx, layer, buf, canvas, layer.blend_mode, opacity, r.pipe().levels)?;
         return Some(());
     }
     // Render in isolation, then matte / preserve transparency, then blend.
-    let iso = place(e, r, ctx, layer, buf, &e.zeros(canvas.width, canvas.height), BlendMode::Normal, 1.0)?;
+    let iso = place(e, r, ctx, layer, buf, &e.zeros(canvas.width, canvas.height), BlendMode::Normal, 1.0, None)?;
     composite_iso(e, r, ctx, layer, iso, canvas, opacity)
 }
 
@@ -222,31 +223,43 @@ fn composite_iso(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, mut is
             && let Some(mb) = r.layer_buf(ctx, m)
         {
             let mo = ctx.opacity(m) as f32;
-            mimg = place(e, r, ctx, m, &mb, &mimg, BlendMode::Normal, mo)?;
+            mimg = place(e, r, ctx, m, &mb, &mimg, BlendMode::Normal, mo, None)?;
         }
         iso = ops::matte(e, &iso, &mimg, kind);
     }
     if layer.preserve_transparency {
         iso = ops::preserve(e, &iso, canvas);
     }
-    *canvas = ops::blend_full(e, canvas, &iso, layer.blend_mode, opacity, layer.id.0 as u32);
+    *canvas = ops::blend_full(e, canvas, &iso, layer.blend_mode, opacity, layer.id.0 as u32, r.pipe().levels);
     Some(())
 }
 
-/// Draw a processed layer buffer onto `target` (transform, motion blur sub-samples).
+/// Draw a processed layer buffer onto `target` (transform, motion blur sub-samples), quantising
-fn place(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, buf: &Arc<Buf>, target: &GpuImage, mode: BlendMode, opacity: f32) -> Option<GpuImage> {
+/// the result to `levels` when set.
+#[allow(clippy::too_many_arguments)]
+fn place(
+    e: &mut Enc,
+    r: &Renderer,
+    ctx: &EvalCtx,
+    layer: &Layer,
+    buf: &Arc<Buf>,
+    target: &GpuImage,
+    mode: BlendMode,
+    opacity: f32,
+    levels: Option<f32>,
+) -> Option<GpuImage> {
     if buf.img.is_empty() || opacity <= 0.0 {
         return Some(target.clone());
     }
     let src = e.g.upload_buf(buf)?;
     let pl = r.placement(ctx, layer, buf);
     if let [m] = pl.matrices.as_slice() {
-        return Some(ops::warp(e, target, &src, m, pl.sampling, mode, opacity, pl.seed, None));
+        return Some(ops::warp_q(e, target, &src, m, pl.sampling, mode, opacity, pl.seed, None, levels));
     }
     let mut acc = e.zeros(target.width, target.height);
     let k = 1.0 / pl.matrices.len() as f32;
     for m in &pl.matrices {
         acc = ops::warp(e, &acc, &src, m, pl.sampling, BlendMode::Normal, 1.0, 0, Some(k));
     }
-    Some(ops::blend_full(e, target, &acc, mode, opacity, pl.seed))
+    Some(ops::blend_full(e, target, &acc, mode, opacity, pl.seed, levels))
 }