From 451f63cc0d706232be34db43f43b00c9695d79c7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 21:08:04 -0400 Subject: [PATCH] M9.11: Blur & Sharpen and Distort effects (CC Radial Blur, Camera-Shake Deblur, CC Flo Motion, Liquify, Rolling Shutter Repair) Liquify stores its strokes as effect data and replays them into a cached displacement mesh; liquify.stroke / liquify.clear commands for agents. Rolling Shutter Repair measures motion with effectcraft-track KLT (new effects -> track same-layer edge). Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + crates/effects/Cargo.toml | 1 + crates/effects/src/blur3.rs | 338 ++++++++++++ crates/effects/src/distort4.rs | 719 ++++++++++++++++++++++++++ crates/effects/src/lib.rs | 6 + crates/engine/src/commands/liquify.rs | 156 ++++++ crates/engine/src/commands/mod.rs | 2 + crates/engine/src/tests_paint.rs | 23 + docs/architecture.md | 2 +- xtask/src/main.rs | 1 + 10 files changed, 1248 insertions(+), 1 deletion(-) create mode 100644 crates/effects/src/blur3.rs create mode 100644 crates/effects/src/distort4.rs create mode 100644 crates/engine/src/commands/liquify.rs diff --git a/Cargo.lock b/Cargo.lock index 635fb0f5..46f34e63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1182,6 +1182,7 @@ dependencies = [ "effectcraft-keyframe", "effectcraft-project", "effectcraft-raster", + "effectcraft-track", "rayon", "serde_json", ] diff --git a/crates/effects/Cargo.toml b/crates/effects/Cargo.toml index db74bf27..e321fa2f 100644 --- a/crates/effects/Cargo.toml +++ b/crates/effects/Cargo.toml @@ -15,6 +15,7 @@ effectcraft-color = { workspace = true } effectcraft-keyframe = { workspace = true } effectcraft-project = { workspace = true } effectcraft-raster = { workspace = true } +effectcraft-track = { workspace = true } rayon = { workspace = true } [dev-dependencies] diff --git a/crates/effects/src/blur3.rs b/crates/effects/src/blur3.rs new file mode 100644 index 00000000..0140ac99 --- /dev/null +++ b/crates/effects/src/blur3.rs @@ -0,0 +1,338 @@ +//! Blur & Sharpen: CC Radial Blur and Camera-Shake Deblur. +//! +//! * **CC Radial Blur** — zoom (straight, fading, centred both ways) and rotational (plain or +//! fading) blurs around a centre point, sampled along the ray / arc through each pixel with a +//! sample count set by Quality. +//! * **Camera-Shake Deblur** — frame-based restoration in the spirit of published video deblurring +//! by "lucky frames": the frame is split into patches; for each patch the sharpness (mean +//! squared gradient) is compared with the same patch in the neighbouring frames within Blur +//! Duration; when a neighbour is clearly sharper, its patch is aligned to the current frame by +//! a small translation search (sum of squared differences on luma) and substituted with +//! feathered patch blending. Deterministic; neighbours come from [`crate::EffectHost::self_at`]. + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::{Image, Px}; +use rayon::prelude::*; + +use crate::util::Plane; +use crate::{Buf, EffectCtx, EffectSpec, num, p, popup, slider}; + +fn spec(id: &'static str, name: &'static str, params: Vec, render: crate::RenderFn) -> EffectSpec { + EffectSpec { id, name, category: "Blur & Sharpen", params, render, gpu: false, float: true } +} + +// ---------------------------------------------------------------- CC Radial Blur + +pub const RADIAL_TYPES: [&str; 5] = ["Straight Zoom", "Fading Zoom", "Centered", "Rotate", "Rotate Fading"]; + +fn cc_radial_blur(ctx: &EffectCtx, mut b: Buf) -> Buf { + let amount = ctx.params.f("amount"); + if amount == 0.0 { + return b; + } + let kind = ctx.params.e("type"); + let quality = ctx.params.f("quality").clamp(1.0, 100.0); + let c = b.to_px(ctx.params.v2("center")); + let src = b.img.clone(); + // Zoom: fraction of the distance to the centre; rotate: degrees of arc. + let zoom = (amount / 100.0).clamp(-1.0, 1.0) * 0.5; + let arc = amount.to_radians() * 0.25; + b.img.rows_mut().for_each(|(y, row)| { + for (x, o) in row.iter_mut().enumerate() { + let (dx, dy) = (x as f64 + 0.5 - c.0, y as f64 + 0.5 - c.1); + let r = (dx * dx + dy * dy).sqrt(); + let span = if kind >= 3 { r * arc.abs() } else { r * zoom.abs() }; + let n = ((span * quality / 50.0).ceil() as usize).clamp(1, 256); + if n <= 1 { + *o = src.get(x as i64, y as i64); + continue; + } + let mut acc = [0.0f32; 4]; + let mut wsum = 0.0f32; + for i in 0..n { + let t = i as f64 / (n - 1) as f64; + let (sx, sy, w) = match kind { + // Toward the centre, equal weights. + 0 => { + let k = 1.0 - zoom * t; + (c.0 + dx * k, c.1 + dy * k, 1.0) + } + // Toward the centre, weights fading with distance along the streak. + 1 => { + let k = 1.0 - zoom * t; + (c.0 + dx * k, c.1 + dy * k, (1.0 - t) as f32 + 0.05) + } + // Both ways around the pixel. + 2 => { + let k = 1.0 + zoom * (t - 0.5); + (c.0 + dx * k, c.1 + dy * k, 1.0) + } + _ => { + let a = arc * (t - 0.5); + let (s, co) = a.sin_cos(); + let w = if kind == 4 { (1.0 - (2.0 * t - 1.0).abs()) as f32 + 0.05 } else { 1.0 }; + (c.0 + dx * co - dy * s, c.1 + dx * s + dy * co, w) + } + }; + let q = src.sample_bilinear(sx, sy); + for k in 0..4 { + acc[k] += q[k] * w; + } + wsum += w; + } + *o = acc.map(|v| v / wsum); + } + }); + b +} + +// ---------------------------------------------------------------- Camera-Shake Deblur + +/// Mean squared luma gradient over a rectangle (sharpness). +fn sharpness(l: &Plane, x0: usize, y0: usize, x1: usize, y1: usize) -> f64 { + let mut s = 0.0f64; + let mut n = 0usize; + for y in y0..y1.saturating_sub(1) { + for x in x0..x1.saturating_sub(1) { + let v = l.get(x, y); + let gx = l.get(x + 1, y) - v; + let gy = l.get(x, y + 1) - v; + s += (gx * gx + gy * gy) as f64; + n += 1; + } + } + if n == 0 { 0.0 } else { s / n as f64 } +} + +/// Best integer translation (dx, dy) within ±`r` aligning `b` to `a` on a patch (SSD). +fn align(a: &Plane, bp: &Plane, x0: usize, y0: usize, x1: usize, y1: usize, r: i64) -> (i64, i64) { + let mut best = (f64::INFINITY, 0i64, 0i64); + let step = ((x1 - x0).max(y1 - y0) / 24).max(1); + for dy in -r..=r { + for dx in -r..=r { + let mut s = 0.0f64; + for y in (y0..y1).step_by(step) { + for x in (x0..x1).step_by(step) { + let d = a.get(x, y) - bp.get_clamped(x as i64 + dx, y as i64 + dy); + s += (d * d) as f64; + } + } + // Prefer the smallest shift on ties. + let s = s * (1.0 + 1e-9 * (dx * dx + dy * dy) as f64); + if s < best.0 { + best = (s, dx, dy); + } + } + } + (best.1, best.2) +} + +fn camera_shake_deblur(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(host) = ctx.env.host else { return b }; + let dur = ctx.params.f("blurDuration").clamp(1.0, 10.0).round() as i64; + let search = (ctx.params.f("searchRange").max(0.0) * b.scale).round() as i64; + let patch = (ctx.params.f("patchSize").max(8.0) * b.scale).round().max(4.0) as usize; + let thresh = 1.0 + ctx.params.f("threshold").max(0.0) / 100.0; + let show = ctx.params.b("showBlurryFrames"); + let fps = ctx.fps(); + let (w, h) = (b.img.width as usize, b.img.height as usize); + if w == 0 || h == 0 { + return b; + } + // Neighbouring frames in the effect's own input, resampled into this buffer's grid. + let mut neigh: Vec<(Image, Plane)> = vec![]; + for k in (-dur..=dur).filter(|k| *k != 0) { + let Some(nb) = host.self_at(ctx.time + k as f64 / fps, ctx.env.effect_index) else { continue }; + let img = if nb.img.width == b.img.width && nb.img.height == b.img.height && nb.offset == b.offset && nb.scale == b.scale { + nb.img + } else { + let k = nb.scale / b.scale.max(1e-9); + crate::util::gen_image(b.img.width, b.img.height, |x, y| { + let lx = (x as f64 + 0.5 - b.offset[0]) * k + nb.offset[0]; + let ly = (y as f64 + 0.5 - b.offset[1]) * k + nb.offset[1]; + nb.img.sample_bilinear(lx, ly) + }) + }; + let l = Plane::luma(&img); + neigh.push((img, l)); + } + if neigh.is_empty() { + return b; + } + let cur_l = Plane::luma(&b.img); + let (nx, ny) = (w.div_ceil(patch), h.div_ceil(patch)); + // Per patch: (neighbour index, shift) of the substitute, or None to keep. + let choice: Vec> = (0..nx * ny) + .into_par_iter() + .map(|i| { + let (px, py) = (i % nx, i / nx); + let (x0, y0) = (px * patch, py * patch); + let (x1, y1) = ((x0 + patch).min(w), (y0 + patch).min(h)); + let s0 = sharpness(&cur_l, x0, y0, x1, y1); + let mut best: Option<(f64, usize)> = None; + for (k, (_, l)) in neigh.iter().enumerate() { + let s = sharpness(l, x0, y0, x1, y1); + if s > s0 * thresh && best.is_none_or(|(bs, _)| s > bs) { + best = Some((s, k)); + } + } + let (_, k) = best?; + let (dx, dy) = align(&cur_l, &neigh[k].1, x0, y0, x1, y1, search); + Some((k, dx, dy)) + }) + .collect(); + if choice.iter().all(Option::is_none) { + return b; + } + let src = b.img.clone(); + // Feathered blending between patch decisions: each pixel mixes the four nearest patch + // centres bilinearly. + let pick = |i: usize, x: usize, y: usize| -> Px { + match choice[i] { + None => src.data[y * w + x], + Some((k, dx, dy)) => neigh[k].0.get_clamped(x as i64 + dx, y as i64 + dy), + } + }; + b.img.rows_mut().for_each(|(y, row)| { + let fy = ((y as f64 + 0.5) / patch as f64 - 0.5).clamp(0.0, (ny - 1) as f64); + let (y0, ty) = (fy.floor() as usize, (fy - fy.floor()) as f32); + let y1 = (y0 + 1).min(ny - 1); + for (x, o) in row.iter_mut().enumerate() { + let fx = ((x as f64 + 0.5) / patch as f64 - 0.5).clamp(0.0, (nx - 1) as f64); + let (x0, tx) = (fx.floor() as usize, (fx - fx.floor()) as f32); + let x1 = (x0 + 1).min(nx - 1); + let ids = [y0 * nx + x0, y0 * nx + x1, y1 * nx + x0, y1 * nx + x1]; + if ids.iter().all(|&i| choice[i].is_none()) { + continue; + } + let ws = [(1.0 - tx) * (1.0 - ty), tx * (1.0 - ty), (1.0 - tx) * ty, tx * ty]; + let mut acc = [0.0f32; 4]; + for (i, wgt) in ids.iter().zip(ws) { + let q = pick(*i, x, y); + for c in 0..4 { + acc[c] += q[c] * wgt; + } + } + if show { + acc[0] = (acc[0] + 0.3 * acc[3]).min(acc[3].max(acc[0])); + } + *o = acc; + } + }); + b +} + +pub fn specs() -> Vec { + vec![ + spec( + "ec.blur.ccradial", + "CC Radial Blur", + vec![ + p("type", "Type", Value::Enum(1), popup(&RADIAL_TYPES)), + p("amount", "Amount", num(10.0), slider(-100.0, 100.0, -100.0, 100.0, 1)), + p("quality", "Quality", num(50.0), slider(1.0, 100.0, 1.0, 100.0, 1)), + p("center", "Center", Value::Vec2([0.5, 0.5]), ParamUi::Point), + ], + cc_radial_blur, + ), + spec( + "ec.blur.camerashakedeblur", + "Camera-Shake Deblur", + vec![ + p("blurDuration", "Blur Duration", num(2.0), slider(1.0, 10.0, 1.0, 10.0, 0)), + p("searchRange", "Search Range", num(8.0), slider(0.0, 64.0, 0.0, 32.0, 0)), + p("patchSize", "Patch Size", num(32.0), slider(8.0, 256.0, 8.0, 128.0, 0)), + p("threshold", "Threshold", num(20.0), slider(0.0, 500.0, 0.0, 100.0, 1)), + p("showBlurryFrames", "Show Blurry Frames", Value::Bool(false), ParamUi::Checkbox), + ], + camera_shake_deblur, + ), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, EffectHost, LayerPixels, run_fx}; + + fn checker(w: u32, h: u32, cell: u32) -> Image { + crate::util::gen_image(w, h, |x, y| { + let v = if ((x as u32 / cell) + (y as u32 / cell)).is_multiple_of(2) { 1.0 } else { 0.0 }; + [v, v, v, 1.0] + }) + } + + #[test] + fn cc_radial_blur_keeps_centre_and_smears_edges() { + let img = checker(64, 64, 4); + for kind in 0..5u32 { + let out = run_fx( + "ec.blur.ccradial", + &[("type", Value::Enum(kind)), ("amount", num(40.0)), ("center", Value::Vec2([32.0, 32.0]))], + img.clone(), + 0.0, + EffectEnv::default(), + ); + // Far from the centre the checker is smeared toward grey. + let edge = out.img.get(2, 2)[0]; + assert!(edge > 0.05 && edge < 0.95, "type {kind}: {edge}"); + let again = run_fx( + "ec.blur.ccradial", + &[("type", Value::Enum(kind)), ("amount", num(40.0)), ("center", Value::Vec2([32.0, 32.0]))], + img.clone(), + 0.0, + EffectEnv::default(), + ); + assert_eq!(out.img.data, again.img.data); + } + // Zero amount is identity; a constant image stays constant. + let flat = Image::filled(16, 16, [0.4, 0.4, 0.4, 1.0]); + let o = run_fx("ec.blur.ccradial", &[("amount", num(80.0))], flat.clone(), 0.0, EffectEnv::default()); + assert!(o.img.data.iter().all(|p| (p[0] - 0.4).abs() < 1e-5)); + let o = run_fx("ec.blur.ccradial", &[("amount", num(0.0))], img.clone(), 0.0, EffectEnv::default()); + assert_eq!(o.img.data, img.data); + } + + /// Frames: sharp checker at even frames, a blurred one at odd frames (shifted by 2 px). + struct Shaky; + impl EffectHost for Shaky { + fn layer(&self, _: u64, _: bool) -> Option { + None + } + fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option> { + None + } + fn self_at(&self, t: f64, _: usize) -> Option { + let f = (t * 30.0).round() as i64; + Some(Buf { img: frame(f), offset: [0.0; 2], scale: 1.0 }) + } + } + + fn frame(f: i64) -> Image { + let sharp = crate::util::gen_image(48, 48, |x, y| { + let v = if (((x + 2) / 6) + (y / 6)) % 2 == 0 { 1.0 } else { 0.0 }; + [v, v, v, 1.0] + }); + if f % 2 == 0 { sharp } else { effectcraft_raster::gaussian_blur(&sharp, 2.5, 2.5, true) } + } + + #[test] + fn camera_shake_deblur_substitutes_sharp_neighbours() { + let host = Shaky; + let env = EffectEnv { host: Some(&host), frame_rate: 30.0, ..Default::default() }; + let blurry = frame(1); + let out = run_fx("ec.blur.camerashakedeblur", &[("patchSize", num(16.0)), ("searchRange", num(3.0))], blurry.clone(), 1.0 / 30.0, env); + let sharp = frame(0); + let err = |a: &Image| a.data.iter().zip(&sharp.data).map(|(p, q)| (p[0] - q[0]).abs()).sum::() / a.data.len() as f32; + assert!(err(&out.img) < err(&blurry) * 0.5, "{} vs {}", err(&out.img), err(&blurry)); + // A sharp frame is left alone (its neighbours are blurrier). + let keep = run_fx("ec.blur.camerashakedeblur", &[], sharp.clone(), 0.0, env); + assert_eq!(keep.img.data, sharp.data); + // Deterministic; no host = pass-through. + let again = run_fx("ec.blur.camerashakedeblur", &[("patchSize", num(16.0)), ("searchRange", num(3.0))], blurry.clone(), 1.0 / 30.0, env); + assert_eq!(out.img.data, again.img.data); + let none = run_fx("ec.blur.camerashakedeblur", &[], blurry.clone(), 0.0, EffectEnv::default()); + assert_eq!(none.img.data, blurry.data); + } +} diff --git a/crates/effects/src/distort4.rs b/crates/effects/src/distort4.rs new file mode 100644 index 00000000..4e6b5040 --- /dev/null +++ b/crates/effects/src/distort4.rs @@ -0,0 +1,719 @@ +//! Distort: CC Flo Motion, Liquify and Rolling Shutter Repair. +//! +//! * **CC Flo Motion** — two knots that pull the image toward themselves (positive amount) or +//! push it away (negative), with a falloff exponent, optional tiling and supersampled +//! antialiasing. +//! * **Liquify** — brush strokes build a displacement mesh (a grid of layer-space offsets). The +//! strokes are the effect's data (the hidden `distortionMesh` text, one stroke per line, so it +//! animates with hold keyframes and is written by the `liquify.stroke` command); replaying +//! them is deterministic and cached. Tools: warp, turbulence, twirl clockwise / counter- +//! clockwise, pucker, bloat, shift pixels, reflection, clone, reconstruction and freeze / thaw. +//! Distortion Percentage scales the mesh and Distortion Mesh Offset moves it. +//! * **Rolling Shutter Repair** — every scanline of a CMOS frame is exposed a little later than +//! the previous one, so moving content shears. Motion is measured between the neighbouring +//! frames with pyramidal Lucas–Kanade (`effectcraft-track`) on a grid of points; each pixel is +//! then resampled from where it was at the frame's mid-exposure time: +//! `out(p) = in(p + v(p)·rate·(s(p) − ½))`, with `s` the scan position (0…1 along the scan +//! direction) and `v` the per-frame velocity (Warp: one smooth velocity per scanline band; +//! Pixel Motion: a dense interpolated field). + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::Image; +use rayon::prelude::*; + +use crate::util::{hash1, point_in_poly}; +use crate::{Buf, EffectCtx, EffectSpec, col, num, p, popup, slider}; + +fn spec(id: &'static str, name: &'static str, params: Vec, render: crate::RenderFn) -> EffectSpec { + EffectSpec { id, name, category: "Distort", params, render, gpu: false, float: true } +} + +// ---------------------------------------------------------------- CC Flo Motion + +fn flo_motion(ctx: &EffectCtx, mut b: Buf) -> Buf { + let knots = [(ctx.params.v2("knot1"), ctx.params.f("amount1") / 100.0), (ctx.params.v2("knot2"), ctx.params.f("amount2") / 100.0)]; + if knots.iter().all(|k| k.1 == 0.0) { + return b; + } + let tile = ctx.params.b("tileEdges"); + let ss = match ctx.params.e("antialiasing") { + 0 => 1, + 1 => 2, + _ => 3, + }; + let falloff = ctx.params.f("falloff").max(0.01); + let (lw, lh) = (ctx.layer_size[0].max(1.0), ctx.layer_size[1].max(1.0)); + let radius = (lw * lw + lh * lh).sqrt() * 0.5; + let src = b.img.clone(); + let (scale, off) = (b.scale, b.offset); + let map = |lx: f64, ly: f64| -> (f64, f64) { + let (mut x, mut y) = (lx, ly); + for (k, a) in knots { + if a == 0.0 { + continue; + } + let (dx, dy) = (x - k[0], y - k[1]); + let r = (dx * dx + dy * dy).sqrt() / radius; + // Weight 1 at the knot, falling off with distance. + let w = (1.0 - r.min(1.0)).powf(falloff); + // Pull: sample closer to the knot (content flows out of it); push: farther. + let s = (1.0 - a.clamp(-4.0, 0.95) * w).max(0.02); + x = k[0] + dx * s; + y = k[1] + dy * s; + } + (x, y) + }; + b.img.rows_mut().for_each(|(y, row)| { + for (x, o) in row.iter_mut().enumerate() { + let mut acc = [0.0f32; 4]; + for sy in 0..ss { + for sx in 0..ss { + let px = x as f64 + (sx as f64 + 0.5) / ss as f64; + let py = y as f64 + (sy as f64 + 0.5) / ss as f64; + let (lx, ly) = map((px - off[0]) / scale, (py - off[1]) / scale); + let (lx, ly) = if tile { (lx.rem_euclid(lw), ly.rem_euclid(lh)) } else { (lx, ly) }; + let q = src.sample_bilinear(lx * scale + off[0], ly * scale + off[1]); + for c in 0..4 { + acc[c] += q[c]; + } + } + } + let n = (ss * ss) as f32; + *o = acc.map(|v| v / n); + } + }); + b +} + +// ---------------------------------------------------------------- Liquify + +/// Liquify tools (the stroke text uses these names). +pub const LIQUIFY_TOOLS: [&str; 12] = [ + "warp", + "turbulence", + "twirlClockwise", + "twirlCounterclockwise", + "pucker", + "bloat", + "shiftPixels", + "reflection", + "clone", + "reconstruction", + "freeze", + "thaw", +]; + +/// One recorded Liquify stroke. +#[derive(Clone, Debug, PartialEq)] +pub struct LiquifyStroke { + pub tool: usize, + /// Brush size (diameter) and pressure (0–100), turbulent jitter (0–100), layer pixels. + pub size: f64, + pub pressure: f64, + pub jitter: f64, + /// Clone offset (source − destination). + pub clone_offset: [f64; 2], + pub points: Vec<[f64; 2]>, +} + +impl LiquifyStroke { + /// Text form: `tool size pressure jitter cloneX cloneY x,y x,y …`. + pub fn to_line(&self) -> String { + let mut s = format!("{} {} {} {} {} {}", LIQUIFY_TOOLS[self.tool], self.size, self.pressure, self.jitter, self.clone_offset[0], self.clone_offset[1]); + for p in &self.points { + s.push_str(&format!(" {},{}", p[0], p[1])); + } + s + } + pub fn parse(line: &str) -> Option { + let mut it = line.split_whitespace(); + let name = it.next()?; + let tool = LIQUIFY_TOOLS.iter().position(|t| t.eq_ignore_ascii_case(name))?; + let mut f = || -> Option { it.next()?.parse().ok() }; + let (size, pressure, jitter, cx, cy) = (f()?, f()?, f()?, f()?, f()?); + let points: Vec<[f64; 2]> = it + .filter_map(|t| { + let (a, b) = t.split_once(',')?; + Some([a.parse().ok()?, b.parse().ok()?]) + }) + .collect(); + (!points.is_empty()).then_some(LiquifyStroke { tool, size, pressure, jitter, clone_offset: [cx, cy], points }) + } +} + +/// Parse the `distortionMesh` text into strokes (bad lines are skipped). +pub fn parse_strokes(s: &str) -> Vec { + s.lines().filter_map(LiquifyStroke::parse).collect() +} + +/// Displacement mesh: offsets (layer pixels) at grid nodes spaced `cell` apart. +#[derive(Clone, Debug, PartialEq)] +pub struct Mesh { + pub nx: usize, + pub ny: usize, + pub cell: f64, + pub d: Vec<[f64; 2]>, + pub freeze: Vec, +} + +impl Mesh { + fn new(w: f64, h: f64, cell: f64) -> Mesh { + let nx = (w / cell).ceil() as usize + 1; + let ny = (h / cell).ceil() as usize + 1; + Mesh { nx, ny, cell, d: vec![[0.0; 2]; nx * ny], freeze: vec![0.0; nx * ny] } + } + /// Bilinear displacement at a layer point (zero outside the mesh). + pub fn at(&self, x: f64, y: f64) -> [f64; 2] { + let fx = x / self.cell; + let fy = y / self.cell; + if fx < 0.0 || fy < 0.0 || fx > (self.nx - 1) as f64 || fy > (self.ny - 1) as f64 { + return [0.0; 2]; + } + let (x0, y0) = ((fx.floor() as usize).min(self.nx - 2), (fy.floor() as usize).min(self.ny - 2)); + let (x1, y1) = ((x0 + 1).min(self.nx - 1), (y0 + 1).min(self.ny - 1)); + let (tx, ty) = (fx - x0 as f64, fy - y0 as f64); + let g = |i: usize, j: usize| self.d[j * self.nx + i]; + let l = |a: [f64; 2], b: [f64; 2], t: f64| [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t]; + l(l(g(x0, y0), g(x1, y0), tx), l(g(x0, y1), g(x1, y1), tx), ty) + } +} + +/// Replay strokes into a displacement mesh over a `w`×`h` layer. `frozen(x, y)` reports +/// mask-frozen points (Freeze Area Mask). +pub fn build_mesh(strokes: &[LiquifyStroke], w: f64, h: f64, frozen: &(dyn Fn(f64, f64) -> bool + Sync)) -> Mesh { + let cell = (w.max(h) / 128.0).clamp(2.0, 16.0); + let mut m = Mesh::new(w, h, cell); + for (i, f) in m.freeze.iter_mut().enumerate() { + let (x, y) = ((i % m.nx) as f64 * cell, (i / m.nx) as f64 * cell); + if frozen(x, y) { + *f = 1.0; + } + } + for (si, s) in strokes.iter().enumerate() { + let r = (s.size * 0.5).max(1.0); + let pr = (s.pressure / 100.0).clamp(0.0, 1.0); + // Dabs every r/4 along the stroke; each dab knows the local motion of the brush. + let mut dabs: Vec<([f64; 2], [f64; 2])> = vec![]; + if s.points.len() == 1 { + dabs.push((s.points[0], [0.0; 2])); + } + for seg in s.points.windows(2) { + let (a, b) = (seg[0], seg[1]); + let len = ((b[0] - a[0]).powi(2) + (b[1] - a[1]).powi(2)).sqrt(); + let n = ((len / (r * 0.25)).ceil() as usize).max(1); + let step = [(b[0] - a[0]) / n as f64, (b[1] - a[1]) / n as f64]; + for k in 1..=n { + dabs.push(([a[0] + step[0] * k as f64, a[1] + step[1] * k as f64], step)); + } + } + for (di, (c, mv)) in dabs.iter().enumerate() { + let (gx0, gx1) = (((c[0] - r) / cell).floor().max(0.0) as usize, (((c[0] + r) / cell).ceil() as usize).min(m.nx - 1)); + let (gy0, gy1) = (((c[1] - r) / cell).floor().max(0.0) as usize, (((c[1] + r) / cell).ceil() as usize).min(m.ny - 1)); + let prev = m.d.clone(); + for gy in gy0..=gy1 { + for gx in gx0..=gx1 { + let (x, y) = (gx as f64 * cell, gy as f64 * cell); + let (dx, dy) = (x - c[0], y - c[1]); + let dist = (dx * dx + dy * dy).sqrt(); + if dist >= r { + continue; + } + let i = gy * m.nx + gx; + // Smooth brush falloff. + let fall = { + let t = 1.0 - dist / r; + t * t * (3.0 - 2.0 * t) + }; + let tool = LIQUIFY_TOOLS[s.tool]; + if tool == "freeze" || tool == "thaw" { + let v = pr * fall; + m.freeze[i] = if tool == "freeze" { m.freeze[i].max(v) } else { (m.freeze[i] - v).max(0.0) }; + continue; + } + let w = pr * fall * (1.0 - m.freeze[i]) * 0.5; + if w <= 0.0 { + continue; + } + let d = &mut m.d[i]; + match tool { + "warp" => { + d[0] -= mv[0] * w * 2.0; + d[1] -= mv[1] * w * 2.0; + } + "turbulence" => { + let j = s.jitter / 100.0 * r * 0.1; + let seed = (si as u32).wrapping_mul(7919) ^ di as u32; + d[0] += (hash1(gx as u32, gy as u32, seed) as f64 * 2.0 - 1.0) * j * w; + d[1] += (hash1(gx as u32, gy as u32, seed ^ 0x55) as f64 * 2.0 - 1.0) * j * w; + d[0] -= mv[0] * w; + d[1] -= mv[1] * w; + } + "twirlClockwise" | "twirlCounterclockwise" => { + let sgn = if tool == "twirlClockwise" { -1.0 } else { 1.0 }; + let a = sgn * 0.08 * w; + let (sn, cs) = a.sin_cos(); + d[0] += dx * cs - dy * sn - dx; + d[1] += dx * sn + dy * cs - dy; + } + "pucker" => { + d[0] += dx * 0.05 * w; + d[1] += dy * 0.05 * w; + } + "bloat" => { + d[0] -= dx * 0.05 * w; + d[1] -= dy * 0.05 * w; + } + "shiftPixels" => { + // Perpendicular to the stroke (to its left). + d[0] -= mv[1] * w * 2.0; + d[1] += mv[0] * w * 2.0; + } + "reflection" => { + // Mirror across the stroke line through the dab. + let l = (mv[0] * mv[0] + mv[1] * mv[1]).sqrt(); + if l > 1e-9 { + let n = [-mv[1] / l, mv[0] / l]; + let k = dx * n[0] + dy * n[1]; + let tgt = [-2.0 * k * n[0], -2.0 * k * n[1]]; + d[0] += (tgt[0] - d[0]) * w; + d[1] += (tgt[1] - d[1]) * w; + } + } + "clone" => { + // Copy the distortion found at the clone source. + let sx = x + s.clone_offset[0]; + let sy = y + s.clone_offset[1]; + let fx = (sx / cell).round().clamp(0.0, (m.nx - 1) as f64) as usize; + let fy = (sy / cell).round().clamp(0.0, (m.ny - 1) as f64) as usize; + let src = prev[fy * m.nx + fx]; + let tgt = [src[0] + s.clone_offset[0], src[1] + s.clone_offset[1]]; + d[0] += (tgt[0] - d[0]) * w; + d[1] += (tgt[1] - d[1]) * w; + } + // Reconstruction: relax toward no distortion. + _ => { + d[0] *= 1.0 - w; + d[1] *= 1.0 - w; + } + } + } + } + } + } + m +} + +type MeshCache = Mutex>>; + +fn cached_mesh(ctx: &EffectCtx) -> Arc { + use std::hash::{Hash, Hasher}; + let text = ctx.params.s("distortionMesh"); + let mask = ctx.params.f("freezeAreaMask").round() as usize; + let poly = mask.checked_sub(1).and_then(|i| ctx.env.masks.get(i)).cloned(); + let mut h = std::collections::hash_map::DefaultHasher::new(); + text.hash(&mut h); + ctx.layer_size[0].to_bits().hash(&mut h); + ctx.layer_size[1].to_bits().hash(&mut h); + if let Some(p) = &poly { + for q in &p.points { + q[0].to_bits().hash(&mut h); + q[1].to_bits().hash(&mut h); + } + p.inverted.hash(&mut h); + } + let key = h.finish(); + static CACHE: OnceLock = OnceLock::new(); + let cache = CACHE.get_or_init(|| Mutex::new(HashMap::new())); + if let Some(m) = cache.lock().ok().and_then(|c| c.get(&key).cloned()) { + return m; + } + let strokes = parse_strokes(text); + let frozen = |x: f64, y: f64| match &poly { + Some(p) => point_in_poly(&p.points, x, y) != p.inverted, + None => false, + }; + let m = Arc::new(build_mesh(&strokes, ctx.layer_size[0].max(1.0), ctx.layer_size[1].max(1.0), &frozen)); + if let Ok(mut c) = cache.lock() { + if c.len() > 32 { + c.clear(); + } + c.insert(key, m.clone()); + } + m +} + +fn liquify(ctx: &EffectCtx, mut b: Buf) -> Buf { + let pct = ctx.params.f("distortionPercentage") / 100.0; + let show_freeze = ctx.params.b("viewFreezeAreaMask"); + let show_mesh = ctx.params.b("viewMesh"); + if ctx.params.s("distortionMesh").trim().is_empty() && !show_freeze && !show_mesh { + return b; + } + let mesh = cached_mesh(ctx); + let off = ctx.params.v2("distortionMeshOffset"); + let mesh_color = ctx.params.color("meshColor"); + let src = b.img.clone(); + let (scale, o) = (b.scale, b.offset); + let line_w = 0.6 / scale.max(1e-6); + b.img.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let lx = (x as f64 + 0.5 - o[0]) / scale; + let ly = (y as f64 + 0.5 - o[1]) / scale; + let d = mesh.at(lx - off[0], ly - off[1]); + let (sx, sy) = (lx + d[0] * pct, ly + d[1] * pct); + let mut c = if d == [0.0; 2] { *px } else { src.sample_bilinear(sx * scale + o[0], sy * scale + o[1]) }; + if show_freeze { + let fx = ((lx - off[0]) / mesh.cell).round().clamp(0.0, (mesh.nx - 1) as f64) as usize; + let fy = ((ly - off[1]) / mesh.cell).round().clamp(0.0, (mesh.ny - 1) as f64) as usize; + let f = mesh.freeze[fy * mesh.nx + fx] as f32; + if f > 0.0 { + c[0] = c[0] * (1.0 - 0.5 * f) + 0.5 * f; + c[3] = c[3].max(0.5 * f); + } + } + if show_mesh { + // Grid lines every 4 mesh cells, displaced with the mesh. + let gs = mesh.cell * 4.0; + let (gx, gy) = ((lx + d[0] * pct - off[0]).rem_euclid(gs), (ly + d[1] * pct - off[1]).rem_euclid(gs)); + if gx < line_w || gy < line_w { + c = [mesh_color[0], mesh_color[1], mesh_color[2], 1.0]; + } + } + *px = c; + } + }); + b +} + +// ---------------------------------------------------------------- Rolling Shutter Repair + +/// Per-pixel velocity field (layer pixels per frame) on a coarse grid, from tracked points. +struct Flow { + nx: usize, + ny: usize, + cell: f64, + v: Vec<[f64; 2]>, +} + +impl Flow { + fn at(&self, x: f64, y: f64) -> [f64; 2] { + let fx = (x / self.cell - 0.5).clamp(0.0, (self.nx - 1) as f64); + let fy = (y / self.cell - 0.5).clamp(0.0, (self.ny - 1) as f64); + let (x0, y0) = (fx.floor() as usize, fy.floor() as usize); + let (x1, y1) = ((x0 + 1).min(self.nx - 1), (y0 + 1).min(self.ny - 1)); + let (tx, ty) = (fx - x0 as f64, fy - y0 as f64); + let g = |i: usize, j: usize| self.v[j * self.nx + i]; + let l = |a: [f64; 2], b: [f64; 2], t: f64| [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t]; + l(l(g(x0, y0), g(x1, y0), tx), l(g(x0, y1), g(x1, y1), tx), ty) + } +} + +fn median(v: &mut [f64]) -> f64 { + v.sort_by(f64::total_cmp); + v[v.len() / 2] +} + +/// Measure motion around the current frame (`cur`) from `prev` to `next` (all in the same +/// buffer grid). `per_row`: Warp method (one velocity per scanline band). +fn measure(cur: &Image, prev: Option<&Image>, next: Option<&Image>, scale: f64, detailed: bool, per_row: bool) -> Option { + use effectcraft_track::klt::{GrayPyramid, LkOpts, analysis_factor, track}; + let factor = analysis_factor(cur.width, cur.height, if detailed { 640 } else { 320 }); + let pc = GrayPyramid::from_image(cur, [0.0; 2], factor, 4); + let (w, h) = (pc.width(), pc.height()); + let n = if detailed { 24 } else { 12 }; + let cell_w = w as f64 / n as f64; + let rows = ((n as f64) * h as f64 / w.max(1) as f64).round().max(2.0) as usize; + let cell_h = h as f64 / rows as f64; + let pts: Vec<[f64; 2]> = (0..rows).flat_map(|j| (0..n).map(move |i| [(i as f64 + 0.5) * cell_w, (j as f64 + 0.5) * cell_h])).collect(); + let o = LkOpts { radius: 4, fb_max: 1.0, ..Default::default() }; + let go = |other: Option<&Image>| other.map(|img| track(&pc, &GrayPyramid::from_image(img, [0.0; 2], factor, 4), &pts, None, &o)); + let (tn, tp) = (go(next), go(prev)); + // Velocity in plane pixels per frame. + let vel: Vec> = (0..pts.len()) + .map(|i| { + let p = pts[i]; + let fw = tn.as_ref().and_then(|t| t[i]).map(|q| [q[0] - p[0], q[1] - p[1]]); + let bw = tp.as_ref().and_then(|t| t[i]).map(|q| [p[0] - q[0], p[1] - q[1]]); + match (fw, bw) { + (Some(a), Some(b)) => Some([(a[0] + b[0]) * 0.5, (a[1] + b[1]) * 0.5]), + (a, b) => a.or(b), + } + }) + .collect(); + let valid: Vec<[f64; 2]> = vel.iter().flatten().copied().collect(); + if valid.is_empty() { + return None; + } + let mut vx: Vec = valid.iter().map(|v| v[0]).collect(); + let mut vy: Vec = valid.iter().map(|v| v[1]).collect(); + let global = [median(&mut vx), median(&mut vy)]; + let k = factor as f64 / scale.max(1e-9); + let mut v = vec![[0.0; 2]; n * rows]; + for j in 0..rows { + // Row (band) median for Warp; per point (outliers replaced) for Pixel Motion. + let band: Vec<[f64; 2]> = (0..n).filter_map(|i| vel[j * n + i]).collect(); + let row_med = if band.is_empty() { + global + } else { + let mut a: Vec = band.iter().map(|v| v[0]).collect(); + let mut b: Vec = band.iter().map(|v| v[1]).collect(); + [median(&mut a), median(&mut b)] + }; + for i in 0..n { + let s = if per_row { + row_med + } else { + match vel[j * n + i] { + Some(q) if (q[0] - row_med[0]).abs() + (q[1] - row_med[1]).abs() < 3.0 + 0.5 * (row_med[0].abs() + row_med[1].abs()) => q, + _ => row_med, + } + }; + v[j * n + i] = [s[0] * k, s[1] * k]; + } + } + // Cell size in layer pixels (grid is uniform in plane pixels; assume square plane pixels). + Some(Flow { nx: n, ny: rows, cell: cell_w * k, v }) +} + +fn same_grid(nb: Buf, b: &Buf) -> Image { + if nb.img.width == b.img.width && nb.img.height == b.img.height && nb.offset == b.offset && nb.scale == b.scale { + return nb.img; + } + let k = nb.scale / b.scale.max(1e-9); + crate::util::gen_image(b.img.width, b.img.height, |x, y| { + let lx = (x as f64 + 0.5 - b.offset[0]) * k + nb.offset[0]; + let ly = (y as f64 + 0.5 - b.offset[1]) * k + nb.offset[1]; + nb.img.sample_bilinear(lx, ly) + }) +} + +fn rolling_shutter(ctx: &EffectCtx, mut b: Buf) -> Buf { + let rate = ctx.params.f("rollingShutterRate") / 100.0; + if rate == 0.0 { + return b; + } + let Some(host) = ctx.env.host else { return b }; + let fps = ctx.fps(); + let prev = host.self_at(ctx.time - 1.0 / fps, ctx.env.effect_index).map(|nb| same_grid(nb, &b)); + let next = host.self_at(ctx.time + 1.0 / fps, ctx.env.effect_index).map(|nb| same_grid(nb, &b)); + if prev.is_none() && next.is_none() { + return b; + } + let per_row = ctx.params.e("method") == 0; + let Some(flow) = measure(&b.img, prev.as_ref(), next.as_ref(), 1.0, ctx.params.b("detailedAnalysis"), per_row) else { return b }; + let dir = ctx.params.e("scanDirection"); + let src = b.img.clone(); + let (w, h) = (b.img.width as f64, b.img.height as f64); + b.img.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let (fx, fy) = (x as f64 + 0.5, y as f64 + 0.5); + let s = match dir { + 1 => 1.0 - fy / h, + 2 => fx / w, + 3 => 1.0 - fx / w, + _ => fy / h, + }; + let tau = rate * (s - 0.5); + let v = flow.at(fx, fy); + *px = src.sample_bilinear_clamped(fx + v[0] * tau, fy + v[1] * tau); + } + }); + b +} + +pub fn specs() -> Vec { + vec![ + spec( + "ec.distort.ccflomotion", + "CC Flo Motion", + vec![ + p("knot1", "Knot 1", Value::Vec2([0.25, 0.5]), ParamUi::Point), + p("amount1", "Amount 1", num(50.0), slider(-400.0, 100.0, -100.0, 100.0, 1)), + p("knot2", "Knot 2", Value::Vec2([0.75, 0.5]), ParamUi::Point), + p("amount2", "Amount 2", num(-50.0), slider(-400.0, 100.0, -100.0, 100.0, 1)), + p("tileEdges", "Tile Edges", Value::Bool(false), ParamUi::Checkbox), + p("antialiasing", "Antialiasing", Value::Enum(1), popup(&["Low", "Medium", "High"])), + p("falloff", "Falloff", num(2.0), slider(0.01, 10.0, 0.1, 5.0, 2)), + ], + flo_motion, + ), + spec( + "ec.distort.liquify", + "Liquify", + vec![ + p("tool", "Tool", Value::Enum(0), popup(&LIQUIFY_TOOLS)), + p("brushSize", "Brush Size", num(64.0), slider(1.0, 600.0, 1.0, 600.0, 0)), + p("brushPressure", "Brush Pressure", num(50.0), slider(1.0, 100.0, 1.0, 100.0, 0)), + p("freezeAreaMask", "Freeze Area Mask", num(0.0), ParamUi::Mask), + p("turbulentJitter", "Turbulent Jitter", num(70.0), slider(1.0, 100.0, 1.0, 100.0, 0)), + p("cloneOffset", "Clone Offset", Value::Vec2([0.0, 0.0]), ParamUi::Hidden), + p("reconstructionMode", "Reconstruction Mode", Value::Enum(0), popup(&["Revert", "Displace", "Amplitwist", "Affine"])), + p("viewFreezeAreaMask", "View Freeze Area Mask", Value::Bool(false), ParamUi::Checkbox), + p("viewMesh", "View Mesh", Value::Bool(false), ParamUi::Checkbox), + p("meshSize", "Mesh Size", Value::Enum(1), popup(&["Small", "Medium", "Large"])), + p("meshColor", "Mesh Color", col(0.5, 0.5, 0.5), ParamUi::Color), + // The strokes (see `LiquifyStroke`); written by the `liquify.stroke` command. + p("distortionMesh", "Distortion Mesh", Value::Str(String::new()), ParamUi::Hidden), + p("distortionMeshOffset", "Distortion Mesh Offset", Value::Vec2([0.0, 0.0]), ParamUi::Hidden), + p("distortionPercentage", "Distortion Percentage", num(100.0), slider(0.0, 200.0, 0.0, 200.0, 1)), + p("distortionDisplayQuality", "Distortion Display Quality", Value::Enum(1), popup(&["Draft", "Smooth"])), + ], + liquify, + ), + spec( + "ec.distort.rollingshutterrepair", + "Rolling Shutter Repair", + vec![ + p("rollingShutterRate", "Rolling Shutter Rate", num(50.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("scanDirection", "Scan Direction", Value::Enum(0), popup(&["Top → Bottom", "Bottom → Top", "Left → Right", "Right → Left"])), + p("method", "Method", Value::Enum(0), popup(&["Warp", "Pixel Motion"])), + p("detailedAnalysis", "Detailed Analysis", Value::Bool(false), ParamUi::Checkbox), + p("pixelMotionDetail", "Pixel Motion Detail", num(20.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + ], + rolling_shutter, + ), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, EffectHost, LayerPixels, run_fx}; + + fn grid_img(w: u32, h: u32) -> Image { + crate::util::gen_image(w, h, |x, y| { + let v = if x % 8 == 0 || y % 8 == 0 { 1.0 } else { 0.2 }; + [v, v * 0.5, x as f32 / w as f32, 1.0] + }) + } + + #[test] + fn flo_motion_pulls_and_pushes() { + let img = grid_img(64, 64); + let out = run_fx( + "ec.distort.ccflomotion", + &[("knot1", Value::Vec2([32.0, 32.0])), ("amount1", num(80.0)), ("amount2", num(0.0))], + img.clone(), + 0.0, + EffectEnv::default(), + ); + // The knot itself is fixed; pixels near it are magnified (sampled closer to the knot). + assert!((out.img.get(32, 32)[2] - img.get(32, 32)[2]).abs() < 0.01); + assert!((out.img.get(40, 32)[2] - img.get(40, 32)[2]).abs() > 0.01); + assert!(out.img.get(40, 32)[2] < img.get(40, 32)[2]); + let none = run_fx("ec.distort.ccflomotion", &[("amount1", num(0.0)), ("amount2", num(0.0))], img.clone(), 0.0, EffectEnv::default()); + assert_eq!(none.img.data, img.data); + let again = run_fx( + "ec.distort.ccflomotion", + &[("knot1", Value::Vec2([32.0, 32.0])), ("amount1", num(80.0)), ("amount2", num(0.0))], + img, + 0.0, + EffectEnv::default(), + ); + assert_eq!(out.img.data, again.img.data); + } + + #[test] + fn liquify_strokes_round_trip_and_warp() { + let s = LiquifyStroke { tool: 0, size: 30.0, pressure: 80.0, jitter: 70.0, clone_offset: [0.0, 0.0], points: vec![[20.0, 32.0], [40.0, 32.0]] }; + assert_eq!(LiquifyStroke::parse(&s.to_line()), Some(s.clone())); + assert!(LiquifyStroke::parse("nope 1 2 3 4 5 1,2").is_none()); + let img = grid_img(64, 64); + let out = run_fx("ec.distort.liquify", &[("distortionMesh", Value::Str(s.to_line()))], img.clone(), 0.0, EffectEnv::default()); + assert_ne!(out.img.data, img.data); + // Warping to the right: a pixel right of the start now shows content from its left. + let at = |i: &Image| i.get(36, 32)[2]; + assert!(at(&out.img) < at(&img), "{} vs {}", at(&out.img), at(&img)); + // Far away, untouched. + assert_eq!(out.img.get(5, 5), img.get(5, 5)); + // Distortion Percentage 0 = identity; deterministic. + let zero = run_fx( + "ec.distort.liquify", + &[("distortionMesh", Value::Str(s.to_line())), ("distortionPercentage", num(0.0))], + img.clone(), + 0.0, + EffectEnv::default(), + ); + assert_eq!(zero.img.data, img.data); + let again = run_fx("ec.distort.liquify", &[("distortionMesh", Value::Str(s.to_line()))], img.clone(), 0.0, EffectEnv::default()); + assert_eq!(out.img.data, again.img.data); + // Reconstruction undoes the warp; freeze protects. + let mut rec = s.clone(); + rec.tool = 9; + rec.pressure = 100.0; + let lines = format!("{}\n{}\n{}\n{}", s.to_line(), rec.to_line(), rec.to_line(), rec.to_line()); + let m1 = build_mesh(&parse_strokes(&s.to_line()), 64.0, 64.0, &|_, _| false); + let m2 = build_mesh(&parse_strokes(&lines), 64.0, 64.0, &|_, _| false); + let mag = |m: &Mesh| m.d.iter().map(|d| d[0].abs() + d[1].abs()).sum::(); + assert!(mag(&m2) < mag(&m1) * 0.5); + let frozen = build_mesh(&parse_strokes(&s.to_line()), 64.0, 64.0, &|_, _| true); + assert_eq!(mag(&frozen), 0.0); + // Every tool changes (or deliberately keeps) the mesh without NaNs. + for t in 0..LIQUIFY_TOOLS.len() { + let st = LiquifyStroke { tool: t, clone_offset: [10.0, 0.0], ..s.clone() }; + let m = build_mesh(&[s.clone(), st], 64.0, 64.0, &|_, _| false); + assert!(m.d.iter().all(|d| d[0].is_finite() && d[1].is_finite()), "{}", LIQUIFY_TOOLS[t]); + } + } + + /// A textured frame panning right at 3 px per frame, captured with a rolling shutter: row + /// `y` is exposed `(y / h − ½)` frames late (rate 100 %). + fn texture(x: f64, y: f64) -> f32 { + let a = ((x * 0.31).sin() * (y * 0.23).cos() + (x * 0.07 + y * 0.11).sin()) as f32; + 0.5 + 0.25 * a + } + fn rs_frame(f: f64, shutter: bool) -> Image { + let (w, h) = (96u32, 64u32); + crate::util::gen_image(w, h, |x, y| { + let tau = if shutter { (y as f64 + 0.5) / h as f64 - 0.5 } else { 0.0 }; + let shift = 3.0 * (f + tau); + let v = texture(x as f64 + 0.5 - shift, y as f64 + 0.5); + [v, v, v, 1.0] + }) + } + struct Pan; + impl EffectHost for Pan { + fn layer(&self, _: u64, _: bool) -> Option { + None + } + fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option> { + None + } + fn self_at(&self, t: f64, _: usize) -> Option { + Some(Buf { img: rs_frame(t * 24.0, true), offset: [0.0; 2], scale: 1.0 }) + } + } + + #[test] + fn rolling_shutter_repair_unskews_a_pan() { + let host = Pan; + let env = EffectEnv { host: Some(&host), frame_rate: 24.0, ..Default::default() }; + let t = 5.0 / 24.0; + let skewed = rs_frame(5.0, true); + let truth = rs_frame(5.0, false); + let err = |a: &Image| { + let mut s = 0.0; + let mut n = 0.0; + for y in 4..60 { + for x in 12..84 { + s += (a.get(x, y)[0] - truth.get(x, y)[0]).abs(); + n += 1.0; + } + } + s / n + }; + for method in [0u32, 1] { + let out = run_fx("ec.distort.rollingshutterrepair", &[("rollingShutterRate", num(100.0)), ("method", Value::Enum(method))], skewed.clone(), t, env); + assert!(err(&out.img) < err(&skewed) * 0.4, "method {method}: {} vs {}", err(&out.img), err(&skewed)); + let again = + run_fx("ec.distort.rollingshutterrepair", &[("rollingShutterRate", num(100.0)), ("method", Value::Enum(method))], skewed.clone(), t, env); + assert_eq!(out.img.data, again.img.data); + } + let none = run_fx("ec.distort.rollingshutterrepair", &[], skewed.clone(), t, EffectEnv::default()); + assert_eq!(none.img.data, skewed.data); + } +} diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 14a43464..746500e6 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -8,6 +8,7 @@ pub mod audio_fx; mod blur2; +mod blur3; mod channel; mod channel2; mod color2; @@ -17,6 +18,7 @@ mod controls; mod distort; mod distort2; mod distort3; +pub mod distort4; mod generate; mod generate2; mod generate3; @@ -356,6 +358,8 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(obsolete::specs()); v.extend(ocio::specs()); v.extend(vr::specs()); + v.extend(blur3::specs()); + v.extend(distort4::specs()); v.extend(time_fx::specs()); v.extend(audio_fx::specs()); v.extend(paint::specs()); @@ -472,6 +476,8 @@ pub const TIME_DEPENDENT: &[&str] = &[ "ec.time.ccwidetime", "ec.time.pixelmotionblur", "ec.vr.digitalglitch", + "ec.blur.camerashakedeblur", + "ec.distort.rollingshutterrepair", ]; /// See [`TIME_DEPENDENT`]. diff --git a/crates/engine/src/commands/liquify.rs b/crates/engine/src/commands/liquify.rs new file mode 100644 index 00000000..748c664c --- /dev/null +++ b/crates/engine/src/commands/liquify.rs @@ -0,0 +1,156 @@ +//! Liquify brush strokes (`liquify.stroke`, `liquify.clear`). +//! +//! A stroke is appended to the layer's Liquify effect (added when the layer has none) as one +//! line of its hidden Distortion Mesh data (see `effectcraft_effects::distort4`). Brush size, +//! pressure, turbulent jitter and clone offset default to the effect's own tool options. When +//! the Distortion Mesh is animated, the stroke goes into a keyframe at the current time (copying +//! the mesh in effect there), as After Effects records mesh keyframes. + +use effectcraft_effects::distort4::{LIQUIFY_TOOLS, LiquifyStroke, parse_strokes}; +use effectcraft_keyframe::{Keyframe, Value as KV}; +use effectcraft_project::build::Ids; +use effectcraft_project::{GroupKind, PropGroup, Uid}; +use serde_json::{Value, json}; + +use super::{CommandSpec, bad, f_p, has_comp, layer_mut, layer_p, str_p}; +use crate::{EngineError, Result, Session, cmd}; + +const ID: &str = "ec.distort.liquify"; + +fn is_liquify(g: &PropGroup) -> bool { + matches!(&g.kind, GroupKind::Effect { effect } if effect == ID) +} + +fn liquify_effect(fx: &mut PropGroup, ids: &mut Ids, layer_size: [f64; 2], want: Option) -> Result { + if let Some(u) = want { + return fx.groups().find(|g| g.uid == u && is_liquify(g)).map(|g| g.uid).ok_or_else(|| bad("liquify.stroke", "no such Liquify effect")); + } + if let Some(g) = fx.groups().filter(|g| is_liquify(g)).last() { + return Ok(g.uid); + } + let spec = effectcraft_effects::find(ID).ok_or_else(|| EngineError::Other("Liquify effect missing".into()))?; + let g = effectcraft_effects::instantiate(spec, ids, "Liquify", layer_size); + let uid = g.uid; + fx.children.push(g.into()); + Ok(uid) +} + +fn num_of(g: &PropGroup, id: &str, default: f64) -> f64 { + g.get(id).map(|p| p.value.as_f64()).unwrap_or(default) +} + +fn stroke(s: &mut Session, p: &Value) -> Result { + let cmd = "liquify.stroke"; + let (cid, lid) = layer_p(s, p, cmd)?; + let tool_name = str_p(p, "tool").unwrap_or("warp"); + let tool = LIQUIFY_TOOLS + .iter() + .position(|t| t.eq_ignore_ascii_case(tool_name)) + .ok_or_else(|| bad(cmd, format!("unknown tool `{tool_name}` ({})", LIQUIFY_TOOLS.join(" | "))))?; + let raw = p.get("points").and_then(Value::as_array).ok_or_else(|| bad(cmd, "missing `points` [[x, y], …] (layer space)"))?; + let mut points = vec![]; + for q in raw { + let a = q.as_array().ok_or_else(|| bad(cmd, "each point is [x, y]"))?; + let (Some(x), Some(y)) = (a.first().and_then(Value::as_f64), a.get(1).and_then(Value::as_f64)) else { + return Err(bad(cmd, "each point is [x, y]")); + }; + points.push([x, y]); + } + if points.is_empty() { + return Err(bad(cmd, "`points` is empty")); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let t = comp.frame_rate.snap_nearest(super::time_p(s, p, Some(comp))); + let layer = comp.layer(lid).ok_or_else(|| bad(cmd, "no layer"))?; + if layer.effects().is_none() || !layer.source.is_av() { + return Err(bad(cmd, "Liquify works on footage, solid, text, shape and precomp layers")); + } + let lt = layer.layer_time(t); + let (w, h) = effectcraft_render::source_size(&s.project, layer); + let size = if w == 0 { [comp.width as f64, comp.height as f64] } else { [w as f64, h as f64] }; + let want = p.get("effect").and_then(Value::as_u64); + let (fx_uid, count) = s.edit("Liquify Stroke", None, |proj, _st| { + let mut next = proj.next_id; + let l = layer_mut(proj, cid, lid)?; + let fx = l.props.sub_mut("effects").ok_or_else(|| bad(cmd, "this layer has no effects"))?; + let mut ids = Ids(&mut next); + let fx_uid = liquify_effect(fx, &mut ids, size, want)?; + proj.next_id = next; + let l = layer_mut(proj, cid, lid)?; + let g = l.props.sub_mut("effects").and_then(|fx| fx.find_group_mut(fx_uid)).ok_or_else(|| bad(cmd, "Liquify effect gone"))?; + let co = g.get("cloneOffset").map(|p| p.value.as_vec2()).unwrap_or([0.0; 2]); + let st = LiquifyStroke { + tool, + size: f_p(p, "size").unwrap_or_else(|| num_of(g, "brushSize", 64.0)).max(1.0), + pressure: f_p(p, "pressure").unwrap_or_else(|| num_of(g, "brushPressure", 50.0)).clamp(1.0, 100.0), + jitter: f_p(p, "jitter").unwrap_or_else(|| num_of(g, "turbulentJitter", 70.0)).clamp(1.0, 100.0), + clone_offset: match p.get("cloneOffset").and_then(Value::as_array) { + Some(a) => [a.first().and_then(Value::as_f64).unwrap_or(0.0), a.get(1).and_then(Value::as_f64).unwrap_or(0.0)], + None => co, + }, + points: points.clone(), + }; + let prop = g.get_mut("distortionMesh").ok_or_else(|| bad(cmd, "Liquify effect has no Distortion Mesh"))?; + let append = |cur: &str| if cur.trim().is_empty() { st.to_line() } else { format!("{}\n{}", cur.trim_end(), st.to_line()) }; + let text = if prop.keys.is_empty() { + let cur = match &prop.value { + KV::Str(x) => x.clone(), + _ => String::new(), + }; + let v = append(&cur); + prop.value = KV::Str(v.clone()); + v + } else { + // Hold keyframes: the mesh in effect at `lt` is the last key at or before it. + let cur = prop.keys.iter().rfind(|k| k.time <= lt).or(prop.keys.first()).map(|k| k.value.clone()); + let cur = match cur { + Some(KV::Str(x)) => x, + _ => String::new(), + }; + let v = append(&cur); + match prop.keys.iter_mut().find(|k| k.time == lt) { + Some(k) => k.value = KV::Str(v.clone()), + None => { + prop.keys.push(Keyframe::new(lt, KV::Str(v.clone()))); + prop.keys.sort_by_key(|k| k.time); + } + } + v + }; + Ok((fx_uid, parse_strokes(&text).len())) + })?; + Ok(json!({"effect": fx_uid, "strokes": count})) +} + +fn clear(s: &mut Session, p: &Value) -> Result { + let cmd = "liquify.clear"; + let (cid, lid) = layer_p(s, p, cmd)?; + let want = p.get("effect").and_then(Value::as_u64); + s.edit("Clear Liquify Mesh", None, |proj, _| { + let l = layer_mut(proj, cid, lid)?; + let fx = l.props.sub_mut("effects").ok_or_else(|| bad(cmd, "no effects"))?; + let uid = + fx.groups().filter(|g| is_liquify(g) && want.is_none_or(|u| u == g.uid)).last().map(|g| g.uid).ok_or_else(|| bad(cmd, "no Liquify effect"))?; + let g = fx.find_group_mut(uid).ok_or_else(|| bad(cmd, "no Liquify effect"))?; + let prop = g.get_mut("distortionMesh").ok_or_else(|| bad(cmd, "no Distortion Mesh"))?; + prop.keys.clear(); + prop.value = KV::Str(String::new()); + Ok(()) + })?; + Ok(Value::Null) +} + +pub fn specs() -> Vec { + vec![ + cmd!( + "liquify.stroke", + "Liquify Stroke", + [], + None, + "{layer?, effect?: uid, tool?: warp|turbulence|twirlClockwise|twirlCounterclockwise|pucker|bloat|shiftPixels|reflection|clone|reconstruction|freeze|thaw, points: [[x,y],…] (layer space), size?, pressure? (1-100), jitter? (1-100), cloneOffset?: [dx,dy], time? (s) | frame?}", + has_comp, + stroke + ), + cmd!("liquify.clear", "Clear Liquify Mesh", [], None, "{layer?, effect?: uid}", has_comp, clear), + ] +} diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 26db85c9..4f0910ba 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -17,6 +17,7 @@ mod layer; mod layer_menu; mod layer_time; mod link; +mod liquify; mod lottie; mod markers; mod mask; @@ -127,6 +128,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(track::specs()); v.extend(paint::specs()); v.extend(puppet::specs()); + v.extend(liquify::specs()); v.extend(settings::specs()); v.extend(stubs::specs()); v diff --git a/crates/engine/src/tests_paint.rs b/crates/engine/src/tests_paint.rs index e274f725..9926e04c 100644 --- a/crates/engine/src/tests_paint.rs +++ b/crates/engine/src/tests_paint.rs @@ -227,3 +227,26 @@ fn project_with_paint_and_puppet_round_trips() { let v: Value = serde_json::from_str(&j).unwrap(); assert!(v.to_string().contains("ec.paint.paint") && v.to_string().contains("ec.distort.puppet")); } + +#[test] +fn liquify_stroke_adds_the_effect_and_warps() { + let (mut s, id) = setup(); + // Layer (5, 30) is comp (55, 50). Before: opaque blue. + assert!(px(&s, 0.0, 55, 50)[3] > 0.99); + let r = s.execute("liquify.stroke", json!({"layer": id, "tool": "warp", "points": [[-20, 30], [10, 30]], "size": 40, "pressure": 100})).unwrap(); + assert_eq!(r["strokes"], 1); + let l = layer(&s, id); + let fx = l.effects().unwrap().groups().next().unwrap().clone(); + assert!(matches!(&fx.kind, effectcraft_project::GroupKind::Effect { effect } if effect == "ec.distort.liquify")); + // The warp pulls transparent pixels from outside the layer over its left edge. + assert!(px(&s, 0.0, 55, 50)[3] < 0.9, "{:?}", px(&s, 0.0, 55, 50)); + // A second stroke goes into the same effect. + let r2 = s.execute("liquify.stroke", json!({"layer": id, "tool": "reconstruction", "points": [[0, 30]], "size": 60, "pressure": 100})).unwrap(); + assert_eq!(r2["strokes"], 2); + assert_eq!(r2["effect"], r["effect"]); + assert!(s.execute("liquify.stroke", json!({"layer": id, "tool": "smudge", "points": [[0, 0]]})).is_err()); + s.execute("edit.undo", json!({})).unwrap(); + s.execute("edit.undo", json!({})).unwrap(); + assert!(layer(&s, id).effects().unwrap().groups().next().is_none()); + assert!(px(&s, 0.0, 55, 50)[3] > 0.99); +} diff --git a/docs/architecture.md b/docs/architecture.md index c3d478e3..f7819a7f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -35,7 +35,7 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L5 | `automation` | The MCP server, headless or bridged to the running app | | L6 | apps `effectcraft`, `effectcraft-cli`, `effectcraft-web` | Desktop app; command-line tool (render, exec, get/set, MCP); the browser app (wasm32, [web.md](web.md)) | -Allowed same-layer edges: `path → keyframe, raster`, `text → path`, `effects → project, text, path`, +Allowed same-layer edges: `path → keyframe, raster`, `text → path`, `effects → project, text, path, track`, `media / expr / export / gpu → render`, `export → media`, `lottie → format`, `host → engine`. ## 2. Time diff --git a/xtask/src/main.rs b/xtask/src/main.rs index ae38adf9..76ff6fa3 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -55,6 +55,7 @@ const SAME_LAYER: &[(&str, &str)] = &[ ("effects", "project"), ("effects", "text"), ("effects", "path"), + ("effects", "track"), ("media", "render"), ("expr", "render"), ("export", "render"),