diff --git a/Cargo.lock b/Cargo.lock index 871c9fd8..94e07457 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1329,6 +1329,7 @@ dependencies = [ "effectcraft-raster", "effectcraft-render", "effectcraft-time", + "exr", "filmcraft-codecs", "filmcraft-color", "filmcraft-frame", @@ -1337,6 +1338,7 @@ dependencies = [ "image", "log", "rayon", + "serde_json", "thiserror 2.0.21", ] @@ -1782,6 +1784,7 @@ dependencies = [ "miniz_oxide 0.8.9", "num-complex", "pulp", + "rayon-core", "smallvec", "zune-inflate", ] diff --git a/Cargo.toml b/Cargo.toml index a5294834..ce22eaac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,7 @@ kurbo = { version = "0.13", features = ["serde"] } tiny-skia = { version = "0.12", default-features = false, features = ["std", "simd"] } skrifa = { version = "0.44", default-features = false, features = ["std"] } harfrust = { version = "0.12", default-features = false, features = ["std"] } +exr = "1.74" image = { version = "0.25", default-features = false, features = ["png", "jpeg"] } # media codecs + containers: FilmCraft's pure-Rust crates, pinned to a revision (plan/adr/0001). diff --git a/crates/effects/Cargo.toml b/crates/effects/Cargo.toml index e321fa2f..ff5d3ed7 100644 --- a/crates/effects/Cargo.toml +++ b/crates/effects/Cargo.toml @@ -17,6 +17,5 @@ effectcraft-project = { workspace = true } effectcraft-raster = { workspace = true } effectcraft-track = { workspace = true } rayon = { workspace = true } - -[dev-dependencies] serde_json = { workspace = true } + diff --git a/crates/effects/src/audio_fx.rs b/crates/effects/src/audio_fx.rs index 855fd10e..bd80519a 100644 --- a/crates/effects/src/audio_fx.rs +++ b/crates/effects/src/audio_fx.rs @@ -1,5 +1,5 @@ -//! Audio effects (Effect > Audio): Backwards, Bass & Treble, Delay, Flange & Chorus, -//! High-Low Pass, Modulator, Parametric EQ, Reverb, Stereo Mixer and Tone. +//! Audio effects (Effect > Audio): Backwards, Bass & Treble, Compressor, Delay, Distortion, +//! Flange & Chorus, Gate, High-Low Pass, Modulator, Parametric EQ, Reverb, Stereo Mixer and Tone. //! //! They are ordinary effect instances on a layer (so their parameters animate and show in //! Effect Controls), pass video through unchanged, and are applied to the layer's audio by the @@ -12,7 +12,10 @@ //! DSP is textbook: second-order sections use the public RBJ "Audio EQ Cookbook" formulas; //! Reverb is a Schroeder/Moorer network (parallel low-pass feedback combs into series //! all-passes, as in the published Freeverb description); chorus/flange/vibrato are modulated -//! fractional delay lines. +//! fractional delay lines. Compressor and Gate are feed-forward dynamics processors on a +//! stereo-linked peak envelope (one-pole attack/release smoothing, the textbook quadratic soft-knee +//! gain computer); Distortion is a static waveshaper (soft clip, hard clip, fold-back, bit crush) +//! followed by a one-pole tone filter. use std::f64::consts::PI; @@ -60,6 +63,20 @@ pub fn specs() -> Vec { p("treble", "Treble", num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 2)), ], ), + spec( + "ec.audio.compressor", + "Compressor", + vec![ + p("threshold", "Threshold (dB)", num(-20.0), slider(-60.0, 0.0, -60.0, 0.0, 2)), + p("ratio", "Ratio", num(4.0), slider(1.0, 30.0, 1.0, 20.0, 2)), + p("attack", "Attack (ms)", num(10.0), slider(0.0, 500.0, 0.0, 200.0, 2)), + p("release", "Release (ms)", num(100.0), slider(1.0, 5000.0, 1.0, 1000.0, 2)), + p("knee", "Knee (dB)", num(0.0), slider(0.0, 24.0, 0.0, 24.0, 2)), + p("makeupGain", "Makeup Gain (dB)", num(0.0), slider(-24.0, 36.0, 0.0, 24.0, 2)), + pp("dryOut", "Dry Out", pct(0.0)), + pp("wetOut", "Wet Out", pct(100.0)), + ], + ), spec( "ec.audio.delay", "Delay", @@ -71,6 +88,19 @@ pub fn specs() -> Vec { pp("wetOut", "Wet Out", pct(75.0)), ], ), + spec( + "ec.audio.distortion", + "Distortion", + vec![ + p("distortionType", "Distortion Type", Value::Enum(0), popup(&["Soft Clip", "Hard Clip", "Fold Back", "Bit Crush"])), + p("drive", "Drive (dB)", num(12.0), slider(0.0, 48.0, 0.0, 48.0, 2)), + p("bitDepth", "Bit Depth", num(8.0), slider(1.0, 16.0, 1.0, 16.0, 0)), + p("tone", "Tone (Hz)", num(8000.0), slider(200.0, 20000.0, 200.0, 20000.0, 1)), + p("outputGain", "Output Gain (dB)", num(-6.0), slider(-48.0, 12.0, -24.0, 12.0, 2)), + pp("dryOut", "Dry Out", pct(0.0)), + pp("wetOut", "Wet Out", pct(100.0)), + ], + ), spec( "ec.audio.flangechorus", "Flange & Chorus", @@ -86,6 +116,17 @@ pub fn specs() -> Vec { pp("wetOut", "Wet Out", pct(50.0)), ], ), + spec( + "ec.audio.gate", + "Gate", + vec![ + p("threshold", "Threshold (dB)", num(-40.0), slider(-90.0, 0.0, -80.0, 0.0, 2)), + p("attack", "Attack (ms)", num(1.0), slider(0.0, 500.0, 0.0, 100.0, 2)), + p("hold", "Hold (ms)", num(50.0), slider(0.0, 5000.0, 0.0, 500.0, 2)), + p("release", "Release (ms)", num(100.0), slider(1.0, 5000.0, 1.0, 1000.0, 2)), + p("range", "Range (dB)", num(-80.0), slider(-120.0, 0.0, -90.0, 0.0, 2)), + ], + ), spec( "ec.audio.highlowpass", "High-Low Pass", @@ -167,6 +208,14 @@ pub fn preroll(id: &str, params: &Params) -> f64 { match id { "ec.audio.basstreble" | "ec.audio.highlowpass" | "ec.audio.parametriceq" => 0.05, "ec.audio.flangechorus" | "ec.audio.modulator" => 0.06, + "ec.audio.distortion" => 0.01, + // The envelope settles within a few attack/release time constants. + "ec.audio.compressor" | "ec.audio.gate" => { + let rel = params.f("release").max(1.0) / 1000.0; + let att = params.f("attack").max(0.0) / 1000.0; + let hold = if id == "ec.audio.gate" { params.f("hold").max(0.0) / 1000.0 } else { 0.0 }; + (9.0 * rel + 9.0 * att + hold + 0.01).min(10.0) + } "ec.audio.delay" => { let d = params.f("delayTime").max(1.0) / 1000.0; let fb = (params.f("feedback") / 100.0).clamp(0.0, 0.99); @@ -231,6 +280,9 @@ pub fn process(id: &str, params: &Params, buf: &mut [f32], rate: u32, start: f64 } } "ec.audio.delay" => delay(params, buf, sr), + "ec.audio.compressor" => compressor(params, buf, sr), + "ec.audio.distortion" => distortion(params, buf, sr), + "ec.audio.gate" => gate(params, buf, sr), "ec.audio.flangechorus" => flange_chorus(params, buf, sr, start), "ec.audio.modulator" => modulator(params, buf, sr, start), "ec.audio.reverb" => reverb(params, buf, sr), @@ -362,6 +414,126 @@ fn delay(params: &Params, buf: &mut [f32], sr: f64) { } } +#[inline] +fn db_to_lin(db: f64) -> f64 { + 10f64.powf(db / 20.0) +} + +#[inline] +fn lin_to_db(v: f64) -> f64 { + 20.0 * v.max(1e-12).log10() +} + +/// One-pole smoothing coefficient for a time constant of `ms` milliseconds. +fn smoothing(ms: f64, sr: f64) -> f64 { + if ms <= 0.0 { 0.0 } else { (-1.0 / (ms / 1000.0 * sr)).exp() } +} + +/// Static compressor curve: gain change (dB, ≤ 0) for an input level `x` dB (quadratic soft +/// knee of width `knee` around the threshold). +pub fn compressor_gain_db(x: f64, threshold: f64, ratio: f64, knee: f64) -> f64 { + let r = ratio.max(1.0); + let over = x - threshold; + let y = if knee > 0.0 && 2.0 * over.abs() <= knee { + x + (1.0 / r - 1.0) * (over + knee / 2.0).powi(2) / (2.0 * knee) + } else if over > 0.0 { + threshold + over / r + } else { + x + }; + y - x +} + +fn compressor(params: &Params, buf: &mut [f32], sr: f64) { + let th = params.f("threshold"); + let ratio = params.f("ratio").max(1.0); + let knee = params.f("knee").max(0.0); + let att = smoothing(params.f("attack"), sr); + let rel = smoothing(params.f("release").max(0.001), sr); + let makeup = params.f("makeupGain"); + let dry = (params.f("dryOut") / 100.0) as f32; + let wet = (params.f("wetOut") / 100.0) as f32; + // Smoothed gain change in dB (≤ 0): attack while the reduction grows, release as it shrinks. + let mut g = 0.0f64; + for f in buf.chunks_exact_mut(2) { + let peak = f[0].abs().max(f[1].abs()) as f64; + let target = compressor_gain_db(lin_to_db(peak), th, ratio, knee); + let k = if target < g { att } else { rel }; + g = target + (g - target) * k; + let lin = db_to_lin(g + makeup) as f32; + for v in f.iter_mut() { + *v = *v * dry + *v * lin * wet; + } + } +} + +fn gate(params: &Params, buf: &mut [f32], sr: f64) { + let th = db_to_lin(params.f("threshold")); + let att = smoothing(params.f("attack"), sr); + let rel = smoothing(params.f("release").max(0.001), sr); + let hold = (params.f("hold").max(0.0) / 1000.0 * sr) as usize; + let floor = db_to_lin(params.f("range").min(0.0)); + // Openness 0..1 (closed..open), mapped to floor..1. + let mut g = 0.0f64; + let mut held = 0usize; + for f in buf.chunks_exact_mut(2) { + let peak = f[0].abs().max(f[1].abs()) as f64; + let open = if peak >= th { + held = hold; + true + } else if held > 0 { + held -= 1; + true + } else { + false + }; + let target = if open { 1.0 } else { 0.0 }; + let k = if target > g { att } else { rel }; + g = target + (g - target) * k; + let lin = (floor + (1.0 - floor) * g) as f32; + f[0] *= lin; + f[1] *= lin; + } +} + +/// The Distortion effect's waveshaper for one sample (`drive` linear gain, `bits` for Bit Crush). +pub fn distort_sample(kind: u32, x: f32, drive: f32, bits: f32) -> f32 { + let v = x * drive; + match kind { + 1 => v.clamp(-1.0, 1.0), + 2 => { + // Fold back: reflect at ±1 (a triangle function of the input). + let t = (v + 1.0).rem_euclid(4.0); + if t < 2.0 { t - 1.0 } else { 3.0 - t } + } + 3 => { + let q = 2f32.powf(bits.clamp(1.0, 16.0) - 1.0); + (v.clamp(-1.0, 1.0) * q).round() / q + } + _ => v.tanh(), + } +} + +fn distortion(params: &Params, buf: &mut [f32], sr: f64) { + let kind = params.e("distortionType"); + let drive = db_to_lin(params.f("drive")) as f32; + let bits = params.f("bitDepth") as f32; + let out = db_to_lin(params.f("outputGain")) as f32; + let dry = (params.f("dryOut") / 100.0) as f32; + let wet = (params.f("wetOut") / 100.0) as f32; + let fc = params.f("tone").clamp(20.0, sr * 0.49); + let a = (-2.0 * PI * fc / sr).exp() as f32; + let mut lp = [0.0f32; 2]; + for f in buf.chunks_exact_mut(2) { + for ch in 0..2 { + let x = f[ch]; + let y = distort_sample(kind, x, drive, bits); + lp[ch] = y + (lp[ch] - y) * a; + f[ch] = x * dry + lp[ch] * out * wet; + } + } +} + fn lfo(kind: u32, phase: f64) -> f64 { let ph = phase.rem_euclid(1.0); match kind { @@ -546,7 +718,10 @@ mod tests { for id in [ "ec.audio.backwards", "ec.audio.basstreble", + "ec.audio.compressor", "ec.audio.delay", + "ec.audio.distortion", + "ec.audio.gate", "ec.audio.flangechorus", "ec.audio.highlowpass", "ec.audio.modulator", @@ -679,4 +854,64 @@ mod tests { assert_eq!(b, vec![2.0, 1.0]); assert!(reverses("ec.audio.backwards")); } + + #[test] + fn compressor_reduces_gain_above_threshold() { + // Static curve: 10 dB over a −20 dB threshold at 4:1 comes out 7.5 dB lower. + assert!((compressor_gain_db(-10.0, -20.0, 4.0, 0.0) + 7.5).abs() < 1e-9); + assert_eq!(compressor_gain_db(-30.0, -20.0, 4.0, 0.0), 0.0); + // The soft knee meets the hard curve at its edges. + let k = 6.0; + assert!(compressor_gain_db(-23.0, -20.0, 4.0, k).abs() < 1e-9); + assert!((compressor_gain_db(-17.0, -20.0, 4.0, k) - compressor_gain_db(-17.0, -20.0, 4.0, 0.0)).abs() < 1e-9); + let sr = 48000.0; + let mut p = defaults("ec.audio.compressor"); + p.values.insert("attack".into(), num(1.0)); + let mut loud = sine(1000.0, sr, 9600); + process("ec.audio.compressor", &p, &mut loud, 48000, 0.0); + // A full-scale sine (peak 0 dB) is pulled down by ~15 dB once the envelope settles. + let peak = loud[18000..].iter().fold(0.0f32, |m, v| m.max(v.abs())); + let want = db_to_lin(compressor_gain_db(0.0, -20.0, 4.0, 0.0)) as f32; + assert!((peak - want).abs() < 0.05, "{peak} vs {want}"); + // Quiet input passes untouched. + let mut quiet: Vec = sine(1000.0, sr, 4800).iter().map(|v| v * 0.01).collect(); + let orig = quiet.clone(); + process("ec.audio.compressor", &p, &mut quiet, 48000, 0.0); + assert!(quiet.iter().zip(&orig).all(|(a, b)| (a - b).abs() < 1e-6)); + } + + #[test] + fn gate_mutes_below_threshold() { + let sr = 48000.0; + let p = defaults("ec.audio.gate"); + // A −60 dB signal under the −40 dB threshold is attenuated by the −80 dB range. + let mut quiet: Vec = sine(500.0, sr, 9600).iter().map(|v| v * 0.001).collect(); + process("ec.audio.gate", &p, &mut quiet, 48000, 0.0); + assert!(rms(&quiet, 4800) < 1e-6, "{}", rms(&quiet, 4800)); + // A loud signal opens the gate. + let mut loud: Vec = sine(500.0, sr, 9600).iter().map(|v| v * 0.5).collect(); + let r0 = rms(&loud, 4800); + process("ec.audio.gate", &p, &mut loud, 48000, 0.0); + assert!((rms(&loud, 4800) - r0).abs() < 0.01); + // Deterministic. + let mut a = sine(300.0, sr, 2000); + let mut b = a.clone(); + process("ec.audio.gate", &p, &mut a, 48000, 0.0); + process("ec.audio.gate", &p, &mut b, 48000, 0.0); + assert_eq!(a, b); + } + + #[test] + fn distortion_shapes_and_bounds() { + assert!((distort_sample(0, 0.5, 1.0, 8.0) - 0.5f32.tanh()).abs() < 1e-6); + assert_eq!(distort_sample(1, 0.8, 2.0, 8.0), 1.0); + assert!((distort_sample(2, 0.7, 2.0, 8.0) - 0.6).abs() < 1e-5); + assert_eq!(distort_sample(3, 0.3, 1.0, 2.0), 0.5); + let p = defaults("ec.audio.distortion"); + let mut b = sine(200.0, 48000.0, 4800); + process("ec.audio.distortion", &p, &mut b, 48000, 0.0); + assert!(b.iter().all(|v| v.abs() <= 1.0)); + assert!(rms(&b, 100) > 0.1); + assert!(preroll("ec.audio.compressor", &defaults("ec.audio.compressor")) > 0.4); + } } diff --git a/crates/effects/src/blur3.rs b/crates/effects/src/blur3.rs new file mode 100644 index 00000000..0140ac99 --- /dev/null +++ b/crates/effects/src/blur3.rs @@ -0,0 +1,338 @@ +//! Blur & Sharpen: CC Radial Blur and Camera-Shake Deblur. +//! +//! * **CC Radial Blur** — zoom (straight, fading, centred both ways) and rotational (plain or +//! fading) blurs around a centre point, sampled along the ray / arc through each pixel with a +//! sample count set by Quality. +//! * **Camera-Shake Deblur** — frame-based restoration in the spirit of published video deblurring +//! by "lucky frames": the frame is split into patches; for each patch the sharpness (mean +//! squared gradient) is compared with the same patch in the neighbouring frames within Blur +//! Duration; when a neighbour is clearly sharper, its patch is aligned to the current frame by +//! a small translation search (sum of squared differences on luma) and substituted with +//! feathered patch blending. Deterministic; neighbours come from [`crate::EffectHost::self_at`]. + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::{Image, Px}; +use rayon::prelude::*; + +use crate::util::Plane; +use crate::{Buf, EffectCtx, EffectSpec, num, p, popup, slider}; + +fn spec(id: &'static str, name: &'static str, params: Vec, render: crate::RenderFn) -> EffectSpec { + EffectSpec { id, name, category: "Blur & Sharpen", params, render, gpu: false, float: true } +} + +// ---------------------------------------------------------------- CC Radial Blur + +pub const RADIAL_TYPES: [&str; 5] = ["Straight Zoom", "Fading Zoom", "Centered", "Rotate", "Rotate Fading"]; + +fn cc_radial_blur(ctx: &EffectCtx, mut b: Buf) -> Buf { + let amount = ctx.params.f("amount"); + if amount == 0.0 { + return b; + } + let kind = ctx.params.e("type"); + let quality = ctx.params.f("quality").clamp(1.0, 100.0); + let c = b.to_px(ctx.params.v2("center")); + let src = b.img.clone(); + // Zoom: fraction of the distance to the centre; rotate: degrees of arc. + let zoom = (amount / 100.0).clamp(-1.0, 1.0) * 0.5; + let arc = amount.to_radians() * 0.25; + b.img.rows_mut().for_each(|(y, row)| { + for (x, o) in row.iter_mut().enumerate() { + let (dx, dy) = (x as f64 + 0.5 - c.0, y as f64 + 0.5 - c.1); + let r = (dx * dx + dy * dy).sqrt(); + let span = if kind >= 3 { r * arc.abs() } else { r * zoom.abs() }; + let n = ((span * quality / 50.0).ceil() as usize).clamp(1, 256); + if n <= 1 { + *o = src.get(x as i64, y as i64); + continue; + } + let mut acc = [0.0f32; 4]; + let mut wsum = 0.0f32; + for i in 0..n { + let t = i as f64 / (n - 1) as f64; + let (sx, sy, w) = match kind { + // Toward the centre, equal weights. + 0 => { + let k = 1.0 - zoom * t; + (c.0 + dx * k, c.1 + dy * k, 1.0) + } + // Toward the centre, weights fading with distance along the streak. + 1 => { + let k = 1.0 - zoom * t; + (c.0 + dx * k, c.1 + dy * k, (1.0 - t) as f32 + 0.05) + } + // Both ways around the pixel. + 2 => { + let k = 1.0 + zoom * (t - 0.5); + (c.0 + dx * k, c.1 + dy * k, 1.0) + } + _ => { + let a = arc * (t - 0.5); + let (s, co) = a.sin_cos(); + let w = if kind == 4 { (1.0 - (2.0 * t - 1.0).abs()) as f32 + 0.05 } else { 1.0 }; + (c.0 + dx * co - dy * s, c.1 + dx * s + dy * co, w) + } + }; + let q = src.sample_bilinear(sx, sy); + for k in 0..4 { + acc[k] += q[k] * w; + } + wsum += w; + } + *o = acc.map(|v| v / wsum); + } + }); + b +} + +// ---------------------------------------------------------------- Camera-Shake Deblur + +/// Mean squared luma gradient over a rectangle (sharpness). +fn sharpness(l: &Plane, x0: usize, y0: usize, x1: usize, y1: usize) -> f64 { + let mut s = 0.0f64; + let mut n = 0usize; + for y in y0..y1.saturating_sub(1) { + for x in x0..x1.saturating_sub(1) { + let v = l.get(x, y); + let gx = l.get(x + 1, y) - v; + let gy = l.get(x, y + 1) - v; + s += (gx * gx + gy * gy) as f64; + n += 1; + } + } + if n == 0 { 0.0 } else { s / n as f64 } +} + +/// Best integer translation (dx, dy) within ±`r` aligning `b` to `a` on a patch (SSD). +fn align(a: &Plane, bp: &Plane, x0: usize, y0: usize, x1: usize, y1: usize, r: i64) -> (i64, i64) { + let mut best = (f64::INFINITY, 0i64, 0i64); + let step = ((x1 - x0).max(y1 - y0) / 24).max(1); + for dy in -r..=r { + for dx in -r..=r { + let mut s = 0.0f64; + for y in (y0..y1).step_by(step) { + for x in (x0..x1).step_by(step) { + let d = a.get(x, y) - bp.get_clamped(x as i64 + dx, y as i64 + dy); + s += (d * d) as f64; + } + } + // Prefer the smallest shift on ties. + let s = s * (1.0 + 1e-9 * (dx * dx + dy * dy) as f64); + if s < best.0 { + best = (s, dx, dy); + } + } + } + (best.1, best.2) +} + +fn camera_shake_deblur(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(host) = ctx.env.host else { return b }; + let dur = ctx.params.f("blurDuration").clamp(1.0, 10.0).round() as i64; + let search = (ctx.params.f("searchRange").max(0.0) * b.scale).round() as i64; + let patch = (ctx.params.f("patchSize").max(8.0) * b.scale).round().max(4.0) as usize; + let thresh = 1.0 + ctx.params.f("threshold").max(0.0) / 100.0; + let show = ctx.params.b("showBlurryFrames"); + let fps = ctx.fps(); + let (w, h) = (b.img.width as usize, b.img.height as usize); + if w == 0 || h == 0 { + return b; + } + // Neighbouring frames in the effect's own input, resampled into this buffer's grid. + let mut neigh: Vec<(Image, Plane)> = vec![]; + for k in (-dur..=dur).filter(|k| *k != 0) { + let Some(nb) = host.self_at(ctx.time + k as f64 / fps, ctx.env.effect_index) else { continue }; + let img = if nb.img.width == b.img.width && nb.img.height == b.img.height && nb.offset == b.offset && nb.scale == b.scale { + nb.img + } else { + let k = nb.scale / b.scale.max(1e-9); + crate::util::gen_image(b.img.width, b.img.height, |x, y| { + let lx = (x as f64 + 0.5 - b.offset[0]) * k + nb.offset[0]; + let ly = (y as f64 + 0.5 - b.offset[1]) * k + nb.offset[1]; + nb.img.sample_bilinear(lx, ly) + }) + }; + let l = Plane::luma(&img); + neigh.push((img, l)); + } + if neigh.is_empty() { + return b; + } + let cur_l = Plane::luma(&b.img); + let (nx, ny) = (w.div_ceil(patch), h.div_ceil(patch)); + // Per patch: (neighbour index, shift) of the substitute, or None to keep. + let choice: Vec> = (0..nx * ny) + .into_par_iter() + .map(|i| { + let (px, py) = (i % nx, i / nx); + let (x0, y0) = (px * patch, py * patch); + let (x1, y1) = ((x0 + patch).min(w), (y0 + patch).min(h)); + let s0 = sharpness(&cur_l, x0, y0, x1, y1); + let mut best: Option<(f64, usize)> = None; + for (k, (_, l)) in neigh.iter().enumerate() { + let s = sharpness(l, x0, y0, x1, y1); + if s > s0 * thresh && best.is_none_or(|(bs, _)| s > bs) { + best = Some((s, k)); + } + } + let (_, k) = best?; + let (dx, dy) = align(&cur_l, &neigh[k].1, x0, y0, x1, y1, search); + Some((k, dx, dy)) + }) + .collect(); + if choice.iter().all(Option::is_none) { + return b; + } + let src = b.img.clone(); + // Feathered blending between patch decisions: each pixel mixes the four nearest patch + // centres bilinearly. + let pick = |i: usize, x: usize, y: usize| -> Px { + match choice[i] { + None => src.data[y * w + x], + Some((k, dx, dy)) => neigh[k].0.get_clamped(x as i64 + dx, y as i64 + dy), + } + }; + b.img.rows_mut().for_each(|(y, row)| { + let fy = ((y as f64 + 0.5) / patch as f64 - 0.5).clamp(0.0, (ny - 1) as f64); + let (y0, ty) = (fy.floor() as usize, (fy - fy.floor()) as f32); + let y1 = (y0 + 1).min(ny - 1); + for (x, o) in row.iter_mut().enumerate() { + let fx = ((x as f64 + 0.5) / patch as f64 - 0.5).clamp(0.0, (nx - 1) as f64); + let (x0, tx) = (fx.floor() as usize, (fx - fx.floor()) as f32); + let x1 = (x0 + 1).min(nx - 1); + let ids = [y0 * nx + x0, y0 * nx + x1, y1 * nx + x0, y1 * nx + x1]; + if ids.iter().all(|&i| choice[i].is_none()) { + continue; + } + let ws = [(1.0 - tx) * (1.0 - ty), tx * (1.0 - ty), (1.0 - tx) * ty, tx * ty]; + let mut acc = [0.0f32; 4]; + for (i, wgt) in ids.iter().zip(ws) { + let q = pick(*i, x, y); + for c in 0..4 { + acc[c] += q[c] * wgt; + } + } + if show { + acc[0] = (acc[0] + 0.3 * acc[3]).min(acc[3].max(acc[0])); + } + *o = acc; + } + }); + b +} + +pub fn specs() -> Vec { + vec![ + spec( + "ec.blur.ccradial", + "CC Radial Blur", + vec![ + p("type", "Type", Value::Enum(1), popup(&RADIAL_TYPES)), + p("amount", "Amount", num(10.0), slider(-100.0, 100.0, -100.0, 100.0, 1)), + p("quality", "Quality", num(50.0), slider(1.0, 100.0, 1.0, 100.0, 1)), + p("center", "Center", Value::Vec2([0.5, 0.5]), ParamUi::Point), + ], + cc_radial_blur, + ), + spec( + "ec.blur.camerashakedeblur", + "Camera-Shake Deblur", + vec![ + p("blurDuration", "Blur Duration", num(2.0), slider(1.0, 10.0, 1.0, 10.0, 0)), + p("searchRange", "Search Range", num(8.0), slider(0.0, 64.0, 0.0, 32.0, 0)), + p("patchSize", "Patch Size", num(32.0), slider(8.0, 256.0, 8.0, 128.0, 0)), + p("threshold", "Threshold", num(20.0), slider(0.0, 500.0, 0.0, 100.0, 1)), + p("showBlurryFrames", "Show Blurry Frames", Value::Bool(false), ParamUi::Checkbox), + ], + camera_shake_deblur, + ), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, EffectHost, LayerPixels, run_fx}; + + fn checker(w: u32, h: u32, cell: u32) -> Image { + crate::util::gen_image(w, h, |x, y| { + let v = if ((x as u32 / cell) + (y as u32 / cell)).is_multiple_of(2) { 1.0 } else { 0.0 }; + [v, v, v, 1.0] + }) + } + + #[test] + fn cc_radial_blur_keeps_centre_and_smears_edges() { + let img = checker(64, 64, 4); + for kind in 0..5u32 { + let out = run_fx( + "ec.blur.ccradial", + &[("type", Value::Enum(kind)), ("amount", num(40.0)), ("center", Value::Vec2([32.0, 32.0]))], + img.clone(), + 0.0, + EffectEnv::default(), + ); + // Far from the centre the checker is smeared toward grey. + let edge = out.img.get(2, 2)[0]; + assert!(edge > 0.05 && edge < 0.95, "type {kind}: {edge}"); + let again = run_fx( + "ec.blur.ccradial", + &[("type", Value::Enum(kind)), ("amount", num(40.0)), ("center", Value::Vec2([32.0, 32.0]))], + img.clone(), + 0.0, + EffectEnv::default(), + ); + assert_eq!(out.img.data, again.img.data); + } + // Zero amount is identity; a constant image stays constant. + let flat = Image::filled(16, 16, [0.4, 0.4, 0.4, 1.0]); + let o = run_fx("ec.blur.ccradial", &[("amount", num(80.0))], flat.clone(), 0.0, EffectEnv::default()); + assert!(o.img.data.iter().all(|p| (p[0] - 0.4).abs() < 1e-5)); + let o = run_fx("ec.blur.ccradial", &[("amount", num(0.0))], img.clone(), 0.0, EffectEnv::default()); + assert_eq!(o.img.data, img.data); + } + + /// Frames: sharp checker at even frames, a blurred one at odd frames (shifted by 2 px). + struct Shaky; + impl EffectHost for Shaky { + fn layer(&self, _: u64, _: bool) -> Option { + None + } + fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option> { + None + } + fn self_at(&self, t: f64, _: usize) -> Option { + let f = (t * 30.0).round() as i64; + Some(Buf { img: frame(f), offset: [0.0; 2], scale: 1.0 }) + } + } + + fn frame(f: i64) -> Image { + let sharp = crate::util::gen_image(48, 48, |x, y| { + let v = if (((x + 2) / 6) + (y / 6)) % 2 == 0 { 1.0 } else { 0.0 }; + [v, v, v, 1.0] + }); + if f % 2 == 0 { sharp } else { effectcraft_raster::gaussian_blur(&sharp, 2.5, 2.5, true) } + } + + #[test] + fn camera_shake_deblur_substitutes_sharp_neighbours() { + let host = Shaky; + let env = EffectEnv { host: Some(&host), frame_rate: 30.0, ..Default::default() }; + let blurry = frame(1); + let out = run_fx("ec.blur.camerashakedeblur", &[("patchSize", num(16.0)), ("searchRange", num(3.0))], blurry.clone(), 1.0 / 30.0, env); + let sharp = frame(0); + let err = |a: &Image| a.data.iter().zip(&sharp.data).map(|(p, q)| (p[0] - q[0]).abs()).sum::() / a.data.len() as f32; + assert!(err(&out.img) < err(&blurry) * 0.5, "{} vs {}", err(&out.img), err(&blurry)); + // A sharp frame is left alone (its neighbours are blurrier). + let keep = run_fx("ec.blur.camerashakedeblur", &[], sharp.clone(), 0.0, env); + assert_eq!(keep.img.data, sharp.data); + // Deterministic; no host = pass-through. + let again = run_fx("ec.blur.camerashakedeblur", &[("patchSize", num(16.0)), ("searchRange", num(3.0))], blurry.clone(), 1.0 / 30.0, env); + assert_eq!(out.img.data, again.img.data); + let none = run_fx("ec.blur.camerashakedeblur", &[], blurry.clone(), 0.0, EffectEnv::default()); + assert_eq!(none.img.data, blurry.data); + } +} diff --git a/crates/effects/src/channel3d.rs b/crates/effects/src/channel3d.rs new file mode 100644 index 00000000..5fd4d715 --- /dev/null +++ b/crates/effects/src/channel3d.rs @@ -0,0 +1,681 @@ +//! 3D Channel effects: 3D Channel Extract, Cryptomatte, Depth Matte, Depth of Field, EXtractoR, +//! Fog 3D, ID Matte and IDentifier. +//! +//! They read the layer's auxiliary channels ([`AuxChannels`], via +//! [`crate::EffectHost::aux`]): multi-layer OpenEXR footage (depth, object/material IDs, +//! Cryptomatte layers, any named channel) or the compositor's depth / layer-ID / Cryptomatte +//! pass for a precomp of a 3D comp. Without auxiliary channels they pass the layer through +//! (EXtractoR still maps the layer's own R, G, B and A). +//! +//! Aux pixels are looked up nearest-neighbour (IDs must not be interpolated) at each buffer +//! pixel's layer position, so render scale and layer padding are respected. + +use std::sync::Arc; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::channels3d::{BACKGROUND_DEPTH, crypto_hash}; +use effectcraft_raster::{AuxChannels, Image, Px, gaussian_blur}; +use rayon::prelude::*; + +use crate::util::{Plane, gauss_plane, hash1, premul, unpremul}; +use crate::{Buf, EffectCtx, EffectSpec, col, num, p, popup, slider}; + +fn spec(id: &'static str, name: &'static str, params: Vec, render: crate::RenderFn) -> EffectSpec { + EffectSpec { id, name, category: "3D Channel", params, render, gpu: false, float: true } +} + +fn aux(ctx: &EffectCtx) -> Option> { + ctx.env.host?.aux() +} + +/// Aux index for each buffer pixel (None outside the aux image). +fn index_map(b: &Buf, a: &AuxChannels) -> Vec> { + let (w, h) = (b.img.width as usize, b.img.height as usize); + let inv = 1.0 / b.scale.max(1e-9); + (0..w * h) + .into_par_iter() + .map(|i| { + let (x, y) = ((i % w) as f64 + 0.5, (i / w) as f64 + 0.5); + a.index_at((x - b.offset[0]) * inv, (y - b.offset[1]) * inv) + }) + .collect() +} + +/// A channel sampled into the buffer grid (`fill` outside). +fn plane_of(b: &Buf, idx: &[Option], ch: &[f32], fill: f32) -> Plane { + Plane { w: b.img.width as usize, h: b.img.height as usize, data: idx.par_iter().map(|i| i.map_or(fill, |i| ch[i])).collect() } +} + +/// Distinct colour for an ID (hash → hue). +fn id_color(id: f32) -> [f32; 3] { + let h = hash1(id.to_bits(), 0x1d, 0x5eed); + let (r, g, b) = effectcraft_color::hsl_to_rgb(h, 0.75, 0.5); + if id == 0.0 { [0.0; 3] } else { [r, g, b] } +} + +// ---------------------------------------------------------------- 3D Channel Extract + +pub const EXTRACT_CHANNELS: [&str; 8] = ["Z-Depth", "Object ID", "Texture UV", "Surface Normals", "Coverage", "Background RGB", "Unclamped RGB", "Material ID"]; + +fn channel_extract(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(a) = aux(ctx) else { return b }; + let idx = index_map(&b, &a); + let (bp, wp) = (ctx.params.f("blackPoint") as f32, ctx.params.f("whitePoint") as f32); + let lv = |v: f32| ((v - bp) / (wp - bp).abs().max(1e-9) * (wp - bp).signum()).clamp(0.0, 1.0); + let lv = |v: f32| if wp >= bp { lv(v) } else { 1.0 - ((v - wp) / (bp - wp).max(1e-9)).clamp(0.0, 1.0) }; + let get3 = |names: [&[&str]; 3]| -> Option<[&[f32]; 3]> { Some([a.find(names[0])?, a.find(names[1])?, a.find(names[2])?]) }; + let img: Option = match ctx.params.e("channel") { + 0 => a.depth().map(|z| map_grey(&b, &idx, |i| lv(z[i]))), + 1 => a.object_id().map(|c| map_grey(&b, &idx, |i| lv(c[i]))), + 7 => a.material_id().map(|c| map_grey(&b, &idx, |i| lv(c[i]))), + 4 => a.coverage().map(|c| map_grey(&b, &idx, |i| lv(c[i]))), + 2 => get3([&["UV.U", "U", "uv.x"], &["UV.V", "V", "uv.y"], &["UV.W", "W", "uv.z"]]) + .or_else(|| Some([a.find(&["UV.U", "U", "uv.x"])?, a.find(&["UV.V", "V", "uv.y"])?, a.find(&["UV.V", "V", "uv.y"])?])) + .map(|c| map_rgb(&b, &idx, |i| [lv(c[0][i]), lv(c[1][i]), 0.0])), + 3 => get3([&["N.X", "Normal.X", "normals.x"], &["N.Y", "Normal.Y", "normals.y"], &["N.Z", "Normal.Z", "normals.z"]]) + .map(|c| map_rgb(&b, &idx, |i| [lv(c[0][i] * 0.5 + 0.5), lv(c[1][i] * 0.5 + 0.5), lv(c[2][i] * 0.5 + 0.5)])), + 5 => get3([&["BG.R", "Background.R"], &["BG.G", "Background.G"], &["BG.B", "Background.B"]]) + .map(|c| map_rgb(&b, &idx, |i| [lv(c[0][i]), lv(c[1][i]), lv(c[2][i])])), + _ => get3([&["R"], &["G"], &["B"]]).map(|c| map_rgb(&b, &idx, |i| [c[0][i], c[1][i], c[2][i]])), + }; + if let Some(i) = img { + b.img = i; + } + b +} + +fn map_grey(b: &Buf, idx: &[Option], f: impl Fn(usize) -> f32 + Sync) -> Image { + map_rgb(b, idx, |i| { + let v = f(i); + [v, v, v] + }) +} + +fn map_rgb(b: &Buf, idx: &[Option], f: impl Fn(usize) -> [f32; 3] + Sync) -> Image { + let mut img = Image::new(b.img.width, b.img.height); + img.data.par_iter_mut().zip(idx.par_iter()).for_each(|(p, i)| { + if let Some(i) = i { + let c = f(*i); + *p = [c[0], c[1], c[2], 1.0]; + } + }); + img +} + +// ---------------------------------------------------------------- Depth Matte + +fn depth_matte(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(a) = aux(ctx) else { return b }; + let Some(z) = a.depth() else { return b }; + let idx = index_map(&b, &a); + let depth = ctx.params.f("depth") as f32; + let feather = ctx.params.f("feather").max(0.0) as f32; + let invert = ctx.params.b("invert"); + // Keeps what lies at or beyond Depth (Invert keeps what is in front), with a soft ramp of + // `feather` depth units centred on the threshold. + let zp = plane_of(&b, &idx, z, BACKGROUND_DEPTH); + b.img.data.par_iter_mut().zip(zp.data.par_iter()).for_each(|(px, &zv)| { + let k = if feather > 0.0 { + ((zv - depth) / feather + 0.5).clamp(0.0, 1.0) + } else if zv >= depth { + 1.0 + } else { + 0.0 + }; + let k = if invert { 1.0 - k } else { k }; + *px = px.map(|v| v * k); + }); + b +} + +// ---------------------------------------------------------------- Depth of Field + +/// Blur radius (pixels) for depth `z`. +pub fn dof_radius(z: f32, focal: f32, thickness: f32, max_r: f32, bias: f32) -> f32 { + let d = ((z - focal).abs() - thickness * 0.5).max(0.0); + let range = focal.abs().max(1.0); + let n = (d / range).min(1.0); + // Bias 50 = linear; higher bias brings blur in sooner. + let gamma = 2f32.powf((50.0 - bias) / 25.0); + max_r * n.powf(gamma) +} + +fn depth_of_field(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(a) = aux(ctx) else { return b }; + let Some(z) = a.depth() else { return b }; + let max_r = (ctx.params.f("maximumRadius").max(0.0) * b.scale) as f32; + if max_r <= 0.0 { + return b; + } + let focal = ctx.params.f("focalPlane") as f32; + let thick = ctx.params.f("focalPlaneThickness").max(0.0) as f32; + let bias = ctx.params.f("focalBias") as f32; + let idx = index_map(&b, &a); + let zp = plane_of(&b, &idx, z, BACKGROUND_DEPTH); + // A stack of blurs at a few radii, interpolated per pixel by its circle of confusion. + const LEVELS: usize = 5; + let stack: Vec = (0..LEVELS) + .into_par_iter() + .map(|k| { + let r = max_r * k as f32 / (LEVELS - 1) as f32; + if r < 0.3 { b.img.clone() } else { gaussian_blur(&b.img, (r / 2.0) as f64, (r / 2.0) as f64, true) } + }) + .collect(); + b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| { + let r = dof_radius(zp.data[i], focal, thick, max_r, bias); + let f = (r / max_r * (LEVELS - 1) as f32).clamp(0.0, (LEVELS - 1) as f32); + let k0 = (f.floor() as usize).min(LEVELS - 2); + let t = f - k0 as f32; + let (a0, a1) = (stack[k0].data[i], stack[k0 + 1].data[i]); + *px = [0, 1, 2, 3].map(|c| a0[c] + (a1[c] - a0[c]) * t); + }); + b +} + +// ---------------------------------------------------------------- Fog 3D + +fn fog_3d(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(a) = aux(ctx) else { return b }; + let Some(z) = a.depth() else { return b }; + let pr = ctx.params; + let fc = pr.color("fogColor"); + let (start, end) = (pr.f("fogStartDepth") as f32, pr.f("fogEndDepth") as f32); + let opacity = (pr.f("fogOpacity") / 100.0) as f32; + let density = (pr.f("scatteringDensity") / 100.0) as f32; + let foggy_bg = pr.b("foggyBackground"); + let contrib = (pr.f("layerContribution") / 100.0) as f32; + let grad = if contrib > 0.0 { ctx.layer_param("gradientLayer", true).map(|o| crate::util::fit_layer(ctx, &b, &o, true)) } else { None }; + let idx = index_map(&b, &a); + let zp = plane_of(&b, &idx, z, BACKGROUND_DEPTH); + b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| { + let zv = zp.data[i]; + let bg = zv >= BACKGROUND_DEPTH * 0.5; + let mut f = if bg { + if foggy_bg { 1.0 } else { 0.0 } + } else { + let t = ((zv - start) / (end - start).abs().max(1e-6)).clamp(0.0, 1.0); + // Denser scattering thickens the fog sooner. + 1.0 - (1.0 - t).powf(1.0 + density * 3.0) + }; + if let Some(g) = &grad { + let (gc, ga) = unpremul(g.data[i]); + let l = effectcraft_color::luminance(gc[0], gc[1], gc[2]) * ga; + f *= 1.0 - contrib + contrib * l; + } + let f = (f * opacity).clamp(0.0, 1.0); + let (c, al) = unpremul(*px); + let al2 = if bg && foggy_bg { al + (1.0 - al) * f } else { al }; + let out = [0, 1, 2].map(|k| c[k] + (fc[k] - c[k]) * f); + *px = premul(out, al2); + }); + b +} + +// ---------------------------------------------------------------- ID Matte / IDentifier + +fn id_channel(a: &AuxChannels, which: u32) -> Option<&[f32]> { + if which == 1 { a.material_id() } else { a.object_id() } +} + +fn id_matte(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(a) = aux(ctx) else { return b }; + let Some(ids) = id_channel(&a, ctx.params.e("auxChannel")) else { return b }; + let idx = index_map(&b, &a); + let sel = ctx.params.f("idSelection").round() as f32; + let cov = if ctx.params.b("useCoverage") { a.coverage() } else { None }; + let mut m = Plane { + w: b.img.width as usize, + h: b.img.height as usize, + data: idx.par_iter().map(|i| i.map_or(0.0, |i| if ids[i].round() == sel { cov.map_or(1.0, |c| c[i].clamp(0.0, 1.0)) } else { 0.0 })).collect(), + }; + let feather = ctx.params.f("feather").max(0.0) * b.scale; + if feather > 0.0 { + m = gauss_plane(&m, feather * 0.5, feather * 0.5); + } + let invert = ctx.params.b("invert"); + b.img.data.par_iter_mut().zip(m.data.par_iter()).for_each(|(px, &k)| { + let k = if invert { 1.0 - k } else { k }; + *px = px.map(|v| v * k); + }); + b +} + +fn identifier(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(a) = aux(ctx) else { return b }; + let Some(ids) = id_channel(&a, ctx.params.e("channelType")) else { return b }; + let idx = index_map(&b, &a); + let sel = ctx.params.f("id").round() as f32; + let display = ctx.params.e("display"); + let max_id = ids.iter().cloned().fold(1.0f32, f32::max); + let src = b.img.clone(); + b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| { + let Some(k) = idx[i] else { + if display != 2 { + *px = [0.0; 4]; + } + return; + }; + let id = ids[k].round(); + let hit = id == sel; + *px = match display { + // Colors: every ID in its own colour, the selected one highlighted white. + 0 => { + let c = if hit && sel != 0.0 { [1.0; 3] } else { id_color(id) }; + [c[0], c[1], c[2], 1.0] + } + 1 => { + let v = if hit { 1.0 } else { 0.0 }; + [v, v, v, 1.0] + } + 2 => { + let s = src.data[i]; + if hit { s } else { [0.0; 4] } + } + _ => { + let v = id / max_id; + [v, v, v, 1.0] + } + }; + }); + b +} + +// ---------------------------------------------------------------- Cryptomatte + +/// Does `name` match a selection pattern (`*` wildcards, case-sensitive like the spec)? +fn wild(pat: &str, name: &str) -> bool { + match pat.split_once('*') { + None => pat == name, + Some((pre, rest)) => { + let Some(tail) = name.strip_prefix(pre) else { return false }; + if rest.is_empty() { + return true; + } + (0..=tail.len()).filter(|&k| tail.is_char_boundary(k)).any(|k| wild(rest, &tail[k..])) + } + } +} + +/// Parse a selection: comma-separated names (optionally quoted) or `` hashes / wildcards. +pub fn selection_hashes(sel: &str, manifest: Option<&[(String, u32)]>) -> Vec { + let mut out = vec![]; + for tok in sel.split(',') { + let t = tok.trim().trim_matches('"').trim_matches('\''); + if t.is_empty() { + continue; + } + if t.contains('*') { + if let Some(m) = manifest { + out.extend(m.iter().filter(|(n, _)| wild(t, n)).map(|(_, h)| *h)); + } + continue; + } + if let Some(hex) = t.strip_prefix("<").and_then(|r| r.strip_suffix(">")) + && let Ok(h) = u32::from_str_radix(hex, 16) + { + out.push(h); + continue; + } + out.push(manifest.and_then(|m| m.iter().find(|(n, _)| n == t).map(|(_, h)| *h)).unwrap_or_else(|| crypto_hash(t))); + } + out +} + +pub const CRYPTO_LAYERS: [&str; 3] = ["CryptoObject", "CryptoMaterial", "CryptoAsset"]; + +fn cryptomatte(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(a) = aux(ctx) else { return b }; + let layers = a.crypto_layers(); + if layers.is_empty() { + return b; + } + let want = CRYPTO_LAYERS[(ctx.params.e("layer") as usize).min(2)]; + let layer = layers.iter().find(|l| l.as_str() == want || l.ends_with(want)).unwrap_or(&layers[0]).clone(); + let manifest = a.manifests.iter().find(|(l, _)| *l == layer).map(|(_, m)| m.as_slice()); + let sel = selection_hashes(ctx.params.s("selection"), manifest); + let idx = index_map(&b, &a); + let display = ctx.params.e("display"); + let matte_only = ctx.params.b("matteOnly"); + let src = b.img.clone(); + b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| { + let ranks = idx[i].map(|k| a.crypto_ranks(&layer, k)).unwrap_or_default(); + let m: f32 = ranks.iter().filter(|(id, c)| *c > 0.0 && sel.contains(&id.to_bits())).map(|(_, c)| *c).sum::().clamp(0.0, 1.0); + if matte_only { + *px = [m, m, m, m]; + return; + } + *px = match display { + // Colors: coverage-weighted ID colours (the spec's preview), selection brighter. + 0 => { + let mut c = [0.0f32; 3]; + for (id, cov) in &ranks { + let k = id_color(*id); + for j in 0..3 { + c[j] += k[j] * cov; + } + } + let c = c.map(|v| v * (1.0 - 0.5 * m) + 0.5 * m); + [c[0], c[1], c[2], 1.0] + } + // Matte: the layer keyed by the selection. + 1 => src.data[i].map(|v| v * m), + // Preview: the layer with the selection tinted. + _ => { + let s = src.data[i]; + [s[0] * (1.0 - 0.5 * m) + 0.5 * m * s[3], s[1] * (1.0 - 0.5 * m) + 0.5 * m * s[3], s[2] * (1.0 - 0.5 * m), s[3]] + } + }; + }); + b +} + +// ---------------------------------------------------------------- EXtractoR + +fn extractor(ctx: &EffectCtx, mut b: Buf) -> Buf { + let a = aux(ctx); + let names = [ctx.params.s("red"), ctx.params.s("green"), ctx.params.s("blue"), ctx.params.s("alpha")]; + let (bp, wp) = (ctx.params.f("blackPoint") as f32, ctx.params.f("whitePoint") as f32); + let unmult = ctx.params.b("unMult"); + let clip = ctx.params.b("clip"); + let idx = a.as_ref().map(|a| index_map(&b, a)); + // Each output channel: an aux channel by name, or the layer's own R/G/B/A. + let own = |n: &str| match n.trim().to_ascii_uppercase().as_str() { + "R" | "RED" => Some(0), + "G" | "GREEN" => Some(1), + "B" | "BLUE" => Some(2), + "A" | "ALPHA" => Some(3), + "" => None, + _ => None, + }; + let src_ch: Vec> = names.iter().map(|n| a.as_ref().and_then(|a| if n.trim().is_empty() { None } else { a.get(n.trim()) })).collect(); + if src_ch.iter().all(Option::is_none) && bp == 0.0 && wp == 1.0 && !unmult && names.iter().enumerate().all(|(k, n)| own(n) == Some(k)) { + return b; + } + let src = b.img.clone(); + let lv = |v: f32| { + let o = (v - bp) / (wp - bp).abs().max(1e-9) * (wp - bp).signum(); + if clip { o.clamp(0.0, 1.0) } else { o } + }; + b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| { + let (sc, sa) = unpremul(src.data[i]); + let straight = [sc[0], sc[1], sc[2], sa]; + let mut o: Px = [0.0; 4]; + for k in 0..4 { + o[k] = match (src_ch[k], idx.as_ref().and_then(|m| m[i])) { + (Some(ch), Some(j)) => ch[j], + (Some(_), None) => 0.0, + (None, _) => match own(names[k]) { + Some(c) => straight[c], + None => { + if k == 3 { + 1.0 + } else { + 0.0 + } + } + }, + }; + if k < 3 { + o[k] = lv(o[k]); + } + } + let al = o[3].clamp(0.0, 1.0); + let c = if unmult && al > 1e-6 { [o[0] / al, o[1] / al, o[2] / al] } else { [o[0], o[1], o[2]] }; + *px = premul(c, al); + }); + b +} + +pub fn specs() -> Vec { + let text = |id: &'static str, name: &'static str, d: &str| p(id, name, Value::Str(d.into()), ParamUi::Text); + vec![ + spec( + "ec.3d.channelextract", + "3D Channel Extract", + vec![ + p("channel", "3D Channel", Value::Enum(0), popup(&EXTRACT_CHANNELS)), + p("blackPoint", "Black Point", num(0.0), slider(-1.0e6, 1.0e6, -10000.0, 10000.0, 2)), + p("whitePoint", "White Point", num(10000.0), slider(-1.0e6, 1.0e6, -10000.0, 10000.0, 2)), + ], + channel_extract, + ), + spec( + "ec.3d.cryptomatte", + "Cryptomatte", + vec![ + p("layer", "Layer", Value::Enum(0), popup(&CRYPTO_LAYERS)), + text("selection", "Selection", ""), + p("display", "Display", Value::Enum(2), popup(&["Colors", "Matte", "Preview"])), + p("matteOnly", "Matte Only", Value::Bool(false), ParamUi::Checkbox), + ], + cryptomatte, + ), + spec( + "ec.3d.depthmatte", + "Depth Matte", + vec![ + p("depth", "Depth", num(0.0), slider(-1.0e6, 1.0e6, -10000.0, 10000.0, 2)), + p("feather", "Feather", num(0.0), slider(0.0, 1.0e6, 0.0, 1000.0, 2)), + p("invert", "Invert", Value::Bool(false), ParamUi::Checkbox), + ], + depth_matte, + ), + spec( + "ec.3d.depthoffield", + "Depth of Field", + vec![ + p("focalPlane", "Focal Plane", num(0.0), slider(-1.0e6, 1.0e6, 0.0, 10000.0, 2)), + p("maximumRadius", "Maximum Radius", num(0.0), slider(0.0, 500.0, 0.0, 50.0, 2)), + p("focalPlaneThickness", "Focal Plane Thickness", num(0.0), slider(0.0, 1.0e6, 0.0, 1000.0, 2)), + p("focalBias", "Focal Bias", num(50.0), slider(0.0, 100.0, 0.0, 100.0, 2)), + ], + depth_of_field, + ), + spec( + "ec.3d.extractor", + "EXtractoR", + vec![ + text("red", "Red", "R"), + text("green", "Green", "G"), + text("blue", "Blue", "B"), + text("alpha", "Alpha", "A"), + p("blackPoint", "Black Point", num(0.0), slider(-1.0e6, 1.0e6, -1.0, 2.0, 3)), + p("whitePoint", "White Point", num(1.0), slider(-1.0e6, 1.0e6, -1.0, 2.0, 3)), + p("unMult", "UnMult", Value::Bool(false), ParamUi::Checkbox), + p("clip", "Clip", Value::Bool(false), ParamUi::Checkbox), + ], + extractor, + ), + spec( + "ec.3d.fog3d", + "Fog 3D", + vec![ + p("fogColor", "Fog Color", col(1.0, 1.0, 1.0), ParamUi::Color), + p("fogStartDepth", "Fog Start Depth", num(0.0), slider(-1.0e6, 1.0e6, 0.0, 10000.0, 2)), + p("fogEndDepth", "Fog End Depth", num(3000.0), slider(-1.0e6, 1.0e6, 0.0, 10000.0, 2)), + p("fogOpacity", "Fog Opacity", num(100.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("scatteringDensity", "Scattering Density", num(50.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("foggyBackground", "Foggy Background", Value::Bool(true), ParamUi::Checkbox), + p("gradientLayer", "Gradient Layer", Value::Layer(None), ParamUi::Layer), + p("layerContribution", "Layer Contribution", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + ], + fog_3d, + ), + spec( + "ec.3d.idmatte", + "ID Matte", + vec![ + p("auxChannel", "Aux. Channel", Value::Enum(0), popup(&["Object ID", "Material ID"])), + p("idSelection", "ID Selection", num(0.0), slider(0.0, 65535.0, 0.0, 100.0, 0)), + p("feather", "Feather", num(0.0), slider(0.0, 100.0, 0.0, 10.0, 2)), + p("invert", "Invert", Value::Bool(false), ParamUi::Checkbox), + p("useCoverage", "Use Coverage", Value::Bool(false), ParamUi::Checkbox), + ], + id_matte, + ), + spec( + "ec.3d.identifier", + "IDentifier", + vec![ + p("channelType", "Channel Type", Value::Enum(0), popup(&["Object ID", "Material ID"])), + p("display", "Display", Value::Enum(0), popup(&["Colors", "Luma Matte", "Alpha Matte", "Raw"])), + p("id", "ID", num(0.0), slider(0.0, 65535.0, 0.0, 100.0, 0)), + ], + identifier, + ), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, EffectHost, LayerPixels, run_fx}; + use effectcraft_raster::channels3d::crypto_float; + + /// A 20×10 layer: left half object 1 at depth 100, right half object 2 at depth 500; the + /// top-right corner is background. Cryptomatte: "left" / "right". + struct AuxHost(Arc); + impl EffectHost for AuxHost { + fn layer(&self, _: u64, _: bool) -> Option { + None + } + fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option> { + None + } + fn aux(&self) -> Option> { + Some(self.0.clone()) + } + } + + fn scene() -> AuxHost { + let (w, h) = (20usize, 10usize); + let mut a = AuxChannels::new(w as u32, h as u32, 1.0); + let (mut z, mut id, mut r, mut g) = (vec![0.0; w * h], vec![0.0; w * h], vec![0.0; w * h], vec![0.0; w * h]); + let (hl, hr) = (crypto_hash("left"), crypto_hash("right")); + for y in 0..h { + for x in 0..w { + let i = y * w + x; + if x >= 15 && y < 3 { + z[i] = BACKGROUND_DEPTH; + } else if x < 10 { + z[i] = 100.0; + id[i] = 1.0; + r[i] = crypto_float(hl); + g[i] = 1.0; + } else { + z[i] = 500.0; + id[i] = 2.0; + r[i] = crypto_float(hr); + g[i] = 1.0; + } + } + } + a.channels = vec![("Z".into(), z), ("ObjectID".into(), id), ("CryptoObject00.R".into(), r), ("CryptoObject00.G".into(), g)]; + a.manifests = vec![("CryptoObject".into(), vec![("left".into(), hl), ("right".into(), hr)])]; + AuxHost(Arc::new(a)) + } + + fn img() -> Image { + Image::filled(20, 10, [0.5, 0.5, 0.5, 1.0]) + } + + fn run(id: &str, vals: &[(&str, Value)], host: &AuxHost) -> Image { + run_fx(id, vals, img(), 0.0, EffectEnv { host: Some(host), ..Default::default() }).img + } + + #[test] + fn passes_through_without_aux_channels() { + for s in specs() { + let out = run_fx(s.id, &[], img(), 0.0, EffectEnv::default()); + assert_eq!(out.img.data, img().data, "{}", s.id); + } + } + + #[test] + fn depth_matte_thresholds() { + let h = scene(); + let o = run("ec.3d.depthmatte", &[("depth", num(300.0))], &h); + assert_eq!(o.get(2, 5)[3], 0.0); + assert_eq!(o.get(12, 5)[3], 1.0); + let inv = run("ec.3d.depthmatte", &[("depth", num(300.0)), ("invert", Value::Bool(true))], &h); + assert_eq!(inv.get(2, 5)[3], 1.0); + assert_eq!(inv.get(12, 5)[3], 0.0); + // Feather: a ramp of 800 units centred on 300 → depth 100 at 0.25, 500 at 0.75. + let f = run("ec.3d.depthmatte", &[("depth", num(300.0)), ("feather", num(800.0))], &h); + assert!((f.get(2, 5)[3] - 0.25).abs() < 1e-5 && (f.get(12, 5)[3] - 0.75).abs() < 1e-5); + } + + #[test] + fn channel_extract_maps_depth_and_ids() { + let h = scene(); + let o = run("ec.3d.channelextract", &[("blackPoint", num(0.0)), ("whitePoint", num(1000.0))], &h); + assert!((o.get(2, 5)[0] - 0.1).abs() < 1e-6 && (o.get(12, 5)[0] - 0.5).abs() < 1e-6); + assert_eq!(o.get(17, 1)[0], 1.0); + let ids = run("ec.3d.channelextract", &[("channel", Value::Enum(1)), ("whitePoint", num(2.0))], &h); + assert_eq!(ids.get(2, 5)[0], 0.5); + } + + #[test] + fn id_matte_and_identifier() { + let h = scene(); + let o = run("ec.3d.idmatte", &[("idSelection", num(2.0))], &h); + assert_eq!(o.get(2, 5)[3], 0.0); + assert_eq!(o.get(12, 5)[3], 1.0); + let l = run("ec.3d.identifier", &[("display", Value::Enum(1)), ("id", num(1.0))], &h); + assert_eq!(l.get(2, 5)[0], 1.0); + assert_eq!(l.get(12, 5)[0], 0.0); + let c = run("ec.3d.identifier", &[], &h); + assert_ne!(c.get(2, 5), c.get(12, 5)); + } + + #[test] + fn cryptomatte_selects_by_name_and_wildcard() { + let h = scene(); + let m = run("ec.3d.cryptomatte", &[("selection", Value::Str("right".into())), ("matteOnly", Value::Bool(true))], &h); + assert_eq!(m.get(2, 5)[3], 0.0); + assert_eq!(m.get(12, 5)[3], 1.0); + let w = run("ec.3d.cryptomatte", &[("selection", Value::Str("l*".into())), ("display", Value::Enum(1))], &h); + assert_eq!(w.get(2, 5)[3], 1.0); + assert_eq!(w.get(12, 5)[3], 0.0); + assert!(wild("*ight", "right") && wild("r*t", "right") && !wild("x*", "right")); + let hex = format!("<{:08x}>", crypto_hash("left")); + assert_eq!(selection_hashes(&hex, None), vec![crypto_hash("left")]); + } + + #[test] + fn fog_and_depth_of_field_follow_depth() { + let h = scene(); + let f = run("ec.3d.fog3d", &[("fogColor", col(1.0, 0.0, 0.0)), ("fogEndDepth", num(1000.0)), ("scatteringDensity", num(0.0))], &h); + // Linear fog: 10 % at depth 100, 50 % at 500, full on the background. + assert!((f.get(2, 5)[0] - 0.55).abs() < 1e-4, "{:?}", f.get(2, 5)); + assert!((f.get(12, 5)[0] - 0.75).abs() < 1e-4); + assert!((f.get(17, 1)[0] - 1.0).abs() < 1e-4 && f.get(17, 1)[1] < 1e-4); + assert!((dof_radius(100.0, 100.0, 0.0, 10.0, 50.0)).abs() < 1e-6); + assert!((dof_radius(200.0, 100.0, 0.0, 10.0, 50.0) - 10.0).abs() < 1e-6); + assert_eq!(dof_radius(140.0, 100.0, 100.0, 10.0, 50.0), 0.0); + // A sharp edge at the far object blurs when focused near. + let mut pic = Image::new(20, 10); + for y in 0..10 { + for x in 0..20 { + let v = if x % 2 == 0 { 1.0 } else { 0.0 }; + pic.set(x, y, [v, v, v, 1.0]); + } + } + let env = EffectEnv { host: Some(&h), ..Default::default() }; + let d = run_fx("ec.3d.depthoffield", &[("focalPlane", num(100.0)), ("maximumRadius", num(4.0))], pic.clone(), 0.0, env).img; + assert_eq!(d.get(4, 5), pic.get(4, 5)); + assert!((d.get(12, 5)[0] - 0.5).abs() < 0.3 && d.get(12, 5) != pic.get(12, 5)); + } + + #[test] + fn extractor_maps_named_channels() { + let h = scene(); + let o = run("ec.3d.extractor", &[("red", Value::Str("Z".into())), ("whitePoint", num(1000.0))], &h); + assert!((o.get(2, 5)[0] - 0.1).abs() < 1e-6); + assert!((o.get(2, 5)[1] - 0.0005).abs() < 1e-6); + // Without aux: the layer's own channels can be swapped. + let s = run_fx("ec.3d.extractor", &[("red", Value::Str("A".into()))], Image::filled(2, 2, [0.2, 0.3, 0.4, 1.0]), 0.0, EffectEnv::default()); + assert_eq!(s.img.get(0, 0), [1.0, 0.3, 0.4, 1.0]); + } +} diff --git a/crates/effects/src/distort4.rs b/crates/effects/src/distort4.rs new file mode 100644 index 00000000..4e6b5040 --- /dev/null +++ b/crates/effects/src/distort4.rs @@ -0,0 +1,719 @@ +//! Distort: CC Flo Motion, Liquify and Rolling Shutter Repair. +//! +//! * **CC Flo Motion** — two knots that pull the image toward themselves (positive amount) or +//! push it away (negative), with a falloff exponent, optional tiling and supersampled +//! antialiasing. +//! * **Liquify** — brush strokes build a displacement mesh (a grid of layer-space offsets). The +//! strokes are the effect's data (the hidden `distortionMesh` text, one stroke per line, so it +//! animates with hold keyframes and is written by the `liquify.stroke` command); replaying +//! them is deterministic and cached. Tools: warp, turbulence, twirl clockwise / counter- +//! clockwise, pucker, bloat, shift pixels, reflection, clone, reconstruction and freeze / thaw. +//! Distortion Percentage scales the mesh and Distortion Mesh Offset moves it. +//! * **Rolling Shutter Repair** — every scanline of a CMOS frame is exposed a little later than +//! the previous one, so moving content shears. Motion is measured between the neighbouring +//! frames with pyramidal Lucas–Kanade (`effectcraft-track`) on a grid of points; each pixel is +//! then resampled from where it was at the frame's mid-exposure time: +//! `out(p) = in(p + v(p)·rate·(s(p) − ½))`, with `s` the scan position (0…1 along the scan +//! direction) and `v` the per-frame velocity (Warp: one smooth velocity per scanline band; +//! Pixel Motion: a dense interpolated field). + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::Image; +use rayon::prelude::*; + +use crate::util::{hash1, point_in_poly}; +use crate::{Buf, EffectCtx, EffectSpec, col, num, p, popup, slider}; + +fn spec(id: &'static str, name: &'static str, params: Vec, render: crate::RenderFn) -> EffectSpec { + EffectSpec { id, name, category: "Distort", params, render, gpu: false, float: true } +} + +// ---------------------------------------------------------------- CC Flo Motion + +fn flo_motion(ctx: &EffectCtx, mut b: Buf) -> Buf { + let knots = [(ctx.params.v2("knot1"), ctx.params.f("amount1") / 100.0), (ctx.params.v2("knot2"), ctx.params.f("amount2") / 100.0)]; + if knots.iter().all(|k| k.1 == 0.0) { + return b; + } + let tile = ctx.params.b("tileEdges"); + let ss = match ctx.params.e("antialiasing") { + 0 => 1, + 1 => 2, + _ => 3, + }; + let falloff = ctx.params.f("falloff").max(0.01); + let (lw, lh) = (ctx.layer_size[0].max(1.0), ctx.layer_size[1].max(1.0)); + let radius = (lw * lw + lh * lh).sqrt() * 0.5; + let src = b.img.clone(); + let (scale, off) = (b.scale, b.offset); + let map = |lx: f64, ly: f64| -> (f64, f64) { + let (mut x, mut y) = (lx, ly); + for (k, a) in knots { + if a == 0.0 { + continue; + } + let (dx, dy) = (x - k[0], y - k[1]); + let r = (dx * dx + dy * dy).sqrt() / radius; + // Weight 1 at the knot, falling off with distance. + let w = (1.0 - r.min(1.0)).powf(falloff); + // Pull: sample closer to the knot (content flows out of it); push: farther. + let s = (1.0 - a.clamp(-4.0, 0.95) * w).max(0.02); + x = k[0] + dx * s; + y = k[1] + dy * s; + } + (x, y) + }; + b.img.rows_mut().for_each(|(y, row)| { + for (x, o) in row.iter_mut().enumerate() { + let mut acc = [0.0f32; 4]; + for sy in 0..ss { + for sx in 0..ss { + let px = x as f64 + (sx as f64 + 0.5) / ss as f64; + let py = y as f64 + (sy as f64 + 0.5) / ss as f64; + let (lx, ly) = map((px - off[0]) / scale, (py - off[1]) / scale); + let (lx, ly) = if tile { (lx.rem_euclid(lw), ly.rem_euclid(lh)) } else { (lx, ly) }; + let q = src.sample_bilinear(lx * scale + off[0], ly * scale + off[1]); + for c in 0..4 { + acc[c] += q[c]; + } + } + } + let n = (ss * ss) as f32; + *o = acc.map(|v| v / n); + } + }); + b +} + +// ---------------------------------------------------------------- Liquify + +/// Liquify tools (the stroke text uses these names). +pub const LIQUIFY_TOOLS: [&str; 12] = [ + "warp", + "turbulence", + "twirlClockwise", + "twirlCounterclockwise", + "pucker", + "bloat", + "shiftPixels", + "reflection", + "clone", + "reconstruction", + "freeze", + "thaw", +]; + +/// One recorded Liquify stroke. +#[derive(Clone, Debug, PartialEq)] +pub struct LiquifyStroke { + pub tool: usize, + /// Brush size (diameter) and pressure (0–100), turbulent jitter (0–100), layer pixels. + pub size: f64, + pub pressure: f64, + pub jitter: f64, + /// Clone offset (source − destination). + pub clone_offset: [f64; 2], + pub points: Vec<[f64; 2]>, +} + +impl LiquifyStroke { + /// Text form: `tool size pressure jitter cloneX cloneY x,y x,y …`. + pub fn to_line(&self) -> String { + let mut s = format!("{} {} {} {} {} {}", LIQUIFY_TOOLS[self.tool], self.size, self.pressure, self.jitter, self.clone_offset[0], self.clone_offset[1]); + for p in &self.points { + s.push_str(&format!(" {},{}", p[0], p[1])); + } + s + } + pub fn parse(line: &str) -> Option { + let mut it = line.split_whitespace(); + let name = it.next()?; + let tool = LIQUIFY_TOOLS.iter().position(|t| t.eq_ignore_ascii_case(name))?; + let mut f = || -> Option { it.next()?.parse().ok() }; + let (size, pressure, jitter, cx, cy) = (f()?, f()?, f()?, f()?, f()?); + let points: Vec<[f64; 2]> = it + .filter_map(|t| { + let (a, b) = t.split_once(',')?; + Some([a.parse().ok()?, b.parse().ok()?]) + }) + .collect(); + (!points.is_empty()).then_some(LiquifyStroke { tool, size, pressure, jitter, clone_offset: [cx, cy], points }) + } +} + +/// Parse the `distortionMesh` text into strokes (bad lines are skipped). +pub fn parse_strokes(s: &str) -> Vec { + s.lines().filter_map(LiquifyStroke::parse).collect() +} + +/// Displacement mesh: offsets (layer pixels) at grid nodes spaced `cell` apart. +#[derive(Clone, Debug, PartialEq)] +pub struct Mesh { + pub nx: usize, + pub ny: usize, + pub cell: f64, + pub d: Vec<[f64; 2]>, + pub freeze: Vec, +} + +impl Mesh { + fn new(w: f64, h: f64, cell: f64) -> Mesh { + let nx = (w / cell).ceil() as usize + 1; + let ny = (h / cell).ceil() as usize + 1; + Mesh { nx, ny, cell, d: vec![[0.0; 2]; nx * ny], freeze: vec![0.0; nx * ny] } + } + /// Bilinear displacement at a layer point (zero outside the mesh). + pub fn at(&self, x: f64, y: f64) -> [f64; 2] { + let fx = x / self.cell; + let fy = y / self.cell; + if fx < 0.0 || fy < 0.0 || fx > (self.nx - 1) as f64 || fy > (self.ny - 1) as f64 { + return [0.0; 2]; + } + let (x0, y0) = ((fx.floor() as usize).min(self.nx - 2), (fy.floor() as usize).min(self.ny - 2)); + let (x1, y1) = ((x0 + 1).min(self.nx - 1), (y0 + 1).min(self.ny - 1)); + let (tx, ty) = (fx - x0 as f64, fy - y0 as f64); + let g = |i: usize, j: usize| self.d[j * self.nx + i]; + let l = |a: [f64; 2], b: [f64; 2], t: f64| [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t]; + l(l(g(x0, y0), g(x1, y0), tx), l(g(x0, y1), g(x1, y1), tx), ty) + } +} + +/// Replay strokes into a displacement mesh over a `w`×`h` layer. `frozen(x, y)` reports +/// mask-frozen points (Freeze Area Mask). +pub fn build_mesh(strokes: &[LiquifyStroke], w: f64, h: f64, frozen: &(dyn Fn(f64, f64) -> bool + Sync)) -> Mesh { + let cell = (w.max(h) / 128.0).clamp(2.0, 16.0); + let mut m = Mesh::new(w, h, cell); + for (i, f) in m.freeze.iter_mut().enumerate() { + let (x, y) = ((i % m.nx) as f64 * cell, (i / m.nx) as f64 * cell); + if frozen(x, y) { + *f = 1.0; + } + } + for (si, s) in strokes.iter().enumerate() { + let r = (s.size * 0.5).max(1.0); + let pr = (s.pressure / 100.0).clamp(0.0, 1.0); + // Dabs every r/4 along the stroke; each dab knows the local motion of the brush. + let mut dabs: Vec<([f64; 2], [f64; 2])> = vec![]; + if s.points.len() == 1 { + dabs.push((s.points[0], [0.0; 2])); + } + for seg in s.points.windows(2) { + let (a, b) = (seg[0], seg[1]); + let len = ((b[0] - a[0]).powi(2) + (b[1] - a[1]).powi(2)).sqrt(); + let n = ((len / (r * 0.25)).ceil() as usize).max(1); + let step = [(b[0] - a[0]) / n as f64, (b[1] - a[1]) / n as f64]; + for k in 1..=n { + dabs.push(([a[0] + step[0] * k as f64, a[1] + step[1] * k as f64], step)); + } + } + for (di, (c, mv)) in dabs.iter().enumerate() { + let (gx0, gx1) = (((c[0] - r) / cell).floor().max(0.0) as usize, (((c[0] + r) / cell).ceil() as usize).min(m.nx - 1)); + let (gy0, gy1) = (((c[1] - r) / cell).floor().max(0.0) as usize, (((c[1] + r) / cell).ceil() as usize).min(m.ny - 1)); + let prev = m.d.clone(); + for gy in gy0..=gy1 { + for gx in gx0..=gx1 { + let (x, y) = (gx as f64 * cell, gy as f64 * cell); + let (dx, dy) = (x - c[0], y - c[1]); + let dist = (dx * dx + dy * dy).sqrt(); + if dist >= r { + continue; + } + let i = gy * m.nx + gx; + // Smooth brush falloff. + let fall = { + let t = 1.0 - dist / r; + t * t * (3.0 - 2.0 * t) + }; + let tool = LIQUIFY_TOOLS[s.tool]; + if tool == "freeze" || tool == "thaw" { + let v = pr * fall; + m.freeze[i] = if tool == "freeze" { m.freeze[i].max(v) } else { (m.freeze[i] - v).max(0.0) }; + continue; + } + let w = pr * fall * (1.0 - m.freeze[i]) * 0.5; + if w <= 0.0 { + continue; + } + let d = &mut m.d[i]; + match tool { + "warp" => { + d[0] -= mv[0] * w * 2.0; + d[1] -= mv[1] * w * 2.0; + } + "turbulence" => { + let j = s.jitter / 100.0 * r * 0.1; + let seed = (si as u32).wrapping_mul(7919) ^ di as u32; + d[0] += (hash1(gx as u32, gy as u32, seed) as f64 * 2.0 - 1.0) * j * w; + d[1] += (hash1(gx as u32, gy as u32, seed ^ 0x55) as f64 * 2.0 - 1.0) * j * w; + d[0] -= mv[0] * w; + d[1] -= mv[1] * w; + } + "twirlClockwise" | "twirlCounterclockwise" => { + let sgn = if tool == "twirlClockwise" { -1.0 } else { 1.0 }; + let a = sgn * 0.08 * w; + let (sn, cs) = a.sin_cos(); + d[0] += dx * cs - dy * sn - dx; + d[1] += dx * sn + dy * cs - dy; + } + "pucker" => { + d[0] += dx * 0.05 * w; + d[1] += dy * 0.05 * w; + } + "bloat" => { + d[0] -= dx * 0.05 * w; + d[1] -= dy * 0.05 * w; + } + "shiftPixels" => { + // Perpendicular to the stroke (to its left). + d[0] -= mv[1] * w * 2.0; + d[1] += mv[0] * w * 2.0; + } + "reflection" => { + // Mirror across the stroke line through the dab. + let l = (mv[0] * mv[0] + mv[1] * mv[1]).sqrt(); + if l > 1e-9 { + let n = [-mv[1] / l, mv[0] / l]; + let k = dx * n[0] + dy * n[1]; + let tgt = [-2.0 * k * n[0], -2.0 * k * n[1]]; + d[0] += (tgt[0] - d[0]) * w; + d[1] += (tgt[1] - d[1]) * w; + } + } + "clone" => { + // Copy the distortion found at the clone source. + let sx = x + s.clone_offset[0]; + let sy = y + s.clone_offset[1]; + let fx = (sx / cell).round().clamp(0.0, (m.nx - 1) as f64) as usize; + let fy = (sy / cell).round().clamp(0.0, (m.ny - 1) as f64) as usize; + let src = prev[fy * m.nx + fx]; + let tgt = [src[0] + s.clone_offset[0], src[1] + s.clone_offset[1]]; + d[0] += (tgt[0] - d[0]) * w; + d[1] += (tgt[1] - d[1]) * w; + } + // Reconstruction: relax toward no distortion. + _ => { + d[0] *= 1.0 - w; + d[1] *= 1.0 - w; + } + } + } + } + } + } + m +} + +type MeshCache = Mutex>>; + +fn cached_mesh(ctx: &EffectCtx) -> Arc { + use std::hash::{Hash, Hasher}; + let text = ctx.params.s("distortionMesh"); + let mask = ctx.params.f("freezeAreaMask").round() as usize; + let poly = mask.checked_sub(1).and_then(|i| ctx.env.masks.get(i)).cloned(); + let mut h = std::collections::hash_map::DefaultHasher::new(); + text.hash(&mut h); + ctx.layer_size[0].to_bits().hash(&mut h); + ctx.layer_size[1].to_bits().hash(&mut h); + if let Some(p) = &poly { + for q in &p.points { + q[0].to_bits().hash(&mut h); + q[1].to_bits().hash(&mut h); + } + p.inverted.hash(&mut h); + } + let key = h.finish(); + static CACHE: OnceLock = OnceLock::new(); + let cache = CACHE.get_or_init(|| Mutex::new(HashMap::new())); + if let Some(m) = cache.lock().ok().and_then(|c| c.get(&key).cloned()) { + return m; + } + let strokes = parse_strokes(text); + let frozen = |x: f64, y: f64| match &poly { + Some(p) => point_in_poly(&p.points, x, y) != p.inverted, + None => false, + }; + let m = Arc::new(build_mesh(&strokes, ctx.layer_size[0].max(1.0), ctx.layer_size[1].max(1.0), &frozen)); + if let Ok(mut c) = cache.lock() { + if c.len() > 32 { + c.clear(); + } + c.insert(key, m.clone()); + } + m +} + +fn liquify(ctx: &EffectCtx, mut b: Buf) -> Buf { + let pct = ctx.params.f("distortionPercentage") / 100.0; + let show_freeze = ctx.params.b("viewFreezeAreaMask"); + let show_mesh = ctx.params.b("viewMesh"); + if ctx.params.s("distortionMesh").trim().is_empty() && !show_freeze && !show_mesh { + return b; + } + let mesh = cached_mesh(ctx); + let off = ctx.params.v2("distortionMeshOffset"); + let mesh_color = ctx.params.color("meshColor"); + let src = b.img.clone(); + let (scale, o) = (b.scale, b.offset); + let line_w = 0.6 / scale.max(1e-6); + b.img.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let lx = (x as f64 + 0.5 - o[0]) / scale; + let ly = (y as f64 + 0.5 - o[1]) / scale; + let d = mesh.at(lx - off[0], ly - off[1]); + let (sx, sy) = (lx + d[0] * pct, ly + d[1] * pct); + let mut c = if d == [0.0; 2] { *px } else { src.sample_bilinear(sx * scale + o[0], sy * scale + o[1]) }; + if show_freeze { + let fx = ((lx - off[0]) / mesh.cell).round().clamp(0.0, (mesh.nx - 1) as f64) as usize; + let fy = ((ly - off[1]) / mesh.cell).round().clamp(0.0, (mesh.ny - 1) as f64) as usize; + let f = mesh.freeze[fy * mesh.nx + fx] as f32; + if f > 0.0 { + c[0] = c[0] * (1.0 - 0.5 * f) + 0.5 * f; + c[3] = c[3].max(0.5 * f); + } + } + if show_mesh { + // Grid lines every 4 mesh cells, displaced with the mesh. + let gs = mesh.cell * 4.0; + let (gx, gy) = ((lx + d[0] * pct - off[0]).rem_euclid(gs), (ly + d[1] * pct - off[1]).rem_euclid(gs)); + if gx < line_w || gy < line_w { + c = [mesh_color[0], mesh_color[1], mesh_color[2], 1.0]; + } + } + *px = c; + } + }); + b +} + +// ---------------------------------------------------------------- Rolling Shutter Repair + +/// Per-pixel velocity field (layer pixels per frame) on a coarse grid, from tracked points. +struct Flow { + nx: usize, + ny: usize, + cell: f64, + v: Vec<[f64; 2]>, +} + +impl Flow { + fn at(&self, x: f64, y: f64) -> [f64; 2] { + let fx = (x / self.cell - 0.5).clamp(0.0, (self.nx - 1) as f64); + let fy = (y / self.cell - 0.5).clamp(0.0, (self.ny - 1) as f64); + let (x0, y0) = (fx.floor() as usize, fy.floor() as usize); + let (x1, y1) = ((x0 + 1).min(self.nx - 1), (y0 + 1).min(self.ny - 1)); + let (tx, ty) = (fx - x0 as f64, fy - y0 as f64); + let g = |i: usize, j: usize| self.v[j * self.nx + i]; + let l = |a: [f64; 2], b: [f64; 2], t: f64| [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t]; + l(l(g(x0, y0), g(x1, y0), tx), l(g(x0, y1), g(x1, y1), tx), ty) + } +} + +fn median(v: &mut [f64]) -> f64 { + v.sort_by(f64::total_cmp); + v[v.len() / 2] +} + +/// Measure motion around the current frame (`cur`) from `prev` to `next` (all in the same +/// buffer grid). `per_row`: Warp method (one velocity per scanline band). +fn measure(cur: &Image, prev: Option<&Image>, next: Option<&Image>, scale: f64, detailed: bool, per_row: bool) -> Option { + use effectcraft_track::klt::{GrayPyramid, LkOpts, analysis_factor, track}; + let factor = analysis_factor(cur.width, cur.height, if detailed { 640 } else { 320 }); + let pc = GrayPyramid::from_image(cur, [0.0; 2], factor, 4); + let (w, h) = (pc.width(), pc.height()); + let n = if detailed { 24 } else { 12 }; + let cell_w = w as f64 / n as f64; + let rows = ((n as f64) * h as f64 / w.max(1) as f64).round().max(2.0) as usize; + let cell_h = h as f64 / rows as f64; + let pts: Vec<[f64; 2]> = (0..rows).flat_map(|j| (0..n).map(move |i| [(i as f64 + 0.5) * cell_w, (j as f64 + 0.5) * cell_h])).collect(); + let o = LkOpts { radius: 4, fb_max: 1.0, ..Default::default() }; + let go = |other: Option<&Image>| other.map(|img| track(&pc, &GrayPyramid::from_image(img, [0.0; 2], factor, 4), &pts, None, &o)); + let (tn, tp) = (go(next), go(prev)); + // Velocity in plane pixels per frame. + let vel: Vec> = (0..pts.len()) + .map(|i| { + let p = pts[i]; + let fw = tn.as_ref().and_then(|t| t[i]).map(|q| [q[0] - p[0], q[1] - p[1]]); + let bw = tp.as_ref().and_then(|t| t[i]).map(|q| [p[0] - q[0], p[1] - q[1]]); + match (fw, bw) { + (Some(a), Some(b)) => Some([(a[0] + b[0]) * 0.5, (a[1] + b[1]) * 0.5]), + (a, b) => a.or(b), + } + }) + .collect(); + let valid: Vec<[f64; 2]> = vel.iter().flatten().copied().collect(); + if valid.is_empty() { + return None; + } + let mut vx: Vec = valid.iter().map(|v| v[0]).collect(); + let mut vy: Vec = valid.iter().map(|v| v[1]).collect(); + let global = [median(&mut vx), median(&mut vy)]; + let k = factor as f64 / scale.max(1e-9); + let mut v = vec![[0.0; 2]; n * rows]; + for j in 0..rows { + // Row (band) median for Warp; per point (outliers replaced) for Pixel Motion. + let band: Vec<[f64; 2]> = (0..n).filter_map(|i| vel[j * n + i]).collect(); + let row_med = if band.is_empty() { + global + } else { + let mut a: Vec = band.iter().map(|v| v[0]).collect(); + let mut b: Vec = band.iter().map(|v| v[1]).collect(); + [median(&mut a), median(&mut b)] + }; + for i in 0..n { + let s = if per_row { + row_med + } else { + match vel[j * n + i] { + Some(q) if (q[0] - row_med[0]).abs() + (q[1] - row_med[1]).abs() < 3.0 + 0.5 * (row_med[0].abs() + row_med[1].abs()) => q, + _ => row_med, + } + }; + v[j * n + i] = [s[0] * k, s[1] * k]; + } + } + // Cell size in layer pixels (grid is uniform in plane pixels; assume square plane pixels). + Some(Flow { nx: n, ny: rows, cell: cell_w * k, v }) +} + +fn same_grid(nb: Buf, b: &Buf) -> Image { + if nb.img.width == b.img.width && nb.img.height == b.img.height && nb.offset == b.offset && nb.scale == b.scale { + return nb.img; + } + let k = nb.scale / b.scale.max(1e-9); + crate::util::gen_image(b.img.width, b.img.height, |x, y| { + let lx = (x as f64 + 0.5 - b.offset[0]) * k + nb.offset[0]; + let ly = (y as f64 + 0.5 - b.offset[1]) * k + nb.offset[1]; + nb.img.sample_bilinear(lx, ly) + }) +} + +fn rolling_shutter(ctx: &EffectCtx, mut b: Buf) -> Buf { + let rate = ctx.params.f("rollingShutterRate") / 100.0; + if rate == 0.0 { + return b; + } + let Some(host) = ctx.env.host else { return b }; + let fps = ctx.fps(); + let prev = host.self_at(ctx.time - 1.0 / fps, ctx.env.effect_index).map(|nb| same_grid(nb, &b)); + let next = host.self_at(ctx.time + 1.0 / fps, ctx.env.effect_index).map(|nb| same_grid(nb, &b)); + if prev.is_none() && next.is_none() { + return b; + } + let per_row = ctx.params.e("method") == 0; + let Some(flow) = measure(&b.img, prev.as_ref(), next.as_ref(), 1.0, ctx.params.b("detailedAnalysis"), per_row) else { return b }; + let dir = ctx.params.e("scanDirection"); + let src = b.img.clone(); + let (w, h) = (b.img.width as f64, b.img.height as f64); + b.img.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let (fx, fy) = (x as f64 + 0.5, y as f64 + 0.5); + let s = match dir { + 1 => 1.0 - fy / h, + 2 => fx / w, + 3 => 1.0 - fx / w, + _ => fy / h, + }; + let tau = rate * (s - 0.5); + let v = flow.at(fx, fy); + *px = src.sample_bilinear_clamped(fx + v[0] * tau, fy + v[1] * tau); + } + }); + b +} + +pub fn specs() -> Vec { + vec![ + spec( + "ec.distort.ccflomotion", + "CC Flo Motion", + vec![ + p("knot1", "Knot 1", Value::Vec2([0.25, 0.5]), ParamUi::Point), + p("amount1", "Amount 1", num(50.0), slider(-400.0, 100.0, -100.0, 100.0, 1)), + p("knot2", "Knot 2", Value::Vec2([0.75, 0.5]), ParamUi::Point), + p("amount2", "Amount 2", num(-50.0), slider(-400.0, 100.0, -100.0, 100.0, 1)), + p("tileEdges", "Tile Edges", Value::Bool(false), ParamUi::Checkbox), + p("antialiasing", "Antialiasing", Value::Enum(1), popup(&["Low", "Medium", "High"])), + p("falloff", "Falloff", num(2.0), slider(0.01, 10.0, 0.1, 5.0, 2)), + ], + flo_motion, + ), + spec( + "ec.distort.liquify", + "Liquify", + vec![ + p("tool", "Tool", Value::Enum(0), popup(&LIQUIFY_TOOLS)), + p("brushSize", "Brush Size", num(64.0), slider(1.0, 600.0, 1.0, 600.0, 0)), + p("brushPressure", "Brush Pressure", num(50.0), slider(1.0, 100.0, 1.0, 100.0, 0)), + p("freezeAreaMask", "Freeze Area Mask", num(0.0), ParamUi::Mask), + p("turbulentJitter", "Turbulent Jitter", num(70.0), slider(1.0, 100.0, 1.0, 100.0, 0)), + p("cloneOffset", "Clone Offset", Value::Vec2([0.0, 0.0]), ParamUi::Hidden), + p("reconstructionMode", "Reconstruction Mode", Value::Enum(0), popup(&["Revert", "Displace", "Amplitwist", "Affine"])), + p("viewFreezeAreaMask", "View Freeze Area Mask", Value::Bool(false), ParamUi::Checkbox), + p("viewMesh", "View Mesh", Value::Bool(false), ParamUi::Checkbox), + p("meshSize", "Mesh Size", Value::Enum(1), popup(&["Small", "Medium", "Large"])), + p("meshColor", "Mesh Color", col(0.5, 0.5, 0.5), ParamUi::Color), + // The strokes (see `LiquifyStroke`); written by the `liquify.stroke` command. + p("distortionMesh", "Distortion Mesh", Value::Str(String::new()), ParamUi::Hidden), + p("distortionMeshOffset", "Distortion Mesh Offset", Value::Vec2([0.0, 0.0]), ParamUi::Hidden), + p("distortionPercentage", "Distortion Percentage", num(100.0), slider(0.0, 200.0, 0.0, 200.0, 1)), + p("distortionDisplayQuality", "Distortion Display Quality", Value::Enum(1), popup(&["Draft", "Smooth"])), + ], + liquify, + ), + spec( + "ec.distort.rollingshutterrepair", + "Rolling Shutter Repair", + vec![ + p("rollingShutterRate", "Rolling Shutter Rate", num(50.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("scanDirection", "Scan Direction", Value::Enum(0), popup(&["Top → Bottom", "Bottom → Top", "Left → Right", "Right → Left"])), + p("method", "Method", Value::Enum(0), popup(&["Warp", "Pixel Motion"])), + p("detailedAnalysis", "Detailed Analysis", Value::Bool(false), ParamUi::Checkbox), + p("pixelMotionDetail", "Pixel Motion Detail", num(20.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + ], + rolling_shutter, + ), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, EffectHost, LayerPixels, run_fx}; + + fn grid_img(w: u32, h: u32) -> Image { + crate::util::gen_image(w, h, |x, y| { + let v = if x % 8 == 0 || y % 8 == 0 { 1.0 } else { 0.2 }; + [v, v * 0.5, x as f32 / w as f32, 1.0] + }) + } + + #[test] + fn flo_motion_pulls_and_pushes() { + let img = grid_img(64, 64); + let out = run_fx( + "ec.distort.ccflomotion", + &[("knot1", Value::Vec2([32.0, 32.0])), ("amount1", num(80.0)), ("amount2", num(0.0))], + img.clone(), + 0.0, + EffectEnv::default(), + ); + // The knot itself is fixed; pixels near it are magnified (sampled closer to the knot). + assert!((out.img.get(32, 32)[2] - img.get(32, 32)[2]).abs() < 0.01); + assert!((out.img.get(40, 32)[2] - img.get(40, 32)[2]).abs() > 0.01); + assert!(out.img.get(40, 32)[2] < img.get(40, 32)[2]); + let none = run_fx("ec.distort.ccflomotion", &[("amount1", num(0.0)), ("amount2", num(0.0))], img.clone(), 0.0, EffectEnv::default()); + assert_eq!(none.img.data, img.data); + let again = run_fx( + "ec.distort.ccflomotion", + &[("knot1", Value::Vec2([32.0, 32.0])), ("amount1", num(80.0)), ("amount2", num(0.0))], + img, + 0.0, + EffectEnv::default(), + ); + assert_eq!(out.img.data, again.img.data); + } + + #[test] + fn liquify_strokes_round_trip_and_warp() { + let s = LiquifyStroke { tool: 0, size: 30.0, pressure: 80.0, jitter: 70.0, clone_offset: [0.0, 0.0], points: vec![[20.0, 32.0], [40.0, 32.0]] }; + assert_eq!(LiquifyStroke::parse(&s.to_line()), Some(s.clone())); + assert!(LiquifyStroke::parse("nope 1 2 3 4 5 1,2").is_none()); + let img = grid_img(64, 64); + let out = run_fx("ec.distort.liquify", &[("distortionMesh", Value::Str(s.to_line()))], img.clone(), 0.0, EffectEnv::default()); + assert_ne!(out.img.data, img.data); + // Warping to the right: a pixel right of the start now shows content from its left. + let at = |i: &Image| i.get(36, 32)[2]; + assert!(at(&out.img) < at(&img), "{} vs {}", at(&out.img), at(&img)); + // Far away, untouched. + assert_eq!(out.img.get(5, 5), img.get(5, 5)); + // Distortion Percentage 0 = identity; deterministic. + let zero = run_fx( + "ec.distort.liquify", + &[("distortionMesh", Value::Str(s.to_line())), ("distortionPercentage", num(0.0))], + img.clone(), + 0.0, + EffectEnv::default(), + ); + assert_eq!(zero.img.data, img.data); + let again = run_fx("ec.distort.liquify", &[("distortionMesh", Value::Str(s.to_line()))], img.clone(), 0.0, EffectEnv::default()); + assert_eq!(out.img.data, again.img.data); + // Reconstruction undoes the warp; freeze protects. + let mut rec = s.clone(); + rec.tool = 9; + rec.pressure = 100.0; + let lines = format!("{}\n{}\n{}\n{}", s.to_line(), rec.to_line(), rec.to_line(), rec.to_line()); + let m1 = build_mesh(&parse_strokes(&s.to_line()), 64.0, 64.0, &|_, _| false); + let m2 = build_mesh(&parse_strokes(&lines), 64.0, 64.0, &|_, _| false); + let mag = |m: &Mesh| m.d.iter().map(|d| d[0].abs() + d[1].abs()).sum::(); + assert!(mag(&m2) < mag(&m1) * 0.5); + let frozen = build_mesh(&parse_strokes(&s.to_line()), 64.0, 64.0, &|_, _| true); + assert_eq!(mag(&frozen), 0.0); + // Every tool changes (or deliberately keeps) the mesh without NaNs. + for t in 0..LIQUIFY_TOOLS.len() { + let st = LiquifyStroke { tool: t, clone_offset: [10.0, 0.0], ..s.clone() }; + let m = build_mesh(&[s.clone(), st], 64.0, 64.0, &|_, _| false); + assert!(m.d.iter().all(|d| d[0].is_finite() && d[1].is_finite()), "{}", LIQUIFY_TOOLS[t]); + } + } + + /// A textured frame panning right at 3 px per frame, captured with a rolling shutter: row + /// `y` is exposed `(y / h − ½)` frames late (rate 100 %). + fn texture(x: f64, y: f64) -> f32 { + let a = ((x * 0.31).sin() * (y * 0.23).cos() + (x * 0.07 + y * 0.11).sin()) as f32; + 0.5 + 0.25 * a + } + fn rs_frame(f: f64, shutter: bool) -> Image { + let (w, h) = (96u32, 64u32); + crate::util::gen_image(w, h, |x, y| { + let tau = if shutter { (y as f64 + 0.5) / h as f64 - 0.5 } else { 0.0 }; + let shift = 3.0 * (f + tau); + let v = texture(x as f64 + 0.5 - shift, y as f64 + 0.5); + [v, v, v, 1.0] + }) + } + struct Pan; + impl EffectHost for Pan { + fn layer(&self, _: u64, _: bool) -> Option { + None + } + fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option> { + None + } + fn self_at(&self, t: f64, _: usize) -> Option { + Some(Buf { img: rs_frame(t * 24.0, true), offset: [0.0; 2], scale: 1.0 }) + } + } + + #[test] + fn rolling_shutter_repair_unskews_a_pan() { + let host = Pan; + let env = EffectEnv { host: Some(&host), frame_rate: 24.0, ..Default::default() }; + let t = 5.0 / 24.0; + let skewed = rs_frame(5.0, true); + let truth = rs_frame(5.0, false); + let err = |a: &Image| { + let mut s = 0.0; + let mut n = 0.0; + for y in 4..60 { + for x in 12..84 { + s += (a.get(x, y)[0] - truth.get(x, y)[0]).abs(); + n += 1.0; + } + } + s / n + }; + for method in [0u32, 1] { + let out = run_fx("ec.distort.rollingshutterrepair", &[("rollingShutterRate", num(100.0)), ("method", Value::Enum(method))], skewed.clone(), t, env); + assert!(err(&out.img) < err(&skewed) * 0.4, "method {method}: {} vs {}", err(&out.img), err(&skewed)); + let again = + run_fx("ec.distort.rollingshutterrepair", &[("rollingShutterRate", num(100.0)), ("method", Value::Enum(method))], skewed.clone(), t, env); + assert_eq!(out.img.data, again.img.data); + } + let none = run_fx("ec.distort.rollingshutterrepair", &[], skewed.clone(), t, EffectEnv::default()); + assert_eq!(none.img.data, skewed.data); + } +} diff --git a/crates/effects/src/keylight.rs b/crates/effects/src/keylight.rs new file mode 100644 index 00000000..980a6169 --- /dev/null +++ b/crates/effects/src/keylight.rs @@ -0,0 +1,455 @@ +//! Key Light — a full screen-colour keyer with the controls of After Effects' bundled +//! "Keylight (1.2)" (that name is a trademark of its vendor, so the effect is registered as +//! **Key Light**; `lookup("Keylight (1.2)")` finds it through [`KEYLIGHT_ALIASES`]). +//! +//! The method is the textbook *screen difference* key, written from the public description of +//! the controls: +//! +//! * colours are first neutralised by the **Alpha Bias** / **Despill Bias** colours (each channel +//! scaled so the bias colour becomes grey — mid grey = no change); +//! * the **screen difference** of a pixel is its screen-primary channel minus a balance-weighted +//! mix of the other two (**Screen Balance**); the raw matte is +//! `1 − Screen Gain × diff(pixel) / diff(screen colour)`; +//! * the **Screen Matte** is then clipped (Clip Black / White, with Clip Rollback restoring edge +//! detail), despotted (morphological open/close: Despot Black / White), shrunk or grown and +//! softened, combined with the **Inside** / **Outside** masks and the source alpha; +//! * the foreground is the source with the screen colour removed in proportion to the +//! transparency (`(c − (1 − α)·S) / α`), then **despilled** (the primary is limited to the +//! balanced secondary), with **Replace Method** colouring areas whose alpha was raised by +//! clipping or the inside mask, and optional foreground and edge colour correction. +//! +//! Views: Source, Source Alpha, Corrected Source, Colour Correction Edges, Screen Matte, Inside +//! Mask, Outside Mask, Combined Matte, Status, Intermediate Result and Final Result. + +use effectcraft_color::luminance; +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::{Image, gaussian_blur}; +use rayon::prelude::*; + +use crate::util::{Plane, gauss_plane, morph_frac, point_in_poly, premul, unpremul}; +use crate::{Buf, EffectCtx, EffectSpec, col, num, p, popup, slider}; + +/// Other names the effect answers to in [`crate::lookup`]. +pub const KEYLIGHT_ALIASES: &[&str] = &["Keylight (1.2)", "Keylight"]; + +pub const VIEWS: [&str; 11] = [ + "Source", + "Source Alpha", + "Corrected Source", + "Colour Correction Edges", + "Screen Matte", + "Inside Mask", + "Outside Mask", + "Combined Matte", + "Status", + "Intermediate Result", + "Final Result", +]; + +const REPLACE: [&str; 4] = ["None", "Source", "Hard Colour", "Soft Colour"]; + +fn pct(d: f64) -> (Value, ParamUi) { + (num(d), slider(0.0, 100.0, 0.0, 100.0, 1)) +} + +fn mask_popup() -> ParamUi { + popup(&["None", "Mask 1", "Mask 2", "Mask 3", "Mask 4", "Mask 5", "Mask 6", "Mask 7", "Mask 8", "Mask 9", "Mask 10"]) +} + +/// Index of the screen's primary channel and the two others (larger-in-screen first). +fn primary(s: [f32; 3]) -> (usize, usize, usize) { + let pi = if s[1] >= s[0] && s[1] >= s[2] { + 1 + } else if s[2] >= s[0] { + 2 + } else { + 0 + }; + let (a, b) = match pi { + 0 => (1, 2), + 1 => (0, 2), + _ => (0, 1), + }; + if s[a] >= s[b] { (pi, a, b) } else { (pi, b, a) } +} + +/// Screen difference: primary minus the balance-weighted secondaries. +#[inline] +pub fn screen_diff(c: [f32; 3], pi: usize, o1: usize, o2: usize, bal: f32) -> f32 { + c[pi] - (bal * c[o1] + (1.0 - bal) * c[o2]) +} + +/// Scale channels so `bias` becomes neutral (grey keeps everything as is). +#[inline] +fn neutralise(c: [f32; 3], bias: [f32; 3]) -> [f32; 3] { + let l = luminance(bias[0], bias[1], bias[2]).max(1e-4); + [0, 1, 2].map(|i| c[i] * l / bias[i].max(1e-4)) +} + +fn mask_coverage(ctx: &EffectCtx, b: &Buf, idx: u32, softness: f64, invert: bool) -> Option { + let shape = ctx.env.masks.get((idx as usize).checked_sub(1)?)?; + let (w, h) = (b.img.width as usize, b.img.height as usize); + let mut pl = Plane::new(w, h); + let inv = 1.0 / b.scale.max(1e-9); + pl.data.par_chunks_mut(w.max(1)).enumerate().for_each(|(y, row)| { + for (x, v) in row.iter_mut().enumerate() { + let lx = (x as f64 + 0.5 - b.offset[0]) * inv; + let ly = (y as f64 + 0.5 - b.offset[1]) * inv; + *v = if point_in_poly(&shape.points, lx, ly) != shape.inverted { 1.0 } else { 0.0 }; + } + }); + let s = softness * b.scale; + if s > 0.0 { + pl = gauss_plane(&pl, s * 0.5, s * 0.5); + } + if invert { + pl = pl.map(|v| 1.0 - v); + } + Some(pl) +} + +/// Saturation / contrast / brightness correction of a straight colour. +fn correct(c: [f32; 3], sat: f32, contrast: f32, bright: f32) -> [f32; 3] { + let l = luminance(c[0], c[1], c[2]); + let c = c.map(|v| l + (v - l) * sat); + c.map(|v| ((v - 0.5) * contrast + 0.5) * bright) +} + +fn key_light(ctx: &EffectCtx, mut b: Buf) -> Buf { + let pr = ctx.params; + let view = pr.e("view"); + if view == 0 { + return b; + } + let sc = pr.color("screenColour"); + let screen = [sc[0], sc[1], sc[2]]; + let (pi, o1, o2) = primary(screen); + let gain = pr.f("screenGain") as f32 / 100.0; + let bal = pr.f("screenBalance") as f32 / 100.0; + let dbias = pr.color("despillBias"); + let abias = if pr.b("lockBiasesTogether") { dbias } else { pr.color("alphaBias") }; + let (dbias, abias) = ([dbias[0], dbias[1], dbias[2]], [abias[0], abias[1], abias[2]]); + let s_alpha = neutralise(screen, abias); + let sd = screen_diff(s_alpha, pi, o1, o2, bal).max(1e-3); + let cb = pr.f("clipBlack") as f32 / 100.0; + let cw = pr.f("clipWhite") as f32 / 100.0; + let rollback = pr.f("clipRollback") * b.scale; + let preblur = pr.f("screenPreblur") * b.scale; + let sg = pr.f("screenShrinkGrow") * b.scale; + let soft = pr.f("screenSoftness") * b.scale; + let despot_b = pr.f("screenDespotBlack") * b.scale; + let despot_w = pr.f("screenDespotWhite") * b.scale; + + let src = b.img.clone(); + let blurred; + let key_src: &Image = if preblur > 0.0 { + blurred = gaussian_blur(&src, preblur * 0.5, preblur * 0.5, true); + &blurred + } else { + &src + }; + // Raw screen matte (opacity of the foreground). + let raw = Plane::from_image(key_src, |px| { + let (c, _) = unpremul(px); + let d = screen_diff(neutralise(c, abias), pi, o1, o2, bal); + (1.0 - gain * d / sd).clamp(0.0, 1.0) + }); + let clip = |v: f32| ((v - cb) / (cw - cb).max(1e-4)).clamp(0.0, 1.0); + let mut m = raw.map(clip); + if rollback > 0.0 { + // Restore edge detail lost by clipping within `rollback` pixels of the clipped matte. + let grown = morph_frac(&m, rollback, true); + let shrunk = morph_frac(&m, rollback, false); + m = Plane { + w: m.w, + h: m.h, + data: (0..m.data.len()) + .into_par_iter() + .map(|i| if grown.data[i] > shrunk.data[i] + 1e-4 { m.data[i].max(raw.data[i]).min(grown.data[i]) } else { m.data[i] }) + .collect(), + }; + } + if despot_b > 0.0 { + // Fill small holes: close (dilate then erode). + m = morph_frac(&morph_frac(&m, despot_b, true), despot_b, false); + } + if despot_w > 0.0 { + // Remove small specks: open (erode then dilate). + m = morph_frac(&morph_frac(&m, despot_w, false), despot_w, true); + } + if sg < 0.0 { + m = morph_frac(&m, -sg, false); + } else if sg > 0.0 { + m = morph_frac(&m, sg, true); + } + if soft > 0.0 { + m = gauss_plane(&m, soft * 0.5, soft * 0.5); + } + let screen_matte = m.clone(); + let inside = mask_coverage(ctx, &b, pr.e("insideMask"), pr.f("insideMaskSoftness"), pr.b("invertInsideMask")); + let outside = mask_coverage(ctx, &b, pr.e("outsideMask"), pr.f("outsideMaskSoftness"), pr.b("invertOutsideMask")); + let src_alpha_mode = pr.e("sourceAlpha"); + let n = src.data.len(); + let combined: Vec = (0..n) + .into_par_iter() + .map(|i| { + let mut a = screen_matte.data[i]; + let sa = src.data[i][3].clamp(0.0, 1.0); + let ins = inside.as_ref().map_or(0.0, |p| p.data[i]); + let ins = if src_alpha_mode == 1 { ins.max(sa) } else { ins }; + a = a.max(ins); + if let Some(o) = &outside { + a *= 1.0 - o.data[i]; + } + if src_alpha_mode == 2 { + a *= sa; + } + a.clamp(0.0, 1.0) + }) + .collect(); + let edges: Option> = pr.b("enableEdgeColourCorrection").then(|| { + let grow = pr.f("edgeGrow") * b.scale; + let hard = (pr.f("edgeHardness") / 100.0) as f32; + let esoft = pr.f("edgeSoftness") * b.scale; + // Edge band: where the (grown) combined matte is partial. + let cm = Plane { w: m.w, h: m.h, data: combined.clone() }; + let band = cm.map(|a| if a > 1e-3 && a < 0.999 { 1.0 } else { 0.0 }); + let mut band = if grow > 0.0 { morph_frac(&band, grow, true) } else { band }; + if esoft > 0.0 { + band = gauss_plane(&band, esoft * 0.5, esoft * 0.5); + } + band.data.iter().map(|v| (v * (1.0 + hard * 4.0)).min(1.0)).collect() + }); + let replace = pr.e("replaceMethod"); + let rc = pr.color("replaceColour"); + let in_replace = pr.e("insideReplaceMethod"); + let in_rc = pr.color("insideReplaceColour"); + let fg_cc = pr.b("enableColourCorrection"); + let (fs, fc, fbr) = (pr.f("saturation") as f32 / 100.0, pr.f("contrast") as f32 / 100.0 + 1.0, pr.f("brightness") as f32 / 100.0 + 1.0); + let (es, ec, ebr) = (pr.f("edgeSaturation") as f32 / 100.0, pr.f("edgeContrast") as f32 / 100.0 + 1.0, pr.f("edgeBrightness") as f32 / 100.0 + 1.0); + let unpremultiply = pr.b("unpremultiplyResult"); + let crops = [pr.f("cropLeft"), pr.f("cropRight"), pr.f("cropTop"), pr.f("cropBottom")].map(|v| v / 100.0); + let (lw, lh) = (ctx.layer_size[0], ctx.layer_size[1]); + let w = b.img.width as usize; + let (scale, off) = (b.scale, b.offset); + b.img.data.par_iter_mut().enumerate().for_each(|(i, px)| { + let (c, _) = unpremul(src.data[i]); + let a = combined[i]; + let sm = screen_matte.data[i]; + // Source crops (layer fractions from each edge). + let (x, y) = ((i % w) as f64 + 0.5, (i / w) as f64 + 0.5); + let (lx, ly) = ((x - off[0]) / scale / lw.max(1e-9), (y - off[1]) / scale / lh.max(1e-9)); + let cropped = lx < crops[0] || lx > 1.0 - crops[1] || ly < crops[2] || ly > 1.0 - crops[3]; + // Screen removal on the raw (unclipped) transparency, then despill. + let ar = raw.data[i].max(1e-4); + let mut fg = [0, 1, 2].map(|k| ((c[k] - (1.0 - ar) * screen[k]) / ar).max(0.0)); + let nfg = neutralise(fg, dbias); + let spill = screen_diff(nfg, pi, o1, o2, bal).max(0.0); + let l0 = (luminance(dbias[0], dbias[1], dbias[2]) / dbias[pi].max(1e-4)).max(1e-4); + fg[pi] -= spill / l0; + let intermediate = fg; + // Replace colour where the alpha was raised beyond the raw screen matte. + let raised = (sm - raw.data[i]).max(0.0); + let raised_in = inside.as_ref().map_or(0.0, |p| (p.data[i] - sm).max(0.0)); + let apply_replace = |fg: [f32; 3], method: u32, rc: [f32; 4], amt: f32| -> [f32; 3] { + if amt <= 0.0 { + return fg; + } + match method { + 1 => [0, 1, 2].map(|k| fg[k] + (c[k] - fg[k]) * amt), + 2 => [rc[0], rc[1], rc[2]], + 3 => { + let l = luminance(c[0], c[1], c[2]) / luminance(rc[0], rc[1], rc[2]).max(1e-4); + [0, 1, 2].map(|k| fg[k] + (rc[k] * l - fg[k]) * amt) + } + _ => fg, + } + }; + fg = apply_replace(fg, replace, rc, raised); + fg = apply_replace(fg, in_replace, in_rc, raised_in); + if fg_cc { + fg = correct(fg, fs, fc, fbr); + } + let edge = edges.as_ref().map_or(0.0, |e| e[i]); + if edge > 0.0 { + let ce = correct(fg, es, ec, ebr); + fg = [0, 1, 2].map(|k| fg[k] + (ce[k] - fg[k]) * edge); + } + let grey = |v: f32| [v, v, v, 1.0]; + *px = match view { + 1 => grey(src.data[i][3]), + 2 => { + let mut cs = neutralise(c, dbias); + let s2 = screen_diff(cs, pi, o1, o2, bal).max(0.0); + cs[pi] -= s2; + if fg_cc { + cs = correct(cs, fs, fc, fbr); + } + [cs[0], cs[1], cs[2], 1.0] + } + 3 => grey(edge), + 4 => grey(sm), + 5 => grey(inside.as_ref().map_or(0.0, |p| p.data[i])), + 6 => grey(outside.as_ref().map_or(0.0, |p| p.data[i])), + 7 => grey(a), + 8 => { + let v = if a <= 1e-3 { + 0.0 + } else if a >= 0.999 { + 1.0 + } else { + 0.5 + }; + grey(v) + } + 9 => premul(intermediate, sm), + _ => { + if cropped { + [0.0; 4] + } else if unpremultiply { + premul(fg, a) + } else { + // Keylight's premultiplied output read as straight colour. + premul(fg.map(|v| v * a), a) + } + } + }; + }); + b +} + +pub fn specs() -> Vec { + let pp = |id: &'static str, name: &'static str, v: (Value, ParamUi)| p(id, name, v.0, v.1); + let cc = |id: &'static str, name: &'static str| p(id, name, num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 1)); + vec![EffectSpec { + id: "ec.keying.keylight", + name: "Key Light", + category: "Keying", + params: vec![ + p("view", "View", Value::Enum(10), popup(&VIEWS)), + p("unpremultiplyResult", "Unpremultiply Result", Value::Bool(true), ParamUi::Checkbox), + p("screenColour", "Screen Colour", col(0.0, 1.0, 0.0), ParamUi::Color), + p("screenGain", "Screen Gain", num(100.0), slider(0.0, 200.0, 0.0, 200.0, 1)), + p("screenBalance", "Screen Balance", num(50.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("despillBias", "Despill Bias", col(0.5, 0.5, 0.5), ParamUi::Color), + p("alphaBias", "Alpha Bias", col(0.5, 0.5, 0.5), ParamUi::Color), + p("lockBiasesTogether", "Lock Biases Together", Value::Bool(true), ParamUi::Checkbox), + p("screenPreblur", "Screen Pre-blur", num(0.0), slider(0.0, 100.0, 0.0, 10.0, 1)), + // Screen Matte + pp("clipBlack", "Clip Black", pct(0.0)), + pp("clipWhite", "Clip White", pct(100.0)), + p("clipRollback", "Clip Rollback", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("screenShrinkGrow", "Screen Shrink/Grow", num(0.0), slider(-100.0, 100.0, -10.0, 10.0, 1)), + p("screenSoftness", "Screen Softness", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("screenDespotBlack", "Screen Despot Black", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("screenDespotWhite", "Screen Despot White", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("replaceMethod", "Replace Method", Value::Enum(3), popup(&REPLACE)), + p("replaceColour", "Replace Colour", col(0.5, 0.5, 0.5), ParamUi::Color), + // Inside Mask + p("insideMask", "Inside Mask", Value::Enum(0), mask_popup()), + p("insideMaskSoftness", "Inside Mask Softness", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("invertInsideMask", "Invert", Value::Bool(false), ParamUi::Checkbox), + p("insideReplaceMethod", "Inside Replace Method", Value::Enum(1), popup(&REPLACE)), + p("insideReplaceColour", "Inside Replace Colour", col(0.5, 0.5, 0.5), ParamUi::Color), + p("sourceAlpha", "Source Alpha", Value::Enum(2), popup(&["Ignore", "Add to Inside Mask", "Normal"])), + // Outside Mask + p("outsideMask", "Outside Mask", Value::Enum(0), mask_popup()), + p("outsideMaskSoftness", "Outside Mask Softness", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("invertOutsideMask", "Invert Outside Mask", Value::Bool(false), ParamUi::Checkbox), + // Foreground Colour Correction + p("enableColourCorrection", "Enable Colour Correction", Value::Bool(false), ParamUi::Checkbox), + p("saturation", "Saturation", num(100.0), slider(0.0, 400.0, 0.0, 200.0, 1)), + cc("contrast", "Contrast"), + cc("brightness", "Brightness"), + // Edge Colour Correction + p("enableEdgeColourCorrection", "Enable Edge Colour Correction", Value::Bool(false), ParamUi::Checkbox), + pp("edgeHardness", "Edge Hardness", pct(0.0)), + p("edgeSoftness", "Edge Softness", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("edgeGrow", "Edge Grow", num(1.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("edgeSaturation", "Edge Saturation", num(100.0), slider(0.0, 400.0, 0.0, 200.0, 1)), + cc("edgeContrast", "Edge Contrast"), + cc("edgeBrightness", "Edge Brightness"), + // Source Crops + pp("cropLeft", "Left", pct(0.0)), + pp("cropRight", "Right", pct(0.0)), + pp("cropTop", "Top", pct(0.0)), + pp("cropBottom", "Bottom", pct(0.0)), + ], + render: key_light, + gpu: false, + float: true, + }] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, MaskShape, run_fx}; + + /// Green screen with a red square (fg) and a half-transparent mix strip. + fn plate() -> Image { + crate::util::gen_image(40, 30, |x, y| { + if (10..20).contains(&x) && (10..20).contains(&y) { + [0.8, 0.2, 0.1, 1.0] + } else if (25..30).contains(&x) { + // 50 % red over green. + [0.4, 0.6, 0.05, 1.0] + } else { + [0.1, 0.9, 0.1, 1.0] + } + }) + } + + fn green() -> Value { + col(0.1, 0.9, 0.1) + } + + #[test] + fn keys_the_screen_and_keeps_the_foreground() { + let out = run_fx("ec.keying.keylight", &[("screenColour", green())], plate(), 0.0, EffectEnv::default()); + let bg = out.img.get(2, 2); + let fg = out.img.get(15, 15); + let mix = out.img.get(27, 5); + assert!(bg[3] < 0.02, "{bg:?}"); + assert!(fg[3] > 0.98 && (fg[0] - 0.8).abs() < 0.05, "{fg:?}"); + assert!(mix[3] > 0.3 && mix[3] < 0.7, "{mix:?}"); + // The recovered edge colour has the green removed (no spill). + let (c, _) = unpremul(mix); + assert!(c[1] <= c[0] + 0.05, "{c:?}"); + // Deterministic. + let again = run_fx("ec.keying.keylight", &[("screenColour", green())], plate(), 0.0, EffectEnv::default()); + assert_eq!(out.img.data, again.img.data); + } + + #[test] + fn clip_gain_views_and_masks() { + // Clip White 60 %: the half-transparent strip becomes solid. + let out = run_fx("ec.keying.keylight", &[("screenColour", green()), ("clipWhite", num(40.0))], plate(), 0.0, EffectEnv::default()); + assert!(out.img.get(27, 5)[3] > 0.99); + // Screen Matte and Status views are grey-scale and opaque. + let sm = run_fx("ec.keying.keylight", &[("screenColour", green()), ("view", Value::Enum(4))], plate(), 0.0, EffectEnv::default()); + assert!(sm.img.get(2, 2)[0] < 0.02 && sm.img.get(15, 15)[0] > 0.98 && sm.img.get(2, 2)[3] == 1.0); + let st = run_fx("ec.keying.keylight", &[("screenColour", green()), ("view", Value::Enum(8))], plate(), 0.0, EffectEnv::default()); + assert_eq!(st.img.get(27, 5)[0], 0.5); + // Source view is the identity. + let src = run_fx("ec.keying.keylight", &[("view", Value::Enum(0))], plate(), 0.0, EffectEnv::default()); + assert_eq!(src.img.data, plate().data); + // Inside mask forces opacity; outside mask forces transparency. + let sq = |x0: f64, y0: f64, x1: f64, y1: f64| MaskShape { + name: "m".into(), + points: vec![[x0, y0], [x1, y0], [x1, y1], [x0, y1]], + closed: true, + inverted: false, + }; + let masks = [sq(0.0, 0.0, 6.0, 6.0), sq(12.0, 12.0, 18.0, 18.0)]; + let env = EffectEnv { masks: &masks, ..Default::default() }; + let m = run_fx("ec.keying.keylight", &[("screenColour", green()), ("insideMask", Value::Enum(1)), ("outsideMask", Value::Enum(2))], plate(), 0.0, env); + assert!(m.img.get(2, 2)[3] > 0.99, "{:?}", m.img.get(2, 2)); + assert!(m.img.get(15, 15)[3] < 0.01); + // Screen gain above 100 % keys more of the strip. + let g = run_fx("ec.keying.keylight", &[("screenColour", green()), ("screenGain", num(150.0))], plate(), 0.0, EffectEnv::default()); + assert!(g.img.get(27, 5)[3] < out.img.get(27, 5)[3]); + assert_eq!(crate::lookup("Keylight (1.2)").map(|s| s.id), Some("ec.keying.keylight")); + } +} diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 6d11e19a..46da720e 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -8,8 +8,10 @@ pub mod audio_fx; mod blur2; +mod blur3; mod channel; mod channel2; +mod channel3d; mod color2; mod color3; mod color_fx; @@ -17,22 +19,27 @@ mod controls; mod distort; mod distort2; mod distort3; +pub mod distort4; mod generate; mod generate2; mod generate3; mod keying; mod keying2; +pub mod keylight; mod matte; mod misc; +pub mod mocha_shape; mod noise; mod noise2; mod obsolete; +mod ocio; pub mod paint; mod perspective; mod perspective2; pub mod puppet; mod sim; mod sim2; +mod sim3; mod stylize2; mod stylize3; mod textfx; @@ -42,6 +49,7 @@ mod transition2; pub mod util; mod utility; mod utility2; +mod vr; pub mod warp_stab; use std::collections::HashMap; @@ -51,7 +59,7 @@ pub use color2::Curve; use effectcraft_keyframe::Value; use effectcraft_project::build::Ids; use effectcraft_project::{GroupKind, ParamUi, PropGroup, Property}; -pub use effectcraft_raster::Image; +pub use effectcraft_raster::{AuxChannels, Image}; /// Effect categories in Effects & Presets order. pub const CATEGORIES: &[&str] = &[ @@ -232,6 +240,13 @@ pub trait EffectHost: Sync { fn layer_at(&self, _id: u64, _comp_time: f64, _masks_and_effects: bool) -> Option { None } + /// Auxiliary 3D channels (depth, object / material IDs, normals, Cryptomatte, any named + /// EXR channel) of the effect's own layer source at the current time, in layer space (see + /// [`AuxChannels`]). `None` when the source has none (the 3D Channel effects then pass the + /// layer through). + fn aux(&self) -> Option> { + None + } } /// Extra context the renderer may supply (all optional; `Default` is "nothing known"). @@ -246,6 +261,10 @@ pub struct EffectEnv<'a> { pub frame_rate: f64, /// Index of the running effect in its layer's stack (for [`EffectHost::self_at`]). pub effect_index: usize, + /// The project's working colour space (`None` = unmanaged, treated as sRGB). + pub working_space: Option, + /// Working-space pixels are linear light (Linearize Working Space). + pub working_linear: bool, } /// What an effect gets to render with. @@ -349,6 +368,14 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(noise2::specs()); v.extend(color3::specs()); v.extend(obsolete::specs()); + v.extend(ocio::specs()); + v.extend(vr::specs()); + v.extend(blur3::specs()); + v.extend(distort4::specs()); + v.extend(keylight::specs()); + v.extend(mocha_shape::specs()); + v.extend(sim3::specs()); + v.extend(channel3d::specs()); v.extend(time_fx::specs()); v.extend(audio_fx::specs()); v.extend(paint::specs()); @@ -370,7 +397,10 @@ pub fn find(id: &str) -> Option<&'static EffectSpec> { /// Find by id or (case-insensitive) display name. pub fn lookup(name_or_id: &str) -> Option<&'static EffectSpec> { - find(name_or_id).or_else(|| registry().iter().find(|s| s.name.eq_ignore_ascii_case(name_or_id))) + find(name_or_id).or_else(|| registry().iter().find(|s| s.name.eq_ignore_ascii_case(name_or_id))).or_else(|| { + // After Effects' third-party display names we register under a generic name. + keylight::KEYLIGHT_ALIASES.iter().any(|a| a.eq_ignore_ascii_case(name_or_id)).then(|| find("ec.keying.keylight")).flatten() + }) } /// A parameter's default value on a layer of `layer_size` (point defaults are fractions of the @@ -492,6 +522,10 @@ pub const TIME_DEPENDENT: &[&str] = &[ "ec.time.ccforcemotionblur", "ec.time.ccwidetime", "ec.time.pixelmotionblur", + "ec.vr.digitalglitch", + "ec.blur.camerashakedeblur", + "ec.distort.rollingshutterrepair", + "ec.obsolete.mochashape", // Each frame gets its own stabilizing warp. warp_stab::ID, ]; diff --git a/crates/effects/src/mocha_shape.rs b/crates/effects/src/mocha_shape.rs new file mode 100644 index 00000000..0fafce9e --- /dev/null +++ b/crates/effects/src/mocha_shape.rs @@ -0,0 +1,243 @@ +//! Obsolete ▸ mocha shape: renders planar-tracked roto shapes as a matte. +//! +//! Mocha's own clipboard / export formats are not publicly specified, so the effect reads shape +//! data in a small documented JSON format of ours (in the hidden `shapeData` parameter, either the +//! JSON itself or a path to a `.json` file), which converters or agents can produce: +//! +//! ```json +//! {"shapes": [{"name": "Shape 1", "closed": true, "inverted": false, "feather": 2.0, +//! "points": [[x, y], …], +//! "frames": [{"time": 0.0, "points": [[x, y], …]}, {"time": 1.0, "points": […]}]}]} +//! ``` +//! +//! Coordinates are layer pixels. `frames` (layer time in seconds, linearly interpolated between +//! keys with the same vertex count, held outside) override the static `points`. Shapes combine +//! with Blend Mode (add / subtract / intersect / difference) into a coverage matte that is used +//! per Render Type: Shape Cutout (layer × matte), Color Composite (shape colour over the layer) +//! or Color Shape Cutout (shape colour inside the matte only). + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use rayon::prelude::*; + +use crate::util::{Plane, gauss_plane, point_in_poly}; +use crate::{Buf, EffectCtx, EffectSpec, col, num, p, popup, slider}; + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct MochaShape { + pub name: String, + pub closed: bool, + pub inverted: bool, + pub feather: f64, + pub points: Vec<[f64; 2]>, + /// (layer time, points), sorted by time. + pub frames: Vec<(f64, Vec<[f64; 2]>)>, +} + +impl MochaShape { + /// Outline at layer time `t`. + pub fn at(&self, t: f64) -> Vec<[f64; 2]> { + if self.frames.is_empty() { + return self.points.clone(); + } + let i = self.frames.partition_point(|f| f.0 <= t); + if i == 0 { + return self.frames[0].1.clone(); + } + if i == self.frames.len() { + return self.frames[i - 1].1.clone(); + } + let (a, b) = (&self.frames[i - 1], &self.frames[i]); + if a.1.len() != b.1.len() { + return a.1.clone(); + } + let u = ((t - a.0) / (b.0 - a.0).max(1e-12)).clamp(0.0, 1.0); + a.1.iter().zip(&b.1).map(|(p, q)| [p[0] + (q[0] - p[0]) * u, p[1] + (q[1] - p[1]) * u]).collect() + } +} + +fn pts(v: &serde_json::Value) -> Vec<[f64; 2]> { + v.as_array() + .map(|a| { + a.iter() + .filter_map(|p| { + let p = p.as_array()?; + Some([p.first()?.as_f64()?, p.get(1)?.as_f64()?]) + }) + .collect() + }) + .unwrap_or_default() +} + +/// Parse shape data JSON (see the module docs). `None` when it is not valid. +pub fn parse_shapes(text: &str) -> Option> { + let v: serde_json::Value = serde_json::from_str(text).ok()?; + let list = v.get("shapes").and_then(|s| s.as_array()).or_else(|| v.as_array())?; + let mut out = vec![]; + for s in list { + let mut frames: Vec<(f64, Vec<[f64; 2]>)> = s + .get("frames") + .and_then(|f| f.as_array()) + .map(|a| a.iter().filter_map(|f| Some((f.get("time")?.as_f64()?, pts(f.get("points")?)))).collect()) + .unwrap_or_default(); + frames.sort_by(|a, b| a.0.total_cmp(&b.0)); + out.push(MochaShape { + name: s.get("name").and_then(|n| n.as_str()).unwrap_or("").to_string(), + closed: s.get("closed").and_then(|c| c.as_bool()).unwrap_or(true), + inverted: s.get("inverted").and_then(|c| c.as_bool()).unwrap_or(false), + feather: s.get("feather").and_then(|c| c.as_f64()).unwrap_or(0.0).max(0.0), + points: s.get("points").map(pts).unwrap_or_default(), + frames, + }); + } + Some(out) +} + +type ShapeCache = Mutex>>>>; + +fn load(src: &str) -> Option>> { + let s = src.trim(); + if s.is_empty() { + return None; + } + static CACHE: OnceLock = OnceLock::new(); + let cache = CACHE.get_or_init(|| Mutex::new(HashMap::new())); + if let Some(v) = cache.lock().ok().and_then(|c| c.get(src).cloned()) { + return v; + } + let parsed = + if s.starts_with('{') || s.starts_with('[') { parse_shapes(s) } else { std::fs::read_to_string(s).ok().and_then(|t| parse_shapes(&t)) }.map(Arc::new); + if let Ok(mut c) = cache.lock() { + if c.len() > 64 { + c.clear(); + } + c.insert(src.to_string(), parsed.clone()); + } + parsed +} + +fn mocha_shape(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(shapes) = load(ctx.params.s("shapeData")) else { return b }; + let (w, h) = (b.img.width as usize, b.img.height as usize); + let blend = ctx.params.e("blendMode"); + let inv = 1.0 / b.scale.max(1e-9); + let mut matte: Option = None; + for s in shapes.iter() { + let outline = s.at(ctx.time); + if outline.len() < 3 || !s.closed { + continue; + } + let mut cov = Plane::new(w, h); + cov.data.par_chunks_mut(w.max(1)).enumerate().for_each(|(y, row)| { + for (x, v) in row.iter_mut().enumerate() { + let lx = (x as f64 + 0.5 - b.offset[0]) * inv; + let ly = (y as f64 + 0.5 - b.offset[1]) * inv; + *v = if point_in_poly(&outline, lx, ly) != s.inverted { 1.0 } else { 0.0 }; + } + }); + if s.feather > 0.0 { + let f = s.feather * b.scale * 0.5; + cov = gauss_plane(&cov, f, f); + } + matte = Some(match matte { + None => cov, + Some(m) => m.zip_map(&cov, |a, c| match blend { + 1 => a * (1.0 - c), + 2 => a * c, + 3 => (a - c).abs(), + _ => a + c - a * c, + }), + }); + } + let Some(mut m) = matte else { return b }; + if ctx.params.b("invert") { + m = m.map(|v| 1.0 - v); + } + let kind = ctx.params.e("renderType"); + let c = ctx.params.color("shapeColour"); + let op = (ctx.params.f("opacity") / 100.0) as f32; + let edge = ctx.params.b("renderEdgeWidth"); + b.img.data.par_iter_mut().zip(m.data.par_iter()).for_each(|(px, &k)| { + let k = if edge { (1.0 - (2.0 * k - 1.0).abs()).clamp(0.0, 1.0) } else { k.clamp(0.0, 1.0) }; + match kind { + 1 => { + let a = k * op; + let s = [c[0] * a, c[1] * a, c[2] * a, a]; + *px = [s[0] + px[0] * (1.0 - a), s[1] + px[1] * (1.0 - a), s[2] + px[2] * (1.0 - a), s[3] + px[3] * (1.0 - a)]; + } + 2 => { + let a = k * op; + *px = [c[0] * a, c[1] * a, c[2] * a, a]; + } + _ => { + let a = 1.0 - (1.0 - k) * op; + *px = px.map(|v| v * a); + } + } + }); + b +} + +pub fn specs() -> Vec { + vec![EffectSpec { + id: "ec.obsolete.mochashape", + name: "mocha shape", + category: "Obsolete", + params: vec![ + p("blendMode", "Blend mode", Value::Enum(0), popup(&["Add", "Subtract", "Intersect", "Difference"])), + p("invert", "Invert", Value::Bool(false), ParamUi::Checkbox), + p("renderEdgeWidth", "Render edge width", Value::Bool(false), ParamUi::Checkbox), + p("renderType", "Render type", Value::Enum(0), popup(&["Shape cutout", "Color composite", "Color shape cutout"])), + p("shapeColour", "Shape colour", col(1.0, 1.0, 1.0), ParamUi::Color), + p("opacity", "Opacity", num(100.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + // JSON shape data (see the module docs) or a path to a .json file. + p("shapeData", "Shape data", Value::Str(String::new()), ParamUi::Hidden), + ], + render: mocha_shape, + gpu: false, + float: true, + }] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, Image, run_fx}; + + const DATA: &str = r#"{"shapes":[{"name":"S","closed":true,"points":[[2,2],[10,2],[10,10],[2,10]], + "frames":[{"time":0,"points":[[2,2],[10,2],[10,10],[2,10]]},{"time":1,"points":[[12,2],[20,2],[20,10],[12,10]]}]}]}"#; + + #[test] + fn parses_and_interpolates() { + let s = parse_shapes(DATA).unwrap(); + assert_eq!(s.len(), 1); + assert_eq!(s[0].at(0.5)[0], [7.0, 2.0]); + assert_eq!(s[0].at(5.0)[0], [12.0, 2.0]); + assert!(parse_shapes("not json").is_none()); + } + + #[test] + fn cuts_out_the_shape_over_time() { + let img = Image::filled(24, 12, [1.0, 0.0, 0.0, 1.0]); + let a = run_fx("ec.obsolete.mochashape", &[("shapeData", Value::Str(DATA.into()))], img.clone(), 0.0, EffectEnv::default()); + assert_eq!(a.img.get(5, 5)[3], 1.0); + assert_eq!(a.img.get(15, 5)[3], 0.0); + let b = run_fx("ec.obsolete.mochashape", &[("shapeData", Value::Str(DATA.into()))], img.clone(), 1.0, EffectEnv::default()); + assert_eq!(b.img.get(5, 5)[3], 0.0); + assert_eq!(b.img.get(15, 5)[3], 1.0); + // Color shape cutout fills with the shape colour; no data = pass-through. + let c = run_fx( + "ec.obsolete.mochashape", + &[("shapeData", Value::Str(DATA.into())), ("renderType", Value::Enum(2)), ("shapeColour", col(0.0, 1.0, 0.0))], + img.clone(), + 0.0, + EffectEnv::default(), + ); + assert_eq!(c.img.get(5, 5), [0.0, 1.0, 0.0, 1.0]); + let none = run_fx("ec.obsolete.mochashape", &[], img.clone(), 0.0, EffectEnv::default()); + assert_eq!(none.img.data, img.data); + } +} diff --git a/crates/effects/src/ocio.rs b/crates/effects/src/ocio.rs new file mode 100644 index 00000000..d868a8f9 --- /dev/null +++ b/crates/effects/src/ocio.rs @@ -0,0 +1,1281 @@ +//! OCIO-style colour transforms (Color Correction ▸ OCIO CDL / Color Space / Display / File / Look +//! Transform), Color Profile Converter (Utility) and Color Stabilizer (Color Correction). +//! +//! The OCIO effects run on a **built-in minimal configuration** written from published +//! standards, not from OpenColorIO's source: +//! +//! * colour spaces are defined by primaries, white point and transfer function: ACES2065-1 (AP0) +//! and ACEScg / ACEScct (AP1) from SMPTE ST 2065-1 and Academy S-2014-004 / S-2016-001, sRGB +//! (IEC 61966-2-1), Rec. 709 / Rec. 2020 with the BT.1886 gamma 2.4 EOTF, Display P3 (SMPTE +//! EG 432-1 primaries, D65, sRGB curve), their linear variants and CIE XYZ D65; +//! * the reference space is CIE XYZ D65; spaces with another white point (the ACES white, +//! x 0.32168 y 0.33767) are adapted with the Bradford chromatic adaptation transform, as in the +//! published ACES configurations; +//! * displays (sRGB, Rec.1886 Rec.709, Display P3, Rec.2100-PQ via SMPTE ST 2084) and views +//! (Standard = clip-free encode, Tone Mapped = extended Reinhard roll-off, Raw); +//! * looks are ASC CDLs applied in ACEScct; +//! * CDL maths follows the ASC CDL v1.2 specification (slope, offset, power, then saturation with +//! Rec. 709 luma weights); the "No Clamp" style mirrors power around zero; +//! * File Transform reads `.cube` (Adobe/Resolve), `.3dl` (Autodesk Lustre/Flame integer +//! lattices), `.csp` (cineSpace, with its per-channel pre-LUT shaper) and ASC `.cc` / `.ccc` / +//! `.cdl` XML, with nearest, trilinear or tetrahedral interpolation and an inverse direction +//! (exact for 1D/CDL, iterative for 3D lattices). + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_color::ColorSpace; +use effectcraft_color::space::{invert, mul, mul_vec, rgb_to_xyz}; +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use rayon::prelude::*; + +use crate::util::unpremul; +use crate::utility::{Lut, parse_cube}; +use crate::{Buf, EffectCtx, EffectSpec, num, p, popup, slider}; + +type M3 = [[f64; 3]; 3]; + +const D65: [f64; 2] = [0.3127, 0.3290]; +const ACES_WHITE: [f64; 2] = [0.32168, 0.33767]; +const AP0: [[f64; 2]; 3] = [[0.7347, 0.2653], [0.0, 1.0], [0.0001, -0.0770]]; +const AP1: [[f64; 2]; 3] = [[0.713, 0.293], [0.165, 0.830], [0.128, 0.044]]; +const REC709: [[f64; 2]; 3] = [[0.640, 0.330], [0.300, 0.600], [0.150, 0.060]]; +const REC2020: [[f64; 2]; 3] = [[0.708, 0.292], [0.170, 0.797], [0.131, 0.046]]; +const P3: [[f64; 2]; 3] = [[0.680, 0.320], [0.265, 0.690], [0.150, 0.060]]; +const IDENTITY: M3 = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]; + +/// Transfer functions (encoded ↔ linear). +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum Tf { + Linear, + Srgb, + Gamma(f64), + AcesCct, + /// SMPTE ST 2084 with 1.0 = 100 cd/m². + Pq, +} + +impl Tf { + pub fn decode(self, v: f64) -> f64 { + match self { + Tf::Linear => v, + Tf::Srgb => { + let a = v.abs(); + let l = if a <= 0.04045 { a / 12.92 } else { ((a + 0.055) / 1.055).powf(2.4) }; + l.copysign(v) + } + Tf::Gamma(g) => v.abs().powf(g).copysign(v), + Tf::AcesCct => { + if v <= 0.155_251_141_552_511 { + (v - 0.072_905_534_195_835_5) / 10.540_237_741_654_5 + } else if v < (65504f64.log2() + 9.72) / 17.52 { + 2f64.powf(v * 17.52 - 9.72) + } else { + 65504.0 + } + } + Tf::Pq => { + let (m1, m2, c1, c2, c3) = pq_consts(); + let e = v.max(0.0).powf(1.0 / m2); + let l = ((e - c1).max(0.0) / (c2 - c3 * e)).powf(1.0 / m1); + l * 100.0 + } + } + } + pub fn encode(self, v: f64) -> f64 { + match self { + Tf::Linear => v, + Tf::Srgb => { + let a = v.abs(); + let e = if a <= 0.003_130_8 { a * 12.92 } else { 1.055 * a.powf(1.0 / 2.4) - 0.055 }; + e.copysign(v) + } + Tf::Gamma(g) => v.abs().powf(1.0 / g).copysign(v), + Tf::AcesCct => { + if v <= 0.007_812_5 { + 10.540_237_741_654_5 * v + 0.072_905_534_195_835_5 + } else { + (v.log2() + 9.72) / 17.52 + } + } + Tf::Pq => { + let (m1, m2, c1, c2, c3) = pq_consts(); + let y = (v / 100.0).max(0.0).powf(m1); + ((c1 + c2 * y) / (1.0 + c3 * y)).powf(m2) + } + } + } +} + +fn pq_consts() -> (f64, f64, f64, f64, f64) { + (2610.0 / 16384.0, 2523.0 / 4096.0 * 128.0, 3424.0 / 4096.0, 2413.0 / 4096.0 * 32.0, 2392.0 / 4096.0 * 32.0) +} + +/// A colour space of the built-in config. +#[derive(Clone, Copy, Debug)] +pub struct Space { + pub name: &'static str, + /// Primaries and white (None = CIE XYZ itself). + pub prims: Option<([[f64; 2]; 3], [f64; 2])>, + pub tf: Tf, + /// Passes data through untouched. + pub raw: bool, +} + +const fn sp(name: &'static str, prims: [[f64; 2]; 3], white: [f64; 2], tf: Tf) -> Space { + Space { name, prims: Some((prims, white)), tf, raw: false } +} + +/// The built-in configuration's colour spaces, in popup order. +pub const SPACES: &[Space] = &[ + sp("ACES2065-1", AP0, ACES_WHITE, Tf::Linear), + sp("ACEScg", AP1, ACES_WHITE, Tf::Linear), + sp("ACEScct", AP1, ACES_WHITE, Tf::AcesCct), + sp("Linear Rec.709 (sRGB)", REC709, D65, Tf::Linear), + sp("sRGB", REC709, D65, Tf::Srgb), + sp("Gamma 2.4 Rec.709", REC709, D65, Tf::Gamma(2.4)), + sp("Linear Rec.2020", REC2020, D65, Tf::Linear), + sp("Gamma 2.4 Rec.2020", REC2020, D65, Tf::Gamma(2.4)), + sp("Linear P3-D65", P3, D65, Tf::Linear), + sp("Display P3", P3, D65, Tf::Srgb), + Space { name: "CIE-XYZ-D65", prims: None, tf: Tf::Linear, raw: false }, + Space { name: "Raw", prims: None, tf: Tf::Linear, raw: true }, +]; + +pub fn space_names() -> Vec<&'static str> { + SPACES.iter().map(|s| s.name).collect() +} + +pub fn space_by_name(n: &str) -> Option<&'static Space> { + SPACES.iter().find(|s| s.name.eq_ignore_ascii_case(n)) +} + +/// Bradford chromatic adaptation from white `src` to white `dst` (xy). +pub fn bradford(src: [f64; 2], dst: [f64; 2]) -> M3 { + if src == dst { + return IDENTITY; + } + const MB: M3 = [[0.8951, 0.2664, -0.1614], [-0.7502, 1.7135, 0.0367], [0.0389, -0.0685, 1.0296]]; + let xyz = |c: [f64; 2]| [c[0] / c[1], 1.0, (1.0 - c[0] - c[1]) / c[1]]; + let s = mul_vec(&MB, xyz(src)); + let d = mul_vec(&MB, xyz(dst)); + let diag = [[d[0] / s[0], 0.0, 0.0], [0.0, d[1] / s[1], 0.0], [0.0, 0.0, d[2] / s[2]]]; + mul(&invert(&MB), &mul(&diag, &MB)) +} + +impl Space { + /// Linear RGB → XYZ D65 (Bradford-adapted when `adapt`, else the space's own white). + pub fn to_ref(self, adapt: bool) -> M3 { + match self.prims { + None => IDENTITY, + Some((p, w)) => { + let m = rgb_to_xyz(p, w); + if adapt { mul(&bradford(w, D65), &m) } else { m } + } + } + } +} + +/// Linear-RGB matrix from space `a` to space `b`. +pub fn matrix(a: &Space, b: &Space) -> M3 { + mul(&invert(&b.to_ref(true)), &a.to_ref(true)) +} + +/// A compiled colour-space conversion. +#[derive(Clone, Copy, Debug)] +pub struct Xform { + dec: Tf, + m: [[f32; 3]; 3], + ident: bool, + enc: Tf, +} + +impl Xform { + pub fn new(a: &Space, b: &Space) -> Xform { + if a.raw || b.raw { + return Xform { dec: Tf::Linear, m: IDENTITY.map(|r| r.map(|v| v as f32)), ident: true, enc: Tf::Linear }; + } + let m = matrix(a, b); + let ident = (0..3).all(|i| (0..3).all(|j| (m[i][j] - IDENTITY[i][j]).abs() < 1e-12)); + Xform { dec: a.tf, m: m.map(|r| r.map(|v| v as f32)), ident, enc: b.tf } + } + #[inline] + pub fn apply(&self, c: [f32; 3]) -> [f32; 3] { + let l = c.map(|v| self.dec.decode(v as f64) as f32); + let l = if self.ident { l } else { [0, 1, 2].map(|i| self.m[i][0] * l[0] + self.m[i][1] * l[1] + self.m[i][2] * l[2]) }; + l.map(|v| self.enc.encode(v as f64) as f32) + } +} + +// ---------------------------------------------------------------- ASC CDL + +/// An ASC Colour Decision List correction. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Cdl { + pub slope: [f64; 3], + pub offset: [f64; 3], + pub power: [f64; 3], + pub sat: f64, +} + +impl Default for Cdl { + fn default() -> Self { + Cdl { slope: [1.0; 3], offset: [0.0; 3], power: [1.0; 3], sat: 1.0 } + } +} + +const LUMA: [f64; 3] = [0.2126, 0.7152, 0.0722]; + +impl Cdl { + /// Forward CDL. `clamp` = the v1.2 style (results clamped to 0..1 after SOP and saturation). + pub fn apply(&self, c: [f64; 3], clamp: bool) -> [f64; 3] { + let mut o = [0.0; 3]; + for i in 0..3 { + let v = c[i] * self.slope[i] + self.offset[i]; + o[i] = if clamp { + v.clamp(0.0, 1.0).powf(self.power[i]) + } else if v >= 0.0 { + v.powf(self.power[i]) + } else { + v + }; + } + let l = o[0] * LUMA[0] + o[1] * LUMA[1] + o[2] * LUMA[2]; + let o = o.map(|v| l + self.sat * (v - l)); + if clamp { o.map(|v| v.clamp(0.0, 1.0)) } else { o } + } + /// Exact inverse of [`Cdl::apply`] (inside the clamped range). + pub fn invert(&self, c: [f64; 3], clamp: bool) -> [f64; 3] { + let c = if clamp { c.map(|v| v.clamp(0.0, 1.0)) } else { c }; + let l = c[0] * LUMA[0] + c[1] * LUMA[1] + c[2] * LUMA[2]; + let s = if self.sat.abs() < 1e-9 { 1e-9 } else { self.sat }; + let o = c.map(|v| l + (v - l) / s); + let mut r = [0.0; 3]; + for i in 0..3 { + let v = if clamp { o[i].clamp(0.0, 1.0) } else { o[i] }; + let pw = self.power[i].max(1e-9); + let u = if v >= 0.0 { v.powf(1.0 / pw) } else { v }; + let sl = if self.slope[i].abs() < 1e-9 { 1e-9 } else { self.slope[i] }; + r[i] = (u - self.offset[i]) / sl; + } + if clamp { r.map(|v| v.clamp(0.0, 1.0)) } else { r } + } +} + +fn tag<'a>(xml: &'a str, name: &str) -> Option<&'a str> { + let open = xml.find(&format!("<{name}"))?; + let gt = open + xml[open..].find('>')?; + if xml[..gt].ends_with('/') { + return Some(""); + } + let close = xml[gt..].find(&format!(""))? + gt; + Some(&xml[gt + 1..close]) +} + +fn nums3(s: &str) -> Option<[f64; 3]> { + let v: Vec = s.split_whitespace().filter_map(|t| t.parse().ok()).collect(); + (v.len() >= 3).then(|| [v[0], v[1], v[2]]) +} + +fn parse_cc(block: &str) -> Cdl { + let mut c = Cdl::default(); + if let Some(s) = tag(block, "Slope").and_then(nums3) { + c.slope = s; + } + if let Some(s) = tag(block, "Offset").and_then(nums3) { + c.offset = s; + } + if let Some(s) = tag(block, "Power").and_then(nums3) { + c.power = s; + } + if let Some(s) = tag(block, "Saturation").and_then(|t| t.trim().parse().ok()) { + c.sat = s; + } + c +} + +/// Parse ASC CDL XML (`.cc`, `.ccc`, `.cdl`): every `` with its `id`. +pub fn parse_cdl_xml(xml: &str) -> Vec<(String, Cdl)> { + let mut out = vec![]; + let mut rest = xml; + while let Some(i) = rest.find("').unwrap_or(after.len()); + let head = &after[..head_end]; + let id = head + .find("id=") + .and_then(|k| { + let q = head[k + 3..].chars().next()?; + let s = &head[k + 4..]; + s.find(q).map(|e| s[..e].to_string()) + }) + .unwrap_or_default(); + let end = after.find("").map(|e| e + "".len()).unwrap_or(after.len()); + out.push((id, parse_cc(&after[..end]))); + rest = &after[end..]; + } + out +} + +// ---------------------------------------------------------------- LUT files + +/// A loaded transform file. +#[derive(Clone, Debug)] +pub enum FileXform { + /// A lattice/1D LUT with an optional per-channel piecewise-linear shaper (cineSpace pre-LUT). + Lut { lut: Lut, shaper: Option<[Vec<(f32, f32)>; 3]> }, + /// CDL corrections by id (in file order). + Cdl(Vec<(String, Cdl)>), +} + +/// Parse an Autodesk `.3dl` lattice (integers, blue fastest; an optional input-mesh line). +pub fn parse_3dl(text: &str) -> Option { + let mut mesh: Option> = None; + let mut rows: Vec<[f64; 3]> = vec![]; + let mut out_bits: Option = None; + for line in text.lines() { + let l = line.trim(); + if l.is_empty() || l.starts_with('#') || l.starts_with('<') { + continue; + } + if l.starts_with(|c: char| c.is_ascii_alphabetic()) { + // "Mesh " (Lustre header); other keywords are ignored. + let t: Vec<&str> = l.split_whitespace().collect(); + if t[0].eq_ignore_ascii_case("mesh") && t.len() >= 3 { + out_bits = t[2].parse().ok(); + } + continue; + } + let v: Vec = l.split_whitespace().filter_map(|t| t.parse().ok()).collect(); + if v.len() > 3 && mesh.is_none() && rows.is_empty() { + mesh = Some(v); + } else if v.len() == 3 { + rows.push([v[0], v[1], v[2]]); + } else { + return None; + } + } + let is_cube = |k: usize| { + let n = (k as f64).cbrt().round() as usize; + n * n * n == k + }; + // A 3-entry input mesh (3³ lattice) looks like a data row: tell them apart by the count. + if mesh.is_none() && !is_cube(rows.len()) && rows.len() == 28 { + mesh = Some(rows.remove(0).to_vec()); + } + let n = match &mesh { + Some(m) => m.len(), + None => (rows.len() as f64).cbrt().round() as usize, + }; + if n < 2 || rows.len() != n * n * n { + return None; + } + let maxv = rows.iter().flat_map(|r| r.iter()).fold(0.0f64, |m, &v| m.max(v)); + let out_max = match out_bits { + Some(b) if b > 0 && b < 32 => (1u64 << b) as f64 - 1.0, + _ => [255.0, 1023.0, 4095.0, 65535.0].into_iter().find(|&m| maxv <= m).unwrap_or(maxv.max(1.0)), + }; + // .3dl rows: red slowest, blue fastest. Our lattice is red fastest. + let mut d3 = vec![[0.0f32; 3]; n * n * n]; + for (k, r) in rows.iter().enumerate() { + let (ri, gi, bi) = (k / (n * n), (k / n) % n, k % n); + d3[ri + gi * n + bi * n * n] = r.map(|v| (v / out_max) as f32); + } + Some(Lut { n1: 0, d1: vec![], n3: n, d3, dmin: [0.0; 3], dmax: [1.0; 3] }) +} + +/// Parse a cineSpace `.csp` LUT (1D or 3D, with its three pre-LUT shapers). +pub fn parse_csp(text: &str) -> Option<(Lut, [Vec<(f32, f32)>; 3])> { + let mut lines = text.lines().map(str::trim).filter(|l| !l.is_empty()); + if !lines.next()?.starts_with("CSPLUTV100") { + return None; + } + let kind = lines.next()?; + let mut lines = lines.peekable(); + if lines.peek().is_some_and(|l| l.starts_with("BEGIN METADATA")) { + for l in lines.by_ref() { + if l.starts_with("END METADATA") { + break; + } + } + } + let nums = |l: &str| -> Option> { l.split_whitespace().map(|t| t.parse().ok()).collect() }; + let mut shaper: [Vec<(f32, f32)>; 3] = Default::default(); + for ch in shaper.iter_mut() { + let n: usize = lines.next()?.parse().ok()?; + let ins = nums(lines.next()?)?; + let outs = nums(lines.next()?)?; + if ins.len() != n || outs.len() != n || n < 2 { + return None; + } + *ch = ins.into_iter().zip(outs).collect(); + } + if kind.starts_with("3D") { + let sz = nums(lines.next()?)?; + if sz.len() != 3 || sz[0] != sz[1] || sz[1] != sz[2] { + return None; + } + let n = sz[0] as usize; + let mut d3 = Vec::with_capacity(n * n * n); + for _ in 0..n * n * n { + let v = nums(lines.next()?)?; + if v.len() != 3 { + return None; + } + d3.push([v[0], v[1], v[2]]); + } + (n >= 2).then_some((Lut { n1: 0, d1: vec![], n3: n, d3, dmin: [0.0; 3], dmax: [1.0; 3] }, shaper)) + } else { + let n: usize = lines.next()?.parse().ok()?; + let mut d1 = Vec::with_capacity(n); + for _ in 0..n { + let v = nums(lines.next()?)?; + if v.len() != 3 { + return None; + } + d1.push([v[0], v[1], v[2]]); + } + (n >= 2).then_some((Lut { n1: n, d1, n3: 0, d3: vec![], dmin: [0.0; 3], dmax: [1.0; 3] }, shaper)) + } +} + +fn shape(curve: &[(f32, f32)], x: f32) -> f32 { + if curve.len() < 2 { + return x; + } + let i = curve.partition_point(|p| p.0 < x).clamp(1, curve.len() - 1); + let (a, b) = (curve[i - 1], curve[i]); + let t = ((x - a.0) / (b.0 - a.0).max(1e-12)).clamp(0.0, 1.0); + a.1 + (b.1 - a.1) * t +} + +fn shape_inv(curve: &[(f32, f32)], y: f32) -> f32 { + let flipped: Vec<(f32, f32)> = curve.iter().map(|&(a, b)| (b, a)).collect(); + shape(&flipped, y) +} + +/// Parse transform text by its format (detected from `ext` or the content). +pub fn parse_file(text: &str, ext: &str) -> Option { + let ext = ext.to_ascii_lowercase(); + if ext == "cc" || ext == "ccc" || ext == "cdl" || text.contains(">>>; + +/// Load (and cache) a transform from a file path or inline content. +pub fn load_file(src: &str) -> Option> { + let src_t = src.trim(); + if src_t.is_empty() { + return None; + } + static CACHE: OnceLock = OnceLock::new(); + let cache = CACHE.get_or_init(|| Mutex::new(HashMap::new())); + if let Ok(m) = cache.lock() + && let Some(v) = m.get(src) + { + return v.clone(); + } + let inline = src.contains('\n'); + let parsed = if inline { + parse_file(src, "") + } else { + let ext = std::path::Path::new(src_t).extension().and_then(|e| e.to_str()).unwrap_or(""); + std::fs::read_to_string(src_t).ok().and_then(|t| parse_file(&t, ext)) + } + .map(Arc::new); + if let Ok(mut m) = cache.lock() { + m.insert(src.to_string(), parsed.clone()); + } + parsed +} + +impl FileXform { + pub fn apply(&self, c: [f32; 3], interp: u32, inverse: bool, ccc_id: &str) -> [f32; 3] { + match self { + FileXform::Cdl(list) => { + let cdl = list.iter().find(|(id, _)| !ccc_id.is_empty() && id == ccc_id).or(list.first()).map(|(_, c)| *c).unwrap_or_default(); + let c64 = c.map(|v| v as f64); + let o = if inverse { cdl.invert(c64, true) } else { cdl.apply(c64, true) }; + o.map(|v| v as f32) + } + FileXform::Lut { lut, shaper } => { + let fwd = |c: [f32; 3]| { + let c = match shaper { + Some(s) => [shape(&s[0], c[0]), shape(&s[1], c[1]), shape(&s[2], c[2])], + None => c, + }; + lut.apply_interp(c, interp) + }; + if !inverse { + return fwd(c); + } + if lut.n3 == 0 { + // 1D: invert each channel exactly through the (monotone) tables. + let mut x = c; + if lut.n1 > 1 { + x = [0, 1, 2].map(|i| { + let curve: Vec<(f32, f32)> = + (0..lut.n1).map(|k| (lut.d1[k][i], lut.dmin[i] + (lut.dmax[i] - lut.dmin[i]) * k as f32 / (lut.n1 - 1) as f32)).collect(); + shape(&curve, c[i]) + }); + } + if let Some(s) = shaper { + x = [shape_inv(&s[0], x[0]), shape_inv(&s[1], x[1]), shape_inv(&s[2], x[2])]; + } + return x; + } + // 3D: damped fixed-point (Newton with an identity Jacobian) from the target. + let mut x = c; + for _ in 0..40 { + let y = fwd(x); + let e = [c[0] - y[0], c[1] - y[1], c[2] - y[2]]; + if e.iter().all(|v| v.abs() < 1e-6) { + break; + } + x = [x[0] + e[0], x[1] + e[1], x[2] + e[2]]; + } + x + } + } + } +} + +// ---------------------------------------------------------------- displays, views and looks + +/// Displays of the built-in config: (name, display primaries/white, encoding). +pub const DISPLAYS: &[(&str, [[f64; 2]; 3], Tf)] = + &[("sRGB", REC709, Tf::Srgb), ("Rec.1886 Rec.709", REC709, Tf::Gamma(2.4)), ("Display P3", P3, Tf::Srgb), ("Rec.2100-PQ", REC2020, Tf::Pq)]; + +pub const VIEWS: &[&str] = &["Standard", "Tone Mapped", "Raw"]; + +/// Built-in looks: CDLs applied in ACEScct. +pub fn looks() -> Vec<(&'static str, Cdl)> { + let grey = 0.413_588_402_492_442_1; // ACEScct of 0.18 + let c = |slope: [f64; 3], offset: [f64; 3], sat: f64| Cdl { slope, offset, power: [1.0; 3], sat }; + vec![ + ("Warm", c([1.04, 1.0, 0.94], [0.0; 3], 1.0)), + ("Cool", c([0.95, 1.0, 1.06], [0.0; 3], 1.0)), + ("High Contrast", c([1.25; 3], [grey * (1.0 - 1.25); 3], 1.0)), + ("Low Contrast", c([0.8; 3], [grey * (1.0 - 0.8); 3], 1.0)), + ("Desaturated", c([1.0; 3], [0.0; 3], 0.6)), + ("Vivid", c([1.0; 3], [0.0; 3], 1.3)), + ] +} + +/// Extended Reinhard roll-off (white point 16): maps 0..∞ into 0..1, ≈ identity near 0. +fn tonemap(v: f64) -> f64 { + let w2 = 256.0; + if v <= 0.0 { v } else { v * (1.0 + v / w2) / (1.0 + v) } +} + +fn tonemap_inv(y: f64) -> f64 { + if y <= 0.0 { + return y; + } + // Solve v² / w² + v (1 − y) − y = 0. + let (a, b, c) = (1.0 / 256.0, 1.0 - y, -y); + (-b + (b * b - 4.0 * a * c).sqrt()) / (2.0 * a) +} + +/// Display transform of an input-space colour (`inverse`: display code values → input space). +pub fn display_xform(input: &Space, display: usize, view: usize, inverse: bool) -> impl Fn([f32; 3]) -> [f32; 3] + Sync { + let (_, prims, tf) = DISPLAYS[display.min(DISPLAYS.len() - 1)]; + let disp = Space { name: "display", prims: Some((prims, D65)), tf: Tf::Linear, raw: false }; + let raw = view == 2 || input.raw; + let m_in = Xform::new(&Space { tf: Tf::Linear, ..*input }, &disp); + let m_out = Xform::new(&disp, &Space { tf: Tf::Linear, ..*input }); + let in_tf = input.tf; + move |c: [f32; 3]| { + if raw { + return c; + } + if !inverse { + let lin = c.map(|v| in_tf.decode(v as f64) as f32); + let d = m_in.apply(lin); + d.map(|v| { + let v = v as f64; + let v = if view == 1 { tonemap(v) } else { v }; + tf.encode(v) as f32 + }) + } else { + let d = c.map(|v| { + let l = tf.decode(v as f64); + (if view == 1 { tonemap_inv(l) } else { l }) as f32 + }); + m_out.apply(d).map(|v| in_tf.encode(v as f64) as f32) + } + } +} + +// ---------------------------------------------------------------- effects + +fn spec(id: &'static str, name: &'static str, cat: &'static str, params: Vec, render: crate::RenderFn) -> EffectSpec { + EffectSpec { id, name, category: cat, params, render, gpu: false, float: true } +} + +fn space_param(id: &'static str, name: &'static str, default: u32) -> crate::ParamSpec { + p(id, name, Value::Enum(default), popup(&space_names())) +} + +fn direction() -> crate::ParamSpec { + p("direction", "Direction", Value::Enum(0), popup(&["Forward", "Inverse"])) +} + +fn map_px(b: &mut Buf, f: impl Fn([f32; 3]) -> [f32; 3] + Sync) { + b.img.data.par_iter_mut().for_each(|px| { + let (c, a) = unpremul(*px); + if a <= 0.0 { + return; + } + let o = f(c); + *px = [o[0] * a, o[1] * a, o[2] * a, a]; + }); +} + +fn space_at(i: u32) -> &'static Space { + &SPACES[(i as usize).min(SPACES.len() - 1)] +} + +fn ocio_cst(ctx: &EffectCtx, mut b: Buf) -> Buf { + let (a, z) = (space_at(ctx.params.e("source")), space_at(ctx.params.e("destination"))); + let (a, z) = if ctx.params.e("direction") == 1 { (z, a) } else { (a, z) }; + if a.name == z.name { + return b; + } + let x = Xform::new(a, z); + map_px(&mut b, |c| x.apply(c)); + b +} + +fn cdl_from_params(pr: &crate::Params) -> Cdl { + Cdl { + slope: [pr.f("slopeRed"), pr.f("slopeGreen"), pr.f("slopeBlue")], + offset: [pr.f("offsetRed"), pr.f("offsetGreen"), pr.f("offsetBlue")], + power: [pr.f("powerRed").max(0.0), pr.f("powerGreen").max(0.0), pr.f("powerBlue").max(0.0)], + sat: pr.f("saturation").max(0.0), + } +} + +fn ocio_cdl(ctx: &EffectCtx, mut b: Buf) -> Buf { + let cdl = cdl_from_params(ctx.params); + if cdl == Cdl::default() { + return b; + } + let clamp = ctx.params.e("style") == 0; + let inv = ctx.params.e("direction") == 1; + map_px(&mut b, |c| { + let c = c.map(|v| v as f64); + let o = if inv { cdl.invert(c, clamp) } else { cdl.apply(c, clamp) }; + o.map(|v| v as f32) + }); + b +} + +fn ocio_display(ctx: &EffectCtx, mut b: Buf) -> Buf { + let input = space_at(ctx.params.e("source")); + let f = display_xform(input, ctx.params.e("display") as usize, ctx.params.e("view") as usize, ctx.params.e("direction") == 1); + map_px(&mut b, f); + b +} + +fn ocio_file(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(x) = load_file(ctx.params.s("file")) else { return b }; + let interp = match ctx.params.e("interpolation") { + 0 => 0, + 1 => 1, + _ => 2, + }; + let inv = ctx.params.e("direction") == 1; + let id = ctx.params.s("cccId").to_string(); + map_px(&mut b, |c| x.apply(c, interp, inv, &id)); + b +} + +/// Parse a looks string: comma/space separated names, `-Name` = inverse. +pub fn parse_looks(s: &str) -> Vec<(Cdl, bool)> { + let all = looks(); + s.split([',', ':']) + .map(str::trim) + .filter(|t| !t.is_empty()) + .filter_map(|t| { + let (inv, n) = match t.strip_prefix('-') { + Some(r) => (true, r.trim()), + None => (false, t.trim_start_matches('+').trim()), + }; + all.iter().find(|(name, _)| name.eq_ignore_ascii_case(n)).map(|(_, c)| (*c, inv)) + }) + .collect() +} + +fn ocio_look(ctx: &EffectCtx, mut b: Buf) -> Buf { + let mut list = parse_looks(ctx.params.s("looks")); + let pick = ctx.params.e("look"); + if pick > 0 + && let Some((_, c)) = looks().get(pick as usize - 1) + { + list.insert(0, (*c, false)); + } + if list.is_empty() { + return b; + } + let (src, dst) = (space_at(ctx.params.e("source")), space_at(ctx.params.e("destination"))); + let inv = ctx.params.e("direction") == 1; + let cct = space_by_name("ACEScct").expect("built-in"); + let (a, z) = if inv { (dst, src) } else { (src, dst) }; + let to_p = Xform::new(a, cct); + let from_p = Xform::new(cct, z); + if inv { + list.reverse(); + } + map_px(&mut b, |c| { + let mut v = to_p.apply(c).map(|x| x as f64); + for (cdl, li) in &list { + v = if *li != inv { cdl.invert(v, false) } else { cdl.apply(v, false) }; + } + from_p.apply(v.map(|x| x as f32)) + }); + b +} + +// ---- Color Profile Converter (Utility) + +pub const PROFILES: &[&str] = &["Project Working Space", "sRGB IEC61966-2.1", "HDTV (Rec. 709)", "Rec. 2020", "Display P3", "ACEScg", "ACES2065-1"]; + +fn profile(ctx: &EffectCtx, i: u32) -> (Space, bool) { + let ws = |cs: ColorSpace| match cs { + ColorSpace::Srgb => sp("sRGB", REC709, D65, Tf::Srgb), + ColorSpace::Rec709 => sp("Rec.709", REC709, D65, Tf::Gamma(2.4)), + ColorSpace::Rec2020 => sp("Rec.2020", REC2020, D65, Tf::Gamma(2.4)), + ColorSpace::DisplayP3 => sp("Display P3", P3, D65, Tf::Srgb), + }; + match i { + 0 => (ws(ctx.env.working_space.unwrap_or(ColorSpace::Srgb)), ctx.env.working_linear), + 1 => (ws(ColorSpace::Srgb), false), + 2 => (ws(ColorSpace::Rec709), false), + 3 => (ws(ColorSpace::Rec2020), false), + 4 => (ws(ColorSpace::DisplayP3), false), + 5 => (*space_by_name("ACEScg").expect("built-in"), true), + _ => (*space_by_name("ACES2065-1").expect("built-in"), true), + } +} + +/// Gamut compression for the perceptual intent: pull out-of-gamut (negative) colours toward +/// their luminance just enough to bring every channel to ≥ 0, rolling off smoothly. +fn compress_gamut(c: [f64; 3], luma: [f64; 3]) -> [f64; 3] { + let l = c[0] * luma[0] + c[1] * luma[1] + c[2] * luma[2]; + if l <= 0.0 { + return c.map(|v| v.max(0.0)); + } + let mn = c.iter().cloned().fold(f64::INFINITY, f64::min); + if mn >= 0.0 { + return c; + } + // Fraction of the way toward grey that reaches zero on the most negative channel. + let t = (l / (l - mn)).clamp(0.0, 1.0); + c.map(|v| l + (v - l) * t) +} + +fn profile_converter(ctx: &EffectCtx, mut b: Buf) -> Buf { + let (inp, in_lin_default) = profile(ctx, ctx.params.e("inputProfile")); + let (out, out_lin_default) = profile(ctx, ctx.params.e("outputProfile")); + let in_lin = ctx.params.b("linearizeInputProfile") || in_lin_default && ctx.params.e("inputProfile") == 0; + let out_lin = ctx.params.b("linearizeOutputProfile") || out_lin_default && ctx.params.e("outputProfile") == 0; + let intent = ctx.params.e("intent"); + let dec = if in_lin { Tf::Linear } else { inp.tf }; + let enc = if out_lin { Tf::Linear } else { out.tf }; + // Absolute colorimetric keeps the source white (no adaptation); the others adapt. + let m = if intent == 3 { mul(&invert(&out.to_ref(false)), &inp.to_ref(false)) } else { matrix(&inp, &out) }; + let identity = dec == enc && (0..3).all(|i| (0..3).all(|j| (m[i][j] - IDENTITY[i][j]).abs() < 1e-9)); + if identity { + return b; + } + let out_luma = out.to_ref(true)[1]; + map_px(&mut b, |c| { + let l = c.map(|v| dec.decode(v as f64)); + let mut o = mul_vec(&m, l); + match intent { + 0 => o = compress_gamut(o, out_luma), + 2 => o = o.map(|v| v.max(0.0)), + _ => {} + } + o.map(|v| enc.encode(v) as f32) + }); + b +} + +// ---- Color Stabilizer + +/// Mean straight colour of a square (half-size `r` layer pixels) around layer point `pt`. +fn sample_mean(b: &Buf, pt: [f64; 2], r: f64) -> [f64; 3] { + let (cx, cy) = b.to_px(pt); + let rr = (r * b.scale).max(0.5); + let (x0, x1) = ((cx - rr).floor().max(0.0) as i64, ((cx + rr).ceil() as i64).min(b.img.width as i64)); + let (y0, y1) = ((cy - rr).floor().max(0.0) as i64, ((cy + rr).ceil() as i64).min(b.img.height as i64)); + let mut acc = [0.0f64; 4]; + for y in y0..y1 { + for x in x0..x1 { + let p = b.img.get(x, y); + for k in 0..4 { + acc[k] += p[k] as f64; + } + } + } + if acc[3] <= 1e-9 { + return [0.0; 3]; + } + [acc[0] / acc[3], acc[1] / acc[3], acc[2] / acc[3]] +} + +/// Monotone piecewise-quadratic curve through (0,0), three samples and (1,1) (linear between +/// them is enough for a stabiliser and stays monotone). +fn curve3(xs: [f64; 3], ys: [f64; 3], v: f64) -> f64 { + let mut pts = vec![(0.0, 0.0)]; + let mut pairs: Vec<(f64, f64)> = xs.iter().cloned().zip(ys.iter().cloned()).collect(); + pairs.sort_by(|a, b| a.0.total_cmp(&b.0)); + for (x, y) in pairs { + if x > pts.last().expect("non-empty").0 + 1e-4 && x < 1.0 - 1e-4 { + pts.push((x, y)); + } + } + pts.push((1.0, 1.0)); + let i = pts.partition_point(|p| p.0 < v).clamp(1, pts.len() - 1); + let (a, c) = (pts[i - 1], pts[i]); + a.1 + (c.1 - a.1) * (v - a.0) / (c.0 - a.0).max(1e-9) +} + +fn color_stabilizer(ctx: &EffectCtx, mut b: Buf) -> Buf { + let Some(host) = ctx.env.host else { return b }; + let Some(reference) = host.self_at(ctx.params.f("referenceFrame"), ctx.env.effect_index) else { return b }; + let r = ctx.params.f("sampleSize").max(0.5); + let pts = [ctx.params.v2("blackPoint"), ctx.params.v2("midPoint"), ctx.params.v2("whitePoint")]; + let cur = pts.map(|pt| sample_mean(&b, pt, r)); + let refv = pts.map(|pt| sample_mean(&reference, pt, r)); + let mode = ctx.params.e("stabilize"); + map_px(&mut b, |c| { + [0, 1, 2].map(|k| { + let v = c[k] as f64; + let o = match mode { + // Brightness: one sample (black point) — offset each channel. + 0 => v + refv[0][k] - cur[0][k], + // Levels: black and white samples — linear map per channel. + 1 => { + let (c0, c1, r0, r1) = (cur[0][k], cur[2][k], refv[0][k], refv[2][k]); + if (c1 - c0).abs() < 1e-6 { v + r0 - c0 } else { r0 + (v - c0) * (r1 - r0) / (c1 - c0) } + } + // Curves: three samples — a curve through them per channel. + _ => curve3([cur[0][k], cur[1][k], cur[2][k]], [refv[0][k], refv[1][k], refv[2][k]], v), + }; + o as f32 + }) + }); + b +} + +fn sl(lo: f64, hi: f64, smin: f64, smax: f64) -> ParamUi { + slider(lo, hi, smin, smax, 3) +} + +pub fn specs() -> Vec { + let cc = "Color Correction"; + let srgb = SPACES.iter().position(|s| s.name == "sRGB").unwrap_or(0) as u32; + let lin709 = SPACES.iter().position(|s| s.name == "Linear Rec.709 (sRGB)").unwrap_or(0) as u32; + let acescg = SPACES.iter().position(|s| s.name == "ACEScg").unwrap_or(0) as u32; + let mut look_opts = vec!["None"]; + look_opts.extend(looks().iter().map(|(n, _)| *n)); + vec![ + spec( + "ec.color.ociocdl", + "OCIO CDL Transform", + cc, + vec![ + p("slopeRed", "Slope Red", num(1.0), sl(0.0, 100.0, 0.0, 4.0)), + p("slopeGreen", "Slope Green", num(1.0), sl(0.0, 100.0, 0.0, 4.0)), + p("slopeBlue", "Slope Blue", num(1.0), sl(0.0, 100.0, 0.0, 4.0)), + p("offsetRed", "Offset Red", num(0.0), sl(-100.0, 100.0, -1.0, 1.0)), + p("offsetGreen", "Offset Green", num(0.0), sl(-100.0, 100.0, -1.0, 1.0)), + p("offsetBlue", "Offset Blue", num(0.0), sl(-100.0, 100.0, -1.0, 1.0)), + p("powerRed", "Power Red", num(1.0), sl(0.0, 100.0, 0.0, 4.0)), + p("powerGreen", "Power Green", num(1.0), sl(0.0, 100.0, 0.0, 4.0)), + p("powerBlue", "Power Blue", num(1.0), sl(0.0, 100.0, 0.0, 4.0)), + p("saturation", "Saturation", num(1.0), sl(0.0, 100.0, 0.0, 4.0)), + p("style", "Style", Value::Enum(0), popup(&["ASC CDL v1.2", "No Clamp"])), + direction(), + ], + ocio_cdl, + ), + spec( + "ec.color.ociocolorspace", + "OCIO Color Space Transform", + cc, + vec![ + p("config", "Configuration", Value::Enum(0), popup(&["Built-in (EffectCraft minimal)"])), + space_param("source", "Source", lin709), + space_param("destination", "Destination", srgb), + direction(), + ], + ocio_cst, + ), + spec( + "ec.color.ociodisplay", + "OCIO Display Transform", + cc, + vec![ + p("config", "Configuration", Value::Enum(0), popup(&["Built-in (EffectCraft minimal)"])), + space_param("source", "Source", lin709), + p("display", "Display", Value::Enum(0), popup(&DISPLAYS.iter().map(|d| d.0).collect::>())), + p("view", "View", Value::Enum(0), popup(VIEWS)), + direction(), + ], + ocio_display, + ), + // `file`: a path to a .cube/.3dl/.csp/.cc/.ccc/.cdl file, or the file's text itself. + spec( + "ec.color.ociofile", + "OCIO File Transform", + cc, + vec![ + p("file", "File", Value::Str(String::new()), ParamUi::Text), + p("cccId", "CCC ID", Value::Str(String::new()), ParamUi::Text), + p("interpolation", "Interpolation", Value::Enum(3), popup(&["Nearest", "Linear", "Tetrahedral", "Best"])), + direction(), + ], + ocio_file, + ), + spec( + "ec.color.ociolook", + "OCIO Look Transform", + cc, + vec![ + p("config", "Configuration", Value::Enum(0), popup(&["Built-in (EffectCraft minimal)"])), + space_param("source", "Source", acescg), + space_param("destination", "Destination", acescg), + p("look", "Look", Value::Enum(0), popup(&look_opts)), + p("looks", "Looks", Value::Str(String::new()), ParamUi::Text), + direction(), + ], + ocio_look, + ), + spec( + "ec.color.colorstabilizer", + "Color Stabilizer", + cc, + vec![ + p("stabilize", "Stabilize", Value::Enum(0), popup(&["Brightness", "Levels", "Curves"])), + p("referenceFrame", "Reference Frame (s)", num(0.0), slider(-100000.0, 100000.0, 0.0, 60.0, 2)), + p("blackPoint", "Black Point", Value::Vec2([0.25, 0.75]), ParamUi::Point), + p("midPoint", "Mid Point", Value::Vec2([0.5, 0.5]), ParamUi::Point), + p("whitePoint", "White Point", Value::Vec2([0.75, 0.25]), ParamUi::Point), + p("sampleSize", "Sample Size", num(5.0), slider(1.0, 100.0, 1.0, 50.0, 0)), + ], + color_stabilizer, + ), + spec( + "ec.utility.colorprofileconverter", + "Color Profile Converter", + "Utility", + vec![ + p("inputProfile", "Input Profile", Value::Enum(0), popup(PROFILES)), + p("linearizeInputProfile", "Linearize Input Profile", Value::Bool(false), ParamUi::Checkbox), + p("outputProfile", "Output Profile", Value::Enum(1), popup(PROFILES)), + p("linearizeOutputProfile", "Linearize Output Profile", Value::Bool(false), ParamUi::Checkbox), + p("intent", "Intent", Value::Enum(1), popup(&["Perceptual", "Relative Colorimetric", "Saturation", "Absolute Colorimetric"])), + ], + profile_converter, + ), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, EffectHost, Image, LayerPixels, run_fx}; + + fn close(a: M3, b: M3, tol: f64) { + for i in 0..3 { + for j in 0..3 { + assert!((a[i][j] - b[i][j]).abs() < tol, "{a:?} vs {b:?}"); + } + } + } + + #[test] + fn aces_matrices_match_published_values() { + let ap0 = space_by_name("ACES2065-1").unwrap(); + let ap1 = space_by_name("ACEScg").unwrap(); + // Academy S-2014-004 AP0 → AP1. + close( + matrix(ap0, ap1), + [ + [1.451_439_316_1, -0.236_510_746_9, -0.214_928_569_3], + [-0.076_553_773_4, 1.176_229_699_8, -0.099_675_926_4], + [0.008_316_148_4, -0.006_032_449_8, 0.997_716_301_4], + ], + 2e-4, + ); + // ACEScg → linear Rec.709 with Bradford adaptation (published ACES config value). + let lin = space_by_name("Linear Rec.709 (sRGB)").unwrap(); + close(matrix(ap1, lin), [[1.705_05, -0.621_79, -0.083_26], [-0.130_26, 1.140_80, -0.010_55], [-0.024_00, -0.128_97, 1.152_97]], 2e-3); + // White maps to white. + let w = mul_vec(&matrix(ap1, lin), [1.0, 1.0, 1.0]); + assert!(w.iter().all(|v| (v - 1.0).abs() < 1e-3), "{w:?}"); + } + + #[test] + fn transfer_functions_round_trip() { + for tf in [Tf::Srgb, Tf::Gamma(2.4), Tf::AcesCct, Tf::Pq] { + for v in [0.0, 0.001, 0.18, 0.5, 1.0, 4.0] { + let r = tf.decode(tf.encode(v)); + assert!((r - v).abs() < 1e-6 * v.max(1.0), "{tf:?} {v} {r}"); + } + } + // ACEScct of 18 % grey (S-2016-001). + assert!((Tf::AcesCct.encode(0.18) - 0.413_588).abs() < 1e-5); + // PQ: 100 nits (1.0) encodes to ≈ 0.508. + assert!((Tf::Pq.encode(1.0) - 0.508).abs() < 1e-3); + } + + #[test] + fn color_space_transform_effect_round_trips() { + let img = Image::filled(4, 4, [0.2, 0.5, 0.7, 1.0]); + let srgb = SPACES.iter().position(|s| s.name == "sRGB").unwrap() as u32; + let cg = SPACES.iter().position(|s| s.name == "ACEScg").unwrap() as u32; + let a = run_fx("ec.color.ociocolorspace", &[("source", Value::Enum(srgb)), ("destination", Value::Enum(cg))], img.clone(), 0.0, EffectEnv::default()); + assert_ne!(a.img.data, img.data); + let b = run_fx("ec.color.ociocolorspace", &[("source", Value::Enum(cg)), ("destination", Value::Enum(srgb))], a.img.clone(), 0.0, EffectEnv::default()); + let c = run_fx( + "ec.color.ociocolorspace", + &[("source", Value::Enum(srgb)), ("destination", Value::Enum(cg)), ("direction", Value::Enum(1))], + a.img, + 0.0, + EffectEnv::default(), + ); + for (x, y) in b.img.data.iter().zip(&img.data).chain(c.img.data.iter().zip(&img.data)) { + assert!((0..4).all(|k| (x[k] - y[k]).abs() < 1e-4), "{x:?} {y:?}"); + } + } + + #[test] + fn cdl_math() { + let cdl = Cdl { slope: [1.2, 1.0, 0.8], offset: [0.05, 0.0, -0.02], power: [1.1, 1.0, 0.9], sat: 0.8 }; + let c = [0.3, 0.5, 0.7]; + // Hand computation of the v1.2 formula. + let sop = [(0.3f64 * 1.2 + 0.05).powf(1.1), 0.5, (0.7f64 * 0.8 - 0.02).powf(0.9)]; + let l = sop[0] * 0.2126 + sop[1] * 0.7152 + sop[2] * 0.0722; + let want = sop.map(|v| l + 0.8 * (v - l)); + let got = cdl.apply(c, true); + for k in 0..3 { + assert!((got[k] - want[k]).abs() < 1e-12); + } + let back = cdl.invert(got, true); + for k in 0..3 { + assert!((back[k] - c[k]).abs() < 1e-9); + } + // v1.2 clamps, No Clamp keeps negatives. + let neg = Cdl { offset: [-0.5; 3], ..Cdl::default() }; + assert_eq!(neg.apply([0.2; 3], true), [0.0; 3]); + assert!((neg.apply([0.2; 3], false)[0] + 0.3).abs() < 1e-12); + let img = Image::filled(2, 2, [0.3, 0.5, 0.7, 1.0]); + let out = run_fx("ec.color.ociocdl", &[("slopeRed", num(1.2)), ("offsetRed", num(0.05)), ("powerRed", num(1.1))], img, 0.0, EffectEnv::default()); + let r = (0.3f64 * 1.2 + 0.05).powf(1.1); + assert!((out.img.get(0, 0)[0] as f64 - r).abs() < 1e-5); + } + + fn identity_lattice(n: usize, f: impl Fn([f32; 3]) -> [f32; 3]) -> Vec<[f32; 3]> { + let mut v = vec![]; + for b in 0..n { + for g in 0..n { + for r in 0..n { + let s = (n - 1) as f32; + v.push(f([r as f32 / s, g as f32 / s, b as f32 / s])); + } + } + } + v + } + + #[test] + fn lut_files_parse_and_interpolate() { + // .cube with a non-linear lattice: trilinear between lattice points. + let f = |c: [f32; 3]| [c[0] * c[0], c[1], 1.0 - c[2]]; + let mut cube = String::from("LUT_3D_SIZE 3\n"); + for r in identity_lattice(3, f) { + cube += &format!("{} {} {}\n", r[0], r[1], r[2]); + } + let Some(FileXform::Lut { lut, .. }) = parse_file(&cube, "cube") else { panic!() }; + // Trilinear between 0.5 (0.25) and 1.0 (1.0) at 0.75 → 0.625. + let o = lut.apply_interp([0.75, 0.3, 0.2], 1); + assert!((o[0] - 0.625).abs() < 1e-6 && (o[1] - 0.3).abs() < 1e-6 && (o[2] - 0.8).abs() < 1e-6, "{o:?}"); + // Nearest snaps. + assert_eq!(lut.apply_interp([0.74, 0.0, 0.0], 0)[0], 0.25); + // .3dl (blue fastest, 10-bit integers, mesh line). + let mut tdl = String::from("0 511 1023\n"); + for r in 0..3 { + for g in 0..3 { + for b in 0..3 { + let v = [r, g, b].map(|i| i as f64 * 511.5); + tdl += &format!("{} {} {}\n", v[0].round(), v[1].round(), (1023.0 - v[2]).round()); + } + } + } + let Some(FileXform::Lut { lut, .. }) = parse_file(&tdl, "3dl") else { panic!() }; + let o = lut.apply_interp([0.25, 0.5, 0.25], 1); + assert!((o[0] - 0.25).abs() < 2e-3 && (o[1] - 0.5).abs() < 2e-3 && (o[2] - 0.75).abs() < 2e-3, "{o:?}"); + // .csp with a squaring pre-LUT on red and an identity 2³ lattice. + let mut csp = String::from("CSPLUTV100\n3D\n\n3\n0 0.5 1\n0 0.25 1\n2\n0 1\n0 1\n2\n0 1\n0 1\n\n2 2 2\n"); + for r in identity_lattice(2, |c| c) { + csp += &format!("{} {} {}\n", r[0], r[1], r[2]); + } + let x = parse_file(&csp, "csp").unwrap(); + let o = x.apply([0.5, 0.4, 0.6], 2, false, ""); + assert!((o[0] - 0.25).abs() < 1e-6 && (o[1] - 0.4).abs() < 1e-6, "{o:?}"); + let inv = x.apply(o, 2, true, ""); + assert!((inv[0] - 0.5).abs() < 1e-4, "{inv:?}"); + // CDL collection: pick by id. + let ccc = r#"2 2 20 0 01 1 1 + 0.5 0.5 0.51"#; + let x = parse_file(ccc, "ccc").unwrap(); + assert!((x.apply([0.4; 3], 2, false, "b")[0] - 0.2).abs() < 1e-6); + assert!((x.apply([0.4; 3], 2, false, "a")[0] - 0.8).abs() < 1e-6); + assert!((x.apply([0.4; 3], 2, false, "")[0] - 0.8).abs() < 1e-6); + // Effect with inline content; inverse of a 3D lattice. + let img = Image::filled(2, 2, [0.6, 0.3, 0.2, 1.0]); + let out = run_fx("ec.color.ociofile", &[("file", Value::Str(cube.clone()))], img.clone(), 0.0, EffectEnv::default()); + let back = run_fx("ec.color.ociofile", &[("file", Value::Str(cube)), ("direction", Value::Enum(1))], out.img, 0.0, EffectEnv::default()); + assert!((back.img.get(0, 0)[0] - 0.6).abs() < 1e-3, "{:?}", back.img.get(0, 0)); + assert!(parse_file("garbage", "3dl").is_none()); + } + + #[test] + fn display_and_look_transforms() { + // Linear 0.18 through the sRGB display / Standard view = sRGB encoding of 0.18. + let img = Image::filled(2, 2, [0.18, 0.18, 0.18, 1.0]); + let out = run_fx("ec.color.ociodisplay", &[], img.clone(), 0.0, EffectEnv::default()); + assert!((out.img.get(0, 0)[0] as f64 - Tf::Srgb.encode(0.18)).abs() < 1e-5); + let back = run_fx("ec.color.ociodisplay", &[("direction", Value::Enum(1))], out.img, 0.0, EffectEnv::default()); + assert!((back.img.get(0, 0)[0] - 0.18).abs() < 1e-5); + // Tone mapped view keeps highlights below 1. + let hot = Image::filled(1, 1, [8.0, 8.0, 8.0, 1.0]); + let tm = run_fx("ec.color.ociodisplay", &[("view", Value::Enum(1))], hot, 0.0, EffectEnv::default()); + assert!(tm.img.get(0, 0)[0] < 1.0); + assert!((tonemap_inv(tonemap(3.0)) - 3.0).abs() < 1e-9); + // Looks: Warm raises red over blue; its inverse undoes it. + let grey = Image::filled(1, 1, [0.18, 0.18, 0.18, 1.0]); + let w = run_fx("ec.color.ociolook", &[("looks", Value::Str("Warm".into()))], grey.clone(), 0.0, EffectEnv::default()); + let px = w.img.get(0, 0); + assert!(px[0] > px[2]); + let u = run_fx("ec.color.ociolook", &[("looks", Value::Str("-Warm".into()))], w.img, 0.0, EffectEnv::default()); + assert!((u.img.get(0, 0)[0] - 0.18).abs() < 1e-4 && (u.img.get(0, 0)[2] - 0.18).abs() < 1e-4); + assert_eq!(parse_looks("Warm, -Cool, nope").len(), 2); + } + + #[test] + fn profile_converter_converts_and_round_trips() { + let img = Image::filled(2, 2, [0.8, 0.4, 0.1, 1.0]); + // sRGB → Rec. 2020 (relative), then back. + let a = run_fx( + "ec.utility.colorprofileconverter", + &[("inputProfile", Value::Enum(1)), ("outputProfile", Value::Enum(3))], + img.clone(), + 0.0, + EffectEnv::default(), + ); + assert_ne!(a.img.data, img.data); + let b = run_fx( + "ec.utility.colorprofileconverter", + &[("inputProfile", Value::Enum(3)), ("outputProfile", Value::Enum(1))], + a.img, + 0.0, + EffectEnv::default(), + ); + assert!((0..3).all(|k| (b.img.get(0, 0)[k] - img.get(0, 0)[k]).abs() < 1e-4)); + // Same profile both sides: identity. Working space defaults to sRGB. + let c = run_fx( + "ec.utility.colorprofileconverter", + &[("inputProfile", Value::Enum(0)), ("outputProfile", Value::Enum(1))], + img.clone(), + 0.0, + EffectEnv::default(), + ); + assert_eq!(c.img.data, img.data); + // Perceptual pulls out-of-gamut colours inside: saturated Rec.2020 green into sRGB. + let g = Image::filled(1, 1, [0.0, 1.0, 0.0, 1.0]); + let rel = run_fx( + "ec.utility.colorprofileconverter", + &[("inputProfile", Value::Enum(3)), ("outputProfile", Value::Enum(1))], + g.clone(), + 0.0, + EffectEnv::default(), + ); + assert!(rel.img.get(0, 0)[0] < 0.0); + let per = run_fx( + "ec.utility.colorprofileconverter", + &[("inputProfile", Value::Enum(3)), ("outputProfile", Value::Enum(1)), ("intent", Value::Enum(0))], + g, + 0.0, + EffectEnv::default(), + ); + assert!(per.img.get(0, 0).iter().all(|v| *v >= -1e-6)); + } + + struct RefHost(Image); + impl EffectHost for RefHost { + fn layer(&self, _: u64, _: bool) -> Option { + None + } + fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option> { + None + } + fn self_at(&self, _: f64, _: usize) -> Option { + Some(Buf { img: self.0.clone(), offset: [0.0; 2], scale: 1.0 }) + } + } + + #[test] + fn color_stabilizer_matches_the_reference_frame() { + let mk = |k: f32, o: f32| { + let mut img = Image::new(40, 40); + for y in 0..40 { + for x in 0..40 { + let v = (x as f32 / 40.0) * k + o; + img.set(x, y, [v, v * 0.9, v * 0.8, 1.0]); + } + } + img + }; + let reference = mk(1.0, 0.0); + let flicker = mk(0.8, 0.1); + let host = RefHost(reference.clone()); + let env = EffectEnv { host: Some(&host), ..Default::default() }; + let pts = [("blackPoint", Value::Vec2([5.0, 20.0])), ("midPoint", Value::Vec2([20.0, 20.0])), ("whitePoint", Value::Vec2([35.0, 20.0]))]; + let mut vals: Vec<(&str, Value)> = pts.to_vec(); + vals.push(("stabilize", Value::Enum(1))); + vals.push(("sampleSize", num(1.0))); + let out = run_fx("ec.color.colorstabilizer", &vals, flicker.clone(), 0.0, env); + for x in [5i64, 20, 35] { + let (o, r) = (out.img.get(x, 20), reference.get(x, 20)); + assert!((o[0] - r[0]).abs() < 0.01, "x {x}: {o:?} vs {r:?}"); + } + // Deterministic, and a no-op without a host. + let again = run_fx("ec.color.colorstabilizer", &vals, flicker.clone(), 0.0, env); + assert_eq!(out.img.data, again.img.data); + let none = run_fx("ec.color.colorstabilizer", &vals, flicker.clone(), 0.0, EffectEnv::default()); + assert_eq!(none.img.data, flicker.data); + // Brightness mode corrects an offset at the black point. + let lifted = mk(1.0, 0.1); + let mut vals2: Vec<(&str, Value)> = pts.to_vec(); + vals2.push(("sampleSize", num(1.0))); + let b = run_fx("ec.color.colorstabilizer", &vals2, lifted, 0.0, env); + assert!((b.img.get(5, 20)[0] - reference.get(5, 20)[0]).abs() < 0.01); + } +} diff --git a/crates/effects/src/sim3.rs b/crates/effects/src/sim3.rs new file mode 100644 index 00000000..66e9e11c --- /dev/null +++ b/crates/effects/src/sim3.rs @@ -0,0 +1,716 @@ +//! Simulation: Particle Playground and CC Hair. +//! +//! **Particle Playground** is a deterministic 2D particle system stepped from layer time 0 at +//! [`SPS`] steps per second through [`SimCache`] (seeking anywhere gives the same frame as +//! playing up to it). Producers: **Cannon** (a stream from a barrel with angle, radius, rate, +//! velocity and random spreads), **Grid** (particles placed on a grid at time 0) and **Layer +//! Exploder** (another layer broken into particles at time 0). Forces: **Gravity** (force, +//! random spread, direction), **Repel** (particles push each other apart within a radius), +//! **Wall** (a mask of the layer the particles bounce off) and the **Persistent Property +//! Mapper** (a map layer's red / green / blue drive particle properties at the particle's +//! position each step). **Layer Map** draws particles with the colours of another layer +//! sampled at each particle's birth position. Map layers are read at the frame being rendered +//! (their pixels are part of the cache key). Particles replace the layer's own pixels, as in +//! After Effects. +//! +//! **CC Hair** grows strands from the layer's opaque pixels (Density per 1 000 px²), each a +//! quadratic curve of Length that droops with Weight, optionally steered and scaled by a hairfall +//! map layer (luminance → length, horizontal/vertical gradient → lean), and shades them with a +//! Kajiya–Kay-style strand model (diffuse ∝ sin(tangent, light), specular from the half vector). + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::{Image, Px}; +use rayon::prelude::*; + +use crate::sim::{Acc, SPS, Shape, Sprite, combine, splat, steps_at}; +use crate::util::{SimCache, hash1, params_key, point_in_poly, unpremul}; +use crate::{Buf, EffectCtx, EffectSpec, col, num, p, popup, slider}; + +fn spec(id: &'static str, name: &'static str, params: Vec, render: crate::RenderFn) -> EffectSpec { + EffectSpec { id, name, category: "Simulation", params, render, gpu: false, float: true } +} + +// ---------------------------------------------------------------- Particle Playground + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct PgParticle { + pub p: [f32; 2], + pub v: [f32; 2], + /// Layer position where the particle was born (Layer Map lookups). + pub origin: [f32; 2], + pub c: [f32; 4], + pub r: f32, + pub mass: f32, + pub friction: f32, + pub force: [f32; 2], + pub id: u32, +} + +#[derive(Clone, Debug, Default)] +pub struct PgState { + pub parts: Vec, + carry: f64, + next_id: u32, +} + +/// A map layer sampled in layer coordinates. +struct MapImg { + img: Image, + /// Layer point → pixel: (x · k₀ + o₀, y · k₁ + o₁). + k: [f64; 2], + o: [f64; 2], +} + +impl MapImg { + fn from(ctx: &EffectCtx, lp: crate::LayerPixels) -> MapImg { + // Fit the map layer to this layer (stretch), like AE's layer maps. + let (ls, os) = (ctx.layer_size, lp.size); + let sx = if ls[0] > 0.0 { os[0] / ls[0] } else { 1.0 }; + let sy = if ls[1] > 0.0 { os[1] / ls[1] } else { 1.0 }; + MapImg { img: lp.buf.img, k: [lp.buf.scale * sx, lp.buf.scale * sy], o: lp.buf.offset } + } + fn at(&self, x: f32, y: f32) -> Px { + let (c, a) = unpremul(self.img.sample_bilinear_clamped(x as f64 * self.k[0] + self.o[0], y as f64 * self.k[1] + self.o[1])); + [c[0], c[1], c[2], a] + } +} + +pub const MAP_TARGETS: [&str; 12] = + ["None", "Red", "Green", "Blue", "Scale", "Opacity", "X Velocity", "Y Velocity", "X Force", "Y Force", "Kinetic Friction", "Mass"]; + +struct Pg { + // Cannon + cannon: bool, + pos: [f32; 2], + barrel_angle: f32, + barrel_radius: f32, + rate: f64, + dir_spread: f32, + vel: f32, + vel_spread: f32, + color: [f32; 4], + radius: f32, + // Gravity + gravity: [f32; 2], + grav_spread: f32, + // Repel + repel: f32, + repel_radius: f32, + // Wall + wall: Option>, + // Property mapper + mapper: Option, + map_to: [(u32, f32, f32); 3], + seed: u32, + bounds: [f32; 4], +} + +impl Pg { + fn spawn(&self, id: u32) -> PgParticle { + let s = self.seed; + let h = |k: u32| hash1(id, k, s) * 2.0 - 1.0; + // AE angles: 0° = up, clockwise. + let a = (self.barrel_angle + h(1) * self.dir_spread * 0.5).to_radians(); + let d = [a.sin(), -a.cos()]; + let perp = [-d[1], d[0]]; + let off = h(2) * self.barrel_radius; + let speed = (self.vel + h(3) * self.vel_spread).max(0.0); + let p = [self.pos[0] + perp[0] * off, self.pos[1] + perp[1] * off]; + PgParticle { p, v: [d[0] * speed, d[1] * speed], origin: p, c: self.color, r: self.radius, mass: 1.0, friction: 0.0, force: [0.0; 2], id } + } + + fn step(&self, st: &mut PgState) { + let dt = (1.0 / SPS) as f32; + let s = self.seed; + // Property mapper (persistent): drive properties from the map at the particle position. + if let Some(m) = &self.mapper { + st.parts.par_iter_mut().for_each(|q| { + let px = m.at(q.p[0], q.p[1]); + for (ch, &(target, lo, hi)) in self.map_to.iter().enumerate() { + let v = lo + (hi - lo) * px[ch]; + match target { + 1 => q.c[0] = v, + 2 => q.c[1] = v, + 3 => q.c[2] = v, + 4 => q.r = self.radius * v.max(0.0), + 5 => q.c[3] = v.clamp(0.0, 1.0), + 6 => q.v[0] = v, + 7 => q.v[1] = v, + 8 => q.force[0] = v, + 9 => q.force[1] = v, + 10 => q.friction = v.clamp(0.0, 1.0), + 11 => q.mass = v.max(0.01), + _ => {} + } + } + }); + } + // Repel: spatial hash, symmetric pairwise push within the radius. + let mut push = vec![[0.0f32; 2]; st.parts.len()]; + if self.repel != 0.0 && self.repel_radius > 0.0 && st.parts.len() > 1 { + let cell = self.repel_radius; + let mut grid: std::collections::HashMap<(i32, i32), Vec> = std::collections::HashMap::new(); + for (i, q) in st.parts.iter().enumerate() { + grid.entry(((q.p[0] / cell).floor() as i32, (q.p[1] / cell).floor() as i32)).or_default().push(i); + } + let parts = &st.parts; + push.par_iter_mut().enumerate().for_each(|(i, f)| { + let q = parts[i]; + let (cx, cy) = ((q.p[0] / cell).floor() as i32, (q.p[1] / cell).floor() as i32); + for gy in cy - 1..=cy + 1 { + for gx in cx - 1..=cx + 1 { + let Some(list) = grid.get(&(gx, gy)) else { continue }; + for &j in list { + if j == i { + continue; + } + let o = parts[j]; + let (dx, dy) = (q.p[0] - o.p[0], q.p[1] - o.p[1]); + let d = (dx * dx + dy * dy).sqrt(); + if d < self.repel_radius && d > 1e-4 { + let k = self.repel * (1.0 - d / self.repel_radius) / d; + f[0] += dx * k; + f[1] += dy * k; + } + } + } + } + }); + } + let wall = self.wall.as_deref(); + st.parts.par_iter_mut().zip(push.par_iter()).for_each(|(q, f)| { + let g = if self.grav_spread > 0.0 { 1.0 + (hash1(q.id, 7, s) * 2.0 - 1.0) * self.grav_spread } else { 1.0 }; + let ax = self.gravity[0] * g + (q.force[0] + f[0]) / q.mass; + let ay = self.gravity[1] * g + (q.force[1] + f[1]) / q.mass; + let damp = 1.0 - q.friction * 0.1; + q.v = [(q.v[0] + ax * dt) * damp, (q.v[1] + ay * dt) * damp]; + let np = [q.p[0] + q.v[0] * dt, q.p[1] + q.v[1] * dt]; + if let Some(w) = wall { + let was = point_in_poly(w, q.p[0] as f64, q.p[1] as f64); + let now = point_in_poly(w, np[0] as f64, np[1] as f64); + if was != now { + // Bounce: reverse the velocity and stay on the same side. + q.v = [-q.v[0] * 0.9, -q.v[1] * 0.9]; + return; + } + } + q.p = np; + }); + let b = self.bounds; + st.parts.retain(|q| q.p[0] > b[0] && q.p[0] < b[2] && q.p[1] > b[1] && q.p[1] < b[3]); + if self.cannon { + st.carry += self.rate / SPS; + let n = st.carry.floor() as u32; + st.carry -= n as f64; + for _ in 0..n { + if st.parts.len() >= 60_000 { + break; + } + let id = st.next_id; + st.next_id = st.next_id.wrapping_add(1); + let mut q = self.spawn(id); + // Sub-step birth offset keeps streams smooth. + let frac = hash1(id, 9, s) * dt; + q.p = [q.p[0] + q.v[0] * frac, q.p[1] + q.v[1] * frac]; + st.parts.push(q); + } + } + } +} + +static PG_CACHE: SimCache = SimCache::new(6); + +fn image_key(img: &Image) -> u64 { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + img.width.hash(&mut h); + img.height.hash(&mut h); + for p in img.data.iter().step_by(7) { + for c in p { + c.to_bits().hash(&mut h); + } + } + h.finish() +} + +fn particle_playground(ctx: &EffectCtx, mut b: Buf) -> Buf { + let pr = ctx.params; + let (lw, lh) = (ctx.layer_size[0] as f32, ctx.layer_size[1] as f32); + let grav_dir = (pr.f("gravityDirection") as f32).to_radians(); + let gf = pr.f("gravityForce") as f32; + let wall = (pr.f("wallBoundary").round() as usize).checked_sub(1).and_then(|i| ctx.env.masks.get(i)).map(|m| m.points.clone()); + let mapper = if pr.b("mapperEnabled") { ctx.layer_param("useLayerAsMap", true).map(|lp| MapImg::from(ctx, lp)) } else { None }; + let pg = Pg { + cannon: pr.b("cannonEnabled"), + pos: { pr.v2("cannonPosition").map(|v| v as f32) }, + barrel_angle: pr.f("barrelAngle") as f32, + barrel_radius: pr.f("barrelRadius") as f32, + rate: pr.f("particlesPerSecond").max(0.0), + dir_spread: pr.f("directionRandomSpread") as f32, + vel: pr.f("velocity") as f32, + vel_spread: pr.f("velocityRandomSpread") as f32, + color: pr.color("cannonColor"), + radius: pr.f("cannonParticleRadius").max(0.0) as f32, + gravity: [grav_dir.sin() * gf, -grav_dir.cos() * gf], + grav_spread: (pr.f("gravityForceRandomSpread") / 100.0).max(0.0) as f32, + repel: pr.f("repelForce") as f32 * 100.0, + repel_radius: pr.f("repelForceRadius").max(0.0) as f32, + wall, + mapper, + map_to: [ + (pr.e("mapRedTo"), pr.f("redMin") as f32, pr.f("redMax") as f32), + (pr.e("mapGreenTo"), pr.f("greenMin") as f32, pr.f("greenMax") as f32), + (pr.e("mapBlueTo"), pr.f("blueMin") as f32, pr.f("blueMax") as f32), + ], + seed: (pr.f("randomSeed") as u32).wrapping_mul(0x9e37_79b9) ^ ctx.seed, + bounds: [-lw * 2.0, -lh * 2.0, lw * 3.0, lh * 3.0], + }; + // Initial particles: Grid and Layer Exploder. + let grid_on = pr.b("gridEnabled"); + let across = pr.f("particlesAcross").max(0.0).round() as u32; + let down = pr.f("particlesDown").max(0.0).round() as u32; + let gpos = pr.v2("gridPosition"); + let (gw, gh) = (pr.f("gridWidth"), pr.f("gridHeight")); + let gcol = pr.color("gridColor"); + let grad = pr.f("gridParticleRadius").max(0.0) as f32; + let exploder = if pr.b("exploderEnabled") { + let host = ctx.env.host; + let lid = pr.get("explodeLayer").and_then(Value::as_layer); + match (host, lid) { + (Some(h), Some(id)) => h.layer_at(id, ctx.env.comp_time - ctx.time, true).or_else(|| h.layer(id, true)), + _ => None, + } + } else { + None + }; + let ex_r = pr.f("radiusOfNewParticles").max(0.5) as f32; + let ex_disp = pr.f("velocityDispersion") as f32; + let layer_map = pr.b("layerMapEnabled").then(|| ctx.layer_param("layerMapLayer", true).map(|lp| MapImg::from(ctx, lp))).flatten(); + + let mut key = params_key(ctx, &Buf { img: Image::new(0, 0), offset: [0.0; 2], scale: 1.0 }, 0x7067); + if let Some(m) = &pg.mapper { + key ^= image_key(&m.img).rotate_left(7); + } + if let Some(e) = &exploder { + key ^= image_key(&e.buf.img).rotate_left(13); + } + if let Some(w) = &pg.wall { + key ^= w.iter().fold(0u64, |a, p| a.rotate_left(5) ^ p[0].to_bits() ^ p[1].to_bits().rotate_left(17)); + } + let seed = pg.seed; + let init = || { + let mut st = PgState::default(); + if grid_on && across > 0 && down > 0 { + for j in 0..down { + for i in 0..across { + let fx = if across > 1 { i as f64 / (across - 1) as f64 - 0.5 } else { 0.0 }; + let fy = if down > 1 { j as f64 / (down - 1) as f64 - 0.5 } else { 0.0 }; + let p = [(gpos[0] + fx * gw) as f32, (gpos[1] + fy * gh) as f32]; + let id = st.next_id; + st.next_id += 1; + st.parts.push(PgParticle { p, v: [0.0; 2], origin: p, c: gcol, r: grad, mass: 1.0, friction: 0.0, force: [0.0; 2], id }); + } + } + } + if let Some(e) = &exploder { + let m = MapImg::from(ctx, e.clone()); + let step = (ex_r * 2.0).max(1.0); + let mut y = step * 0.5; + while y < lh { + let mut x = step * 0.5; + while x < lw { + let c = m.at(x, y); + if c[3] > 0.05 { + let id = st.next_id; + st.next_id += 1; + let h = |k: u32| hash1(id, k, seed) * 2.0 - 1.0; + let (dx, dy) = (x - lw * 0.5, y - lh * 0.5); + let l = (dx * dx + dy * dy).sqrt().max(1.0); + let v = [dx / l * ex_disp + h(1) * ex_disp, dy / l * ex_disp + h(2) * ex_disp]; + st.parts.push(PgParticle { p: [x, y], v, origin: [x, y], c, r: ex_r, mass: 1.0, friction: 0.0, force: [0.0; 2], id }); + } + x += step; + } + y += step; + } + } + st + }; + let st = PG_CACHE.run(key, steps_at(ctx.time), init, |st, _| pg.step(st)); + let s = b.scale as f32; + let sprites: Vec = st + .parts + .iter() + .map(|q| { + let (x, y) = b.to_px([q.p[0] as f64, q.p[1] as f64]); + let c = match &layer_map { + Some(m) => { + let mc = m.at(q.origin[0], q.origin[1]); + [mc[0], mc[1], mc[2], mc[3] * q.c[3]] + } + None => q.c, + }; + Sprite::new(x as f32, y as f32, q.r * s, c, Shape::Disc) + }) + .collect(); + let fx = splat(b.img.width, b.img.height, &sprites, Acc::Over); + // Particles replace the layer's own pixels. + b.img = combine(&b.img, &fx, 5); + b +} + +// ---------------------------------------------------------------- CC Hair + +fn cc_hair(ctx: &EffectCtx, mut b: Buf) -> Buf { + let pr = ctx.params; + let len = pr.f("length").max(0.0) as f32; + let density = pr.f("density").max(0.0) as f32 / 100.0; + if len <= 0.0 || density <= 0.0 { + return b; + } + let thick = pr.f("thickness").max(0.05) as f32; + let weight = pr.f("weight") as f32; + let const_mass = pr.b("constantMass"); + let map = ctx.layer_param("mapLayer", true).map(|lp| MapImg::from(ctx, lp)); + let map_strength = (pr.f("mapStrength") / 100.0) as f32; + let map_soft = pr.f("mapSoftness").max(0.0) as f32; + let noise = (pr.f("addNoise") / 100.0) as f32; + let hc = pr.color("hairColor"); + let bright = (pr.f("brightness") / 100.0) as f32; + let opacity = (pr.f("opacity") / 100.0) as f32; + let inherit = (pr.f("colorInheritance") / 100.0) as f32; + let light_i = (pr.f("lightIntensity") / 100.0) as f32; + let ld = (pr.f("lightDirection") as f32).to_radians(); + let light = [ld.sin(), -ld.cos()]; + let (amb, dif, spe) = ((pr.f("ambient") / 100.0) as f32, (pr.f("diffuse") / 100.0) as f32, (pr.f("specular") / 100.0) as f32); + let rough = (pr.f("roughness") / 100.0).clamp(0.01, 1.0) as f32; + let seed = ctx.seed ^ (pr.f("randomSeed") as u32).wrapping_mul(0x85eb_ca6b); + let (lw, lh) = (ctx.layer_size[0] as f32, ctx.layer_size[1] as f32); + // Roots: a jittered grid over the layer, density per 1 000 px². + let count = ((lw * lh / 1000.0) * density * 10.0).min(200_000.0) as u32; + let src = b.img.clone(); + let segs = 6u32; + let strands: Vec> = (0..count) + .into_par_iter() + .filter_map(|i| { + let rx = hash1(i, 1, seed) * lw; + let ry = hash1(i, 2, seed) * lh; + let (bx, by) = b.to_px([rx as f64, ry as f64]); + let root = src.sample_bilinear(bx, by); + if root[3] < 0.5 { + return None; + } + let (rc, _) = unpremul(root); + let mut l = len * (0.75 + 0.5 * hash1(i, 3, seed)); + // Lean: random, plus the map's local gradient (hairfall map). + let mut lean = [(hash1(i, 4, seed) * 2.0 - 1.0) * 0.6, -1.0 + hash1(i, 5, seed) * 0.4]; + if let Some(m) = &map { + let lum = |x: f32, y: f32| { + let c = m.at(x, y); + (0.2126 * c[0] + 0.7152 * c[1] + 0.0722 * c[2]) * c[3] + }; + let d = map_soft.max(1.0); + let (gx, gy) = (lum(rx + d, ry) - lum(rx - d, ry), lum(rx, ry + d) - lum(rx, ry - d)); + l *= 1.0 - map_strength + map_strength * lum(rx, ry); + lean = [lean[0] + gx * map_strength * 4.0, lean[1] + gy * map_strength * 4.0]; + if noise > 0.0 { + lean[0] += (hash1(i, 6, seed) * 2.0 - 1.0) * noise; + } + } + let ll = (lean[0] * lean[0] + lean[1] * lean[1]).sqrt().max(1e-4); + let dir = [lean[0] / ll, lean[1] / ll]; + // Droop: heavier (or thinner with Constant Mass off) hair bends more. + let droop = weight * if const_mass { 1.0 } else { 1.0 / thick.max(0.2) } * l; + let base = [rc[0] + (hc[0] - rc[0]) * (1.0 - inherit), rc[1] + (hc[1] - rc[1]) * (1.0 - inherit), rc[2] + (hc[2] - rc[2]) * (1.0 - inherit)]; + let pt = |s: f32| [rx + dir[0] * l * s, ry + dir[1] * l * s + droop * s * s]; + let mut out = Vec::with_capacity(segs as usize); + for k in 0..segs { + let (s0, s1) = (k as f32 / segs as f32, (k + 1) as f32 / segs as f32); + let (a, c) = (pt(s0), pt(s1)); + let t = { + let (tx, ty) = (c[0] - a[0], c[1] - a[1]); + let tl = (tx * tx + ty * ty).sqrt().max(1e-6); + [tx / tl, ty / tl] + }; + // Strand shading (Kajiya–Kay): diffuse ∝ sin(T, L), specular from T·L. + let tl = t[0] * light[0] + t[1] * light[1]; + let diff = (1.0 - tl * tl).max(0.0).sqrt(); + let spec = (1.0 - tl.abs()).powf(1.0 / rough) * spe; + let shade = (amb + dif * diff * light_i) * bright; + let tip = 1.0 - 0.6 * s1; + let c3 = [base[0] * shade + spec * light_i, base[1] * shade + spec * light_i, base[2] * shade + spec * light_i]; + let (pa, pc) = (b.to_px([a[0] as f64, a[1] as f64]), b.to_px([c[0] as f64, c[1] as f64])); + out.push(Sprite { + x: pa.0 as f32, + y: pa.1 as f32, + r: thick * 0.5 * b.scale as f32 * tip, + c: [c3[0], c3[1], c3[2], opacity], + shape: Shape::Line { dx: (pc.0 - pa.0) as f32, dy: (pc.1 - pa.1) as f32 }, + rot: 0.0, + }); + } + Some(out) + }) + .collect(); + let sprites: Vec = strands.into_iter().flatten().collect(); + let fx = splat(b.img.width, b.img.height, &sprites, Acc::Over); + b.img = combine(&b.img, &fx, 0); + b +} + +pub fn specs() -> Vec { + let pt = |x: f64, y: f64| Value::Vec2([x, y]); + let px = |d: f64, max: f64| (num(d), slider(0.0, max * 10.0, 0.0, max, 1)); + let sp = |id: &'static str, name: &'static str, v: (Value, ParamUi)| p(id, name, v.0, v.1); + let mut pg = vec![ + // Cannon + p("cannonEnabled", "Cannon", Value::Bool(true), ParamUi::Checkbox), + p("cannonPosition", "Position", pt(0.5, 0.9), ParamUi::Point), + p("barrelAngle", "Barrel Angle", num(0.0), ParamUi::Angle), + sp("barrelRadius", "Barrel Radius", px(0.0, 100.0)), + sp("particlesPerSecond", "Particles Per Second", px(60.0, 500.0)), + sp("directionRandomSpread", "Direction Random Spread", px(20.0, 360.0)), + sp("velocity", "Velocity", px(130.0, 1000.0)), + sp("velocityRandomSpread", "Velocity Random Spread", px(20.0, 500.0)), + p("cannonColor", "Color", col(1.0, 0.0, 0.0), ParamUi::Color), + sp("cannonParticleRadius", "Particle Radius", px(2.0, 50.0)), + // Grid + p("gridEnabled", "Grid", Value::Bool(true), ParamUi::Checkbox), + p("gridPosition", "Grid Position", pt(0.5, 0.5), ParamUi::Point), + sp("gridWidth", "Width", px(100.0, 2000.0)), + sp("gridHeight", "Height", px(100.0, 2000.0)), + sp("particlesAcross", "Particles Across", px(0.0, 100.0)), + sp("particlesDown", "Particles Down", px(0.0, 100.0)), + p("gridColor", "Grid Color", col(1.0, 1.0, 1.0), ParamUi::Color), + sp("gridParticleRadius", "Grid Particle Radius", px(2.0, 50.0)), + // Layer Exploder + p("exploderEnabled", "Layer Exploder", Value::Bool(true), ParamUi::Checkbox), + p("explodeLayer", "Explode Layer", Value::Layer(None), ParamUi::Layer), + sp("radiusOfNewParticles", "Radius of New Particles", px(2.0, 50.0)), + sp("velocityDispersion", "Velocity Dispersion", px(20.0, 500.0)), + // Layer Map + p("layerMapEnabled", "Layer Map", Value::Bool(true), ParamUi::Checkbox), + p("layerMapLayer", "Use Layer", Value::Layer(None), ParamUi::Layer), + // Gravity + sp("gravityForce", "Force", px(108.0, 1000.0)), + sp("gravityForceRandomSpread", "Force Random Spread", px(0.0, 100.0)), + p("gravityDirection", "Direction", num(180.0), ParamUi::Angle), + // Repel + p("repelForce", "Repel Force", num(0.0), slider(-100.0, 100.0, -10.0, 10.0, 2)), + sp("repelForceRadius", "Force Radius", px(0.0, 100.0)), + // Wall + p("wallBoundary", "Boundary", num(0.0), ParamUi::Mask), + // Persistent Property Mapper + p("mapperEnabled", "Persistent Property Mapper", Value::Bool(true), ParamUi::Checkbox), + p("useLayerAsMap", "Use Layer As Map", Value::Layer(None), ParamUi::Layer), + ]; + for (to, mn, mx, n_to, n_min, n_max) in [ + ("mapRedTo", "redMin", "redMax", "Map Red To", "Red Min", "Red Max"), + ("mapGreenTo", "greenMin", "greenMax", "Map Green To", "Green Min", "Green Max"), + ("mapBlueTo", "blueMin", "blueMax", "Map Blue To", "Blue Min", "Blue Max"), + ] { + pg.push(p(to, n_to, Value::Enum(0), popup(&MAP_TARGETS))); + pg.push(p(mn, n_min, num(0.0), slider(-10000.0, 10000.0, -100.0, 100.0, 2))); + pg.push(p(mx, n_max, num(1.0), slider(-10000.0, 10000.0, -100.0, 100.0, 2))); + } + pg.push(p("randomSeed", "Random Seed", num(0.0), slider(0.0, 10000.0, 0.0, 1000.0, 0))); + let pc = |d: f64| (num(d), slider(0.0, 100.0, 0.0, 100.0, 1)); + vec![ + spec("ec.sim.particleplayground", "Particle Playground", pg, particle_playground), + spec( + "ec.sim.cchair", + "CC Hair", + vec![ + sp("length", "Length", px(30.0, 200.0)), + p("thickness", "Thickness", num(1.0), slider(0.05, 20.0, 0.1, 5.0, 2)), + p("weight", "Weight", num(0.2), slider(-10.0, 10.0, -2.0, 2.0, 2)), + p("constantMass", "Constant Mass", Value::Bool(false), ParamUi::Checkbox), + sp("density", "Density", px(100.0, 1000.0)), + // Hairfall Map + sp("mapStrength", "Map Strength", pc(0.0)), + p("mapLayer", "Map Layer", Value::Layer(None), ParamUi::Layer), + sp("mapSoftness", "Map Softness", px(0.0, 100.0)), + sp("addNoise", "Add Noise", pc(0.0)), + // Hair Color + p("hairColor", "Color", col(0.35, 0.25, 0.15), ParamUi::Color), + sp("brightness", "Brightness", px(100.0, 400.0)), + sp("opacity", "Opacity", pc(100.0)), + sp("colorInheritance", "Color Inheritance", pc(0.0)), + // Light + sp("lightIntensity", "Light Intensity", px(100.0, 400.0)), + p("lightDirection", "Light Direction", num(-45.0), ParamUi::Angle), + // Shading + sp("ambient", "Ambient", pc(40.0)), + sp("diffuse", "Diffuse", pc(60.0)), + sp("specular", "Specular", pc(30.0)), + sp("roughness", "Roughness", pc(20.0)), + p("randomSeed", "Random Seed", num(0.0), slider(0.0, 10000.0, 0.0, 1000.0, 0)), + ], + cc_hair, + ), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, MaskShape, run_fx}; + + fn run(vals: &[(&str, Value)], t: f64) -> Image { + run_fx("ec.sim.particleplayground", vals, Image::filled(80, 60, [0.2, 0.2, 0.2, 1.0]), t, EffectEnv::default()).img + } + + fn cannon(t: f64) -> Image { + run(&[("cannonPosition", Value::Vec2([40.0, 50.0])), ("velocity", num(60.0)), ("gravityForce", num(30.0))], t) + } + + #[test] + fn cannon_stream_is_seek_consistent() { + let a = cannon(1.0); + // Particles replace the layer: some coverage, mostly transparent, red. + let cov: f32 = a.data.iter().map(|p| p[3]).sum(); + assert!(cov > 5.0, "{cov}"); + assert!(a.data.iter().any(|p| p[3] > 0.5 && p[0] > 0.5 && p[1] < 0.1)); + // Seek back and forth: the same frame every time. + let _ = cannon(2.0); + let b = cannon(1.0); + assert_eq!(a.data, b.data); + let c = cannon(0.5); + assert_ne!(a.data, c.data); + // Nothing has been fired at time 0. + let z = cannon(0.0); + assert!(z.data.iter().all(|p| p[3] == 0.0)); + } + + #[test] + fn grid_falls_with_gravity_and_bounces_off_a_wall() { + let grid = |t: f64, env: EffectEnv| { + run_fx( + "ec.sim.particleplayground", + &[ + ("cannonEnabled", Value::Bool(false)), + ("gridPosition", Value::Vec2([40.0, 10.0])), + ("gridWidth", num(40.0)), + ("gridHeight", num(0.0)), + ("particlesAcross", num(5.0)), + ("particlesDown", num(1.0)), + ("gravityForce", num(100.0)), + ("wallBoundary", num(1.0)), + ], + Image::new(80, 60), + t, + env, + ) + .img + }; + // Alpha-weighted mean height of the particles. + let row_of = |img: &Image| { + let (mut s, mut n) = (0.0f64, 0.0f64); + for y in 0..60 { + for x in 0..80 { + let a = img.get(x, y)[3] as f64; + s += (y as f64 + 0.5) * a; + n += a; + } + } + (s / n.max(1e-9)).round() as i64 + }; + let t0 = grid(0.0, EffectEnv::default()); + let t1 = grid(0.5, EffectEnv::default()); + assert_eq!(row_of(&t0), 10); + // ½ g t² = 12.5 px. + assert!((row_of(&t1) - 22).abs() <= 2, "{}", row_of(&t1)); + // A wall (mask 1) whose floor is at y = 15 keeps them above it. + let masks = [MaskShape { name: "w".into(), points: vec![[0.0, 0.0], [80.0, 0.0], [80.0, 15.0], [0.0, 15.0]], closed: true, inverted: false }]; + let env = EffectEnv { masks: &masks, ..Default::default() }; + let w = grid(1.5, env); + assert!(row_of(&w) <= 16, "{}", row_of(&w)); + } + + #[test] + fn repel_spreads_particles() { + let spread = |force: f64| { + let img = run( + &[ + ("cannonEnabled", Value::Bool(false)), + ("gridPosition", Value::Vec2([40.0, 30.0])), + ("gridWidth", num(6.0)), + ("gridHeight", num(6.0)), + ("particlesAcross", num(3.0)), + ("particlesDown", num(3.0)), + ("gravityForce", num(0.0)), + ("repelForce", num(force)), + ("repelForceRadius", num(20.0)), + ], + 0.5, + ); + let (mut x0, mut x1) = (80, 0); + for y in 0..60 { + for x in 0..80 { + if img.get(x, y)[3] > 0.3 { + x0 = x0.min(x); + x1 = x1.max(x); + } + } + } + x1 - x0 + }; + assert!(spread(0.2) > spread(0.0) + 4, "{} vs {}", spread(0.2), spread(0.0)); + } + + #[test] + fn property_mapper_and_layer_map_read_layers() { + // Without a host, the layer-based features are inert (deterministic output). + let a = run(&[("mapRedTo", Value::Enum(6)), ("redMin", num(-50.0)), ("redMax", num(50.0))], 0.7); + let b = run(&[("mapRedTo", Value::Enum(6)), ("redMin", num(-50.0)), ("redMax", num(50.0))], 0.7); + assert_eq!(a.data, b.data); + let st = Pg { + cannon: false, + pos: [0.0; 2], + barrel_angle: 0.0, + barrel_radius: 0.0, + rate: 0.0, + dir_spread: 0.0, + vel: 0.0, + vel_spread: 0.0, + color: [1.0; 4], + radius: 2.0, + gravity: [0.0; 2], + grav_spread: 0.0, + repel: 0.0, + repel_radius: 0.0, + wall: None, + mapper: Some(MapImg { img: Image::filled(4, 4, [1.0, 0.0, 0.5, 1.0]), k: [1.0; 2], o: [0.0; 2] }), + map_to: [(6, 0.0, 120.0), (0, 0.0, 1.0), (4, 0.0, 2.0)], + seed: 1, + bounds: [-1e6, -1e6, 1e6, 1e6], + }; + let mut s = PgState::default(); + s.parts.push(PgParticle { p: [1.0, 1.0], v: [0.0; 2], origin: [1.0, 1.0], c: [1.0; 4], r: 2.0, mass: 1.0, friction: 0.0, force: [0.0; 2], id: 0 }); + st.step(&mut s); + // Red 1 → X velocity 120 px/s; blue 0.5 → scale 1 (radius 2). + assert!((s.parts[0].v[0] - 120.0).abs() < 1e-3); + assert!((s.parts[0].r - 2.0).abs() < 1e-3); + } + + #[test] + fn cc_hair_grows_from_opaque_pixels() { + let mut img = Image::new(60, 60); + for y in 20..40 { + for x in 20..40 { + img.set(x, y, [0.8, 0.6, 0.4, 1.0]); + } + } + let out = run_fx("ec.sim.cchair", &[("length", num(12.0)), ("density", num(300.0))], img.clone(), 0.0, EffectEnv::default()); + // Hair reaches beyond the square (above it: hair leans up, then droops). + let outside: f32 = (0..60).flat_map(|x| (0..20).map(move |y| (x, y))).map(|(x, y)| out.img.get(x, y)[3]).sum(); + assert!(outside > 1.0, "{outside}"); + // Nothing grows from transparent areas far away. + assert_eq!(out.img.get(2, 58)[3], 0.0); + let again = run_fx("ec.sim.cchair", &[("length", num(12.0)), ("density", num(300.0))], img.clone(), 3.0, EffectEnv::default()); + assert_eq!(out.img.data, again.img.data); + let none = run_fx("ec.sim.cchair", &[("length", num(0.0))], img.clone(), 0.0, EffectEnv::default()); + assert_eq!(none.img.data, img.data); + } +} diff --git a/crates/effects/src/utility.rs b/crates/effects/src/utility.rs index 49b7ed36..7689050d 100644 --- a/crates/effects/src/utility.rs +++ b/crates/effects/src/utility.rs @@ -153,7 +153,7 @@ fn cineon(ctx: &EffectCtx, mut b: Buf) -> Buf { // ---- Colour LUTs (.cube) ---- #[derive(Clone, Debug, PartialEq)] -pub(crate) struct Lut { +pub struct Lut { pub n1: usize, pub d1: Vec<[f32; 3]>, pub n3: usize, @@ -163,7 +163,7 @@ pub(crate) struct Lut { } /// Parse `.cube` text (TITLE, LUT_1D_SIZE, LUT_3D_SIZE, DOMAIN_MIN/MAX, data red-fastest). -pub(crate) fn parse_cube(text: &str) -> Option { +pub fn parse_cube(text: &str) -> Option { let mut lut = Lut { n1: 0, d1: vec![], n3: 0, d3: vec![], dmin: [0.0; 3], dmax: [1.0; 3] }; let mut rows: Vec<[f32; 3]> = Vec::new(); let three = |it: &mut std::str::SplitWhitespace| -> Option<[f32; 3]> { @@ -238,6 +238,57 @@ impl Lut { } c } + /// Apply with a choice of 3D interpolation: 0 nearest, 1 trilinear, otherwise tetrahedral. + pub fn apply_interp(&self, c: [f32; 3], interp: u32) -> [f32; 3] { + let mut c = c; + if self.n1 > 1 { + let t = self.norm(c); + c = [0, 1, 2].map(|i| { + let x = t[i] * (self.n1 - 1) as f32; + if interp == 0 { + return self.d1[(x.round() as usize).min(self.n1 - 1)][i]; + } + let i0 = (x.floor() as usize).min(self.n1 - 2); + let f = x - i0 as f32; + self.d1[i0][i] + (self.d1[i0 + 1][i] - self.d1[i0][i]) * f + }); + } + if self.n3 > 1 { + let t = self.norm(c); + c = match interp { + 0 => self.nearest(t), + 1 => self.trilinear(t), + _ => self.tetra(t), + }; + } + c + } + fn nearest(&self, t: [f32; 3]) -> [f32; 3] { + let n = self.n3; + let s = (n - 1) as f32; + let i = |x: f32| ((x * s).round() as usize).min(n - 1); + self.d3[i(t[0]) + i(t[1]) * n + i(t[2]) * n * n] + } + /// Trilinear interpolation on the lattice (red fastest). + pub fn trilinear(&self, t: [f32; 3]) -> [f32; 3] { + let n = self.n3; + let s = (n - 1) as f32; + let idx = |x: f32| { + let v = x * s; + let i = (v.floor() as usize).min(n - 2); + (i, v - i as f32) + }; + let (r, fr) = idx(t[0]); + let (g, fg) = idx(t[1]); + let (b, fb) = idx(t[2]); + let at = |dr: usize, dg: usize, db: usize| self.d3[(r + dr) + (g + dg) * n + (b + db) * n * n]; + let l = |a: [f32; 3], b: [f32; 3], f: f32| [a[0] + (b[0] - a[0]) * f, a[1] + (b[1] - a[1]) * f, a[2] + (b[2] - a[2]) * f]; + let c00 = l(at(0, 0, 0), at(1, 0, 0), fr); + let c10 = l(at(0, 1, 0), at(1, 1, 0), fr); + let c01 = l(at(0, 0, 1), at(1, 0, 1), fr); + let c11 = l(at(0, 1, 1), at(1, 1, 1), fr); + l(l(c00, c10, fg), l(c01, c11, fg), fb) + } fn tetra(&self, t: [f32; 3]) -> [f32; 3] { let n = self.n3; let s = (n - 1) as f32; diff --git a/crates/effects/src/vr.rs b/crates/effects/src/vr.rs new file mode 100644 index 00000000..ba7e6d9d --- /dev/null +++ b/crates/effects/src/vr.rs @@ -0,0 +1,1188 @@ +//! Immersive Video effects (VR Blur, VR Chromatic Aberrations, VR Color Gradients, VR Converter, +//! VR De-Noise, VR Digital Glitch, VR Fractal Noise, VR Glow, VR Plane to Sphere, VR Rotate +//! Sphere, VR Sharpen, VR Sphere To Plane). +//! +//! All of them treat the layer as a 360° equirectangular frame (longitude across, latitude down; +//! the front, +Z, at the centre) and are **seam-aware**: filters wrap around the left/right edge, +//! continue across the poles (a column continues on the opposite meridian), and spatial filters +//! widen their horizontal footprint by 1/cos(latitude) so a blur is the same size on the sphere +//! everywhere. Stereo footage (Frame Layout: over/under or side by side) is processed per eye. +//! +//! VR Converter maps between equirectangular, cube maps (4:3 cross, 3:2 and 6:1 strips), an +//! angular fisheye ("sphere map") and a rectilinear 2D view, through a common view direction. + +use std::f64::consts::PI; + +use effectcraft_color::{BlendMode, blend_pixel, luminance}; +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::{Image, Px}; +use rayon::prelude::*; + +use crate::generate::value_noise; +use crate::util::{hash1, unpremul}; +use crate::{Buf, EffectCtx, EffectSpec, col, num, p, popup, slider}; + +type V3 = [f64; 3]; + +fn spec(id: &'static str, name: &'static str, params: Vec, render: crate::RenderFn) -> EffectSpec { + EffectSpec { id, name, category: "Immersive Video", params, render, gpu: false, float: true } +} + +pub const FRAME_LAYOUTS: [&str; 3] = ["Monoscopic", "Stereo - Over/Under", "Stereo - Side by Side"]; + +fn layout_param() -> crate::ParamSpec { + p("frameLayout", "Frame Layout", Value::Enum(0), popup(&FRAME_LAYOUTS)) +} + +/// Eye rectangles (x, y, w, h) of a frame. +fn eyes(w: u32, h: u32, layout: u32) -> Vec<(u32, u32, u32, u32)> { + match layout { + 1 if h >= 2 => vec![(0, 0, w, h / 2), (0, h / 2, w, h - h / 2)], + 2 if w >= 2 => vec![(0, 0, w / 2, h), (w / 2, 0, w - w / 2, h)], + _ => vec![(0, 0, w, h)], + } +} + +/// Run `f` on every eye of `img` and reassemble. +fn per_eye(img: &Image, layout: u32, f: impl Fn(&Image, usize) -> Image + Sync) -> Image { + let rects = eyes(img.width, img.height, layout); + if rects.len() == 1 { + return f(img, 0); + } + let outs: Vec = rects.par_iter().enumerate().map(|(i, &(x, y, w, h))| f(&img.crop(x as i64, y as i64, w, h), i)).collect(); + let mut out = Image::new(img.width, img.height); + for (&(x0, y0, w, _), o) in rects.iter().zip(&outs) { + for (yy, row) in o.data.chunks(w.max(1) as usize).enumerate() { + let start = out.idx(x0, y0 + yy as u32); + out.data[start..start + row.len()].copy_from_slice(row); + } + } + out +} + +// ---------------------------------------------------------------- sphere geometry + +#[inline] +fn norm(v: V3) -> V3 { + let l = (v[0] * v[0] + v[1] * v[1] + v[2] * v[2]).sqrt().max(1e-300); + [v[0] / l, v[1] / l, v[2] / l] +} + +#[inline] +fn dot(a: V3, b: V3) -> f64 { + a[0] * b[0] + a[1] * b[1] + a[2] * b[2] +} + +/// Direction (y up, +z front) of an equirectangular position in pixels. +#[inline] +pub fn equi_dir(x: f64, y: f64, w: f64, h: f64) -> V3 { + let lon = (x / w - 0.5) * 2.0 * PI; + let lat = (0.5 - y / h) * PI; + [lat.cos() * lon.sin(), lat.sin(), lat.cos() * lon.cos()] +} + +/// Equirectangular pixel position of a direction. +#[inline] +pub fn equi_px(d: V3, w: f64, h: f64) -> (f64, f64) { + let lon = d[0].atan2(d[2]); + let lat = d[1].clamp(-1.0, 1.0).asin(); + ((lon / (2.0 * PI) + 0.5) * w, (0.5 - lat / PI) * h) +} + +/// Bilinear sample of an equirectangular image: wraps horizontally, continues over the poles. +pub fn sample_equi(img: &Image, x: f64, y: f64) -> Px { + let (w, h) = (img.width as i64, img.height as i64); + if w == 0 || h == 0 { + return [0.0; 4]; + } + let fx = x - 0.5; + let fy = y - 0.5; + let (x0, y0) = (fx.floor(), fy.floor()); + let (tx, ty) = ((fx - x0) as f32, (fy - y0) as f32); + let at = |xi: i64, yi: i64| -> Px { + let (mut xi, mut yi) = (xi, yi); + if yi < 0 { + yi = -1 - yi; + xi += w / 2; + } else if yi >= h { + yi = 2 * h - 1 - yi; + xi += w / 2; + } + img.data[(yi.clamp(0, h - 1) * w + xi.rem_euclid(w)) as usize] + }; + let (x0, y0) = (x0 as i64, y0 as i64); + let (a, b, c, d) = (at(x0, y0), at(x0 + 1, y0), at(x0, y0 + 1), at(x0 + 1, y0 + 1)); + let mut o = [0.0f32; 4]; + for k in 0..4 { + let top = a[k] + (b[k] - a[k]) * tx; + let bot = c[k] + (d[k] - c[k]) * tx; + o[k] = top + (bot - top) * ty; + } + o +} + +/// Rotation matrix (row-major) for tilt (about X), pan (about Y) and roll (about Z), degrees: +/// R = Ry(pan) · Rx(tilt) · Rz(roll). +pub fn rotation(tilt: f64, pan: f64, roll: f64) -> [[f64; 3]; 3] { + let (a, b, c) = (tilt.to_radians(), pan.to_radians(), roll.to_radians()); + let rx = [[1.0, 0.0, 0.0], [0.0, a.cos(), -a.sin()], [0.0, a.sin(), a.cos()]]; + let ry = [[b.cos(), 0.0, b.sin()], [0.0, 1.0, 0.0], [-b.sin(), 0.0, b.cos()]]; + let rz = [[c.cos(), -c.sin(), 0.0], [c.sin(), c.cos(), 0.0], [0.0, 0.0, 1.0]]; + mm(&ry, &mm(&rx, &rz)) +} + +fn mm(a: &[[f64; 3]; 3], b: &[[f64; 3]; 3]) -> [[f64; 3]; 3] { + let mut o = [[0.0; 3]; 3]; + for i in 0..3 { + for j in 0..3 { + o[i][j] = (0..3).map(|k| a[i][k] * b[k][j]).sum(); + } + } + o +} + +#[inline] +fn mv(m: &[[f64; 3]; 3], v: V3) -> V3 { + [dot(m[0], v), dot(m[1], v), dot(m[2], v)] +} + +#[inline] +fn mtv(m: &[[f64; 3]; 3], v: V3) -> V3 { + [m[0][0] * v[0] + m[1][0] * v[1] + m[2][0] * v[2], m[0][1] * v[0] + m[1][1] * v[1] + m[2][1] * v[2], m[0][2] * v[0] + m[1][2] * v[1] + m[2][2] * v[2]] +} + +// ---------------------------------------------------------------- projections (VR Converter) + +pub const PROJECTIONS: [&str; 6] = ["Equirectangular", "Cube-map 4:3", "Cube-map 3:2", "Cube-map 6:1", "Sphere Map", "2D Rectilinear"]; + +/// Cube faces: (forward, right, up). +const FACES: [(V3, V3, V3); 6] = [ + ([1.0, 0.0, 0.0], [0.0, 0.0, -1.0], [0.0, 1.0, 0.0]), // +X right + ([-1.0, 0.0, 0.0], [0.0, 0.0, 1.0], [0.0, 1.0, 0.0]), // -X left + ([0.0, 1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, -1.0]), // +Y top + ([0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]), // -Y bottom + ([0.0, 0.0, 1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0]), // +Z front + ([0.0, 0.0, -1.0], [-1.0, 0.0, 0.0], [0.0, 1.0, 0.0]), // -Z back +]; + +/// Grid size and the (column, row) of each face (+X, −X, +Y, −Y, +Z, −Z) for a cube layout. +fn cube_grid(kind: u32) -> ((u32, u32), [(u32, u32); 6]) { + match kind { + // Horizontal cross: top over the front; left, front, right, back across the middle. + 1 => ((4, 3), [(2, 1), (0, 1), (1, 0), (1, 2), (1, 1), (3, 1)]), + // 3 × 2: right, left, top / bottom, front, back. + 2 => ((3, 2), [(0, 0), (1, 0), (2, 0), (0, 1), (1, 1), (2, 1)]), + // 6 × 1 strip in face order. + _ => ((6, 1), [(0, 0), (1, 0), (2, 0), (3, 0), (4, 0), (5, 0)]), + } +} + +/// A projection: frame position (pixels) ↔ view direction. +#[derive(Clone, Copy, Debug)] +pub struct Proj { + pub kind: u32, + /// Horizontal field of view in degrees (sphere map and 2D only). + pub fov: f64, +} + +impl Proj { + /// Direction seen at frame position (x, y) of a `w`×`h` frame (None outside the image). + pub fn dir(&self, x: f64, y: f64, w: f64, h: f64) -> Option { + match self.kind { + 0 => Some(equi_dir(x, y, w, h)), + 1..=3 => { + let ((gw, gh), cells) = cube_grid(self.kind); + let (cw, ch) = (w / gw as f64, h / gh as f64); + let (cx, cy) = ((x / cw).floor(), (y / ch).floor()); + let f = cells.iter().position(|&(c, r)| c as f64 == cx && r as f64 == cy)?; + let a = (x - cx * cw) / cw * 2.0 - 1.0; + let b = (y - cy * ch) / ch * 2.0 - 1.0; + let (fw, r, u) = FACES[f]; + Some(norm([fw[0] + a * r[0] - b * u[0], fw[1] + a * r[1] - b * u[1], fw[2] + a * r[2] - b * u[2]])) + } + 4 => { + // Angular fisheye centred on +Z; radius ∝ angle, `fov` across the image width. + let half = (self.fov.clamp(1.0, 360.0) / 2.0).to_radians(); + let s = w.min(h) / 2.0; + let (u, v) = ((x - w / 2.0) / s, (h / 2.0 - y) / s); + let r = (u * u + v * v).sqrt(); + if r > 1.0 { + return None; + } + let th = r * half; + let (cu, cv) = if r > 1e-12 { (u / r, v / r) } else { (0.0, 0.0) }; + Some([th.sin() * cu, th.sin() * cv, th.cos()]) + } + _ => { + let t = (self.fov.clamp(1.0, 179.0) / 2.0).to_radians().tan(); + let u = (x / w * 2.0 - 1.0) * t; + let v = (1.0 - y / h * 2.0) * t * h / w; + Some(norm([u, v, 1.0])) + } + } + } + + /// Frame position (pixels) of direction `d`, and for cube maps the face rectangle to clamp + /// samples into. None when the direction is not visible. + pub fn pos(&self, d: V3, w: f64, h: f64) -> Option<(f64, f64, Option<[f64; 4]>)> { + match self.kind { + 0 => { + let (x, y) = equi_px(d, w, h); + Some((x, y, None)) + } + 1..=3 => { + let f = (0..6).max_by(|&i, &j| dot(d, FACES[i].0).total_cmp(&dot(d, FACES[j].0)))?; + let (fw, r, u) = FACES[f]; + let k = dot(d, fw); + let (a, b) = (dot(d, r) / k, -dot(d, u) / k); + let ((gw, gh), cells) = cube_grid(self.kind); + let (cw, ch) = (w / gw as f64, h / gh as f64); + let (c, rr) = cells[f]; + let (x0, y0) = (c as f64 * cw, rr as f64 * ch); + Some((x0 + (a + 1.0) * 0.5 * cw, y0 + (b + 1.0) * 0.5 * ch, Some([x0, y0, x0 + cw, y0 + ch]))) + } + 4 => { + let half = (self.fov.clamp(1.0, 360.0) / 2.0).to_radians(); + let th = d[2].clamp(-1.0, 1.0).acos(); + if th > half + 1e-9 { + return None; + } + let s = w.min(h) / 2.0; + let r = th / half; + let l = (d[0] * d[0] + d[1] * d[1]).sqrt(); + let (cu, cv) = if l > 1e-12 { (d[0] / l, d[1] / l) } else { (0.0, 0.0) }; + Some((w / 2.0 + cu * r * s, h / 2.0 - cv * r * s, None)) + } + _ => { + if d[2] <= 1e-9 { + return None; + } + let t = (self.fov.clamp(1.0, 179.0) / 2.0).to_radians().tan(); + let u = d[0] / d[2] / t; + let v = d[1] / d[2] / (t * h / w); + if u.abs() > 1.0 || v.abs() > 1.0 { + return None; + } + Some(((u + 1.0) * 0.5 * w, (1.0 - v) * 0.5 * h, None)) + } + } + } + + /// Sample image `img` (in this projection) in direction `d`. + pub fn sample(&self, img: &Image, d: V3) -> Px { + let (w, h) = (img.width as f64, img.height as f64); + match self.pos(d, w, h) { + None => [0.0; 4], + Some((x, y, None)) => { + if self.kind == 0 { + sample_equi(img, x, y) + } else { + img.sample_bilinear_clamped(x, y) + } + } + Some((x, y, Some(r))) => { + // Stay inside the face so neighbouring faces don't bleed in. + let x = x.clamp(r[0] + 0.5, r[2] - 0.5); + let y = y.clamp(r[1] + 0.5, r[3] - 0.5); + img.sample_bilinear_clamped(x, y) + } + } + } +} + +/// Re-project `src` (in `from`) into a `w`×`h` frame in `to`, rotating the view by `rot` +/// (output direction → source direction = rotᵀ · d). +pub fn convert(src: &Image, from: Proj, to: Proj, w: u32, h: u32, rot: Option<&[[f64; 3]; 3]>) -> Image { + let mut out = Image::new(w, h); + let (fw, fh) = (w as f64, h as f64); + out.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + *px = match to.dir(x as f64 + 0.5, y as f64 + 0.5, fw, fh) { + Some(d) => { + let d = match rot { + Some(m) => mtv(m, d), + None => d, + }; + from.sample(src, d) + } + None => [0.0; 4], + }; + } + }); + out +} + +// ---------------------------------------------------------------- seam-aware filters + +/// Horizontal box mean with wrap-around, radius `r` (pixels, ≤ w/2) on one row. +fn box_row_wrap(src: &[Px], dst: &mut [Px], r: usize) { + let n = src.len(); + if n == 0 { + return; + } + if r == 0 { + dst.copy_from_slice(src); + return; + } + let r = r.min((n - 1) / 2); + if r == 0 { + dst.copy_from_slice(src); + return; + } + let k = (2 * r + 1) as f64; + let mut acc = [0.0f64; 4]; + for i in 0..=2 * r { + let p = src[(i + n - r) % n]; + for c in 0..4 { + acc[c] += p[c] as f64; + } + } + for x in 0..n { + for c in 0..4 { + dst[x][c] = (acc[c] / k) as f32; + } + let add = src[(x + r + 1) % n]; + let sub = src[(x + n - r) % n]; + for c in 0..4 { + acc[c] += add[c] as f64 - sub[c] as f64; + } + } +} + +/// Vertical box mean with pole continuation, radius `r`, over all columns. +fn box_cols_pole(img: &Image, r: usize) -> Image { + let (w, h) = (img.width as usize, img.height as usize); + if r == 0 || h == 0 { + return img.clone(); + } + let at = |x: usize, y: i64| -> Px { + let (mut x, mut y) = (x, y); + if y < 0 { + y = -1 - y; + x = (x + w / 2) % w; + } else if y >= h as i64 { + y = 2 * h as i64 - 1 - y; + x = (x + w / 2) % w; + } + img.data[y.clamp(0, h as i64 - 1) as usize * w + x] + }; + let k = (2 * r + 1) as f64; + let cols: Vec> = (0..w) + .into_par_iter() + .map(|x| { + let mut out = vec![[0.0f32; 4]; h]; + let mut acc = [0.0f64; 4]; + for j in -(r as i64)..=(r as i64) { + let p = at(x, j); + for c in 0..4 { + acc[c] += p[c] as f64; + } + } + for y in 0..h { + for c in 0..4 { + out[y][c] = (acc[c] / k) as f32; + } + let add = at(x, y as i64 + r as i64 + 1); + let sub = at(x, y as i64 - r as i64); + for c in 0..4 { + acc[c] += add[c] as f64 - sub[c] as f64; + } + } + out + }) + .collect(); + let mut out = Image::new(img.width, img.height); + out.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + *px = cols[x][y]; + } + }); + out +} + +/// Gaussian-like blur of an equirectangular image with standard deviation `sigma` pixels at +/// the equator (3 box passes per axis), wrapping at the seam and continuing over the poles. +pub fn sphere_blur(img: &Image, sigma: f64) -> Image { + if sigma < 0.1 || img.is_empty() { + return img.clone(); + } + let (w, h) = (img.width as usize, img.height as usize); + let radii = crate::util::box_radii(sigma, 3); + let mut cur = img.clone(); + let mut tmp = Image::new(img.width, img.height); + for &r in &radii { + tmp.rows_mut().for_each(|(y, row)| { + let lat = (0.5 - (y as f64 + 0.5) / h as f64) * PI; + let rr = ((r as f64) / lat.cos().max(1e-3)).round().min((w / 2) as f64) as usize; + box_row_wrap(&cur.data[y * w..(y + 1) * w], row, rr); + }); + std::mem::swap(&mut cur, &mut tmp); + } + for &r in &radii { + cur = box_cols_pole(&cur, r); + } + cur +} + +// ---------------------------------------------------------------- effects + +fn layout(ctx: &EffectCtx) -> u32 { + ctx.params.e("frameLayout") +} + +fn vr_blur(ctx: &EffectCtx, mut b: Buf) -> Buf { + let s = ctx.params.f("blurriness").max(0.0) * b.scale * 0.5; + if s < 0.1 { + return b; + } + b.img = per_eye(&b.img, layout(ctx), |e, _| sphere_blur(e, s)); + b +} + +fn vr_sharpen(ctx: &EffectCtx, mut b: Buf) -> Buf { + let amt = ctx.params.f("sharpenAmount") as f32 / 100.0; + if amt == 0.0 { + return b; + } + let s = (1.5 * b.scale).max(0.5); + b.img = per_eye(&b.img, layout(ctx), |e, _| { + let bl = sphere_blur(e, s); + let mut o = e.clone(); + o.data.par_iter_mut().zip(bl.data.par_iter()).for_each(|(p, q)| { + for c in 0..3 { + p[c] = (p[c] + (p[c] - q[c]) * amt).clamp(0.0, p[3].max(0.0) * 4.0); + } + }); + o + }); + b +} + +fn vr_denoise(ctx: &EffectCtx, mut b: Buf) -> Buf { + let level = ctx.params.f("noiseLevel").max(0.0) / 100.0; + if level <= 0.0 { + return b; + } + let r = ((1.0 + level * 4.0) * b.scale).max(1.0) as usize; + let eps = (level * level * 0.02) as f32; + let detail = (ctx.params.f("detail") / 100.0).clamp(0.0, 1.0) as f32; + b.img = per_eye(&b.img, layout(ctx), |e, _| { + // Guided filter (self-guided, per channel) on a frame padded with wrapped / pole-continued + // pixels so the seam and poles are filtered like any other place. + let pad = r * 2 + 1; + let padded = wrap_pad(e, pad); + let guide = crate::util::Plane::luma(&padded); + let ch = crate::util::split(&padded); + let f: Vec = ch.par_iter().map(|c| crate::util::guided_filter(&guide, c, r, eps.max(1e-6))).collect(); + let joined = crate::util::join(&[f[0].clone(), f[1].clone(), f[2].clone(), f[3].clone()]); + let mut o = joined.crop(pad as i64, pad as i64, e.width, e.height); + o.data.par_iter_mut().zip(e.data.par_iter()).for_each(|(p, s)| { + for c in 0..4 { + p[c] = p[c] + (s[c] - p[c]) * detail; + } + p[3] = p[3].clamp(0.0, 1.0); + for c in 0..3 { + p[c] = p[c].max(0.0); + } + }); + o + }); + b +} + +/// Pad an equirectangular image by `pad` pixels: wrapped columns, pole-continued rows. +pub fn wrap_pad(img: &Image, pad: usize) -> Image { + let (w, h) = (img.width as i64, img.height as i64); + let p = pad as i64; + let mut out = Image::new((w + 2 * p) as u32, (h + 2 * p) as u32); + out.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let (mut sx, mut sy) = (x as i64 - p, y as i64 - p); + if sy < 0 { + sy = -1 - sy; + sx += w / 2; + } else if sy >= h { + sy = 2 * h - 1 - sy; + sx += w / 2; + } + *px = img.data[(sy.clamp(0, h - 1) * w + sx.rem_euclid(w.max(1))) as usize]; + } + }); + out +} + +fn vr_glow(ctx: &EffectCtx, mut b: Buf) -> Buf { + let th = ctx.params.f("luminanceThreshold") as f32 / 100.0; + let radius = ctx.params.f("glowRadius").max(0.0) * b.scale; + let bright = ctx.params.f("glowBrightness") as f32 / 100.0; + let sat = ctx.params.f("glowSaturation") as f32 / 100.0; + let tint = ctx.params.b("useTintColor"); + let tc = ctx.params.color("tintColor"); + if bright <= 0.0 { + return b; + } + b.img = per_eye(&b.img, layout(ctx), |e, _| { + let mut hi = e.clone(); + hi.data.par_iter_mut().for_each(|p| { + let (c, a) = unpremul(*p); + let l = luminance(c[0], c[1], c[2]); + let k = if l > th { ((l - th) / (1.0 - th).max(1e-3)).min(4.0) } else { 0.0 }; + let mut c = c.map(|v| v * k); + let lc = luminance(c[0], c[1], c[2]); + c = c.map(|v| lc + (v - lc) * sat); + if tint { + c = [lc * tc[0], lc * tc[1], lc * tc[2]]; + } + *p = [c[0] * a, c[1] * a, c[2] * a, a * k.min(1.0)]; + }); + let g = sphere_blur(&hi, (radius * 0.5).max(0.5)); + let mut o = e.clone(); + o.data.par_iter_mut().zip(g.data.par_iter()).for_each(|(p, q)| { + for c in 0..3 { + p[c] += q[c] * bright; + } + p[3] = (p[3] + q[3] * bright).min(1.0); + }); + o + }); + b +} + +fn rot_params(ctx: &EffectCtx) -> [[f64; 3]; 3] { + rotation(ctx.params.f("tilt"), ctx.params.f("pan"), ctx.params.f("roll")) +} + +fn vr_rotate(ctx: &EffectCtx, mut b: Buf) -> Buf { + let (t, pn, r) = (ctx.params.f("tilt"), ctx.params.f("pan"), ctx.params.f("roll")); + if t == 0.0 && pn == 0.0 && r == 0.0 { + return b; + } + let m = rot_params(ctx); + let eq = Proj { kind: 0, fov: 360.0 }; + b.img = per_eye(&b.img, layout(ctx), |e, _| convert(e, eq, eq, e.width, e.height, Some(&m))); + b +} + +fn vr_converter(ctx: &EffectCtx, mut b: Buf) -> Buf { + let from = Proj { kind: ctx.params.e("sourceProjection"), fov: ctx.params.f("sourceHorizontalFov") }; + let to = Proj { kind: ctx.params.e("targetProjection"), fov: ctx.params.f("targetHorizontalFov") }; + let (t, pn, r) = (ctx.params.f("tilt"), ctx.params.f("pan"), ctx.params.f("roll")); + if from.kind == to.kind && (from.kind <= 3 || from.fov == to.fov) && t == 0.0 && pn == 0.0 && r == 0.0 { + return b; + } + let m = rot_params(ctx); + b.img = per_eye(&b.img, layout(ctx), |e, _| convert(e, from, to, e.width, e.height, Some(&m))); + b +} + +fn vr_sphere_to_plane(ctx: &EffectCtx, mut b: Buf) -> Buf { + let fov = ctx.params.f("horizontalFov"); + let m = rot_params(ctx); + let eq = Proj { kind: 0, fov: 360.0 }; + let flat = Proj { kind: 5, fov }; + b.img = per_eye(&b.img, layout(ctx), |e, _| convert(e, eq, flat, e.width, e.height, Some(&m))); + b +} + +fn vr_plane_to_sphere(ctx: &EffectCtx, mut b: Buf) -> Buf { + // The layer is a flat picture placed on the sphere: horizontal FOV = 90° × scale. + let fov = (90.0 * ctx.params.f("scale") / 100.0).clamp(1.0, 179.0); + let m = rot_params(ctx); + let feather = (ctx.params.f("feather") / 100.0).clamp(0.0, 1.0); + let flat = Proj { kind: 5, fov }; + let eq = Proj { kind: 0, fov: 360.0 }; + let src = b.img.clone(); + let mut out = convert(&src, flat, eq, src.width, src.height, Some(&m)); + if feather > 0.0 { + let (w, h) = (src.width as f64, src.height as f64); + let t = (fov / 2.0).to_radians().tan(); + out.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let d = mtv(&m, equi_dir(x as f64 + 0.5, y as f64 + 0.5, w, h)); + if d[2] <= 0.0 { + continue; + } + let u = (d[0] / d[2] / t).abs(); + let v = (d[1] / d[2] / (t * h / w)).abs(); + let edge = 1.0 - u.max(v); + let k = (edge / feather).clamp(0.0, 1.0) as f32; + for c in px.iter_mut() { + *c *= k; + } + } + }); + } + b.img = out; + b +} + +/// Angle (radians) between two directions. +#[inline] +fn angle(a: V3, b: V3) -> f64 { + dot(a, b).clamp(-1.0, 1.0).acos() +} + +fn vr_chromatic(ctx: &EffectCtx, mut b: Buf) -> Buf { + let k = [ctx.params.f("aberrationRed"), ctx.params.f("aberrationGreen"), ctx.params.f("aberrationBlue")].map(|v| v / 100.0 * 0.1); + if k.iter().all(|v| *v == 0.0) { + return b; + } + let center = mv(&rotation(ctx.params.f("centerTilt"), ctx.params.f("centerPan"), 0.0), [0.0, 0.0, 1.0]); + let falloff = (ctx.params.f("falloff") / 100.0).clamp(0.0, 1.0); + b.img = per_eye(&b.img, layout(ctx), |e, _| { + let (w, h) = (e.width as f64, e.height as f64); + let mut o = Image::new(e.width, e.height); + o.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let d = equi_dir(x as f64 + 0.5, y as f64 + 0.5, w, h); + let th = angle(d, center); + // Axis perpendicular to centre and d: rotate d about it toward/away from centre. + let ax = [center[1] * d[2] - center[2] * d[1], center[2] * d[0] - center[0] * d[2], center[0] * d[1] - center[1] * d[0]]; + let al = dot(ax, ax).sqrt(); + let weight = (th / PI).powf(1.0 + falloff * 3.0); + let mut out = [0.0f32; 4]; + for c in 0..3 { + let s = if al < 1e-9 { + e.data[(y * e.width as usize) + x] + } else { + let new_th = th * (1.0 - k[c] * weight); + // Direction at angle new_th from the centre in the plane of (centre, d). + let perp = norm([d[0] - center[0] * th.cos(), d[1] - center[1] * th.cos(), d[2] - center[2] * th.cos()]); + let nd = [ + center[0] * new_th.cos() + perp[0] * new_th.sin(), + center[1] * new_th.cos() + perp[1] * new_th.sin(), + center[2] * new_th.cos() + perp[2] * new_th.sin(), + ]; + let (sx, sy) = equi_px(nd, w, h); + sample_equi(e, sx, sy) + }; + out[c] = s[c]; + out[3] = out[3].max(s[3]); + } + *px = out; + } + }); + o + }); + b +} + +const BLEND_OPTS: [&str; 6] = ["None", "Normal", "Add", "Multiply", "Screen", "Overlay"]; + +fn blend_mode(i: u32) -> Option { + match i { + 0 => None, + 2 => Some(BlendMode::Add), + 3 => Some(BlendMode::Multiply), + 4 => Some(BlendMode::Screen), + 5 => Some(BlendMode::Overlay), + _ => Some(BlendMode::Normal), + } +} + +/// Composite a generated eye image over the original with a blend mode and opacity (mode None +/// replaces the layer). +fn composite(orig: &Image, layer: &Image, mode: u32, opacity: f32) -> Image { + let mut o = orig.clone(); + o.data.par_iter_mut().zip(layer.data.par_iter()).for_each(|(d, s)| { + let s = [s[0] * opacity, s[1] * opacity, s[2] * opacity, s[3] * opacity]; + *d = match blend_mode(mode) { + None => s, + Some(m) => { + let mut r = blend_pixel(m, *d, s, 0.5); + // Keep the layer's own coverage: generated pixels show only where it is opaque. + let a = d[3]; + if a < 1.0 { + let k = a / r[3].max(1e-6); + if k < 1.0 { + r = r.map(|v| v * k); + } + } + r + } + }; + }); + o +} + +fn vr_gradients(ctx: &EffectCtx, mut b: Buf) -> Buf { + let pr = ctx.params; + let blend = pr.f("blend").max(0.1); + let opacity = (pr.f("opacity") / 100.0) as f32; + let mode = pr.e("blendingMode"); + let (lw, lh) = (ctx.layer_size[0].max(1.0), ctx.layer_size[1].max(1.0)); + let pts: Vec<(V3, [f32; 4])> = (1..=5) + .filter(|i| pr.b(&format!("enablePoint{i}"))) + .map(|i| { + let p = pr.v2(&format!("point{i}")); + (equi_dir(p[0], p[1], lw, lh), pr.color(&format!("color{i}"))) + }) + .collect(); + if pts.is_empty() { + return b; + } + b.img = per_eye(&b.img, layout(ctx), |e, _| { + let (w, h) = (e.width as f64, e.height as f64); + let mut g = Image::new(e.width, e.height); + g.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let d = equi_dir(x as f64 + 0.5, y as f64 + 0.5, w, h); + // Inverse-distance weighting on great-circle distance. + let mut acc = [0.0f64; 3]; + let mut ws = 0.0; + let mut exact = None; + for (pd, c) in &pts { + let a = angle(d, *pd); + if a < 1e-6 { + exact = Some(*c); + break; + } + let wgt = 1.0 / a.powf(blend); + for k in 0..3 { + acc[k] += c[k] as f64 * wgt; + } + ws += wgt; + } + let c = exact.map(|c| [c[0], c[1], c[2]]).unwrap_or([(acc[0] / ws) as f32, (acc[1] / ws) as f32, (acc[2] / ws) as f32]); + *px = [c[0], c[1], c[2], 1.0]; + } + }); + composite(e, &g, mode, opacity) + }); + b +} + +fn vr_fractal(ctx: &EffectCtx, mut b: Buf) -> Buf { + let pr = ctx.params; + let kind = pr.e("fractalType"); + let invert = pr.b("invert"); + let contrast = pr.f("contrast") as f32 / 100.0; + let brightness = pr.f("brightness") as f32 / 100.0; + let scale = (pr.f("scale") / 100.0).max(0.01) as f32; + let octaves = pr.f("complexity").clamp(1.0, 20.0); + let evo = (pr.f("evolution") / 360.0) as f32; + let opacity = (pr.f("opacity") / 100.0) as f32; + let mode = pr.e("blendingMode"); + let seed = pr.f("randomSeed") as u32; + let base = 4.0 / scale; + b.img = per_eye(&b.img, layout(ctx), |e, _| { + let (w, h) = (e.width as f64, e.height as f64); + let mut g = Image::new(e.width, e.height); + g.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let d = equi_dir(x as f64 + 0.5, y as f64 + 0.5, w, h); + let mut v = 0.0f32; + let mut amp = 1.0f32; + let mut freq = base; + let mut norm = 0.0f32; + let n_oct = octaves.ceil() as usize; + for o in 0..n_oct { + let wgt = if o + 1 == n_oct && octaves.fract() > 0.0 { octaves.fract() as f32 } else { 1.0 }; + let q = [d[0] as f32 * freq + evo * 0.37, d[1] as f32 * freq + evo * 0.71, d[2] as f32 * freq + evo * 0.59]; + let n = value_noise(q[0] + o as f32 * 17.3, q[1], q[2], seed.wrapping_add(o as u32)); + let n = match kind { + 1 => (n * 2.0 - 1.0).abs(), + 2 => 1.0 - (n * 2.0 - 1.0).abs(), + _ => n, + }; + v += n * amp * wgt; + norm += amp * wgt; + amp *= 0.5; + freq *= 2.0; + } + let mut v = v / norm.max(1e-6); + v = (v - 0.5) * contrast + 0.5 + brightness; + if invert { + v = 1.0 - v; + } + let v = v.clamp(0.0, 1.0); + *px = [v, v, v, 1.0]; + } + }); + composite(e, &g, mode, opacity) + }); + b +} + +fn vr_glitch(ctx: &EffectCtx, mut b: Buf) -> Buf { + let pr = ctx.params; + let amp = (pr.f("masterAmplitude") / 100.0).max(0.0); + if amp <= 0.0 { + return b; + } + let rate = pr.f("distortionRate").max(0.0); + let geo = pr.f("geometricDistortion") / 100.0 * amp; + let complexity = pr.f("distortionComplexity").clamp(1.0, 100.0); + let rgb = pr.f("colorDistortion") / 100.0 * amp; + let scan = (pr.f("scanlines") / 100.0 * amp) as f32; + let seed = (pr.f("randomSeed") as u32) ^ ctx.seed; + // The glitch pattern changes `rate` times a second (held between changes). + let slot = if rate > 0.0 { (ctx.time * rate).floor() as i64 } else { 0 } as u32; + b.img = per_eye(&b.img, layout(ctx), |e, eye| { + let (w, h) = (e.width as f64, e.height as f64); + let bands = complexity.round() as u32; + let mut o = Image::new(e.width, e.height); + o.rows_mut().for_each(|(y, row)| { + let band = ((y as f64 / h) * bands as f64) as u32; + let r0 = hash1(band, slot, seed ^ (eye as u32 * 977)); + let on = r0 < 0.35; + let shift = if on { (hash1(band, slot + 1, seed) as f64 * 2.0 - 1.0) * geo * w * 0.15 } else { 0.0 }; + let split = if on { rgb * w * 0.01 * (1.0 + hash1(band, slot + 2, seed) as f64) } else { 0.0 }; + let dark = if y % 2 == 0 { 1.0 - scan * 0.5 } else { 1.0 }; + for (x, px) in row.iter_mut().enumerate() { + let sx = x as f64 + 0.5 - shift; + let sy = y as f64 + 0.5; + // Horizontal shifts wrap around the seam (the frame is a full turn). + let g = sample_equi(e, sx, sy); + let r = sample_equi(e, sx - split, sy); + let bb = sample_equi(e, sx + split, sy); + *px = [r[0] * dark, g[1] * dark, bb[2] * dark, g[3].max(r[3]).max(bb[3])]; + } + }); + o + }); + b +} + +fn ang(id: &'static str, name: &'static str) -> crate::ParamSpec { + p(id, name, num(0.0), ParamUi::Angle) +} + +pub fn specs() -> Vec { + let pct = |d: f64| (num(d), slider(0.0, 100.0, 0.0, 100.0, 1)); + let mut grad = vec![layout_param()]; + let defaults = [ + ([0.25, 0.5], [1.0, 0.0, 0.0]), + ([0.5, 0.25], [1.0, 1.0, 0.0]), + ([0.75, 0.5], [0.0, 0.0, 1.0]), + ([0.5, 0.75], [0.0, 1.0, 0.0]), + ([0.0, 0.5], [1.0, 0.0, 1.0]), + ]; + const IDS: [[&str; 6]; 5] = [ + ["enablePoint1", "Enable Point 1", "point1", "Point 1", "color1", "Color 1"], + ["enablePoint2", "Enable Point 2", "point2", "Point 2", "color2", "Color 2"], + ["enablePoint3", "Enable Point 3", "point3", "Point 3", "color3", "Color 3"], + ["enablePoint4", "Enable Point 4", "point4", "Point 4", "color4", "Color 4"], + ["enablePoint5", "Enable Point 5", "point5", "Point 5", "color5", "Color 5"], + ]; + for (i, (ids, (pt, c))) in IDS.iter().zip(defaults).enumerate() { + grad.push(p(ids[0], ids[1], Value::Bool(i < 4), ParamUi::Checkbox)); + grad.push(p(ids[2], ids[3], Value::Vec2(pt), ParamUi::Point)); + grad.push(p(ids[4], ids[5], col(c[0], c[1], c[2]), ParamUi::Color)); + } + grad.push(p("blend", "Blend", num(2.0), slider(0.1, 10.0, 0.5, 6.0, 2))); + let (v, u) = pct(100.0); + grad.push(p("opacity", "Opacity", v, u)); + grad.push(p("blendingMode", "Blending Mode", Value::Enum(0), popup(&BLEND_OPTS))); + vec![ + spec("ec.vr.blur", "VR Blur", vec![layout_param(), p("blurriness", "Blurriness", num(10.0), slider(0.0, 1000.0, 0.0, 100.0, 1))], vr_blur), + spec( + "ec.vr.chromaticaberrations", + "VR Chromatic Aberrations", + vec![ + layout_param(), + ang("centerTilt", "Center Tilt"), + ang("centerPan", "Center Pan"), + p("aberrationRed", "Aberration (Red)", num(10.0), slider(-100.0, 100.0, -100.0, 100.0, 1)), + p("aberrationGreen", "Aberration (Green)", num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 1)), + p("aberrationBlue", "Aberration (Blue)", num(-10.0), slider(-100.0, 100.0, -100.0, 100.0, 1)), + p("falloff", "Falloff", num(50.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + ], + vr_chromatic, + ), + spec("ec.vr.colorgradients", "VR Color Gradients", grad, vr_gradients), + spec( + "ec.vr.converter", + "VR Converter", + vec![ + layout_param(), + p("sourceProjection", "Source Projection", Value::Enum(0), popup(&PROJECTIONS)), + p("sourceHorizontalFov", "Source Horizontal FOV", num(90.0), slider(1.0, 360.0, 1.0, 360.0, 1)), + p("targetProjection", "Target Projection", Value::Enum(2), popup(&PROJECTIONS)), + p("targetHorizontalFov", "Target Horizontal FOV", num(90.0), slider(1.0, 360.0, 1.0, 360.0, 1)), + ang("tilt", "Tilt (X Axis)"), + ang("pan", "Pan (Y Axis)"), + ang("roll", "Roll (Z Axis)"), + ], + vr_converter, + ), + spec( + "ec.vr.denoise", + "VR De-Noise", + vec![ + layout_param(), + p("noiseLevel", "Noise Level", num(20.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("detail", "Detail", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + ], + vr_denoise, + ), + spec( + "ec.vr.digitalglitch", + "VR Digital Glitch", + vec![ + layout_param(), + p("masterAmplitude", "Master Amplitude", num(100.0), slider(0.0, 500.0, 0.0, 200.0, 1)), + p("distortionRate", "Distortion Rate", num(5.0), slider(0.0, 60.0, 0.0, 30.0, 2)), + p("distortionComplexity", "Distortion Complexity", num(16.0), slider(1.0, 100.0, 1.0, 64.0, 0)), + p("geometricDistortion", "Geometric Distortion", num(50.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("colorDistortion", "Color Distortion", num(50.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("scanlines", "Scanlines", num(20.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("randomSeed", "Random Seed", num(0.0), slider(0.0, 10000.0, 0.0, 1000.0, 0)), + ], + vr_glitch, + ), + spec( + "ec.vr.fractalnoise", + "VR Fractal Noise", + vec![ + layout_param(), + p("fractalType", "Fractal Type", Value::Enum(0), popup(&["Basic", "Turbulent Smooth", "Turbulent Sharp"])), + p("invert", "Invert", Value::Bool(false), ParamUi::Checkbox), + p("contrast", "Contrast", num(100.0), slider(0.0, 10000.0, 0.0, 400.0, 1)), + p("brightness", "Brightness", num(0.0), slider(-10000.0, 10000.0, -200.0, 200.0, 1)), + p("scale", "Scale", num(100.0), slider(1.0, 10000.0, 10.0, 600.0, 1)), + p("complexity", "Complexity", num(6.0), slider(1.0, 20.0, 1.0, 20.0, 1)), + ang("evolution", "Evolution"), + p("randomSeed", "Random Seed", num(0.0), slider(0.0, 10000.0, 0.0, 1000.0, 0)), + p("opacity", "Opacity", num(100.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("blendingMode", "Blending Mode", Value::Enum(0), popup(&BLEND_OPTS)), + ], + vr_fractal, + ), + spec( + "ec.vr.glow", + "VR Glow", + vec![ + layout_param(), + p("luminanceThreshold", "Luminance Threshold", num(60.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + p("glowRadius", "Glow Radius", num(20.0), slider(0.0, 1000.0, 0.0, 200.0, 1)), + p("glowBrightness", "Glow Brightness", num(100.0), slider(0.0, 1000.0, 0.0, 400.0, 1)), + p("glowSaturation", "Glow Saturation", num(100.0), slider(0.0, 400.0, 0.0, 200.0, 1)), + p("useTintColor", "Use Tint Color", Value::Bool(false), ParamUi::Checkbox), + p("tintColor", "Tint Color", col(1.0, 0.8, 0.5), ParamUi::Color), + ], + vr_glow, + ), + spec( + "ec.vr.planetosphere", + "VR Plane to Sphere", + vec![ + layout_param(), + p("scale", "Scale", num(100.0), slider(1.0, 198.0, 1.0, 198.0, 1)), + ang("tilt", "Tilt (X Axis)"), + ang("pan", "Pan (Y Axis)"), + ang("roll", "Roll (Z Axis)"), + p("feather", "Feather", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 1)), + ], + vr_plane_to_sphere, + ), + spec( + "ec.vr.rotatesphere", + "VR Rotate Sphere", + vec![layout_param(), ang("tilt", "Tilt (X Axis)"), ang("pan", "Pan (Y Axis)"), ang("roll", "Roll (Z Axis)")], + vr_rotate, + ), + spec( + "ec.vr.sharpen", + "VR Sharpen", + vec![layout_param(), p("sharpenAmount", "Sharpen Amount", num(20.0), slider(0.0, 500.0, 0.0, 100.0, 1))], + vr_sharpen, + ), + spec( + "ec.vr.spheretoplane", + "VR Sphere To Plane", + vec![ + layout_param(), + p("horizontalFov", "Horizontal FOV", num(90.0), slider(1.0, 179.0, 10.0, 170.0, 1)), + ang("tilt", "Tilt (X Axis)"), + ang("pan", "Pan (Y Axis)"), + ang("roll", "Roll (Z Axis)"), + ], + vr_sphere_to_plane, + ), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, run_fx}; + + /// A smooth function of the view direction rendered as an equirectangular image. + fn sphere_img(w: u32, h: u32) -> Image { + let mut img = Image::new(w, h); + for y in 0..h { + for x in 0..w { + let d = equi_dir(x as f64 + 0.5, y as f64 + 0.5, w as f64, h as f64); + img.set(x, y, [(0.5 + 0.4 * d[0]) as f32, (0.5 + 0.4 * d[1]) as f32, (0.5 + 0.4 * d[2]) as f32, 1.0]); + } + } + img + } + + #[test] + fn projections_are_inverse() { + for kind in 0..6u32 { + let pr = Proj { kind, fov: if kind == 4 { 360.0 } else { 100.0 } }; + let (w, h) = (240.0, 120.0); + for (x, y) in [(10.5, 20.5), (100.5, 60.5), (230.5, 110.5), (130.5, 33.5)] { + let Some(d) = pr.dir(x, y, w, h) else { continue }; + let (px, py, _) = pr.pos(d, w, h).unwrap(); + assert!((px - x).abs() < 1e-6 && (py - y).abs() < 1e-6, "kind {kind}: ({x},{y}) → ({px},{py})"); + } + } + } + + #[test] + fn converter_round_trips_equirect_through_cube_maps() { + let img = sphere_img(128, 64); + for target in [1u32, 2, 3] { + let cube = run_fx("ec.vr.converter", &[("targetProjection", Value::Enum(target))], img.clone(), 0.0, EffectEnv::default()); + let back = run_fx( + "ec.vr.converter", + &[("sourceProjection", Value::Enum(target)), ("targetProjection", Value::Enum(0))], + cube.img.clone(), + 0.0, + EffectEnv::default(), + ); + let mut worst = 0.0f32; + let mut sum = 0.0f32; + for (a, b) in back.img.data.iter().zip(&img.data) { + let e = (0..3).map(|k| (a[k] - b[k]).abs()).fold(0.0, f32::max); + worst = worst.max(e); + sum += e; + } + let mean = sum / img.data.len() as f32; + // The 6:1 strip squeezes each face into a narrow cell, so its seams are coarser. + let tol = if target == 3 { 0.25 } else { 0.08 }; + assert!(mean < 0.01 && worst < tol, "target {target}: mean {mean} worst {worst}"); + // Cube faces carry the expected directions: the front face centre looks at +Z. + let pr = Proj { kind: target, fov: 90.0 }; + let (x, y, _) = pr.pos([0.0, 0.0, 1.0], 128.0, 64.0).unwrap(); + let px = cube.img.sample_bilinear_clamped(x, y); + assert!((px[2] - 0.9).abs() < 0.02, "{px:?}"); + } + } + + #[test] + fn rotate_sphere_pans_and_round_trips() { + let img = sphere_img(96, 48); + let a = run_fx("ec.vr.rotatesphere", &[("pan", num(90.0))], img.clone(), 0.0, EffectEnv::default()); + let b = run_fx("ec.vr.rotatesphere", &[("pan", num(-90.0))], a.img.clone(), 0.0, EffectEnv::default()); + let err: f32 = b.img.data.iter().zip(&img.data).map(|(p, q)| (p[0] - q[0]).abs()).sum::() / img.data.len() as f32; + assert!(err < 0.01, "{err}"); + // A 90° pan is a horizontal shift by a quarter turn. + let (x, y) = (10u32, 24u32); + let shifted = img.get(((x + 24) % 96) as i64, y as i64); + let got = a.img.get(x as i64, y as i64); + let alt = img.get(((x + 72) % 96) as i64, y as i64); + assert!((got[0] - shifted[0]).abs() < 0.02 || (got[0] - alt[0]).abs() < 0.02); + } + + #[test] + fn blur_is_seamless() { + // A frame whose left and right edges differ: after blurring, the two edge columns agree + // (they are neighbours on the sphere). + let mut img = Image::new(64, 32); + for y in 0..32 { + for x in 0..64 { + let v = if x < 4 { + 1.0 + } else if x >= 60 { + 0.0 + } else { + 0.5 + }; + img.set(x, y, [v, v, v, 1.0]); + } + } + let out = run_fx("ec.vr.blur", &[("blurriness", num(8.0))], img.clone(), 0.0, EffectEnv::default()); + let (l, r) = (out.img.get(0, 16)[0], out.img.get(63, 16)[0]); + assert!((l - r).abs() < 0.2, "{l} {r}"); + assert!(r > 0.2 && l < 0.85); + // Pole rows are uniform (every pixel of the top row is the same point). + let top: Vec = (0..64).map(|x| out.img.get(x, 0)[0]).collect(); + let spread = top.iter().cloned().fold(f32::MIN, f32::max) - top.iter().cloned().fold(f32::MAX, f32::min); + assert!(spread < 0.1, "{spread}"); + // Constant images stay constant. + let flat = Image::filled(32, 16, [0.3, 0.3, 0.3, 1.0]); + let o = run_fx("ec.vr.blur", &[], flat, 0.0, EffectEnv::default()); + assert!(o.img.data.iter().all(|p| (p[0] - 0.3).abs() < 1e-4)); + } + + #[test] + fn stereo_layouts_process_each_eye() { + let mut img = Image::new(32, 32); + for y in 0..32 { + for x in 0..32 { + let v = if y < 16 { 0.2 } else { 0.8 }; + img.set(x, y, [v, v, v, 1.0]); + } + } + // Over/under: blurring each eye keeps the eyes separate. + let o = run_fx("ec.vr.blur", &[("frameLayout", Value::Enum(1)), ("blurriness", num(10.0))], img.clone(), 0.0, EffectEnv::default()); + assert!((o.img.get(5, 15)[0] - 0.2).abs() < 1e-4 && (o.img.get(5, 16)[0] - 0.8).abs() < 1e-4); + let mono = run_fx("ec.vr.blur", &[("blurriness", num(10.0))], img, 0.0, EffectEnv::default()); + assert!((mono.img.get(5, 15)[0] - 0.2).abs() > 0.01); + } + + #[test] + fn sphere_to_plane_and_back() { + let img = sphere_img(128, 64); + // The centre of a 2D view looking forward sees +Z (blue 0.9). + let v = run_fx("ec.vr.spheretoplane", &[], img.clone(), 0.0, EffectEnv::default()); + let c = v.img.get(64, 32); + assert!((c[2] - 0.9).abs() < 0.02, "{c:?}"); + // Looking up (tilt 90°): +Y (green 0.9). + let up = run_fx("ec.vr.spheretoplane", &[("tilt", num(-90.0))], img.clone(), 0.0, EffectEnv::default()); + let u = up.img.get(64, 32); + let dn = run_fx("ec.vr.spheretoplane", &[("tilt", num(90.0))], img, 0.0, EffectEnv::default()); + let d = dn.img.get(64, 32); + assert!((u[1] - 0.9).abs() < 0.03 || (d[1] - 0.9).abs() < 0.03, "{u:?} {d:?}"); + // Plane to sphere puts the picture in front, transparent behind. + let pic = Image::filled(64, 32, [1.0, 0.0, 0.0, 1.0]); + let s = run_fx("ec.vr.planetosphere", &[], pic, 0.0, EffectEnv::default()); + assert!(s.img.get(32, 16)[3] > 0.99 && s.img.get(1, 16)[3] < 0.01); + } + + #[test] + fn generators_and_filters_are_deterministic_and_seamless() { + let img = sphere_img(64, 32); + for (id, vals) in [ + ("ec.vr.fractalnoise", vec![]), + ("ec.vr.colorgradients", vec![]), + ("ec.vr.glow", vec![("luminanceThreshold", num(50.0))]), + ("ec.vr.sharpen", vec![]), + ("ec.vr.denoise", vec![]), + ("ec.vr.chromaticaberrations", vec![]), + ("ec.vr.digitalglitch", vec![]), + ] { + let a = run_fx(id, &vals, img.clone(), 0.5, EffectEnv::default()); + let b = run_fx(id, &vals, img.clone(), 0.5, EffectEnv::default()); + assert_eq!(a.img.data, b.img.data, "{id}"); + assert!(a.img.data.iter().all(|p| p.iter().all(|v| v.is_finite())), "{id}"); + } + // Fractal noise is continuous across the seam: the first and last columns are close. + let n = run_fx("ec.vr.fractalnoise", &[], img.clone(), 0.0, EffectEnv::default()); + let mut worst = 0.0f32; + for y in 0..32 { + worst = worst.max((n.img.get(0, y)[0] - n.img.get(63, y)[0]).abs()); + } + assert!(worst < 0.15, "{worst}"); + // Gradients reach the point colour at the point. + let pts = [ + ("point1", Value::Vec2([16.0, 16.0])), + ("point2", Value::Vec2([32.0, 8.0])), + ("point3", Value::Vec2([48.0, 16.0])), + ("point4", Value::Vec2([32.0, 24.0])), + ]; + let g = run_fx("ec.vr.colorgradients", &pts, img.clone(), 0.0, EffectEnv::default()); + let px = g.img.get(16, 16); + assert!(px[0] > 0.8 && px[1] < 0.3, "{px:?}"); + // Glitch changes over time. + let g0 = run_fx("ec.vr.digitalglitch", &[], img.clone(), 0.0, EffectEnv::default()); + let g1 = run_fx("ec.vr.digitalglitch", &[], img.clone(), 3.3, EffectEnv::default()); + assert_ne!(g0.img.data, g1.img.data); + } +} diff --git a/crates/engine/src/commands/liquify.rs b/crates/engine/src/commands/liquify.rs new file mode 100644 index 00000000..748c664c --- /dev/null +++ b/crates/engine/src/commands/liquify.rs @@ -0,0 +1,156 @@ +//! Liquify brush strokes (`liquify.stroke`, `liquify.clear`). +//! +//! A stroke is appended to the layer's Liquify effect (added when the layer has none) as one +//! line of its hidden Distortion Mesh data (see `effectcraft_effects::distort4`). Brush size, +//! pressure, turbulent jitter and clone offset default to the effect's own tool options. When +//! the Distortion Mesh is animated, the stroke goes into a keyframe at the current time (copying +//! the mesh in effect there), as After Effects records mesh keyframes. + +use effectcraft_effects::distort4::{LIQUIFY_TOOLS, LiquifyStroke, parse_strokes}; +use effectcraft_keyframe::{Keyframe, Value as KV}; +use effectcraft_project::build::Ids; +use effectcraft_project::{GroupKind, PropGroup, Uid}; +use serde_json::{Value, json}; + +use super::{CommandSpec, bad, f_p, has_comp, layer_mut, layer_p, str_p}; +use crate::{EngineError, Result, Session, cmd}; + +const ID: &str = "ec.distort.liquify"; + +fn is_liquify(g: &PropGroup) -> bool { + matches!(&g.kind, GroupKind::Effect { effect } if effect == ID) +} + +fn liquify_effect(fx: &mut PropGroup, ids: &mut Ids, layer_size: [f64; 2], want: Option) -> Result { + if let Some(u) = want { + return fx.groups().find(|g| g.uid == u && is_liquify(g)).map(|g| g.uid).ok_or_else(|| bad("liquify.stroke", "no such Liquify effect")); + } + if let Some(g) = fx.groups().filter(|g| is_liquify(g)).last() { + return Ok(g.uid); + } + let spec = effectcraft_effects::find(ID).ok_or_else(|| EngineError::Other("Liquify effect missing".into()))?; + let g = effectcraft_effects::instantiate(spec, ids, "Liquify", layer_size); + let uid = g.uid; + fx.children.push(g.into()); + Ok(uid) +} + +fn num_of(g: &PropGroup, id: &str, default: f64) -> f64 { + g.get(id).map(|p| p.value.as_f64()).unwrap_or(default) +} + +fn stroke(s: &mut Session, p: &Value) -> Result { + let cmd = "liquify.stroke"; + let (cid, lid) = layer_p(s, p, cmd)?; + let tool_name = str_p(p, "tool").unwrap_or("warp"); + let tool = LIQUIFY_TOOLS + .iter() + .position(|t| t.eq_ignore_ascii_case(tool_name)) + .ok_or_else(|| bad(cmd, format!("unknown tool `{tool_name}` ({})", LIQUIFY_TOOLS.join(" | "))))?; + let raw = p.get("points").and_then(Value::as_array).ok_or_else(|| bad(cmd, "missing `points` [[x, y], …] (layer space)"))?; + let mut points = vec![]; + for q in raw { + let a = q.as_array().ok_or_else(|| bad(cmd, "each point is [x, y]"))?; + let (Some(x), Some(y)) = (a.first().and_then(Value::as_f64), a.get(1).and_then(Value::as_f64)) else { + return Err(bad(cmd, "each point is [x, y]")); + }; + points.push([x, y]); + } + if points.is_empty() { + return Err(bad(cmd, "`points` is empty")); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let t = comp.frame_rate.snap_nearest(super::time_p(s, p, Some(comp))); + let layer = comp.layer(lid).ok_or_else(|| bad(cmd, "no layer"))?; + if layer.effects().is_none() || !layer.source.is_av() { + return Err(bad(cmd, "Liquify works on footage, solid, text, shape and precomp layers")); + } + let lt = layer.layer_time(t); + let (w, h) = effectcraft_render::source_size(&s.project, layer); + let size = if w == 0 { [comp.width as f64, comp.height as f64] } else { [w as f64, h as f64] }; + let want = p.get("effect").and_then(Value::as_u64); + let (fx_uid, count) = s.edit("Liquify Stroke", None, |proj, _st| { + let mut next = proj.next_id; + let l = layer_mut(proj, cid, lid)?; + let fx = l.props.sub_mut("effects").ok_or_else(|| bad(cmd, "this layer has no effects"))?; + let mut ids = Ids(&mut next); + let fx_uid = liquify_effect(fx, &mut ids, size, want)?; + proj.next_id = next; + let l = layer_mut(proj, cid, lid)?; + let g = l.props.sub_mut("effects").and_then(|fx| fx.find_group_mut(fx_uid)).ok_or_else(|| bad(cmd, "Liquify effect gone"))?; + let co = g.get("cloneOffset").map(|p| p.value.as_vec2()).unwrap_or([0.0; 2]); + let st = LiquifyStroke { + tool, + size: f_p(p, "size").unwrap_or_else(|| num_of(g, "brushSize", 64.0)).max(1.0), + pressure: f_p(p, "pressure").unwrap_or_else(|| num_of(g, "brushPressure", 50.0)).clamp(1.0, 100.0), + jitter: f_p(p, "jitter").unwrap_or_else(|| num_of(g, "turbulentJitter", 70.0)).clamp(1.0, 100.0), + clone_offset: match p.get("cloneOffset").and_then(Value::as_array) { + Some(a) => [a.first().and_then(Value::as_f64).unwrap_or(0.0), a.get(1).and_then(Value::as_f64).unwrap_or(0.0)], + None => co, + }, + points: points.clone(), + }; + let prop = g.get_mut("distortionMesh").ok_or_else(|| bad(cmd, "Liquify effect has no Distortion Mesh"))?; + let append = |cur: &str| if cur.trim().is_empty() { st.to_line() } else { format!("{}\n{}", cur.trim_end(), st.to_line()) }; + let text = if prop.keys.is_empty() { + let cur = match &prop.value { + KV::Str(x) => x.clone(), + _ => String::new(), + }; + let v = append(&cur); + prop.value = KV::Str(v.clone()); + v + } else { + // Hold keyframes: the mesh in effect at `lt` is the last key at or before it. + let cur = prop.keys.iter().rfind(|k| k.time <= lt).or(prop.keys.first()).map(|k| k.value.clone()); + let cur = match cur { + Some(KV::Str(x)) => x, + _ => String::new(), + }; + let v = append(&cur); + match prop.keys.iter_mut().find(|k| k.time == lt) { + Some(k) => k.value = KV::Str(v.clone()), + None => { + prop.keys.push(Keyframe::new(lt, KV::Str(v.clone()))); + prop.keys.sort_by_key(|k| k.time); + } + } + v + }; + Ok((fx_uid, parse_strokes(&text).len())) + })?; + Ok(json!({"effect": fx_uid, "strokes": count})) +} + +fn clear(s: &mut Session, p: &Value) -> Result { + let cmd = "liquify.clear"; + let (cid, lid) = layer_p(s, p, cmd)?; + let want = p.get("effect").and_then(Value::as_u64); + s.edit("Clear Liquify Mesh", None, |proj, _| { + let l = layer_mut(proj, cid, lid)?; + let fx = l.props.sub_mut("effects").ok_or_else(|| bad(cmd, "no effects"))?; + let uid = + fx.groups().filter(|g| is_liquify(g) && want.is_none_or(|u| u == g.uid)).last().map(|g| g.uid).ok_or_else(|| bad(cmd, "no Liquify effect"))?; + let g = fx.find_group_mut(uid).ok_or_else(|| bad(cmd, "no Liquify effect"))?; + let prop = g.get_mut("distortionMesh").ok_or_else(|| bad(cmd, "no Distortion Mesh"))?; + prop.keys.clear(); + prop.value = KV::Str(String::new()); + Ok(()) + })?; + Ok(Value::Null) +} + +pub fn specs() -> Vec { + vec![ + cmd!( + "liquify.stroke", + "Liquify Stroke", + [], + None, + "{layer?, effect?: uid, tool?: warp|turbulence|twirlClockwise|twirlCounterclockwise|pucker|bloat|shiftPixels|reflection|clone|reconstruction|freeze|thaw, points: [[x,y],…] (layer space), size?, pressure? (1-100), jitter? (1-100), cloneOffset?: [dx,dy], time? (s) | frame?}", + has_comp, + stroke + ), + cmd!("liquify.clear", "Clear Liquify Mesh", [], None, "{layer?, effect?: uid}", has_comp, clear), + ] +} diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index e75a558f..1a144723 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -17,6 +17,7 @@ mod layer; mod layer_menu; mod layer_time; mod link; +mod liquify; mod lottie; mod markers; mod mask; @@ -133,6 +134,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(warp_cmds::specs()); v.extend(paint::specs()); v.extend(puppet::specs()); + v.extend(liquify::specs()); v.extend(settings::specs()); v.extend(stubs::specs()); v diff --git a/crates/engine/src/tests_paint.rs b/crates/engine/src/tests_paint.rs index e274f725..9926e04c 100644 --- a/crates/engine/src/tests_paint.rs +++ b/crates/engine/src/tests_paint.rs @@ -227,3 +227,26 @@ fn project_with_paint_and_puppet_round_trips() { let v: Value = serde_json::from_str(&j).unwrap(); assert!(v.to_string().contains("ec.paint.paint") && v.to_string().contains("ec.distort.puppet")); } + +#[test] +fn liquify_stroke_adds_the_effect_and_warps() { + let (mut s, id) = setup(); + // Layer (5, 30) is comp (55, 50). Before: opaque blue. + assert!(px(&s, 0.0, 55, 50)[3] > 0.99); + let r = s.execute("liquify.stroke", json!({"layer": id, "tool": "warp", "points": [[-20, 30], [10, 30]], "size": 40, "pressure": 100})).unwrap(); + assert_eq!(r["strokes"], 1); + let l = layer(&s, id); + let fx = l.effects().unwrap().groups().next().unwrap().clone(); + assert!(matches!(&fx.kind, effectcraft_project::GroupKind::Effect { effect } if effect == "ec.distort.liquify")); + // The warp pulls transparent pixels from outside the layer over its left edge. + assert!(px(&s, 0.0, 55, 50)[3] < 0.9, "{:?}", px(&s, 0.0, 55, 50)); + // A second stroke goes into the same effect. + let r2 = s.execute("liquify.stroke", json!({"layer": id, "tool": "reconstruction", "points": [[0, 30]], "size": 60, "pressure": 100})).unwrap(); + assert_eq!(r2["strokes"], 2); + assert_eq!(r2["effect"], r["effect"]); + assert!(s.execute("liquify.stroke", json!({"layer": id, "tool": "smudge", "points": [[0, 0]]})).is_err()); + s.execute("edit.undo", json!({})).unwrap(); + s.execute("edit.undo", json!({})).unwrap(); + assert!(layer(&s, id).effects().unwrap().groups().next().is_none()); + assert!(px(&s, 0.0, 55, 50)[3] > 0.99); +} diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 010a6414..adc3edce 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -20,7 +20,9 @@ filmcraft-media = { workspace = true } filmcraft-frame = { workspace = true } filmcraft-color = { workspace = true } filmcraft-time = { workspace = true } +exr = { workspace = true } image = { workspace = true, features = ["gif", "webp", "tiff", "bmp", "exr"] } rayon = { workspace = true } +serde_json = { workspace = true } thiserror = { workspace = true } log = { workspace = true } diff --git a/crates/media/src/exr_channels.rs b/crates/media/src/exr_channels.rs new file mode 100644 index 00000000..ba0034c3 --- /dev/null +++ b/crates/media/src/exr_channels.rs @@ -0,0 +1,152 @@ +//! Auxiliary channels from multi-layer OpenEXR files (depth, object / material IDs, normals, +//! Cryptomatte, …) for the 3D Channel effects, read with the pure-Rust `exr` crate. +//! +//! Every channel of every layer becomes a plane named `layer.channel` (or just `channel` for +//! the unnamed layer), placed in the display window (pixels outside a layer's data window are +//! 0, or the background depth for depth channels). Cryptomatte manifests come from the +//! `cryptomatte//name` and `cryptomatte//manifest` header attributes (the published +//! Cryptomatte metadata convention: the manifest is a JSON object of name → hex hash). + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_raster::AuxChannels; +use effectcraft_raster::channels3d::BACKGROUND_DEPTH; + +fn is_depth(name: &str) -> bool { + let last = name.rsplit('.').next().unwrap_or(name); + last.eq_ignore_ascii_case("z") || name.to_ascii_lowercase().contains("depth") +} + +/// Parse a Cryptomatte manifest (`{"name": "hexhash", …}`). +pub fn parse_manifest(json: &str) -> Vec<(String, u32)> { + let Ok(serde_json::Value::Object(m)) = serde_json::from_str::(json) else { return vec![] }; + let mut v: Vec<(String, u32)> = m.iter().filter_map(|(k, h)| Some((k.clone(), u32::from_str_radix(h.as_str()?.trim(), 16).ok()?))).collect(); + v.sort(); + v +} + +/// Read all channels of an EXR file's bytes. `None` when it is not a readable EXR. +pub fn read_exr_channels(bytes: &[u8]) -> Option { + use exr::prelude::*; + let img = read().no_deep_data().largest_resolution_level().all_channels().all_layers().all_attributes().from_buffered(std::io::Cursor::new(bytes)).ok()?; + let dw = img.attributes.display_window; + let (w, h) = (dw.size.0, dw.size.1); + if w == 0 || h == 0 { + return None; + } + let mut aux = AuxChannels::new(w as u32, h as u32, 1.0); + let mut crypto: HashMap, Option)> = HashMap::new(); + let mut collect_meta = |other: &HashMap| { + for (k, v) in other { + let key = k.to_string(); + let Some(rest) = key.strip_prefix("cryptomatte/") else { continue }; + let Some((id, field)) = rest.split_once('/') else { continue }; + let AttributeValue::Text(t) = v else { continue }; + let e = crypto.entry(id.to_string()).or_default(); + match field { + "name" => e.0 = Some(t.to_string()), + "manifest" => e.1 = Some(t.to_string()), + _ => {} + } + } + }; + collect_meta(&img.attributes.other); + for layer in img.layer_data.iter() { + collect_meta(&layer.attributes.other); + let prefix = layer.attributes.layer_name.as_ref().map(|t| t.to_string()).filter(|s| !s.is_empty()); + let (lw, lh) = (layer.size.0, layer.size.1); + let pos = layer.attributes.layer_position; + let (ox, oy) = (pos.0 as i64 - dw.position.0 as i64, pos.1 as i64 - dw.position.1 as i64); + for ch in layer.channel_data.list.iter() { + let name = match &prefix { + Some(p) => format!("{p}.{}", ch.name), + None => ch.name.to_string(), + }; + let fill = if is_depth(&name) { BACKGROUND_DEPTH } else { 0.0 }; + let mut plane = vec![fill; w * h]; + let vals: Vec = ch.sample_data.values_as_f32().collect(); + if vals.len() < lw * lh { + continue; + } + for y in 0..lh { + let ty = y as i64 + oy; + if ty < 0 || ty >= h as i64 { + continue; + } + for x in 0..lw { + let tx = x as i64 + ox; + if tx < 0 || tx >= w as i64 { + continue; + } + plane[ty as usize * w + tx as usize] = vals[y * lw + x]; + } + } + aux.channels.push((name, plane)); + } + } + for (_, (name, manifest)) in crypto { + if let (Some(n), Some(m)) = (name, manifest) { + aux.manifests.push((n, parse_manifest(&m))); + } + } + aux.manifests.sort(); + Some(aux) +} + +type Cache = Mutex>)>>; + +/// Read (and cache, a few files) the channels of the EXR at `path` via `read`. +pub(crate) fn cached(path: &str, read: impl FnOnce() -> Option>) -> Option> { + static CACHE: OnceLock = OnceLock::new(); + let cache = CACHE.get_or_init(|| Mutex::new(Vec::new())); + if let Some(v) = cache.lock().ok().and_then(|c| c.iter().find(|(p, _)| p == path).map(|(_, v)| v.clone())) { + return v; + } + let v = read().and_then(|b| read_exr_channels(&b)).map(Arc::new); + if let Ok(mut c) = cache.lock() { + if c.len() >= 4 { + c.remove(0); + } + c.push((path.to_string(), v.clone())); + } + v +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_layered_channels_and_cryptomatte_metadata() { + use exr::prelude::*; + let (w, h) = (4usize, 2usize); + let z: Vec = (0..w * h).map(|i| i as f32 * 10.0).collect(); + let id: Vec = (0..w * h).map(|i| (i % 2) as f32 + 1.0).collect(); + let r: Vec = vec![0.5; w * h]; + let channels = AnyChannels::sort( + vec![ + AnyChannel::new("R", FlatSamples::F32(r)), + AnyChannel::new("Z", FlatSamples::F32(z.clone())), + AnyChannel::new("ObjectID", FlatSamples::F32(id.clone())), + ] + .into(), + ); + let mut layer = Layer::new((w, h), LayerAttributes::named("depth"), Encoding::FAST_LOSSLESS, channels); + layer.attributes.other.insert(Text::from("cryptomatte/abc1234/name"), AttributeValue::Text(Text::from("CryptoObject"))); + layer + .attributes + .other + .insert(Text::from("cryptomatte/abc1234/manifest"), AttributeValue::Text(Text::from(r#"{"bunny":"13851a76","floor":"0a1b2c3d"}"#))); + let image = Image::from_layer(layer); + let mut bytes = std::io::Cursor::new(Vec::new()); + image.write().to_buffered(&mut bytes).unwrap(); + let aux = read_exr_channels(bytes.get_ref()).unwrap(); + assert_eq!((aux.width, aux.height), (4, 2)); + assert_eq!(aux.get("depth.Z").unwrap(), z.as_slice()); + assert_eq!(aux.depth().unwrap(), z.as_slice()); + assert_eq!(aux.object_id().unwrap(), id.as_slice()); + assert_eq!(aux.manifests, vec![("CryptoObject".to_string(), vec![("bunny".to_string(), 0x1385_1a76), ("floor".to_string(), 0x0a1b_2c3d)])]); + assert!(read_exr_channels(b"not an exr").is_none()); + } +} diff --git a/crates/media/src/lib.rs b/crates/media/src/lib.rs index fcc872d1..862fbfc2 100644 --- a/crates/media/src/lib.rs +++ b/crates/media/src/lib.rs @@ -13,6 +13,7 @@ //! (PNG, JPEG, GIF, WebP, TIFF, BMP, OpenEXR). mod convert; +pub mod exr_channels; mod pool; mod probe; diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index a7898fc6..b6632fba 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -503,6 +503,18 @@ impl FootageSource for MediaPool { } } + fn aux(&self, _item: ItemId, footage: &Footage, t: Tick) -> Option> { + if footage.missing || !footage.has_video { + return None; + } + let loc = Inner::locate(footage, t); + if loc.media_t.is_some() || !loc.key.path.to_ascii_lowercase().ends_with(".exr") { + return None; + } + let path = loc.key.path.clone(); + crate::exr_channels::cached(&path, || self.inner.read(&path).ok()) + } + fn audio(&self, _item: ItemId, footage: &Footage, t: Tick, frames: usize, rate: u32) -> Option> { if !footage.has_audio || footage.missing { return None; diff --git a/crates/raster/src/channels3d.rs b/crates/raster/src/channels3d.rs new file mode 100644 index 00000000..0402349b --- /dev/null +++ b/crates/raster/src/channels3d.rs @@ -0,0 +1,206 @@ +//! Auxiliary (3D) channels that travel with a layer's pixels: depth, object / material IDs, +//! surface normals, texture UVs, coverage and Cryptomatte ID/coverage pairs. +//! +//! They come from multi-layer OpenEXR footage (any channel by its full `layer.channel` name) +//! or from the compositor's own render passes for precomps of 3D comps. The 3D Channel effects +//! read them through `EffectHost::aux`. +//! +//! Conventions (shared by the EXR reader and the render pass): +//! +//! * channel planes are row-major `width × height` `f32`s covering the layer's source rectangle, +//! with `scale` aux pixels per layer pixel; +//! * depth is camera-space distance in pixels (`Z`), background `BACKGROUND_DEPTH`; +//! * `ObjectID` / `MaterialID` are small integers (0 = background); +//! * Cryptomatte layers follow the published Cryptomatte specification (Psyop, 2015): a layer +//! named `CryptoObject` stores rank pairs in `CryptoObject00.R/G/B/A`, `CryptoObject01.R/…` +//! (ID float, coverage, ID float, coverage, …), IDs are MurmurHash3 (x86, 32-bit, seed 0) of +//! the object name reinterpreted as a float with the exponent nudged off 0 and 255, and the +//! manifest maps names to hashes. + +/// Depth written where nothing was rendered. +pub const BACKGROUND_DEPTH: f32 = 1.0e10; + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct AuxChannels { + pub width: u32, + pub height: u32, + /// Aux pixels per layer pixel. + pub scale: f64, + /// Named planes (`Z`, `ObjectID`, `CryptoObject00.R`, …). + pub channels: Vec<(String, Vec)>, + /// Cryptomatte manifests: (layer name, [(object name, hash)]). + pub manifests: Vec<(String, Vec<(String, u32)>)>, +} + +impl AuxChannels { + pub fn new(width: u32, height: u32, scale: f64) -> AuxChannels { + AuxChannels { width, height, scale, channels: vec![], manifests: vec![] } + } + + /// A channel by exact name (case-insensitive). + pub fn get(&self, name: &str) -> Option<&[f32]> { + self.channels.iter().find(|(n, _)| n.eq_ignore_ascii_case(name)).map(|(_, v)| v.as_slice()) + } + + /// The first channel present among `names` (also matching `*.name` suffixes, so `Z` finds + /// `depth.Z`). + pub fn find(&self, names: &[&str]) -> Option<&[f32]> { + for n in names { + if let Some(v) = self.get(n) { + return Some(v); + } + } + for n in names { + let suffix = format!(".{}", n.to_ascii_lowercase()); + if let Some((_, v)) = self.channels.iter().find(|(c, _)| c.to_ascii_lowercase().ends_with(&suffix)) { + return Some(v); + } + } + None + } + + pub fn names(&self) -> Vec<&str> { + self.channels.iter().map(|(n, _)| n.as_str()).collect() + } + + pub fn depth(&self) -> Option<&[f32]> { + self.find(&["Z", "depth", "Depth", "zDepth", "Z-Depth"]) + } + pub fn object_id(&self) -> Option<&[f32]> { + self.find(&["ObjectID", "object_id", "objectId", "id"]) + } + pub fn material_id(&self) -> Option<&[f32]> { + self.find(&["MaterialID", "material_id", "materialId"]) + } + pub fn coverage(&self) -> Option<&[f32]> { + self.find(&["Coverage", "coverage"]) + } + + /// Cryptomatte layer names present (prefixes with a `00.R` channel). + pub fn crypto_layers(&self) -> Vec { + let mut v: Vec = + self.channels.iter().filter_map(|(n, _)| n.strip_suffix("00.R").or_else(|| n.strip_suffix("00.r")).map(str::to_string)).collect(); + v.sort(); + v.dedup(); + v + } + + /// Rank pairs (id, coverage) of Cryptomatte layer `layer` at aux pixel index `i`. + pub fn crypto_ranks(&self, layer: &str, i: usize) -> Vec<(f32, f32)> { + let mut out = vec![]; + for k in 0.. { + let (r, g, b, a) = ( + self.get(&format!("{layer}{k:02}.R")), + self.get(&format!("{layer}{k:02}.G")), + self.get(&format!("{layer}{k:02}.B")), + self.get(&format!("{layer}{k:02}.A")), + ); + let (Some(r), Some(g)) = (r, g) else { break }; + out.push((r[i], g[i])); + if let (Some(b), Some(a)) = (b, a) { + out.push((b[i], a[i])); + } + } + out + } + + /// Aux pixel index of a layer-space point (nearest; `None` outside). + #[inline] + pub fn index_at(&self, lx: f64, ly: f64) -> Option { + let x = (lx * self.scale).floor(); + let y = (ly * self.scale).floor(); + if x < 0.0 || y < 0.0 || x >= self.width as f64 || y >= self.height as f64 { + return None; + } + Some(y as usize * self.width as usize + x as usize) + } + + /// Manifest entry name for a hash in Cryptomatte layer `layer`. + pub fn manifest_name(&self, layer: &str, hash: u32) -> Option<&str> { + self.manifests.iter().find(|(l, _)| l == layer)?.1.iter().find(|(_, h)| *h == hash).map(|(n, _)| n.as_str()) + } +} + +/// MurmurHash3 x86 32-bit (public-domain algorithm by Austin Appleby). +pub fn murmur3_32(data: &[u8], seed: u32) -> u32 { + const C1: u32 = 0xcc9e_2d51; + const C2: u32 = 0x1b87_3593; + let mut h = seed; + let chunks = data.chunks_exact(4); + let tail = chunks.remainder(); + for c in chunks { + let mut k = u32::from_le_bytes([c[0], c[1], c[2], c[3]]); + k = k.wrapping_mul(C1).rotate_left(15).wrapping_mul(C2); + h ^= k; + h = h.rotate_left(13).wrapping_mul(5).wrapping_add(0xe654_6b64); + } + let mut k = 0u32; + for (i, b) in tail.iter().enumerate() { + k ^= (*b as u32) << (8 * i); + } + if !tail.is_empty() { + k = k.wrapping_mul(C1).rotate_left(15).wrapping_mul(C2); + h ^= k; + } + h ^= data.len() as u32; + h ^= h >> 16; + h = h.wrapping_mul(0x85eb_ca6b); + h ^= h >> 13; + h = h.wrapping_mul(0xc2b2_ae35); + h ^= h >> 16; + h +} + +/// Cryptomatte hash of a name (the manifest value). +pub fn crypto_hash(name: &str) -> u32 { + let h = murmur3_32(name.as_bytes(), 0); + // Keep the float finite and normal: flip the low exponent bit when it is all 0s or 1s. + let e = (h >> 23) & 0xff; + if e == 0 || e == 255 { h ^ (1 << 23) } else { h } +} + +/// The ID float stored in Cryptomatte channels for a hash. +pub fn crypto_float(hash: u32) -> f32 { + f32::from_bits(hash) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn murmur3_reference_values() { + // Published MurmurHash3_x86_32 test vectors. + assert_eq!(murmur3_32(b"", 0), 0); + assert_eq!(murmur3_32(b"", 1), 0x514e_28b7); + assert_eq!(murmur3_32(b"", 0xffff_ffff), 0x81f1_6f39); + assert_eq!(murmur3_32(&[0, 0, 0, 0], 0), 0x2362_f9de); + assert_eq!(murmur3_32(b"a", 0x9747_b28c), 0x7fa0_9ea6); + assert_eq!(murmur3_32(b"aaaa", 0x9747_b28c), 0x5a97_808a); + assert_eq!(murmur3_32(b"Hello, world!", 0x9747_b28c), 0x2488_4cba); + assert_eq!(murmur3_32(b"The quick brown fox jumps over the lazy dog", 0x9747_b28c), 0x2fa8_26cd); + } + + #[test] + fn crypto_floats_are_normal_and_round_trip() { + for n in ["bunny", "set", "ground", "a", ""] { + let h = crypto_hash(n); + let f = crypto_float(h); + assert!(f.is_finite() && (f == 0.0 || f.is_normal() || f.abs() > 0.0), "{n}"); + assert_eq!(f.to_bits(), h); + } + } + + #[test] + fn lookup_helpers() { + let mut a = AuxChannels::new(2, 1, 1.0); + a.channels.push(("depth.Z".into(), vec![5.0, 6.0])); + a.channels.push(("CryptoObject00.R".into(), vec![1.0, 2.0])); + a.channels.push(("CryptoObject00.G".into(), vec![0.5, 1.0])); + assert_eq!(a.depth(), Some(&[5.0, 6.0][..])); + assert_eq!(a.crypto_layers(), vec!["CryptoObject".to_string()]); + assert_eq!(a.crypto_ranks("CryptoObject", 0), vec![(1.0, 0.5)]); + assert_eq!(a.index_at(1.5, 0.2), Some(1)); + assert_eq!(a.index_at(2.5, 0.2), None); + } +} diff --git a/crates/raster/src/lib.rs b/crates/raster/src/lib.rs index dfe53848..4f244199 100644 --- a/crates/raster/src/lib.rs +++ b/crates/raster/src/lib.rs @@ -4,6 +4,7 @@ //! heavy operation is row-parallel with rayon. pub mod blur; +pub mod channels3d; pub mod flow; pub mod warp; @@ -11,6 +12,7 @@ use effectcraft_color::{BlendMode, blend_pixel}; use rayon::prelude::*; pub use blur::{box_blur, directional_blur, gaussian_blur, radial_blur}; +pub use channels3d::AuxChannels; pub use warp::{Sampling, WarpOpts, accumulate_warp, composite_warp, composite_warp_reference, resample}; pub type Px = [f32; 4]; diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index d557357b..64e2684b 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -39,6 +39,12 @@ pub trait FootageSource: Send + Sync { fn audio(&self, _item: ItemId, _footage: &Footage, _t: Tick, _frames: usize, _rate: u32) -> Option> { None } + /// Auxiliary 3D channels of `item` at source time `t` (multi-layer OpenEXR: depth, IDs, + /// Cryptomatte, any named channel), with `scale` relative to the file's pixels. `None` when + /// the footage has none. + fn aux(&self, _item: ItemId, _footage: &Footage, _t: Tick) -> Option> { + None + } /// The parsed 3D model of a [`effectcraft_project::FootageKind::Model`] item (Advanced 3D). fn model(&self, _item: ItemId, _footage: &Footage) -> Option> { None @@ -111,6 +117,29 @@ impl EffectHost for FxHost<'_, '_, '_> { sub.layer_input(&self.ctx.at(t), self.layer, n).map(|b| (*b).clone()) } + fn aux(&self) -> Option> { + match &self.layer.source { + LayerSource::Footage { item } => { + let ItemKind::Footage(f) = &self.r.project.item(*item)?.kind else { return None }; + let a = self.r.footage.aux(*item, f, self.ctx.source_time(self.layer))?; + // Aux pixels per layer pixel (the layer is the footage at its nominal size). + if f.width > 0 && a.width > 0 { + let mut a = (*a).clone(); + a.scale = a.width as f64 / f.width as f64; + return Some(Arc::new(a)); + } + Some(a) + } + LayerSource::Comp { item } => { + if self.r.depth > MAX_FX_DEPTH || self.r.project.comp_contains(*item, self.ctx.comp_id) { + return None; + } + self.r.nested().comp_aux(*item, self.ctx.source_time(self.layer)).map(Arc::new) + } + _ => None, + } + } + fn layer_at(&self, id: u64, comp_time: f64, masks_and_effects: bool) -> Option { let other = self.ctx.layer(effectcraft_project::LayerId(id))?; if other.id == self.layer.id || self.r.depth > MAX_FX_DEPTH { @@ -537,8 +566,15 @@ impl<'a> Renderer<'a> { let layer_size = if size.0 == 0 { [ctx.comp.width as f64, ctx.comp.height as f64] } else { [size.0 as f64, size.1 as f64] }; let mask_shapes = masks::shapes(ctx, layer); let host = FxHost { r: self, ctx, layer, index: Default::default() }; - let env = - EffectEnv { masks: &mask_shapes, host: Some(&host), comp_time: ctx.time.seconds(), frame_rate: ctx.comp.frame_rate.as_f64(), effect_index: 0 }; + let env = EffectEnv { + masks: &mask_shapes, + host: Some(&host), + comp_time: ctx.time.seconds(), + frame_rate: ctx.comp.frame_rate.as_f64(), + effect_index: 0, + working_space: self.pipe.space, + working_linear: self.pipe.linear, + }; // Video effects in stack order (index, group, spec); disabled and audio effects skipped. let stack: Vec<(usize, &effectcraft_project::PropGroup, &'static effectcraft_effects::EffectSpec)> = fx .groups() @@ -1178,6 +1214,13 @@ impl<'a> Renderer<'a> { self.depth } + /// The auxiliary 3D channels of comp `comp_id` at comp time `t` (depth, layer IDs, normals, + /// UVs, Cryptomatte; see `three_d::compose::aux_pass`), at the renderer's output scale. + pub fn comp_aux(&self, comp_id: ItemId, t: Tick) -> Option { + let ctx = self.eval_ctx(comp_id, t)?; + Some(three_d::compose::aux_pass(self, &ctx)) + } + /// Evaluation context of comp `comp_id` at comp time `t`. pub fn eval_ctx(&self, comp_id: ItemId, t: Tick) -> Option> { Some(self.ctx(comp_id, self.project.comp(comp_id)?, t)) @@ -1298,6 +1341,8 @@ pub fn kurbo_path(sp: &effectcraft_keyframe::ShapePath) -> kurbo::BezPath { #[cfg(test)] mod tests_audio_fx; #[cfg(test)] +mod tests_aux; +#[cfg(test)] mod tests_collapse; #[cfg(test)] mod tests_color; diff --git a/crates/render/src/tests_audio_fx.rs b/crates/render/src/tests_audio_fx.rs index dfd49ccc..a249955f 100644 --- a/crates/render/src/tests_audio_fx.rs +++ b/crates/render/src/tests_audio_fx.rs @@ -174,7 +174,17 @@ fn blocks_match_one_long_mix() { let x = ((t * 12345.678).sin() * 43758.5453).fract() as f32; (x, -x) } - for id in ["ec.audio.reverb", "ec.audio.basstreble", "ec.audio.flangechorus", "ec.audio.modulator", "ec.audio.delay", "ec.audio.parametriceq"] { + for id in [ + "ec.audio.reverb", + "ec.audio.basstreble", + "ec.audio.flangechorus", + "ec.audio.modulator", + "ec.audio.delay", + "ec.audio.parametriceq", + "ec.audio.compressor", + "ec.audio.gate", + "ec.audio.distortion", + ] { let vals: Vec<(&str, Value)> = match id { "ec.audio.basstreble" => vec![("bass", Value::Scalar(50.0))], "ec.audio.parametriceq" => vec![("band1BoostCut", Value::Scalar(9.0))], diff --git a/crates/render/src/tests_aux.rs b/crates/render/src/tests_aux.rs new file mode 100644 index 00000000..31bb1881 --- /dev/null +++ b/crates/render/src/tests_aux.rs @@ -0,0 +1,98 @@ +//! Auxiliary 3D channels: the comp's depth / ID / Cryptomatte pass and the 3D Channel effects on +//! a precomp of a 3D comp. + +use effectcraft_color::Label; +use effectcraft_keyframe::Value; +use effectcraft_project::build::{self, Ids}; +use effectcraft_project::{Comp, ItemId, ItemKind, Layer, LayerSource, Project, Solid}; +use effectcraft_raster::channels3d::{BACKGROUND_DEPTH, crypto_hash}; +use effectcraft_time::{FrameRate, Tick}; + +use crate::{NoFootage, RenderOpts, Renderer, render_frame}; + +const W: u32 = 200; +const H: u32 = 100; + +fn zoom() -> f64 { + W as f64 * 50.0 / 36.0 +} + +fn solid3(p: &mut Project, comp: &Comp, name: &str, color: [f32; 3], pos: [f64; 3]) -> Layer { + let sid = p.add_item(name, Label::Red, None, ItemKind::Solid(Solid { color, width: 60, height: 60, pixel_aspect: 1.0 })); + let mut l = build::layer(p, comp, name, LayerSource::Solid { item: sid }, (60, 60), None); + l.name = name.to_string(); + l.switches.three_d = true; + l.props.prop_mut("transform/position").unwrap().value = Value::Vec3(pos); + l +} + +/// Inner 3D comp: "Near" (red, z 0, left) above "Far" (green, z 500, right). Outer comp: the +/// precomp layer, optionally with an effect. +fn scene(effect: Option<(&str, Vec<(&str, Value)>)>) -> (Project, ItemId, ItemId) { + let mut p = Project::default(); + p.settings.bit_depth = effectcraft_project::BitDepth::Bpc32; + let inner = Comp::new(W, H, FrameRate::FPS_30, Tick::from_seconds_f64(2.0)); + let iid = p.add_item("Inner", Label::Sandstone, None, ItemKind::Comp(inner.clone().into())); + let near = solid3(&mut p, &inner, "Near", [1.0, 0.0, 0.0], [60.0, 50.0, 0.0]); + let far = solid3(&mut p, &inner, "Far", [0.0, 1.0, 0.0], [140.0, 50.0, 500.0]); + p.comp_mut(iid).unwrap().layers.push(near); + p.comp_mut(iid).unwrap().layers.push(far); + let outer = Comp::new(W, H, FrameRate::FPS_30, Tick::from_seconds_f64(2.0)); + let oid = p.add_item("Outer", Label::Sandstone, None, ItemKind::Comp(outer.clone().into())); + let mut pre = build::layer(&mut p, &outer, "Inner", LayerSource::Comp { item: iid }, (W, H), None); + if let Some((id, vals)) = effect { + let spec = effectcraft_effects::find(id).unwrap(); + let mut next = p.next_id; + let mut g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), spec.name, [W as f64, H as f64]); + p.next_id = next; + for (k, v) in vals { + g.prop_mut(k).unwrap().value = v.clone(); + } + pre.props.sub_mut("effects").unwrap().children.push(g.into()); + } + p.comp_mut(oid).unwrap().layers.push(pre); + (p, iid, oid) +} + +#[test] +fn comp_aux_pass_has_depth_ids_and_cryptomatte() { + let (p, iid, _) = scene(None); + let r = Renderer::new(&p, &NoFootage, RenderOpts::default()); + let a = r.comp_aux(iid, Tick::ZERO).unwrap(); + assert_eq!((a.width, a.height), (W, H)); + let at = |name: &str, x: usize, y: usize| a.get(name).unwrap()[y * W as usize + x]; + // Near plane at the camera's zoom distance, the far one 500 px behind it. + assert!((at("Z", 60, 50) as f64 - zoom()).abs() < 1e-2, "{}", at("Z", 60, 50)); + assert!((at("Z", 114, 50) as f64 - (zoom() + 500.0)).abs() < 1e-2, "{}", at("Z", 114, 50)); + assert_eq!(at("Z", 5, 5), BACKGROUND_DEPTH); + assert_eq!((at("ObjectID", 60, 50), at("ObjectID", 114, 50), at("ObjectID", 5, 5)), (1.0, 2.0, 0.0)); + assert_eq!(at("Coverage", 60, 50), 1.0); + // Planes face the camera. + assert!((at("N.Z", 60, 50) + 1.0).abs() < 1e-5); + // Cryptomatte rank 0 = the layer under the pixel, full coverage; manifest lists names. + assert_eq!(at("CryptoObject00.R", 60, 50).to_bits(), crypto_hash("Near")); + assert_eq!(at("CryptoObject00.G", 60, 50), 1.0); + assert_eq!(a.manifest_name("CryptoObject", crypto_hash("Far")), Some("Far")); +} + +#[test] +fn three_d_channel_effects_work_on_a_precomp_of_a_3d_comp() { + let near = (60, 50); + let far = (114, 50); + let alpha = |p: &Project, oid: ItemId, (x, y): (i64, i64)| render_frame(p, oid, Tick::ZERO, 1.0).get(x, y)[3]; + let (p, _, oid) = scene(Some(("ec.3d.idmatte", vec![("idSelection", Value::Scalar(2.0))]))); + assert_eq!(alpha(&p, oid, near), 0.0); + assert!(alpha(&p, oid, far) > 0.99); + let (p, _, oid) = scene(Some(("ec.3d.depthmatte", vec![("depth", Value::Scalar(zoom() + 250.0))]))); + assert_eq!(alpha(&p, oid, near), 0.0); + assert!(alpha(&p, oid, far) > 0.99); + let (p, _, oid) = scene(Some(("ec.3d.cryptomatte", vec![("selection", Value::Str("Near".into())), ("display", Value::Enum(1))]))); + assert!(alpha(&p, oid, near) > 0.99); + assert_eq!(alpha(&p, oid, far), 0.0); + // Half resolution: the pass follows the render scale. + let (p, iid, _) = scene(None); + let r = Renderer::new(&p, &NoFootage, RenderOpts { scale: 0.5, ..Default::default() }); + let a = r.comp_aux(iid, Tick::ZERO).unwrap(); + assert_eq!((a.width, a.height, a.scale), (W / 2, H / 2, 0.5)); + assert_eq!(a.get("ObjectID").unwrap()[25 * 100 + 30], 1.0); +} diff --git a/crates/render/src/three_d/compose.rs b/crates/render/src/three_d/compose.rs index e6fc9209..2647fadd 100644 --- a/crates/render/src/three_d/compose.rs +++ b/crates/render/src/three_d/compose.rs @@ -589,3 +589,182 @@ fn occlusion(c: &Item, p: Vec3, l: &LightState) -> [f32; 3] { } out } + +// ------------------------------------------------------------------ auxiliary render pass + +/// One surface seen through an output pixel (aux pass). +#[derive(Clone, Copy)] +struct AuxFrag { + z: f32, + a: f32, + obj: f32, + mat: f32, + n: [f32; 3], + uv: [f32; 2], + obj_hash: u32, + mat_hash: u32, +} + +/// The comp's auxiliary channels at the context time, at output resolution: camera-space depth +/// (`Z`), `ObjectID` (layer number), `MaterialID` (source item id), `Coverage`, camera-space +/// normals (`N.X/Y/Z`), texture coordinates (`UV.U/V`) and Cryptomatte `CryptoObject` (layer +/// names) / `CryptoMaterial` (source names) with two ranks each, plus their manifests. +/// +/// Layers are gathered in stack order (top first); a run of 3D layers contributes its planes +/// sorted near → far at each pixel, a 2D layer its coverage on the plane of the comp. The +/// front-most surface with ≥ 50 % opacity (else the strongest one) provides depth, IDs, normal +/// and UV; Cryptomatte coverage is each surface's visible share (`α · Π(1 − α_front)`). +pub(crate) fn aux_pass(r: &Renderer, ctx: &EvalCtx) -> effectcraft_raster::AuxChannels { + use effectcraft_raster::channels3d::{BACKGROUND_DEPTH, crypto_float, crypto_hash}; + let s = r.opts.scale; + let (w, h) = (((ctx.comp.width as f64 * s).round() as u32).max(1), ((ctx.comp.height as f64 * s).round() as u32).max(1)); + let out = (w, h); + let (wu, n) = (w as usize, w as usize * h as usize); + let cam = camera_for(r, ctx); + let visible: Vec<&Layer> = r.visible_layers(ctx).into_iter().filter(|l| !l.switches.adjustment).collect(); + let number = |l: &Layer| ctx.comp.layers.iter().position(|x| x.id == l.id).map_or(0.0, |i| i as f32 + 1.0); + let source_name = |l: &Layer| l.source.item().and_then(|i| r.project.item(i)).map(|it| it.name.clone()).unwrap_or_else(|| l.name.clone()); + let source_id = |l: &Layer| l.source.item().map_or(0.0, |i| i.0 as f32); + let mut manifest_obj: Vec<(String, u32)> = vec![]; + let mut manifest_mat: Vec<(String, u32)> = vec![]; + for l in &visible { + let (on, mn) = (l.name.clone(), source_name(l)); + if !manifest_obj.iter().any(|(x, _)| *x == on) { + manifest_obj.push((on.clone(), crypto_hash(&on))); + } + if !manifest_mat.iter().any(|(x, _)| *x == mn) { + manifest_mat.push((mn.clone(), crypto_hash(&mn))); + } + } + // Groups in stack order: runs of 3D layers, single 2D layers. + let mut groups: Vec<&[&Layer]> = vec![]; + let mut i = 0; + while i < visible.len() { + let mut j = i + 1; + if visible[i].is_3d() { + while j < visible.len() && visible[j].is_3d() { + j += 1; + } + } + groups.push(&visible[i..j]); + i = j; + } + let mut frags: Vec> = vec![Vec::new(); n]; + let plane_z = cam.depth(vec3(ctx.comp.width as f64 * 0.5, ctx.comp.height as f64 * 0.5, 0.0)) as f32; + for g in groups.iter().rev() { + if g[0].is_3d() { + let items: Vec = g.iter().enumerate().flat_map(|(k, l)| prepare_all(r, ctx, l, k, out, true)).collect(); + let meta: Vec<(f32, f32, u32, u32, [f32; 3], [f64; 2])> = items + .iter() + .map(|it| { + let nrm = it.geos.first().map_or(vec3(0.0, 0.0, -1.0), |gm| cam.view.apply_vec(gm.normal)); + let mut nv = [nrm.x as f32, nrm.y as f32, nrm.z as f32]; + if nv[2] > 0.0 { + nv = nv.map(|v| -v); + } + let (sw, sh) = crate::source_size(r.project, it.layer); + let size = if sw == 0 { [ctx.comp.width as f64, ctx.comp.height as f64] } else { [sw as f64, sh as f64] }; + (number(it.layer), source_id(it.layer), crypto_hash(&it.layer.name), crypto_hash(&source_name(it.layer)), nv, size) + }) + .collect(); + frags.par_chunks_mut(wu).enumerate().for_each(|(y, row)| { + let yi = y as i64; + let mut local: Vec = Vec::new(); + for (x, cell) in row.iter_mut().enumerate() { + let xi = x as i64; + local.clear(); + for (it, m) in items.iter().zip(&meta) { + if xi < it.bbox[0] || xi >= it.bbox[2] || yi < it.bbox[1] || yi >= it.bbox[3] { + continue; + } + let Some(gm) = it.geos.first() else { continue }; + let Some((u, v, z)) = unproject(gm, x as f64 + 0.5, y as f64 + 0.5) else { continue }; + let a = it.texel(u, v)[3] * it.opacity; + if a > 1e-4 { + let uv = [(u / m.5[0]) as f32, (v / m.5[1]) as f32]; + local.push(AuxFrag { z: z as f32, a: a.min(1.0), obj: m.0, mat: m.1, n: m.4, uv, obj_hash: m.2, mat_hash: m.3 }); + } + } + local.sort_by(|p, q| p.z.total_cmp(&q.z)); + cell.extend_from_slice(&local); + } + }); + } else { + let l = g[0]; + let mut img = Image::new(w, h); + if let Some(b) = r.layer_buf(ctx, l) { + r.place(ctx, l, &b, &mut img, BlendMode::Normal, ctx.opacity(l) as f32); + } + let (obj, mat, oh, mh) = (number(l), source_id(l), crypto_hash(&l.name), crypto_hash(&source_name(l))); + frags.par_chunks_mut(wu).zip(img.data.par_chunks(wu)).enumerate().for_each(|(y, (row, px))| { + for (x, (cell, p)) in row.iter_mut().zip(px).enumerate() { + if p[3] > 1e-4 { + let uv = [(x as f32 + 0.5) / w as f32, (y as f32 + 0.5) / h as f32]; + cell.push(AuxFrag { z: plane_z, a: p[3].min(1.0), obj, mat, n: [0.0, 0.0, -1.0], uv, obj_hash: oh, mat_hash: mh }); + } + } + }); + } + } + // Resolve per pixel. + const NAMES: [&str; 9] = ["Z", "ObjectID", "MaterialID", "Coverage", "N.X", "N.Y", "N.Z", "UV.U", "UV.V"]; + const CRYPTO: usize = 16; + let resolved: Vec<([f32; 9], [f32; CRYPTO])> = frags + .par_iter() + .map(|fs| { + let mut o = [BACKGROUND_DEPTH, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]; + let mut crypto = [0.0f32; CRYPTO]; + if fs.is_empty() { + return (o, crypto); + } + let mut t = 1.0f32; + let mut obj: Vec<(u32, f32)> = vec![]; + let mut mat: Vec<(u32, f32)> = vec![]; + let mut best: Option<(usize, f32)> = None; + let mut front: Option = None; + for (k, f) in fs.iter().enumerate() { + let c = f.a * t; + if front.is_none() && f.a >= 0.5 { + front = Some(k); + } + if best.is_none_or(|(_, bc)| c > bc) { + best = Some((k, c)); + } + for (list, hsh) in [(&mut obj, f.obj_hash), (&mut mat, f.mat_hash)] { + match list.iter_mut().find(|(x, _)| *x == hsh) { + Some(e) => e.1 += c, + None => list.push((hsh, c)), + } + } + t *= 1.0 - f.a; + if t <= 1e-5 { + break; + } + } + let f = fs[front.or(best.map(|b| b.0)).unwrap_or(0)]; + o = [f.z, f.obj, f.mat, 1.0 - t, f.n[0], f.n[1], f.n[2], f.uv[0], f.uv[1]]; + for (base, list) in [(0usize, &mut obj), (8usize, &mut mat)] { + list.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0))); + for (rank, (hsh, c)) in list.iter().take(4).enumerate() { + crypto[base + rank * 2] = crypto_float(*hsh); + crypto[base + rank * 2 + 1] = *c; + } + } + (o, crypto) + }) + .collect(); + let mut aux = effectcraft_raster::AuxChannels::new(w, h, s); + for (k, name) in NAMES.iter().enumerate() { + aux.channels.push((name.to_string(), resolved.iter().map(|r| r.0[k]).collect())); + } + for (base, layer) in [(0usize, "CryptoObject"), (8usize, "CryptoMaterial")] { + for rank_pair in 0..2 { + for (c, ch) in ["R", "G", "B", "A"].iter().enumerate() { + let k = base + rank_pair * 4 + c; + aux.channels.push((format!("{layer}{rank_pair:02}.{ch}"), resolved.iter().map(|r| r.1[k]).collect())); + } + } + } + aux.manifests = vec![("CryptoObject".into(), manifest_obj), ("CryptoMaterial".into(), manifest_mat)]; + aux +} diff --git a/docs/parity.md b/docs/parity.md index aecb3f13..419b407c 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -15,7 +15,7 @@ rendered, does not count as done. | P1 (professional daily use) | ≈ 38% | | P2 (long tail) | ≈ 17% | | Features done / partial / missing | 18 / 60 / 14 of 92 | -| **Effects** | **257 of 298** After Effects 2026 effects by name (86%); ≈ 70% allowing for simplified implementations | +| **Effects** | **297 of 298** After Effects 2026 effects by name (99.7%, after M9.11 and Warp Stabilizer; was 257); only the 3D Camera Tracker is left | | Remaining work | ≈ **167 agent-hours** (139 h of features + ≈ 20% for 1:1 polish against After Effects) | | Wall-clock estimate | ≈ **42–50 hours** with five agents working in parallel and one integrating; ≈ 31–35 h for P0 + P1 only | @@ -76,8 +76,8 @@ library groundwork for mask tracking and Warp Stabilizer. What is left is concentrated in large systems: Roto Brush and Refine Edge, the 3D camera tracker, the Warp Stabilizer and mask-tracking UI on top of the new tracking library, a JavaScript scripting object model, SVG/PSD import, WebM, a disk cache, -the missing effect categories (3D Channel, Immersive Video, OCIO), native macOS menus, and -wasm threads. +native macOS menus, and wasm threads. (The missing effect categories, 3D Channel, Immersive Video +and OCIO, landed in M9.11.) ## By area @@ -94,22 +94,34 @@ wasm threads. | Text | ≈ 86% | 1.3 | vertical type is basic (upright characters, no tate-chu-yoko), no OpenType feature panel, no extruded strokes (extruded, bevelled text landed in M7.6); per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API landed (M9.9–M9.10) | | Web | 60% | 2.5 | threads, browser storage, audio | | 3D | 72% | 12.0 | 3D camera tracker, multi-view layouts, Advanced 3D motion blur and blend modes, cameras/lights from models; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) landed in M7.4–M7.6 | -| Effects | 58% | 13.6 | Effect Controls widgets (angle dial, point crosshair, eyedropper, curves), GPU versions of more effects, 41 missing effects | +| Effects | ≈ 75% | 5.0 | GPU versions of more effects and the 3D Camera Tracker (every other After Effects effect exists since M9.11 and M12.5) | | Interface | 55% | 10.4 | drag-to-dock and floating panels, preferences, native macOS menus (viewer rulers/snapping/channels/snapshots landed in M0.13) | | Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves | | Masks & roto | 52% | 10.0 | Roto Brush, variable-width mask feather points (mask tracking and Mask Interpolation landed in M6.6) | | Preview | 55% | 6.1 | GPU 3D and adjustment layers, disk cache (region of interest, snapshots, exposure and Fast Previews landed in M0.13) | -| Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking, Subspace Warp's mesh warp and rolling-shutter repair (point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) | +| Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking and Subspace Warp's mesh warp (Rolling Shutter Repair landed in M9.11; point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | ## Effects still missing -Immersive Video (all 12), 3D Channel (all 8, including Cryptomatte), Color Stabilizer and the five -OCIO transforms, CC Flo Motion, Liquify, Rolling Shutter Repair, Compressor, -Distortion and Gate (audio), Camera-Shake Deblur, CC Radial Blur, CC Hair, Particle Playground, -Keylight (our Screen Key stands in), Color Profile Converter and the 3D Camera Tracker. Boris FX +The 3D Camera Tracker (built on the tracking library by other work). Boris FX Mocha and Cineware are third-party and not counted. +### Added in M9.11 (39 effects) + +| Category | Effects | Notes | +|---|---|---| +| 3D Channel | 3D Channel Extract, Cryptomatte, Depth Matte, Depth of Field, EXtractoR, Fog 3D, ID Matte, IDentifier | Read auxiliary channels: multi-layer OpenEXR (any `layer.channel`, Cryptomatte manifests from the header) or the compositor's own depth / layer-ID / normal / UV / Cryptomatte pass for precomps of 3D comps. | +| Immersive Video | VR Blur, VR Chromatic Aberrations, VR Color Gradients, VR Converter, VR De-Noise, VR Digital Glitch, VR Fractal Noise, VR Glow, VR Plane to Sphere, VR Rotate Sphere, VR Sharpen, VR Sphere To Plane | Seam-aware equirectangular processing (wrap-around, pole continuation, latitude-scaled filters), mono / over-under / side-by-side layouts; VR Converter between equirectangular, cube maps (4:3, 3:2, 6:1), sphere map and 2D. | +| Color Correction | OCIO CDL / Color Space / Display / File / Look Transform, Color Stabilizer | Built-in minimal OCIO-style config (ACES2065-1, ACEScg, ACEScct, sRGB, Rec.709, Rec.2020, Display P3, linear variants, XYZ) from published matrices and transfer functions; `.cube` / `.3dl` / `.csp` LUTs and ASC `.cc` / `.ccc` / `.cdl`. Custom `.ocio` config files are not read yet. | +| Distort | CC Flo Motion, Liquify, Rolling Shutter Repair | Liquify strokes are effect data replayed into a displacement mesh (`liquify.stroke` / `liquify.clear` commands; no interactive viewer brush yet). Rolling Shutter Repair uses the KLT tracker. | +| Blur & Sharpen | Camera-Shake Deblur, CC Radial Blur | Deblur substitutes aligned patches from sharper neighbouring frames. | +| Audio | Compressor, Distortion, Gate | Applied in the mixdown like the other audio effects. | +| Simulation | CC Hair, Particle Playground | Particle Playground: cannon, grid, layer exploder, layer map, gravity, repel, wall, persistent property mapper (no Particle Exploder, text particles or ephemeral mapper yet). | +| Keying | Key Light | The full Keylight 1.2 control set under a generic name ("Keylight" is a vendor trademark); `lookup("Keylight (1.2)")` finds it. | +| Utility | Color Profile Converter | Our colour spaces and ACES; rendering intents (perceptual gamut compression, relative / absolute colorimetric, saturation). | +| Obsolete | mocha shape | Mocha's export format is not public: reads a documented JSON shape format instead. | + ## Highest-value gaps, in order 1. ~~On-canvas text editing and per-character styles~~ (landed: M9.9–M9.10).