From 07330bca6b15da7653a95c7255a0e923c85837b4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:20:25 -0400 Subject: [PATCH 1/2] M6.7: roto brush segmentation + propagation Graph-cut segmentation (Boykov-Kolmogorov max-flow, GrabCut GMMs, coarse to fine), optical-flow propagation with band re-segmentation, correction strokes, Roto Brush & Refine Edge effect with content-keyed derived cache, roto.* commands (stroke/propagate/span/freeze/unfreeze/clearStrokes/ cancel/options/status), refine-edge matting (guided filter + closed-form) and decontamination, Freeze job. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + crates/effects/Cargo.toml | 3 +- crates/effects/src/lib.rs | 6 +- crates/effects/src/roto.rs | 698 ++++++++++++++++++++++ crates/engine/src/commands/edit.rs | 3 + crates/engine/src/commands/mod.rs | 2 + crates/engine/src/commands/roto_cmds.rs | 424 +++++++++++++ crates/engine/src/lib.rs | 22 + crates/engine/src/roto.rs | 569 ++++++++++++++++++ crates/engine/src/tests_roto.rs | 288 +++++++++ crates/track/Cargo.toml | 2 +- crates/track/src/lib.rs | 1 + crates/track/src/roto/gmm.rs | 228 +++++++ crates/track/src/roto/matting.rs | 465 +++++++++++++++ crates/track/src/roto/maxflow.rs | 438 ++++++++++++++ crates/track/src/roto/mod.rs | 754 ++++++++++++++++++++++++ crates/track/src/roto/refine.rs | 198 +++++++ crates/track/src/roto/rle.rs | 136 +++++ crates/track/src/roto/tests.rs | 331 +++++++++++ 19 files changed, 4565 insertions(+), 4 deletions(-) create mode 100644 crates/effects/src/roto.rs create mode 100644 crates/engine/src/commands/roto_cmds.rs create mode 100644 crates/engine/src/roto.rs create mode 100644 crates/engine/src/tests_roto.rs create mode 100644 crates/track/src/roto/gmm.rs create mode 100644 crates/track/src/roto/matting.rs create mode 100644 crates/track/src/roto/maxflow.rs create mode 100644 crates/track/src/roto/mod.rs create mode 100644 crates/track/src/roto/refine.rs create mode 100644 crates/track/src/roto/rle.rs create mode 100644 crates/track/src/roto/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 46f34e63..544b6615 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1184,6 +1184,7 @@ dependencies = [ "effectcraft-raster", "effectcraft-track", "rayon", + "serde", "serde_json", ] diff --git a/crates/effects/Cargo.toml b/crates/effects/Cargo.toml index e321fa2f..20068a93 100644 --- a/crates/effects/Cargo.toml +++ b/crates/effects/Cargo.toml @@ -17,6 +17,5 @@ effectcraft-project = { workspace = true } effectcraft-raster = { workspace = true } effectcraft-track = { workspace = true } rayon = { workspace = true } - -[dev-dependencies] +serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 6d11e19a..2c3255d7 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -31,6 +31,7 @@ pub mod paint; mod perspective; mod perspective2; pub mod puppet; +pub mod roto; mod sim; mod sim2; mod stylize2; @@ -354,6 +355,7 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(paint::specs()); v.extend(puppet::specs()); v.extend(warp_stab::specs()); + v.extend(roto::specs()); v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); for s in v.iter_mut() { if GPU_EFFECTS.contains(&s.id) { @@ -426,7 +428,7 @@ pub fn instantiate(spec: &EffectSpec, ids: &mut Ids, instance_name: &str, layer_ /// Display names of nested parameter groups by match id. fn group_name(m: &str) -> &str { - warp_stab::GROUPS.iter().find(|(id, _)| *id == m).map(|(_, n)| *n).unwrap_or(m) + warp_stab::GROUPS.iter().chain(roto::GROUPS).find(|(id, _)| *id == m).map(|(_, n)| *n).unwrap_or(m) } /// The nested group at `path` (`borders/autoScale`) under `g`, created on first use. @@ -494,6 +496,8 @@ pub const TIME_DEPENDENT: &[&str] = &[ "ec.time.pixelmotionblur", // Each frame gets its own stabilizing warp. warp_stab::ID, + // Each frame has its own segmentation. + roto::ID, ]; /// See [`TIME_DEPENDENT`]. diff --git a/crates/effects/src/roto.rs b/crates/effects/src/roto.rs new file mode 100644 index 00000000..a1f93a92 --- /dev/null +++ b/crates/effects/src/roto.rs @@ -0,0 +1,698 @@ +//! Roto Brush & Refine Edge (applied by the Roto Brush tool; Effect ▸ Matte). +//! +//! The instance stores the user's strokes, base frame and segmentation span as JSON in its hidden +//! **Strokes** parameter ([`RotoData`]); the segmentation itself is *derived* data: each frame's +//! result is a deterministic function of its chain key ([`RotoData::chain_keys`] seeded with +//! [`seed`]: what the frames are, the Roto Brush settings and the strokes from the base frame +//! out to the frame), so it lives in a process-wide content-keyed cache ([`cached`]) that the +//! engine's propagation job fills at full resolution and rendering fills on demand (reading +//! neighbouring frames with [`crate::EffectHost::self_at`]). Undo, redo and reopening a project +//! therefore never see stale mattes. **Freeze** stores the final mattes in the hidden **Frozen** +//! parameter ([`FrozenCache`]); they are used while their key still matches. +//! +//! Rendering: binary segmentation → Roto Brush Matte (Shift Edge, Reduce Chatter, Feather, +//! Contrast) → Refine Edge matting inside the Refine Edge band (guided filter + closed-form +//! matting with Higher Quality) → Refine Edge Matte adjustments → matte motion blur → +//! decontamination of the edge colours. Outside the segmentation span the layer is transparent; +//! without strokes the effect passes its input through. +//! +//! *Version* is stored but all versions use the same (classic, graph-cut) engine; *Quality ▸ +//! Best* segments at a higher working resolution with more iterations. + +use std::collections::{BTreeMap, HashMap, VecDeque}; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::Image; +use effectcraft_track::roto::refine::{self, MatteParams}; +use effectcraft_track::roto::{self as rb, FrameSeg, RotoData, SegOpts, rle}; +use rayon::prelude::*; +use serde::{Deserialize, Serialize}; + +use crate::warp_stab::param; +use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; + +pub use effectcraft_track::roto::{Stroke, StrokeKind}; + +/// The effect's id. +pub const ID: &str = "ec.matte.rotobrush"; +/// Hidden parameters. +pub const STROKES: &str = "strokes"; +pub const INPUT_KEY: &str = "inputKey"; +pub const FROZEN: &str = "frozen"; + +/// Default segmentation span each side of the base frame (frames). +pub const DEFAULT_SPAN: i64 = 20; + +/// Display names of the instance's parameter groups. +pub const GROUPS: &[(&str, &str)] = &[("rotoBrushMatte", "Roto Brush Matte"), ("refineEdgeMatte", "Refine Edge Matte")]; + +pub const VERSIONS: [&str; 3] = ["1.0", "2.0", "3.0"]; +pub const QUALITIES: [&str; 2] = ["Standard", "Best"]; + +pub fn specs() -> Vec { + vec![EffectSpec { + id: ID, + name: "Roto Brush & Refine Edge", + category: "Matte", + params: vec![ + p("version", "Version", Value::Enum(2), popup(&VERSIONS)), + p("quality", "Quality", Value::Enum(0), popup(&QUALITIES)), + p("rotoBrushMatte/searchRadius", "Search Radius", num(15.0), slider(0.0, 100.0, 0.0, 50.0, 1)), + p("rotoBrushMatte/motionThreshold", "Motion Threshold", num(10.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("rotoBrushMatte/motionDamping", "Motion Damping", num(20.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("rotoBrushMatte/viewSearchRegion", "View Search Region", Value::Bool(false), ParamUi::Checkbox), + p("rotoBrushMatte/feather", "Feather", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("rotoBrushMatte/contrast", "Contrast", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("rotoBrushMatte/shiftEdge", "Shift Edge", num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 0)), + p("rotoBrushMatte/reduceChatter", "Reduce Chatter", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("refineEdgeMatte/smooth", "Smooth", num(1.0), slider(0.0, 100.0, 0.0, 10.0, 1)), + p("refineEdgeMatte/feather", "Feather", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("refineEdgeMatte/contrast", "Contrast", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("refineEdgeMatte/shiftEdge", "Shift Edge", num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 0)), + p("refineEdgeMatte/reduceChatter", "Reduce Chatter", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("refineEdgeMatte/useMotionBlur", "Use Motion Blur", Value::Bool(false), ParamUi::Checkbox), + p("refineEdgeMatte/motionBlurSamples", "Motion Blur Samples", num(11.0), slider(1.0, 64.0, 1.0, 32.0, 0)), + p("refineEdgeMatte/shutterAngle", "Shutter Angle", num(180.0), slider(0.0, 720.0, 0.0, 360.0, 0)), + p("refineEdgeMatte/higherQuality", "Higher Quality", Value::Bool(true), ParamUi::Checkbox), + p("refineEdgeMatte/decontaminateEdgeColors", "Decontaminate Edge Colors", Value::Bool(true), ParamUi::Checkbox), + p("refineEdgeMatte/decontaminationAmount", "Decontamination Amount", num(100.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("refineEdgeMatte/extendWhereSmoothed", "Extend Where Smoothed", Value::Bool(true), ParamUi::Checkbox), + p("refineEdgeMatte/increaseDecontaminationRadius", "Increase Decontamination Radius", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("refineEdgeMatte/viewDecontaminationMap", "View Decontamination Map", Value::Bool(false), ParamUi::Checkbox), + p(STROKES, "Strokes", Value::Str(String::new()), ParamUi::Hidden), + p(INPUT_KEY, "Input Key", Value::Str(String::new()), ParamUi::Hidden), + p(FROZEN, "Frozen", Value::Str(String::new()), ParamUi::Hidden), + ], + render, + gpu: false, + float: true, + }] +} + +fn f(params: &Params, id: &str) -> f64 { + param(params, id).map(Value::as_f64).unwrap_or(0.0) +} +fn e(params: &Params, id: &str) -> u32 { + param(params, id).map(Value::as_enum).unwrap_or(0) +} +fn b(params: &Params, id: &str) -> bool { + param(params, id).map(Value::as_bool).unwrap_or(false) +} +fn s<'a>(params: &'a Params, id: &str) -> &'a str { + match param(params, id) { + Some(Value::Str(s)) => s, + _ => "", + } +} + +fn fnv(h: &mut u64, bytes: &[u8]) { + for b in bytes { + *h ^= *b as u64; + *h = h.wrapping_mul(0x0100_0000_01b3); + } +} + +/// The instance's strokes (parsed once per content). +pub fn data(params: &Params) -> Arc { + static C: OnceLock>>> = OnceLock::new(); + let j = s(params, STROKES); + let mut h = 0xcbf2_9ce4_8422_2325; + fnv(&mut h, j.as_bytes()); + let c = C.get_or_init(Default::default); + if let Some(d) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&h) { + return d.clone(); + } + let d = Arc::new(RotoData::from_json(j)); + let mut m = c.lock().unwrap_or_else(|e| e.into_inner()); + if m.len() > 64 { + m.clear(); + } + m.insert(h, d.clone()); + d +} + +/// Segmentation settings at `scale` (pixels per layer pixel). +pub fn seg_opts(params: &Params, scale: f64) -> SegOpts { + SegOpts { + scale, + search_radius: f(params, "rotoBrushMatte/searchRadius").max(0.0), + motion_threshold: f(params, "rotoBrushMatte/motionThreshold"), + motion_damping: f(params, "rotoBrushMatte/motionDamping"), + best: e(params, "quality") == 1, + } +} + +/// Matte settings. +pub fn matte_params(params: &Params) -> MatteParams { + MatteParams { + rb_feather: f(params, "rotoBrushMatte/feather").max(0.0), + rb_contrast: f(params, "rotoBrushMatte/contrast"), + rb_shift: f(params, "rotoBrushMatte/shiftEdge"), + rb_chatter: f(params, "rotoBrushMatte/reduceChatter"), + smooth: f(params, "refineEdgeMatte/smooth").max(0.0), + feather: f(params, "refineEdgeMatte/feather").max(0.0), + contrast: f(params, "refineEdgeMatte/contrast"), + shift: f(params, "refineEdgeMatte/shiftEdge"), + chatter: f(params, "refineEdgeMatte/reduceChatter"), + motion_blur: b(params, "refineEdgeMatte/useMotionBlur"), + mb_samples: f(params, "refineEdgeMatte/motionBlurSamples").round().clamp(1.0, 64.0) as usize, + shutter_angle: f(params, "refineEdgeMatte/shutterAngle"), + higher_quality: b(params, "refineEdgeMatte/higherQuality"), + decontaminate: b(params, "refineEdgeMatte/decontaminateEdgeColors"), + decontamination: f(params, "refineEdgeMatte/decontaminationAmount"), + extend_where_smoothed: b(params, "refineEdgeMatte/extendWhereSmoothed"), + increase_radius: f(params, "refineEdgeMatte/increaseDecontaminationRadius").max(0.0), + } +} + +/// Seed of the chain keys: what the frames are (the engine-maintained Input Key), the layer +/// size, the frame rate and the segmentation settings. +pub fn seed(params: &Params, layer_size: [f64; 2], fps: f64) -> u64 { + let mut h = 0xcbf2_9ce4_8422_2325; + fnv(&mut h, s(params, INPUT_KEY).as_bytes()); + for v in [ + layer_size[0], + layer_size[1], + fps, + f(params, "rotoBrushMatte/searchRadius"), + f(params, "rotoBrushMatte/motionThreshold"), + f(params, "rotoBrushMatte/motionDamping"), + ] { + fnv(&mut h, &v.to_bits().to_le_bytes()); + } + fnv(&mut h, &[e(params, "quality") as u8, e(params, "version") as u8]); + h +} + +/// The layer frame shown at layer time `t` seconds. +pub fn frame_of(t: f64, fps: f64) -> i64 { + (t * fps + 1e-6).floor() as i64 +} + +// ------------------------------------------------------------------ derived-data cache + +struct SegCache { + map: HashMap<(u64, u64), Arc>, + order: VecDeque<(u64, u64)>, + bytes: usize, +} + +const CACHE_BYTES: usize = 768 << 20; + +fn cache() -> &'static Mutex { + static C: OnceLock> = OnceLock::new(); + C.get_or_init(|| Mutex::new(SegCache { map: HashMap::new(), order: VecDeque::new(), bytes: 0 })) +} + +/// A cached segmentation (chain key, scale). +pub fn cached(key: u64, scale: f64) -> Option> { + cache().lock().unwrap_or_else(|e| e.into_inner()).map.get(&(key, scale.to_bits())).cloned() +} + +pub fn store(key: u64, scale: f64, seg: Arc) { + let mut c = cache().lock().unwrap_or_else(|e| e.into_inner()); + let k = (key, scale.to_bits()); + if c.map.contains_key(&k) { + return; + } + c.bytes += seg.matte.len() + seg.refine.len(); + c.map.insert(k, seg); + c.order.push_back(k); + while c.bytes > CACHE_BYTES { + let Some(old) = c.order.pop_front() else { break }; + if let Some(s) = c.map.remove(&old) { + c.bytes -= s.matte.len() + s.refine.len(); + } + } +} + +/// Drop the cached segmentations of these chain keys (every scale). +pub fn forget(keys: impl IntoIterator) { + let keys: std::collections::HashSet = keys.into_iter().collect(); + let mut c = cache().lock().unwrap_or_else(|e| e.into_inner()); + let gone: Vec<(u64, u64)> = c.map.keys().filter(|k| keys.contains(&k.0)).copied().collect(); + for k in gone { + if let Some(s) = c.map.remove(&k) { + c.bytes -= s.matte.len() + s.refine.len(); + } + } + c.order.retain(|k| !keys.contains(&k.0)); +} + +/// Drop every cached segmentation (Edit ▸ Purge). +pub fn purge() { + let mut c = cache().lock().unwrap_or_else(|e| e.into_inner()); + c.map.clear(); + c.order.clear(); + c.bytes = 0; +} + +/// The layer-space grid (`round(size · scale)` pixels) of a buffer. +pub fn layer_grid(buf: &Buf, layer_size: [f64; 2]) -> Image { + let s = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + let (w, h) = (((layer_size[0] * s).round() as u32).max(1), ((layer_size[1] * s).round() as u32).max(1)); + let (ox, oy) = (buf.offset[0].round() as i64, buf.offset[1].round() as i64); + let mut img = Image::new(w, h); + img.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + *px = buf.img.get(x as i64 + ox, y as i64 + oy); + } + }); + img +} + +/// Everything needed to compute an instance's segmentations. +pub struct Chain { + pub data: Arc, + pub keys: BTreeMap, + pub opts: SegOpts, + pub refine_radius: f64, +} + +impl Chain { + pub fn new(params: &Params, layer_size: [f64; 2], fps: f64, scale: f64) -> Chain { + let data = data(params); + let keys = data.chain_keys(seed(params, layer_size, fps)); + Chain { refine_radius: data.refine_radius(), data, keys, opts: seg_opts(params, scale) } + } + + /// Frame `f`'s segmentation at this chain's scale: cached, or computed from the nearest + /// cached frame towards the base (full-resolution results serve smaller scales). + /// `frame(g)`: the effect's input on frame g on the layer grid at this scale. + pub fn ensure(&self, f: i64, frame: &mut dyn FnMut(i64) -> Option>, cancel: &dyn Fn() -> bool) -> Option> { + let sc = self.opts.scale; + let lookup = |g: i64| -> Option> { + let k = *self.keys.get(&g)?; + if let Some(s) = cached(k, sc) { + return Some(s); + } + if sc != 1.0 { + let full = cached(k, 1.0)?; + let s = Arc::new(downscale(&full, sc)); + store(k, sc, s.clone()); + return Some(s); + } + None + }; + if let Some(s) = lookup(f) { + return Some(s); + } + let base = self.data.base?; + if !self.data.in_span(f) { + return None; + } + // Walk towards the base to the nearest available frame. + let mut path = vec![f]; + let mut g = f; + let mut start: Option> = None; + while let Some(src) = self.data.source_of(g) { + if let Some(s) = lookup(src) { + start = Some(s); + break; + } + path.push(src); + g = src; + } + path.reverse(); + let mut prev_img: Option> = None; + let mut cur = start; + for g in path { + if cancel() { + return None; + } + let img = frame(g)?; + let strokes = self.data.strokes_at(g); + let seg = match (&cur, g == base) { + (_, true) | (None, _) => rb::segment(&img, &strokes, None, &self.opts, self.refine_radius), + (Some(p), false) => { + let src = self.data.source_of(g)?; + let pimg = match prev_img.take() { + Some(i) => i, + None => frame(src)?, + }; + if pimg.width != img.width || pimg.height != img.height { + return None; + } + rb::propagate(&pimg, p, &img, &strokes, &self.opts, self.refine_radius) + } + }; + let seg = Arc::new(seg); + store(*self.keys.get(&g)?, sc, seg.clone()); + cur = Some(seg); + prev_img = Some(img); + } + cur + } + + /// Whether frame `f` is computed (at full resolution or at `scale`). + pub fn is_cached(&self, f: i64, scale: f64) -> bool { + self.keys.get(&f).is_some_and(|k| cached(*k, 1.0).is_some() || cached(*k, scale).is_some()) + } +} + +/// Nearest-neighbour resampling of a segmentation to another scale. +pub fn downscale(s: &FrameSeg, scale: f64) -> FrameSeg { + let (w, h) = (((s.w as f64) * scale).round().max(1.0) as usize, ((s.h as f64) * scale).round().max(1.0) as usize); + resample(s, w, h) +} + +fn resample(s: &FrameSeg, w: usize, h: usize) -> FrameSeg { + if s.w == w && s.h == h { + return s.clone(); + } + let mut out = FrameSeg { w, h, matte: vec![0; w * h], refine: vec![0; w * h], fg: s.fg.clone(), bg: s.bg.clone() }; + if s.w == 0 || s.h == 0 { + return out; + } + for y in 0..h { + let sy = (((y as f64 + 0.5) * s.h as f64 / h as f64) as usize).min(s.h - 1); + for x in 0..w { + let sx = (((x as f64 + 0.5) * s.w as f64 / w as f64) as usize).min(s.w - 1); + out.matte[y * w + x] = s.matte[sy * s.w + sx]; + out.refine[y * w + x] = s.refine.get(sy * s.w + sx).copied().unwrap_or(0); + } + } + out +} + +/// The final matte of frame `seg` (with its neighbours), on its own grid. +pub fn final_alpha( + img: &Image, + seg: &FrameSeg, + prev: Option<&FrameSeg>, + next: Option<&FrameSeg>, + mp: &MatteParams, + refine_radius: f64, + scale: f64, +) -> Vec { + let rgb = rb::rgb_of(img); + refine::matte_alpha(&rgb, seg, prev, next, mp, refine_radius * scale, scale) +} + +fn needs_neighbours(mp: &MatteParams) -> bool { + mp.rb_chatter > 0.0 || mp.chatter > 0.0 || mp.motion_blur +} + +// ------------------------------------------------------------------ Freeze + +/// Frozen mattes (Freeze): the final 8-bit alpha and the Refine Edge band of every frame of +/// the span at full resolution, valid while `key` matches [`frozen_key`]. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct FrozenCache { + pub key: String, + pub size: [usize; 2], + pub frames: BTreeMap, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct FrozenFrame { + /// Alpha (RLE, base64). + pub a: String, + /// Refine Edge band (RLE, base64). + pub b: String, +} + +impl FrozenCache { + pub fn from_json(s: &str) -> Option { + if s.trim().is_empty() { + return None; + } + serde_json::from_str(s).ok() + } + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } +} + +/// What frozen mattes depend on: every chain key and the matte settings. +pub fn frozen_key(params: &Params, layer_size: [f64; 2], fps: f64) -> String { + let ch = Chain::new(params, layer_size, fps, 1.0); + let mut h = 0xcbf2_9ce4_8422_2325; + for (f, k) in &ch.keys { + fnv(&mut h, &f.to_le_bytes()); + fnv(&mut h, &k.to_le_bytes()); + } + fnv(&mut h, format!("{:?}", matte_params(params)).as_bytes()); + format!("{h:016x}") +} + +/// Whether the instance is frozen (has frozen mattes). +pub fn is_frozen(params: &Params) -> bool { + !s(params, FROZEN).is_empty() +} + +type FrozenParsed = Mutex>>; + +fn frozen(params: &Params) -> Option> { + static C: OnceLock = OnceLock::new(); + let j = s(params, FROZEN); + if j.is_empty() { + return None; + } + let mut h = 0xcbf2_9ce4_8422_2325; + fnv(&mut h, j.as_bytes()); + let c = C.get_or_init(Default::default); + if let Some(d) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&h) { + return Some(d.clone()); + } + let d = Arc::new(FrozenCache::from_json(j)?); + let mut m = c.lock().unwrap_or_else(|e| e.into_inner()); + if m.len() > 8 { + m.clear(); + } + m.insert(h, d.clone()); + Some(d) +} + +/// Encode one frame for [`FrozenCache`]. +pub fn freeze_frame(alpha: &[f32], band: &[u8]) -> FrozenFrame { + let a: Vec = alpha.iter().map(|v| (v.clamp(0.0, 1.0) * 255.0).round() as u8).collect(); + FrozenFrame { a: rle::encode(&a), b: rle::encode(band) } +} + +// ------------------------------------------------------------------ render + +/// A frame's matte (alpha and band) on the layer grid of `img` at `scale`. +fn matte_for(ctx: &EffectCtx, f: i64, img: &Arc, scale: f64) -> Option<(Vec, Vec, f64)> { + let (w, h) = (img.width as usize, img.height as usize); + // Frozen mattes. + if let Some(fc) = frozen(ctx.params) + && fc.key == frozen_key(ctx.params, ctx.layer_size, ctx.fps()) + && let Some(fr) = fc.frames.get(&f) + { + let a = rle::decode(&fr.a)?; + let bnd = rle::decode(&fr.b)?; + let [fw, fh] = fc.size; + if a.len() == fw * fh && bnd.len() == fw * fh { + let af: Vec = a.iter().map(|v| *v as f32 / 255.0).collect(); + let (alpha, band) = if fw == w && fh == h { + (af, bnd) + } else { + let mut al = vec![0.0f32; w * h]; + let mut bd = vec![0u8; w * h]; + for y in 0..h { + for x in 0..w { + let (sx, sy) = ((x as f64 + 0.5) * fw as f64 / w as f64, (y as f64 + 0.5) * fh as f64 / h as f64); + al[y * w + x] = rb::matting::sample(&af, fw, fh, sx, sy); + bd[y * w + x] = bnd[(sy as usize).min(fh - 1) * fw + (sx as usize).min(fw - 1)]; + } + } + (al, bd) + }; + let rr = data(ctx.params).refine_radius(); + return Some((alpha, band, rr)); + } + } + let host = ctx.env.host; + let chain = Chain::new(ctx.params, ctx.layer_size, ctx.fps(), scale); + let fps = ctx.fps(); + let effects = ctx.env.effect_index; + let layer_size = ctx.layer_size; + let me = img.clone(); + let mut frame = |g: i64| -> Option> { + if g == f { + return Some(me.clone()); + } + let buf = host?.self_at(g as f64 / fps, effects)?; + let im = layer_grid(&buf, layer_size); + (im.width as usize == w && im.height as usize == h).then(|| Arc::new(im)) + }; + let seg = chain.ensure(f, &mut frame, &|| false)?; + let seg = if seg.w == w && seg.h == h { (*seg).clone() } else { resample(&seg, w, h) }; + let mp = matte_params(ctx.params); + let (mut prev, mut next) = (None, None); + if needs_neighbours(&mp) { + for (slot, g) in [(&mut prev, f - 1), (&mut next, f + 1)] { + if chain.data.in_span(g) + && let Some(s) = chain.ensure(g, &mut frame, &|| false) + { + *slot = Some(if s.w == w && s.h == h { (*s).clone() } else { resample(&s, w, h) }); + } + } + } + let alpha = final_alpha(img, &seg, prev.as_ref(), next.as_ref(), &mp, chain.refine_radius, scale); + Some((alpha, seg.refine, chain.refine_radius)) +} + +fn render(ctx: &EffectCtx, buf: Buf) -> Buf { + let d = data(ctx.params); + if d.is_empty() || ctx.adjustment { + return buf; + } + let f = frame_of(ctx.time, ctx.fps()); + let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + let mut out = Buf { img: Image::new(buf.img.width, buf.img.height), offset: buf.offset, scale: buf.scale }; + if !d.in_span(f) { + return out; + } + let img = Arc::new(layer_grid(&buf, ctx.layer_size)); + let Some((alpha, band, rr)) = matte_for(ctx, f, &img, sc) else { return out }; + let (w, h) = (img.width as usize, img.height as usize); + let mp = matte_params(ctx.params); + let mut rgb = rb::rgb_of(&img); + let map = refine::decontaminate(&mut rgb, &alpha, &band, w, h, &mp, rr * sc, sc); + let view_map = b(ctx.params, "refineEdgeMatte/viewDecontaminationMap"); + let (ox, oy) = (buf.offset[0].round() as i64, buf.offset[1].round() as i64); + for y in 0..h { + for x in 0..w { + let (bx, by) = (x as i64 + ox, y as i64 + oy); + if bx < 0 || by < 0 || bx >= out.img.width as i64 || by >= out.img.height as i64 { + continue; + } + let i = y * w + x; + let a_in = img.data[i][3]; + let px = if view_map { + let v = map[i]; + [v, v, v, 1.0] + } else { + let a = alpha[i] * a_in; + [rgb[i][0] * a, rgb[i][1] * a, rgb[i][2] * a, a] + }; + out.img.set(bx as u32, by as u32, px); + } + } + out +} + +/// The final matte of the effect at layer frame `f` on the full-resolution layer grid of +/// `input` (the effect's input buffer at scale 1), computing what is missing. For Freeze and +/// the Layer panel overlays. +pub fn frame_matte( + params: &Params, + layer_size: [f64; 2], + fps: f64, + f: i64, + frame: &mut dyn FnMut(i64) -> Option>, + cancel: &dyn Fn() -> bool, +) -> Option<(Vec, Vec)> { + let chain = Chain::new(params, layer_size, fps, 1.0); + let img = frame(f)?; + let seg = chain.ensure(f, frame, cancel)?; + let mp = matte_params(params); + let (mut prev, mut next) = (None, None); + if needs_neighbours(&mp) { + if chain.data.in_span(f - 1) { + prev = chain.ensure(f - 1, frame, cancel); + } + if chain.data.in_span(f + 1) { + next = chain.ensure(f + 1, frame, cancel); + } + } + let alpha = final_alpha(&img, &seg, prev.as_deref(), next.as_deref(), &mp, chain.refine_radius, 1.0); + Some((alpha, seg.refine.clone())) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, EffectHost, LayerPixels}; + + /// A host serving a moving red disk over a blue/green background as the effect's input. + struct Seq; + fn frame(t: f64) -> Image { + let fi = (t * 25.0 + 1e-6).floor(); + let c = [40.0 + 2.0 * fi, 40.0]; + let mut img = Image::new(100, 80); + for y in 0..80 { + for x in 0..100 { + let d = ((x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1])) - 15.0; + let a = (0.5 - d).clamp(0.0, 1.0) as f32; + let tex = ((x * 7 + y * 13) % 9) as f32 / 60.0; + let fg = [0.85 + tex, 0.2, 0.1]; + let bg = [0.1, 0.4 + tex, 0.7]; + img.set(x, y, [fg[0] * a + bg[0] * (1.0 - a), fg[1] * a + bg[1] * (1.0 - a), fg[2] * a + bg[2] * (1.0 - a), 1.0]); + } + } + img + } + impl EffectHost for Seq { + fn layer(&self, _: u64, _: bool) -> Option { + None + } + fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option> { + None + } + fn self_at(&self, t: f64, _: usize) -> Option { + Some(Buf { img: frame(t), offset: [0.0; 2], scale: 1.0 }) + } + } + + fn params_with(d: &RotoData) -> Params { + let s = &specs()[0]; + let mut params = Params { values: s.params.iter().map(|p| (p.id.to_string(), p.default.clone())).collect() }; + params.values.insert(STROKES.into(), Value::Str(d.to_json())); + params + } + + #[test] + fn renders_the_segmented_disk_on_demand() { + let mut d = RotoData::default(); + d.add(Stroke { kind: StrokeKind::Fg, frame: 0, radius: 3.0, points: vec![[35.0, 40.0], [45.0, 40.0]] }, 20, [0, 24]); + d.add(Stroke { kind: StrokeKind::Bg, frame: 0, radius: 3.0, points: vec![[5.0, 5.0], [95.0, 5.0], [95.0, 75.0]] }, 20, [0, 24]); + let params = params_with(&d); + let host = Seq; + let run = |t: f64| { + let ctx = EffectCtx { + params: ¶ms, + time: t, + layer_size: [100.0, 80.0], + seed: 1, + adjustment: false, + env: EffectEnv { host: Some(&host), frame_rate: 25.0, ..Default::default() }, + }; + crate::apply(&specs()[0], &ctx, Buf { img: frame(t), offset: [0.0; 2], scale: 1.0 }).img + }; + // Frame 5 (propagated through frames 1–4). + let out = run(5.0 / 25.0 + 0.001); + let c = [50.0, 40.0]; + let mut wrong = 0; + for y in 0..80u32 { + for x in 0..100u32 { + let d = (x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1]); + let a = out.data[(y * 100 + x) as usize][3]; + if (d < 14.0 && a < 0.5) || (d > 16.0 && a > 0.5) { + wrong += 1; + } + } + } + assert!(wrong < 20, "{wrong} wrong pixels"); + // Outside the span: transparent. + let mut d2 = (*data(¶ms)).clone(); + d2.span = [0, 3]; + let p2 = params_with(&d2); + let ctx = EffectCtx { + params: &p2, + time: 0.3, + layer_size: [100.0, 80.0], + seed: 1, + adjustment: false, + env: EffectEnv { host: Some(&host), frame_rate: 25.0, ..Default::default() }, + }; + let o = crate::apply(&specs()[0], &ctx, Buf { img: frame(0.3), offset: [0.0; 2], scale: 1.0 }).img; + assert!(o.data.iter().all(|p| p[3] == 0.0)); + } +} diff --git a/crates/engine/src/commands/edit.rs b/crates/engine/src/commands/edit.rs index e7ca4bd7..8ebd78db 100644 --- a/crates/engine/src/commands/edit.rs +++ b/crates/engine/src/commands/edit.rs @@ -525,6 +525,9 @@ fn purge_caches(s: &mut Session, p: &Value) -> Result { s.snapshot = None; s.state.viewer.show_snapshot = false; } + if matches!(what.as_str(), "all" | "image" | "memory" | "memoryAndDisk") { + effectcraft_effects::roto::purge(); + } s.events.push(crate::Event::PurgeCaches); s.toast(format!("Purged {what} cache")); Ok(json!({"purged": what})) diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 4537752f..ac451872 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -28,6 +28,7 @@ mod prop; pub mod puppet; mod query; mod render_queue; +pub mod roto_cmds; mod settings; mod stubs; mod styles; @@ -129,6 +130,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(track::specs()); v.extend(mask_interp::specs()); v.extend(warp_cmds::specs()); + v.extend(roto_cmds::specs()); v.extend(paint::specs()); v.extend(puppet::specs()); v.extend(settings::specs()); diff --git a/crates/engine/src/commands/roto_cmds.rs b/crates/engine/src/commands/roto_cmds.rs new file mode 100644 index 00000000..5b9db20f --- /dev/null +++ b/crates/engine/src/commands/roto_cmds.rs @@ -0,0 +1,424 @@ +//! Roto Brush & Refine Edge commands: strokes (the Roto Brush and Refine Edge tools in the Layer +//! panel), propagation, the segmentation span, Freeze / Unfreeze, tool options and status. +//! +//! `roto.stroke` records a stroke into the layer's Roto Brush & Refine Edge effect (applying the +//! effect on the first stroke, as After Effects does); the first foreground stroke sets the base +//! frame and a span of 20 frames each side. Strokes on a frozen effect are refused (Unfreeze +//! first). Every edit is one undo step. + +use effectcraft_effects::roto::{self as fx, FROZEN, Stroke, StrokeKind}; +use effectcraft_keyframe::Value as KV; +use effectcraft_project::build::Ids; +use effectcraft_project::{ItemId, Layer, LayerId, PropGroup, Uid}; +use effectcraft_track::roto::{RotoData, iou, rle}; +use serde_json::{Value, json}; + +use super::{CommandSpec, always, b_p, bad, f_p, has_comp, layer_mut, layer_p, str_p}; +use crate::roto::{self as rt, Direction, RotoOptions, RotoTask, VIEWS, is_roto, params_static}; +use crate::{EngineError, Result, Session, cmd, query}; + +pub fn specs() -> Vec { + vec![ + cmd!( + "roto.stroke", + "Roto Brush Stroke", + [], + None, + "{layer?, kind?: fg|bg|refine|refineErase, points: [[x, y], …] (layer pixels), frame? (layer frame; default current), radius? (layer px; default the tool's diameter / 2), effect?}", + has_comp, + stroke + ), + cmd!( + "roto.propagate", + "Propagate Roto Brush", + [], + None, + "{layer?, effect?, direction?: forward|backward|both, to? (layer frame: extend the span), wait?: block until done}", + has_comp, + propagate + ), + cmd!("roto.span", "Set Segmentation Span", [], None, "{layer?, effect?, start?, end? (layer frames)}", has_comp, span), + cmd!("roto.freeze", "Freeze", [], None, "{layer?, effect?, wait?}", has_comp, freeze), + cmd!("roto.unfreeze", "Unfreeze", [], None, "{layer?, effect?}", has_comp, unfreeze), + cmd!("roto.clearStrokes", "Remove Roto Brush Strokes", [], None, "{layer?, effect?, frame? (only this layer frame), kind?}", has_comp, clear_strokes), + cmd!("roto.cancel", "Stop Roto Brush", [], None, "{}", is_running, |s, _| Ok(json!({"stopped": s.stop_roto()}))), + cmd!( + "roto.options", + "Roto Brush Options", + [], + None, + "{diameter?, refineDiameter?, view?: alphaBoundary|alpha|alphaOverlay|none, overlayColor?: [r,g,b], overlayOpacity?, boundaryColor?: [r,g,b], autoPropagate?}", + always, + options + ), + query!( + "roto.status", + "Roto Brush Status", + "{layer?, effect?, frame? (layer frame: also report its matte), matte?: include the matte (RLE, base64), compute?: compute the frame if needed, compareTo?: RLE matte to report the IoU against}", + status + ), + ] +} + +fn is_running(s: &Session) -> std::result::Result<(), String> { + if s.is_roto_running() { Ok(()) } else { Err("no Roto Brush job is running".into()) } +} + +/// The Roto Brush instance targeted by `p`: `layer` / `effect` (uid, name or 1-based index), +/// else a selected one, else the first on the selected (or any) layer. +pub(crate) fn roto_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> { + let cid = super::comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let candidates: Vec = match p.get("layer") { + Some(v) => vec![super::resolve_layer(comp, v).ok_or_else(|| bad(cmd, format!("no layer {v}")))?], + None => { + let mut v: Vec = s.state.selected_props.iter().map(|(l, _)| *l).collect(); + v.extend(s.state.selected_layers.iter().copied()); + v.extend(comp.layers.iter().map(|l| l.id)); + v + } + }; + for lid in candidates { + let Some(fxg) = comp.layer(lid).and_then(Layer::effects) else { continue }; + let g = match p.get("effect") { + Some(Value::Number(n)) => { + let n = n.as_u64().unwrap_or(0); + fxg.groups().find(|g| g.uid == n).or_else(|| fxg.groups().nth((n as usize).saturating_sub(1))) + } + Some(Value::String(name)) => fxg.groups().find(|g| &g.name == name), + _ => s + .state + .selected_props + .iter() + .find_map(|(l, u)| (*l == lid).then(|| fxg.groups().find(|g| g.uid == *u && is_roto(g))).flatten()) + .or_else(|| fxg.groups().find(|g| is_roto(g))), + }; + if let Some(g) = g.filter(|g| is_roto(g)) { + return Ok((cid, lid, g.uid)); + } + if p.get("layer").is_some() { + break; + } + } + Err(bad(cmd, "no Roto Brush & Refine Edge effect found (paint a stroke with the Roto Brush tool first)")) +} + +fn group(s: &Session, cid: ItemId, lid: LayerId, uid: Uid) -> Result<(&Layer, &PropGroup)> { + let l = s.project.comp(cid).and_then(|c| c.layer(lid)).ok_or(EngineError::NoComp)?; + let g = l.props.find_group(uid).ok_or_else(|| EngineError::Other("the effect is gone".into()))?; + Ok((l, g)) +} + +fn data_of(g: &PropGroup) -> RotoData { + fx::data(¶ms_static(g)).as_ref().clone() +} + +fn frozen_of(g: &PropGroup) -> bool { + matches!(g.get(FROZEN).map(|p| &p.value), Some(KV::Str(s)) if !s.is_empty()) +} + +/// Write an instance's data (one undo step). +fn write_data(s: &mut Session, label: &str, cid: ItemId, lid: LayerId, uid: Uid, d: &RotoData) -> Result<()> { + let j = d.to_json(); + s.edit(label, None, |proj, _| { + let g = layer_mut(proj, cid, lid)?.props.find_group_mut(uid).ok_or_else(|| EngineError::Other("the effect is gone".into()))?; + if let Some(pr) = g.get_mut(fx::STROKES) { + pr.value = KV::Str(j); + } + Ok(()) + }) +} + +/// Current layer frame of `layer` (or `frame` from the params). +fn frame_p(s: &Session, p: &Value, cid: ItemId, layer: &Layer) -> i64 { + if let Some(f) = f_p(p, "frame") { + return f.round() as i64; + } + let comp = s.project.comp(cid); + let fps = comp.map(|c| c.frame_rate.as_f64()).unwrap_or(30.0); + fx::frame_of(layer.layer_time(s.time_of(cid)).seconds(), fps) +} + +fn points_p(p: &Value, cmd: &str) -> Result> { + let raw = p.get("points").and_then(Value::as_array).ok_or_else(|| bad(cmd, "missing `points` [[x, y], …] (layer pixels)"))?; + let mut v = vec![]; + for q in raw { + let a = q.as_array().ok_or_else(|| bad(cmd, "each point is [x, y]"))?; + let (Some(x), Some(y)) = (a.first().and_then(Value::as_f64), a.get(1).and_then(Value::as_f64)) else { + return Err(bad(cmd, "each point is [x, y]")); + }; + v.push([x, y]); + } + if v.is_empty() { + return Err(bad(cmd, "`points` is empty")); + } + Ok(v) +} + +fn stroke(s: &mut Session, p: &Value) -> Result { + let cmd = "roto.stroke"; + let (cid, lid) = match roto_p(s, p, cmd) { + Ok((c, l, _)) => (c, l), + Err(_) => layer_p(s, p, cmd)?, + }; + let kind = match str_p(p, "kind") { + Some(k) => StrokeKind::from_name(k).ok_or_else(|| bad(cmd, format!("unknown kind `{k}` (fg | bg | refine | refineErase)")))?, + None => StrokeKind::Fg, + }; + let points = points_p(p, cmd)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or_else(|| bad(cmd, "no layer"))?; + if !layer.has_video() || layer.switches.adjustment || layer.effects().is_none() { + return Err(bad(cmd, "the Roto Brush works on footage, solid and composition layers")); + } + let frame = frame_p(s, p, cid, layer); + let limits = rt::frame_limits(comp, layer); + if frame < limits[0] || frame > limits[1] { + return Err(bad(cmd, format!("frame {frame} is outside the layer ({}–{})", limits[0], limits[1]))); + } + let o = &s.state.roto; + let def_r = if matches!(kind, StrokeKind::Refine | StrokeKind::RefineErase) { o.refine_diameter / 2.0 } else { o.diameter / 2.0 }; + let radius = f_p(p, "radius").unwrap_or(def_r).clamp(0.5, 2500.0); + let size = rt::layer_size(&s.project, comp, layer); + // The instance: the requested / first one, else a new one. + let existing = roto_p(s, p, cmd).ok().filter(|(c, l, _)| *c == cid && *l == lid).map(|x| x.2); + if let Some(uid) = existing { + let (_, g) = group(s, cid, lid, uid)?; + if frozen_of(g) { + return Err(bad(cmd, "the Roto Brush is frozen: Unfreeze it to edit strokes")); + } + } + if existing.is_none() && !kind.is_segmentation() { + return Err(bad(cmd, "draw a foreground stroke with the Roto Brush tool first")); + } + let st = Stroke { kind, frame, radius, points }; + let label = if kind.is_segmentation() { "Roto Brush Stroke" } else { "Refine Edge Stroke" }; + let (uid, d) = s.edit(label, None, |proj, _| { + let mut next = proj.next_id; + let l = layer_mut(proj, cid, lid)?; + let fxg = l.props.sub_mut("effects").ok_or_else(|| bad(cmd, "this layer has no effects"))?; + let uid = match existing { + Some(u) => u, + None => { + let spec = effectcraft_effects::find(fx::ID).ok_or_else(|| EngineError::Other("Roto Brush effect missing".into()))?; + let n = fxg.groups().filter(|g| is_roto(g)).count(); + let name = if n == 0 { spec.name.to_string() } else { format!("{} {}", spec.name, n + 1) }; + let g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), &name, size); + let u = g.uid; + fxg.children.push(g.into()); + u + } + }; + let g = fxg.find_group_mut(uid).ok_or_else(|| bad(cmd, "effect gone"))?; + let mut d = data_of(g); + d.add(st, fx::DEFAULT_SPAN, limits); + if let Some(pr) = g.get_mut(fx::STROKES) { + pr.value = KV::Str(d.to_json()); + } + proj.next_id = proj.next_id.max(next); + Ok((uid, d)) + })?; + if !s.state.selected_props.iter().any(|(l, u)| *l == lid && *u == uid) { + s.state.selected_layers = vec![lid]; + } + Ok(json!({ + "layer": lid.0, "effect": uid, "frame": frame, "kind": kind, "radius": radius, + "base": d.base, "span": d.span, "strokes": d.strokes.len(), + })) +} + +fn propagate(s: &mut Session, p: &Value) -> Result { + let cmd = "roto.propagate"; + let (cid, lid, uid) = roto_p(s, p, cmd)?; + let dir = match str_p(p, "direction") { + Some(d) => Direction::from_name(d).ok_or_else(|| bad(cmd, format!("unknown direction `{d}` (forward | backward | both)")))?, + None => Direction::Both, + }; + if let Some(to) = f_p(p, "to") { + let to = to.round() as i64; + let (l, g) = group(s, cid, lid, uid)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let lim = rt::frame_limits(comp, l); + let mut d = data_of(g); + let to = to.clamp(lim[0], lim[1]); + let before = d.span; + d.span[0] = d.span[0].min(to); + d.span[1] = d.span[1].max(to); + if d.span != before { + if frozen_of(g) { + return Err(bad(cmd, "the Roto Brush is frozen: Unfreeze it to change the span")); + } + write_data(s, "Extend Segmentation Span", cid, lid, uid, &d)?; + } + } + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_roto(cid, lid, uid, RotoTask::Propagate, dir, wait).map_err(|e| bad(cmd, e))?; + let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_roto_running(), "progress": s.roto_progress()}); + if wait { + out["status"] = status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?; + } + Ok(out) +} + +fn span(s: &mut Session, p: &Value) -> Result { + let cmd = "roto.span"; + let (cid, lid, uid) = roto_p(s, p, cmd)?; + let (l, g) = group(s, cid, lid, uid)?; + if frozen_of(g) { + return Err(bad(cmd, "the Roto Brush is frozen: Unfreeze it to change the span")); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let lim = rt::frame_limits(comp, l); + let mut d = data_of(g); + let base = d.base.ok_or_else(|| bad(cmd, "no strokes yet"))?; + let first = d.strokes.iter().map(|s| s.frame).min().unwrap_or(base); + let last = d.strokes.iter().map(|s| s.frame).max().unwrap_or(base); + if let Some(a) = f_p(p, "start") { + d.span[0] = (a.round() as i64).clamp(lim[0], first); + } + if let Some(b) = f_p(p, "end") { + d.span[1] = (b.round() as i64).clamp(last, lim[1]); + } + write_data(s, "Segmentation Span", cid, lid, uid, &d)?; + Ok(json!({"layer": lid.0, "effect": uid, "span": d.span, "base": base})) +} + +fn freeze(s: &mut Session, p: &Value) -> Result { + let cmd = "roto.freeze"; + let (cid, lid, uid) = roto_p(s, p, cmd)?; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_roto(cid, lid, uid, RotoTask::Freeze, Direction::Both, wait).map_err(|e| bad(cmd, e))?; + let (_, g) = group(s, cid, lid, uid)?; + Ok(json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_roto_running(), "frozen": frozen_of(g), "progress": s.roto_progress()})) +} + +fn unfreeze(s: &mut Session, p: &Value) -> Result { + let cmd = "roto.unfreeze"; + let (cid, lid, uid) = roto_p(s, p, cmd)?; + let (_, g) = group(s, cid, lid, uid)?; + if !frozen_of(g) { + return Ok(json!({"layer": lid.0, "effect": uid, "frozen": false})); + } + s.edit("Unfreeze", None, |proj, _| { + let g = layer_mut(proj, cid, lid)?.props.find_group_mut(uid).ok_or_else(|| EngineError::Other("the effect is gone".into()))?; + if let Some(pr) = g.get_mut(FROZEN) { + pr.value = KV::Str(String::new()); + } + Ok(()) + })?; + Ok(json!({"layer": lid.0, "effect": uid, "frozen": false})) +} + +fn clear_strokes(s: &mut Session, p: &Value) -> Result { + let cmd = "roto.clearStrokes"; + let (cid, lid, uid) = roto_p(s, p, cmd)?; + let (_, g) = group(s, cid, lid, uid)?; + if frozen_of(g) { + return Err(bad(cmd, "the Roto Brush is frozen: Unfreeze it to edit strokes")); + } + let mut d = data_of(g); + let frame = f_p(p, "frame").map(|f| f.round() as i64); + let kind = match str_p(p, "kind") { + Some(k) => Some(StrokeKind::from_name(k).ok_or_else(|| bad(cmd, format!("unknown kind `{k}`")))?), + None => None, + }; + let n0 = d.strokes.len(); + d.strokes.retain(|st| !(frame.is_none_or(|f| st.frame == f) && kind.is_none_or(|k| st.kind == k))); + if d.strokes.iter().all(|s| !s.kind.is_segmentation()) { + d.base = None; + } else if d.base.is_some_and(|b| !d.strokes.iter().any(|s| s.frame == b && s.kind.is_segmentation())) { + // The base moves to the earliest remaining stroked frame. + d.base = d.strokes.iter().filter(|s| s.kind.is_segmentation()).map(|s| s.frame).min(); + } + let removed = n0 - d.strokes.len(); + if removed > 0 { + write_data(s, "Remove Roto Brush Strokes", cid, lid, uid, &d)?; + } + Ok(json!({"layer": lid.0, "effect": uid, "removed": removed, "strokes": d.strokes.len(), "base": d.base})) +} + +fn options(s: &mut Session, p: &Value) -> Result { + let cmd = "roto.options"; + let mut o: RotoOptions = s.state.roto.clone(); + for (k, f) in [("diameter", &mut o.diameter as &mut f64), ("refineDiameter", &mut o.refine_diameter), ("overlayOpacity", &mut o.overlay_opacity)] { + if let Some(v) = f_p(p, k) { + *f = v; + } + } + o.diameter = o.diameter.clamp(1.0, 5000.0); + o.refine_diameter = o.refine_diameter.clamp(1.0, 5000.0); + o.overlay_opacity = o.overlay_opacity.clamp(0.0, 100.0); + if let Some(v) = str_p(p, "view") { + let v = VIEWS.iter().find(|x| x.eq_ignore_ascii_case(v)).ok_or_else(|| bad(cmd, format!("unknown view `{v}` ({})", VIEWS.join(" | "))))?; + o.view = v.to_string(); + } + for (k, c) in [("overlayColor", &mut o.overlay_color), ("boundaryColor", &mut o.boundary_color)] { + if let Some(a) = p.get(k).and_then(Value::as_array) { + if a.len() < 3 { + return Err(bad(cmd, format!("`{k}` is [r, g, b] (0–1)"))); + } + for (i, v) in a.iter().take(3).enumerate() { + c[i] = v.as_f64().unwrap_or(0.0).clamp(0.0, 1.0); + } + } + } + if let Some(b) = b_p(p, "autoPropagate") { + o.auto_propagate = b; + } + s.state.roto = o.clone(); + Ok(serde_json::to_value(o).unwrap_or(Value::Null)) +} + +fn status(s: &mut Session, p: &Value) -> Result { + s.poll_roto(false); + let mut out = + json!({"running": s.is_roto_running(), "progress": s.roto_progress(), "banner": s.roto_progress().map(|p| p.banner()), "options": s.state.roto}); + let Ok((cid, lid, uid)) = roto_p(s, p, "roto.status") else { return Ok(out) }; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let (layer, g) = group(s, cid, lid, uid)?; + let (layer, g) = (layer.clone(), g.clone()); + let d = data_of(&g); + let computed = s.roto_computed(cid, lid, uid); + out["layer"] = json!(lid.0); + out["effect"] = json!(uid); + out["base"] = json!(d.base); + out["span"] = json!(d.span); + out["strokes"] = json!(d.strokes.len()); + out["strokeFrames"] = json!(d.stroke_frames()); + out["computed"] = json!(computed.iter().filter(|(_, c)| **c).map(|(f, _)| *f).collect::>()); + out["frozen"] = json!(frozen_of(&g)); + out["pending"] = json!(s.roto_pending.contains(&(cid, lid, uid))); + out["limits"] = json!(rt::frame_limits(&comp, &layer)); + let frame = frame_p(s, p, cid, &layer); + out["frame"] = json!(frame); + let want_matte = b_p(p, "matte").unwrap_or(false); + let compute = b_p(p, "compute").unwrap_or(want_matte); + if (p.get("frame").is_some() || want_matte) && d.in_span(frame) { + let size = rt::layer_size(&s.project, &comp, &layer); + let fps = comp.frame_rate.as_f64(); + let ps = params_static(&g); + let chain = fx::Chain::new(&ps, size, fps, 1.0); + let seg = match chain.keys.get(&frame).and_then(|k| fx::cached(*k, 1.0)) { + Some(sg) => Some(sg), + None if compute => { + let index = layer.effects().and_then(|f| f.groups().position(|x| x.uid == uid)).unwrap_or(0); + let mut src = rt::FrameSource::new(&s.project, s.footage.as_ref(), s.expr.as_deref(), &s.layer_cache, cid, &comp, &layer, index, size); + chain.ensure(frame, &mut |f| src.get(f), &|| false) + } + None => None, + }; + if let Some(sg) = seg { + out["area"] = json!(sg.area()); + out["centroid"] = json!(sg.centroid()); + out["size"] = json!([sg.w, sg.h]); + out["refineArea"] = json!(sg.refine.iter().filter(|v| **v != 0).count()); + if want_matte { + out["matte"] = json!(rle::encode(&sg.matte)); + } + if let Some(gt) = p.get("compareTo").and_then(Value::as_str).and_then(rle::decode) { + out["iou"] = json!(iou(&sg.matte, >)); + } + } + } + Ok(out) +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index ba34488a..fa2e6fe4 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -17,6 +17,7 @@ pub mod mask_track; pub mod menus; pub mod prefs; pub mod render_queue; +pub mod roto; mod session_settings; pub mod shortcuts; pub mod tracking; @@ -41,6 +42,7 @@ pub use effectcraft_project as project; pub use effectcraft_render as render; pub use effectcraft_text as text; pub use effectcraft_time as time; +pub use effectcraft_track as track; pub use render_queue::{ExportJob, ExportResult, Exporter, JobState}; #[derive(Debug, thiserror::Error)] @@ -176,6 +178,9 @@ pub struct EditorState { /// Paint and Brushes panel options (Brush, Clone Stamp and Eraser tools). #[serde(default)] pub paint: commands::paint::PaintOptions, + /// Roto Brush and Refine Edge tool options. + #[serde(default)] + pub roto: roto::RotoOptions, /// Puppet tool options for new meshes. #[serde(default)] pub puppet: commands::puppet::PuppetOptions, @@ -263,6 +268,10 @@ pub struct Session { pub warp_job: Option, /// Warp Stabilizers waiting for (re-)analysis: (comp, layer, effect uid). pub warp_pending: Vec<(ItemId, LayerId, Uid)>, + /// The running (or finished, not yet polled) Roto Brush propagation / Freeze. + pub roto_job: Option, + /// Roto Brush instances edited since their last propagation: (comp, layer, effect uid). + pub roto_pending: Vec<(ItemId, LayerId, Uid)>, pub events: Vec, /// Commands executed: (id, params). pub journal: Vec<(String, Value)>, @@ -308,6 +317,8 @@ impl Default for Session { mask_job: None, warp_job: None, warp_pending: vec![], + roto_job: None, + roto_pending: vec![], events: vec![], journal: vec![], layer_cache: Arc::new(LayerCache::default()), @@ -374,6 +385,12 @@ impl Session { self.warp_pending.push(w); } } + // Roto Brush Input Keys follow the frames; edited instances propagate again. + for r in roto::sync(&before, &mut p) { + if !self.roto_pending.contains(&r) { + self.roto_pending.push(r); + } + } let same = merge.is_some() && merge.map(str::to_string) == self.history.merge_key; if !same { self.history.undo.push((label.to_string(), before)); @@ -551,6 +568,9 @@ impl Session { self.stop_warp(); self.warp_job = None; self.warp_pending.clear(); + self.stop_roto(); + self.roto_job = None; + self.roto_pending.clear(); p.fix_next_id(); self.project = Arc::new(p); self.history = History::default(); @@ -615,3 +635,5 @@ pub fn text_families() -> Vec { } #[cfg(test)] mod tests_paint; +#[cfg(test)] +mod tests_roto; diff --git a/crates/engine/src/roto.rs b/crates/engine/src/roto.rs new file mode 100644 index 00000000..5b2f635e --- /dev/null +++ b/crates/engine/src/roto.rs @@ -0,0 +1,569 @@ +//! Roto Brush & Refine Edge runtime: propagation and Freeze jobs, Input Key maintenance and the +//! tool options. +//! +//! Segmentations are derived data in `effectcraft_effects::roto`'s content-keyed cache (see that +//! module). `roto.propagate` fills it at full resolution in the background (or blocking with +//! `wait`), frame by frame outward from the base frame, rendering the effect's input +//! (`Renderer::layer_input`, source → masks → the effects above it). `roto.freeze` runs the same +//! pass and stores the final mattes in the effect (one undo step). [`sync`], run after every +//! edit, keeps each instance's hidden Input Key equal to a signature of what its frames are made +//! of (the layer's source and timing, its masks and the effects above it) and drops frozen +//! mattes made from other frames. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use effectcraft_effects::roto::{self as fx, FROZEN, FrozenCache, INPUT_KEY}; +use effectcraft_effects::{Params, flatten_params}; +use effectcraft_keyframe::Value; +use effectcraft_project::{GroupKind, ItemId, ItemKind, Layer, LayerId, Project, PropGroup, Uid}; +use effectcraft_raster::Image; +use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; +use effectcraft_time::Tick; +use serde::{Deserialize, Serialize}; + +use crate::tracking::lock; +use crate::{Event, Session}; + +/// Roto Brush / Refine Edge tool options (serde: `roto.options`). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct RotoOptions { + /// Roto Brush tip diameter (layer pixels). + pub diameter: f64, + /// Refine Edge tip diameter (layer pixels). + pub refine_diameter: f64, + /// Layer panel view: `alphaBoundary` (pink outline), `alpha`, `alphaOverlay`, `none`. + pub view: String, + /// Alpha Overlay colour (straight RGB 0–1) and opacity (percent). + pub overlay_color: [f64; 3], + pub overlay_opacity: f64, + /// Alpha Boundary colour. + pub boundary_color: [f64; 3], + /// Start propagating in the background after each stroke. + pub auto_propagate: bool, +} + +impl Default for RotoOptions { + fn default() -> Self { + RotoOptions { + diameter: 20.0, + refine_diameter: 10.0, + view: "alphaBoundary".into(), + overlay_color: [1.0, 0.0, 0.0], + overlay_opacity: 50.0, + boundary_color: [1.0, 0.25, 0.75], + auto_propagate: true, + } + } +} + +pub const VIEWS: [&str; 4] = ["alphaBoundary", "alpha", "alphaOverlay", "none"]; + +/// What a job does. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum RotoTask { + Propagate, + Freeze, +} + +/// Which way to propagate. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Direction { + Forward, + Backward, + #[default] + Both, +} + +impl Direction { + pub fn from_name(s: &str) -> Option { + Some(match s.to_ascii_lowercase().as_str() { + "forward" | "fwd" => Direction::Forward, + "backward" | "back" => Direction::Backward, + "both" | "all" => Direction::Both, + _ => return None, + }) + } +} + +/// Live job progress (serde for agents / the control channel). +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct RotoProgress { + pub task: Option, + pub done: u64, + pub total: u64, + pub elapsed: f64, + pub fps: f64, + pub finished: bool, + pub cancelled: bool, + pub error: Option, +} + +impl RotoProgress { + pub fn banner(&self) -> String { + let pc = (100 * self.done).checked_div(self.total).unwrap_or(0); + match self.task { + Some(RotoTask::Freeze) => format!("Freezing Roto Brush & Refine Edge: {pc}%"), + _ => format!("Propagating Roto Brush: {pc}%"), + } + } +} + +#[derive(Default)] +pub struct RotoShared { + pub cancel: AtomicBool, + pub state: Mutex, + frozen: Mutex>, +} + +pub struct RotoJob { + pub shared: Arc, + thread: Option>, + pub comp: ItemId, + pub layer: LayerId, + pub effect: Uid, + pub task: RotoTask, + /// The strokes the job was started with (a Freeze result is dropped when they changed). + strokes: String, +} + +impl RotoJob { + pub fn is_finished(&self) -> bool { + lock(&self.shared.state).finished + } + pub fn progress(&self) -> RotoProgress { + lock(&self.shared.state).clone() + } +} + +pub(crate) fn is_roto(g: &PropGroup) -> bool { + matches!(&g.kind, GroupKind::Effect { effect } if effect == fx::ID) +} + +fn str_of(g: &PropGroup, m: &str) -> String { + match g.get(m).map(|p| &p.value) { + Some(Value::Str(s)) => s.clone(), + _ => String::new(), + } +} + +fn set_str(g: &mut PropGroup, m: &str, v: String) { + if let Some(p) = g.get_mut(m) { + p.value = Value::Str(v); + p.keys.clear(); + } +} + +fn fnv(s: &str) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for b in s.as_bytes() { + h ^= *b as u64; + h = h.wrapping_mul(0x0100_0000_01b3); + } + h +} + +/// What the effect's input frames are made of: the layer's source and timing, its masks and the +/// effects above the Roto Brush. +pub fn input_key(p: &Project, comp: ItemId, layer: &Layer, uid: Uid) -> String { + let mut s = crate::warp::signature(p, comp, layer); + if let Some(m) = layer.masks() { + s.push_str(&format!("{m:?}")); + } + if let Some(fx) = layer.effects() { + for g in fx.groups() { + if g.uid == uid { + break; + } + s.push_str(&format!("{g:?}")); + } + } + format!("{:016x}", fnv(&s)) +} + +/// After an edit: refresh Input Keys; frozen mattes made from other frames are dropped. +/// Returns the instances whose segmentation changed (to propagate again). +pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> { + let mut todo = vec![]; + let mut set = vec![]; + for (cid, it) in &after.items { + let ItemKind::Comp(c) = &it.kind else { continue }; + for l in &c.layers { + let Some(fxg) = l.effects() else { continue }; + for g in fxg.groups().filter(|g| is_roto(g)) { + let key = input_key(after, *cid, l, g.uid); + let old = before.comp(*cid).and_then(|bc| bc.layer(l.id)).and_then(|ol| ol.props.find_group(g.uid)); + if str_of(g, INPUT_KEY) != key { + set.push((*cid, l.id, g.uid, key)); + todo.push((*cid, l.id, g.uid)); + } else if old.is_none_or(|o| str_of(o, fx::STROKES) != str_of(g, fx::STROKES)) { + todo.push((*cid, l.id, g.uid)); + } + } + } + } + for (cid, lid, uid, key) in set { + if let Some(g) = after.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)) { + let had = !str_of(g, INPUT_KEY).is_empty(); + set_str(g, INPUT_KEY, key); + if had { + set_str(g, FROZEN, String::new()); + } + } + } + todo.retain(|(c, l, u)| { + after.comp(*c).and_then(|c| c.layer(*l)).and_then(|l| l.props.find_group(*u)).is_some_and(|g| !fx::data(¶ms_static(g)).is_empty()) + }); + todo +} + +/// Parameters of an instance with static values (strokes and keys are never animated). +pub(crate) fn params_static(g: &PropGroup) -> Params { + flatten_params(g, &mut |p| p.value.clone()) +} + +/// Evaluated parameters of an instance at comp time `t`. +pub(crate) fn params_at(p: &Project, cid: ItemId, layer: &Layer, g: &PropGroup, t: Tick, expr: Option<&dyn ExprHost>) -> Params { + match p.comp(cid) { + Some(c) => { + let ctx = EvalCtx { expr, ..EvalCtx::new(p, cid, c, t) }; + flatten_params(g, &mut |pr| ctx.value(layer, pr)) + } + None => params_static(g), + } +} + +/// The layer's source size (layer pixels). +pub(crate) fn layer_size(p: &Project, comp: &effectcraft_project::Comp, layer: &Layer) -> [f64; 2] { + let (w, h) = effectcraft_render::source_size(p, layer); + if w == 0 { [comp.width as f64, comp.height as f64] } else { [w as f64, h as f64] } +} + +/// First and last layer frame of the layer inside the comp (inclusive). +pub fn frame_limits(comp: &effectcraft_project::Comp, layer: &Layer) -> [i64; 2] { + let fps = comp.frame_rate.as_f64().max(1e-6); + let lo = layer.layer_time(layer.in_point.max(Tick::ZERO)).seconds(); + let hi = layer.layer_time(layer.out_point.min(comp.duration)).seconds(); + let a = fx::frame_of(lo, fps); + let b = (fx::frame_of(hi, fps) - 1).max(a); + [a, b] +} + +/// Comp time of layer frame `f`. +pub(crate) fn comp_time_of(comp: &effectcraft_project::Comp, layer: &Layer, f: i64) -> Tick { + layer.comp_time(comp.frame_rate.tick_of(f)) +} + +/// What the worker needs, detached from the session. +struct Work { + project: Arc, + footage: Arc, + expr: Option>, + cache: Arc, + comp: ItemId, + layer: LayerId, + effect_index: usize, + params: Params, + size: [f64; 2], + fps: f64, + frames: Vec, + task: RotoTask, +} + +/// Renders the effect's input frames for a worker, keeping the last few. +pub(crate) struct FrameSource<'a> { + r: Renderer<'a>, + ctx0: EvalCtx<'a>, + comp: &'a effectcraft_project::Comp, + layer: &'a Layer, + index: usize, + size: [f64; 2], + memo: HashMap>, +} + +impl<'a> FrameSource<'a> { + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + project: &'a Project, + footage: &'a dyn FootageSource, + expr: Option<&'a dyn ExprHost>, + cache: &'a LayerCache, + cid: ItemId, + comp: &'a effectcraft_project::Comp, + layer: &'a Layer, + index: usize, + size: [f64; 2], + ) -> FrameSource<'a> { + let mut r = Renderer::new(project, footage, RenderOpts { scale: 1.0, ..Default::default() }); + r.expr = expr; + r.cache = Some(cache); + let ctx0 = EvalCtx { expr, ..EvalCtx::new(project, cid, comp, Tick::ZERO) }; + FrameSource { r, ctx0, comp, layer, index, size, memo: HashMap::new() } + } + + pub(crate) fn get(&mut self, f: i64) -> Option> { + if let Some(i) = self.memo.get(&f) { + return Some(i.clone()); + } + let t = comp_time_of(self.comp, self.layer, f); + let buf = self.r.layer_input(&self.ctx0.at(t), self.layer, self.index)?; + let buf = if (buf.scale - 1.0).abs() > 1e-9 && buf.scale > 0.0 { + let w = (buf.img.width as f64 / buf.scale).round().max(1.0) as u32; + let h = (buf.img.height as f64 / buf.scale).round().max(1.0) as u32; + effectcraft_effects::Buf { + img: effectcraft_raster::resample(&buf.img, w, h), + offset: [buf.offset[0] / buf.scale, buf.offset[1] / buf.scale], + scale: 1.0, + } + } else { + (*buf).clone() + }; + let img = Arc::new(fx::layer_grid(&buf, self.size)); + if self.memo.len() > 4 { + self.memo.clear(); + } + self.memo.insert(f, img.clone()); + Some(img) + } +} + +fn run_work(w: Work, shared: &RotoShared) { + let t0 = web_time::Instant::now(); + let fail = |msg: &str| { + let mut s = lock(&shared.state); + s.error = Some(msg.to_string()); + s.finished = true; + }; + let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") }; + let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") }; + let mut src = FrameSource::new(&w.project, w.footage.as_ref(), w.expr.as_deref(), &w.cache, w.comp, comp, layer, w.effect_index, w.size); + { + let mut s = lock(&shared.state); + s.task = Some(w.task); + s.total = w.frames.len() as u64; + } + let cancel = || shared.cancel.load(Ordering::Relaxed); + let chain = fx::Chain::new(&w.params, w.size, w.fps, 1.0); + let mut frozen = FrozenCache { key: fx::frozen_key(&w.params, w.size, w.fps), ..Default::default() }; + for (k, f) in w.frames.iter().enumerate() { + if cancel() { + let mut s = lock(&shared.state); + s.cancelled = true; + s.finished = true; + return; + } + let mut get = |g: i64| src.get(g); + let ok = match w.task { + RotoTask::Propagate => chain.ensure(*f, &mut get, &cancel).is_some(), + RotoTask::Freeze => match fx::frame_matte(&w.params, w.size, w.fps, *f, &mut get, &cancel) { + Some((a, b)) => { + if let Some(img) = src.get(*f) { + frozen.size = [img.width as usize, img.height as usize]; + } + frozen.frames.insert(*f, fx::freeze_frame(&a, &b)); + true + } + None => false, + }, + }; + if !ok && !cancel() { + return fail(&format!("frame {f} could not be segmented")); + } + let mut s = lock(&shared.state); + s.done = k as u64 + 1; + s.elapsed = t0.elapsed().as_secs_f64(); + s.fps = if s.elapsed > 0.0 { s.done as f64 / s.elapsed } else { 0.0 }; + } + if w.task == RotoTask::Freeze { + *lock(&shared.frozen) = Some(frozen); + } + let mut s = lock(&shared.state); + s.elapsed = t0.elapsed().as_secs_f64(); + s.finished = true; +} + +/// Frames to compute, outward from the base frame. +pub(crate) fn job_frames(d: &effectcraft_track::roto::RotoData, dir: Direction) -> Vec { + let Some(base) = d.base else { return vec![] }; + let mut v = vec![base]; + if dir != Direction::Backward { + v.extend(base + 1..=d.span[1]); + } + if dir != Direction::Forward { + v.extend((d.span[0]..base).rev()); + } + v +} + +impl Session { + pub fn is_roto_running(&self) -> bool { + self.roto_job.as_ref().is_some_and(|j| !j.is_finished()) + } + + pub fn roto_progress(&self) -> Option { + self.roto_job.as_ref().map(|j| j.progress()) + } + + /// (comp, layer, effect uid, task) of the running job. + pub fn roto_target(&self) -> Option<(ItemId, LayerId, Uid, RotoTask)> { + self.roto_job.as_ref().map(|j| (j.comp, j.layer, j.effect, j.task)) + } + + pub fn stop_roto(&mut self) -> bool { + match &self.roto_job { + Some(j) if !j.is_finished() => { + j.shared.cancel.store(true, Ordering::Relaxed); + true + } + _ => false, + } + } + + /// Start propagating (or freezing) Roto Brush `effect` on `layer`. `wait`: block until done + /// (wasm32 always does). Returns the number of frames. + pub fn start_roto(&mut self, comp: ItemId, layer: LayerId, effect: Uid, task: RotoTask, dir: Direction, wait: bool) -> Result { + if self.is_roto_running() { + if !wait { + return Err("a Roto Brush job is already running".into()); + } + self.stop_roto(); + if let Some(mut j) = self.roto_job.take() + && let Some(t) = j.thread.take() + { + let _ = t.join(); + } + } + self.poll_roto_job(); + self.roto_pending.retain(|x| *x != (comp, layer, effect)); + let c = self.project.comp(comp).ok_or("no composition")?; + let l = c.layer(layer).ok_or("no layer")?; + let fxg = l.effects().ok_or("the layer has no effects")?; + let index = fxg.groups().position(|g| g.uid == effect).ok_or("no such effect")?; + let g = fxg.groups().nth(index).filter(|g| is_roto(g)).ok_or("not a Roto Brush & Refine Edge effect")?; + let size = layer_size(&self.project, c, l); + let fps = c.frame_rate.as_f64(); + let pstat = params_static(g); + let d = fx::data(&pstat); + if d.is_empty() { + return Err("draw a foreground stroke with the Roto Brush tool first".into()); + } + let t = comp_time_of(c, l, d.base.unwrap_or(0)); + let params = params_at(&self.project, comp, l, g, t, self.expr.as_deref()); + let frames = job_frames(&d, if task == RotoTask::Freeze { Direction::Both } else { dir }); + let n = frames.len(); + let work = Work { + project: self.project.clone(), + footage: self.footage.clone(), + expr: self.expr.clone(), + cache: self.layer_cache.clone(), + comp, + layer, + effect_index: index, + params, + size, + fps, + frames, + task, + }; + let strokes = str_of(g, fx::STROKES); + let wait = wait || cfg!(target_arch = "wasm32"); + let shared = Arc::new(RotoShared::default()); + let thread = if wait { + run_work(work, &shared); + None + } else { + let sh = shared.clone(); + Some(std::thread::Builder::new().name("roto".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?) + }; + self.roto_job = Some(RotoJob { shared, thread, comp, layer, effect, task, strokes }); + self.events.push(Event::ProjectChanged { revision: self.revision }); + self.poll_roto_job(); + Ok(n) + } + + /// Finish a done job: Freeze writes the frozen mattes (one undo step). + fn poll_roto_job(&mut self) -> bool { + let Some(job) = &self.roto_job else { return false }; + if !job.is_finished() { + return false; + } + let mut job = self.roto_job.take().expect("job"); + if let Some(t) = job.thread.take() { + let _ = t.join(); + } + let st = job.progress(); + if let Some(e) = &st.error { + self.events.push(Event::Toast { message: format!("Roto Brush: {e}"), error: true }); + self.bump(); + return true; + } + if st.cancelled { + self.events.push(Event::Toast { message: "Roto Brush: stopped".into(), error: false }); + self.bump(); + return true; + } + match job.task { + RotoTask::Propagate => { + self.events.push(Event::Toast { message: format!("Roto Brush: propagated {} frame(s) in {:.1} s", st.done, st.elapsed), error: false }); + // Rendered frames depend on the cache: redraw. + self.bump(); + } + RotoTask::Freeze => { + let Some(fc) = lock(&job.shared.frozen).take() else { return true }; + let (cid, lid, uid, strokes) = (job.comp, job.layer, job.effect, job.strokes.clone()); + let r = self.edit("Freeze", None, |p, _| { + let g = p + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .ok_or_else(|| crate::EngineError::Other("the Roto Brush effect was removed".into()))?; + if str_of(g, fx::STROKES) != strokes { + return Err(crate::EngineError::Other("the strokes changed while freezing".into())); + } + set_str(g, FROZEN, fc.to_json()); + Ok(()) + }); + let msg = match r { + Ok(()) => format!("Roto Brush: froze {} frame(s) in {:.1} s", fc.frames.len(), st.elapsed), + Err(e) => format!("Roto Brush: {e}"), + }; + self.events.push(Event::Toast { message: msg, error: false }); + } + } + true + } + + /// Frontends call this every frame: finishes jobs and (with `auto`) propagates edited + /// instances in the background. + pub fn poll_roto(&mut self, auto: bool) -> bool { + let changed = self.poll_roto_job(); + if auto && self.roto_job.is_none() && self.state.roto.auto_propagate { + while let Some((c, l, e)) = self.roto_pending.first().copied() { + self.roto_pending.remove(0); + if self.start_roto(c, l, e, RotoTask::Propagate, Direction::Both, false).is_ok() { + return true; + } + } + } + changed + } + + /// The computed frames of an instance (full resolution), by layer frame. + pub fn roto_computed(&self, comp: ItemId, layer: LayerId, effect: Uid) -> BTreeMap { + let mut out = BTreeMap::new(); + let Some(c) = self.project.comp(comp) else { return out }; + let Some(l) = c.layer(layer) else { return out }; + let Some(g) = l.props.find_group(effect) else { return out }; + let ps = params_static(g); + let ch = fx::Chain::new(&ps, layer_size(&self.project, c, l), c.frame_rate.as_f64(), 1.0); + for f in ch.keys.keys() { + out.insert(*f, ch.is_cached(*f, 1.0)); + } + out + } +} diff --git a/crates/engine/src/tests_roto.rs b/crates/engine/src/tests_roto.rs new file mode 100644 index 00000000..79403626 --- /dev/null +++ b/crates/engine/src/tests_roto.rs @@ -0,0 +1,288 @@ +//! Roto Brush & Refine Edge end to end through the commands, on synthetic footage: a textured +//! disk moving over a textured background. + +use effectcraft_project::LayerId; +use effectcraft_raster::Image; +use effectcraft_render::RenderOpts; +use effectcraft_track::roto::rle; +use serde_json::{Value, json}; + +use crate::Session; +use crate::tests_track::{H, W, frame_time, setup}; + +const R: f64 = 40.0; + +fn noise(x: f64, y: f64, seed: u32) -> f64 { + fn h(i: i64, j: i64, s: u32) -> f64 { + let mut v = (i.wrapping_mul(73856093) ^ j.wrapping_mul(19349663) ^ (s as i64).wrapping_mul(83492791)) as u64; + v ^= v >> 13; + v = v.wrapping_mul(0x5bd1e995); + v ^= v >> 15; + (v % 10_000) as f64 / 10_000.0 + } + let mut out = 0.0; + for (cell, amp) in [(9.0, 0.6), (3.5, 0.4)] { + let (fx, fy) = (x / cell, y / cell); + let (i, j) = (fx.floor() as i64, fy.floor() as i64); + let (tx, ty) = (fx - i as f64, fy - j as f64); + let (sx, sy) = (tx * tx * (3.0 - 2.0 * tx), ty * ty * (3.0 - 2.0 * ty)); + let a = h(i, j, seed) + (h(i + 1, j, seed) - h(i, j, seed)) * sx; + let b = h(i, j + 1, seed) + (h(i + 1, j + 1, seed) - h(i, j + 1, seed)) * sx; + out += amp * (a + (b - a) * sy); + } + out +} + +/// Disk centre on frame `f`. +fn center(f: u32) -> [f64; 2] { + let t = f as f64; + [90.0 + 4.0 * t, 110.0 + 1.5 * t + 5.0 * (t * 0.3).sin()] +} + +/// Disk centre with a jump at frame 8 (propagation loses it there). +fn jumping(f: u32) -> [f64; 2] { + if f < 8 { center(f) } else { [230.0 + 2.0 * (f - 8) as f64, 150.0] } +} + +fn draw(c: [f64; 2]) -> Image { + let mut img = Image::new(W, H); + for y in 0..H { + for x in 0..W { + let p = [x as f64 + 0.5, y as f64 + 0.5]; + let d = (p[0] - c[0]).hypot(p[1] - c[1]); + let a = (R + 0.5 - d).clamp(0.0, 1.0) as f32; + let n = noise(p[0] - c[0], p[1] - c[1], 11); + let f = [(0.75 + 0.25 * n) as f32, (0.35 + 0.3 * n) as f32, (0.1 + 0.15 * n) as f32]; + let (bn, bm) = (noise(p[0], p[1], 5), noise(p[0] * 1.7, p[1] * 1.7, 9)); + let b = [(0.05 + 0.25 * bm) as f32, (0.3 + 0.4 * bn) as f32, (0.45 + 0.4 * bm) as f32]; + img.set(x, y, [f[0] * a + b[0] * (1.0 - a), f[1] * a + b[1] * (1.0 - a), f[2] * a + b[2] * (1.0 - a), 1.0]); + } + } + img +} + +fn gt(c: [f64; 2]) -> String { + let m: Vec = (0..W * H).map(|i| (((i % W) as f64 + 0.5 - c[0]).hypot((i / W) as f64 + 0.5 - c[1]) <= R) as u8).collect(); + rle::encode(&m) +} + +fn line(a: [f64; 2], b: [f64; 2]) -> Value { + json!((0..=8).map(|k| [a[0] + (b[0] - a[0]) * k as f64 / 8.0, a[1] + (b[1] - a[1]) * k as f64 / 8.0]).collect::>()) +} + +/// A session with the clip (the Track solid removed) and foreground / background strokes on +/// frame 0. `name`: the footage file name (segmentations are cached process-wide by content key, +/// which includes the footage's identity: different clips need different names). +fn painted(path: fn(u32) -> [f64; 2], name: &str) -> (Session, LayerId, u64) { + let (mut s, clip, solid) = setup(move |f| draw(path(f))); + let cid = s.active_comp_id().unwrap(); + s.edit("remove solid", None, |p, _| { + p.comp_mut(cid).unwrap().layers.retain(|l| l.id != solid); + for it in p.items.values_mut() { + if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind { + f.path = name.into(); + } + } + Ok(()) + }) + .unwrap(); + s.execute("time.set", json!({"time": 0})).unwrap(); + let c = path(0); + let r = + s.execute_checked("roto.stroke", json!({"layer": clip.0, "points": line([c[0] - 22.0, c[1] - 8.0], [c[0] + 22.0, c[1] + 8.0]), "radius": 6})).unwrap(); + let uid = r["effect"].as_u64().unwrap(); + assert_eq!(r["base"], 0); + assert_eq!(r["span"], json!([0, 20])); + for (a, b) in [([c[0] - 60.0, c[1] - 58.0], [c[0] + 70.0, c[1] - 58.0]), ([c[0] + 70.0, c[1] + 58.0], [c[0] - 60.0, c[1] + 58.0])] { + s.execute_checked("roto.stroke", json!({"layer": clip.0, "kind": "bg", "points": line(a, b), "radius": 6})).unwrap(); + } + (s, clip, uid) +} + +/// Drop an instance's derived segmentations from the process-wide cache. +fn forget(s: &Session, clip: LayerId, uid: u64) { + let comp = s.active_comp().unwrap(); + let l = comp.layer(clip).unwrap(); + let g = l.props.find_group(uid).unwrap(); + let size = crate::roto::layer_size(&s.project, comp, l); + let ch = effectcraft_effects::roto::Chain::new(&crate::roto::params_static(g), size, comp.frame_rate.as_f64(), 1.0); + effectcraft_effects::roto::forget(ch.keys.values().copied()); +} + +fn iou_at(s: &mut Session, clip: LayerId, f: u32, c: [f64; 2]) -> f64 { + let st = s.execute("roto.status", json!({"layer": clip.0, "frame": f, "compute": true, "compareTo": gt(c)})).unwrap(); + st["iou"].as_f64().unwrap_or_else(|| panic!("no matte on frame {f}: {st}")) +} + +fn alpha_at(s: &Session, f: u32) -> Vec { + let img = s.render(s.active_comp_id().unwrap(), frame_time(f), RenderOpts { scale: 1.0, ..Default::default() }); + img.data.iter().map(|p| p[3]).collect() +} + +#[test] +fn roto_brush_segments_and_propagates() { + let (mut s, clip, uid) = painted(center, "disk-a.mov"); + let l = s.active_comp().unwrap().layer(clip).unwrap().clone(); + assert_eq!(l.effects().unwrap().groups().filter(|g| crate::roto::is_roto(g)).count(), 1, "one effect for all strokes"); + let j0 = iou_at(&mut s, clip, 0, center(0)); + assert!(j0 > 0.95, "base frame IoU {j0}"); + // Propagate the span (blocking). + let r = s.execute_checked("roto.propagate", json!({"layer": clip.0, "direction": "both", "wait": true})).unwrap(); + assert_eq!(r["frames"], 21, "{r}"); + assert_eq!(r["status"]["computed"].as_array().unwrap().len(), 21); + let mut worst: f64 = 1.0; + for f in 0..=20 { + worst = worst.min(iou_at(&mut s, clip, f, center(f))); + } + assert!(worst > 0.9, "worst IoU over the span {worst}"); + // The render applies the matte; outside the span the layer is transparent. + let a = alpha_at(&s, 12); + let c = center(12); + let mut wrong = 0; + for (i, v) in a.iter().enumerate() { + let d = (((i as u32 % W) as f64 + 0.5 - c[0]).hypot((i as u32 / W) as f64 + 0.5 - c[1])) - R; + if (d < -3.0 && *v < 0.5) || (d > 3.0 && *v > 0.5) { + wrong += 1; + } + } + assert!(wrong < 150, "{wrong} wrong pixels in the render"); + assert!(alpha_at(&s, 23).iter().all(|v| *v == 0.0)); + // Extending the span propagates further. + let r = s.execute("roto.propagate", json!({"layer": clip.0, "direction": "forward", "to": 24, "wait": true})).unwrap(); + assert_eq!(r["status"]["span"], json!([0, 24])); + assert!(iou_at(&mut s, clip, 24, center(24)) > 0.9); + // Undo / redo of strokes and the span. + let n = |s: &Session| { + let st = s.active_comp().unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().get(effectcraft_effects::roto::STROKES).unwrap().value.clone(); + match st { + effectcraft_keyframe::Value::Str(j) => effectcraft_track::roto::RotoData::from_json(&j), + _ => panic!(), + } + }; + assert_eq!(n(&s).span, [0, 24]); + assert!(s.undo()); + assert_eq!(n(&s).span, [0, 20]); + assert_eq!(n(&s).strokes.len(), 3); + assert!(s.undo()); + assert_eq!(n(&s).strokes.len(), 2); + assert!(s.redo()); + assert!(s.redo()); + assert_eq!(n(&s).strokes.len(), 3); + assert_eq!(n(&s).span, [0, 24]); + // Serde keeps the strokes; the reopened project renders the same matte. + let back = effectcraft_project::Project::from_json(&s.project.to_json()).unwrap(); + let g = back.comp(s.active_comp_id().unwrap()).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().clone(); + assert_eq!( + crate::roto::params_static(&g).get("strokes"), + crate::roto::params_static(s.active_comp().unwrap().layer(clip).unwrap().props.find_group(uid).unwrap()).get("strokes") + ); + // Bad parameters are reported. + assert!(s.execute_checked("roto.stroke", json!({"layer": clip.0, "kind": "sideways", "points": [[1, 1]]})).is_err()); + assert!(s.execute_checked("roto.propagate", json!({"layer": clip.0, "bogus": 1})).is_err()); + assert!(s.execute_checked("roto.stroke", json!({"layer": clip.0, "points": [[1, 1]], "frame": 99})).is_err()); +} + +#[test] +fn correction_strokes_restart_propagation() { + let (mut s, clip, _) = painted(jumping, "disk-jump.mov"); + s.execute("roto.propagate", json!({"layer": clip.0, "wait": true})).unwrap(); + assert!(iou_at(&mut s, clip, 6, jumping(6)) > 0.9); + let bad = iou_at(&mut s, clip, 8, jumping(8)); + assert!(bad < 0.5, "the jump should break propagation ({bad})"); + // Correct frame 8: paint the disk in and its old place out. + let (c, old) = (jumping(8), center(8)); + s.execute("roto.stroke", json!({"layer": clip.0, "frame": 8, "points": line([c[0] - 20.0, c[1]], [c[0] + 20.0, c[1]]), "radius": 6})).unwrap(); + s.execute( + "roto.stroke", + json!({"layer": clip.0, "frame": 8, "kind": "bg", "points": line([old[0] - 15.0, old[1] - 15.0], [old[0] + 15.0, old[1] + 15.0]), "radius": 8}), + ) + .unwrap(); + let st = s.execute("roto.status", json!({"layer": clip.0})).unwrap(); + let computed: Vec = serde_json::from_value(st["computed"].clone()).unwrap(); + assert!(computed.contains(&7) && !computed.contains(&8) && !computed.contains(&9), "frames from the correction on are recomputed: {computed:?}"); + assert_eq!(st["strokeFrames"], json!([0, 8])); + s.execute("roto.propagate", json!({"layer": clip.0, "direction": "forward", "wait": true})).unwrap(); + for f in 8..=20 { + let j = iou_at(&mut s, clip, f, jumping(f)); + assert!(j > 0.9, "frame {f}: IoU {j}"); + } + // Clearing the correction brings the bad frame back. + let r = s.execute("roto.clearStrokes", json!({"layer": clip.0, "frame": 8})).unwrap(); + assert_eq!(r["removed"], 2); + assert!(iou_at(&mut s, clip, 8, jumping(8)) < 0.5); +} + +#[test] +fn freeze_caches_the_mattes() { + let (mut s, clip, uid) = painted(center, "disk-freeze.mov"); + // A refine band along the edge on the base frame. + let c = center(0); + let ring: Vec<[f64; 2]> = (0..=40).map(|k| k as f64 / 40.0 * std::f64::consts::TAU).map(|a| [c[0] + R * a.cos(), c[1] + R * a.sin()]).collect(); + s.execute_checked("roto.stroke", json!({"layer": clip.0, "kind": "refine", "points": ring, "radius": 5})).unwrap(); + s.execute("roto.span", json!({"layer": clip.0, "end": 3})).unwrap(); + let live: Vec> = (0..=3).map(|f| alpha_at(&s, f)).collect(); + // Freeze in the background and wait for it. + let r = s.execute_checked("roto.freeze", json!({"layer": clip.0})).unwrap(); + assert_eq!(r["frames"], 4); + let t0 = std::time::Instant::now(); + while s.roto_job.is_some() { + assert!(t0.elapsed().as_secs() < 300); + std::thread::sleep(std::time::Duration::from_millis(5)); + s.poll_roto(false); + } + let st = s.execute("roto.status", json!({"layer": clip.0})).unwrap(); + assert_eq!(st["frozen"], true, "{st}"); + // Frozen renders match the live ones (8-bit alpha), even without the derived segmentations. + forget(&s, clip, uid); + s.layer_cache = Default::default(); + for (f, want) in live.iter().enumerate() { + let got = alpha_at(&s, f as u32); + let worst = got.iter().zip(want).map(|(a, b)| (a - b).abs()).fold(0.0f32, f32::max); + assert!(worst <= 1.0 / 255.0 + 1e-4, "frame {f}: {worst}"); + } + assert!(st["computed"].as_array().unwrap().len() >= 4); + // Frozen effects refuse strokes; Unfreeze is undoable. + assert!(s.execute("roto.stroke", json!({"layer": clip.0, "points": [[10, 10]]})).is_err()); + s.execute("roto.unfreeze", json!({"layer": clip.0})).unwrap(); + assert_eq!(s.execute("roto.status", json!({"layer": clip.0})).unwrap()["frozen"], false); + assert!(s.undo()); + assert_eq!(s.execute("roto.status", json!({"layer": clip.0})).unwrap()["frozen"], true); + // Changing the source frames (the layer's timing) drops the frozen mattes. + let cid = s.active_comp_id().unwrap(); + s.edit("shift", None, |p, _| { + let l = p.comp_mut(cid).unwrap().layer_mut(clip).unwrap(); + l.start_time = frame_time(1); + Ok(()) + }) + .unwrap(); + assert_eq!(s.execute("roto.status", json!({"layer": clip.0, "effect": uid})).unwrap()["frozen"], false); +} + +#[test] +fn propagation_is_deterministic_and_runs_in_background() { + let (mut s, clip, uid) = painted(center, "disk-bg.mov"); + s.execute("roto.span", json!({"layer": clip.0, "end": 5})).unwrap(); + s.execute("roto.propagate", json!({"layer": clip.0})).unwrap(); + let t0 = std::time::Instant::now(); + while s.roto_job.is_some() { + assert!(t0.elapsed().as_secs() < 300); + std::thread::sleep(std::time::Duration::from_millis(5)); + s.poll_roto(false); + } + let st = s.execute("roto.status", json!({"layer": clip.0})).unwrap(); + assert_eq!(st["computed"], json!([0, 1, 2, 3, 4, 5]), "{st}"); + let a = s.execute("roto.status", json!({"layer": clip.0, "frame": 5, "matte": true})).unwrap()["matte"].clone(); + // Recomputing from scratch gives the same matte. + forget(&s, clip, uid); + let b = s.execute("roto.status", json!({"layer": clip.0, "frame": 5, "matte": true})).unwrap()["matte"].clone(); + assert!(a.is_string()); + assert_eq!(a, b); + // Tool options. + let o = s.execute_checked("roto.options", json!({"diameter": 30, "view": "alphaOverlay"})).unwrap(); + assert_eq!(o["view"], "alphaOverlay"); + assert_eq!(s.state.roto.diameter, 30.0); + assert!(s.execute("roto.options", json!({"view": "sideways"})).is_err()); + // The default radius follows the tool diameter. + let r = s.execute("roto.stroke", json!({"layer": clip.0, "points": [[90, 110]]})).unwrap(); + assert_eq!(r["radius"], 15.0); +} diff --git a/crates/track/Cargo.toml b/crates/track/Cargo.toml index ba449c92..70e08ae0 100644 --- a/crates/track/Cargo.toml +++ b/crates/track/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-track" -description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, and corner-pin / transform solves" +description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, corner-pin / transform solves, stabilization and Roto Brush segmentation (graph cut, matting)" version.workspace = true edition.workspace = true license.workspace = true diff --git a/crates/track/src/lib.rs b/crates/track/src/lib.rs index d7b239f5..629097cf 100644 --- a/crates/track/src/lib.rs +++ b/crates/track/src/lib.rs @@ -22,6 +22,7 @@ pub mod fit; pub mod klt; pub mod mask; pub mod plane; +pub mod roto; pub mod solve; pub mod stabilize; diff --git a/crates/track/src/roto/gmm.rs b/crates/track/src/roto/gmm.rs new file mode 100644 index 00000000..c0fe95b0 --- /dev/null +++ b/crates/track/src/roto/gmm.rs @@ -0,0 +1,228 @@ +//! Gaussian mixture colour models (full 3 × 3 covariances), as used by GrabCut (C. Rother, +//! V. Kolmogorov and A. Blake, "GrabCut: Interactive Foreground Extraction using Iterated Graph +//! Cuts", SIGGRAPH 2004). +//! +//! Fitting is deterministic: components are initialised by recursive splitting of the cluster +//! with the largest variance along its principal axis (Orchard & Bouman, "Color Quantization of +//! Images", IEEE TSP 1991), then refined by a few hard-assignment EM iterations. + +use serde::{Deserialize, Serialize}; + +/// One Gaussian: weight, mean, inverse covariance and log normaliser. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Gauss { + pub weight: f64, + pub mean: [f64; 3], + pub inv: [[f64; 3]; 3], + /// −log(weight) + ½·log(det Σ) (the 2π term is dropped: it cancels between models). + pub log_norm: f64, +} + +/// A colour model. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Gmm { + pub comps: Vec, +} + +/// Regularisation added to every covariance's diagonal (colour noise floor). +const REG: f64 = 2e-4; + +fn det3(m: &[[f64; 3]; 3]) -> f64 { + m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0]) + m[0][2] * (m[1][0] * m[2][1] - m[1][1] * m[2][0]) +} + +fn inv3(m: &[[f64; 3]; 3]) -> Option<[[f64; 3]; 3]> { + let d = det3(m); + if d.abs() < 1e-30 { + return None; + } + let mut r = [[0.0; 3]; 3]; + r[0][0] = (m[1][1] * m[2][2] - m[1][2] * m[2][1]) / d; + r[0][1] = (m[0][2] * m[2][1] - m[0][1] * m[2][2]) / d; + r[0][2] = (m[0][1] * m[1][2] - m[0][2] * m[1][1]) / d; + r[1][0] = (m[1][2] * m[2][0] - m[1][0] * m[2][2]) / d; + r[1][1] = (m[0][0] * m[2][2] - m[0][2] * m[2][0]) / d; + r[1][2] = (m[0][2] * m[1][0] - m[0][0] * m[1][2]) / d; + r[2][0] = (m[1][0] * m[2][1] - m[1][1] * m[2][0]) / d; + r[2][1] = (m[0][1] * m[2][0] - m[0][0] * m[2][1]) / d; + r[2][2] = (m[0][0] * m[1][1] - m[0][1] * m[1][0]) / d; + Some(r) +} + +/// Sufficient statistics of a cluster. +#[derive(Clone, Copy, Default)] +struct Stats { + n: f64, + s: [f64; 3], + ss: [[f64; 3]; 3], +} + +impl Stats { + fn add(&mut self, c: [f32; 3]) { + let c = [c[0] as f64, c[1] as f64, c[2] as f64]; + self.n += 1.0; + for i in 0..3 { + self.s[i] += c[i]; + for j in 0..3 { + self.ss[i][j] += c[i] * c[j]; + } + } + } + fn mean(&self) -> [f64; 3] { + let n = self.n.max(1.0); + [self.s[0] / n, self.s[1] / n, self.s[2] / n] + } + fn cov(&self) -> [[f64; 3]; 3] { + let n = self.n.max(1.0); + let m = self.mean(); + let mut c = [[0.0; 3]; 3]; + for i in 0..3 { + for j in 0..3 { + c[i][j] = self.ss[i][j] / n - m[i] * m[j]; + } + c[i][i] += REG; + } + c + } +} + +/// Principal eigenvector and eigenvalue of a symmetric 3 × 3 matrix (power iteration). +fn principal(c: &[[f64; 3]; 3]) -> ([f64; 3], f64) { + let mut v = [0.577, 0.577, 0.577]; + let mut lambda = 0.0; + for _ in 0..40 { + let w = [ + c[0][0] * v[0] + c[0][1] * v[1] + c[0][2] * v[2], + c[1][0] * v[0] + c[1][1] * v[1] + c[1][2] * v[2], + c[2][0] * v[0] + c[2][1] * v[1] + c[2][2] * v[2], + ]; + let n = (w[0] * w[0] + w[1] * w[1] + w[2] * w[2]).sqrt(); + if n < 1e-30 { + break; + } + lambda = n; + v = [w[0] / n, w[1] / n, w[2] / n]; + } + (v, lambda) +} + +impl Gmm { + /// Fit `k` components to `samples` (at most `max_samples` are used, taken at a fixed stride). + pub fn fit(samples: &[[f32; 3]], k: usize, max_samples: usize) -> Gmm { + if samples.is_empty() { + return Gmm::default(); + } + let stride = samples.len().div_ceil(max_samples.max(1)).max(1); + let xs: Vec<[f32; 3]> = samples.iter().step_by(stride).copied().collect(); + // Orchard–Bouman splitting. + let mut label = vec![0usize; xs.len()]; + let mut ncl = 1; + while ncl < k { + let mut st = vec![Stats::default(); ncl]; + for (x, l) in xs.iter().zip(&label) { + st[*l].add(*x); + } + let (mut best, mut best_l, mut axis) = (usize::MAX, 0.0, [0.0; 3]); + for (c, s) in st.iter().enumerate() { + if s.n < 2.0 { + continue; + } + let (v, l) = principal(&s.cov()); + if l > best_l { + best = c; + best_l = l; + axis = v; + } + } + if best == usize::MAX || best_l <= REG * 1.01 { + break; + } + let m = st[best].mean(); + let thr = axis[0] * m[0] + axis[1] * m[1] + axis[2] * m[2]; + for (x, l) in xs.iter().zip(label.iter_mut()) { + if *l == best && axis[0] * x[0] as f64 + axis[1] * x[1] as f64 + axis[2] * x[2] as f64 > thr { + *l = ncl; + } + } + ncl += 1; + } + let mut g = Gmm::from_labels(&xs, &label, ncl); + // Hard EM. + for _ in 0..3 { + for (x, l) in xs.iter().zip(label.iter_mut()) { + *l = g.best(*x); + } + let ng = Gmm::from_labels(&xs, &label, g.comps.len()); + if ng.comps.is_empty() { + break; + } + g = ng; + } + g + } + + fn from_labels(xs: &[[f32; 3]], label: &[usize], k: usize) -> Gmm { + let mut st = vec![Stats::default(); k]; + for (x, l) in xs.iter().zip(label) { + st[*l].add(*x); + } + let total: f64 = st.iter().map(|s| s.n).sum::().max(1.0); + let comps = st + .iter() + .filter(|s| s.n >= 1.0) + .filter_map(|s| { + let c = s.cov(); + let inv = inv3(&c)?; + let det = det3(&c).max(1e-300); + let weight = s.n / total; + Some(Gauss { weight, mean: s.mean(), inv, log_norm: -weight.ln() + 0.5 * det.ln() }) + }) + .collect(); + Gmm { comps } + } + + #[inline] + fn comp_cost(g: &Gauss, c: [f32; 3]) -> f64 { + let d = [c[0] as f64 - g.mean[0], c[1] as f64 - g.mean[1], c[2] as f64 - g.mean[2]]; + let mut q = 0.0; + for i in 0..3 { + q += d[i] * (g.inv[i][0] * d[0] + g.inv[i][1] * d[1] + g.inv[i][2] * d[2]); + } + g.log_norm + 0.5 * q + } + + /// The most likely component of colour `c`. + pub fn best(&self, c: [f32; 3]) -> usize { + let mut b = 0; + let mut bv = f64::INFINITY; + for (i, g) in self.comps.iter().enumerate() { + let v = Self::comp_cost(g, c); + if v < bv { + bv = v; + b = i; + } + } + b + } + + /// −log p(c) (up to a constant shared by all models), clamped to a finite range. + pub fn cost(&self, c: [f32; 3]) -> f64 { + if self.comps.is_empty() { + return 10.0; + } + // log-sum-exp over components. + let mut m = f64::INFINITY; + let mut costs = [0.0f64; 16]; + let n = self.comps.len().min(16); + for (i, g) in self.comps.iter().take(n).enumerate() { + costs[i] = Self::comp_cost(g, c); + m = m.min(costs[i]); + } + let s: f64 = costs[..n].iter().map(|v| (m - v).exp()).sum(); + (m - s.ln()).clamp(-20.0, 60.0) + } + + pub fn is_empty(&self) -> bool { + self.comps.is_empty() + } +} diff --git a/crates/track/src/roto/matting.rs b/crates/track/src/roto/matting.rs new file mode 100644 index 00000000..439d9c43 --- /dev/null +++ b/crates/track/src/roto/matting.rs @@ -0,0 +1,465 @@ +//! Matte utilities: exact Euclidean distance transforms (Felzenszwalb & Huttenlocher, +//! "Distance Transforms of Sampled Functions", Theory of Computing 8, 2012), box and Gaussian +//! filters, the colour **guided filter** (K. He, J. Sun and X. Tang, "Guided Image Filtering", +//! ECCV 2010 / IEEE PAMI 2013) used for edge matting, foreground colour estimation +//! (decontamination, from the compositing equation I = αF + (1 − α)B with B and F estimated from +//! nearby pure background / foreground pixels), and the Refine Edge matte adjustments. + +use rayon::prelude::*; + +const BIG: f32 = 1e12; + +/// 1-D squared distance transform of `f` (lower envelope of parabolas). +fn dt1(f: &[f32], d: &mut [f32], v: &mut [usize], z: &mut [f64]) { + let n = f.len(); + if n == 0 { + return; + } + let ff = |q: usize| f[q] as f64 + (q * q) as f64; + let mut k = 0usize; + v[0] = 0; + z[0] = f64::NEG_INFINITY; + z[1] = f64::INFINITY; + for q in 1..n { + let mut s = (ff(q) - ff(v[k])) / (2.0 * (q as f64 - v[k] as f64)); + while s <= z[k] { + if k == 0 { + break; + } + k -= 1; + s = (ff(q) - ff(v[k])) / (2.0 * (q as f64 - v[k] as f64)); + } + k += 1; + v[k] = q; + z[k] = s; + z[k + 1] = f64::INFINITY; + } + k = 0; + for (q, dq) in d.iter_mut().enumerate().take(n) { + while z[k + 1] < q as f64 { + k += 1; + } + let p = v[k]; + let dd = q as f64 - p as f64; + *dq = (dd * dd + f[p] as f64) as f32; + } +} + +/// Euclidean distance from every pixel to the nearest pixel where `seed` is true (BIG.sqrt() +/// when there is none). +pub fn distance(seed: &[bool], w: usize, h: usize) -> Vec { + if w == 0 || h == 0 { + return vec![]; + } + let mut g: Vec = seed.iter().map(|s| if *s { 0.0 } else { BIG }).collect(); + // Columns. + let cols: Vec> = (0..w) + .into_par_iter() + .map(|x| { + let f: Vec = (0..h).map(|y| g[y * w + x]).collect(); + let mut d = vec![0.0; h]; + let (mut v, mut z) = (vec![0usize; h], vec![0.0f64; h + 1]); + dt1(&f, &mut d, &mut v, &mut z); + d + }) + .collect(); + for (x, c) in cols.iter().enumerate() { + for y in 0..h { + g[y * w + x] = c[y]; + } + } + // Rows. + g.par_chunks_mut(w).for_each(|row| { + let f = row.to_vec(); + let (mut v, mut z) = (vec![0usize; w], vec![0.0f64; w + 1]); + dt1(&f, row, &mut v, &mut z); + }); + g.par_iter_mut().for_each(|v| *v = v.sqrt()); + g +} + +/// Signed distance to a binary matte's edge: positive inside (distance to the nearest +/// background pixel centre, minus ½), negative outside. +pub fn signed_distance(m: &[u8], w: usize, h: usize) -> Vec { + let fg: Vec = m.iter().map(|v| *v != 0).collect(); + let bg: Vec = m.iter().map(|v| *v == 0).collect(); + let din = distance(&bg, w, h); + let dout = distance(&fg, w, h); + m.iter().enumerate().map(|(i, v)| if *v != 0 { din[i] - 0.5 } else { 0.5 - dout[i] }).collect() +} + +/// Mean over the (2r+1)² window (clamped at the edges: averages only the pixels inside). +pub fn box_mean(src: &[f32], w: usize, h: usize, r: usize) -> Vec { + if w == 0 || h == 0 { + return vec![]; + } + // Horizontal window sums per row. + let mut tmp = vec![0.0f64; w * h]; + tmp.par_chunks_mut(w).enumerate().for_each(|(y, row)| { + let s = &src[y * w..y * w + w]; + let mut acc = 0.0f64; + for x in 0..r.min(w) { + acc += s[x] as f64; + } + for (x, o) in row.iter_mut().enumerate() { + if x + r < w { + acc += s[x + r] as f64; + } + if x > r { + acc -= s[x - r - 1] as f64; + } + *o = acc; + } + }); + // Vertical prefix sums (row by row, cache friendly), then window differences. + let mut pre = vec![0.0f64; w * (h + 1)]; + for y in 0..h { + let (done, rest) = pre.split_at_mut((y + 1) * w); + let prev = &done[y * w..]; + let cur = &mut rest[..w]; + let t = &tmp[y * w..y * w + w]; + for x in 0..w { + cur[x] = prev[x] + t[x]; + } + } + let mut out = vec![0.0f32; w * h]; + out.par_chunks_mut(w).enumerate().for_each(|(y, row)| { + let a = y.saturating_sub(r); + let b = (y + r + 1).min(h); + let cy = (b - a) as f64; + for (x, o) in row.iter_mut().enumerate() { + let cx = ((x + r + 1).min(w) - x.saturating_sub(r)) as f64; + *o = ((pre[b * w + x] - pre[a * w + x]) / (cx * cy)) as f32; + } + }); + out +} + +/// Approximate Gaussian blur (three box passes) with standard deviation `sigma` pixels. +pub fn gauss(src: &[f32], w: usize, h: usize, sigma: f64) -> Vec { + if sigma < 0.3 { + return src.to_vec(); + } + // Box radius whose three passes match the variance: 3·((2r+1)² − 1)/12 = σ². + let r = (((4.0 * sigma * sigma + 1.0).sqrt() - 1.0) / 2.0).round().max(1.0) as usize; + let a = box_mean(src, w, h, r); + let b = box_mean(&a, w, h, r); + box_mean(&b, w, h, r) +} + +fn inv3(m: [[f32; 3]; 3]) -> [[f32; 3]; 3] { + let d = m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0]) + + m[0][2] * (m[1][0] * m[2][1] - m[1][1] * m[2][0]); + let d = if d.abs() < 1e-20 { 1e-20 } else { d }; + [ + [(m[1][1] * m[2][2] - m[1][2] * m[2][1]) / d, (m[0][2] * m[2][1] - m[0][1] * m[2][2]) / d, (m[0][1] * m[1][2] - m[0][2] * m[1][1]) / d], + [(m[1][2] * m[2][0] - m[1][0] * m[2][2]) / d, (m[0][0] * m[2][2] - m[0][2] * m[2][0]) / d, (m[0][2] * m[1][0] - m[0][0] * m[1][2]) / d], + [(m[1][0] * m[2][1] - m[1][1] * m[2][0]) / d, (m[0][1] * m[2][0] - m[0][0] * m[2][1]) / d, (m[0][0] * m[1][1] - m[0][1] * m[1][0]) / d], + ] +} + +/// The colour guided filter of `p` with guide `img` (RGB, w × h), window radius `r`, +/// regularisation `eps` (He et al. 2010, section 3.5). +pub fn guided_filter(img: &[[f32; 3]], p: &[f32], w: usize, h: usize, r: usize, eps: f32) -> Vec { + let n = w * h; + let ch = |c: usize| -> Vec { img.iter().map(|v| v[c]).collect() }; + let (ir, ig, ib) = (ch(0), ch(1), ch(2)); + let mul = |a: &[f32], b: &[f32]| -> Vec { a.iter().zip(b).map(|(x, y)| x * y).collect() }; + let bm = |v: &[f32]| box_mean(v, w, h, r); + let (mr, mg, mb, mp) = (bm(&ir), bm(&ig), bm(&ib), bm(p)); + let (mrp, mgp, mbp) = (bm(&mul(&ir, p)), bm(&mul(&ig, p)), bm(&mul(&ib, p))); + let (vrr, vrg, vrb) = (bm(&mul(&ir, &ir)), bm(&mul(&ir, &ig)), bm(&mul(&ir, &ib))); + let (vgg, vgb, vbb) = (bm(&mul(&ig, &ig)), bm(&mul(&ig, &ib)), bm(&mul(&ib, &ib))); + let mut a = vec![[0.0f32; 3]; n]; + let mut b = vec![0.0f32; n]; + a.par_iter_mut().zip(b.par_iter_mut()).enumerate().for_each(|(i, (ai, bi))| { + let cov = [mrp[i] - mr[i] * mp[i], mgp[i] - mg[i] * mp[i], mbp[i] - mb[i] * mp[i]]; + let s = [ + [vrr[i] - mr[i] * mr[i] + eps, vrg[i] - mr[i] * mg[i], vrb[i] - mr[i] * mb[i]], + [vrg[i] - mr[i] * mg[i], vgg[i] - mg[i] * mg[i] + eps, vgb[i] - mg[i] * mb[i]], + [vrb[i] - mr[i] * mb[i], vgb[i] - mg[i] * mb[i], vbb[i] - mb[i] * mb[i] + eps], + ]; + let inv = inv3(s); + let av = [ + cov[0] * inv[0][0] + cov[1] * inv[1][0] + cov[2] * inv[2][0], + cov[0] * inv[0][1] + cov[1] * inv[1][1] + cov[2] * inv[2][1], + cov[0] * inv[0][2] + cov[1] * inv[1][2] + cov[2] * inv[2][2], + ]; + *ai = av; + *bi = mp[i] - av[0] * mr[i] - av[1] * mg[i] - av[2] * mb[i]; + }); + let (a0, a1, a2): (Vec, Vec, Vec) = (a.iter().map(|v| v[0]).collect(), a.iter().map(|v| v[1]).collect(), a.iter().map(|v| v[2]).collect()); + let (ma0, ma1, ma2, mbb) = (bm(&a0), bm(&a1), bm(&a2), bm(&b)); + (0..n).into_par_iter().map(|i| ma0[i] * ir[i] + ma1[i] * ig[i] + ma2[i] * ib[i] + mbb[i]).collect() +} + +/// Closed-form matting (A. Levin, D. Lischinski and Y. Weiss, "A Closed-Form Solution to Natural +/// Image Matting", IEEE PAMI 30(2), 2008): the alpha that minimises αᵀLα with the matting +/// Laplacian `L` of the colour image, with `known` pixels fixed to `init`. Solved by conjugate +/// gradients over the unknown pixels, starting from `init`; `L·x` is evaluated with box filters +/// (K. He, J. Sun and X. Tang, "Fast Matting Using Large Kernel Matting Laplacian Matrices", +/// CVPR 2010): `(L·x)ᵢ = Σ_{k∋i} (xᵢ − a_kᵀIᵢ − b_k)` with `(a_k, b_k)` the ridge-regression +/// fit of `x` to the colours in window k. +pub fn closed_form(img: &[[f32; 3]], init: &[f32], known: &[bool], w: usize, h: usize, r: usize, eps: f32, iters: usize) -> Vec { + let n = w * h; + if n == 0 || known.iter().all(|k| *k) { + return init.to_vec(); + } + let ch = |c: usize| -> Vec { img.iter().map(|v| v[c]).collect() }; + let ic = [ch(0), ch(1), ch(2)]; + let bm = |v: &[f32]| box_mean(v, w, h, r); + let mu = [bm(&ic[0]), bm(&ic[1]), bm(&ic[2])]; + let prod = |a: usize, b: usize| -> Vec { ic[a].iter().zip(&ic[b]).map(|(x, y)| x * y).collect() }; + let mut cov = [[vec![], vec![], vec![]], [vec![], vec![], vec![]], [vec![], vec![], vec![]]]; + for a in 0..3 { + for b in a..3 { + let m = bm(&prod(a, b)); + cov[a][b] = m.clone(); + cov[b][a] = m; + } + } + let inv: Vec<[[f32; 3]; 3]> = (0..n) + .into_par_iter() + .map(|i| { + let mut s = [[0.0f32; 3]; 3]; + for a in 0..3 { + for b in 0..3 { + s[a][b] = cov[a][b][i] - mu[a][i] * mu[b][i] + if a == b { eps } else { 0.0 }; + } + } + inv3(s) + }) + .collect(); + // Pixels per window (= windows per pixel). + let cnt: Vec = { + let (rr, mut out) = (r as i64, vec![0.0f32; n]); + for (i, o) in out.iter_mut().enumerate() { + let (x, y) = ((i % w) as i64, (i / w) as i64); + let cx = (x + rr).min(w as i64 - 1) - (x - rr).max(0) + 1; + let cy = (y + rr).min(h as i64 - 1) - (y - rr).max(0) + 1; + *o = (cx * cy) as f32; + } + out + }; + let lap = |x: &[f32]| -> Vec { + let pm = bm(x); + let ipm: Vec> = (0..3).map(|c| bm(&ic[c].iter().zip(x).map(|(a, b)| a * b).collect::>())).collect(); + let ab: Vec<[f32; 4]> = (0..n) + .into_par_iter() + .map(|i| { + let cv = [ipm[0][i] - mu[0][i] * pm[i], ipm[1][i] - mu[1][i] * pm[i], ipm[2][i] - mu[2][i] * pm[i]]; + let m = &inv[i]; + let a = [ + m[0][0] * cv[0] + m[0][1] * cv[1] + m[0][2] * cv[2], + m[1][0] * cv[0] + m[1][1] * cv[1] + m[1][2] * cv[2], + m[2][0] * cv[0] + m[2][1] * cv[1] + m[2][2] * cv[2], + ]; + [a[0], a[1], a[2], pm[i] - a[0] * mu[0][i] - a[1] * mu[1][i] - a[2] * mu[2][i]] + }) + .collect(); + let sa: Vec> = (0..3).map(|c| bm(&ab.iter().map(|v| v[c]).collect::>())).collect(); + let sb = bm(&ab.iter().map(|v| v[3]).collect::>()); + (0..n).into_par_iter().map(|i| cnt[i] * (x[i] - sa[0][i] * ic[0][i] - sa[1][i] * ic[1][i] - sa[2][i] * ic[2][i] - sb[i])).collect() + }; + // L_UU x = −(L α_K)_U. + let a0: Vec = (0..n).map(|i| if known[i] { init[i] } else { 0.0 }).collect(); + let la0 = lap(&a0); + let mut x: Vec = (0..n).map(|i| if known[i] { 0.0 } else { init[i] }).collect(); + let lx = lap(&x); + let mut res: Vec = (0..n).map(|i| if known[i] { 0.0 } else { (-la0[i] - lx[i]) as f64 }).collect(); + let mut p = res.clone(); + let mut rs: f64 = res.iter().map(|v| v * v).sum(); + let rs0 = rs.max(1e-30); + for _ in 0..iters { + if rs <= rs0 * 1e-8 { + break; + } + let pf: Vec = p.iter().map(|v| *v as f32).collect(); + let ap: Vec = lap(&pf).iter().enumerate().map(|(i, v)| if known[i] { 0.0 } else { *v as f64 }).collect(); + let pap: f64 = p.iter().zip(&ap).map(|(a, b)| a * b).sum(); + if pap <= 1e-30 { + break; + } + let alpha = rs / pap; + for i in 0..n { + x[i] += (alpha * p[i]) as f32; + res[i] -= alpha * ap[i]; + } + let rs_new: f64 = res.iter().map(|v| v * v).sum(); + let beta = rs_new / rs; + rs = rs_new; + for i in 0..n { + p[i] = res[i] + beta * p[i]; + } + } + (0..n).map(|i| if known[i] { init[i] } else { x[i].clamp(0.0, 1.0) }).collect() +} + +/// Contrast (0–100 %) around ½ on a soft alpha. +pub fn contrast(a: &mut [f32], pc: f64) { + if pc <= 0.0 { + return; + } + let k = 1.0 / (1.0 - 0.99 * (pc / 100.0).clamp(0.0, 1.0)) as f32; + a.par_iter_mut().for_each(|v| *v = (0.5 + (*v - 0.5) * k).clamp(0.0, 1.0)); +} + +/// Shift Edge (−100–100 %) on a soft alpha: positive values grow the matte (a gamma curve on α). +pub fn shift_soft(a: &mut [f32], pc: f64) { + if pc.abs() < 1e-9 { + return; + } + let g = 2f32.powf((-pc / 50.0) as f32); + a.par_iter_mut().for_each(|v| *v = v.clamp(0.0, 1.0).powf(g)); +} + +/// Estimate the pure foreground colour of every pixel listed in `region` (alpha strictly +/// between 0 and 1) and mix it in by `amount` (0–1). `radius`: neighbourhood (pixels) that pure +/// background / foreground estimates are taken from. Returns the decontamination map (how much +/// each pixel's colour changed, 0–1). +pub fn decontaminate(img: &mut [[f32; 3]], alpha: &[f32], region: &[bool], w: usize, h: usize, radius: usize, amount: f32) -> Vec { + let n = w * h; + let mut map = vec![0.0f32; n]; + if !region.iter().any(|r| *r) || amount <= 0.0 { + return map; + } + let lo: Vec = alpha.iter().map(|a| if *a < 0.02 { 1.0 } else { 0.0 }).collect(); + let hi: Vec = alpha.iter().map(|a| if *a > 0.98 { 1.0 } else { 0.0 }).collect(); + // Windowed means of pure background / foreground colours, widening where none is near. + let est = |m: &[f32]| -> Vec> { + let mut out: Vec> = vec![None; n]; + let mut r = radius.max(2); + for _ in 0..4 { + let cnt = box_mean(m, w, h, r); + let sums: Vec> = (0..3).map(|c| box_mean(&img.iter().zip(m).map(|(p, k)| p[c] * k).collect::>(), w, h, r)).collect(); + let mut missing = false; + for i in 0..n { + if region[i] && out[i].is_none() { + if cnt[i] > 1e-4 { + out[i] = Some([sums[0][i] / cnt[i], sums[1][i] / cnt[i], sums[2][i] / cnt[i]]); + } else { + missing = true; + } + } + } + if !missing { + break; + } + r *= 2; + } + out + }; + let bgc = est(&lo); + let fgc = est(&hi); + for i in 0..n { + if !region[i] { + continue; + } + let a = alpha[i].clamp(0.0, 1.0); + if a <= 0.0 || a >= 1.0 { + continue; + } + let c = img[i]; + let f = match (bgc[i], fgc[i]) { + (Some(b), fbar) => { + let mut f = [0.0f32; 3]; + for k in 0..3 { + f[k] = ((c[k] - (1.0 - a) * b[k]) / a).clamp(0.0, 1.0); + } + // Low alphas amplify noise: lean on the nearby foreground colour there. + if let Some(fb) = fbar { + let t = ((0.25 - a) / 0.25).clamp(0.0, 1.0); + for k in 0..3 { + f[k] += (fb[k] - f[k]) * t; + } + } + f + } + (None, Some(fb)) => fb, + (None, None) => c, + }; + let mut d = 0.0f32; + for k in 0..3 { + let v = c[k] + (f[k] - c[k]) * amount; + d = d.max((v - c[k]).abs()); + img[i][k] = v; + } + map[i] = d.min(1.0); + } + map +} + +/// Sample a float plane bilinearly at continuous coordinates (pixel centres at +0.5), clamped. +pub fn sample(p: &[f32], w: usize, h: usize, x: f64, y: f64) -> f32 { + if w == 0 || h == 0 { + return 0.0; + } + let fx = (x - 0.5).clamp(0.0, (w - 1) as f64); + let fy = (y - 0.5).clamp(0.0, (h - 1) as f64); + let (x0, y0) = (fx.floor() as usize, fy.floor() as usize); + let (x1, y1) = ((x0 + 1).min(w - 1), (y0 + 1).min(h - 1)); + let (tx, ty) = ((fx - x0 as f64) as f32, (fy - y0 as f64) as f32); + let g = |x: usize, y: usize| p[y * w + x]; + let top = g(x0, y0) + (g(x1, y0) - g(x0, y0)) * tx; + let bot = g(x0, y1) + (g(x1, y1) - g(x0, y1)) * tx; + top + (bot - top) * ty +} + +/// Average of `a` sampled along the segment `[-v/2, v/2]` with `samples` taps (motion blur of a +/// matte moving by `v` pixels during the shutter). +pub fn motion_blur(a: &[f32], w: usize, h: usize, v: [f64; 2], samples: usize) -> Vec { + let n = samples.max(2); + if v[0].hypot(v[1]) < 0.25 { + return a.to_vec(); + } + (0..w * h) + .into_par_iter() + .map(|i| { + let (x, y) = ((i % w) as f64 + 0.5, (i / w) as f64 + 0.5); + let mut s = 0.0; + for k in 0..n { + let t = k as f64 / (n - 1) as f64 - 0.5; + s += sample(a, w, h, x - v[0] * t, y - v[1] * t); + } + s / n as f32 + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn distance_matches_brute_force() { + let (w, h) = (23, 17); + let seed: Vec = (0..w * h).map(|i| (i * 7919) % 31 == 0).collect(); + let d = distance(&seed, w, h); + for y in 0..h { + for x in 0..w { + let mut b = f32::INFINITY; + for j in 0..h { + for i in 0..w { + if seed[j * w + i] { + b = b.min(((x as f32 - i as f32).powi(2) + (y as f32 - j as f32).powi(2)).sqrt()); + } + } + } + assert!((d[y * w + x] - b).abs() < 1e-3, "{x},{y}: {} vs {b}", d[y * w + x]); + } + } + } + + #[test] + fn guided_filter_recovers_a_linear_blend() { + // I = αF + (1 − α)B with constant F, B: the filtered binary matte is close to α. + let (w, h) = (64, 8); + let alpha: Vec = (0..w * h).map(|i| (((i % w) as f32 - 26.0) / 12.0).clamp(0.0, 1.0)).collect(); + let (f, b) = ([0.9f32, 0.2, 0.1], [0.1f32, 0.3, 0.8]); + let img: Vec<[f32; 3]> = alpha.iter().map(|a| [a * f[0] + (1.0 - a) * b[0], a * f[1] + (1.0 - a) * b[1], a * f[2] + (1.0 - a) * b[2]]).collect(); + let p: Vec = alpha.iter().map(|a| if *a >= 0.5 { 1.0 } else { 0.0 }).collect(); + let q = guided_filter(&img, &p, w, h, 8, 1e-5); + let err: f32 = q.iter().zip(&alpha).map(|(a, b)| (a.clamp(0.0, 1.0) - b).abs()).sum::() / (w * h) as f32; + assert!(err < 0.03, "mean error {err}"); + } +} diff --git a/crates/track/src/roto/maxflow.rs b/crates/track/src/roto/maxflow.rs new file mode 100644 index 00000000..dee4b010 --- /dev/null +++ b/crates/track/src/roto/maxflow.rs @@ -0,0 +1,438 @@ +//! Min-cut / max-flow on sparse graphs with source and sink terminals, by the +//! Boykov–Kolmogorov algorithm (Y. Boykov and V. Kolmogorov, "An Experimental Comparison of +//! Min-Cut/Max-Flow Algorithms for Energy Minimization in Vision", IEEE PAMI 26(9), 2004). +//! +//! Two search trees grow from the terminals (S from the source, T from the sink) through +//! non-saturated edges; when they touch, the path found is augmented; saturated tree edges turn +//! the nodes below them into *orphans*, which the adoption stage re-attaches to their tree (or +//! frees). Trees are reused between augmentations, which is what makes the method fast on the +//! grid graphs of image segmentation. The timestamp/distance heuristic of section 3.2.3 keeps +//! adoption paths short. +//! +//! Capacities are integers (callers scale their energies), so saturation tests are exact and +//! the result is deterministic. + +use std::collections::VecDeque; + +const NONE: u32 = u32::MAX; +const TERMINAL: u32 = u32::MAX - 1; +const ORPHAN: u32 = u32::MAX - 2; + +#[derive(Clone, Copy)] +struct Node { + first: u32, + /// Arc to the parent (from this node), or TERMINAL / ORPHAN / NONE (free). + parent: u32, + /// Residual terminal capacity: > 0 towards the source, < 0 towards the sink. + tr_cap: i64, + is_sink: bool, + active: bool, + ts: u32, + dist: u32, +} + +#[derive(Clone, Copy)] +struct Arc { + head: u32, + next: u32, + r_cap: i32, +} + +/// A graph for one min-cut computation. +pub struct Graph { + nodes: Vec, + arcs: Vec, + flow: i64, + time: u32, + queue: VecDeque, + orphans: VecDeque, +} + +/// Which side of the cut a node ended on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Segment { + Source, + Sink, +} + +impl Graph { + pub fn new(nodes: usize, edges_hint: usize) -> Graph { + Graph { + nodes: vec![Node { first: NONE, parent: NONE, tr_cap: 0, is_sink: false, active: false, ts: 0, dist: 0 }; nodes], + arcs: Vec::with_capacity(edges_hint * 2), + flow: 0, + time: 0, + queue: VecDeque::new(), + orphans: VecDeque::new(), + } + } + + pub fn len(&self) -> usize { + self.nodes.len() + } + pub fn is_empty(&self) -> bool { + self.nodes.is_empty() + } + + /// Terminal capacities of node `i`: `source` is paid when `i` ends on the sink side, `sink` + /// when it ends on the source side. + pub fn add_tweights(&mut self, i: usize, source: i64, sink: i64) { + let n = &mut self.nodes[i]; + let d = n.tr_cap; + let (s, t) = if d > 0 { (source + d, sink) } else { (source, sink - d) }; + self.flow += s.min(t); + n.tr_cap = s - t; + } + + /// An edge between `i` and `j` with capacity `cap` from i to j and `rev` from j to i. + pub fn add_edge(&mut self, i: usize, j: usize, cap: i32, rev: i32) { + debug_assert!(i != j); + let a = self.arcs.len() as u32; + self.arcs.push(Arc { head: j as u32, next: self.nodes[i].first, r_cap: cap }); + self.nodes[i].first = a; + self.arcs.push(Arc { head: i as u32, next: self.nodes[j].first, r_cap: rev }); + self.nodes[j].first = a + 1; + } + + #[inline] + fn set_active(&mut self, i: u32) { + let n = &mut self.nodes[i as usize]; + if !n.active { + n.active = true; + self.queue.push_back(i); + } + } + + /// Compute the maximum flow (= the minimum cut's cost). + pub fn maxflow(&mut self) -> i64 { + for i in 0..self.nodes.len() { + let n = &mut self.nodes[i]; + if n.tr_cap != 0 { + n.is_sink = n.tr_cap < 0; + n.parent = TERMINAL; + n.ts = 0; + n.dist = 1; + self.set_active(i as u32); + } + } + let mut current: Option = None; + loop { + let i = match current.take() { + Some(i) if self.nodes[i as usize].parent != NONE => i, + _ => loop { + let Some(i) = self.queue.pop_front() else { return self.flow }; + self.nodes[i as usize].active = false; + if self.nodes[i as usize].parent != NONE { + break i; + } + }, + }; + // Growth. + let mut found = NONE; + let ni = self.nodes[i as usize]; + let mut a = ni.first; + if !ni.is_sink { + while a != NONE { + let arc = self.arcs[a as usize]; + if arc.r_cap > 0 { + let j = arc.head as usize; + let nj = self.nodes[j]; + if nj.parent == NONE { + let n = &mut self.nodes[j]; + n.is_sink = false; + n.parent = a ^ 1; + n.ts = ni.ts; + n.dist = ni.dist + 1; + self.set_active(j as u32); + } else if nj.is_sink { + found = a; + break; + } else if nj.ts <= ni.ts && nj.dist > ni.dist { + let n = &mut self.nodes[j]; + n.parent = a ^ 1; + n.ts = ni.ts; + n.dist = ni.dist + 1; + } + } + a = arc.next; + } + } else { + while a != NONE { + let arc = self.arcs[a as usize]; + if self.arcs[(a ^ 1) as usize].r_cap > 0 { + let j = arc.head as usize; + let nj = self.nodes[j]; + if nj.parent == NONE { + let n = &mut self.nodes[j]; + n.is_sink = true; + n.parent = a ^ 1; + n.ts = ni.ts; + n.dist = ni.dist + 1; + self.set_active(j as u32); + } else if !nj.is_sink { + found = a ^ 1; + break; + } else if nj.ts <= ni.ts && nj.dist > ni.dist { + let n = &mut self.nodes[j]; + n.parent = a ^ 1; + n.ts = ni.ts; + n.dist = ni.dist + 1; + } + } + a = arc.next; + } + } + self.time = self.time.wrapping_add(1); + if found != NONE { + // Keep `i` current: it may have more paths. + current = Some(i); + self.augment(found); + self.adopt(); + } + } + } + + fn augment(&mut self, middle: u32) { + let mut b = self.arcs[middle as usize].r_cap as i64; + // Source side. + let mut i = self.arcs[(middle ^ 1) as usize].head; + loop { + let a = self.nodes[i as usize].parent; + if a == TERMINAL { + break; + } + b = b.min(self.arcs[(a ^ 1) as usize].r_cap as i64); + i = self.arcs[a as usize].head; + } + b = b.min(self.nodes[i as usize].tr_cap); + // Sink side. + let mut i = self.arcs[middle as usize].head; + loop { + let a = self.nodes[i as usize].parent; + if a == TERMINAL { + break; + } + b = b.min(self.arcs[a as usize].r_cap as i64); + i = self.arcs[a as usize].head; + } + b = b.min(-self.nodes[i as usize].tr_cap); + let bi = b as i32; + self.arcs[(middle ^ 1) as usize].r_cap += bi; + self.arcs[middle as usize].r_cap -= bi; + // Source side. + let mut i = self.arcs[(middle ^ 1) as usize].head; + loop { + let a = self.nodes[i as usize].parent; + if a == TERMINAL { + break; + } + self.arcs[a as usize].r_cap += bi; + self.arcs[(a ^ 1) as usize].r_cap -= bi; + if self.arcs[(a ^ 1) as usize].r_cap == 0 { + self.make_orphan(i); + } + i = self.arcs[a as usize].head; + } + self.nodes[i as usize].tr_cap -= b; + if self.nodes[i as usize].tr_cap == 0 { + self.make_orphan(i); + } + // Sink side. + let mut i = self.arcs[middle as usize].head; + loop { + let a = self.nodes[i as usize].parent; + if a == TERMINAL { + break; + } + self.arcs[(a ^ 1) as usize].r_cap += bi; + self.arcs[a as usize].r_cap -= bi; + if self.arcs[a as usize].r_cap == 0 { + self.make_orphan(i); + } + i = self.arcs[a as usize].head; + } + self.nodes[i as usize].tr_cap += b; + if self.nodes[i as usize].tr_cap == 0 { + self.make_orphan(i); + } + self.flow += b; + } + + fn make_orphan(&mut self, i: u32) { + self.nodes[i as usize].parent = ORPHAN; + self.orphans.push_back(i); + } + + /// Whether node `j`'s path to its terminal is intact; returns its distance (marking the + /// path with the current time) or None. + fn origin_dist(&mut self, j: u32) -> Option { + let mut d = 0u32; + let mut k = j; + loop { + let n = self.nodes[k as usize]; + if n.ts == self.time { + d += n.dist; + break; + } + let a = n.parent; + d += 1; + if a == TERMINAL { + let n = &mut self.nodes[k as usize]; + n.ts = self.time; + n.dist = 1; + break; + } + if a == ORPHAN || a == NONE { + return None; + } + k = self.arcs[a as usize].head; + } + // Mark the path. + let total = d; + let mut k = j; + let mut dd = d; + while self.nodes[k as usize].ts != self.time { + let n = &mut self.nodes[k as usize]; + n.ts = self.time; + n.dist = dd; + dd -= 1; + k = self.arcs[n.parent as usize].head; + } + Some(total) + } + + fn adopt(&mut self) { + while let Some(i) = self.orphans.pop_front() { + let sink = self.nodes[i as usize].is_sink; + let mut best = NONE; + let mut best_d = u32::MAX; + let mut a = self.nodes[i as usize].first; + while a != NONE { + let arc = self.arcs[a as usize]; + // S orphan: the parent must push flow into i (residual on j→i = sister of i→j). + // T orphan: i must push flow into the parent (residual on i→j). + let cap = if sink { arc.r_cap } else { self.arcs[(a ^ 1) as usize].r_cap }; + if cap > 0 { + let j = arc.head; + let nj = self.nodes[j as usize]; + if nj.is_sink == sink + && nj.parent != NONE + && let Some(d) = self.origin_dist(j) + && d < best_d + { + best = a; + best_d = d; + } + } + a = arc.next; + } + if best != NONE { + let n = &mut self.nodes[i as usize]; + n.parent = best; + n.ts = self.time; + n.dist = best_d + 1; + continue; + } + // No parent: free the node; its children become orphans, neighbours that could + // reach it become active. + self.nodes[i as usize].ts = 0; + let mut a = self.nodes[i as usize].first; + while a != NONE { + let arc = self.arcs[a as usize]; + let j = arc.head; + let nj = self.nodes[j as usize]; + if nj.is_sink == sink && nj.parent != NONE { + let cap = if sink { arc.r_cap } else { self.arcs[(a ^ 1) as usize].r_cap }; + if cap > 0 { + self.set_active(j); + } + let pa = nj.parent; + if pa != TERMINAL && pa != ORPHAN && self.arcs[pa as usize].head == i { + self.make_orphan(j); + } + } + a = arc.next; + } + self.nodes[i as usize].parent = NONE; + } + } + + /// The side of node `i` after [`Graph::maxflow`] (free nodes go to the sink). + pub fn segment(&self, i: usize) -> Segment { + let n = &self.nodes[i]; + if n.parent != NONE && !n.is_sink { Segment::Source } else { Segment::Sink } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Brute-force min cut over all labelings of a small graph. + fn brute(n: usize, t: &[(i64, i64)], e: &[(usize, usize, i32, i32)]) -> i64 { + let mut best = i64::MAX; + for m in 0..(1u32 << n) { + let src = |i: usize| m & (1 << i) != 0; + let mut c = 0i64; + for (i, (s, k)) in t.iter().enumerate() { + c += if src(i) { *k } else { *s }; + } + for (i, j, a, b) in e { + if src(*i) && !src(*j) { + c += *a as i64; + } + if src(*j) && !src(*i) { + c += *b as i64; + } + } + best = best.min(c); + } + best + } + + #[test] + fn matches_brute_force_on_random_graphs() { + let mut s = 12345u64; + let mut rnd = |m: u64| { + s ^= s << 13; + s ^= s >> 7; + s ^= s << 17; + s % m + }; + for _ in 0..300 { + let n = 2 + rnd(9) as usize; + let t: Vec<(i64, i64)> = (0..n).map(|_| (rnd(20) as i64, rnd(20) as i64)).collect(); + let mut e = vec![]; + for _ in 0..rnd(20) { + let (i, j) = (rnd(n as u64) as usize, rnd(n as u64) as usize); + if i != j { + e.push((i, j, rnd(15) as i32, rnd(15) as i32)); + } + } + let mut g = Graph::new(n, e.len()); + for (i, (a, b)) in t.iter().enumerate() { + g.add_tweights(i, *a, *b); + } + for (i, j, a, b) in &e { + g.add_edge(*i, *j, *a, *b); + } + let f = g.maxflow(); + assert_eq!(f, brute(n, &t, &e)); + // The labeling realises the cut. + let src = |i: usize| g.segment(i) == Segment::Source; + let mut c = 0i64; + for (i, (a, b)) in t.iter().enumerate() { + c += if src(i) { *b } else { *a }; + } + for (i, j, a, b) in &e { + if src(*i) && !src(*j) { + c += *a as i64; + } + if src(*j) && !src(*i) { + c += *b as i64; + } + } + assert_eq!(c, f); + } + } +} diff --git a/crates/track/src/roto/mod.rs b/crates/track/src/roto/mod.rs new file mode 100644 index 00000000..393f743e --- /dev/null +++ b/crates/track/src/roto/mod.rs @@ -0,0 +1,754 @@ +//! Roto Brush segmentation and propagation, clean-room from published methods: +//! +//! - **Interactive graph-cut segmentation** (Y. Boykov and M.-P. Jolly, "Interactive Graph Cuts +//! for Optimal Boundary & Region Segmentation of Objects in N-D Images", ICCV 2001): the user's +//! foreground / background strokes are hard constraints, every other pixel gets a data cost +//! from colour models and neighbouring pixels a contrast-sensitive smoothness cost; the +//! minimum s–t cut ([`maxflow`], Boykov–Kolmogorov) is the segmentation. +//! - **Iterated colour models** (C. Rother, V. Kolmogorov and A. Blake, "GrabCut", SIGGRAPH +//! 2004): Gaussian mixtures ([`gmm`]) are fitted to the strokes, then re-fitted to the +//! segmentation and the cut repeated; the 8-connected n-link weight is +//! `γ/dist · exp(−β‖Iₚ − I_q‖²)` with `β = 1 / (2⟨‖Iₚ − I_q‖²⟩)`. +//! - **Coarse to fine**: large frames are segmented at a reduced size, then re-cut at full +//! resolution in a narrow band around the upsampled boundary (Lombaert et al., "A Multilevel +//! Banded Graph Cuts Method for Fast Image Segmentation", ICCV 2005). +//! - **Propagation** (in the spirit of X. Bai et al., "Video SnapCut", SIGGRAPH 2009): the +//! previous frame's matte is warped to the next frame by optical flow +//! (`effectcraft_raster::flow::block_flow`), and the frame re-segmented by graph cut only in a +//! band of **Search Radius** pixels around the warped boundary, with the previous frame's +//! colour models and the warped matte as a shape prior. Correction strokes on a frame re-cut +//! the whole frame with the warped matte as a soft prior, and propagation restarts from there. +//! +//! Mattes are in layer pixels at a given scale (1 = full resolution). Strokes are stored in +//! layer pixels ([`RotoData`], serde) and keyed per frame so that each frame's result is a +//! deterministic function of its *chain*: the base frame's strokes, then every frame's strokes +//! on the way out to it ([`RotoData::chain_keys`]). + +pub mod gmm; +pub mod matting; +pub mod maxflow; +pub mod refine; +pub mod rle; + +use std::collections::BTreeMap; + +use effectcraft_raster::Image; +use rayon::prelude::*; +use serde::{Deserialize, Serialize}; + +pub use gmm::Gmm; +use maxflow::{Graph, Segment}; + +/// What a stroke paints. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum StrokeKind { + /// Roto Brush (green): foreground. + Fg, + /// Roto Brush with Alt (red): background. + Bg, + /// Refine Edge tool: mark a band for edge matting. + Refine, + /// Refine Edge tool with Alt: remove from the band. + RefineErase, +} + +impl StrokeKind { + pub fn from_name(s: &str) -> Option { + Some(match s.to_ascii_lowercase().as_str() { + "fg" | "foreground" => StrokeKind::Fg, + "bg" | "background" => StrokeKind::Bg, + "refine" | "refineedge" => StrokeKind::Refine, + "refineerase" | "refine-erase" | "erase" => StrokeKind::RefineErase, + _ => return None, + }) + } + pub fn is_segmentation(self) -> bool { + matches!(self, StrokeKind::Fg | StrokeKind::Bg) + } +} + +/// One stroke: a polyline swept by a disc of `radius` layer pixels, on layer frame `frame`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Stroke { + pub kind: StrokeKind, + pub frame: i64, + pub radius: f64, + pub points: Vec<[f64; 2]>, +} + +/// A Roto Brush instance's user data: strokes, the base frame and the segmentation span +/// (inclusive layer frames). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct RotoData { + pub base: Option, + pub span: [i64; 2], + pub strokes: Vec, +} + +fn fnv(h: &mut u64, bytes: &[u8]) { + for b in bytes { + *h ^= *b as u64; + *h = h.wrapping_mul(0x0100_0000_01b3); + } +} + +impl RotoData { + pub fn from_json(s: &str) -> RotoData { + if s.trim().is_empty() { + return RotoData::default(); + } + serde_json::from_str(s).unwrap_or_default() + } + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } + pub fn is_empty(&self) -> bool { + self.base.is_none() || !self.strokes.iter().any(|s| s.kind.is_segmentation()) + } + pub fn strokes_at(&self, frame: i64) -> Vec<&Stroke> { + self.strokes.iter().filter(|s| s.frame == frame).collect() + } + /// Frames that carry strokes. + pub fn stroke_frames(&self) -> Vec { + let mut v: Vec = self.strokes.iter().map(|s| s.frame).collect(); + v.sort_unstable(); + v.dedup(); + v + } + pub fn in_span(&self, f: i64) -> bool { + self.base.is_some() && f >= self.span[0] && f <= self.span[1] + } + /// Add a stroke: the first segmentation stroke sets the base frame and a span of + /// `default_span` frames each side (clamped to `limits`); strokes outside the span extend it. + pub fn add(&mut self, s: Stroke, default_span: i64, limits: [i64; 2]) { + let f = s.frame; + match self.base { + None if s.kind.is_segmentation() => { + self.base = Some(f); + self.span = [(f - default_span).max(limits[0]), (f + default_span).min(limits[1]).max(f)]; + } + Some(_) => { + self.span[0] = self.span[0].min(f); + self.span[1] = self.span[1].max(f); + } + None => {} + } + self.strokes.push(s); + } + /// Largest Refine Edge stroke radius (layer pixels). + pub fn refine_radius(&self) -> f64 { + self.strokes.iter().filter(|s| matches!(s.kind, StrokeKind::Refine)).map(|s| s.radius).fold(0.0, f64::max) + } + /// Content key of every frame in the span: the base frame's from `seed` (what the frames + /// and settings are) and its strokes, every other frame's from its neighbour towards the + /// base and its own strokes. + pub fn chain_keys(&self, seed: u64) -> BTreeMap { + let mut out = BTreeMap::new(); + let Some(base) = self.base else { return out }; + let key = |prev: u64, f: i64| { + let mut h = prev ^ 0x9e37_79b9_7f4a_7c15; + fnv(&mut h, &f.to_le_bytes()); + for s in self.strokes.iter().filter(|s| s.frame == f) { + fnv(&mut h, serde_json::to_string(s).unwrap_or_default().as_bytes()); + } + h + }; + let k0 = key(seed, base); + out.insert(base, k0); + let mut k = k0; + for f in base + 1..=self.span[1] { + k = key(k, f); + out.insert(f, k); + } + k = k0; + for f in (self.span[0]..base).rev() { + k = key(k, f); + out.insert(f, k); + } + out + } + /// The neighbour a frame is propagated from (None for the base frame and outside the span). + pub fn source_of(&self, f: i64) -> Option { + let base = self.base?; + if !self.in_span(f) || f == base { + return None; + } + Some(if f > base { f - 1 } else { f + 1 }) + } +} + +/// Segmentation settings (the effect's Roto Brush parameters, in frame pixels). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct SegOpts { + /// Pixels of the matte per layer pixel. + pub scale: f64, + /// Search Radius (layer pixels). + pub search_radius: f64, + /// Motion Threshold / Motion Damping, percent. + pub motion_threshold: f64, + pub motion_damping: f64, + /// Quality ▸ Best. + pub best: bool, +} + +impl Default for SegOpts { + fn default() -> Self { + SegOpts { scale: 1.0, search_radius: 15.0, motion_threshold: 10.0, motion_damping: 20.0, best: false } + } +} + +/// One frame's segmentation: the binary matte, the Refine Edge band and the colour models. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct FrameSeg { + pub w: usize, + pub h: usize, + pub matte: Vec, + pub refine: Vec, + pub fg: Gmm, + pub bg: Gmm, +} + +impl FrameSeg { + pub fn empty(w: usize, h: usize) -> FrameSeg { + FrameSeg { w, h, matte: vec![0; w * h], refine: vec![0; w * h], ..Default::default() } + } + pub fn area(&self) -> usize { + self.matte.iter().filter(|v| **v != 0).count() + } + /// Centroid of the matte (pixels), if any. + pub fn centroid(&self) -> Option<[f64; 2]> { + let (mut sx, mut sy, mut n) = (0.0, 0.0, 0.0); + for (i, v) in self.matte.iter().enumerate() { + if *v != 0 { + sx += (i % self.w) as f64 + 0.5; + sy += (i / self.w) as f64 + 0.5; + n += 1.0; + } + } + (n > 0.0).then(|| [sx / n, sy / n]) + } +} + +/// Straight RGB of an image (transparent pixels are black). +pub fn rgb_of(img: &Image) -> Vec<[f32; 3]> { + img.data.par_iter().map(|p| if p[3] > 1e-6 { [p[0] / p[3], p[1] / p[3], p[2] / p[3]] } else { [0.0; 3] }).collect() +} + +/// Pixels covered by `strokes` (of the given kinds) at `scale`. +pub fn rasterize(strokes: &[&Stroke], kinds: &[StrokeKind], scale: f64, w: usize, h: usize) -> Vec { + let mut m = vec![false; w * h]; + for s in strokes.iter().filter(|s| kinds.contains(&s.kind)) { + let r = (s.radius * scale).max(0.5); + let pts: Vec<[f64; 2]> = s.points.iter().map(|p| [p[0] * scale, p[1] * scale]).collect(); + let segs: Vec<([f64; 2], [f64; 2])> = if pts.len() == 1 { vec![(pts[0], pts[0])] } else { pts.windows(2).map(|w| (w[0], w[1])).collect() }; + for (a, b) in segs { + let x0 = (a[0].min(b[0]) - r).floor().max(0.0) as usize; + let x1 = ((a[0].max(b[0]) + r).ceil().max(0.0) as usize).min(w); + let y0 = (a[1].min(b[1]) - r).floor().max(0.0) as usize; + let y1 = ((a[1].max(b[1]) + r).ceil().max(0.0) as usize).min(h); + let d = [b[0] - a[0], b[1] - a[1]]; + let l2 = d[0] * d[0] + d[1] * d[1]; + for y in y0..y1 { + for x in x0..x1 { + let p = [x as f64 + 0.5, y as f64 + 0.5]; + let t = if l2 > 0.0 { (((p[0] - a[0]) * d[0] + (p[1] - a[1]) * d[1]) / l2).clamp(0.0, 1.0) } else { 0.0 }; + let q = [a[0] + d[0] * t, a[1] + d[1] * t]; + if (p[0] - q[0]).hypot(p[1] - q[1]) <= r { + m[y * w + x] = true; + } + } + } + } + } + m +} + +/// Hard constraints: 1 = foreground, 2 = background, 0 = free. +const HARD_FG: u8 = 1; +const HARD_BG: u8 = 2; + +/// Graph-cut energy weights. +const GAMMA: f64 = 50.0; +const PRIOR: f64 = 1.0; +/// Fixed-point scale of the energies. +const K: f64 = 16.0; +const INF: i64 = 1 << 40; + +const NB: [(i64, i64, bool); 8] = [(1, 0, true), (0, 1, true), (1, 1, true), (-1, 1, true), (-1, 0, false), (0, -1, false), (-1, -1, false), (1, -1, false)]; + +fn beta(img: &[[f32; 3]], free: &[bool], w: usize, h: usize) -> f64 { + let (mut s, mut n) = (0.0f64, 0.0f64); + for y in 0..h { + for x in 0..w { + let i = y * w + x; + if !free[i] { + continue; + } + let c = img[i]; + for (dx, dy) in [(1usize, 0usize), (0, 1)] { + let (nx, ny) = (x + dx, y + dy); + if nx < w && ny < h { + let q = img[ny * w + nx]; + s += ((c[0] - q[0]).powi(2) + (c[1] - q[1]).powi(2) + (c[2] - q[2]).powi(2)) as f64; + n += 1.0; + } + } + } + } + if s <= 1e-12 { 1.0 } else { n / (2.0 * s) } +} + +/// One min-cut over the `free` pixels; others keep `labels`. `prior`: P(foreground) per pixel. +fn cut(img: &[[f32; 3]], w: usize, h: usize, labels: &mut [u8], free: &[bool], hard: &[u8], fg: &Gmm, bg: &Gmm, prior: Option<&[f32]>) { + let mut idx = vec![u32::MAX; w * h]; + let mut nodes = vec![]; + for i in 0..w * h { + if free[i] { + idx[i] = nodes.len() as u32; + nodes.push(i); + } + } + if nodes.is_empty() { + return; + } + let b = beta(img, free, w, h); + let costs: Vec<(f64, f64)> = nodes + .par_iter() + .map(|&i| { + let c = img[i]; + let (mut cf, mut cb) = (fg.cost(c), bg.cost(c)); + if let Some(p) = prior { + let pf = (p[i] as f64).clamp(0.02, 0.98); + cf += PRIOR * -pf.ln(); + cb += PRIOR * -(1.0 - pf).ln(); + } + (cf, cb) + }) + .collect(); + let mut g = Graph::new(nodes.len(), nodes.len() * 4); + for (n, &i) in nodes.iter().enumerate() { + let (x, y) = ((i % w) as i64, (i / w) as i64); + let (cf, cb) = costs[n]; + let m = cf.min(cb); + let (mut src, mut snk) = (((cb - m) * K).round() as i64, ((cf - m) * K).round() as i64); + match hard[i] { + HARD_FG => (src, snk) = (INF, 0), + HARD_BG => (src, snk) = (0, INF), + _ => {} + } + g.add_tweights(n, src, snk); + let c = img[i]; + for (dx, dy, fwd) in NB { + let (nx, ny) = (x + dx, y + dy); + if nx < 0 || ny < 0 || nx >= w as i64 || ny >= h as i64 { + continue; + } + let j = ny as usize * w + nx as usize; + let q = img[j]; + let d2 = ((c[0] - q[0]).powi(2) + (c[1] - q[1]).powi(2) + (c[2] - q[2]).powi(2)) as f64; + let dist = if dx != 0 && dy != 0 { std::f64::consts::SQRT_2 } else { 1.0 }; + let wt = ((GAMMA / dist * (-b * d2).exp()) * K).round() as i32; + if wt <= 0 { + continue; + } + if idx[j] == u32::MAX { + // Fixed neighbour: disagreeing with it costs the edge. + if labels[j] != 0 { + g.add_tweights(n, wt as i64, 0); + } else { + g.add_tweights(n, 0, wt as i64); + } + } else if fwd { + g.add_edge(n, idx[j] as usize, wt, wt); + } + } + } + g.maxflow(); + for (n, &i) in nodes.iter().enumerate() { + labels[i] = (g.segment(n) == Segment::Source) as u8; + } +} + +/// Colour samples of the pixels where `sel` holds. +fn samples(img: &[[f32; 3]], sel: impl Fn(usize) -> bool) -> Vec<[f32; 3]> { + img.iter().enumerate().filter(|(i, _)| sel(*i)).map(|(_, c)| *c).collect() +} + +const COMPONENTS: usize = 5; +const MAX_SAMPLES: usize = 24_000; + +fn fit_models(img: &[[f32; 3]], labels: &[u8]) -> (Gmm, Gmm) { + let fg = samples(img, |i| labels[i] != 0); + let bg = samples(img, |i| labels[i] == 0); + (Gmm::fit(&fg, COMPONENTS, MAX_SAMPLES), Gmm::fit(&bg, COMPONENTS, MAX_SAMPLES)) +} + +/// Keep the foreground components (8-connected) that touch `seeds`; fill small background +/// holes that touch neither the frame edge nor a background seed. +fn clean(labels: &mut [u8], seeds: &[bool], bg_seeds: &[bool], w: usize, h: usize) { + let n = w * h; + let mut comp = vec![u32::MAX; n]; + let mut stack = vec![]; + let mut fg_area = 0usize; + for start in 0..n { + if comp[start] != u32::MAX { + continue; + } + let lab = labels[start]; + let id = start as u32; + comp[start] = id; + stack.push(start); + let mut members = vec![]; + let (mut seeded, mut edge, mut bgs) = (false, false, false); + while let Some(i) = stack.pop() { + members.push(i); + seeded |= seeds[i]; + bgs |= bg_seeds[i]; + let (x, y) = ((i % w) as i64, (i / w) as i64); + edge |= x == 0 || y == 0 || x == w as i64 - 1 || y == h as i64 - 1; + for (dx, dy, _) in NB { + let (nx, ny) = (x + dx, y + dy); + if nx < 0 || ny < 0 || nx >= w as i64 || ny >= h as i64 { + continue; + } + let j = ny as usize * w + nx as usize; + if comp[j] == u32::MAX && labels[j] == lab { + comp[j] = id; + stack.push(j); + } + } + } + if lab != 0 { + if seeded { + fg_area += members.len(); + } else { + for i in members { + labels[i] = 0; + } + } + } else if !edge && !bgs { + // Remember holes as negative candidates (resolved below once fg_area is known). + for i in members { + comp[i] = u32::MAX - 1; + } + } + } + // Fill holes smaller than 2 % of the foreground. + let mut seen = vec![false; n]; + for start in 0..n { + if comp[start] != u32::MAX - 1 || seen[start] || labels[start] != 0 { + continue; + } + let mut members = vec![start]; + seen[start] = true; + let mut k = 0; + while k < members.len() { + let i = members[k]; + k += 1; + let (x, y) = ((i % w) as i64, (i / w) as i64); + for (dx, dy, _) in NB { + let (nx, ny) = (x + dx, y + dy); + if nx < 0 || ny < 0 || nx >= w as i64 || ny >= h as i64 { + continue; + } + let j = ny as usize * w + nx as usize; + if !seen[j] && comp[j] == u32::MAX - 1 && labels[j] == 0 { + seen[j] = true; + members.push(j); + } + } + } + // A hole is enclosed by foreground that survived the component filter. + let enclosed = members.iter().all(|&i| { + let (x, y) = ((i % w) as i64, (i / w) as i64); + NB.iter().all(|(dx, dy, _)| { + let (nx, ny) = (x + dx, y + dy); + if nx < 0 || ny < 0 || nx >= w as i64 || ny >= h as i64 { + return false; + } + let j = ny as usize * w + nx as usize; + labels[j] != 0 || comp[j] == u32::MAX - 1 + }) + }); + if enclosed && (members.len() as f64) < 0.02 * fg_area as f64 { + for i in members { + labels[i] = 1; + } + } + } +} + +fn downsample_rgb(img: &[[f32; 3]], w: usize, h: usize, f: usize) -> (Vec<[f32; 3]>, usize, usize) { + let (cw, ch) = (w.div_ceil(f), h.div_ceil(f)); + let out = (0..cw * ch) + .into_par_iter() + .map(|i| { + let (cx, cy) = (i % cw, i / cw); + let mut s = [0.0f32; 3]; + let mut n = 0.0; + for y in cy * f..((cy + 1) * f).min(h) { + for x in cx * f..((cx + 1) * f).min(w) { + let c = img[y * w + x]; + s[0] += c[0]; + s[1] += c[1]; + s[2] += c[2]; + n += 1.0; + } + } + [s[0] / n, s[1] / n, s[2] / n] + }) + .collect(); + (out, cw, ch) +} + +/// What a frame is segmented with besides its strokes: the matte propagated from its neighbour. +pub struct Prior { + /// Warped neighbour matte, soft (0–1). + pub soft: Vec, + /// The neighbour's colour models. + pub fg: Gmm, + pub bg: Gmm, + /// Warped Refine Edge band. + pub refine: Vec, +} + +/// Segment a frame from its strokes (and a propagated prior): GrabCut iterations on a reduced +/// copy, then a full-resolution cut in a band around the boundary. +pub fn segment(img: &Image, strokes: &[&Stroke], prior: Option<&Prior>, opts: &SegOpts, refine_radius: f64) -> FrameSeg { + let (w, h) = (img.width as usize, img.height as usize); + if w == 0 || h == 0 { + return FrameSeg::empty(w, h); + } + let rgb = rgb_of(img); + let fgs = rasterize(strokes, &[StrokeKind::Fg], opts.scale, w, h); + let bgs = rasterize(strokes, &[StrokeKind::Bg], opts.scale, w, h); + let hard: Vec = fgs + .iter() + .zip(&bgs) + .map(|(f, b)| { + if *b { + HARD_BG + } else if *f { + HARD_FG + } else { + 0 + } + }) + .collect(); + let soft_prior: Option> = prior.map(|p| matting::gauss(&p.soft, w, h, (opts.search_radius * opts.scale / 4.0).max(1.0))); + // Initial colour models: the strokes (and the propagated matte). + let (fg0, bg0) = match prior { + Some(p) => { + let fgc = samples(&rgb, |i| (hard[i] == HARD_FG) || (hard[i] == 0 && p.soft[i] >= 0.5)); + let bgc = samples(&rgb, |i| (hard[i] == HARD_BG) || (hard[i] == 0 && p.soft[i] < 0.5)); + let f = Gmm::fit(&fgc, COMPONENTS, MAX_SAMPLES); + let b = Gmm::fit(&bgc, COMPONENTS, MAX_SAMPLES); + (if f.is_empty() { p.fg.clone() } else { f }, if b.is_empty() { p.bg.clone() } else { b }) + } + None => { + let fgc = samples(&rgb, |i| hard[i] == HARD_FG); + let mut bgc = samples(&rgb, |i| hard[i] == HARD_BG); + if bgc.is_empty() { + // No background strokes: what is far from the foreground strokes. + let d = matting::distance(&fgs, w, h); + let r = strokes.iter().filter(|s| s.kind == StrokeKind::Fg).map(|s| s.radius).fold(1.0, f64::max) * opts.scale; + let far = (4.0 * r).max(0.25 * w.max(h) as f64) as f32; + bgc = samples(&rgb, |i| d[i] > far); + if bgc.is_empty() { + bgc = samples(&rgb, |i| hard[i] != HARD_FG); + } + } + (Gmm::fit(&fgc, COMPONENTS, MAX_SAMPLES), Gmm::fit(&bgc, COMPONENTS, MAX_SAMPLES)) + } + }; + let iters = if opts.best { 4 } else { 3 }; + let work = if opts.best { 960 } else { 480 }; + let f = w.max(h).div_ceil(work).max(1); + let mut labels: Vec = match &soft_prior { + Some(p) => p.iter().map(|v| (*v >= 0.5) as u8).collect(), + None => fgs.iter().map(|v| *v as u8).collect(), + }; + for (l, hd) in labels.iter_mut().zip(&hard) { + if *hd != 0 { + *l = (*hd == HARD_FG) as u8; + } + } + let (mut fgm, mut bgm) = (fg0, bg0); + if f > 1 { + let (crgb, cw, ch) = downsample_rgb(&rgb, w, h, f); + let mut chard = vec![0u8; cw * ch]; + let mut cprior = soft_prior.as_ref().map(|_| vec![0.0f32; cw * ch]); + for cy in 0..ch { + for cx in 0..cw { + let (mut anyf, mut anyb, mut s, mut n) = (false, false, 0.0f32, 0.0f32); + for y in cy * f..((cy + 1) * f).min(h) { + for x in cx * f..((cx + 1) * f).min(w) { + let i = y * w + x; + anyf |= hard[i] == HARD_FG; + anyb |= hard[i] == HARD_BG; + if let Some(p) = &soft_prior { + s += p[i]; + } + n += 1.0; + } + } + chard[cy * cw + cx] = match (anyf, anyb) { + (true, false) => HARD_FG, + (false, true) => HARD_BG, + _ => 0, + }; + if let Some(cp) = &mut cprior { + cp[cy * cw + cx] = s / n; + } + } + } + let mut cl: Vec = match &cprior { + Some(p) => p.iter().map(|v| (*v >= 0.5) as u8).collect(), + None => chard.iter().map(|v| (*v == HARD_FG) as u8).collect(), + }; + let free = vec![true; cw * ch]; + for it in 0..iters { + cut(&crgb, cw, ch, &mut cl, &free, &chard, &fgm, &bgm, cprior.as_deref()); + if it + 1 < iters { + let (a, b) = fit_models(&crgb, &cl); + if !a.is_empty() && !b.is_empty() { + (fgm, bgm) = (a, b); + } + } + } + for y in 0..h { + for x in 0..w { + let i = y * w + x; + labels[i] = if hard[i] != 0 { (hard[i] == HARD_FG) as u8 } else { cl[(y / f) * cw + x / f] }; + } + } + let (a, b) = fit_models(&rgb, &labels); + if !a.is_empty() && !b.is_empty() { + (fgm, bgm) = (a, b); + } + // Full-resolution band. + let sd = matting::signed_distance(&labels, w, h); + let band = f as f32 + 1.5; + let free: Vec = sd.iter().map(|d| d.abs() <= band).collect(); + cut(&rgb, w, h, &mut labels, &free, &hard, &fgm, &bgm, soft_prior.as_deref()); + } else { + let free = vec![true; w * h]; + for it in 0..iters { + cut(&rgb, w, h, &mut labels, &free, &hard, &fgm, &bgm, soft_prior.as_deref()); + if it + 1 < iters { + let (a, b) = fit_models(&rgb, &labels); + if !a.is_empty() && !b.is_empty() { + (fgm, bgm) = (a, b); + } + } + } + } + // Keep what the strokes (or the prior's core) select. + let mut seeds = fgs.clone(); + if let Some(p) = &soft_prior { + let core = core_of(p, w, h, opts.search_radius * opts.scale); + for (s, c) in seeds.iter_mut().zip(core) { + *s |= c; + } + } + clean(&mut labels, &seeds, &bgs, w, h); + finish(img, &rgb, labels, strokes, prior.map(|p| p.refine.as_slice()), opts, refine_radius) +} + +/// Pixels deep inside a soft matte (more than `r` from its edge), else the matte itself. +fn core_of(soft: &[f32], w: usize, h: usize, r: f64) -> Vec { + let bin: Vec = soft.iter().map(|v| (*v >= 0.5) as u8).collect(); + let sd = matting::signed_distance(&bin, w, h); + let core: Vec = sd.iter().map(|d| *d as f64 > r).collect(); + if core.iter().any(|c| *c) { core } else { bin.iter().map(|v| *v != 0).collect() } +} + +/// Final models and Refine Edge band of a segmented frame. +fn finish(_img: &Image, rgb: &[[f32; 3]], labels: Vec, strokes: &[&Stroke], warped_refine: Option<&[u8]>, opts: &SegOpts, refine_radius: f64) -> FrameSeg { + let (w, h) = (_img.width as usize, _img.height as usize); + let (fg, bg) = fit_models(rgb, &labels); + let add = rasterize(strokes, &[StrokeKind::Refine], opts.scale, w, h); + let erase = rasterize(strokes, &[StrokeKind::RefineErase], opts.scale, w, h); + let near: Option> = warped_refine.filter(|r| r.iter().any(|v| *v != 0)).map(|_| matting::signed_distance(&labels, w, h)); + let rr = (refine_radius * opts.scale + 2.0) as f32; + let refine: Vec = (0..w * h) + .map(|i| { + let carried = match (warped_refine, &near) { + (Some(r), Some(sd)) => r[i] != 0 && sd[i].abs() <= rr, + _ => false, + }; + ((carried || add[i]) && !erase[i]) as u8 + }) + .collect(); + FrameSeg { w, h, matte: labels, refine, fg, bg } +} + +/// Warp the previous frame's segmentation onto the next frame by optical flow (next → prev), +/// with Motion Threshold / Damping applied to the vectors. +pub fn warp_prior(prev_img: &Image, prev: &FrameSeg, next_img: &Image, opts: &SegOpts) -> Prior { + let (w, h) = (next_img.width as usize, next_img.height as usize); + let flow = effectcraft_raster::flow::block_flow(next_img, prev_img, 8, 4); + let thr = (opts.motion_threshold / 100.0 * opts.search_radius * opts.scale).max(0.0); + let damp = (1.0 - opts.motion_damping / 100.0).clamp(0.0, 1.0); + let src: Vec = prev.matte.iter().map(|v| *v as f32).collect(); + let pw = prev.w.min(w); + let ph = prev.h.min(h); + let res: Vec<(f32, u8)> = (0..w * h) + .into_par_iter() + .map(|i| { + let (x, y) = ((i % w) as f64 + 0.5, (i / w) as f64 + 0.5); + let v = flow.at(x, y); + let (mut vx, mut vy) = (v[0] as f64, v[1] as f64); + if vx.hypot(vy) < thr { + vx *= damp; + vy *= damp; + } + let (sx, sy) = (x + vx, y + vy); + let soft = if pw == prev.w && ph == prev.h { matting::sample(&src, prev.w, prev.h, sx, sy) } else { 0.0 }; + let (ix, iy) = (sx.floor().clamp(0.0, (prev.w.max(1) - 1) as f64) as usize, sy.floor().clamp(0.0, (prev.h.max(1) - 1) as f64) as usize); + let r = prev.refine.get(iy * prev.w + ix).copied().unwrap_or(0); + (soft, r) + }) + .collect(); + Prior { soft: res.iter().map(|r| r.0).collect(), fg: prev.fg.clone(), bg: prev.bg.clone(), refine: res.iter().map(|r| r.1).collect() } +} + +/// Propagate the previous frame's segmentation to the next frame: warp it, then re-cut in a +/// band of Search Radius pixels around the warped boundary. Strokes on the next frame re-cut +/// the whole frame with the warped matte as a soft prior instead. +pub fn propagate(prev_img: &Image, prev: &FrameSeg, next_img: &Image, strokes: &[&Stroke], opts: &SegOpts, refine_radius: f64) -> FrameSeg { + let (w, h) = (next_img.width as usize, next_img.height as usize); + let prior = warp_prior(prev_img, prev, next_img, opts); + if strokes.iter().any(|s| s.kind.is_segmentation()) { + return segment(next_img, strokes, Some(&prior), opts, refine_radius); + } + let rgb = rgb_of(next_img); + let mut labels: Vec = prior.soft.iter().map(|v| (*v >= 0.5) as u8).collect(); + let sd = matting::signed_distance(&labels, w, h); + let r = (opts.search_radius * opts.scale).max(2.0) as f32; + let free: Vec = sd.iter().map(|d| d.abs() <= r).collect(); + let hard = vec![0u8; w * h]; + let soft = matting::gauss(&prior.soft, w, h, (r as f64 / 4.0).max(1.0)); + let (fg, bg) = if prev.fg.is_empty() || prev.bg.is_empty() { fit_models(&rgb, &labels) } else { (prev.fg.clone(), prev.bg.clone()) }; + cut(&rgb, w, h, &mut labels, &free, &hard, &fg, &bg, Some(&soft)); + let seeds = core_of(&prior.soft, w, h, r as f64); + clean(&mut labels, &seeds, &vec![false; w * h], w, h); + finish(next_img, &rgb, labels, strokes, Some(&prior.refine), opts, refine_radius) +} + +/// Intersection over union of two binary mattes. +pub fn iou(a: &[u8], b: &[u8]) -> f64 { + let (mut i, mut u) = (0usize, 0usize); + for (x, y) in a.iter().zip(b) { + let (x, y) = (*x != 0, *y != 0); + i += (x && y) as usize; + u += (x || y) as usize; + } + if u == 0 { 1.0 } else { i as f64 / u as f64 } +} + +#[cfg(test)] +mod tests; diff --git a/crates/track/src/roto/refine.rs b/crates/track/src/roto/refine.rs new file mode 100644 index 00000000..2a498e28 --- /dev/null +++ b/crates/track/src/roto/refine.rs @@ -0,0 +1,198 @@ +//! From a frame's binary segmentation to the final soft matte: the Roto Brush Matte and Refine +//! Edge Matte adjustments, edge matting inside the Refine Edge band (colour guided filter), +//! Reduce Chatter (temporal averaging with the neighbouring frames), matte motion blur and edge +//! colour decontamination. + +use super::FrameSeg; +use super::matting::{self, contrast, gauss, guided_filter, shift_soft, signed_distance}; + +/// The effect's matte parameters (After Effects' units: pixels and percent). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct MatteParams { + // Roto Brush Matte. + pub rb_feather: f64, + pub rb_contrast: f64, + pub rb_shift: f64, + pub rb_chatter: f64, + // Refine Edge Matte. + pub smooth: f64, + pub feather: f64, + pub contrast: f64, + pub shift: f64, + pub chatter: f64, + pub motion_blur: bool, + pub mb_samples: usize, + pub shutter_angle: f64, + /// Higher Quality: closed-form matting after the guided-filter estimate. + pub higher_quality: bool, + pub decontaminate: bool, + pub decontamination: f64, + pub extend_where_smoothed: bool, + pub increase_radius: f64, +} + +impl Default for MatteParams { + fn default() -> Self { + MatteParams { + rb_feather: 0.0, + rb_contrast: 0.0, + rb_shift: 0.0, + rb_chatter: 0.0, + smooth: 1.0, + feather: 0.0, + contrast: 0.0, + shift: 0.0, + chatter: 0.0, + motion_blur: false, + mb_samples: 11, + shutter_angle: 180.0, + higher_quality: true, + decontaminate: true, + decontamination: 100.0, + extend_where_smoothed: true, + increase_radius: 0.0, + } + } +} + +fn shifted(seg: &FrameSeg, px: f64) -> Vec { + if px.abs() < 0.25 { + return seg.matte.iter().map(|v| *v as f32).collect(); + } + let sd = signed_distance(&seg.matte, seg.w, seg.h); + sd.iter().map(|d| ((*d as f64) > -px) as u8 as f32).collect() +} + +/// The soft matte of `seg` (all planes `seg.w × seg.h`), with `rgb` the frame's straight colour. +/// `prev` / `next`: the neighbouring frames' segmentations (Reduce Chatter, motion blur). +/// `band_radius`: the Refine Edge brush radius in these pixels; `scale`: pixels per layer pixel. +pub fn matte_alpha( + rgb: &[[f32; 3]], + seg: &FrameSeg, + prev: Option<&FrameSeg>, + next: Option<&FrameSeg>, + p: &MatteParams, + band_radius: f64, + scale: f64, +) -> Vec { + let (w, h) = (seg.w, seg.h); + let n = w * h; + let same = |s: &&FrameSeg| s.w == w && s.h == h; + let (prev, next) = (prev.filter(same), next.filter(same)); + // Roto Brush Matte: Shift Edge (100 % = 10 px), Reduce Chatter, Feather, Contrast. + let shift_px = p.rb_shift / 100.0 * 10.0 * scale; + let mut a = shifted(seg, shift_px); + let nb: Vec> = [prev, next].into_iter().flatten().map(|s| shifted(s, shift_px)).collect(); + if p.rb_chatter > 0.0 && !nb.is_empty() { + let k = (p.rb_chatter / 100.0).clamp(0.0, 1.0) as f32 * (nb.len() as f32 / (nb.len() as f32 + 1.0)); + for i in 0..n { + let m = nb.iter().map(|v| v[i]).sum::() / nb.len() as f32; + a[i] += (m - a[i]) * k; + } + } + if p.rb_feather > 0.0 { + a = gauss(&a, w, h, p.rb_feather * scale / 2.0); + } + contrast(&mut a, p.rb_contrast); + // Refine Edge band: guided-filter matting. + if seg.refine.iter().any(|v| *v != 0) && seg.refine.len() == n { + let r = (band_radius * 1.0).round().max(2.0) as usize; + let eps = (2e-4 * (1.0 + p.smooth.max(0.0))) as f32; + // Work on the band's bounding box (plus the filter's reach). + let (mut x0, mut y0, mut x1, mut y1) = (w, h, 0, 0); + for (i, v) in seg.refine.iter().enumerate() { + if *v != 0 { + let (x, y) = (i % w, i / w); + x0 = x0.min(x); + y0 = y0.min(y); + x1 = x1.max(x + 1); + y1 = y1.max(y + 1); + } + } + let pad = 2 * r + 2; + let (x0, y0, x1, y1) = (x0.saturating_sub(pad), y0.saturating_sub(pad), (x1 + pad).min(w), (y1 + pad).min(h)); + let (cw, ch) = (x1 - x0, y1 - y0); + let crop = |src: &dyn Fn(usize) -> f32| -> Vec { (0..cw * ch).map(|j| src((y0 + j / cw) * w + x0 + j % cw)).collect() }; + let cimg: Vec<[f32; 3]> = (0..cw * ch).map(|j| rgb[(y0 + j / cw) * w + x0 + j % cw]).collect(); + let cp = crop(&|i| a[i]); + let band: Vec = crop(&|i| seg.refine[i] as f32).iter().map(|v| *v > 0.0).collect(); + let outside: Vec = band.iter().map(|b| !b).collect(); + // Guided filter estimate, then closed-form matting with everything outside the band known. + let mut q = guided_filter(&cimg, &cp, cw, ch, r, eps); + for j in 0..cw * ch { + q[j] = if band[j] { q[j].clamp(0.0, 1.0) } else { cp[j] }; + } + if p.higher_quality { + q = matting::closed_form(&cimg, &q, &outside, cw, ch, 1, 1e-5 * (1.0 + p.smooth.max(0.0)) as f32, 50); + } + // Refine Edge Matte adjustments. + if p.smooth > 1.0 { + q = gauss(&q, cw, ch, (p.smooth - 1.0) * 0.5 * scale); + } + if p.feather > 0.0 { + q = gauss(&q, cw, ch, p.feather * scale / 2.0); + } + contrast(&mut q, p.contrast); + shift_soft(&mut q, p.shift); + // Blend in, fading over the band's last two pixels. + let din = matting::distance(&outside, cw, ch); + for j in 0..cw * ch { + if !band[j] { + continue; + } + let wgt = (din[j] / 2.0).clamp(0.0, 1.0); + let i = (y0 + j / cw) * w + x0 + j % cw; + a[i] += (q[j] - a[i]) * wgt; + } + // Refine-stage Reduce Chatter: towards the neighbours' mattes inside the band. + if p.chatter > 0.0 && !nb.is_empty() { + let k = (p.chatter / 100.0).clamp(0.0, 1.0) as f32 * 0.5; + let soft: Vec> = nb.iter().map(|v| gauss(v, w, h, (r as f64 / 2.0).max(1.0))).collect(); + for i in 0..n { + if seg.refine[i] != 0 { + let m = soft.iter().map(|v| v[i]).sum::() / soft.len() as f32; + a[i] += (m - a[i]) * k; + } + } + } + } + // Motion blur: the matte's motion between the neighbouring frames. + if p.motion_blur { + let c = seg.centroid(); + let v = match (prev.and_then(FrameSeg::centroid), next.and_then(FrameSeg::centroid), c) { + (Some(a), Some(b), _) => Some([(b[0] - a[0]) / 2.0, (b[1] - a[1]) / 2.0]), + (Some(a), None, Some(c)) => Some([c[0] - a[0], c[1] - a[1]]), + (None, Some(b), Some(c)) => Some([b[0] - c[0], b[1] - c[1]]), + _ => None, + }; + if let Some(v) = v { + let s = p.shutter_angle / 360.0; + a = matting::motion_blur(&a, w, h, [v[0] * s, v[1] * s], p.mb_samples); + } + } + a.iter_mut().for_each(|v| *v = v.clamp(0.0, 1.0)); + a +} + +/// Decontaminate edge colours of `rgb` in place for the matte `alpha` (see +/// [`matting::decontaminate`]); returns the decontamination map. `band`: the Refine Edge band +/// (pixels where decontamination always applies). +pub fn decontaminate(rgb: &mut [[f32; 3]], alpha: &[f32], band: &[u8], w: usize, h: usize, p: &MatteParams, band_radius: f64, scale: f64) -> Vec { + if !p.decontaminate { + return vec![0.0; w * h]; + } + let grow = p.increase_radius * scale; + let in_band: Vec = if band.len() == w * h && band.iter().any(|v| *v != 0) { + if grow > 0.5 { + let d = matting::distance(&band.iter().map(|v| *v != 0).collect::>(), w, h); + d.iter().map(|v| (*v as f64) <= grow).collect() + } else { + band.iter().map(|v| *v != 0).collect() + } + } else { + vec![false; w * h] + }; + let region: Vec = (0..w * h).map(|i| alpha[i] > 0.004 && alpha[i] < 0.996 && (in_band[i] || p.extend_where_smoothed)).collect(); + let radius = (band_radius + grow).max(4.0 * scale).round() as usize; + matting::decontaminate(rgb, alpha, ®ion, w, h, radius, (p.decontamination / 100.0).clamp(0.0, 1.0) as f32) +} diff --git a/crates/track/src/roto/rle.rs b/crates/track/src/roto/rle.rs new file mode 100644 index 00000000..308b2689 --- /dev/null +++ b/crates/track/src/roto/rle.rs @@ -0,0 +1,136 @@ +//! Compact text encodings of mattes for project storage: run-length coding (LEB128 varints) +//! wrapped in standard base64 (RFC 4648 §4). + +const B64: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +pub fn base64_encode(data: &[u8]) -> String { + let mut out = String::with_capacity(data.len().div_ceil(3) * 4); + for c in data.chunks(3) { + let b = [c[0], *c.get(1).unwrap_or(&0), *c.get(2).unwrap_or(&0)]; + let n = (b[0] as u32) << 16 | (b[1] as u32) << 8 | b[2] as u32; + out.push(B64[(n >> 18) as usize & 63] as char); + out.push(B64[(n >> 12) as usize & 63] as char); + out.push(if c.len() > 1 { B64[(n >> 6) as usize & 63] as char } else { '=' }); + out.push(if c.len() > 2 { B64[n as usize & 63] as char } else { '=' }); + } + out +} + +pub fn base64_decode(s: &str) -> Option> { + let val = |c: u8| -> Option { + Some(match c { + b'A'..=b'Z' => (c - b'A') as u32, + b'a'..=b'z' => (c - b'a' + 26) as u32, + b'0'..=b'9' => (c - b'0' + 52) as u32, + b'+' => 62, + b'/' => 63, + _ => return None, + }) + }; + let bytes: Vec = s.bytes().filter(|c| !c.is_ascii_whitespace()).collect(); + if !bytes.len().is_multiple_of(4) { + return None; + } + let mut out = Vec::with_capacity(bytes.len() / 4 * 3); + for c in bytes.chunks(4) { + let pad = c.iter().rev().take_while(|b| **b == b'=').count(); + let mut n = 0u32; + for (i, b) in c.iter().enumerate() { + n |= if *b == b'=' { 0 } else { val(*b)? } << (18 - 6 * i); + } + out.push((n >> 16) as u8); + if pad < 2 { + out.push((n >> 8) as u8); + } + if pad < 1 { + out.push(n as u8); + } + } + Some(out) +} + +fn put(out: &mut Vec, mut v: u64) { + loop { + let b = (v & 0x7f) as u8; + v >>= 7; + if v == 0 { + out.push(b); + return; + } + out.push(b | 0x80); + } +} + +fn get(data: &[u8], pos: &mut usize) -> Option { + let mut v = 0u64; + let mut shift = 0; + loop { + let b = *data.get(*pos)?; + *pos += 1; + v |= ((b & 0x7f) as u64) << shift; + if b & 0x80 == 0 { + return Some(v); + } + shift += 7; + if shift > 63 { + return None; + } + } +} + +/// Encode a byte plane as (value, run length) pairs. +pub fn encode(values: &[u8]) -> String { + let mut out = vec![]; + put(&mut out, values.len() as u64); + let mut i = 0; + while i < values.len() { + let v = values[i]; + let mut j = i + 1; + while j < values.len() && values[j] == v { + j += 1; + } + out.push(v); + put(&mut out, (j - i) as u64); + i = j; + } + base64_encode(&out) +} + +/// Decode [`encode`]'s output (None when malformed). +pub fn decode(s: &str) -> Option> { + let data = base64_decode(s)?; + let mut pos = 0; + let n = get(&data, &mut pos)? as usize; + if n > 1 << 30 { + return None; + } + let mut out = Vec::with_capacity(n); + while out.len() < n { + let v = *data.get(pos)?; + pos += 1; + let run = get(&data, &mut pos)? as usize; + if out.len() + run > n { + return None; + } + out.resize(out.len() + run, v); + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trips() { + for n in [0usize, 1, 2, 3, 4, 5, 100, 1000] { + let v: Vec = (0..n).map(|i| ((i / 7) % 3) as u8 * 100).collect(); + assert_eq!(decode(&encode(&v)).unwrap(), v); + let raw: Vec = (0..n).map(|i| (i * 31) as u8).collect(); + assert_eq!(base64_decode(&base64_encode(&raw)).unwrap(), raw); + } + assert_eq!(base64_encode(b"Man"), "TWFu"); + assert_eq!(base64_encode(b"Ma"), "TWE="); + assert!(decode("!!!!").is_none()); + } +} diff --git a/crates/track/src/roto/tests.rs b/crates/track/src/roto/tests.rs new file mode 100644 index 00000000..bec3f378 --- /dev/null +++ b/crates/track/src/roto/tests.rs @@ -0,0 +1,331 @@ +//! Roto Brush on synthetic sequences: a textured disk moving over a textured background. + +use super::refine::{MatteParams, decontaminate, matte_alpha}; +use super::*; + +pub(crate) const W: u32 = 200; +pub(crate) const H: u32 = 150; +pub(crate) const R: f64 = 32.0; + +/// Smooth value noise in 0..1. +pub(crate) fn noise(x: f64, y: f64, seed: u32) -> f64 { + fn h(i: i64, j: i64, s: u32) -> f64 { + let mut v = (i.wrapping_mul(73856093) ^ j.wrapping_mul(19349663) ^ (s as i64).wrapping_mul(83492791)) as u64; + v ^= v >> 13; + v = v.wrapping_mul(0x5bd1e995); + v ^= v >> 15; + (v % 10_000) as f64 / 10_000.0 + } + let mut out = 0.0; + for (cell, amp) in [(9.0, 0.6), (3.5, 0.4)] { + let (fx, fy) = (x / cell, y / cell); + let (i, j) = (fx.floor() as i64, fy.floor() as i64); + let (tx, ty) = (fx - i as f64, fy - j as f64); + let (sx, sy) = (tx * tx * (3.0 - 2.0 * tx), ty * ty * (3.0 - 2.0 * ty)); + let a = h(i, j, seed) + (h(i + 1, j, seed) - h(i, j, seed)) * sx; + let b = h(i, j + 1, seed) + (h(i + 1, j + 1, seed) - h(i, j + 1, seed)) * sx; + out += amp * (a + (b - a) * sy); + } + out +} + +pub(crate) fn fg_color(x: f64, y: f64) -> [f32; 3] { + let n = noise(x, y, 11); + [(0.75 + 0.25 * n) as f32, (0.35 + 0.3 * n) as f32, (0.1 + 0.15 * n) as f32] +} + +pub(crate) fn bg_color(x: f64, y: f64) -> [f32; 3] { + let n = noise(x, y, 5); + let m = noise(x * 1.7, y * 1.7, 9); + [(0.05 + 0.25 * m) as f32, (0.3 + 0.4 * n) as f32, (0.45 + 0.4 * m) as f32] +} + +pub(crate) fn center(f: i64) -> [f64; 2] { + let t = f as f64; + [60.0 + 3.1 * t, 70.0 + 1.3 * t + 4.0 * (t * 0.3).sin()] +} + +/// Frame `f`: the disk (texture moving with it) over the static background, with a 1-pixel +/// anti-aliased edge; `c` overrides the centre. +pub(crate) fn frame_at(c: [f64; 2]) -> (Image, Vec) { + let mut img = Image::new(W, H); + let mut gt = vec![0u8; (W * H) as usize]; + for y in 0..H { + for x in 0..W { + let p = [x as f64 + 0.5, y as f64 + 0.5]; + let d = (p[0] - c[0]).hypot(p[1] - c[1]); + let a = (R + 0.5 - d).clamp(0.0, 1.0) as f32; + let f = fg_color(p[0] - c[0], p[1] - c[1]); + let b = bg_color(p[0], p[1]); + img.set(x, y, [f[0] * a + b[0] * (1.0 - a), f[1] * a + b[1] * (1.0 - a), f[2] * a + b[2] * (1.0 - a), 1.0]); + gt[(y * W + x) as usize] = (d <= R) as u8; + } + } + (img, gt) +} + +fn line(kind: StrokeKind, frame: i64, a: [f64; 2], b: [f64; 2], radius: f64) -> Stroke { + Stroke { kind, frame, radius, points: (0..=10).map(|k| [a[0] + (b[0] - a[0]) * k as f64 / 10.0, a[1] + (b[1] - a[1]) * k as f64 / 10.0]).collect() } +} + +/// The strokes an artist would paint on frame `f`: one through the disk, one around it. +pub(crate) fn base_strokes(f: i64) -> Vec { + let c = center(f); + vec![ + line(StrokeKind::Fg, f, [c[0] - 18.0, c[1] - 6.0], [c[0] + 18.0, c[1] + 6.0], 5.0), + line(StrokeKind::Bg, f, [c[0] - 50.0, c[1] - 45.0], [c[0] + 50.0, c[1] - 45.0], 5.0), + line(StrokeKind::Bg, f, [c[0] + 50.0, c[1] + 45.0], [c[0] - 45.0, c[1] + 45.0], 5.0), + ] +} + +#[test] +fn strokes_segment_the_base_frame() { + let (img, gt) = frame_at(center(0)); + let st = base_strokes(0); + let refs: Vec<&Stroke> = st.iter().collect(); + let seg = segment(&img, &refs, None, &SegOpts::default(), 0.0); + let j = iou(&seg.matte, >); + assert!(j > 0.95, "IoU {j}"); + // Deterministic. + let again = segment(&img, &refs, None, &SegOpts::default(), 0.0); + assert_eq!(seg, again); +} + +#[test] +fn coarse_to_fine_segments_large_frames() { + // A 2× upscaled frame goes through the reduced-size path (Standard works at 480 px). + let (small, _) = frame_at(center(0)); + let img = effectcraft_raster::resample(&small, W * 3, H * 3); + let c = center(0); + let st: Vec = base_strokes(0) + .into_iter() + .map(|mut s| { + s.points.iter_mut().for_each(|p| *p = [p[0] * 3.0, p[1] * 3.0]); + s.radius *= 3.0; + s + }) + .collect(); + let refs: Vec<&Stroke> = st.iter().collect(); + let seg = segment(&img, &refs, None, &SegOpts::default(), 0.0); + let gt: Vec = + (0..W * H * 9).map(|i| ((((i % (W * 3)) as f64 + 0.5) / 3.0 - c[0]).hypot((((i / (W * 3)) as f64 + 0.5) / 3.0) - c[1]) <= R) as u8).collect(); + let j = iou(&seg.matte, >); + assert!(j > 0.95, "IoU {j}"); +} + +#[test] +fn propagation_follows_the_disk_for_20_frames() { + let (img0, _) = frame_at(center(0)); + let st = base_strokes(0); + let refs: Vec<&Stroke> = st.iter().collect(); + let opts = SegOpts::default(); + let mut seg = segment(&img0, &refs, None, &opts, 0.0); + let mut prev = img0; + let mut worst: f64 = 1.0; + for f in 1..=20 { + let (img, gt) = frame_at(center(f)); + seg = propagate(&prev, &seg, &img, &[], &opts, 0.0); + let j = iou(&seg.matte, >); + worst = worst.min(j); + prev = img; + } + assert!(worst > 0.9, "worst IoU {worst}"); +} + +#[test] +fn a_correction_stroke_fixes_a_bad_frame() { + let opts = SegOpts::default(); + let (img0, _) = frame_at(center(0)); + let st = base_strokes(0); + let refs: Vec<&Stroke> = st.iter().collect(); + let seg0 = segment(&img0, &refs, None, &opts, 0.0); + // The disk jumps far away on frame 1: propagation cannot follow. + let jump = [150.0, 90.0]; + let (img1, gt1) = frame_at(jump); + let bad = propagate(&img0, &seg0, &img1, &[], &opts, 0.0); + assert!(iou(&bad.matte, >1) < 0.5, "the jump should break propagation"); + // Paint the disk in and the old place out. + let c0 = center(0); + let fix = [ + line(StrokeKind::Fg, 1, [jump[0] - 15.0, jump[1]], [jump[0] + 15.0, jump[1]], 5.0), + line(StrokeKind::Bg, 1, [c0[0] - 10.0, c0[1] - 10.0], [c0[0] + 10.0, c0[1] + 10.0], 6.0), + ]; + let fr: Vec<&Stroke> = fix.iter().collect(); + let good = propagate(&img0, &seg0, &img1, &fr, &opts, 0.0); + let j = iou(&good.matte, >1); + assert!(j > 0.95, "corrected IoU {j}"); + // And propagation continues from the corrected frame. + let c2 = [jump[0] + 3.0, jump[1] + 1.0]; + let (img2, gt2) = frame_at(c2); + let next = propagate(&img1, &good, &img2, &[], &opts, 0.0); + assert!(iou(&next.matte, >2) > 0.93); +} + +/// A soft-edged disk (alpha ramp over `width` pixels) over a background. +fn soft_frame(c: [f64; 2], width: f64, fg: [f32; 3], bg: [f32; 3]) -> (Image, Vec) { + let mut img = Image::new(W, H); + let mut gt = vec![0.0f32; (W * H) as usize]; + for y in 0..H { + for x in 0..W { + let p = [x as f64 + 0.5, y as f64 + 0.5]; + let d = (p[0] - c[0]).hypot(p[1] - c[1]); + let a = ((R + width / 2.0 - d) / width).clamp(0.0, 1.0) as f32; + // Faint texture on both layers. + let tf = (noise(p[0], p[1], 3) as f32 - 0.5) * 0.04; + let tb = (noise(p[0], p[1], 4) as f32 - 0.5) * 0.04; + let f = [fg[0] + tf, fg[1] + tf, fg[2] + tf]; + let b = [bg[0] + tb, bg[1] + tb, bg[2] + tb]; + img.set(x, y, [f[0] * a + b[0] * (1.0 - a), f[1] * a + b[1] * (1.0 - a), f[2] * a + b[2] * (1.0 - a), 1.0]); + gt[(y * W + x) as usize] = a; + } + } + (img, gt) +} + +fn ring(frame: i64, c: [f64; 2], r: f64, radius: f64) -> Stroke { + Stroke { + kind: StrokeKind::Refine, + frame, + radius, + points: (0..=48).map(|k| k as f64 / 48.0 * std::f64::consts::TAU).map(|a| [c[0] + r * a.cos(), c[1] + r * a.sin()]).collect(), + } +} + +#[test] +fn refine_edge_recovers_a_soft_edge_and_decontaminates() { + let c = [100.0, 75.0]; + let (fgc, bgc) = ([0.9f32, 0.25, 0.1], [0.1f32, 0.3, 0.9]); + let (img, gt) = soft_frame(c, 10.0, fgc, bgc); + let mut st = vec![ + line(StrokeKind::Fg, 0, [c[0] - 15.0, c[1]], [c[0] + 15.0, c[1]], 5.0), + line(StrokeKind::Bg, 0, [c[0] - 60.0, c[1] - 55.0], [c[0] + 60.0, c[1] - 55.0], 5.0), + ]; + st.push(ring(0, c, R, 9.0)); + let refs: Vec<&Stroke> = st.iter().collect(); + let seg = segment(&img, &refs, None, &SegOpts::default(), 9.0); + assert!(seg.refine.iter().filter(|v| **v != 0).count() > 1000); + let p = MatteParams::default(); + let rgb = rgb_of(&img); + let a = matte_alpha(&rgb, &seg, None, None, &p, 9.0, 1.0); + // Inside the ramp the matte is fractional and close to the truth. + let (mut err, mut n, mut frac) = (0.0f64, 0usize, 0usize); + let (mut err_bin, mut n_bin) = (0.0f64, 0usize); + for i in 0..a.len() { + if gt[i] > 0.0 && gt[i] < 1.0 { + err += (a[i] - gt[i]).abs() as f64; + err_bin += (seg.matte[i] as f32 - gt[i]).abs() as f64; + n += 1; + n_bin += 1; + frac += (a[i] > 0.05 && a[i] < 0.95) as usize; + } + } + let (err, err_bin) = (err / n as f64, err_bin / n_bin as f64); + assert!(err < 0.03, "mean alpha error {err} (binary {err_bin})"); + assert!(err < err_bin * 0.4); + assert!(frac as f64 > 0.6 * n as f64, "{frac}/{n} fractional"); + // Decontamination: edge pixels' colour is the foreground's, not a blue blend. + let mut col = rgb.clone(); + let map = decontaminate(&mut col, &a, &seg.refine, W as usize, H as usize, &p, 9.0, 1.0); + let (mut blue_before, mut blue_after, mut m) = (0.0f64, 0.0f64, 0usize); + for i in 0..a.len() { + if a[i] > 0.2 && a[i] < 0.8 { + blue_before += rgb[i][2] as f64; + blue_after += col[i][2] as f64; + m += 1; + } + } + let (bb, ba) = (blue_before / m as f64, blue_after / m as f64); + assert!(ba < 0.2 && bb > 0.35, "blue before {bb}, after {ba}"); + assert!(map.iter().any(|v| *v > 0.2)); + // Without the band, the binary matte passes through. + let mut seg2 = seg.clone(); + seg2.refine.iter_mut().for_each(|v| *v = 0); + let a2 = matte_alpha(&rgb, &seg2, None, None, &p, 9.0, 1.0); + assert!(a2.iter().zip(&seg.matte).all(|(x, y)| (*x - *y as f32).abs() < 1e-6)); +} + +#[test] +fn reduce_chatter_and_motion_blur_use_the_neighbours() { + let (w, h) = (40usize, 20usize); + let mk = |x0: usize| FrameSeg { + w, + h, + matte: (0..w * h).map(|i| (i % w >= x0 && i % w < x0 + 10) as u8).collect(), + refine: vec![0; w * h], + ..Default::default() + }; + let (a, b, c) = (mk(10), mk(14), mk(18)); + let rgb = vec![[0.5f32; 3]; w * h]; + let p = MatteParams { rb_chatter: 100.0, ..Default::default() }; + let al = matte_alpha(&rgb, &b, Some(&a), Some(&c), &p, 0.0, 1.0); + // Pixel 12 is only in the previous frame: partially on. + assert!(al[12] > 0.2 && al[12] < 0.5, "{}", al[12]); + let p = MatteParams { motion_blur: true, ..Default::default() }; + let al = matte_alpha(&rgb, &b, Some(&a), Some(&c), &p, 0.0, 1.0); + assert!(al[14] > 0.3 && al[14] < 0.9, "{}", al[14]); + assert!(al[19] > 0.99); +} + +#[test] +fn chain_keys_restart_at_correction_frames() { + let mut d = RotoData::default(); + for s in base_strokes(5) { + d.add(s, 20, [0, 30]); + } + assert_eq!(d.base, Some(5)); + assert_eq!(d.span, [0, 25]); + let k1 = d.chain_keys(1); + d.add(line(StrokeKind::Fg, 12, [0.0, 0.0], [1.0, 1.0], 3.0), 20, [0, 30]); + let k2 = d.chain_keys(1); + for f in 0..=11 { + assert_eq!(k1[&f], k2[&f], "frame {f}"); + } + for f in 12..=25 { + assert_ne!(k1[&f], k2[&f], "frame {f}"); + } + assert_ne!(d.chain_keys(2)[&5], k2[&5]); + let back = RotoData::from_json(&d.to_json()); + assert_eq!(back, d); + assert_eq!(d.source_of(12), Some(11)); + assert_eq!(d.source_of(3), Some(4)); + assert_eq!(d.source_of(5), None); +} + +/// `cargo test -p effectcraft-track --release roto_perf -- --ignored --nocapture` +#[test] +#[ignore] +fn roto_perf_1080p() { + let up = |c: [f64; 2]| { + let (img, _) = frame_at(c); + effectcraft_raster::resample(&img, 1920, 1080) + }; + let s = 1920.0 / W as f64; + let st: Vec = base_strokes(0) + .into_iter() + .map(|mut k| { + k.points.iter_mut().for_each(|p| *p = [p[0] * s, p[1] * s]); + k.radius *= s; + k + }) + .collect(); + let refs: Vec<&Stroke> = st.iter().collect(); + let opts = SegOpts { search_radius: 15.0 * s / 4.0, ..Default::default() }; + let a = up(center(0)); + let t = std::time::Instant::now(); + let seg = segment(&a, &refs, None, &opts, 0.0); + eprintln!("segment 1080p: {:?}", t.elapsed()); + let b = up(center(1)); + let t = std::time::Instant::now(); + let next = propagate(&a, &seg, &b, &[], &opts, 0.0); + eprintln!("propagate 1080p: {:?} (area {})", t.elapsed(), next.area()); + let mut seg2 = next.clone(); + let c = center(1); + let ring: Vec<[f64; 2]> = (0..=64).map(|k| k as f64 / 64.0 * std::f64::consts::TAU).map(|t| [(c[0] + R * t.cos()) * s, (c[1] + R * t.sin()) * s]).collect(); + let st = [Stroke { kind: StrokeKind::Refine, frame: 1, radius: 20.0, points: ring }]; + let r = rasterize(&st.iter().collect::>(), &[StrokeKind::Refine], 1.0, 1920, 1080); + seg2.refine = r.iter().map(|v| *v as u8).collect(); + let rgb = rgb_of(&b); + let t = std::time::Instant::now(); + let _ = matte_alpha(&rgb, &seg2, None, None, &MatteParams::default(), 20.0, 1.0); + eprintln!("refine 1080p: {:?}", t.elapsed()); +} From ed72e67155e17a2b1349b66462b26bccf5ef06bc Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:27:22 -0400 Subject: [PATCH 2/2] M6.7: refine edge + freeze + UI Refine Edge tool (Alt+W cycles Roto Brush / Refine Edge), Layer panel roto strokes with matte overlays (Alpha Boundary in pink, Alpha, Alpha Overlay, View Search Region), segmentation span bar, Freeze/Propagate in the Layer panel and Effect Controls, background propagation polling, docs. Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 2 +- crates/engine/src/commands/roto_cmds.rs | 4 +- crates/ui-egui/src/icons.rs | 10 + crates/ui-egui/src/lib.rs | 1 + crates/ui-egui/src/menus.rs | 3 +- crates/ui-egui/src/panels/effect_controls.rs | 31 ++ crates/ui-egui/src/panels/layer_panel.rs | 39 ++- crates/ui-egui/src/panels/mod.rs | 1 + crates/ui-egui/src/panels/roto_tool.rs | 331 +++++++++++++++++++ crates/ui-egui/src/state.rs | 21 +- docs/agents.md | 22 ++ docs/architecture.md | 13 + docs/parity.md | 4 +- 13 files changed, 467 insertions(+), 15 deletions(-) create mode 100644 crates/ui-egui/src/panels/roto_tool.rs diff --git a/ROADMAP.md b/ROADMAP.md index a3ed979c..55b05822 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -32,6 +32,6 @@ Measured feature by feature in [docs/parity.md](docs/parity.md) (end of 2 Octobe - **≈ 100 agent-hours of work remain by the audit's estimate.** With five Claude Opus 5.5 agents in parallel that is **≈ 8–10 hours of wall-clock time at the pace measured so far, up to ≈ 25 hours** by the audit's conservative per-feature figures. -- Hardest remaining pieces: Roto Brush, the 3D camera tracker, Advanced 3D, a scripting object model. +- Hardest remaining pieces: the 3D camera tracker, Advanced 3D, a scripting object model (Roto Brush & Refine Edge landed in M6.7). Come tell us what matters most to you on [Discord](https://discord.gg/artcraft). diff --git a/crates/engine/src/commands/roto_cmds.rs b/crates/engine/src/commands/roto_cmds.rs index 5b9db20f..95326b42 100644 --- a/crates/engine/src/commands/roto_cmds.rs +++ b/crates/engine/src/commands/roto_cmds.rs @@ -378,14 +378,12 @@ fn status(s: &mut Session, p: &Value) -> Result { let (layer, g) = group(s, cid, lid, uid)?; let (layer, g) = (layer.clone(), g.clone()); let d = data_of(&g); - let computed = s.roto_computed(cid, lid, uid); out["layer"] = json!(lid.0); out["effect"] = json!(uid); out["base"] = json!(d.base); out["span"] = json!(d.span); out["strokes"] = json!(d.strokes.len()); out["strokeFrames"] = json!(d.stroke_frames()); - out["computed"] = json!(computed.iter().filter(|(_, c)| **c).map(|(f, _)| *f).collect::>()); out["frozen"] = json!(frozen_of(&g)); out["pending"] = json!(s.roto_pending.contains(&(cid, lid, uid))); out["limits"] = json!(rt::frame_limits(&comp, &layer)); @@ -420,5 +418,7 @@ fn status(s: &mut Session, p: &Value) -> Result { } } } + let computed = s.roto_computed(cid, lid, uid); + out["computed"] = json!(computed.iter().filter(|(_, c)| **c).map(|(f, _)| *f).collect::>()); Ok(out) } diff --git a/crates/ui-egui/src/icons.rs b/crates/ui-egui/src/icons.rs index d28cb950..d2de8fc3 100644 --- a/crates/ui-egui/src/icons.rs +++ b/crates/ui-egui/src/icons.rs @@ -33,6 +33,7 @@ pub enum Icon { Clone, Eraser, RotoBrush, + RefineEdge, Puppet, // switches & timeline Eye, @@ -282,6 +283,15 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) { pen.ellipse(6.0, 10.0, 3.6, 3.6); pen.dot(6.0, 10.0, 1.0); } + RefineEdge => { + // A brush over a soft (dashed) edge. + pen.line(&[(13.5, 2.5), (9.0, 7.0)]); + pen.ellipse(7.0, 9.0, 2.6, 2.6); + pen.line(&[(2.0, 14.0), (3.5, 12.5)]); + pen.line(&[(5.0, 14.0), (6.5, 12.5)]); + pen.line(&[(8.0, 14.0), (9.5, 12.5)]); + pen.line(&[(11.0, 14.0), (12.5, 12.5)]); + } Puppet => { pen.circle(8.0, 3.6, 1.6); pen.line(&[(8.0, 5.2), (8.0, 10.0)]); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 071c1948..71cb9f45 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -725,6 +725,7 @@ impl EffectcraftApp { // Warp Stabilizer: finish analyses and start queued (re-)analyses in the background. if self.session.warp_job.is_some() || !self.session.warp_pending.is_empty() { self.session.poll_warp(true); + self.session.poll_roto(true); ctx.request_repaint_after(std::time::Duration::from_millis(100)); } self.apply_prefs(&ctx); diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index da5eaec2..b14ade47 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -72,7 +72,7 @@ pub const UI_COMMANDS: &[UiCommand] = &[ uic!("tool.pen", "Pen Tool", [], Some("G")), uic!("tool.type", "Type Tools", [], Some("Cmd+T")), uic!("tool.brush", "Brush Tools", [], Some("Cmd+B")), - uic!("tool.rotoBrush", "Roto Brush Tool", [], Some("Alt+W")), + uic!("tool.rotoBrush", "Roto Brush & Refine Edge Tools", [], Some("Alt+W")), uic!("tool.puppet", "Puppet Tools", [], Some("Cmd+P")), uic!("app.newComp", "New Composition...", [], None), uic!("app.compSettings", "Composition Settings...", [], None), @@ -139,6 +139,7 @@ pub fn invoke(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, params: V "pen" => Some(9), "type" => Some(10), "brush" => Some(11), + "rotoBrush" => Some(14), "puppet" => Some(15), _ => None, }; diff --git a/crates/ui-egui/src/panels/effect_controls.rs b/crates/ui-egui/src/panels/effect_controls.rs index 0a19d57f..03afd94c 100644 --- a/crates/ui-egui/src/panels/effect_controls.rs +++ b/crates/ui-egui/src/panels/effect_controls.rs @@ -537,6 +537,35 @@ fn warp_editor(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, l p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim); } +/// Roto Brush & Refine Edge: Freeze / Unfreeze and Propagate buttons and the segmentation status. +fn roto_editor(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, layer: &Layer, g: &PropGroup, r: Rect, actions: &mut Actions) { + use effectcraft_engine::effects::roto as fx; + let t = app.tokens; + let params = effectcraft_engine::effects::flatten_params(g, &mut |pr| pr.value.clone()); + let frozen = fx::is_frozen(¶ms); + let d = fx::data(¶ms); + let running = app.session.roto_target().is_some_and(|(_, l, u, _)| l == layer.id && u == g.uid) && app.session.is_roto_running(); + let b1 = Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 4.0), vec2(84.0, 22.0)); + let b2 = Rect::from_min_size(pos2(b1.max.x + 8.0, b1.min.y), vec2(84.0, 22.0)); + let label = if frozen { "Unfreeze" } else { "Freeze" }; + if widgets::text_button(ui, b1, label, frozen, &t, egui::Id::new(("roto-freeze-ec", g.uid))).clicked() && !running { + let id = if frozen { "roto.unfreeze" } else { "roto.freeze" }; + actions.push((id.into(), json!({"layer": layer.id.0, "effect": g.uid}))); + } + app.auto.add(&format!("effectControls.roto.{}.freeze", g.uid), b1, label); + let (l2, id2) = if running { ("Stop", "roto.cancel") } else { ("Propagate", "roto.propagate") }; + if widgets::text_button(ui, b2, l2, false, &t, egui::Id::new(("roto-prop-ec", g.uid))).clicked() && !frozen { + actions.push((id2.into(), json!({"layer": layer.id.0, "effect": g.uid}))); + } + app.auto.add(&format!("effectControls.roto.{}.propagate", g.uid), b2, l2); + let status = match (app.session.roto_progress(), d.base) { + (Some(pr), _) if running => pr.banner(), + (_, None) => "No strokes: paint with the Roto Brush tool (Alt+W) in the Layer panel".to_string(), + (_, Some(b)) => format!("Base frame {b}, span {}–{}{}", d.span[0], d.span[1], if frozen { " (frozen)" } else { "" }), + }; + p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim); +} + // --------------------------------------------------------------------------------------------- // Curves / Levels editors @@ -545,6 +574,7 @@ fn editor_height(effect: &str, width: f32) -> f32 { match effect { "ec.color.curves" => 34.0 + curves_size(width) + 26.0, effectcraft_engine::effects::warp_stab::ID => 50.0, + effectcraft_engine::effects::roto::ID => 50.0, "ec.color.levels" | "ec.color.levelsic" => 34.0 + 80.0 + 58.0, "ec.color.autolevels" | "ec.color.autocontrast" | "ec.color.autocolor" => 24.0 + 80.0 + 12.0, _ => 0.0, @@ -920,6 +950,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { match effect.as_str() { "ec.color.curves" => curves_editor(app, ui, &bp, &layer, g, &ectx, er, &mut actions), effectcraft_engine::effects::warp_stab::ID => warp_editor(app, ui, &bp, &layer, g, er, &mut actions), + effectcraft_engine::effects::roto::ID => roto_editor(app, ui, &bp, &layer, g, er, &mut actions), "ec.color.levels" | "ec.color.levelsic" => levels_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions), _ => histogram_only(app, ui, &bp, g, er), } diff --git a/crates/ui-egui/src/panels/layer_panel.rs b/crates/ui-egui/src/panels/layer_panel.rs index e53e268e..c20a0fba 100644 --- a/crates/ui-egui/src/panels/layer_panel.rs +++ b/crates/ui-egui/src/panels/layer_panel.rs @@ -3,9 +3,13 @@ //! //! The image is the layer's source and masks followed by its effects up to the View menu's //! choice (by default the last Paint effect, as After Effects shows "Paint"). Dragging with a -//! paint tool records the stroke's points in layer space and, on release, runs `paint.stroke` +//! paint tools records the stroke's points in layer space and, on release, runs `paint.stroke` //! at the current time (the drag's duration feeds Write On). Alt/Option-click with the Clone //! Stamp sets the clone source. +//! +//! With a Roto Brush & Refine Edge effect the panel shows the effect's input with the matte +//! overlay, the segmentation span bar and the view / Freeze buttons; the Roto Brush and Refine +//! Edge tools paint their strokes here (see [`super::roto_tool`]). use effectcraft_engine::effects::paint; use effectcraft_engine::project::{Layer, LayerId}; @@ -46,9 +50,15 @@ pub fn current_layer(app: &EffectcraftApp) -> Option { comp.layer(id).cloned() } -/// Number of effects shown by default: through the last Paint effect (none when there isn't one). -fn default_view(layer: &Layer) -> usize { - layer.effects().and_then(|fx| fx.groups().enumerate().filter(|(_, g)| paint::is_paint(g)).map(|(i, _)| i + 1).last()).unwrap_or(0) +/// Number of effects shown by default: the Roto Brush's input while a Roto Brush tool is active +/// (or when there is no Paint effect), else through the last Paint effect (none when there +/// isn't one). +fn default_view(layer: &Layer, tool: Tool) -> usize { + let paint = layer.effects().and_then(|fx| fx.groups().enumerate().filter(|(_, g)| paint::is_paint(g)).map(|(i, _)| i + 1).last()); + match super::roto_tool::roto_index(layer) { + Some(r) if tool.roto_kind(false).is_some() || paint.is_none() => r, + _ => paint.unwrap_or(0), + } } pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { @@ -64,7 +74,8 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let comp = app.session.project.comp(cid).cloned().unwrap_or_else(|| effectcraft_engine::project::Comp::new(1, 1, Default::default(), Default::default())); let time = app.session.time(); let fx_count = layer.effects().map(|f| f.groups().count()).unwrap_or(0); - let view = app.ui.layer_view.map(|v| v.min(fx_count)).unwrap_or_else(|| default_view(&layer)); + let view = app.ui.layer_view.map(|v| v.min(fx_count)).unwrap_or_else(|| default_view(&layer, app.ui.tool)); + let roto = super::roto_tool::roto_index(&layer); // Title strip. let top = Rect::from_min_size(rect.min, vec2(rect.width(), 24.0)); @@ -90,9 +101,17 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } let tc = super::timecode(&app.session, &comp, time); p.text(pos2(bar.max.x - 10.0, bar.center().y), Align2::RIGHT_CENTER, tc, Tokens::mono(12.0), t.timecode); + // Roto Brush: view / Freeze buttons and the segmentation span bar. + let mut canvas_bottom = bar.min.y; + if roto.is_some() { + super::roto_tool::bar_buttons(app, ui, &p, bar, vr.max.x + 12.0, &layer); + let sb = Rect::from_min_max(pos2(rect.min.x, bar.min.y - 18.0), pos2(rect.max.x, bar.min.y)); + super::roto_tool::span_bar(app, ui, &p, sb, &comp, &layer); + canvas_bottom = sb.min.y; + } // Canvas. - let area = Rect::from_min_max(pos2(rect.min.x, top.max.y), pos2(rect.max.x, bar.min.y)); + let area = Rect::from_min_max(pos2(rect.min.x, top.max.y), pos2(rect.max.x, canvas_bottom)); p.rect_filled(area, 0.0, t.pasteboard); let (w, h) = effectcraft_engine::render::source_size(&app.session.project, &layer); let (lw, lh) = if w == 0 { (comp.width as f64, comp.height as f64) } else { (w as f64, h as f64) }; @@ -139,10 +158,18 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let r = Rect::from_min_max(to_screen([tx.rect[0], tx.rect[1]]), to_screen([tx.rect[2], tx.rect[3]])); painter.image(tx.tex.id(), r, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE); } + if roto.is_some_and(|r| r == view) { + super::roto_tool::draw_overlay(app, &ctx, &painter, &comp, &layer, scale, &to_screen); + } painter.rect_stroke(frame, 0.0, Stroke::new(1.0, Color32::from_black_alpha(160)), StrokeKind::Outside); app.auto.add("layerPanel.canvas", frame, &layer.name); app.auto.add("layerPanel.area", area, "Layer panel"); + // Interaction: Roto Brush / Refine Edge. + if app.ui.tool.roto_kind(false).is_some() { + super::roto_tool::interact(app, ui, &painter, area, &layer, zoom, &to_screen, &to_layer); + return; + } // Interaction: paint tools. let resp = ui.interact(area, egui::Id::new("layer-panel-interact"), Sense::click_and_drag()); let kind = app.ui.tool.paint_kind(); diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 3f737502..8a8ce0cf 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -21,6 +21,7 @@ pub mod project; pub mod properties; pub mod puppet_tool; pub mod render_queue; +pub mod roto_tool; pub mod settings; pub mod shortcut_editor; pub mod text_panels; diff --git a/crates/ui-egui/src/panels/roto_tool.rs b/crates/ui-egui/src/panels/roto_tool.rs new file mode 100644 index 00000000..25354ee3 --- /dev/null +++ b/crates/ui-egui/src/panels/roto_tool.rs @@ -0,0 +1,331 @@ +//! Roto Brush and Refine Edge tools in the Layer panel: green foreground strokes, Alt/Option red +//! background strokes, Refine Edge strokes along soft edges; the matte overlay (Alpha Boundary in +//! pink, Alpha, Alpha Overlay), the segmentation span bar and the Freeze button. + +use effectcraft_engine::effects::roto as fx; +use effectcraft_engine::project::{Comp, Layer}; +use effectcraft_engine::render::{EvalCtx, RenderOpts, Renderer}; +use egui::{Align2, Color32, Pos2, Rect, Sense, Stroke, StrokeKind, pos2, vec2}; +use serde_json::json; + +use crate::theme::Tokens; +use crate::{EffectcraftApp, widgets}; + +/// Position of the layer's first Roto Brush & Refine Edge effect. +pub fn roto_index(layer: &Layer) -> Option { + layer.effects()?.groups().position(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == fx::ID)) +} + +fn roto_group(layer: &Layer) -> Option<&effectcraft_engine::project::PropGroup> { + layer.effects()?.groups().nth(roto_index(layer)?) +} + +/// A stroke being drawn. +#[derive(Clone, Debug)] +struct Drag { + layer: u64, + kind: &'static str, + points: Vec<[f64; 2]>, +} + +#[derive(Clone)] +struct Overlay { + key: (u64, u64, i64, u32, String), + tex: egui::TextureHandle, + rect: [f64; 4], +} + +fn c32(c: [f64; 3], a: u8) -> Color32 { + Color32::from_rgba_unmultiplied((c[0] * 255.0) as u8, (c[1] * 255.0) as u8, (c[2] * 255.0) as u8, a) +} + +/// Draw the matte overlay of the Roto Brush on `layer` (the panel shows the effect's input). +pub fn draw_overlay( + app: &mut EffectcraftApp, + ctx: &egui::Context, + painter: &egui::Painter, + comp: &Comp, + layer: &Layer, + scale: f64, + to_screen: &dyn Fn([f64; 2]) -> Pos2, +) { + let Some(idx) = roto_index(layer) else { return }; + let o = app.session.state.roto.clone(); + if o.view == "none" { + return; + } + let Some(cid) = app.session.active_comp_id() else { return }; + let time = app.session.time(); + let key = ( + app.session.revision, + layer.id.0, + time.0, + (scale * 1000.0) as u32, + format!("{}{:?}{:?}{}", o.view, o.overlay_color, o.boundary_color, o.overlay_opacity), + ); + let id = egui::Id::new("roto-overlay"); + let cached: Option = ctx.data(|d| d.get_temp(id)); + let ov = match cached { + Some(c) if c.key == key => Some(c), + _ => { + let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp, time, expr: app.session.expr.as_deref() }; + let mut r = Renderer::new(&app.session.project, &*app.session.footage, RenderOpts { scale, ..Default::default() }); + r.expr = app.session.expr.as_deref(); + r.cache = Some(&app.session.layer_cache); + let (Some(inp), Some(out)) = (r.layer_input(&ectx, layer, idx), r.layer_input(&ectx, layer, idx + 1)) else { return }; + let (w, h) = (inp.img.width as usize, inp.img.height as usize); + let a_at = |x: i64, y: i64| -> f32 { + // Output pixel at the same layer point. + let lx = (x as f64 + 0.5 - inp.offset[0]) / inp.scale; + let ly = (y as f64 + 0.5 - inp.offset[1]) / inp.scale; + let (ox, oy) = out.to_px([lx, ly]); + out.img.get(ox.floor() as i64, oy.floor() as i64)[3] + }; + let alpha: Vec = (0..w * h).map(|i| a_at((i % w) as i64, (i / w) as i64)).collect(); + let inside = |x: i64, y: i64| x >= 0 && y >= 0 && x < w as i64 && y < h as i64 && alpha[y as usize * w + x as usize] >= 0.5; + let mut px = vec![Color32::TRANSPARENT; w * h]; + let bc = c32(o.boundary_color, 255); + let oc = c32(o.overlay_color, (o.overlay_opacity / 100.0 * 255.0).clamp(0.0, 255.0) as u8); + // View Search Region: the band around the boundary (yellow). + let search = roto_group(layer).filter(|g| g.prop("rotoBrushMatte/viewSearchRegion").is_some_and(|p| p.value.as_bool())).map(|g| { + let r = g.prop("rotoBrushMatte/searchRadius").map(|p| p.value.as_f64()).unwrap_or(15.0) * scale; + let bin: Vec = alpha.iter().map(|a| (*a >= 0.5) as u8).collect(); + let sd = effectcraft_engine::track::roto::matting::signed_distance(&bin, w, h); + (sd, r as f32) + }); + for y in 0..h as i64 { + for x in 0..w as i64 { + let i = y as usize * w + x as usize; + let a = alpha[i]; + px[i] = match o.view.as_str() { + "alpha" => { + let v = (a.clamp(0.0, 1.0) * 255.0) as u8; + Color32::from_rgb(v, v, v) + } + "alphaOverlay" if a < 0.5 => oc, + "alphaOverlay" => Color32::TRANSPARENT, + _ => { + let me = a >= 0.5; + let edge = me && [(-1, 0), (1, 0), (0, -1), (0, 1)].iter().any(|(dx, dy)| !inside(x + dx, y + dy)); + if edge { bc } else { Color32::TRANSPARENT } + } + }; + if let Some((sd, r)) = &search { + let d = sd[i].abs(); + if (d - r).abs() < 0.75 { + px[i] = Color32::from_rgb(255, 220, 0); + } + } + } + } + let img = egui::ColorImage { size: [w, h], pixels: px, source_size: egui::Vec2::new(w as f32, h as f32) }; + let x0 = -inp.offset[0] / inp.scale; + let y0 = -inp.offset[1] / inp.scale; + let rect = [x0, y0, x0 + w as f64 / inp.scale, y0 + h as f64 / inp.scale]; + let tex = ctx.load_texture("roto-overlay", img, egui::TextureOptions::NEAREST); + let c = Overlay { key, tex, rect }; + ctx.data_mut(|d| d.insert_temp(id, c.clone())); + Some(c) + } + }; + if let Some(ov) = ov { + let r = Rect::from_min_max(to_screen([ov.rect[0], ov.rect[1]]), to_screen([ov.rect[2], ov.rect[3]])); + painter.image(ov.tex.id(), r, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE); + } +} + +/// Pointer handling for the Roto Brush / Refine Edge tools. Returns true when it handled input. +pub fn interact( + app: &mut EffectcraftApp, + ui: &mut egui::Ui, + painter: &egui::Painter, + area: Rect, + layer: &Layer, + zoom: f64, + to_screen: &dyn Fn([f64; 2]) -> Pos2, + to_layer: &dyn Fn(Pos2) -> [f64; 2], +) -> bool { + let ctx = ui.ctx().clone(); + let mods = ui.input(|i| i.modifiers); + let Some(kind) = app.ui.tool.roto_kind(mods.alt) else { return false }; + let resp = ui.interact(area, egui::Id::new("layer-panel-roto"), Sense::click_and_drag()); + let o = app.session.state.roto.clone(); + let refine = kind.starts_with("refine"); + let dia = if refine { o.refine_diameter } else { o.diameter }; + let col = match kind { + "fg" => Color32::from_rgb(40, 220, 60), + "bg" => Color32::from_rgb(235, 50, 40), + "refine" => Color32::from_rgb(110, 200, 255), + _ => Color32::from_rgb(255, 160, 60), + }; + if let Some(hp) = resp.hover_pos() { + let lp = to_layer(hp); + app.pointer_comp = Some([lp[0] as f32, lp[1] as f32]); + ctx.set_cursor_icon(egui::CursorIcon::Crosshair); + let r = (dia * 0.5 * zoom) as f32; + painter.circle_stroke(hp, r, Stroke::new(1.5, col)); + let sign = if kind == "bg" || kind == "refineErase" { "−" } else { "+" }; + painter.text(hp, Align2::CENTER_CENTER, sign, Tokens::semibold(12.0), col); + } + let did = egui::Id::new("layer-panel-roto-drag"); + if resp.drag_started() + && let Some(pos) = ui.input(|i| i.pointer.press_origin()).or(resp.interact_pointer_pos()) + { + ctx.data_mut(|d| d.insert_temp(did, Drag { layer: layer.id.0, kind, points: vec![to_layer(pos)] })); + } + if let Some(mut d) = ctx.data(|d| d.get_temp::(did)) { + if let Some(pos) = resp.interact_pointer_pos() { + let lp = to_layer(pos); + let last = d.points.last().copied().unwrap_or(lp); + if (lp[0] - last[0]).hypot(lp[1] - last[1]) * zoom >= 1.5 { + d.points.push(lp); + ctx.data_mut(|dd| dd.insert_temp(did, d.clone())); + } + } + let width = ((dia * zoom) as f32).max(1.0); + let pts: Vec = d.points.iter().map(|q| to_screen(*q)).collect(); + let c = col.gamma_multiply(0.6); + if pts.len() == 1 { + painter.circle_filled(pts[0], width / 2.0, c); + } else { + painter.add(egui::Shape::line(pts, Stroke::new(width, c))); + } + if resp.drag_stopped() { + ctx.data_mut(|dd| dd.remove::(did)); + run_stroke(app, d.layer, d.kind, &d.points, dia / 2.0); + } + } else if resp.clicked() + && let Some(pos) = resp.interact_pointer_pos() + { + run_stroke(app, layer.id.0, kind, &[to_layer(pos)], dia / 2.0); + } + true +} + +fn run_stroke(app: &mut EffectcraftApp, layer: u64, kind: &str, points: &[[f64; 2]], radius: f64) { + let r = app.session.execute("roto.stroke", json!({"layer": layer, "kind": kind, "points": points, "radius": radius})); + if let Err(e) = r { + app.ui.status = e.to_string(); + } +} + +/// The segmentation span bar (above the panel's bottom bar): the layer's frames, the span, the +/// base frame (yellow), computed frames (green) and stroke frames. Click to go to a frame; drag +/// the span's ends to change it. +pub fn span_bar(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, r: Rect, comp: &Comp, layer: &Layer) { + let t = app.tokens; + let Some(g) = roto_group(layer) else { return }; + let Some(cid) = app.session.active_comp_id() else { return }; + let uid = g.uid; + let params = effectcraft_engine::effects::flatten_params(g, &mut |p| p.value.clone()); + let d = fx::data(¶ms); + p.rect_filled(r, 0.0, t.panel_bg); + let lim = effectcraft_engine::roto::frame_limits(comp, layer); + let n = (lim[1] - lim[0] + 1).max(1) as f32; + let track = Rect::from_min_max(pos2(r.min.x + 10.0, r.min.y + 4.0), pos2(r.max.x - 10.0, r.max.y - 4.0)); + let x_of = |f: f64| track.min.x + ((f - lim[0] as f64) / n as f64) as f32 * track.width(); + let f_of = |x: f32| (lim[0] as f64 + ((x - track.min.x) / track.width()).clamp(0.0, 0.9999) as f64 * n as f64).floor() as i64; + p.rect_filled(track, 2.0, t.pasteboard); + if let Some(base) = d.base { + let sp = Rect::from_min_max(pos2(x_of(d.span[0] as f64), track.min.y), pos2(x_of(d.span[1] as f64 + 1.0), track.max.y)); + let frozen = fx::is_frozen(¶ms); + p.rect_filled(sp, 2.0, if frozen { Color32::from_rgb(60, 110, 190) } else { Color32::from_rgb(90, 90, 96) }); + for (f, done) in app.session.roto_computed(cid, layer.id, uid) { + if done { + let cr = Rect::from_min_max(pos2(x_of(f as f64), track.max.y - 4.0), pos2(x_of(f as f64 + 1.0), track.max.y)); + p.rect_filled(cr, 0.0, Color32::from_rgb(70, 200, 90)); + } + } + for f in d.stroke_frames() { + let x = x_of(f as f64 + 0.5); + p.line_segment([pos2(x, track.min.y), pos2(x, track.min.y + 5.0)], Stroke::new(2.0, Color32::WHITE)); + } + let bx = x_of(base as f64 + 0.5); + p.rect_filled(Rect::from_center_size(pos2(bx, track.center().y), vec2(6.0, track.height())), 1.0, Color32::from_rgb(240, 200, 40)); + } + // Current time. + let fps = comp.frame_rate.as_f64(); + let cur = fx::frame_of(layer.layer_time(app.session.time()).seconds(), fps); + let cx = x_of(cur as f64 + 0.5); + p.line_segment([pos2(cx, r.min.y), pos2(cx, r.max.y)], Stroke::new(1.0, t.timecode)); + p.rect_stroke(track, 2.0, Stroke::new(1.0, Color32::from_black_alpha(140)), StrokeKind::Outside); + app.auto.add("layerPanel.rotoSpan", track, "Segmentation span"); + let resp = ui.interact(track, egui::Id::new("roto-span-bar"), Sense::click_and_drag()); + let hid = egui::Id::new("roto-span-handle"); + if resp.drag_started() + && let (Some(pos), Some(_)) = (resp.interact_pointer_pos(), d.base) + { + let (a, b) = (x_of(d.span[0] as f64), x_of(d.span[1] as f64 + 1.0)); + let h = if (pos.x - a).abs() < 6.0 { + 1 + } else if (pos.x - b).abs() < 6.0 { + 2 + } else { + 0 + }; + ui.ctx().data_mut(|m| m.insert_temp(hid, h)); + } + let handle: i32 = ui.ctx().data(|m| m.get_temp(hid)).unwrap_or(0); + if let Some(pos) = resp.interact_pointer_pos() { + let f = f_of(pos.x); + if handle == 0 && (resp.clicked() || resp.dragged()) { + let t = layer.comp_time(comp.frame_rate.tick_of(f)); + let _ = app.session.execute("time.set", json!({"time": t.seconds()})); + } + if handle != 0 && resp.dragged() { + p.line_segment([pos2(pos.x, track.min.y), pos2(pos.x, track.max.y)], Stroke::new(2.0, Color32::from_rgb(240, 200, 40))); + } + if handle != 0 && resp.drag_stopped() { + let k = if handle == 1 { "start" } else { "end" }; + if let Err(e) = app.session.execute("roto.span", json!({"layer": layer.id.0, "effect": uid, k: f})) { + app.ui.status = e.to_string(); + } + } + } + if resp.drag_stopped() { + ui.ctx().data_mut(|m| m.remove::(hid)); + } +} + +/// View-mode toggles and Freeze / Propagate buttons in the Layer panel's bottom bar, from `x`. +pub fn bar_buttons(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, bar: Rect, x: f32, layer: &Layer) { + let t = app.tokens; + let mut x = x; + let view = app.session.state.roto.view.clone(); + for (id, label) in [("alpha", "Alpha"), ("alphaBoundary", "Boundary"), ("alphaOverlay", "Overlay")] { + let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(66.0, 20.0)); + let on = view == id; + if widgets::text_button(ui, r, label, on, &t, egui::Id::new(("roto-view", id))).clicked() { + let v = if on { "none" } else { id }; + let _ = app.session.execute("roto.options", json!({"view": v})); + } + app.auto.add(&format!("layerPanel.roto.view.{id}"), r, label); + x += 70.0; + } + let Some(g) = roto_group(layer) else { return }; + let uid = g.uid; + let params = effectcraft_engine::effects::flatten_params(g, &mut |p| p.value.clone()); + let frozen = fx::is_frozen(¶ms); + let running = app.session.is_roto_running(); + x += 8.0; + let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(74.0, 20.0)); + let label = if frozen { "Unfreeze" } else { "Freeze" }; + if widgets::text_button(ui, r, label, frozen, &t, egui::Id::new(("roto-freeze", uid))).clicked() { + let id = if frozen { "roto.unfreeze" } else { "roto.freeze" }; + if let Err(e) = app.session.execute(id, json!({"layer": layer.id.0, "effect": uid})) { + app.ui.status = e.to_string(); + } + } + app.auto.add("layerPanel.roto.freeze", r, label); + x += 80.0; + if running { + let r = Rect::from_min_size(pos2(x, bar.center().y - 10.0), vec2(60.0, 20.0)); + if widgets::text_button(ui, r, "Stop", false, &t, egui::Id::new("roto-stop")).clicked() { + let _ = app.session.execute("roto.cancel", json!({})); + } + app.auto.add("layerPanel.roto.stop", r, "Stop"); + if let Some(pr) = app.session.roto_progress() { + p.text(pos2(r.max.x + 8.0, bar.center().y), Align2::LEFT_CENTER, pr.banner(), Tokens::ui(11.5), t.text_dim); + } + } +} diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index d1350103..b89ddadb 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -34,6 +34,7 @@ pub enum Tool { Clone, Eraser, RotoBrush, + RefineEdge, Puppet, PuppetStarch, PuppetBend, @@ -58,10 +59,10 @@ impl Tool { &[Tool::Brush], &[Tool::Clone], &[Tool::Eraser], - &[Tool::RotoBrush], + &[Tool::RotoBrush, Tool::RefineEdge], &[Tool::Puppet, Tool::PuppetStarch, Tool::PuppetBend, Tool::PuppetAdvanced, Tool::PuppetOverlap], ]; - pub const ALL: [Tool; 29] = [ + pub const ALL: [Tool; 30] = [ Tool::Selection, Tool::Hand, Tool::Zoom, @@ -86,6 +87,7 @@ impl Tool { Tool::Clone, Tool::Eraser, Tool::RotoBrush, + Tool::RefineEdge, Tool::Puppet, Tool::PuppetStarch, Tool::PuppetBend, @@ -118,6 +120,7 @@ impl Tool { Tool::Clone => "Clone Stamp Tool", Tool::Eraser => "Eraser Tool", Tool::RotoBrush => "Roto Brush Tool", + Tool::RefineEdge => "Refine Edge Tool", Tool::Puppet => "Puppet Position Pin Tool", Tool::PuppetStarch => "Puppet Starch Pin Tool", Tool::PuppetBend => "Puppet Bend Pin Tool", @@ -139,7 +142,7 @@ impl Tool { Tool::Pen | Tool::PenAdd | Tool::PenDelete | Tool::PenConvert | Tool::MaskFeather => Some("G"), Tool::Type | Tool::TypeVertical => Some("Cmd+T"), Tool::Brush | Tool::Clone | Tool::Eraser => Some("Cmd+B"), - Tool::RotoBrush => Some("Alt+W"), + Tool::RotoBrush | Tool::RefineEdge => Some("Alt+W"), Tool::Puppet | Tool::PuppetStarch | Tool::PuppetBend | Tool::PuppetAdvanced | Tool::PuppetOverlap => Some("Cmd+P"), } } @@ -169,6 +172,7 @@ impl Tool { Tool::Clone => Icon::Clone, Tool::Eraser => Icon::Eraser, Tool::RotoBrush => Icon::RotoBrush, + Tool::RefineEdge => Icon::RefineEdge, Tool::Puppet | Tool::PuppetStarch | Tool::PuppetBend | Tool::PuppetAdvanced | Tool::PuppetOverlap => Icon::Puppet, } } @@ -187,6 +191,17 @@ impl Tool { _ => return None, }) } + /// Roto Brush / Refine Edge: the stroke kind they paint (`roto.stroke` kind), without and + /// with Alt/Option. + pub fn roto_kind(self, alt: bool) -> Option<&'static str> { + Some(match (self, alt) { + (Tool::RotoBrush, false) => "fg", + (Tool::RotoBrush, true) => "bg", + (Tool::RefineEdge, false) => "refine", + (Tool::RefineEdge, true) => "refineErase", + _ => return None, + }) + } /// Paint tools, with their stroke kind (`paint.stroke` kind). pub fn paint_kind(self) -> Option<&'static str> { Some(match self { diff --git a/docs/agents.md b/docs/agents.md index 510bc356..5c1b4187 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -146,6 +146,28 @@ effectcraft-cli exec warp.analyze '{"layer":"#1","wait":true}' shaky.ecproj --sa effectcraft-cli exec warp.status '{"layer":"#1"}' shaky.ecproj --json ``` +### Roto Brush & Refine Edge + +The Roto Brush tool (Alt+W cycles Roto Brush / Refine Edge) paints in the Layer panel; agents use +`roto.stroke {layer, kind: fg|bg|refine|refineErase, points: [[x, y], …], frame?, radius?}` +(layer pixels, layer frames). The first foreground stroke applies the Roto Brush & Refine Edge +effect and sets the base frame with a span of 20 frames each side. `roto.propagate {layer, +direction?: forward|backward|both, to?, wait?}` segments the span (in the background unless +`wait`); strokes on any other frame correct it and propagation restarts from there. +`roto.span {start?, end?}`, `roto.freeze {wait?}` / `roto.unfreeze`, `roto.clearStrokes {frame?, +kind?}`, `roto.cancel` and `roto.options {diameter?, refineDiameter?, view?: alphaBoundary|alpha| +alphaOverlay|none}` complete the set; every edit is one undo step. `roto.status {layer, frame?, +matte?, compute?, compareTo?}` reports the base frame, span, computed and stroked frames, frozen +state, job progress and, for a frame, the matte's area, centroid, RLE matte and IoU against a +reference. Matte settings are ordinary properties (`effects/#1/rotoBrushMatte/searchRadius`, +`effects/#1/refineEdgeMatte/decontaminateEdgeColors`, …). + +```sh +effectcraft-cli run clip.ecproj roto.stroke '{"layer":"#1","points":[[300,200],[360,230]],"radius":10}' \ + roto.stroke '{"layer":"#1","kind":"bg","points":[[40,40],[600,40]],"radius":12}' \ + roto.propagate '{"layer":"#1","wait":true}' roto.freeze '{"layer":"#1","wait":true}' --save +``` + ### Text: styles and editing Source Text holds character style runs and per-paragraph settings. `layer.setText` changes the diff --git a/docs/architecture.md b/docs/architecture.md index 033ed6cb..9f121608 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -176,6 +176,19 @@ auto-scale; cached per analysis and settings) and warps each frame; Synthesize E borders from neighbouring frames read with `EffectHost::self_at`. The effect lives in the effects crate, hence the `effects → track` edge. +**Roto Brush & Refine Edge** (`effectcraft_track::roto`, `effects::roto`, `engine::roto`): +strokes (foreground, background, Refine Edge) are stored as JSON in the effect's hidden Strokes +parameter with the base frame and segmentation span. Each frame is segmented by graph cut +(Boykov–Jolly hard constraints, GrabCut colour mixtures, our own Boykov–Kolmogorov max-flow, +coarse to fine) and propagated to the next frame by warping the matte with block optical flow and +re-cutting in a Search Radius band; frames with correction strokes are re-cut with the warped +matte as a soft prior. Refine Edge bands get guided-filter + closed-form matting and +foreground-colour decontamination. Segmentations are derived data: each frame's chain key (Input +Key, settings, strokes from the base frame out) indexes a process-wide cache that renders fill on +demand (`self_at`) and `roto.propagate` fills in the background; `Session::edit` keeps the Input +Key in step with the layer's source, masks and upstream effects. Freeze stores final 8-bit mattes +in the effect. + ## 5. Expressions Expressions are JavaScript, run by boa, with After Effects' object model (`thisComp`, `thisLayer`, diff --git a/docs/parity.md b/docs/parity.md index d259fec2..6e926dc7 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -73,7 +73,7 @@ library groundwork for mask tracking and Warp Stabilizer. | Remaining (audit-scale agent-hours) | ≈ 167 | ≈ 100 | | Wall clock, five parallel agents | 42–50 h | **≈ 8–10 h at today's measured pace; ≈ 25 h by the audit's conservative figures** | -What is left is concentrated in large systems: Roto Brush and Refine Edge, the 3D camera tracker, +What is left is concentrated in large systems (Roto Brush and Refine Edge landed in M6.7): the 3D camera tracker, Advanced 3D (models, PBR, image-based lighting), the Warp Stabilizer and mask-tracking UI on top of the new tracking library, a JavaScript scripting object model, SVG/PSD import, WebM, a disk cache, the missing effect categories (3D Channel, Immersive Video, OCIO), native macOS menus, and @@ -97,7 +97,7 @@ wasm threads. | Effects | 58% | 13.6 | Effect Controls widgets (angle dial, point crosshair, eyedropper, curves), GPU versions of more effects, 41 missing effects | | Interface | 55% | 10.4 | drag-to-dock and floating panels, preferences, native macOS menus (viewer rulers/snapping/channels/snapshots landed in M0.13) | | Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves | -| Masks & roto | 52% | 10.0 | Roto Brush, variable-width mask feather points (mask tracking and Mask Interpolation landed in M6.6) | +| Masks & roto | 68% | 6.0 | variable-width mask feather points, Roto Brush's learned (3.0) segmentation model (mask tracking and Mask Interpolation landed in M6.6; Roto Brush & Refine Edge with graph-cut segmentation, flow propagation, edge matting, decontamination and Freeze in M6.7) | | Preview | 55% | 6.1 | GPU 3D and adjustment layers, disk cache (region of interest, snapshots, exposure and Fast Previews landed in M0.13) | | Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking, Subspace Warp's mesh warp and rolling-shutter repair (point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser |