From 953fbbc730bee402cdaf7913498758138cf22eb7 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 11 Oct 2026 13:32:10 +0200 Subject: [PATCH 1/8] #783: Object Matte effect from a point / box prompt, tracked Problem: After Effects 26.2's Object Matte (pick an object with a click, optionally a box, get a matte that follows it) had no counterpart; only stroke-driven Roto Brush existed. Fix: an Object Matte effect (Effect > Matte, ec.matte.objectmatte) turns the Object Point (and optional box) on a Reference Frame into Roto Brush strokes and segments them with Roto Brush's engine, content-keyed cache and the model chosen in Settings > Roto Brush (MobileSAM, classical graph cut without it). Track Object carries the matte frame by frame up to 600 frames each way; otherwise every frame is segmented on its own. Feather, Contrast, Shift Edge, Invert and a Matte view. The engine keeps its Input Key current so upstream edits recompute it. objectMatte.apply creates one from agents / the CLI. Unusable prompts pass the layer through. Co-Authored-By: Claude Opus 5.5 --- crates/effects/src/lib.rs | 13 +- crates/effects/src/object_matte.rs | 468 ++++++++++++++++++++++++ crates/effects/src/roto.rs | 8 +- crates/engine/src/commands/roto_cmds.rs | 90 +++++ crates/engine/src/roto.rs | 20 +- crates/engine/src/tests_roto.rs | 75 ++++ docs/effects.md | 7 +- docs/gaps.md | 2 +- docs/target-app-parity.md | 4 +- 9 files changed, 674 insertions(+), 13 deletions(-) create mode 100644 crates/effects/src/object_matte.rs diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 7e968297..d32cfb2e 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -40,6 +40,7 @@ pub mod mocha_shape; mod noise; mod noise2; mod noise3; +pub mod object_matte; mod obsolete; mod ocio; pub mod ocio_config; @@ -497,6 +498,7 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(camera_tracker::specs()); v.extend(face_track::specs()); v.extend(roto::specs()); + v.extend(object_matte::specs()); v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); for s in v.iter_mut() { if GPU_EFFECTS.contains(&s.id) { @@ -805,7 +807,15 @@ pub const PARAM_GROUPS: &[(&str, &str)] = &[ /// Display names of nested parameter groups by match id. fn group_name(m: &str) -> &str { - warp_stab::GROUPS.iter().chain(roto::GROUPS).chain(keylight::GROUPS).chain(PARAM_GROUPS).find(|(id, _)| *id == m).map(|(_, n)| *n).unwrap_or(m) + warp_stab::GROUPS + .iter() + .chain(roto::GROUPS) + .chain(object_matte::GROUPS) + .chain(keylight::GROUPS) + .chain(PARAM_GROUPS) + .find(|(id, _)| *id == m) + .map(|(_, n)| *n) + .unwrap_or(m) } /// The nested group at `path` (`borders/autoScale`) under `g`, created on first use. @@ -1180,6 +1190,7 @@ pub const TIME_DEPENDENT: &[&str] = &[ camera_tracker::ID, // Each frame has its own segmentation. roto::ID, + object_matte::ID, ]; /// See [`TIME_DEPENDENT`]. diff --git a/crates/effects/src/object_matte.rs b/crates/effects/src/object_matte.rs new file mode 100644 index 00000000..3b7b7c28 --- /dev/null +++ b/crates/effects/src/object_matte.rs @@ -0,0 +1,468 @@ +//! Object Matte (Effect ▸ Matte): isolate one object from a click, and follow it through the +//! shot (After Effects 26.2's Object Matte, from its public description: pick an object with a +//! point, optionally a box around it, and get a matte that tracks it). +//! +//! The prompt is the **Object Point** (layer pixels) and, with **Use Box**, a box around the +//! object. On the **Reference Frame** it is segmented by the same engine as Roto Brush: the +//! trained model chosen in Settings ▸ Roto Brush (MobileSAM, `effectcraft_segment`) when one is +//! installed, the classical graph cut otherwise (the point is a foreground stroke, the box's +//! edges, or without a box the layer's border, background strokes). With **Track Object** the +//! matte is then carried frame by frame up to **Frames Before** / **Frames After** away (optical +//! flow, re-segmented by the model prompted with the carried matte); without it every frame is +//! segmented on its own from the prompt (useful when the point is animated). +//! +//! Like Roto Brush, the segmentation is derived data in Roto Brush's content-keyed cache +//! ([`crate::roto::cached`]): the chain seed holds the engine-maintained Input Key (what the +//! frames are made of), the layer size, the frame rate, the mode and the model, and every +//! frame's key the prompt, so edits upstream or to the prompt recompute it and undo never sees a +//! stale matte. **Feather**, **Contrast** and **Shift Edge** shape the matte, **Invert** keeps +//! everything but the object and **View ▸ Matte** shows the matte in grey. + +use std::sync::Arc; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::Image; +use effectcraft_segment::MaskModel; +use effectcraft_track::roto::refine::MatteParams; +use effectcraft_track::roto::{RotoData, SegOpts, Stroke, StrokeKind}; +use rayon::prelude::*; + +use crate::roto::{self, Chain, INPUT_KEY}; +use crate::warp_stab::param; +use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; + +/// The effect's id. +pub const ID: &str = "ec.matte.objectmatte"; + +/// Display names of the instance's parameter groups. +pub const GROUPS: &[(&str, &str)] = &[("objectTracking", "Tracking"), ("objectMatte", "Matte")]; + +pub const VIEWS: [&str; 2] = ["Final Output", "Matte"]; + +/// Most frames a matte is carried from the reference frame each way. +pub const MAX_REACH: f64 = 600.0; +/// Default reach each way (frames). +pub const DEFAULT_REACH: f64 = 30.0; + +/// Radius (layer pixels) of the foreground stroke the Object Point becomes. +const POINT_RADIUS: f64 = 3.0; +/// Radius (layer pixels) of the background strokes along the box / the layer's border. +const EDGE_RADIUS: f64 = 2.0; + +pub fn specs() -> Vec { + vec![EffectSpec { + id: ID, + name: "Object Matte", + category: "Matte", + params: vec![ + p("objectPoint", "Object Point", Value::Vec2([0.5, 0.5]), ParamUi::Point), + p("useBox", "Use Box", Value::Bool(false), ParamUi::Checkbox), + p("boxTopLeft", "Box Top Left", Value::Vec2([0.25, 0.25]), ParamUi::Point), + p("boxBottomRight", "Box Bottom Right", Value::Vec2([0.75, 0.75]), ParamUi::Point), + p("objectTracking/trackObject", "Track Object", Value::Bool(true), ParamUi::Checkbox), + p("objectTracking/referenceFrame", "Reference Frame", num(0.0), ParamUi::Number), + p("objectTracking/framesBefore", "Frames Before", num(DEFAULT_REACH), slider(0.0, MAX_REACH, 0.0, 120.0, 0)), + p("objectTracking/framesAfter", "Frames After", num(DEFAULT_REACH), slider(0.0, MAX_REACH, 0.0, 120.0, 0)), + p("objectMatte/feather", "Feather", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("objectMatte/contrast", "Contrast", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("objectMatte/shiftEdge", "Shift Edge", num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 0)), + p("invert", "Invert", Value::Bool(false), ParamUi::Checkbox), + p("view", "View", Value::Enum(0), popup(&VIEWS)), + p(INPUT_KEY, "Input Key", Value::Str(String::new()), ParamUi::Hidden), + ], + render, + gpu: false, + float: true, + }] +} + +fn f(params: &Params, id: &str) -> f64 { + param(params, id).map(Value::as_f64).unwrap_or(0.0) +} +fn b(params: &Params, id: &str) -> bool { + param(params, id).map(Value::as_bool).unwrap_or(false) +} +fn v2(params: &Params, id: &str) -> Option<[f64; 2]> { + let v = param(params, id)?.as_vec2(); + v.iter().all(|c| c.is_finite()).then_some(v) +} +fn s<'a>(params: &'a Params, id: &str) -> &'a str { + match param(params, id) { + Some(Value::Str(s)) => s, + _ => "", + } +} + +fn fnv(h: &mut u64, bytes: &[u8]) { + for b in bytes { + *h ^= *b as u64; + *h = h.wrapping_mul(0x0100_0000_01b3); + } +} + +/// A frame count parameter (finite, clamped to `lo..=hi`). +fn frames(params: &Params, id: &str, lo: f64, hi: f64) -> i64 { + let v = f(params, id); + if v.is_finite() { v.round().clamp(lo, hi) as i64 } else { 0 } +} + +/// The reference frame (layer frame). +pub fn reference_frame(params: &Params) -> i64 { + frames(params, "objectTracking/referenceFrame", -1e9, 1e9) +} + +/// Whether the matte is carried from the reference frame (else every frame is segmented on its +/// own). +pub fn tracks(params: &Params) -> bool { + b(params, "objectTracking/trackObject") +} + +/// The prompt as Roto Brush strokes on layer frame `frame`: the Object Point as a foreground +/// stroke; the box's edges as background strokes; without a box, the layer's border when there +/// is no model (the graph cut needs background samples; a model does not, and a border stroke +/// would cut objects touching the frame edge). `None` when the point is unusable. +pub fn prompt_strokes(params: &Params, layer_size: [f64; 2], frame: i64, with_model: bool) -> Option> { + let [w, h] = layer_size; + if !(w.is_finite() && h.is_finite() && w >= 1.0 && h >= 1.0) { + return None; + } + let pt = v2(params, "objectPoint")?; + let inside = |q: [f64; 2]| q[0] >= 0.0 && q[1] >= 0.0 && q[0] <= w && q[1] <= h; + if !inside(pt) { + return None; + } + let mut out = vec![Stroke { kind: StrokeKind::Fg, frame, radius: POINT_RADIUS, points: vec![pt] }]; + let rect = |x0: f64, y0: f64, x1: f64, y1: f64| vec![[x0, y0], [x1, y0], [x1, y1], [x0, y1], [x0, y0]]; + let boxed = b(params, "useBox").then(|| Some((v2(params, "boxTopLeft")?, v2(params, "boxBottomRight")?))).flatten(); + match boxed { + Some((a, c)) => { + let (x0, x1) = (a[0].min(c[0]).clamp(0.0, w), a[0].max(c[0]).clamp(0.0, w)); + let (y0, y1) = (a[1].min(c[1]).clamp(0.0, h), a[1].max(c[1]).clamp(0.0, h)); + // The point must lie inside the box, clear of its edges, for the box to mean anything. + let m = POINT_RADIUS + EDGE_RADIUS; + if pt[0] - m <= x0 || pt[0] + m >= x1 || pt[1] - m <= y0 || pt[1] + m >= y1 { + return None; + } + out.push(Stroke { kind: StrokeKind::Bg, frame, radius: EDGE_RADIUS, points: rect(x0, y0, x1, y1) }); + } + None if !with_model => { + let i = EDGE_RADIUS; + if w <= 4.0 * i || h <= 4.0 * i { + return None; + } + out.push(Stroke { kind: StrokeKind::Bg, frame, radius: EDGE_RADIUS, points: rect(i, i, w - i, h - i) }); + } + None => {} + } + Some(out) +} + +/// The segmentation data the matte of layer frame `f` comes from: with Track Object, the prompt +/// on the reference frame and the span it is carried over; otherwise the prompt on `f` alone. +pub fn roto_data(params: &Params, layer_size: [f64; 2], f: i64, with_model: bool) -> Option { + if tracks(params) { + let base = reference_frame(params); + let before = frames(params, "objectTracking/framesBefore", 0.0, MAX_REACH); + let after = frames(params, "objectTracking/framesAfter", 0.0, MAX_REACH); + let strokes = prompt_strokes(params, layer_size, base, with_model)?; + Some(RotoData { base: Some(base), span: [base.saturating_sub(before), base.saturating_add(after)], strokes }) + } else { + Some(RotoData { base: Some(f), span: [f, f], strokes: prompt_strokes(params, layer_size, f, with_model)? }) + } +} + +/// Seed of the chain keys: what the frames are (the Input Key), the layer size, the frame rate, +/// the mode and the model. The prompt is in each frame's key. +pub fn seed(params: &Params, layer_size: [f64; 2], fps: f64, model: Option<&dyn MaskModel>) -> u64 { + let mut h = 0xcbf2_9ce4_8422_2325; + fnv(&mut h, ID.as_bytes()); + fnv(&mut h, s(params, INPUT_KEY).as_bytes()); + for v in [layer_size[0], layer_size[1], fps] { + fnv(&mut h, &v.to_bits().to_le_bytes()); + } + fnv(&mut h, &[tracks(params) as u8]); + fnv(&mut h, model.map_or(effectcraft_segment::CLASSICAL, |m| m.info().id).as_bytes()); + h +} + +/// Segmentation settings at `scale` (Roto Brush's defaults). +fn seg_opts(scale: f64) -> SegOpts { + SegOpts { scale, search_radius: 15.0, motion_threshold: 10.0, motion_damping: 20.0, best: false } +} + +/// Matte settings: Feather, Contrast, Shift Edge (no Refine Edge band). +pub fn matte_params(params: &Params) -> MatteParams { + let num = |id: &str| { + let v = f(params, id); + if v.is_finite() { v } else { 0.0 } + }; + MatteParams { + rb_feather: num("objectMatte/feather").clamp(0.0, 100.0), + rb_contrast: num("objectMatte/contrast").clamp(0.0, 100.0), + rb_shift: num("objectMatte/shiftEdge").clamp(-100.0, 100.0), + decontaminate: false, + ..MatteParams::default() + } +} + +/// What a frame's matte is: the object's alpha on the layer grid, or why there is none. +#[derive(Clone, Debug, PartialEq)] +pub enum FrameMatte { + /// The object's alpha (0–1) on the frame's layer grid. + Alpha(Vec), + /// The prompt is unusable (point outside the layer, outside the box…): the layer passes. + NoPrompt, + /// The frame is beyond the tracked range, or its matte could not be computed. + Outside, +} + +/// The matte of layer frame `f` on the layer grid of `img` (the effect's input on that frame at +/// `scale`), computing what is missing. `frame(g)`: the input on frame g on the same grid. +#[allow(clippy::too_many_arguments)] +pub fn frame_matte( + params: &Params, + layer_size: [f64; 2], + fps: f64, + f: i64, + scale: f64, + img: &Image, + model: Option>, + frame: &mut dyn FnMut(i64) -> Option>, + cancel: &dyn Fn() -> bool, +) -> FrameMatte { + let Some(data) = roto_data(params, layer_size, f, model.is_some()) else { return FrameMatte::NoPrompt }; + if !data.in_span(f) { + return FrameMatte::Outside; + } + let keys = data.chain_keys(seed(params, layer_size, fps, model.as_deref())); + let chain = Chain { data: Arc::new(data), keys, opts: seg_opts(scale), refine_radius: 0.0, model }; + let Some(seg) = chain.ensure(f, frame, cancel) else { return FrameMatte::Outside }; + let (w, h) = (img.width as usize, img.height as usize); + let seg = if seg.w == w && seg.h == h { (*seg).clone() } else { roto::resample(&seg, w, h) }; + FrameMatte::Alpha(roto::final_alpha(img, &seg, None, None, &matte_params(params), 0.0, scale)) +} + +fn render(ctx: &EffectCtx, buf: Buf) -> Buf { + if ctx.adjustment { + return buf; + } + let fps = ctx.fps(); + let f = roto::frame_of(ctx.time, fps); + let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + let img = Arc::new(roto::layer_grid(&buf, ctx.layer_size)); + let (w, h) = (img.width as usize, img.height as usize); + let host = ctx.env.host; + let effects = ctx.env.effect_index; + let layer_size = ctx.layer_size; + let me = img.clone(); + let mut frame = |g: i64| -> Option> { + if g == f { + return Some(me.clone()); + } + let other = host?.self_at(g as f64 / fps, effects)?; + let im = roto::layer_grid(&other, layer_size); + (im.width as usize == w && im.height as usize == h).then(|| Arc::new(im)) + }; + let alpha = match frame_matte(ctx.params, layer_size, fps, f, sc, &img, roto::installed_model(), &mut frame, &|| false) { + FrameMatte::Alpha(a) if a.len() == w * h => Some(a), + FrameMatte::NoPrompt => return buf, + _ => None, + }; + compose(&buf, alpha.as_deref(), w, h, b(ctx.params, "invert"), param(ctx.params, "view").map(Value::as_enum).unwrap_or(0) == 1) +} + +/// The output: the input keyed by the matte (`None`: no object, all zero), or the matte itself. +fn compose(buf: &Buf, alpha: Option<&[f32]>, w: usize, h: usize, invert: bool, view_matte: bool) -> Buf { + let (ox, oy) = (buf.offset[0].round() as i64, buf.offset[1].round() as i64); + let mut out = Buf { img: Image::new(buf.img.width, buf.img.height), offset: buf.offset, scale: buf.scale }; + out.img.rows_mut().for_each(|(by, row)| { + for (bx, px) in row.iter_mut().enumerate() { + let (x, y) = (bx as i64 - ox, by as i64 - oy); + let a = if x >= 0 && y >= 0 && (x as usize) < w && (y as usize) < h { + alpha.and_then(|a| a.get(y as usize * w + x as usize)).copied().unwrap_or(0.0) + } else { + 0.0 + }; + let a = if invert { 1.0 - a } else { a }.clamp(0.0, 1.0); + let src = buf.img.get(bx as i64, by as i64); + *px = if view_matte { + let v = a * src[3]; + [v, v, v, 1.0] + } else { + [src[0] * a, src[1] * a, src[2] * a, src[3] * a] + }; + } + }); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, EffectHost, LayerPixels}; + + /// A red disk of radius 15 moving 2 px a frame right over a textured blue background + /// (100×80, 25 fps), plus a still green square in the corner. + fn frame_at(fi: i64) -> Image { + let c = [40.0 + 2.0 * fi as f64, 40.0]; + let mut img = Image::new(100, 80); + for y in 0..80 { + for x in 0..100 { + let d = ((x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1])) - 15.0; + let a = (0.5 - d).clamp(0.0, 1.0) as f32; + let tex = ((x * 7 + y * 13) % 9) as f32 / 60.0; + let bg = if x >= 80 && y >= 60 { [0.1, 0.8, 0.2] } else { [0.1, 0.4 + tex, 0.7] }; + let fg = [0.85 + tex, 0.2, 0.1]; + img.set(x, y, [fg[0] * a + bg[0] * (1.0 - a), fg[1] * a + bg[1] * (1.0 - a), fg[2] * a + bg[2] * (1.0 - a), 1.0]); + } + } + img + } + fn frame_of_time(t: f64) -> Image { + frame_at((t * 25.0 + 1e-6).floor() as i64) + } + + struct Seq; + impl EffectHost for Seq { + fn layer(&self, _: u64, _: bool) -> Option { + None + } + fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option> { + None + } + fn self_at(&self, t: f64, _: usize) -> Option { + Some(Buf { img: frame_of_time(t), offset: [0.0; 2], scale: 1.0 }) + } + } + + fn params(vals: &[(&str, Value)]) -> Params { + let s = &specs()[0]; + let mut p = Params { values: s.params.iter().map(|p| (p.id.to_string(), crate::default_value(p, [100.0, 80.0]))).collect() }; + for (k, v) in vals { + p.values.insert(k.to_string(), v.clone()); + } + p + } + + fn run(p: &Params, fi: i64) -> Image { + let host = Seq; + let t = fi as f64 / 25.0 + 0.001; + let ctx = EffectCtx { + params: p, + time: t, + layer_size: [100.0, 80.0], + seed: 1, + adjustment: false, + env: EffectEnv { host: Some(&host), frame_rate: 25.0, ..Default::default() }, + }; + crate::apply(&specs()[0], &ctx, Buf { img: frame_of_time(t), offset: [0.0; 2], scale: 1.0 }).img + } + + /// Pixels whose alpha disagrees with the disk centred at `cx` (1 px of slack at the edge). + fn wrong(out: &Image, cx: f64) -> usize { + let mut n = 0; + for y in 0..80u32 { + for x in 0..100u32 { + let d = (x as f64 + 0.5 - cx).hypot(y as f64 + 0.5 - 40.0); + let a = out.data[(y * 100 + x) as usize][3]; + if (d < 14.0 && a < 0.5) || (d > 16.0 && a > 0.5) { + n += 1; + } + } + } + n + } + + #[test] + fn a_click_isolates_the_object_and_tracks_it() { + let p = params(&[("objectPoint", Value::Vec2([40.0, 40.0]))]); + // The reference frame: the disk, nothing else. + let out = run(&p, 0); + assert!(wrong(&out, 40.0) < 20, "{} wrong pixels on the reference frame", wrong(&out, 40.0)); + // Six frames on (carried through frames 1–5), the disk has moved 12 px and the matte with it. + let out = run(&p, 6); + assert!(wrong(&out, 52.0) < 30, "{} wrong pixels on frame 6", wrong(&out, 52.0)); + // Beyond Frames After: no matte. + let p2 = params(&[("objectPoint", Value::Vec2([40.0, 40.0])), ("objectTracking/framesAfter", Value::Scalar(3.0))]); + assert!(run(&p2, 6).data.iter().all(|px| px[3] == 0.0)); + // Invert keeps everything but the disk; View ▸ Matte shows it in grey, opaque. + let inv = run(¶ms(&[("objectPoint", Value::Vec2([40.0, 40.0])), ("invert", Value::Bool(true))]), 0); + assert!(inv.data[40 * 100 + 40][3] < 0.5 && inv.data[5 * 100 + 95][3] > 0.5); + let m = run(¶ms(&[("objectPoint", Value::Vec2([40.0, 40.0])), ("view", Value::Enum(1))]), 0); + assert!(m.data[40 * 100 + 40][0] > 0.5 && m.data[5 * 100 + 95][0] < 0.5 && m.data.iter().all(|px| px[3] == 1.0)); + } + + #[test] + fn without_tracking_every_frame_follows_the_prompt() { + // The point animated onto the disk at frame 9: frame 9 is segmented from it alone. + let p = params(&[("objectPoint", Value::Vec2([58.0, 40.0])), ("objectTracking/trackObject", Value::Bool(false))]); + let out = run(&p, 9); + assert!(wrong(&out, 58.0) < 20, "{} wrong pixels", wrong(&out, 58.0)); + } + + #[test] + fn a_box_and_bad_prompts() { + // A box around the green square, the point in it: the square, not the disk. + let p = params(&[ + ("objectPoint", Value::Vec2([90.0, 70.0])), + ("useBox", Value::Bool(true)), + ("boxTopLeft", Value::Vec2([74.0, 54.0])), + ("boxBottomRight", Value::Vec2([100.0, 80.0])), + ]); + let out = run(&p, 0); + assert!(out.data[70 * 100 + 90][3] > 0.5 && out.data[40 * 100 + 40][3] < 0.5); + // Unusable prompts pass the layer through instead of failing: a point off the layer, a + // non-finite point, a point outside its box. + let src = frame_at(0); + for vals in [ + vec![("objectPoint", Value::Vec2([-5.0, 40.0]))], + vec![("objectPoint", Value::Vec2([f64::NAN, 40.0]))], + vec![ + ("objectPoint", Value::Vec2([40.0, 40.0])), + ("useBox", Value::Bool(true)), + ("boxTopLeft", Value::Vec2([60.0, 10.0])), + ("boxBottomRight", Value::Vec2([90.0, 30.0])), + ], + ] { + let p = params(&vals); + assert_eq!(run(&p, 0).data, src.data, "{vals:?}"); + } + // A huge reference frame or reach doesn't overflow. + let p = params(&[("objectTracking/referenceFrame", Value::Scalar(1e300)), ("objectTracking/framesBefore", Value::Scalar(f64::INFINITY))]); + assert!(run(&p, 0).data.iter().all(|px| px[3] == 0.0)); + } + + /// A stand-in model (the weights are an optional download): a disk around the first + /// foreground point, radius 15, so the test sees the model's answer in the matte. + #[derive(Default)] + struct Stub { + calls: std::sync::atomic::AtomicUsize, + } + impl MaskModel for Stub { + fn info(&self) -> &'static effectcraft_segment::ModelInfo { + &effectcraft_segment::MOBILE_SAM + } + fn segment(&self, _: &[[f32; 3]], w: usize, h: usize, prompt: &effectcraft_segment::Prompt) -> effectcraft_segment::Result> { + self.calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let Some(([cx, cy], _)) = prompt.points.iter().find(|(_, fg)| *fg) else { return Err("no point".into()) }; + Ok((0..w * h).map(|i| (((i % w) as f32 + 0.5 - cx).hypot((i / w) as f32 + 0.5 - cy) < 15.0) as u8 as f32).collect()) + } + } + + #[test] + fn segments_with_the_installed_model_without_a_border_stroke() { + let p = params(&[("objectPoint", Value::Vec2([40.0, 40.0]))]); + // With a model, no border stroke: an object touching the frame edge stays whole. + assert_eq!(prompt_strokes(&p, [100.0, 80.0], 0, true).map(|s| s.len()), Some(1)); + assert_eq!(prompt_strokes(&p, [100.0, 80.0], 0, false).map(|s| s.len()), Some(2)); + // The model's answer is part of the cache key, and drives the matte. + let stub = Arc::new(Stub::default()); + assert_ne!(seed(&p, [100.0, 80.0], 25.0, Some(stub.as_ref())), seed(&p, [100.0, 80.0], 25.0, None)); + let img = frame_at(0); + let mut frame = |g: i64| Some(Arc::new(frame_at(g))); + let FrameMatte::Alpha(a) = frame_matte(&p, [100.0, 80.0], 25.0, 0, 1.0, &img, Some(stub.clone()), &mut frame, &|| false) else { panic!("no matte") }; + assert!(a[40 * 100 + 40] > 0.5 && a[5 * 100 + 5] < 0.5); + assert_eq!(stub.calls.load(std::sync::atomic::Ordering::Relaxed), 1); + } +} diff --git a/crates/effects/src/roto.rs b/crates/effects/src/roto.rs index d514e80a..cd2d240e 100644 --- a/crates/effects/src/roto.rs +++ b/crates/effects/src/roto.rs @@ -195,6 +195,12 @@ pub fn model_for(params: &Params) -> Option> { if e(params, "version") == 0 { return None; } + installed_model() +} + +/// The trained model chosen in Settings ▸ Roto Brush, if one is installed (Roto Brush 2.0 / 3.0 +/// and Object Matte segment with it). +pub fn installed_model() -> Option> { model_slot().read().ok().and_then(|m| m.clone()) } @@ -392,7 +398,7 @@ pub fn downscale(s: &FrameSeg, scale: f64) -> FrameSeg { resample(s, w, h) } -fn resample(s: &FrameSeg, w: usize, h: usize) -> FrameSeg { +pub(crate) fn resample(s: &FrameSeg, w: usize, h: usize) -> FrameSeg { if s.w == w && s.h == h { return s.clone(); } diff --git a/crates/engine/src/commands/roto_cmds.rs b/crates/engine/src/commands/roto_cmds.rs index 95326b42..ff459d0b 100644 --- a/crates/engine/src/commands/roto_cmds.rs +++ b/crates/engine/src/commands/roto_cmds.rs @@ -1,6 +1,9 @@ //! Roto Brush & Refine Edge commands: strokes (the Roto Brush and Refine Edge tools in the Layer //! panel), propagation, the segmentation span, Freeze / Unfreeze, tool options and status. //! +//! `objectMatte.apply` adds an Object Matte effect (Effect ▸ Matte) from a point / box prompt on +//! a frame; it segments with the same engine and model as Roto Brush. +//! //! `roto.stroke` records a stroke into the layer's Roto Brush & Refine Edge effect (applying the //! effect on the first stroke, as After Effects does); the first foreground stroke sets the base //! frame and a span of 20 frames each side. Strokes on a frozen effect are refused (Unfreeze @@ -51,6 +54,15 @@ pub fn specs() -> Vec { always, options ), + cmd!( + "objectMatte.apply", + "Object Matte", + [], + None, + "{layer?, point: [x, y] (layer pixels: the object), box?: [x0, y0, x1, y1] (layer pixels, around the object), frame? (reference layer frame; default current), track?: follow the object from that frame (default true), before?, after? (frames to follow it; default 30, up to 600)}", + has_comp, + object_matte_apply + ), query!( "roto.status", "Roto Brush Status", @@ -422,3 +434,81 @@ fn status(s: &mut Session, p: &Value) -> Result { out["computed"] = json!(computed.iter().filter(|(_, c)| **c).map(|(f, _)| *f).collect::>()); Ok(out) } + +/// Two finite numbers. +fn xy_p(p: &Value, k: &str, n: usize, cmd: &str) -> Result>> { + let Some(v) = p.get(k).filter(|v| !v.is_null()) else { return Ok(None) }; + let a = v.as_array().filter(|a| a.len() == n).ok_or_else(|| bad(cmd, format!("`{k}` is {n} numbers (layer pixels)")))?; + let out: Vec = a.iter().filter_map(Value::as_f64).filter(|x| x.is_finite()).collect(); + if out.len() != n { + return Err(bad(cmd, format!("`{k}` is {n} numbers (layer pixels)"))); + } + Ok(Some(out)) +} + +fn object_matte_apply(s: &mut Session, p: &Value) -> Result { + use effectcraft_effects::object_matte as om; + let cmd = "objectMatte.apply"; + let (cid, lid) = layer_p(s, p, cmd)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or_else(|| bad(cmd, "no layer"))?; + if !layer.has_video() || layer.switches.adjustment || layer.effects().is_none() { + return Err(bad(cmd, "Object Matte works on footage, solid and composition layers")); + } + let size = rt::layer_size(&s.project, comp, layer); + let point = xy_p(p, "point", 2, cmd)?.ok_or_else(|| bad(cmd, "missing `point` [x, y] (layer pixels)"))?; + let inside = |x: f64, y: f64| (0.0..=size[0]).contains(&x) && (0.0..=size[1]).contains(&y); + if !inside(point[0], point[1]) { + return Err(bad(cmd, format!("`point` is outside the layer ({}×{})", size[0], size[1]))); + } + let bx = xy_p(p, "box", 4, cmd)?; + if let Some(b) = &bx { + let (x0, x1, y0, y1) = (b[0].min(b[2]), b[0].max(b[2]), b[1].min(b[3]), b[1].max(b[3])); + if !(point[0] > x0 && point[0] < x1 && point[1] > y0 && point[1] < y1) { + return Err(bad(cmd, "`point` must be inside `box`")); + } + } + let frame = frame_p(s, p, cid, layer); + let limits = rt::frame_limits(comp, layer); + if frame < limits[0] || frame > limits[1] { + return Err(bad(cmd, format!("frame {frame} is outside the layer ({}–{})", limits[0], limits[1]))); + } + let track = b_p(p, "track").unwrap_or(true); + let reach = |k: &str| f_p(p, k).filter(|v| v.is_finite()).unwrap_or(om::DEFAULT_REACH).round().clamp(0.0, om::MAX_REACH); + let (before, after) = (reach("before"), reach("after")); + let spec = effectcraft_effects::find(om::ID).ok_or_else(|| EngineError::Other("Object Matte effect missing".into()))?; + let uid = s.edit("Object Matte", None, |proj, _| { + let mut next = proj.next_id; + let l = layer_mut(proj, cid, lid)?; + let fxg = l.props.sub_mut("effects").ok_or_else(|| bad(cmd, "this layer has no effects"))?; + let n = fxg.groups().filter(|g| rt::is_object_matte(g)).count(); + let name = if n == 0 { spec.name.to_string() } else { format!("{} {}", spec.name, n + 1) }; + let mut g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), &name, size); + let mut set = |path: &str, v: KV| { + let (grp, leaf) = path.rsplit_once('/').map_or((None, path), |(a, b)| (Some(a), b)); + let target = match grp { + Some(sub) => g.sub_mut(sub), + None => Some(&mut g), + }; + if let Some(pr) = target.and_then(|t| t.get_mut(leaf)) { + pr.value = v; + } + }; + set("objectPoint", KV::Vec2([point[0], point[1]])); + if let Some(b) = &bx { + set("useBox", KV::Bool(true)); + set("boxTopLeft", KV::Vec2([b[0].min(b[2]), b[1].min(b[3])])); + set("boxBottomRight", KV::Vec2([b[0].max(b[2]), b[1].max(b[3])])); + } + set("objectTracking/trackObject", KV::Bool(track)); + set("objectTracking/referenceFrame", KV::Scalar(frame as f64)); + set("objectTracking/framesBefore", KV::Scalar(before)); + set("objectTracking/framesAfter", KV::Scalar(after)); + let u = g.uid; + fxg.children.push(g.into()); + proj.next_id = proj.next_id.max(next); + Ok(u) + })?; + s.state.selected_layers = vec![lid]; + Ok(json!({"layer": lid.0, "effect": uid, "frame": frame, "track": track, "span": [frame - before as i64, frame + after as i64]})) +} diff --git a/crates/engine/src/roto.rs b/crates/engine/src/roto.rs index ca92a74b..b8c2c649 100644 --- a/crates/engine/src/roto.rs +++ b/crates/engine/src/roto.rs @@ -147,6 +147,12 @@ pub(crate) fn is_roto(g: &PropGroup) -> bool { matches!(&g.kind, GroupKind::Effect { effect } if effect == fx::ID) } +/// An Object Matte instance (its Input Key is kept current like Roto Brush's; it segments on +/// demand while rendering). +pub(crate) fn is_object_matte(g: &PropGroup) -> bool { + matches!(&g.kind, GroupKind::Effect { effect } if effect == effectcraft_effects::object_matte::ID) +} + fn str_of(g: &PropGroup, m: &str) -> String { match g.get(m).map(|p| &p.value) { Some(Value::Str(s)) => s.clone(), @@ -188,8 +194,9 @@ pub fn input_key(p: &Project, comp: ItemId, layer: &Layer, uid: Uid) -> String { format!("{:016x}", fnv(&s)) } -/// After an edit: refresh Input Keys; frozen mattes made from other frames are dropped. -/// Returns the instances whose segmentation changed (to propagate again). +/// After an edit: refresh Input Keys (Roto Brush and Object Matte); frozen mattes made from +/// other frames are dropped. Returns the Roto Brush instances whose segmentation changed (to +/// propagate again). pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> { let mut todo = vec![]; let mut set = vec![]; @@ -197,13 +204,16 @@ pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerI let ItemKind::Comp(c) = &it.kind else { continue }; for l in &c.layers { let Some(fxg) = l.effects() else { continue }; - for g in fxg.groups().filter(|g| is_roto(g)) { + for g in fxg.groups().filter(|g| is_roto(g) || is_object_matte(g)) { let key = input_key(after, *cid, l, g.uid); + let roto = is_roto(g); let old = before.comp(*cid).and_then(|bc| bc.layer(l.id)).and_then(|ol| ol.props.find_group(g.uid)); if str_of(g, INPUT_KEY) != key { set.push((*cid, l.id, g.uid, key)); - todo.push((*cid, l.id, g.uid)); - } else if old.is_none_or(|o| str_of(o, fx::STROKES) != str_of(g, fx::STROKES)) { + if roto { + todo.push((*cid, l.id, g.uid)); + } + } else if roto && old.is_none_or(|o| str_of(o, fx::STROKES) != str_of(g, fx::STROKES)) { todo.push((*cid, l.id, g.uid)); } } diff --git a/crates/engine/src/tests_roto.rs b/crates/engine/src/tests_roto.rs index c1cbd2c8..cdf58540 100644 --- a/crates/engine/src/tests_roto.rs +++ b/crates/engine/src/tests_roto.rs @@ -351,3 +351,78 @@ fn offloaded_propagation_streams_segmentations_back() { s.execute("roto.propagate", json!({"layer": clip.0, "wait": true})).unwrap(); assert!(s.offloaded.is_empty()); } + +/// Object Matte: one click on the disk on frame 0 isolates it there and on the frames it is +/// followed to; the Input Key follows the layer's frames; bad prompts are refused. +#[test] +fn object_matte_from_a_click() { + hold_roto_cache_test_lock(); + let (mut s, clip, solid) = setup(|f| draw(center(f))); + let cid = s.active_comp_id().unwrap(); + s.edit("remove solid", None, |p, _| { + p.comp_mut(cid).unwrap().layers.retain(|l| l.id != solid); + for it in p.items.values_mut() { + if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind { + f.path = "object-matte.mov".into(); + } + } + Ok(()) + }) + .unwrap(); + s.execute("time.set", json!({"time": 0})).unwrap(); + let c = center(0); + let r = s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "point": c, "after": 6})).unwrap(); + assert_eq!(r["frame"], 0); + assert_eq!(r["span"], json!([-30, 6])); + let uid = r["effect"].as_u64().unwrap(); + let key = |s: &Session| { + let g = s.active_comp().unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().clone(); + crate::roto::params_static(&g).get(effectcraft_effects::roto::INPUT_KEY).cloned() + }; + let k0 = key(&s); + assert!(matches!(&k0, Some(effectcraft_keyframe::Value::Str(k)) if !k.is_empty()), "{k0:?}"); + let wrong = |s: &Session, f: u32| { + let c = center(f); + alpha_at(s, f) + .iter() + .enumerate() + .filter(|(i, v)| { + let d = (((*i as u32 % W) as f64 + 0.5 - c[0]).hypot((*i as u32 / W) as f64 + 0.5 - c[1])) - R; + (d < -3.0 && **v < 0.5) || (d > 3.0 && **v > 0.5) + }) + .count() + }; + let w0 = wrong(&s, 0); + assert!(w0 < 40, "{w0} wrong pixels on the reference frame"); + let w4 = wrong(&s, 4); + assert!(w4 < 60, "{w4} wrong pixels on frame 4"); + // Past Frames After: transparent. + assert!(alpha_at(&s, 8).iter().all(|v| *v == 0.0)); + // Other frames under the effect: a new Input Key (the mattes are recomputed). + s.edit("swap clip", None, |p, _| { + for it in p.items.values_mut() { + if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind { + f.path = "object-matte-2.mov".into(); + } + } + Ok(()) + }) + .unwrap(); + assert_ne!(key(&s), k0); + // Bad prompts are refused. + for bad in [ + json!({"layer": clip.0}), + json!({"layer": clip.0, "point": [-10, 5]}), + json!({"layer": clip.0, "point": [10, "x"]}), + json!({"layer": clip.0, "point": c, "box": [0, 0, 10, 10]}), + json!({"layer": clip.0, "point": c, "frame": 999}), + ] { + assert!(s.execute_checked("objectMatte.apply", bad.clone()).is_err(), "{bad}"); + } + // A box and a static prompt are accepted; the second instance is numbered. + let r = s + .execute_checked("objectMatte.apply", json!({"layer": clip.0, "point": c, "box": [c[0] + 50.0, c[1] + 50.0, c[0] - 50.0, c[1] - 50.0], "track": false})) + .unwrap(); + let g = s.active_comp().unwrap().layer(clip).unwrap().props.find_group(r["effect"].as_u64().unwrap()).unwrap().clone(); + assert_eq!(g.name, "Object Matte 2"); +} diff --git a/docs/effects.md b/docs/effects.md index e498cf1a..84f16ef8 100644 --- a/docs/effects.md +++ b/docs/effects.md @@ -3,11 +3,11 @@ -EffectCraft ships 306 effects with 3014 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. +EffectCraft ships 307 effects with 3028 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. - **GPU**: 280 effects also run on the GPU compositor with identical results. -- **32**: 306 effects process 32-bit float (HDR, overbright) pixels without clamping. -- **Status**: 306 implemented in full, 0 partial (what is missing is listed). +- **32**: 307 effects process 32-bit float (HDR, overbright) pixels without clamping. +- **Status**: 307 implemented in full, 0 partial (what is missing is listed). Effects are addressed by id (`ec..`) or display name in commands, scripts and the control channel, and every parameter by its id (`effects/#1/blurriness`). @@ -255,6 +255,7 @@ Effects are addressed by id (`ec..`) or display name in commands |---|---|---:|:-:|:-:|---| | Matte Choker | `ec.matte.mattechoker` | 7 | GPU | 32 | Implemented | | Mocha shape | `ec.obsolete.mochashape` | 6 | | 32 | Implemented | +| Object Matte | `ec.matte.objectmatte` | 13 | | 32 | Implemented | | Refine Hard Matte | `ec.matte.refinehard` | 16 | GPU | 32 | Implemented | | Refine Soft Matte | `ec.matte.refinesoft` | 18 | GPU | 32 | Implemented | | Roto Brush & Refine Edge | `ec.matte.rotobrush` | 24 | | 32 | Implemented | diff --git a/docs/gaps.md b/docs/gaps.md index acb22a63..577f7bd6 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -70,7 +70,7 @@ are grouped by area; the issue tracker has the rest. | 11 | **Text rendering and layout** | #453 CJK text in comps, #316 paragraph alignment inverted, #483 squeezed at 4:3, #417 soft at Fit, #583 font search, #550 toolbar font options | Text is in almost every motion graphic | 10–20 h | G2 · [ui-parity.md](ui-parity.md) | | 12 | **3D interaction and depth** | No 3D transform gizmos (#392, #347); Z position (#489); camera aperture (#260); reflections (#261); bevel options (#315); 3D issue reopened (#307) | Advanced 3D is After Effects' growth area since 2023 | 20–35 h | G2 · [ui-parity.md](ui-parity.md) | | 13 | **3D model formats and materials** | After Effects 26 imports FBX, USD/USDZ, STL besides glTF and OBJ (`usd_plugins`), and Substance 3D materials with mesh displacement (26.0–26.2); we read glTF and OBJ | Product shots and 3D scenes come as FBX/USDZ | 25–45 h | — · [file-format-parity.md](file-format-parity.md) | -| 14 | **Object Matte** (26.2, disk cache in 26.5) | Not present; MobileSAM is already in `effectcraft-segment` and could back it | The new one-click roto; Roto Brush covers part of it | 15–30 h | G7 · [target-app-parity.md](target-app-parity.md) | +| 14 | **Object Matte** (26.2, disk cache in 26.5) | First slice (#783): the Object Matte effect (Effect ▸ Matte) and `objectMatte.apply` turn a point (and optional box) on a reference frame into a matte and follow the object up to 600 frames each way, or segment every frame on its own; Roto Brush's engine, cache and model (MobileSAM from Settings ▸ Roto Brush, classical graph cut without it). Missing: the click / hover tool in the viewer, a background propagation job, the disk cache, comparison with After Effects | The new one-click roto | 10–20 h | G7 · [target-app-parity.md](target-app-parity.md) | | 15 | **Hardware video decode and encode** | No VideoToolbox, NVENC, Quick Sync or VA-API; #498 asks for it; FilmCraft has VideoToolbox / NVENC / VA-API paths to borrow | 4K playback and export speed | 25–45 h | G5/G6 · [hardware-parity.md](hardware-parity.md) | | 16 | **Localization depth** | Menus in 6 languages; panels in 4; effect names and ~1,650 effect parameter names untranslated; French, German, Korean, Italian, Russian (After Effects ships them) absent; #599 Hangul boxes on macOS | After Effects is used heavily in Japan, China, Korea, Brazil, Europe | 60–110 h + native review | G9 · [localization-parity.md](localization-parity.md) | | 17 | **Media Browser lists files import refuses** | `media_browser.rs` lists TGA, DPX, HDR, AVI, MXF; `media::is_importable` has none of them; #431 any image imports as a sequence; #324 MP3 goes silent | Confusing failures on common VFX formats (DPX, TGA) | 15–30 h | G6 · [file-format-parity.md](file-format-parity.md) | diff --git a/docs/target-app-parity.md b/docs/target-app-parity.md index 12d37245..f0ed90b6 100644 --- a/docs/target-app-parity.md +++ b/docs/target-app-parity.md @@ -122,7 +122,7 @@ compositing and effects dominate; audio is light). Hours are to full parity for | Text and typography | 7% | 94% | 50% | 20–35 | CJK text in comps renders wrongly (#453); paragraph alignment inverted (#316); squeezed text at 4:3 (#483); font search (#583); soft text at Fit (#417); variable-font filter (26.3) | | Effects and animation presets | 14% | 85% | 40% | 70–130 | All 306 effects exist, but rendered output is unmeasured against After Effects; 8 built-in presets against After Effects' 621; open effect bugs (#580 crash, #514, #526, #463, #369, #473); effect colour labels (26.5); see [effects-parity.md](effects-parity.md) | | 3D (Classic and Advanced) | 6% | 82% | 35% | 35–65 | No FBX/USD/STL models, no Substance materials, mesh displacement unverified; no 3D transform gizmos (#392, #347); Z position (#489); reflections (#261); camera aperture (#260); bevel options (#315) | -| Tracking and AI-assisted tools | 5% | 80% | 35% | 35–70 | No Object Matte (26.2); Roto Brush (MobileSAM) and face tracking (MediaPipe) unmeasured against After Effects; camera-tracker view (#505) | +| Tracking and AI-assisted tools | 5% | 80% | 35% | 35–70 | Object Matte (26.2) is a first slice: an effect and command from a point / box prompt, no viewer tool or disk cache; Roto Brush (MobileSAM) and face tracking (MediaPipe) unmeasured against After Effects; camera-tracker view (#505) | | Preview and cache | 7% | 92% | 40% | 25–50 | Playback stutters on cached frames (#595, #304); slow scrubbing (#329); late audio (#554, #504, #335); no benchmark against After Effects | | Render queue and export | 7% | 85% | 55% | 25–45 | No hardware encoding; ProRes 4444 colour tag (#340); rendering over an existing file (#325); default macOS output location (#446); uncompressed single-file output (#372); HEVC/AV1 without B-frames | | Audio | 2% | 85% | 45% | 6–12 | Audio sync and crackle (#335); audio to keyframes depth | @@ -144,7 +144,7 @@ Feature depth (≈ 50%) feeds the overall *ready* number with the other dimensio | Stability | 45% | 40–80 | Open crash reports #580, #591, #501, #426, #341, #306, #234, #511; no CI runs the tests on pull requests | [gaps.md](gaps.md#g3-stability-and-a-green-main) | | Platforms | 65% | 30–60 | Releases for macOS, Windows x64/x86/ARM64, Linux, FreeBSD and the web (more than After Effects), but Linux and Windows users hit basic bugs | [gaps.md](gaps.md#g2-real-user-reliability-on-every-platform) | | Ecosystem and plug-ins | 20% | 40–80 | After Effects scripts and expressions mostly carry over; no After Effects SDK plug-ins, `.ffx` presets or `.mogrt`; our WebAssembly plug-in API has no third-party plug-ins yet | [plugins.md](plugins.md) | -| AI features | 35% | 40–80 (overlaps Tracking) | Roto Brush (MobileSAM) and face tracking (MediaPipe) as optional downloads; no Object Matte; the After Effects bundle ships Whisper, BERT and Marian tokenizers and a shot-cut model we have no equivalent of | [gaps.md](gaps.md) | +| AI features | 35% | 40–80 (overlaps Tracking) | Roto Brush (MobileSAM) and face tracking (MediaPipe) as optional downloads; Object Matte (point / box prompt, same model) without its viewer tool or disk cache; the After Effects bundle ships Whisper, BERT and Marian tokenizers and a shot-cut model we have no equivalent of | [gaps.md](gaps.md) | | Agent control (beyond After Effects) | ahead | — | Every command is drivable over MCP, the control channel and the CLI | [agents.md](agents.md) | **Totals.** Summing the rows double-counts: AI overlaps Tracking, performance overlaps Preview, From c21a93e45eed34cd0a6bbab3ece518b2e43b281a Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 11 Oct 2026 13:33:17 +0200 Subject: [PATCH 2/8] #783: app.capabilities counts Object Matte (307 registered effects) Problem: the hand-maintained effect count in app.capabilities still said 306 after Object Matte was registered. Fix: bump it to 307. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/commands/query.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/engine/src/commands/query.rs b/crates/engine/src/commands/query.rs index 56139b6c..2872d9a5 100644 --- a/crates/engine/src/commands/query.rs +++ b/crates/engine/src/commands/query.rs @@ -191,7 +191,7 @@ fn capabilities(s: &mut Session, _: &Value) -> Result { } /// The headline of `docs/target-app-parity.md` (kept in sync by hand at each re-measure). -const PARITY: &str = "≈91% feature breadth and ≈45% ready for real work (estimated parity with After Effects 2026 26.5, stage alpha); all After Effects effects present (306 registered)"; +const PARITY: &str = "≈91% feature breadth and ≈45% ready for real work (estimated parity with After Effects 2026 26.5, stage alpha); all After Effects effects present (307 registered)"; fn state(s: &mut Session, _: &Value) -> Result { Ok(serde_json::to_value(&s.state).unwrap_or_default()) From 4fc7b5235196732a66aa8efa4a3a3b02c48658a0 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 11 Oct 2026 13:38:10 +0200 Subject: [PATCH 3/8] #783: Object Matte passes its box to the model as a box prompt Problem: with a trained model installed, Object Matte's optional box reached MobileSAM only as background points along the box's edges (an approximation), although the model's prompt encoder takes a box prompt directly. Fix: a new Roto Brush stroke kind, Box (two opposite corners), that stroke_prompt turns into the model prompt's bbox (scaled to the matte grid, corners in any order, degenerate or non-finite boxes ignored). The graph cut never rasterizes it. Object Matte emits it instead of the edge strokes when a model segments; the classical fallback keeps the edge strokes. A prior matte's box no longer replaces a box the user drew on that frame. Tests: effects object_matte::the_box_reaches_the_model_as_a_box_prompt (a stand-in model sees the box, at full and half resolution, and no edge points); track roto::tests::box_stroke_is_the_model_box_prompt. Co-Authored-By: Claude Opus 5.5 --- crates/effects/src/object_matte.rs | 56 ++++++++++++++++++++++++++---- crates/track/src/roto/mod.rs | 28 +++++++++++---- crates/track/src/roto/tests.rs | 22 ++++++++++++ 3 files changed, 94 insertions(+), 12 deletions(-) diff --git a/crates/effects/src/object_matte.rs b/crates/effects/src/object_matte.rs index 3b7b7c28..9ee33a90 100644 --- a/crates/effects/src/object_matte.rs +++ b/crates/effects/src/object_matte.rs @@ -5,8 +5,9 @@ //! The prompt is the **Object Point** (layer pixels) and, with **Use Box**, a box around the //! object. On the **Reference Frame** it is segmented by the same engine as Roto Brush: the //! trained model chosen in Settings ▸ Roto Brush (MobileSAM, `effectcraft_segment`) when one is -//! installed, the classical graph cut otherwise (the point is a foreground stroke, the box's -//! edges, or without a box the layer's border, background strokes). With **Track Object** the +//! installed, the classical graph cut otherwise. The model gets the point as a foreground point +//! and the box as its box prompt; the graph cut gets the point as a foreground stroke and the +//! box's edges (without a box, the layer's border) as background strokes. With **Track Object** the //! matte is then carried frame by frame up to **Frames Before** / **Frames After** away (optical //! flow, re-segmented by the model prompted with the carried matte); without it every frame is //! segmented on its own from the prompt (useful when the point is animated). @@ -119,9 +120,10 @@ pub fn tracks(params: &Params) -> bool { } /// The prompt as Roto Brush strokes on layer frame `frame`: the Object Point as a foreground -/// stroke; the box's edges as background strokes; without a box, the layer's border when there -/// is no model (the graph cut needs background samples; a model does not, and a border stroke -/// would cut objects touching the frame edge). `None` when the point is unusable. +/// stroke; the box as a box stroke (the model's box prompt) with a model, its edges as +/// background strokes for the graph cut without one; without a box, the layer's border when +/// there is no model (the graph cut needs background samples; a model does not, and a border +/// stroke would cut objects touching the frame edge). `None` when the point is unusable. pub fn prompt_strokes(params: &Params, layer_size: [f64; 2], frame: i64, with_model: bool) -> Option> { let [w, h] = layer_size; if !(w.is_finite() && h.is_finite() && w >= 1.0 && h >= 1.0) { @@ -144,7 +146,12 @@ pub fn prompt_strokes(params: &Params, layer_size: [f64; 2], frame: i64, with_mo if pt[0] - m <= x0 || pt[0] + m >= x1 || pt[1] - m <= y0 || pt[1] + m >= y1 { return None; } - out.push(Stroke { kind: StrokeKind::Bg, frame, radius: EDGE_RADIUS, points: rect(x0, y0, x1, y1) }); + out.push(if with_model { + // The model takes the box itself as its box prompt. + Stroke { kind: StrokeKind::Box, frame, radius: 0.0, points: vec![[x0, y0], [x1, y1]] } + } else { + Stroke { kind: StrokeKind::Bg, frame, radius: EDGE_RADIUS, points: rect(x0, y0, x1, y1) } + }); } None if !with_model => { let i = EDGE_RADIUS; @@ -438,6 +445,8 @@ mod tests { #[derive(Default)] struct Stub { calls: std::sync::atomic::AtomicUsize, + /// The box prompt of the last call. + bbox: std::sync::Mutex>, } impl MaskModel for Stub { fn info(&self) -> &'static effectcraft_segment::ModelInfo { @@ -445,6 +454,7 @@ mod tests { } fn segment(&self, _: &[[f32; 3]], w: usize, h: usize, prompt: &effectcraft_segment::Prompt) -> effectcraft_segment::Result> { self.calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + *self.bbox.lock().unwrap() = prompt.bbox; let Some(([cx, cy], _)) = prompt.points.iter().find(|(_, fg)| *fg) else { return Err("no point".into()) }; Ok((0..w * h).map(|i| (((i % w) as f32 + 0.5 - cx).hypot((i / w) as f32 + 0.5 - cy) < 15.0) as u8 as f32).collect()) } @@ -464,5 +474,39 @@ mod tests { let FrameMatte::Alpha(a) = frame_matte(&p, [100.0, 80.0], 25.0, 0, 1.0, &img, Some(stub.clone()), &mut frame, &|| false) else { panic!("no matte") }; assert!(a[40 * 100 + 40] > 0.5 && a[5 * 100 + 5] < 0.5); assert_eq!(stub.calls.load(std::sync::atomic::Ordering::Relaxed), 1); + assert_eq!(*stub.bbox.lock().unwrap(), None); + } + + #[test] + fn the_box_reaches_the_model_as_a_box_prompt() { + let p = params(&[ + ("objectPoint", Value::Vec2([90.0, 70.0])), + ("useBox", Value::Bool(true)), + ("boxTopLeft", Value::Vec2([100.0, 80.0])), + ("boxBottomRight", Value::Vec2([74.0, 54.0])), + ]); + // With a model: the point and a box stroke (corners ordered); without: the box's edges + // as a background stroke, as before. + let with = prompt_strokes(&p, [100.0, 80.0], 0, true).unwrap(); + assert_eq!(with.iter().map(|s| s.kind).collect::>(), [StrokeKind::Fg, StrokeKind::Box]); + assert_eq!(with[1].points, [[74.0, 54.0], [100.0, 80.0]]); + let without = prompt_strokes(&p, [100.0, 80.0], 0, false).unwrap(); + assert_eq!(without.iter().map(|s| s.kind).collect::>(), [StrokeKind::Fg, StrokeKind::Bg]); + // The model is called with the box itself (matte pixels at scale 1) and only one + // foreground point: no background points along the edges. + let stub = Arc::new(Stub::default()); + let img = frame_at(0); + let mut frame = |g: i64| Some(Arc::new(frame_at(g))); + let FrameMatte::Alpha(a) = frame_matte(&p, [100.0, 80.0], 25.0, 0, 1.0, &img, Some(stub.clone()), &mut frame, &|| false) else { panic!("no matte") }; + assert_eq!(*stub.bbox.lock().unwrap(), Some([74.0, 54.0, 100.0, 80.0])); + assert!(a[70 * 100 + 90] > 0.5 && a[40 * 100 + 40] < 0.5); + // At half resolution the box is scaled with the frame (another Input Key, so the + // full-resolution result above is not reused). + let mut p = p; + p.values.insert(INPUT_KEY.into(), Value::Str("half".into())); + let half = effectcraft_raster::resample(&img, 50, 40); + let mut frame = |g: i64| Some(Arc::new(effectcraft_raster::resample(&frame_at(g), 50, 40))); + let _ = frame_matte(&p, [100.0, 80.0], 25.0, 0, 0.5, &half, Some(stub.clone()), &mut frame, &|| false); + assert_eq!(*stub.bbox.lock().unwrap(), Some([37.0, 27.0, 50.0, 40.0])); } } diff --git a/crates/track/src/roto/mod.rs b/crates/track/src/roto/mod.rs index 7138a93a..4efb4335 100644 --- a/crates/track/src/roto/mod.rs +++ b/crates/track/src/roto/mod.rs @@ -59,6 +59,9 @@ pub enum StrokeKind { Refine, /// Refine Edge tool with Alt: remove from the band. RefineErase, + /// A box around the object (Object Matte): two opposite corners in `points`. Trained models + /// get it as their box prompt; the graph cut ignores it. + Box, } impl StrokeKind { @@ -830,8 +833,9 @@ fn points_along(s: &Stroke, scale: f64, max: usize) -> Vec<[f32; 2]> { out } -/// A model's prompt from the strokes: points along the foreground and background strokes. -fn stroke_prompt(strokes: &[&Stroke], scale: f64) -> Prompt { +/// A model's prompt from the strokes: points along the foreground and background strokes, and +/// the box of a [`StrokeKind::Box`] stroke (matte pixels). +pub fn stroke_prompt(strokes: &[&Stroke], scale: f64) -> Prompt { let mut points = vec![]; for (kind, fg) in [(StrokeKind::Fg, true), (StrokeKind::Bg, false)] { let ss: Vec<&&Stroke> = strokes.iter().filter(|s| s.kind == kind).collect(); @@ -840,11 +844,20 @@ fn stroke_prompt(strokes: &[&Stroke], scale: f64) -> Prompt { points.extend(points_along(s, scale, per).into_iter().map(|p| (p, fg))); } } - Prompt { points, ..Default::default() } + let bbox = strokes.iter().filter(|s| s.kind == StrokeKind::Box).find_map(|s| box_of(s, scale)); + Prompt { points, bbox, ..Default::default() } } -/// Add a prior matte to a prompt: its box (a little larger), up to four points spread through -/// its core, and the matte itself. +/// The box `[x0, y0, x1, y1]` (matte pixels) of a box stroke: its first two points as opposite +/// corners; `None` when they are missing, not finite or the box is empty. +fn box_of(s: &Stroke, scale: f64) -> Option<[f32; 4]> { + let (a, c) = (s.points.first()?, s.points.get(1)?); + let v = [a[0].min(c[0]), a[1].min(c[1]), a[0].max(c[0]), a[1].max(c[1])].map(|v| (v * scale) as f32); + (v.iter().all(|x| x.is_finite()) && v[2] > v[0] && v[3] > v[1]).then_some(v) +} + +/// Add a prior matte to a prompt: its box (a little larger; unless the prompt has one), up to +/// four points spread through its core, and the matte itself. fn add_prior_prompt(prompt: &mut Prompt, soft: &[f32], w: usize, h: usize) -> bool { let bin: Vec = soft.iter().map(|v| (*v >= 0.5) as u8).collect(); let (mut x0, mut y0, mut x1, mut y1) = (usize::MAX, usize::MAX, 0usize, 0usize); @@ -858,7 +871,10 @@ fn add_prior_prompt(prompt: &mut Prompt, soft: &[f32], w: usize, h: usize) -> bo return false; } let (mx, my) = (((x1 - x0) as f32 * 0.05).max(4.0), ((y1 - y0) as f32 * 0.05).max(4.0)); - prompt.bbox = Some([(x0 as f32 - mx).max(0.0), (y0 as f32 - my).max(0.0), (x1 as f32 + 1.0 + mx).min(w as f32), (y1 as f32 + 1.0 + my).min(h as f32)]); + // A box the user drew on this frame wins over the prior's. + if prompt.bbox.is_none() { + prompt.bbox = Some([(x0 as f32 - mx).max(0.0), (y0 as f32 - my).max(0.0), (x1 as f32 + 1.0 + mx).min(w as f32), (y1 as f32 + 1.0 + my).min(h as f32)]); + } // Farthest-point picks among the core (deep inside), starting from the deepest pixel. let sd = matting::signed_distance(&bin, w, h); let deep = sd.iter().copied().fold(0.0f32, f32::max); diff --git a/crates/track/src/roto/tests.rs b/crates/track/src/roto/tests.rs index f92e67d5..65d1743f 100644 --- a/crates/track/src/roto/tests.rs +++ b/crates/track/src/roto/tests.rs @@ -428,3 +428,25 @@ fn mobilesam_segments_and_propagates_the_disk() { eprintln!("MobileSAM: base IoU {jb:.4}, worst propagated {worst:.4} (classic {cworst:.4}), {:.0} ms/frame", t.elapsed().as_secs_f64() * 1000.0 / 20.0); assert!(jb > 0.95 && worst > 0.93 && worst >= cworst - 0.01, "base {jb}, worst {worst} vs classic {cworst}"); } + +/// A box stroke becomes the model's box prompt (scaled, corners in any order); the graph cut +/// ignores it; a prior's box does not replace it. +#[test] +fn box_stroke_is_the_model_box_prompt() { + let fg = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 2.0, points: vec![[50.0, 40.0]] }; + let bx = Stroke { kind: StrokeKind::Box, frame: 0, radius: 0.0, points: vec![[80.0, 60.0], [20.0, 10.0]] }; + let p = stroke_prompt(&[&fg, &bx], 0.5); + assert_eq!(p.bbox, Some([10.0, 5.0, 40.0, 30.0])); + assert_eq!(p.points, vec![([25.0, 20.0], true)]); + // Degenerate or non-finite boxes are no prompt. + for pts in [vec![[20.0, 10.0]], vec![[20.0, 10.0], [20.0, 50.0]], vec![[f64::NAN, 10.0], [80.0, 60.0]]] { + let b = Stroke { points: pts, ..bx.clone() }; + assert_eq!(stroke_prompt(&[&fg, &b], 1.0).bbox, None); + } + assert!(!StrokeKind::Box.is_segmentation()); + assert!(rasterize(&[&bx], &[StrokeKind::Fg, StrokeKind::Bg], 1.0, 100, 80).iter().all(|v| !v)); + let mut q = stroke_prompt(&[&fg, &bx], 1.0); + let soft: Vec = (0..100 * 80).map(|i| ((i % 100) < 10) as u8 as f32).collect(); + assert!(add_prior_prompt(&mut q, &soft, 100, 80)); + assert_eq!(q.bbox, Some([20.0, 10.0, 80.0, 60.0])); +} From 0c695cff7740241a38621faa334a33c164f2e197 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 11 Oct 2026 13:50:01 +0200 Subject: [PATCH 4/8] #783: Object Matte tool in the Composition viewer Problem: Object Matte could only be applied through the objectMatte.apply command; After Effects 26.2 picks the object directly in the viewer (click it, or drag a box around it). Fix: an Object Matte tool (with Roto Brush & Refine Edge in the Alt+W slot, icon drawn in code). In the Composition viewer a click sets a point prompt and a drag a box prompt (its centre the point) on the footage, solid or comp layer under the pointer, converted to layer pixels through the layer's transform. The layer's Object Matte (the selected one, else its last) is re-prompted; Shift adds another. Everything goes through objectMatte.apply, which gained `effect` to re-prompt an instance (point, box or no box, reference frame; the reach kept unless given), so each prompt is one undo step; the effect is then selected, so its point controls show in the viewer. Hover draws a crosshair and the outline of the layer that would get the matte (a live matte preview would need a segmentation per pointer move). Automation ids: viewer.objectMatte.crosshair / .target / .box, tools.ObjectMatte. Tests: ui-egui ui_object_matte (hover, click, drag box re-prompts the same effect, Shift-click adds one, undo steps); engine tests_roto::object_matte_reprompt_updates_the_instance. Checked with the headless snapshot example (hover crosshair, box drag isolating a disk). Co-Authored-By: Claude Opus 5.5 --- crates/effects/src/object_matte.rs | 8 +- crates/engine/src/commands/roto_cmds.rs | 72 +++++++-- crates/engine/src/tests_roto.rs | 39 +++++ crates/ui-egui/src/icons.rs | 10 ++ crates/ui-egui/src/panels/mod.rs | 1 + .../ui-egui/src/panels/object_matte_tool.rs | 128 ++++++++++++++++ crates/ui-egui/src/panels/viewer.rs | 18 ++- crates/ui-egui/src/state.rs | 11 +- crates/ui-egui/tests/ui_object_matte.rs | 142 ++++++++++++++++++ 9 files changed, 409 insertions(+), 20 deletions(-) create mode 100644 crates/ui-egui/src/panels/object_matte_tool.rs create mode 100644 crates/ui-egui/tests/ui_object_matte.rs diff --git a/crates/effects/src/object_matte.rs b/crates/effects/src/object_matte.rs index 9ee33a90..a7db562a 100644 --- a/crates/effects/src/object_matte.rs +++ b/crates/effects/src/object_matte.rs @@ -113,6 +113,11 @@ pub fn reference_frame(params: &Params) -> i64 { frames(params, "objectTracking/referenceFrame", -1e9, 1e9) } +/// Frames Before / After (finite, clamped to `0..=MAX_REACH`). +pub fn reach(params: &Params) -> [i64; 2] { + [frames(params, "objectTracking/framesBefore", 0.0, MAX_REACH), frames(params, "objectTracking/framesAfter", 0.0, MAX_REACH)] +} + /// Whether the matte is carried from the reference frame (else every frame is segmented on its /// own). pub fn tracks(params: &Params) -> bool { @@ -170,8 +175,7 @@ pub fn prompt_strokes(params: &Params, layer_size: [f64; 2], frame: i64, with_mo pub fn roto_data(params: &Params, layer_size: [f64; 2], f: i64, with_model: bool) -> Option { if tracks(params) { let base = reference_frame(params); - let before = frames(params, "objectTracking/framesBefore", 0.0, MAX_REACH); - let after = frames(params, "objectTracking/framesAfter", 0.0, MAX_REACH); + let [before, after] = reach(params); let strokes = prompt_strokes(params, layer_size, base, with_model)?; Some(RotoData { base: Some(base), span: [base.saturating_sub(before), base.saturating_add(after)], strokes }) } else { diff --git a/crates/engine/src/commands/roto_cmds.rs b/crates/engine/src/commands/roto_cmds.rs index ff459d0b..f4f33d53 100644 --- a/crates/engine/src/commands/roto_cmds.rs +++ b/crates/engine/src/commands/roto_cmds.rs @@ -59,7 +59,7 @@ pub fn specs() -> Vec { "Object Matte", [], None, - "{layer?, point: [x, y] (layer pixels: the object), box?: [x0, y0, x1, y1] (layer pixels, around the object), frame? (reference layer frame; default current), track?: follow the object from that frame (default true), before?, after? (frames to follow it; default 30, up to 600)}", + "{layer?, effect? (uid: re-prompt this Object Matte instead of adding one), point: [x, y] (layer pixels: the object), box?: [x0, y0, x1, y1] (layer pixels, around the object), frame? (reference layer frame; default current), track?: follow the object from that frame (default true), before?, after? (frames to follow it; default 30, up to 600)}", has_comp, object_matte_apply ), @@ -455,6 +455,18 @@ fn object_matte_apply(s: &mut Session, p: &Value) -> Result { if !layer.has_video() || layer.switches.adjustment || layer.effects().is_none() { return Err(bad(cmd, "Object Matte works on footage, solid and composition layers")); } + // Re-prompt an existing instance (the Object Matte tool on a layer that has one). + let existing = match p.get("effect").filter(|v| !v.is_null()) { + Some(v) => { + let uid = v.as_u64().ok_or_else(|| bad(cmd, "`effect` is an Object Matte effect's uid"))?; + layer + .effects() + .and_then(|f| f.groups().find(|g| g.uid == uid && rt::is_object_matte(g))) + .ok_or_else(|| bad(cmd, format!("no Object Matte effect {uid} on this layer")))?; + Some(uid) + } + None => None, + }; let size = rt::layer_size(&s.project, comp, layer); let point = xy_p(p, "point", 2, cmd)?.ok_or_else(|| bad(cmd, "missing `point` [x, y] (layer pixels)"))?; let inside = |x: f64, y: f64| (0.0..=size[0]).contains(&x) && (0.0..=size[1]).contains(&y); @@ -473,42 +485,74 @@ fn object_matte_apply(s: &mut Session, p: &Value) -> Result { if frame < limits[0] || frame > limits[1] { return Err(bad(cmd, format!("frame {frame} is outside the layer ({}–{})", limits[0], limits[1]))); } - let track = b_p(p, "track").unwrap_or(true); - let reach = |k: &str| f_p(p, k).filter(|v| v.is_finite()).unwrap_or(om::DEFAULT_REACH).round().clamp(0.0, om::MAX_REACH); + // A new instance gets the defaults; a re-prompted one keeps what is not given. + let track = b_p(p, "track").or(existing.is_none().then_some(true)); + let reach = |k: &str| { + let v = f_p(p, k).filter(|v| v.is_finite()); + let v = if existing.is_none() { Some(v.unwrap_or(om::DEFAULT_REACH)) } else { v }; + v.map(|v| v.round().clamp(0.0, om::MAX_REACH)) + }; let (before, after) = (reach("before"), reach("after")); let spec = effectcraft_effects::find(om::ID).ok_or_else(|| EngineError::Other("Object Matte effect missing".into()))?; - let uid = s.edit("Object Matte", None, |proj, _| { + let label = if existing.is_some() { "Object Matte Prompt" } else { "Object Matte" }; + let uid = s.edit(label, None, |proj, _| { let mut next = proj.next_id; let l = layer_mut(proj, cid, lid)?; let fxg = l.props.sub_mut("effects").ok_or_else(|| bad(cmd, "this layer has no effects"))?; - let n = fxg.groups().filter(|g| rt::is_object_matte(g)).count(); - let name = if n == 0 { spec.name.to_string() } else { format!("{} {}", spec.name, n + 1) }; - let mut g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), &name, size); + let mut fresh = None; + let g = match existing { + Some(u) => fxg.find_group_mut(u).ok_or_else(|| bad(cmd, "the Object Matte effect is gone"))?, + None => { + let n = fxg.groups().filter(|g| rt::is_object_matte(g)).count(); + let name = if n == 0 { spec.name.to_string() } else { format!("{} {}", spec.name, n + 1) }; + fresh.insert(effectcraft_effects::instantiate(spec, &mut Ids(&mut next), &name, size)) + } + }; let mut set = |path: &str, v: KV| { let (grp, leaf) = path.rsplit_once('/').map_or((None, path), |(a, b)| (Some(a), b)); let target = match grp { Some(sub) => g.sub_mut(sub), - None => Some(&mut g), + None => Some(&mut *g), }; + // The prompt is static: a re-prompt replaces any keyframes. if let Some(pr) = target.and_then(|t| t.get_mut(leaf)) { pr.value = v; + pr.keys.clear(); } }; set("objectPoint", KV::Vec2([point[0], point[1]])); + set("useBox", KV::Bool(bx.is_some())); if let Some(b) = &bx { - set("useBox", KV::Bool(true)); set("boxTopLeft", KV::Vec2([b[0].min(b[2]), b[1].min(b[3])])); set("boxBottomRight", KV::Vec2([b[0].max(b[2]), b[1].max(b[3])])); } - set("objectTracking/trackObject", KV::Bool(track)); + if let Some(t) = track { + set("objectTracking/trackObject", KV::Bool(t)); + } set("objectTracking/referenceFrame", KV::Scalar(frame as f64)); - set("objectTracking/framesBefore", KV::Scalar(before)); - set("objectTracking/framesAfter", KV::Scalar(after)); + if let Some(v) = before { + set("objectTracking/framesBefore", KV::Scalar(v)); + } + if let Some(v) = after { + set("objectTracking/framesAfter", KV::Scalar(v)); + } let u = g.uid; - fxg.children.push(g.into()); + if let Some(g) = fresh { + fxg.children.push(g.into()); + } proj.next_id = proj.next_id.max(next); Ok(u) })?; s.state.selected_layers = vec![lid]; - Ok(json!({"layer": lid.0, "effect": uid, "frame": frame, "track": track, "span": [frame - before as i64, frame + after as i64]})) + let (track, before, after) = object_matte_reach(s, cid, lid, uid); + Ok(json!({"layer": lid.0, "effect": uid, "frame": frame, "track": track, "updated": existing.is_some(), "span": [frame - before, frame + after]})) +} + +/// Track Object and Frames Before / After of an Object Matte instance (static values). +fn object_matte_reach(s: &Session, cid: ItemId, lid: LayerId, uid: Uid) -> (bool, i64, i64) { + use effectcraft_effects::object_matte as om; + let Ok((_, g)) = group(s, cid, lid, uid) else { return (true, 0, 0) }; + let ps = params_static(g); + let [before, after] = om::reach(&ps); + (om::tracks(&ps), before, after) } diff --git a/crates/engine/src/tests_roto.rs b/crates/engine/src/tests_roto.rs index cdf58540..19eee110 100644 --- a/crates/engine/src/tests_roto.rs +++ b/crates/engine/src/tests_roto.rs @@ -426,3 +426,42 @@ fn object_matte_from_a_click() { let g = s.active_comp().unwrap().layer(clip).unwrap().props.find_group(r["effect"].as_u64().unwrap()).unwrap().clone(); assert_eq!(g.name, "Object Matte 2"); } + +/// `objectMatte.apply` with `effect` re-prompts that instance in one undo step: the new point and +/// box, the box dropped when none is given, the reach kept unless given. +#[test] +fn object_matte_reprompt_updates_the_instance() { + let (mut s, clip, solid) = setup(|f| draw(center(f))); + let cid = s.active_comp_id().unwrap(); + s.edit("remove solid", None, |p, _| { + p.comp_mut(cid).unwrap().layers.retain(|l| l.id != solid); + Ok(()) + }) + .unwrap(); + let c = center(0); + let r = s + .execute_checked( + "objectMatte.apply", + json!({"layer": clip.0, "point": c, "box": [c[0] - 50.0, c[1] - 50.0, c[0] + 50.0, c[1] + 50.0], "after": 6, "frame": 0}), + ) + .unwrap(); + let uid = r["effect"].as_u64().unwrap(); + let count = |s: &Session| s.active_comp().unwrap().layer(clip).unwrap().effects().unwrap().groups().count(); + let ps = |s: &Session| crate::roto::params_static(s.active_comp().unwrap().layer(clip).unwrap().props.find_group(uid).unwrap()); + let undo0 = s.history.undo.len(); + let r = s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "effect": uid, "point": [c[0] + 4.0, c[1]], "frame": 2})).unwrap(); + assert_eq!((r["effect"].as_u64(), r["updated"].as_bool()), (Some(uid), Some(true))); + assert_eq!(r["span"], json!([-28, 8])); + assert_eq!(count(&s), 1); + assert_eq!(s.history.undo.len(), undo0 + 1); + let p = ps(&s); + assert_eq!(p.get("objectPoint"), Some(&effectcraft_keyframe::Value::Vec2([c[0] + 4.0, c[1]]))); + assert_eq!(p.get("useBox"), Some(&effectcraft_keyframe::Value::Bool(false))); + assert_eq!(effectcraft_effects::object_matte::reference_frame(&p), 2); + assert_eq!(effectcraft_effects::object_matte::reach(&p), [30, 6]); + // Undo restores the first prompt. + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(ps(&s).get("useBox"), Some(&effectcraft_keyframe::Value::Bool(true))); + // An unknown or non-Object Matte effect is refused. + assert!(s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "effect": 999_999, "point": c})).is_err()); +} diff --git a/crates/ui-egui/src/icons.rs b/crates/ui-egui/src/icons.rs index 52066f85..91dd0f85 100644 --- a/crates/ui-egui/src/icons.rs +++ b/crates/ui-egui/src/icons.rs @@ -34,6 +34,8 @@ pub enum Icon { Eraser, RotoBrush, RefineEdge, + /// Object Matte: a selection box's corners around a solid object, with a click point. + ObjectMatte, /// Puppet Position Pin: a pushpin. The other pin tools add a badge at the lower right. Puppet, PuppetAdvanced, @@ -314,6 +316,14 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) { pen.line(&[(8.0, 14.0), (9.5, 12.5)]); pen.line(&[(11.0, 14.0), (12.5, 12.5)]); } + ObjectMatte => { + // Corner brackets of a box around a filled object, and the click on it. + for (x, y, dx, dy) in [(2.0, 2.0, 1.0, 1.0), (14.0, 2.0, -1.0, 1.0), (14.0, 14.0, -1.0, -1.0), (2.0, 14.0, 1.0, -1.0)] { + pen.line(&[(x, y + 3.0 * dy), (x, y), (x + 3.0 * dx, y)]); + } + pen.dot(8.0, 8.0, 2.2); + pen.circle(8.0, 8.0, 4.4); + } Puppet => pen.pushpin(8.0), PuppetAdvanced => { // Rotation ring with its square scale handle. diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 2f0ca804..da2e05d0 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -28,6 +28,7 @@ pub mod learn; pub mod markers_ui; pub mod media_panels; pub mod misc; +pub mod object_matte_tool; pub mod paint_panels; pub mod panel_kit; pub mod path_vr_panels; diff --git a/crates/ui-egui/src/panels/object_matte_tool.rs b/crates/ui-egui/src/panels/object_matte_tool.rs new file mode 100644 index 00000000..70c9df8e --- /dev/null +++ b/crates/ui-egui/src/panels/object_matte_tool.rs @@ -0,0 +1,128 @@ +//! The Object Matte tool in the Composition viewer (After Effects 26.2's Object Matte, from its +//! public description): click an object to isolate it, or drag a box around it. The layer under +//! the pointer gets an Object Matte effect (`objectMatte.apply`), or its Object Matte is +//! re-prompted (the selected one, else its last); Shift adds another. Every prompt is one undo +//! step, and the effect is selected so its point controls show in the viewer. +//! +//! While hovering, a crosshair and the outline of the layer that would get the matte preview the +//! click: segmenting on every pointer move would be far too slow for a hover preview (a model +//! pass or a graph cut per move), so the matte itself appears once the prompt is set. + +use effectcraft_engine::geom::vec2 as gv2; +use effectcraft_engine::project::Layer; +use effectcraft_engine::render::EvalCtx; +use egui::{Color32, Pos2, Rect, Stroke, StrokeKind}; +use serde_json::json; + +use super::viewer::{ViewerMap, fx2c, layers_at}; +use crate::EffectcraftApp; + +/// Drags shorter than this (screen points) are clicks: a point prompt, no box. +pub const MIN_BOX: f32 = 4.0; + +/// Whether Object Matte can work on `layer` (footage, solid or composition layers with pixels). +fn eligible(layer: &Layer) -> bool { + layer.has_video() && !layer.switches.adjustment && layer.effects().is_some() +} + +/// The layer a prompt at comp point `cpt` goes to: a selected one under it, else the topmost. +pub fn target<'a>(app: &EffectcraftApp, ectx: &EvalCtx<'a>, cpt: [f64; 2]) -> Option<&'a Layer> { + let hits: Vec<&Layer> = layers_at(ectx, cpt).filter(|l| eligible(l)).collect(); + let selected = &app.session.state.selected_layers; + hits.iter().find(|l| selected.contains(&l.id)).or(hits.first()).copied() +} + +/// The layer's effect-space rectangle (its pixels) on screen. +fn outline(ectx: &EvalCtx, map: &ViewerMap, layer: &Layer) -> [Pos2; 4] { + let [w, h] = ectx.effect_bounds(layer).0; + let m = fx2c(ectx, layer); + [[0.0, 0.0], [w, 0.0], [w, h], [0.0, h]].map(|p| { + let c = m.apply(gv2(p[0], p[1])); + map.to_screen([c.x, c.y]) + }) +} + +/// Hover preview at `hp` (screen): the outline of the layer that would get the matte and a +/// crosshair where the object point would go. +pub fn hover(app: &mut EffectcraftApp, painter: &egui::Painter, map: &ViewerMap, ectx: &EvalCtx, hp: Pos2) { + let accent = app.tokens.accent; + let cpt = map.to_comp(hp); + if let Some(layer) = target(app, ectx, cpt) { + let q = outline(ectx, map, layer); + let mut pts = q.to_vec(); + pts.push(q[0]); + painter.add(egui::Shape::line(pts, Stroke::new(1.5, accent))); + let r = Rect::from_points(&q); + app.auto.add("viewer.objectMatte.target", r, &layer.name); + } + let (gap, arm) = (4.0, 11.0); + for (dx, dy) in [(1.0, 0.0), (-1.0, 0.0), (0.0, 1.0), (0.0, -1.0)] { + let d = egui::vec2(dx, dy); + for (w, c) in [(3.0, Color32::from_black_alpha(160)), (1.5, Color32::WHITE)] { + painter.line_segment([hp + d * gap, hp + d * arm], Stroke::new(w, c)); + } + } + painter.circle_stroke(hp, 2.0, Stroke::new(1.5, accent)); + app.auto.add("viewer.objectMatte.crosshair", Rect::from_center_size(hp, egui::vec2(2.0 * arm, 2.0 * arm)), "Object Matte point"); +} + +/// The box being dragged (screen `a`–`b`). +pub fn draw_box(app: &mut EffectcraftApp, painter: &egui::Painter, a: Pos2, b: Pos2) { + let r = Rect::from_two_pos(a, b); + let accent = app.tokens.accent; + painter.rect_stroke(r, 0.0, Stroke::new(2.5, Color32::from_black_alpha(140)), StrokeKind::Middle); + painter.rect_stroke(r, 0.0, Stroke::new(1.5, accent), StrokeKind::Middle); + painter.circle_filled(r.center(), 2.5, accent); + app.auto.add("viewer.objectMatte.box", r, "Object Matte box"); +} + +/// Set the prompt from a click at `a` (screen), or a box dragged from `a` to `b`: on the +/// target layer's Object Matte (re-prompted), or a new one with `add` or when it has none. +pub fn apply(app: &mut EffectcraftApp, ectx: &EvalCtx, map: &ViewerMap, a: Pos2, b: Option, add: bool) { + let boxed = b.filter(|b| (b.x - a.x).abs() >= MIN_BOX && (b.y - a.y).abs() >= MIN_BOX); + let center = boxed.map_or(a, |b| Rect::from_two_pos(a, b).center()); + let Some(layer) = target(app, ectx, map.to_comp(center)) else { + app.ui.status = "Object Matte: click an object on a footage, solid or composition layer".into(); + return; + }; + let Some(inv) = fx2c(ectx, layer).inverse() else { return }; + let to_layer = |p: Pos2| { + let c = map.to_comp(p); + let l = inv.apply(gv2(c[0], c[1])); + [l.x, l.y] + }; + let point = to_layer(center); + let mut q = json!({"layer": layer.id.0, "point": point}); + if let Some(b) = boxed { + // The box in layer pixels: the bounds of its corners (the layer may be turned). + let r = Rect::from_two_pos(a, b); + let c = [r.left_top(), r.right_top(), r.right_bottom(), r.left_bottom()].map(to_layer); + let (x0, x1) = c.iter().map(|p| p[0]).fold((f64::INFINITY, f64::NEG_INFINITY), |(lo, hi), v| (lo.min(v), hi.max(v))); + let (y0, y1) = c.iter().map(|p| p[1]).fold((f64::INFINITY, f64::NEG_INFINITY), |(lo, hi), v| (lo.min(v), hi.max(v))); + q["box"] = json!([x0, y0, x1, y1]); + } + if !add && let Some(uid) = existing(app, layer) { + q["effect"] = json!(uid); + } + let lid = layer.id; + match app.session.execute("objectMatte.apply", q) { + Ok(r) => { + if let Some(uid) = r["effect"].as_u64() { + let _ = app.session.execute("prop.select", json!({"layer": lid.0, "prop": uid})); + } + } + Err(e) => app.ui.status = e.to_string(), + } +} + +/// The Object Matte a prompt on `layer` re-prompts: the selected one, else the layer's last. +fn existing(app: &EffectcraftApp, layer: &Layer) -> Option { + let fx = layer.effects()?; + let om: Vec = fx.groups().filter(|g| is_object_matte(g)).map(|g| g.uid).collect(); + let sel = app.session.state.selected_props.iter().rev().find(|(l, u)| *l == layer.id && om.contains(u)).map(|(_, u)| *u); + sel.or(om.last().copied()) +} + +fn is_object_matte(g: &effectcraft_engine::project::PropGroup) -> bool { + matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == effectcraft_engine::effects::object_matte::ID) +} diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index 5fe7697c..4ea3b8c3 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -164,6 +164,10 @@ enum Gesture { PinMarquee { start: Pos2, }, + /// Object Matte tool: a box around the object (screen point where the drag began). + ObjectMatteBox { + start: Pos2, + }, /// Puppet tool Record mode (⌘/Ctrl-drag a pin): the drag is sampled in real time (seconds /// since it began, effect space) and becomes keyframes on release (`puppet.recordPin`); the /// current time runs along while recording. @@ -1164,6 +1168,14 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } t if t.is_shape() => egui::CursorIcon::Crosshair, _ if app.ui.viewer.roi_draw => egui::CursorIcon::Crosshair, + Tool::ObjectMatte => { + // The tool draws its own crosshair (and the layer it would matte) while no box + // is being dragged. + if ui.data(|d| d.get_temp::(egui::Id::new("viewer-gesture"))).is_none() { + super::object_matte_tool::hover(app, &painter, &map, &ectx, hp); + } + egui::CursorIcon::None + } t if t.puppet_kind().is_some() => match super::puppet_tool::hit(&pin_hits, hp) { Some((_, super::puppet_tool::PinPart::Ring)) => egui::CursorIcon::Alias, Some((_, super::puppet_tool::PinPart::Scale)) => egui::CursorIcon::ResizeNwSe, @@ -1293,6 +1305,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { // From where the button went down: by the time the pointer has moved far enough to // count as a drag it has already left the press point, and the shape is drawn there. t if t.is_shape() => Some(Gesture::Create { tool: t, start: map.to_comp(press) }), + Tool::ObjectMatte => Some(Gesture::ObjectMatteBox { start: press }), t if t.puppet_kind().is_some() && super::puppet_tool::hit(&pin_hits, press).is_none() => { // Off the pins: Alt or a press outside the art drags a marquee that selects pins. (mods.alt || puppet_layer(app, cpt).is_none()).then_some(Gesture::PinMarquee { start: press }) @@ -1609,6 +1622,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { q["merge"] = json!(merge); let _ = app.session.execute("track.setPoint", q); } + Gesture::ObjectMatteBox { start } => super::object_matte_tool::draw_box(app, &painter, start, pos), Gesture::Marquee { start } | Gesture::PinMarquee { start } => { let r = Rect::from_two_pos(start, pos); painter.rect_filled(r, 0.0, t.accent.gamma_multiply(0.12)); @@ -1649,6 +1663,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } } Gesture::PuppetPin { layer, pin, .. } => super::puppet_tool::reveal_pin(app, layer, pin), + Gesture::ObjectMatteBox { start } => super::object_matte_tool::apply(app, &ectx, &map, start, Some(pos), mods.shift), Gesture::Create { tool, start } => { let end = map.to_comp(pos); create_shape(app, tool, start, end, mods); @@ -1752,6 +1767,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { // Clicking away from the free-transform box ends it. ov::end_free_transform(&ctx); } + Tool::ObjectMatte => super::object_matte_tool::apply(app, &ectx, &map, pos, None, mods.shift), t if t.puppet_kind().is_some() => { if let Some((h, _)) = super::puppet_tool::hit(&pin_hits, pos) { // Click selects the pin; Shift-click adds or removes it. @@ -2075,7 +2091,7 @@ pub(crate) fn selectable_layers(comp: &Comp, time: Tick) -> impl Iterator(ectx: &'b EvalCtx<'a>, cpt: [f64; 2]) -> impl Iterator + 'b { +pub(crate) fn layers_at<'a, 'b>(ectx: &'b EvalCtx<'a>, cpt: [f64; 2]) -> impl Iterator + 'b { selectable_layers(ectx.comp, ectx.time).filter(move |l| layer_quad(ectx, l).is_some_and(|(_, q, _)| point_in_quad(cpt, &q))) } diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index 0cce2c11..7c49c786 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -35,6 +35,8 @@ pub enum Tool { Eraser, RotoBrush, RefineEdge, + /// Object Matte: click an object (or drag a box around it) in the Composition viewer. + ObjectMatte, Puppet, PuppetStarch, PuppetBend, @@ -59,10 +61,10 @@ impl Tool { &[Tool::Brush], &[Tool::Clone], &[Tool::Eraser], - &[Tool::RotoBrush, Tool::RefineEdge], + &[Tool::RotoBrush, Tool::RefineEdge, Tool::ObjectMatte], &[Tool::Puppet, Tool::PuppetStarch, Tool::PuppetBend, Tool::PuppetAdvanced, Tool::PuppetOverlap], ]; - pub const ALL: [Tool; 30] = [ + pub const ALL: [Tool; 31] = [ Tool::Selection, Tool::Hand, Tool::Zoom, @@ -88,6 +90,7 @@ impl Tool { Tool::Eraser, Tool::RotoBrush, Tool::RefineEdge, + Tool::ObjectMatte, Tool::Puppet, Tool::PuppetStarch, Tool::PuppetBend, @@ -121,6 +124,7 @@ impl Tool { Tool::Eraser => "Eraser Tool", Tool::RotoBrush => "Roto Brush Tool", Tool::RefineEdge => "Refine Edge Tool", + Tool::ObjectMatte => "Object Matte Tool", Tool::Puppet => "Puppet Position Pin Tool", Tool::PuppetStarch => "Puppet Starch Pin Tool", Tool::PuppetBend => "Puppet Bend Pin Tool", @@ -142,7 +146,7 @@ impl Tool { Tool::Pen | Tool::PenAdd | Tool::PenDelete | Tool::PenConvert | Tool::MaskFeather => Some("G"), Tool::Type | Tool::TypeVertical => Some("Cmd+T"), Tool::Brush | Tool::Clone | Tool::Eraser => Some("Cmd+B"), - Tool::RotoBrush | Tool::RefineEdge => Some("Alt+W"), + Tool::RotoBrush | Tool::RefineEdge | Tool::ObjectMatte => Some("Alt+W"), Tool::Puppet | Tool::PuppetStarch | Tool::PuppetBend | Tool::PuppetAdvanced | Tool::PuppetOverlap => Some("Cmd+P"), } } @@ -173,6 +177,7 @@ impl Tool { Tool::Eraser => Icon::Eraser, Tool::RotoBrush => Icon::RotoBrush, Tool::RefineEdge => Icon::RefineEdge, + Tool::ObjectMatte => Icon::ObjectMatte, Tool::Puppet => Icon::Puppet, Tool::PuppetStarch => Icon::PuppetStarch, Tool::PuppetBend => Icon::PuppetBend, diff --git a/crates/ui-egui/tests/ui_object_matte.rs b/crates/ui-egui/tests/ui_object_matte.rs new file mode 100644 index 00000000..483d271e --- /dev/null +++ b/crates/ui-egui/tests/ui_object_matte.rs @@ -0,0 +1,142 @@ +//! Headless checks for the Object Matte tool in the Composition viewer: the hover preview, a +//! click (point prompt), a drag (box prompt) re-prompting the same effect, Shift adding another, +//! one undo step each (egui_kittest, UI logic only). + +use effectcraft_engine::Session; +use effectcraft_engine::effects::flatten_params; +use effectcraft_engine::effects::object_matte as om; +use effectcraft_engine::keyframe::Value; +use effectcraft_engine::project::{GroupKind, LayerId, PropGroup}; +use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::state::Tool; +use egui::{Event, Modifiers, Pos2, Rect, pos2, vec2}; +use egui_kittest::Harness; +use serde_json::json; + +fn harness() -> Harness<'static, EffectcraftApp> { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Matte", "width": 640, "height": 360, "duration": 4})).unwrap(); + s.execute("layer.newSolid", json!({"name": "Plate", "color": "#406080", "width": 640, "height": 360})).unwrap(); + s.execute("layer.newSolid", json!({"name": "Box", "color": "#e04020", "width": 80, "height": 80})).unwrap(); + s.execute("edit.deselectAll", json!({})).unwrap(); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).build_eframe(move |_| EffectcraftApp::new(s)); + h.run_steps(3); + h +} + +fn rect(h: &Harness<'_, EffectcraftApp>, id: &str) -> Rect { + let e = h.state().auto.find(id).unwrap_or_else(|| panic!("no {id}")); + Rect::from_min_size(pos2(e.rect[0], e.rect[1]), vec2(e.rect[2], e.rect[3])) +} + +/// Comp pixel → screen point. +fn screen(h: &Harness<'_, EffectcraftApp>, p: [f32; 2]) -> Pos2 { + let c = rect(h, "viewer.comp"); + let z = c.width() / 640.0; + c.min + vec2(p[0] * z, p[1] * z) +} + +fn press(h: &mut Harness<'_, EffectcraftApp>, p: Pos2, pressed: bool, modifiers: Modifiers) { + h.input_mut().events.push(Event::ModifiersChanged(modifiers)); + h.input_mut().events.push(Event::PointerButton { pos: p, button: egui::PointerButton::Primary, pressed, modifiers }); + h.step(); +} + +fn click(h: &mut Harness<'_, EffectcraftApp>, p: Pos2, modifiers: Modifiers) { + h.input_mut().events.push(Event::PointerMoved(p)); + h.step(); + press(h, p, true, modifiers); + press(h, p, false, modifiers); + h.input_mut().events.push(Event::ModifiersChanged(Modifiers::NONE)); + h.run_steps(2); +} + +fn drag(h: &mut Harness<'_, EffectcraftApp>, from: Pos2, to: Pos2) -> Option { + h.input_mut().events.push(Event::PointerMoved(from)); + press(h, from, true, Modifiers::NONE); + let mut drawn = None; + for i in 1..=8 { + h.input_mut().events.push(Event::PointerMoved(from + (to - from) * (i as f32 / 8.0))); + h.step(); + drawn = h.state().auto.find("viewer.objectMatte.box").map(|e| Rect::from_min_size(pos2(e.rect[0], e.rect[1]), vec2(e.rect[2], e.rect[3]))); + } + press(h, to, false, Modifiers::NONE); + h.run_steps(2); + drawn +} + +fn box_layer(h: &Harness<'_, EffectcraftApp>) -> LayerId { + h.state().session.active_comp().unwrap().layers.iter().find(|l| l.name == "Box").unwrap().id +} + +/// The Box layer's Object Matte effects. +fn mattes(h: &Harness<'_, EffectcraftApp>) -> Vec { + let comp = h.state().session.active_comp().unwrap(); + let l = comp.layer(box_layer(h)).unwrap(); + l.effects().unwrap().groups().filter(|g| matches!(&g.kind, GroupKind::Effect { effect } if effect == om::ID)).cloned().collect() +} + +fn v2(g: &PropGroup, id: &str) -> [f64; 2] { + flatten_params(g, &mut |p| p.value.clone()).get(id).map(Value::as_vec2).unwrap() +} + +fn near(a: [f64; 2], b: [f64; 2]) -> bool { + (a[0] - b[0]).abs() < 1.5 && (a[1] - b[1]).abs() < 1.5 +} + +#[test] +fn object_matte_tool_prompts_with_a_click_and_a_box() { + let mut h = harness(); + assert_eq!(Tool::from_name("objectMatte"), Some(Tool::ObjectMatte)); + assert!(Tool::SLOTS.iter().any(|s| s.contains(&Tool::RotoBrush) && s.contains(&Tool::ObjectMatte))); + h.state_mut().ui.tool = Tool::ObjectMatte; + h.run_steps(2); + // Hover: a crosshair at the pointer and the outline of the layer that would get the matte. + let center = screen(&h, [320.0, 180.0]); + h.input_mut().events.push(Event::PointerMoved(center)); + h.run_steps(2); + assert!(rect(&h, "viewer.objectMatte.crosshair").contains(center)); + let target = h.state().auto.find("viewer.objectMatte.target").expect("no target outline"); + assert_eq!(target.label, "Box"); + assert!(mattes(&h).is_empty(), "hovering must not apply anything"); + let undo0 = h.state().session.history.undo.len(); + + // Click: a point prompt in layer pixels on the layer under the pointer, selected. + click(&mut h, center, Modifiers::NONE); + let m = mattes(&h); + assert_eq!(m.len(), 1); + assert!(near(v2(&m[0], "objectPoint"), [40.0, 40.0]), "{:?}", v2(&m[0], "objectPoint")); + let uid = m[0].uid; + let st = &h.state().session.state; + assert!(st.selected_props.contains(&(box_layer(&h), uid)), "the new effect is selected"); + assert_eq!(h.state().session.history.undo.len(), undo0 + 1); + + // Drag: a box (drawn while dragging) re-prompts the same effect; its centre is the point. + // (It starts clear of the selected effect's point controls, which take drags of their own.) + let (a, b) = (screen(&h, [286.0, 150.0]), screen(&h, [356.0, 214.0])); + let drawn = drag(&mut h, a, b).expect("no box drawn while dragging"); + assert!(drawn.width() > 20.0); + let m = mattes(&h); + assert_eq!(m.len(), 1); + assert_eq!(m[0].uid, uid); + let ps = flatten_params(&m[0], &mut |p| p.value.clone()); + assert_eq!(ps.get("useBox"), Some(&Value::Bool(true))); + assert!(near(v2(&m[0], "boxTopLeft"), [6.0, 10.0]), "{:?}", v2(&m[0], "boxTopLeft")); + assert!(near(v2(&m[0], "boxBottomRight"), [76.0, 74.0]), "{:?}", v2(&m[0], "boxBottomRight")); + assert!(near(v2(&m[0], "objectPoint"), [41.0, 42.0]), "{:?}", v2(&m[0], "objectPoint")); + assert_eq!(h.state().session.history.undo.len(), undo0 + 2); + + // Shift-click adds a second Object Matte. + let p = screen(&h, [310.0, 170.0]); + click(&mut h, p, Modifiers::SHIFT); + let m = mattes(&h); + assert_eq!(m.len(), 2); + assert_eq!(m[1].name, "Object Matte 2"); + assert!(near(v2(&m[1], "objectPoint"), [30.0, 30.0])); + + // Undo takes the prompts back one at a time. + h.state_mut().session.execute("edit.undo", json!({})).unwrap(); + assert_eq!(mattes(&h).len(), 1); + h.state_mut().session.execute("edit.undo", json!({})).unwrap(); + assert_eq!(flatten_params(&mattes(&h)[0], &mut |p| p.value.clone()).get("useBox"), Some(&Value::Bool(false))); +} From 560767dfe79e2cea59da0814d950db37c4b8e420 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 11 Oct 2026 14:02:50 +0200 Subject: [PATCH 5/8] #783: Object Matte tracks in the background and freezes Problem: Object Matte computed its frames only on demand while rendering, so playing a shot stalled on every untracked frame, there was no progress or way to stop, and nothing kept the mattes once computed. Fix: Object Matte runs through Roto Brush's job machinery (one job slot, cancellable with roto.cancel, a worker thread on the desktop, a Web Worker in the browser, never the UI thread). objectMatte.propagate computes the frames around the reference frame (clipped to the layer) outward, filling the same content-keyed segmentation cache the renderer reads; the engine's sync now queues an instance for background tracking when its frames or prompt change. objectMatte.freeze stores the final mattes in a new hidden Frozen parameter (Roto Brush's FrozenCache format) under a key of the input, prompt, span, model and matte settings; renders use them without segmenting while the key matches; a frozen prompt is refused until objectMatte.unfreeze. Freeze is one undo step and is dropped if the prompt changed meanwhile. objectMatte.status reports progress, frozen state and computed frames. Effect Controls shows Freeze / Unfreeze and Track / Stop with a status line (progress banner). The frontend now polls Roto Brush / Object Matte jobs on their own: before, a finished job (and so a Freeze result) was only picked up while a Warp Stabilizer job was active. Job progress is set before the worker starts. Tests: engine tests_roto::object_matte_background_job_cancel_and_freeze (queued by the edit, returns at once with progress, cancel keeps a partial result, completion computes all 25 frames, frozen renders recompute nothing, unfreeze recomputes, undo restores); effects object_matte job_frames and frozen-matte tests (stored pattern used, key mismatch and damaged data ignored); ui-egui ui_object_matte Effect Controls Freeze end to end. docs/agents.md: Object Matte section; docs/effects.md regenerated. Co-Authored-By: Claude Opus 5.5 --- crates/effects/src/object_matte.rs | 193 ++++++++++++++++++- crates/engine/src/commands/roto_cmds.rs | 104 +++++++++- crates/engine/src/offload.rs | 3 +- crates/engine/src/roto.rs | 177 +++++++++++++---- crates/engine/src/tests_roto.rs | 81 ++++++++ crates/ui-egui/src/i18n/ui.rs | 4 + crates/ui-egui/src/lib.rs | 6 + crates/ui-egui/src/panels/effect_controls.rs | 41 +++- crates/ui-egui/tests/ui_object_matte.rs | 43 +++++ docs/agents.md | 18 ++ docs/effects.md | 2 +- 11 files changed, 618 insertions(+), 54 deletions(-) diff --git a/crates/effects/src/object_matte.rs b/crates/effects/src/object_matte.rs index a7db562a..32a196af 100644 --- a/crates/effects/src/object_matte.rs +++ b/crates/effects/src/object_matte.rs @@ -16,8 +16,12 @@ //! ([`crate::roto::cached`]): the chain seed holds the engine-maintained Input Key (what the //! frames are made of), the layer size, the frame rate, the mode and the model, and every //! frame's key the prompt, so edits upstream or to the prompt recompute it and undo never sees a -//! stale matte. **Feather**, **Contrast** and **Shift Edge** shape the matte, **Invert** keeps -//! everything but the object and **View ▸ Matte** shows the matte in grey. +//! stale matte. The engine fills that cache in a background job (`objectMatte.propagate`, +//! outward from the reference frame over [`job_frames`]) and **Freeze** (`objectMatte.freeze`) +//! stores the final mattes in the hidden **Frozen** parameter ([`FrozenCache`], as Roto Brush +//! does): they are used, without segmenting, while [`frozen_key`] still matches. **Feather**, +//! **Contrast** and **Shift Edge** shape the matte, **Invert** keeps everything but the object +//! and **View ▸ Matte** shows the matte in grey. use std::sync::Arc; @@ -26,10 +30,10 @@ use effectcraft_project::ParamUi; use effectcraft_raster::Image; use effectcraft_segment::MaskModel; use effectcraft_track::roto::refine::MatteParams; -use effectcraft_track::roto::{RotoData, SegOpts, Stroke, StrokeKind}; +use effectcraft_track::roto::{FrameSeg, RotoData, SegOpts, Stroke, StrokeKind, rle}; use rayon::prelude::*; -use crate::roto::{self, Chain, INPUT_KEY}; +use crate::roto::{self, Chain, FROZEN, FrozenCache, INPUT_KEY}; use crate::warp_stab::param; use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; @@ -71,6 +75,7 @@ pub fn specs() -> Vec { p("invert", "Invert", Value::Bool(false), ParamUi::Checkbox), p("view", "View", Value::Enum(0), popup(&VIEWS)), p(INPUT_KEY, "Input Key", Value::Str(String::new()), ParamUi::Hidden), + p(FROZEN, "Frozen", Value::Str(String::new()), ParamUi::Hidden), ], render, gpu: false, @@ -197,6 +202,127 @@ pub fn seed(params: &Params, layer_size: [f64; 2], fps: f64, model: Option<&dyn h } +/// The layer frames a background job computes, outward from the reference frame: the tracked +/// span (Frames Before / After around it), clipped to `limits` (the layer's frames). Without +/// Track Object, the same frames, each segmented on its own. +pub fn job_frames(params: &Params, limits: [i64; 2]) -> Vec { + let base = reference_frame(params); + let [before, after] = reach(params); + let (lo, hi) = (base.saturating_sub(before).max(limits[0]), base.saturating_add(after).min(limits[1])); + if base < limits[0] || base > limits[1] { + return vec![]; + } + let mut v = vec![base]; + v.extend(base.saturating_add(1)..=hi); + v.extend((lo..base).rev()); + v +} + +/// The segmentation chain the matte of layer frame `f` comes from (`None`: no usable prompt). +pub fn chain(params: &Params, layer_size: [f64; 2], fps: f64, f: i64, scale: f64, model: Option>) -> Option { + let data = roto_data(params, layer_size, f, model.is_some())?; + let keys = data.chain_keys(seed(params, layer_size, fps, model.as_deref())); + Some(Chain { data: Arc::new(data), keys, opts: seg_opts(scale), refine_radius: 0.0, model }) +} + +/// Frame `f`'s segmentation at full resolution, computing what is missing (the background +/// job's step). `None` outside the tracked span, without a prompt, or when cancelled. +pub fn ensure( + params: &Params, + layer_size: [f64; 2], + fps: f64, + f: i64, + model: Option>, + frame: &mut dyn FnMut(i64) -> Option>, + cancel: &dyn Fn() -> bool, +) -> Option> { + let ch = chain(params, layer_size, fps, f, 1.0, model)?; + if !ch.data.in_span(f) { + return None; + } + ch.ensure(f, frame, cancel) +} + +/// The chain key of each of `frames` (what a computed segmentation is cached under). +pub fn frame_keys(params: &Params, layer_size: [f64; 2], fps: f64, frames: &[i64], model: Option>) -> Vec<(i64, u64)> { + if tracks(params) { + let Some(ch) = chain(params, layer_size, fps, 0, 1.0, model) else { return vec![] }; + return frames.iter().filter_map(|f| Some((*f, *ch.keys.get(f)?))).collect(); + } + frames.iter().filter_map(|f| Some((*f, *chain(params, layer_size, fps, *f, 1.0, model.clone())?.keys.get(f)?))).collect() +} + +/// What frozen mattes depend on: what the frames are (the seed: Input Key, size, rate, mode, +/// model), the prompt, the tracked span and the matte settings. +pub fn frozen_key(params: &Params, layer_size: [f64; 2], fps: f64, model: Option<&dyn MaskModel>) -> String { + let mut h = seed(params, layer_size, fps, model); + let base = reference_frame(params); + if let Some(st) = prompt_strokes(params, layer_size, base, model.is_some()) { + fnv(&mut h, serde_json::to_string(&st).unwrap_or_default().as_bytes()); + } + for v in [base, reach(params)[0], reach(params)[1]] { + fnv(&mut h, &v.to_le_bytes()); + } + fnv(&mut h, format!("{:?}", matte_params(params)).as_bytes()); + format!("{h:016x}") +} + +/// Whether the instance is frozen (has frozen mattes). +pub fn is_frozen(params: &Params) -> bool { + !s(params, FROZEN).is_empty() +} + +/// Frame `f`'s frozen matte on a `w`×`h` grid, when the instance is frozen, the mattes still +/// match ([`frozen_key`]) and `f` is one of them. +pub fn frozen_alpha(params: &Params, layer_size: [f64; 2], fps: f64, f: i64, w: usize, h: usize, model: Option<&dyn MaskModel>) -> Option> { + let j = s(params, FROZEN); + if j.is_empty() { + return None; + } + let fc = frozen_cache(j)?; + if fc.key != frozen_key(params, layer_size, fps, model) { + return None; + } + let fr = fc.frames.get(&f)?; + let a = rle::decode(&fr.a)?; + let [fw, fh] = fc.size; + if fw == 0 || fh == 0 || a.len() != fw.checked_mul(fh)? { + return None; + } + let af: Vec = a.iter().map(|v| *v as f32 / 255.0).collect(); + if fw == w && fh == h { + return Some(af); + } + let mut out = vec![0.0f32; w * h]; + for y in 0..h { + for x in 0..w { + let (sx, sy) = ((x as f64 + 0.5) * fw as f64 / w as f64, (y as f64 + 0.5) * fh as f64 / h as f64); + out[y * w + x] = effectcraft_track::roto::matting::sample(&af, fw, fh, sx, sy); + } + } + Some(out) +} + +/// The parsed Frozen parameter (parsed once per content). +fn frozen_cache(j: &str) -> Option> { + use std::collections::HashMap; + use std::sync::{Mutex, OnceLock}; + static C: OnceLock>>> = OnceLock::new(); + let mut h = 0xcbf2_9ce4_8422_2325; + fnv(&mut h, j.as_bytes()); + let c = C.get_or_init(Default::default); + if let Some(d) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&h) { + return Some(d.clone()); + } + let d = Arc::new(FrozenCache::from_json(j)?); + let mut m = c.lock().unwrap_or_else(|e| e.into_inner()); + if m.len() > 8 { + m.clear(); + } + m.insert(h, d.clone()); + Some(d) +} + /// Segmentation settings at `scale` (Roto Brush's defaults). fn seg_opts(scale: f64) -> SegOpts { SegOpts { scale, search_radius: 15.0, motion_threshold: 10.0, motion_damping: 20.0, best: false } @@ -242,12 +368,10 @@ pub fn frame_matte( frame: &mut dyn FnMut(i64) -> Option>, cancel: &dyn Fn() -> bool, ) -> FrameMatte { - let Some(data) = roto_data(params, layer_size, f, model.is_some()) else { return FrameMatte::NoPrompt }; - if !data.in_span(f) { + let Some(chain) = chain(params, layer_size, fps, f, scale, model) else { return FrameMatte::NoPrompt }; + if !chain.data.in_span(f) { return FrameMatte::Outside; } - let keys = data.chain_keys(seed(params, layer_size, fps, model.as_deref())); - let chain = Chain { data: Arc::new(data), keys, opts: seg_opts(scale), refine_radius: 0.0, model }; let Some(seg) = chain.ensure(f, frame, cancel) else { return FrameMatte::Outside }; let (w, h) = (img.width as usize, img.height as usize); let seg = if seg.w == w && seg.h == h { (*seg).clone() } else { roto::resample(&seg, w, h) }; @@ -275,7 +399,12 @@ fn render(ctx: &EffectCtx, buf: Buf) -> Buf { let im = roto::layer_grid(&other, layer_size); (im.width as usize == w && im.height as usize == h).then(|| Arc::new(im)) }; - let alpha = match frame_matte(ctx.params, layer_size, fps, f, sc, &img, roto::installed_model(), &mut frame, &|| false) { + let model = roto::installed_model(); + // Frozen mattes are used as they are (no segmentation). + if let Some(a) = frozen_alpha(ctx.params, layer_size, fps, f, w, h, model.as_deref()) { + return compose(&buf, Some(&a), w, h, b(ctx.params, "invert"), param(ctx.params, "view").map(Value::as_enum).unwrap_or(0) == 1); + } + let alpha = match frame_matte(ctx.params, layer_size, fps, f, sc, &img, model, &mut frame, &|| false) { FrameMatte::Alpha(a) if a.len() == w * h => Some(a), FrameMatte::NoPrompt => return buf, _ => None, @@ -513,4 +642,50 @@ mod tests { let _ = frame_matte(&p, [100.0, 80.0], 25.0, 0, 0.5, &half, Some(stub.clone()), &mut frame, &|| false); assert_eq!(*stub.bbox.lock().unwrap(), Some([37.0, 27.0, 50.0, 40.0])); } + + #[test] + fn job_frames_go_outward_from_the_reference_frame_inside_the_layer() { + let p = params(&[ + ("objectTracking/referenceFrame", Value::Scalar(5.0)), + ("objectTracking/framesBefore", Value::Scalar(2.0)), + ("objectTracking/framesAfter", Value::Scalar(3.0)), + ]); + assert_eq!(job_frames(&p, [0, 100]), [5, 6, 7, 8, 4, 3]); + assert_eq!(job_frames(&p, [4, 6]), [5, 6, 4]); + assert!(job_frames(&p, [10, 20]).is_empty()); + let huge = params(&[("objectTracking/referenceFrame", Value::Scalar(1e300)), ("objectTracking/framesAfter", Value::Scalar(f64::INFINITY))]); + assert!(job_frames(&huge, [0, 100]).is_empty()); + } + + /// Frozen mattes are used as stored while their key matches (no segmentation: a stored + /// pattern the segmentation could never produce comes out), and ignored once it doesn't. + #[test] + fn frozen_mattes_are_used_while_their_key_matches() { + let mut p = params(&[("objectPoint", Value::Vec2([40.0, 40.0])), ("objectTracking/framesAfter", Value::Scalar(3.0))]); + p.values.insert(INPUT_KEY.into(), Value::Str("frozen-test".into())); + // Left half opaque on frame 2. + let alpha: Vec = (0..100 * 80).map(|i| ((i % 100) < 50) as u8 as f32).collect(); + let mut fc = FrozenCache { key: frozen_key(&p, [100.0, 80.0], 25.0, None), size: [100, 80], ..Default::default() }; + fc.frames.insert(2, roto::freeze_frame(&alpha, &[])); + p.values.insert(FROZEN.into(), Value::Str(fc.to_json())); + assert!(is_frozen(&p)); + let out = run(&p, 2); + assert!(out.data[40 * 100 + 10][3] > 0.99 && out.data[40 * 100 + 90][3] < 0.01, "the frozen matte is used"); + // At half resolution too. + assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 50, 40, None).map(|a| (a[20 * 50 + 5], a[20 * 50 + 45])), Some((1.0, 0.0))); + // A frame without a frozen matte is computed (the disk). + assert!(wrong(&run(&p, 1), 42.0) < 30); + // Another prompt: the frozen mattes no longer apply. + p.values.insert("objectPoint".into(), Value::Vec2([41.0, 40.0])); + assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 100, 80, None), None); + assert!(wrong(&run(&p, 2), 44.0) < 30); + // Damaged frozen data is ignored, not a crash. + p.values.insert(FROZEN.into(), Value::Str("{\"key\": 3, \"frames\": [".into())); + assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 100, 80, None), None); + let mut fc2 = fc.clone(); + fc2.size = [usize::MAX, 2]; + p.values.insert("objectPoint".into(), Value::Vec2([40.0, 40.0])); + p.values.insert(FROZEN.into(), Value::Str(fc2.to_json())); + assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 100, 80, None), None); + } } diff --git a/crates/engine/src/commands/roto_cmds.rs b/crates/engine/src/commands/roto_cmds.rs index f4f33d53..ee9cd3a2 100644 --- a/crates/engine/src/commands/roto_cmds.rs +++ b/crates/engine/src/commands/roto_cmds.rs @@ -63,6 +63,23 @@ pub fn specs() -> Vec { has_comp, object_matte_apply ), + cmd!( + "objectMatte.propagate", + "Track Object Matte", + [], + None, + "{layer?, effect?, direction?: forward|backward|both, wait?: block until done} (a background job like Roto Brush propagation; roto.cancel stops it)", + has_comp, + object_matte_propagate + ), + cmd!("objectMatte.freeze", "Freeze Object Matte", [], None, "{layer?, effect?, wait?}", has_comp, object_matte_freeze), + cmd!("objectMatte.unfreeze", "Unfreeze Object Matte", [], None, "{layer?, effect?}", has_comp, object_matte_unfreeze), + query!( + "objectMatte.status", + "Object Matte Status", + "{layer?, effect?}: the job's progress, whether the instance is frozen, its frames and which are computed", + object_matte_status + ), query!( "roto.status", "Roto Brush Status", @@ -79,6 +96,16 @@ fn is_running(s: &Session) -> std::result::Result<(), String> { /// The Roto Brush instance targeted by `p`: `layer` / `effect` (uid, name or 1-based index), /// else a selected one, else the first on the selected (or any) layer. pub(crate) fn roto_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> { + instance_lookup(s, p, cmd, is_roto)?.ok_or_else(|| bad(cmd, "no Roto Brush & Refine Edge effect found (paint a stroke with the Roto Brush tool first)")) +} + +/// The Object Matte instance targeted by `p` (as [`roto_p`]). +fn object_matte_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> { + instance_lookup(s, p, cmd, rt::is_object_matte)?.ok_or_else(|| bad(cmd, "no Object Matte effect found (click an object with the Object Matte tool first)")) +} + +/// The instance of a kind (`is`) targeted by `p`; `None` when there is none. +fn instance_lookup(s: &Session, p: &Value, cmd: &str, is: fn(&PropGroup) -> bool) -> Result> { let cid = super::comp_id(s, p)?; let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; let candidates: Vec = match p.get("layer") { @@ -102,17 +129,17 @@ pub(crate) fn roto_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, Layer .state .selected_props .iter() - .find_map(|(l, u)| (*l == lid).then(|| fxg.groups().find(|g| g.uid == *u && is_roto(g))).flatten()) - .or_else(|| fxg.groups().find(|g| is_roto(g))), + .find_map(|(l, u)| (*l == lid).then(|| fxg.groups().find(|g| g.uid == *u && is(g))).flatten()) + .or_else(|| fxg.groups().find(|g| is(g))), }; - if let Some(g) = g.filter(|g| is_roto(g)) { - return Ok((cid, lid, g.uid)); + if let Some(g) = g.filter(|g| is(g)) { + return Ok(Some((cid, lid, g.uid))); } if p.get("layer").is_some() { break; } } - Err(bad(cmd, "no Roto Brush & Refine Edge effect found (paint a stroke with the Roto Brush tool first)")) + Ok(None) } fn group(s: &Session, cid: ItemId, lid: LayerId, uid: Uid) -> Result<(&Layer, &PropGroup)> { @@ -459,10 +486,13 @@ fn object_matte_apply(s: &mut Session, p: &Value) -> Result { let existing = match p.get("effect").filter(|v| !v.is_null()) { Some(v) => { let uid = v.as_u64().ok_or_else(|| bad(cmd, "`effect` is an Object Matte effect's uid"))?; - layer + let g = layer .effects() .and_then(|f| f.groups().find(|g| g.uid == uid && rt::is_object_matte(g))) .ok_or_else(|| bad(cmd, format!("no Object Matte effect {uid} on this layer")))?; + if frozen_of(g) { + return Err(bad(cmd, "the Object Matte is frozen: Unfreeze it to change the prompt")); + } Some(uid) } None => None, @@ -556,3 +586,65 @@ fn object_matte_reach(s: &Session, cid: ItemId, lid: LayerId, uid: Uid) -> (bool let [before, after] = om::reach(&ps); (om::tracks(&ps), before, after) } + +fn object_matte_propagate(s: &mut Session, p: &Value) -> Result { + let cmd = "objectMatte.propagate"; + let (cid, lid, uid) = object_matte_p(s, p, cmd)?; + let dir = match str_p(p, "direction") { + Some(d) => Direction::from_name(d).ok_or_else(|| bad(cmd, format!("unknown direction `{d}` (forward | backward | both)")))?, + None => Direction::Both, + }; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_roto(cid, lid, uid, RotoTask::Propagate, dir, wait).map_err(|e| bad(cmd, e))?; + let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_roto_running(), "progress": s.roto_progress()}); + if wait { + out["status"] = object_matte_status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?; + } + Ok(out) +} + +fn object_matte_freeze(s: &mut Session, p: &Value) -> Result { + let cmd = "objectMatte.freeze"; + let (cid, lid, uid) = object_matte_p(s, p, cmd)?; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_roto(cid, lid, uid, RotoTask::Freeze, Direction::Both, wait).map_err(|e| bad(cmd, e))?; + let (_, g) = group(s, cid, lid, uid)?; + Ok(json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_roto_running(), "frozen": frozen_of(g), "progress": s.roto_progress()})) +} + +fn object_matte_unfreeze(s: &mut Session, p: &Value) -> Result { + let cmd = "objectMatte.unfreeze"; + let (cid, lid, uid) = object_matte_p(s, p, cmd)?; + let (_, g) = group(s, cid, lid, uid)?; + if frozen_of(g) { + s.edit("Unfreeze", None, |proj, _| { + let g = layer_mut(proj, cid, lid)?.props.find_group_mut(uid).ok_or_else(|| EngineError::Other("the effect is gone".into()))?; + if let Some(pr) = g.get_mut(FROZEN) { + pr.value = KV::Str(String::new()); + } + Ok(()) + })?; + } + Ok(json!({"layer": lid.0, "effect": uid, "frozen": false})) +} + +fn object_matte_status(s: &mut Session, p: &Value) -> Result { + s.poll_roto(false); + let running = s.is_roto_running(); + let mut out = json!({"running": running, "progress": s.roto_progress(), "banner": s.roto_progress().map(|p| p.banner())}); + let Ok((cid, lid, uid)) = object_matte_p(s, p, "objectMatte.status") else { return Ok(out) }; + let (_, g) = group(s, cid, lid, uid)?; + let frozen = frozen_of(g); + let (track, before, after) = object_matte_reach(s, cid, lid, uid); + let computed = s.roto_computed(cid, lid, uid); + out["layer"] = json!(lid.0); + out["effect"] = json!(uid); + out["frozen"] = json!(frozen); + out["track"] = json!(track); + out["reach"] = json!([before, after]); + out["mine"] = json!(running && s.roto_target().is_some_and(|(c, l, u, _)| (c, l, u) == (cid, lid, uid))); + out["pending"] = json!(s.roto_pending.contains(&(cid, lid, uid))); + out["frames"] = json!(computed.keys().collect::>()); + out["computed"] = json!(computed.iter().filter(|(_, c)| **c).map(|(f, _)| *f).collect::>()); + Ok(out) +} diff --git a/crates/engine/src/offload.rs b/crates/engine/src/offload.rs index 23fa2b38..561ebe6c 100644 --- a/crates/engine/src/offload.rs +++ b/crates/engine/src/offload.rs @@ -561,7 +561,8 @@ async fn fill(s: &mut Session, plan: crate::commands::content_fill::FillPlan, po /// Roto Brush propagation in a worker: the segmentations stream back (at most every 250 ms) /// as they are computed, so stopping the job keeps what was propagated. async fn propagate(s: &mut Session, comp: ItemId, layer: LayerId, effect: Uid, dir: crate::roto::Direction, post: Post) -> Result<(), String> { - let keys: Vec = s.roto_chain(comp, layer, effect).map(|c| c.keys.values().copied().collect()).ok_or("no Roto Brush strokes")?; + let keys: Vec = + s.roto_keys(comp, layer, effect).map(|k| k.into_iter().map(|(_, k)| k).collect()).ok_or("no Roto Brush strokes or Object Matte prompt")?; let sent = Rc::new(RefCell::new(std::collections::HashSet::new())); let (p, k2, s2) = (post.clone(), keys.clone(), sent.clone()); let mut last = None::; diff --git a/crates/engine/src/roto.rs b/crates/engine/src/roto.rs index b8c2c649..49f53d39 100644 --- a/crates/engine/src/roto.rs +++ b/crates/engine/src/roto.rs @@ -10,12 +10,18 @@ //! edit, keeps each instance's hidden Input Key equal to a signature of what its frames are made //! of (the layer's source and timing, its masks and the effects above it) and drops frozen //! mattes made from other frames. +//! +//! Object Matte instances run through the same jobs (`objectMatte.propagate` / `.freeze`): the +//! frames are [`om::job_frames`] around the reference frame, each segmented through +//! [`om::ensure`]; Freeze stores [`om::frame_matte`]'s mattes under [`om::frozen_key`]. An edit +//! to an instance's frames or prompt queues it for background tracking like a Roto Brush. use crate::offload::JobKind; use std::collections::{BTreeMap, HashMap}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; +use effectcraft_effects::object_matte as om; use effectcraft_effects::roto::{self as fx, FROZEN, FrozenCache, INPUT_KEY}; use effectcraft_effects::{Params, flatten_params}; use effectcraft_keyframe::Value; @@ -104,14 +110,18 @@ pub struct RotoProgress { pub finished: bool, pub cancelled: bool, pub error: Option, + /// The job works on an Object Matte (else a Roto Brush & Refine Edge). + pub object_matte: bool, } impl RotoProgress { pub fn banner(&self) -> String { let pc = (100 * self.done).checked_div(self.total).unwrap_or(0); - match self.task { - Some(RotoTask::Freeze) => format!("Freezing Roto Brush & Refine Edge: {pc}%"), - _ => format!("Propagating Roto Brush: {pc}%"), + match (self.task, self.object_matte) { + (Some(RotoTask::Freeze), false) => format!("Freezing Roto Brush & Refine Edge: {pc}%"), + (Some(RotoTask::Freeze), true) => format!("Freezing Object Matte: {pc}%"), + (_, false) => format!("Propagating Roto Brush: {pc}%"), + (_, true) => format!("Tracking Object Matte: {pc}%"), } } } @@ -130,8 +140,11 @@ pub struct RotoJob { pub layer: LayerId, pub effect: Uid, pub task: RotoTask, - /// The strokes the job was started with (a Freeze result is dropped when they changed). + /// What the job was started from: the strokes (Roto Brush) or the prompt (Object Matte); a + /// Freeze result is dropped when it changed. strokes: String, + /// The job works on an Object Matte. + pub object_matte: bool, } impl RotoJob { @@ -153,6 +166,20 @@ pub(crate) fn is_object_matte(g: &PropGroup) -> bool { matches!(&g.kind, GroupKind::Effect { effect } if effect == effectcraft_effects::object_matte::ID) } +/// What an Object Matte's segmentation is made from besides its frames: the prompt and tracking +/// parameters (static values), for queueing it after edits and guarding a Freeze. +pub(crate) fn om_prompt(g: &PropGroup) -> String { + let ps = params_static(g); + let mut keys: Vec<&String> = ps.values.keys().filter(|k| ["objectPoint", "useBox", "box", "objectTracking/"].iter().any(|p| k.starts_with(p))).collect(); + keys.sort(); + keys.iter().map(|k| format!("{k}={:?};", ps.values.get(*k))).collect() +} + +/// The job-relevant guard of an instance: its strokes (Roto Brush) or prompt (Object Matte). +fn guard_of(g: &PropGroup) -> String { + if is_object_matte(g) { om_prompt(g) } else { str_of(g, fx::STROKES) } +} + fn str_of(g: &PropGroup, m: &str) -> String { match g.get(m).map(|p| &p.value) { Some(Value::Str(s)) => s.clone(), @@ -195,8 +222,8 @@ pub fn input_key(p: &Project, comp: ItemId, layer: &Layer, uid: Uid) -> String { } /// After an edit: refresh Input Keys (Roto Brush and Object Matte); frozen mattes made from -/// other frames are dropped. Returns the Roto Brush instances whose segmentation changed (to -/// propagate again). +/// other frames are dropped. Returns the instances whose segmentation changed (to propagate +/// again): new frames, strokes (Roto Brush) or prompt (Object Matte). pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> { let mut todo = vec![]; let mut set = vec![]; @@ -206,14 +233,11 @@ pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerI let Some(fxg) = l.effects() else { continue }; for g in fxg.groups().filter(|g| is_roto(g) || is_object_matte(g)) { let key = input_key(after, *cid, l, g.uid); - let roto = is_roto(g); let old = before.comp(*cid).and_then(|bc| bc.layer(l.id)).and_then(|ol| ol.props.find_group(g.uid)); if str_of(g, INPUT_KEY) != key { set.push((*cid, l.id, g.uid, key)); - if roto { - todo.push((*cid, l.id, g.uid)); - } - } else if roto && old.is_none_or(|o| str_of(o, fx::STROKES) != str_of(g, fx::STROKES)) { + todo.push((*cid, l.id, g.uid)); + } else if old.is_none_or(|o| guard_of(o) != guard_of(g)) { todo.push((*cid, l.id, g.uid)); } } @@ -229,7 +253,11 @@ pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerI } } todo.retain(|(c, l, u)| { - after.comp(*c).and_then(|c| c.layer(*l)).and_then(|l| l.props.find_group(*u)).is_some_and(|g| !fx::data(¶ms_static(g)).is_empty()) + after + .comp(*c) + .and_then(|c| c.layer(*l)) + .and_then(|l| l.props.find_group(*u)) + .is_some_and(|g| is_object_matte(g) || !fx::data(¶ms_static(g)).is_empty()) }); todo } @@ -284,6 +312,7 @@ struct Work { fps: f64, frames: Vec, task: RotoTask, + object_matte: bool, } /// Renders the effect's input frames for a worker, keeping the last few. With a browser job @@ -400,10 +429,13 @@ async fn run_work(w: Work, shared: &RotoShared) { let mut s = lock(&shared.state); s.task = Some(w.task); s.total = w.frames.len() as u64; + s.object_matte = w.object_matte; } let cancel = || shared.cancel.load(Ordering::Relaxed); let chain = fx::Chain::new(&w.params, w.size, w.fps, 1.0); - let mut frozen = FrozenCache { key: fx::frozen_key(&w.params, w.size, w.fps), ..Default::default() }; + let model = fx::installed_model(); + let key = if w.object_matte { om::frozen_key(&w.params, w.size, w.fps, model.as_deref()) } else { fx::frozen_key(&w.params, w.size, w.fps) }; + let mut frozen = FrozenCache { key, ..Default::default() }; for (k, f) in w.frames.iter().enumerate() { if cancel() { let mut s = lock(&shared.state); @@ -414,9 +446,21 @@ async fn run_work(w: Work, shared: &RotoShared) { // The frame and its neighbours (what a step reads), on the GPU. src.prefetch(&[*f, f - 1, f + 1]).await; let mut get = |g: i64| src.get(g); - let ok = match w.task { - RotoTask::Propagate => chain.ensure(*f, &mut get, &cancel).is_some(), - RotoTask::Freeze => match fx::frame_matte(&w.params, w.size, w.fps, *f, &mut get, &cancel) { + let ok = match (w.task, w.object_matte) { + (RotoTask::Propagate, true) => om::ensure(&w.params, w.size, w.fps, *f, model.clone(), &mut get, &cancel).is_some(), + (RotoTask::Freeze, true) => { + let img = get(*f); + match img.map(|img| (om::frame_matte(&w.params, w.size, w.fps, *f, 1.0, &img, model.clone(), &mut get, &cancel), img)) { + Some((om::FrameMatte::Alpha(a), img)) => { + frozen.size = [img.width as usize, img.height as usize]; + frozen.frames.insert(*f, fx::freeze_frame(&a, &[])); + true + } + _ => false, + } + } + (RotoTask::Propagate, false) => chain.ensure(*f, &mut get, &cancel).is_some(), + (RotoTask::Freeze, false) => match fx::frame_matte(&w.params, w.size, w.fps, *f, &mut get, &cancel) { Some((a, b)) => { if let Some(img) = src.get(*f) { frozen.size = [img.width as usize, img.height as usize]; @@ -471,12 +515,15 @@ impl Session { self.roto_job.as_ref().map(|j| j.progress()).or_else(|| { self.offloaded_roto().map(|(j, task)| { let p = &j.progress; + let object_matte = + self.project.comp(j.comp).and_then(|c| c.layer(j.layer)).and_then(|l| l.props.find_group(j.uid)).is_some_and(is_object_matte); RotoProgress { task: Some(task), done: p.done, total: p.total, elapsed: p.elapsed, fps: crate::offload::rate(p.done, p.elapsed), + object_matte, ..Default::default() } }) @@ -488,7 +535,8 @@ impl Session { self.roto_job.as_ref().map(|j| (j.comp, j.layer, j.effect, j.task)).or_else(|| self.offloaded_roto().map(|(j, task)| (j.comp, j.layer, j.uid, task))) } - /// The segmentation chain a propagation of `effect` computes (its frames' cache keys). + /// The segmentation chain a propagation of Roto Brush `effect` computes (its frames' cache + /// keys). pub fn roto_chain(&self, comp: ItemId, layer: LayerId, effect: Uid) -> Option { let c = self.project.comp(comp)?; let l = c.layer(layer)?; @@ -499,6 +547,27 @@ impl Session { Some(fx::Chain::new(¶ms, layer_size(&self.project, c, l), c.frame_rate.as_f64(), 1.0)) } + /// The cache keys of the segmentations a propagation of `effect` (Roto Brush or Object + /// Matte) computes, by layer frame. + pub fn roto_keys(&self, comp: ItemId, layer: LayerId, effect: Uid) -> Option> { + if let Some(ch) = self.roto_chain(comp, layer, effect) { + return Some(ch.keys.into_iter().collect()); + } + let c = self.project.comp(comp)?; + let l = c.layer(layer)?; + let g = l.effects()?.groups().find(|g| g.uid == effect && is_object_matte(g))?; + let params = self.om_params(comp, l, g)?; + let frames = om::job_frames(¶ms, frame_limits(c, l)); + Some(om::frame_keys(¶ms, layer_size(&self.project, c, l), c.frame_rate.as_f64(), &frames, fx::installed_model())) + } + + /// An Object Matte's parameters at its reference frame (what its jobs segment with). + fn om_params(&self, comp: ItemId, l: &Layer, g: &PropGroup) -> Option { + let c = self.project.comp(comp)?; + let t = comp_time_of(c, l, om::reference_frame(¶ms_static(g))); + Some(params_at(&self.project, comp, l, g, t, self.expr.as_deref())) + } + pub fn stop_roto(&mut self) -> bool { if self.cancel_offloaded(JobKind::RotoFreeze) || self.cancel_offloaded(JobKind::RotoPropagate) { return true; @@ -532,17 +601,36 @@ impl Session { let l = c.layer(layer).ok_or("no layer")?; let fxg = l.effects().ok_or("the layer has no effects")?; let index = fxg.groups().position(|g| g.uid == effect).ok_or("no such effect")?; - let g = fxg.groups().nth(index).filter(|g| is_roto(g)).ok_or("not a Roto Brush & Refine Edge effect")?; + let g = fxg.groups().nth(index).filter(|g| is_roto(g) || is_object_matte(g)).ok_or("not a Roto Brush & Refine Edge or Object Matte effect")?; + let object_matte = is_object_matte(g); let size = layer_size(&self.project, c, l); let fps = c.frame_rate.as_f64(); - let pstat = params_static(g); - let d = fx::data(&pstat); - if d.is_empty() { - return Err("draw a foreground stroke with the Roto Brush tool first".into()); - } - let t = comp_time_of(c, l, d.base.unwrap_or(0)); - let params = params_at(&self.project, comp, l, g, t, self.expr.as_deref()); - let frames = job_frames(&d, if task == RotoTask::Freeze { Direction::Both } else { dir }); + let (params, frames) = if object_matte { + let params = self.om_params(comp, l, g).ok_or("no composition")?; + let base = om::reference_frame(¶ms); + if om::roto_data(¶ms, size, base, fx::installed_model().is_some()).is_none() { + return Err("set the object first: click it with the Object Matte tool (its point is off the layer or outside its box)".into()); + } + let all = om::job_frames(¶ms, frame_limits(c, l)); + if all.is_empty() { + return Err(format!("the reference frame {base} is outside the layer")); + } + // Forward / backward from the reference frame, as for a Roto Brush. + let frames = all + .into_iter() + .filter(|f| task == RotoTask::Freeze || dir == Direction::Both || *f == base || (*f > base) == (dir == Direction::Forward)) + .collect(); + (params, frames) + } else { + let pstat = params_static(g); + let d = fx::data(&pstat); + if d.is_empty() { + return Err("draw a foreground stroke with the Roto Brush tool first".into()); + } + let t = comp_time_of(c, l, d.base.unwrap_or(0)); + let params = params_at(&self.project, comp, l, g, t, self.expr.as_deref()); + (params, job_frames(&d, if task == RotoTask::Freeze { Direction::Both } else { dir })) + }; let n = frames.len(); let work = Work { project: self.project.clone(), @@ -557,8 +645,9 @@ impl Session { fps, frames, task, + object_matte, }; - let strokes = str_of(g, fx::STROKES); + let strokes = guard_of(g); if !wait && self.offloads() { drop(work); self.offload_analysis(match task { @@ -569,6 +658,13 @@ impl Session { } let wait = wait || cfg!(target_arch = "wasm32"); let shared = Arc::new(RotoShared::default()); + { + // Known before the worker starts (the progress shows at once). + let mut st = lock(&shared.state); + st.task = Some(task); + st.total = n as u64; + st.object_matte = object_matte; + } let thread = if wait { crate::offload::run_or_queue({ let sh = shared.clone(); @@ -584,7 +680,7 @@ impl Session { .map_err(|e| e.to_string())?, ) }; - self.roto_job = Some(RotoJob { shared, thread, comp, layer, effect, task, strokes }); + self.roto_job = Some(RotoJob { shared, thread, comp, layer, effect, task, strokes, object_matte }); self.events.push(Event::ProjectChanged { revision: self.revision }); self.poll_roto_job(); Ok(n) @@ -601,40 +697,43 @@ impl Session { let _ = t.join(); } let st = job.progress(); + let name = if job.object_matte { "Object Matte" } else { "Roto Brush" }; if let Some(e) = &st.error { - self.events.push(Event::Toast { message: format!("Roto Brush: {e}"), error: true }); + self.events.push(Event::Toast { message: format!("{name}: {e}"), error: true }); self.bump(); return true; } if st.cancelled { - self.events.push(Event::Toast { message: "Roto Brush: stopped".into(), error: false }); + self.events.push(Event::Toast { message: format!("{name}: stopped"), error: false }); self.bump(); return true; } match job.task { RotoTask::Propagate => { - self.events.push(Event::Toast { message: format!("Roto Brush: propagated {} frame(s) in {:.1} s", st.done, st.elapsed), error: false }); + let verb = if job.object_matte { "tracked" } else { "propagated" }; + self.events.push(Event::Toast { message: format!("{name}: {verb} {} frame(s) in {:.1} s", st.done, st.elapsed), error: false }); // Rendered frames depend on the cache: redraw. self.bump(); } RotoTask::Freeze => { let Some(fc) = lock(&job.shared.frozen).take() else { return true }; let (cid, lid, uid, strokes) = (job.comp, job.layer, job.effect, job.strokes.clone()); + let what = if job.object_matte { "prompt" } else { "strokes" }; let r = self.edit("Freeze", None, |p, _| { let g = p .comp_mut(cid) .and_then(|c| c.layer_mut(lid)) .and_then(|l| l.props.find_group_mut(uid)) - .ok_or_else(|| crate::EngineError::Other("the Roto Brush effect was removed".into()))?; - if str_of(g, fx::STROKES) != strokes { - return Err(crate::EngineError::Other("the strokes changed while freezing".into())); + .ok_or_else(|| crate::EngineError::Other(format!("the {name} effect was removed")))?; + if guard_of(g) != strokes { + return Err(crate::EngineError::Other(format!("the {what} changed while freezing"))); } set_str(g, FROZEN, fc.to_json()); Ok(()) }); let msg = match r { - Ok(()) => format!("Roto Brush: froze {} frame(s) in {:.1} s", fc.frames.len(), st.elapsed), - Err(e) => format!("Roto Brush: {e}"), + Ok(()) => format!("{name}: froze {} frame(s) in {:.1} s", fc.frames.len(), st.elapsed), + Err(e) => format!("{name}: {e}"), }; self.events.push(Event::Toast { message: msg, error: false }); } @@ -664,6 +763,12 @@ impl Session { let Some(c) = self.project.comp(comp) else { return out }; let Some(l) = c.layer(layer) else { return out }; let Some(g) = l.props.find_group(effect) else { return out }; + if is_object_matte(g) { + for (f, k) in self.roto_keys(comp, layer, effect).unwrap_or_default() { + out.insert(f, fx::cached(k, 1.0).is_some()); + } + return out; + } let ps = params_static(g); let ch = fx::Chain::new(&ps, layer_size(&self.project, c, l), c.frame_rate.as_f64(), 1.0); for f in ch.keys.keys() { diff --git a/crates/engine/src/tests_roto.rs b/crates/engine/src/tests_roto.rs index 19eee110..edcab5c1 100644 --- a/crates/engine/src/tests_roto.rs +++ b/crates/engine/src/tests_roto.rs @@ -465,3 +465,84 @@ fn object_matte_reprompt_updates_the_instance() { // An unknown or non-Object Matte effect is refused. assert!(s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "effect": 999_999, "point": c})).is_err()); } + +/// Object Matte runs in the background like Roto Brush propagation: queued by the edit, a job +/// that returns at once and reports progress, stops when cancelled (keeping what it computed), +/// computes the tracked frames when left to run, and Freeze stores mattes that renders then use +/// without segmenting again. +#[test] +fn object_matte_background_job_cancel_and_freeze() { + hold_roto_cache_test_lock(); + let (mut s, clip, solid) = setup(|f| draw(center(f))); + let cid = s.active_comp_id().unwrap(); + s.edit("remove solid", None, |p, _| { + p.comp_mut(cid).unwrap().layers.retain(|l| l.id != solid); + for it in p.items.values_mut() { + if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind { + f.path = "object-matte-job.mov".into(); + } + } + Ok(()) + }) + .unwrap(); + let c = center(0); + let r = s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "point": c, "before": 0, "after": 24, "frame": 0})).unwrap(); + let uid = r["effect"].as_u64().unwrap(); + assert!(s.roto_pending.contains(&(cid, clip, uid)), "the new prompt is queued for tracking"); + let wait_done = |s: &mut Session| { + let t = std::time::Instant::now(); + while s.is_roto_running() { + assert!(t.elapsed().as_secs() < 120, "the job never finished"); + std::thread::sleep(std::time::Duration::from_millis(5)); + s.poll_roto(false); + } + s.poll_roto(false); + }; + let toast = |s: &Session, m: &str| s.events.iter().any(|e| matches!(e, crate::Event::Toast { message, .. } if message.starts_with(m))); + let status = |s: &mut Session| s.execute("objectMatte.status", json!({"layer": clip.0, "effect": uid})).unwrap(); + + // Cancelled at once: it stops early, and says so. + let r = s.execute_checked("objectMatte.propagate", json!({"layer": clip.0, "effect": uid})).unwrap(); + assert_eq!(r["frames"], 25); + assert!(!s.roto_pending.contains(&(cid, clip, uid))); + let _ = s.execute("roto.cancel", json!({})); + wait_done(&mut s); + assert!(toast(&s, "Object Matte: stopped"), "{:?}", s.events); + let st = status(&mut s); + assert_eq!(st["frames"].as_array().unwrap().len(), 25, "{st}"); + let early = st["computed"].as_array().unwrap().len(); + assert!(early < 25, "{st}"); + + // Left to run: the call returns while the job works (progress visible), then every tracked + // frame is computed. + let r = s.execute_checked("objectMatte.propagate", json!({"layer": clip.0, "effect": uid})).unwrap(); + assert_eq!(r["running"], true, "{r}"); + let pr = s.roto_progress().unwrap(); + assert!(pr.object_matte && pr.banner().starts_with("Tracking Object Matte"), "{pr:?}"); + // One job at a time. + assert!(s.execute_checked("objectMatte.propagate", json!({"layer": clip.0, "effect": uid})).is_err()); + wait_done(&mut s); + assert!(toast(&s, "Object Matte: tracked"), "{:?}", s.events); + assert_eq!(status(&mut s)["computed"].as_array().unwrap().len(), 25); + let before = alpha_at(&s, 4); + + // Freeze, then drop the segmentations: renders use the frozen mattes and compute nothing. + let r = s.execute_checked("objectMatte.freeze", json!({"layer": clip.0, "effect": uid, "wait": true})).unwrap(); + assert_eq!(r["frozen"], true, "{r}"); + let keys: Vec = s.roto_keys(cid, clip, uid).unwrap().into_iter().map(|(_, k)| k).collect(); + assert_eq!(keys.len(), 25); + effectcraft_effects::roto::forget(keys.iter().copied()); + let frozen = alpha_at(&s, 4); + let diff = before.iter().zip(&frozen).filter(|(a, b)| (**a - **b).abs() > 0.02).count(); + assert!(diff < 30, "{diff} pixels differ from the computed matte"); + assert!(status(&mut s)["computed"].as_array().unwrap().is_empty(), "a frozen frame was segmented again"); + assert_eq!(status(&mut s)["frozen"], true); + // A frozen prompt is refused; Unfreeze computes again. + assert!(s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "effect": uid, "point": c})).is_err()); + s.execute_checked("objectMatte.unfreeze", json!({"layer": clip.0, "effect": uid})).unwrap(); + let _ = alpha_at(&s, 4); + assert!(status(&mut s)["computed"].as_array().unwrap().iter().any(|f| f == 4)); + // Undo brings the frozen mattes back. + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(status(&mut s)["frozen"], true); +} diff --git a/crates/ui-egui/src/i18n/ui.rs b/crates/ui-egui/src/i18n/ui.rs index f7c77905..117d4b70 100644 --- a/crates/ui-egui/src/i18n/ui.rs +++ b/crates/ui-egui/src/i18n/ui.rs @@ -508,6 +508,7 @@ const JAPANESE: &[(&str, &str)] = &[ ("⌘/Ctrl-drag a pin to record its motion", "⌘/Ctrl+ドラッグでピンの動きを記録"), ("▣ {}", "▣ {}"), ("⚠ {} is used by:", "⚠ {}は次のコマンドで使用中:"), + ("Track", "トラック"), ]; const SIMPLIFIED_CHINESE: &[(&str, &str)] = &[ @@ -1033,6 +1034,7 @@ const SIMPLIFIED_CHINESE: &[(&str, &str)] = &[ ("After Effects project", "After Effects 项目"), ("Lottie animation", "Lottie 动画"), ("Project", "项目"), + ("Track", "跟踪"), ]; const TRADITIONAL_CHINESE: &[(&str, &str)] = &[ @@ -1553,6 +1555,7 @@ const TRADITIONAL_CHINESE: &[(&str, &str)] = &[ ("After Effects project", "After Effects 專案"), ("Lottie animation", "Lottie 動畫"), ("Project", "專案"), + ("Track", "追蹤"), ]; /// Ukrainian (uk): original translations of the same interface rows as the Chinese catalogs. @@ -1988,6 +1991,7 @@ const UKRAINIAN: &[(&str, &str)] = &[ ("⌘/Ctrl-drag a pin to record its motion", "Перетягніть опорну точку з ⌘/Ctrl, щоб записати її рух"), ("▣ {}", "▣ {}"), ("⚠ {} is used by:", "⚠ {} використовується в:"), + ("Track", "Відстежити"), ]; #[cfg(test)] diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index a0e05e00..a6b5501c 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -1580,6 +1580,12 @@ impl EffectcraftApp { self.session.poll_roto(true); ctx.request_repaint_after(std::time::Duration::from_millis(100)); } + // Roto Brush propagation and Object Matte tracking / Freeze: finish jobs (a Freeze writes + // its mattes) and start queued ones in the background. + if self.session.roto_job.is_some() || !self.session.roto_pending.is_empty() { + self.session.poll_roto(true); + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } // 3D Camera Tracker: likewise (tracking, then solving). if self.session.camera_job.is_some() || !self.session.camera_pending.is_empty() { self.session.poll_camera(true); diff --git a/crates/ui-egui/src/panels/effect_controls.rs b/crates/ui-egui/src/panels/effect_controls.rs index 501adf89..a63b5a83 100644 --- a/crates/ui-egui/src/panels/effect_controls.rs +++ b/crates/ui-egui/src/panels/effect_controls.rs @@ -787,6 +787,44 @@ fn roto_editor(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, l p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim); } +/// Object Matte: Freeze / Unfreeze and Track / Stop buttons (the background job that computes +/// the matte over the tracked frames) and its status. +fn object_matte_editor(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, layer: &Layer, g: &PropGroup, r: Rect, actions: &mut Actions) { + use effectcraft_engine::effects::object_matte as om; + let t = app.tokens; + let params = effectcraft_engine::effects::flatten_params(g, &mut |pr| pr.value.clone()); + let frozen = om::is_frozen(¶ms); + let mine = app.session.roto_target().is_some_and(|(_, l, u, _)| l == layer.id && u == g.uid); + let running = mine && app.session.is_roto_running(); + let busy = app.session.is_roto_running(); + let b1 = Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 4.0), vec2(84.0, 22.0)); + let b2 = Rect::from_min_size(pos2(b1.max.x + 8.0, b1.min.y), vec2(84.0, 22.0)); + let label = if frozen { "Unfreeze" } else { "Freeze" }; + if widgets::text_button(ui, b1, crate::i18n::tr(label), frozen, &t, egui::Id::new(("om-freeze-ec", g.uid))).clicked() && !busy { + let id = if frozen { "objectMatte.unfreeze" } else { "objectMatte.freeze" }; + actions.push((id.into(), json!({"layer": layer.id.0, "effect": g.uid}))); + } + app.auto.add(&format!("effectControls.objectMatte.{}.freeze", g.uid), b1, label); + let (l2, id2) = if running { ("Stop", "roto.cancel") } else { ("Track", "objectMatte.propagate") }; + if widgets::text_button(ui, b2, crate::i18n::tr(l2), false, &t, egui::Id::new(("om-track-ec", g.uid))).clicked() && !frozen && (running || !busy) { + actions.push((id2.into(), json!({"layer": layer.id.0, "effect": g.uid}))); + } + app.auto.add(&format!("effectControls.objectMatte.{}.track", g.uid), b2, l2); + let base = om::reference_frame(¶ms); + let status = match app.session.roto_progress() { + Some(pr) if running => pr.banner(), + _ if frozen => format!("Frozen from reference frame {base}"), + _ if app.session.roto_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => "Waiting to track".to_string(), + _ => format!("Reference frame {base}: Track computes the matte ahead of playback"), + }; + p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, &status, Tokens::ui(11.5), t.text_dim); + app.auto.add( + &format!("effectControls.objectMatte.{}.status", g.uid), + Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 30.0), vec2(r.width() - 24.0, 16.0)), + &status, + ); +} + // --------------------------------------------------------------------------------------------- // Curves / Levels editors @@ -797,7 +835,7 @@ fn editor_height(effect: &str, g: &PropGroup, width: f32) -> f32 { "ec.color.curves" => 34.0 + curves_size(width) + 26.0, effectcraft_engine::effects::warp_stab::ID => 50.0, effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::EDITOR_HEIGHT, - effectcraft_engine::effects::roto::ID => 50.0, + effectcraft_engine::effects::roto::ID | effectcraft_engine::effects::object_matte::ID => 50.0, "ec.color.levels" | "ec.color.levelsic" => 34.0 + 80.0 + 58.0, "ec.color.autolevels" | "ec.color.autocontrast" | "ec.color.autocolor" => 24.0 + 80.0 + 12.0, _ => 0.0, @@ -1207,6 +1245,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { effectcraft_engine::effects::warp_stab::ID => warp_editor(app, ui, &bp, &layer, g, er, &mut actions), effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::editor(app, ui, &bp, &layer, g, er, &mut actions), effectcraft_engine::effects::roto::ID => roto_editor(app, ui, &bp, &layer, g, er, &mut actions), + effectcraft_engine::effects::object_matte::ID => object_matte_editor(app, ui, &bp, &layer, g, er, &mut actions), "ec.color.levels" | "ec.color.levelsic" => levels_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions), super::fx_editors::GLOW | super::fx_editors::RESHAPE | super::fx_editors::EXTRACTOR => { super::fx_editors::header_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions) diff --git a/crates/ui-egui/tests/ui_object_matte.rs b/crates/ui-egui/tests/ui_object_matte.rs index 483d271e..5f30ba03 100644 --- a/crates/ui-egui/tests/ui_object_matte.rs +++ b/crates/ui-egui/tests/ui_object_matte.rs @@ -8,6 +8,7 @@ use effectcraft_engine::effects::object_matte as om; use effectcraft_engine::keyframe::Value; use effectcraft_engine::project::{GroupKind, LayerId, PropGroup}; use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::dock::PanelKind; use effectcraft_ui_egui::state::Tool; use egui::{Event, Modifiers, Pos2, Rect, pos2, vec2}; use egui_kittest::Harness; @@ -140,3 +141,45 @@ fn object_matte_tool_prompts_with_a_click_and_a_box() { h.state_mut().session.execute("edit.undo", json!({})).unwrap(); assert_eq!(flatten_params(&mattes(&h)[0], &mut |p| p.value.clone()).get("useBox"), Some(&Value::Bool(false))); } + +/// Effect Controls: the Object Matte's Track and Freeze buttons run the background job and its +/// status line shows where it is; the UI keeps drawing frames while it runs and finishes the +/// job itself (a Freeze writes its mattes without any other job running). +#[test] +fn object_matte_effect_controls_track_and_freeze() { + let mut h = harness(); + h.state_mut().show_panel(PanelKind::EffectControls); + h.state_mut().ui.tool = Tool::ObjectMatte; + h.run_steps(2); + let center = screen(&h, [320.0, 180.0]); + click(&mut h, center, Modifiers::NONE); + let uid = mattes(&h)[0].uid; + h.run_steps(2); + let status = |h: &Harness<'_, EffectcraftApp>| h.state().auto.find(&format!("effectControls.objectMatte.{uid}.status")).map(|e| e.label.clone()); + assert!(status(&h).is_some(), "no Object Matte status line"); + // Freeze: a background job (the frames keep coming), then the effect is frozen. + let b = rect(&h, &format!("effectControls.objectMatte.{uid}.freeze")); + // Let any automatic tracking finish first (one job at a time). + let frozen = |h: &Harness<'_, EffectcraftApp>| om::is_frozen(&flatten_params(&mattes(h)[0], &mut |p| p.value.clone())); + let t = std::time::Instant::now(); + while h.state().session.is_roto_running() { + assert!(t.elapsed().as_secs() < 120); + h.step(); + } + click(&mut h, b.center(), Modifiers::NONE); + let t = std::time::Instant::now(); + while !frozen(&h) { + assert!( + t.elapsed().as_secs() < 20, + "never frozen: {:?} / {} / running {} / {:?}", + status(&h), + h.state().ui.status, + h.state().session.is_roto_running(), + h.state().session.roto_progress() + ); + h.step(); + } + h.run_steps(2); + assert!(status(&h).is_some_and(|s| s.starts_with("Frozen")), "{:?}", status(&h)); + assert_eq!(h.state().auto.find(&format!("effectControls.objectMatte.{uid}.freeze")).map(|e| e.label.as_str()), Some("Unfreeze")); +} diff --git a/docs/agents.md b/docs/agents.md index 6a3b9e85..d94d46bd 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -471,6 +471,24 @@ effectcraft-cli run clip.ecproj roto.stroke '{"layer":"#1","points":[[300,200],[ roto.propagate '{"layer":"#1","wait":true}' roto.freeze '{"layer":"#1","wait":true}' --save ``` +### Object Matte + +The Object Matte tool (third in the Alt+W slot) works in the Composition viewer: click an object, +or drag a box around it; Shift adds another Object Matte instead of re-prompting the layer's. +Agents use `objectMatte.apply {layer, point: [x, y], box?: [x0, y0, x1, y1], frame?, track?, +before?, after?, effect?}` (layer pixels, layer frames; `effect` re-prompts that instance), one +undo step each. With a trained model installed (Settings ▸ Roto Brush) the box is the model's box +prompt. `objectMatte.propagate {layer, effect?, direction?, wait?}` tracks the matte over the +frames around the reference frame in the background (`roto.cancel` stops it; edits to the prompt +or the layer's frames queue it again), `objectMatte.freeze {wait?}` / `objectMatte.unfreeze` +store or drop the final mattes, and `objectMatte.status {layer, effect?}` reports the job's +progress, the frozen state, the tracked frames and which are computed. + +```sh +effectcraft-cli run clip.ecproj objectMatte.apply '{"layer":"#1","point":[320,200],"box":[250,120,400,300]}' \ + objectMatte.propagate '{"layer":"#1","wait":true}' objectMatte.freeze '{"layer":"#1","wait":true}' --save +``` + ### Text: styles and editing Source Text holds character style runs and per-paragraph settings. `layer.setText` changes the diff --git a/docs/effects.md b/docs/effects.md index 84f16ef8..8a02b489 100644 --- a/docs/effects.md +++ b/docs/effects.md @@ -3,7 +3,7 @@ -EffectCraft ships 307 effects with 3028 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. +EffectCraft ships 307 effects with 3029 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. - **GPU**: 280 effects also run on the GPU compositor with identical results. - **32**: 307 effects process 32-bit float (HDR, overbright) pixels without clamping. From 3466538d0eb3288cf46f644958cd2cc295fc8a10 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 11 Oct 2026 14:06:33 +0200 Subject: [PATCH 6/8] #783: Object Matte mattes persist in the disk cache Problem: computed Object Matte segmentations lived only in the in-memory cache, so every restart or memory eviction segmented the shot again (After Effects 26.5 keeps Object Matte results in its disk cache). Fix: the persistent disk cache gets a third entry kind, Matte (folder `mattes`, LZ4-compressed FrameSeg bytes, same checksum, atomic writes, LRU eviction under the one size limit, purged with the rest). Entries are keyed by the segmentation's chain key and scale; chain keys already hash the Input Key (the layer's source, timing, masks and effects above), the prompt, the mode and the model, so any change simply misses. The effects crate gets a SegStore trait behind its memory cache: Chain::ensure reads from it on a memory miss and writes what it computes. The engine installs the session's DiskCache as the store (held weakly, so switching the disk cache off drops it). Roto Brush segmentations go through the same chain and are persisted too. Frozen mattes already persist in the project (the Frozen parameter). Tests: engine tests_roto::object_matte_mattes_persist_in_the_disk_cache (a second session on the same folder and project gets all tracked frames back without being able to read a single input frame, renders identically, and a moved prompt finds nothing); render disk_cache segmentations_round_trip_and_reopen (round trip, reopen, damaged file is a miss, size limit evicts). Co-Authored-By: Claude Opus 5.5 --- crates/effects/src/roto.rs | 50 ++++++++++++++--- crates/engine/src/session_settings.rs | 4 ++ crates/engine/src/tests_roto.rs | 66 +++++++++++++++++++++++ crates/render/src/disk_cache.rs | 78 +++++++++++++++++++++++++-- docs/agents.md | 4 +- 5 files changed, 191 insertions(+), 11 deletions(-) diff --git a/crates/effects/src/roto.rs b/crates/effects/src/roto.rs index cd2d240e..a6c80e74 100644 --- a/crates/effects/src/roto.rs +++ b/crates/effects/src/roto.rs @@ -7,7 +7,8 @@ //! out to the frame), so it lives in a process-wide content-keyed cache ([`cached`]) that the //! engine's propagation job fills at full resolution and rendering fills on demand (reading //! neighbouring frames with [`crate::EffectHost::self_at`]). Undo, redo and reopening a project -//! therefore never see stale mattes. **Freeze** stores the final mattes in the hidden **Frozen** +//! therefore never see stale mattes. Behind the memory cache an optional [`SegStore`] (the +//! persistent disk cache, set by the engine) keeps computed segmentations across restarts. **Freeze** stores the final mattes in the hidden **Frozen** //! parameter ([`FrozenCache`]); they are used while their key still matches. //! //! Rendering: binary segmentation → Roto Brush Matte (Shift Edge, Reduce Chatter, Feather, @@ -23,21 +24,21 @@ //! higher working resolution with more iterations. use std::collections::{BTreeMap, HashMap, VecDeque}; -use std::sync::{Arc, Mutex, OnceLock, RwLock}; +use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak}; use effectcraft_keyframe::Value; use effectcraft_project::ParamUi; use effectcraft_raster::Image; use effectcraft_segment::MaskModel; use effectcraft_track::roto::refine::{self, MatteParams}; -use effectcraft_track::roto::{self as rb, FrameSeg, RotoData, SegOpts, rle}; +use effectcraft_track::roto::{self as rb, RotoData, SegOpts, rle}; use rayon::prelude::*; use serde::{Deserialize, Serialize}; use crate::warp_stab::param; use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; -pub use effectcraft_track::roto::{Stroke, StrokeKind}; +pub use effectcraft_track::roto::{FrameSeg, Stroke, StrokeKind}; /// The effect's id. pub const ID: &str = "ec.matte.rotobrush"; @@ -266,6 +267,37 @@ pub fn store(key: u64, scale: f64, seg: Arc) { } } +/// A second-level store for segmentations, keyed like the memory cache (chain key, scale): the +/// persistent disk cache, so computed mattes survive the memory cache and restarts. +pub trait SegStore: Send + Sync { + fn get_seg(&self, key: u64, scale: f64) -> Option; + fn put_seg(&self, key: u64, scale: f64, seg: &FrameSeg); +} + +fn seg_store_slot() -> &'static RwLock>> { + static S: OnceLock>>> = OnceLock::new(); + S.get_or_init(|| RwLock::new(None)) +} + +/// Use `store` behind the memory cache (held weakly: once its owner drops it, e.g. the disk +/// cache is switched off, it is no longer used). `None` turns it off. +pub fn set_seg_store(store: Option>) { + if let Ok(mut s) = seg_store_slot().write() { + *s = store; + } +} + +fn seg_store() -> Option> { + seg_store_slot().read().ok()?.as_ref()?.upgrade() +} + +/// A segmentation from the second-level store, put into the memory cache. +fn from_store(key: u64, scale: f64) -> Option> { + let seg = Arc::new(seg_store()?.get_seg(key, scale)?); + store(key, scale, seg.clone()); + Some(seg) +} + /// Drop the cached segmentations of these chain keys (every scale). pub fn forget(keys: impl IntoIterator) { let keys: std::collections::HashSet = keys.into_iter().collect(); @@ -325,11 +357,11 @@ impl Chain { let sc = self.opts.scale; let lookup = |g: i64| -> Option> { let k = *self.keys.get(&g)?; - if let Some(s) = cached(k, sc) { + if let Some(s) = cached(k, sc).or_else(|| from_store(k, sc)) { return Some(s); } if sc != 1.0 { - let full = cached(k, 1.0)?; + let full = cached(k, 1.0).or_else(|| from_store(k, 1.0))?; let s = Arc::new(downscale(&full, sc)); store(k, sc, s.clone()); return Some(s); @@ -379,7 +411,11 @@ impl Chain { } }; let seg = Arc::new(seg); - store(*self.keys.get(&g)?, sc, seg.clone()); + let k = *self.keys.get(&g)?; + store(k, sc, seg.clone()); + if let Some(st) = seg_store() { + st.put_seg(k, sc, &seg); + } cur = Some(seg); prev_img = Some(img); } diff --git a/crates/engine/src/session_settings.rs b/crates/engine/src/session_settings.rs index 34f24786..55c8e415 100644 --- a/crates/engine/src/session_settings.rs +++ b/crates/engine/src/session_settings.rs @@ -132,6 +132,10 @@ impl Session { }, }; self.layer_cache.set_disk(Some((dc.clone(), footage_salt(&self.project)))); + // Roto Brush / Object Matte segmentations persist there too (held weakly: switching the + // disk cache off drops it). + let store: std::sync::Arc = dc.clone(); + effectcraft_effects::roto::set_seg_store(Some(std::sync::Arc::downgrade(&store))); self.disk_cache = Some(dc); } diff --git a/crates/engine/src/tests_roto.rs b/crates/engine/src/tests_roto.rs index edcab5c1..4b82c8c3 100644 --- a/crates/engine/src/tests_roto.rs +++ b/crates/engine/src/tests_roto.rs @@ -546,3 +546,69 @@ fn object_matte_background_job_cancel_and_freeze() { s.execute("edit.undo", json!({})).unwrap(); assert_eq!(status(&mut s)["frozen"], true); } + +/// Computed Object Matte segmentations go to the persistent disk cache: a second session on the +/// same folder and project gets them back without segmenting (its frames cannot even be read +/// here), and another prompt (other keys) finds nothing. +#[test] +fn object_matte_mattes_persist_in_the_disk_cache() { + use effectcraft_effects::object_matte as om; + use effectcraft_render::disk_cache::Kind; + hold_roto_cache_test_lock(); + let dir = std::env::temp_dir().join(format!("effectcraft-om-disk-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let enable = |s: &mut Session| { + s.execute_checked("prefs.set", json!({"values": {"disk.diskCacheFolder": dir.to_string_lossy(), "disk.diskCacheEnabled": true}})).unwrap(); + }; + let (mut s, clip, solid) = setup(|f| draw(center(f))); + enable(&mut s); + let cid = s.active_comp_id().unwrap(); + s.edit("remove solid", None, |p, _| { + p.comp_mut(cid).unwrap().layers.retain(|l| l.id != solid); + for it in p.items.values_mut() { + if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind { + f.path = "object-matte-disk.mov".into(); + } + } + Ok(()) + }) + .unwrap(); + let c = center(0); + let r = s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "point": c, "before": 0, "after": 5, "frame": 0})).unwrap(); + let uid = r["effect"].as_u64().unwrap(); + s.execute_checked("objectMatte.propagate", json!({"layer": clip.0, "effect": uid, "wait": true})).unwrap(); + let keys = s.roto_keys(cid, clip, uid).unwrap(); + assert_eq!(keys.len(), 6); + let dc = s.disk_cache.clone().unwrap(); + dc.flush(); + assert!(dc.keys(Kind::Matte).len() >= 6, "{:?}", dc.stats()); + let before = alpha_at(&s, 3); + let saved = s.project.clone(); + drop(dc); + drop(s); + effectcraft_effects::roto::forget(keys.iter().map(|(_, k)| *k)); + + // "Reopen": a new session on the same folder and project. + let (mut s2, _, _) = setup(|f| draw(center(f))); + enable(&mut s2); + s2.project = saved; + s2.bump(); + s2.open_comp(cid); + assert_eq!(s2.roto_keys(cid, clip, uid).unwrap(), keys, "the same input key, so the same keys"); + let comp = s2.active_comp().unwrap().clone(); + let l = comp.layer(clip).unwrap(); + let ps = crate::roto::params_static(l.props.find_group(uid).unwrap()); + let size = crate::roto::layer_size(&s2.project, &comp, l); + // No frames to segment from: whatever comes back was read from disk. + for (f, _) in &keys { + assert!(om::ensure(&ps, size, 25.0, *f, None, &mut |_| None, &|| false).is_some(), "frame {f} not reused from disk"); + } + assert!(s2.disk_cache.as_ref().unwrap().stats().hits >= 6); + let again = alpha_at(&s2, 3); + assert_eq!(again, before, "the disk-cached matte renders the same"); + // Another prompt: other keys, nothing on disk. + let mut moved = ps.clone(); + moved.values.insert("objectPoint".into(), effectcraft_keyframe::Value::Vec2([c[0] + 3.0, c[1]])); + assert!(om::ensure(&moved, size, 25.0, 0, None, &mut |_| None, &|| false).is_none()); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/render/src/disk_cache.rs b/crates/render/src/disk_cache.rs index da46708e..4140b792 100644 --- a/crates/render/src/disk_cache.rs +++ b/crates/render/src/disk_cache.rs @@ -8,7 +8,9 @@ //! project therefore never reads stale pixels: its keys simply change. //! * Payloads are LZ4-compressed (`lz4_flex`, MIT): frames as 8-bit premultiplied RGBA, layer //! buffers as `f32` RGBA with the bytes shuffled into planes (exact, so cached and fresh -//! renders are bit-identical). Every file ends in a checksum; damaged files read as misses and +//! renders are bit-identical), Roto Brush / Object Matte segmentations as +//! `FrameSeg::to_bytes` (keyed by their content-keyed chain key and scale, so they are +//! reused whenever the same frames, settings and prompt come back, also after a restart). Every file ends in a checksum; damaged files read as misses and //! are deleted. //! * Size limit with least-recently-used eviction. The in-memory index is rebuilt from the //! folder when the cache opens (file modification times order it), so the cache survives @@ -37,6 +39,8 @@ const QUEUE_LIMIT: usize = 64; pub enum Kind { Frame, Layer, + /// A Roto Brush / Object Matte segmentation. + Matte, } impl Kind { @@ -45,12 +49,14 @@ impl Kind { match self { Kind::Frame => "frames", Kind::Layer => "layers", + Kind::Matte => "mattes", } } fn tag(self) -> u8 { match self { Kind::Frame => 1, Kind::Layer => 2, + Kind::Matte => 3, } } } @@ -319,7 +325,7 @@ impl DiskCache { /// Rebuild the index from the folder (oldest files first). pub fn rescan(&self) { let mut found: Vec<(std::time::SystemTime, Kind, u128, u64)> = vec![]; - for kind in [Kind::Frame, Kind::Layer] { + for kind in [Kind::Frame, Kind::Layer, Kind::Matte] { let Ok(shards) = std::fs::read_dir(self.dir.join(kind.dir())) else { continue }; for shard in shards.flatten() { let Ok(files) = std::fs::read_dir(shard.path()) else { continue }; @@ -403,7 +409,7 @@ impl DiskCache { if let Ok(mut g) = self.index.lock() { g.clear(); } - for kind in [Kind::Frame, Kind::Layer] { + for kind in [Kind::Frame, Kind::Layer, Kind::Matte] { let _ = std::fs::remove_dir_all(self.dir.join(kind.dir())); } } @@ -555,6 +561,22 @@ impl DiskCache { self.write_now(Kind::Layer, key, &encode_layer(buf)); } + /// A segmentation (chain key, scale) written by [`DiskCache::put_seg`]. + pub fn get_seg(&self, key: u64, scale: f64) -> Option { + let p = self.read(Kind::Matte, seg_key(key, scale))?; + let raw = lz4_flex::decompress_size_prepended(&p).ok()?; + effectcraft_effects::roto::FrameSeg::from_bytes(&raw) + } + + /// Queue a segmentation for writing. + pub fn put_seg(&self, key: u64, scale: f64, seg: &effectcraft_effects::roto::FrameSeg) { + let k = seg_key(key, scale); + if self.contains(Kind::Matte, k) { + return; + } + self.enqueue(Kind::Matte, k, lz4_flex::compress_prepend_size(&seg.to_bytes())); + } + /// Write a frame synchronously (tests). pub fn put_frame_now(&self, key: u128, width: u32, height: u32, rgba: &[u8]) { self.write_now(Kind::Frame, key, &encode_frame(width, height, rgba)); @@ -627,6 +649,24 @@ pub fn read_layer_entry(data: &[u8]) -> Option { decode_layer(verify(data, Kind::Layer)?) } +/// The disk key of a segmentation: its chain key and scale. +fn seg_key(key: u64, scale: f64) -> u128 { + let mut h = Hash128::default(); + h.write(b"segmentation-v1"); + h.write_u64(key); + h.write_u64(scale.to_bits()); + h.finish() +} + +impl effectcraft_effects::roto::SegStore for DiskCache { + fn get_seg(&self, key: u64, scale: f64) -> Option { + DiskCache::get_seg(self, key, scale) + } + fn put_seg(&self, key: u64, scale: f64, seg: &effectcraft_effects::roto::FrameSeg) { + DiskCache::put_seg(self, key, scale, seg); + } +} + impl crate::cache::LayerStore for DiskCache { fn get_layer(&self, key: u128) -> Option> { DiskCache::get_layer(self, key).map(Arc::new) @@ -889,6 +929,38 @@ mod tests { assert_eq!(parse_entry_name(&format!("{key:032x}.tmp.1")), None); } + /// Segmentations round-trip by (chain key, scale), survive a reopen (rescan), are bounded by + /// the size limit like every entry, and a damaged file is a miss. + #[test] + fn segmentations_round_trip_and_reopen() { + use effectcraft_effects::roto::FrameSeg; + let dir = tmpdir("seg"); + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + let seg = FrameSeg { w: 7, h: 5, matte: (0..35).map(|i| (i % 3 == 0) as u8).collect(), refine: vec![0; 35], ..Default::default() }; + assert!(dc.get_seg(42, 1.0).is_none()); + dc.put_seg(42, 1.0, &seg); + dc.flush(); + assert_eq!(dc.get_seg(42, 1.0), Some(seg.clone())); + assert!(dc.get_seg(42, 0.5).is_none() && dc.get_seg(43, 1.0).is_none()); + assert_eq!(dc.keys(Kind::Matte).len(), 1); + drop(dc); + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + assert_eq!(dc.get_seg(42, 1.0), Some(seg.clone())); + // Damaged on disk: a miss, and the entry is forgotten. + let path = dc.path(Kind::Matte, seg_key(42, 1.0)); + let mut d = std::fs::read(&path).unwrap(); + d[18] ^= 0x55; + std::fs::write(&path, d).unwrap(); + assert!(dc.get_seg(42, 1.0).is_none()); + assert!(!dc.contains(Kind::Matte, seg_key(42, 1.0))); + // The size limit evicts segmentations too. + dc.put_seg(1, 1.0, &seg); + dc.flush(); + dc.set_max_bytes(1); + assert!(dc.keys(Kind::Matte).is_empty()); + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn hit_miss_and_exact_round_trip() { let dir = tmpdir("hit"); diff --git a/docs/agents.md b/docs/agents.md index d94d46bd..8bf3cc87 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -482,7 +482,9 @@ prompt. `objectMatte.propagate {layer, effect?, direction?, wait?}` tracks the m frames around the reference frame in the background (`roto.cancel` stops it; edits to the prompt or the layer's frames queue it again), `objectMatte.freeze {wait?}` / `objectMatte.unfreeze` store or drop the final mattes, and `objectMatte.status {layer, effect?}` reports the job's -progress, the frozen state, the tracked frames and which are computed. +progress, the frozen state, the tracked frames and which are computed. With the disk cache on (Settings ▸ +Media & Disk Cache), computed Object Matte and Roto Brush segmentations are also kept there, keyed +by what they are made of, so reopening a project reuses them. ```sh effectcraft-cli run clip.ecproj objectMatte.apply '{"layer":"#1","point":[320,200],"box":[250,120,400,300]}' \ From a8de199acac3c079be916bbed2ba3ad4f64797f6 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 11 Oct 2026 14:07:22 +0200 Subject: [PATCH 7/8] #783: Object Matte fidelity plan; gaps and parity notes updated Problem: Object Matte had no After Effects comparison, and none can be made on this Linux machine (no After Effects). The gap notes still listed the viewer tool, background job and disk cache as missing. Fix: docs/fidelity/object-matte.md states plainly that nothing is measured yet and pins down the comparison for when an After Effects 26.2+ reference is available: four original or openly licensed clips (the synthetic moving disk from the engine tests, the disk with a jump, two objects with a box prompt, one CC0/CC BY real clip), the prompt and settings, where reference mattes stay (gitignored), and the metrics: per-frame IoU against After Effects (headline), IoU against ground truth, boundary distance, soft-edge error, temporal stability, drift and time per frame, plus a proposed pass bar left to the maintainers. Linked from docs/fidelity/README.md. docs/gaps.md (#14) and docs/target-app-parity.md now list what landed and leave the comparison as the one remaining item. Co-Authored-By: Claude Opus 5.5 --- docs/fidelity/README.md | 3 ++ docs/fidelity/object-matte.md | 73 +++++++++++++++++++++++++++++++++++ docs/gaps.md | 4 +- docs/target-app-parity.md | 4 +- 4 files changed, 80 insertions(+), 4 deletions(-) create mode 100644 docs/fidelity/object-matte.md diff --git a/docs/fidelity/README.md b/docs/fidelity/README.md index 025997d4..5afafee4 100644 --- a/docs/fidelity/README.md +++ b/docs/fidelity/README.md @@ -3,6 +3,9 @@ The [AE 2026 replacement acceptance gates](ae-2026-acceptance.md) define the broader target across workflows, rendered output, work-project compatibility, performance and platforms. +[Object Matte](object-matte.md) has a comparison plan (clips, prompt, per-frame IoU and the other +metrics) but no measurement yet. + The first G1 comparison measures eight original scalar Rotation cases against a live After Effects 2026 session through AEsync 2.0.4: Easy Ease, overlapping influences, asymmetric ease, overshoot, descending values, equal endpoints, 100%/100% influence and 0.1%/100% influence. Each case has diff --git a/docs/fidelity/object-matte.md b/docs/fidelity/object-matte.md new file mode 100644 index 00000000..da4e3bd3 --- /dev/null +++ b/docs/fidelity/object-matte.md @@ -0,0 +1,73 @@ +# Object Matte: comparison plan (not measured yet) + +**Status: untested.** There is no After Effects 26.2+ comparison of Object Matte yet. Nothing on +this page is a measurement. It lists what to compare once an After Effects reference session +is available, so the first run can be repeated and scored without guesswork. + +EffectCraft side, as of #783: the Object Matte effect (`ec.matte.objectmatte`), the Object Matte +tool in the Composition viewer, `objectMatte.apply` / `.propagate` / `.freeze` / +`.status`, and segmentations kept in the disk cache. The point (and an optional box) is +segmented by Roto Brush's engine: MobileSAM when it is installed (Settings ▸ Roto Brush, with +the box as the model's box prompt), otherwise the classical graph cut. The result is tracked +frame by frame with optical flow, and the model re-prompts each frame from the carried matte. +Record which engine was active, because the two are separate cases. + +## Test clips + +All original, generated or openly licensed; none of them goes into the repository. + +1. **Synthetic disk**: the moving textured disk over a textured background from + `crates/engine/src/tests_roto.rs` (`draw(center(f))`, 320×240, 25 fps, 25 frames, radius + 40 px, about 4 px a frame to the right). Export it as a PNG sequence. Its ground truth is + analytic (`gt(center(f))`). +2. **Synthetic disk with a jump**: the same disk with `jumping(f)` (it jumps at frame 8). This + checks how each tool handles losing the object. +3. **Two objects**: the disk plus a second still, differently coloured square that touches it + on some frames, prompted once with a point only and once with a box around the disk. This + checks the box prompt. +4. **Real footage**: one openly licensed clip (CC0 or CC BY) with a person or object moving + against clutter, at least 1080p and 5 s long. Record its URL, licence and SHA-256 next to the + results, and keep the clip itself out of the repository. + +## Prompt and settings + +- Reference frame 0. The point at the object's centre, in layer pixels (clip 1: `[90, 110]`). + The box, where used, is `[centre − 55, centre + 55]` on each axis. +- Track forward to the last frame (Frames After = clip length − 1, Frames Before = 0). +- Matte refinement at its defaults on both sides: Feather 0, Contrast 0, Shift Edge 0, no + Invert. Output the matte itself (View ▸ Matte in EffectCraft, the alpha channel in After + Effects) as a full-resolution PNG sequence. +- Record the After Effects build, the OS, the GPU, and the exact click position in After + Effects. It is a viewer click, so convert it to layer pixels with the comp's zoom. + +Keep the After Effects mattes in gitignored `plan/aftereffects/ref/object-matte/`. Only an +aggregate results file (like `keyframe-ease.json`) and the comparison script may be committed. + +## Metrics (per frame, then mean / worst) + +- **IoU against After Effects**: binarize both mattes at alpha 0.5, then intersection over + union. This is the headline number. `effectcraft_track::roto::iou` computes it, and + `roto.status` already reports it for Roto Brush through `compareTo`; Object Matte needs the + same for exported mattes. +- **IoU against ground truth** for the synthetic clips, for both applications. It tells + "differs from After Effects" apart from "worse than After Effects". +- **Boundary error**: the mean and 95th-percentile distance (px) between the two mattes' 0.5 + contours, which shows edge disagreement that IoU dilutes on large objects. +- **Soft-edge error**: the mean absolute alpha difference inside a 3 px band around the After + Effects contour. +- **Temporal stability**: the frame-to-frame IoU change of each matte on clip 1, where the + object moves rigidly, so jitter shows. +- **Drift**: the frame where IoU against ground truth first drops below 0.9 (clips 1 and 2). +- **Time**: seconds per frame to track the clip, warm and cold, on the same machine, with the + disk cache on and off. + +Proposed pass bar (a maintainer decision, not yet agreed): mean IoU against After Effects +≥ 0.95 and worst ≥ 0.90 on clips 1 and 3, with MobileSAM. With the classical engine the numbers +are reported but not gated. + +## Reproduce (to write) + +The script does not exist yet. It should: generate the synthetic clips; run EffectCraft +headless (`effectcraft-cli run … objectMatte.apply … objectMatte.propagate {"wait":true}`, then +render the matte); drive After Effects through ExtendScript or AEsync to apply Object Matte with +the same prompt and render its alpha; then compute the metrics above into a JSON results file. diff --git a/docs/gaps.md b/docs/gaps.md index 577f7bd6..e9366c73 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -42,7 +42,7 @@ because 179 issues were closed with regression tests between 5 and 10 October. | Real-user experience | ≈ 60%, uneven by platform | (in G2) | 179 issues closed 5–10 October, nearly all with regression tests ([log below](#real-user-report-log-510-october)); 134 open, many from Linux and Windows. Most checking still happens on macOS | | Performance | unknown against After Effects, ≈ 35% | 50–100 | Internal numbers only (e.g. Advanced 3D 290 ms/frame at 1080p on the GPU on an M4 Pro). Users report stutter on cached frames (#595, #304), slow scrubbing (#329), late audio (#554). No 4K / hundreds-of-layers project tested | | Media formats | ≈ 75% | (in file formats) | H.264, ProRes, HEVC, AV1, VP9, image sequences and audio exist. The HEVC / AV1 encoders have no B-frames, multi-reference or SAO / CDEF, so files are larger than from mature encoders. Camera formats (BRAW, R3D, ProRes RAW, variable-frame-rate phone video) are unverified | -| AI-assisted tools | ≈ 35% | 40–80 | Roto Brush 2.0 / 3.0 with MobileSAM (M13.35) scores IoU 0.989 on the base frame of our synthetic moving-disc test and ≥ 0.980 over 20 propagated frames (classic: 0.973). Face tracking with MediaPipe Face Landmarker (M13.36) matches Google's own pipeline to 0.85 px on a test portrait. Neither is compared with After Effects; Object Matte (26.2) is missing | +| AI-assisted tools | ≈ 35% | 40–80 | Roto Brush 2.0 / 3.0 with MobileSAM (M13.35) scores IoU 0.989 on the base frame of our synthetic moving-disc test and ≥ 0.980 over 20 propagated frames (classic: 0.973). Face tracking with MediaPipe Face Landmarker (M13.36) matches Google's own pipeline to 0.85 px on a test portrait. Neither is compared with After Effects; Object Matte (26.2) has a first slice, not compared either | | Maturity | early | — | First commit 1 October 2026. ≈ 329,000 lines of almost entirely agent-written Rust, 2,704 tests, 313 GitHub issues. After Effects has around 30 years of edge cases behind it | ## Every known gap, ranked @@ -70,7 +70,7 @@ are grouped by area; the issue tracker has the rest. | 11 | **Text rendering and layout** | #453 CJK text in comps, #316 paragraph alignment inverted, #483 squeezed at 4:3, #417 soft at Fit, #583 font search, #550 toolbar font options | Text is in almost every motion graphic | 10–20 h | G2 · [ui-parity.md](ui-parity.md) | | 12 | **3D interaction and depth** | No 3D transform gizmos (#392, #347); Z position (#489); camera aperture (#260); reflections (#261); bevel options (#315); 3D issue reopened (#307) | Advanced 3D is After Effects' growth area since 2023 | 20–35 h | G2 · [ui-parity.md](ui-parity.md) | | 13 | **3D model formats and materials** | After Effects 26 imports FBX, USD/USDZ, STL besides glTF and OBJ (`usd_plugins`), and Substance 3D materials with mesh displacement (26.0–26.2); we read glTF and OBJ | Product shots and 3D scenes come as FBX/USDZ | 25–45 h | — · [file-format-parity.md](file-format-parity.md) | -| 14 | **Object Matte** (26.2, disk cache in 26.5) | First slice (#783): the Object Matte effect (Effect ▸ Matte) and `objectMatte.apply` turn a point (and optional box) on a reference frame into a matte and follow the object up to 600 frames each way, or segment every frame on its own; Roto Brush's engine, cache and model (MobileSAM from Settings ▸ Roto Brush, classical graph cut without it). Missing: the click / hover tool in the viewer, a background propagation job, the disk cache, comparison with After Effects | The new one-click roto | 10–20 h | G7 · [target-app-parity.md](target-app-parity.md) | +| 14 | **Object Matte** (26.2, disk cache in 26.5) | First slice (#783): the Object Matte effect (Effect ▸ Matte) and `objectMatte.apply` turn a point (and optional box) on a reference frame into a matte and follow the object up to 600 frames each way, or segment every frame on its own; Roto Brush's engine, cache and model (MobileSAM from Settings ▸ Roto Brush, classical graph cut without it). Since then: the Object Matte tool in the viewer (click / drag a box, hover crosshair), the box as the model's box prompt, background tracking with progress and cancel, Freeze, and segmentations kept in the disk cache (26.5). Missing: a comparison with After Effects ([plan](fidelity/object-matte.md)) | The new one-click roto | 10–20 h | G7 · [target-app-parity.md](target-app-parity.md) | | 15 | **Hardware video decode and encode** | No VideoToolbox, NVENC, Quick Sync or VA-API; #498 asks for it; FilmCraft has VideoToolbox / NVENC / VA-API paths to borrow | 4K playback and export speed | 25–45 h | G5/G6 · [hardware-parity.md](hardware-parity.md) | | 16 | **Localization depth** | Menus in 6 languages; panels in 4; effect names and ~1,650 effect parameter names untranslated; French, German, Korean, Italian, Russian (After Effects ships them) absent; #599 Hangul boxes on macOS | After Effects is used heavily in Japan, China, Korea, Brazil, Europe | 60–110 h + native review | G9 · [localization-parity.md](localization-parity.md) | | 17 | **Media Browser lists files import refuses** | `media_browser.rs` lists TGA, DPX, HDR, AVI, MXF; `media::is_importable` has none of them; #431 any image imports as a sequence; #324 MP3 goes silent | Confusing failures on common VFX formats (DPX, TGA) | 15–30 h | G6 · [file-format-parity.md](file-format-parity.md) | diff --git a/docs/target-app-parity.md b/docs/target-app-parity.md index f0ed90b6..c8ef95a0 100644 --- a/docs/target-app-parity.md +++ b/docs/target-app-parity.md @@ -122,7 +122,7 @@ compositing and effects dominate; audio is light). Hours are to full parity for | Text and typography | 7% | 94% | 50% | 20–35 | CJK text in comps renders wrongly (#453); paragraph alignment inverted (#316); squeezed text at 4:3 (#483); font search (#583); soft text at Fit (#417); variable-font filter (26.3) | | Effects and animation presets | 14% | 85% | 40% | 70–130 | All 306 effects exist, but rendered output is unmeasured against After Effects; 8 built-in presets against After Effects' 621; open effect bugs (#580 crash, #514, #526, #463, #369, #473); effect colour labels (26.5); see [effects-parity.md](effects-parity.md) | | 3D (Classic and Advanced) | 6% | 82% | 35% | 35–65 | No FBX/USD/STL models, no Substance materials, mesh displacement unverified; no 3D transform gizmos (#392, #347); Z position (#489); reflections (#261); camera aperture (#260); bevel options (#315) | -| Tracking and AI-assisted tools | 5% | 80% | 35% | 35–70 | Object Matte (26.2) is a first slice: an effect and command from a point / box prompt, no viewer tool or disk cache; Roto Brush (MobileSAM) and face tracking (MediaPipe) unmeasured against After Effects; camera-tracker view (#505) | +| Tracking and AI-assisted tools | 5% | 80% | 35% | 35–70 | Object Matte (26.2) has the effect, the viewer tool (click / box), background tracking, Freeze and the 26.5 disk cache, not yet compared with After Effects; Roto Brush (MobileSAM) and face tracking (MediaPipe) unmeasured against After Effects; camera-tracker view (#505) | | Preview and cache | 7% | 92% | 40% | 25–50 | Playback stutters on cached frames (#595, #304); slow scrubbing (#329); late audio (#554, #504, #335); no benchmark against After Effects | | Render queue and export | 7% | 85% | 55% | 25–45 | No hardware encoding; ProRes 4444 colour tag (#340); rendering over an existing file (#325); default macOS output location (#446); uncompressed single-file output (#372); HEVC/AV1 without B-frames | | Audio | 2% | 85% | 45% | 6–12 | Audio sync and crackle (#335); audio to keyframes depth | @@ -144,7 +144,7 @@ Feature depth (≈ 50%) feeds the overall *ready* number with the other dimensio | Stability | 45% | 40–80 | Open crash reports #580, #591, #501, #426, #341, #306, #234, #511; no CI runs the tests on pull requests | [gaps.md](gaps.md#g3-stability-and-a-green-main) | | Platforms | 65% | 30–60 | Releases for macOS, Windows x64/x86/ARM64, Linux, FreeBSD and the web (more than After Effects), but Linux and Windows users hit basic bugs | [gaps.md](gaps.md#g2-real-user-reliability-on-every-platform) | | Ecosystem and plug-ins | 20% | 40–80 | After Effects scripts and expressions mostly carry over; no After Effects SDK plug-ins, `.ffx` presets or `.mogrt`; our WebAssembly plug-in API has no third-party plug-ins yet | [plugins.md](plugins.md) | -| AI features | 35% | 40–80 (overlaps Tracking) | Roto Brush (MobileSAM) and face tracking (MediaPipe) as optional downloads; Object Matte (point / box prompt, same model) without its viewer tool or disk cache; the After Effects bundle ships Whisper, BERT and Marian tokenizers and a shot-cut model we have no equivalent of | [gaps.md](gaps.md) | +| AI features | 35% | 40–80 (overlaps Tracking) | Roto Brush (MobileSAM) and face tracking (MediaPipe) as optional downloads; Object Matte (point / box prompt, same model; viewer tool, background tracking, Freeze, disk cache) unmeasured against After Effects; the After Effects bundle ships Whisper, BERT and Marian tokenizers and a shot-cut model we have no equivalent of | [gaps.md](gaps.md) | | Agent control (beyond After Effects) | ahead | — | Every command is drivable over MCP, the control channel and the CLI | [agents.md](agents.md) | **Totals.** Summing the rows double-counts: AI overlaps Tracking, performance overlaps Preview, From 925760c9c85aba319461b7499df38e706de00c57 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 12 Oct 2026 19:08:21 +0200 Subject: [PATCH 8/8] Web disk cache: handle the Matte kind (mattes/ folder) Co-Authored-By: Claude Sonnet 5.5 --- apps/effectcraft-web/js/host.js | 3 ++- apps/effectcraft-web/src/diskcache.rs | 12 ++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/apps/effectcraft-web/js/host.js b/apps/effectcraft-web/js/host.js index 793900be..3db6ad35 100644 --- a/apps/effectcraft-web/js/host.js +++ b/apps/effectcraft-web/js/host.js @@ -404,7 +404,7 @@ export function frameWorkerStart(base, onMessage, gpu) { // ends in a checksum) and read the layer buffers they prefetch; the page lists, reads and // deletes. const CACHE_ROOT = "effectcraft-cache"; -const CACHE_KINDS = ["frames", "layers"]; +const CACHE_KINDS = ["frames", "layers", "mattes"]; const cacheDirHandles = {}; async function cacheDir(kind = "frames") { @@ -419,6 +419,7 @@ async function cacheDir(kind = "frames") { /// The folder and file name of an entry name (`layers/.ecc` or `.ecc`). async function entryDir(name) { if (name.startsWith("layers/")) return [await cacheDir("layers"), name.slice(7)]; + if (name.startsWith("mattes/")) return [await cacheDir("mattes"), name.slice(7)]; return [await cacheDir("frames"), name]; } diff --git a/apps/effectcraft-web/src/diskcache.rs b/apps/effectcraft-web/src/diskcache.rs index d96e6f13..ef9d46f8 100644 --- a/apps/effectcraft-web/src/diskcache.rs +++ b/apps/effectcraft-web/src/diskcache.rs @@ -105,19 +105,23 @@ pub fn init() { }); } -/// An entry's name in `js/host.js`'s cache folders (`layers/` for layer buffers). +/// An entry's name in `js/host.js`'s cache folders (`layers/` for layer buffers, `mattes/` for segmentations). pub(crate) fn name_of(kind: Kind, key: u128) -> String { match kind { Kind::Frame => disk_cache::entry_name(key), Kind::Layer => format!("layers/{}", disk_cache::entry_name(key)), + Kind::Matte => format!("mattes/{}", disk_cache::entry_name(key)), } } fn parse_name(name: &str) -> Option<(Kind, u128)> { - match name.strip_prefix("layers/") { - Some(n) => Some((Kind::Layer, disk_cache::parse_entry_name(n)?)), - None => Some((Kind::Frame, disk_cache::parse_entry_name(name)?)), + if let Some(n) = name.strip_prefix("layers/") { + return Some((Kind::Layer, disk_cache::parse_entry_name(n)?)); } + if let Some(n) = name.strip_prefix("mattes/") { + return Some((Kind::Matte, disk_cache::parse_entry_name(n)?)); + } + Some((Kind::Frame, disk_cache::parse_entry_name(name)?)) } /// Layer entries (32 hex digits) a frame worker should read before its next frame: recent