diff --git a/apps/effectcraft-web/js/host.js b/apps/effectcraft-web/js/host.js index 793900be..3db6ad35 100644 --- a/apps/effectcraft-web/js/host.js +++ b/apps/effectcraft-web/js/host.js @@ -404,7 +404,7 @@ export function frameWorkerStart(base, onMessage, gpu) { // ends in a checksum) and read the layer buffers they prefetch; the page lists, reads and // deletes. const CACHE_ROOT = "effectcraft-cache"; -const CACHE_KINDS = ["frames", "layers"]; +const CACHE_KINDS = ["frames", "layers", "mattes"]; const cacheDirHandles = {}; async function cacheDir(kind = "frames") { @@ -419,6 +419,7 @@ async function cacheDir(kind = "frames") { /// The folder and file name of an entry name (`layers/.ecc` or `.ecc`). async function entryDir(name) { if (name.startsWith("layers/")) return [await cacheDir("layers"), name.slice(7)]; + if (name.startsWith("mattes/")) return [await cacheDir("mattes"), name.slice(7)]; return [await cacheDir("frames"), name]; } diff --git a/apps/effectcraft-web/src/diskcache.rs b/apps/effectcraft-web/src/diskcache.rs index d96e6f13..ef9d46f8 100644 --- a/apps/effectcraft-web/src/diskcache.rs +++ b/apps/effectcraft-web/src/diskcache.rs @@ -105,19 +105,23 @@ pub fn init() { }); } -/// An entry's name in `js/host.js`'s cache folders (`layers/` for layer buffers). +/// An entry's name in `js/host.js`'s cache folders (`layers/` for layer buffers, `mattes/` for segmentations). pub(crate) fn name_of(kind: Kind, key: u128) -> String { match kind { Kind::Frame => disk_cache::entry_name(key), Kind::Layer => format!("layers/{}", disk_cache::entry_name(key)), + Kind::Matte => format!("mattes/{}", disk_cache::entry_name(key)), } } fn parse_name(name: &str) -> Option<(Kind, u128)> { - match name.strip_prefix("layers/") { - Some(n) => Some((Kind::Layer, disk_cache::parse_entry_name(n)?)), - None => Some((Kind::Frame, disk_cache::parse_entry_name(name)?)), + if let Some(n) = name.strip_prefix("layers/") { + return Some((Kind::Layer, disk_cache::parse_entry_name(n)?)); } + if let Some(n) = name.strip_prefix("mattes/") { + return Some((Kind::Matte, disk_cache::parse_entry_name(n)?)); + } + Some((Kind::Frame, disk_cache::parse_entry_name(name)?)) } /// Layer entries (32 hex digits) a frame worker should read before its next frame: recent diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 2304ce45..40ed79a7 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -40,6 +40,7 @@ pub mod mocha_shape; mod noise; mod noise2; mod noise3; +pub mod object_matte; mod obsolete; mod ocio; pub mod ocio_config; @@ -533,6 +534,7 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(camera_tracker::specs()); v.extend(face_track::specs()); v.extend(roto::specs()); + v.extend(object_matte::specs()); v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); for s in v.iter_mut() { if GPU_EFFECTS.contains(&s.id) { @@ -841,7 +843,15 @@ pub const PARAM_GROUPS: &[(&str, &str)] = &[ /// Display names of nested parameter groups by match id. fn group_name(m: &str) -> &str { - warp_stab::GROUPS.iter().chain(roto::GROUPS).chain(keylight::GROUPS).chain(PARAM_GROUPS).find(|(id, _)| *id == m).map(|(_, n)| *n).unwrap_or(m) + warp_stab::GROUPS + .iter() + .chain(roto::GROUPS) + .chain(object_matte::GROUPS) + .chain(keylight::GROUPS) + .chain(PARAM_GROUPS) + .find(|(id, _)| *id == m) + .map(|(_, n)| *n) + .unwrap_or(m) } /// The nested group at `path` (`borders/autoScale`) under `g`, created on first use. @@ -1216,6 +1226,7 @@ pub const TIME_DEPENDENT: &[&str] = &[ camera_tracker::ID, // Each frame has its own segmentation. roto::ID, + object_matte::ID, ]; /// See [`TIME_DEPENDENT`]. diff --git a/crates/effects/src/object_matte.rs b/crates/effects/src/object_matte.rs new file mode 100644 index 00000000..32a196af --- /dev/null +++ b/crates/effects/src/object_matte.rs @@ -0,0 +1,691 @@ +//! Object Matte (Effect ▸ Matte): isolate one object from a click, and follow it through the +//! shot (After Effects 26.2's Object Matte, from its public description: pick an object with a +//! point, optionally a box around it, and get a matte that tracks it). +//! +//! The prompt is the **Object Point** (layer pixels) and, with **Use Box**, a box around the +//! object. On the **Reference Frame** it is segmented by the same engine as Roto Brush: the +//! trained model chosen in Settings ▸ Roto Brush (MobileSAM, `effectcraft_segment`) when one is +//! installed, the classical graph cut otherwise. The model gets the point as a foreground point +//! and the box as its box prompt; the graph cut gets the point as a foreground stroke and the +//! box's edges (without a box, the layer's border) as background strokes. With **Track Object** the +//! matte is then carried frame by frame up to **Frames Before** / **Frames After** away (optical +//! flow, re-segmented by the model prompted with the carried matte); without it every frame is +//! segmented on its own from the prompt (useful when the point is animated). +//! +//! Like Roto Brush, the segmentation is derived data in Roto Brush's content-keyed cache +//! ([`crate::roto::cached`]): the chain seed holds the engine-maintained Input Key (what the +//! frames are made of), the layer size, the frame rate, the mode and the model, and every +//! frame's key the prompt, so edits upstream or to the prompt recompute it and undo never sees a +//! stale matte. The engine fills that cache in a background job (`objectMatte.propagate`, +//! outward from the reference frame over [`job_frames`]) and **Freeze** (`objectMatte.freeze`) +//! stores the final mattes in the hidden **Frozen** parameter ([`FrozenCache`], as Roto Brush +//! does): they are used, without segmenting, while [`frozen_key`] still matches. **Feather**, +//! **Contrast** and **Shift Edge** shape the matte, **Invert** keeps everything but the object +//! and **View ▸ Matte** shows the matte in grey. + +use std::sync::Arc; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::Image; +use effectcraft_segment::MaskModel; +use effectcraft_track::roto::refine::MatteParams; +use effectcraft_track::roto::{FrameSeg, RotoData, SegOpts, Stroke, StrokeKind, rle}; +use rayon::prelude::*; + +use crate::roto::{self, Chain, FROZEN, FrozenCache, INPUT_KEY}; +use crate::warp_stab::param; +use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; + +/// The effect's id. +pub const ID: &str = "ec.matte.objectmatte"; + +/// Display names of the instance's parameter groups. +pub const GROUPS: &[(&str, &str)] = &[("objectTracking", "Tracking"), ("objectMatte", "Matte")]; + +pub const VIEWS: [&str; 2] = ["Final Output", "Matte"]; + +/// Most frames a matte is carried from the reference frame each way. +pub const MAX_REACH: f64 = 600.0; +/// Default reach each way (frames). +pub const DEFAULT_REACH: f64 = 30.0; + +/// Radius (layer pixels) of the foreground stroke the Object Point becomes. +const POINT_RADIUS: f64 = 3.0; +/// Radius (layer pixels) of the background strokes along the box / the layer's border. +const EDGE_RADIUS: f64 = 2.0; + +pub fn specs() -> Vec { + vec![EffectSpec { + id: ID, + name: "Object Matte", + category: "Matte", + params: vec![ + p("objectPoint", "Object Point", Value::Vec2([0.5, 0.5]), ParamUi::Point), + p("useBox", "Use Box", Value::Bool(false), ParamUi::Checkbox), + p("boxTopLeft", "Box Top Left", Value::Vec2([0.25, 0.25]), ParamUi::Point), + p("boxBottomRight", "Box Bottom Right", Value::Vec2([0.75, 0.75]), ParamUi::Point), + p("objectTracking/trackObject", "Track Object", Value::Bool(true), ParamUi::Checkbox), + p("objectTracking/referenceFrame", "Reference Frame", num(0.0), ParamUi::Number), + p("objectTracking/framesBefore", "Frames Before", num(DEFAULT_REACH), slider(0.0, MAX_REACH, 0.0, 120.0, 0)), + p("objectTracking/framesAfter", "Frames After", num(DEFAULT_REACH), slider(0.0, MAX_REACH, 0.0, 120.0, 0)), + p("objectMatte/feather", "Feather", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("objectMatte/contrast", "Contrast", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("objectMatte/shiftEdge", "Shift Edge", num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 0)), + p("invert", "Invert", Value::Bool(false), ParamUi::Checkbox), + p("view", "View", Value::Enum(0), popup(&VIEWS)), + p(INPUT_KEY, "Input Key", Value::Str(String::new()), ParamUi::Hidden), + p(FROZEN, "Frozen", Value::Str(String::new()), ParamUi::Hidden), + ], + render, + gpu: false, + float: true, + }] +} + +fn f(params: &Params, id: &str) -> f64 { + param(params, id).map(Value::as_f64).unwrap_or(0.0) +} +fn b(params: &Params, id: &str) -> bool { + param(params, id).map(Value::as_bool).unwrap_or(false) +} +fn v2(params: &Params, id: &str) -> Option<[f64; 2]> { + let v = param(params, id)?.as_vec2(); + v.iter().all(|c| c.is_finite()).then_some(v) +} +fn s<'a>(params: &'a Params, id: &str) -> &'a str { + match param(params, id) { + Some(Value::Str(s)) => s, + _ => "", + } +} + +fn fnv(h: &mut u64, bytes: &[u8]) { + for b in bytes { + *h ^= *b as u64; + *h = h.wrapping_mul(0x0100_0000_01b3); + } +} + +/// A frame count parameter (finite, clamped to `lo..=hi`). +fn frames(params: &Params, id: &str, lo: f64, hi: f64) -> i64 { + let v = f(params, id); + if v.is_finite() { v.round().clamp(lo, hi) as i64 } else { 0 } +} + +/// The reference frame (layer frame). +pub fn reference_frame(params: &Params) -> i64 { + frames(params, "objectTracking/referenceFrame", -1e9, 1e9) +} + +/// Frames Before / After (finite, clamped to `0..=MAX_REACH`). +pub fn reach(params: &Params) -> [i64; 2] { + [frames(params, "objectTracking/framesBefore", 0.0, MAX_REACH), frames(params, "objectTracking/framesAfter", 0.0, MAX_REACH)] +} + +/// Whether the matte is carried from the reference frame (else every frame is segmented on its +/// own). +pub fn tracks(params: &Params) -> bool { + b(params, "objectTracking/trackObject") +} + +/// The prompt as Roto Brush strokes on layer frame `frame`: the Object Point as a foreground +/// stroke; the box as a box stroke (the model's box prompt) with a model, its edges as +/// background strokes for the graph cut without one; without a box, the layer's border when +/// there is no model (the graph cut needs background samples; a model does not, and a border +/// stroke would cut objects touching the frame edge). `None` when the point is unusable. +pub fn prompt_strokes(params: &Params, layer_size: [f64; 2], frame: i64, with_model: bool) -> Option> { + let [w, h] = layer_size; + if !(w.is_finite() && h.is_finite() && w >= 1.0 && h >= 1.0) { + return None; + } + let pt = v2(params, "objectPoint")?; + let inside = |q: [f64; 2]| q[0] >= 0.0 && q[1] >= 0.0 && q[0] <= w && q[1] <= h; + if !inside(pt) { + return None; + } + let mut out = vec![Stroke { kind: StrokeKind::Fg, frame, radius: POINT_RADIUS, points: vec![pt] }]; + let rect = |x0: f64, y0: f64, x1: f64, y1: f64| vec![[x0, y0], [x1, y0], [x1, y1], [x0, y1], [x0, y0]]; + let boxed = b(params, "useBox").then(|| Some((v2(params, "boxTopLeft")?, v2(params, "boxBottomRight")?))).flatten(); + match boxed { + Some((a, c)) => { + let (x0, x1) = (a[0].min(c[0]).clamp(0.0, w), a[0].max(c[0]).clamp(0.0, w)); + let (y0, y1) = (a[1].min(c[1]).clamp(0.0, h), a[1].max(c[1]).clamp(0.0, h)); + // The point must lie inside the box, clear of its edges, for the box to mean anything. + let m = POINT_RADIUS + EDGE_RADIUS; + if pt[0] - m <= x0 || pt[0] + m >= x1 || pt[1] - m <= y0 || pt[1] + m >= y1 { + return None; + } + out.push(if with_model { + // The model takes the box itself as its box prompt. + Stroke { kind: StrokeKind::Box, frame, radius: 0.0, points: vec![[x0, y0], [x1, y1]] } + } else { + Stroke { kind: StrokeKind::Bg, frame, radius: EDGE_RADIUS, points: rect(x0, y0, x1, y1) } + }); + } + None if !with_model => { + let i = EDGE_RADIUS; + if w <= 4.0 * i || h <= 4.0 * i { + return None; + } + out.push(Stroke { kind: StrokeKind::Bg, frame, radius: EDGE_RADIUS, points: rect(i, i, w - i, h - i) }); + } + None => {} + } + Some(out) +} + +/// The segmentation data the matte of layer frame `f` comes from: with Track Object, the prompt +/// on the reference frame and the span it is carried over; otherwise the prompt on `f` alone. +pub fn roto_data(params: &Params, layer_size: [f64; 2], f: i64, with_model: bool) -> Option { + if tracks(params) { + let base = reference_frame(params); + let [before, after] = reach(params); + let strokes = prompt_strokes(params, layer_size, base, with_model)?; + Some(RotoData { base: Some(base), span: [base.saturating_sub(before), base.saturating_add(after)], strokes }) + } else { + Some(RotoData { base: Some(f), span: [f, f], strokes: prompt_strokes(params, layer_size, f, with_model)? }) + } +} + +/// Seed of the chain keys: what the frames are (the Input Key), the layer size, the frame rate, +/// the mode and the model. The prompt is in each frame's key. +pub fn seed(params: &Params, layer_size: [f64; 2], fps: f64, model: Option<&dyn MaskModel>) -> u64 { + let mut h = 0xcbf2_9ce4_8422_2325; + fnv(&mut h, ID.as_bytes()); + fnv(&mut h, s(params, INPUT_KEY).as_bytes()); + for v in [layer_size[0], layer_size[1], fps] { + fnv(&mut h, &v.to_bits().to_le_bytes()); + } + fnv(&mut h, &[tracks(params) as u8]); + fnv(&mut h, model.map_or(effectcraft_segment::CLASSICAL, |m| m.info().id).as_bytes()); + h +} + +/// The layer frames a background job computes, outward from the reference frame: the tracked +/// span (Frames Before / After around it), clipped to `limits` (the layer's frames). Without +/// Track Object, the same frames, each segmented on its own. +pub fn job_frames(params: &Params, limits: [i64; 2]) -> Vec { + let base = reference_frame(params); + let [before, after] = reach(params); + let (lo, hi) = (base.saturating_sub(before).max(limits[0]), base.saturating_add(after).min(limits[1])); + if base < limits[0] || base > limits[1] { + return vec![]; + } + let mut v = vec![base]; + v.extend(base.saturating_add(1)..=hi); + v.extend((lo..base).rev()); + v +} + +/// The segmentation chain the matte of layer frame `f` comes from (`None`: no usable prompt). +pub fn chain(params: &Params, layer_size: [f64; 2], fps: f64, f: i64, scale: f64, model: Option>) -> Option { + let data = roto_data(params, layer_size, f, model.is_some())?; + let keys = data.chain_keys(seed(params, layer_size, fps, model.as_deref())); + Some(Chain { data: Arc::new(data), keys, opts: seg_opts(scale), refine_radius: 0.0, model }) +} + +/// Frame `f`'s segmentation at full resolution, computing what is missing (the background +/// job's step). `None` outside the tracked span, without a prompt, or when cancelled. +pub fn ensure( + params: &Params, + layer_size: [f64; 2], + fps: f64, + f: i64, + model: Option>, + frame: &mut dyn FnMut(i64) -> Option>, + cancel: &dyn Fn() -> bool, +) -> Option> { + let ch = chain(params, layer_size, fps, f, 1.0, model)?; + if !ch.data.in_span(f) { + return None; + } + ch.ensure(f, frame, cancel) +} + +/// The chain key of each of `frames` (what a computed segmentation is cached under). +pub fn frame_keys(params: &Params, layer_size: [f64; 2], fps: f64, frames: &[i64], model: Option>) -> Vec<(i64, u64)> { + if tracks(params) { + let Some(ch) = chain(params, layer_size, fps, 0, 1.0, model) else { return vec![] }; + return frames.iter().filter_map(|f| Some((*f, *ch.keys.get(f)?))).collect(); + } + frames.iter().filter_map(|f| Some((*f, *chain(params, layer_size, fps, *f, 1.0, model.clone())?.keys.get(f)?))).collect() +} + +/// What frozen mattes depend on: what the frames are (the seed: Input Key, size, rate, mode, +/// model), the prompt, the tracked span and the matte settings. +pub fn frozen_key(params: &Params, layer_size: [f64; 2], fps: f64, model: Option<&dyn MaskModel>) -> String { + let mut h = seed(params, layer_size, fps, model); + let base = reference_frame(params); + if let Some(st) = prompt_strokes(params, layer_size, base, model.is_some()) { + fnv(&mut h, serde_json::to_string(&st).unwrap_or_default().as_bytes()); + } + for v in [base, reach(params)[0], reach(params)[1]] { + fnv(&mut h, &v.to_le_bytes()); + } + fnv(&mut h, format!("{:?}", matte_params(params)).as_bytes()); + format!("{h:016x}") +} + +/// Whether the instance is frozen (has frozen mattes). +pub fn is_frozen(params: &Params) -> bool { + !s(params, FROZEN).is_empty() +} + +/// Frame `f`'s frozen matte on a `w`×`h` grid, when the instance is frozen, the mattes still +/// match ([`frozen_key`]) and `f` is one of them. +pub fn frozen_alpha(params: &Params, layer_size: [f64; 2], fps: f64, f: i64, w: usize, h: usize, model: Option<&dyn MaskModel>) -> Option> { + let j = s(params, FROZEN); + if j.is_empty() { + return None; + } + let fc = frozen_cache(j)?; + if fc.key != frozen_key(params, layer_size, fps, model) { + return None; + } + let fr = fc.frames.get(&f)?; + let a = rle::decode(&fr.a)?; + let [fw, fh] = fc.size; + if fw == 0 || fh == 0 || a.len() != fw.checked_mul(fh)? { + return None; + } + let af: Vec = a.iter().map(|v| *v as f32 / 255.0).collect(); + if fw == w && fh == h { + return Some(af); + } + let mut out = vec![0.0f32; w * h]; + for y in 0..h { + for x in 0..w { + let (sx, sy) = ((x as f64 + 0.5) * fw as f64 / w as f64, (y as f64 + 0.5) * fh as f64 / h as f64); + out[y * w + x] = effectcraft_track::roto::matting::sample(&af, fw, fh, sx, sy); + } + } + Some(out) +} + +/// The parsed Frozen parameter (parsed once per content). +fn frozen_cache(j: &str) -> Option> { + use std::collections::HashMap; + use std::sync::{Mutex, OnceLock}; + static C: OnceLock>>> = OnceLock::new(); + let mut h = 0xcbf2_9ce4_8422_2325; + fnv(&mut h, j.as_bytes()); + let c = C.get_or_init(Default::default); + if let Some(d) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&h) { + return Some(d.clone()); + } + let d = Arc::new(FrozenCache::from_json(j)?); + let mut m = c.lock().unwrap_or_else(|e| e.into_inner()); + if m.len() > 8 { + m.clear(); + } + m.insert(h, d.clone()); + Some(d) +} + +/// Segmentation settings at `scale` (Roto Brush's defaults). +fn seg_opts(scale: f64) -> SegOpts { + SegOpts { scale, search_radius: 15.0, motion_threshold: 10.0, motion_damping: 20.0, best: false } +} + +/// Matte settings: Feather, Contrast, Shift Edge (no Refine Edge band). +pub fn matte_params(params: &Params) -> MatteParams { + let num = |id: &str| { + let v = f(params, id); + if v.is_finite() { v } else { 0.0 } + }; + MatteParams { + rb_feather: num("objectMatte/feather").clamp(0.0, 100.0), + rb_contrast: num("objectMatte/contrast").clamp(0.0, 100.0), + rb_shift: num("objectMatte/shiftEdge").clamp(-100.0, 100.0), + decontaminate: false, + ..MatteParams::default() + } +} + +/// What a frame's matte is: the object's alpha on the layer grid, or why there is none. +#[derive(Clone, Debug, PartialEq)] +pub enum FrameMatte { + /// The object's alpha (0–1) on the frame's layer grid. + Alpha(Vec), + /// The prompt is unusable (point outside the layer, outside the box…): the layer passes. + NoPrompt, + /// The frame is beyond the tracked range, or its matte could not be computed. + Outside, +} + +/// The matte of layer frame `f` on the layer grid of `img` (the effect's input on that frame at +/// `scale`), computing what is missing. `frame(g)`: the input on frame g on the same grid. +#[allow(clippy::too_many_arguments)] +pub fn frame_matte( + params: &Params, + layer_size: [f64; 2], + fps: f64, + f: i64, + scale: f64, + img: &Image, + model: Option>, + frame: &mut dyn FnMut(i64) -> Option>, + cancel: &dyn Fn() -> bool, +) -> FrameMatte { + let Some(chain) = chain(params, layer_size, fps, f, scale, model) else { return FrameMatte::NoPrompt }; + if !chain.data.in_span(f) { + return FrameMatte::Outside; + } + let Some(seg) = chain.ensure(f, frame, cancel) else { return FrameMatte::Outside }; + let (w, h) = (img.width as usize, img.height as usize); + let seg = if seg.w == w && seg.h == h { (*seg).clone() } else { roto::resample(&seg, w, h) }; + FrameMatte::Alpha(roto::final_alpha(img, &seg, None, None, &matte_params(params), 0.0, scale)) +} + +fn render(ctx: &EffectCtx, buf: Buf) -> Buf { + if ctx.adjustment { + return buf; + } + let fps = ctx.fps(); + let f = roto::frame_of(ctx.time, fps); + let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + let img = Arc::new(roto::layer_grid(&buf, ctx.layer_size)); + let (w, h) = (img.width as usize, img.height as usize); + let host = ctx.env.host; + let effects = ctx.env.effect_index; + let layer_size = ctx.layer_size; + let me = img.clone(); + let mut frame = |g: i64| -> Option> { + if g == f { + return Some(me.clone()); + } + let other = host?.self_at(g as f64 / fps, effects)?; + let im = roto::layer_grid(&other, layer_size); + (im.width as usize == w && im.height as usize == h).then(|| Arc::new(im)) + }; + let model = roto::installed_model(); + // Frozen mattes are used as they are (no segmentation). + if let Some(a) = frozen_alpha(ctx.params, layer_size, fps, f, w, h, model.as_deref()) { + return compose(&buf, Some(&a), w, h, b(ctx.params, "invert"), param(ctx.params, "view").map(Value::as_enum).unwrap_or(0) == 1); + } + let alpha = match frame_matte(ctx.params, layer_size, fps, f, sc, &img, model, &mut frame, &|| false) { + FrameMatte::Alpha(a) if a.len() == w * h => Some(a), + FrameMatte::NoPrompt => return buf, + _ => None, + }; + compose(&buf, alpha.as_deref(), w, h, b(ctx.params, "invert"), param(ctx.params, "view").map(Value::as_enum).unwrap_or(0) == 1) +} + +/// The output: the input keyed by the matte (`None`: no object, all zero), or the matte itself. +fn compose(buf: &Buf, alpha: Option<&[f32]>, w: usize, h: usize, invert: bool, view_matte: bool) -> Buf { + let (ox, oy) = (buf.offset[0].round() as i64, buf.offset[1].round() as i64); + let mut out = Buf { img: Image::new(buf.img.width, buf.img.height), offset: buf.offset, scale: buf.scale }; + out.img.rows_mut().for_each(|(by, row)| { + for (bx, px) in row.iter_mut().enumerate() { + let (x, y) = (bx as i64 - ox, by as i64 - oy); + let a = if x >= 0 && y >= 0 && (x as usize) < w && (y as usize) < h { + alpha.and_then(|a| a.get(y as usize * w + x as usize)).copied().unwrap_or(0.0) + } else { + 0.0 + }; + let a = if invert { 1.0 - a } else { a }.clamp(0.0, 1.0); + let src = buf.img.get(bx as i64, by as i64); + *px = if view_matte { + let v = a * src[3]; + [v, v, v, 1.0] + } else { + [src[0] * a, src[1] * a, src[2] * a, src[3] * a] + }; + } + }); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, EffectHost, LayerPixels}; + + /// A red disk of radius 15 moving 2 px a frame right over a textured blue background + /// (100×80, 25 fps), plus a still green square in the corner. + fn frame_at(fi: i64) -> Image { + let c = [40.0 + 2.0 * fi as f64, 40.0]; + let mut img = Image::new(100, 80); + for y in 0..80 { + for x in 0..100 { + let d = ((x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1])) - 15.0; + let a = (0.5 - d).clamp(0.0, 1.0) as f32; + let tex = ((x * 7 + y * 13) % 9) as f32 / 60.0; + let bg = if x >= 80 && y >= 60 { [0.1, 0.8, 0.2] } else { [0.1, 0.4 + tex, 0.7] }; + let fg = [0.85 + tex, 0.2, 0.1]; + img.set(x, y, [fg[0] * a + bg[0] * (1.0 - a), fg[1] * a + bg[1] * (1.0 - a), fg[2] * a + bg[2] * (1.0 - a), 1.0]); + } + } + img + } + fn frame_of_time(t: f64) -> Image { + frame_at((t * 25.0 + 1e-6).floor() as i64) + } + + struct Seq; + impl EffectHost for Seq { + fn layer(&self, _: u64, _: bool) -> Option { + None + } + fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option> { + None + } + fn self_at(&self, t: f64, _: usize) -> Option { + Some(Buf { img: frame_of_time(t), offset: [0.0; 2], scale: 1.0 }) + } + } + + fn params(vals: &[(&str, Value)]) -> Params { + let s = &specs()[0]; + let mut p = Params { values: s.params.iter().map(|p| (p.id.to_string(), crate::default_value(p, [100.0, 80.0]))).collect() }; + for (k, v) in vals { + p.values.insert(k.to_string(), v.clone()); + } + p + } + + fn run(p: &Params, fi: i64) -> Image { + let host = Seq; + let t = fi as f64 / 25.0 + 0.001; + let ctx = EffectCtx { + params: p, + time: t, + layer_size: [100.0, 80.0], + seed: 1, + adjustment: false, + env: EffectEnv { host: Some(&host), frame_rate: 25.0, ..Default::default() }, + }; + crate::apply(&specs()[0], &ctx, Buf { img: frame_of_time(t), offset: [0.0; 2], scale: 1.0 }).img + } + + /// Pixels whose alpha disagrees with the disk centred at `cx` (1 px of slack at the edge). + fn wrong(out: &Image, cx: f64) -> usize { + let mut n = 0; + for y in 0..80u32 { + for x in 0..100u32 { + let d = (x as f64 + 0.5 - cx).hypot(y as f64 + 0.5 - 40.0); + let a = out.data[(y * 100 + x) as usize][3]; + if (d < 14.0 && a < 0.5) || (d > 16.0 && a > 0.5) { + n += 1; + } + } + } + n + } + + #[test] + fn a_click_isolates_the_object_and_tracks_it() { + let p = params(&[("objectPoint", Value::Vec2([40.0, 40.0]))]); + // The reference frame: the disk, nothing else. + let out = run(&p, 0); + assert!(wrong(&out, 40.0) < 20, "{} wrong pixels on the reference frame", wrong(&out, 40.0)); + // Six frames on (carried through frames 1–5), the disk has moved 12 px and the matte with it. + let out = run(&p, 6); + assert!(wrong(&out, 52.0) < 30, "{} wrong pixels on frame 6", wrong(&out, 52.0)); + // Beyond Frames After: no matte. + let p2 = params(&[("objectPoint", Value::Vec2([40.0, 40.0])), ("objectTracking/framesAfter", Value::Scalar(3.0))]); + assert!(run(&p2, 6).data.iter().all(|px| px[3] == 0.0)); + // Invert keeps everything but the disk; View ▸ Matte shows it in grey, opaque. + let inv = run(¶ms(&[("objectPoint", Value::Vec2([40.0, 40.0])), ("invert", Value::Bool(true))]), 0); + assert!(inv.data[40 * 100 + 40][3] < 0.5 && inv.data[5 * 100 + 95][3] > 0.5); + let m = run(¶ms(&[("objectPoint", Value::Vec2([40.0, 40.0])), ("view", Value::Enum(1))]), 0); + assert!(m.data[40 * 100 + 40][0] > 0.5 && m.data[5 * 100 + 95][0] < 0.5 && m.data.iter().all(|px| px[3] == 1.0)); + } + + #[test] + fn without_tracking_every_frame_follows_the_prompt() { + // The point animated onto the disk at frame 9: frame 9 is segmented from it alone. + let p = params(&[("objectPoint", Value::Vec2([58.0, 40.0])), ("objectTracking/trackObject", Value::Bool(false))]); + let out = run(&p, 9); + assert!(wrong(&out, 58.0) < 20, "{} wrong pixels", wrong(&out, 58.0)); + } + + #[test] + fn a_box_and_bad_prompts() { + // A box around the green square, the point in it: the square, not the disk. + let p = params(&[ + ("objectPoint", Value::Vec2([90.0, 70.0])), + ("useBox", Value::Bool(true)), + ("boxTopLeft", Value::Vec2([74.0, 54.0])), + ("boxBottomRight", Value::Vec2([100.0, 80.0])), + ]); + let out = run(&p, 0); + assert!(out.data[70 * 100 + 90][3] > 0.5 && out.data[40 * 100 + 40][3] < 0.5); + // Unusable prompts pass the layer through instead of failing: a point off the layer, a + // non-finite point, a point outside its box. + let src = frame_at(0); + for vals in [ + vec![("objectPoint", Value::Vec2([-5.0, 40.0]))], + vec![("objectPoint", Value::Vec2([f64::NAN, 40.0]))], + vec![ + ("objectPoint", Value::Vec2([40.0, 40.0])), + ("useBox", Value::Bool(true)), + ("boxTopLeft", Value::Vec2([60.0, 10.0])), + ("boxBottomRight", Value::Vec2([90.0, 30.0])), + ], + ] { + let p = params(&vals); + assert_eq!(run(&p, 0).data, src.data, "{vals:?}"); + } + // A huge reference frame or reach doesn't overflow. + let p = params(&[("objectTracking/referenceFrame", Value::Scalar(1e300)), ("objectTracking/framesBefore", Value::Scalar(f64::INFINITY))]); + assert!(run(&p, 0).data.iter().all(|px| px[3] == 0.0)); + } + + /// A stand-in model (the weights are an optional download): a disk around the first + /// foreground point, radius 15, so the test sees the model's answer in the matte. + #[derive(Default)] + struct Stub { + calls: std::sync::atomic::AtomicUsize, + /// The box prompt of the last call. + bbox: std::sync::Mutex>, + } + impl MaskModel for Stub { + fn info(&self) -> &'static effectcraft_segment::ModelInfo { + &effectcraft_segment::MOBILE_SAM + } + fn segment(&self, _: &[[f32; 3]], w: usize, h: usize, prompt: &effectcraft_segment::Prompt) -> effectcraft_segment::Result> { + self.calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + *self.bbox.lock().unwrap() = prompt.bbox; + let Some(([cx, cy], _)) = prompt.points.iter().find(|(_, fg)| *fg) else { return Err("no point".into()) }; + Ok((0..w * h).map(|i| (((i % w) as f32 + 0.5 - cx).hypot((i / w) as f32 + 0.5 - cy) < 15.0) as u8 as f32).collect()) + } + } + + #[test] + fn segments_with_the_installed_model_without_a_border_stroke() { + let p = params(&[("objectPoint", Value::Vec2([40.0, 40.0]))]); + // With a model, no border stroke: an object touching the frame edge stays whole. + assert_eq!(prompt_strokes(&p, [100.0, 80.0], 0, true).map(|s| s.len()), Some(1)); + assert_eq!(prompt_strokes(&p, [100.0, 80.0], 0, false).map(|s| s.len()), Some(2)); + // The model's answer is part of the cache key, and drives the matte. + let stub = Arc::new(Stub::default()); + assert_ne!(seed(&p, [100.0, 80.0], 25.0, Some(stub.as_ref())), seed(&p, [100.0, 80.0], 25.0, None)); + let img = frame_at(0); + let mut frame = |g: i64| Some(Arc::new(frame_at(g))); + let FrameMatte::Alpha(a) = frame_matte(&p, [100.0, 80.0], 25.0, 0, 1.0, &img, Some(stub.clone()), &mut frame, &|| false) else { panic!("no matte") }; + assert!(a[40 * 100 + 40] > 0.5 && a[5 * 100 + 5] < 0.5); + assert_eq!(stub.calls.load(std::sync::atomic::Ordering::Relaxed), 1); + assert_eq!(*stub.bbox.lock().unwrap(), None); + } + + #[test] + fn the_box_reaches_the_model_as_a_box_prompt() { + let p = params(&[ + ("objectPoint", Value::Vec2([90.0, 70.0])), + ("useBox", Value::Bool(true)), + ("boxTopLeft", Value::Vec2([100.0, 80.0])), + ("boxBottomRight", Value::Vec2([74.0, 54.0])), + ]); + // With a model: the point and a box stroke (corners ordered); without: the box's edges + // as a background stroke, as before. + let with = prompt_strokes(&p, [100.0, 80.0], 0, true).unwrap(); + assert_eq!(with.iter().map(|s| s.kind).collect::>(), [StrokeKind::Fg, StrokeKind::Box]); + assert_eq!(with[1].points, [[74.0, 54.0], [100.0, 80.0]]); + let without = prompt_strokes(&p, [100.0, 80.0], 0, false).unwrap(); + assert_eq!(without.iter().map(|s| s.kind).collect::>(), [StrokeKind::Fg, StrokeKind::Bg]); + // The model is called with the box itself (matte pixels at scale 1) and only one + // foreground point: no background points along the edges. + let stub = Arc::new(Stub::default()); + let img = frame_at(0); + let mut frame = |g: i64| Some(Arc::new(frame_at(g))); + let FrameMatte::Alpha(a) = frame_matte(&p, [100.0, 80.0], 25.0, 0, 1.0, &img, Some(stub.clone()), &mut frame, &|| false) else { panic!("no matte") }; + assert_eq!(*stub.bbox.lock().unwrap(), Some([74.0, 54.0, 100.0, 80.0])); + assert!(a[70 * 100 + 90] > 0.5 && a[40 * 100 + 40] < 0.5); + // At half resolution the box is scaled with the frame (another Input Key, so the + // full-resolution result above is not reused). + let mut p = p; + p.values.insert(INPUT_KEY.into(), Value::Str("half".into())); + let half = effectcraft_raster::resample(&img, 50, 40); + let mut frame = |g: i64| Some(Arc::new(effectcraft_raster::resample(&frame_at(g), 50, 40))); + let _ = frame_matte(&p, [100.0, 80.0], 25.0, 0, 0.5, &half, Some(stub.clone()), &mut frame, &|| false); + assert_eq!(*stub.bbox.lock().unwrap(), Some([37.0, 27.0, 50.0, 40.0])); + } + + #[test] + fn job_frames_go_outward_from_the_reference_frame_inside_the_layer() { + let p = params(&[ + ("objectTracking/referenceFrame", Value::Scalar(5.0)), + ("objectTracking/framesBefore", Value::Scalar(2.0)), + ("objectTracking/framesAfter", Value::Scalar(3.0)), + ]); + assert_eq!(job_frames(&p, [0, 100]), [5, 6, 7, 8, 4, 3]); + assert_eq!(job_frames(&p, [4, 6]), [5, 6, 4]); + assert!(job_frames(&p, [10, 20]).is_empty()); + let huge = params(&[("objectTracking/referenceFrame", Value::Scalar(1e300)), ("objectTracking/framesAfter", Value::Scalar(f64::INFINITY))]); + assert!(job_frames(&huge, [0, 100]).is_empty()); + } + + /// Frozen mattes are used as stored while their key matches (no segmentation: a stored + /// pattern the segmentation could never produce comes out), and ignored once it doesn't. + #[test] + fn frozen_mattes_are_used_while_their_key_matches() { + let mut p = params(&[("objectPoint", Value::Vec2([40.0, 40.0])), ("objectTracking/framesAfter", Value::Scalar(3.0))]); + p.values.insert(INPUT_KEY.into(), Value::Str("frozen-test".into())); + // Left half opaque on frame 2. + let alpha: Vec = (0..100 * 80).map(|i| ((i % 100) < 50) as u8 as f32).collect(); + let mut fc = FrozenCache { key: frozen_key(&p, [100.0, 80.0], 25.0, None), size: [100, 80], ..Default::default() }; + fc.frames.insert(2, roto::freeze_frame(&alpha, &[])); + p.values.insert(FROZEN.into(), Value::Str(fc.to_json())); + assert!(is_frozen(&p)); + let out = run(&p, 2); + assert!(out.data[40 * 100 + 10][3] > 0.99 && out.data[40 * 100 + 90][3] < 0.01, "the frozen matte is used"); + // At half resolution too. + assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 50, 40, None).map(|a| (a[20 * 50 + 5], a[20 * 50 + 45])), Some((1.0, 0.0))); + // A frame without a frozen matte is computed (the disk). + assert!(wrong(&run(&p, 1), 42.0) < 30); + // Another prompt: the frozen mattes no longer apply. + p.values.insert("objectPoint".into(), Value::Vec2([41.0, 40.0])); + assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 100, 80, None), None); + assert!(wrong(&run(&p, 2), 44.0) < 30); + // Damaged frozen data is ignored, not a crash. + p.values.insert(FROZEN.into(), Value::Str("{\"key\": 3, \"frames\": [".into())); + assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 100, 80, None), None); + let mut fc2 = fc.clone(); + fc2.size = [usize::MAX, 2]; + p.values.insert("objectPoint".into(), Value::Vec2([40.0, 40.0])); + p.values.insert(FROZEN.into(), Value::Str(fc2.to_json())); + assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 100, 80, None), None); + } +} diff --git a/crates/effects/src/roto.rs b/crates/effects/src/roto.rs index d514e80a..a6c80e74 100644 --- a/crates/effects/src/roto.rs +++ b/crates/effects/src/roto.rs @@ -7,7 +7,8 @@ //! out to the frame), so it lives in a process-wide content-keyed cache ([`cached`]) that the //! engine's propagation job fills at full resolution and rendering fills on demand (reading //! neighbouring frames with [`crate::EffectHost::self_at`]). Undo, redo and reopening a project -//! therefore never see stale mattes. **Freeze** stores the final mattes in the hidden **Frozen** +//! therefore never see stale mattes. Behind the memory cache an optional [`SegStore`] (the +//! persistent disk cache, set by the engine) keeps computed segmentations across restarts. **Freeze** stores the final mattes in the hidden **Frozen** //! parameter ([`FrozenCache`]); they are used while their key still matches. //! //! Rendering: binary segmentation → Roto Brush Matte (Shift Edge, Reduce Chatter, Feather, @@ -23,21 +24,21 @@ //! higher working resolution with more iterations. use std::collections::{BTreeMap, HashMap, VecDeque}; -use std::sync::{Arc, Mutex, OnceLock, RwLock}; +use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak}; use effectcraft_keyframe::Value; use effectcraft_project::ParamUi; use effectcraft_raster::Image; use effectcraft_segment::MaskModel; use effectcraft_track::roto::refine::{self, MatteParams}; -use effectcraft_track::roto::{self as rb, FrameSeg, RotoData, SegOpts, rle}; +use effectcraft_track::roto::{self as rb, RotoData, SegOpts, rle}; use rayon::prelude::*; use serde::{Deserialize, Serialize}; use crate::warp_stab::param; use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; -pub use effectcraft_track::roto::{Stroke, StrokeKind}; +pub use effectcraft_track::roto::{FrameSeg, Stroke, StrokeKind}; /// The effect's id. pub const ID: &str = "ec.matte.rotobrush"; @@ -195,6 +196,12 @@ pub fn model_for(params: &Params) -> Option> { if e(params, "version") == 0 { return None; } + installed_model() +} + +/// The trained model chosen in Settings ▸ Roto Brush, if one is installed (Roto Brush 2.0 / 3.0 +/// and Object Matte segment with it). +pub fn installed_model() -> Option> { model_slot().read().ok().and_then(|m| m.clone()) } @@ -260,6 +267,37 @@ pub fn store(key: u64, scale: f64, seg: Arc) { } } +/// A second-level store for segmentations, keyed like the memory cache (chain key, scale): the +/// persistent disk cache, so computed mattes survive the memory cache and restarts. +pub trait SegStore: Send + Sync { + fn get_seg(&self, key: u64, scale: f64) -> Option; + fn put_seg(&self, key: u64, scale: f64, seg: &FrameSeg); +} + +fn seg_store_slot() -> &'static RwLock>> { + static S: OnceLock>>> = OnceLock::new(); + S.get_or_init(|| RwLock::new(None)) +} + +/// Use `store` behind the memory cache (held weakly: once its owner drops it, e.g. the disk +/// cache is switched off, it is no longer used). `None` turns it off. +pub fn set_seg_store(store: Option>) { + if let Ok(mut s) = seg_store_slot().write() { + *s = store; + } +} + +fn seg_store() -> Option> { + seg_store_slot().read().ok()?.as_ref()?.upgrade() +} + +/// A segmentation from the second-level store, put into the memory cache. +fn from_store(key: u64, scale: f64) -> Option> { + let seg = Arc::new(seg_store()?.get_seg(key, scale)?); + store(key, scale, seg.clone()); + Some(seg) +} + /// Drop the cached segmentations of these chain keys (every scale). pub fn forget(keys: impl IntoIterator) { let keys: std::collections::HashSet = keys.into_iter().collect(); @@ -319,11 +357,11 @@ impl Chain { let sc = self.opts.scale; let lookup = |g: i64| -> Option> { let k = *self.keys.get(&g)?; - if let Some(s) = cached(k, sc) { + if let Some(s) = cached(k, sc).or_else(|| from_store(k, sc)) { return Some(s); } if sc != 1.0 { - let full = cached(k, 1.0)?; + let full = cached(k, 1.0).or_else(|| from_store(k, 1.0))?; let s = Arc::new(downscale(&full, sc)); store(k, sc, s.clone()); return Some(s); @@ -373,7 +411,11 @@ impl Chain { } }; let seg = Arc::new(seg); - store(*self.keys.get(&g)?, sc, seg.clone()); + let k = *self.keys.get(&g)?; + store(k, sc, seg.clone()); + if let Some(st) = seg_store() { + st.put_seg(k, sc, &seg); + } cur = Some(seg); prev_img = Some(img); } @@ -392,7 +434,7 @@ pub fn downscale(s: &FrameSeg, scale: f64) -> FrameSeg { resample(s, w, h) } -fn resample(s: &FrameSeg, w: usize, h: usize) -> FrameSeg { +pub(crate) fn resample(s: &FrameSeg, w: usize, h: usize) -> FrameSeg { if s.w == w && s.h == h { return s.clone(); } diff --git a/crates/engine/src/commands/query.rs b/crates/engine/src/commands/query.rs index c755bda9..2ea8048f 100644 --- a/crates/engine/src/commands/query.rs +++ b/crates/engine/src/commands/query.rs @@ -199,7 +199,7 @@ fn capabilities(s: &mut Session, _: &Value) -> Result { } /// The headline of `docs/target-app-parity.md` (kept in sync by hand at each re-measure). -const PARITY: &str = "≈91% feature breadth and ≈45% ready for real work (estimated parity with After Effects 2026 26.5, stage alpha); all After Effects effects present (306 registered)"; +const PARITY: &str = "≈91% feature breadth and ≈45% ready for real work (estimated parity with After Effects 2026 26.5, stage alpha); all After Effects effects present (307 registered)"; fn state(s: &mut Session, _: &Value) -> Result { Ok(serde_json::to_value(&s.state).unwrap_or_default()) diff --git a/crates/engine/src/commands/roto_cmds.rs b/crates/engine/src/commands/roto_cmds.rs index 1df62bcb..a6d389ca 100644 --- a/crates/engine/src/commands/roto_cmds.rs +++ b/crates/engine/src/commands/roto_cmds.rs @@ -1,6 +1,9 @@ //! Roto Brush & Refine Edge commands: strokes (the Roto Brush and Refine Edge tools in the Layer //! panel), propagation, the segmentation span, Freeze / Unfreeze, tool options and status. //! +//! `objectMatte.apply` adds an Object Matte effect (Effect ▸ Matte) from a point / box prompt on +//! a frame; it segments with the same engine and model as Roto Brush. +//! //! `roto.stroke` records a stroke into the layer's Roto Brush & Refine Edge effect (applying the //! effect on the first stroke, as After Effects does); the first foreground stroke sets the base //! frame and a span of 20 frames each side. Strokes on a frozen effect are refused (Unfreeze @@ -51,6 +54,32 @@ pub fn specs() -> Vec { always, options ), + cmd!( + "objectMatte.apply", + "Object Matte", + [], + None, + "{layer?, effect? (uid: re-prompt this Object Matte instead of adding one), point: [x, y] (layer pixels: the object), box?: [x0, y0, x1, y1] (layer pixels, around the object), frame? (reference layer frame; default current), track?: follow the object from that frame (default true), before?, after? (frames to follow it; default 30, up to 600)}", + has_comp, + object_matte_apply + ), + cmd!( + "objectMatte.propagate", + "Track Object Matte", + [], + None, + "{layer?, effect?, direction?: forward|backward|both, wait?: block until done} (a background job like Roto Brush propagation; roto.cancel stops it)", + has_comp, + object_matte_propagate + ), + cmd!("objectMatte.freeze", "Freeze Object Matte", [], None, "{layer?, effect?, wait?}", has_comp, object_matte_freeze), + cmd!("objectMatte.unfreeze", "Unfreeze Object Matte", [], None, "{layer?, effect?}", has_comp, object_matte_unfreeze), + query!( + "objectMatte.status", + "Object Matte Status", + "{layer?, effect?}: the job's progress, whether the instance is frozen, its frames and which are computed", + object_matte_status + ), query!( "roto.status", "Roto Brush Status", @@ -67,6 +96,16 @@ fn is_running(s: &Session) -> std::result::Result<(), String> { /// The Roto Brush instance targeted by `p`: `layer` / `effect` (uid, name or 1-based index), /// else a selected one, else the first on the selected (or any) layer. pub(crate) fn roto_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> { + instance_lookup(s, p, cmd, is_roto)?.ok_or_else(|| bad(cmd, "no Roto Brush & Refine Edge effect found (paint a stroke with the Roto Brush tool first)")) +} + +/// The Object Matte instance targeted by `p` (as [`roto_p`]). +fn object_matte_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> { + instance_lookup(s, p, cmd, rt::is_object_matte)?.ok_or_else(|| bad(cmd, "no Object Matte effect found (click an object with the Object Matte tool first)")) +} + +/// The instance of a kind (`is`) targeted by `p`; `None` when there is none. +fn instance_lookup(s: &Session, p: &Value, cmd: &str, is: fn(&PropGroup) -> bool) -> Result> { let cid = super::comp_id(s, p)?; let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; let candidates: Vec = match p.get("layer") { @@ -90,17 +129,17 @@ pub(crate) fn roto_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, Layer .state .selected_props .iter() - .find_map(|(l, u)| (*l == lid).then(|| fxg.groups().find(|g| g.uid == *u && is_roto(g))).flatten()) - .or_else(|| fxg.groups().find(|g| is_roto(g))), + .find_map(|(l, u)| (*l == lid).then(|| fxg.groups().find(|g| g.uid == *u && is(g))).flatten()) + .or_else(|| fxg.groups().find(|g| is(g))), }; - if let Some(g) = g.filter(|g| is_roto(g)) { - return Ok((cid, lid, g.uid)); + if let Some(g) = g.filter(|g| is(g)) { + return Ok(Some((cid, lid, g.uid))); } if p.get("layer").is_some() { break; } } - Err(bad(cmd, "no Roto Brush & Refine Edge effect found (paint a stroke with the Roto Brush tool first)")) + Ok(None) } fn group(s: &Session, cid: ItemId, lid: LayerId, uid: Uid) -> Result<(&Layer, &PropGroup)> { @@ -428,3 +467,190 @@ fn status(s: &mut Session, p: &Value) -> Result { out["computed"] = json!(computed.iter().filter(|(_, c)| **c).map(|(f, _)| *f).collect::>()); Ok(out) } + +/// Two finite numbers. +fn xy_p(p: &Value, k: &str, n: usize, cmd: &str) -> Result>> { + let Some(v) = p.get(k).filter(|v| !v.is_null()) else { return Ok(None) }; + let a = v.as_array().filter(|a| a.len() == n).ok_or_else(|| bad(cmd, format!("`{k}` is {n} numbers (layer pixels)")))?; + let out: Vec = a.iter().filter_map(Value::as_f64).filter(|x| x.is_finite()).collect(); + if out.len() != n { + return Err(bad(cmd, format!("`{k}` is {n} numbers (layer pixels)"))); + } + Ok(Some(out)) +} + +fn object_matte_apply(s: &mut Session, p: &Value) -> Result { + use effectcraft_effects::object_matte as om; + let cmd = "objectMatte.apply"; + let (cid, lid) = layer_p(s, p, cmd)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or_else(|| bad(cmd, "no layer"))?; + if !layer.has_video() || layer.switches.adjustment || layer.effects().is_none() { + return Err(bad(cmd, "Object Matte works on footage, solid and composition layers")); + } + // Re-prompt an existing instance (the Object Matte tool on a layer that has one). + let existing = match p.get("effect").filter(|v| !v.is_null()) { + Some(v) => { + let uid = v.as_u64().ok_or_else(|| bad(cmd, "`effect` is an Object Matte effect's uid"))?; + let g = layer + .effects() + .and_then(|f| f.groups().find(|g| g.uid == uid && rt::is_object_matte(g))) + .ok_or_else(|| bad(cmd, format!("no Object Matte effect {uid} on this layer")))?; + if frozen_of(g) { + return Err(bad(cmd, "the Object Matte is frozen: Unfreeze it to change the prompt")); + } + Some(uid) + } + None => None, + }; + let size = rt::layer_size(&s.project, comp, layer); + let point = xy_p(p, "point", 2, cmd)?.ok_or_else(|| bad(cmd, "missing `point` [x, y] (layer pixels)"))?; + let inside = |x: f64, y: f64| (0.0..=size[0]).contains(&x) && (0.0..=size[1]).contains(&y); + if !inside(point[0], point[1]) { + return Err(bad(cmd, format!("`point` is outside the layer ({}×{})", size[0], size[1]))); + } + let bx = xy_p(p, "box", 4, cmd)?; + if let Some(b) = &bx { + let (x0, x1, y0, y1) = (b[0].min(b[2]), b[0].max(b[2]), b[1].min(b[3]), b[1].max(b[3])); + if !(point[0] > x0 && point[0] < x1 && point[1] > y0 && point[1] < y1) { + return Err(bad(cmd, "`point` must be inside `box`")); + } + } + let frame = frame_p(s, p, cid, layer); + let limits = rt::frame_limits(comp, layer); + if frame < limits[0] || frame > limits[1] { + return Err(bad(cmd, format!("frame {frame} is outside the layer ({}–{})", limits[0], limits[1]))); + } + // A new instance gets the defaults; a re-prompted one keeps what is not given. + let track = b_p(p, "track").or(existing.is_none().then_some(true)); + let reach = |k: &str| { + let v = f_p(p, k).filter(|v| v.is_finite()); + let v = if existing.is_none() { Some(v.unwrap_or(om::DEFAULT_REACH)) } else { v }; + v.map(|v| v.round().clamp(0.0, om::MAX_REACH)) + }; + let (before, after) = (reach("before"), reach("after")); + let spec = effectcraft_effects::find(om::ID).ok_or_else(|| EngineError::Other("Object Matte effect missing".into()))?; + let label = if existing.is_some() { "Object Matte Prompt" } else { "Object Matte" }; + let uid = s.edit(label, None, |proj, _| { + let mut next = proj.next_id; + let l = layer_mut(proj, cid, lid)?; + let fxg = l.props.sub_mut("effects").ok_or_else(|| bad(cmd, "this layer has no effects"))?; + let mut fresh = None; + let g = match existing { + Some(u) => fxg.find_group_mut(u).ok_or_else(|| bad(cmd, "the Object Matte effect is gone"))?, + None => { + let n = fxg.groups().filter(|g| rt::is_object_matte(g)).count(); + let name = if n == 0 { spec.name.to_string() } else { format!("{} {}", spec.name, n + 1) }; + fresh.insert(effectcraft_effects::instantiate(spec, &mut Ids(&mut next), &name, size)) + } + }; + let mut set = |path: &str, v: KV| { + let (grp, leaf) = path.rsplit_once('/').map_or((None, path), |(a, b)| (Some(a), b)); + let target = match grp { + Some(sub) => g.sub_mut(sub), + None => Some(&mut *g), + }; + // The prompt is static: a re-prompt replaces any keyframes. + if let Some(pr) = target.and_then(|t| t.get_mut(leaf)) { + pr.value = v; + pr.keys.clear(); + } + }; + set("objectPoint", KV::Vec2([point[0], point[1]])); + set("useBox", KV::Bool(bx.is_some())); + if let Some(b) = &bx { + set("boxTopLeft", KV::Vec2([b[0].min(b[2]), b[1].min(b[3])])); + set("boxBottomRight", KV::Vec2([b[0].max(b[2]), b[1].max(b[3])])); + } + if let Some(t) = track { + set("objectTracking/trackObject", KV::Bool(t)); + } + set("objectTracking/referenceFrame", KV::Scalar(frame as f64)); + if let Some(v) = before { + set("objectTracking/framesBefore", KV::Scalar(v)); + } + if let Some(v) = after { + set("objectTracking/framesAfter", KV::Scalar(v)); + } + let u = g.uid; + if let Some(g) = fresh { + fxg.children.push(g.into()); + } + proj.next_id = proj.next_id.max(next); + Ok(u) + })?; + s.state.selected_layers = vec![lid]; + let (track, before, after) = object_matte_reach(s, cid, lid, uid); + Ok(json!({"layer": lid.0, "effect": uid, "frame": frame, "track": track, "updated": existing.is_some(), "span": [frame - before, frame + after]})) +} + +/// Track Object and Frames Before / After of an Object Matte instance (static values). +fn object_matte_reach(s: &Session, cid: ItemId, lid: LayerId, uid: Uid) -> (bool, i64, i64) { + use effectcraft_effects::object_matte as om; + let Ok((_, g)) = group(s, cid, lid, uid) else { return (true, 0, 0) }; + let ps = params_static(g); + let [before, after] = om::reach(&ps); + (om::tracks(&ps), before, after) +} + +fn object_matte_propagate(s: &mut Session, p: &Value) -> Result { + let cmd = "objectMatte.propagate"; + let (cid, lid, uid) = object_matte_p(s, p, cmd)?; + let dir = match str_p(p, "direction") { + Some(d) => Direction::from_name(d).ok_or_else(|| bad(cmd, format!("unknown direction `{d}` (forward | backward | both)")))?, + None => Direction::Both, + }; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_roto(cid, lid, uid, RotoTask::Propagate, dir, wait).map_err(|e| bad(cmd, e))?; + let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_roto_running(), "progress": s.roto_progress()}); + if wait { + out["status"] = object_matte_status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?; + } + Ok(out) +} + +fn object_matte_freeze(s: &mut Session, p: &Value) -> Result { + let cmd = "objectMatte.freeze"; + let (cid, lid, uid) = object_matte_p(s, p, cmd)?; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_roto(cid, lid, uid, RotoTask::Freeze, Direction::Both, wait).map_err(|e| bad(cmd, e))?; + let (_, g) = group(s, cid, lid, uid)?; + Ok(json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_roto_running(), "frozen": frozen_of(g), "progress": s.roto_progress()})) +} + +fn object_matte_unfreeze(s: &mut Session, p: &Value) -> Result { + let cmd = "objectMatte.unfreeze"; + let (cid, lid, uid) = object_matte_p(s, p, cmd)?; + let (_, g) = group(s, cid, lid, uid)?; + if frozen_of(g) { + s.edit("Unfreeze", None, |proj, _| { + let g = layer_mut(proj, cid, lid)?.props.find_group_mut(uid).ok_or_else(|| EngineError::Other("the effect is gone".into()))?; + if let Some(pr) = g.get_mut(FROZEN) { + pr.value = KV::Str(String::new()); + } + Ok(()) + })?; + } + Ok(json!({"layer": lid.0, "effect": uid, "frozen": false})) +} + +fn object_matte_status(s: &mut Session, p: &Value) -> Result { + s.poll_roto(false); + let running = s.is_roto_running(); + let mut out = json!({"running": running, "progress": s.roto_progress(), "banner": s.roto_progress().map(|p| p.banner())}); + let Ok((cid, lid, uid)) = object_matte_p(s, p, "objectMatte.status") else { return Ok(out) }; + let (_, g) = group(s, cid, lid, uid)?; + let frozen = frozen_of(g); + let (track, before, after) = object_matte_reach(s, cid, lid, uid); + let computed = s.roto_computed(cid, lid, uid); + out["layer"] = json!(lid.0); + out["effect"] = json!(uid); + out["frozen"] = json!(frozen); + out["track"] = json!(track); + out["reach"] = json!([before, after]); + out["mine"] = json!(running && s.roto_target().is_some_and(|(c, l, u, _)| (c, l, u) == (cid, lid, uid))); + out["pending"] = json!(s.roto_pending.contains(&(cid, lid, uid))); + out["frames"] = json!(computed.keys().collect::>()); + out["computed"] = json!(computed.iter().filter(|(_, c)| **c).map(|(f, _)| *f).collect::>()); + Ok(out) +} diff --git a/crates/engine/src/offload.rs b/crates/engine/src/offload.rs index 23fa2b38..561ebe6c 100644 --- a/crates/engine/src/offload.rs +++ b/crates/engine/src/offload.rs @@ -561,7 +561,8 @@ async fn fill(s: &mut Session, plan: crate::commands::content_fill::FillPlan, po /// Roto Brush propagation in a worker: the segmentations stream back (at most every 250 ms) /// as they are computed, so stopping the job keeps what was propagated. async fn propagate(s: &mut Session, comp: ItemId, layer: LayerId, effect: Uid, dir: crate::roto::Direction, post: Post) -> Result<(), String> { - let keys: Vec = s.roto_chain(comp, layer, effect).map(|c| c.keys.values().copied().collect()).ok_or("no Roto Brush strokes")?; + let keys: Vec = + s.roto_keys(comp, layer, effect).map(|k| k.into_iter().map(|(_, k)| k).collect()).ok_or("no Roto Brush strokes or Object Matte prompt")?; let sent = Rc::new(RefCell::new(std::collections::HashSet::new())); let (p, k2, s2) = (post.clone(), keys.clone(), sent.clone()); let mut last = None::; diff --git a/crates/engine/src/roto.rs b/crates/engine/src/roto.rs index ca92a74b..49f53d39 100644 --- a/crates/engine/src/roto.rs +++ b/crates/engine/src/roto.rs @@ -10,12 +10,18 @@ //! edit, keeps each instance's hidden Input Key equal to a signature of what its frames are made //! of (the layer's source and timing, its masks and the effects above it) and drops frozen //! mattes made from other frames. +//! +//! Object Matte instances run through the same jobs (`objectMatte.propagate` / `.freeze`): the +//! frames are [`om::job_frames`] around the reference frame, each segmented through +//! [`om::ensure`]; Freeze stores [`om::frame_matte`]'s mattes under [`om::frozen_key`]. An edit +//! to an instance's frames or prompt queues it for background tracking like a Roto Brush. use crate::offload::JobKind; use std::collections::{BTreeMap, HashMap}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; +use effectcraft_effects::object_matte as om; use effectcraft_effects::roto::{self as fx, FROZEN, FrozenCache, INPUT_KEY}; use effectcraft_effects::{Params, flatten_params}; use effectcraft_keyframe::Value; @@ -104,14 +110,18 @@ pub struct RotoProgress { pub finished: bool, pub cancelled: bool, pub error: Option, + /// The job works on an Object Matte (else a Roto Brush & Refine Edge). + pub object_matte: bool, } impl RotoProgress { pub fn banner(&self) -> String { let pc = (100 * self.done).checked_div(self.total).unwrap_or(0); - match self.task { - Some(RotoTask::Freeze) => format!("Freezing Roto Brush & Refine Edge: {pc}%"), - _ => format!("Propagating Roto Brush: {pc}%"), + match (self.task, self.object_matte) { + (Some(RotoTask::Freeze), false) => format!("Freezing Roto Brush & Refine Edge: {pc}%"), + (Some(RotoTask::Freeze), true) => format!("Freezing Object Matte: {pc}%"), + (_, false) => format!("Propagating Roto Brush: {pc}%"), + (_, true) => format!("Tracking Object Matte: {pc}%"), } } } @@ -130,8 +140,11 @@ pub struct RotoJob { pub layer: LayerId, pub effect: Uid, pub task: RotoTask, - /// The strokes the job was started with (a Freeze result is dropped when they changed). + /// What the job was started from: the strokes (Roto Brush) or the prompt (Object Matte); a + /// Freeze result is dropped when it changed. strokes: String, + /// The job works on an Object Matte. + pub object_matte: bool, } impl RotoJob { @@ -147,6 +160,26 @@ pub(crate) fn is_roto(g: &PropGroup) -> bool { matches!(&g.kind, GroupKind::Effect { effect } if effect == fx::ID) } +/// An Object Matte instance (its Input Key is kept current like Roto Brush's; it segments on +/// demand while rendering). +pub(crate) fn is_object_matte(g: &PropGroup) -> bool { + matches!(&g.kind, GroupKind::Effect { effect } if effect == effectcraft_effects::object_matte::ID) +} + +/// What an Object Matte's segmentation is made from besides its frames: the prompt and tracking +/// parameters (static values), for queueing it after edits and guarding a Freeze. +pub(crate) fn om_prompt(g: &PropGroup) -> String { + let ps = params_static(g); + let mut keys: Vec<&String> = ps.values.keys().filter(|k| ["objectPoint", "useBox", "box", "objectTracking/"].iter().any(|p| k.starts_with(p))).collect(); + keys.sort(); + keys.iter().map(|k| format!("{k}={:?};", ps.values.get(*k))).collect() +} + +/// The job-relevant guard of an instance: its strokes (Roto Brush) or prompt (Object Matte). +fn guard_of(g: &PropGroup) -> String { + if is_object_matte(g) { om_prompt(g) } else { str_of(g, fx::STROKES) } +} + fn str_of(g: &PropGroup, m: &str) -> String { match g.get(m).map(|p| &p.value) { Some(Value::Str(s)) => s.clone(), @@ -188,8 +221,9 @@ pub fn input_key(p: &Project, comp: ItemId, layer: &Layer, uid: Uid) -> String { format!("{:016x}", fnv(&s)) } -/// After an edit: refresh Input Keys; frozen mattes made from other frames are dropped. -/// Returns the instances whose segmentation changed (to propagate again). +/// After an edit: refresh Input Keys (Roto Brush and Object Matte); frozen mattes made from +/// other frames are dropped. Returns the instances whose segmentation changed (to propagate +/// again): new frames, strokes (Roto Brush) or prompt (Object Matte). pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> { let mut todo = vec![]; let mut set = vec![]; @@ -197,13 +231,13 @@ pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerI let ItemKind::Comp(c) = &it.kind else { continue }; for l in &c.layers { let Some(fxg) = l.effects() else { continue }; - for g in fxg.groups().filter(|g| is_roto(g)) { + for g in fxg.groups().filter(|g| is_roto(g) || is_object_matte(g)) { let key = input_key(after, *cid, l, g.uid); let old = before.comp(*cid).and_then(|bc| bc.layer(l.id)).and_then(|ol| ol.props.find_group(g.uid)); if str_of(g, INPUT_KEY) != key { set.push((*cid, l.id, g.uid, key)); todo.push((*cid, l.id, g.uid)); - } else if old.is_none_or(|o| str_of(o, fx::STROKES) != str_of(g, fx::STROKES)) { + } else if old.is_none_or(|o| guard_of(o) != guard_of(g)) { todo.push((*cid, l.id, g.uid)); } } @@ -219,7 +253,11 @@ pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerI } } todo.retain(|(c, l, u)| { - after.comp(*c).and_then(|c| c.layer(*l)).and_then(|l| l.props.find_group(*u)).is_some_and(|g| !fx::data(¶ms_static(g)).is_empty()) + after + .comp(*c) + .and_then(|c| c.layer(*l)) + .and_then(|l| l.props.find_group(*u)) + .is_some_and(|g| is_object_matte(g) || !fx::data(¶ms_static(g)).is_empty()) }); todo } @@ -274,6 +312,7 @@ struct Work { fps: f64, frames: Vec, task: RotoTask, + object_matte: bool, } /// Renders the effect's input frames for a worker, keeping the last few. With a browser job @@ -390,10 +429,13 @@ async fn run_work(w: Work, shared: &RotoShared) { let mut s = lock(&shared.state); s.task = Some(w.task); s.total = w.frames.len() as u64; + s.object_matte = w.object_matte; } let cancel = || shared.cancel.load(Ordering::Relaxed); let chain = fx::Chain::new(&w.params, w.size, w.fps, 1.0); - let mut frozen = FrozenCache { key: fx::frozen_key(&w.params, w.size, w.fps), ..Default::default() }; + let model = fx::installed_model(); + let key = if w.object_matte { om::frozen_key(&w.params, w.size, w.fps, model.as_deref()) } else { fx::frozen_key(&w.params, w.size, w.fps) }; + let mut frozen = FrozenCache { key, ..Default::default() }; for (k, f) in w.frames.iter().enumerate() { if cancel() { let mut s = lock(&shared.state); @@ -404,9 +446,21 @@ async fn run_work(w: Work, shared: &RotoShared) { // The frame and its neighbours (what a step reads), on the GPU. src.prefetch(&[*f, f - 1, f + 1]).await; let mut get = |g: i64| src.get(g); - let ok = match w.task { - RotoTask::Propagate => chain.ensure(*f, &mut get, &cancel).is_some(), - RotoTask::Freeze => match fx::frame_matte(&w.params, w.size, w.fps, *f, &mut get, &cancel) { + let ok = match (w.task, w.object_matte) { + (RotoTask::Propagate, true) => om::ensure(&w.params, w.size, w.fps, *f, model.clone(), &mut get, &cancel).is_some(), + (RotoTask::Freeze, true) => { + let img = get(*f); + match img.map(|img| (om::frame_matte(&w.params, w.size, w.fps, *f, 1.0, &img, model.clone(), &mut get, &cancel), img)) { + Some((om::FrameMatte::Alpha(a), img)) => { + frozen.size = [img.width as usize, img.height as usize]; + frozen.frames.insert(*f, fx::freeze_frame(&a, &[])); + true + } + _ => false, + } + } + (RotoTask::Propagate, false) => chain.ensure(*f, &mut get, &cancel).is_some(), + (RotoTask::Freeze, false) => match fx::frame_matte(&w.params, w.size, w.fps, *f, &mut get, &cancel) { Some((a, b)) => { if let Some(img) = src.get(*f) { frozen.size = [img.width as usize, img.height as usize]; @@ -461,12 +515,15 @@ impl Session { self.roto_job.as_ref().map(|j| j.progress()).or_else(|| { self.offloaded_roto().map(|(j, task)| { let p = &j.progress; + let object_matte = + self.project.comp(j.comp).and_then(|c| c.layer(j.layer)).and_then(|l| l.props.find_group(j.uid)).is_some_and(is_object_matte); RotoProgress { task: Some(task), done: p.done, total: p.total, elapsed: p.elapsed, fps: crate::offload::rate(p.done, p.elapsed), + object_matte, ..Default::default() } }) @@ -478,7 +535,8 @@ impl Session { self.roto_job.as_ref().map(|j| (j.comp, j.layer, j.effect, j.task)).or_else(|| self.offloaded_roto().map(|(j, task)| (j.comp, j.layer, j.uid, task))) } - /// The segmentation chain a propagation of `effect` computes (its frames' cache keys). + /// The segmentation chain a propagation of Roto Brush `effect` computes (its frames' cache + /// keys). pub fn roto_chain(&self, comp: ItemId, layer: LayerId, effect: Uid) -> Option { let c = self.project.comp(comp)?; let l = c.layer(layer)?; @@ -489,6 +547,27 @@ impl Session { Some(fx::Chain::new(¶ms, layer_size(&self.project, c, l), c.frame_rate.as_f64(), 1.0)) } + /// The cache keys of the segmentations a propagation of `effect` (Roto Brush or Object + /// Matte) computes, by layer frame. + pub fn roto_keys(&self, comp: ItemId, layer: LayerId, effect: Uid) -> Option> { + if let Some(ch) = self.roto_chain(comp, layer, effect) { + return Some(ch.keys.into_iter().collect()); + } + let c = self.project.comp(comp)?; + let l = c.layer(layer)?; + let g = l.effects()?.groups().find(|g| g.uid == effect && is_object_matte(g))?; + let params = self.om_params(comp, l, g)?; + let frames = om::job_frames(¶ms, frame_limits(c, l)); + Some(om::frame_keys(¶ms, layer_size(&self.project, c, l), c.frame_rate.as_f64(), &frames, fx::installed_model())) + } + + /// An Object Matte's parameters at its reference frame (what its jobs segment with). + fn om_params(&self, comp: ItemId, l: &Layer, g: &PropGroup) -> Option { + let c = self.project.comp(comp)?; + let t = comp_time_of(c, l, om::reference_frame(¶ms_static(g))); + Some(params_at(&self.project, comp, l, g, t, self.expr.as_deref())) + } + pub fn stop_roto(&mut self) -> bool { if self.cancel_offloaded(JobKind::RotoFreeze) || self.cancel_offloaded(JobKind::RotoPropagate) { return true; @@ -522,17 +601,36 @@ impl Session { let l = c.layer(layer).ok_or("no layer")?; let fxg = l.effects().ok_or("the layer has no effects")?; let index = fxg.groups().position(|g| g.uid == effect).ok_or("no such effect")?; - let g = fxg.groups().nth(index).filter(|g| is_roto(g)).ok_or("not a Roto Brush & Refine Edge effect")?; + let g = fxg.groups().nth(index).filter(|g| is_roto(g) || is_object_matte(g)).ok_or("not a Roto Brush & Refine Edge or Object Matte effect")?; + let object_matte = is_object_matte(g); let size = layer_size(&self.project, c, l); let fps = c.frame_rate.as_f64(); - let pstat = params_static(g); - let d = fx::data(&pstat); - if d.is_empty() { - return Err("draw a foreground stroke with the Roto Brush tool first".into()); - } - let t = comp_time_of(c, l, d.base.unwrap_or(0)); - let params = params_at(&self.project, comp, l, g, t, self.expr.as_deref()); - let frames = job_frames(&d, if task == RotoTask::Freeze { Direction::Both } else { dir }); + let (params, frames) = if object_matte { + let params = self.om_params(comp, l, g).ok_or("no composition")?; + let base = om::reference_frame(¶ms); + if om::roto_data(¶ms, size, base, fx::installed_model().is_some()).is_none() { + return Err("set the object first: click it with the Object Matte tool (its point is off the layer or outside its box)".into()); + } + let all = om::job_frames(¶ms, frame_limits(c, l)); + if all.is_empty() { + return Err(format!("the reference frame {base} is outside the layer")); + } + // Forward / backward from the reference frame, as for a Roto Brush. + let frames = all + .into_iter() + .filter(|f| task == RotoTask::Freeze || dir == Direction::Both || *f == base || (*f > base) == (dir == Direction::Forward)) + .collect(); + (params, frames) + } else { + let pstat = params_static(g); + let d = fx::data(&pstat); + if d.is_empty() { + return Err("draw a foreground stroke with the Roto Brush tool first".into()); + } + let t = comp_time_of(c, l, d.base.unwrap_or(0)); + let params = params_at(&self.project, comp, l, g, t, self.expr.as_deref()); + (params, job_frames(&d, if task == RotoTask::Freeze { Direction::Both } else { dir })) + }; let n = frames.len(); let work = Work { project: self.project.clone(), @@ -547,8 +645,9 @@ impl Session { fps, frames, task, + object_matte, }; - let strokes = str_of(g, fx::STROKES); + let strokes = guard_of(g); if !wait && self.offloads() { drop(work); self.offload_analysis(match task { @@ -559,6 +658,13 @@ impl Session { } let wait = wait || cfg!(target_arch = "wasm32"); let shared = Arc::new(RotoShared::default()); + { + // Known before the worker starts (the progress shows at once). + let mut st = lock(&shared.state); + st.task = Some(task); + st.total = n as u64; + st.object_matte = object_matte; + } let thread = if wait { crate::offload::run_or_queue({ let sh = shared.clone(); @@ -574,7 +680,7 @@ impl Session { .map_err(|e| e.to_string())?, ) }; - self.roto_job = Some(RotoJob { shared, thread, comp, layer, effect, task, strokes }); + self.roto_job = Some(RotoJob { shared, thread, comp, layer, effect, task, strokes, object_matte }); self.events.push(Event::ProjectChanged { revision: self.revision }); self.poll_roto_job(); Ok(n) @@ -591,40 +697,43 @@ impl Session { let _ = t.join(); } let st = job.progress(); + let name = if job.object_matte { "Object Matte" } else { "Roto Brush" }; if let Some(e) = &st.error { - self.events.push(Event::Toast { message: format!("Roto Brush: {e}"), error: true }); + self.events.push(Event::Toast { message: format!("{name}: {e}"), error: true }); self.bump(); return true; } if st.cancelled { - self.events.push(Event::Toast { message: "Roto Brush: stopped".into(), error: false }); + self.events.push(Event::Toast { message: format!("{name}: stopped"), error: false }); self.bump(); return true; } match job.task { RotoTask::Propagate => { - self.events.push(Event::Toast { message: format!("Roto Brush: propagated {} frame(s) in {:.1} s", st.done, st.elapsed), error: false }); + let verb = if job.object_matte { "tracked" } else { "propagated" }; + self.events.push(Event::Toast { message: format!("{name}: {verb} {} frame(s) in {:.1} s", st.done, st.elapsed), error: false }); // Rendered frames depend on the cache: redraw. self.bump(); } RotoTask::Freeze => { let Some(fc) = lock(&job.shared.frozen).take() else { return true }; let (cid, lid, uid, strokes) = (job.comp, job.layer, job.effect, job.strokes.clone()); + let what = if job.object_matte { "prompt" } else { "strokes" }; let r = self.edit("Freeze", None, |p, _| { let g = p .comp_mut(cid) .and_then(|c| c.layer_mut(lid)) .and_then(|l| l.props.find_group_mut(uid)) - .ok_or_else(|| crate::EngineError::Other("the Roto Brush effect was removed".into()))?; - if str_of(g, fx::STROKES) != strokes { - return Err(crate::EngineError::Other("the strokes changed while freezing".into())); + .ok_or_else(|| crate::EngineError::Other(format!("the {name} effect was removed")))?; + if guard_of(g) != strokes { + return Err(crate::EngineError::Other(format!("the {what} changed while freezing"))); } set_str(g, FROZEN, fc.to_json()); Ok(()) }); let msg = match r { - Ok(()) => format!("Roto Brush: froze {} frame(s) in {:.1} s", fc.frames.len(), st.elapsed), - Err(e) => format!("Roto Brush: {e}"), + Ok(()) => format!("{name}: froze {} frame(s) in {:.1} s", fc.frames.len(), st.elapsed), + Err(e) => format!("{name}: {e}"), }; self.events.push(Event::Toast { message: msg, error: false }); } @@ -654,6 +763,12 @@ impl Session { let Some(c) = self.project.comp(comp) else { return out }; let Some(l) = c.layer(layer) else { return out }; let Some(g) = l.props.find_group(effect) else { return out }; + if is_object_matte(g) { + for (f, k) in self.roto_keys(comp, layer, effect).unwrap_or_default() { + out.insert(f, fx::cached(k, 1.0).is_some()); + } + return out; + } let ps = params_static(g); let ch = fx::Chain::new(&ps, layer_size(&self.project, c, l), c.frame_rate.as_f64(), 1.0); for f in ch.keys.keys() { diff --git a/crates/engine/src/session_settings.rs b/crates/engine/src/session_settings.rs index 34f24786..55c8e415 100644 --- a/crates/engine/src/session_settings.rs +++ b/crates/engine/src/session_settings.rs @@ -132,6 +132,10 @@ impl Session { }, }; self.layer_cache.set_disk(Some((dc.clone(), footage_salt(&self.project)))); + // Roto Brush / Object Matte segmentations persist there too (held weakly: switching the + // disk cache off drops it). + let store: std::sync::Arc = dc.clone(); + effectcraft_effects::roto::set_seg_store(Some(std::sync::Arc::downgrade(&store))); self.disk_cache = Some(dc); } diff --git a/crates/engine/src/tests_roto.rs b/crates/engine/src/tests_roto.rs index bef0936f..33500b4e 100644 --- a/crates/engine/src/tests_roto.rs +++ b/crates/engine/src/tests_roto.rs @@ -380,3 +380,264 @@ fn offloaded_propagation_streams_segmentations_back() { s.execute("roto.propagate", json!({"layer": clip.0, "wait": true})).unwrap(); assert!(s.offloaded.is_empty()); } + +/// Object Matte: one click on the disk on frame 0 isolates it there and on the frames it is +/// followed to; the Input Key follows the layer's frames; bad prompts are refused. +#[test] +fn object_matte_from_a_click() { + hold_roto_cache_test_lock(); + let (mut s, clip, solid) = setup(|f| draw(center(f))); + let cid = s.active_comp_id().unwrap(); + s.edit("remove solid", None, |p, _| { + p.comp_mut(cid).unwrap().layers.retain(|l| l.id != solid); + for it in p.items.values_mut() { + if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind { + f.path = "object-matte.mov".into(); + } + } + Ok(()) + }) + .unwrap(); + s.execute("time.set", json!({"time": 0})).unwrap(); + let c = center(0); + let r = s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "point": c, "after": 6})).unwrap(); + assert_eq!(r["frame"], 0); + assert_eq!(r["span"], json!([-30, 6])); + let uid = r["effect"].as_u64().unwrap(); + let key = |s: &Session| { + let g = s.active_comp().unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().clone(); + crate::roto::params_static(&g).get(effectcraft_effects::roto::INPUT_KEY).cloned() + }; + let k0 = key(&s); + assert!(matches!(&k0, Some(effectcraft_keyframe::Value::Str(k)) if !k.is_empty()), "{k0:?}"); + let wrong = |s: &Session, f: u32| { + let c = center(f); + alpha_at(s, f) + .iter() + .enumerate() + .filter(|(i, v)| { + let d = (((*i as u32 % W) as f64 + 0.5 - c[0]).hypot((*i as u32 / W) as f64 + 0.5 - c[1])) - R; + (d < -3.0 && **v < 0.5) || (d > 3.0 && **v > 0.5) + }) + .count() + }; + let w0 = wrong(&s, 0); + assert!(w0 < 40, "{w0} wrong pixels on the reference frame"); + let w4 = wrong(&s, 4); + assert!(w4 < 60, "{w4} wrong pixels on frame 4"); + // Past Frames After: transparent. + assert!(alpha_at(&s, 8).iter().all(|v| *v == 0.0)); + // Other frames under the effect: a new Input Key (the mattes are recomputed). + s.edit("swap clip", None, |p, _| { + for it in p.items.values_mut() { + if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind { + f.path = "object-matte-2.mov".into(); + } + } + Ok(()) + }) + .unwrap(); + assert_ne!(key(&s), k0); + // Bad prompts are refused. + for bad in [ + json!({"layer": clip.0}), + json!({"layer": clip.0, "point": [-10, 5]}), + json!({"layer": clip.0, "point": [10, "x"]}), + json!({"layer": clip.0, "point": c, "box": [0, 0, 10, 10]}), + json!({"layer": clip.0, "point": c, "frame": 999}), + ] { + assert!(s.execute_checked("objectMatte.apply", bad.clone()).is_err(), "{bad}"); + } + // A box and a static prompt are accepted; the second instance is numbered. + let r = s + .execute_checked("objectMatte.apply", json!({"layer": clip.0, "point": c, "box": [c[0] + 50.0, c[1] + 50.0, c[0] - 50.0, c[1] - 50.0], "track": false})) + .unwrap(); + let g = s.active_comp().unwrap().layer(clip).unwrap().props.find_group(r["effect"].as_u64().unwrap()).unwrap().clone(); + assert_eq!(g.name, "Object Matte 2"); +} + +/// `objectMatte.apply` with `effect` re-prompts that instance in one undo step: the new point and +/// box, the box dropped when none is given, the reach kept unless given. +#[test] +fn object_matte_reprompt_updates_the_instance() { + let (mut s, clip, solid) = setup(|f| draw(center(f))); + let cid = s.active_comp_id().unwrap(); + s.edit("remove solid", None, |p, _| { + p.comp_mut(cid).unwrap().layers.retain(|l| l.id != solid); + Ok(()) + }) + .unwrap(); + let c = center(0); + let r = s + .execute_checked( + "objectMatte.apply", + json!({"layer": clip.0, "point": c, "box": [c[0] - 50.0, c[1] - 50.0, c[0] + 50.0, c[1] + 50.0], "after": 6, "frame": 0}), + ) + .unwrap(); + let uid = r["effect"].as_u64().unwrap(); + let count = |s: &Session| s.active_comp().unwrap().layer(clip).unwrap().effects().unwrap().groups().count(); + let ps = |s: &Session| crate::roto::params_static(s.active_comp().unwrap().layer(clip).unwrap().props.find_group(uid).unwrap()); + let undo0 = s.history.undo.len(); + let r = s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "effect": uid, "point": [c[0] + 4.0, c[1]], "frame": 2})).unwrap(); + assert_eq!((r["effect"].as_u64(), r["updated"].as_bool()), (Some(uid), Some(true))); + assert_eq!(r["span"], json!([-28, 8])); + assert_eq!(count(&s), 1); + assert_eq!(s.history.undo.len(), undo0 + 1); + let p = ps(&s); + assert_eq!(p.get("objectPoint"), Some(&effectcraft_keyframe::Value::Vec2([c[0] + 4.0, c[1]]))); + assert_eq!(p.get("useBox"), Some(&effectcraft_keyframe::Value::Bool(false))); + assert_eq!(effectcraft_effects::object_matte::reference_frame(&p), 2); + assert_eq!(effectcraft_effects::object_matte::reach(&p), [30, 6]); + // Undo restores the first prompt. + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(ps(&s).get("useBox"), Some(&effectcraft_keyframe::Value::Bool(true))); + // An unknown or non-Object Matte effect is refused. + assert!(s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "effect": 999_999, "point": c})).is_err()); +} + +/// Object Matte runs in the background like Roto Brush propagation: queued by the edit, a job +/// that returns at once and reports progress, stops when cancelled (keeping what it computed), +/// computes the tracked frames when left to run, and Freeze stores mattes that renders then use +/// without segmenting again. +#[test] +fn object_matte_background_job_cancel_and_freeze() { + hold_roto_cache_test_lock(); + let (mut s, clip, solid) = setup(|f| draw(center(f))); + let cid = s.active_comp_id().unwrap(); + s.edit("remove solid", None, |p, _| { + p.comp_mut(cid).unwrap().layers.retain(|l| l.id != solid); + for it in p.items.values_mut() { + if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind { + f.path = "object-matte-job.mov".into(); + } + } + Ok(()) + }) + .unwrap(); + let c = center(0); + let r = s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "point": c, "before": 0, "after": 24, "frame": 0})).unwrap(); + let uid = r["effect"].as_u64().unwrap(); + assert!(s.roto_pending.contains(&(cid, clip, uid)), "the new prompt is queued for tracking"); + let wait_done = |s: &mut Session| { + let t = std::time::Instant::now(); + while s.is_roto_running() { + assert!(t.elapsed().as_secs() < 120, "the job never finished"); + std::thread::sleep(std::time::Duration::from_millis(5)); + s.poll_roto(false); + } + s.poll_roto(false); + }; + let toast = |s: &Session, m: &str| s.events.iter().any(|e| matches!(e, crate::Event::Toast { message, .. } if message.starts_with(m))); + let status = |s: &mut Session| s.execute("objectMatte.status", json!({"layer": clip.0, "effect": uid})).unwrap(); + + // Cancelled at once: it stops early, and says so. + let r = s.execute_checked("objectMatte.propagate", json!({"layer": clip.0, "effect": uid})).unwrap(); + assert_eq!(r["frames"], 25); + assert!(!s.roto_pending.contains(&(cid, clip, uid))); + let _ = s.execute("roto.cancel", json!({})); + wait_done(&mut s); + assert!(toast(&s, "Object Matte: stopped"), "{:?}", s.events); + let st = status(&mut s); + assert_eq!(st["frames"].as_array().unwrap().len(), 25, "{st}"); + let early = st["computed"].as_array().unwrap().len(); + assert!(early < 25, "{st}"); + + // Left to run: the call returns while the job works (progress visible), then every tracked + // frame is computed. + let r = s.execute_checked("objectMatte.propagate", json!({"layer": clip.0, "effect": uid})).unwrap(); + assert_eq!(r["running"], true, "{r}"); + let pr = s.roto_progress().unwrap(); + assert!(pr.object_matte && pr.banner().starts_with("Tracking Object Matte"), "{pr:?}"); + // One job at a time. + assert!(s.execute_checked("objectMatte.propagate", json!({"layer": clip.0, "effect": uid})).is_err()); + wait_done(&mut s); + assert!(toast(&s, "Object Matte: tracked"), "{:?}", s.events); + assert_eq!(status(&mut s)["computed"].as_array().unwrap().len(), 25); + let before = alpha_at(&s, 4); + + // Freeze, then drop the segmentations: renders use the frozen mattes and compute nothing. + let r = s.execute_checked("objectMatte.freeze", json!({"layer": clip.0, "effect": uid, "wait": true})).unwrap(); + assert_eq!(r["frozen"], true, "{r}"); + let keys: Vec = s.roto_keys(cid, clip, uid).unwrap().into_iter().map(|(_, k)| k).collect(); + assert_eq!(keys.len(), 25); + effectcraft_effects::roto::forget(keys.iter().copied()); + let frozen = alpha_at(&s, 4); + let diff = before.iter().zip(&frozen).filter(|(a, b)| (**a - **b).abs() > 0.02).count(); + assert!(diff < 30, "{diff} pixels differ from the computed matte"); + assert!(status(&mut s)["computed"].as_array().unwrap().is_empty(), "a frozen frame was segmented again"); + assert_eq!(status(&mut s)["frozen"], true); + // A frozen prompt is refused; Unfreeze computes again. + assert!(s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "effect": uid, "point": c})).is_err()); + s.execute_checked("objectMatte.unfreeze", json!({"layer": clip.0, "effect": uid})).unwrap(); + let _ = alpha_at(&s, 4); + assert!(status(&mut s)["computed"].as_array().unwrap().iter().any(|f| f == 4)); + // Undo brings the frozen mattes back. + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(status(&mut s)["frozen"], true); +} + +/// Computed Object Matte segmentations go to the persistent disk cache: a second session on the +/// same folder and project gets them back without segmenting (its frames cannot even be read +/// here), and another prompt (other keys) finds nothing. +#[test] +fn object_matte_mattes_persist_in_the_disk_cache() { + use effectcraft_effects::object_matte as om; + use effectcraft_render::disk_cache::Kind; + hold_roto_cache_test_lock(); + let dir = std::env::temp_dir().join(format!("effectcraft-om-disk-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let enable = |s: &mut Session| { + s.execute_checked("prefs.set", json!({"values": {"disk.diskCacheFolder": dir.to_string_lossy(), "disk.diskCacheEnabled": true}})).unwrap(); + }; + let (mut s, clip, solid) = setup(|f| draw(center(f))); + enable(&mut s); + let cid = s.active_comp_id().unwrap(); + s.edit("remove solid", None, |p, _| { + p.comp_mut(cid).unwrap().layers.retain(|l| l.id != solid); + for it in p.items.values_mut() { + if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind { + f.path = "object-matte-disk.mov".into(); + } + } + Ok(()) + }) + .unwrap(); + let c = center(0); + let r = s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "point": c, "before": 0, "after": 5, "frame": 0})).unwrap(); + let uid = r["effect"].as_u64().unwrap(); + s.execute_checked("objectMatte.propagate", json!({"layer": clip.0, "effect": uid, "wait": true})).unwrap(); + let keys = s.roto_keys(cid, clip, uid).unwrap(); + assert_eq!(keys.len(), 6); + let dc = s.disk_cache.clone().unwrap(); + dc.flush(); + assert!(dc.keys(Kind::Matte).len() >= 6, "{:?}", dc.stats()); + let before = alpha_at(&s, 3); + let saved = s.project.clone(); + drop(dc); + drop(s); + effectcraft_effects::roto::forget(keys.iter().map(|(_, k)| *k)); + + // "Reopen": a new session on the same folder and project. + let (mut s2, _, _) = setup(|f| draw(center(f))); + enable(&mut s2); + s2.project = saved; + s2.bump(); + s2.open_comp(cid); + assert_eq!(s2.roto_keys(cid, clip, uid).unwrap(), keys, "the same input key, so the same keys"); + let comp = s2.active_comp().unwrap().clone(); + let l = comp.layer(clip).unwrap(); + let ps = crate::roto::params_static(l.props.find_group(uid).unwrap()); + let size = crate::roto::layer_size(&s2.project, &comp, l); + // No frames to segment from: whatever comes back was read from disk. + for (f, _) in &keys { + assert!(om::ensure(&ps, size, 25.0, *f, None, &mut |_| None, &|| false).is_some(), "frame {f} not reused from disk"); + } + assert!(s2.disk_cache.as_ref().unwrap().stats().hits >= 6); + let again = alpha_at(&s2, 3); + assert_eq!(again, before, "the disk-cached matte renders the same"); + // Another prompt: other keys, nothing on disk. + let mut moved = ps.clone(); + moved.values.insert("objectPoint".into(), effectcraft_keyframe::Value::Vec2([c[0] + 3.0, c[1]])); + assert!(om::ensure(&moved, size, 25.0, 0, None, &mut |_| None, &|| false).is_none()); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/render/src/disk_cache.rs b/crates/render/src/disk_cache.rs index da46708e..4140b792 100644 --- a/crates/render/src/disk_cache.rs +++ b/crates/render/src/disk_cache.rs @@ -8,7 +8,9 @@ //! project therefore never reads stale pixels: its keys simply change. //! * Payloads are LZ4-compressed (`lz4_flex`, MIT): frames as 8-bit premultiplied RGBA, layer //! buffers as `f32` RGBA with the bytes shuffled into planes (exact, so cached and fresh -//! renders are bit-identical). Every file ends in a checksum; damaged files read as misses and +//! renders are bit-identical), Roto Brush / Object Matte segmentations as +//! `FrameSeg::to_bytes` (keyed by their content-keyed chain key and scale, so they are +//! reused whenever the same frames, settings and prompt come back, also after a restart). Every file ends in a checksum; damaged files read as misses and //! are deleted. //! * Size limit with least-recently-used eviction. The in-memory index is rebuilt from the //! folder when the cache opens (file modification times order it), so the cache survives @@ -37,6 +39,8 @@ const QUEUE_LIMIT: usize = 64; pub enum Kind { Frame, Layer, + /// A Roto Brush / Object Matte segmentation. + Matte, } impl Kind { @@ -45,12 +49,14 @@ impl Kind { match self { Kind::Frame => "frames", Kind::Layer => "layers", + Kind::Matte => "mattes", } } fn tag(self) -> u8 { match self { Kind::Frame => 1, Kind::Layer => 2, + Kind::Matte => 3, } } } @@ -319,7 +325,7 @@ impl DiskCache { /// Rebuild the index from the folder (oldest files first). pub fn rescan(&self) { let mut found: Vec<(std::time::SystemTime, Kind, u128, u64)> = vec![]; - for kind in [Kind::Frame, Kind::Layer] { + for kind in [Kind::Frame, Kind::Layer, Kind::Matte] { let Ok(shards) = std::fs::read_dir(self.dir.join(kind.dir())) else { continue }; for shard in shards.flatten() { let Ok(files) = std::fs::read_dir(shard.path()) else { continue }; @@ -403,7 +409,7 @@ impl DiskCache { if let Ok(mut g) = self.index.lock() { g.clear(); } - for kind in [Kind::Frame, Kind::Layer] { + for kind in [Kind::Frame, Kind::Layer, Kind::Matte] { let _ = std::fs::remove_dir_all(self.dir.join(kind.dir())); } } @@ -555,6 +561,22 @@ impl DiskCache { self.write_now(Kind::Layer, key, &encode_layer(buf)); } + /// A segmentation (chain key, scale) written by [`DiskCache::put_seg`]. + pub fn get_seg(&self, key: u64, scale: f64) -> Option { + let p = self.read(Kind::Matte, seg_key(key, scale))?; + let raw = lz4_flex::decompress_size_prepended(&p).ok()?; + effectcraft_effects::roto::FrameSeg::from_bytes(&raw) + } + + /// Queue a segmentation for writing. + pub fn put_seg(&self, key: u64, scale: f64, seg: &effectcraft_effects::roto::FrameSeg) { + let k = seg_key(key, scale); + if self.contains(Kind::Matte, k) { + return; + } + self.enqueue(Kind::Matte, k, lz4_flex::compress_prepend_size(&seg.to_bytes())); + } + /// Write a frame synchronously (tests). pub fn put_frame_now(&self, key: u128, width: u32, height: u32, rgba: &[u8]) { self.write_now(Kind::Frame, key, &encode_frame(width, height, rgba)); @@ -627,6 +649,24 @@ pub fn read_layer_entry(data: &[u8]) -> Option { decode_layer(verify(data, Kind::Layer)?) } +/// The disk key of a segmentation: its chain key and scale. +fn seg_key(key: u64, scale: f64) -> u128 { + let mut h = Hash128::default(); + h.write(b"segmentation-v1"); + h.write_u64(key); + h.write_u64(scale.to_bits()); + h.finish() +} + +impl effectcraft_effects::roto::SegStore for DiskCache { + fn get_seg(&self, key: u64, scale: f64) -> Option { + DiskCache::get_seg(self, key, scale) + } + fn put_seg(&self, key: u64, scale: f64, seg: &effectcraft_effects::roto::FrameSeg) { + DiskCache::put_seg(self, key, scale, seg); + } +} + impl crate::cache::LayerStore for DiskCache { fn get_layer(&self, key: u128) -> Option> { DiskCache::get_layer(self, key).map(Arc::new) @@ -889,6 +929,38 @@ mod tests { assert_eq!(parse_entry_name(&format!("{key:032x}.tmp.1")), None); } + /// Segmentations round-trip by (chain key, scale), survive a reopen (rescan), are bounded by + /// the size limit like every entry, and a damaged file is a miss. + #[test] + fn segmentations_round_trip_and_reopen() { + use effectcraft_effects::roto::FrameSeg; + let dir = tmpdir("seg"); + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + let seg = FrameSeg { w: 7, h: 5, matte: (0..35).map(|i| (i % 3 == 0) as u8).collect(), refine: vec![0; 35], ..Default::default() }; + assert!(dc.get_seg(42, 1.0).is_none()); + dc.put_seg(42, 1.0, &seg); + dc.flush(); + assert_eq!(dc.get_seg(42, 1.0), Some(seg.clone())); + assert!(dc.get_seg(42, 0.5).is_none() && dc.get_seg(43, 1.0).is_none()); + assert_eq!(dc.keys(Kind::Matte).len(), 1); + drop(dc); + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + assert_eq!(dc.get_seg(42, 1.0), Some(seg.clone())); + // Damaged on disk: a miss, and the entry is forgotten. + let path = dc.path(Kind::Matte, seg_key(42, 1.0)); + let mut d = std::fs::read(&path).unwrap(); + d[18] ^= 0x55; + std::fs::write(&path, d).unwrap(); + assert!(dc.get_seg(42, 1.0).is_none()); + assert!(!dc.contains(Kind::Matte, seg_key(42, 1.0))); + // The size limit evicts segmentations too. + dc.put_seg(1, 1.0, &seg); + dc.flush(); + dc.set_max_bytes(1); + assert!(dc.keys(Kind::Matte).is_empty()); + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn hit_miss_and_exact_round_trip() { let dir = tmpdir("hit"); diff --git a/crates/track/src/roto/mod.rs b/crates/track/src/roto/mod.rs index 7138a93a..4efb4335 100644 --- a/crates/track/src/roto/mod.rs +++ b/crates/track/src/roto/mod.rs @@ -59,6 +59,9 @@ pub enum StrokeKind { Refine, /// Refine Edge tool with Alt: remove from the band. RefineErase, + /// A box around the object (Object Matte): two opposite corners in `points`. Trained models + /// get it as their box prompt; the graph cut ignores it. + Box, } impl StrokeKind { @@ -830,8 +833,9 @@ fn points_along(s: &Stroke, scale: f64, max: usize) -> Vec<[f32; 2]> { out } -/// A model's prompt from the strokes: points along the foreground and background strokes. -fn stroke_prompt(strokes: &[&Stroke], scale: f64) -> Prompt { +/// A model's prompt from the strokes: points along the foreground and background strokes, and +/// the box of a [`StrokeKind::Box`] stroke (matte pixels). +pub fn stroke_prompt(strokes: &[&Stroke], scale: f64) -> Prompt { let mut points = vec![]; for (kind, fg) in [(StrokeKind::Fg, true), (StrokeKind::Bg, false)] { let ss: Vec<&&Stroke> = strokes.iter().filter(|s| s.kind == kind).collect(); @@ -840,11 +844,20 @@ fn stroke_prompt(strokes: &[&Stroke], scale: f64) -> Prompt { points.extend(points_along(s, scale, per).into_iter().map(|p| (p, fg))); } } - Prompt { points, ..Default::default() } + let bbox = strokes.iter().filter(|s| s.kind == StrokeKind::Box).find_map(|s| box_of(s, scale)); + Prompt { points, bbox, ..Default::default() } } -/// Add a prior matte to a prompt: its box (a little larger), up to four points spread through -/// its core, and the matte itself. +/// The box `[x0, y0, x1, y1]` (matte pixels) of a box stroke: its first two points as opposite +/// corners; `None` when they are missing, not finite or the box is empty. +fn box_of(s: &Stroke, scale: f64) -> Option<[f32; 4]> { + let (a, c) = (s.points.first()?, s.points.get(1)?); + let v = [a[0].min(c[0]), a[1].min(c[1]), a[0].max(c[0]), a[1].max(c[1])].map(|v| (v * scale) as f32); + (v.iter().all(|x| x.is_finite()) && v[2] > v[0] && v[3] > v[1]).then_some(v) +} + +/// Add a prior matte to a prompt: its box (a little larger; unless the prompt has one), up to +/// four points spread through its core, and the matte itself. fn add_prior_prompt(prompt: &mut Prompt, soft: &[f32], w: usize, h: usize) -> bool { let bin: Vec = soft.iter().map(|v| (*v >= 0.5) as u8).collect(); let (mut x0, mut y0, mut x1, mut y1) = (usize::MAX, usize::MAX, 0usize, 0usize); @@ -858,7 +871,10 @@ fn add_prior_prompt(prompt: &mut Prompt, soft: &[f32], w: usize, h: usize) -> bo return false; } let (mx, my) = (((x1 - x0) as f32 * 0.05).max(4.0), ((y1 - y0) as f32 * 0.05).max(4.0)); - prompt.bbox = Some([(x0 as f32 - mx).max(0.0), (y0 as f32 - my).max(0.0), (x1 as f32 + 1.0 + mx).min(w as f32), (y1 as f32 + 1.0 + my).min(h as f32)]); + // A box the user drew on this frame wins over the prior's. + if prompt.bbox.is_none() { + prompt.bbox = Some([(x0 as f32 - mx).max(0.0), (y0 as f32 - my).max(0.0), (x1 as f32 + 1.0 + mx).min(w as f32), (y1 as f32 + 1.0 + my).min(h as f32)]); + } // Farthest-point picks among the core (deep inside), starting from the deepest pixel. let sd = matting::signed_distance(&bin, w, h); let deep = sd.iter().copied().fold(0.0f32, f32::max); diff --git a/crates/track/src/roto/tests.rs b/crates/track/src/roto/tests.rs index f92e67d5..65d1743f 100644 --- a/crates/track/src/roto/tests.rs +++ b/crates/track/src/roto/tests.rs @@ -428,3 +428,25 @@ fn mobilesam_segments_and_propagates_the_disk() { eprintln!("MobileSAM: base IoU {jb:.4}, worst propagated {worst:.4} (classic {cworst:.4}), {:.0} ms/frame", t.elapsed().as_secs_f64() * 1000.0 / 20.0); assert!(jb > 0.95 && worst > 0.93 && worst >= cworst - 0.01, "base {jb}, worst {worst} vs classic {cworst}"); } + +/// A box stroke becomes the model's box prompt (scaled, corners in any order); the graph cut +/// ignores it; a prior's box does not replace it. +#[test] +fn box_stroke_is_the_model_box_prompt() { + let fg = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 2.0, points: vec![[50.0, 40.0]] }; + let bx = Stroke { kind: StrokeKind::Box, frame: 0, radius: 0.0, points: vec![[80.0, 60.0], [20.0, 10.0]] }; + let p = stroke_prompt(&[&fg, &bx], 0.5); + assert_eq!(p.bbox, Some([10.0, 5.0, 40.0, 30.0])); + assert_eq!(p.points, vec![([25.0, 20.0], true)]); + // Degenerate or non-finite boxes are no prompt. + for pts in [vec![[20.0, 10.0]], vec![[20.0, 10.0], [20.0, 50.0]], vec![[f64::NAN, 10.0], [80.0, 60.0]]] { + let b = Stroke { points: pts, ..bx.clone() }; + assert_eq!(stroke_prompt(&[&fg, &b], 1.0).bbox, None); + } + assert!(!StrokeKind::Box.is_segmentation()); + assert!(rasterize(&[&bx], &[StrokeKind::Fg, StrokeKind::Bg], 1.0, 100, 80).iter().all(|v| !v)); + let mut q = stroke_prompt(&[&fg, &bx], 1.0); + let soft: Vec = (0..100 * 80).map(|i| ((i % 100) < 10) as u8 as f32).collect(); + assert!(add_prior_prompt(&mut q, &soft, 100, 80)); + assert_eq!(q.bbox, Some([20.0, 10.0, 80.0, 60.0])); +} diff --git a/crates/ui-egui/src/i18n/ui.rs b/crates/ui-egui/src/i18n/ui.rs index 537854bd..f499f910 100644 --- a/crates/ui-egui/src/i18n/ui.rs +++ b/crates/ui-egui/src/i18n/ui.rs @@ -627,6 +627,7 @@ const JAPANESE: &[(&str, &str)] = &[ ("Project", "プロジェクト"), ("Total Offset", "合計オフセット"), ("Per Layer", "レイヤーごと"), + ("Track", "トラック"), ]; const SIMPLIFIED_CHINESE: &[(&str, &str)] = &[ @@ -1352,6 +1353,7 @@ const SIMPLIFIED_CHINESE: &[(&str, &str)] = &[ ("Stored, but not used by EffectCraft yet", "已保存,但 EffectCraft 尚未使用"), ("Total Offset", "总偏移"), ("Per Layer", "每个图层"), + ("Track", "跟踪"), ]; const TRADITIONAL_CHINESE: &[(&str, &str)] = &[ @@ -2072,6 +2074,7 @@ const TRADITIONAL_CHINESE: &[(&str, &str)] = &[ ("Stored, but not used by EffectCraft yet", "已儲存,但 EffectCraft 尚未使用"), ("Total Offset", "總偏移"), ("Per Layer", "每個圖層"), + ("Track", "追蹤"), ]; /// Ukrainian (uk): original translations of the same interface rows as the Chinese catalogs. @@ -2539,6 +2542,7 @@ const UKRAINIAN: &[(&str, &str)] = &[ ("⌘/Ctrl-drag a pin to record its motion", "Перетягніть опорну точку з ⌘/Ctrl, щоб записати її рух"), ("▣ {}", "▣ {}"), ("⚠ {} is used by:", "⚠ {} використовується в:"), + ("Track", "Відстежити"), // Strings a panel used to hand to a helper as a bare literal: the helper draws them (the // viewer's toolbar tooltips and popup items, the solid dialog's checkbox), so the lookup has // to reach the catalog from inside the helper. Kept at the end so the catalogs stay parallel. diff --git a/crates/ui-egui/src/icons.rs b/crates/ui-egui/src/icons.rs index 976e58ab..949ea377 100644 --- a/crates/ui-egui/src/icons.rs +++ b/crates/ui-egui/src/icons.rs @@ -35,6 +35,8 @@ pub enum Icon { Eraser, RotoBrush, RefineEdge, + /// Object Matte: a selection box's corners around a solid object, with a click point. + ObjectMatte, /// Puppet Position Pin: a pushpin. The other pin tools add a badge at the lower right. Puppet, PuppetAdvanced, @@ -348,6 +350,14 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) { pen.line(&[(8.0, 14.0), (9.5, 12.5)]); pen.line(&[(11.0, 14.0), (12.5, 12.5)]); } + ObjectMatte => { + // Corner brackets of a box around a filled object, and the click on it. + for (x, y, dx, dy) in [(2.0, 2.0, 1.0, 1.0), (14.0, 2.0, -1.0, 1.0), (14.0, 14.0, -1.0, -1.0), (2.0, 14.0, 1.0, -1.0)] { + pen.line(&[(x, y + 3.0 * dy), (x, y), (x + 3.0 * dx, y)]); + } + pen.dot(8.0, 8.0, 2.2); + pen.circle(8.0, 8.0, 4.4); + } Puppet => pen.pushpin(8.0), PuppetAdvanced => { // Rotation ring with its square scale handle. diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 0ffe2928..da89fd7a 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -1635,6 +1635,12 @@ impl EffectcraftApp { self.session.poll_roto(true); ctx.request_repaint_after(std::time::Duration::from_millis(100)); } + // Roto Brush propagation and Object Matte tracking / Freeze: finish jobs (a Freeze writes + // its mattes) and start queued ones in the background. + if self.session.roto_job.is_some() || !self.session.roto_pending.is_empty() { + self.session.poll_roto(true); + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } // 3D Camera Tracker: likewise (tracking, then solving). if self.session.camera_job.is_some() || !self.session.camera_pending.is_empty() { self.session.poll_camera(true); diff --git a/crates/ui-egui/src/panels/effect_controls.rs b/crates/ui-egui/src/panels/effect_controls.rs index 29bf154b..b3c17156 100644 --- a/crates/ui-egui/src/panels/effect_controls.rs +++ b/crates/ui-egui/src/panels/effect_controls.rs @@ -805,6 +805,44 @@ fn roto_editor(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, l p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim); } +/// Object Matte: Freeze / Unfreeze and Track / Stop buttons (the background job that computes +/// the matte over the tracked frames) and its status. +fn object_matte_editor(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, layer: &Layer, g: &PropGroup, r: Rect, actions: &mut Actions) { + use effectcraft_engine::effects::object_matte as om; + let t = app.tokens; + let params = effectcraft_engine::effects::flatten_params(g, &mut |pr| pr.value.clone()); + let frozen = om::is_frozen(¶ms); + let mine = app.session.roto_target().is_some_and(|(_, l, u, _)| l == layer.id && u == g.uid); + let running = mine && app.session.is_roto_running(); + let busy = app.session.is_roto_running(); + let b1 = Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 4.0), vec2(84.0, 22.0)); + let b2 = Rect::from_min_size(pos2(b1.max.x + 8.0, b1.min.y), vec2(84.0, 22.0)); + let label = if frozen { "Unfreeze" } else { "Freeze" }; + if widgets::text_button(ui, b1, crate::i18n::tr(label), frozen, &t, egui::Id::new(("om-freeze-ec", g.uid))).clicked() && !busy { + let id = if frozen { "objectMatte.unfreeze" } else { "objectMatte.freeze" }; + actions.push((id.into(), json!({"layer": layer.id.0, "effect": g.uid}))); + } + app.auto.add(&format!("effectControls.objectMatte.{}.freeze", g.uid), b1, label); + let (l2, id2) = if running { ("Stop", "roto.cancel") } else { ("Track", "objectMatte.propagate") }; + if widgets::text_button(ui, b2, crate::i18n::tr(l2), false, &t, egui::Id::new(("om-track-ec", g.uid))).clicked() && !frozen && (running || !busy) { + actions.push((id2.into(), json!({"layer": layer.id.0, "effect": g.uid}))); + } + app.auto.add(&format!("effectControls.objectMatte.{}.track", g.uid), b2, l2); + let base = om::reference_frame(¶ms); + let status = match app.session.roto_progress() { + Some(pr) if running => pr.banner(), + _ if frozen => format!("Frozen from reference frame {base}"), + _ if app.session.roto_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => "Waiting to track".to_string(), + _ => format!("Reference frame {base}: Track computes the matte ahead of playback"), + }; + p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, &status, Tokens::ui(11.5), t.text_dim); + app.auto.add( + &format!("effectControls.objectMatte.{}.status", g.uid), + Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 30.0), vec2(r.width() - 24.0, 16.0)), + &status, + ); +} + // --------------------------------------------------------------------------------------------- // Curves / Levels editors @@ -815,7 +853,7 @@ fn editor_height(effect: &str, g: &PropGroup, width: f32) -> f32 { "ec.color.curves" => 34.0 + curves_size(width) + 26.0, effectcraft_engine::effects::warp_stab::ID => 50.0, effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::EDITOR_HEIGHT, - effectcraft_engine::effects::roto::ID => 50.0, + effectcraft_engine::effects::roto::ID | effectcraft_engine::effects::object_matte::ID => 50.0, "ec.color.levels" | "ec.color.levelsic" => 34.0 + 80.0 + 58.0, "ec.color.autolevels" | "ec.color.autocontrast" | "ec.color.autocolor" => 24.0 + 80.0 + 12.0, _ => 0.0, @@ -1290,6 +1328,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { effectcraft_engine::effects::warp_stab::ID => warp_editor(app, ui, &bp, &layer, g, er, &mut actions), effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::editor(app, ui, &bp, &layer, g, er, &mut actions), effectcraft_engine::effects::roto::ID => roto_editor(app, ui, &bp, &layer, g, er, &mut actions), + effectcraft_engine::effects::object_matte::ID => object_matte_editor(app, ui, &bp, &layer, g, er, &mut actions), "ec.color.levels" | "ec.color.levelsic" => levels_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions), super::fx_editors::GLOW | super::fx_editors::RESHAPE | super::fx_editors::EXTRACTOR => { super::fx_editors::header_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions) diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index cc7910b3..1b6ef8c3 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -30,6 +30,7 @@ pub mod learn; pub mod markers_ui; pub mod media_panels; pub mod misc; +pub mod object_matte_tool; pub mod paint_panels; pub mod panel_kit; pub mod path_vr_panels; diff --git a/crates/ui-egui/src/panels/object_matte_tool.rs b/crates/ui-egui/src/panels/object_matte_tool.rs new file mode 100644 index 00000000..70c9df8e --- /dev/null +++ b/crates/ui-egui/src/panels/object_matte_tool.rs @@ -0,0 +1,128 @@ +//! The Object Matte tool in the Composition viewer (After Effects 26.2's Object Matte, from its +//! public description): click an object to isolate it, or drag a box around it. The layer under +//! the pointer gets an Object Matte effect (`objectMatte.apply`), or its Object Matte is +//! re-prompted (the selected one, else its last); Shift adds another. Every prompt is one undo +//! step, and the effect is selected so its point controls show in the viewer. +//! +//! While hovering, a crosshair and the outline of the layer that would get the matte preview the +//! click: segmenting on every pointer move would be far too slow for a hover preview (a model +//! pass or a graph cut per move), so the matte itself appears once the prompt is set. + +use effectcraft_engine::geom::vec2 as gv2; +use effectcraft_engine::project::Layer; +use effectcraft_engine::render::EvalCtx; +use egui::{Color32, Pos2, Rect, Stroke, StrokeKind}; +use serde_json::json; + +use super::viewer::{ViewerMap, fx2c, layers_at}; +use crate::EffectcraftApp; + +/// Drags shorter than this (screen points) are clicks: a point prompt, no box. +pub const MIN_BOX: f32 = 4.0; + +/// Whether Object Matte can work on `layer` (footage, solid or composition layers with pixels). +fn eligible(layer: &Layer) -> bool { + layer.has_video() && !layer.switches.adjustment && layer.effects().is_some() +} + +/// The layer a prompt at comp point `cpt` goes to: a selected one under it, else the topmost. +pub fn target<'a>(app: &EffectcraftApp, ectx: &EvalCtx<'a>, cpt: [f64; 2]) -> Option<&'a Layer> { + let hits: Vec<&Layer> = layers_at(ectx, cpt).filter(|l| eligible(l)).collect(); + let selected = &app.session.state.selected_layers; + hits.iter().find(|l| selected.contains(&l.id)).or(hits.first()).copied() +} + +/// The layer's effect-space rectangle (its pixels) on screen. +fn outline(ectx: &EvalCtx, map: &ViewerMap, layer: &Layer) -> [Pos2; 4] { + let [w, h] = ectx.effect_bounds(layer).0; + let m = fx2c(ectx, layer); + [[0.0, 0.0], [w, 0.0], [w, h], [0.0, h]].map(|p| { + let c = m.apply(gv2(p[0], p[1])); + map.to_screen([c.x, c.y]) + }) +} + +/// Hover preview at `hp` (screen): the outline of the layer that would get the matte and a +/// crosshair where the object point would go. +pub fn hover(app: &mut EffectcraftApp, painter: &egui::Painter, map: &ViewerMap, ectx: &EvalCtx, hp: Pos2) { + let accent = app.tokens.accent; + let cpt = map.to_comp(hp); + if let Some(layer) = target(app, ectx, cpt) { + let q = outline(ectx, map, layer); + let mut pts = q.to_vec(); + pts.push(q[0]); + painter.add(egui::Shape::line(pts, Stroke::new(1.5, accent))); + let r = Rect::from_points(&q); + app.auto.add("viewer.objectMatte.target", r, &layer.name); + } + let (gap, arm) = (4.0, 11.0); + for (dx, dy) in [(1.0, 0.0), (-1.0, 0.0), (0.0, 1.0), (0.0, -1.0)] { + let d = egui::vec2(dx, dy); + for (w, c) in [(3.0, Color32::from_black_alpha(160)), (1.5, Color32::WHITE)] { + painter.line_segment([hp + d * gap, hp + d * arm], Stroke::new(w, c)); + } + } + painter.circle_stroke(hp, 2.0, Stroke::new(1.5, accent)); + app.auto.add("viewer.objectMatte.crosshair", Rect::from_center_size(hp, egui::vec2(2.0 * arm, 2.0 * arm)), "Object Matte point"); +} + +/// The box being dragged (screen `a`–`b`). +pub fn draw_box(app: &mut EffectcraftApp, painter: &egui::Painter, a: Pos2, b: Pos2) { + let r = Rect::from_two_pos(a, b); + let accent = app.tokens.accent; + painter.rect_stroke(r, 0.0, Stroke::new(2.5, Color32::from_black_alpha(140)), StrokeKind::Middle); + painter.rect_stroke(r, 0.0, Stroke::new(1.5, accent), StrokeKind::Middle); + painter.circle_filled(r.center(), 2.5, accent); + app.auto.add("viewer.objectMatte.box", r, "Object Matte box"); +} + +/// Set the prompt from a click at `a` (screen), or a box dragged from `a` to `b`: on the +/// target layer's Object Matte (re-prompted), or a new one with `add` or when it has none. +pub fn apply(app: &mut EffectcraftApp, ectx: &EvalCtx, map: &ViewerMap, a: Pos2, b: Option, add: bool) { + let boxed = b.filter(|b| (b.x - a.x).abs() >= MIN_BOX && (b.y - a.y).abs() >= MIN_BOX); + let center = boxed.map_or(a, |b| Rect::from_two_pos(a, b).center()); + let Some(layer) = target(app, ectx, map.to_comp(center)) else { + app.ui.status = "Object Matte: click an object on a footage, solid or composition layer".into(); + return; + }; + let Some(inv) = fx2c(ectx, layer).inverse() else { return }; + let to_layer = |p: Pos2| { + let c = map.to_comp(p); + let l = inv.apply(gv2(c[0], c[1])); + [l.x, l.y] + }; + let point = to_layer(center); + let mut q = json!({"layer": layer.id.0, "point": point}); + if let Some(b) = boxed { + // The box in layer pixels: the bounds of its corners (the layer may be turned). + let r = Rect::from_two_pos(a, b); + let c = [r.left_top(), r.right_top(), r.right_bottom(), r.left_bottom()].map(to_layer); + let (x0, x1) = c.iter().map(|p| p[0]).fold((f64::INFINITY, f64::NEG_INFINITY), |(lo, hi), v| (lo.min(v), hi.max(v))); + let (y0, y1) = c.iter().map(|p| p[1]).fold((f64::INFINITY, f64::NEG_INFINITY), |(lo, hi), v| (lo.min(v), hi.max(v))); + q["box"] = json!([x0, y0, x1, y1]); + } + if !add && let Some(uid) = existing(app, layer) { + q["effect"] = json!(uid); + } + let lid = layer.id; + match app.session.execute("objectMatte.apply", q) { + Ok(r) => { + if let Some(uid) = r["effect"].as_u64() { + let _ = app.session.execute("prop.select", json!({"layer": lid.0, "prop": uid})); + } + } + Err(e) => app.ui.status = e.to_string(), + } +} + +/// The Object Matte a prompt on `layer` re-prompts: the selected one, else the layer's last. +fn existing(app: &EffectcraftApp, layer: &Layer) -> Option { + let fx = layer.effects()?; + let om: Vec = fx.groups().filter(|g| is_object_matte(g)).map(|g| g.uid).collect(); + let sel = app.session.state.selected_props.iter().rev().find(|(l, u)| *l == layer.id && om.contains(u)).map(|(_, u)| *u); + sel.or(om.last().copied()) +} + +fn is_object_matte(g: &effectcraft_engine::project::PropGroup) -> bool { + matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == effectcraft_engine::effects::object_matte::ID) +} diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index 30dc02b3..6d47ce40 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -168,6 +168,10 @@ enum Gesture { PinMarquee { start: Pos2, }, + /// Object Matte tool: a box around the object (screen point where the drag began). + ObjectMatteBox { + start: Pos2, + }, /// Puppet tool Record mode (⌘/Ctrl-drag a pin): the drag is sampled in real time (seconds /// since it began, effect space) and becomes keyframes on release (`puppet.recordPin`); the /// current time runs along while recording. @@ -1313,6 +1317,14 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } t if t.is_shape() => egui::CursorIcon::Crosshair, _ if app.ui.viewer.roi_draw => egui::CursorIcon::Crosshair, + Tool::ObjectMatte => { + // The tool draws its own crosshair (and the layer it would matte) while no box + // is being dragged. + if ui.data(|d| d.get_temp::(egui::Id::new("viewer-gesture"))).is_none() { + super::object_matte_tool::hover(app, &painter, &map, &ectx, hp); + } + egui::CursorIcon::None + } t if t.puppet_kind().is_some() => match super::puppet_tool::hit(&pin_hits, hp) { Some((_, super::puppet_tool::PinPart::Ring)) => egui::CursorIcon::Alias, Some((_, super::puppet_tool::PinPart::Scale)) => egui::CursorIcon::ResizeNwSe, @@ -1443,6 +1455,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { // From where the button went down: by the time the pointer has moved far enough to // count as a drag it has already left the press point, and the shape is drawn there. t if t.is_shape() => Some(Gesture::Create { tool: t, start: map.to_comp(press) }), + Tool::ObjectMatte => Some(Gesture::ObjectMatteBox { start: press }), t if t.puppet_kind().is_some() && super::puppet_tool::hit(&pin_hits, press).is_none() => { // Off the pins: Alt or a press outside the art drags a marquee that selects pins. (mods.alt || puppet_layer(app, cpt).is_none()).then_some(Gesture::PinMarquee { start: press }) @@ -1766,6 +1779,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { q["merge"] = json!(merge); let _ = app.session.execute("track.setPoint", q); } + Gesture::ObjectMatteBox { start } => super::object_matte_tool::draw_box(app, &painter, start, pos), Gesture::Marquee { start } | Gesture::PinMarquee { start } => { let r = Rect::from_two_pos(start, pos); painter.rect_filled(r, 0.0, t.accent.gamma_multiply(0.12)); @@ -1806,6 +1820,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } } Gesture::PuppetPin { layer, pin, .. } => super::puppet_tool::reveal_pin(app, layer, pin), + Gesture::ObjectMatteBox { start } => super::object_matte_tool::apply(app, &ectx, &map, start, Some(pos), mods.shift), Gesture::Create { tool, start } => { let end = map.to_comp(pos); create_shape(app, tool, start, end, mods); @@ -1909,6 +1924,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { // Clicking away from the free-transform box ends it. ov::end_free_transform(&ctx); } + Tool::ObjectMatte => super::object_matte_tool::apply(app, &ectx, &map, pos, None, mods.shift), t if t.puppet_kind().is_some() => { if let Some((h, _)) = super::puppet_tool::hit(&pin_hits, pos) { // Click selects the pin; Shift-click adds or removes it. @@ -2270,7 +2286,7 @@ pub(crate) fn selectable_layers(comp: &Comp, time: Tick) -> impl Iterator(ectx: &'b EvalCtx<'a>, cpt: [f64; 2]) -> impl Iterator + 'b { +pub(crate) fn layers_at<'a, 'b>(ectx: &'b EvalCtx<'a>, cpt: [f64; 2]) -> impl Iterator + 'b { selectable_layers(ectx.comp, ectx.time).filter(move |l| layer_quad(ectx, l).is_some_and(|(_, q, _)| point_in_quad(cpt, &q))) } diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index 3cd997a9..830cd75c 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -35,6 +35,8 @@ pub enum Tool { Eraser, RotoBrush, RefineEdge, + /// Object Matte: click an object (or drag a box around it) in the Composition viewer. + ObjectMatte, Puppet, PuppetStarch, PuppetBend, @@ -59,10 +61,10 @@ impl Tool { &[Tool::Brush], &[Tool::Clone], &[Tool::Eraser], - &[Tool::RotoBrush, Tool::RefineEdge], + &[Tool::RotoBrush, Tool::RefineEdge, Tool::ObjectMatte], &[Tool::Puppet, Tool::PuppetStarch, Tool::PuppetBend, Tool::PuppetAdvanced, Tool::PuppetOverlap], ]; - pub const ALL: [Tool; 30] = [ + pub const ALL: [Tool; 31] = [ Tool::Selection, Tool::Hand, Tool::Zoom, @@ -88,6 +90,7 @@ impl Tool { Tool::Eraser, Tool::RotoBrush, Tool::RefineEdge, + Tool::ObjectMatte, Tool::Puppet, Tool::PuppetStarch, Tool::PuppetBend, @@ -121,6 +124,7 @@ impl Tool { Tool::Eraser => "Eraser Tool", Tool::RotoBrush => "Roto Brush Tool", Tool::RefineEdge => "Refine Edge Tool", + Tool::ObjectMatte => "Object Matte Tool", Tool::Puppet => "Puppet Position Pin Tool", Tool::PuppetStarch => "Puppet Starch Pin Tool", Tool::PuppetBend => "Puppet Bend Pin Tool", @@ -142,7 +146,7 @@ impl Tool { Tool::Pen | Tool::PenAdd | Tool::PenDelete | Tool::PenConvert | Tool::MaskFeather => Some("G"), Tool::Type | Tool::TypeVertical => Some("Cmd+T"), Tool::Brush | Tool::Clone | Tool::Eraser => Some("Cmd+B"), - Tool::RotoBrush | Tool::RefineEdge => Some("Alt+W"), + Tool::RotoBrush | Tool::RefineEdge | Tool::ObjectMatte => Some("Alt+W"), Tool::Puppet | Tool::PuppetStarch | Tool::PuppetBend | Tool::PuppetAdvanced | Tool::PuppetOverlap => Some("Cmd+P"), } } @@ -173,6 +177,7 @@ impl Tool { Tool::Eraser => Icon::Eraser, Tool::RotoBrush => Icon::RotoBrush, Tool::RefineEdge => Icon::RefineEdge, + Tool::ObjectMatte => Icon::ObjectMatte, Tool::Puppet => Icon::Puppet, Tool::PuppetStarch => Icon::PuppetStarch, Tool::PuppetBend => Icon::PuppetBend, diff --git a/crates/ui-egui/tests/ui_object_matte.rs b/crates/ui-egui/tests/ui_object_matte.rs new file mode 100644 index 00000000..5f30ba03 --- /dev/null +++ b/crates/ui-egui/tests/ui_object_matte.rs @@ -0,0 +1,185 @@ +//! Headless checks for the Object Matte tool in the Composition viewer: the hover preview, a +//! click (point prompt), a drag (box prompt) re-prompting the same effect, Shift adding another, +//! one undo step each (egui_kittest, UI logic only). + +use effectcraft_engine::Session; +use effectcraft_engine::effects::flatten_params; +use effectcraft_engine::effects::object_matte as om; +use effectcraft_engine::keyframe::Value; +use effectcraft_engine::project::{GroupKind, LayerId, PropGroup}; +use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::dock::PanelKind; +use effectcraft_ui_egui::state::Tool; +use egui::{Event, Modifiers, Pos2, Rect, pos2, vec2}; +use egui_kittest::Harness; +use serde_json::json; + +fn harness() -> Harness<'static, EffectcraftApp> { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Matte", "width": 640, "height": 360, "duration": 4})).unwrap(); + s.execute("layer.newSolid", json!({"name": "Plate", "color": "#406080", "width": 640, "height": 360})).unwrap(); + s.execute("layer.newSolid", json!({"name": "Box", "color": "#e04020", "width": 80, "height": 80})).unwrap(); + s.execute("edit.deselectAll", json!({})).unwrap(); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).build_eframe(move |_| EffectcraftApp::new(s)); + h.run_steps(3); + h +} + +fn rect(h: &Harness<'_, EffectcraftApp>, id: &str) -> Rect { + let e = h.state().auto.find(id).unwrap_or_else(|| panic!("no {id}")); + Rect::from_min_size(pos2(e.rect[0], e.rect[1]), vec2(e.rect[2], e.rect[3])) +} + +/// Comp pixel → screen point. +fn screen(h: &Harness<'_, EffectcraftApp>, p: [f32; 2]) -> Pos2 { + let c = rect(h, "viewer.comp"); + let z = c.width() / 640.0; + c.min + vec2(p[0] * z, p[1] * z) +} + +fn press(h: &mut Harness<'_, EffectcraftApp>, p: Pos2, pressed: bool, modifiers: Modifiers) { + h.input_mut().events.push(Event::ModifiersChanged(modifiers)); + h.input_mut().events.push(Event::PointerButton { pos: p, button: egui::PointerButton::Primary, pressed, modifiers }); + h.step(); +} + +fn click(h: &mut Harness<'_, EffectcraftApp>, p: Pos2, modifiers: Modifiers) { + h.input_mut().events.push(Event::PointerMoved(p)); + h.step(); + press(h, p, true, modifiers); + press(h, p, false, modifiers); + h.input_mut().events.push(Event::ModifiersChanged(Modifiers::NONE)); + h.run_steps(2); +} + +fn drag(h: &mut Harness<'_, EffectcraftApp>, from: Pos2, to: Pos2) -> Option { + h.input_mut().events.push(Event::PointerMoved(from)); + press(h, from, true, Modifiers::NONE); + let mut drawn = None; + for i in 1..=8 { + h.input_mut().events.push(Event::PointerMoved(from + (to - from) * (i as f32 / 8.0))); + h.step(); + drawn = h.state().auto.find("viewer.objectMatte.box").map(|e| Rect::from_min_size(pos2(e.rect[0], e.rect[1]), vec2(e.rect[2], e.rect[3]))); + } + press(h, to, false, Modifiers::NONE); + h.run_steps(2); + drawn +} + +fn box_layer(h: &Harness<'_, EffectcraftApp>) -> LayerId { + h.state().session.active_comp().unwrap().layers.iter().find(|l| l.name == "Box").unwrap().id +} + +/// The Box layer's Object Matte effects. +fn mattes(h: &Harness<'_, EffectcraftApp>) -> Vec { + let comp = h.state().session.active_comp().unwrap(); + let l = comp.layer(box_layer(h)).unwrap(); + l.effects().unwrap().groups().filter(|g| matches!(&g.kind, GroupKind::Effect { effect } if effect == om::ID)).cloned().collect() +} + +fn v2(g: &PropGroup, id: &str) -> [f64; 2] { + flatten_params(g, &mut |p| p.value.clone()).get(id).map(Value::as_vec2).unwrap() +} + +fn near(a: [f64; 2], b: [f64; 2]) -> bool { + (a[0] - b[0]).abs() < 1.5 && (a[1] - b[1]).abs() < 1.5 +} + +#[test] +fn object_matte_tool_prompts_with_a_click_and_a_box() { + let mut h = harness(); + assert_eq!(Tool::from_name("objectMatte"), Some(Tool::ObjectMatte)); + assert!(Tool::SLOTS.iter().any(|s| s.contains(&Tool::RotoBrush) && s.contains(&Tool::ObjectMatte))); + h.state_mut().ui.tool = Tool::ObjectMatte; + h.run_steps(2); + // Hover: a crosshair at the pointer and the outline of the layer that would get the matte. + let center = screen(&h, [320.0, 180.0]); + h.input_mut().events.push(Event::PointerMoved(center)); + h.run_steps(2); + assert!(rect(&h, "viewer.objectMatte.crosshair").contains(center)); + let target = h.state().auto.find("viewer.objectMatte.target").expect("no target outline"); + assert_eq!(target.label, "Box"); + assert!(mattes(&h).is_empty(), "hovering must not apply anything"); + let undo0 = h.state().session.history.undo.len(); + + // Click: a point prompt in layer pixels on the layer under the pointer, selected. + click(&mut h, center, Modifiers::NONE); + let m = mattes(&h); + assert_eq!(m.len(), 1); + assert!(near(v2(&m[0], "objectPoint"), [40.0, 40.0]), "{:?}", v2(&m[0], "objectPoint")); + let uid = m[0].uid; + let st = &h.state().session.state; + assert!(st.selected_props.contains(&(box_layer(&h), uid)), "the new effect is selected"); + assert_eq!(h.state().session.history.undo.len(), undo0 + 1); + + // Drag: a box (drawn while dragging) re-prompts the same effect; its centre is the point. + // (It starts clear of the selected effect's point controls, which take drags of their own.) + let (a, b) = (screen(&h, [286.0, 150.0]), screen(&h, [356.0, 214.0])); + let drawn = drag(&mut h, a, b).expect("no box drawn while dragging"); + assert!(drawn.width() > 20.0); + let m = mattes(&h); + assert_eq!(m.len(), 1); + assert_eq!(m[0].uid, uid); + let ps = flatten_params(&m[0], &mut |p| p.value.clone()); + assert_eq!(ps.get("useBox"), Some(&Value::Bool(true))); + assert!(near(v2(&m[0], "boxTopLeft"), [6.0, 10.0]), "{:?}", v2(&m[0], "boxTopLeft")); + assert!(near(v2(&m[0], "boxBottomRight"), [76.0, 74.0]), "{:?}", v2(&m[0], "boxBottomRight")); + assert!(near(v2(&m[0], "objectPoint"), [41.0, 42.0]), "{:?}", v2(&m[0], "objectPoint")); + assert_eq!(h.state().session.history.undo.len(), undo0 + 2); + + // Shift-click adds a second Object Matte. + let p = screen(&h, [310.0, 170.0]); + click(&mut h, p, Modifiers::SHIFT); + let m = mattes(&h); + assert_eq!(m.len(), 2); + assert_eq!(m[1].name, "Object Matte 2"); + assert!(near(v2(&m[1], "objectPoint"), [30.0, 30.0])); + + // Undo takes the prompts back one at a time. + h.state_mut().session.execute("edit.undo", json!({})).unwrap(); + assert_eq!(mattes(&h).len(), 1); + h.state_mut().session.execute("edit.undo", json!({})).unwrap(); + assert_eq!(flatten_params(&mattes(&h)[0], &mut |p| p.value.clone()).get("useBox"), Some(&Value::Bool(false))); +} + +/// Effect Controls: the Object Matte's Track and Freeze buttons run the background job and its +/// status line shows where it is; the UI keeps drawing frames while it runs and finishes the +/// job itself (a Freeze writes its mattes without any other job running). +#[test] +fn object_matte_effect_controls_track_and_freeze() { + let mut h = harness(); + h.state_mut().show_panel(PanelKind::EffectControls); + h.state_mut().ui.tool = Tool::ObjectMatte; + h.run_steps(2); + let center = screen(&h, [320.0, 180.0]); + click(&mut h, center, Modifiers::NONE); + let uid = mattes(&h)[0].uid; + h.run_steps(2); + let status = |h: &Harness<'_, EffectcraftApp>| h.state().auto.find(&format!("effectControls.objectMatte.{uid}.status")).map(|e| e.label.clone()); + assert!(status(&h).is_some(), "no Object Matte status line"); + // Freeze: a background job (the frames keep coming), then the effect is frozen. + let b = rect(&h, &format!("effectControls.objectMatte.{uid}.freeze")); + // Let any automatic tracking finish first (one job at a time). + let frozen = |h: &Harness<'_, EffectcraftApp>| om::is_frozen(&flatten_params(&mattes(h)[0], &mut |p| p.value.clone())); + let t = std::time::Instant::now(); + while h.state().session.is_roto_running() { + assert!(t.elapsed().as_secs() < 120); + h.step(); + } + click(&mut h, b.center(), Modifiers::NONE); + let t = std::time::Instant::now(); + while !frozen(&h) { + assert!( + t.elapsed().as_secs() < 20, + "never frozen: {:?} / {} / running {} / {:?}", + status(&h), + h.state().ui.status, + h.state().session.is_roto_running(), + h.state().session.roto_progress() + ); + h.step(); + } + h.run_steps(2); + assert!(status(&h).is_some_and(|s| s.starts_with("Frozen")), "{:?}", status(&h)); + assert_eq!(h.state().auto.find(&format!("effectControls.objectMatte.{uid}.freeze")).map(|e| e.label.as_str()), Some("Unfreeze")); +} diff --git a/docs/agents.md b/docs/agents.md index df1e7ba8..a1759768 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -483,6 +483,26 @@ effectcraft-cli run clip.ecproj roto.stroke '{"layer":"#1","points":[[300,200],[ roto.propagate '{"layer":"#1","wait":true}' roto.freeze '{"layer":"#1","wait":true}' --save ``` +### Object Matte + +The Object Matte tool (third in the Alt+W slot) works in the Composition viewer: click an object, +or drag a box around it; Shift adds another Object Matte instead of re-prompting the layer's. +Agents use `objectMatte.apply {layer, point: [x, y], box?: [x0, y0, x1, y1], frame?, track?, +before?, after?, effect?}` (layer pixels, layer frames; `effect` re-prompts that instance), one +undo step each. With a trained model installed (Settings ▸ Roto Brush) the box is the model's box +prompt. `objectMatte.propagate {layer, effect?, direction?, wait?}` tracks the matte over the +frames around the reference frame in the background (`roto.cancel` stops it; edits to the prompt +or the layer's frames queue it again), `objectMatte.freeze {wait?}` / `objectMatte.unfreeze` +store or drop the final mattes, and `objectMatte.status {layer, effect?}` reports the job's +progress, the frozen state, the tracked frames and which are computed. With the disk cache on (Settings ▸ +Media & Disk Cache), computed Object Matte and Roto Brush segmentations are also kept there, keyed +by what they are made of, so reopening a project reuses them. + +```sh +effectcraft-cli run clip.ecproj objectMatte.apply '{"layer":"#1","point":[320,200],"box":[250,120,400,300]}' \ + objectMatte.propagate '{"layer":"#1","wait":true}' objectMatte.freeze '{"layer":"#1","wait":true}' --save +``` + ### Text: styles and editing Source Text holds character style runs and per-paragraph settings. CR (`\r`), LF (`\n`) and diff --git a/docs/effects.md b/docs/effects.md index 7b8f479e..0bfc6a7b 100644 --- a/docs/effects.md +++ b/docs/effects.md @@ -3,11 +3,11 @@ -EffectCraft ships 306 effects with 3017 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. +EffectCraft ships 307 effects with 3032 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. - **GPU**: 280 effects also run on the GPU compositor with identical results. -- **32**: 306 effects process 32-bit float (HDR, overbright) pixels without clamping. -- **Status**: 306 implemented in full, 0 partial (what is missing is listed). +- **32**: 307 effects process 32-bit float (HDR, overbright) pixels without clamping. +- **Status**: 307 implemented in full, 0 partial (what is missing is listed). Effects are addressed by id (`ec..`) or display name in commands, scripts and the control channel, and every parameter by its id (`effects/#1/blurriness`). @@ -255,6 +255,7 @@ Effects are addressed by id (`ec..`) or display name in commands |---|---|---:|:-:|:-:|---| | Matte Choker | `ec.matte.mattechoker` | 7 | GPU | 32 | Implemented | | Mocha shape | `ec.obsolete.mochashape` | 6 | | 32 | Implemented | +| Object Matte | `ec.matte.objectmatte` | 13 | | 32 | Implemented | | Refine Hard Matte | `ec.matte.refinehard` | 16 | GPU | 32 | Implemented | | Refine Soft Matte | `ec.matte.refinesoft` | 18 | GPU | 32 | Implemented | | Roto Brush & Refine Edge | `ec.matte.rotobrush` | 24 | | 32 | Implemented | diff --git a/docs/fidelity/README.md b/docs/fidelity/README.md index 025997d4..5afafee4 100644 --- a/docs/fidelity/README.md +++ b/docs/fidelity/README.md @@ -3,6 +3,9 @@ The [AE 2026 replacement acceptance gates](ae-2026-acceptance.md) define the broader target across workflows, rendered output, work-project compatibility, performance and platforms. +[Object Matte](object-matte.md) has a comparison plan (clips, prompt, per-frame IoU and the other +metrics) but no measurement yet. + The first G1 comparison measures eight original scalar Rotation cases against a live After Effects 2026 session through AEsync 2.0.4: Easy Ease, overlapping influences, asymmetric ease, overshoot, descending values, equal endpoints, 100%/100% influence and 0.1%/100% influence. Each case has diff --git a/docs/fidelity/object-matte.md b/docs/fidelity/object-matte.md new file mode 100644 index 00000000..da4e3bd3 --- /dev/null +++ b/docs/fidelity/object-matte.md @@ -0,0 +1,73 @@ +# Object Matte: comparison plan (not measured yet) + +**Status: untested.** There is no After Effects 26.2+ comparison of Object Matte yet. Nothing on +this page is a measurement. It lists what to compare once an After Effects reference session +is available, so the first run can be repeated and scored without guesswork. + +EffectCraft side, as of #783: the Object Matte effect (`ec.matte.objectmatte`), the Object Matte +tool in the Composition viewer, `objectMatte.apply` / `.propagate` / `.freeze` / +`.status`, and segmentations kept in the disk cache. The point (and an optional box) is +segmented by Roto Brush's engine: MobileSAM when it is installed (Settings ▸ Roto Brush, with +the box as the model's box prompt), otherwise the classical graph cut. The result is tracked +frame by frame with optical flow, and the model re-prompts each frame from the carried matte. +Record which engine was active, because the two are separate cases. + +## Test clips + +All original, generated or openly licensed; none of them goes into the repository. + +1. **Synthetic disk**: the moving textured disk over a textured background from + `crates/engine/src/tests_roto.rs` (`draw(center(f))`, 320×240, 25 fps, 25 frames, radius + 40 px, about 4 px a frame to the right). Export it as a PNG sequence. Its ground truth is + analytic (`gt(center(f))`). +2. **Synthetic disk with a jump**: the same disk with `jumping(f)` (it jumps at frame 8). This + checks how each tool handles losing the object. +3. **Two objects**: the disk plus a second still, differently coloured square that touches it + on some frames, prompted once with a point only and once with a box around the disk. This + checks the box prompt. +4. **Real footage**: one openly licensed clip (CC0 or CC BY) with a person or object moving + against clutter, at least 1080p and 5 s long. Record its URL, licence and SHA-256 next to the + results, and keep the clip itself out of the repository. + +## Prompt and settings + +- Reference frame 0. The point at the object's centre, in layer pixels (clip 1: `[90, 110]`). + The box, where used, is `[centre − 55, centre + 55]` on each axis. +- Track forward to the last frame (Frames After = clip length − 1, Frames Before = 0). +- Matte refinement at its defaults on both sides: Feather 0, Contrast 0, Shift Edge 0, no + Invert. Output the matte itself (View ▸ Matte in EffectCraft, the alpha channel in After + Effects) as a full-resolution PNG sequence. +- Record the After Effects build, the OS, the GPU, and the exact click position in After + Effects. It is a viewer click, so convert it to layer pixels with the comp's zoom. + +Keep the After Effects mattes in gitignored `plan/aftereffects/ref/object-matte/`. Only an +aggregate results file (like `keyframe-ease.json`) and the comparison script may be committed. + +## Metrics (per frame, then mean / worst) + +- **IoU against After Effects**: binarize both mattes at alpha 0.5, then intersection over + union. This is the headline number. `effectcraft_track::roto::iou` computes it, and + `roto.status` already reports it for Roto Brush through `compareTo`; Object Matte needs the + same for exported mattes. +- **IoU against ground truth** for the synthetic clips, for both applications. It tells + "differs from After Effects" apart from "worse than After Effects". +- **Boundary error**: the mean and 95th-percentile distance (px) between the two mattes' 0.5 + contours, which shows edge disagreement that IoU dilutes on large objects. +- **Soft-edge error**: the mean absolute alpha difference inside a 3 px band around the After + Effects contour. +- **Temporal stability**: the frame-to-frame IoU change of each matte on clip 1, where the + object moves rigidly, so jitter shows. +- **Drift**: the frame where IoU against ground truth first drops below 0.9 (clips 1 and 2). +- **Time**: seconds per frame to track the clip, warm and cold, on the same machine, with the + disk cache on and off. + +Proposed pass bar (a maintainer decision, not yet agreed): mean IoU against After Effects +≥ 0.95 and worst ≥ 0.90 on clips 1 and 3, with MobileSAM. With the classical engine the numbers +are reported but not gated. + +## Reproduce (to write) + +The script does not exist yet. It should: generate the synthetic clips; run EffectCraft +headless (`effectcraft-cli run … objectMatte.apply … objectMatte.propagate {"wait":true}`, then +render the matte); drive After Effects through ExtendScript or AEsync to apply Object Matte with +the same prompt and render its alpha; then compute the metrics above into a JSON results file. diff --git a/docs/gaps.md b/docs/gaps.md index 25da664f..d7b8211c 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -70,7 +70,7 @@ are grouped by area; the issue tracker has the rest. | 11 | **Text rendering and layout** | Open: #453 CJK text in comps, #316 paragraph alignment inverted (maintainers' tests show Left/Right correct; asked for a project), #417 soft at Fit (mostly fixed by #444), #550 toolbar font options. Closed: #483, #583 | Text is in almost every motion graphic | 10–20 h | G2 · [ui-parity.md](ui-parity.md) | | 12 | **3D interaction and depth** | No 3D transform gizmos (#392, #347); camera aperture (#260); reflections (#261); extruded surface materials beyond Front / Bevel / Side / Back Color (fb5a5e9a; #315 closed); #307 reopened (the viewer's renderer button now shows and switches the renderer, #645); Z position (#489) closed | Advanced 3D is After Effects' growth area since 2023 | 20–35 h | G2 · [ui-parity.md](ui-parity.md) | | 13 | **3D model formats and materials** | After Effects 26 imports FBX, USD/USDZ, STL besides glTF and OBJ (`usd_plugins`), and Substance 3D materials with mesh displacement (26.0–26.2); we read glTF, OBJ, STL (binary and ASCII, with facet colours) and FBX 7.x (binary and ASCII: meshes, hierarchy, Lambert/Phong materials and textures; no FBX animation or skinning yet) | Product shots and 3D scenes come as FBX/USDZ | 25–45 h | — · [file-format-parity.md](file-format-parity.md) | -| 14 | **Object Matte** (26.2, disk cache in 26.5) | Not present; MobileSAM is already in `effectcraft-segment` and could back it | The new one-click roto; Roto Brush covers part of it | 15–30 h | G7 · [target-app-parity.md](target-app-parity.md) | +| 14 | **Object Matte** (26.2, disk cache in 26.5) | First slice (#783): the Object Matte effect (Effect ▸ Matte) and `objectMatte.apply` turn a point (and optional box) on a reference frame into a matte and follow the object up to 600 frames each way, or segment every frame on its own; Roto Brush's engine, cache and model (MobileSAM from Settings ▸ Roto Brush, classical graph cut without it). Since then: the Object Matte tool in the viewer (click / drag a box, hover crosshair), the box as the model's box prompt, background tracking with progress and cancel, Freeze, and segmentations kept in the disk cache (26.5). Missing: a comparison with After Effects ([plan](fidelity/object-matte.md)) | The new one-click roto | 10–20 h | G7 · [target-app-parity.md](target-app-parity.md) | | 15 | **Hardware video decode and encode** | No VideoToolbox, NVENC, Quick Sync or VA-API; #498 asks for it (answered: CUDA / NVENC need Nvidia's proprietary SDK through FFI, which the no-`unsafe` rule forbids); open PR #819 adds hardware decoding via `filmcraft-platform`; FilmCraft has VideoToolbox / NVENC / VA-API paths to borrow | 4K playback and export speed | 25–45 h | G5/G6 · [hardware-parity.md](hardware-parity.md) | | 16 | **Localization depth** | Menus in 10 languages (ja, zh-hans, zh-hant, uk, es, pt-br, ru, fr, de, cs: 624 rows each); panels in 6 (ja, zh-hans, zh-hant, uk, pt-br, cs); effect names and ~1,650 effect parameter names untranslated; Korean and Italian (After Effects ships them) absent; Hangul probe added (#599 closed) | After Effects is used heavily in Japan, China, Korea, Brazil, Europe | 60–110 h + native review | G9 · [localization-parity.md](localization-parity.md) | | 17 | **Media Browser lists files import refuses** | `media_browser.rs` lists AVI; `media::is_importable` has none of them (Targa and Radiance HDR import since #745; DPX and Cineon since #756, log shown as stored: no Cineon Settings conversion yet; MXF OP1a / OP-Atom since #767; FilmCraft has no AVI demuxer); #431 and #324 closed | Confusing failures on common VFX formats | 15–30 h | G6 · [file-format-parity.md](file-format-parity.md) | diff --git a/docs/target-app-parity.md b/docs/target-app-parity.md index 2f312cd2..89464bcd 100644 --- a/docs/target-app-parity.md +++ b/docs/target-app-parity.md @@ -122,7 +122,7 @@ compositing and effects dominate; audio is light). Hours are to full parity for | Text and typography | 7% | 94% | 50% | 20–35 | CJK text in comps renders wrongly (#453); paragraph alignment inverted (#316); soft text at Fit (#417, mostly fixed by #444); toolbar font options (#550); #483 and #583 are closed; variable-font filter landed (#741) | | Effects and animation presets | 14% | 85% | 40% | 70–130 | All 306 effects exist, but rendered output is unmeasured against After Effects; 54 built-in presets (text, Transitions, Backgrounds) against After Effects' 621 ([effects-parity.md](effects-parity.md)); open effect bugs (#580 crash, #463, #369, #473, #474; #514 and #526 are closed); effect colour labels (26.5); see [effects-parity.md](effects-parity.md) | | 3D (Classic and Advanced) | 6% | 82% | 35% | 35–65 | No FBX/USD/STL models, no Substance materials, mesh displacement unverified; no 3D transform gizmos (#392, #347); reflections (#261); camera aperture (#260); #489 and #315 are closed | -| Tracking and AI-assisted tools | 5% | 80% | 35% | 35–70 | No Object Matte (26.2); Roto Brush (MobileSAM) and face tracking (MediaPipe) unmeasured against After Effects; camera-tracker view (#505) | +| Tracking and AI-assisted tools | 5% | 80% | 35% | 35–70 | Object Matte (26.2) has the effect, the viewer tool (click / box), background tracking, Freeze and the 26.5 disk cache, not yet compared with After Effects; Roto Brush (MobileSAM) and face tracking (MediaPipe) unmeasured against After Effects; camera-tracker view (#505) | | Preview and cache | 7% | 92% | 40% | 25–50 | Playback stutters on cached frames (#595, #304); late audio (#504, #335); #329 and #554 are closed; no benchmark against After Effects | | Render queue and export | 7% | 85% | 55% | 25–45 | No hardware encoding; ProRes 4444 colour tag (#340); rendering over an existing file (#325); #446 and #372 (8-bit uncompressed QuickTime, #778) are closed; HEVC/AV1 without B-frames | | Audio | 2% | 85% | 45% | 6–12 | Audio sync and crackle (#335); audio to keyframes depth | @@ -144,7 +144,7 @@ Feature depth (≈ 50%) feeds the overall *ready* number with the other dimensio | Stability | 45% | 40–80 | Open crash reports #580, #501, #426, #341, #234, #879 (#591, #306 and #511 are closed); no CI runs the tests on pull requests | [gaps.md](gaps.md#g3-stability-and-a-green-main) | | Platforms | 65% | 30–60 | Releases for macOS, Windows x64/x86/ARM64, Linux, FreeBSD and the web (more than After Effects), but Linux and Windows users hit basic bugs | [gaps.md](gaps.md#g2-real-user-reliability-on-every-platform) | | Ecosystem and plug-ins | 20% | 40–80 | After Effects scripts and expressions mostly carry over; no After Effects SDK plug-ins, `.ffx` presets or `.mogrt`; our WebAssembly plug-in API has no third-party plug-ins yet | [plugins.md](plugins.md) | -| AI features | 35% | 40–80 (overlaps Tracking) | Roto Brush (MobileSAM) and face tracking (MediaPipe) as optional downloads; no Object Matte; the After Effects bundle ships Whisper, BERT and Marian tokenizers and a shot-cut model we have no equivalent of | [gaps.md](gaps.md) | +| AI features | 35% | 40–80 (overlaps Tracking) | Roto Brush (MobileSAM) and face tracking (MediaPipe) as optional downloads; Object Matte (point / box prompt, same model; viewer tool, background tracking, Freeze, disk cache) unmeasured against After Effects; the After Effects bundle ships Whisper, BERT and Marian tokenizers and a shot-cut model we have no equivalent of | [gaps.md](gaps.md) | | Agent control (beyond After Effects) | ahead | — | Every command is drivable over MCP, the control channel and the CLI | [agents.md](agents.md) | **Totals.** Summing the rows double-counts: AI overlaps Tracking, performance overlaps Preview,