From 953fbbc730bee402cdaf7913498758138cf22eb7 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 11 Oct 2026 13:32:10 +0200 Subject: [PATCH] #783: Object Matte effect from a point / box prompt, tracked Problem: After Effects 26.2's Object Matte (pick an object with a click, optionally a box, get a matte that follows it) had no counterpart; only stroke-driven Roto Brush existed. Fix: an Object Matte effect (Effect > Matte, ec.matte.objectmatte) turns the Object Point (and optional box) on a Reference Frame into Roto Brush strokes and segments them with Roto Brush's engine, content-keyed cache and the model chosen in Settings > Roto Brush (MobileSAM, classical graph cut without it). Track Object carries the matte frame by frame up to 600 frames each way; otherwise every frame is segmented on its own. Feather, Contrast, Shift Edge, Invert and a Matte view. The engine keeps its Input Key current so upstream edits recompute it. objectMatte.apply creates one from agents / the CLI. Unusable prompts pass the layer through. Co-Authored-By: Claude Opus 5.5 --- crates/effects/src/lib.rs | 13 +- crates/effects/src/object_matte.rs | 468 ++++++++++++++++++++++++ crates/effects/src/roto.rs | 8 +- crates/engine/src/commands/roto_cmds.rs | 90 +++++ crates/engine/src/roto.rs | 20 +- crates/engine/src/tests_roto.rs | 75 ++++ docs/effects.md | 7 +- docs/gaps.md | 2 +- docs/target-app-parity.md | 4 +- 9 files changed, 674 insertions(+), 13 deletions(-) create mode 100644 crates/effects/src/object_matte.rs diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 7e968297..d32cfb2e 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -40,6 +40,7 @@ pub mod mocha_shape; mod noise; mod noise2; mod noise3; +pub mod object_matte; mod obsolete; mod ocio; pub mod ocio_config; @@ -497,6 +498,7 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(camera_tracker::specs()); v.extend(face_track::specs()); v.extend(roto::specs()); + v.extend(object_matte::specs()); v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); for s in v.iter_mut() { if GPU_EFFECTS.contains(&s.id) { @@ -805,7 +807,15 @@ pub const PARAM_GROUPS: &[(&str, &str)] = &[ /// Display names of nested parameter groups by match id. fn group_name(m: &str) -> &str { - warp_stab::GROUPS.iter().chain(roto::GROUPS).chain(keylight::GROUPS).chain(PARAM_GROUPS).find(|(id, _)| *id == m).map(|(_, n)| *n).unwrap_or(m) + warp_stab::GROUPS + .iter() + .chain(roto::GROUPS) + .chain(object_matte::GROUPS) + .chain(keylight::GROUPS) + .chain(PARAM_GROUPS) + .find(|(id, _)| *id == m) + .map(|(_, n)| *n) + .unwrap_or(m) } /// The nested group at `path` (`borders/autoScale`) under `g`, created on first use. @@ -1180,6 +1190,7 @@ pub const TIME_DEPENDENT: &[&str] = &[ camera_tracker::ID, // Each frame has its own segmentation. roto::ID, + object_matte::ID, ]; /// See [`TIME_DEPENDENT`]. diff --git a/crates/effects/src/object_matte.rs b/crates/effects/src/object_matte.rs new file mode 100644 index 00000000..3b7b7c28 --- /dev/null +++ b/crates/effects/src/object_matte.rs @@ -0,0 +1,468 @@ +//! Object Matte (Effect ▸ Matte): isolate one object from a click, and follow it through the +//! shot (After Effects 26.2's Object Matte, from its public description: pick an object with a +//! point, optionally a box around it, and get a matte that tracks it). +//! +//! The prompt is the **Object Point** (layer pixels) and, with **Use Box**, a box around the +//! object. On the **Reference Frame** it is segmented by the same engine as Roto Brush: the +//! trained model chosen in Settings ▸ Roto Brush (MobileSAM, `effectcraft_segment`) when one is +//! installed, the classical graph cut otherwise (the point is a foreground stroke, the box's +//! edges, or without a box the layer's border, background strokes). With **Track Object** the +//! matte is then carried frame by frame up to **Frames Before** / **Frames After** away (optical +//! flow, re-segmented by the model prompted with the carried matte); without it every frame is +//! segmented on its own from the prompt (useful when the point is animated). +//! +//! Like Roto Brush, the segmentation is derived data in Roto Brush's content-keyed cache +//! ([`crate::roto::cached`]): the chain seed holds the engine-maintained Input Key (what the +//! frames are made of), the layer size, the frame rate, the mode and the model, and every +//! frame's key the prompt, so edits upstream or to the prompt recompute it and undo never sees a +//! stale matte. **Feather**, **Contrast** and **Shift Edge** shape the matte, **Invert** keeps +//! everything but the object and **View ▸ Matte** shows the matte in grey. + +use std::sync::Arc; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::Image; +use effectcraft_segment::MaskModel; +use effectcraft_track::roto::refine::MatteParams; +use effectcraft_track::roto::{RotoData, SegOpts, Stroke, StrokeKind}; +use rayon::prelude::*; + +use crate::roto::{self, Chain, INPUT_KEY}; +use crate::warp_stab::param; +use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; + +/// The effect's id. +pub const ID: &str = "ec.matte.objectmatte"; + +/// Display names of the instance's parameter groups. +pub const GROUPS: &[(&str, &str)] = &[("objectTracking", "Tracking"), ("objectMatte", "Matte")]; + +pub const VIEWS: [&str; 2] = ["Final Output", "Matte"]; + +/// Most frames a matte is carried from the reference frame each way. +pub const MAX_REACH: f64 = 600.0; +/// Default reach each way (frames). +pub const DEFAULT_REACH: f64 = 30.0; + +/// Radius (layer pixels) of the foreground stroke the Object Point becomes. +const POINT_RADIUS: f64 = 3.0; +/// Radius (layer pixels) of the background strokes along the box / the layer's border. +const EDGE_RADIUS: f64 = 2.0; + +pub fn specs() -> Vec { + vec![EffectSpec { + id: ID, + name: "Object Matte", + category: "Matte", + params: vec![ + p("objectPoint", "Object Point", Value::Vec2([0.5, 0.5]), ParamUi::Point), + p("useBox", "Use Box", Value::Bool(false), ParamUi::Checkbox), + p("boxTopLeft", "Box Top Left", Value::Vec2([0.25, 0.25]), ParamUi::Point), + p("boxBottomRight", "Box Bottom Right", Value::Vec2([0.75, 0.75]), ParamUi::Point), + p("objectTracking/trackObject", "Track Object", Value::Bool(true), ParamUi::Checkbox), + p("objectTracking/referenceFrame", "Reference Frame", num(0.0), ParamUi::Number), + p("objectTracking/framesBefore", "Frames Before", num(DEFAULT_REACH), slider(0.0, MAX_REACH, 0.0, 120.0, 0)), + p("objectTracking/framesAfter", "Frames After", num(DEFAULT_REACH), slider(0.0, MAX_REACH, 0.0, 120.0, 0)), + p("objectMatte/feather", "Feather", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)), + p("objectMatte/contrast", "Contrast", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)), + p("objectMatte/shiftEdge", "Shift Edge", num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 0)), + p("invert", "Invert", Value::Bool(false), ParamUi::Checkbox), + p("view", "View", Value::Enum(0), popup(&VIEWS)), + p(INPUT_KEY, "Input Key", Value::Str(String::new()), ParamUi::Hidden), + ], + render, + gpu: false, + float: true, + }] +} + +fn f(params: &Params, id: &str) -> f64 { + param(params, id).map(Value::as_f64).unwrap_or(0.0) +} +fn b(params: &Params, id: &str) -> bool { + param(params, id).map(Value::as_bool).unwrap_or(false) +} +fn v2(params: &Params, id: &str) -> Option<[f64; 2]> { + let v = param(params, id)?.as_vec2(); + v.iter().all(|c| c.is_finite()).then_some(v) +} +fn s<'a>(params: &'a Params, id: &str) -> &'a str { + match param(params, id) { + Some(Value::Str(s)) => s, + _ => "", + } +} + +fn fnv(h: &mut u64, bytes: &[u8]) { + for b in bytes { + *h ^= *b as u64; + *h = h.wrapping_mul(0x0100_0000_01b3); + } +} + +/// A frame count parameter (finite, clamped to `lo..=hi`). +fn frames(params: &Params, id: &str, lo: f64, hi: f64) -> i64 { + let v = f(params, id); + if v.is_finite() { v.round().clamp(lo, hi) as i64 } else { 0 } +} + +/// The reference frame (layer frame). +pub fn reference_frame(params: &Params) -> i64 { + frames(params, "objectTracking/referenceFrame", -1e9, 1e9) +} + +/// Whether the matte is carried from the reference frame (else every frame is segmented on its +/// own). +pub fn tracks(params: &Params) -> bool { + b(params, "objectTracking/trackObject") +} + +/// The prompt as Roto Brush strokes on layer frame `frame`: the Object Point as a foreground +/// stroke; the box's edges as background strokes; without a box, the layer's border when there +/// is no model (the graph cut needs background samples; a model does not, and a border stroke +/// would cut objects touching the frame edge). `None` when the point is unusable. +pub fn prompt_strokes(params: &Params, layer_size: [f64; 2], frame: i64, with_model: bool) -> Option> { + let [w, h] = layer_size; + if !(w.is_finite() && h.is_finite() && w >= 1.0 && h >= 1.0) { + return None; + } + let pt = v2(params, "objectPoint")?; + let inside = |q: [f64; 2]| q[0] >= 0.0 && q[1] >= 0.0 && q[0] <= w && q[1] <= h; + if !inside(pt) { + return None; + } + let mut out = vec![Stroke { kind: StrokeKind::Fg, frame, radius: POINT_RADIUS, points: vec![pt] }]; + let rect = |x0: f64, y0: f64, x1: f64, y1: f64| vec![[x0, y0], [x1, y0], [x1, y1], [x0, y1], [x0, y0]]; + let boxed = b(params, "useBox").then(|| Some((v2(params, "boxTopLeft")?, v2(params, "boxBottomRight")?))).flatten(); + match boxed { + Some((a, c)) => { + let (x0, x1) = (a[0].min(c[0]).clamp(0.0, w), a[0].max(c[0]).clamp(0.0, w)); + let (y0, y1) = (a[1].min(c[1]).clamp(0.0, h), a[1].max(c[1]).clamp(0.0, h)); + // The point must lie inside the box, clear of its edges, for the box to mean anything. + let m = POINT_RADIUS + EDGE_RADIUS; + if pt[0] - m <= x0 || pt[0] + m >= x1 || pt[1] - m <= y0 || pt[1] + m >= y1 { + return None; + } + out.push(Stroke { kind: StrokeKind::Bg, frame, radius: EDGE_RADIUS, points: rect(x0, y0, x1, y1) }); + } + None if !with_model => { + let i = EDGE_RADIUS; + if w <= 4.0 * i || h <= 4.0 * i { + return None; + } + out.push(Stroke { kind: StrokeKind::Bg, frame, radius: EDGE_RADIUS, points: rect(i, i, w - i, h - i) }); + } + None => {} + } + Some(out) +} + +/// The segmentation data the matte of layer frame `f` comes from: with Track Object, the prompt +/// on the reference frame and the span it is carried over; otherwise the prompt on `f` alone. +pub fn roto_data(params: &Params, layer_size: [f64; 2], f: i64, with_model: bool) -> Option { + if tracks(params) { + let base = reference_frame(params); + let before = frames(params, "objectTracking/framesBefore", 0.0, MAX_REACH); + let after = frames(params, "objectTracking/framesAfter", 0.0, MAX_REACH); + let strokes = prompt_strokes(params, layer_size, base, with_model)?; + Some(RotoData { base: Some(base), span: [base.saturating_sub(before), base.saturating_add(after)], strokes }) + } else { + Some(RotoData { base: Some(f), span: [f, f], strokes: prompt_strokes(params, layer_size, f, with_model)? }) + } +} + +/// Seed of the chain keys: what the frames are (the Input Key), the layer size, the frame rate, +/// the mode and the model. The prompt is in each frame's key. +pub fn seed(params: &Params, layer_size: [f64; 2], fps: f64, model: Option<&dyn MaskModel>) -> u64 { + let mut h = 0xcbf2_9ce4_8422_2325; + fnv(&mut h, ID.as_bytes()); + fnv(&mut h, s(params, INPUT_KEY).as_bytes()); + for v in [layer_size[0], layer_size[1], fps] { + fnv(&mut h, &v.to_bits().to_le_bytes()); + } + fnv(&mut h, &[tracks(params) as u8]); + fnv(&mut h, model.map_or(effectcraft_segment::CLASSICAL, |m| m.info().id).as_bytes()); + h +} + +/// Segmentation settings at `scale` (Roto Brush's defaults). +fn seg_opts(scale: f64) -> SegOpts { + SegOpts { scale, search_radius: 15.0, motion_threshold: 10.0, motion_damping: 20.0, best: false } +} + +/// Matte settings: Feather, Contrast, Shift Edge (no Refine Edge band). +pub fn matte_params(params: &Params) -> MatteParams { + let num = |id: &str| { + let v = f(params, id); + if v.is_finite() { v } else { 0.0 } + }; + MatteParams { + rb_feather: num("objectMatte/feather").clamp(0.0, 100.0), + rb_contrast: num("objectMatte/contrast").clamp(0.0, 100.0), + rb_shift: num("objectMatte/shiftEdge").clamp(-100.0, 100.0), + decontaminate: false, + ..MatteParams::default() + } +} + +/// What a frame's matte is: the object's alpha on the layer grid, or why there is none. +#[derive(Clone, Debug, PartialEq)] +pub enum FrameMatte { + /// The object's alpha (0–1) on the frame's layer grid. + Alpha(Vec), + /// The prompt is unusable (point outside the layer, outside the box…): the layer passes. + NoPrompt, + /// The frame is beyond the tracked range, or its matte could not be computed. + Outside, +} + +/// The matte of layer frame `f` on the layer grid of `img` (the effect's input on that frame at +/// `scale`), computing what is missing. `frame(g)`: the input on frame g on the same grid. +#[allow(clippy::too_many_arguments)] +pub fn frame_matte( + params: &Params, + layer_size: [f64; 2], + fps: f64, + f: i64, + scale: f64, + img: &Image, + model: Option>, + frame: &mut dyn FnMut(i64) -> Option>, + cancel: &dyn Fn() -> bool, +) -> FrameMatte { + let Some(data) = roto_data(params, layer_size, f, model.is_some()) else { return FrameMatte::NoPrompt }; + if !data.in_span(f) { + return FrameMatte::Outside; + } + let keys = data.chain_keys(seed(params, layer_size, fps, model.as_deref())); + let chain = Chain { data: Arc::new(data), keys, opts: seg_opts(scale), refine_radius: 0.0, model }; + let Some(seg) = chain.ensure(f, frame, cancel) else { return FrameMatte::Outside }; + let (w, h) = (img.width as usize, img.height as usize); + let seg = if seg.w == w && seg.h == h { (*seg).clone() } else { roto::resample(&seg, w, h) }; + FrameMatte::Alpha(roto::final_alpha(img, &seg, None, None, &matte_params(params), 0.0, scale)) +} + +fn render(ctx: &EffectCtx, buf: Buf) -> Buf { + if ctx.adjustment { + return buf; + } + let fps = ctx.fps(); + let f = roto::frame_of(ctx.time, fps); + let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + let img = Arc::new(roto::layer_grid(&buf, ctx.layer_size)); + let (w, h) = (img.width as usize, img.height as usize); + let host = ctx.env.host; + let effects = ctx.env.effect_index; + let layer_size = ctx.layer_size; + let me = img.clone(); + let mut frame = |g: i64| -> Option> { + if g == f { + return Some(me.clone()); + } + let other = host?.self_at(g as f64 / fps, effects)?; + let im = roto::layer_grid(&other, layer_size); + (im.width as usize == w && im.height as usize == h).then(|| Arc::new(im)) + }; + let alpha = match frame_matte(ctx.params, layer_size, fps, f, sc, &img, roto::installed_model(), &mut frame, &|| false) { + FrameMatte::Alpha(a) if a.len() == w * h => Some(a), + FrameMatte::NoPrompt => return buf, + _ => None, + }; + compose(&buf, alpha.as_deref(), w, h, b(ctx.params, "invert"), param(ctx.params, "view").map(Value::as_enum).unwrap_or(0) == 1) +} + +/// The output: the input keyed by the matte (`None`: no object, all zero), or the matte itself. +fn compose(buf: &Buf, alpha: Option<&[f32]>, w: usize, h: usize, invert: bool, view_matte: bool) -> Buf { + let (ox, oy) = (buf.offset[0].round() as i64, buf.offset[1].round() as i64); + let mut out = Buf { img: Image::new(buf.img.width, buf.img.height), offset: buf.offset, scale: buf.scale }; + out.img.rows_mut().for_each(|(by, row)| { + for (bx, px) in row.iter_mut().enumerate() { + let (x, y) = (bx as i64 - ox, by as i64 - oy); + let a = if x >= 0 && y >= 0 && (x as usize) < w && (y as usize) < h { + alpha.and_then(|a| a.get(y as usize * w + x as usize)).copied().unwrap_or(0.0) + } else { + 0.0 + }; + let a = if invert { 1.0 - a } else { a }.clamp(0.0, 1.0); + let src = buf.img.get(bx as i64, by as i64); + *px = if view_matte { + let v = a * src[3]; + [v, v, v, 1.0] + } else { + [src[0] * a, src[1] * a, src[2] * a, src[3] * a] + }; + } + }); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EffectEnv, EffectHost, LayerPixels}; + + /// A red disk of radius 15 moving 2 px a frame right over a textured blue background + /// (100×80, 25 fps), plus a still green square in the corner. + fn frame_at(fi: i64) -> Image { + let c = [40.0 + 2.0 * fi as f64, 40.0]; + let mut img = Image::new(100, 80); + for y in 0..80 { + for x in 0..100 { + let d = ((x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1])) - 15.0; + let a = (0.5 - d).clamp(0.0, 1.0) as f32; + let tex = ((x * 7 + y * 13) % 9) as f32 / 60.0; + let bg = if x >= 80 && y >= 60 { [0.1, 0.8, 0.2] } else { [0.1, 0.4 + tex, 0.7] }; + let fg = [0.85 + tex, 0.2, 0.1]; + img.set(x, y, [fg[0] * a + bg[0] * (1.0 - a), fg[1] * a + bg[1] * (1.0 - a), fg[2] * a + bg[2] * (1.0 - a), 1.0]); + } + } + img + } + fn frame_of_time(t: f64) -> Image { + frame_at((t * 25.0 + 1e-6).floor() as i64) + } + + struct Seq; + impl EffectHost for Seq { + fn layer(&self, _: u64, _: bool) -> Option { + None + } + fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option> { + None + } + fn self_at(&self, t: f64, _: usize) -> Option { + Some(Buf { img: frame_of_time(t), offset: [0.0; 2], scale: 1.0 }) + } + } + + fn params(vals: &[(&str, Value)]) -> Params { + let s = &specs()[0]; + let mut p = Params { values: s.params.iter().map(|p| (p.id.to_string(), crate::default_value(p, [100.0, 80.0]))).collect() }; + for (k, v) in vals { + p.values.insert(k.to_string(), v.clone()); + } + p + } + + fn run(p: &Params, fi: i64) -> Image { + let host = Seq; + let t = fi as f64 / 25.0 + 0.001; + let ctx = EffectCtx { + params: p, + time: t, + layer_size: [100.0, 80.0], + seed: 1, + adjustment: false, + env: EffectEnv { host: Some(&host), frame_rate: 25.0, ..Default::default() }, + }; + crate::apply(&specs()[0], &ctx, Buf { img: frame_of_time(t), offset: [0.0; 2], scale: 1.0 }).img + } + + /// Pixels whose alpha disagrees with the disk centred at `cx` (1 px of slack at the edge). + fn wrong(out: &Image, cx: f64) -> usize { + let mut n = 0; + for y in 0..80u32 { + for x in 0..100u32 { + let d = (x as f64 + 0.5 - cx).hypot(y as f64 + 0.5 - 40.0); + let a = out.data[(y * 100 + x) as usize][3]; + if (d < 14.0 && a < 0.5) || (d > 16.0 && a > 0.5) { + n += 1; + } + } + } + n + } + + #[test] + fn a_click_isolates_the_object_and_tracks_it() { + let p = params(&[("objectPoint", Value::Vec2([40.0, 40.0]))]); + // The reference frame: the disk, nothing else. + let out = run(&p, 0); + assert!(wrong(&out, 40.0) < 20, "{} wrong pixels on the reference frame", wrong(&out, 40.0)); + // Six frames on (carried through frames 1–5), the disk has moved 12 px and the matte with it. + let out = run(&p, 6); + assert!(wrong(&out, 52.0) < 30, "{} wrong pixels on frame 6", wrong(&out, 52.0)); + // Beyond Frames After: no matte. + let p2 = params(&[("objectPoint", Value::Vec2([40.0, 40.0])), ("objectTracking/framesAfter", Value::Scalar(3.0))]); + assert!(run(&p2, 6).data.iter().all(|px| px[3] == 0.0)); + // Invert keeps everything but the disk; View ▸ Matte shows it in grey, opaque. + let inv = run(¶ms(&[("objectPoint", Value::Vec2([40.0, 40.0])), ("invert", Value::Bool(true))]), 0); + assert!(inv.data[40 * 100 + 40][3] < 0.5 && inv.data[5 * 100 + 95][3] > 0.5); + let m = run(¶ms(&[("objectPoint", Value::Vec2([40.0, 40.0])), ("view", Value::Enum(1))]), 0); + assert!(m.data[40 * 100 + 40][0] > 0.5 && m.data[5 * 100 + 95][0] < 0.5 && m.data.iter().all(|px| px[3] == 1.0)); + } + + #[test] + fn without_tracking_every_frame_follows_the_prompt() { + // The point animated onto the disk at frame 9: frame 9 is segmented from it alone. + let p = params(&[("objectPoint", Value::Vec2([58.0, 40.0])), ("objectTracking/trackObject", Value::Bool(false))]); + let out = run(&p, 9); + assert!(wrong(&out, 58.0) < 20, "{} wrong pixels", wrong(&out, 58.0)); + } + + #[test] + fn a_box_and_bad_prompts() { + // A box around the green square, the point in it: the square, not the disk. + let p = params(&[ + ("objectPoint", Value::Vec2([90.0, 70.0])), + ("useBox", Value::Bool(true)), + ("boxTopLeft", Value::Vec2([74.0, 54.0])), + ("boxBottomRight", Value::Vec2([100.0, 80.0])), + ]); + let out = run(&p, 0); + assert!(out.data[70 * 100 + 90][3] > 0.5 && out.data[40 * 100 + 40][3] < 0.5); + // Unusable prompts pass the layer through instead of failing: a point off the layer, a + // non-finite point, a point outside its box. + let src = frame_at(0); + for vals in [ + vec![("objectPoint", Value::Vec2([-5.0, 40.0]))], + vec![("objectPoint", Value::Vec2([f64::NAN, 40.0]))], + vec![ + ("objectPoint", Value::Vec2([40.0, 40.0])), + ("useBox", Value::Bool(true)), + ("boxTopLeft", Value::Vec2([60.0, 10.0])), + ("boxBottomRight", Value::Vec2([90.0, 30.0])), + ], + ] { + let p = params(&vals); + assert_eq!(run(&p, 0).data, src.data, "{vals:?}"); + } + // A huge reference frame or reach doesn't overflow. + let p = params(&[("objectTracking/referenceFrame", Value::Scalar(1e300)), ("objectTracking/framesBefore", Value::Scalar(f64::INFINITY))]); + assert!(run(&p, 0).data.iter().all(|px| px[3] == 0.0)); + } + + /// A stand-in model (the weights are an optional download): a disk around the first + /// foreground point, radius 15, so the test sees the model's answer in the matte. + #[derive(Default)] + struct Stub { + calls: std::sync::atomic::AtomicUsize, + } + impl MaskModel for Stub { + fn info(&self) -> &'static effectcraft_segment::ModelInfo { + &effectcraft_segment::MOBILE_SAM + } + fn segment(&self, _: &[[f32; 3]], w: usize, h: usize, prompt: &effectcraft_segment::Prompt) -> effectcraft_segment::Result> { + self.calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let Some(([cx, cy], _)) = prompt.points.iter().find(|(_, fg)| *fg) else { return Err("no point".into()) }; + Ok((0..w * h).map(|i| (((i % w) as f32 + 0.5 - cx).hypot((i / w) as f32 + 0.5 - cy) < 15.0) as u8 as f32).collect()) + } + } + + #[test] + fn segments_with_the_installed_model_without_a_border_stroke() { + let p = params(&[("objectPoint", Value::Vec2([40.0, 40.0]))]); + // With a model, no border stroke: an object touching the frame edge stays whole. + assert_eq!(prompt_strokes(&p, [100.0, 80.0], 0, true).map(|s| s.len()), Some(1)); + assert_eq!(prompt_strokes(&p, [100.0, 80.0], 0, false).map(|s| s.len()), Some(2)); + // The model's answer is part of the cache key, and drives the matte. + let stub = Arc::new(Stub::default()); + assert_ne!(seed(&p, [100.0, 80.0], 25.0, Some(stub.as_ref())), seed(&p, [100.0, 80.0], 25.0, None)); + let img = frame_at(0); + let mut frame = |g: i64| Some(Arc::new(frame_at(g))); + let FrameMatte::Alpha(a) = frame_matte(&p, [100.0, 80.0], 25.0, 0, 1.0, &img, Some(stub.clone()), &mut frame, &|| false) else { panic!("no matte") }; + assert!(a[40 * 100 + 40] > 0.5 && a[5 * 100 + 5] < 0.5); + assert_eq!(stub.calls.load(std::sync::atomic::Ordering::Relaxed), 1); + } +} diff --git a/crates/effects/src/roto.rs b/crates/effects/src/roto.rs index d514e80a..cd2d240e 100644 --- a/crates/effects/src/roto.rs +++ b/crates/effects/src/roto.rs @@ -195,6 +195,12 @@ pub fn model_for(params: &Params) -> Option> { if e(params, "version") == 0 { return None; } + installed_model() +} + +/// The trained model chosen in Settings ▸ Roto Brush, if one is installed (Roto Brush 2.0 / 3.0 +/// and Object Matte segment with it). +pub fn installed_model() -> Option> { model_slot().read().ok().and_then(|m| m.clone()) } @@ -392,7 +398,7 @@ pub fn downscale(s: &FrameSeg, scale: f64) -> FrameSeg { resample(s, w, h) } -fn resample(s: &FrameSeg, w: usize, h: usize) -> FrameSeg { +pub(crate) fn resample(s: &FrameSeg, w: usize, h: usize) -> FrameSeg { if s.w == w && s.h == h { return s.clone(); } diff --git a/crates/engine/src/commands/roto_cmds.rs b/crates/engine/src/commands/roto_cmds.rs index 95326b42..ff459d0b 100644 --- a/crates/engine/src/commands/roto_cmds.rs +++ b/crates/engine/src/commands/roto_cmds.rs @@ -1,6 +1,9 @@ //! Roto Brush & Refine Edge commands: strokes (the Roto Brush and Refine Edge tools in the Layer //! panel), propagation, the segmentation span, Freeze / Unfreeze, tool options and status. //! +//! `objectMatte.apply` adds an Object Matte effect (Effect ▸ Matte) from a point / box prompt on +//! a frame; it segments with the same engine and model as Roto Brush. +//! //! `roto.stroke` records a stroke into the layer's Roto Brush & Refine Edge effect (applying the //! effect on the first stroke, as After Effects does); the first foreground stroke sets the base //! frame and a span of 20 frames each side. Strokes on a frozen effect are refused (Unfreeze @@ -51,6 +54,15 @@ pub fn specs() -> Vec { always, options ), + cmd!( + "objectMatte.apply", + "Object Matte", + [], + None, + "{layer?, point: [x, y] (layer pixels: the object), box?: [x0, y0, x1, y1] (layer pixels, around the object), frame? (reference layer frame; default current), track?: follow the object from that frame (default true), before?, after? (frames to follow it; default 30, up to 600)}", + has_comp, + object_matte_apply + ), query!( "roto.status", "Roto Brush Status", @@ -422,3 +434,81 @@ fn status(s: &mut Session, p: &Value) -> Result { out["computed"] = json!(computed.iter().filter(|(_, c)| **c).map(|(f, _)| *f).collect::>()); Ok(out) } + +/// Two finite numbers. +fn xy_p(p: &Value, k: &str, n: usize, cmd: &str) -> Result>> { + let Some(v) = p.get(k).filter(|v| !v.is_null()) else { return Ok(None) }; + let a = v.as_array().filter(|a| a.len() == n).ok_or_else(|| bad(cmd, format!("`{k}` is {n} numbers (layer pixels)")))?; + let out: Vec = a.iter().filter_map(Value::as_f64).filter(|x| x.is_finite()).collect(); + if out.len() != n { + return Err(bad(cmd, format!("`{k}` is {n} numbers (layer pixels)"))); + } + Ok(Some(out)) +} + +fn object_matte_apply(s: &mut Session, p: &Value) -> Result { + use effectcraft_effects::object_matte as om; + let cmd = "objectMatte.apply"; + let (cid, lid) = layer_p(s, p, cmd)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or_else(|| bad(cmd, "no layer"))?; + if !layer.has_video() || layer.switches.adjustment || layer.effects().is_none() { + return Err(bad(cmd, "Object Matte works on footage, solid and composition layers")); + } + let size = rt::layer_size(&s.project, comp, layer); + let point = xy_p(p, "point", 2, cmd)?.ok_or_else(|| bad(cmd, "missing `point` [x, y] (layer pixels)"))?; + let inside = |x: f64, y: f64| (0.0..=size[0]).contains(&x) && (0.0..=size[1]).contains(&y); + if !inside(point[0], point[1]) { + return Err(bad(cmd, format!("`point` is outside the layer ({}×{})", size[0], size[1]))); + } + let bx = xy_p(p, "box", 4, cmd)?; + if let Some(b) = &bx { + let (x0, x1, y0, y1) = (b[0].min(b[2]), b[0].max(b[2]), b[1].min(b[3]), b[1].max(b[3])); + if !(point[0] > x0 && point[0] < x1 && point[1] > y0 && point[1] < y1) { + return Err(bad(cmd, "`point` must be inside `box`")); + } + } + let frame = frame_p(s, p, cid, layer); + let limits = rt::frame_limits(comp, layer); + if frame < limits[0] || frame > limits[1] { + return Err(bad(cmd, format!("frame {frame} is outside the layer ({}–{})", limits[0], limits[1]))); + } + let track = b_p(p, "track").unwrap_or(true); + let reach = |k: &str| f_p(p, k).filter(|v| v.is_finite()).unwrap_or(om::DEFAULT_REACH).round().clamp(0.0, om::MAX_REACH); + let (before, after) = (reach("before"), reach("after")); + let spec = effectcraft_effects::find(om::ID).ok_or_else(|| EngineError::Other("Object Matte effect missing".into()))?; + let uid = s.edit("Object Matte", None, |proj, _| { + let mut next = proj.next_id; + let l = layer_mut(proj, cid, lid)?; + let fxg = l.props.sub_mut("effects").ok_or_else(|| bad(cmd, "this layer has no effects"))?; + let n = fxg.groups().filter(|g| rt::is_object_matte(g)).count(); + let name = if n == 0 { spec.name.to_string() } else { format!("{} {}", spec.name, n + 1) }; + let mut g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), &name, size); + let mut set = |path: &str, v: KV| { + let (grp, leaf) = path.rsplit_once('/').map_or((None, path), |(a, b)| (Some(a), b)); + let target = match grp { + Some(sub) => g.sub_mut(sub), + None => Some(&mut g), + }; + if let Some(pr) = target.and_then(|t| t.get_mut(leaf)) { + pr.value = v; + } + }; + set("objectPoint", KV::Vec2([point[0], point[1]])); + if let Some(b) = &bx { + set("useBox", KV::Bool(true)); + set("boxTopLeft", KV::Vec2([b[0].min(b[2]), b[1].min(b[3])])); + set("boxBottomRight", KV::Vec2([b[0].max(b[2]), b[1].max(b[3])])); + } + set("objectTracking/trackObject", KV::Bool(track)); + set("objectTracking/referenceFrame", KV::Scalar(frame as f64)); + set("objectTracking/framesBefore", KV::Scalar(before)); + set("objectTracking/framesAfter", KV::Scalar(after)); + let u = g.uid; + fxg.children.push(g.into()); + proj.next_id = proj.next_id.max(next); + Ok(u) + })?; + s.state.selected_layers = vec![lid]; + Ok(json!({"layer": lid.0, "effect": uid, "frame": frame, "track": track, "span": [frame - before as i64, frame + after as i64]})) +} diff --git a/crates/engine/src/roto.rs b/crates/engine/src/roto.rs index ca92a74b..b8c2c649 100644 --- a/crates/engine/src/roto.rs +++ b/crates/engine/src/roto.rs @@ -147,6 +147,12 @@ pub(crate) fn is_roto(g: &PropGroup) -> bool { matches!(&g.kind, GroupKind::Effect { effect } if effect == fx::ID) } +/// An Object Matte instance (its Input Key is kept current like Roto Brush's; it segments on +/// demand while rendering). +pub(crate) fn is_object_matte(g: &PropGroup) -> bool { + matches!(&g.kind, GroupKind::Effect { effect } if effect == effectcraft_effects::object_matte::ID) +} + fn str_of(g: &PropGroup, m: &str) -> String { match g.get(m).map(|p| &p.value) { Some(Value::Str(s)) => s.clone(), @@ -188,8 +194,9 @@ pub fn input_key(p: &Project, comp: ItemId, layer: &Layer, uid: Uid) -> String { format!("{:016x}", fnv(&s)) } -/// After an edit: refresh Input Keys; frozen mattes made from other frames are dropped. -/// Returns the instances whose segmentation changed (to propagate again). +/// After an edit: refresh Input Keys (Roto Brush and Object Matte); frozen mattes made from +/// other frames are dropped. Returns the Roto Brush instances whose segmentation changed (to +/// propagate again). pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> { let mut todo = vec![]; let mut set = vec![]; @@ -197,13 +204,16 @@ pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerI let ItemKind::Comp(c) = &it.kind else { continue }; for l in &c.layers { let Some(fxg) = l.effects() else { continue }; - for g in fxg.groups().filter(|g| is_roto(g)) { + for g in fxg.groups().filter(|g| is_roto(g) || is_object_matte(g)) { let key = input_key(after, *cid, l, g.uid); + let roto = is_roto(g); let old = before.comp(*cid).and_then(|bc| bc.layer(l.id)).and_then(|ol| ol.props.find_group(g.uid)); if str_of(g, INPUT_KEY) != key { set.push((*cid, l.id, g.uid, key)); - todo.push((*cid, l.id, g.uid)); - } else if old.is_none_or(|o| str_of(o, fx::STROKES) != str_of(g, fx::STROKES)) { + if roto { + todo.push((*cid, l.id, g.uid)); + } + } else if roto && old.is_none_or(|o| str_of(o, fx::STROKES) != str_of(g, fx::STROKES)) { todo.push((*cid, l.id, g.uid)); } } diff --git a/crates/engine/src/tests_roto.rs b/crates/engine/src/tests_roto.rs index c1cbd2c8..cdf58540 100644 --- a/crates/engine/src/tests_roto.rs +++ b/crates/engine/src/tests_roto.rs @@ -351,3 +351,78 @@ fn offloaded_propagation_streams_segmentations_back() { s.execute("roto.propagate", json!({"layer": clip.0, "wait": true})).unwrap(); assert!(s.offloaded.is_empty()); } + +/// Object Matte: one click on the disk on frame 0 isolates it there and on the frames it is +/// followed to; the Input Key follows the layer's frames; bad prompts are refused. +#[test] +fn object_matte_from_a_click() { + hold_roto_cache_test_lock(); + let (mut s, clip, solid) = setup(|f| draw(center(f))); + let cid = s.active_comp_id().unwrap(); + s.edit("remove solid", None, |p, _| { + p.comp_mut(cid).unwrap().layers.retain(|l| l.id != solid); + for it in p.items.values_mut() { + if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind { + f.path = "object-matte.mov".into(); + } + } + Ok(()) + }) + .unwrap(); + s.execute("time.set", json!({"time": 0})).unwrap(); + let c = center(0); + let r = s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "point": c, "after": 6})).unwrap(); + assert_eq!(r["frame"], 0); + assert_eq!(r["span"], json!([-30, 6])); + let uid = r["effect"].as_u64().unwrap(); + let key = |s: &Session| { + let g = s.active_comp().unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().clone(); + crate::roto::params_static(&g).get(effectcraft_effects::roto::INPUT_KEY).cloned() + }; + let k0 = key(&s); + assert!(matches!(&k0, Some(effectcraft_keyframe::Value::Str(k)) if !k.is_empty()), "{k0:?}"); + let wrong = |s: &Session, f: u32| { + let c = center(f); + alpha_at(s, f) + .iter() + .enumerate() + .filter(|(i, v)| { + let d = (((*i as u32 % W) as f64 + 0.5 - c[0]).hypot((*i as u32 / W) as f64 + 0.5 - c[1])) - R; + (d < -3.0 && **v < 0.5) || (d > 3.0 && **v > 0.5) + }) + .count() + }; + let w0 = wrong(&s, 0); + assert!(w0 < 40, "{w0} wrong pixels on the reference frame"); + let w4 = wrong(&s, 4); + assert!(w4 < 60, "{w4} wrong pixels on frame 4"); + // Past Frames After: transparent. + assert!(alpha_at(&s, 8).iter().all(|v| *v == 0.0)); + // Other frames under the effect: a new Input Key (the mattes are recomputed). + s.edit("swap clip", None, |p, _| { + for it in p.items.values_mut() { + if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind { + f.path = "object-matte-2.mov".into(); + } + } + Ok(()) + }) + .unwrap(); + assert_ne!(key(&s), k0); + // Bad prompts are refused. + for bad in [ + json!({"layer": clip.0}), + json!({"layer": clip.0, "point": [-10, 5]}), + json!({"layer": clip.0, "point": [10, "x"]}), + json!({"layer": clip.0, "point": c, "box": [0, 0, 10, 10]}), + json!({"layer": clip.0, "point": c, "frame": 999}), + ] { + assert!(s.execute_checked("objectMatte.apply", bad.clone()).is_err(), "{bad}"); + } + // A box and a static prompt are accepted; the second instance is numbered. + let r = s + .execute_checked("objectMatte.apply", json!({"layer": clip.0, "point": c, "box": [c[0] + 50.0, c[1] + 50.0, c[0] - 50.0, c[1] - 50.0], "track": false})) + .unwrap(); + let g = s.active_comp().unwrap().layer(clip).unwrap().props.find_group(r["effect"].as_u64().unwrap()).unwrap().clone(); + assert_eq!(g.name, "Object Matte 2"); +} diff --git a/docs/effects.md b/docs/effects.md index e498cf1a..84f16ef8 100644 --- a/docs/effects.md +++ b/docs/effects.md @@ -3,11 +3,11 @@ -EffectCraft ships 306 effects with 3014 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. +EffectCraft ships 307 effects with 3028 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. - **GPU**: 280 effects also run on the GPU compositor with identical results. -- **32**: 306 effects process 32-bit float (HDR, overbright) pixels without clamping. -- **Status**: 306 implemented in full, 0 partial (what is missing is listed). +- **32**: 307 effects process 32-bit float (HDR, overbright) pixels without clamping. +- **Status**: 307 implemented in full, 0 partial (what is missing is listed). Effects are addressed by id (`ec..`) or display name in commands, scripts and the control channel, and every parameter by its id (`effects/#1/blurriness`). @@ -255,6 +255,7 @@ Effects are addressed by id (`ec..`) or display name in commands |---|---|---:|:-:|:-:|---| | Matte Choker | `ec.matte.mattechoker` | 7 | GPU | 32 | Implemented | | Mocha shape | `ec.obsolete.mochashape` | 6 | | 32 | Implemented | +| Object Matte | `ec.matte.objectmatte` | 13 | | 32 | Implemented | | Refine Hard Matte | `ec.matte.refinehard` | 16 | GPU | 32 | Implemented | | Refine Soft Matte | `ec.matte.refinesoft` | 18 | GPU | 32 | Implemented | | Roto Brush & Refine Edge | `ec.matte.rotobrush` | 24 | | 32 | Implemented | diff --git a/docs/gaps.md b/docs/gaps.md index acb22a63..577f7bd6 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -70,7 +70,7 @@ are grouped by area; the issue tracker has the rest. | 11 | **Text rendering and layout** | #453 CJK text in comps, #316 paragraph alignment inverted, #483 squeezed at 4:3, #417 soft at Fit, #583 font search, #550 toolbar font options | Text is in almost every motion graphic | 10–20 h | G2 · [ui-parity.md](ui-parity.md) | | 12 | **3D interaction and depth** | No 3D transform gizmos (#392, #347); Z position (#489); camera aperture (#260); reflections (#261); bevel options (#315); 3D issue reopened (#307) | Advanced 3D is After Effects' growth area since 2023 | 20–35 h | G2 · [ui-parity.md](ui-parity.md) | | 13 | **3D model formats and materials** | After Effects 26 imports FBX, USD/USDZ, STL besides glTF and OBJ (`usd_plugins`), and Substance 3D materials with mesh displacement (26.0–26.2); we read glTF and OBJ | Product shots and 3D scenes come as FBX/USDZ | 25–45 h | — · [file-format-parity.md](file-format-parity.md) | -| 14 | **Object Matte** (26.2, disk cache in 26.5) | Not present; MobileSAM is already in `effectcraft-segment` and could back it | The new one-click roto; Roto Brush covers part of it | 15–30 h | G7 · [target-app-parity.md](target-app-parity.md) | +| 14 | **Object Matte** (26.2, disk cache in 26.5) | First slice (#783): the Object Matte effect (Effect ▸ Matte) and `objectMatte.apply` turn a point (and optional box) on a reference frame into a matte and follow the object up to 600 frames each way, or segment every frame on its own; Roto Brush's engine, cache and model (MobileSAM from Settings ▸ Roto Brush, classical graph cut without it). Missing: the click / hover tool in the viewer, a background propagation job, the disk cache, comparison with After Effects | The new one-click roto | 10–20 h | G7 · [target-app-parity.md](target-app-parity.md) | | 15 | **Hardware video decode and encode** | No VideoToolbox, NVENC, Quick Sync or VA-API; #498 asks for it; FilmCraft has VideoToolbox / NVENC / VA-API paths to borrow | 4K playback and export speed | 25–45 h | G5/G6 · [hardware-parity.md](hardware-parity.md) | | 16 | **Localization depth** | Menus in 6 languages; panels in 4; effect names and ~1,650 effect parameter names untranslated; French, German, Korean, Italian, Russian (After Effects ships them) absent; #599 Hangul boxes on macOS | After Effects is used heavily in Japan, China, Korea, Brazil, Europe | 60–110 h + native review | G9 · [localization-parity.md](localization-parity.md) | | 17 | **Media Browser lists files import refuses** | `media_browser.rs` lists TGA, DPX, HDR, AVI, MXF; `media::is_importable` has none of them; #431 any image imports as a sequence; #324 MP3 goes silent | Confusing failures on common VFX formats (DPX, TGA) | 15–30 h | G6 · [file-format-parity.md](file-format-parity.md) | diff --git a/docs/target-app-parity.md b/docs/target-app-parity.md index 12d37245..f0ed90b6 100644 --- a/docs/target-app-parity.md +++ b/docs/target-app-parity.md @@ -122,7 +122,7 @@ compositing and effects dominate; audio is light). Hours are to full parity for | Text and typography | 7% | 94% | 50% | 20–35 | CJK text in comps renders wrongly (#453); paragraph alignment inverted (#316); squeezed text at 4:3 (#483); font search (#583); soft text at Fit (#417); variable-font filter (26.3) | | Effects and animation presets | 14% | 85% | 40% | 70–130 | All 306 effects exist, but rendered output is unmeasured against After Effects; 8 built-in presets against After Effects' 621; open effect bugs (#580 crash, #514, #526, #463, #369, #473); effect colour labels (26.5); see [effects-parity.md](effects-parity.md) | | 3D (Classic and Advanced) | 6% | 82% | 35% | 35–65 | No FBX/USD/STL models, no Substance materials, mesh displacement unverified; no 3D transform gizmos (#392, #347); Z position (#489); reflections (#261); camera aperture (#260); bevel options (#315) | -| Tracking and AI-assisted tools | 5% | 80% | 35% | 35–70 | No Object Matte (26.2); Roto Brush (MobileSAM) and face tracking (MediaPipe) unmeasured against After Effects; camera-tracker view (#505) | +| Tracking and AI-assisted tools | 5% | 80% | 35% | 35–70 | Object Matte (26.2) is a first slice: an effect and command from a point / box prompt, no viewer tool or disk cache; Roto Brush (MobileSAM) and face tracking (MediaPipe) unmeasured against After Effects; camera-tracker view (#505) | | Preview and cache | 7% | 92% | 40% | 25–50 | Playback stutters on cached frames (#595, #304); slow scrubbing (#329); late audio (#554, #504, #335); no benchmark against After Effects | | Render queue and export | 7% | 85% | 55% | 25–45 | No hardware encoding; ProRes 4444 colour tag (#340); rendering over an existing file (#325); default macOS output location (#446); uncompressed single-file output (#372); HEVC/AV1 without B-frames | | Audio | 2% | 85% | 45% | 6–12 | Audio sync and crackle (#335); audio to keyframes depth | @@ -144,7 +144,7 @@ Feature depth (≈ 50%) feeds the overall *ready* number with the other dimensio | Stability | 45% | 40–80 | Open crash reports #580, #591, #501, #426, #341, #306, #234, #511; no CI runs the tests on pull requests | [gaps.md](gaps.md#g3-stability-and-a-green-main) | | Platforms | 65% | 30–60 | Releases for macOS, Windows x64/x86/ARM64, Linux, FreeBSD and the web (more than After Effects), but Linux and Windows users hit basic bugs | [gaps.md](gaps.md#g2-real-user-reliability-on-every-platform) | | Ecosystem and plug-ins | 20% | 40–80 | After Effects scripts and expressions mostly carry over; no After Effects SDK plug-ins, `.ffx` presets or `.mogrt`; our WebAssembly plug-in API has no third-party plug-ins yet | [plugins.md](plugins.md) | -| AI features | 35% | 40–80 (overlaps Tracking) | Roto Brush (MobileSAM) and face tracking (MediaPipe) as optional downloads; no Object Matte; the After Effects bundle ships Whisper, BERT and Marian tokenizers and a shot-cut model we have no equivalent of | [gaps.md](gaps.md) | +| AI features | 35% | 40–80 (overlaps Tracking) | Roto Brush (MobileSAM) and face tracking (MediaPipe) as optional downloads; Object Matte (point / box prompt, same model) without its viewer tool or disk cache; the After Effects bundle ships Whisper, BERT and Marian tokenizers and a shot-cut model we have no equivalent of | [gaps.md](gaps.md) | | Agent control (beyond After Effects) | ahead | — | Every command is drivable over MCP, the control channel and the CLI | [agents.md](agents.md) | **Totals.** Summing the rows double-counts: AI overlaps Tracking, performance overlaps Preview,