Merge pull request #792 from vangel76/gap-object-matte
# Conflicts: # crates/ui-egui/src/i18n/ui.rs # docs/gaps.md # docs/target-app-parity.md
This commit is contained in:
Brandon Thomas
commit
4574fb62bc
|
|
@ -404,7 +404,7 @@ export function frameWorkerStart(base, onMessage, gpu) {
|
|||
// ends in a checksum) and read the layer buffers they prefetch; the page lists, reads and
|
||||
// deletes.
|
||||
const CACHE_ROOT = "effectcraft-cache";
|
||||
const CACHE_KINDS = ["frames", "layers"];
|
||||
const CACHE_KINDS = ["frames", "layers", "mattes"];
|
||||
const cacheDirHandles = {};
|
||||
|
||||
async function cacheDir(kind = "frames") {
|
||||
|
|
@ -419,6 +419,7 @@ async function cacheDir(kind = "frames") {
|
|||
/// The folder and file name of an entry name (`layers/<hex>.ecc` or `<hex>.ecc`).
|
||||
async function entryDir(name) {
|
||||
if (name.startsWith("layers/")) return [await cacheDir("layers"), name.slice(7)];
|
||||
if (name.startsWith("mattes/")) return [await cacheDir("mattes"), name.slice(7)];
|
||||
return [await cacheDir("frames"), name];
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -105,19 +105,23 @@ pub fn init() {
|
|||
});
|
||||
}
|
||||
|
||||
/// An entry's name in `js/host.js`'s cache folders (`layers/` for layer buffers).
|
||||
/// An entry's name in `js/host.js`'s cache folders (`layers/` for layer buffers, `mattes/` for segmentations).
|
||||
pub(crate) fn name_of(kind: Kind, key: u128) -> String {
|
||||
match kind {
|
||||
Kind::Frame => disk_cache::entry_name(key),
|
||||
Kind::Layer => format!("layers/{}", disk_cache::entry_name(key)),
|
||||
Kind::Matte => format!("mattes/{}", disk_cache::entry_name(key)),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_name(name: &str) -> Option<(Kind, u128)> {
|
||||
match name.strip_prefix("layers/") {
|
||||
Some(n) => Some((Kind::Layer, disk_cache::parse_entry_name(n)?)),
|
||||
None => Some((Kind::Frame, disk_cache::parse_entry_name(name)?)),
|
||||
if let Some(n) = name.strip_prefix("layers/") {
|
||||
return Some((Kind::Layer, disk_cache::parse_entry_name(n)?));
|
||||
}
|
||||
if let Some(n) = name.strip_prefix("mattes/") {
|
||||
return Some((Kind::Matte, disk_cache::parse_entry_name(n)?));
|
||||
}
|
||||
Some((Kind::Frame, disk_cache::parse_entry_name(name)?))
|
||||
}
|
||||
|
||||
/// Layer entries (32 hex digits) a frame worker should read before its next frame: recent
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ pub mod mocha_shape;
|
|||
mod noise;
|
||||
mod noise2;
|
||||
mod noise3;
|
||||
pub mod object_matte;
|
||||
mod obsolete;
|
||||
mod ocio;
|
||||
pub mod ocio_config;
|
||||
|
|
@ -533,6 +534,7 @@ pub fn registry() -> &'static [EffectSpec] {
|
|||
v.extend(camera_tracker::specs());
|
||||
v.extend(face_track::specs());
|
||||
v.extend(roto::specs());
|
||||
v.extend(object_matte::specs());
|
||||
v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name)));
|
||||
for s in v.iter_mut() {
|
||||
if GPU_EFFECTS.contains(&s.id) {
|
||||
|
|
@ -841,7 +843,15 @@ pub const PARAM_GROUPS: &[(&str, &str)] = &[
|
|||
|
||||
/// Display names of nested parameter groups by match id.
|
||||
fn group_name(m: &str) -> &str {
|
||||
warp_stab::GROUPS.iter().chain(roto::GROUPS).chain(keylight::GROUPS).chain(PARAM_GROUPS).find(|(id, _)| *id == m).map(|(_, n)| *n).unwrap_or(m)
|
||||
warp_stab::GROUPS
|
||||
.iter()
|
||||
.chain(roto::GROUPS)
|
||||
.chain(object_matte::GROUPS)
|
||||
.chain(keylight::GROUPS)
|
||||
.chain(PARAM_GROUPS)
|
||||
.find(|(id, _)| *id == m)
|
||||
.map(|(_, n)| *n)
|
||||
.unwrap_or(m)
|
||||
}
|
||||
|
||||
/// The nested group at `path` (`borders/autoScale`) under `g`, created on first use.
|
||||
|
|
@ -1216,6 +1226,7 @@ pub const TIME_DEPENDENT: &[&str] = &[
|
|||
camera_tracker::ID,
|
||||
// Each frame has its own segmentation.
|
||||
roto::ID,
|
||||
object_matte::ID,
|
||||
];
|
||||
|
||||
/// See [`TIME_DEPENDENT`].
|
||||
|
|
|
|||
|
|
@ -0,0 +1,691 @@
|
|||
//! Object Matte (Effect ▸ Matte): isolate one object from a click, and follow it through the
|
||||
//! shot (After Effects 26.2's Object Matte, from its public description: pick an object with a
|
||||
//! point, optionally a box around it, and get a matte that tracks it).
|
||||
//!
|
||||
//! The prompt is the **Object Point** (layer pixels) and, with **Use Box**, a box around the
|
||||
//! object. On the **Reference Frame** it is segmented by the same engine as Roto Brush: the
|
||||
//! trained model chosen in Settings ▸ Roto Brush (MobileSAM, `effectcraft_segment`) when one is
|
||||
//! installed, the classical graph cut otherwise. The model gets the point as a foreground point
|
||||
//! and the box as its box prompt; the graph cut gets the point as a foreground stroke and the
|
||||
//! box's edges (without a box, the layer's border) as background strokes. With **Track Object** the
|
||||
//! matte is then carried frame by frame up to **Frames Before** / **Frames After** away (optical
|
||||
//! flow, re-segmented by the model prompted with the carried matte); without it every frame is
|
||||
//! segmented on its own from the prompt (useful when the point is animated).
|
||||
//!
|
||||
//! Like Roto Brush, the segmentation is derived data in Roto Brush's content-keyed cache
|
||||
//! ([`crate::roto::cached`]): the chain seed holds the engine-maintained Input Key (what the
|
||||
//! frames are made of), the layer size, the frame rate, the mode and the model, and every
|
||||
//! frame's key the prompt, so edits upstream or to the prompt recompute it and undo never sees a
|
||||
//! stale matte. The engine fills that cache in a background job (`objectMatte.propagate`,
|
||||
//! outward from the reference frame over [`job_frames`]) and **Freeze** (`objectMatte.freeze`)
|
||||
//! stores the final mattes in the hidden **Frozen** parameter ([`FrozenCache`], as Roto Brush
|
||||
//! does): they are used, without segmenting, while [`frozen_key`] still matches. **Feather**,
|
||||
//! **Contrast** and **Shift Edge** shape the matte, **Invert** keeps everything but the object
|
||||
//! and **View ▸ Matte** shows the matte in grey.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use effectcraft_keyframe::Value;
|
||||
use effectcraft_project::ParamUi;
|
||||
use effectcraft_raster::Image;
|
||||
use effectcraft_segment::MaskModel;
|
||||
use effectcraft_track::roto::refine::MatteParams;
|
||||
use effectcraft_track::roto::{FrameSeg, RotoData, SegOpts, Stroke, StrokeKind, rle};
|
||||
use rayon::prelude::*;
|
||||
|
||||
use crate::roto::{self, Chain, FROZEN, FrozenCache, INPUT_KEY};
|
||||
use crate::warp_stab::param;
|
||||
use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider};
|
||||
|
||||
/// The effect's id.
|
||||
pub const ID: &str = "ec.matte.objectmatte";
|
||||
|
||||
/// Display names of the instance's parameter groups.
|
||||
pub const GROUPS: &[(&str, &str)] = &[("objectTracking", "Tracking"), ("objectMatte", "Matte")];
|
||||
|
||||
pub const VIEWS: [&str; 2] = ["Final Output", "Matte"];
|
||||
|
||||
/// Most frames a matte is carried from the reference frame each way.
|
||||
pub const MAX_REACH: f64 = 600.0;
|
||||
/// Default reach each way (frames).
|
||||
pub const DEFAULT_REACH: f64 = 30.0;
|
||||
|
||||
/// Radius (layer pixels) of the foreground stroke the Object Point becomes.
|
||||
const POINT_RADIUS: f64 = 3.0;
|
||||
/// Radius (layer pixels) of the background strokes along the box / the layer's border.
|
||||
const EDGE_RADIUS: f64 = 2.0;
|
||||
|
||||
pub fn specs() -> Vec<EffectSpec> {
|
||||
vec![EffectSpec {
|
||||
id: ID,
|
||||
name: "Object Matte",
|
||||
category: "Matte",
|
||||
params: vec![
|
||||
p("objectPoint", "Object Point", Value::Vec2([0.5, 0.5]), ParamUi::Point),
|
||||
p("useBox", "Use Box", Value::Bool(false), ParamUi::Checkbox),
|
||||
p("boxTopLeft", "Box Top Left", Value::Vec2([0.25, 0.25]), ParamUi::Point),
|
||||
p("boxBottomRight", "Box Bottom Right", Value::Vec2([0.75, 0.75]), ParamUi::Point),
|
||||
p("objectTracking/trackObject", "Track Object", Value::Bool(true), ParamUi::Checkbox),
|
||||
p("objectTracking/referenceFrame", "Reference Frame", num(0.0), ParamUi::Number),
|
||||
p("objectTracking/framesBefore", "Frames Before", num(DEFAULT_REACH), slider(0.0, MAX_REACH, 0.0, 120.0, 0)),
|
||||
p("objectTracking/framesAfter", "Frames After", num(DEFAULT_REACH), slider(0.0, MAX_REACH, 0.0, 120.0, 0)),
|
||||
p("objectMatte/feather", "Feather", num(0.0), slider(0.0, 100.0, 0.0, 20.0, 1)),
|
||||
p("objectMatte/contrast", "Contrast", num(0.0), slider(0.0, 100.0, 0.0, 100.0, 0)),
|
||||
p("objectMatte/shiftEdge", "Shift Edge", num(0.0), slider(-100.0, 100.0, -100.0, 100.0, 0)),
|
||||
p("invert", "Invert", Value::Bool(false), ParamUi::Checkbox),
|
||||
p("view", "View", Value::Enum(0), popup(&VIEWS)),
|
||||
p(INPUT_KEY, "Input Key", Value::Str(String::new()), ParamUi::Hidden),
|
||||
p(FROZEN, "Frozen", Value::Str(String::new()), ParamUi::Hidden),
|
||||
],
|
||||
render,
|
||||
gpu: false,
|
||||
float: true,
|
||||
}]
|
||||
}
|
||||
|
||||
fn f(params: &Params, id: &str) -> f64 {
|
||||
param(params, id).map(Value::as_f64).unwrap_or(0.0)
|
||||
}
|
||||
fn b(params: &Params, id: &str) -> bool {
|
||||
param(params, id).map(Value::as_bool).unwrap_or(false)
|
||||
}
|
||||
fn v2(params: &Params, id: &str) -> Option<[f64; 2]> {
|
||||
let v = param(params, id)?.as_vec2();
|
||||
v.iter().all(|c| c.is_finite()).then_some(v)
|
||||
}
|
||||
fn s<'a>(params: &'a Params, id: &str) -> &'a str {
|
||||
match param(params, id) {
|
||||
Some(Value::Str(s)) => s,
|
||||
_ => "",
|
||||
}
|
||||
}
|
||||
|
||||
fn fnv(h: &mut u64, bytes: &[u8]) {
|
||||
for b in bytes {
|
||||
*h ^= *b as u64;
|
||||
*h = h.wrapping_mul(0x0100_0000_01b3);
|
||||
}
|
||||
}
|
||||
|
||||
/// A frame count parameter (finite, clamped to `lo..=hi`).
|
||||
fn frames(params: &Params, id: &str, lo: f64, hi: f64) -> i64 {
|
||||
let v = f(params, id);
|
||||
if v.is_finite() { v.round().clamp(lo, hi) as i64 } else { 0 }
|
||||
}
|
||||
|
||||
/// The reference frame (layer frame).
|
||||
pub fn reference_frame(params: &Params) -> i64 {
|
||||
frames(params, "objectTracking/referenceFrame", -1e9, 1e9)
|
||||
}
|
||||
|
||||
/// Frames Before / After (finite, clamped to `0..=MAX_REACH`).
|
||||
pub fn reach(params: &Params) -> [i64; 2] {
|
||||
[frames(params, "objectTracking/framesBefore", 0.0, MAX_REACH), frames(params, "objectTracking/framesAfter", 0.0, MAX_REACH)]
|
||||
}
|
||||
|
||||
/// Whether the matte is carried from the reference frame (else every frame is segmented on its
|
||||
/// own).
|
||||
pub fn tracks(params: &Params) -> bool {
|
||||
b(params, "objectTracking/trackObject")
|
||||
}
|
||||
|
||||
/// The prompt as Roto Brush strokes on layer frame `frame`: the Object Point as a foreground
|
||||
/// stroke; the box as a box stroke (the model's box prompt) with a model, its edges as
|
||||
/// background strokes for the graph cut without one; without a box, the layer's border when
|
||||
/// there is no model (the graph cut needs background samples; a model does not, and a border
|
||||
/// stroke would cut objects touching the frame edge). `None` when the point is unusable.
|
||||
pub fn prompt_strokes(params: &Params, layer_size: [f64; 2], frame: i64, with_model: bool) -> Option<Vec<Stroke>> {
|
||||
let [w, h] = layer_size;
|
||||
if !(w.is_finite() && h.is_finite() && w >= 1.0 && h >= 1.0) {
|
||||
return None;
|
||||
}
|
||||
let pt = v2(params, "objectPoint")?;
|
||||
let inside = |q: [f64; 2]| q[0] >= 0.0 && q[1] >= 0.0 && q[0] <= w && q[1] <= h;
|
||||
if !inside(pt) {
|
||||
return None;
|
||||
}
|
||||
let mut out = vec![Stroke { kind: StrokeKind::Fg, frame, radius: POINT_RADIUS, points: vec![pt] }];
|
||||
let rect = |x0: f64, y0: f64, x1: f64, y1: f64| vec![[x0, y0], [x1, y0], [x1, y1], [x0, y1], [x0, y0]];
|
||||
let boxed = b(params, "useBox").then(|| Some((v2(params, "boxTopLeft")?, v2(params, "boxBottomRight")?))).flatten();
|
||||
match boxed {
|
||||
Some((a, c)) => {
|
||||
let (x0, x1) = (a[0].min(c[0]).clamp(0.0, w), a[0].max(c[0]).clamp(0.0, w));
|
||||
let (y0, y1) = (a[1].min(c[1]).clamp(0.0, h), a[1].max(c[1]).clamp(0.0, h));
|
||||
// The point must lie inside the box, clear of its edges, for the box to mean anything.
|
||||
let m = POINT_RADIUS + EDGE_RADIUS;
|
||||
if pt[0] - m <= x0 || pt[0] + m >= x1 || pt[1] - m <= y0 || pt[1] + m >= y1 {
|
||||
return None;
|
||||
}
|
||||
out.push(if with_model {
|
||||
// The model takes the box itself as its box prompt.
|
||||
Stroke { kind: StrokeKind::Box, frame, radius: 0.0, points: vec![[x0, y0], [x1, y1]] }
|
||||
} else {
|
||||
Stroke { kind: StrokeKind::Bg, frame, radius: EDGE_RADIUS, points: rect(x0, y0, x1, y1) }
|
||||
});
|
||||
}
|
||||
None if !with_model => {
|
||||
let i = EDGE_RADIUS;
|
||||
if w <= 4.0 * i || h <= 4.0 * i {
|
||||
return None;
|
||||
}
|
||||
out.push(Stroke { kind: StrokeKind::Bg, frame, radius: EDGE_RADIUS, points: rect(i, i, w - i, h - i) });
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// The segmentation data the matte of layer frame `f` comes from: with Track Object, the prompt
|
||||
/// on the reference frame and the span it is carried over; otherwise the prompt on `f` alone.
|
||||
pub fn roto_data(params: &Params, layer_size: [f64; 2], f: i64, with_model: bool) -> Option<RotoData> {
|
||||
if tracks(params) {
|
||||
let base = reference_frame(params);
|
||||
let [before, after] = reach(params);
|
||||
let strokes = prompt_strokes(params, layer_size, base, with_model)?;
|
||||
Some(RotoData { base: Some(base), span: [base.saturating_sub(before), base.saturating_add(after)], strokes })
|
||||
} else {
|
||||
Some(RotoData { base: Some(f), span: [f, f], strokes: prompt_strokes(params, layer_size, f, with_model)? })
|
||||
}
|
||||
}
|
||||
|
||||
/// Seed of the chain keys: what the frames are (the Input Key), the layer size, the frame rate,
|
||||
/// the mode and the model. The prompt is in each frame's key.
|
||||
pub fn seed(params: &Params, layer_size: [f64; 2], fps: f64, model: Option<&dyn MaskModel>) -> u64 {
|
||||
let mut h = 0xcbf2_9ce4_8422_2325;
|
||||
fnv(&mut h, ID.as_bytes());
|
||||
fnv(&mut h, s(params, INPUT_KEY).as_bytes());
|
||||
for v in [layer_size[0], layer_size[1], fps] {
|
||||
fnv(&mut h, &v.to_bits().to_le_bytes());
|
||||
}
|
||||
fnv(&mut h, &[tracks(params) as u8]);
|
||||
fnv(&mut h, model.map_or(effectcraft_segment::CLASSICAL, |m| m.info().id).as_bytes());
|
||||
h
|
||||
}
|
||||
|
||||
/// The layer frames a background job computes, outward from the reference frame: the tracked
|
||||
/// span (Frames Before / After around it), clipped to `limits` (the layer's frames). Without
|
||||
/// Track Object, the same frames, each segmented on its own.
|
||||
pub fn job_frames(params: &Params, limits: [i64; 2]) -> Vec<i64> {
|
||||
let base = reference_frame(params);
|
||||
let [before, after] = reach(params);
|
||||
let (lo, hi) = (base.saturating_sub(before).max(limits[0]), base.saturating_add(after).min(limits[1]));
|
||||
if base < limits[0] || base > limits[1] {
|
||||
return vec![];
|
||||
}
|
||||
let mut v = vec![base];
|
||||
v.extend(base.saturating_add(1)..=hi);
|
||||
v.extend((lo..base).rev());
|
||||
v
|
||||
}
|
||||
|
||||
/// The segmentation chain the matte of layer frame `f` comes from (`None`: no usable prompt).
|
||||
pub fn chain(params: &Params, layer_size: [f64; 2], fps: f64, f: i64, scale: f64, model: Option<Arc<dyn MaskModel>>) -> Option<Chain> {
|
||||
let data = roto_data(params, layer_size, f, model.is_some())?;
|
||||
let keys = data.chain_keys(seed(params, layer_size, fps, model.as_deref()));
|
||||
Some(Chain { data: Arc::new(data), keys, opts: seg_opts(scale), refine_radius: 0.0, model })
|
||||
}
|
||||
|
||||
/// Frame `f`'s segmentation at full resolution, computing what is missing (the background
|
||||
/// job's step). `None` outside the tracked span, without a prompt, or when cancelled.
|
||||
pub fn ensure(
|
||||
params: &Params,
|
||||
layer_size: [f64; 2],
|
||||
fps: f64,
|
||||
f: i64,
|
||||
model: Option<Arc<dyn MaskModel>>,
|
||||
frame: &mut dyn FnMut(i64) -> Option<Arc<Image>>,
|
||||
cancel: &dyn Fn() -> bool,
|
||||
) -> Option<Arc<FrameSeg>> {
|
||||
let ch = chain(params, layer_size, fps, f, 1.0, model)?;
|
||||
if !ch.data.in_span(f) {
|
||||
return None;
|
||||
}
|
||||
ch.ensure(f, frame, cancel)
|
||||
}
|
||||
|
||||
/// The chain key of each of `frames` (what a computed segmentation is cached under).
|
||||
pub fn frame_keys(params: &Params, layer_size: [f64; 2], fps: f64, frames: &[i64], model: Option<Arc<dyn MaskModel>>) -> Vec<(i64, u64)> {
|
||||
if tracks(params) {
|
||||
let Some(ch) = chain(params, layer_size, fps, 0, 1.0, model) else { return vec![] };
|
||||
return frames.iter().filter_map(|f| Some((*f, *ch.keys.get(f)?))).collect();
|
||||
}
|
||||
frames.iter().filter_map(|f| Some((*f, *chain(params, layer_size, fps, *f, 1.0, model.clone())?.keys.get(f)?))).collect()
|
||||
}
|
||||
|
||||
/// What frozen mattes depend on: what the frames are (the seed: Input Key, size, rate, mode,
|
||||
/// model), the prompt, the tracked span and the matte settings.
|
||||
pub fn frozen_key(params: &Params, layer_size: [f64; 2], fps: f64, model: Option<&dyn MaskModel>) -> String {
|
||||
let mut h = seed(params, layer_size, fps, model);
|
||||
let base = reference_frame(params);
|
||||
if let Some(st) = prompt_strokes(params, layer_size, base, model.is_some()) {
|
||||
fnv(&mut h, serde_json::to_string(&st).unwrap_or_default().as_bytes());
|
||||
}
|
||||
for v in [base, reach(params)[0], reach(params)[1]] {
|
||||
fnv(&mut h, &v.to_le_bytes());
|
||||
}
|
||||
fnv(&mut h, format!("{:?}", matte_params(params)).as_bytes());
|
||||
format!("{h:016x}")
|
||||
}
|
||||
|
||||
/// Whether the instance is frozen (has frozen mattes).
|
||||
pub fn is_frozen(params: &Params) -> bool {
|
||||
!s(params, FROZEN).is_empty()
|
||||
}
|
||||
|
||||
/// Frame `f`'s frozen matte on a `w`×`h` grid, when the instance is frozen, the mattes still
|
||||
/// match ([`frozen_key`]) and `f` is one of them.
|
||||
pub fn frozen_alpha(params: &Params, layer_size: [f64; 2], fps: f64, f: i64, w: usize, h: usize, model: Option<&dyn MaskModel>) -> Option<Vec<f32>> {
|
||||
let j = s(params, FROZEN);
|
||||
if j.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let fc = frozen_cache(j)?;
|
||||
if fc.key != frozen_key(params, layer_size, fps, model) {
|
||||
return None;
|
||||
}
|
||||
let fr = fc.frames.get(&f)?;
|
||||
let a = rle::decode(&fr.a)?;
|
||||
let [fw, fh] = fc.size;
|
||||
if fw == 0 || fh == 0 || a.len() != fw.checked_mul(fh)? {
|
||||
return None;
|
||||
}
|
||||
let af: Vec<f32> = a.iter().map(|v| *v as f32 / 255.0).collect();
|
||||
if fw == w && fh == h {
|
||||
return Some(af);
|
||||
}
|
||||
let mut out = vec![0.0f32; w * h];
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
let (sx, sy) = ((x as f64 + 0.5) * fw as f64 / w as f64, (y as f64 + 0.5) * fh as f64 / h as f64);
|
||||
out[y * w + x] = effectcraft_track::roto::matting::sample(&af, fw, fh, sx, sy);
|
||||
}
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// The parsed Frozen parameter (parsed once per content).
|
||||
fn frozen_cache(j: &str) -> Option<Arc<FrozenCache>> {
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
static C: OnceLock<Mutex<HashMap<u64, Arc<FrozenCache>>>> = OnceLock::new();
|
||||
let mut h = 0xcbf2_9ce4_8422_2325;
|
||||
fnv(&mut h, j.as_bytes());
|
||||
let c = C.get_or_init(Default::default);
|
||||
if let Some(d) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&h) {
|
||||
return Some(d.clone());
|
||||
}
|
||||
let d = Arc::new(FrozenCache::from_json(j)?);
|
||||
let mut m = c.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if m.len() > 8 {
|
||||
m.clear();
|
||||
}
|
||||
m.insert(h, d.clone());
|
||||
Some(d)
|
||||
}
|
||||
|
||||
/// Segmentation settings at `scale` (Roto Brush's defaults).
|
||||
fn seg_opts(scale: f64) -> SegOpts {
|
||||
SegOpts { scale, search_radius: 15.0, motion_threshold: 10.0, motion_damping: 20.0, best: false }
|
||||
}
|
||||
|
||||
/// Matte settings: Feather, Contrast, Shift Edge (no Refine Edge band).
|
||||
pub fn matte_params(params: &Params) -> MatteParams {
|
||||
let num = |id: &str| {
|
||||
let v = f(params, id);
|
||||
if v.is_finite() { v } else { 0.0 }
|
||||
};
|
||||
MatteParams {
|
||||
rb_feather: num("objectMatte/feather").clamp(0.0, 100.0),
|
||||
rb_contrast: num("objectMatte/contrast").clamp(0.0, 100.0),
|
||||
rb_shift: num("objectMatte/shiftEdge").clamp(-100.0, 100.0),
|
||||
decontaminate: false,
|
||||
..MatteParams::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// What a frame's matte is: the object's alpha on the layer grid, or why there is none.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum FrameMatte {
|
||||
/// The object's alpha (0–1) on the frame's layer grid.
|
||||
Alpha(Vec<f32>),
|
||||
/// The prompt is unusable (point outside the layer, outside the box…): the layer passes.
|
||||
NoPrompt,
|
||||
/// The frame is beyond the tracked range, or its matte could not be computed.
|
||||
Outside,
|
||||
}
|
||||
|
||||
/// The matte of layer frame `f` on the layer grid of `img` (the effect's input on that frame at
|
||||
/// `scale`), computing what is missing. `frame(g)`: the input on frame g on the same grid.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn frame_matte(
|
||||
params: &Params,
|
||||
layer_size: [f64; 2],
|
||||
fps: f64,
|
||||
f: i64,
|
||||
scale: f64,
|
||||
img: &Image,
|
||||
model: Option<Arc<dyn MaskModel>>,
|
||||
frame: &mut dyn FnMut(i64) -> Option<Arc<Image>>,
|
||||
cancel: &dyn Fn() -> bool,
|
||||
) -> FrameMatte {
|
||||
let Some(chain) = chain(params, layer_size, fps, f, scale, model) else { return FrameMatte::NoPrompt };
|
||||
if !chain.data.in_span(f) {
|
||||
return FrameMatte::Outside;
|
||||
}
|
||||
let Some(seg) = chain.ensure(f, frame, cancel) else { return FrameMatte::Outside };
|
||||
let (w, h) = (img.width as usize, img.height as usize);
|
||||
let seg = if seg.w == w && seg.h == h { (*seg).clone() } else { roto::resample(&seg, w, h) };
|
||||
FrameMatte::Alpha(roto::final_alpha(img, &seg, None, None, &matte_params(params), 0.0, scale))
|
||||
}
|
||||
|
||||
fn render(ctx: &EffectCtx, buf: Buf) -> Buf {
|
||||
if ctx.adjustment {
|
||||
return buf;
|
||||
}
|
||||
let fps = ctx.fps();
|
||||
let f = roto::frame_of(ctx.time, fps);
|
||||
let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 };
|
||||
let img = Arc::new(roto::layer_grid(&buf, ctx.layer_size));
|
||||
let (w, h) = (img.width as usize, img.height as usize);
|
||||
let host = ctx.env.host;
|
||||
let effects = ctx.env.effect_index;
|
||||
let layer_size = ctx.layer_size;
|
||||
let me = img.clone();
|
||||
let mut frame = |g: i64| -> Option<Arc<Image>> {
|
||||
if g == f {
|
||||
return Some(me.clone());
|
||||
}
|
||||
let other = host?.self_at(g as f64 / fps, effects)?;
|
||||
let im = roto::layer_grid(&other, layer_size);
|
||||
(im.width as usize == w && im.height as usize == h).then(|| Arc::new(im))
|
||||
};
|
||||
let model = roto::installed_model();
|
||||
// Frozen mattes are used as they are (no segmentation).
|
||||
if let Some(a) = frozen_alpha(ctx.params, layer_size, fps, f, w, h, model.as_deref()) {
|
||||
return compose(&buf, Some(&a), w, h, b(ctx.params, "invert"), param(ctx.params, "view").map(Value::as_enum).unwrap_or(0) == 1);
|
||||
}
|
||||
let alpha = match frame_matte(ctx.params, layer_size, fps, f, sc, &img, model, &mut frame, &|| false) {
|
||||
FrameMatte::Alpha(a) if a.len() == w * h => Some(a),
|
||||
FrameMatte::NoPrompt => return buf,
|
||||
_ => None,
|
||||
};
|
||||
compose(&buf, alpha.as_deref(), w, h, b(ctx.params, "invert"), param(ctx.params, "view").map(Value::as_enum).unwrap_or(0) == 1)
|
||||
}
|
||||
|
||||
/// The output: the input keyed by the matte (`None`: no object, all zero), or the matte itself.
|
||||
fn compose(buf: &Buf, alpha: Option<&[f32]>, w: usize, h: usize, invert: bool, view_matte: bool) -> Buf {
|
||||
let (ox, oy) = (buf.offset[0].round() as i64, buf.offset[1].round() as i64);
|
||||
let mut out = Buf { img: Image::new(buf.img.width, buf.img.height), offset: buf.offset, scale: buf.scale };
|
||||
out.img.rows_mut().for_each(|(by, row)| {
|
||||
for (bx, px) in row.iter_mut().enumerate() {
|
||||
let (x, y) = (bx as i64 - ox, by as i64 - oy);
|
||||
let a = if x >= 0 && y >= 0 && (x as usize) < w && (y as usize) < h {
|
||||
alpha.and_then(|a| a.get(y as usize * w + x as usize)).copied().unwrap_or(0.0)
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
let a = if invert { 1.0 - a } else { a }.clamp(0.0, 1.0);
|
||||
let src = buf.img.get(bx as i64, by as i64);
|
||||
*px = if view_matte {
|
||||
let v = a * src[3];
|
||||
[v, v, v, 1.0]
|
||||
} else {
|
||||
[src[0] * a, src[1] * a, src[2] * a, src[3] * a]
|
||||
};
|
||||
}
|
||||
});
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{EffectEnv, EffectHost, LayerPixels};
|
||||
|
||||
/// A red disk of radius 15 moving 2 px a frame right over a textured blue background
|
||||
/// (100×80, 25 fps), plus a still green square in the corner.
|
||||
fn frame_at(fi: i64) -> Image {
|
||||
let c = [40.0 + 2.0 * fi as f64, 40.0];
|
||||
let mut img = Image::new(100, 80);
|
||||
for y in 0..80 {
|
||||
for x in 0..100 {
|
||||
let d = ((x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1])) - 15.0;
|
||||
let a = (0.5 - d).clamp(0.0, 1.0) as f32;
|
||||
let tex = ((x * 7 + y * 13) % 9) as f32 / 60.0;
|
||||
let bg = if x >= 80 && y >= 60 { [0.1, 0.8, 0.2] } else { [0.1, 0.4 + tex, 0.7] };
|
||||
let fg = [0.85 + tex, 0.2, 0.1];
|
||||
img.set(x, y, [fg[0] * a + bg[0] * (1.0 - a), fg[1] * a + bg[1] * (1.0 - a), fg[2] * a + bg[2] * (1.0 - a), 1.0]);
|
||||
}
|
||||
}
|
||||
img
|
||||
}
|
||||
fn frame_of_time(t: f64) -> Image {
|
||||
frame_at((t * 25.0 + 1e-6).floor() as i64)
|
||||
}
|
||||
|
||||
struct Seq;
|
||||
impl EffectHost for Seq {
|
||||
fn layer(&self, _: u64, _: bool) -> Option<LayerPixels> {
|
||||
None
|
||||
}
|
||||
fn audio(&self, _: u64, _: f64, _: usize, _: u32) -> Option<Vec<f32>> {
|
||||
None
|
||||
}
|
||||
fn self_at(&self, t: f64, _: usize) -> Option<Buf> {
|
||||
Some(Buf { img: frame_of_time(t), offset: [0.0; 2], scale: 1.0 })
|
||||
}
|
||||
}
|
||||
|
||||
fn params(vals: &[(&str, Value)]) -> Params {
|
||||
let s = &specs()[0];
|
||||
let mut p = Params { values: s.params.iter().map(|p| (p.id.to_string(), crate::default_value(p, [100.0, 80.0]))).collect() };
|
||||
for (k, v) in vals {
|
||||
p.values.insert(k.to_string(), v.clone());
|
||||
}
|
||||
p
|
||||
}
|
||||
|
||||
fn run(p: &Params, fi: i64) -> Image {
|
||||
let host = Seq;
|
||||
let t = fi as f64 / 25.0 + 0.001;
|
||||
let ctx = EffectCtx {
|
||||
params: p,
|
||||
time: t,
|
||||
layer_size: [100.0, 80.0],
|
||||
seed: 1,
|
||||
adjustment: false,
|
||||
env: EffectEnv { host: Some(&host), frame_rate: 25.0, ..Default::default() },
|
||||
};
|
||||
crate::apply(&specs()[0], &ctx, Buf { img: frame_of_time(t), offset: [0.0; 2], scale: 1.0 }).img
|
||||
}
|
||||
|
||||
/// Pixels whose alpha disagrees with the disk centred at `cx` (1 px of slack at the edge).
|
||||
fn wrong(out: &Image, cx: f64) -> usize {
|
||||
let mut n = 0;
|
||||
for y in 0..80u32 {
|
||||
for x in 0..100u32 {
|
||||
let d = (x as f64 + 0.5 - cx).hypot(y as f64 + 0.5 - 40.0);
|
||||
let a = out.data[(y * 100 + x) as usize][3];
|
||||
if (d < 14.0 && a < 0.5) || (d > 16.0 && a > 0.5) {
|
||||
n += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
n
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_click_isolates_the_object_and_tracks_it() {
|
||||
let p = params(&[("objectPoint", Value::Vec2([40.0, 40.0]))]);
|
||||
// The reference frame: the disk, nothing else.
|
||||
let out = run(&p, 0);
|
||||
assert!(wrong(&out, 40.0) < 20, "{} wrong pixels on the reference frame", wrong(&out, 40.0));
|
||||
// Six frames on (carried through frames 1–5), the disk has moved 12 px and the matte with it.
|
||||
let out = run(&p, 6);
|
||||
assert!(wrong(&out, 52.0) < 30, "{} wrong pixels on frame 6", wrong(&out, 52.0));
|
||||
// Beyond Frames After: no matte.
|
||||
let p2 = params(&[("objectPoint", Value::Vec2([40.0, 40.0])), ("objectTracking/framesAfter", Value::Scalar(3.0))]);
|
||||
assert!(run(&p2, 6).data.iter().all(|px| px[3] == 0.0));
|
||||
// Invert keeps everything but the disk; View ▸ Matte shows it in grey, opaque.
|
||||
let inv = run(¶ms(&[("objectPoint", Value::Vec2([40.0, 40.0])), ("invert", Value::Bool(true))]), 0);
|
||||
assert!(inv.data[40 * 100 + 40][3] < 0.5 && inv.data[5 * 100 + 95][3] > 0.5);
|
||||
let m = run(¶ms(&[("objectPoint", Value::Vec2([40.0, 40.0])), ("view", Value::Enum(1))]), 0);
|
||||
assert!(m.data[40 * 100 + 40][0] > 0.5 && m.data[5 * 100 + 95][0] < 0.5 && m.data.iter().all(|px| px[3] == 1.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_tracking_every_frame_follows_the_prompt() {
|
||||
// The point animated onto the disk at frame 9: frame 9 is segmented from it alone.
|
||||
let p = params(&[("objectPoint", Value::Vec2([58.0, 40.0])), ("objectTracking/trackObject", Value::Bool(false))]);
|
||||
let out = run(&p, 9);
|
||||
assert!(wrong(&out, 58.0) < 20, "{} wrong pixels", wrong(&out, 58.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_box_and_bad_prompts() {
|
||||
// A box around the green square, the point in it: the square, not the disk.
|
||||
let p = params(&[
|
||||
("objectPoint", Value::Vec2([90.0, 70.0])),
|
||||
("useBox", Value::Bool(true)),
|
||||
("boxTopLeft", Value::Vec2([74.0, 54.0])),
|
||||
("boxBottomRight", Value::Vec2([100.0, 80.0])),
|
||||
]);
|
||||
let out = run(&p, 0);
|
||||
assert!(out.data[70 * 100 + 90][3] > 0.5 && out.data[40 * 100 + 40][3] < 0.5);
|
||||
// Unusable prompts pass the layer through instead of failing: a point off the layer, a
|
||||
// non-finite point, a point outside its box.
|
||||
let src = frame_at(0);
|
||||
for vals in [
|
||||
vec![("objectPoint", Value::Vec2([-5.0, 40.0]))],
|
||||
vec![("objectPoint", Value::Vec2([f64::NAN, 40.0]))],
|
||||
vec![
|
||||
("objectPoint", Value::Vec2([40.0, 40.0])),
|
||||
("useBox", Value::Bool(true)),
|
||||
("boxTopLeft", Value::Vec2([60.0, 10.0])),
|
||||
("boxBottomRight", Value::Vec2([90.0, 30.0])),
|
||||
],
|
||||
] {
|
||||
let p = params(&vals);
|
||||
assert_eq!(run(&p, 0).data, src.data, "{vals:?}");
|
||||
}
|
||||
// A huge reference frame or reach doesn't overflow.
|
||||
let p = params(&[("objectTracking/referenceFrame", Value::Scalar(1e300)), ("objectTracking/framesBefore", Value::Scalar(f64::INFINITY))]);
|
||||
assert!(run(&p, 0).data.iter().all(|px| px[3] == 0.0));
|
||||
}
|
||||
|
||||
/// A stand-in model (the weights are an optional download): a disk around the first
|
||||
/// foreground point, radius 15, so the test sees the model's answer in the matte.
|
||||
#[derive(Default)]
|
||||
struct Stub {
|
||||
calls: std::sync::atomic::AtomicUsize,
|
||||
/// The box prompt of the last call.
|
||||
bbox: std::sync::Mutex<Option<[f32; 4]>>,
|
||||
}
|
||||
impl MaskModel for Stub {
|
||||
fn info(&self) -> &'static effectcraft_segment::ModelInfo {
|
||||
&effectcraft_segment::MOBILE_SAM
|
||||
}
|
||||
fn segment(&self, _: &[[f32; 3]], w: usize, h: usize, prompt: &effectcraft_segment::Prompt) -> effectcraft_segment::Result<Vec<f32>> {
|
||||
self.calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
*self.bbox.lock().unwrap() = prompt.bbox;
|
||||
let Some(([cx, cy], _)) = prompt.points.iter().find(|(_, fg)| *fg) else { return Err("no point".into()) };
|
||||
Ok((0..w * h).map(|i| (((i % w) as f32 + 0.5 - cx).hypot((i / w) as f32 + 0.5 - cy) < 15.0) as u8 as f32).collect())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn segments_with_the_installed_model_without_a_border_stroke() {
|
||||
let p = params(&[("objectPoint", Value::Vec2([40.0, 40.0]))]);
|
||||
// With a model, no border stroke: an object touching the frame edge stays whole.
|
||||
assert_eq!(prompt_strokes(&p, [100.0, 80.0], 0, true).map(|s| s.len()), Some(1));
|
||||
assert_eq!(prompt_strokes(&p, [100.0, 80.0], 0, false).map(|s| s.len()), Some(2));
|
||||
// The model's answer is part of the cache key, and drives the matte.
|
||||
let stub = Arc::new(Stub::default());
|
||||
assert_ne!(seed(&p, [100.0, 80.0], 25.0, Some(stub.as_ref())), seed(&p, [100.0, 80.0], 25.0, None));
|
||||
let img = frame_at(0);
|
||||
let mut frame = |g: i64| Some(Arc::new(frame_at(g)));
|
||||
let FrameMatte::Alpha(a) = frame_matte(&p, [100.0, 80.0], 25.0, 0, 1.0, &img, Some(stub.clone()), &mut frame, &|| false) else { panic!("no matte") };
|
||||
assert!(a[40 * 100 + 40] > 0.5 && a[5 * 100 + 5] < 0.5);
|
||||
assert_eq!(stub.calls.load(std::sync::atomic::Ordering::Relaxed), 1);
|
||||
assert_eq!(*stub.bbox.lock().unwrap(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_box_reaches_the_model_as_a_box_prompt() {
|
||||
let p = params(&[
|
||||
("objectPoint", Value::Vec2([90.0, 70.0])),
|
||||
("useBox", Value::Bool(true)),
|
||||
("boxTopLeft", Value::Vec2([100.0, 80.0])),
|
||||
("boxBottomRight", Value::Vec2([74.0, 54.0])),
|
||||
]);
|
||||
// With a model: the point and a box stroke (corners ordered); without: the box's edges
|
||||
// as a background stroke, as before.
|
||||
let with = prompt_strokes(&p, [100.0, 80.0], 0, true).unwrap();
|
||||
assert_eq!(with.iter().map(|s| s.kind).collect::<Vec<_>>(), [StrokeKind::Fg, StrokeKind::Box]);
|
||||
assert_eq!(with[1].points, [[74.0, 54.0], [100.0, 80.0]]);
|
||||
let without = prompt_strokes(&p, [100.0, 80.0], 0, false).unwrap();
|
||||
assert_eq!(without.iter().map(|s| s.kind).collect::<Vec<_>>(), [StrokeKind::Fg, StrokeKind::Bg]);
|
||||
// The model is called with the box itself (matte pixels at scale 1) and only one
|
||||
// foreground point: no background points along the edges.
|
||||
let stub = Arc::new(Stub::default());
|
||||
let img = frame_at(0);
|
||||
let mut frame = |g: i64| Some(Arc::new(frame_at(g)));
|
||||
let FrameMatte::Alpha(a) = frame_matte(&p, [100.0, 80.0], 25.0, 0, 1.0, &img, Some(stub.clone()), &mut frame, &|| false) else { panic!("no matte") };
|
||||
assert_eq!(*stub.bbox.lock().unwrap(), Some([74.0, 54.0, 100.0, 80.0]));
|
||||
assert!(a[70 * 100 + 90] > 0.5 && a[40 * 100 + 40] < 0.5);
|
||||
// At half resolution the box is scaled with the frame (another Input Key, so the
|
||||
// full-resolution result above is not reused).
|
||||
let mut p = p;
|
||||
p.values.insert(INPUT_KEY.into(), Value::Str("half".into()));
|
||||
let half = effectcraft_raster::resample(&img, 50, 40);
|
||||
let mut frame = |g: i64| Some(Arc::new(effectcraft_raster::resample(&frame_at(g), 50, 40)));
|
||||
let _ = frame_matte(&p, [100.0, 80.0], 25.0, 0, 0.5, &half, Some(stub.clone()), &mut frame, &|| false);
|
||||
assert_eq!(*stub.bbox.lock().unwrap(), Some([37.0, 27.0, 50.0, 40.0]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn job_frames_go_outward_from_the_reference_frame_inside_the_layer() {
|
||||
let p = params(&[
|
||||
("objectTracking/referenceFrame", Value::Scalar(5.0)),
|
||||
("objectTracking/framesBefore", Value::Scalar(2.0)),
|
||||
("objectTracking/framesAfter", Value::Scalar(3.0)),
|
||||
]);
|
||||
assert_eq!(job_frames(&p, [0, 100]), [5, 6, 7, 8, 4, 3]);
|
||||
assert_eq!(job_frames(&p, [4, 6]), [5, 6, 4]);
|
||||
assert!(job_frames(&p, [10, 20]).is_empty());
|
||||
let huge = params(&[("objectTracking/referenceFrame", Value::Scalar(1e300)), ("objectTracking/framesAfter", Value::Scalar(f64::INFINITY))]);
|
||||
assert!(job_frames(&huge, [0, 100]).is_empty());
|
||||
}
|
||||
|
||||
/// Frozen mattes are used as stored while their key matches (no segmentation: a stored
|
||||
/// pattern the segmentation could never produce comes out), and ignored once it doesn't.
|
||||
#[test]
|
||||
fn frozen_mattes_are_used_while_their_key_matches() {
|
||||
let mut p = params(&[("objectPoint", Value::Vec2([40.0, 40.0])), ("objectTracking/framesAfter", Value::Scalar(3.0))]);
|
||||
p.values.insert(INPUT_KEY.into(), Value::Str("frozen-test".into()));
|
||||
// Left half opaque on frame 2.
|
||||
let alpha: Vec<f32> = (0..100 * 80).map(|i| ((i % 100) < 50) as u8 as f32).collect();
|
||||
let mut fc = FrozenCache { key: frozen_key(&p, [100.0, 80.0], 25.0, None), size: [100, 80], ..Default::default() };
|
||||
fc.frames.insert(2, roto::freeze_frame(&alpha, &[]));
|
||||
p.values.insert(FROZEN.into(), Value::Str(fc.to_json()));
|
||||
assert!(is_frozen(&p));
|
||||
let out = run(&p, 2);
|
||||
assert!(out.data[40 * 100 + 10][3] > 0.99 && out.data[40 * 100 + 90][3] < 0.01, "the frozen matte is used");
|
||||
// At half resolution too.
|
||||
assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 50, 40, None).map(|a| (a[20 * 50 + 5], a[20 * 50 + 45])), Some((1.0, 0.0)));
|
||||
// A frame without a frozen matte is computed (the disk).
|
||||
assert!(wrong(&run(&p, 1), 42.0) < 30);
|
||||
// Another prompt: the frozen mattes no longer apply.
|
||||
p.values.insert("objectPoint".into(), Value::Vec2([41.0, 40.0]));
|
||||
assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 100, 80, None), None);
|
||||
assert!(wrong(&run(&p, 2), 44.0) < 30);
|
||||
// Damaged frozen data is ignored, not a crash.
|
||||
p.values.insert(FROZEN.into(), Value::Str("{\"key\": 3, \"frames\": [".into()));
|
||||
assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 100, 80, None), None);
|
||||
let mut fc2 = fc.clone();
|
||||
fc2.size = [usize::MAX, 2];
|
||||
p.values.insert("objectPoint".into(), Value::Vec2([40.0, 40.0]));
|
||||
p.values.insert(FROZEN.into(), Value::Str(fc2.to_json()));
|
||||
assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 100, 80, None), None);
|
||||
}
|
||||
}
|
||||
|
|
@ -7,7 +7,8 @@
|
|||
//! out to the frame), so it lives in a process-wide content-keyed cache ([`cached`]) that the
|
||||
//! engine's propagation job fills at full resolution and rendering fills on demand (reading
|
||||
//! neighbouring frames with [`crate::EffectHost::self_at`]). Undo, redo and reopening a project
|
||||
//! therefore never see stale mattes. **Freeze** stores the final mattes in the hidden **Frozen**
|
||||
//! therefore never see stale mattes. Behind the memory cache an optional [`SegStore`] (the
|
||||
//! persistent disk cache, set by the engine) keeps computed segmentations across restarts. **Freeze** stores the final mattes in the hidden **Frozen**
|
||||
//! parameter ([`FrozenCache`]); they are used while their key still matches.
|
||||
//!
|
||||
//! Rendering: binary segmentation → Roto Brush Matte (Shift Edge, Reduce Chatter, Feather,
|
||||
|
|
@ -23,21 +24,21 @@
|
|||
//! higher working resolution with more iterations.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap, VecDeque};
|
||||
use std::sync::{Arc, Mutex, OnceLock, RwLock};
|
||||
use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak};
|
||||
|
||||
use effectcraft_keyframe::Value;
|
||||
use effectcraft_project::ParamUi;
|
||||
use effectcraft_raster::Image;
|
||||
use effectcraft_segment::MaskModel;
|
||||
use effectcraft_track::roto::refine::{self, MatteParams};
|
||||
use effectcraft_track::roto::{self as rb, FrameSeg, RotoData, SegOpts, rle};
|
||||
use effectcraft_track::roto::{self as rb, RotoData, SegOpts, rle};
|
||||
use rayon::prelude::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::warp_stab::param;
|
||||
use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider};
|
||||
|
||||
pub use effectcraft_track::roto::{Stroke, StrokeKind};
|
||||
pub use effectcraft_track::roto::{FrameSeg, Stroke, StrokeKind};
|
||||
|
||||
/// The effect's id.
|
||||
pub const ID: &str = "ec.matte.rotobrush";
|
||||
|
|
@ -195,6 +196,12 @@ pub fn model_for(params: &Params) -> Option<Arc<dyn MaskModel>> {
|
|||
if e(params, "version") == 0 {
|
||||
return None;
|
||||
}
|
||||
installed_model()
|
||||
}
|
||||
|
||||
/// The trained model chosen in Settings ▸ Roto Brush, if one is installed (Roto Brush 2.0 / 3.0
|
||||
/// and Object Matte segment with it).
|
||||
pub fn installed_model() -> Option<Arc<dyn MaskModel>> {
|
||||
model_slot().read().ok().and_then(|m| m.clone())
|
||||
}
|
||||
|
||||
|
|
@ -260,6 +267,37 @@ pub fn store(key: u64, scale: f64, seg: Arc<FrameSeg>) {
|
|||
}
|
||||
}
|
||||
|
||||
/// A second-level store for segmentations, keyed like the memory cache (chain key, scale): the
|
||||
/// persistent disk cache, so computed mattes survive the memory cache and restarts.
|
||||
pub trait SegStore: Send + Sync {
|
||||
fn get_seg(&self, key: u64, scale: f64) -> Option<FrameSeg>;
|
||||
fn put_seg(&self, key: u64, scale: f64, seg: &FrameSeg);
|
||||
}
|
||||
|
||||
fn seg_store_slot() -> &'static RwLock<Option<Weak<dyn SegStore>>> {
|
||||
static S: OnceLock<RwLock<Option<Weak<dyn SegStore>>>> = OnceLock::new();
|
||||
S.get_or_init(|| RwLock::new(None))
|
||||
}
|
||||
|
||||
/// Use `store` behind the memory cache (held weakly: once its owner drops it, e.g. the disk
|
||||
/// cache is switched off, it is no longer used). `None` turns it off.
|
||||
pub fn set_seg_store(store: Option<Weak<dyn SegStore>>) {
|
||||
if let Ok(mut s) = seg_store_slot().write() {
|
||||
*s = store;
|
||||
}
|
||||
}
|
||||
|
||||
fn seg_store() -> Option<Arc<dyn SegStore>> {
|
||||
seg_store_slot().read().ok()?.as_ref()?.upgrade()
|
||||
}
|
||||
|
||||
/// A segmentation from the second-level store, put into the memory cache.
|
||||
fn from_store(key: u64, scale: f64) -> Option<Arc<FrameSeg>> {
|
||||
let seg = Arc::new(seg_store()?.get_seg(key, scale)?);
|
||||
store(key, scale, seg.clone());
|
||||
Some(seg)
|
||||
}
|
||||
|
||||
/// Drop the cached segmentations of these chain keys (every scale).
|
||||
pub fn forget(keys: impl IntoIterator<Item = u64>) {
|
||||
let keys: std::collections::HashSet<u64> = keys.into_iter().collect();
|
||||
|
|
@ -319,11 +357,11 @@ impl Chain {
|
|||
let sc = self.opts.scale;
|
||||
let lookup = |g: i64| -> Option<Arc<FrameSeg>> {
|
||||
let k = *self.keys.get(&g)?;
|
||||
if let Some(s) = cached(k, sc) {
|
||||
if let Some(s) = cached(k, sc).or_else(|| from_store(k, sc)) {
|
||||
return Some(s);
|
||||
}
|
||||
if sc != 1.0 {
|
||||
let full = cached(k, 1.0)?;
|
||||
let full = cached(k, 1.0).or_else(|| from_store(k, 1.0))?;
|
||||
let s = Arc::new(downscale(&full, sc));
|
||||
store(k, sc, s.clone());
|
||||
return Some(s);
|
||||
|
|
@ -373,7 +411,11 @@ impl Chain {
|
|||
}
|
||||
};
|
||||
let seg = Arc::new(seg);
|
||||
store(*self.keys.get(&g)?, sc, seg.clone());
|
||||
let k = *self.keys.get(&g)?;
|
||||
store(k, sc, seg.clone());
|
||||
if let Some(st) = seg_store() {
|
||||
st.put_seg(k, sc, &seg);
|
||||
}
|
||||
cur = Some(seg);
|
||||
prev_img = Some(img);
|
||||
}
|
||||
|
|
@ -392,7 +434,7 @@ pub fn downscale(s: &FrameSeg, scale: f64) -> FrameSeg {
|
|||
resample(s, w, h)
|
||||
}
|
||||
|
||||
fn resample(s: &FrameSeg, w: usize, h: usize) -> FrameSeg {
|
||||
pub(crate) fn resample(s: &FrameSeg, w: usize, h: usize) -> FrameSeg {
|
||||
if s.w == w && s.h == h {
|
||||
return s.clone();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -199,7 +199,7 @@ fn capabilities(s: &mut Session, _: &Value) -> Result<Value> {
|
|||
}
|
||||
|
||||
/// The headline of `docs/target-app-parity.md` (kept in sync by hand at each re-measure).
|
||||
const PARITY: &str = "≈91% feature breadth and ≈45% ready for real work (estimated parity with After Effects 2026 26.5, stage alpha); all After Effects effects present (306 registered)";
|
||||
const PARITY: &str = "≈91% feature breadth and ≈45% ready for real work (estimated parity with After Effects 2026 26.5, stage alpha); all After Effects effects present (307 registered)";
|
||||
|
||||
fn state(s: &mut Session, _: &Value) -> Result<Value> {
|
||||
Ok(serde_json::to_value(&s.state).unwrap_or_default())
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
//! Roto Brush & Refine Edge commands: strokes (the Roto Brush and Refine Edge tools in the Layer
|
||||
//! panel), propagation, the segmentation span, Freeze / Unfreeze, tool options and status.
|
||||
//!
|
||||
//! `objectMatte.apply` adds an Object Matte effect (Effect ▸ Matte) from a point / box prompt on
|
||||
//! a frame; it segments with the same engine and model as Roto Brush.
|
||||
//!
|
||||
//! `roto.stroke` records a stroke into the layer's Roto Brush & Refine Edge effect (applying the
|
||||
//! effect on the first stroke, as After Effects does); the first foreground stroke sets the base
|
||||
//! frame and a span of 20 frames each side. Strokes on a frozen effect are refused (Unfreeze
|
||||
|
|
@ -51,6 +54,32 @@ pub fn specs() -> Vec<CommandSpec> {
|
|||
always,
|
||||
options
|
||||
),
|
||||
cmd!(
|
||||
"objectMatte.apply",
|
||||
"Object Matte",
|
||||
[],
|
||||
None,
|
||||
"{layer?, effect? (uid: re-prompt this Object Matte instead of adding one), point: [x, y] (layer pixels: the object), box?: [x0, y0, x1, y1] (layer pixels, around the object), frame? (reference layer frame; default current), track?: follow the object from that frame (default true), before?, after? (frames to follow it; default 30, up to 600)}",
|
||||
has_comp,
|
||||
object_matte_apply
|
||||
),
|
||||
cmd!(
|
||||
"objectMatte.propagate",
|
||||
"Track Object Matte",
|
||||
[],
|
||||
None,
|
||||
"{layer?, effect?, direction?: forward|backward|both, wait?: block until done} (a background job like Roto Brush propagation; roto.cancel stops it)",
|
||||
has_comp,
|
||||
object_matte_propagate
|
||||
),
|
||||
cmd!("objectMatte.freeze", "Freeze Object Matte", [], None, "{layer?, effect?, wait?}", has_comp, object_matte_freeze),
|
||||
cmd!("objectMatte.unfreeze", "Unfreeze Object Matte", [], None, "{layer?, effect?}", has_comp, object_matte_unfreeze),
|
||||
query!(
|
||||
"objectMatte.status",
|
||||
"Object Matte Status",
|
||||
"{layer?, effect?}: the job's progress, whether the instance is frozen, its frames and which are computed",
|
||||
object_matte_status
|
||||
),
|
||||
query!(
|
||||
"roto.status",
|
||||
"Roto Brush Status",
|
||||
|
|
@ -67,6 +96,16 @@ fn is_running(s: &Session) -> std::result::Result<(), String> {
|
|||
/// The Roto Brush instance targeted by `p`: `layer` / `effect` (uid, name or 1-based index),
|
||||
/// else a selected one, else the first on the selected (or any) layer.
|
||||
pub(crate) fn roto_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> {
|
||||
instance_lookup(s, p, cmd, is_roto)?.ok_or_else(|| bad(cmd, "no Roto Brush & Refine Edge effect found (paint a stroke with the Roto Brush tool first)"))
|
||||
}
|
||||
|
||||
/// The Object Matte instance targeted by `p` (as [`roto_p`]).
|
||||
fn object_matte_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> {
|
||||
instance_lookup(s, p, cmd, rt::is_object_matte)?.ok_or_else(|| bad(cmd, "no Object Matte effect found (click an object with the Object Matte tool first)"))
|
||||
}
|
||||
|
||||
/// The instance of a kind (`is`) targeted by `p`; `None` when there is none.
|
||||
fn instance_lookup(s: &Session, p: &Value, cmd: &str, is: fn(&PropGroup) -> bool) -> Result<Option<(ItemId, LayerId, Uid)>> {
|
||||
let cid = super::comp_id(s, p)?;
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let candidates: Vec<LayerId> = match p.get("layer") {
|
||||
|
|
@ -90,17 +129,17 @@ pub(crate) fn roto_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, Layer
|
|||
.state
|
||||
.selected_props
|
||||
.iter()
|
||||
.find_map(|(l, u)| (*l == lid).then(|| fxg.groups().find(|g| g.uid == *u && is_roto(g))).flatten())
|
||||
.or_else(|| fxg.groups().find(|g| is_roto(g))),
|
||||
.find_map(|(l, u)| (*l == lid).then(|| fxg.groups().find(|g| g.uid == *u && is(g))).flatten())
|
||||
.or_else(|| fxg.groups().find(|g| is(g))),
|
||||
};
|
||||
if let Some(g) = g.filter(|g| is_roto(g)) {
|
||||
return Ok((cid, lid, g.uid));
|
||||
if let Some(g) = g.filter(|g| is(g)) {
|
||||
return Ok(Some((cid, lid, g.uid)));
|
||||
}
|
||||
if p.get("layer").is_some() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(bad(cmd, "no Roto Brush & Refine Edge effect found (paint a stroke with the Roto Brush tool first)"))
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn group(s: &Session, cid: ItemId, lid: LayerId, uid: Uid) -> Result<(&Layer, &PropGroup)> {
|
||||
|
|
@ -428,3 +467,190 @@ fn status(s: &mut Session, p: &Value) -> Result<Value> {
|
|||
out["computed"] = json!(computed.iter().filter(|(_, c)| **c).map(|(f, _)| *f).collect::<Vec<_>>());
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Two finite numbers.
|
||||
fn xy_p(p: &Value, k: &str, n: usize, cmd: &str) -> Result<Option<Vec<f64>>> {
|
||||
let Some(v) = p.get(k).filter(|v| !v.is_null()) else { return Ok(None) };
|
||||
let a = v.as_array().filter(|a| a.len() == n).ok_or_else(|| bad(cmd, format!("`{k}` is {n} numbers (layer pixels)")))?;
|
||||
let out: Vec<f64> = a.iter().filter_map(Value::as_f64).filter(|x| x.is_finite()).collect();
|
||||
if out.len() != n {
|
||||
return Err(bad(cmd, format!("`{k}` is {n} numbers (layer pixels)")));
|
||||
}
|
||||
Ok(Some(out))
|
||||
}
|
||||
|
||||
fn object_matte_apply(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
use effectcraft_effects::object_matte as om;
|
||||
let cmd = "objectMatte.apply";
|
||||
let (cid, lid) = layer_p(s, p, cmd)?;
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let layer = comp.layer(lid).ok_or_else(|| bad(cmd, "no layer"))?;
|
||||
if !layer.has_video() || layer.switches.adjustment || layer.effects().is_none() {
|
||||
return Err(bad(cmd, "Object Matte works on footage, solid and composition layers"));
|
||||
}
|
||||
// Re-prompt an existing instance (the Object Matte tool on a layer that has one).
|
||||
let existing = match p.get("effect").filter(|v| !v.is_null()) {
|
||||
Some(v) => {
|
||||
let uid = v.as_u64().ok_or_else(|| bad(cmd, "`effect` is an Object Matte effect's uid"))?;
|
||||
let g = layer
|
||||
.effects()
|
||||
.and_then(|f| f.groups().find(|g| g.uid == uid && rt::is_object_matte(g)))
|
||||
.ok_or_else(|| bad(cmd, format!("no Object Matte effect {uid} on this layer")))?;
|
||||
if frozen_of(g) {
|
||||
return Err(bad(cmd, "the Object Matte is frozen: Unfreeze it to change the prompt"));
|
||||
}
|
||||
Some(uid)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
let size = rt::layer_size(&s.project, comp, layer);
|
||||
let point = xy_p(p, "point", 2, cmd)?.ok_or_else(|| bad(cmd, "missing `point` [x, y] (layer pixels)"))?;
|
||||
let inside = |x: f64, y: f64| (0.0..=size[0]).contains(&x) && (0.0..=size[1]).contains(&y);
|
||||
if !inside(point[0], point[1]) {
|
||||
return Err(bad(cmd, format!("`point` is outside the layer ({}×{})", size[0], size[1])));
|
||||
}
|
||||
let bx = xy_p(p, "box", 4, cmd)?;
|
||||
if let Some(b) = &bx {
|
||||
let (x0, x1, y0, y1) = (b[0].min(b[2]), b[0].max(b[2]), b[1].min(b[3]), b[1].max(b[3]));
|
||||
if !(point[0] > x0 && point[0] < x1 && point[1] > y0 && point[1] < y1) {
|
||||
return Err(bad(cmd, "`point` must be inside `box`"));
|
||||
}
|
||||
}
|
||||
let frame = frame_p(s, p, cid, layer);
|
||||
let limits = rt::frame_limits(comp, layer);
|
||||
if frame < limits[0] || frame > limits[1] {
|
||||
return Err(bad(cmd, format!("frame {frame} is outside the layer ({}–{})", limits[0], limits[1])));
|
||||
}
|
||||
// A new instance gets the defaults; a re-prompted one keeps what is not given.
|
||||
let track = b_p(p, "track").or(existing.is_none().then_some(true));
|
||||
let reach = |k: &str| {
|
||||
let v = f_p(p, k).filter(|v| v.is_finite());
|
||||
let v = if existing.is_none() { Some(v.unwrap_or(om::DEFAULT_REACH)) } else { v };
|
||||
v.map(|v| v.round().clamp(0.0, om::MAX_REACH))
|
||||
};
|
||||
let (before, after) = (reach("before"), reach("after"));
|
||||
let spec = effectcraft_effects::find(om::ID).ok_or_else(|| EngineError::Other("Object Matte effect missing".into()))?;
|
||||
let label = if existing.is_some() { "Object Matte Prompt" } else { "Object Matte" };
|
||||
let uid = s.edit(label, None, |proj, _| {
|
||||
let mut next = proj.next_id;
|
||||
let l = layer_mut(proj, cid, lid)?;
|
||||
let fxg = l.props.sub_mut("effects").ok_or_else(|| bad(cmd, "this layer has no effects"))?;
|
||||
let mut fresh = None;
|
||||
let g = match existing {
|
||||
Some(u) => fxg.find_group_mut(u).ok_or_else(|| bad(cmd, "the Object Matte effect is gone"))?,
|
||||
None => {
|
||||
let n = fxg.groups().filter(|g| rt::is_object_matte(g)).count();
|
||||
let name = if n == 0 { spec.name.to_string() } else { format!("{} {}", spec.name, n + 1) };
|
||||
fresh.insert(effectcraft_effects::instantiate(spec, &mut Ids(&mut next), &name, size))
|
||||
}
|
||||
};
|
||||
let mut set = |path: &str, v: KV| {
|
||||
let (grp, leaf) = path.rsplit_once('/').map_or((None, path), |(a, b)| (Some(a), b));
|
||||
let target = match grp {
|
||||
Some(sub) => g.sub_mut(sub),
|
||||
None => Some(&mut *g),
|
||||
};
|
||||
// The prompt is static: a re-prompt replaces any keyframes.
|
||||
if let Some(pr) = target.and_then(|t| t.get_mut(leaf)) {
|
||||
pr.value = v;
|
||||
pr.keys.clear();
|
||||
}
|
||||
};
|
||||
set("objectPoint", KV::Vec2([point[0], point[1]]));
|
||||
set("useBox", KV::Bool(bx.is_some()));
|
||||
if let Some(b) = &bx {
|
||||
set("boxTopLeft", KV::Vec2([b[0].min(b[2]), b[1].min(b[3])]));
|
||||
set("boxBottomRight", KV::Vec2([b[0].max(b[2]), b[1].max(b[3])]));
|
||||
}
|
||||
if let Some(t) = track {
|
||||
set("objectTracking/trackObject", KV::Bool(t));
|
||||
}
|
||||
set("objectTracking/referenceFrame", KV::Scalar(frame as f64));
|
||||
if let Some(v) = before {
|
||||
set("objectTracking/framesBefore", KV::Scalar(v));
|
||||
}
|
||||
if let Some(v) = after {
|
||||
set("objectTracking/framesAfter", KV::Scalar(v));
|
||||
}
|
||||
let u = g.uid;
|
||||
if let Some(g) = fresh {
|
||||
fxg.children.push(g.into());
|
||||
}
|
||||
proj.next_id = proj.next_id.max(next);
|
||||
Ok(u)
|
||||
})?;
|
||||
s.state.selected_layers = vec![lid];
|
||||
let (track, before, after) = object_matte_reach(s, cid, lid, uid);
|
||||
Ok(json!({"layer": lid.0, "effect": uid, "frame": frame, "track": track, "updated": existing.is_some(), "span": [frame - before, frame + after]}))
|
||||
}
|
||||
|
||||
/// Track Object and Frames Before / After of an Object Matte instance (static values).
|
||||
fn object_matte_reach(s: &Session, cid: ItemId, lid: LayerId, uid: Uid) -> (bool, i64, i64) {
|
||||
use effectcraft_effects::object_matte as om;
|
||||
let Ok((_, g)) = group(s, cid, lid, uid) else { return (true, 0, 0) };
|
||||
let ps = params_static(g);
|
||||
let [before, after] = om::reach(&ps);
|
||||
(om::tracks(&ps), before, after)
|
||||
}
|
||||
|
||||
fn object_matte_propagate(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
let cmd = "objectMatte.propagate";
|
||||
let (cid, lid, uid) = object_matte_p(s, p, cmd)?;
|
||||
let dir = match str_p(p, "direction") {
|
||||
Some(d) => Direction::from_name(d).ok_or_else(|| bad(cmd, format!("unknown direction `{d}` (forward | backward | both)")))?,
|
||||
None => Direction::Both,
|
||||
};
|
||||
let wait = b_p(p, "wait").unwrap_or(false);
|
||||
let n = s.start_roto(cid, lid, uid, RotoTask::Propagate, dir, wait).map_err(|e| bad(cmd, e))?;
|
||||
let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_roto_running(), "progress": s.roto_progress()});
|
||||
if wait {
|
||||
out["status"] = object_matte_status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?;
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn object_matte_freeze(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
let cmd = "objectMatte.freeze";
|
||||
let (cid, lid, uid) = object_matte_p(s, p, cmd)?;
|
||||
let wait = b_p(p, "wait").unwrap_or(false);
|
||||
let n = s.start_roto(cid, lid, uid, RotoTask::Freeze, Direction::Both, wait).map_err(|e| bad(cmd, e))?;
|
||||
let (_, g) = group(s, cid, lid, uid)?;
|
||||
Ok(json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_roto_running(), "frozen": frozen_of(g), "progress": s.roto_progress()}))
|
||||
}
|
||||
|
||||
fn object_matte_unfreeze(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
let cmd = "objectMatte.unfreeze";
|
||||
let (cid, lid, uid) = object_matte_p(s, p, cmd)?;
|
||||
let (_, g) = group(s, cid, lid, uid)?;
|
||||
if frozen_of(g) {
|
||||
s.edit("Unfreeze", None, |proj, _| {
|
||||
let g = layer_mut(proj, cid, lid)?.props.find_group_mut(uid).ok_or_else(|| EngineError::Other("the effect is gone".into()))?;
|
||||
if let Some(pr) = g.get_mut(FROZEN) {
|
||||
pr.value = KV::Str(String::new());
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
}
|
||||
Ok(json!({"layer": lid.0, "effect": uid, "frozen": false}))
|
||||
}
|
||||
|
||||
fn object_matte_status(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
s.poll_roto(false);
|
||||
let running = s.is_roto_running();
|
||||
let mut out = json!({"running": running, "progress": s.roto_progress(), "banner": s.roto_progress().map(|p| p.banner())});
|
||||
let Ok((cid, lid, uid)) = object_matte_p(s, p, "objectMatte.status") else { return Ok(out) };
|
||||
let (_, g) = group(s, cid, lid, uid)?;
|
||||
let frozen = frozen_of(g);
|
||||
let (track, before, after) = object_matte_reach(s, cid, lid, uid);
|
||||
let computed = s.roto_computed(cid, lid, uid);
|
||||
out["layer"] = json!(lid.0);
|
||||
out["effect"] = json!(uid);
|
||||
out["frozen"] = json!(frozen);
|
||||
out["track"] = json!(track);
|
||||
out["reach"] = json!([before, after]);
|
||||
out["mine"] = json!(running && s.roto_target().is_some_and(|(c, l, u, _)| (c, l, u) == (cid, lid, uid)));
|
||||
out["pending"] = json!(s.roto_pending.contains(&(cid, lid, uid)));
|
||||
out["frames"] = json!(computed.keys().collect::<Vec<_>>());
|
||||
out["computed"] = json!(computed.iter().filter(|(_, c)| **c).map(|(f, _)| *f).collect::<Vec<_>>());
|
||||
Ok(out)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -561,7 +561,8 @@ async fn fill(s: &mut Session, plan: crate::commands::content_fill::FillPlan, po
|
|||
/// Roto Brush propagation in a worker: the segmentations stream back (at most every 250 ms)
|
||||
/// as they are computed, so stopping the job keeps what was propagated.
|
||||
async fn propagate(s: &mut Session, comp: ItemId, layer: LayerId, effect: Uid, dir: crate::roto::Direction, post: Post) -> Result<(), String> {
|
||||
let keys: Vec<u64> = s.roto_chain(comp, layer, effect).map(|c| c.keys.values().copied().collect()).ok_or("no Roto Brush strokes")?;
|
||||
let keys: Vec<u64> =
|
||||
s.roto_keys(comp, layer, effect).map(|k| k.into_iter().map(|(_, k)| k).collect()).ok_or("no Roto Brush strokes or Object Matte prompt")?;
|
||||
let sent = Rc::new(RefCell::new(std::collections::HashSet::new()));
|
||||
let (p, k2, s2) = (post.clone(), keys.clone(), sent.clone());
|
||||
let mut last = None::<web_time::Instant>;
|
||||
|
|
|
|||
|
|
@ -10,12 +10,18 @@
|
|||
//! edit, keeps each instance's hidden Input Key equal to a signature of what its frames are made
|
||||
//! of (the layer's source and timing, its masks and the effects above it) and drops frozen
|
||||
//! mattes made from other frames.
|
||||
//!
|
||||
//! Object Matte instances run through the same jobs (`objectMatte.propagate` / `.freeze`): the
|
||||
//! frames are [`om::job_frames`] around the reference frame, each segmented through
|
||||
//! [`om::ensure`]; Freeze stores [`om::frame_matte`]'s mattes under [`om::frozen_key`]. An edit
|
||||
//! to an instance's frames or prompt queues it for background tracking like a Roto Brush.
|
||||
|
||||
use crate::offload::JobKind;
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use effectcraft_effects::object_matte as om;
|
||||
use effectcraft_effects::roto::{self as fx, FROZEN, FrozenCache, INPUT_KEY};
|
||||
use effectcraft_effects::{Params, flatten_params};
|
||||
use effectcraft_keyframe::Value;
|
||||
|
|
@ -104,14 +110,18 @@ pub struct RotoProgress {
|
|||
pub finished: bool,
|
||||
pub cancelled: bool,
|
||||
pub error: Option<String>,
|
||||
/// The job works on an Object Matte (else a Roto Brush & Refine Edge).
|
||||
pub object_matte: bool,
|
||||
}
|
||||
|
||||
impl RotoProgress {
|
||||
pub fn banner(&self) -> String {
|
||||
let pc = (100 * self.done).checked_div(self.total).unwrap_or(0);
|
||||
match self.task {
|
||||
Some(RotoTask::Freeze) => format!("Freezing Roto Brush & Refine Edge: {pc}%"),
|
||||
_ => format!("Propagating Roto Brush: {pc}%"),
|
||||
match (self.task, self.object_matte) {
|
||||
(Some(RotoTask::Freeze), false) => format!("Freezing Roto Brush & Refine Edge: {pc}%"),
|
||||
(Some(RotoTask::Freeze), true) => format!("Freezing Object Matte: {pc}%"),
|
||||
(_, false) => format!("Propagating Roto Brush: {pc}%"),
|
||||
(_, true) => format!("Tracking Object Matte: {pc}%"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -130,8 +140,11 @@ pub struct RotoJob {
|
|||
pub layer: LayerId,
|
||||
pub effect: Uid,
|
||||
pub task: RotoTask,
|
||||
/// The strokes the job was started with (a Freeze result is dropped when they changed).
|
||||
/// What the job was started from: the strokes (Roto Brush) or the prompt (Object Matte); a
|
||||
/// Freeze result is dropped when it changed.
|
||||
strokes: String,
|
||||
/// The job works on an Object Matte.
|
||||
pub object_matte: bool,
|
||||
}
|
||||
|
||||
impl RotoJob {
|
||||
|
|
@ -147,6 +160,26 @@ pub(crate) fn is_roto(g: &PropGroup) -> bool {
|
|||
matches!(&g.kind, GroupKind::Effect { effect } if effect == fx::ID)
|
||||
}
|
||||
|
||||
/// An Object Matte instance (its Input Key is kept current like Roto Brush's; it segments on
|
||||
/// demand while rendering).
|
||||
pub(crate) fn is_object_matte(g: &PropGroup) -> bool {
|
||||
matches!(&g.kind, GroupKind::Effect { effect } if effect == effectcraft_effects::object_matte::ID)
|
||||
}
|
||||
|
||||
/// What an Object Matte's segmentation is made from besides its frames: the prompt and tracking
|
||||
/// parameters (static values), for queueing it after edits and guarding a Freeze.
|
||||
pub(crate) fn om_prompt(g: &PropGroup) -> String {
|
||||
let ps = params_static(g);
|
||||
let mut keys: Vec<&String> = ps.values.keys().filter(|k| ["objectPoint", "useBox", "box", "objectTracking/"].iter().any(|p| k.starts_with(p))).collect();
|
||||
keys.sort();
|
||||
keys.iter().map(|k| format!("{k}={:?};", ps.values.get(*k))).collect()
|
||||
}
|
||||
|
||||
/// The job-relevant guard of an instance: its strokes (Roto Brush) or prompt (Object Matte).
|
||||
fn guard_of(g: &PropGroup) -> String {
|
||||
if is_object_matte(g) { om_prompt(g) } else { str_of(g, fx::STROKES) }
|
||||
}
|
||||
|
||||
fn str_of(g: &PropGroup, m: &str) -> String {
|
||||
match g.get(m).map(|p| &p.value) {
|
||||
Some(Value::Str(s)) => s.clone(),
|
||||
|
|
@ -188,8 +221,9 @@ pub fn input_key(p: &Project, comp: ItemId, layer: &Layer, uid: Uid) -> String {
|
|||
format!("{:016x}", fnv(&s))
|
||||
}
|
||||
|
||||
/// After an edit: refresh Input Keys; frozen mattes made from other frames are dropped.
|
||||
/// Returns the instances whose segmentation changed (to propagate again).
|
||||
/// After an edit: refresh Input Keys (Roto Brush and Object Matte); frozen mattes made from
|
||||
/// other frames are dropped. Returns the instances whose segmentation changed (to propagate
|
||||
/// again): new frames, strokes (Roto Brush) or prompt (Object Matte).
|
||||
pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> {
|
||||
let mut todo = vec![];
|
||||
let mut set = vec![];
|
||||
|
|
@ -197,13 +231,13 @@ pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerI
|
|||
let ItemKind::Comp(c) = &it.kind else { continue };
|
||||
for l in &c.layers {
|
||||
let Some(fxg) = l.effects() else { continue };
|
||||
for g in fxg.groups().filter(|g| is_roto(g)) {
|
||||
for g in fxg.groups().filter(|g| is_roto(g) || is_object_matte(g)) {
|
||||
let key = input_key(after, *cid, l, g.uid);
|
||||
let old = before.comp(*cid).and_then(|bc| bc.layer(l.id)).and_then(|ol| ol.props.find_group(g.uid));
|
||||
if str_of(g, INPUT_KEY) != key {
|
||||
set.push((*cid, l.id, g.uid, key));
|
||||
todo.push((*cid, l.id, g.uid));
|
||||
} else if old.is_none_or(|o| str_of(o, fx::STROKES) != str_of(g, fx::STROKES)) {
|
||||
} else if old.is_none_or(|o| guard_of(o) != guard_of(g)) {
|
||||
todo.push((*cid, l.id, g.uid));
|
||||
}
|
||||
}
|
||||
|
|
@ -219,7 +253,11 @@ pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerI
|
|||
}
|
||||
}
|
||||
todo.retain(|(c, l, u)| {
|
||||
after.comp(*c).and_then(|c| c.layer(*l)).and_then(|l| l.props.find_group(*u)).is_some_and(|g| !fx::data(¶ms_static(g)).is_empty())
|
||||
after
|
||||
.comp(*c)
|
||||
.and_then(|c| c.layer(*l))
|
||||
.and_then(|l| l.props.find_group(*u))
|
||||
.is_some_and(|g| is_object_matte(g) || !fx::data(¶ms_static(g)).is_empty())
|
||||
});
|
||||
todo
|
||||
}
|
||||
|
|
@ -274,6 +312,7 @@ struct Work {
|
|||
fps: f64,
|
||||
frames: Vec<i64>,
|
||||
task: RotoTask,
|
||||
object_matte: bool,
|
||||
}
|
||||
|
||||
/// Renders the effect's input frames for a worker, keeping the last few. With a browser job
|
||||
|
|
@ -390,10 +429,13 @@ async fn run_work(w: Work, shared: &RotoShared) {
|
|||
let mut s = lock(&shared.state);
|
||||
s.task = Some(w.task);
|
||||
s.total = w.frames.len() as u64;
|
||||
s.object_matte = w.object_matte;
|
||||
}
|
||||
let cancel = || shared.cancel.load(Ordering::Relaxed);
|
||||
let chain = fx::Chain::new(&w.params, w.size, w.fps, 1.0);
|
||||
let mut frozen = FrozenCache { key: fx::frozen_key(&w.params, w.size, w.fps), ..Default::default() };
|
||||
let model = fx::installed_model();
|
||||
let key = if w.object_matte { om::frozen_key(&w.params, w.size, w.fps, model.as_deref()) } else { fx::frozen_key(&w.params, w.size, w.fps) };
|
||||
let mut frozen = FrozenCache { key, ..Default::default() };
|
||||
for (k, f) in w.frames.iter().enumerate() {
|
||||
if cancel() {
|
||||
let mut s = lock(&shared.state);
|
||||
|
|
@ -404,9 +446,21 @@ async fn run_work(w: Work, shared: &RotoShared) {
|
|||
// The frame and its neighbours (what a step reads), on the GPU.
|
||||
src.prefetch(&[*f, f - 1, f + 1]).await;
|
||||
let mut get = |g: i64| src.get(g);
|
||||
let ok = match w.task {
|
||||
RotoTask::Propagate => chain.ensure(*f, &mut get, &cancel).is_some(),
|
||||
RotoTask::Freeze => match fx::frame_matte(&w.params, w.size, w.fps, *f, &mut get, &cancel) {
|
||||
let ok = match (w.task, w.object_matte) {
|
||||
(RotoTask::Propagate, true) => om::ensure(&w.params, w.size, w.fps, *f, model.clone(), &mut get, &cancel).is_some(),
|
||||
(RotoTask::Freeze, true) => {
|
||||
let img = get(*f);
|
||||
match img.map(|img| (om::frame_matte(&w.params, w.size, w.fps, *f, 1.0, &img, model.clone(), &mut get, &cancel), img)) {
|
||||
Some((om::FrameMatte::Alpha(a), img)) => {
|
||||
frozen.size = [img.width as usize, img.height as usize];
|
||||
frozen.frames.insert(*f, fx::freeze_frame(&a, &[]));
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
(RotoTask::Propagate, false) => chain.ensure(*f, &mut get, &cancel).is_some(),
|
||||
(RotoTask::Freeze, false) => match fx::frame_matte(&w.params, w.size, w.fps, *f, &mut get, &cancel) {
|
||||
Some((a, b)) => {
|
||||
if let Some(img) = src.get(*f) {
|
||||
frozen.size = [img.width as usize, img.height as usize];
|
||||
|
|
@ -461,12 +515,15 @@ impl Session {
|
|||
self.roto_job.as_ref().map(|j| j.progress()).or_else(|| {
|
||||
self.offloaded_roto().map(|(j, task)| {
|
||||
let p = &j.progress;
|
||||
let object_matte =
|
||||
self.project.comp(j.comp).and_then(|c| c.layer(j.layer)).and_then(|l| l.props.find_group(j.uid)).is_some_and(is_object_matte);
|
||||
RotoProgress {
|
||||
task: Some(task),
|
||||
done: p.done,
|
||||
total: p.total,
|
||||
elapsed: p.elapsed,
|
||||
fps: crate::offload::rate(p.done, p.elapsed),
|
||||
object_matte,
|
||||
..Default::default()
|
||||
}
|
||||
})
|
||||
|
|
@ -478,7 +535,8 @@ impl Session {
|
|||
self.roto_job.as_ref().map(|j| (j.comp, j.layer, j.effect, j.task)).or_else(|| self.offloaded_roto().map(|(j, task)| (j.comp, j.layer, j.uid, task)))
|
||||
}
|
||||
|
||||
/// The segmentation chain a propagation of `effect` computes (its frames' cache keys).
|
||||
/// The segmentation chain a propagation of Roto Brush `effect` computes (its frames' cache
|
||||
/// keys).
|
||||
pub fn roto_chain(&self, comp: ItemId, layer: LayerId, effect: Uid) -> Option<fx::Chain> {
|
||||
let c = self.project.comp(comp)?;
|
||||
let l = c.layer(layer)?;
|
||||
|
|
@ -489,6 +547,27 @@ impl Session {
|
|||
Some(fx::Chain::new(¶ms, layer_size(&self.project, c, l), c.frame_rate.as_f64(), 1.0))
|
||||
}
|
||||
|
||||
/// The cache keys of the segmentations a propagation of `effect` (Roto Brush or Object
|
||||
/// Matte) computes, by layer frame.
|
||||
pub fn roto_keys(&self, comp: ItemId, layer: LayerId, effect: Uid) -> Option<Vec<(i64, u64)>> {
|
||||
if let Some(ch) = self.roto_chain(comp, layer, effect) {
|
||||
return Some(ch.keys.into_iter().collect());
|
||||
}
|
||||
let c = self.project.comp(comp)?;
|
||||
let l = c.layer(layer)?;
|
||||
let g = l.effects()?.groups().find(|g| g.uid == effect && is_object_matte(g))?;
|
||||
let params = self.om_params(comp, l, g)?;
|
||||
let frames = om::job_frames(¶ms, frame_limits(c, l));
|
||||
Some(om::frame_keys(¶ms, layer_size(&self.project, c, l), c.frame_rate.as_f64(), &frames, fx::installed_model()))
|
||||
}
|
||||
|
||||
/// An Object Matte's parameters at its reference frame (what its jobs segment with).
|
||||
fn om_params(&self, comp: ItemId, l: &Layer, g: &PropGroup) -> Option<Params> {
|
||||
let c = self.project.comp(comp)?;
|
||||
let t = comp_time_of(c, l, om::reference_frame(¶ms_static(g)));
|
||||
Some(params_at(&self.project, comp, l, g, t, self.expr.as_deref()))
|
||||
}
|
||||
|
||||
pub fn stop_roto(&mut self) -> bool {
|
||||
if self.cancel_offloaded(JobKind::RotoFreeze) || self.cancel_offloaded(JobKind::RotoPropagate) {
|
||||
return true;
|
||||
|
|
@ -522,17 +601,36 @@ impl Session {
|
|||
let l = c.layer(layer).ok_or("no layer")?;
|
||||
let fxg = l.effects().ok_or("the layer has no effects")?;
|
||||
let index = fxg.groups().position(|g| g.uid == effect).ok_or("no such effect")?;
|
||||
let g = fxg.groups().nth(index).filter(|g| is_roto(g)).ok_or("not a Roto Brush & Refine Edge effect")?;
|
||||
let g = fxg.groups().nth(index).filter(|g| is_roto(g) || is_object_matte(g)).ok_or("not a Roto Brush & Refine Edge or Object Matte effect")?;
|
||||
let object_matte = is_object_matte(g);
|
||||
let size = layer_size(&self.project, c, l);
|
||||
let fps = c.frame_rate.as_f64();
|
||||
let pstat = params_static(g);
|
||||
let d = fx::data(&pstat);
|
||||
if d.is_empty() {
|
||||
return Err("draw a foreground stroke with the Roto Brush tool first".into());
|
||||
}
|
||||
let t = comp_time_of(c, l, d.base.unwrap_or(0));
|
||||
let params = params_at(&self.project, comp, l, g, t, self.expr.as_deref());
|
||||
let frames = job_frames(&d, if task == RotoTask::Freeze { Direction::Both } else { dir });
|
||||
let (params, frames) = if object_matte {
|
||||
let params = self.om_params(comp, l, g).ok_or("no composition")?;
|
||||
let base = om::reference_frame(¶ms);
|
||||
if om::roto_data(¶ms, size, base, fx::installed_model().is_some()).is_none() {
|
||||
return Err("set the object first: click it with the Object Matte tool (its point is off the layer or outside its box)".into());
|
||||
}
|
||||
let all = om::job_frames(¶ms, frame_limits(c, l));
|
||||
if all.is_empty() {
|
||||
return Err(format!("the reference frame {base} is outside the layer"));
|
||||
}
|
||||
// Forward / backward from the reference frame, as for a Roto Brush.
|
||||
let frames = all
|
||||
.into_iter()
|
||||
.filter(|f| task == RotoTask::Freeze || dir == Direction::Both || *f == base || (*f > base) == (dir == Direction::Forward))
|
||||
.collect();
|
||||
(params, frames)
|
||||
} else {
|
||||
let pstat = params_static(g);
|
||||
let d = fx::data(&pstat);
|
||||
if d.is_empty() {
|
||||
return Err("draw a foreground stroke with the Roto Brush tool first".into());
|
||||
}
|
||||
let t = comp_time_of(c, l, d.base.unwrap_or(0));
|
||||
let params = params_at(&self.project, comp, l, g, t, self.expr.as_deref());
|
||||
(params, job_frames(&d, if task == RotoTask::Freeze { Direction::Both } else { dir }))
|
||||
};
|
||||
let n = frames.len();
|
||||
let work = Work {
|
||||
project: self.project.clone(),
|
||||
|
|
@ -547,8 +645,9 @@ impl Session {
|
|||
fps,
|
||||
frames,
|
||||
task,
|
||||
object_matte,
|
||||
};
|
||||
let strokes = str_of(g, fx::STROKES);
|
||||
let strokes = guard_of(g);
|
||||
if !wait && self.offloads() {
|
||||
drop(work);
|
||||
self.offload_analysis(match task {
|
||||
|
|
@ -559,6 +658,13 @@ impl Session {
|
|||
}
|
||||
let wait = wait || cfg!(target_arch = "wasm32");
|
||||
let shared = Arc::new(RotoShared::default());
|
||||
{
|
||||
// Known before the worker starts (the progress shows at once).
|
||||
let mut st = lock(&shared.state);
|
||||
st.task = Some(task);
|
||||
st.total = n as u64;
|
||||
st.object_matte = object_matte;
|
||||
}
|
||||
let thread = if wait {
|
||||
crate::offload::run_or_queue({
|
||||
let sh = shared.clone();
|
||||
|
|
@ -574,7 +680,7 @@ impl Session {
|
|||
.map_err(|e| e.to_string())?,
|
||||
)
|
||||
};
|
||||
self.roto_job = Some(RotoJob { shared, thread, comp, layer, effect, task, strokes });
|
||||
self.roto_job = Some(RotoJob { shared, thread, comp, layer, effect, task, strokes, object_matte });
|
||||
self.events.push(Event::ProjectChanged { revision: self.revision });
|
||||
self.poll_roto_job();
|
||||
Ok(n)
|
||||
|
|
@ -591,40 +697,43 @@ impl Session {
|
|||
let _ = t.join();
|
||||
}
|
||||
let st = job.progress();
|
||||
let name = if job.object_matte { "Object Matte" } else { "Roto Brush" };
|
||||
if let Some(e) = &st.error {
|
||||
self.events.push(Event::Toast { message: format!("Roto Brush: {e}"), error: true });
|
||||
self.events.push(Event::Toast { message: format!("{name}: {e}"), error: true });
|
||||
self.bump();
|
||||
return true;
|
||||
}
|
||||
if st.cancelled {
|
||||
self.events.push(Event::Toast { message: "Roto Brush: stopped".into(), error: false });
|
||||
self.events.push(Event::Toast { message: format!("{name}: stopped"), error: false });
|
||||
self.bump();
|
||||
return true;
|
||||
}
|
||||
match job.task {
|
||||
RotoTask::Propagate => {
|
||||
self.events.push(Event::Toast { message: format!("Roto Brush: propagated {} frame(s) in {:.1} s", st.done, st.elapsed), error: false });
|
||||
let verb = if job.object_matte { "tracked" } else { "propagated" };
|
||||
self.events.push(Event::Toast { message: format!("{name}: {verb} {} frame(s) in {:.1} s", st.done, st.elapsed), error: false });
|
||||
// Rendered frames depend on the cache: redraw.
|
||||
self.bump();
|
||||
}
|
||||
RotoTask::Freeze => {
|
||||
let Some(fc) = lock(&job.shared.frozen).take() else { return true };
|
||||
let (cid, lid, uid, strokes) = (job.comp, job.layer, job.effect, job.strokes.clone());
|
||||
let what = if job.object_matte { "prompt" } else { "strokes" };
|
||||
let r = self.edit("Freeze", None, |p, _| {
|
||||
let g = p
|
||||
.comp_mut(cid)
|
||||
.and_then(|c| c.layer_mut(lid))
|
||||
.and_then(|l| l.props.find_group_mut(uid))
|
||||
.ok_or_else(|| crate::EngineError::Other("the Roto Brush effect was removed".into()))?;
|
||||
if str_of(g, fx::STROKES) != strokes {
|
||||
return Err(crate::EngineError::Other("the strokes changed while freezing".into()));
|
||||
.ok_or_else(|| crate::EngineError::Other(format!("the {name} effect was removed")))?;
|
||||
if guard_of(g) != strokes {
|
||||
return Err(crate::EngineError::Other(format!("the {what} changed while freezing")));
|
||||
}
|
||||
set_str(g, FROZEN, fc.to_json());
|
||||
Ok(())
|
||||
});
|
||||
let msg = match r {
|
||||
Ok(()) => format!("Roto Brush: froze {} frame(s) in {:.1} s", fc.frames.len(), st.elapsed),
|
||||
Err(e) => format!("Roto Brush: {e}"),
|
||||
Ok(()) => format!("{name}: froze {} frame(s) in {:.1} s", fc.frames.len(), st.elapsed),
|
||||
Err(e) => format!("{name}: {e}"),
|
||||
};
|
||||
self.events.push(Event::Toast { message: msg, error: false });
|
||||
}
|
||||
|
|
@ -654,6 +763,12 @@ impl Session {
|
|||
let Some(c) = self.project.comp(comp) else { return out };
|
||||
let Some(l) = c.layer(layer) else { return out };
|
||||
let Some(g) = l.props.find_group(effect) else { return out };
|
||||
if is_object_matte(g) {
|
||||
for (f, k) in self.roto_keys(comp, layer, effect).unwrap_or_default() {
|
||||
out.insert(f, fx::cached(k, 1.0).is_some());
|
||||
}
|
||||
return out;
|
||||
}
|
||||
let ps = params_static(g);
|
||||
let ch = fx::Chain::new(&ps, layer_size(&self.project, c, l), c.frame_rate.as_f64(), 1.0);
|
||||
for f in ch.keys.keys() {
|
||||
|
|
|
|||
|
|
@ -132,6 +132,10 @@ impl Session {
|
|||
},
|
||||
};
|
||||
self.layer_cache.set_disk(Some((dc.clone(), footage_salt(&self.project))));
|
||||
// Roto Brush / Object Matte segmentations persist there too (held weakly: switching the
|
||||
// disk cache off drops it).
|
||||
let store: std::sync::Arc<dyn effectcraft_effects::roto::SegStore> = dc.clone();
|
||||
effectcraft_effects::roto::set_seg_store(Some(std::sync::Arc::downgrade(&store)));
|
||||
self.disk_cache = Some(dc);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -380,3 +380,264 @@ fn offloaded_propagation_streams_segmentations_back() {
|
|||
s.execute("roto.propagate", json!({"layer": clip.0, "wait": true})).unwrap();
|
||||
assert!(s.offloaded.is_empty());
|
||||
}
|
||||
|
||||
/// Object Matte: one click on the disk on frame 0 isolates it there and on the frames it is
|
||||
/// followed to; the Input Key follows the layer's frames; bad prompts are refused.
|
||||
#[test]
|
||||
fn object_matte_from_a_click() {
|
||||
hold_roto_cache_test_lock();
|
||||
let (mut s, clip, solid) = setup(|f| draw(center(f)));
|
||||
let cid = s.active_comp_id().unwrap();
|
||||
s.edit("remove solid", None, |p, _| {
|
||||
p.comp_mut(cid).unwrap().layers.retain(|l| l.id != solid);
|
||||
for it in p.items.values_mut() {
|
||||
if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind {
|
||||
f.path = "object-matte.mov".into();
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
s.execute("time.set", json!({"time": 0})).unwrap();
|
||||
let c = center(0);
|
||||
let r = s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "point": c, "after": 6})).unwrap();
|
||||
assert_eq!(r["frame"], 0);
|
||||
assert_eq!(r["span"], json!([-30, 6]));
|
||||
let uid = r["effect"].as_u64().unwrap();
|
||||
let key = |s: &Session| {
|
||||
let g = s.active_comp().unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().clone();
|
||||
crate::roto::params_static(&g).get(effectcraft_effects::roto::INPUT_KEY).cloned()
|
||||
};
|
||||
let k0 = key(&s);
|
||||
assert!(matches!(&k0, Some(effectcraft_keyframe::Value::Str(k)) if !k.is_empty()), "{k0:?}");
|
||||
let wrong = |s: &Session, f: u32| {
|
||||
let c = center(f);
|
||||
alpha_at(s, f)
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(i, v)| {
|
||||
let d = (((*i as u32 % W) as f64 + 0.5 - c[0]).hypot((*i as u32 / W) as f64 + 0.5 - c[1])) - R;
|
||||
(d < -3.0 && **v < 0.5) || (d > 3.0 && **v > 0.5)
|
||||
})
|
||||
.count()
|
||||
};
|
||||
let w0 = wrong(&s, 0);
|
||||
assert!(w0 < 40, "{w0} wrong pixels on the reference frame");
|
||||
let w4 = wrong(&s, 4);
|
||||
assert!(w4 < 60, "{w4} wrong pixels on frame 4");
|
||||
// Past Frames After: transparent.
|
||||
assert!(alpha_at(&s, 8).iter().all(|v| *v == 0.0));
|
||||
// Other frames under the effect: a new Input Key (the mattes are recomputed).
|
||||
s.edit("swap clip", None, |p, _| {
|
||||
for it in p.items.values_mut() {
|
||||
if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind {
|
||||
f.path = "object-matte-2.mov".into();
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert_ne!(key(&s), k0);
|
||||
// Bad prompts are refused.
|
||||
for bad in [
|
||||
json!({"layer": clip.0}),
|
||||
json!({"layer": clip.0, "point": [-10, 5]}),
|
||||
json!({"layer": clip.0, "point": [10, "x"]}),
|
||||
json!({"layer": clip.0, "point": c, "box": [0, 0, 10, 10]}),
|
||||
json!({"layer": clip.0, "point": c, "frame": 999}),
|
||||
] {
|
||||
assert!(s.execute_checked("objectMatte.apply", bad.clone()).is_err(), "{bad}");
|
||||
}
|
||||
// A box and a static prompt are accepted; the second instance is numbered.
|
||||
let r = s
|
||||
.execute_checked("objectMatte.apply", json!({"layer": clip.0, "point": c, "box": [c[0] + 50.0, c[1] + 50.0, c[0] - 50.0, c[1] - 50.0], "track": false}))
|
||||
.unwrap();
|
||||
let g = s.active_comp().unwrap().layer(clip).unwrap().props.find_group(r["effect"].as_u64().unwrap()).unwrap().clone();
|
||||
assert_eq!(g.name, "Object Matte 2");
|
||||
}
|
||||
|
||||
/// `objectMatte.apply` with `effect` re-prompts that instance in one undo step: the new point and
|
||||
/// box, the box dropped when none is given, the reach kept unless given.
|
||||
#[test]
|
||||
fn object_matte_reprompt_updates_the_instance() {
|
||||
let (mut s, clip, solid) = setup(|f| draw(center(f)));
|
||||
let cid = s.active_comp_id().unwrap();
|
||||
s.edit("remove solid", None, |p, _| {
|
||||
p.comp_mut(cid).unwrap().layers.retain(|l| l.id != solid);
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
let c = center(0);
|
||||
let r = s
|
||||
.execute_checked(
|
||||
"objectMatte.apply",
|
||||
json!({"layer": clip.0, "point": c, "box": [c[0] - 50.0, c[1] - 50.0, c[0] + 50.0, c[1] + 50.0], "after": 6, "frame": 0}),
|
||||
)
|
||||
.unwrap();
|
||||
let uid = r["effect"].as_u64().unwrap();
|
||||
let count = |s: &Session| s.active_comp().unwrap().layer(clip).unwrap().effects().unwrap().groups().count();
|
||||
let ps = |s: &Session| crate::roto::params_static(s.active_comp().unwrap().layer(clip).unwrap().props.find_group(uid).unwrap());
|
||||
let undo0 = s.history.undo.len();
|
||||
let r = s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "effect": uid, "point": [c[0] + 4.0, c[1]], "frame": 2})).unwrap();
|
||||
assert_eq!((r["effect"].as_u64(), r["updated"].as_bool()), (Some(uid), Some(true)));
|
||||
assert_eq!(r["span"], json!([-28, 8]));
|
||||
assert_eq!(count(&s), 1);
|
||||
assert_eq!(s.history.undo.len(), undo0 + 1);
|
||||
let p = ps(&s);
|
||||
assert_eq!(p.get("objectPoint"), Some(&effectcraft_keyframe::Value::Vec2([c[0] + 4.0, c[1]])));
|
||||
assert_eq!(p.get("useBox"), Some(&effectcraft_keyframe::Value::Bool(false)));
|
||||
assert_eq!(effectcraft_effects::object_matte::reference_frame(&p), 2);
|
||||
assert_eq!(effectcraft_effects::object_matte::reach(&p), [30, 6]);
|
||||
// Undo restores the first prompt.
|
||||
s.execute("edit.undo", json!({})).unwrap();
|
||||
assert_eq!(ps(&s).get("useBox"), Some(&effectcraft_keyframe::Value::Bool(true)));
|
||||
// An unknown or non-Object Matte effect is refused.
|
||||
assert!(s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "effect": 999_999, "point": c})).is_err());
|
||||
}
|
||||
|
||||
/// Object Matte runs in the background like Roto Brush propagation: queued by the edit, a job
|
||||
/// that returns at once and reports progress, stops when cancelled (keeping what it computed),
|
||||
/// computes the tracked frames when left to run, and Freeze stores mattes that renders then use
|
||||
/// without segmenting again.
|
||||
#[test]
|
||||
fn object_matte_background_job_cancel_and_freeze() {
|
||||
hold_roto_cache_test_lock();
|
||||
let (mut s, clip, solid) = setup(|f| draw(center(f)));
|
||||
let cid = s.active_comp_id().unwrap();
|
||||
s.edit("remove solid", None, |p, _| {
|
||||
p.comp_mut(cid).unwrap().layers.retain(|l| l.id != solid);
|
||||
for it in p.items.values_mut() {
|
||||
if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind {
|
||||
f.path = "object-matte-job.mov".into();
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
let c = center(0);
|
||||
let r = s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "point": c, "before": 0, "after": 24, "frame": 0})).unwrap();
|
||||
let uid = r["effect"].as_u64().unwrap();
|
||||
assert!(s.roto_pending.contains(&(cid, clip, uid)), "the new prompt is queued for tracking");
|
||||
let wait_done = |s: &mut Session| {
|
||||
let t = std::time::Instant::now();
|
||||
while s.is_roto_running() {
|
||||
assert!(t.elapsed().as_secs() < 120, "the job never finished");
|
||||
std::thread::sleep(std::time::Duration::from_millis(5));
|
||||
s.poll_roto(false);
|
||||
}
|
||||
s.poll_roto(false);
|
||||
};
|
||||
let toast = |s: &Session, m: &str| s.events.iter().any(|e| matches!(e, crate::Event::Toast { message, .. } if message.starts_with(m)));
|
||||
let status = |s: &mut Session| s.execute("objectMatte.status", json!({"layer": clip.0, "effect": uid})).unwrap();
|
||||
|
||||
// Cancelled at once: it stops early, and says so.
|
||||
let r = s.execute_checked("objectMatte.propagate", json!({"layer": clip.0, "effect": uid})).unwrap();
|
||||
assert_eq!(r["frames"], 25);
|
||||
assert!(!s.roto_pending.contains(&(cid, clip, uid)));
|
||||
let _ = s.execute("roto.cancel", json!({}));
|
||||
wait_done(&mut s);
|
||||
assert!(toast(&s, "Object Matte: stopped"), "{:?}", s.events);
|
||||
let st = status(&mut s);
|
||||
assert_eq!(st["frames"].as_array().unwrap().len(), 25, "{st}");
|
||||
let early = st["computed"].as_array().unwrap().len();
|
||||
assert!(early < 25, "{st}");
|
||||
|
||||
// Left to run: the call returns while the job works (progress visible), then every tracked
|
||||
// frame is computed.
|
||||
let r = s.execute_checked("objectMatte.propagate", json!({"layer": clip.0, "effect": uid})).unwrap();
|
||||
assert_eq!(r["running"], true, "{r}");
|
||||
let pr = s.roto_progress().unwrap();
|
||||
assert!(pr.object_matte && pr.banner().starts_with("Tracking Object Matte"), "{pr:?}");
|
||||
// One job at a time.
|
||||
assert!(s.execute_checked("objectMatte.propagate", json!({"layer": clip.0, "effect": uid})).is_err());
|
||||
wait_done(&mut s);
|
||||
assert!(toast(&s, "Object Matte: tracked"), "{:?}", s.events);
|
||||
assert_eq!(status(&mut s)["computed"].as_array().unwrap().len(), 25);
|
||||
let before = alpha_at(&s, 4);
|
||||
|
||||
// Freeze, then drop the segmentations: renders use the frozen mattes and compute nothing.
|
||||
let r = s.execute_checked("objectMatte.freeze", json!({"layer": clip.0, "effect": uid, "wait": true})).unwrap();
|
||||
assert_eq!(r["frozen"], true, "{r}");
|
||||
let keys: Vec<u64> = s.roto_keys(cid, clip, uid).unwrap().into_iter().map(|(_, k)| k).collect();
|
||||
assert_eq!(keys.len(), 25);
|
||||
effectcraft_effects::roto::forget(keys.iter().copied());
|
||||
let frozen = alpha_at(&s, 4);
|
||||
let diff = before.iter().zip(&frozen).filter(|(a, b)| (**a - **b).abs() > 0.02).count();
|
||||
assert!(diff < 30, "{diff} pixels differ from the computed matte");
|
||||
assert!(status(&mut s)["computed"].as_array().unwrap().is_empty(), "a frozen frame was segmented again");
|
||||
assert_eq!(status(&mut s)["frozen"], true);
|
||||
// A frozen prompt is refused; Unfreeze computes again.
|
||||
assert!(s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "effect": uid, "point": c})).is_err());
|
||||
s.execute_checked("objectMatte.unfreeze", json!({"layer": clip.0, "effect": uid})).unwrap();
|
||||
let _ = alpha_at(&s, 4);
|
||||
assert!(status(&mut s)["computed"].as_array().unwrap().iter().any(|f| f == 4));
|
||||
// Undo brings the frozen mattes back.
|
||||
s.execute("edit.undo", json!({})).unwrap();
|
||||
assert_eq!(status(&mut s)["frozen"], true);
|
||||
}
|
||||
|
||||
/// Computed Object Matte segmentations go to the persistent disk cache: a second session on the
|
||||
/// same folder and project gets them back without segmenting (its frames cannot even be read
|
||||
/// here), and another prompt (other keys) finds nothing.
|
||||
#[test]
|
||||
fn object_matte_mattes_persist_in_the_disk_cache() {
|
||||
use effectcraft_effects::object_matte as om;
|
||||
use effectcraft_render::disk_cache::Kind;
|
||||
hold_roto_cache_test_lock();
|
||||
let dir = std::env::temp_dir().join(format!("effectcraft-om-disk-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
let enable = |s: &mut Session| {
|
||||
s.execute_checked("prefs.set", json!({"values": {"disk.diskCacheFolder": dir.to_string_lossy(), "disk.diskCacheEnabled": true}})).unwrap();
|
||||
};
|
||||
let (mut s, clip, solid) = setup(|f| draw(center(f)));
|
||||
enable(&mut s);
|
||||
let cid = s.active_comp_id().unwrap();
|
||||
s.edit("remove solid", None, |p, _| {
|
||||
p.comp_mut(cid).unwrap().layers.retain(|l| l.id != solid);
|
||||
for it in p.items.values_mut() {
|
||||
if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind {
|
||||
f.path = "object-matte-disk.mov".into();
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
let c = center(0);
|
||||
let r = s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "point": c, "before": 0, "after": 5, "frame": 0})).unwrap();
|
||||
let uid = r["effect"].as_u64().unwrap();
|
||||
s.execute_checked("objectMatte.propagate", json!({"layer": clip.0, "effect": uid, "wait": true})).unwrap();
|
||||
let keys = s.roto_keys(cid, clip, uid).unwrap();
|
||||
assert_eq!(keys.len(), 6);
|
||||
let dc = s.disk_cache.clone().unwrap();
|
||||
dc.flush();
|
||||
assert!(dc.keys(Kind::Matte).len() >= 6, "{:?}", dc.stats());
|
||||
let before = alpha_at(&s, 3);
|
||||
let saved = s.project.clone();
|
||||
drop(dc);
|
||||
drop(s);
|
||||
effectcraft_effects::roto::forget(keys.iter().map(|(_, k)| *k));
|
||||
|
||||
// "Reopen": a new session on the same folder and project.
|
||||
let (mut s2, _, _) = setup(|f| draw(center(f)));
|
||||
enable(&mut s2);
|
||||
s2.project = saved;
|
||||
s2.bump();
|
||||
s2.open_comp(cid);
|
||||
assert_eq!(s2.roto_keys(cid, clip, uid).unwrap(), keys, "the same input key, so the same keys");
|
||||
let comp = s2.active_comp().unwrap().clone();
|
||||
let l = comp.layer(clip).unwrap();
|
||||
let ps = crate::roto::params_static(l.props.find_group(uid).unwrap());
|
||||
let size = crate::roto::layer_size(&s2.project, &comp, l);
|
||||
// No frames to segment from: whatever comes back was read from disk.
|
||||
for (f, _) in &keys {
|
||||
assert!(om::ensure(&ps, size, 25.0, *f, None, &mut |_| None, &|| false).is_some(), "frame {f} not reused from disk");
|
||||
}
|
||||
assert!(s2.disk_cache.as_ref().unwrap().stats().hits >= 6);
|
||||
let again = alpha_at(&s2, 3);
|
||||
assert_eq!(again, before, "the disk-cached matte renders the same");
|
||||
// Another prompt: other keys, nothing on disk.
|
||||
let mut moved = ps.clone();
|
||||
moved.values.insert("objectPoint".into(), effectcraft_keyframe::Value::Vec2([c[0] + 3.0, c[1]]));
|
||||
assert!(om::ensure(&moved, size, 25.0, 0, None, &mut |_| None, &|| false).is_none());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,9 @@
|
|||
//! project therefore never reads stale pixels: its keys simply change.
|
||||
//! * Payloads are LZ4-compressed (`lz4_flex`, MIT): frames as 8-bit premultiplied RGBA, layer
|
||||
//! buffers as `f32` RGBA with the bytes shuffled into planes (exact, so cached and fresh
|
||||
//! renders are bit-identical). Every file ends in a checksum; damaged files read as misses and
|
||||
//! renders are bit-identical), Roto Brush / Object Matte segmentations as
|
||||
//! `FrameSeg::to_bytes` (keyed by their content-keyed chain key and scale, so they are
|
||||
//! reused whenever the same frames, settings and prompt come back, also after a restart). Every file ends in a checksum; damaged files read as misses and
|
||||
//! are deleted.
|
||||
//! * Size limit with least-recently-used eviction. The in-memory index is rebuilt from the
|
||||
//! folder when the cache opens (file modification times order it), so the cache survives
|
||||
|
|
@ -37,6 +39,8 @@ const QUEUE_LIMIT: usize = 64;
|
|||
pub enum Kind {
|
||||
Frame,
|
||||
Layer,
|
||||
/// A Roto Brush / Object Matte segmentation.
|
||||
Matte,
|
||||
}
|
||||
|
||||
impl Kind {
|
||||
|
|
@ -45,12 +49,14 @@ impl Kind {
|
|||
match self {
|
||||
Kind::Frame => "frames",
|
||||
Kind::Layer => "layers",
|
||||
Kind::Matte => "mattes",
|
||||
}
|
||||
}
|
||||
fn tag(self) -> u8 {
|
||||
match self {
|
||||
Kind::Frame => 1,
|
||||
Kind::Layer => 2,
|
||||
Kind::Matte => 3,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -319,7 +325,7 @@ impl DiskCache {
|
|||
/// Rebuild the index from the folder (oldest files first).
|
||||
pub fn rescan(&self) {
|
||||
let mut found: Vec<(std::time::SystemTime, Kind, u128, u64)> = vec![];
|
||||
for kind in [Kind::Frame, Kind::Layer] {
|
||||
for kind in [Kind::Frame, Kind::Layer, Kind::Matte] {
|
||||
let Ok(shards) = std::fs::read_dir(self.dir.join(kind.dir())) else { continue };
|
||||
for shard in shards.flatten() {
|
||||
let Ok(files) = std::fs::read_dir(shard.path()) else { continue };
|
||||
|
|
@ -403,7 +409,7 @@ impl DiskCache {
|
|||
if let Ok(mut g) = self.index.lock() {
|
||||
g.clear();
|
||||
}
|
||||
for kind in [Kind::Frame, Kind::Layer] {
|
||||
for kind in [Kind::Frame, Kind::Layer, Kind::Matte] {
|
||||
let _ = std::fs::remove_dir_all(self.dir.join(kind.dir()));
|
||||
}
|
||||
}
|
||||
|
|
@ -555,6 +561,22 @@ impl DiskCache {
|
|||
self.write_now(Kind::Layer, key, &encode_layer(buf));
|
||||
}
|
||||
|
||||
/// A segmentation (chain key, scale) written by [`DiskCache::put_seg`].
|
||||
pub fn get_seg(&self, key: u64, scale: f64) -> Option<effectcraft_effects::roto::FrameSeg> {
|
||||
let p = self.read(Kind::Matte, seg_key(key, scale))?;
|
||||
let raw = lz4_flex::decompress_size_prepended(&p).ok()?;
|
||||
effectcraft_effects::roto::FrameSeg::from_bytes(&raw)
|
||||
}
|
||||
|
||||
/// Queue a segmentation for writing.
|
||||
pub fn put_seg(&self, key: u64, scale: f64, seg: &effectcraft_effects::roto::FrameSeg) {
|
||||
let k = seg_key(key, scale);
|
||||
if self.contains(Kind::Matte, k) {
|
||||
return;
|
||||
}
|
||||
self.enqueue(Kind::Matte, k, lz4_flex::compress_prepend_size(&seg.to_bytes()));
|
||||
}
|
||||
|
||||
/// Write a frame synchronously (tests).
|
||||
pub fn put_frame_now(&self, key: u128, width: u32, height: u32, rgba: &[u8]) {
|
||||
self.write_now(Kind::Frame, key, &encode_frame(width, height, rgba));
|
||||
|
|
@ -627,6 +649,24 @@ pub fn read_layer_entry(data: &[u8]) -> Option<Buf> {
|
|||
decode_layer(verify(data, Kind::Layer)?)
|
||||
}
|
||||
|
||||
/// The disk key of a segmentation: its chain key and scale.
|
||||
fn seg_key(key: u64, scale: f64) -> u128 {
|
||||
let mut h = Hash128::default();
|
||||
h.write(b"segmentation-v1");
|
||||
h.write_u64(key);
|
||||
h.write_u64(scale.to_bits());
|
||||
h.finish()
|
||||
}
|
||||
|
||||
impl effectcraft_effects::roto::SegStore for DiskCache {
|
||||
fn get_seg(&self, key: u64, scale: f64) -> Option<effectcraft_effects::roto::FrameSeg> {
|
||||
DiskCache::get_seg(self, key, scale)
|
||||
}
|
||||
fn put_seg(&self, key: u64, scale: f64, seg: &effectcraft_effects::roto::FrameSeg) {
|
||||
DiskCache::put_seg(self, key, scale, seg);
|
||||
}
|
||||
}
|
||||
|
||||
impl crate::cache::LayerStore for DiskCache {
|
||||
fn get_layer(&self, key: u128) -> Option<Arc<Buf>> {
|
||||
DiskCache::get_layer(self, key).map(Arc::new)
|
||||
|
|
@ -889,6 +929,38 @@ mod tests {
|
|||
assert_eq!(parse_entry_name(&format!("{key:032x}.tmp.1")), None);
|
||||
}
|
||||
|
||||
/// Segmentations round-trip by (chain key, scale), survive a reopen (rescan), are bounded by
|
||||
/// the size limit like every entry, and a damaged file is a miss.
|
||||
#[test]
|
||||
fn segmentations_round_trip_and_reopen() {
|
||||
use effectcraft_effects::roto::FrameSeg;
|
||||
let dir = tmpdir("seg");
|
||||
let dc = DiskCache::open(&dir, 1 << 30).unwrap();
|
||||
let seg = FrameSeg { w: 7, h: 5, matte: (0..35).map(|i| (i % 3 == 0) as u8).collect(), refine: vec![0; 35], ..Default::default() };
|
||||
assert!(dc.get_seg(42, 1.0).is_none());
|
||||
dc.put_seg(42, 1.0, &seg);
|
||||
dc.flush();
|
||||
assert_eq!(dc.get_seg(42, 1.0), Some(seg.clone()));
|
||||
assert!(dc.get_seg(42, 0.5).is_none() && dc.get_seg(43, 1.0).is_none());
|
||||
assert_eq!(dc.keys(Kind::Matte).len(), 1);
|
||||
drop(dc);
|
||||
let dc = DiskCache::open(&dir, 1 << 30).unwrap();
|
||||
assert_eq!(dc.get_seg(42, 1.0), Some(seg.clone()));
|
||||
// Damaged on disk: a miss, and the entry is forgotten.
|
||||
let path = dc.path(Kind::Matte, seg_key(42, 1.0));
|
||||
let mut d = std::fs::read(&path).unwrap();
|
||||
d[18] ^= 0x55;
|
||||
std::fs::write(&path, d).unwrap();
|
||||
assert!(dc.get_seg(42, 1.0).is_none());
|
||||
assert!(!dc.contains(Kind::Matte, seg_key(42, 1.0)));
|
||||
// The size limit evicts segmentations too.
|
||||
dc.put_seg(1, 1.0, &seg);
|
||||
dc.flush();
|
||||
dc.set_max_bytes(1);
|
||||
assert!(dc.keys(Kind::Matte).is_empty());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hit_miss_and_exact_round_trip() {
|
||||
let dir = tmpdir("hit");
|
||||
|
|
|
|||
|
|
@ -59,6 +59,9 @@ pub enum StrokeKind {
|
|||
Refine,
|
||||
/// Refine Edge tool with Alt: remove from the band.
|
||||
RefineErase,
|
||||
/// A box around the object (Object Matte): two opposite corners in `points`. Trained models
|
||||
/// get it as their box prompt; the graph cut ignores it.
|
||||
Box,
|
||||
}
|
||||
|
||||
impl StrokeKind {
|
||||
|
|
@ -830,8 +833,9 @@ fn points_along(s: &Stroke, scale: f64, max: usize) -> Vec<[f32; 2]> {
|
|||
out
|
||||
}
|
||||
|
||||
/// A model's prompt from the strokes: points along the foreground and background strokes.
|
||||
fn stroke_prompt(strokes: &[&Stroke], scale: f64) -> Prompt {
|
||||
/// A model's prompt from the strokes: points along the foreground and background strokes, and
|
||||
/// the box of a [`StrokeKind::Box`] stroke (matte pixels).
|
||||
pub fn stroke_prompt(strokes: &[&Stroke], scale: f64) -> Prompt {
|
||||
let mut points = vec![];
|
||||
for (kind, fg) in [(StrokeKind::Fg, true), (StrokeKind::Bg, false)] {
|
||||
let ss: Vec<&&Stroke> = strokes.iter().filter(|s| s.kind == kind).collect();
|
||||
|
|
@ -840,11 +844,20 @@ fn stroke_prompt(strokes: &[&Stroke], scale: f64) -> Prompt {
|
|||
points.extend(points_along(s, scale, per).into_iter().map(|p| (p, fg)));
|
||||
}
|
||||
}
|
||||
Prompt { points, ..Default::default() }
|
||||
let bbox = strokes.iter().filter(|s| s.kind == StrokeKind::Box).find_map(|s| box_of(s, scale));
|
||||
Prompt { points, bbox, ..Default::default() }
|
||||
}
|
||||
|
||||
/// Add a prior matte to a prompt: its box (a little larger), up to four points spread through
|
||||
/// its core, and the matte itself.
|
||||
/// The box `[x0, y0, x1, y1]` (matte pixels) of a box stroke: its first two points as opposite
|
||||
/// corners; `None` when they are missing, not finite or the box is empty.
|
||||
fn box_of(s: &Stroke, scale: f64) -> Option<[f32; 4]> {
|
||||
let (a, c) = (s.points.first()?, s.points.get(1)?);
|
||||
let v = [a[0].min(c[0]), a[1].min(c[1]), a[0].max(c[0]), a[1].max(c[1])].map(|v| (v * scale) as f32);
|
||||
(v.iter().all(|x| x.is_finite()) && v[2] > v[0] && v[3] > v[1]).then_some(v)
|
||||
}
|
||||
|
||||
/// Add a prior matte to a prompt: its box (a little larger; unless the prompt has one), up to
|
||||
/// four points spread through its core, and the matte itself.
|
||||
fn add_prior_prompt(prompt: &mut Prompt, soft: &[f32], w: usize, h: usize) -> bool {
|
||||
let bin: Vec<u8> = soft.iter().map(|v| (*v >= 0.5) as u8).collect();
|
||||
let (mut x0, mut y0, mut x1, mut y1) = (usize::MAX, usize::MAX, 0usize, 0usize);
|
||||
|
|
@ -858,7 +871,10 @@ fn add_prior_prompt(prompt: &mut Prompt, soft: &[f32], w: usize, h: usize) -> bo
|
|||
return false;
|
||||
}
|
||||
let (mx, my) = (((x1 - x0) as f32 * 0.05).max(4.0), ((y1 - y0) as f32 * 0.05).max(4.0));
|
||||
prompt.bbox = Some([(x0 as f32 - mx).max(0.0), (y0 as f32 - my).max(0.0), (x1 as f32 + 1.0 + mx).min(w as f32), (y1 as f32 + 1.0 + my).min(h as f32)]);
|
||||
// A box the user drew on this frame wins over the prior's.
|
||||
if prompt.bbox.is_none() {
|
||||
prompt.bbox = Some([(x0 as f32 - mx).max(0.0), (y0 as f32 - my).max(0.0), (x1 as f32 + 1.0 + mx).min(w as f32), (y1 as f32 + 1.0 + my).min(h as f32)]);
|
||||
}
|
||||
// Farthest-point picks among the core (deep inside), starting from the deepest pixel.
|
||||
let sd = matting::signed_distance(&bin, w, h);
|
||||
let deep = sd.iter().copied().fold(0.0f32, f32::max);
|
||||
|
|
|
|||
|
|
@ -428,3 +428,25 @@ fn mobilesam_segments_and_propagates_the_disk() {
|
|||
eprintln!("MobileSAM: base IoU {jb:.4}, worst propagated {worst:.4} (classic {cworst:.4}), {:.0} ms/frame", t.elapsed().as_secs_f64() * 1000.0 / 20.0);
|
||||
assert!(jb > 0.95 && worst > 0.93 && worst >= cworst - 0.01, "base {jb}, worst {worst} vs classic {cworst}");
|
||||
}
|
||||
|
||||
/// A box stroke becomes the model's box prompt (scaled, corners in any order); the graph cut
|
||||
/// ignores it; a prior's box does not replace it.
|
||||
#[test]
|
||||
fn box_stroke_is_the_model_box_prompt() {
|
||||
let fg = Stroke { kind: StrokeKind::Fg, frame: 0, radius: 2.0, points: vec![[50.0, 40.0]] };
|
||||
let bx = Stroke { kind: StrokeKind::Box, frame: 0, radius: 0.0, points: vec![[80.0, 60.0], [20.0, 10.0]] };
|
||||
let p = stroke_prompt(&[&fg, &bx], 0.5);
|
||||
assert_eq!(p.bbox, Some([10.0, 5.0, 40.0, 30.0]));
|
||||
assert_eq!(p.points, vec![([25.0, 20.0], true)]);
|
||||
// Degenerate or non-finite boxes are no prompt.
|
||||
for pts in [vec![[20.0, 10.0]], vec![[20.0, 10.0], [20.0, 50.0]], vec![[f64::NAN, 10.0], [80.0, 60.0]]] {
|
||||
let b = Stroke { points: pts, ..bx.clone() };
|
||||
assert_eq!(stroke_prompt(&[&fg, &b], 1.0).bbox, None);
|
||||
}
|
||||
assert!(!StrokeKind::Box.is_segmentation());
|
||||
assert!(rasterize(&[&bx], &[StrokeKind::Fg, StrokeKind::Bg], 1.0, 100, 80).iter().all(|v| !v));
|
||||
let mut q = stroke_prompt(&[&fg, &bx], 1.0);
|
||||
let soft: Vec<f32> = (0..100 * 80).map(|i| ((i % 100) < 10) as u8 as f32).collect();
|
||||
assert!(add_prior_prompt(&mut q, &soft, 100, 80));
|
||||
assert_eq!(q.bbox, Some([20.0, 10.0, 80.0, 60.0]));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -627,6 +627,7 @@ const JAPANESE: &[(&str, &str)] = &[
|
|||
("Project", "プロジェクト"),
|
||||
("Total Offset", "合計オフセット"),
|
||||
("Per Layer", "レイヤーごと"),
|
||||
("Track", "トラック"),
|
||||
];
|
||||
|
||||
const SIMPLIFIED_CHINESE: &[(&str, &str)] = &[
|
||||
|
|
@ -1352,6 +1353,7 @@ const SIMPLIFIED_CHINESE: &[(&str, &str)] = &[
|
|||
("Stored, but not used by EffectCraft yet", "已保存，但 EffectCraft 尚未使用"),
|
||||
("Total Offset", "总偏移"),
|
||||
("Per Layer", "每个图层"),
|
||||
("Track", "跟踪"),
|
||||
];
|
||||
|
||||
const TRADITIONAL_CHINESE: &[(&str, &str)] = &[
|
||||
|
|
@ -2072,6 +2074,7 @@ const TRADITIONAL_CHINESE: &[(&str, &str)] = &[
|
|||
("Stored, but not used by EffectCraft yet", "已儲存，但 EffectCraft 尚未使用"),
|
||||
("Total Offset", "總偏移"),
|
||||
("Per Layer", "每個圖層"),
|
||||
("Track", "追蹤"),
|
||||
];
|
||||
|
||||
/// Ukrainian (uk): original translations of the same interface rows as the Chinese catalogs.
|
||||
|
|
@ -2539,6 +2542,7 @@ const UKRAINIAN: &[(&str, &str)] = &[
|
|||
("⌘/Ctrl-drag a pin to record its motion", "Перетягніть опорну точку з ⌘/Ctrl, щоб записати її рух"),
|
||||
("▣ {}", "▣ {}"),
|
||||
("⚠ {} is used by:", "⚠ {} використовується в:"),
|
||||
("Track", "Відстежити"),
|
||||
// Strings a panel used to hand to a helper as a bare literal: the helper draws them (the
|
||||
// viewer's toolbar tooltips and popup items, the solid dialog's checkbox), so the lookup has
|
||||
// to reach the catalog from inside the helper. Kept at the end so the catalogs stay parallel.
|
||||
|
|
|
|||
|
|
@ -35,6 +35,8 @@ pub enum Icon {
|
|||
Eraser,
|
||||
RotoBrush,
|
||||
RefineEdge,
|
||||
/// Object Matte: a selection box's corners around a solid object, with a click point.
|
||||
ObjectMatte,
|
||||
/// Puppet Position Pin: a pushpin. The other pin tools add a badge at the lower right.
|
||||
Puppet,
|
||||
PuppetAdvanced,
|
||||
|
|
@ -348,6 +350,14 @@ pub fn paint(painter: &Painter, rect: Rect, icon: Icon, color: Color32) {
|
|||
pen.line(&[(8.0, 14.0), (9.5, 12.5)]);
|
||||
pen.line(&[(11.0, 14.0), (12.5, 12.5)]);
|
||||
}
|
||||
ObjectMatte => {
|
||||
// Corner brackets of a box around a filled object, and the click on it.
|
||||
for (x, y, dx, dy) in [(2.0, 2.0, 1.0, 1.0), (14.0, 2.0, -1.0, 1.0), (14.0, 14.0, -1.0, -1.0), (2.0, 14.0, 1.0, -1.0)] {
|
||||
pen.line(&[(x, y + 3.0 * dy), (x, y), (x + 3.0 * dx, y)]);
|
||||
}
|
||||
pen.dot(8.0, 8.0, 2.2);
|
||||
pen.circle(8.0, 8.0, 4.4);
|
||||
}
|
||||
Puppet => pen.pushpin(8.0),
|
||||
PuppetAdvanced => {
|
||||
// Rotation ring with its square scale handle.
|
||||
|
|
|
|||
|
|
@ -1635,6 +1635,12 @@ impl EffectcraftApp {
|
|||
self.session.poll_roto(true);
|
||||
ctx.request_repaint_after(std::time::Duration::from_millis(100));
|
||||
}
|
||||
// Roto Brush propagation and Object Matte tracking / Freeze: finish jobs (a Freeze writes
|
||||
// its mattes) and start queued ones in the background.
|
||||
if self.session.roto_job.is_some() || !self.session.roto_pending.is_empty() {
|
||||
self.session.poll_roto(true);
|
||||
ctx.request_repaint_after(std::time::Duration::from_millis(100));
|
||||
}
|
||||
// 3D Camera Tracker: likewise (tracking, then solving).
|
||||
if self.session.camera_job.is_some() || !self.session.camera_pending.is_empty() {
|
||||
self.session.poll_camera(true);
|
||||
|
|
|
|||
|
|
@ -805,6 +805,44 @@ fn roto_editor(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, l
|
|||
p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim);
|
||||
}
|
||||
|
||||
/// Object Matte: Freeze / Unfreeze and Track / Stop buttons (the background job that computes
|
||||
/// the matte over the tracked frames) and its status.
|
||||
fn object_matte_editor(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, layer: &Layer, g: &PropGroup, r: Rect, actions: &mut Actions) {
|
||||
use effectcraft_engine::effects::object_matte as om;
|
||||
let t = app.tokens;
|
||||
let params = effectcraft_engine::effects::flatten_params(g, &mut |pr| pr.value.clone());
|
||||
let frozen = om::is_frozen(¶ms);
|
||||
let mine = app.session.roto_target().is_some_and(|(_, l, u, _)| l == layer.id && u == g.uid);
|
||||
let running = mine && app.session.is_roto_running();
|
||||
let busy = app.session.is_roto_running();
|
||||
let b1 = Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 4.0), vec2(84.0, 22.0));
|
||||
let b2 = Rect::from_min_size(pos2(b1.max.x + 8.0, b1.min.y), vec2(84.0, 22.0));
|
||||
let label = if frozen { "Unfreeze" } else { "Freeze" };
|
||||
if widgets::text_button(ui, b1, crate::i18n::tr(label), frozen, &t, egui::Id::new(("om-freeze-ec", g.uid))).clicked() && !busy {
|
||||
let id = if frozen { "objectMatte.unfreeze" } else { "objectMatte.freeze" };
|
||||
actions.push((id.into(), json!({"layer": layer.id.0, "effect": g.uid})));
|
||||
}
|
||||
app.auto.add(&format!("effectControls.objectMatte.{}.freeze", g.uid), b1, label);
|
||||
let (l2, id2) = if running { ("Stop", "roto.cancel") } else { ("Track", "objectMatte.propagate") };
|
||||
if widgets::text_button(ui, b2, crate::i18n::tr(l2), false, &t, egui::Id::new(("om-track-ec", g.uid))).clicked() && !frozen && (running || !busy) {
|
||||
actions.push((id2.into(), json!({"layer": layer.id.0, "effect": g.uid})));
|
||||
}
|
||||
app.auto.add(&format!("effectControls.objectMatte.{}.track", g.uid), b2, l2);
|
||||
let base = om::reference_frame(¶ms);
|
||||
let status = match app.session.roto_progress() {
|
||||
Some(pr) if running => pr.banner(),
|
||||
_ if frozen => format!("Frozen from reference frame {base}"),
|
||||
_ if app.session.roto_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => "Waiting to track".to_string(),
|
||||
_ => format!("Reference frame {base}: Track computes the matte ahead of playback"),
|
||||
};
|
||||
p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, &status, Tokens::ui(11.5), t.text_dim);
|
||||
app.auto.add(
|
||||
&format!("effectControls.objectMatte.{}.status", g.uid),
|
||||
Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 30.0), vec2(r.width() - 24.0, 16.0)),
|
||||
&status,
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Curves / Levels editors
|
||||
|
||||
|
|
@ -815,7 +853,7 @@ fn editor_height(effect: &str, g: &PropGroup, width: f32) -> f32 {
|
|||
"ec.color.curves" => 34.0 + curves_size(width) + 26.0,
|
||||
effectcraft_engine::effects::warp_stab::ID => 50.0,
|
||||
effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::EDITOR_HEIGHT,
|
||||
effectcraft_engine::effects::roto::ID => 50.0,
|
||||
effectcraft_engine::effects::roto::ID | effectcraft_engine::effects::object_matte::ID => 50.0,
|
||||
"ec.color.levels" | "ec.color.levelsic" => 34.0 + 80.0 + 58.0,
|
||||
"ec.color.autolevels" | "ec.color.autocontrast" | "ec.color.autocolor" => 24.0 + 80.0 + 12.0,
|
||||
_ => 0.0,
|
||||
|
|
@ -1290,6 +1328,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
|
|||
effectcraft_engine::effects::warp_stab::ID => warp_editor(app, ui, &bp, &layer, g, er, &mut actions),
|
||||
effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::editor(app, ui, &bp, &layer, g, er, &mut actions),
|
||||
effectcraft_engine::effects::roto::ID => roto_editor(app, ui, &bp, &layer, g, er, &mut actions),
|
||||
effectcraft_engine::effects::object_matte::ID => object_matte_editor(app, ui, &bp, &layer, g, er, &mut actions),
|
||||
"ec.color.levels" | "ec.color.levelsic" => levels_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions),
|
||||
super::fx_editors::GLOW | super::fx_editors::RESHAPE | super::fx_editors::EXTRACTOR => {
|
||||
super::fx_editors::header_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions)
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ pub mod learn;
|
|||
pub mod markers_ui;
|
||||
pub mod media_panels;
|
||||
pub mod misc;
|
||||
pub mod object_matte_tool;
|
||||
pub mod paint_panels;
|
||||
pub mod panel_kit;
|
||||
pub mod path_vr_panels;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,128 @@
|
|||
//! The Object Matte tool in the Composition viewer (After Effects 26.2's Object Matte, from its
|
||||
//! public description): click an object to isolate it, or drag a box around it. The layer under
|
||||
//! the pointer gets an Object Matte effect (`objectMatte.apply`), or its Object Matte is
|
||||
//! re-prompted (the selected one, else its last); Shift adds another. Every prompt is one undo
|
||||
//! step, and the effect is selected so its point controls show in the viewer.
|
||||
//!
|
||||
//! While hovering, a crosshair and the outline of the layer that would get the matte preview the
|
||||
//! click: segmenting on every pointer move would be far too slow for a hover preview (a model
|
||||
//! pass or a graph cut per move), so the matte itself appears once the prompt is set.
|
||||
|
||||
use effectcraft_engine::geom::vec2 as gv2;
|
||||
use effectcraft_engine::project::Layer;
|
||||
use effectcraft_engine::render::EvalCtx;
|
||||
use egui::{Color32, Pos2, Rect, Stroke, StrokeKind};
|
||||
use serde_json::json;
|
||||
|
||||
use super::viewer::{ViewerMap, fx2c, layers_at};
|
||||
use crate::EffectcraftApp;
|
||||
|
||||
/// Drags shorter than this (screen points) are clicks: a point prompt, no box.
|
||||
pub const MIN_BOX: f32 = 4.0;
|
||||
|
||||
/// Whether Object Matte can work on `layer` (footage, solid or composition layers with pixels).
|
||||
fn eligible(layer: &Layer) -> bool {
|
||||
layer.has_video() && !layer.switches.adjustment && layer.effects().is_some()
|
||||
}
|
||||
|
||||
/// The layer a prompt at comp point `cpt` goes to: a selected one under it, else the topmost.
|
||||
pub fn target<'a>(app: &EffectcraftApp, ectx: &EvalCtx<'a>, cpt: [f64; 2]) -> Option<&'a Layer> {
|
||||
let hits: Vec<&Layer> = layers_at(ectx, cpt).filter(|l| eligible(l)).collect();
|
||||
let selected = &app.session.state.selected_layers;
|
||||
hits.iter().find(|l| selected.contains(&l.id)).or(hits.first()).copied()
|
||||
}
|
||||
|
||||
/// The layer's effect-space rectangle (its pixels) on screen.
|
||||
fn outline(ectx: &EvalCtx, map: &ViewerMap, layer: &Layer) -> [Pos2; 4] {
|
||||
let [w, h] = ectx.effect_bounds(layer).0;
|
||||
let m = fx2c(ectx, layer);
|
||||
[[0.0, 0.0], [w, 0.0], [w, h], [0.0, h]].map(|p| {
|
||||
let c = m.apply(gv2(p[0], p[1]));
|
||||
map.to_screen([c.x, c.y])
|
||||
})
|
||||
}
|
||||
|
||||
/// Hover preview at `hp` (screen): the outline of the layer that would get the matte and a
|
||||
/// crosshair where the object point would go.
|
||||
pub fn hover(app: &mut EffectcraftApp, painter: &egui::Painter, map: &ViewerMap, ectx: &EvalCtx, hp: Pos2) {
|
||||
let accent = app.tokens.accent;
|
||||
let cpt = map.to_comp(hp);
|
||||
if let Some(layer) = target(app, ectx, cpt) {
|
||||
let q = outline(ectx, map, layer);
|
||||
let mut pts = q.to_vec();
|
||||
pts.push(q[0]);
|
||||
painter.add(egui::Shape::line(pts, Stroke::new(1.5, accent)));
|
||||
let r = Rect::from_points(&q);
|
||||
app.auto.add("viewer.objectMatte.target", r, &layer.name);
|
||||
}
|
||||
let (gap, arm) = (4.0, 11.0);
|
||||
for (dx, dy) in [(1.0, 0.0), (-1.0, 0.0), (0.0, 1.0), (0.0, -1.0)] {
|
||||
let d = egui::vec2(dx, dy);
|
||||
for (w, c) in [(3.0, Color32::from_black_alpha(160)), (1.5, Color32::WHITE)] {
|
||||
painter.line_segment([hp + d * gap, hp + d * arm], Stroke::new(w, c));
|
||||
}
|
||||
}
|
||||
painter.circle_stroke(hp, 2.0, Stroke::new(1.5, accent));
|
||||
app.auto.add("viewer.objectMatte.crosshair", Rect::from_center_size(hp, egui::vec2(2.0 * arm, 2.0 * arm)), "Object Matte point");
|
||||
}
|
||||
|
||||
/// The box being dragged (screen `a`–`b`).
|
||||
pub fn draw_box(app: &mut EffectcraftApp, painter: &egui::Painter, a: Pos2, b: Pos2) {
|
||||
let r = Rect::from_two_pos(a, b);
|
||||
let accent = app.tokens.accent;
|
||||
painter.rect_stroke(r, 0.0, Stroke::new(2.5, Color32::from_black_alpha(140)), StrokeKind::Middle);
|
||||
painter.rect_stroke(r, 0.0, Stroke::new(1.5, accent), StrokeKind::Middle);
|
||||
painter.circle_filled(r.center(), 2.5, accent);
|
||||
app.auto.add("viewer.objectMatte.box", r, "Object Matte box");
|
||||
}
|
||||
|
||||
/// Set the prompt from a click at `a` (screen), or a box dragged from `a` to `b`: on the
|
||||
/// target layer's Object Matte (re-prompted), or a new one with `add` or when it has none.
|
||||
pub fn apply(app: &mut EffectcraftApp, ectx: &EvalCtx, map: &ViewerMap, a: Pos2, b: Option<Pos2>, add: bool) {
|
||||
let boxed = b.filter(|b| (b.x - a.x).abs() >= MIN_BOX && (b.y - a.y).abs() >= MIN_BOX);
|
||||
let center = boxed.map_or(a, |b| Rect::from_two_pos(a, b).center());
|
||||
let Some(layer) = target(app, ectx, map.to_comp(center)) else {
|
||||
app.ui.status = "Object Matte: click an object on a footage, solid or composition layer".into();
|
||||
return;
|
||||
};
|
||||
let Some(inv) = fx2c(ectx, layer).inverse() else { return };
|
||||
let to_layer = |p: Pos2| {
|
||||
let c = map.to_comp(p);
|
||||
let l = inv.apply(gv2(c[0], c[1]));
|
||||
[l.x, l.y]
|
||||
};
|
||||
let point = to_layer(center);
|
||||
let mut q = json!({"layer": layer.id.0, "point": point});
|
||||
if let Some(b) = boxed {
|
||||
// The box in layer pixels: the bounds of its corners (the layer may be turned).
|
||||
let r = Rect::from_two_pos(a, b);
|
||||
let c = [r.left_top(), r.right_top(), r.right_bottom(), r.left_bottom()].map(to_layer);
|
||||
let (x0, x1) = c.iter().map(|p| p[0]).fold((f64::INFINITY, f64::NEG_INFINITY), |(lo, hi), v| (lo.min(v), hi.max(v)));
|
||||
let (y0, y1) = c.iter().map(|p| p[1]).fold((f64::INFINITY, f64::NEG_INFINITY), |(lo, hi), v| (lo.min(v), hi.max(v)));
|
||||
q["box"] = json!([x0, y0, x1, y1]);
|
||||
}
|
||||
if !add && let Some(uid) = existing(app, layer) {
|
||||
q["effect"] = json!(uid);
|
||||
}
|
||||
let lid = layer.id;
|
||||
match app.session.execute("objectMatte.apply", q) {
|
||||
Ok(r) => {
|
||||
if let Some(uid) = r["effect"].as_u64() {
|
||||
let _ = app.session.execute("prop.select", json!({"layer": lid.0, "prop": uid}));
|
||||
}
|
||||
}
|
||||
Err(e) => app.ui.status = e.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The Object Matte a prompt on `layer` re-prompts: the selected one, else the layer's last.
|
||||
fn existing(app: &EffectcraftApp, layer: &Layer) -> Option<u64> {
|
||||
let fx = layer.effects()?;
|
||||
let om: Vec<u64> = fx.groups().filter(|g| is_object_matte(g)).map(|g| g.uid).collect();
|
||||
let sel = app.session.state.selected_props.iter().rev().find(|(l, u)| *l == layer.id && om.contains(u)).map(|(_, u)| *u);
|
||||
sel.or(om.last().copied())
|
||||
}
|
||||
|
||||
fn is_object_matte(g: &effectcraft_engine::project::PropGroup) -> bool {
|
||||
matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == effectcraft_engine::effects::object_matte::ID)
|
||||
}
|
||||
|
|
@ -168,6 +168,10 @@ enum Gesture {
|
|||
PinMarquee {
|
||||
start: Pos2,
|
||||
},
|
||||
/// Object Matte tool: a box around the object (screen point where the drag began).
|
||||
ObjectMatteBox {
|
||||
start: Pos2,
|
||||
},
|
||||
/// Puppet tool Record mode (⌘/Ctrl-drag a pin): the drag is sampled in real time (seconds
|
||||
/// since it began, effect space) and becomes keyframes on release (`puppet.recordPin`); the
|
||||
/// current time runs along while recording.
|
||||
|
|
@ -1313,6 +1317,14 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
|
|||
}
|
||||
t if t.is_shape() => egui::CursorIcon::Crosshair,
|
||||
_ if app.ui.viewer.roi_draw => egui::CursorIcon::Crosshair,
|
||||
Tool::ObjectMatte => {
|
||||
// The tool draws its own crosshair (and the layer it would matte) while no box
|
||||
// is being dragged.
|
||||
if ui.data(|d| d.get_temp::<Gesture>(egui::Id::new("viewer-gesture"))).is_none() {
|
||||
super::object_matte_tool::hover(app, &painter, &map, &ectx, hp);
|
||||
}
|
||||
egui::CursorIcon::None
|
||||
}
|
||||
t if t.puppet_kind().is_some() => match super::puppet_tool::hit(&pin_hits, hp) {
|
||||
Some((_, super::puppet_tool::PinPart::Ring)) => egui::CursorIcon::Alias,
|
||||
Some((_, super::puppet_tool::PinPart::Scale)) => egui::CursorIcon::ResizeNwSe,
|
||||
|
|
@ -1443,6 +1455,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
|
|||
// From where the button went down: by the time the pointer has moved far enough to
|
||||
// count as a drag it has already left the press point, and the shape is drawn there.
|
||||
t if t.is_shape() => Some(Gesture::Create { tool: t, start: map.to_comp(press) }),
|
||||
Tool::ObjectMatte => Some(Gesture::ObjectMatteBox { start: press }),
|
||||
t if t.puppet_kind().is_some() && super::puppet_tool::hit(&pin_hits, press).is_none() => {
|
||||
// Off the pins: Alt or a press outside the art drags a marquee that selects pins.
|
||||
(mods.alt || puppet_layer(app, cpt).is_none()).then_some(Gesture::PinMarquee { start: press })
|
||||
|
|
@ -1766,6 +1779,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
|
|||
q["merge"] = json!(merge);
|
||||
let _ = app.session.execute("track.setPoint", q);
|
||||
}
|
||||
Gesture::ObjectMatteBox { start } => super::object_matte_tool::draw_box(app, &painter, start, pos),
|
||||
Gesture::Marquee { start } | Gesture::PinMarquee { start } => {
|
||||
let r = Rect::from_two_pos(start, pos);
|
||||
painter.rect_filled(r, 0.0, t.accent.gamma_multiply(0.12));
|
||||
|
|
@ -1806,6 +1820,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
|
|||
}
|
||||
}
|
||||
Gesture::PuppetPin { layer, pin, .. } => super::puppet_tool::reveal_pin(app, layer, pin),
|
||||
Gesture::ObjectMatteBox { start } => super::object_matte_tool::apply(app, &ectx, &map, start, Some(pos), mods.shift),
|
||||
Gesture::Create { tool, start } => {
|
||||
let end = map.to_comp(pos);
|
||||
create_shape(app, tool, start, end, mods);
|
||||
|
|
@ -1909,6 +1924,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
|
|||
// Clicking away from the free-transform box ends it.
|
||||
ov::end_free_transform(&ctx);
|
||||
}
|
||||
Tool::ObjectMatte => super::object_matte_tool::apply(app, &ectx, &map, pos, None, mods.shift),
|
||||
t if t.puppet_kind().is_some() => {
|
||||
if let Some((h, _)) = super::puppet_tool::hit(&pin_hits, pos) {
|
||||
// Click selects the pin; Shift-click adds or removes it.
|
||||
|
|
@ -2270,7 +2286,7 @@ pub(crate) fn selectable_layers(comp: &Comp, time: Tick) -> impl Iterator<Item =
|
|||
}
|
||||
|
||||
/// Selectable layers under a comp point, front to back.
|
||||
fn layers_at<'a, 'b>(ectx: &'b EvalCtx<'a>, cpt: [f64; 2]) -> impl Iterator<Item = &'a Layer> + 'b {
|
||||
pub(crate) fn layers_at<'a, 'b>(ectx: &'b EvalCtx<'a>, cpt: [f64; 2]) -> impl Iterator<Item = &'a Layer> + 'b {
|
||||
selectable_layers(ectx.comp, ectx.time).filter(move |l| layer_quad(ectx, l).is_some_and(|(_, q, _)| point_in_quad(cpt, &q)))
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -35,6 +35,8 @@ pub enum Tool {
|
|||
Eraser,
|
||||
RotoBrush,
|
||||
RefineEdge,
|
||||
/// Object Matte: click an object (or drag a box around it) in the Composition viewer.
|
||||
ObjectMatte,
|
||||
Puppet,
|
||||
PuppetStarch,
|
||||
PuppetBend,
|
||||
|
|
@ -59,10 +61,10 @@ impl Tool {
|
|||
&[Tool::Brush],
|
||||
&[Tool::Clone],
|
||||
&[Tool::Eraser],
|
||||
&[Tool::RotoBrush, Tool::RefineEdge],
|
||||
&[Tool::RotoBrush, Tool::RefineEdge, Tool::ObjectMatte],
|
||||
&[Tool::Puppet, Tool::PuppetStarch, Tool::PuppetBend, Tool::PuppetAdvanced, Tool::PuppetOverlap],
|
||||
];
|
||||
pub const ALL: [Tool; 30] = [
|
||||
pub const ALL: [Tool; 31] = [
|
||||
Tool::Selection,
|
||||
Tool::Hand,
|
||||
Tool::Zoom,
|
||||
|
|
@ -88,6 +90,7 @@ impl Tool {
|
|||
Tool::Eraser,
|
||||
Tool::RotoBrush,
|
||||
Tool::RefineEdge,
|
||||
Tool::ObjectMatte,
|
||||
Tool::Puppet,
|
||||
Tool::PuppetStarch,
|
||||
Tool::PuppetBend,
|
||||
|
|
@ -121,6 +124,7 @@ impl Tool {
|
|||
Tool::Eraser => "Eraser Tool",
|
||||
Tool::RotoBrush => "Roto Brush Tool",
|
||||
Tool::RefineEdge => "Refine Edge Tool",
|
||||
Tool::ObjectMatte => "Object Matte Tool",
|
||||
Tool::Puppet => "Puppet Position Pin Tool",
|
||||
Tool::PuppetStarch => "Puppet Starch Pin Tool",
|
||||
Tool::PuppetBend => "Puppet Bend Pin Tool",
|
||||
|
|
@ -142,7 +146,7 @@ impl Tool {
|
|||
Tool::Pen | Tool::PenAdd | Tool::PenDelete | Tool::PenConvert | Tool::MaskFeather => Some("G"),
|
||||
Tool::Type | Tool::TypeVertical => Some("Cmd+T"),
|
||||
Tool::Brush | Tool::Clone | Tool::Eraser => Some("Cmd+B"),
|
||||
Tool::RotoBrush | Tool::RefineEdge => Some("Alt+W"),
|
||||
Tool::RotoBrush | Tool::RefineEdge | Tool::ObjectMatte => Some("Alt+W"),
|
||||
Tool::Puppet | Tool::PuppetStarch | Tool::PuppetBend | Tool::PuppetAdvanced | Tool::PuppetOverlap => Some("Cmd+P"),
|
||||
}
|
||||
}
|
||||
|
|
@ -173,6 +177,7 @@ impl Tool {
|
|||
Tool::Eraser => Icon::Eraser,
|
||||
Tool::RotoBrush => Icon::RotoBrush,
|
||||
Tool::RefineEdge => Icon::RefineEdge,
|
||||
Tool::ObjectMatte => Icon::ObjectMatte,
|
||||
Tool::Puppet => Icon::Puppet,
|
||||
Tool::PuppetStarch => Icon::PuppetStarch,
|
||||
Tool::PuppetBend => Icon::PuppetBend,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,185 @@
|
|||
//! Headless checks for the Object Matte tool in the Composition viewer: the hover preview, a
|
||||
//! click (point prompt), a drag (box prompt) re-prompting the same effect, Shift adding another,
|
||||
//! one undo step each (egui_kittest, UI logic only).
|
||||
|
||||
use effectcraft_engine::Session;
|
||||
use effectcraft_engine::effects::flatten_params;
|
||||
use effectcraft_engine::effects::object_matte as om;
|
||||
use effectcraft_engine::keyframe::Value;
|
||||
use effectcraft_engine::project::{GroupKind, LayerId, PropGroup};
|
||||
use effectcraft_ui_egui::EffectcraftApp;
|
||||
use effectcraft_ui_egui::dock::PanelKind;
|
||||
use effectcraft_ui_egui::state::Tool;
|
||||
use egui::{Event, Modifiers, Pos2, Rect, pos2, vec2};
|
||||
use egui_kittest::Harness;
|
||||
use serde_json::json;
|
||||
|
||||
fn harness() -> Harness<'static, EffectcraftApp> {
|
||||
let mut s = Session::default();
|
||||
s.execute("comp.new", json!({"name": "Matte", "width": 640, "height": 360, "duration": 4})).unwrap();
|
||||
s.execute("layer.newSolid", json!({"name": "Plate", "color": "#406080", "width": 640, "height": 360})).unwrap();
|
||||
s.execute("layer.newSolid", json!({"name": "Box", "color": "#e04020", "width": 80, "height": 80})).unwrap();
|
||||
s.execute("edit.deselectAll", json!({})).unwrap();
|
||||
let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).build_eframe(move |_| EffectcraftApp::new(s));
|
||||
h.run_steps(3);
|
||||
h
|
||||
}
|
||||
|
||||
fn rect(h: &Harness<'_, EffectcraftApp>, id: &str) -> Rect {
|
||||
let e = h.state().auto.find(id).unwrap_or_else(|| panic!("no {id}"));
|
||||
Rect::from_min_size(pos2(e.rect[0], e.rect[1]), vec2(e.rect[2], e.rect[3]))
|
||||
}
|
||||
|
||||
/// Comp pixel → screen point.
|
||||
fn screen(h: &Harness<'_, EffectcraftApp>, p: [f32; 2]) -> Pos2 {
|
||||
let c = rect(h, "viewer.comp");
|
||||
let z = c.width() / 640.0;
|
||||
c.min + vec2(p[0] * z, p[1] * z)
|
||||
}
|
||||
|
||||
fn press(h: &mut Harness<'_, EffectcraftApp>, p: Pos2, pressed: bool, modifiers: Modifiers) {
|
||||
h.input_mut().events.push(Event::ModifiersChanged(modifiers));
|
||||
h.input_mut().events.push(Event::PointerButton { pos: p, button: egui::PointerButton::Primary, pressed, modifiers });
|
||||
h.step();
|
||||
}
|
||||
|
||||
fn click(h: &mut Harness<'_, EffectcraftApp>, p: Pos2, modifiers: Modifiers) {
|
||||
h.input_mut().events.push(Event::PointerMoved(p));
|
||||
h.step();
|
||||
press(h, p, true, modifiers);
|
||||
press(h, p, false, modifiers);
|
||||
h.input_mut().events.push(Event::ModifiersChanged(Modifiers::NONE));
|
||||
h.run_steps(2);
|
||||
}
|
||||
|
||||
fn drag(h: &mut Harness<'_, EffectcraftApp>, from: Pos2, to: Pos2) -> Option<Rect> {
|
||||
h.input_mut().events.push(Event::PointerMoved(from));
|
||||
press(h, from, true, Modifiers::NONE);
|
||||
let mut drawn = None;
|
||||
for i in 1..=8 {
|
||||
h.input_mut().events.push(Event::PointerMoved(from + (to - from) * (i as f32 / 8.0)));
|
||||
h.step();
|
||||
drawn = h.state().auto.find("viewer.objectMatte.box").map(|e| Rect::from_min_size(pos2(e.rect[0], e.rect[1]), vec2(e.rect[2], e.rect[3])));
|
||||
}
|
||||
press(h, to, false, Modifiers::NONE);
|
||||
h.run_steps(2);
|
||||
drawn
|
||||
}
|
||||
|
||||
fn box_layer(h: &Harness<'_, EffectcraftApp>) -> LayerId {
|
||||
h.state().session.active_comp().unwrap().layers.iter().find(|l| l.name == "Box").unwrap().id
|
||||
}
|
||||
|
||||
/// The Box layer's Object Matte effects.
|
||||
fn mattes(h: &Harness<'_, EffectcraftApp>) -> Vec<PropGroup> {
|
||||
let comp = h.state().session.active_comp().unwrap();
|
||||
let l = comp.layer(box_layer(h)).unwrap();
|
||||
l.effects().unwrap().groups().filter(|g| matches!(&g.kind, GroupKind::Effect { effect } if effect == om::ID)).cloned().collect()
|
||||
}
|
||||
|
||||
fn v2(g: &PropGroup, id: &str) -> [f64; 2] {
|
||||
flatten_params(g, &mut |p| p.value.clone()).get(id).map(Value::as_vec2).unwrap()
|
||||
}
|
||||
|
||||
fn near(a: [f64; 2], b: [f64; 2]) -> bool {
|
||||
(a[0] - b[0]).abs() < 1.5 && (a[1] - b[1]).abs() < 1.5
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_matte_tool_prompts_with_a_click_and_a_box() {
|
||||
let mut h = harness();
|
||||
assert_eq!(Tool::from_name("objectMatte"), Some(Tool::ObjectMatte));
|
||||
assert!(Tool::SLOTS.iter().any(|s| s.contains(&Tool::RotoBrush) && s.contains(&Tool::ObjectMatte)));
|
||||
h.state_mut().ui.tool = Tool::ObjectMatte;
|
||||
h.run_steps(2);
|
||||
// Hover: a crosshair at the pointer and the outline of the layer that would get the matte.
|
||||
let center = screen(&h, [320.0, 180.0]);
|
||||
h.input_mut().events.push(Event::PointerMoved(center));
|
||||
h.run_steps(2);
|
||||
assert!(rect(&h, "viewer.objectMatte.crosshair").contains(center));
|
||||
let target = h.state().auto.find("viewer.objectMatte.target").expect("no target outline");
|
||||
assert_eq!(target.label, "Box");
|
||||
assert!(mattes(&h).is_empty(), "hovering must not apply anything");
|
||||
let undo0 = h.state().session.history.undo.len();
|
||||
|
||||
// Click: a point prompt in layer pixels on the layer under the pointer, selected.
|
||||
click(&mut h, center, Modifiers::NONE);
|
||||
let m = mattes(&h);
|
||||
assert_eq!(m.len(), 1);
|
||||
assert!(near(v2(&m[0], "objectPoint"), [40.0, 40.0]), "{:?}", v2(&m[0], "objectPoint"));
|
||||
let uid = m[0].uid;
|
||||
let st = &h.state().session.state;
|
||||
assert!(st.selected_props.contains(&(box_layer(&h), uid)), "the new effect is selected");
|
||||
assert_eq!(h.state().session.history.undo.len(), undo0 + 1);
|
||||
|
||||
// Drag: a box (drawn while dragging) re-prompts the same effect; its centre is the point.
|
||||
// (It starts clear of the selected effect's point controls, which take drags of their own.)
|
||||
let (a, b) = (screen(&h, [286.0, 150.0]), screen(&h, [356.0, 214.0]));
|
||||
let drawn = drag(&mut h, a, b).expect("no box drawn while dragging");
|
||||
assert!(drawn.width() > 20.0);
|
||||
let m = mattes(&h);
|
||||
assert_eq!(m.len(), 1);
|
||||
assert_eq!(m[0].uid, uid);
|
||||
let ps = flatten_params(&m[0], &mut |p| p.value.clone());
|
||||
assert_eq!(ps.get("useBox"), Some(&Value::Bool(true)));
|
||||
assert!(near(v2(&m[0], "boxTopLeft"), [6.0, 10.0]), "{:?}", v2(&m[0], "boxTopLeft"));
|
||||
assert!(near(v2(&m[0], "boxBottomRight"), [76.0, 74.0]), "{:?}", v2(&m[0], "boxBottomRight"));
|
||||
assert!(near(v2(&m[0], "objectPoint"), [41.0, 42.0]), "{:?}", v2(&m[0], "objectPoint"));
|
||||
assert_eq!(h.state().session.history.undo.len(), undo0 + 2);
|
||||
|
||||
// Shift-click adds a second Object Matte.
|
||||
let p = screen(&h, [310.0, 170.0]);
|
||||
click(&mut h, p, Modifiers::SHIFT);
|
||||
let m = mattes(&h);
|
||||
assert_eq!(m.len(), 2);
|
||||
assert_eq!(m[1].name, "Object Matte 2");
|
||||
assert!(near(v2(&m[1], "objectPoint"), [30.0, 30.0]));
|
||||
|
||||
// Undo takes the prompts back one at a time.
|
||||
h.state_mut().session.execute("edit.undo", json!({})).unwrap();
|
||||
assert_eq!(mattes(&h).len(), 1);
|
||||
h.state_mut().session.execute("edit.undo", json!({})).unwrap();
|
||||
assert_eq!(flatten_params(&mattes(&h)[0], &mut |p| p.value.clone()).get("useBox"), Some(&Value::Bool(false)));
|
||||
}
|
||||
|
||||
/// Effect Controls: the Object Matte's Track and Freeze buttons run the background job and its
|
||||
/// status line shows where it is; the UI keeps drawing frames while it runs and finishes the
|
||||
/// job itself (a Freeze writes its mattes without any other job running).
|
||||
#[test]
|
||||
fn object_matte_effect_controls_track_and_freeze() {
|
||||
let mut h = harness();
|
||||
h.state_mut().show_panel(PanelKind::EffectControls);
|
||||
h.state_mut().ui.tool = Tool::ObjectMatte;
|
||||
h.run_steps(2);
|
||||
let center = screen(&h, [320.0, 180.0]);
|
||||
click(&mut h, center, Modifiers::NONE);
|
||||
let uid = mattes(&h)[0].uid;
|
||||
h.run_steps(2);
|
||||
let status = |h: &Harness<'_, EffectcraftApp>| h.state().auto.find(&format!("effectControls.objectMatte.{uid}.status")).map(|e| e.label.clone());
|
||||
assert!(status(&h).is_some(), "no Object Matte status line");
|
||||
// Freeze: a background job (the frames keep coming), then the effect is frozen.
|
||||
let b = rect(&h, &format!("effectControls.objectMatte.{uid}.freeze"));
|
||||
// Let any automatic tracking finish first (one job at a time).
|
||||
let frozen = |h: &Harness<'_, EffectcraftApp>| om::is_frozen(&flatten_params(&mattes(h)[0], &mut |p| p.value.clone()));
|
||||
let t = std::time::Instant::now();
|
||||
while h.state().session.is_roto_running() {
|
||||
assert!(t.elapsed().as_secs() < 120);
|
||||
h.step();
|
||||
}
|
||||
click(&mut h, b.center(), Modifiers::NONE);
|
||||
let t = std::time::Instant::now();
|
||||
while !frozen(&h) {
|
||||
assert!(
|
||||
t.elapsed().as_secs() < 20,
|
||||
"never frozen: {:?} / {} / running {} / {:?}",
|
||||
status(&h),
|
||||
h.state().ui.status,
|
||||
h.state().session.is_roto_running(),
|
||||
h.state().session.roto_progress()
|
||||
);
|
||||
h.step();
|
||||
}
|
||||
h.run_steps(2);
|
||||
assert!(status(&h).is_some_and(|s| s.starts_with("Frozen")), "{:?}", status(&h));
|
||||
assert_eq!(h.state().auto.find(&format!("effectControls.objectMatte.{uid}.freeze")).map(|e| e.label.as_str()), Some("Unfreeze"));
|
||||
}
|
||||
|
|
@ -483,6 +483,26 @@ effectcraft-cli run clip.ecproj roto.stroke '{"layer":"#1","points":[[300,200],[
|
|||
roto.propagate '{"layer":"#1","wait":true}' roto.freeze '{"layer":"#1","wait":true}' --save
|
||||
```
|
||||
|
||||
### Object Matte
|
||||
|
||||
The Object Matte tool (third in the Alt+W slot) works in the Composition viewer: click an object,
|
||||
or drag a box around it; Shift adds another Object Matte instead of re-prompting the layer's.
|
||||
Agents use `objectMatte.apply {layer, point: [x, y], box?: [x0, y0, x1, y1], frame?, track?,
|
||||
before?, after?, effect?}` (layer pixels, layer frames; `effect` re-prompts that instance), one
|
||||
undo step each. With a trained model installed (Settings ▸ Roto Brush) the box is the model's box
|
||||
prompt. `objectMatte.propagate {layer, effect?, direction?, wait?}` tracks the matte over the
|
||||
frames around the reference frame in the background (`roto.cancel` stops it; edits to the prompt
|
||||
or the layer's frames queue it again), `objectMatte.freeze {wait?}` / `objectMatte.unfreeze`
|
||||
store or drop the final mattes, and `objectMatte.status {layer, effect?}` reports the job's
|
||||
progress, the frozen state, the tracked frames and which are computed. With the disk cache on (Settings ▸
|
||||
Media & Disk Cache), computed Object Matte and Roto Brush segmentations are also kept there, keyed
|
||||
by what they are made of, so reopening a project reuses them.
|
||||
|
||||
```sh
|
||||
effectcraft-cli run clip.ecproj objectMatte.apply '{"layer":"#1","point":[320,200],"box":[250,120,400,300]}' \
|
||||
objectMatte.propagate '{"layer":"#1","wait":true}' objectMatte.freeze '{"layer":"#1","wait":true}' --save
|
||||
```
|
||||
|
||||
### Text: styles and editing
|
||||
|
||||
Source Text holds character style runs and per-paragraph settings. CR (`\r`), LF (`\n`) and
|
||||
|
|
|
|||
|
|
@ -3,11 +3,11 @@
|
|||
<!-- Generated from the effect registry (crates/effects/src/catalog.rs); do not edit by hand.
|
||||
Regenerate: UPDATE_DOCS=1 cargo test -p effectcraft-effects --lib effects_doc_is_current -->
|
||||
|
||||
EffectCraft ships 306 effects with 3017 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over.
|
||||
EffectCraft ships 307 effects with 3032 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over.
|
||||
|
||||
- **GPU**: 280 effects also run on the GPU compositor with identical results.
|
||||
- **32**: 306 effects process 32-bit float (HDR, overbright) pixels without clamping.
|
||||
- **Status**: 306 implemented in full, 0 partial (what is missing is listed).
|
||||
- **32**: 307 effects process 32-bit float (HDR, overbright) pixels without clamping.
|
||||
- **Status**: 307 implemented in full, 0 partial (what is missing is listed).
|
||||
|
||||
Effects are addressed by id (`ec.<category>.<name>`) or display name in commands, scripts and the control channel, and every parameter by its id (`effects/#1/blurriness`).
|
||||
|
||||
|
|
@ -255,6 +255,7 @@ Effects are addressed by id (`ec.<category>.<name>`) or display name in commands
|
|||
|---|---|---:|:-:|:-:|---|
|
||||
| Matte Choker | `ec.matte.mattechoker` | 7 | GPU | 32 | Implemented |
|
||||
| Mocha shape | `ec.obsolete.mochashape` | 6 | | 32 | Implemented |
|
||||
| Object Matte | `ec.matte.objectmatte` | 13 | | 32 | Implemented |
|
||||
| Refine Hard Matte | `ec.matte.refinehard` | 16 | GPU | 32 | Implemented |
|
||||
| Refine Soft Matte | `ec.matte.refinesoft` | 18 | GPU | 32 | Implemented |
|
||||
| Roto Brush & Refine Edge | `ec.matte.rotobrush` | 24 | | 32 | Implemented |
|
||||
|
|
|
|||
|
|
@ -3,6 +3,9 @@
|
|||
The [AE 2026 replacement acceptance gates](ae-2026-acceptance.md) define the broader
|
||||
target across workflows, rendered output, work-project compatibility, performance and platforms.
|
||||
|
||||
[Object Matte](object-matte.md) has a comparison plan (clips, prompt, per-frame IoU and the other
|
||||
metrics) but no measurement yet.
|
||||
|
||||
The first G1 comparison measures eight original scalar Rotation cases against a live After Effects
|
||||
2026 session through AEsync 2.0.4: Easy Ease, overlapping influences, asymmetric ease, overshoot,
|
||||
descending values, equal endpoints, 100%/100% influence and 0.1%/100% influence. Each case has
|
||||
|
|
|
|||
|
|
@ -0,0 +1,73 @@
|
|||
# Object Matte: comparison plan (not measured yet)
|
||||
|
||||
**Status: untested.** There is no After Effects 26.2+ comparison of Object Matte yet. Nothing on
|
||||
this page is a measurement. It lists what to compare once an After Effects reference session
|
||||
is available, so the first run can be repeated and scored without guesswork.
|
||||
|
||||
EffectCraft side, as of #783: the Object Matte effect (`ec.matte.objectmatte`), the Object Matte
|
||||
tool in the Composition viewer, `objectMatte.apply` / `.propagate` / `.freeze` /
|
||||
`.status`, and segmentations kept in the disk cache. The point (and an optional box) is
|
||||
segmented by Roto Brush's engine: MobileSAM when it is installed (Settings ▸ Roto Brush, with
|
||||
the box as the model's box prompt), otherwise the classical graph cut. The result is tracked
|
||||
frame by frame with optical flow, and the model re-prompts each frame from the carried matte.
|
||||
Record which engine was active, because the two are separate cases.
|
||||
|
||||
## Test clips
|
||||
|
||||
All original, generated or openly licensed; none of them goes into the repository.
|
||||
|
||||
1. **Synthetic disk**: the moving textured disk over a textured background from
|
||||
`crates/engine/src/tests_roto.rs` (`draw(center(f))`, 320×240, 25 fps, 25 frames, radius
|
||||
40 px, about 4 px a frame to the right). Export it as a PNG sequence. Its ground truth is
|
||||
analytic (`gt(center(f))`).
|
||||
2. **Synthetic disk with a jump**: the same disk with `jumping(f)` (it jumps at frame 8). This
|
||||
checks how each tool handles losing the object.
|
||||
3. **Two objects**: the disk plus a second still, differently coloured square that touches it
|
||||
on some frames, prompted once with a point only and once with a box around the disk. This
|
||||
checks the box prompt.
|
||||
4. **Real footage**: one openly licensed clip (CC0 or CC BY) with a person or object moving
|
||||
against clutter, at least 1080p and 5 s long. Record its URL, licence and SHA-256 next to the
|
||||
results, and keep the clip itself out of the repository.
|
||||
|
||||
## Prompt and settings
|
||||
|
||||
- Reference frame 0. The point at the object's centre, in layer pixels (clip 1: `[90, 110]`).
|
||||
The box, where used, is `[centre − 55, centre + 55]` on each axis.
|
||||
- Track forward to the last frame (Frames After = clip length − 1, Frames Before = 0).
|
||||
- Matte refinement at its defaults on both sides: Feather 0, Contrast 0, Shift Edge 0, no
|
||||
Invert. Output the matte itself (View ▸ Matte in EffectCraft, the alpha channel in After
|
||||
Effects) as a full-resolution PNG sequence.
|
||||
- Record the After Effects build, the OS, the GPU, and the exact click position in After
|
||||
Effects. It is a viewer click, so convert it to layer pixels with the comp's zoom.
|
||||
|
||||
Keep the After Effects mattes in gitignored `plan/aftereffects/ref/object-matte/`. Only an
|
||||
aggregate results file (like `keyframe-ease.json`) and the comparison script may be committed.
|
||||
|
||||
## Metrics (per frame, then mean / worst)
|
||||
|
||||
- **IoU against After Effects**: binarize both mattes at alpha 0.5, then intersection over
|
||||
union. This is the headline number. `effectcraft_track::roto::iou` computes it, and
|
||||
`roto.status` already reports it for Roto Brush through `compareTo`; Object Matte needs the
|
||||
same for exported mattes.
|
||||
- **IoU against ground truth** for the synthetic clips, for both applications. It tells
|
||||
"differs from After Effects" apart from "worse than After Effects".
|
||||
- **Boundary error**: the mean and 95th-percentile distance (px) between the two mattes' 0.5
|
||||
contours, which shows edge disagreement that IoU dilutes on large objects.
|
||||
- **Soft-edge error**: the mean absolute alpha difference inside a 3 px band around the After
|
||||
Effects contour.
|
||||
- **Temporal stability**: the frame-to-frame IoU change of each matte on clip 1, where the
|
||||
object moves rigidly, so jitter shows.
|
||||
- **Drift**: the frame where IoU against ground truth first drops below 0.9 (clips 1 and 2).
|
||||
- **Time**: seconds per frame to track the clip, warm and cold, on the same machine, with the
|
||||
disk cache on and off.
|
||||
|
||||
Proposed pass bar (a maintainer decision, not yet agreed): mean IoU against After Effects
|
||||
≥ 0.95 and worst ≥ 0.90 on clips 1 and 3, with MobileSAM. With the classical engine the numbers
|
||||
are reported but not gated.
|
||||
|
||||
## Reproduce (to write)
|
||||
|
||||
The script does not exist yet. It should: generate the synthetic clips; run EffectCraft
|
||||
headless (`effectcraft-cli run … objectMatte.apply … objectMatte.propagate {"wait":true}`, then
|
||||
render the matte); drive After Effects through ExtendScript or AEsync to apply Object Matte with
|
||||
the same prompt and render its alpha; then compute the metrics above into a JSON results file.
|
||||
|
|
@ -70,7 +70,7 @@ are grouped by area; the issue tracker has the rest.
|
|||
| 11 | **Text rendering and layout** | Open: #453 CJK text in comps, #316 paragraph alignment inverted (maintainers' tests show Left/Right correct; asked for a project), #417 soft at Fit (mostly fixed by #444), #550 toolbar font options. Closed: #483, #583 | Text is in almost every motion graphic | 10–20 h | G2 · [ui-parity.md](ui-parity.md) |
|
||||
| 12 | **3D interaction and depth** | No 3D transform gizmos (#392, #347); camera aperture (#260); reflections (#261); extruded surface materials beyond Front / Bevel / Side / Back Color (fb5a5e9a; #315 closed); #307 reopened (the viewer's renderer button now shows and switches the renderer, #645); Z position (#489) closed | Advanced 3D is After Effects' growth area since 2023 | 20–35 h | G2 · [ui-parity.md](ui-parity.md) |
|
||||
| 13 | **3D model formats and materials** | After Effects 26 imports FBX, USD/USDZ, STL besides glTF and OBJ (`usd_plugins`), and Substance 3D materials with mesh displacement (26.0–26.2); we read glTF, OBJ, STL (binary and ASCII, with facet colours) and FBX 7.x (binary and ASCII: meshes, hierarchy, Lambert/Phong materials and textures; no FBX animation or skinning yet) | Product shots and 3D scenes come as FBX/USDZ | 25–45 h | — · [file-format-parity.md](file-format-parity.md) |
|
||||
| 14 | **Object Matte** (26.2, disk cache in 26.5) | Not present; MobileSAM is already in `effectcraft-segment` and could back it | The new one-click roto; Roto Brush covers part of it | 15–30 h | G7 · [target-app-parity.md](target-app-parity.md) |
|
||||
| 14 | **Object Matte** (26.2, disk cache in 26.5) | First slice (#783): the Object Matte effect (Effect ▸ Matte) and `objectMatte.apply` turn a point (and optional box) on a reference frame into a matte and follow the object up to 600 frames each way, or segment every frame on its own; Roto Brush's engine, cache and model (MobileSAM from Settings ▸ Roto Brush, classical graph cut without it). Since then: the Object Matte tool in the viewer (click / drag a box, hover crosshair), the box as the model's box prompt, background tracking with progress and cancel, Freeze, and segmentations kept in the disk cache (26.5). Missing: a comparison with After Effects ([plan](fidelity/object-matte.md)) | The new one-click roto | 10–20 h | G7 · [target-app-parity.md](target-app-parity.md) |
|
||||
| 15 | **Hardware video decode and encode** | No VideoToolbox, NVENC, Quick Sync or VA-API; #498 asks for it (answered: CUDA / NVENC need Nvidia's proprietary SDK through FFI, which the no-`unsafe` rule forbids); open PR #819 adds hardware decoding via `filmcraft-platform`; FilmCraft has VideoToolbox / NVENC / VA-API paths to borrow | 4K playback and export speed | 25–45 h | G5/G6 · [hardware-parity.md](hardware-parity.md) |
|
||||
| 16 | **Localization depth** | Menus in 10 languages (ja, zh-hans, zh-hant, uk, es, pt-br, ru, fr, de, cs: 624 rows each); panels in 6 (ja, zh-hans, zh-hant, uk, pt-br, cs); effect names and ~1,650 effect parameter names untranslated; Korean and Italian (After Effects ships them) absent; Hangul probe added (#599 closed) | After Effects is used heavily in Japan, China, Korea, Brazil, Europe | 60–110 h + native review | G9 · [localization-parity.md](localization-parity.md) |
|
||||
| 17 | **Media Browser lists files import refuses** | `media_browser.rs` lists AVI; `media::is_importable` has none of them (Targa and Radiance HDR import since #745; DPX and Cineon since #756, log shown as stored: no Cineon Settings conversion yet; MXF OP1a / OP-Atom since #767; FilmCraft has no AVI demuxer); #431 and #324 closed | Confusing failures on common VFX formats | 15–30 h | G6 · [file-format-parity.md](file-format-parity.md) |
|
||||
|
|
|
|||
|
|
@ -122,7 +122,7 @@ compositing and effects dominate; audio is light). Hours are to full parity for
|
|||
| Text and typography | 7% | 94% | 50% | 20–35 | CJK text in comps renders wrongly (#453); paragraph alignment inverted (#316); soft text at Fit (#417, mostly fixed by #444); toolbar font options (#550); #483 and #583 are closed; variable-font filter landed (#741) |
|
||||
| Effects and animation presets | 14% | 85% | 40% | 70–130 | All 306 effects exist, but rendered output is unmeasured against After Effects; 54 built-in presets (text, Transitions, Backgrounds) against After Effects' 621 ([effects-parity.md](effects-parity.md)); open effect bugs (#580 crash, #463, #369, #473, #474; #514 and #526 are closed); effect colour labels (26.5); see [effects-parity.md](effects-parity.md) |
|
||||
| 3D (Classic and Advanced) | 6% | 82% | 35% | 35–65 | No FBX/USD/STL models, no Substance materials, mesh displacement unverified; no 3D transform gizmos (#392, #347); reflections (#261); camera aperture (#260); #489 and #315 are closed |
|
||||
| Tracking and AI-assisted tools | 5% | 80% | 35% | 35–70 | No Object Matte (26.2); Roto Brush (MobileSAM) and face tracking (MediaPipe) unmeasured against After Effects; camera-tracker view (#505) |
|
||||
| Tracking and AI-assisted tools | 5% | 80% | 35% | 35–70 | Object Matte (26.2) has the effect, the viewer tool (click / box), background tracking, Freeze and the 26.5 disk cache, not yet compared with After Effects; Roto Brush (MobileSAM) and face tracking (MediaPipe) unmeasured against After Effects; camera-tracker view (#505) |
|
||||
| Preview and cache | 7% | 92% | 40% | 25–50 | Playback stutters on cached frames (#595, #304); late audio (#504, #335); #329 and #554 are closed; no benchmark against After Effects |
|
||||
| Render queue and export | 7% | 85% | 55% | 25–45 | No hardware encoding; ProRes 4444 colour tag (#340); rendering over an existing file (#325); #446 and #372 (8-bit uncompressed QuickTime, #778) are closed; HEVC/AV1 without B-frames |
|
||||
| Audio | 2% | 85% | 45% | 6–12 | Audio sync and crackle (#335); audio to keyframes depth |
|
||||
|
|
@ -144,7 +144,7 @@ Feature depth (≈ 50%) feeds the overall *ready* number with the other dimensio
|
|||
| Stability | 45% | 40–80 | Open crash reports #580, #501, #426, #341, #234, #879 (#591, #306 and #511 are closed); no CI runs the tests on pull requests | [gaps.md](gaps.md#g3-stability-and-a-green-main) |
|
||||
| Platforms | 65% | 30–60 | Releases for macOS, Windows x64/x86/ARM64, Linux, FreeBSD and the web (more than After Effects), but Linux and Windows users hit basic bugs | [gaps.md](gaps.md#g2-real-user-reliability-on-every-platform) |
|
||||
| Ecosystem and plug-ins | 20% | 40–80 | After Effects scripts and expressions mostly carry over; no After Effects SDK plug-ins, `.ffx` presets or `.mogrt`; our WebAssembly plug-in API has no third-party plug-ins yet | [plugins.md](plugins.md) |
|
||||
| AI features | 35% | 40–80 (overlaps Tracking) | Roto Brush (MobileSAM) and face tracking (MediaPipe) as optional downloads; no Object Matte; the After Effects bundle ships Whisper, BERT and Marian tokenizers and a shot-cut model we have no equivalent of | [gaps.md](gaps.md) |
|
||||
| AI features | 35% | 40–80 (overlaps Tracking) | Roto Brush (MobileSAM) and face tracking (MediaPipe) as optional downloads; Object Matte (point / box prompt, same model; viewer tool, background tracking, Freeze, disk cache) unmeasured against After Effects; the After Effects bundle ships Whisper, BERT and Marian tokenizers and a shot-cut model we have no equivalent of | [gaps.md](gaps.md) |
|
||||
| Agent control (beyond After Effects) | ahead | — | Every command is drivable over MCP, the control channel and the CLI | [agents.md](agents.md) |
|
||||
|
||||
**Totals.** Summing the rows double-counts: AI overlaps Tracking, performance overlaps Preview,
|
||||
|
|
|
|||
Loading…
Reference in New Issue