diff --git a/crates/effects/src/object_matte.rs b/crates/effects/src/object_matte.rs index a7db562a..32a196af 100644 --- a/crates/effects/src/object_matte.rs +++ b/crates/effects/src/object_matte.rs @@ -16,8 +16,12 @@ //! ([`crate::roto::cached`]): the chain seed holds the engine-maintained Input Key (what the //! frames are made of), the layer size, the frame rate, the mode and the model, and every //! frame's key the prompt, so edits upstream or to the prompt recompute it and undo never sees a -//! stale matte. **Feather**, **Contrast** and **Shift Edge** shape the matte, **Invert** keeps -//! everything but the object and **View ▸ Matte** shows the matte in grey. +//! stale matte. The engine fills that cache in a background job (`objectMatte.propagate`, +//! outward from the reference frame over [`job_frames`]) and **Freeze** (`objectMatte.freeze`) +//! stores the final mattes in the hidden **Frozen** parameter ([`FrozenCache`], as Roto Brush +//! does): they are used, without segmenting, while [`frozen_key`] still matches. **Feather**, +//! **Contrast** and **Shift Edge** shape the matte, **Invert** keeps everything but the object +//! and **View ▸ Matte** shows the matte in grey. use std::sync::Arc; @@ -26,10 +30,10 @@ use effectcraft_project::ParamUi; use effectcraft_raster::Image; use effectcraft_segment::MaskModel; use effectcraft_track::roto::refine::MatteParams; -use effectcraft_track::roto::{RotoData, SegOpts, Stroke, StrokeKind}; +use effectcraft_track::roto::{FrameSeg, RotoData, SegOpts, Stroke, StrokeKind, rle}; use rayon::prelude::*; -use crate::roto::{self, Chain, INPUT_KEY}; +use crate::roto::{self, Chain, FROZEN, FrozenCache, INPUT_KEY}; use crate::warp_stab::param; use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; @@ -71,6 +75,7 @@ pub fn specs() -> Vec { p("invert", "Invert", Value::Bool(false), ParamUi::Checkbox), p("view", "View", Value::Enum(0), popup(&VIEWS)), p(INPUT_KEY, "Input Key", Value::Str(String::new()), ParamUi::Hidden), + p(FROZEN, "Frozen", Value::Str(String::new()), ParamUi::Hidden), ], render, gpu: false, @@ -197,6 +202,127 @@ pub fn seed(params: &Params, layer_size: [f64; 2], fps: f64, model: Option<&dyn h } +/// The layer frames a background job computes, outward from the reference frame: the tracked +/// span (Frames Before / After around it), clipped to `limits` (the layer's frames). Without +/// Track Object, the same frames, each segmented on its own. +pub fn job_frames(params: &Params, limits: [i64; 2]) -> Vec { + let base = reference_frame(params); + let [before, after] = reach(params); + let (lo, hi) = (base.saturating_sub(before).max(limits[0]), base.saturating_add(after).min(limits[1])); + if base < limits[0] || base > limits[1] { + return vec![]; + } + let mut v = vec![base]; + v.extend(base.saturating_add(1)..=hi); + v.extend((lo..base).rev()); + v +} + +/// The segmentation chain the matte of layer frame `f` comes from (`None`: no usable prompt). +pub fn chain(params: &Params, layer_size: [f64; 2], fps: f64, f: i64, scale: f64, model: Option>) -> Option { + let data = roto_data(params, layer_size, f, model.is_some())?; + let keys = data.chain_keys(seed(params, layer_size, fps, model.as_deref())); + Some(Chain { data: Arc::new(data), keys, opts: seg_opts(scale), refine_radius: 0.0, model }) +} + +/// Frame `f`'s segmentation at full resolution, computing what is missing (the background +/// job's step). `None` outside the tracked span, without a prompt, or when cancelled. +pub fn ensure( + params: &Params, + layer_size: [f64; 2], + fps: f64, + f: i64, + model: Option>, + frame: &mut dyn FnMut(i64) -> Option>, + cancel: &dyn Fn() -> bool, +) -> Option> { + let ch = chain(params, layer_size, fps, f, 1.0, model)?; + if !ch.data.in_span(f) { + return None; + } + ch.ensure(f, frame, cancel) +} + +/// The chain key of each of `frames` (what a computed segmentation is cached under). +pub fn frame_keys(params: &Params, layer_size: [f64; 2], fps: f64, frames: &[i64], model: Option>) -> Vec<(i64, u64)> { + if tracks(params) { + let Some(ch) = chain(params, layer_size, fps, 0, 1.0, model) else { return vec![] }; + return frames.iter().filter_map(|f| Some((*f, *ch.keys.get(f)?))).collect(); + } + frames.iter().filter_map(|f| Some((*f, *chain(params, layer_size, fps, *f, 1.0, model.clone())?.keys.get(f)?))).collect() +} + +/// What frozen mattes depend on: what the frames are (the seed: Input Key, size, rate, mode, +/// model), the prompt, the tracked span and the matte settings. +pub fn frozen_key(params: &Params, layer_size: [f64; 2], fps: f64, model: Option<&dyn MaskModel>) -> String { + let mut h = seed(params, layer_size, fps, model); + let base = reference_frame(params); + if let Some(st) = prompt_strokes(params, layer_size, base, model.is_some()) { + fnv(&mut h, serde_json::to_string(&st).unwrap_or_default().as_bytes()); + } + for v in [base, reach(params)[0], reach(params)[1]] { + fnv(&mut h, &v.to_le_bytes()); + } + fnv(&mut h, format!("{:?}", matte_params(params)).as_bytes()); + format!("{h:016x}") +} + +/// Whether the instance is frozen (has frozen mattes). +pub fn is_frozen(params: &Params) -> bool { + !s(params, FROZEN).is_empty() +} + +/// Frame `f`'s frozen matte on a `w`×`h` grid, when the instance is frozen, the mattes still +/// match ([`frozen_key`]) and `f` is one of them. +pub fn frozen_alpha(params: &Params, layer_size: [f64; 2], fps: f64, f: i64, w: usize, h: usize, model: Option<&dyn MaskModel>) -> Option> { + let j = s(params, FROZEN); + if j.is_empty() { + return None; + } + let fc = frozen_cache(j)?; + if fc.key != frozen_key(params, layer_size, fps, model) { + return None; + } + let fr = fc.frames.get(&f)?; + let a = rle::decode(&fr.a)?; + let [fw, fh] = fc.size; + if fw == 0 || fh == 0 || a.len() != fw.checked_mul(fh)? { + return None; + } + let af: Vec = a.iter().map(|v| *v as f32 / 255.0).collect(); + if fw == w && fh == h { + return Some(af); + } + let mut out = vec![0.0f32; w * h]; + for y in 0..h { + for x in 0..w { + let (sx, sy) = ((x as f64 + 0.5) * fw as f64 / w as f64, (y as f64 + 0.5) * fh as f64 / h as f64); + out[y * w + x] = effectcraft_track::roto::matting::sample(&af, fw, fh, sx, sy); + } + } + Some(out) +} + +/// The parsed Frozen parameter (parsed once per content). +fn frozen_cache(j: &str) -> Option> { + use std::collections::HashMap; + use std::sync::{Mutex, OnceLock}; + static C: OnceLock>>> = OnceLock::new(); + let mut h = 0xcbf2_9ce4_8422_2325; + fnv(&mut h, j.as_bytes()); + let c = C.get_or_init(Default::default); + if let Some(d) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&h) { + return Some(d.clone()); + } + let d = Arc::new(FrozenCache::from_json(j)?); + let mut m = c.lock().unwrap_or_else(|e| e.into_inner()); + if m.len() > 8 { + m.clear(); + } + m.insert(h, d.clone()); + Some(d) +} + /// Segmentation settings at `scale` (Roto Brush's defaults). fn seg_opts(scale: f64) -> SegOpts { SegOpts { scale, search_radius: 15.0, motion_threshold: 10.0, motion_damping: 20.0, best: false } @@ -242,12 +368,10 @@ pub fn frame_matte( frame: &mut dyn FnMut(i64) -> Option>, cancel: &dyn Fn() -> bool, ) -> FrameMatte { - let Some(data) = roto_data(params, layer_size, f, model.is_some()) else { return FrameMatte::NoPrompt }; - if !data.in_span(f) { + let Some(chain) = chain(params, layer_size, fps, f, scale, model) else { return FrameMatte::NoPrompt }; + if !chain.data.in_span(f) { return FrameMatte::Outside; } - let keys = data.chain_keys(seed(params, layer_size, fps, model.as_deref())); - let chain = Chain { data: Arc::new(data), keys, opts: seg_opts(scale), refine_radius: 0.0, model }; let Some(seg) = chain.ensure(f, frame, cancel) else { return FrameMatte::Outside }; let (w, h) = (img.width as usize, img.height as usize); let seg = if seg.w == w && seg.h == h { (*seg).clone() } else { roto::resample(&seg, w, h) }; @@ -275,7 +399,12 @@ fn render(ctx: &EffectCtx, buf: Buf) -> Buf { let im = roto::layer_grid(&other, layer_size); (im.width as usize == w && im.height as usize == h).then(|| Arc::new(im)) }; - let alpha = match frame_matte(ctx.params, layer_size, fps, f, sc, &img, roto::installed_model(), &mut frame, &|| false) { + let model = roto::installed_model(); + // Frozen mattes are used as they are (no segmentation). + if let Some(a) = frozen_alpha(ctx.params, layer_size, fps, f, w, h, model.as_deref()) { + return compose(&buf, Some(&a), w, h, b(ctx.params, "invert"), param(ctx.params, "view").map(Value::as_enum).unwrap_or(0) == 1); + } + let alpha = match frame_matte(ctx.params, layer_size, fps, f, sc, &img, model, &mut frame, &|| false) { FrameMatte::Alpha(a) if a.len() == w * h => Some(a), FrameMatte::NoPrompt => return buf, _ => None, @@ -513,4 +642,50 @@ mod tests { let _ = frame_matte(&p, [100.0, 80.0], 25.0, 0, 0.5, &half, Some(stub.clone()), &mut frame, &|| false); assert_eq!(*stub.bbox.lock().unwrap(), Some([37.0, 27.0, 50.0, 40.0])); } + + #[test] + fn job_frames_go_outward_from_the_reference_frame_inside_the_layer() { + let p = params(&[ + ("objectTracking/referenceFrame", Value::Scalar(5.0)), + ("objectTracking/framesBefore", Value::Scalar(2.0)), + ("objectTracking/framesAfter", Value::Scalar(3.0)), + ]); + assert_eq!(job_frames(&p, [0, 100]), [5, 6, 7, 8, 4, 3]); + assert_eq!(job_frames(&p, [4, 6]), [5, 6, 4]); + assert!(job_frames(&p, [10, 20]).is_empty()); + let huge = params(&[("objectTracking/referenceFrame", Value::Scalar(1e300)), ("objectTracking/framesAfter", Value::Scalar(f64::INFINITY))]); + assert!(job_frames(&huge, [0, 100]).is_empty()); + } + + /// Frozen mattes are used as stored while their key matches (no segmentation: a stored + /// pattern the segmentation could never produce comes out), and ignored once it doesn't. + #[test] + fn frozen_mattes_are_used_while_their_key_matches() { + let mut p = params(&[("objectPoint", Value::Vec2([40.0, 40.0])), ("objectTracking/framesAfter", Value::Scalar(3.0))]); + p.values.insert(INPUT_KEY.into(), Value::Str("frozen-test".into())); + // Left half opaque on frame 2. + let alpha: Vec = (0..100 * 80).map(|i| ((i % 100) < 50) as u8 as f32).collect(); + let mut fc = FrozenCache { key: frozen_key(&p, [100.0, 80.0], 25.0, None), size: [100, 80], ..Default::default() }; + fc.frames.insert(2, roto::freeze_frame(&alpha, &[])); + p.values.insert(FROZEN.into(), Value::Str(fc.to_json())); + assert!(is_frozen(&p)); + let out = run(&p, 2); + assert!(out.data[40 * 100 + 10][3] > 0.99 && out.data[40 * 100 + 90][3] < 0.01, "the frozen matte is used"); + // At half resolution too. + assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 50, 40, None).map(|a| (a[20 * 50 + 5], a[20 * 50 + 45])), Some((1.0, 0.0))); + // A frame without a frozen matte is computed (the disk). + assert!(wrong(&run(&p, 1), 42.0) < 30); + // Another prompt: the frozen mattes no longer apply. + p.values.insert("objectPoint".into(), Value::Vec2([41.0, 40.0])); + assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 100, 80, None), None); + assert!(wrong(&run(&p, 2), 44.0) < 30); + // Damaged frozen data is ignored, not a crash. + p.values.insert(FROZEN.into(), Value::Str("{\"key\": 3, \"frames\": [".into())); + assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 100, 80, None), None); + let mut fc2 = fc.clone(); + fc2.size = [usize::MAX, 2]; + p.values.insert("objectPoint".into(), Value::Vec2([40.0, 40.0])); + p.values.insert(FROZEN.into(), Value::Str(fc2.to_json())); + assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 100, 80, None), None); + } } diff --git a/crates/engine/src/commands/roto_cmds.rs b/crates/engine/src/commands/roto_cmds.rs index f4f33d53..ee9cd3a2 100644 --- a/crates/engine/src/commands/roto_cmds.rs +++ b/crates/engine/src/commands/roto_cmds.rs @@ -63,6 +63,23 @@ pub fn specs() -> Vec { has_comp, object_matte_apply ), + cmd!( + "objectMatte.propagate", + "Track Object Matte", + [], + None, + "{layer?, effect?, direction?: forward|backward|both, wait?: block until done} (a background job like Roto Brush propagation; roto.cancel stops it)", + has_comp, + object_matte_propagate + ), + cmd!("objectMatte.freeze", "Freeze Object Matte", [], None, "{layer?, effect?, wait?}", has_comp, object_matte_freeze), + cmd!("objectMatte.unfreeze", "Unfreeze Object Matte", [], None, "{layer?, effect?}", has_comp, object_matte_unfreeze), + query!( + "objectMatte.status", + "Object Matte Status", + "{layer?, effect?}: the job's progress, whether the instance is frozen, its frames and which are computed", + object_matte_status + ), query!( "roto.status", "Roto Brush Status", @@ -79,6 +96,16 @@ fn is_running(s: &Session) -> std::result::Result<(), String> { /// The Roto Brush instance targeted by `p`: `layer` / `effect` (uid, name or 1-based index), /// else a selected one, else the first on the selected (or any) layer. pub(crate) fn roto_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> { + instance_lookup(s, p, cmd, is_roto)?.ok_or_else(|| bad(cmd, "no Roto Brush & Refine Edge effect found (paint a stroke with the Roto Brush tool first)")) +} + +/// The Object Matte instance targeted by `p` (as [`roto_p`]). +fn object_matte_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> { + instance_lookup(s, p, cmd, rt::is_object_matte)?.ok_or_else(|| bad(cmd, "no Object Matte effect found (click an object with the Object Matte tool first)")) +} + +/// The instance of a kind (`is`) targeted by `p`; `None` when there is none. +fn instance_lookup(s: &Session, p: &Value, cmd: &str, is: fn(&PropGroup) -> bool) -> Result> { let cid = super::comp_id(s, p)?; let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; let candidates: Vec = match p.get("layer") { @@ -102,17 +129,17 @@ pub(crate) fn roto_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, Layer .state .selected_props .iter() - .find_map(|(l, u)| (*l == lid).then(|| fxg.groups().find(|g| g.uid == *u && is_roto(g))).flatten()) - .or_else(|| fxg.groups().find(|g| is_roto(g))), + .find_map(|(l, u)| (*l == lid).then(|| fxg.groups().find(|g| g.uid == *u && is(g))).flatten()) + .or_else(|| fxg.groups().find(|g| is(g))), }; - if let Some(g) = g.filter(|g| is_roto(g)) { - return Ok((cid, lid, g.uid)); + if let Some(g) = g.filter(|g| is(g)) { + return Ok(Some((cid, lid, g.uid))); } if p.get("layer").is_some() { break; } } - Err(bad(cmd, "no Roto Brush & Refine Edge effect found (paint a stroke with the Roto Brush tool first)")) + Ok(None) } fn group(s: &Session, cid: ItemId, lid: LayerId, uid: Uid) -> Result<(&Layer, &PropGroup)> { @@ -459,10 +486,13 @@ fn object_matte_apply(s: &mut Session, p: &Value) -> Result { let existing = match p.get("effect").filter(|v| !v.is_null()) { Some(v) => { let uid = v.as_u64().ok_or_else(|| bad(cmd, "`effect` is an Object Matte effect's uid"))?; - layer + let g = layer .effects() .and_then(|f| f.groups().find(|g| g.uid == uid && rt::is_object_matte(g))) .ok_or_else(|| bad(cmd, format!("no Object Matte effect {uid} on this layer")))?; + if frozen_of(g) { + return Err(bad(cmd, "the Object Matte is frozen: Unfreeze it to change the prompt")); + } Some(uid) } None => None, @@ -556,3 +586,65 @@ fn object_matte_reach(s: &Session, cid: ItemId, lid: LayerId, uid: Uid) -> (bool let [before, after] = om::reach(&ps); (om::tracks(&ps), before, after) } + +fn object_matte_propagate(s: &mut Session, p: &Value) -> Result { + let cmd = "objectMatte.propagate"; + let (cid, lid, uid) = object_matte_p(s, p, cmd)?; + let dir = match str_p(p, "direction") { + Some(d) => Direction::from_name(d).ok_or_else(|| bad(cmd, format!("unknown direction `{d}` (forward | backward | both)")))?, + None => Direction::Both, + }; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_roto(cid, lid, uid, RotoTask::Propagate, dir, wait).map_err(|e| bad(cmd, e))?; + let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_roto_running(), "progress": s.roto_progress()}); + if wait { + out["status"] = object_matte_status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?; + } + Ok(out) +} + +fn object_matte_freeze(s: &mut Session, p: &Value) -> Result { + let cmd = "objectMatte.freeze"; + let (cid, lid, uid) = object_matte_p(s, p, cmd)?; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_roto(cid, lid, uid, RotoTask::Freeze, Direction::Both, wait).map_err(|e| bad(cmd, e))?; + let (_, g) = group(s, cid, lid, uid)?; + Ok(json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_roto_running(), "frozen": frozen_of(g), "progress": s.roto_progress()})) +} + +fn object_matte_unfreeze(s: &mut Session, p: &Value) -> Result { + let cmd = "objectMatte.unfreeze"; + let (cid, lid, uid) = object_matte_p(s, p, cmd)?; + let (_, g) = group(s, cid, lid, uid)?; + if frozen_of(g) { + s.edit("Unfreeze", None, |proj, _| { + let g = layer_mut(proj, cid, lid)?.props.find_group_mut(uid).ok_or_else(|| EngineError::Other("the effect is gone".into()))?; + if let Some(pr) = g.get_mut(FROZEN) { + pr.value = KV::Str(String::new()); + } + Ok(()) + })?; + } + Ok(json!({"layer": lid.0, "effect": uid, "frozen": false})) +} + +fn object_matte_status(s: &mut Session, p: &Value) -> Result { + s.poll_roto(false); + let running = s.is_roto_running(); + let mut out = json!({"running": running, "progress": s.roto_progress(), "banner": s.roto_progress().map(|p| p.banner())}); + let Ok((cid, lid, uid)) = object_matte_p(s, p, "objectMatte.status") else { return Ok(out) }; + let (_, g) = group(s, cid, lid, uid)?; + let frozen = frozen_of(g); + let (track, before, after) = object_matte_reach(s, cid, lid, uid); + let computed = s.roto_computed(cid, lid, uid); + out["layer"] = json!(lid.0); + out["effect"] = json!(uid); + out["frozen"] = json!(frozen); + out["track"] = json!(track); + out["reach"] = json!([before, after]); + out["mine"] = json!(running && s.roto_target().is_some_and(|(c, l, u, _)| (c, l, u) == (cid, lid, uid))); + out["pending"] = json!(s.roto_pending.contains(&(cid, lid, uid))); + out["frames"] = json!(computed.keys().collect::>()); + out["computed"] = json!(computed.iter().filter(|(_, c)| **c).map(|(f, _)| *f).collect::>()); + Ok(out) +} diff --git a/crates/engine/src/offload.rs b/crates/engine/src/offload.rs index 23fa2b38..561ebe6c 100644 --- a/crates/engine/src/offload.rs +++ b/crates/engine/src/offload.rs @@ -561,7 +561,8 @@ async fn fill(s: &mut Session, plan: crate::commands::content_fill::FillPlan, po /// Roto Brush propagation in a worker: the segmentations stream back (at most every 250 ms) /// as they are computed, so stopping the job keeps what was propagated. async fn propagate(s: &mut Session, comp: ItemId, layer: LayerId, effect: Uid, dir: crate::roto::Direction, post: Post) -> Result<(), String> { - let keys: Vec = s.roto_chain(comp, layer, effect).map(|c| c.keys.values().copied().collect()).ok_or("no Roto Brush strokes")?; + let keys: Vec = + s.roto_keys(comp, layer, effect).map(|k| k.into_iter().map(|(_, k)| k).collect()).ok_or("no Roto Brush strokes or Object Matte prompt")?; let sent = Rc::new(RefCell::new(std::collections::HashSet::new())); let (p, k2, s2) = (post.clone(), keys.clone(), sent.clone()); let mut last = None::; diff --git a/crates/engine/src/roto.rs b/crates/engine/src/roto.rs index b8c2c649..49f53d39 100644 --- a/crates/engine/src/roto.rs +++ b/crates/engine/src/roto.rs @@ -10,12 +10,18 @@ //! edit, keeps each instance's hidden Input Key equal to a signature of what its frames are made //! of (the layer's source and timing, its masks and the effects above it) and drops frozen //! mattes made from other frames. +//! +//! Object Matte instances run through the same jobs (`objectMatte.propagate` / `.freeze`): the +//! frames are [`om::job_frames`] around the reference frame, each segmented through +//! [`om::ensure`]; Freeze stores [`om::frame_matte`]'s mattes under [`om::frozen_key`]. An edit +//! to an instance's frames or prompt queues it for background tracking like a Roto Brush. use crate::offload::JobKind; use std::collections::{BTreeMap, HashMap}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; +use effectcraft_effects::object_matte as om; use effectcraft_effects::roto::{self as fx, FROZEN, FrozenCache, INPUT_KEY}; use effectcraft_effects::{Params, flatten_params}; use effectcraft_keyframe::Value; @@ -104,14 +110,18 @@ pub struct RotoProgress { pub finished: bool, pub cancelled: bool, pub error: Option, + /// The job works on an Object Matte (else a Roto Brush & Refine Edge). + pub object_matte: bool, } impl RotoProgress { pub fn banner(&self) -> String { let pc = (100 * self.done).checked_div(self.total).unwrap_or(0); - match self.task { - Some(RotoTask::Freeze) => format!("Freezing Roto Brush & Refine Edge: {pc}%"), - _ => format!("Propagating Roto Brush: {pc}%"), + match (self.task, self.object_matte) { + (Some(RotoTask::Freeze), false) => format!("Freezing Roto Brush & Refine Edge: {pc}%"), + (Some(RotoTask::Freeze), true) => format!("Freezing Object Matte: {pc}%"), + (_, false) => format!("Propagating Roto Brush: {pc}%"), + (_, true) => format!("Tracking Object Matte: {pc}%"), } } } @@ -130,8 +140,11 @@ pub struct RotoJob { pub layer: LayerId, pub effect: Uid, pub task: RotoTask, - /// The strokes the job was started with (a Freeze result is dropped when they changed). + /// What the job was started from: the strokes (Roto Brush) or the prompt (Object Matte); a + /// Freeze result is dropped when it changed. strokes: String, + /// The job works on an Object Matte. + pub object_matte: bool, } impl RotoJob { @@ -153,6 +166,20 @@ pub(crate) fn is_object_matte(g: &PropGroup) -> bool { matches!(&g.kind, GroupKind::Effect { effect } if effect == effectcraft_effects::object_matte::ID) } +/// What an Object Matte's segmentation is made from besides its frames: the prompt and tracking +/// parameters (static values), for queueing it after edits and guarding a Freeze. +pub(crate) fn om_prompt(g: &PropGroup) -> String { + let ps = params_static(g); + let mut keys: Vec<&String> = ps.values.keys().filter(|k| ["objectPoint", "useBox", "box", "objectTracking/"].iter().any(|p| k.starts_with(p))).collect(); + keys.sort(); + keys.iter().map(|k| format!("{k}={:?};", ps.values.get(*k))).collect() +} + +/// The job-relevant guard of an instance: its strokes (Roto Brush) or prompt (Object Matte). +fn guard_of(g: &PropGroup) -> String { + if is_object_matte(g) { om_prompt(g) } else { str_of(g, fx::STROKES) } +} + fn str_of(g: &PropGroup, m: &str) -> String { match g.get(m).map(|p| &p.value) { Some(Value::Str(s)) => s.clone(), @@ -195,8 +222,8 @@ pub fn input_key(p: &Project, comp: ItemId, layer: &Layer, uid: Uid) -> String { } /// After an edit: refresh Input Keys (Roto Brush and Object Matte); frozen mattes made from -/// other frames are dropped. Returns the Roto Brush instances whose segmentation changed (to -/// propagate again). +/// other frames are dropped. Returns the instances whose segmentation changed (to propagate +/// again): new frames, strokes (Roto Brush) or prompt (Object Matte). pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> { let mut todo = vec![]; let mut set = vec![]; @@ -206,14 +233,11 @@ pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerI let Some(fxg) = l.effects() else { continue }; for g in fxg.groups().filter(|g| is_roto(g) || is_object_matte(g)) { let key = input_key(after, *cid, l, g.uid); - let roto = is_roto(g); let old = before.comp(*cid).and_then(|bc| bc.layer(l.id)).and_then(|ol| ol.props.find_group(g.uid)); if str_of(g, INPUT_KEY) != key { set.push((*cid, l.id, g.uid, key)); - if roto { - todo.push((*cid, l.id, g.uid)); - } - } else if roto && old.is_none_or(|o| str_of(o, fx::STROKES) != str_of(g, fx::STROKES)) { + todo.push((*cid, l.id, g.uid)); + } else if old.is_none_or(|o| guard_of(o) != guard_of(g)) { todo.push((*cid, l.id, g.uid)); } } @@ -229,7 +253,11 @@ pub(crate) fn sync(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerI } } todo.retain(|(c, l, u)| { - after.comp(*c).and_then(|c| c.layer(*l)).and_then(|l| l.props.find_group(*u)).is_some_and(|g| !fx::data(¶ms_static(g)).is_empty()) + after + .comp(*c) + .and_then(|c| c.layer(*l)) + .and_then(|l| l.props.find_group(*u)) + .is_some_and(|g| is_object_matte(g) || !fx::data(¶ms_static(g)).is_empty()) }); todo } @@ -284,6 +312,7 @@ struct Work { fps: f64, frames: Vec, task: RotoTask, + object_matte: bool, } /// Renders the effect's input frames for a worker, keeping the last few. With a browser job @@ -400,10 +429,13 @@ async fn run_work(w: Work, shared: &RotoShared) { let mut s = lock(&shared.state); s.task = Some(w.task); s.total = w.frames.len() as u64; + s.object_matte = w.object_matte; } let cancel = || shared.cancel.load(Ordering::Relaxed); let chain = fx::Chain::new(&w.params, w.size, w.fps, 1.0); - let mut frozen = FrozenCache { key: fx::frozen_key(&w.params, w.size, w.fps), ..Default::default() }; + let model = fx::installed_model(); + let key = if w.object_matte { om::frozen_key(&w.params, w.size, w.fps, model.as_deref()) } else { fx::frozen_key(&w.params, w.size, w.fps) }; + let mut frozen = FrozenCache { key, ..Default::default() }; for (k, f) in w.frames.iter().enumerate() { if cancel() { let mut s = lock(&shared.state); @@ -414,9 +446,21 @@ async fn run_work(w: Work, shared: &RotoShared) { // The frame and its neighbours (what a step reads), on the GPU. src.prefetch(&[*f, f - 1, f + 1]).await; let mut get = |g: i64| src.get(g); - let ok = match w.task { - RotoTask::Propagate => chain.ensure(*f, &mut get, &cancel).is_some(), - RotoTask::Freeze => match fx::frame_matte(&w.params, w.size, w.fps, *f, &mut get, &cancel) { + let ok = match (w.task, w.object_matte) { + (RotoTask::Propagate, true) => om::ensure(&w.params, w.size, w.fps, *f, model.clone(), &mut get, &cancel).is_some(), + (RotoTask::Freeze, true) => { + let img = get(*f); + match img.map(|img| (om::frame_matte(&w.params, w.size, w.fps, *f, 1.0, &img, model.clone(), &mut get, &cancel), img)) { + Some((om::FrameMatte::Alpha(a), img)) => { + frozen.size = [img.width as usize, img.height as usize]; + frozen.frames.insert(*f, fx::freeze_frame(&a, &[])); + true + } + _ => false, + } + } + (RotoTask::Propagate, false) => chain.ensure(*f, &mut get, &cancel).is_some(), + (RotoTask::Freeze, false) => match fx::frame_matte(&w.params, w.size, w.fps, *f, &mut get, &cancel) { Some((a, b)) => { if let Some(img) = src.get(*f) { frozen.size = [img.width as usize, img.height as usize]; @@ -471,12 +515,15 @@ impl Session { self.roto_job.as_ref().map(|j| j.progress()).or_else(|| { self.offloaded_roto().map(|(j, task)| { let p = &j.progress; + let object_matte = + self.project.comp(j.comp).and_then(|c| c.layer(j.layer)).and_then(|l| l.props.find_group(j.uid)).is_some_and(is_object_matte); RotoProgress { task: Some(task), done: p.done, total: p.total, elapsed: p.elapsed, fps: crate::offload::rate(p.done, p.elapsed), + object_matte, ..Default::default() } }) @@ -488,7 +535,8 @@ impl Session { self.roto_job.as_ref().map(|j| (j.comp, j.layer, j.effect, j.task)).or_else(|| self.offloaded_roto().map(|(j, task)| (j.comp, j.layer, j.uid, task))) } - /// The segmentation chain a propagation of `effect` computes (its frames' cache keys). + /// The segmentation chain a propagation of Roto Brush `effect` computes (its frames' cache + /// keys). pub fn roto_chain(&self, comp: ItemId, layer: LayerId, effect: Uid) -> Option { let c = self.project.comp(comp)?; let l = c.layer(layer)?; @@ -499,6 +547,27 @@ impl Session { Some(fx::Chain::new(¶ms, layer_size(&self.project, c, l), c.frame_rate.as_f64(), 1.0)) } + /// The cache keys of the segmentations a propagation of `effect` (Roto Brush or Object + /// Matte) computes, by layer frame. + pub fn roto_keys(&self, comp: ItemId, layer: LayerId, effect: Uid) -> Option> { + if let Some(ch) = self.roto_chain(comp, layer, effect) { + return Some(ch.keys.into_iter().collect()); + } + let c = self.project.comp(comp)?; + let l = c.layer(layer)?; + let g = l.effects()?.groups().find(|g| g.uid == effect && is_object_matte(g))?; + let params = self.om_params(comp, l, g)?; + let frames = om::job_frames(¶ms, frame_limits(c, l)); + Some(om::frame_keys(¶ms, layer_size(&self.project, c, l), c.frame_rate.as_f64(), &frames, fx::installed_model())) + } + + /// An Object Matte's parameters at its reference frame (what its jobs segment with). + fn om_params(&self, comp: ItemId, l: &Layer, g: &PropGroup) -> Option { + let c = self.project.comp(comp)?; + let t = comp_time_of(c, l, om::reference_frame(¶ms_static(g))); + Some(params_at(&self.project, comp, l, g, t, self.expr.as_deref())) + } + pub fn stop_roto(&mut self) -> bool { if self.cancel_offloaded(JobKind::RotoFreeze) || self.cancel_offloaded(JobKind::RotoPropagate) { return true; @@ -532,17 +601,36 @@ impl Session { let l = c.layer(layer).ok_or("no layer")?; let fxg = l.effects().ok_or("the layer has no effects")?; let index = fxg.groups().position(|g| g.uid == effect).ok_or("no such effect")?; - let g = fxg.groups().nth(index).filter(|g| is_roto(g)).ok_or("not a Roto Brush & Refine Edge effect")?; + let g = fxg.groups().nth(index).filter(|g| is_roto(g) || is_object_matte(g)).ok_or("not a Roto Brush & Refine Edge or Object Matte effect")?; + let object_matte = is_object_matte(g); let size = layer_size(&self.project, c, l); let fps = c.frame_rate.as_f64(); - let pstat = params_static(g); - let d = fx::data(&pstat); - if d.is_empty() { - return Err("draw a foreground stroke with the Roto Brush tool first".into()); - } - let t = comp_time_of(c, l, d.base.unwrap_or(0)); - let params = params_at(&self.project, comp, l, g, t, self.expr.as_deref()); - let frames = job_frames(&d, if task == RotoTask::Freeze { Direction::Both } else { dir }); + let (params, frames) = if object_matte { + let params = self.om_params(comp, l, g).ok_or("no composition")?; + let base = om::reference_frame(¶ms); + if om::roto_data(¶ms, size, base, fx::installed_model().is_some()).is_none() { + return Err("set the object first: click it with the Object Matte tool (its point is off the layer or outside its box)".into()); + } + let all = om::job_frames(¶ms, frame_limits(c, l)); + if all.is_empty() { + return Err(format!("the reference frame {base} is outside the layer")); + } + // Forward / backward from the reference frame, as for a Roto Brush. + let frames = all + .into_iter() + .filter(|f| task == RotoTask::Freeze || dir == Direction::Both || *f == base || (*f > base) == (dir == Direction::Forward)) + .collect(); + (params, frames) + } else { + let pstat = params_static(g); + let d = fx::data(&pstat); + if d.is_empty() { + return Err("draw a foreground stroke with the Roto Brush tool first".into()); + } + let t = comp_time_of(c, l, d.base.unwrap_or(0)); + let params = params_at(&self.project, comp, l, g, t, self.expr.as_deref()); + (params, job_frames(&d, if task == RotoTask::Freeze { Direction::Both } else { dir })) + }; let n = frames.len(); let work = Work { project: self.project.clone(), @@ -557,8 +645,9 @@ impl Session { fps, frames, task, + object_matte, }; - let strokes = str_of(g, fx::STROKES); + let strokes = guard_of(g); if !wait && self.offloads() { drop(work); self.offload_analysis(match task { @@ -569,6 +658,13 @@ impl Session { } let wait = wait || cfg!(target_arch = "wasm32"); let shared = Arc::new(RotoShared::default()); + { + // Known before the worker starts (the progress shows at once). + let mut st = lock(&shared.state); + st.task = Some(task); + st.total = n as u64; + st.object_matte = object_matte; + } let thread = if wait { crate::offload::run_or_queue({ let sh = shared.clone(); @@ -584,7 +680,7 @@ impl Session { .map_err(|e| e.to_string())?, ) }; - self.roto_job = Some(RotoJob { shared, thread, comp, layer, effect, task, strokes }); + self.roto_job = Some(RotoJob { shared, thread, comp, layer, effect, task, strokes, object_matte }); self.events.push(Event::ProjectChanged { revision: self.revision }); self.poll_roto_job(); Ok(n) @@ -601,40 +697,43 @@ impl Session { let _ = t.join(); } let st = job.progress(); + let name = if job.object_matte { "Object Matte" } else { "Roto Brush" }; if let Some(e) = &st.error { - self.events.push(Event::Toast { message: format!("Roto Brush: {e}"), error: true }); + self.events.push(Event::Toast { message: format!("{name}: {e}"), error: true }); self.bump(); return true; } if st.cancelled { - self.events.push(Event::Toast { message: "Roto Brush: stopped".into(), error: false }); + self.events.push(Event::Toast { message: format!("{name}: stopped"), error: false }); self.bump(); return true; } match job.task { RotoTask::Propagate => { - self.events.push(Event::Toast { message: format!("Roto Brush: propagated {} frame(s) in {:.1} s", st.done, st.elapsed), error: false }); + let verb = if job.object_matte { "tracked" } else { "propagated" }; + self.events.push(Event::Toast { message: format!("{name}: {verb} {} frame(s) in {:.1} s", st.done, st.elapsed), error: false }); // Rendered frames depend on the cache: redraw. self.bump(); } RotoTask::Freeze => { let Some(fc) = lock(&job.shared.frozen).take() else { return true }; let (cid, lid, uid, strokes) = (job.comp, job.layer, job.effect, job.strokes.clone()); + let what = if job.object_matte { "prompt" } else { "strokes" }; let r = self.edit("Freeze", None, |p, _| { let g = p .comp_mut(cid) .and_then(|c| c.layer_mut(lid)) .and_then(|l| l.props.find_group_mut(uid)) - .ok_or_else(|| crate::EngineError::Other("the Roto Brush effect was removed".into()))?; - if str_of(g, fx::STROKES) != strokes { - return Err(crate::EngineError::Other("the strokes changed while freezing".into())); + .ok_or_else(|| crate::EngineError::Other(format!("the {name} effect was removed")))?; + if guard_of(g) != strokes { + return Err(crate::EngineError::Other(format!("the {what} changed while freezing"))); } set_str(g, FROZEN, fc.to_json()); Ok(()) }); let msg = match r { - Ok(()) => format!("Roto Brush: froze {} frame(s) in {:.1} s", fc.frames.len(), st.elapsed), - Err(e) => format!("Roto Brush: {e}"), + Ok(()) => format!("{name}: froze {} frame(s) in {:.1} s", fc.frames.len(), st.elapsed), + Err(e) => format!("{name}: {e}"), }; self.events.push(Event::Toast { message: msg, error: false }); } @@ -664,6 +763,12 @@ impl Session { let Some(c) = self.project.comp(comp) else { return out }; let Some(l) = c.layer(layer) else { return out }; let Some(g) = l.props.find_group(effect) else { return out }; + if is_object_matte(g) { + for (f, k) in self.roto_keys(comp, layer, effect).unwrap_or_default() { + out.insert(f, fx::cached(k, 1.0).is_some()); + } + return out; + } let ps = params_static(g); let ch = fx::Chain::new(&ps, layer_size(&self.project, c, l), c.frame_rate.as_f64(), 1.0); for f in ch.keys.keys() { diff --git a/crates/engine/src/tests_roto.rs b/crates/engine/src/tests_roto.rs index 19eee110..edcab5c1 100644 --- a/crates/engine/src/tests_roto.rs +++ b/crates/engine/src/tests_roto.rs @@ -465,3 +465,84 @@ fn object_matte_reprompt_updates_the_instance() { // An unknown or non-Object Matte effect is refused. assert!(s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "effect": 999_999, "point": c})).is_err()); } + +/// Object Matte runs in the background like Roto Brush propagation: queued by the edit, a job +/// that returns at once and reports progress, stops when cancelled (keeping what it computed), +/// computes the tracked frames when left to run, and Freeze stores mattes that renders then use +/// without segmenting again. +#[test] +fn object_matte_background_job_cancel_and_freeze() { + hold_roto_cache_test_lock(); + let (mut s, clip, solid) = setup(|f| draw(center(f))); + let cid = s.active_comp_id().unwrap(); + s.edit("remove solid", None, |p, _| { + p.comp_mut(cid).unwrap().layers.retain(|l| l.id != solid); + for it in p.items.values_mut() { + if let effectcraft_project::ItemKind::Footage(f) = &mut it.kind { + f.path = "object-matte-job.mov".into(); + } + } + Ok(()) + }) + .unwrap(); + let c = center(0); + let r = s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "point": c, "before": 0, "after": 24, "frame": 0})).unwrap(); + let uid = r["effect"].as_u64().unwrap(); + assert!(s.roto_pending.contains(&(cid, clip, uid)), "the new prompt is queued for tracking"); + let wait_done = |s: &mut Session| { + let t = std::time::Instant::now(); + while s.is_roto_running() { + assert!(t.elapsed().as_secs() < 120, "the job never finished"); + std::thread::sleep(std::time::Duration::from_millis(5)); + s.poll_roto(false); + } + s.poll_roto(false); + }; + let toast = |s: &Session, m: &str| s.events.iter().any(|e| matches!(e, crate::Event::Toast { message, .. } if message.starts_with(m))); + let status = |s: &mut Session| s.execute("objectMatte.status", json!({"layer": clip.0, "effect": uid})).unwrap(); + + // Cancelled at once: it stops early, and says so. + let r = s.execute_checked("objectMatte.propagate", json!({"layer": clip.0, "effect": uid})).unwrap(); + assert_eq!(r["frames"], 25); + assert!(!s.roto_pending.contains(&(cid, clip, uid))); + let _ = s.execute("roto.cancel", json!({})); + wait_done(&mut s); + assert!(toast(&s, "Object Matte: stopped"), "{:?}", s.events); + let st = status(&mut s); + assert_eq!(st["frames"].as_array().unwrap().len(), 25, "{st}"); + let early = st["computed"].as_array().unwrap().len(); + assert!(early < 25, "{st}"); + + // Left to run: the call returns while the job works (progress visible), then every tracked + // frame is computed. + let r = s.execute_checked("objectMatte.propagate", json!({"layer": clip.0, "effect": uid})).unwrap(); + assert_eq!(r["running"], true, "{r}"); + let pr = s.roto_progress().unwrap(); + assert!(pr.object_matte && pr.banner().starts_with("Tracking Object Matte"), "{pr:?}"); + // One job at a time. + assert!(s.execute_checked("objectMatte.propagate", json!({"layer": clip.0, "effect": uid})).is_err()); + wait_done(&mut s); + assert!(toast(&s, "Object Matte: tracked"), "{:?}", s.events); + assert_eq!(status(&mut s)["computed"].as_array().unwrap().len(), 25); + let before = alpha_at(&s, 4); + + // Freeze, then drop the segmentations: renders use the frozen mattes and compute nothing. + let r = s.execute_checked("objectMatte.freeze", json!({"layer": clip.0, "effect": uid, "wait": true})).unwrap(); + assert_eq!(r["frozen"], true, "{r}"); + let keys: Vec = s.roto_keys(cid, clip, uid).unwrap().into_iter().map(|(_, k)| k).collect(); + assert_eq!(keys.len(), 25); + effectcraft_effects::roto::forget(keys.iter().copied()); + let frozen = alpha_at(&s, 4); + let diff = before.iter().zip(&frozen).filter(|(a, b)| (**a - **b).abs() > 0.02).count(); + assert!(diff < 30, "{diff} pixels differ from the computed matte"); + assert!(status(&mut s)["computed"].as_array().unwrap().is_empty(), "a frozen frame was segmented again"); + assert_eq!(status(&mut s)["frozen"], true); + // A frozen prompt is refused; Unfreeze computes again. + assert!(s.execute_checked("objectMatte.apply", json!({"layer": clip.0, "effect": uid, "point": c})).is_err()); + s.execute_checked("objectMatte.unfreeze", json!({"layer": clip.0, "effect": uid})).unwrap(); + let _ = alpha_at(&s, 4); + assert!(status(&mut s)["computed"].as_array().unwrap().iter().any(|f| f == 4)); + // Undo brings the frozen mattes back. + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(status(&mut s)["frozen"], true); +} diff --git a/crates/ui-egui/src/i18n/ui.rs b/crates/ui-egui/src/i18n/ui.rs index f7c77905..117d4b70 100644 --- a/crates/ui-egui/src/i18n/ui.rs +++ b/crates/ui-egui/src/i18n/ui.rs @@ -508,6 +508,7 @@ const JAPANESE: &[(&str, &str)] = &[ ("⌘/Ctrl-drag a pin to record its motion", "⌘/Ctrl+ドラッグでピンの動きを記録"), ("▣ {}", "▣ {}"), ("⚠ {} is used by:", "⚠ {}は次のコマンドで使用中:"), + ("Track", "トラック"), ]; const SIMPLIFIED_CHINESE: &[(&str, &str)] = &[ @@ -1033,6 +1034,7 @@ const SIMPLIFIED_CHINESE: &[(&str, &str)] = &[ ("After Effects project", "After Effects 项目"), ("Lottie animation", "Lottie 动画"), ("Project", "项目"), + ("Track", "跟踪"), ]; const TRADITIONAL_CHINESE: &[(&str, &str)] = &[ @@ -1553,6 +1555,7 @@ const TRADITIONAL_CHINESE: &[(&str, &str)] = &[ ("After Effects project", "After Effects 專案"), ("Lottie animation", "Lottie 動畫"), ("Project", "專案"), + ("Track", "追蹤"), ]; /// Ukrainian (uk): original translations of the same interface rows as the Chinese catalogs. @@ -1988,6 +1991,7 @@ const UKRAINIAN: &[(&str, &str)] = &[ ("⌘/Ctrl-drag a pin to record its motion", "Перетягніть опорну точку з ⌘/Ctrl, щоб записати її рух"), ("▣ {}", "▣ {}"), ("⚠ {} is used by:", "⚠ {} використовується в:"), + ("Track", "Відстежити"), ]; #[cfg(test)] diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index a0e05e00..a6b5501c 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -1580,6 +1580,12 @@ impl EffectcraftApp { self.session.poll_roto(true); ctx.request_repaint_after(std::time::Duration::from_millis(100)); } + // Roto Brush propagation and Object Matte tracking / Freeze: finish jobs (a Freeze writes + // its mattes) and start queued ones in the background. + if self.session.roto_job.is_some() || !self.session.roto_pending.is_empty() { + self.session.poll_roto(true); + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } // 3D Camera Tracker: likewise (tracking, then solving). if self.session.camera_job.is_some() || !self.session.camera_pending.is_empty() { self.session.poll_camera(true); diff --git a/crates/ui-egui/src/panels/effect_controls.rs b/crates/ui-egui/src/panels/effect_controls.rs index 501adf89..a63b5a83 100644 --- a/crates/ui-egui/src/panels/effect_controls.rs +++ b/crates/ui-egui/src/panels/effect_controls.rs @@ -787,6 +787,44 @@ fn roto_editor(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, l p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim); } +/// Object Matte: Freeze / Unfreeze and Track / Stop buttons (the background job that computes +/// the matte over the tracked frames) and its status. +fn object_matte_editor(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter, layer: &Layer, g: &PropGroup, r: Rect, actions: &mut Actions) { + use effectcraft_engine::effects::object_matte as om; + let t = app.tokens; + let params = effectcraft_engine::effects::flatten_params(g, &mut |pr| pr.value.clone()); + let frozen = om::is_frozen(¶ms); + let mine = app.session.roto_target().is_some_and(|(_, l, u, _)| l == layer.id && u == g.uid); + let running = mine && app.session.is_roto_running(); + let busy = app.session.is_roto_running(); + let b1 = Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 4.0), vec2(84.0, 22.0)); + let b2 = Rect::from_min_size(pos2(b1.max.x + 8.0, b1.min.y), vec2(84.0, 22.0)); + let label = if frozen { "Unfreeze" } else { "Freeze" }; + if widgets::text_button(ui, b1, crate::i18n::tr(label), frozen, &t, egui::Id::new(("om-freeze-ec", g.uid))).clicked() && !busy { + let id = if frozen { "objectMatte.unfreeze" } else { "objectMatte.freeze" }; + actions.push((id.into(), json!({"layer": layer.id.0, "effect": g.uid}))); + } + app.auto.add(&format!("effectControls.objectMatte.{}.freeze", g.uid), b1, label); + let (l2, id2) = if running { ("Stop", "roto.cancel") } else { ("Track", "objectMatte.propagate") }; + if widgets::text_button(ui, b2, crate::i18n::tr(l2), false, &t, egui::Id::new(("om-track-ec", g.uid))).clicked() && !frozen && (running || !busy) { + actions.push((id2.into(), json!({"layer": layer.id.0, "effect": g.uid}))); + } + app.auto.add(&format!("effectControls.objectMatte.{}.track", g.uid), b2, l2); + let base = om::reference_frame(¶ms); + let status = match app.session.roto_progress() { + Some(pr) if running => pr.banner(), + _ if frozen => format!("Frozen from reference frame {base}"), + _ if app.session.roto_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => "Waiting to track".to_string(), + _ => format!("Reference frame {base}: Track computes the matte ahead of playback"), + }; + p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, &status, Tokens::ui(11.5), t.text_dim); + app.auto.add( + &format!("effectControls.objectMatte.{}.status", g.uid), + Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 30.0), vec2(r.width() - 24.0, 16.0)), + &status, + ); +} + // --------------------------------------------------------------------------------------------- // Curves / Levels editors @@ -797,7 +835,7 @@ fn editor_height(effect: &str, g: &PropGroup, width: f32) -> f32 { "ec.color.curves" => 34.0 + curves_size(width) + 26.0, effectcraft_engine::effects::warp_stab::ID => 50.0, effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::EDITOR_HEIGHT, - effectcraft_engine::effects::roto::ID => 50.0, + effectcraft_engine::effects::roto::ID | effectcraft_engine::effects::object_matte::ID => 50.0, "ec.color.levels" | "ec.color.levelsic" => 34.0 + 80.0 + 58.0, "ec.color.autolevels" | "ec.color.autocontrast" | "ec.color.autocolor" => 24.0 + 80.0 + 12.0, _ => 0.0, @@ -1207,6 +1245,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { effectcraft_engine::effects::warp_stab::ID => warp_editor(app, ui, &bp, &layer, g, er, &mut actions), effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::editor(app, ui, &bp, &layer, g, er, &mut actions), effectcraft_engine::effects::roto::ID => roto_editor(app, ui, &bp, &layer, g, er, &mut actions), + effectcraft_engine::effects::object_matte::ID => object_matte_editor(app, ui, &bp, &layer, g, er, &mut actions), "ec.color.levels" | "ec.color.levelsic" => levels_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions), super::fx_editors::GLOW | super::fx_editors::RESHAPE | super::fx_editors::EXTRACTOR => { super::fx_editors::header_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions) diff --git a/crates/ui-egui/tests/ui_object_matte.rs b/crates/ui-egui/tests/ui_object_matte.rs index 483d271e..5f30ba03 100644 --- a/crates/ui-egui/tests/ui_object_matte.rs +++ b/crates/ui-egui/tests/ui_object_matte.rs @@ -8,6 +8,7 @@ use effectcraft_engine::effects::object_matte as om; use effectcraft_engine::keyframe::Value; use effectcraft_engine::project::{GroupKind, LayerId, PropGroup}; use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::dock::PanelKind; use effectcraft_ui_egui::state::Tool; use egui::{Event, Modifiers, Pos2, Rect, pos2, vec2}; use egui_kittest::Harness; @@ -140,3 +141,45 @@ fn object_matte_tool_prompts_with_a_click_and_a_box() { h.state_mut().session.execute("edit.undo", json!({})).unwrap(); assert_eq!(flatten_params(&mattes(&h)[0], &mut |p| p.value.clone()).get("useBox"), Some(&Value::Bool(false))); } + +/// Effect Controls: the Object Matte's Track and Freeze buttons run the background job and its +/// status line shows where it is; the UI keeps drawing frames while it runs and finishes the +/// job itself (a Freeze writes its mattes without any other job running). +#[test] +fn object_matte_effect_controls_track_and_freeze() { + let mut h = harness(); + h.state_mut().show_panel(PanelKind::EffectControls); + h.state_mut().ui.tool = Tool::ObjectMatte; + h.run_steps(2); + let center = screen(&h, [320.0, 180.0]); + click(&mut h, center, Modifiers::NONE); + let uid = mattes(&h)[0].uid; + h.run_steps(2); + let status = |h: &Harness<'_, EffectcraftApp>| h.state().auto.find(&format!("effectControls.objectMatte.{uid}.status")).map(|e| e.label.clone()); + assert!(status(&h).is_some(), "no Object Matte status line"); + // Freeze: a background job (the frames keep coming), then the effect is frozen. + let b = rect(&h, &format!("effectControls.objectMatte.{uid}.freeze")); + // Let any automatic tracking finish first (one job at a time). + let frozen = |h: &Harness<'_, EffectcraftApp>| om::is_frozen(&flatten_params(&mattes(h)[0], &mut |p| p.value.clone())); + let t = std::time::Instant::now(); + while h.state().session.is_roto_running() { + assert!(t.elapsed().as_secs() < 120); + h.step(); + } + click(&mut h, b.center(), Modifiers::NONE); + let t = std::time::Instant::now(); + while !frozen(&h) { + assert!( + t.elapsed().as_secs() < 20, + "never frozen: {:?} / {} / running {} / {:?}", + status(&h), + h.state().ui.status, + h.state().session.is_roto_running(), + h.state().session.roto_progress() + ); + h.step(); + } + h.run_steps(2); + assert!(status(&h).is_some_and(|s| s.starts_with("Frozen")), "{:?}", status(&h)); + assert_eq!(h.state().auto.find(&format!("effectControls.objectMatte.{uid}.freeze")).map(|e| e.label.as_str()), Some("Unfreeze")); +} diff --git a/docs/agents.md b/docs/agents.md index 6a3b9e85..d94d46bd 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -471,6 +471,24 @@ effectcraft-cli run clip.ecproj roto.stroke '{"layer":"#1","points":[[300,200],[ roto.propagate '{"layer":"#1","wait":true}' roto.freeze '{"layer":"#1","wait":true}' --save ``` +### Object Matte + +The Object Matte tool (third in the Alt+W slot) works in the Composition viewer: click an object, +or drag a box around it; Shift adds another Object Matte instead of re-prompting the layer's. +Agents use `objectMatte.apply {layer, point: [x, y], box?: [x0, y0, x1, y1], frame?, track?, +before?, after?, effect?}` (layer pixels, layer frames; `effect` re-prompts that instance), one +undo step each. With a trained model installed (Settings ▸ Roto Brush) the box is the model's box +prompt. `objectMatte.propagate {layer, effect?, direction?, wait?}` tracks the matte over the +frames around the reference frame in the background (`roto.cancel` stops it; edits to the prompt +or the layer's frames queue it again), `objectMatte.freeze {wait?}` / `objectMatte.unfreeze` +store or drop the final mattes, and `objectMatte.status {layer, effect?}` reports the job's +progress, the frozen state, the tracked frames and which are computed. + +```sh +effectcraft-cli run clip.ecproj objectMatte.apply '{"layer":"#1","point":[320,200],"box":[250,120,400,300]}' \ + objectMatte.propagate '{"layer":"#1","wait":true}' objectMatte.freeze '{"layer":"#1","wait":true}' --save +``` + ### Text: styles and editing Source Text holds character style runs and per-paragraph settings. `layer.setText` changes the diff --git a/docs/effects.md b/docs/effects.md index 84f16ef8..8a02b489 100644 --- a/docs/effects.md +++ b/docs/effects.md @@ -3,7 +3,7 @@ -EffectCraft ships 307 effects with 3028 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. +EffectCraft ships 307 effects with 3029 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. - **GPU**: 280 effects also run on the GPU compositor with identical results. - **32**: 307 effects process 32-bit float (HDR, overbright) pixels without clamping.