diff --git a/Cargo.lock b/Cargo.lock index 8519ae6b..b3f5fdaf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1197,8 +1197,10 @@ dependencies = [ "effectcraft-render", "effectcraft-text", "effectcraft-time", + "effectcraft-track", "image", "log", + "rayon", "serde", "serde_json", "thiserror 2.0.21", @@ -1368,6 +1370,15 @@ dependencies = [ "serde", ] +[[package]] +name = "effectcraft-track" +version = "0.1.0" +dependencies = [ + "effectcraft-raster", + "rayon", + "serde", +] + [[package]] name = "effectcraft-ui-egui" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index a22b766c..38cd14e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ effectcraft-keyframe = { path = "crates/keyframe" } effectcraft-project = { path = "crates/project" } effectcraft-text = { path = "crates/text" } effectcraft-effects = { path = "crates/effects" } +effectcraft-track = { path = "crates/track" } effectcraft-render = { path = "crates/render" } effectcraft-media = { path = "crates/media" } effectcraft-export = { path = "crates/export" } @@ -109,6 +110,9 @@ opt-level = 3 [profile.dev.package.effectcraft-effects] opt-level = 3 +[profile.dev.package.effectcraft-track] +opt-level = 3 + [profile.dev.package.effectcraft-path] opt-level = 3 diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 815b2ab9..9d77eea4 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -20,9 +20,11 @@ effectcraft-path = { workspace = true } effectcraft-text = { workspace = true } effectcraft-effects = { workspace = true } effectcraft-render = { workspace = true } +effectcraft-track = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } log = { workspace = true } +rayon = { workspace = true } image = { workspace = true } web-time = { workspace = true } diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 6301e440..7833f818 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -24,6 +24,7 @@ mod styles; mod text_anim; mod three_d; mod time; +mod track; mod view; use std::sync::OnceLock; @@ -102,6 +103,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(file_more::specs()); v.extend(comp_more::specs()); v.extend(frontend::specs()); + v.extend(track::specs()); v.extend(stubs::specs()); v }) diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index b8b97575..76cbd18c 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -83,9 +83,7 @@ pub fn specs() -> Vec { stub!("text.animatorFontAxes", "Variable Font Axes", ["Animation", "Animate Text"], "{}"), stub!("track.camera", "Track Camera", ["Animation"], "{}"), stub!("track.warpStabilizer", "Warp Stabilizer VFX", ["Animation"], "{}"), - stub!("track.motion", "Track Motion", ["Animation"], "{}"), stub!("track.mask", "Track Mask", ["Animation"], "{}"), - stub!("track.property", "Track this Property", ["Animation"], "{}"), // Workspaces and panels that don't exist yet. stub!("window.saveWorkspace", "Save Changes to this Workspace", ["Window", "Workspace"], "{}"), stub!("window.saveWorkspaceAs", "Save as New Workspace...", ["Window", "Workspace"], "{name}"), diff --git a/crates/engine/src/commands/track.rs b/crates/engine/src/commands/track.rs new file mode 100644 index 00000000..2588c061 --- /dev/null +++ b/crates/engine/src/commands/track.rs @@ -0,0 +1,584 @@ +//! Motion tracking commands (Animation ▸ Track Motion / Stabilize Motion / Track this Property and +//! the Tracker panel): create trackers, set type / target / options, move track points, analyze +//! (background job or blocking), apply, reset. +//! +//! A tracker is addressed by `layer` (the tracked layer, a.k.a. Motion Source) and `tracker` +//! (group uid, name or 1-based index); both default to the Tracker panel's Current Track. + +use effectcraft_project::build::Ids; +use effectcraft_project::tracking::{self, LowConfidence, TrackChannel, TrackKind, TrackerSettings}; +use effectcraft_project::{ItemId, Layer, LayerId, Node, PropGroup, Uid, Value}; +use effectcraft_time::Tick; +use serde_json::json; + +use super::{CommandSpec, b_p, bad, comp_id, f_p, frontend, has_comp, layer_mut, layer_p, resolve_layer, str_p}; +use crate::tracking::{Dims, Direction, Work, point_values}; +use crate::{EngineError, Result, Session, cmd, query}; + +type V = serde_json::Value; + +pub fn specs() -> Vec { + vec![ + cmd!("track.motion", "Track Motion", ["Animation"], None, "{layer?}", can_track, |s, p| new_with(s, p, TrackKind::Transform, "track.motion")), + cmd!("track.stabilize", "Stabilize Motion", ["Animation"], None, "{layer?}", can_track, |s, p| new_with(s, p, TrackKind::Stabilize, "track.stabilize")), + cmd!( + "track.new", + "New Tracker", + [], + None, + "{layer?, kind?: transform|stabilize|affine|perspective|raw, position?, rotation?, scale?, target?: layer}", + can_track, + new_tracker + ), + cmd!("track.property", "Track this Property", ["Animation"], None, "{layer?, source?: layer}", has_prop_selected, track_property), + cmd!("track.select", "Current Track", [], None, "{layer?, tracker?: uid|name|#n}", has_comp, select), + cmd!( + "track.setType", + "Track Type", + [], + None, + "{layer?, tracker?, kind?: transform|stabilize|affine|perspective|raw, position?, rotation?, scale?}", + has_track, + set_type + ), + cmd!("track.setTarget", "Edit Target", [], None, "{layer?, tracker?, target: layer|null}", has_track, set_target), + cmd!( + "track.options", + "Motion Tracker Options", + [], + None, + "{layer?, tracker?, name?, channel?: rgb|luminance|saturation, blur? (px, 0 = off), enhance?, subpixel?, adaptEveryFrame?, threshold? (%), action?: continue|stop|extrapolate|adapt, trackShape?}", + has_track, + options + ), + cmd!( + "track.setPoint", + "Move Track Point", + [], + None, + "{layer?, tracker?, point: n (1-based), center?: [x,y], featureSize?: [w,h], searchOffset?: [x,y], searchSize?: [w,h], attachOffset?: [x,y], move?: [dx,dy], time? (s)}", + has_track, + set_point + ), + cmd!( + "track.analyze", + "Analyze", + [], + None, + "{layer?, tracker?, direction?: forward|backward|frameForward|frameBackward, start? (s), end? (s), wait?: block until done}", + has_track, + analyze + ), + cmd!("track.stop", "Stop Analysis", [], None, "{}", is_tracking, |s, _| Ok(json!({"stopped": s.stop_track()}))), + cmd!("track.apply", "Apply", [], None, "{layer?, tracker?, dimensions?: xy|x|y}", has_track, apply), + cmd!("track.reset", "Reset", [], None, "{layer?, tracker?}", has_track, reset), + cmd!("track.delete", "Delete Tracker", [], None, "{layer?, tracker?}", has_track, delete), + cmd!("track.editTargetDialog", "Edit Target...", [], None, "{}", has_track, |s, p| frontend(s, "track.editTargetDialog", p)), + cmd!("track.optionsDialog", "Options...", [], None, "{}", has_track, |s, p| frontend(s, "track.optionsDialog", p)), + query!("track.status", "Tracker Status", "{layer?, tracker?}", status), + ] +} + +// ---------- enablement ---------- + +fn trackable(l: &Layer) -> bool { + l.has_video() && !matches!(l.source, effectcraft_project::LayerSource::Text | effectcraft_project::LayerSource::Shape) +} + +fn can_track(s: &Session) -> std::result::Result<(), String> { + has_comp(s)?; + let c = s.active_comp().ok_or("no composition is open")?; + match s.state.selected_layers.first().and_then(|l| c.layer(*l)) { + Some(l) if trackable(l) => Ok(()), + Some(_) => Err("select a footage, composition or solid layer to track".into()), + None => Err("select a layer to track".into()), + } +} + +fn has_prop_selected(s: &Session) -> std::result::Result<(), String> { + has_comp(s)?; + if s.state.selected_props.is_empty() { Err("select a property to track".into()) } else { Ok(()) } +} + +fn has_track(s: &Session) -> std::result::Result<(), String> { + current(s, &json!({}), "track").map(|_| ()).map_err(|e| e.to_string().replace("invalid parameters for `track`: ", "")) +} + +fn is_tracking(s: &Session) -> std::result::Result<(), String> { + if s.is_tracking() { Ok(()) } else { Err("no track analysis is running".into()) } +} + +// ---------- addressing ---------- + +/// (comp, tracked layer, tracker uid) from `layer` / `tracker` or the Current Track. +pub(crate) fn current(s: &Session, p: &V, cmd: &str) -> Result<(ItemId, LayerId, Uid)> { + let cid = comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let lid = match p.get("layer") { + Some(v) => resolve_layer(comp, v).ok_or_else(|| bad(cmd, format!("no layer {v}")))?, + None => match s.state.current_track { + Some((l, _)) if comp.layer(l).is_some() => l, + _ => s + .state + .selected_layers + .iter() + .copied() + .find(|l| comp.layer(*l).is_some_and(|l| l.trackers().next().is_some())) + .ok_or_else(|| bad(cmd, "no current track: use Track Motion first"))?, + }, + }; + let layer = comp.layer(lid).ok_or_else(|| bad(cmd, "no layer"))?; + let uid = match p.get("tracker") { + Some(V::Number(n)) => { + let n = n.as_u64().unwrap_or(0); + layer.tracker(n).map(|(g, _)| g.uid).or_else(|| layer.trackers().nth((n as usize).saturating_sub(1)).map(|(g, _)| g.uid)) + } + Some(V::String(name)) => match name.strip_prefix('#').and_then(|i| i.parse::().ok()) { + Some(i) => layer.trackers().nth(i.saturating_sub(1)).map(|(g, _)| g.uid), + None => layer.trackers().find(|(g, _)| &g.name == name).map(|(g, _)| g.uid), + }, + _ => match s.state.current_track { + Some((l, t)) if l == lid && layer.tracker(t).is_some() => Some(t), + _ => layer.trackers().next().map(|(g, _)| g.uid), + }, + } + .ok_or_else(|| bad(cmd, format!("layer `{}` has no such tracker", layer.name)))?; + Ok((cid, lid, uid)) +} + +fn tracker_mut(proj: &mut effectcraft_project::Project, cid: ItemId, lid: LayerId, uid: Uid) -> Result<&mut PropGroup> { + layer_mut(proj, cid, lid)?.props.find_group_mut(uid).ok_or_else(|| EngineError::Other("tracker gone".into())) +} + +fn layer_size(s: &Session, cid: ItemId, l: &Layer) -> [f64; 2] { + let (w, h) = effectcraft_render::source_size(&s.project, l); + if w == 0 { + let c = s.project.comp(cid); + c.map(|c| [c.width as f64, c.height as f64]).unwrap_or([1920.0, 1080.0]) + } else { + [w as f64, h as f64] + } +} + +/// The layer directly above `lid` (the default Motion Target). +fn layer_above(comp: &effectcraft_project::Comp, lid: LayerId) -> Option { + let i = comp.layers.iter().position(|l| l.id == lid)?; + i.checked_sub(1).map(|i| comp.layers[i].id) +} + +fn kind_p(p: &V, cmd: &str) -> Result> { + match str_p(p, "kind") { + Some(k) => TrackKind::from_name(k).map(Some).ok_or_else(|| bad(cmd, format!("unknown kind `{k}` (transform|stabilize|affine|perspective|raw)"))), + None => Ok(None), + } +} + +// ---------- create ---------- + +fn new_with(s: &mut Session, p: &V, kind: TrackKind, cmd: &str) -> Result { + let mut q = p.clone(); + q["kind"] = json!(kind.id()); + create(s, &q, cmd) +} + +fn new_tracker(s: &mut Session, p: &V) -> Result { + create(s, p, "track.new") +} + +fn create(s: &mut Session, p: &V, cmd: &str) -> Result { + let (cid, lid) = layer_p(s, p, cmd)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + if !trackable(layer) { + return Err(bad(cmd, format!("layer `{}` has no pixels to track", layer.name))); + } + let kind = kind_p(p, cmd)?.unwrap_or_default(); + let mut st = TrackerSettings::new(kind); + st.position = b_p(p, "position").unwrap_or(true); + st.rotation = b_p(p, "rotation").unwrap_or(false); + st.scale = b_p(p, "scale").unwrap_or(false); + // Motion Target: the layer above the tracked layer (Stabilize applies to the layer itself). + st.target = match p.get("target") { + Some(V::Null) => None, + Some(v) => Some(resolve_layer(comp, v).ok_or_else(|| bad(cmd, format!("no target layer {v}")))?), + None if kind == TrackKind::Stabilize => Some(lid), + None => layer_above(comp, lid), + }; + let size = layer_size(s, cid, layer); + let n_existing = layer.trackers().count(); + let name = format!("Tracker {}", n_existing + 1); + let pts = tracking::default_points(kind, st.point_count(), size); + let uid = s.edit(&format!("New {}", name), None, |proj, state| { + let mut next = proj.next_id; + let l = layer_mut(proj, cid, lid)?; + let mut ids = Ids(&mut next); + let g = tracking::tracker(&mut ids, &name, st, &pts); + let uid = g.uid; + l.motion_trackers_mut(&mut ids).children.push(g.into()); + proj.next_id = next; + state.current_track = Some((lid, uid)); + state.selected_layers = vec![lid]; + Ok(uid) + })?; + let _ = frontend(s, "window.panel", &json!({"panel": "tracker"})); + Ok(json!({"tracker": uid, "name": name, "layer": lid.0, "points": pts.len()})) +} + +/// Track this Property: track another layer and drive the selected property's layer. +fn track_property(s: &mut Session, p: &V) -> Result { + let cid = comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let target = match p.get("layer") { + Some(v) => resolve_layer(comp, v).ok_or_else(|| bad("track.property", format!("no layer {v}")))?, + None => s.state.selected_props.first().map(|(l, _)| *l).ok_or_else(|| bad("track.property", "select a property first"))?, + }; + let source = match p.get("source") { + Some(v) => resolve_layer(comp, v).ok_or_else(|| bad("track.property", format!("no layer {v}")))?, + None => { + let ti = comp.layers.iter().position(|l| l.id == target).unwrap_or(0); + comp.layers + .iter() + .skip(ti + 1) + .chain(comp.layers.iter().take(ti)) + .find(|l| l.id != target && trackable(l)) + .map(|l| l.id) + .ok_or_else(|| bad("track.property", "no layer to track"))? + } + }; + create(s, &json!({"layer": source.0, "kind": "transform", "target": target.0, "comp": cid.0}), "track.property") +} + +// ---------- settings ---------- + +fn select(s: &mut Session, p: &V) -> Result { + let (_, lid, uid) = current(s, p, "track.select")?; + s.state.current_track = Some((lid, uid)); + s.bump(); + Ok(json!({"layer": lid.0, "tracker": uid})) +} + +fn set_type(s: &mut Session, p: &V) -> Result { + let (cid, lid, uid) = current(s, p, "track.setType")?; + let kind = kind_p(p, "track.setType")?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let size = layer_size(s, cid, layer); + let above = layer_above(comp, lid); + let n = s.edit("Track Type", None, |proj, _| { + let mut next = proj.next_id; + let g = tracker_mut(proj, cid, lid, uid)?; + let st = g.tracker_settings_mut().ok_or_else(|| bad("track.setType", "not a tracker"))?; + if let Some(k) = kind { + if k == TrackKind::Stabilize { + st.target = Some(lid); + } else if st.kind == TrackKind::Stabilize && st.target == Some(lid) { + st.target = above; + } + st.kind = k; + } + for (k, f) in [("position", &mut st.position), ("rotation", &mut st.rotation), ("scale", &mut st.scale)] { + if let Some(v) = b_p(p, k) { + *f = v; + } + } + let (kind, n) = (st.kind, st.point_count()); + // Add or remove Track Points to match the type. + let have = g.track_points().count(); + if have > n { + let mut seen = 0; + g.children.retain(|c| match c { + Node::Group(tp) if tp.match_id == tracking::TRACK_POINT => { + seen += 1; + seen <= n + } + _ => true, + }); + } else if have < n { + let defaults = tracking::default_points(kind, n, size); + let mut ids = Ids(&mut next); + for (i, d) in defaults.iter().enumerate().skip(have) { + g.children.push(tracking::track_point(&mut ids, &format!("Track Point {}", i + 1), d).into()); + } + } + proj.next_id = next; + Ok(n) + })?; + Ok(json!({"points": n})) +} + +fn set_target(s: &mut Session, p: &V) -> Result { + let (cid, lid, uid) = current(s, p, "track.setTarget")?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let target = match p.get("target") { + None | Some(V::Null) => None, + Some(v) => Some(resolve_layer(comp, v).ok_or_else(|| bad("track.setTarget", format!("no layer {v}")))?), + }; + s.edit("Edit Target", None, |proj, _| { + let st = tracker_mut(proj, cid, lid, uid)?.tracker_settings_mut().ok_or_else(|| bad("track.setTarget", "not a tracker"))?; + if st.kind == TrackKind::Stabilize && target.is_some_and(|t| t != lid) { + return Err(bad("track.setTarget", "Stabilize applies to the tracked layer")); + } + st.target = target; + Ok(()) + })?; + Ok(json!({"target": target.map(|t| t.0)})) +} + +fn options(s: &mut Session, p: &V) -> Result { + let (cid, lid, uid) = current(s, p, "track.options")?; + let channel = match str_p(p, "channel").map(|c| c.to_ascii_lowercase()) { + None => None, + Some(c) => Some(match c.as_str() { + "rgb" => TrackChannel::Rgb, + "luminance" | "luma" => TrackChannel::Luminance, + "saturation" => TrackChannel::Saturation, + _ => return Err(bad("track.options", "channel: rgb|luminance|saturation")), + }), + }; + let action = match str_p(p, "action") { + None => None, + Some(a) => Some(match effectcraft_track::ConfidenceAction::from_name(a) { + Some(effectcraft_track::ConfidenceAction::Continue) => LowConfidence::Continue, + Some(effectcraft_track::ConfidenceAction::Stop) => LowConfidence::Stop, + Some(effectcraft_track::ConfidenceAction::Extrapolate) => LowConfidence::Extrapolate, + Some(effectcraft_track::ConfidenceAction::Adapt) => LowConfidence::Adapt, + None => return Err(bad("track.options", "action: continue|stop|extrapolate|adapt")), + }), + }; + let r = s.edit("Motion Tracker Options", None, |proj, _| { + let g = tracker_mut(proj, cid, lid, uid)?; + if let Some(n) = str_p(p, "name") { + g.name = n.to_string(); + } + let o = &mut g.tracker_settings_mut().ok_or_else(|| bad("track.options", "not a tracker"))?.options; + if let Some(c) = channel { + o.channel = c; + } + if let Some(b) = f_p(p, "blur") { + o.blur = b.clamp(0.0, 100.0); + } + if let Some(v) = b_p(p, "enhance") { + o.enhance = v; + } + if let Some(v) = b_p(p, "subpixel") { + o.subpixel = v; + } + if let Some(v) = b_p(p, "adaptEveryFrame") { + o.adapt_every_frame = v; + } + if let Some(v) = f_p(p, "threshold") { + o.threshold = v.clamp(0.0, 100.0); + } + if let Some(a) = action { + o.action = a; + } + if let Some(v) = b_p(p, "trackShape") { + o.track_shape = v; + } + Ok(serde_json::to_value(&*o).unwrap_or(V::Null)) + })?; + Ok(r) +} + +fn vec2_p(p: &V, k: &str) -> Option<[f64; 2]> { + let a = p.get(k)?.as_array()?; + Some([a.first()?.as_f64()?, a.get(1)?.as_f64()?]) +} + +fn set_point(s: &mut Session, p: &V) -> Result { + let (cid, lid, uid) = current(s, p, "track.setPoint")?; + let n = p.get("point").and_then(V::as_u64).unwrap_or(1).max(1) as usize; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let t = f_p(p, "time").map(Tick::from_seconds_f64).map(|t| comp.frame_rate.snap_nearest(t)).unwrap_or_else(|| s.time()); + let lt = layer.layer_time(t); + let merge = super::merge_p(p).map(str::to_string); + s.edit("Move Track Point", merge.as_deref(), |proj, _| { + let g = tracker_mut(proj, cid, lid, uid)?; + let tp_uid = g.track_points().nth(n - 1).map(|g| g.uid).ok_or_else(|| bad("track.setPoint", format!("no track point {n}")))?; + let tp = g.find_group_mut(tp_uid).ok_or_else(|| bad("track.setPoint", "no track point"))?; + let cur = point_values(tp, lt); + let mut center = vec2_p(p, "center").unwrap_or(cur.spec.center); + if let Some(d) = vec2_p(p, "move") { + center = [center[0] + d[0], center[1] + d[1]]; + } + let set = |tp: &mut PropGroup, m: &str, v: [f64; 2]| { + if let Some(pr) = tp.get_mut(m) { + pr.set_value_at(lt, Value::Vec2(v)); + } + }; + set(tp, "featureCenter", center); + for (k, m) in [("featureSize", "featureSize"), ("searchOffset", "searchOffset"), ("searchSize", "searchSize"), ("attachOffset", "attachPointOffset")] { + if let Some(v) = vec2_p(p, k) { + let v = if k.ends_with("Size") { [v[0].max(4.0), v[1].max(4.0)] } else { v }; + set(tp, m, v); + } + } + let off = tp.get("attachPointOffset").map(|pr| pr.value_at(lt).as_vec2()).unwrap_or([0.0; 2]); + set(tp, "attachPoint", [center[0] + off[0], center[1] + off[1]]); + Ok(()) + })?; + Ok(V::Null) +} + +// ---------- analyze / apply / reset ---------- + +fn analyze(s: &mut Session, p: &V) -> Result { + let (cid, lid, uid) = current(s, p, "track.analyze")?; + let dir = match str_p(p, "direction") { + Some(d) => Direction::from_name(d).ok_or_else(|| bad("track.analyze", "direction: forward|backward|frameForward|frameBackward"))?, + None => Direction::Forward, + }; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let (tg, settings) = layer.tracker(uid).ok_or_else(|| bad("track.analyze", "no tracker"))?; + let fd = comp.frame_duration(); + let now = comp.frame_rate.snap_nearest(if s.state.active_comp == Some(cid) { s.time() } else { Tick::ZERO }); + let start = f_p(p, "start").map(|v| comp.frame_rate.snap_nearest(Tick::from_seconds_f64(v))); + let end = f_p(p, "end").map(|v| comp.frame_rate.snap_nearest(Tick::from_seconds_f64(v))); + let lo = layer.in_point.max(Tick::ZERO); + let hi = layer.out_point.min(comp.duration) - fd; + let mut times = vec![]; + match dir { + Direction::Forward | Direction::FrameForward => { + let mut t = start.unwrap_or(now).clamp(lo, hi); + let stop = if dir == Direction::FrameForward { t + fd } else { end.unwrap_or(hi).min(hi) }; + while t <= stop && t <= hi { + times.push(t); + t += fd; + } + } + Direction::Backward | Direction::FrameBackward => { + let mut t = end.unwrap_or(now).clamp(lo, hi); + let stop = if dir == Direction::FrameBackward { t - fd } else { start.unwrap_or(lo).max(lo) }; + while t >= stop && t >= lo { + times.push(t); + t -= fd; + } + } + } + if times.len() < 2 { + return Err(bad("track.analyze", "nothing to analyze in that direction (at the layer's end)")); + } + let lt0 = layer.layer_time(times[0]); + let points: Vec<_> = tg.track_points().map(|g| point_values(g, lt0)).collect(); + if points.is_empty() { + return Err(bad("track.analyze", "the tracker has no track points")); + } + let work = Work { + project: s.project.clone(), + footage: s.footage.clone(), + expr: s.expr.clone(), + cache: s.layer_cache.clone(), + comp: cid, + layer: lid, + settings: settings.clone(), + points, + times: times.clone(), + }; + s.state.current_track = Some((lid, uid)); + let wait = b_p(p, "wait").unwrap_or(false); + s.start_track(work, uid, dir, wait).map_err(EngineError::Other)?; + let prog = s.track_progress(); + Ok(json!({ + "frames": times.len() - 1, + "start": times[0].seconds(), + "end": times.last().map(|t| t.seconds()), + "running": s.is_tracking(), + "progress": prog, + })) +} + +fn apply(s: &mut Session, p: &V) -> Result { + let (cid, lid, uid) = current(s, p, "track.apply")?; + let dims = match str_p(p, "dimensions") { + Some(d) => Dims::from_name(d).ok_or_else(|| bad("track.apply", "dimensions: xy|x|y"))?, + None => Dims::XY, + }; + if s.is_tracking() { + return Err(bad("track.apply", "wait for the analysis to finish")); + } + let n = s.edit("Apply Track", None, |proj, _| crate::tracking::apply(proj, cid, lid, uid, dims).map_err(|e| bad("track.apply", e)))?; + Ok(json!({"frames": n})) +} + +fn reset(s: &mut Session, p: &V) -> Result { + let (cid, lid, uid) = current(s, p, "track.reset")?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let size = layer_size(s, cid, layer); + s.edit("Reset Track", None, |proj, _| { + let g = tracker_mut(proj, cid, lid, uid)?; + let st = g.tracker_settings().cloned().unwrap_or_default(); + let n = g.track_points().count(); + let defaults = tracking::default_points(st.kind, n, size); + let mut k = 0; + for c in &mut g.children { + let Node::Group(tp) = c else { continue }; + if tp.match_id != tracking::TRACK_POINT { + continue; + } + let d = defaults.get(k).copied().unwrap_or(defaults[0]); + k += 1; + for (m, v) in [ + ("featureCenter", Value::Vec2(d.center)), + ("featureSize", Value::Vec2(d.feature_size)), + ("searchOffset", Value::Vec2(d.search_offset)), + ("searchSize", Value::Vec2(d.search_size)), + ("confidence", Value::Scalar(0.0)), + ("attachPoint", Value::Vec2(d.center)), + ("attachPointOffset", Value::Vec2(d.attach_offset)), + ] { + if let Some(pr) = tp.get_mut(m) { + pr.keys.clear(); + pr.value = v; + } + } + } + Ok(()) + })?; + Ok(V::Null) +} + +fn delete(s: &mut Session, p: &V) -> Result { + let (cid, lid, uid) = current(s, p, "track.delete")?; + s.edit("Delete Tracker", None, |proj, st| { + let l = layer_mut(proj, cid, lid)?; + if let Some(mt) = l.props.sub_mut(tracking::MOTION_TRACKERS) { + mt.children.retain(|c| c.uid() != uid); + if mt.children.is_empty() { + l.props.children.retain(|c| c.match_id() != tracking::MOTION_TRACKERS); + } + } + if st.current_track == Some((lid, uid)) { + st.current_track = None; + } + Ok(()) + })?; + Ok(V::Null) +} + +fn status(s: &mut Session, p: &V) -> Result { + s.poll_track(); + let progress = s.track_progress(); + let cur = current(s, p, "track.status").ok(); + let mut out = json!({"running": s.is_tracking(), "progress": progress}); + if let Some((cid, lid, uid)) = cur + && let Some(layer) = s.project.comp(cid).and_then(|c| c.layer(lid)) + && let Some((g, st)) = layer.tracker(uid) + { + let points: Vec = g + .track_points() + .map(|tp| { + let keys = tp.get("featureCenter").map(|p| p.keys.len()).unwrap_or(0); + let v = point_values(tp, layer.layer_time(s.time())); + json!({"name": tp.name, "uid": tp.uid, "center": v.spec.center, "featureSize": v.spec.feature_size, "searchOffset": v.spec.search_offset, "searchSize": v.spec.search_size, "attachOffset": v.attach_offset, "keys": keys}) + }) + .collect(); + let trackers: Vec = layer.trackers().map(|(g, _)| json!({"uid": g.uid, "name": g.name})).collect(); + out["layer"] = json!(lid.0); + out["tracker"] = json!({"uid": uid, "name": g.name, "settings": st, "points": points}); + out["trackers"] = json!(trackers); + } + Ok(out) +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 78c63947..fd3af5d3 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -13,6 +13,7 @@ pub mod demo; pub mod links; pub mod menus; pub mod render_queue; +pub mod tracking; use std::sync::Arc; @@ -157,6 +158,9 @@ pub struct EditorState { /// Viewer 3D view per comp (Active Camera / Front / … / Custom View 3 and edited view cameras). #[serde(default)] pub views3d: std::collections::BTreeMap, + /// Tracker panel ▸ Current Track: (tracked layer, tracker group uid) in the active comp. + #[serde(default)] + pub current_track: Option<(LayerId, Uid)>, } /// What Copy with Property Links / Copy Expression Only put on the clipboard. @@ -207,6 +211,8 @@ pub struct Session { pub exporter: Option>, /// The running (or finished, not yet polled) render. pub render_job: Option, + /// The running (or finished, not yet polled) track analysis. + pub track_job: Option, pub events: Vec, /// Commands executed: (id, params). pub journal: Vec<(String, Value)>, @@ -230,6 +236,7 @@ impl Default for Session { importer: None, exporter: None, render_job: None, + track_job: None, events: vec![], journal: vec![], layer_cache: Arc::new(LayerCache::default()), @@ -336,6 +343,11 @@ impl Session { self.state.selected_keys.retain(|k| ids.contains(&k.layer)); self.state.selected_vertices.retain(|v| ids.contains(&v.layer)); } + if let Some((l, t)) = self.state.current_track + && self.active_comp().and_then(|c| c.layer(l)).and_then(|l| l.tracker(t)).is_none() + { + self.state.current_track = None; + } self.state.project_selection.retain(|i| p.item(*i).is_some()); } @@ -380,6 +392,7 @@ impl Session { self.state.selected_props.clear(); self.state.selected_keys.clear(); self.state.selected_vertices.clear(); + self.state.current_track = None; } self.events.push(Event::OpenComp(id)); } @@ -437,6 +450,8 @@ impl Session { job.wait(); } self.render_job = None; + self.stop_track(); + self.track_job = None; p.fix_next_id(); self.project = Arc::new(p); self.history = History::default(); @@ -467,6 +482,8 @@ mod tests_styles; mod tests_text; #[cfg(test)] mod tests_timeline; +#[cfg(test)] +mod tests_track; /// Font families available to text layers (bundled + scanned system fonts). /// Text animation presets: (id, name), for menus. diff --git a/crates/engine/src/menus.rs b/crates/engine/src/menus.rs index 8965da25..dc641003 100644 --- a/crates/engine/src/menus.rs +++ b/crates/engine/src/menus.rs @@ -788,6 +788,7 @@ Animation Track Camera | track.camera Warp Stabilizer VFX | track.warpStabilizer Track Motion | track.motion + Stabilize Motion | track.stabilize Track Mask | track.mask Track this Property | track.property --- diff --git a/crates/engine/src/tests_track.rs b/crates/engine/src/tests_track.rs new file mode 100644 index 00000000..b3c2a174 --- /dev/null +++ b/crates/engine/src/tests_track.rs @@ -0,0 +1,431 @@ +//! Motion tracking through the commands: synthetic footage generated in-test (a textured patch on +//! a known path, or a whole frame under a known homography), analysis (blocking and background), +//! apply (Transform, Stabilize, Corner Pin), undo/redo and serde. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use effectcraft_color::Label; +use effectcraft_project::build; +use effectcraft_project::tracking::TrackKind; +use effectcraft_project::{Footage, FootageKind, ItemId, ItemKind, LayerId, LayerSource}; +use effectcraft_raster::Image; +use effectcraft_render::{EvalCtx, FootageSource}; +use effectcraft_time::{FrameRate, Tick}; +use effectcraft_track::Homography; +use serde_json::json; + +use crate::Session; + +const W: u32 = 320; +const H: u32 = 240; +const FPS: u32 = 25; + +fn texture(x: f64, y: f64, seed: u32) -> f32 { + let mut v = 0.0f64; + let mut s = seed.wrapping_mul(2654435761).wrapping_add(12345); + let mut rnd = || { + s ^= s << 13; + s ^= s >> 17; + s ^= s << 5; + (s % 10_000) as f64 / 10_000.0 + }; + for _ in 0..6 { + let ang = rnd() * std::f64::consts::TAU; + let period = 9.0 + rnd() * 25.0; + let phase = rnd() * std::f64::consts::TAU; + let k = std::f64::consts::TAU / period; + v += (k * (x * ang.cos() + y * ang.sin()) + phase).sin() * (0.4 + rnd()) * 0.1; + } + (0.5 + v).clamp(0.0, 1.0) as f32 +} + +/// Footage computed per frame (cached). +struct Synth { + make: Box Image + Send + Sync>, + cache: Mutex>>, +} + +impl FootageSource for Synth { + fn frame(&self, _: ItemId, _: &Footage, t: Tick) -> Option> { + let f = (t.seconds() * FPS as f64).round().max(0.0) as u32; + let mut c = self.cache.lock().unwrap(); + Some(c.entry(f).or_insert_with(|| Arc::new((self.make)(f))).clone()) + } +} + +/// Patch pose on frame `f`: centre, rotation (degrees), scale. +type Pose = fn(u32) -> ([f64; 2], f64, f64); + +fn patch_frames(pose: Pose) -> impl Fn(u32) -> Image + Send + Sync { + move |f| { + let (c, rot, sc) = pose(f); + let (sn, cs) = rot.to_radians().sin_cos(); + let mut img = Image::new(W, H); + for y in 0..H { + for x in 0..W { + let p = [x as f64 + 0.5, y as f64 + 0.5]; + let d = [(p[0] - c[0]) / sc, (p[1] - c[1]) / sc]; + let l = [cs * d[0] + sn * d[1], -sn * d[0] + cs * d[1]]; + let v = if l[0].abs() <= 50.0 && l[1].abs() <= 50.0 { texture(l[0], l[1], 7) } else { 0.6 * texture(p[0] * 0.7, p[1] * 0.7, 3) }; + img.set(x, y, [v, v, v, 1.0]); + } + } + img + } +} + +/// A comp with the synthetic clip (layer `clip`) and a 100 × 100 solid above it (layer `solid`). +fn setup(make: impl Fn(u32) -> Image + Send + Sync + 'static) -> (Session, LayerId, LayerId) { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Track", "width": W, "height": H, "frameRate": FPS, "duration": 1})).unwrap(); + let cid = s.active_comp_id().unwrap(); + let clip = s + .edit("clip", None, |p, _| { + let fr = FrameRate::new(FPS as i64, 1); + let footage = Footage { + path: "synthetic.mov".into(), + kind: FootageKind::Video, + width: W, + height: H, + pixel_aspect: 1.0, + frame_rate: fr, + native_rate: None, + duration: Tick::from_seconds_f64(1.0), + has_video: true, + has_audio: false, + alpha: Default::default(), + premul_color: [0.0; 3], + loop_count: 1, + codec: String::new(), + missing: false, + sequence: vec![], + }; + let item = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(footage)); + let comp = p.comp(cid).unwrap().clone(); + let l = build::layer(p, &comp, "Clip", LayerSource::Footage { item }, (W, H), None); + let id = l.id; + p.comp_mut(cid).unwrap().layers.insert(0, l); + Ok(id) + }) + .unwrap(); + s.footage = Arc::new(Synth { make: Box::new(make), cache: Mutex::default() }); + let solid = s.execute("layer.newSolid", json!({"width": 100, "height": 100, "color": "#ff0000"})).unwrap()["layer"].as_u64().unwrap(); + s.execute("layer.select", json!({"layers": [clip.0]})).unwrap(); + (s, clip, LayerId(solid)) +} + +fn tracker_points(s: &Session, layer: LayerId) -> Vec { + let l = s.active_comp().unwrap().layer(layer).unwrap(); + l.trackers().next().unwrap().0.track_points().cloned().collect() +} + +fn frame_time(f: u32) -> Tick { + FrameRate::new(FPS as i64, 1).tick_of(f as i64) +} + +fn dist(a: [f64; 2], b: [f64; 2]) -> f64 { + ((a[0] - b[0]).powi(2) + (a[1] - b[1]).powi(2)).sqrt() +} + +fn translate_pose(f: u32) -> ([f64; 2], f64, f64) { + let t = f as f64; + ([150.0 + 2.37 * t + 4.0 * (t * 0.4).sin(), 110.0 + 1.13 * t], 0.0, 1.0) +} + +#[test] +fn track_motion_analyze_apply_transform_undo_redo() { + let (mut s, clip, solid) = setup(patch_frames(translate_pose)); + // The menu entry and the Tracker panel's Track Motion button. + assert!(s.is_enabled("track.motion")); + let r = s.execute("track.motion", json!({})).unwrap(); + assert_eq!(r["points"], 1); + let st = s.execute("track.status", json!({})).unwrap(); + assert_eq!(st["tracker"]["settings"]["target"], json!(solid.0), "{st}"); + // Put the track point on the patch and analyze to the end (blocking). + s.execute("track.setPoint", json!({"point": 1, "center": translate_pose(0).0, "featureSize": [36, 36], "searchSize": [80, 80]})).unwrap(); + let r = s.execute("track.analyze", json!({"direction": "forward", "wait": true})).unwrap(); + assert_eq!(r["frames"], 24); + s.poll_track(); + assert!(s.track_job.is_none()); + let tp = &tracker_points(&s, clip)[0]; + let fc = tp.get("featureCenter").unwrap(); + assert_eq!(fc.keys.len(), 25); + let conf = tp.get("confidence").unwrap(); + let mut worst: f64 = 0.0; + for f in 0..25 { + let t = frame_time(f); + worst = worst.max(dist(fc.value_at(t).as_vec2(), translate_pose(f).0)); + assert!(conf.value_at(t).as_f64() > 90.0); + } + assert!(worst < 0.25, "worst {worst}"); + // CTI moved to the last analysed frame. + assert_eq!(s.time(), frame_time(24)); + // Apply: the solid's position follows the attach point. + s.execute("track.apply", json!({})).unwrap(); + let pos = s.active_comp().unwrap().layer(solid).unwrap().props.prop("transform/position").unwrap().clone(); + assert_eq!(pos.keys.len(), 25); + for f in [0, 7, 24] { + let v = pos.value_at(frame_time(f)).as_vec2(); + assert!(dist(v, translate_pose(f).0) < 0.25, "frame {f}: {v:?}"); + } + // Undo the apply, then the analysis; redo both. + assert!(s.undo()); + assert!(s.active_comp().unwrap().layer(solid).unwrap().props.prop("transform/position").unwrap().keys.is_empty()); + assert!(s.undo()); + assert!(tracker_points(&s, clip)[0].get("featureCenter").unwrap().keys.is_empty()); + assert!(s.redo()); + assert_eq!(tracker_points(&s, clip)[0].get("featureCenter").unwrap().keys.len(), 25); + assert!(s.redo()); + assert_eq!(s.active_comp().unwrap().layer(solid).unwrap().props.prop("transform/position").unwrap().keys.len(), 25); + // X only keeps the solid's Y. + s.undo(); + s.execute("track.apply", json!({"dimensions": "x"})).unwrap(); + let pos = s.active_comp().unwrap().layer(solid).unwrap().props.prop("transform/position").unwrap().clone(); + let v = pos.value_at(frame_time(10)).as_vec2(); + assert!((v[0] - translate_pose(10).0[0]).abs() < 0.25 && (v[1] - 120.0).abs() < 1e-9, "{v:?}"); + // Serde round trip keeps the tracker. + let json = s.project.to_json(); + let back = effectcraft_project::Project::from_json(&json).unwrap(); + let l = back.comp(s.active_comp_id().unwrap()).unwrap().layer(clip).unwrap(); + let (g, st) = l.trackers().next().unwrap(); + assert_eq!(st.kind, TrackKind::Transform); + assert_eq!(g.track_points().next().unwrap().get("featureCenter").unwrap().keys.len(), 25); + assert_eq!(l.props.children[0].match_id(), "motionTrackers"); +} + +#[test] +fn stabilize_keeps_the_feature_fixed() { + let (mut s, clip, _) = setup(patch_frames(translate_pose)); + s.execute("track.stabilize", json!({})).unwrap(); + s.execute("track.setPoint", json!({"point": 1, "center": translate_pose(0).0, "featureSize": [36, 36], "searchSize": [80, 80]})).unwrap(); + s.execute("time.set", json!({"time": 0})).unwrap(); + s.execute("track.analyze", json!({"wait": true})).unwrap(); + s.execute("track.apply", json!({})).unwrap(); + let cid = s.active_comp_id().unwrap(); + let comp = s.project.comp(cid).unwrap().clone(); + let l = comp.layer(clip).unwrap(); + let at = |f: u32| { + let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(f)); + let (m, _) = ctx.layer_to_comp(l); + let c = translate_pose(f).0; + let q = m.apply(effectcraft_geom::vec2(c[0], c[1])); + [q.x, q.y] + }; + let p0 = at(0); + // The layer itself didn't move on the first frame. + assert!(dist(p0, translate_pose(0).0) < 1e-6, "{p0:?}"); + let worst = (1..25).map(|f| dist(at(f), p0)).fold(0.0, f64::max); + assert!(worst < 0.25, "feature drifts by {worst} px"); +} + +fn rotating_pose(f: u32) -> ([f64; 2], f64, f64) { + let t = f as f64; + ([160.0 + 1.2 * t, 120.0 - 0.7 * t], 1.5 * t, 1.0 + 0.008 * t) +} + +#[test] +fn transform_rotation_scale_and_background_job() { + let (mut s, clip, solid) = setup(patch_frames(rotating_pose)); + s.execute("track.new", json!({"kind": "transform", "rotation": true, "scale": true})).unwrap(); + assert_eq!(tracker_points(&s, clip).len(), 2); + let (c, _, _) = rotating_pose(0); + s.execute("track.setPoint", json!({"point": 1, "center": [c[0] - 26.0, c[1]], "featureSize": [26, 26], "searchSize": [56, 56]})).unwrap(); + s.execute("track.setPoint", json!({"point": 2, "center": [c[0] + 26.0, c[1]], "featureSize": [26, 26], "searchSize": [56, 56]})).unwrap(); + // Background analysis with progress, polled like the UI does. + let r = s.execute("track.analyze", json!({"direction": "forward", "end": 0.6})).unwrap(); + assert_eq!(r["frames"], 15); + let t0 = std::time::Instant::now(); + while s.is_tracking() { + assert!(t0.elapsed().as_secs() < 120); + std::thread::sleep(std::time::Duration::from_millis(5)); + s.poll_track(); + } + s.poll_track(); + let st = s.execute("track.status", json!({})).unwrap(); + assert_eq!(st["running"], false); + assert_eq!(st["tracker"]["points"][0]["keys"], 16); + s.execute("track.apply", json!({})).unwrap(); + let tr = s.active_comp().unwrap().layer(solid).unwrap().props.sub("transform").unwrap().clone(); + let rot = tr.get("rotation").unwrap(); + let sc = tr.get("scale").unwrap(); + for f in [5, 10, 15] { + let (_, r, k) = rotating_pose(f); + let got = rot.value_at(frame_time(f)).as_f64(); + assert!((got - r).abs() < 0.3, "frame {f}: rotation {got} vs {r}"); + let gs = sc.value_at(frame_time(f)).as_vec3()[0]; + assert!((gs - 100.0 * k).abs() < 0.5, "frame {f}: scale {gs} vs {}", 100.0 * k); + } +} + +#[test] +fn perspective_corner_pin_follows_a_known_homography() { + // Frame f shows the texture under H_f (identity → `end` over 12 frames). + let end = Homography([[1.04, 0.05, 6.0], [-0.03, 0.97, 4.0], [0.00012, -0.00008, 1.0]]); + let hf = move |f: u32| { + let k = f as f64 / 12.0; + let mut m = [[0.0; 3]; 3]; + for i in 0..3 { + for j in 0..3 { + let id = if i == j { 1.0 } else { 0.0 }; + m[i][j] = id + (end.0[i][j] - id) * k; + } + } + Homography(m) + }; + let inv = move |f: u32, p: [f64; 2]| { + let h = hf(f); + // Solve H x = p by mapping four points back (exact for a homography). + let src = [[0.0, 0.0], [W as f64, 0.0], [0.0, H as f64], [W as f64, H as f64]]; + let dst = src.map(|q| h.apply(q)); + Homography::from_points(dst, src).unwrap().apply(p) + }; + let make = move |f: u32| { + let h = hf(f); + let src = [[0.0, 0.0], [W as f64, 0.0], [0.0, H as f64], [W as f64, H as f64]]; + let back = Homography::from_points(src.map(|q| h.apply(q)), src).unwrap(); + let _ = inv; + let mut img = Image::new(W, H); + for y in 0..H { + for x in 0..W { + let q = back.apply([x as f64 + 0.5, y as f64 + 0.5]); + let v = texture(q[0], q[1], 11); + img.set(x, y, [v, v, v, 1.0]); + } + } + img + }; + let (mut s, clip, solid) = setup(make); + s.execute("track.new", json!({"kind": "perspective"})).unwrap(); + let corners = [[90.0, 70.0], [230.0, 70.0], [90.0, 170.0], [230.0, 170.0]]; + for (i, c) in corners.iter().enumerate() { + s.execute("track.setPoint", json!({"point": i + 1, "center": c, "featureSize": [30, 30], "searchSize": [60, 60]})).unwrap(); + } + s.execute("track.analyze", json!({"wait": true, "end": 12.0 / FPS as f64})).unwrap(); + let pts = tracker_points(&s, clip); + for (i, tp) in pts.iter().enumerate() { + let got = tp.get("featureCenter").unwrap().value_at(frame_time(12)).as_vec2(); + assert!(dist(got, end.apply(corners[i])) < 0.3, "point {i}: {got:?} vs {:?}", end.apply(corners[i])); + } + s.execute("track.apply", json!({})).unwrap(); + let l = s.active_comp().unwrap().layer(solid).unwrap().clone(); + let fx = l.effects().unwrap().groups().next().unwrap().clone(); + assert_eq!(fx.match_id, "ec.distort.cornerpin"); + // The solid is 100 × 100, centred: comp = layer + (110, 70). + for (m, i) in [("ul", 0), ("ur", 1), ("ll", 2), ("lr", 3)] { + let p = fx.get(m).unwrap(); + assert_eq!(p.keys.len(), 13); + for f in [6, 12] { + let want = hf(f).apply(corners[i]); + let got = p.value_at(frame_time(f)).as_vec2(); + assert!(dist([got[0] + 110.0, got[1] + 70.0], want) < 0.35, "{m} frame {f}: {got:?} vs {want:?}"); + } + } +} + +#[test] +fn backward_frame_steps_options_and_reset() { + let (mut s, clip, _) = setup(patch_frames(translate_pose)); + s.execute("track.motion", json!({})).unwrap(); + s.execute("time.set", json!({"time": 10.0 / FPS as f64})).unwrap(); + s.execute("track.setPoint", json!({"point": 1, "center": translate_pose(10).0, "featureSize": [36, 36], "searchSize": [80, 80]})).unwrap(); + s.execute("track.options", json!({"channel": "rgb", "blur": 2, "threshold": 70, "action": "extrapolate", "name": "Patch"})).unwrap(); + let st = s.execute("track.status", json!({})).unwrap(); + assert_eq!(st["tracker"]["name"], "Patch"); + assert_eq!(st["tracker"]["settings"]["options"]["channel"], "rgb"); + s.execute("track.analyze", json!({"direction": "frameBackward", "wait": true})).unwrap(); + s.execute("track.analyze", json!({"direction": "backward", "wait": true})).unwrap(); + assert_eq!(s.time(), Tick::ZERO); + s.execute("track.analyze", json!({"direction": "frameForward", "wait": true, "start": 10.0 / FPS as f64})).unwrap(); + let fc = tracker_points(&s, clip)[0].get("featureCenter").unwrap().clone(); + assert_eq!(fc.keys.len(), 12); + for f in 0..12 { + assert!(dist(fc.value_at(frame_time(f)).as_vec2(), translate_pose(f).0) < 0.25, "frame {f}"); + } + // Unknown params are rejected for agents. + assert!(s.execute_checked("track.analyze", json!({"dir": "forward"})).is_err()); + s.execute("track.setType", json!({"kind": "perspective"})).unwrap(); + assert_eq!(tracker_points(&s, clip).len(), 4); + s.execute("track.setType", json!({"kind": "transform", "rotation": false})).unwrap(); + assert_eq!(tracker_points(&s, clip).len(), 1); + s.execute("track.reset", json!({})).unwrap(); + assert!(tracker_points(&s, clip)[0].get("featureCenter").unwrap().keys.is_empty()); + s.execute("track.delete", json!({})).unwrap(); + assert!(s.active_comp().unwrap().layer(clip).unwrap().motion_trackers().is_none()); + assert!(s.execute("track.analyze", json!({})).is_err()); +} + +/// End-to-end analysis speed at 1080p for one track point: footage frames through the renderer's +/// layer source and the tracker (frames are prepared before timing). +#[test] +fn analyze_1080p_single_point_speed() { + let (w, h) = (1920u32, 1080u32); + let mut base = Image::new(w + 64, h + 64); + for y in 0..base.height { + for x in 0..base.width { + let v = 0.5 + 0.25 * ((x as f32 * 0.21).sin() * (y as f32 * 0.13).cos()) + 0.15 * ((x + 2 * y) as f32 * 0.07).sin(); + base.set(x, y, [v, v, v, 1.0]); + } + } + let frames: Vec> = (0..31u32) + .map(|f| { + let (dx, dy) = (f as i64, (f / 2) as i64); + let mut img = Image::new(w, h); + for y in 0..h { + for x in 0..w { + img.set(x, y, base.get(x as i64 + 32 - dx, y as i64 + 32 - dy)); + } + } + Arc::new(img) + }) + .collect(); + struct Pre(Vec>); + impl FootageSource for Pre { + fn frame(&self, _: ItemId, _: &Footage, t: Tick) -> Option> { + let f = (t.seconds() * FPS as f64).round().max(0.0) as usize; + self.0.get(f.min(self.0.len() - 1)).cloned() + } + } + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "HD", "width": w, "height": h, "frameRate": FPS, "duration": 2})).unwrap(); + let cid = s.active_comp_id().unwrap(); + let clip = s + .edit("clip", None, |p, _| { + let footage = Footage { + path: "hd.mov".into(), + kind: FootageKind::Video, + width: w, + height: h, + pixel_aspect: 1.0, + frame_rate: FrameRate::new(FPS as i64, 1), + native_rate: None, + duration: Tick::from_seconds_f64(2.0), + has_video: true, + has_audio: false, + alpha: Default::default(), + premul_color: [0.0; 3], + loop_count: 1, + codec: String::new(), + missing: false, + sequence: vec![], + }; + let item = p.add_item("hd", Label::Aqua, None, ItemKind::Footage(footage)); + let comp = p.comp(cid).unwrap().clone(); + let l = build::layer(p, &comp, "HD", LayerSource::Footage { item }, (w, h), None); + let id = l.id; + p.comp_mut(cid).unwrap().layers.insert(0, l); + Ok(id) + }) + .unwrap(); + s.footage = Arc::new(Pre(frames)); + s.execute("track.motion", json!({"layer": clip.0})).unwrap(); + s.execute("track.setPoint", json!({"point": 1, "center": [960, 540], "featureSize": [48, 48], "searchSize": [128, 128]})).unwrap(); + let t0 = std::time::Instant::now(); + s.execute("track.analyze", json!({"wait": true, "end": 30.0 / FPS as f64})).unwrap(); + let secs = t0.elapsed().as_secs_f64(); + let fps = 30.0 / secs; + eprintln!("1080p single-point analysis: {fps:.0} frames/s end to end ({:.1} ms/frame)", secs * 1e3 / 30.0); + let fc = tracker_points(&s, clip)[0].get("featureCenter").unwrap().clone(); + assert!(dist(fc.value_at(frame_time(30)).as_vec2(), [990.0, 555.0]) < 0.25); + assert!(fps > 30.0, "{fps} frames/s"); +} diff --git a/crates/engine/src/tracking.rs b/crates/engine/src/tracking.rs new file mode 100644 index 00000000..43e24cb8 --- /dev/null +++ b/crates/engine/src/tracking.rs @@ -0,0 +1,607 @@ +//! Motion tracking runtime: the analysis job (background thread with progress and cancel, or +//! blocking for the CLI / agents / wasm), writing track results into the tracker's Track Point +//! properties, and applying tracks (Transform, Stabilize, Corner Pin) to layers. +//! +//! The algorithms are in `effectcraft-track`; the property layout in +//! `effectcraft_project::tracking`. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use effectcraft_geom::{Mat3, vec2}; +use effectcraft_project::tracking::{LowConfidence, TrackChannel, TrackKind, TrackerOptions, TrackerSettings}; +use effectcraft_project::{Comp, ItemId, Keyframe, Layer, LayerId, Project, PropGroup, Uid, Value}; +use effectcraft_raster::Image; +use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; +use effectcraft_time::Tick; +use effectcraft_track as trk; +use serde::{Deserialize, Serialize}; + +use crate::{Event, Session}; + +/// Tracker panel ▸ Analyze buttons. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Direction { + #[default] + Forward, + Backward, + FrameForward, + FrameBackward, +} + +impl Direction { + pub fn from_name(s: &str) -> Option { + let n = s.to_ascii_lowercase().replace([' ', '_', '-'], ""); + Some(match n.as_str() { + "forward" | "fwd" => Direction::Forward, + "backward" | "back" | "bwd" => Direction::Backward, + "frameforward" | "oneframeforward" | "1frameforward" => Direction::FrameForward, + "framebackward" | "oneframebackward" | "1framebackward" => Direction::FrameBackward, + _ => return None, + }) + } + pub fn forward(self) -> bool { + matches!(self, Direction::Forward | Direction::FrameForward) + } +} + +/// Track options for the algorithms. +pub fn track_options(o: &TrackerOptions) -> trk::TrackOptions { + trk::TrackOptions { + channel: match o.channel { + TrackChannel::Rgb => trk::Channel::Rgb, + TrackChannel::Luminance => trk::Channel::Luminance, + TrackChannel::Saturation => trk::Channel::Saturation, + }, + blur: o.blur, + enhance: o.enhance, + subpixel: o.subpixel, + adapt_every_frame: o.adapt_every_frame, + threshold: o.threshold, + action: match o.action { + LowConfidence::Continue => trk::ConfidenceAction::Continue, + LowConfidence::Stop => trk::ConfidenceAction::Stop, + LowConfidence::Extrapolate => trk::ConfidenceAction::Extrapolate, + LowConfidence::Adapt => trk::ConfidenceAction::Adapt, + }, + track_shape: o.track_shape, + } +} + +/// A track point's regions and attach offset read from its properties at a layer time. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct PointValues { + pub uid: Uid, + pub spec: trk::PointSpec, + pub attach_offset: [f64; 2], +} + +pub fn point_values(tp: &PropGroup, lt: Tick) -> PointValues { + let v = |m: &str, d: [f64; 2]| tp.get(m).map(|p| p.value_at(lt).as_vec2()).unwrap_or(d); + PointValues { + uid: tp.uid, + spec: trk::PointSpec { + center: v("featureCenter", [0.0; 2]), + feature_size: v("featureSize", [32.0; 2]), + search_offset: v("searchOffset", [0.0; 2]), + search_size: v("searchSize", [64.0; 2]), + }, + attach_offset: v("attachPointOffset", [0.0; 2]), + } +} + +/// Attach points of a frame from the feature centres, given the start frame's centres and attach +/// points: two-point transforms rotate and scale the offsets with the track, Parallel Corner Pin +/// maps them through the 3-point affine, Perspective Corner Pin through the 4-point homography. +pub fn attach_points(kind: TrackKind, rot_scale: bool, ref_c: &[[f64; 2]], ref_a: &[[f64; 2]], cur: &[[f64; 2]]) -> Vec<[f64; 2]> { + let translate = || cur.iter().zip(ref_c.iter().zip(ref_a)).map(|(c, (r, a))| [c[0] + a[0] - r[0], c[1] + a[1] - r[1]]).collect::>(); + let through = |h: Option| match h { + Some(h) => ref_a.iter().map(|a| h.apply(*a)).collect(), + None => translate(), + }; + match kind { + TrackKind::Perspective if cur.len() >= 4 => { + let h = trk::Homography::from_points([ref_c[0], ref_c[1], ref_c[2], ref_c[3]], [cur[0], cur[1], cur[2], cur[3]]); + through(h) + } + TrackKind::Affine if cur.len() >= 3 => through(trk::Homography::affine_from_points([ref_c[0], ref_c[1], ref_c[2]], [cur[0], cur[1], cur[2]])), + _ if rot_scale && cur.len() >= 2 => { + let Some(s) = trk::Homography::similarity_from_points([ref_c[0], ref_c[1]], [cur[0], cur[1]]) else { return translate() }; + let m = s.0; + cur.iter() + .zip(ref_c.iter().zip(ref_a)) + .map(|(c, (r, a))| { + let d = [a[0] - r[0], a[1] - r[1]]; + [c[0] + m[0][0] * d[0] + m[0][1] * d[1], c[1] + m[1][0] * d[0] + m[1][1] * d[1]] + }) + .collect() + } + _ => translate(), + } +} + +/// Live analysis progress (serde for agents / the control channel). +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct TrackProgress { + pub done: u64, + pub total: u64, + pub elapsed: f64, + /// Analysed frames per second. + pub fps: f64, + pub finished: bool, + pub cancelled: bool, + /// Stopped by "If Confidence is Below … ▸ Stop Tracking". + pub low_confidence_stop: bool, + pub error: Option, + /// Comp time (seconds) of the last analysed frame. + pub time: f64, +} + +/// One analysed frame. +#[derive(Clone, Debug)] +pub struct FrameResult { + pub comp_time: Tick, + pub layer_time: Tick, + /// Per point: (point group uid, result, attach point). + pub points: Vec<(Uid, trk::PointResult, [f64; 2])>, +} + +#[derive(Default)] +pub struct TrackShared { + pub cancel: AtomicBool, + pub state: Mutex, + pub frames: Mutex>, +} + +fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, T> { + m.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// A running (or finished, not yet polled) analysis. +pub struct TrackJob { + pub shared: Arc, + thread: Option>, + pub comp: ItemId, + pub layer: LayerId, + pub tracker: Uid, + pub direction: Direction, +} + +impl TrackJob { + pub fn is_finished(&self) -> bool { + lock(&self.shared.state).finished + } + pub fn progress(&self) -> TrackProgress { + lock(&self.shared.state).clone() + } + pub fn cancel(&self) { + self.shared.cancel.store(true, Ordering::Relaxed); + } +} + +/// What the worker needs, detached from the session. +pub(crate) struct Work { + pub project: Arc, + pub footage: Arc, + pub expr: Option>, + pub cache: Arc, + pub comp: ItemId, + pub layer: LayerId, + pub settings: TrackerSettings, + pub points: Vec, + /// Comp times to analyse; the first is the start frame. + pub times: Vec, +} + +/// The layer's source frame at comp time `t` at 100 % (layer pixels + offset = image pixels). +/// Footage at its native size is used straight from the footage source (no copy). +fn source_frame( + r: &Renderer, + project: &Project, + cid: ItemId, + comp: &Comp, + layer: &Layer, + t: Tick, + expr: Option<&dyn ExprHost>, +) -> Option<(Arc, [f64; 2])> { + let ctx = EvalCtx { project, comp_id: cid, comp, time: t, expr }; + if let effectcraft_project::LayerSource::Footage { item } = &layer.source + && let Some(effectcraft_project::ItemKind::Footage(f)) = project.item(*item).map(|i| &i.kind) + && f.has_video + && let Some(img) = r.footage.frame(*item, f, ctx.source_time(layer)) + && img.width == f.width + && img.height == f.height + { + return Some((img, [0.0; 2])); + } + let buf = r.layer_source(&ctx, layer)?; + if (buf.scale - 1.0).abs() > 1e-9 && buf.scale > 0.0 { + // Footage decoded at another size: bring it to layer pixels. + let w = (buf.img.width as f64 / buf.scale).round().max(1.0) as u32; + let h = (buf.img.height as f64 / buf.scale).round().max(1.0) as u32; + let off = [buf.offset[0] / buf.scale, buf.offset[1] / buf.scale]; + return Some((Arc::new(effectcraft_raster::resample(&buf.img, w, h)), off)); + } + Some((Arc::new(buf.img), buf.offset)) +} + +pub(crate) fn run_work(w: Work, shared: &TrackShared) { + let t0 = web_time::Instant::now(); + let fail = |msg: &str| { + let mut s = lock(&shared.state); + s.error = Some(msg.to_string()); + s.finished = true; + }; + let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") }; + let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") }; + let mut r = Renderer::new(&w.project, w.footage.as_ref(), RenderOpts { scale: 1.0, ..Default::default() }); + r.expr = w.expr.as_deref(); + r.cache = Some(&w.cache); + let r = &r; + let frame_at = |t: Tick| source_frame(r, &w.project, w.comp, comp, layer, t, w.expr.as_deref()); + { + let mut s = lock(&shared.state); + s.total = w.times.len().saturating_sub(1) as u64; + } + let Some(first) = frame_at(w.times[0]) else { return fail("the layer has no pixels to track") }; + let specs: Vec = w.points.iter().map(|p| p.spec).collect(); + let mut tracker = trk::Tracker::new(track_options(&w.settings.options), &specs, &trk::Frame { img: &first.0, offset: first.1 }); + let ref_c: Vec<[f64; 2]> = specs.iter().map(|s| s.center).collect(); + let ref_a: Vec<[f64; 2]> = w.points.iter().map(|p| [p.spec.center[0] + p.attach_offset[0], p.spec.center[1] + p.attach_offset[1]]).collect(); + let rot_scale = w.settings.rotation || w.settings.scale; + // The start frame keeps its regions and gets keys too. + lock(&shared.frames).push(FrameResult { + comp_time: w.times[0], + layer_time: layer.layer_time(w.times[0]), + points: w + .points + .iter() + .zip(&ref_a) + .map(|(p, a)| (p.uid, trk::PointResult { center: p.spec.center, confidence: 100.0, rotation: 0.0, scale: 1.0, status: trk::Status::Tracked }, *a)) + .collect(), + }); + let mut next = if w.times.len() > 1 { frame_at(w.times[1]) } else { None }; + for k in 1..w.times.len() { + if shared.cancel.load(Ordering::Relaxed) { + lock(&shared.state).cancelled = true; + break; + } + let Some(cur) = next.take() else { + lock(&shared.state).error = Some(format!("no frame at {:.3} s", w.times[k].seconds())); + break; + }; + // Track this frame while the next one renders. + let (res, nf) = + rayon::join(|| tracker.step(&trk::Frame { img: &cur.0, offset: cur.1 }), || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None }); + next = nf; + if res.iter().any(|p| p.status == trk::Status::Stopped) { + lock(&shared.state).low_confidence_stop = true; + break; + } + let centers: Vec<[f64; 2]> = res.iter().map(|p| p.center).collect(); + let attach = attach_points(w.settings.kind, rot_scale, &ref_c, &ref_a, ¢ers); + lock(&shared.frames).push(FrameResult { + comp_time: w.times[k], + layer_time: layer.layer_time(w.times[k]), + points: w.points.iter().zip(res).zip(attach).map(|((p, r), a)| (p.uid, r, a)).collect(), + }); + let mut s = lock(&shared.state); + s.done = k as u64; + s.elapsed = t0.elapsed().as_secs_f64(); + s.fps = if s.elapsed > 0.0 { k as f64 / s.elapsed } else { 0.0 }; + s.time = w.times[k].seconds(); + } + let mut s = lock(&shared.state); + s.elapsed = t0.elapsed().as_secs_f64(); + s.finished = true; +} + +/// Set (or add) a key at layer time `t`. +pub(crate) fn key(g: &mut PropGroup, m: &str, t: Tick, v: Value) { + if let Some(p) = g.get_mut(m) { + effectcraft_keyframe::set_key(&mut p.keys, Keyframe::new(t, v)); + } +} + +/// Write analysed frames into the tracker's points. +fn write_frames(p: &mut Project, job: &TrackJob, frames: &[FrameResult]) { + let Some(comp) = p.comp_mut(job.comp) else { return }; + let Some(layer) = comp.layer_mut(job.layer) else { return }; + for f in frames { + for (uid, r, a) in &f.points { + let Some(tp) = layer.props.find_group_mut(*uid) else { continue }; + key(tp, "featureCenter", f.layer_time, Value::Vec2(r.center)); + key(tp, "confidence", f.layer_time, Value::Scalar(r.confidence)); + key(tp, "attachPoint", f.layer_time, Value::Vec2(*a)); + } + } +} + +impl Session { + /// Whether a track analysis is running. + pub fn is_tracking(&self) -> bool { + self.track_job.as_ref().is_some_and(|j| !j.is_finished()) + } + + /// Live progress of the running (or just finished) analysis. + pub fn track_progress(&self) -> Option { + self.track_job.as_ref().map(|j| j.progress()) + } + + /// Cancel the running analysis (results so far are kept). + pub fn stop_track(&mut self) -> bool { + match &self.track_job { + Some(j) if !j.is_finished() => { + j.cancel(); + true + } + _ => false, + } + } + + /// Start analysing. `wait`: block until done (wasm32 always does). One undo step covers the + /// whole analysis. + pub(crate) fn start_track(&mut self, work: Work, tracker: Uid, direction: Direction, wait: bool) -> Result<(), String> { + if self.is_tracking() { + return Err("a track analysis is already running".into()); + } + self.poll_track(); + let wait = wait || cfg!(target_arch = "wasm32"); + self.history.undo.push(("Analyze Track".into(), self.project.clone())); + self.history.redo.clear(); + self.history.merge_key = None; + let shared = Arc::new(TrackShared::default()); + let (comp, layer) = (work.comp, work.layer); + let thread = if wait { + run_work(work, &shared); + None + } else { + let sh = shared.clone(); + Some(std::thread::Builder::new().name("track-analyze".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?) + }; + self.track_job = Some(TrackJob { shared, thread, comp, layer, tracker, direction }); + self.poll_track(); + Ok(()) + } + + /// Write new results into the project and move the CTI to the last analysed frame; drop the + /// job once it finished. Frontends call this every frame while tracking. + pub fn poll_track(&mut self) -> bool { + let Some(job) = &self.track_job else { return false }; + let frames = std::mem::take(&mut *lock(&job.shared.frames)); + let finished = job.is_finished(); + let changed = !frames.is_empty(); + if changed { + write_frames(Arc::make_mut(&mut self.project), job, &frames); + if self.state.active_comp == Some(job.comp) + && let Some(last) = frames.last() + { + self.state.times.insert(job.comp, last.comp_time); + } + self.bump(); + } + if finished { + let mut job = self.track_job.take().expect("job"); + if let Some(t) = job.thread.take() { + let _ = t.join(); + } + let st = job.progress(); + let msg = match (&st.error, st.cancelled, st.low_confidence_stop) { + (Some(e), _, _) => format!("Tracking stopped: {e}"), + (None, true, _) => format!("Tracking stopped after {} frame(s)", st.done), + (None, false, true) => format!("Tracking stopped: confidence below threshold after {} frame(s)", st.done), + _ => format!("Tracked {} frame(s) in {:.1} s", st.done, st.elapsed), + }; + self.events.push(Event::Toast { message: msg, error: st.error.is_some() }); + } + changed || finished + } +} + +// ---------------------------------------------------------------- apply + +/// Apply Dimensions. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Dims { + #[default] + XY, + X, + Y, +} + +impl Dims { + pub fn from_name(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '_', '-'], "").as_str() { + "xy" | "xandy" | "both" => Some(Dims::XY), + "x" | "xonly" => Some(Dims::X), + "y" | "yonly" => Some(Dims::Y), + _ => None, + } + } + fn mix(self, old: [f64; 3], new: [f64; 2]) -> [f64; 3] { + match self { + Dims::XY => [new[0], new[1], old[2]], + Dims::X => [new[0], old[1], old[2]], + Dims::Y => [old[0], new[1], old[2]], + } + } +} + +fn apply_m(m: &Mat3, p: [f64; 2]) -> [f64; 2] { + let q = m.apply(vec2(p[0], p[1])); + [q.x, q.y] +} + +/// Parent space of a layer (comp space when unparented), as a 2D matrix from comp space. +fn comp_to_parent(ctx: &EvalCtx, layer: &Layer) -> Mat3 { + match layer.parent.and_then(|p| ctx.comp.layer(p)) { + Some(p) => ctx.layer_to_comp(p).0.inverse().unwrap_or(Mat3::IDENTITY), + None => Mat3::IDENTITY, + } +} + +/// Per analysed time: (source layer time, attach points in source layer space). +pub fn attach_track(tracker: &PropGroup) -> Vec<(Tick, Vec<[f64; 2]>)> { + let pts: Vec<&PropGroup> = tracker.track_points().collect(); + let Some(first) = pts.first().and_then(|g| g.get("attachPoint")) else { return vec![] }; + first.keys.iter().map(|k| (k.time, pts.iter().map(|g| g.get("attachPoint").map(|p| p.value_at(k.time).as_vec2()).unwrap_or([0.0; 2])).collect())).collect() +} + +/// Apply a tracker (Tracker panel ▸ Apply). Returns the number of keyframed frames. +pub fn apply(p: &mut Project, cid: ItemId, src: LayerId, tracker: Uid, dims: Dims) -> Result { + let comp = p.comp(cid).ok_or("no composition")?.clone(); + let source = comp.layer(src).ok_or("no source layer")?.clone(); + let (tg, settings) = source.tracker(tracker).ok_or("no such tracker")?; + let settings = settings.clone(); + let track = attach_track(tg); + if track.is_empty() { + return Err("analyze the track first".into()); + } + let target_id = match settings.kind { + TrackKind::Stabilize => src, + TrackKind::Raw => return Err("a Raw track has nothing to apply".into()), + _ => settings.target.ok_or("set a Motion Target first (Edit Target…)")?, + }; + let target = comp.layer(target_id).ok_or("the motion target layer is gone")?.clone(); + let ctx0 = EvalCtx::new(p, cid, &comp, Tick::ZERO); + let t_first = source.comp_time(track[0].0); + let rot_scale = settings.point_count() >= 2 && track[0].1.len() >= 2; + let (ang0, len0) = if rot_scale { effectcraft_track::angle_and_length(track[0].1[0], track[0].1[1]) } else { (0.0, 1.0) }; + // (target layer time, values) to write. + let mut pos_keys = vec![]; + let mut anchor_keys = vec![]; + let mut rot_keys = vec![]; + let mut scale_keys = vec![]; + let mut corner_keys = vec![]; + let ttr = target.transform().ok_or("the target has no transform")?; + let c0 = ctx0.at(t_first); + let rot0 = c0.f(&target, ttr, "rotation", 0.0); + let scale0 = c0.v3(&target, ttr, "scale", [100.0; 3]); + let mut prev_ang = 0.0; + // Stabilize: the feature stays where it was on the first tracked frame. + let stab_pos = { + let (m, _) = c0.layer_to_comp(&source); + apply_m(&comp_to_parent(&c0, &source), apply_m(&m, track[0].1[0])) + }; + for (lt, pts) in &track { + let ct = source.comp_time(*lt); + let ctx = ctx0.at(ct); + let tt = target.layer_time(ct); + let (ang, len) = if rot_scale { effectcraft_track::angle_and_length(pts[0], pts[1]) } else { (ang0, len0) }; + let dang = effectcraft_track::unwrap_degrees(prev_ang, ang - ang0); + prev_ang = dang; + let ratio = if len0 > 1e-9 { len / len0 } else { 1.0 }; + match settings.kind { + TrackKind::Stabilize => { + if settings.position { + let old = ctx.v3(&target, ttr, "anchor", [0.0; 3]); + anchor_keys.push((tt, dims.mix(old, pts[0]))); + let oldp = ctx.position(&target, ttr); + pos_keys.push((tt, dims.mix(oldp, stab_pos))); + } + if settings.rotation && rot_scale { + rot_keys.push((tt, rot0 - dang)); + } + if settings.scale && rot_scale { + scale_keys.push((tt, [scale0[0] / ratio, scale0[1] / ratio, scale0[2]])); + } + } + TrackKind::Transform => { + let (m, _) = ctx.layer_to_comp(&source); + if settings.position { + let c = apply_m(&m, pts[0]); + let pp = apply_m(&comp_to_parent(&ctx, &target), c); + pos_keys.push((tt, dims.mix(ctx.position(&target, ttr), pp))); + } + if settings.rotation && rot_scale { + rot_keys.push((tt, rot0 + dang)); + } + if settings.scale && rot_scale { + scale_keys.push((tt, [scale0[0] * ratio, scale0[1] * ratio, scale0[2]])); + } + } + TrackKind::Affine | TrackKind::Perspective => { + let (ms, _) = ctx.layer_to_comp(&source); + let (mt, _) = ctx.layer_to_comp(&target); + let inv = mt.inverse().ok_or("the target layer's transform is not invertible")?; + let n = pts.len(); + if n < 3 { + return Err("corner pin tracks need 3 or 4 track points".into()); + } + let lr = + if settings.kind == TrackKind::Affine || n < 4 { [pts[1][0] + pts[2][0] - pts[0][0], pts[1][1] + pts[2][1] - pts[0][1]] } else { pts[3] }; + // Track Points 1–4: upper left, upper right, lower left, lower right. + let map = |q: [f64; 2]| apply_m(&inv, apply_m(&ms, q)); + corner_keys.push((tt, [map(pts[0]), map(pts[1]), map(pts[2]), map(lr)])); + } + TrackKind::Raw => {} + } + } + let n = track.len(); + let tsize = effectcraft_render::source_size(p, &target); + let mut ids_next = p.next_id; + let layer = p.comp_mut(cid).and_then(|c| c.layer_mut(target_id)).ok_or("no target layer")?; + if !corner_keys.is_empty() { + let spec = effectcraft_effects::find("ec.distort.cornerpin").ok_or("Corner Pin is not available")?; + let size = tsize; + let size = if size.0 == 0 { [comp.width as f64, comp.height as f64] } else { [size.0 as f64, size.1 as f64] }; + let fx = layer.props.sub_mut("effects").ok_or("the target layer can't have effects")?; + let same = fx.groups().filter(|g| g.match_id == spec.id).count(); + let name = if same == 0 { spec.name.to_string() } else { format!("{} {}", spec.name, same + 1) }; + let mut g = effectcraft_effects::instantiate(spec, &mut effectcraft_project::build::Ids(&mut ids_next), &name, size); + for (t, c) in &corner_keys { + for (m, v) in ["ul", "ur", "ll", "lr"].iter().zip(c) { + key(&mut g, m, *t, Value::Vec2(*v)); + } + } + fx.children.push(g.into()); + } + let tr = layer.transform_mut().ok_or("no transform")?; + for (t, v) in anchor_keys { + key(tr, "anchor", t, Value::Vec3(v)); + } + let separated = tr.get("positionX").is_some(); + for (t, v) in pos_keys { + if separated { + key(tr, "positionX", t, Value::Scalar(v[0])); + key(tr, "positionY", t, Value::Scalar(v[1])); + } else { + key(tr, "position", t, Value::Vec3(v)); + } + } + for (t, v) in rot_keys { + key(tr, "rotation", t, Value::Scalar(v)); + } + for (t, v) in scale_keys { + key(tr, "scale", t, Value::Vec3(v)); + } + p.next_id = ids_next; + Ok(n) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn attach_points_follow_the_track() { + let rc = [[0.0, 0.0], [10.0, 0.0]]; + let ra = [[0.0, 5.0], [10.0, 5.0]]; + // Rotated 90° and doubled. + let cur = [[100.0, 100.0], [100.0, 120.0]]; + let a = attach_points(TrackKind::Transform, true, &rc, &ra, &cur); + assert!((a[0][0] - 90.0).abs() < 1e-9 && (a[0][1] - 100.0).abs() < 1e-9, "{a:?}"); + let a = attach_points(TrackKind::Transform, false, &rc, &ra, &cur); + assert_eq!(a[0], [100.0, 105.0]); + let h = trk::Homography([[1.1, 0.1, 5.0], [0.05, 0.9, -3.0], [0.001, 0.0005, 1.0]]); + let rc4 = [[0.0, 0.0], [100.0, 0.0], [0.0, 100.0], [100.0, 100.0]]; + let ra4 = [[10.0, 10.0], [90.0, 10.0], [10.0, 90.0], [90.0, 90.0]]; + let cur4: Vec<[f64; 2]> = rc4.iter().map(|p| h.apply(*p)).collect(); + let a = attach_points(TrackKind::Perspective, false, &rc4, &ra4, &cur4); + for (got, want) in a.iter().zip(ra4.iter().map(|p| h.apply(*p))) { + assert!((got[0] - want[0]).abs() < 1e-6 && (got[1] - want[1]).abs() < 1e-6); + } + } +} diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 9e8cf949..164d25b8 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -8,6 +8,7 @@ pub mod build; pub mod props; pub mod render_queue; pub mod styles; +pub mod tracking; use std::collections::BTreeMap; use std::sync::Arc; diff --git a/crates/project/src/props.rs b/crates/project/src/props.rs index 89081632..496547d8 100644 --- a/crates/project/src/props.rs +++ b/crates/project/src/props.rs @@ -170,6 +170,10 @@ pub enum GroupKind { /// Effect spec id, e.g. `ec.blur.gaussian`. effect: String, }, + /// A motion tracker (Motion Trackers ▸ Tracker n) with its Tracker panel settings. + Tracker { + settings: Box, + }, } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] diff --git a/crates/project/src/tracking.rs b/crates/project/src/tracking.rs new file mode 100644 index 00000000..5e7f8f1d --- /dev/null +++ b/crates/project/src/tracking.rs @@ -0,0 +1,288 @@ +//! Motion trackers in the property tree, laid out like After Effects': +//! +//! ```text +//! Motion Trackers (match `motionTrackers`, first group of the layer) +//! Tracker 1 (GroupKind::Tracker: type, target, options) +//! Track Point 1 (match `trackPoint`) +//! Feature Center, Feature Size, Search Offset, Search Size, +//! Confidence, Attach Point, Attach Point Offset +//! ``` +//! +//! Feature Center, Confidence and Attach Point get one keyframe per analysed frame (layer time). +//! The tracking algorithms live in `effectcraft-track`; applying tracks lives in the engine. + +use effectcraft_keyframe::Value; +use serde::{Deserialize, Serialize}; + +use crate::build::Ids; +use crate::props::{GroupKind, ParamUi, PropGroup}; +use crate::{Layer, LayerId, Uid}; + +/// Tracker panel ▸ Track Type. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum TrackKind { + Stabilize, + #[default] + Transform, + /// Parallel (affine) corner pin: three points, the fourth corner completes the parallelogram. + #[serde(alias = "parallel", alias = "parallelCornerPin")] + Affine, + /// Perspective corner pin: four points. + #[serde(alias = "perspectiveCornerPin")] + Perspective, + /// Raw: track data only (nothing to apply). + Raw, +} + +impl TrackKind { + pub const ALL: [TrackKind; 5] = [TrackKind::Stabilize, TrackKind::Transform, TrackKind::Affine, TrackKind::Perspective, TrackKind::Raw]; + pub fn label(self) -> &'static str { + match self { + TrackKind::Stabilize => "Stabilize", + TrackKind::Transform => "Transform", + TrackKind::Affine => "Parallel Corner Pin", + TrackKind::Perspective => "Perspective Corner Pin", + TrackKind::Raw => "Raw", + } + } + pub fn id(self) -> &'static str { + match self { + TrackKind::Stabilize => "stabilize", + TrackKind::Transform => "transform", + TrackKind::Affine => "affine", + TrackKind::Perspective => "perspective", + TrackKind::Raw => "raw", + } + } + pub fn from_name(s: &str) -> Option { + let n = s.to_ascii_lowercase().replace([' ', '_', '-'], ""); + match n.as_str() { + "parallel" | "parallelcornerpin" | "affinecornerpin" => return Some(TrackKind::Affine), + "perspectivecornerpin" => return Some(TrackKind::Perspective), + _ => {} + } + TrackKind::ALL.into_iter().find(|k| k.id() == n || k.label().to_ascii_lowercase().replace(' ', "") == n) + } + pub fn is_corner_pin(self) -> bool { + matches!(self, TrackKind::Affine | TrackKind::Perspective) + } + /// Number of track points for this type (Transform/Stabilize: 2 with rotation or scale). + pub fn point_count(self, rotation_or_scale: bool) -> usize { + match self { + TrackKind::Affine => 3, + TrackKind::Perspective => 4, + _ if rotation_or_scale => 2, + _ => 1, + } + } +} + +/// Motion Tracker Options ▸ Channel. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum TrackChannel { + Rgb, + #[default] + Luminance, + Saturation, +} + +/// Motion Tracker Options ▸ "If Confidence is Below … %". +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum LowConfidence { + Continue, + Stop, + Extrapolate, + #[default] + Adapt, +} + +/// Motion Tracker Options. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct TrackerOptions { + pub channel: TrackChannel, + /// Process Before Match ▸ Blur radius in pixels (0 = off). + pub blur: f64, + /// Process Before Match ▸ Enhance. + pub enhance: bool, + pub subpixel: bool, + /// Adapt Feature on Every Frame. + pub adapt_every_frame: bool, + /// Confidence threshold, percent. + pub threshold: f64, + pub action: LowConfidence, + /// Follow each feature's rotation and scale while matching (EffectCraft extension). + pub track_shape: bool, +} + +impl Default for TrackerOptions { + fn default() -> Self { + TrackerOptions { + channel: TrackChannel::Luminance, + blur: 0.0, + enhance: false, + subpixel: true, + adapt_every_frame: false, + threshold: 80.0, + action: LowConfidence::Adapt, + track_shape: true, + } + } +} + +/// A tracker's non-animatable settings (Tracker panel). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct TrackerSettings { + pub kind: TrackKind, + pub position: bool, + pub rotation: bool, + pub scale: bool, + /// Motion Target (None: no target; Stabilize always applies to the tracked layer). + pub target: Option, + pub options: TrackerOptions, +} + +impl TrackerSettings { + pub fn new(kind: TrackKind) -> TrackerSettings { + TrackerSettings { kind, position: true, ..Default::default() } + } + pub fn point_count(&self) -> usize { + self.kind.point_count(self.rotation || self.scale) + } +} + +/// Initial regions of a track point (layer pixels). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct PointInit { + pub center: [f64; 2], + pub feature_size: [f64; 2], + pub search_offset: [f64; 2], + pub search_size: [f64; 2], + pub attach_offset: [f64; 2], +} + +pub const MOTION_TRACKERS: &str = "motionTrackers"; +pub const TRACK_POINT: &str = "trackPoint"; + +/// The (empty) Motion Trackers group. +pub fn motion_trackers(ids: &mut Ids) -> PropGroup { + ids.group(MOTION_TRACKERS, "Motion Trackers") +} + +/// One Track Point group. +pub fn track_point(ids: &mut Ids, name: &str, p: &PointInit) -> PropGroup { + let pt = |ids: &mut Ids, m: &str, n: &str, v: [f64; 2]| ids.prop(m, n, Value::Vec2(v)).with_ui(ParamUi::Point).spatial(); + let vec = |ids: &mut Ids, m: &str, n: &str, v: [f64; 2]| ids.prop(m, n, Value::Vec2(v)).with_ui(ParamUi::Point); + let attach = [p.center[0] + p.attach_offset[0], p.center[1] + p.attach_offset[1]]; + let fc = pt(ids, "featureCenter", "Feature Center", p.center); + let fs = vec(ids, "featureSize", "Feature Size", p.feature_size); + let so = vec(ids, "searchOffset", "Search Offset", p.search_offset); + let ss = vec(ids, "searchSize", "Search Size", p.search_size); + let cf = ids.prop("confidence", "Confidence", Value::Scalar(0.0)).with_ui(ParamUi::Percent); + let ap = pt(ids, "attachPoint", "Attach Point", attach); + let ao = vec(ids, "attachPointOffset", "Attach Point Offset", p.attach_offset); + let mut g = ids.group(TRACK_POINT, name).with(fc).with(fs).with(so).with(ss).with(cf).with(ap).with(ao); + g.kind = GroupKind::Indexed; + g +} + +/// A Tracker group with its points. +pub fn tracker(ids: &mut Ids, name: &str, settings: TrackerSettings, points: &[PointInit]) -> PropGroup { + let mut g = ids.group("tracker", name); + g.kind = GroupKind::Tracker { settings: Box::new(settings) }; + for (i, p) in points.iter().enumerate() { + let tp = track_point(ids, &format!("Track Point {}", i + 1), p); + g.children.push(tp.into()); + } + g +} + +/// Default track point layout for a tracker type, centred on a layer of size `size`. +pub fn default_points(kind: TrackKind, n: usize, size: [f64; 2]) -> Vec { + let m = size[0].min(size[1]).max(1.0); + let f = (m * 0.04).clamp(16.0, 80.0).round(); + let s = (f * 2.0).round(); + let c = [size[0] / 2.0, size[1] / 2.0]; + let mk = |p: [f64; 2]| PointInit { center: p, feature_size: [f, f], search_offset: [0.0; 2], search_size: [s, s], attach_offset: [0.0; 2] }; + let (dx, dy) = ((size[0] * 0.18).max(2.0 * s), (size[1] * 0.18).max(2.0 * s)); + match (kind, n) { + (TrackKind::Perspective, _) | (_, 4) => { + vec![mk([c[0] - dx, c[1] - dy]), mk([c[0] + dx, c[1] - dy]), mk([c[0] - dx, c[1] + dy]), mk([c[0] + dx, c[1] + dy])] + } + (TrackKind::Affine, _) | (_, 3) => vec![mk([c[0] - dx, c[1] - dy]), mk([c[0] + dx, c[1] - dy]), mk([c[0] - dx, c[1] + dy])], + (_, 2) => vec![mk([c[0] - dx, c[1]]), mk([c[0] + dx, c[1]])], + _ => vec![mk(c)], + } +} + +impl Layer { + /// The Motion Trackers group, if the layer has one. + pub fn motion_trackers(&self) -> Option<&PropGroup> { + self.props.sub(MOTION_TRACKERS) + } + /// Trackers of the layer (groups with [`GroupKind::Tracker`]). + pub fn trackers(&self) -> impl Iterator { + self.motion_trackers().into_iter().flat_map(|g| g.groups()).filter_map(|g| match &g.kind { + GroupKind::Tracker { settings } => Some((g, settings.as_ref())), + _ => None, + }) + } + pub fn tracker(&self, uid: Uid) -> Option<(&PropGroup, &TrackerSettings)> { + self.trackers().find(|(g, _)| g.uid == uid) + } + /// The Motion Trackers group, created (first in the layer's tree, like AE) when missing. + pub fn motion_trackers_mut(&mut self, ids: &mut Ids) -> &mut PropGroup { + if self.props.sub(MOTION_TRACKERS).is_none() { + self.props.children.insert(0, motion_trackers(ids).into()); + } + self.props.sub_mut(MOTION_TRACKERS).expect("motion trackers group") + } +} + +impl PropGroup { + /// Tracker settings of a tracker group. + pub fn tracker_settings(&self) -> Option<&TrackerSettings> { + match &self.kind { + GroupKind::Tracker { settings } => Some(settings), + _ => None, + } + } + pub fn tracker_settings_mut(&mut self) -> Option<&mut TrackerSettings> { + match &mut self.kind { + GroupKind::Tracker { settings } => Some(settings), + _ => None, + } + } + /// Track Point groups of a tracker group, in order. + pub fn track_points(&self) -> impl Iterator { + self.groups().filter(|g| g.match_id == TRACK_POINT) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tracker_tree_and_serde() { + let mut n = 1; + let mut ids = Ids(&mut n); + let pts = default_points(TrackKind::Perspective, 4, [1920.0, 1080.0]); + let g = tracker(&mut ids, "Tracker 1", TrackerSettings::new(TrackKind::Perspective), &pts); + assert_eq!(g.track_points().count(), 4); + let tp = g.track_points().next().unwrap(); + let names: Vec<&str> = tp.props().map(|p| p.name.as_str()).collect(); + assert_eq!(names, ["Feature Center", "Feature Size", "Search Offset", "Search Size", "Confidence", "Attach Point", "Attach Point Offset"]); + let json = serde_json::to_string(&g).unwrap(); + let back: PropGroup = serde_json::from_str(&json).unwrap(); + assert_eq!(back, g); + assert_eq!(back.tracker_settings().unwrap().kind, TrackKind::Perspective); + assert_eq!(TrackKind::from_name("Parallel Corner Pin"), Some(TrackKind::Affine)); + assert_eq!(TrackKind::from_name("perspective"), Some(TrackKind::Perspective)); + assert_eq!(TrackerSettings { rotation: true, ..TrackerSettings::new(TrackKind::Transform) }.point_count(), 2); + } +} diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 68d41982..8f3282dc 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -341,6 +341,13 @@ impl<'a> Renderer<'a> { Some(flat) } + /// A layer's source pixels at the context time, before masks, effects and the transform + /// (what the motion tracker analyses, like After Effects' Layer panel). `None` for layers + /// without pixels. + pub fn layer_source(&self, ctx: &EvalCtx, layer: &Layer) -> Option { + self.source(ctx, layer) + } + /// Layer pixels after source → masks → effects (no layer styles). pub fn content_buf(&self, ctx: &EvalCtx, layer: &Layer) -> Option> { self.content_buf_timed(ctx, layer, None).map(|(b, _)| b) diff --git a/crates/track/Cargo.toml b/crates/track/Cargo.toml new file mode 100644 index 00000000..f8cd71d1 --- /dev/null +++ b/crates/track/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "effectcraft-track" +description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, and corner-pin / transform solves" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +effectcraft-raster = { workspace = true } +rayon = { workspace = true } +serde = { workspace = true } diff --git a/crates/track/src/lib.rs b/crates/track/src/lib.rs new file mode 100644 index 00000000..d106b209 --- /dev/null +++ b/crates/track/src/lib.rs @@ -0,0 +1,639 @@ +//! Motion tracking, clean-room from textbook methods: +//! +//! - each **track point** has a *feature region* (the pattern to follow) and a *search region* +//! (where to look for it in the next frame), as in After Effects' Tracker; +//! - the pattern is located by **normalized cross-correlation** (Lewis, "Fast Normalized +//! Cross-Correlation", 1995) searched **coarse to fine over an image pyramid**; +//! - the integer match is refined to **sub-pixel** precision (and, optionally, the feature's +//! rotation and scale) by **Lucas–Kanade** Gauss–Newton image alignment (Lucas & Kanade 1981; +//! Baker & Matthews, "Lucas-Kanade 20 Years On", 2004) with a gain/bias photometric model; +//! - **confidence** is the correlation of the template with the matched region, in percent; +//! when it drops below a threshold the tracker continues, stops, extrapolates the motion or +//! adapts the feature (re-samples the template), like AE's Motion Tracker Options. +//! +//! Only the pixels around each search region are converted and pre-processed, so tracking cost is +//! independent of the frame size. Points are tracked in parallel and the coarse correlation search +//! is parallel over candidate rows (rayon). +//! +//! Coordinates are *layer* pixels: the frame image's pixel `(i, j)` covers `[i, i+1) × [j, j+1)` +//! after subtracting the frame's `offset` (see [`Frame`]). + +pub mod plane; +pub mod solve; + +use effectcraft_raster::Image; +use rayon::prelude::*; +use serde::{Deserialize, Serialize}; + +pub use plane::{Channel, Plane}; +pub use solve::{Homography, angle_and_length, unwrap_degrees}; + +/// What to do when a frame's confidence is below the threshold. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ConfidenceAction { + /// Keep the match anyway. + Continue, + /// Stop the analysis before this frame. + Stop, + /// Ignore the match and continue the previous motion. + Extrapolate, + /// Keep the match and re-sample the feature from this frame. + #[default] + Adapt, +} + +impl ConfidenceAction { + pub const ALL: [ConfidenceAction; 4] = [ConfidenceAction::Continue, ConfidenceAction::Stop, ConfidenceAction::Extrapolate, ConfidenceAction::Adapt]; + pub fn label(self) -> &'static str { + match self { + ConfidenceAction::Continue => "Continue Tracking", + ConfidenceAction::Stop => "Stop Tracking", + ConfidenceAction::Extrapolate => "Extrapolate Motion", + ConfidenceAction::Adapt => "Adapt Feature", + } + } + pub fn from_name(s: &str) -> Option { + let n = s.to_ascii_lowercase().replace([' ', '_', '-'], ""); + ConfidenceAction::ALL.into_iter().find(|a| { + format!("{a:?}").to_ascii_lowercase() == n + || a.label().to_ascii_lowercase().replace(' ', "") == n + || a.label().to_ascii_lowercase().replace(' ', "").starts_with(&n) + }) + } +} + +/// Motion Tracker Options that affect matching. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct TrackOptions { + pub channel: Channel, + /// Pre-process blur radius in pixels (0 = off). + pub blur: f64, + /// Pre-process edge enhancement. + pub enhance: bool, + /// Sub-pixel positioning (else whole-pixel matches). + pub subpixel: bool, + /// Re-sample the feature on every frame. + pub adapt_every_frame: bool, + /// Confidence threshold in percent for `action`. + pub threshold: f64, + pub action: ConfidenceAction, + /// Also estimate each feature's rotation and scale while matching (follows rotating or + /// zooming features without adapting). + pub track_shape: bool, +} + +impl Default for TrackOptions { + fn default() -> Self { + TrackOptions { + channel: Channel::Luminance, + blur: 0.0, + enhance: false, + subpixel: true, + adapt_every_frame: false, + threshold: 80.0, + action: ConfidenceAction::Adapt, + track_shape: true, + } + } +} + +/// One track point's regions (layer pixels). +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PointSpec { + pub center: [f64; 2], + /// Feature region width and height. + pub feature_size: [f64; 2], + /// Search region centre relative to the feature centre. + pub search_offset: [f64; 2], + /// Search region width and height. + pub search_size: [f64; 2], +} + +/// A frame to track in: layer pixels map to image pixels by adding `offset`. +#[derive(Clone, Copy)] +pub struct Frame<'a> { + pub img: &'a Image, + pub offset: [f64; 2], +} + +impl<'a> Frame<'a> { + pub fn new(img: &'a Image) -> Frame<'a> { + Frame { img, offset: [0.0; 2] } + } +} + +/// How a point's position on a frame was obtained. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Status { + Tracked, + /// Tracked, and the feature was re-sampled from this frame. + Adapted, + /// Low confidence: the previous motion was continued. + Extrapolated, + /// Low confidence with the Stop action: no result for this frame. + Stopped, +} + +/// A point's result on one frame. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PointResult { + /// Feature centre (layer pixels). + pub center: [f64; 2], + /// Match confidence, 0–100 %. + pub confidence: f64, + /// Feature rotation (degrees) and scale (1 = start) relative to the start frame (when + /// `track_shape` is on). + pub rotation: f64, + pub scale: f64, + pub status: Status, +} + +/// A template sampled on a unit grid around the feature centre at one pyramid level. +#[derive(Clone, Debug)] +struct Patch { + r: [i32; 2], + ch: usize, + raw: Vec, + /// Zero-mean values and their sum of squares. + zm: Vec, + ss: f64, +} + +impl Patch { + fn sample(plane: &Plane, c: [f64; 2], a: [f64; 2], r: [i32; 2]) -> Patch { + let ch = plane.ch; + let mut raw = Vec::with_capacity(((2 * r[0] + 1) * (2 * r[1] + 1)) as usize * ch); + let mut v = [0.0f32; 3]; + for gy in -r[1]..=r[1] { + for gx in -r[0]..=r[0] { + let (ux, uy) = (gx as f64, gy as f64); + plane.sample(c[0] + a[0] * ux - a[1] * uy, c[1] + a[1] * ux + a[0] * uy, &mut v); + raw.extend_from_slice(&v[..ch]); + } + } + let mean = raw.iter().map(|x| *x as f64).sum::() / raw.len().max(1) as f64; + let zm: Vec = raw.iter().map(|x| (*x as f64 - mean) as f32).collect(); + let ss = zm.iter().map(|x| (*x as f64) * (*x as f64)).sum(); + Patch { r, ch, raw, zm, ss } + } +} + +/// Normalized cross-correlation of `patch` with `plane` sampled at `c + A u` (A = similarity +/// `[[a0, -a1], [a1, a0]]`). Pure translations share one set of bilinear weights. +fn ncc(plane: &Plane, p: &Patch, c: [f64; 2], a: [f64; 2]) -> f64 { + if p.ss <= 1e-12 { + return 0.0; + } + let ch = p.ch; + let (mut s, mut s2, mut st) = (0.0f64, 0.0f64, 0.0f64); + let identity = (a[0] - 1.0).abs() < 1e-9 && a[1].abs() < 1e-9; + let fx = c[0] - 0.5; + let fy = c[1] - 0.5; + let (bx, by) = (fx.floor(), fy.floor()); + let inside = bx as i64 - p.r[0] as i64 >= 0 + && by as i64 - p.r[1] as i64 >= 0 + && (bx as i64 + p.r[0] as i64 + 1) < plane.w as i64 + && (by as i64 + p.r[1] as i64 + 1) < plane.h as i64; + if identity && inside { + let (tx, ty) = ((fx - bx) as f32, (fy - by) as f32); + let (w00, w10, w01, w11) = ((1.0 - tx) * (1.0 - ty), tx * (1.0 - ty), (1.0 - tx) * ty, tx * ty); + let row = plane.w * ch; + let mut k = 0; + for gy in -p.r[1]..=p.r[1] { + let y = (by as i64 + gy as i64) as usize; + let base = y * row; + for gx in -p.r[0]..=p.r[0] { + let x = (bx as i64 + gx as i64) as usize; + let i00 = base + x * ch; + for cc in 0..ch { + let v = + w00 * plane.data[i00 + cc] + w10 * plane.data[i00 + ch + cc] + w01 * plane.data[i00 + row + cc] + w11 * plane.data[i00 + row + ch + cc]; + let v = v as f64; + s += v; + s2 += v * v; + st += v * p.zm[k] as f64; + k += 1; + } + } + } + } else { + let mut v = [0.0f32; 3]; + let mut k = 0; + for gy in -p.r[1]..=p.r[1] { + for gx in -p.r[0]..=p.r[0] { + let (ux, uy) = (gx as f64, gy as f64); + plane.sample(c[0] + a[0] * ux - a[1] * uy, c[1] + a[1] * ux + a[0] * uy, &mut v); + for cc in 0..ch { + let x = v[cc] as f64; + s += x; + s2 += x * x; + st += x * p.zm[k] as f64; + k += 1; + } + } + } + } + let n = p.zm.len() as f64; + let var = s2 - s * s / n; + if var <= 1e-12 { + return 0.0; + } + (st / (var * p.ss).sqrt()).clamp(-1.0, 1.0) +} + +/// The pre-processed pixels around one point: a pyramid of a crop. Plane coordinates are image +/// pixels minus `origin`, divided by `2^level`. +struct Crop { + origin: [f64; 2], + levels: Vec, + grad0: (Plane, Plane), +} + +impl Crop { + /// Crop image pixels `[c - ext, c + ext]` (image pixel coordinates), pre-processed. + fn new(frame: &Frame, opts: &TrackOptions, c: [f64; 2], ext: [f64; 2], levels: usize) -> Crop { + let sigma = opts.blur.max(0.0) / 2.0; + let margin = (sigma * 3.0).ceil() + 3.0 + if opts.enhance { 6.0 } else { 0.0 } + (1 << levels) as f64; + let x0 = (c[0] - ext[0] - margin).floor(); + let y0 = (c[1] - ext[1] - margin).floor(); + let w = ((2.0 * (ext[0] + margin)).ceil() as usize + 2).max(4); + let h = ((2.0 * (ext[1] + margin)).ceil() as usize + 2).max(4); + let mut p = Plane::from_image(frame.img, opts.channel, x0 as i64, y0 as i64, w, h); + if sigma > 0.05 { + p = p.gaussian(sigma); + } + if opts.enhance { + p = p.enhance(); + } + let grad0 = p.gradients(); + let mut lv = vec![p]; + for _ in 0..levels { + let d = lv.last().map(Plane::downsample).unwrap_or_default(); + lv.push(d); + } + Crop { origin: [x0, y0], levels: lv, grad0 } + } + fn to_level(&self, p: [f64; 2], k: usize) -> [f64; 2] { + let s = (1u32 << k) as f64; + [(p[0] - self.origin[0]) / s, (p[1] - self.origin[1]) / s] + } + fn level_to_image(&self, p: [f64; 2], k: usize) -> [f64; 2] { + let s = (1u32 << k) as f64; + [p[0] * s + self.origin[0], p[1] * s + self.origin[1]] + } +} + +/// Per-point tracking state. +#[derive(Clone, Debug)] +struct PointState { + spec: PointSpec, + /// Current centre in image pixels. + center: [f64; 2], + /// Current similarity `[s cos θ, s sin θ]` relative to the template's frame. + shape: [f64; 2], + velocity: [f64; 2], + templates: Vec, + levels: usize, +} + +fn half(v: [f64; 2]) -> [f64; 2] { + [(v[0] / 2.0).max(1.0), (v[1] / 2.0).max(1.0)] +} + +impl PointState { + fn feature_r(&self) -> [i32; 2] { + let h = half(self.spec.feature_size); + [h[0].round().max(2.0) as i32, h[1].round().max(2.0) as i32] + } + fn level_r(&self, k: usize) -> [i32; 2] { + let r = self.feature_r(); + let s = (1 << k) as f64; + [((r[0] as f64 / s).ceil() as i32).max(2), ((r[1] as f64 / s).ceil() as i32).max(2)] + } + /// Search range around the predicted centre (pixels, per axis). + fn range(&self) -> [f64; 2] { + let f = half(self.spec.feature_size); + let s = half(self.spec.search_size); + [(s[0] - f[0]).max(2.0), (s[1] - f[1]).max(2.0)] + } + fn extent(&self) -> f64 { + let r = self.feature_r(); + let s = (self.shape[0] * self.shape[0] + self.shape[1] * self.shape[1]).sqrt().max(1.0); + ((r[0] * r[0] + r[1] * r[1]) as f64).sqrt() * s + 2.0 + } + + fn new(spec: PointSpec, frame: &Frame, opts: &TrackOptions) -> PointState { + let c = [spec.center[0] + frame.offset[0], spec.center[1] + frame.offset[1]]; + let mut st = PointState { spec, center: c, shape: [1.0, 0.0], velocity: [0.0; 2], templates: vec![], levels: 0 }; + let range = st.range(); + let fr = st.feature_r(); + let mut l = 0; + while l < 4 && range[0].min(range[1]) / (2u32 << l) as f64 >= 3.0 && (fr[0].min(fr[1]) as f64) / (2u32 << l) as f64 >= 3.0 { + l += 1; + } + st.levels = l; + let e = st.extent(); + let crop = Crop::new(frame, opts, c, [e, e], l); + st.resample(&crop); + st + } + + /// Re-sample the templates from `crop` at the current centre and shape. + fn resample(&mut self, crop: &Crop) { + self.templates = (0..=self.levels).map(|k| Patch::sample(&crop.levels[k], crop.to_level(self.center, k), self.shape, self.level_r(k))).collect(); + } + + fn step(&mut self, frame: &Frame, opts: &TrackOptions) -> PointResult { + let prev = self.center; + let predicted = [prev[0] + self.velocity[0], prev[1] + self.velocity[1]]; + let sc = [predicted[0] + self.spec.search_offset[0], predicted[1] + self.spec.search_offset[1]]; + let range = self.range(); + let e = self.extent(); + let crop = Crop::new(frame, opts, sc, [range[0] + e, range[1] + e], self.levels); + let top = self.levels; + // Coarse: exhaustive correlation over the search range at the top level. + let s = (1u32 << top) as f64; + let ctop = crop.to_level(sc, top); + let (rx, ry) = ((range[0] / s).ceil() as i32, (range[1] / s).ceil() as i32); + let plane = &crop.levels[top]; + let tpl = &self.templates[top]; + let shape = self.shape; + let mut cands: Vec<(f64, [f64; 2])> = (-ry..=ry) + .into_par_iter() + .flat_map_iter(|j| { + (-rx..=rx).map(move |i| { + let c = [ctop[0] + i as f64, ctop[1] + j as f64]; + (ncc(plane, tpl, c, shape), c) + }) + }) + .collect(); + // Best score first; ties (flat or repetitive regions) go to the candidate nearest the + // prediction so featureless areas don't drift. + let cp = crop.to_level(predicted, top); + let d2 = |c: &[f64; 2]| (c[0] - cp[0]).powi(2) + (c[1] - cp[1]).powi(2); + cands.sort_by(|a, b| if (a.0 - b.0).abs() < 1e-9 { d2(&a.1).total_cmp(&d2(&b.1)) } else { b.0.total_cmp(&a.0) }); + let mut seeds: Vec<[f64; 2]> = vec![]; + for (_, c) in &cands { + if seeds.iter().all(|q| (q[0] - c[0]).abs() > 1.5 || (q[1] - c[1]).abs() > 1.5) { + seeds.push(*c); + } + if seeds.len() == 3 { + break; + } + } + // Fine: refine each seed down the pyramid in a 5 × 5 neighbourhood. + let mut best = (f64::MIN, sc); + for seed in seeds { + let mut c = seed; + let mut score = f64::MIN; + for k in (0..top).rev() { + c = [c[0] * 2.0, c[1] * 2.0]; + let (pl, tp) = (&crop.levels[k], &self.templates[k]); + // Start from the projected match; neighbours must beat it (no drift on ties). + let mut lb = (ncc(pl, tp, c, shape), c); + for j in -2..=2 { + for i in -2..=2 { + if i == 0 && j == 0 { + continue; + } + let q = [c[0] + i as f64, c[1] + j as f64]; + let v = ncc(pl, tp, q, shape); + if v > lb.0 + 1e-9 { + lb = (v, q); + } + } + } + (score, c) = lb; + } + if top == 0 { + score = ncc(&crop.levels[0], &self.templates[0], c, shape); + } + if score > best.0 + 1e-9 { + best = (score, crop.level_to_image(c, 0)); + } + } + let c0 = crop.to_level(best.1, 0); + // Sub-pixel (and shape) refinement. + let (mut c, mut a) = (c0, shape); + if opts.subpixel { + let mut ok = false; + if opts.track_shape + && let Some((cr, ar)) = lk_refine(&crop, &self.templates[0], c0, shape, true) + { + (c, a) = (cr, ar); + ok = true; + } + if !ok { + if let Some((cr, _)) = lk_refine(&crop, &self.templates[0], c0, shape, false) { + c = cr; + } else { + c = parabolic(&crop.levels[0], &self.templates[0], c0, shape); + } + } + } + // A featureless template can't be matched: stay put. + if self.templates[0].ss < 1e-6 { + (c, a) = (crop.to_level(predicted, 0), shape); + } + let mut conf = ncc(&crop.levels[0], &self.templates[0], c, a); + // Keep the coarse match if refinement made it worse. + let coarse = ncc(&crop.levels[0], &self.templates[0], c0, shape); + if coarse > conf + 0.02 { + (c, a, conf) = (c0, shape, coarse); + } + let confidence = (conf.max(0.0) * 100.0).min(100.0); + let found = crop.level_to_image(c, 0); + let low = confidence < opts.threshold; + let mut status = Status::Tracked; + if low { + match opts.action { + ConfidenceAction::Stop => { + return self.result(prev, frame, confidence, Status::Stopped); + } + ConfidenceAction::Extrapolate => { + self.center = predicted; + return self.result(predicted, frame, confidence, Status::Extrapolated); + } + ConfidenceAction::Continue => {} + ConfidenceAction::Adapt => status = Status::Adapted, + } + } + // Only confident matches predict the next search position. + self.velocity = if low { [0.0; 2] } else { [found[0] - prev[0], found[1] - prev[1]] }; + self.center = found; + self.shape = a; + if opts.adapt_every_frame || status == Status::Adapted { + status = Status::Adapted; + self.resample(&crop); + } + self.result(found, frame, confidence, status) + } + + fn result(&self, c: [f64; 2], frame: &Frame, confidence: f64, status: Status) -> PointResult { + PointResult { + center: [c[0] - frame.offset[0], c[1] - frame.offset[1]], + confidence, + rotation: self.shape[1].atan2(self.shape[0]).to_degrees(), + scale: (self.shape[0] * self.shape[0] + self.shape[1] * self.shape[1]).sqrt(), + status, + } + } +} + +/// Sub-pixel peak from 1-D parabola fits of the correlation around `c`. +fn parabolic(plane: &Plane, p: &Patch, c: [f64; 2], a: [f64; 2]) -> [f64; 2] { + let f = |dx: f64, dy: f64| ncc(plane, p, [c[0] + dx, c[1] + dy], a); + let m = f(0.0, 0.0); + let off = |l: f64, r: f64| { + let d = l - 2.0 * m + r; + if d.abs() < 1e-12 { 0.0 } else { (0.5 * (l - r) / d).clamp(-0.5, 0.5) } + }; + [c[0] + off(f(-1.0, 0.0), f(1.0, 0.0)), c[1] + off(f(0.0, -1.0), f(0.0, 1.0))] +} + +/// Forward-additive Lucas–Kanade (Gauss–Newton) alignment of the level-0 template with gain and +/// bias: minimise Σ (I(c + A u) − g T(u) − b)² over the centre `c` and, with `shape`, the +/// similarity `A`. Returns `None` when it diverges. +fn lk_refine(crop: &Crop, p: &Patch, c0: [f64; 2], a0: [f64; 2], shape: bool) -> Option<([f64; 2], [f64; 2])> { + let plane = &crop.levels[0]; + let (gxp, gyp) = &crop.grad0; + let ch = p.ch; + let (mut c, mut a) = (c0, a0); + let mut g = 1.0f64; + let mut b; + // Initial photometric estimate from means/variances. + { + let mut v = [0.0f32; 3]; + let (mut si, mut si2, mut st, mut st2, mut n) = (0.0, 0.0, 0.0, 0.0, 0.0); + let mut k = 0; + for gy in -p.r[1]..=p.r[1] { + for gx in -p.r[0]..=p.r[0] { + let (ux, uy) = (gx as f64, gy as f64); + plane.sample(c[0] + a[0] * ux - a[1] * uy, c[1] + a[1] * ux + a[0] * uy, &mut v); + for cc in 0..ch { + let (i, t) = (v[cc] as f64, p.raw[k] as f64); + si += i; + si2 += i * i; + st += t; + st2 += t * t; + n += 1.0; + k += 1; + } + } + } + let (mi, mt) = (si / n, st / n); + let (vi, vt) = ((si2 / n - mi * mi).max(0.0), (st2 / n - mt * mt).max(0.0)); + if vt > 1e-12 && vi > 1e-12 { + g = (vi / vt).sqrt(); + } + b = mi - g * mt; + } + let np = if shape { 6 } else { 4 }; + let (mut v, mut vx, mut vy) = ([0.0f32; 3], [0.0f32; 3], [0.0f32; 3]); + for _ in 0..30 { + let mut h = [[0.0f64; 6]; 6]; + let mut rhs = [0.0f64; 6]; + let mut k = 0; + for gy in -p.r[1]..=p.r[1] { + for gx in -p.r[0]..=p.r[0] { + let (ux, uy) = (gx as f64, gy as f64); + let q = [c[0] + a[0] * ux - a[1] * uy, c[1] + a[1] * ux + a[0] * uy]; + plane.sample(q[0], q[1], &mut v); + gxp.sample(q[0], q[1], &mut vx); + gyp.sample(q[0], q[1], &mut vy); + for cc in 0..ch { + let t = p.raw[k] as f64; + k += 1; + let (ix, iy) = (vx[cc] as f64, vy[cc] as f64); + let r = v[cc] as f64 - g * t - b; + let j: [f64; 6] = if shape { [ix, iy, ix * ux + iy * uy, -ix * uy + iy * ux, -t, -1.0] } else { [ix, iy, -t, -1.0, 0.0, 0.0] }; + for m in 0..np { + rhs[m] -= j[m] * r; + for n in m..np { + h[m][n] += j[m] * j[n]; + } + } + } + } + } + for m in 0..np { + for n in 0..m { + h[m][n] = h[n][m]; + } + h[m][m] *= 1.0 + 1e-6; + h[m][m] += 1e-9; + } + let d = if shape { + solve::solve_linear(h, rhs)? + } else { + let hh = [ + [h[0][0], h[0][1], h[0][2], h[0][3]], + [h[1][0], h[1][1], h[1][2], h[1][3]], + [h[2][0], h[2][1], h[2][2], h[2][3]], + [h[3][0], h[3][1], h[3][2], h[3][3]], + ]; + let x = solve::solve_linear(hh, [rhs[0], rhs[1], rhs[2], rhs[3]])?; + [x[0], x[1], 0.0, 0.0, x[2], x[3]] + }; + // Limit a single step (Gauss-Newton overshoot on poorly textured features). + let step = (d[0] * d[0] + d[1] * d[1]).sqrt(); + let k = if step > 1.0 { 1.0 / step } else { 1.0 }; + c[0] += d[0] * k; + c[1] += d[1] * k; + if shape { + a[0] += d[2] * k; + a[1] += d[3] * k; + } + g += d[4] * k; + b += d[5] * k; + if !(c[0].is_finite() && c[1].is_finite() && a[0].is_finite() && a[1].is_finite()) { + return None; + } + if step < 1e-4 && (!shape || (d[2].abs() + d[3].abs()) < 1e-6) { + break; + } + } + let moved = ((c[0] - c0[0]).powi(2) + (c[1] - c0[1]).powi(2)).sqrt(); + let s0 = (a0[0] * a0[0] + a0[1] * a0[1]).sqrt(); + let s1 = (a[0] * a[0] + a[1] * a[1]).sqrt(); + if moved > 2.5 || !(0.75..=1.33).contains(&(s1 / s0.max(1e-9))) || g <= 0.0 { + return None; + } + Some((c, a)) +} + +/// Tracks a set of points frame by frame (the points of one tracker). +#[derive(Clone, Debug)] +pub struct Tracker { + pub opts: TrackOptions, + points: Vec, +} + +impl Tracker { + /// Start tracking `points` as they appear on `frame` (the start frame). + pub fn new(opts: TrackOptions, points: &[PointSpec], frame: &Frame) -> Tracker { + let pts = points.par_iter().map(|s| PointState::new(*s, frame, &opts)).collect(); + Tracker { opts, points: pts } + } + + /// Track every point into the next frame (in either time direction). + pub fn step(&mut self, frame: &Frame) -> Vec { + let opts = &self.opts; + self.points.par_iter_mut().map(|p| p.step(frame, opts)).collect() + } + + pub fn len(&self) -> usize { + self.points.len() + } + pub fn is_empty(&self) -> bool { + self.points.is_empty() + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/track/src/plane.rs b/crates/track/src/plane.rs new file mode 100644 index 00000000..bd2d987d --- /dev/null +++ b/crates/track/src/plane.rs @@ -0,0 +1,243 @@ +//! Single- or multi-channel float planes: the tracker's working images (a crop of the frame in +//! the matching channel, pre-processed), their pyramids and bilinear sampling. +//! +//! Continuous coordinates follow the raster crate: pixel `i` covers `[i, i + 1)` and its centre is +//! at `i + 0.5`. Downsampling by 2 keeps that convention (level-1 coordinate = level-0 / 2). + +use effectcraft_raster::Image; +use serde::{Deserialize, Serialize}; + +/// What the tracker compares (Motion Tracker Options ▸ Channel). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Channel { + /// All three colour channels. + Rgb, + /// Rec. 709 luma. + #[default] + Luminance, + /// HSV saturation. + Saturation, +} + +impl Channel { + pub const ALL: [Channel; 3] = [Channel::Rgb, Channel::Luminance, Channel::Saturation]; + pub fn label(self) -> &'static str { + match self { + Channel::Rgb => "RGB", + Channel::Luminance => "Luminance", + Channel::Saturation => "Saturation", + } + } + pub fn from_name(s: &str) -> Option { + Channel::ALL.into_iter().find(|c| c.label().eq_ignore_ascii_case(s) || format!("{c:?}").eq_ignore_ascii_case(s)) + } + pub fn channels(self) -> usize { + if self == Channel::Rgb { 3 } else { 1 } + } +} + +/// An interleaved float plane. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Plane { + pub w: usize, + pub h: usize, + pub ch: usize, + pub data: Vec, +} + +impl Plane { + pub fn new(w: usize, h: usize, ch: usize) -> Plane { + Plane { w, h, ch, data: vec![0.0; w * h * ch] } + } + + #[inline] + fn idx(&self, x: usize, y: usize) -> usize { + (y * self.w + x) * self.ch + } + + /// Value at integer pixel `(x, y)` with edges repeated. + #[inline] + pub fn at(&self, x: isize, y: isize, c: usize) -> f32 { + let x = x.clamp(0, self.w as isize - 1) as usize; + let y = y.clamp(0, self.h as isize - 1) as usize; + self.data[self.idx(x, y) + c] + } + + /// Bilinear sample at continuous coordinates (edges repeated) into `out[..ch]`. + #[inline] + pub fn sample(&self, x: f64, y: f64, out: &mut [f32]) { + let fx = x - 0.5; + let fy = y - 0.5; + let x0 = fx.floor(); + let y0 = fy.floor(); + let tx = (fx - x0) as f32; + let ty = (fy - y0) as f32; + let (x0, y0) = (x0 as isize, y0 as isize); + let inside = x0 >= 0 && y0 >= 0 && (x0 as usize) + 1 < self.w && (y0 as usize) + 1 < self.h; + if inside { + let i00 = self.idx(x0 as usize, y0 as usize); + let i10 = i00 + self.ch; + let i01 = i00 + self.w * self.ch; + let i11 = i01 + self.ch; + for c in 0..self.ch { + let top = self.data[i00 + c] + (self.data[i10 + c] - self.data[i00 + c]) * tx; + let bot = self.data[i01 + c] + (self.data[i11 + c] - self.data[i01 + c]) * tx; + out[c] = top + (bot - top) * ty; + } + } else { + for c in 0..self.ch { + let a = self.at(x0, y0, c); + let b = self.at(x0 + 1, y0, c); + let cc = self.at(x0, y0 + 1, c); + let d = self.at(x0 + 1, y0 + 1, c); + let top = a + (b - a) * tx; + let bot = cc + (d - cc) * tx; + out[c] = top + (bot - top) * ty; + } + } + } + + /// Crop `w × h` pixels at `(x0, y0)` of `img` in the given channel (edges repeated outside). + pub fn from_image(img: &Image, channel: Channel, x0: i64, y0: i64, w: usize, h: usize) -> Plane { + let ch = channel.channels(); + let mut p = Plane::new(w, h, ch); + if img.is_empty() { + return p; + } + for y in 0..h { + for x in 0..w { + let px = img.get_clamped(x0 + x as i64, y0 + y as i64); + let i = (y * w + x) * ch; + match channel { + Channel::Rgb => p.data[i..i + 3].copy_from_slice(&px[..3]), + Channel::Luminance => p.data[i] = 0.2126 * px[0] + 0.7152 * px[1] + 0.0722 * px[2], + Channel::Saturation => { + let mx = px[0].max(px[1]).max(px[2]); + let mn = px[0].min(px[1]).min(px[2]); + p.data[i] = if mx > 1e-6 { (mx - mn) / mx } else { 0.0 }; + } + } + } + } + p + } + + /// Separable Gaussian blur (edges repeated). + pub fn gaussian(&self, sigma: f64) -> Plane { + if sigma <= 0.05 { + return self.clone(); + } + let r = (sigma * 3.0).ceil() as isize; + let mut k: Vec = (-r..=r).map(|i| (-(i * i) as f64 / (2.0 * sigma * sigma)).exp() as f32).collect(); + let s: f32 = k.iter().sum(); + k.iter_mut().for_each(|v| *v /= s); + let mut tmp = Plane::new(self.w, self.h, self.ch); + for y in 0..self.h { + for x in 0..self.w { + for c in 0..self.ch { + let mut acc = 0.0; + for (j, kv) in k.iter().enumerate() { + acc += kv * self.at(x as isize + j as isize - r, y as isize, c); + } + let i = tmp.idx(x, y) + c; + tmp.data[i] = acc; + } + } + } + let mut out = Plane::new(self.w, self.h, self.ch); + for y in 0..self.h { + for x in 0..self.w { + for c in 0..self.ch { + let mut acc = 0.0; + for (j, kv) in k.iter().enumerate() { + acc += kv * tmp.at(x as isize, y as isize + j as isize - r, c); + } + let i = out.idx(x, y) + c; + out.data[i] = acc; + } + } + } + out + } + + /// Edge enhancement (unsharp mask): `x + 2 (x - blur(x))`. + pub fn enhance(&self) -> Plane { + let b = self.gaussian(1.5); + let mut out = self.clone(); + for (o, (x, y)) in out.data.iter_mut().zip(self.data.iter().zip(&b.data)) { + *o = x + 2.0 * (x - y); + } + out + } + + /// Half resolution (2 × 2 box average; odd sizes round up with the edge repeated). + pub fn downsample(&self) -> Plane { + let w = self.w.div_ceil(2).max(1); + let h = self.h.div_ceil(2).max(1); + let mut out = Plane::new(w, h, self.ch); + for y in 0..h { + for x in 0..w { + for c in 0..self.ch { + let (sx, sy) = (2 * x as isize, 2 * y as isize); + let v = self.at(sx, sy, c) + self.at(sx + 1, sy, c) + self.at(sx, sy + 1, c) + self.at(sx + 1, sy + 1, c); + let i = out.idx(x, y) + c; + out.data[i] = v * 0.25; + } + } + } + out + } + + /// Central-difference gradients `(d/dx, d/dy)`. + pub fn gradients(&self) -> (Plane, Plane) { + let mut gx = Plane::new(self.w, self.h, self.ch); + let mut gy = Plane::new(self.w, self.h, self.ch); + for y in 0..self.h { + for x in 0..self.w { + for c in 0..self.ch { + let (xi, yi) = (x as isize, y as isize); + let i = self.idx(x, y) + c; + gx.data[i] = 0.5 * (self.at(xi + 1, yi, c) - self.at(xi - 1, yi, c)); + gy.data[i] = 0.5 * (self.at(xi, yi + 1, c) - self.at(xi, yi - 1, c)); + } + } + } + (gx, gy) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sampling_and_pyramid_share_coordinates() { + let mut p = Plane::new(8, 8, 1); + for y in 0..8 { + for x in 0..8 { + p.data[y * 8 + x] = x as f32 + 10.0 * y as f32; + } + } + let mut o = [0.0]; + p.sample(2.5, 3.5, &mut o); + assert_eq!(o[0], 32.0); + p.sample(3.0, 3.5, &mut o); + assert!((o[0] - 32.5).abs() < 1e-6); + // A linear ramp keeps its value at matching coordinates on the next level. + let d = p.downsample(); + d.sample(1.5, 1.5, &mut o); + let mut o0 = [0.0]; + p.sample(3.0, 3.0, &mut o0); + assert!((o[0] - o0[0]).abs() < 1e-5, "{} vs {}", o[0], o0[0]); + } + + #[test] + fn channels() { + let img = Image::filled(2, 2, [1.0, 0.5, 0.0, 1.0]); + assert_eq!(Plane::from_image(&img, Channel::Rgb, 0, 0, 2, 2).data[..3], [1.0, 0.5, 0.0]); + assert!((Plane::from_image(&img, Channel::Luminance, 0, 0, 2, 2).data[0] - (0.2126 + 0.3576)).abs() < 1e-6); + assert_eq!(Plane::from_image(&img, Channel::Saturation, 0, 0, 2, 2).data[0], 1.0); + assert_eq!(Channel::from_name("luminance"), Some(Channel::Luminance)); + } +} diff --git a/crates/track/src/solve.rs b/crates/track/src/solve.rs new file mode 100644 index 00000000..bf3f2eb5 --- /dev/null +++ b/crates/track/src/solve.rs @@ -0,0 +1,176 @@ +//! Small geometric solves for applying tracks: a 2-point similarity (Transform track rotation and +//! scale), a 3-point affine (Affine Corner Pin) and a 4-point homography (Perspective Corner Pin, +//! the direct linear transform with `h33 = 1`, Hartley & Zisserman §4.1). + +/// Solve `a x = b` (n × n, row-major) by Gaussian elimination with partial pivoting. +pub fn solve_linear(mut a: [[f64; N]; N], mut b: [f64; N]) -> Option<[f64; N]> { + for col in 0..N { + let piv = (col..N).max_by(|&i, &j| a[i][col].abs().total_cmp(&a[j][col].abs()))?; + if a[piv][col].abs() < 1e-12 { + return None; + } + a.swap(col, piv); + b.swap(col, piv); + for r in col + 1..N { + let f = a[r][col] / a[col][col]; + if f == 0.0 { + continue; + } + for c in col..N { + a[r][c] -= f * a[col][c]; + } + b[r] -= f * b[col]; + } + } + let mut x = [0.0; N]; + for r in (0..N).rev() { + let mut s = b[r]; + for c in r + 1..N { + s -= a[r][c] * x[c]; + } + x[r] = s / a[r][r]; + } + x.iter().all(|v| v.is_finite()).then_some(x) +} + +/// A 3 × 3 projective transform (row-major), mapping `[x, y, 1]`. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Homography(pub [[f64; 3]; 3]); + +impl Homography { + pub const IDENTITY: Homography = Homography([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]); + + pub fn apply(&self, p: [f64; 2]) -> [f64; 2] { + let m = &self.0; + let w = m[2][0] * p[0] + m[2][1] * p[1] + m[2][2]; + let w = if w.abs() < 1e-12 { 1e-12 } else { w }; + [(m[0][0] * p[0] + m[0][1] * p[1] + m[0][2]) / w, (m[1][0] * p[0] + m[1][1] * p[1] + m[1][2]) / w] + } + + /// The homography taking `src[i]` to `dst[i]` (four points, no three collinear). + pub fn from_points(src: [[f64; 2]; 4], dst: [[f64; 2]; 4]) -> Option { + // Normalise both sets (centroid at 0, mean distance √2) for conditioning. + let (ns, ts) = normalise(&src); + let (nd, td) = normalise(&dst); + let mut a = [[0.0; 8]; 8]; + let mut b = [0.0; 8]; + for i in 0..4 { + let ([x, y], [u, v]) = (ns[i], nd[i]); + a[2 * i] = [x, y, 1.0, 0.0, 0.0, 0.0, -u * x, -u * y]; + b[2 * i] = u; + a[2 * i + 1] = [0.0, 0.0, 0.0, x, y, 1.0, -v * x, -v * y]; + b[2 * i + 1] = v; + } + let h = solve_linear(a, b)?; + let hn = [[h[0], h[1], h[2]], [h[3], h[4], h[5]], [h[6], h[7], 1.0]]; + // H = Td⁻¹ · Hn · Ts + let m = mul(inv_similarity(td), mul(hn, ts)); + let s = m[2][2]; + if s.abs() < 1e-12 { + return None; + } + Some(Homography(m.map(|r| r.map(|v| v / s)))) + } + + /// The affine map taking three points to three points. + pub fn affine_from_points(src: [[f64; 2]; 3], dst: [[f64; 2]; 3]) -> Option { + let mut a = [[0.0; 3]; 3]; + for i in 0..3 { + a[i] = [src[i][0], src[i][1], 1.0]; + } + let x = solve_linear(a, [dst[0][0], dst[1][0], dst[2][0]])?; + let y = solve_linear(a, [dst[0][1], dst[1][1], dst[2][1]])?; + Some(Homography([x, y, [0.0, 0.0, 1.0]])) + } + + /// The similarity (rotation, uniform scale, translation) taking two points to two points. + pub fn similarity_from_points(src: [[f64; 2]; 2], dst: [[f64; 2]; 2]) -> Option { + let (ds, dd) = (sub(src[1], src[0]), sub(dst[1], dst[0])); + let n = ds[0] * ds[0] + ds[1] * ds[1]; + if n < 1e-12 { + return None; + } + // dd = [[a, -b], [b, a]] ds + let a = (dd[0] * ds[0] + dd[1] * ds[1]) / n; + let b = (dd[1] * ds[0] - dd[0] * ds[1]) / n; + let tx = dst[0][0] - (a * src[0][0] - b * src[0][1]); + let ty = dst[0][1] - (b * src[0][0] + a * src[0][1]); + Some(Homography([[a, -b, tx], [b, a, ty], [0.0, 0.0, 1.0]])) + } +} + +/// Angle (degrees, y down = clockwise positive like After Effects' Rotation) and length of `b - a`. +pub fn angle_and_length(a: [f64; 2], b: [f64; 2]) -> (f64, f64) { + let d = sub(b, a); + (d[1].atan2(d[0]).to_degrees(), (d[0] * d[0] + d[1] * d[1]).sqrt()) +} + +/// `angle` moved by multiples of 360° to be closest to `prev` (continuous rotation curves). +pub fn unwrap_degrees(prev: f64, angle: f64) -> f64 { + angle + ((prev - angle) / 360.0).round() * 360.0 +} + +fn sub(a: [f64; 2], b: [f64; 2]) -> [f64; 2] { + [a[0] - b[0], a[1] - b[1]] +} + +fn mul(a: [[f64; 3]; 3], b: [[f64; 3]; 3]) -> [[f64; 3]; 3] { + let mut o = [[0.0; 3]; 3]; + for i in 0..3 { + for j in 0..3 { + o[i][j] = (0..3).map(|k| a[i][k] * b[k][j]).sum(); + } + } + o +} + +fn normalise(p: &[[f64; 2]; 4]) -> ([[f64; 2]; 4], [[f64; 3]; 3]) { + let c = [p.iter().map(|q| q[0]).sum::() / 4.0, p.iter().map(|q| q[1]).sum::() / 4.0]; + let d = p.iter().map(|q| ((q[0] - c[0]).powi(2) + (q[1] - c[1]).powi(2)).sqrt()).sum::() / 4.0; + let s = if d > 1e-12 { std::f64::consts::SQRT_2 / d } else { 1.0 }; + let t = [[s, 0.0, -s * c[0]], [0.0, s, -s * c[1]], [0.0, 0.0, 1.0]]; + (p.map(|q| [s * (q[0] - c[0]), s * (q[1] - c[1])]), t) +} + +fn inv_similarity(t: [[f64; 3]; 3]) -> [[f64; 3]; 3] { + let s = t[0][0]; + [[1.0 / s, 0.0, -t[0][2] / s], [0.0, 1.0 / s, -t[1][2] / s], [0.0, 0.0, 1.0]] +} + +#[cfg(test)] +mod tests { + use super::*; + + fn close(a: [f64; 2], b: [f64; 2], tol: f64) -> bool { + (a[0] - b[0]).abs() < tol && (a[1] - b[1]).abs() < tol + } + + #[test] + fn homography_recovers_known_projection() { + let h = Homography([[0.9, 0.12, 30.0], [-0.08, 1.1, -12.0], [0.0004, -0.0002, 1.0]]); + let src = [[100.0, 100.0], [700.0, 120.0], [680.0, 500.0], [90.0, 520.0]]; + let dst = src.map(|p| h.apply(p)); + let got = Homography::from_points(src, dst).unwrap(); + for i in 0..3 { + for j in 0..3 { + assert!((got.0[i][j] - h.0[i][j]).abs() < 1e-6 * (1.0 + h.0[i][j].abs()), "{:?}", got.0); + } + } + for p in [[0.0, 0.0], [400.0, 300.0], [1000.0, 50.0]] { + assert!(close(got.apply(p), h.apply(p), 1e-6)); + } + // Degenerate (collinear) input has no solution. + assert!(Homography::from_points([[0.0, 0.0], [1.0, 1.0], [2.0, 2.0], [3.0, 3.0]], dst).is_none()); + } + + #[test] + fn affine_and_similarity() { + let a = Homography::affine_from_points([[0.0, 0.0], [10.0, 0.0], [0.0, 10.0]], [[5.0, 5.0], [15.0, 7.0], [3.0, 15.0]]).unwrap(); + assert!(close(a.apply([10.0, 10.0]), [13.0, 17.0], 1e-9)); + let s = Homography::similarity_from_points([[0.0, 0.0], [10.0, 0.0]], [[1.0, 1.0], [1.0, 21.0]]).unwrap(); + assert!(close(s.apply([0.0, 10.0]), [-19.0, 1.0], 1e-9)); + let (ang, len) = angle_and_length([1.0, 1.0], [1.0, 21.0]); + assert!((ang - 90.0).abs() < 1e-9 && (len - 20.0).abs() < 1e-9); + assert!((unwrap_degrees(350.0, -5.0) - 355.0).abs() < 1e-9); + } +} diff --git a/crates/track/src/tests.rs b/crates/track/src/tests.rs new file mode 100644 index 00000000..1e91e2e6 --- /dev/null +++ b/crates/track/src/tests.rs @@ -0,0 +1,256 @@ +//! Synthetic footage: a textured patch moving along a known path (with rotation and scale) over a +//! textured background, with noise. Ground truth is exact, so recovered positions are compared +//! against it directly. + +use super::*; + +/// Smooth, band-limited texture (periods of 9–40 px). +fn texture(x: f64, y: f64, seed: u32) -> [f32; 3] { + let mut v = [0.0f64; 3]; + let mut s = seed.wrapping_mul(2654435761).wrapping_add(12345); + let mut rnd = || { + s ^= s << 13; + s ^= s >> 17; + s ^= s << 5; + (s % 10_000) as f64 / 10_000.0 + }; + for _ in 0..7 { + let ang = rnd() * std::f64::consts::TAU; + let period = 9.0 + rnd() * 31.0; + let phase = rnd() * std::f64::consts::TAU; + let k = std::f64::consts::TAU / period; + let w = (k * (x * ang.cos() + y * ang.sin()) + phase).sin(); + let c = [rnd(), rnd(), rnd()]; + for i in 0..3 { + v[i] += w * (0.3 + c[i]) * 0.12; + } + } + v.map(|q| (0.5 + q).clamp(0.0, 1.0) as f32) +} + +fn noise(x: u32, y: u32, f: u32) -> f32 { + effectcraft_raster::hash_noise(x, y.wrapping_add(f.wrapping_mul(7919)), 99) - 0.5 +} + +/// Patch pose on a frame: centre, rotation (degrees), scale. +#[derive(Clone, Copy)] +struct Pose { + c: [f64; 2], + rot: f64, + scale: f64, +} + +impl Pose { + /// Where the patch-local point `u` lands in the frame. + fn map(&self, u: [f64; 2]) -> [f64; 2] { + let (s, c) = self.rot.to_radians().sin_cos(); + [self.c[0] + self.scale * (c * u[0] - s * u[1]), self.c[1] + self.scale * (s * u[0] + c * u[1])] + } + fn local(&self, p: [f64; 2]) -> [f64; 2] { + let (s, c) = self.rot.to_radians().sin_cos(); + let d = [(p[0] - self.c[0]) / self.scale, (p[1] - self.c[1]) / self.scale]; + [c * d[0] + s * d[1], -s * d[0] + c * d[1]] + } +} + +/// Render a frame: background texture, the patch (half size `half`) on top, Gaussian-ish noise +/// of amplitude `amp`, and an optional flat occluder rectangle. +fn frame(w: u32, h: u32, pose: Pose, half: f64, amp: f32, f: u32, occluder: Option<[f64; 4]>) -> Image { + let mut img = Image::new(w, h); + let ss = [0.25, 0.75]; + for y in 0..h { + for x in 0..w { + let mut acc = [0.0f32; 3]; + for sy in ss { + for sx in ss { + let p = [x as f64 + sx, y as f64 + sy]; + let l = pose.local(p); + let t = if l[0].abs() <= half && l[1].abs() <= half { texture(l[0], l[1], 7) } else { texture(p[0] * 0.7, p[1] * 0.7, 3).map(|v| v * 0.6) }; + for i in 0..3 { + acc[i] += t[i] * 0.25; + } + } + } + let n = noise(x, y, f) * amp; + let mut px = [acc[0] + n, acc[1] + n, acc[2] + n, 1.0]; + if let Some([ox, oy, ow, oh]) = occluder { + let (cx, cy) = (x as f64 + 0.5, y as f64 + 0.5); + if cx >= ox && cx < ox + ow && cy >= oy && cy < oy + oh { + px = [0.4, 0.4, 0.4, 1.0]; + } + } + img.set(x, y, px); + } + } + img +} + +fn spec(c: [f64; 2], feature: f64, search: f64) -> PointSpec { + PointSpec { center: c, feature_size: [feature, feature], search_offset: [0.0, 0.0], search_size: [search, search] } +} + +fn dist(a: [f64; 2], b: [f64; 2]) -> f64 { + ((a[0] - b[0]).powi(2) + (a[1] - b[1]).powi(2)).sqrt() +} + +#[test] +fn recovers_translation_path_subpixel() { + let path = |f: usize| { + let t = f as f64; + Pose { c: [100.0 + 4.37 * t + 6.0 * (t * 0.3).sin(), 90.0 + 2.71 * t - 3.0 * (t * 0.21).cos()], rot: 0.0, scale: 1.0 } + }; + let f0 = frame(320, 240, path(0), 40.0, 0.02, 0, None); + let mut tr = Tracker::new(TrackOptions::default(), &[spec(path(0).c, 31.0, 71.0)], &Frame::new(&f0)); + let mut worst: f64 = 0.0; + for f in 1..24 { + let img = frame(320, 240, path(f), 40.0, 0.02, f as u32, None); + let r = tr.step(&Frame::new(&img))[0]; + let e = dist(r.center, path(f).c); + worst = worst.max(e); + assert!(r.confidence > 90.0, "frame {f}: confidence {}", r.confidence); + assert_eq!(r.status, Status::Tracked); + } + eprintln!("worst error {worst:.4} px"); + assert!(worst < 0.25, "worst error {worst} px"); +} + +#[test] +fn layer_offset_and_channels() { + let pose = |f: usize| Pose { c: [120.0 + 3.3 * f as f64, 100.0 - 1.7 * f as f64], rot: 0.0, scale: 1.0 }; + for channel in Channel::ALL { + for (blur, enhance) in [(0.0, false), (3.0, false), (0.0, true)] { + let opts = TrackOptions { channel, blur, enhance, ..Default::default() }; + let f0 = frame(240, 200, pose(0), 36.0, 0.01, 0, None); + // Layer coordinates are image pixels minus the offset. + let off = [10.0, -5.0]; + let c0 = [pose(0).c[0] - off[0], pose(0).c[1] - off[1]]; + let mut tr = Tracker::new(opts, &[spec(c0, 29.0, 61.0)], &Frame { img: &f0, offset: off }); + for f in 1..6 { + let img = frame(240, 200, pose(f), 36.0, 0.01, f as u32, None); + let r = tr.step(&Frame { img: &img, offset: off })[0]; + let want = [pose(f).c[0] - off[0], pose(f).c[1] - off[1]]; + assert!(dist(r.center, want) < 0.3, "{channel:?} blur {blur} enhance {enhance} frame {f}: {:?} vs {want:?}", r.center); + } + } + } +} + +#[test] +fn two_points_recover_rotation_and_scale() { + let pose = |f: usize| Pose { c: [160.0 + 1.5 * f as f64, 120.0 + 0.8 * f as f64], rot: 1.8 * f as f64, scale: 1.0 + 0.01 * f as f64 }; + let u = [[-26.0, 0.0], [26.0, 0.0]]; + let p0 = pose(0); + let f0 = frame(320, 240, p0, 52.0, 0.01, 0, None); + let mut tr = Tracker::new(TrackOptions::default(), &u.map(|q| spec(p0.map(q), 23.0, 51.0)), &Frame::new(&f0)); + let (a0, l0) = angle_and_length(p0.map(u[0]), p0.map(u[1])); + let mut worst: f64 = 0.0; + for f in 1..20 { + let p = pose(f); + let img = frame(320, 240, p, 52.0, 0.01, f as u32, None); + let r = tr.step(&Frame::new(&img)); + for k in 0..2 { + worst = worst.max(dist(r[k].center, p.map(u[k]))); + } + let (a, l) = angle_and_length(r[0].center, r[1].center); + assert!((unwrap_degrees(0.0, a - a0) - p.rot).abs() < 0.5, "frame {f}: rotation {} vs {}", a - a0, p.rot); + assert!((l / l0 - p.scale).abs() < 0.01, "frame {f}: scale {} vs {}", l / l0, p.scale); + // The per-feature shape estimate follows too. + assert!((r[0].rotation - p.rot).abs() < 1.0, "frame {f}: feature rotation {}", r[0].rotation); + assert!((r[0].scale - p.scale).abs() < 0.02, "frame {f}: feature scale {}", r[0].scale); + } + eprintln!("worst error {worst:.4} px"); + assert!(worst < 0.25, "worst error {worst} px"); +} + +#[test] +fn confidence_drops_on_occlusion_and_actions() { + let pose = |f: usize| Pose { c: [110.0 + 2.0 * f as f64, 100.0], rot: 0.0, scale: 1.0 }; + let occ = |f: usize| (f == 5).then(|| [pose(f).c[0] - 12.0, pose(f).c[1] - 30.0, 40.0, 60.0]); + let run = |action: ConfidenceAction| { + let f0 = frame(240, 200, pose(0), 36.0, 0.01, 0, None); + let opts = TrackOptions { action, ..Default::default() }; + let mut tr = Tracker::new(opts, &[spec(pose(0).c, 31.0, 61.0)], &Frame::new(&f0)); + (1..9) + .map(|f| { + let img = frame(240, 200, pose(f), 36.0, 0.01, f as u32, occ(f)); + tr.step(&Frame::new(&img))[0] + }) + .collect::>() + }; + let r = run(ConfidenceAction::Continue); + assert!(r[3].confidence > 95.0, "{}", r[3].confidence); + assert!(r[4].confidence < 70.0, "occluded frame confidence {}", r[4].confidence); + let r = run(ConfidenceAction::Extrapolate); + assert_eq!(r[4].status, Status::Extrapolated); + assert!(dist(r[4].center, pose(5).c) < 0.5, "extrapolated {:?}", r[4].center); + // Back on track after the occlusion. + assert!(dist(r[6].center, pose(7).c) < 0.25 && r[6].status == Status::Tracked); + let r = run(ConfidenceAction::Stop); + assert_eq!(r[4].status, Status::Stopped); + let r = run(ConfidenceAction::Adapt); + assert_eq!(r[4].status, Status::Adapted); +} + +#[test] +fn backward_and_whole_pixel() { + let pose = |f: usize| Pose { c: [100.0 + 3.25 * f as f64, 100.0 + 1.5 * f as f64], rot: 0.0, scale: 1.0 }; + let last = 6; + let fl = frame(240, 200, pose(last), 36.0, 0.0, 0, None); + let opts = TrackOptions { subpixel: false, ..Default::default() }; + let mut tr = Tracker::new(opts, &[spec(pose(last).c, 31.0, 61.0)], &Frame::new(&fl)); + for f in (0..last).rev() { + let img = frame(240, 200, pose(f), 36.0, 0.0, 0, None); + let r = tr.step(&Frame::new(&img))[0]; + let want = pose(f).c; + // Whole-pixel displacements of the start position. + let d = [r.center[0] - pose(last).c[0], r.center[1] - pose(last).c[1]]; + assert!((d[0] - d[0].round()).abs() < 1e-9 && (d[1] - d[1].round()).abs() < 1e-9); + assert!(dist(r.center, want) < 0.75, "frame {f}: {:?} vs {want:?}", r.center); + } +} + +/// 1080p, one point: tracking cost per frame (frames are generated outside the timing). +#[test] +fn performance_1080p_single_point() { + let w = 1920; + let h = 1080; + // A cheap textured frame (the full synthetic renderer is slow at 1080p in debug builds). + let make = |dx: f64, dy: f64| { + let mut img = Image::new(w, h); + for y in 0..h { + for x in 0..w { + let (fx, fy) = (x as f64 + 0.5 - dx, y as f64 + 0.5 - dy); + let v = 0.5 + 0.2 * (fx * 0.31).sin() * (fy * 0.17).cos() + 0.15 * ((fx + fy) * 0.11).sin() + 0.1 * ((fx * 0.05).sin() * (fy * 0.23).sin()); + img.set(x, y, [v as f32, v as f32, v as f32, 1.0]); + } + } + img + }; + let frames: Vec = (0..6).map(|f| make(2.3 * f as f64, 1.1 * f as f64)).collect(); + let c0 = [960.0, 540.0]; + let s = PointSpec { center: c0, feature_size: [48.0, 48.0], search_offset: [0.0; 2], search_size: [128.0, 128.0] }; + let t0 = std::time::Instant::now(); + let mut tr = Tracker::new(TrackOptions::default(), &[s], &Frame::new(&frames[0])); + let mut last = PointResult { center: c0, confidence: 0.0, rotation: 0.0, scale: 1.0, status: Status::Tracked }; + for f in frames.iter().skip(1) { + last = tr.step(&Frame::new(f))[0]; + } + let per = t0.elapsed().as_secs_f64() / 5.0; + eprintln!("1080p single-point track: {:.2} ms/frame ({:.0} fps)", per * 1e3, 1.0 / per); + assert!(dist(last.center, [c0[0] + 2.3 * 5.0, c0[1] + 1.1 * 5.0]) < 0.25, "{:?}", last.center); + assert!(per < 0.1, "{per} s per frame"); +} + +#[test] +fn featureless_regions_stay_put() { + let flat = Image::filled(200, 160, [0.3, 0.3, 0.3, 1.0]); + for action in ConfidenceAction::ALL { + let opts = TrackOptions { action, ..Default::default() }; + let mut tr = Tracker::new(opts, &[spec([100.0, 80.0], 21.0, 61.0)], &Frame::new(&flat)); + for _ in 0..10 { + let r = tr.step(&Frame::new(&flat))[0]; + assert!(dist(r.center, [100.0, 80.0]) < 1e-9, "{action:?}: drifted to {:?}", r.center); + assert!(r.confidence < 1.0); + } + } +} diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index e1e110f4..0dbb0020 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -57,6 +57,12 @@ pub enum Dialog { KeyVelocity, KeyInterpolation, TimeStretch, + /// Tracker ▸ Options… + TrackOptions, + /// Tracker ▸ Edit Target… + TrackTarget, + /// Tracker ▸ Apply (Transform / Stabilize): Apply Dimensions. + TrackApply, } /// Host hooks provided by the native app (file pickers etc.). @@ -536,6 +542,10 @@ impl EffectcraftApp { self.session.poll_render(); ctx.request_repaint_after(std::time::Duration::from_millis(100)); } + if self.session.track_job.is_some() { + self.session.poll_track(); + ctx.request_repaint_after(std::time::Duration::from_millis(50)); + } self.handle_events(&ctx); if let Some(rx) = self.command_inbox.take() { while let Ok(id) = rx.try_recv() { diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index a81ac54a..a938ab4a 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -446,6 +446,14 @@ pub fn frontend(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, p: Valu app.show_panel(PanelKind::Flowchart); Value::Null } + "track.editTargetDialog" => { + crate::panels::tracker::open_target(app)?; + Value::Null + } + "track.optionsDialog" => { + crate::panels::tracker::open_options(app)?; + Value::Null + } "layer.openLayer" => { app.show_panel(PanelKind::Layer); Value::Null diff --git a/crates/ui-egui/src/panels/dialogs.rs b/crates/ui-egui/src/panels/dialogs.rs index a92c3db3..3e8ae984 100644 --- a/crates/ui-egui/src/panels/dialogs.rs +++ b/crates/ui-egui/src/panels/dialogs.rs @@ -31,6 +31,9 @@ pub struct DialogState { pub velocity: super::key_dialogs::VelocityDraft, pub interp: super::key_dialogs::InterpDraft, pub stretch: super::key_dialogs::StretchDraft, + pub track_options: super::tracker::OptionsDraft, + pub track_target: super::tracker::TargetDraft, + pub track_apply: super::tracker::ApplyDraft, } pub fn open_new_comp(app: &mut EffectcraftApp) { @@ -108,6 +111,9 @@ pub fn show(app: &mut EffectcraftApp, ctx: &egui::Context) { Dialog::KeyVelocity => super::key_dialogs::velocity(app, ctx, &t), Dialog::KeyInterpolation => super::key_dialogs::interpolation(app, ctx, &t), Dialog::TimeStretch => super::key_dialogs::time_stretch(app, ctx, &t), + Dialog::TrackOptions => super::tracker::options_dialog(app, ctx, &t), + Dialog::TrackTarget => super::tracker::target_dialog(app, ctx, &t), + Dialog::TrackApply => super::tracker::apply_dialog(app, ctx, &t), } } diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 22837cb3..ebd30a55 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -16,6 +16,7 @@ pub mod properties; pub mod render_queue; pub mod text_panels; pub mod timeline; +pub mod tracker; pub mod viewer; pub mod waveform; @@ -70,6 +71,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: PanelKind, rect: Rec PanelKind::Markers => misc::markers(app, ui, rect), PanelKind::Wiggler => misc::wiggler(app, ui, rect), PanelKind::RenderQueue => render_queue::show(app, ui, rect), + PanelKind::Tracker => tracker::show(app, ui, rect), other => misc::placeholder(app, ui, rect, other), } } diff --git a/crates/ui-egui/src/panels/tracker.rs b/crates/ui-egui/src/panels/tracker.rs new file mode 100644 index 00000000..ad84665b --- /dev/null +++ b/crates/ui-egui/src/panels/tracker.rs @@ -0,0 +1,722 @@ +//! Tracker panel (Window ▸ Tracker), its Motion Tracker Options / Motion Target / Apply Options +//! dialogs, and the track point widgets drawn in the viewer (feature region, search region, +//! attach point crosshair, track path) with their drag gestures. +//! +//! Everything goes through `track.*` commands, so agents can do the same. + +use effectcraft_engine::geom::{Mat3, vec2 as gv2}; +use effectcraft_engine::project::tracking::{LowConfidence, TrackChannel, TrackKind, TrackerOptions}; +use effectcraft_engine::project::{Layer, LayerId, Uid}; +use effectcraft_engine::time::Tick; +use effectcraft_engine::tracking::point_values; +use egui::{Align2, Color32, Pos2, Rect, Sense, Stroke, StrokeKind, pos2, vec2}; +use serde_json::json; + +use super::viewer::ViewerMap; +use crate::theme::Tokens; +use crate::{Dialog, EffectcraftApp, widgets}; + +const ROW: f32 = 26.0; + +/// Motion Tracker Options dialog state. +#[derive(Clone, Debug, Default)] +pub struct OptionsDraft { + pub name: String, + pub opts: TrackerOptions, + pub blur_on: bool, +} + +/// Motion Target dialog state. +#[derive(Clone, Debug, Default)] +pub struct TargetDraft { + pub target: Option, +} + +/// Apply Options dialog state (0 = X and Y, 1 = X only, 2 = Y only). +#[derive(Clone, Debug, Default)] +pub struct ApplyDraft { + pub dims: usize, +} + +/// The current track: (tracked layer, tracker uid), resolved like the engine does. +fn current(app: &EffectcraftApp) -> Option<(LayerId, Uid)> { + let comp = app.session.active_comp()?; + if let Some((l, t)) = app.session.state.current_track + && comp.layer(l).and_then(|l| l.tracker(t)).is_some() + { + return Some((l, t)); + } + let src = motion_source(app)?; + let l = comp.layer(src)?; + l.trackers().next().map(|(g, _)| (l.id, g.uid)) +} + +fn source_id() -> egui::Id { + egui::Id::new("tracker-motion-source") +} + +/// Motion Source: the current track's layer, the chosen source, or the selected layer. +fn motion_source(app: &EffectcraftApp) -> Option { + let comp = app.session.active_comp()?; + if let Some((l, _)) = app.session.state.current_track + && comp.layer(l).is_some() + { + return Some(l); + } + let chosen: Option = app.ui.tracker_source; + if let Some(l) = chosen.map(LayerId).filter(|l| comp.layer(*l).is_some()) { + return Some(l); + } + app.session.state.selected_layers.first().copied().filter(|l| comp.layer(*l).is_some_and(trackable)) +} + +fn trackable(l: &Layer) -> bool { + use effectcraft_engine::project::LayerSource; + l.has_video() && !matches!(l.source, LayerSource::Text | LayerSource::Shape) +} + +fn label(p: &egui::Painter, pos: Pos2, s: &str, t: &Tokens) { + p.text(pos, Align2::LEFT_CENTER, s, Tokens::ui(12.0), t.text_dim); +} + +/// A button that can be disabled (drawn dim and inert). +fn button(app: &mut EffectcraftApp, ui: &mut egui::Ui, r: Rect, text: &str, enabled: bool, id: &str) -> bool { + let t = app.tokens; + app.auto.add(id, r, text); + if !enabled { + ui.painter().rect_filled(r, r.height() / 2.0, Color32::from_rgb(0x2a, 0x2a, 0x2a)); + ui.painter().text(r.center(), Align2::CENTER_CENTER, text, Tokens::medium(12.0), t.text_faint); + return false; + } + widgets::text_button(ui, r, text, false, &t, egui::Id::new(id)).clicked() +} + +fn run(app: &mut EffectcraftApp, ctx: &egui::Context, cmd: &str, p: serde_json::Value) { + if let Err(e) = crate::menus::invoke(app, ctx, cmd, p) { + app.ui.status = e; + } +} + +/// Analyze button glyphs: ◀| ◀ ▶ |▶ (and ■ while analysing in that direction). +fn analyze_glyph(p: &egui::Painter, r: Rect, forward: bool, frame: bool, stop: bool, col: Color32) { + let c = r.center(); + if stop { + p.rect_filled(Rect::from_center_size(c, vec2(9.0, 9.0)), 1.0, col); + return; + } + let s = if forward { 1.0 } else { -1.0 }; + let tri = vec![pos2(c.x - 4.0 * s, c.y - 5.0), pos2(c.x + 4.0 * s, c.y), pos2(c.x - 4.0 * s, c.y + 5.0)]; + p.add(egui::Shape::convex_polygon(tri, col, Stroke::NONE)); + if frame { + let x = c.x + 5.5 * s; + p.line_segment([pos2(x, c.y - 5.0), pos2(x, c.y + 5.0)], Stroke::new(1.6, col)); + } +} + +pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let ctx = ui.ctx().clone(); + if app.session.track_job.is_some() { + app.session.poll_track(); + ctx.request_repaint_after(std::time::Duration::from_millis(50)); + } + let p = ui.painter().with_clip_rect(rect); + let x0 = rect.min.x + 10.0; + let w = (rect.width() - 20.0).max(120.0); + let half = (w - 6.0) / 2.0; + let mut y = rect.min.y + 10.0; + let has_comp = app.session.active_comp().is_some(); + let source = motion_source(app); + let cur = current(app); + let can_track = has_comp && source.is_some(); + + // Track Camera / Warp Stabilizer (not available) and Track Motion / Stabilize Motion. + let r1 = Rect::from_min_size(pos2(x0, y), vec2(half, 22.0)); + let r2 = Rect::from_min_size(pos2(x0 + half + 6.0, y), vec2(half, 22.0)); + button(app, ui, r1, "Track Camera", false, "tracker.trackCamera"); + button(app, ui, r2, "Warp Stabilizer", false, "tracker.warpStabilizer"); + y += 28.0; + let r1 = Rect::from_min_size(pos2(x0, y), vec2(half, 22.0)); + let r2 = Rect::from_min_size(pos2(x0 + half + 6.0, y), vec2(half, 22.0)); + let src_p = source.map(|l| json!({"layer": l.0})).unwrap_or_else(|| json!({})); + if button(app, ui, r1, "Track Motion", can_track, "tracker.trackMotion") { + run(app, &ctx, "track.motion", src_p.clone()); + } + if button(app, ui, r2, "Stabilize Motion", can_track, "tracker.stabilizeMotion") { + run(app, &ctx, "track.stabilize", src_p.clone()); + } + y += 34.0; + let Some(comp) = app.session.active_comp().cloned() else { + p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, "Open a composition to track motion.", Tokens::ui(12.0), t.text_faint); + return; + }; + let field_x = x0 + 100.0; + let field_w = (w - 100.0).max(80.0); + + // Motion Source. + label(&p, pos2(x0, y + 10.0), "Motion Source:", &t); + let srcs: Vec<(LayerId, String)> = + comp.layers.iter().enumerate().filter(|(_, l)| trackable(l)).map(|(i, l)| (l.id, format!("{}. {}", i + 1, l.name))).collect(); + let dd = Rect::from_min_size(pos2(field_x, y), vec2(field_w, 20.0)); + let name = source.and_then(|s| srcs.iter().find(|(l, _)| *l == s)).map(|(_, n)| n.clone()).unwrap_or_else(|| "None".into()); + let did = egui::Id::new("tracker-source-dd"); + if widgets::dropdown(ui, dd, &name, &t, did).clicked() { + widgets::open_popup(ui, did); + } + app.auto.add("tracker.motionSource", dd, "Motion Source"); + let mut opts: Vec = vec!["None".into()]; + opts.extend(srcs.iter().map(|(_, n)| n.clone())); + let cur_i = source.and_then(|s| srcs.iter().position(|(l, _)| *l == s)).map(|i| i + 1).or(Some(0)); + if let Some(i) = widgets::popup_menu(ui, did, dd.left_bottom(), &opts, cur_i) { + let chosen = i.checked_sub(1).and_then(|i| srcs.get(i)).map(|(l, _)| *l); + app.ui.tracker_source = chosen.map(|l| l.0); + ctx.data_mut(|d| d.remove::(source_id())); + match chosen.and_then(|l| comp.layer(l)).and_then(|l| l.trackers().next().map(|(g, _)| (l.id, g.uid))) { + Some((l, g)) => run(app, &ctx, "track.select", json!({"layer": l.0, "tracker": g})), + None => app.session.state.current_track = None, + } + } + y += ROW; + + // Current Track. + label(&p, pos2(x0, y + 10.0), "Current Track:", &t); + let trackers: Vec<(Uid, String)> = + source.and_then(|s| comp.layer(s)).map(|l| l.trackers().map(|(g, _)| (g.uid, g.name.clone())).collect()).unwrap_or_default(); + let dd = Rect::from_min_size(pos2(field_x, y), vec2(field_w, 20.0)); + let cname = cur.and_then(|(_, u)| trackers.iter().find(|(g, _)| *g == u)).map(|(_, n)| n.clone()).unwrap_or_else(|| "None".into()); + let did = egui::Id::new("tracker-current-dd"); + if widgets::dropdown(ui, dd, &cname, &t, did).clicked() && !trackers.is_empty() { + widgets::open_popup(ui, did); + } + app.auto.add("tracker.currentTrack", dd, "Current Track"); + let names: Vec = trackers.iter().map(|(_, n)| n.clone()).collect(); + let ci = cur.and_then(|(_, u)| trackers.iter().position(|(g, _)| *g == u)); + if let Some(i) = widgets::popup_menu(ui, did, dd.left_bottom(), &names, ci) + && let (Some(src), Some((uid, _))) = (source, trackers.get(i)) + { + run(app, &ctx, "track.select", json!({"layer": src.0, "tracker": uid})); + } + y += ROW; + + let settings = cur.and_then(|(l, u)| comp.layer(l).and_then(|l| l.tracker(u)).map(|(_, s)| s.clone())); + let enabled = settings.is_some(); + let addr = cur.map(|(l, u)| json!({"layer": l.0, "tracker": u})).unwrap_or_else(|| json!({})); + let with = |extra: serde_json::Value| { + let mut v = addr.clone(); + if let (Some(o), Some(e)) = (v.as_object_mut(), extra.as_object()) { + for (k, x) in e { + o.insert(k.clone(), x.clone()); + } + } + v + }; + + // Track Type. + label(&p, pos2(x0, y + 10.0), "Track Type:", &t); + let dd = Rect::from_min_size(pos2(field_x, y), vec2(field_w, 20.0)); + let kind = settings.as_ref().map(|s| s.kind).unwrap_or_default(); + let did = egui::Id::new("tracker-type-dd"); + if widgets::dropdown(ui, dd, if enabled { kind.label() } else { "" }, &t, did).clicked() && enabled { + widgets::open_popup(ui, did); + } + app.auto.add("tracker.trackType", dd, "Track Type"); + let kinds: Vec = TrackKind::ALL.iter().map(|k| k.label().to_string()).collect(); + if let Some(i) = widgets::popup_menu(ui, did, dd.left_bottom(), &kinds, TrackKind::ALL.iter().position(|k| *k == kind)) { + run(app, &ctx, "track.setType", with(json!({"kind": TrackKind::ALL[i].id()}))); + } + y += ROW; + + // Position / Rotation / Scale. + let transformish = matches!(kind, TrackKind::Transform | TrackKind::Stabilize) && enabled; + let mut xx = x0; + for (key, text, on) in [ + ("position", "Position", settings.as_ref().is_some_and(|s| s.position)), + ("rotation", "Rotation", settings.as_ref().is_some_and(|s| s.rotation)), + ("scale", "Scale", settings.as_ref().is_some_and(|s| s.scale)), + ] { + let cr = Rect::from_min_size(pos2(xx, y + 2.0), vec2(16.0, 16.0)); + let resp = widgets::checkbox(ui, cr, on && transformish, &t, egui::Id::new(("tracker-cb", key))); + app.auto.add(&format!("tracker.{key}"), cr, text); + p.text(pos2(cr.max.x + 4.0, cr.center().y), Align2::LEFT_CENTER, text, Tokens::ui(12.0), if transformish { t.text } else { t.text_faint }); + if resp.clicked() && transformish { + run(app, &ctx, "track.setType", with(json!({ key: !on }))); + } + xx += 84.0; + } + y += ROW; + + // Motion Target. + let target_name = settings + .as_ref() + .and_then(|s| s.target) + .and_then(|l| comp.layer(l).map(|l| format!("{}. {}", comp.layers.iter().position(|x| x.id == l.id).unwrap_or(0) + 1, l.name))); + label(&p, pos2(x0, y + 10.0), "Motion Target:", &t); + let tr = Rect::from_min_size(pos2(field_x, y), vec2(field_w, 20.0)); + p.text( + pos2(tr.min.x, tr.center().y), + Align2::LEFT_CENTER, + target_name.unwrap_or_else(|| if enabled && kind != TrackKind::Raw { "None".into() } else { String::new() }), + Tokens::ui(12.0), + t.text, + ); + app.auto.add("tracker.motionTarget", tr, "Motion Target"); + y += ROW; + let r1 = Rect::from_min_size(pos2(x0, y), vec2(half, 22.0)); + let r2 = Rect::from_min_size(pos2(x0 + half + 6.0, y), vec2(half, 22.0)); + if button(app, ui, r1, "Edit Target...", enabled && kind != TrackKind::Raw && kind != TrackKind::Stabilize, "tracker.editTarget") { + run(app, &ctx, "track.editTargetDialog", json!({})); + } + if button(app, ui, r2, "Options...", enabled, "tracker.options") { + run(app, &ctx, "track.optionsDialog", json!({})); + } + y += 32.0; + + // Analyze. + label(&p, pos2(x0, y + 11.0), "Analyze:", &t); + let tracking = app.session.is_tracking(); + let running_dir = app.session.track_job.as_ref().map(|j| j.direction); + let mut bx = field_x; + for (dir, fwd, frame, id, tip) in [ + (effectcraft_engine::tracking::Direction::FrameBackward, false, true, "frameBackward", "Analyze 1 frame backward"), + (effectcraft_engine::tracking::Direction::Backward, false, false, "backward", "Analyze backward"), + (effectcraft_engine::tracking::Direction::Forward, true, false, "forward", "Analyze forward"), + (effectcraft_engine::tracking::Direction::FrameForward, true, true, "frameForward", "Analyze 1 frame forward"), + ] { + let r = Rect::from_min_size(pos2(bx, y), vec2(30.0, 22.0)); + let resp = ui.interact(r, egui::Id::new(("tracker-analyze", id)), Sense::click()).on_hover_text(tip); + let stop = tracking && running_dir == Some(dir) && !frame; + let on = enabled && (!tracking || stop); + p.rect_filled(r, 4.0, if resp.hovered() && on { t.hover } else { t.field_bg }); + p.rect_stroke(r, 4.0, Stroke::new(1.0, t.field_border), StrokeKind::Inside); + analyze_glyph(&p, r, fwd, frame, stop, if on { t.text } else { t.text_faint }); + app.auto.add(&format!("tracker.analyze.{id}"), r, tip); + if resp.clicked() && on { + if stop { + run(app, &ctx, "track.stop", json!({})); + } else { + run(app, &ctx, "track.analyze", with(json!({ "direction": id }))); + } + } + bx += 34.0; + } + y += 32.0; + + // Reset / Apply. + let r1 = Rect::from_min_size(pos2(x0, y), vec2(half, 22.0)); + let r2 = Rect::from_min_size(pos2(x0 + half + 6.0, y), vec2(half, 22.0)); + if button(app, ui, r1, "Reset", enabled && !tracking, "tracker.reset") { + run(app, &ctx, "track.reset", addr.clone()); + } + if button(app, ui, r2, "Apply", enabled && !tracking && kind != TrackKind::Raw, "tracker.apply") { + if matches!(kind, TrackKind::Transform | TrackKind::Stabilize) { + app.dialog_state.track_apply = ApplyDraft::default(); + app.dialog = Some(Dialog::TrackApply); + } else { + run(app, &ctx, "track.apply", addr.clone()); + } + } + y += 32.0; + + // Progress. + if let Some(pr) = app.session.track_progress() { + let bar = Rect::from_min_size(pos2(x0, y), vec2(w, 6.0)); + p.rect_filled(bar, 3.0, t.field_bg); + let f = if pr.total > 0 { pr.done as f32 / pr.total as f32 } else { 0.0 }; + p.rect_filled(Rect::from_min_size(bar.min, vec2(bar.width() * f.clamp(0.0, 1.0), 6.0)), 3.0, t.accent); + app.auto.add("tracker.progress", bar, "Analysis progress"); + p.text( + pos2(x0, y + 18.0), + Align2::LEFT_CENTER, + format!("Analyzing {} / {} frames ({:.0} fps)", pr.done, pr.total, pr.fps), + Tokens::ui(11.0), + t.text_dim, + ); + } +} + +// ---------------------------------------------------------------- dialogs + +pub fn open_options(app: &mut EffectcraftApp) -> Result<(), String> { + let (l, u) = current(app).ok_or("no current track")?; + let comp = app.session.active_comp().ok_or("no composition")?; + let (g, s) = comp.layer(l).and_then(|l| l.tracker(u)).ok_or("no tracker")?; + app.dialog_state.track_options = OptionsDraft { name: g.name.clone(), opts: s.options.clone(), blur_on: s.options.blur > 0.0 }; + if app.dialog_state.track_options.opts.blur <= 0.0 { + app.dialog_state.track_options.opts.blur = 2.0; + } + app.dialog = Some(Dialog::TrackOptions); + Ok(()) +} + +pub fn open_target(app: &mut EffectcraftApp) -> Result<(), String> { + let (l, u) = current(app).ok_or("no current track")?; + let comp = app.session.active_comp().ok_or("no composition")?; + let (_, s) = comp.layer(l).and_then(|l| l.tracker(u)).ok_or("no tracker")?; + app.dialog_state.track_target = TargetDraft { target: s.target.map(|l| l.0) }; + app.dialog = Some(Dialog::TrackTarget); + Ok(()) +} + +fn ok_cancel(ui: &mut egui::Ui, app: &mut EffectcraftApp, t: &Tokens, prefix: &str) -> (bool, bool) { + let (mut ok, mut cancel) = (false, false); + ui.add_space(14.0); + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + let r = ui.add(egui::Button::new(egui::RichText::new(" OK ").color(Color32::WHITE)).fill(t.accent)); + app.auto.add(&format!("{prefix}.ok"), r.rect, "OK"); + ok = r.clicked(); + let r = ui.button("Cancel"); + app.auto.add(&format!("{prefix}.cancel"), r.rect, "Cancel"); + cancel = r.clicked(); + }); + (ok, cancel) +} + +pub fn options_dialog(app: &mut EffectcraftApp, ctx: &egui::Context, t: &Tokens) { + let mut d = app.dialog_state.track_options.clone(); + let (mut ok, mut cancel) = (false, false); + super::dialogs::modal(ctx, "Motion Tracker Options", vec2(460.0, 470.0), t, |ui| { + ui.horizontal(|ui| { + ui.label("Track Name:"); + let r = ui.text_edit_singleline(&mut d.name); + app.auto.add("dialog.trackOptions.name", r.rect, "Track Name"); + }); + ui.add_space(8.0); + ui.label(egui::RichText::new("Channel").font(Tokens::medium(12.5))); + ui.horizontal(|ui| { + for (c, l) in [(TrackChannel::Rgb, "RGB"), (TrackChannel::Luminance, "Luminance"), (TrackChannel::Saturation, "Saturation")] { + let r = ui.radio_value(&mut d.opts.channel, c, l); + app.auto.add(&format!("dialog.trackOptions.channel.{}", l.to_ascii_lowercase()), r.rect, l); + } + }); + ui.add_space(8.0); + ui.label(egui::RichText::new("Process Before Match").font(Tokens::medium(12.5))); + ui.horizontal(|ui| { + let r = ui.checkbox(&mut d.blur_on, "Blur"); + app.auto.add("dialog.trackOptions.blurOn", r.rect, "Blur"); + let r = ui.add_enabled(d.blur_on, egui::DragValue::new(&mut d.opts.blur).speed(0.1).range(0.5..=50.0).suffix(" pixels")); + app.auto.add("dialog.trackOptions.blur", r.rect, "Blur pixels"); + let r = ui.checkbox(&mut d.opts.enhance, "Enhance"); + app.auto.add("dialog.trackOptions.enhance", r.rect, "Enhance"); + }); + ui.add_space(8.0); + let r = ui.checkbox(&mut d.opts.subpixel, "Subpixel Positioning"); + app.auto.add("dialog.trackOptions.subpixel", r.rect, "Subpixel Positioning"); + let r = ui.checkbox(&mut d.opts.adapt_every_frame, "Adapt Feature On Every Frame"); + app.auto.add("dialog.trackOptions.adaptEveryFrame", r.rect, "Adapt Feature On Every Frame"); + let r = ui.checkbox(&mut d.opts.track_shape, "Follow Feature Rotation and Scale"); + app.auto.add("dialog.trackOptions.trackShape", r.rect, "Follow Feature Rotation and Scale"); + ui.add_space(8.0); + ui.horizontal(|ui| { + let actions = [ + (LowConfidence::Continue, "Continue Tracking"), + (LowConfidence::Stop, "Stop Tracking"), + (LowConfidence::Extrapolate, "Extrapolate Motion"), + (LowConfidence::Adapt, "Adapt Feature"), + ]; + let cur = actions.iter().find(|(a, _)| *a == d.opts.action).map(|(_, l)| *l).unwrap_or("Adapt Feature"); + let r = egui::ComboBox::from_id_salt("track-action") + .selected_text(cur) + .show_ui(ui, |ui| { + for (a, l) in actions { + ui.selectable_value(&mut d.opts.action, a, l); + } + }) + .response; + app.auto.add("dialog.trackOptions.action", r.rect, "If Confidence is Below action"); + ui.label("If Confidence is Below"); + let r = ui.add(egui::DragValue::new(&mut d.opts.threshold).speed(0.5).range(0.0..=100.0).suffix(" %")); + app.auto.add("dialog.trackOptions.threshold", r.rect, "Confidence threshold"); + }); + (ok, cancel) = ok_cancel(ui, app, t, "dialog.trackOptions"); + }); + app.dialog_state.track_options = d.clone(); + if ok || super::dialogs::ui_enter(ctx) { + let o = &d.opts; + let action = match o.action { + LowConfidence::Continue => "continue", + LowConfidence::Stop => "stop", + LowConfidence::Extrapolate => "extrapolate", + LowConfidence::Adapt => "adapt", + }; + let channel = match o.channel { + TrackChannel::Rgb => "rgb", + TrackChannel::Luminance => "luminance", + TrackChannel::Saturation => "saturation", + }; + let p = json!({ + "name": d.name, "channel": channel, "blur": if d.blur_on { o.blur } else { 0.0 }, "enhance": o.enhance, + "subpixel": o.subpixel, "adaptEveryFrame": o.adapt_every_frame, "threshold": o.threshold, "action": action, "trackShape": o.track_shape, + }); + if let Err(e) = app.session.execute("track.options", p) { + app.ui.status = e.to_string(); + } + cancel = true; + } + if cancel { + app.dialog = None; + } +} + +pub fn target_dialog(app: &mut EffectcraftApp, ctx: &egui::Context, t: &Tokens) { + let mut d = app.dialog_state.track_target.clone(); + let (mut ok, mut cancel) = (false, false); + let layers: Vec<(u64, String)> = + app.session.active_comp().map(|c| c.layers.iter().enumerate().map(|(i, l)| (l.id.0, format!("{}. {}", i + 1, l.name))).collect()).unwrap_or_default(); + super::dialogs::modal(ctx, "Motion Target", vec2(400.0, 200.0), t, |ui| { + ui.label("Apply Motion To:"); + ui.horizontal(|ui| { + ui.label("Layer:"); + let cur = d.target.and_then(|id| layers.iter().find(|(l, _)| *l == id)).map(|(_, n)| n.clone()).unwrap_or_else(|| "None".into()); + let r = egui::ComboBox::from_id_salt("track-target") + .selected_text(cur) + .width(240.0) + .show_ui(ui, |ui| { + ui.selectable_value(&mut d.target, None, "None"); + for (id, n) in &layers { + ui.selectable_value(&mut d.target, Some(*id), n); + } + }) + .response; + app.auto.add("dialog.trackTarget.layer", r.rect, "Layer"); + }); + (ok, cancel) = ok_cancel(ui, app, t, "dialog.trackTarget"); + }); + app.dialog_state.track_target = d.clone(); + if ok || super::dialogs::ui_enter(ctx) { + if let Err(e) = app.session.execute("track.setTarget", json!({"target": d.target})) { + app.ui.status = e.to_string(); + } + cancel = true; + } + if cancel { + app.dialog = None; + } +} + +pub fn apply_dialog(app: &mut EffectcraftApp, ctx: &egui::Context, t: &Tokens) { + let mut d = app.dialog_state.track_apply.clone(); + let (mut ok, mut cancel) = (false, false); + super::dialogs::modal(ctx, "Motion Tracker Apply Options", vec2(380.0, 170.0), t, |ui| { + ui.horizontal(|ui| { + ui.label("Apply Dimensions:"); + let labels = ["X and Y", "X only", "Y only"]; + let r = egui::ComboBox::from_id_salt("track-apply-dims") + .selected_text(labels[d.dims.min(2)]) + .show_ui(ui, |ui| { + for (i, l) in labels.iter().enumerate() { + ui.selectable_value(&mut d.dims, i, *l); + } + }) + .response; + app.auto.add("dialog.trackApply.dimensions", r.rect, "Apply Dimensions"); + }); + (ok, cancel) = ok_cancel(ui, app, t, "dialog.trackApply"); + }); + app.dialog_state.track_apply = d.clone(); + if ok || super::dialogs::ui_enter(ctx) { + let dims = ["xy", "x", "y"][d.dims.min(2)]; + if let Err(e) = app.session.execute("track.apply", json!({"dimensions": dims})) { + app.ui.status = e.to_string(); + } + cancel = true; + } + if cancel { + app.dialog = None; + } +} + +// ---------------------------------------------------------------- viewer widgets + +/// Which part of a track point a drag grabbed. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Part { + /// Inside the feature region: move the whole point. + Point, + /// Inside the search region: move only the search region. + Search, + /// A feature region corner (0..4: UL, UR, LR, LL). + FeatureCorner(usize), + SearchCorner(usize), + Attach, +} + +/// A track point widget hit area (screen space). +#[derive(Clone, Copy, Debug)] +pub struct Hit { + pub point: usize, + pub part: Part, + pub rect: Rect, + /// Screen quad for inside tests (feature / search region). + pub quad: Option<[Pos2; 4]>, +} + +/// A track point drag in progress. +#[derive(Clone, Debug)] +pub struct Drag { + pub point: usize, + pub part: Part, + /// Comp → tracked-layer matrix and the layer-space press point. + pub inv: Mat3, + pub start: [f64; 2], + pub center: [f64; 2], + pub feature: [f64; 2], + pub search_offset: [f64; 2], + pub search: [f64; 2], + pub attach: [f64; 2], +} + +const TRACK_COL: Color32 = Color32::from_rgb(0xf0, 0xc8, 0x3c); + +fn quad_contains(q: &[Pos2; 4], p: Pos2) -> bool { + let mut sign = 0.0f32; + for i in 0..4 { + let (a, b) = (q[i], q[(i + 1) % 4]); + let c = (b.x - a.x) * (p.y - a.y) - (b.y - a.y) * (p.x - a.x); + if c.abs() < 1e-6 { + continue; + } + if sign == 0.0 { + sign = c.signum(); + } else if c.signum() != sign { + return false; + } + } + true +} + +/// Draw the current track's points (layer → comp matrix `m`) and return their hit areas, from the +/// most to the least specific. +pub fn draw_overlay(app: &mut EffectcraftApp, painter: &egui::Painter, map: &ViewerMap, m: &Mat3, layer: &Layer, tracker: Uid, comp_time: Tick) -> Vec { + let Some((g, _)) = layer.tracker(tracker) else { return vec![] }; + let lt = layer.layer_time(comp_time); + let scr = |p: [f64; 2]| { + let c = m.apply(gv2(p[0], p[1])); + map.to_screen([c.x, c.y]) + }; + let mut hits = vec![]; + let mut inside = vec![]; + for (i, tp) in g.track_points().enumerate() { + let v = point_values(tp, lt); + let c = v.spec.center; + let attach = tp + .get("attachPoint") + .filter(|p| !p.keys.is_empty()) + .map(|p| p.value_at(lt).as_vec2()) + .unwrap_or([c[0] + v.attach_offset[0], c[1] + v.attach_offset[1]]); + let rect_q = |cx: [f64; 2], s: [f64; 2]| { + let (hx, hy) = (s[0] / 2.0, s[1] / 2.0); + [scr([cx[0] - hx, cx[1] - hy]), scr([cx[0] + hx, cx[1] - hy]), scr([cx[0] + hx, cx[1] + hy]), scr([cx[0] - hx, cx[1] + hy])] + }; + // Track path (feature centre keys). + if let Some(fc) = tp.get("featureCenter") + && fc.keys.len() > 1 + { + let pts: Vec = fc.keys.iter().map(|k| scr(k.value.as_vec2())).collect(); + painter.add(egui::Shape::line(pts.clone(), Stroke::new(1.0, TRACK_COL.gamma_multiply(0.6)))); + for q in &pts { + painter.circle_filled(*q, 1.6, TRACK_COL); + } + } + let fq = rect_q(c, v.spec.feature_size); + let sc = [c[0] + v.spec.search_offset[0], c[1] + v.spec.search_offset[1]]; + let sq = rect_q(sc, v.spec.search_size); + painter.add(egui::Shape::closed_line(sq.to_vec(), Stroke::new(1.0, TRACK_COL.gamma_multiply(0.7)))); + painter.add(egui::Shape::closed_line(fq.to_vec(), Stroke::new(1.0, TRACK_COL))); + painter.circle_filled(scr(c), 2.0, TRACK_COL); + let a = scr(attach); + for d in [vec2(7.0, 0.0), vec2(0.0, 7.0)] { + painter.line_segment([a - d, a + d], Stroke::new(1.4, Color32::BLACK)); + painter.line_segment([a - d, a + d], Stroke::new(1.0, TRACK_COL)); + } + painter.text(sq[0] + vec2(0.0, -3.0), Align2::LEFT_BOTTOM, &tp.name, Tokens::ui(10.5), TRACK_COL); + // Corner handles only when the region is big enough on screen to see inside it. + let fbig = fq[0].distance(fq[2]) > 28.0; + let sbig = sq[0].distance(sq[2]) > 28.0; + let n = i + 1; + let hr = Rect::from_center_size(a, vec2(12.0, 12.0)); + hits.push(Hit { point: n, part: Part::Attach, rect: hr, quad: None }); + app.auto.add(&format!("viewer.track.{n}.attach"), hr, "Attach Point"); + for (k, q) in fq.iter().enumerate() { + let r = Rect::from_center_size(*q, vec2(5.0, 5.0)); + if fbig { + painter.rect_filled(r, 0.0, TRACK_COL); + } + hits.push(Hit { point: n, part: Part::FeatureCorner(k), rect: r.expand(2.0), quad: None }); + app.auto.add(&format!("viewer.track.{n}.feature.{k}"), r, "Feature region corner"); + } + for (k, q) in sq.iter().enumerate() { + let r = Rect::from_center_size(*q, vec2(5.0, 5.0)); + if sbig { + painter.rect_stroke(r, 0.0, Stroke::new(1.0, TRACK_COL), StrokeKind::Middle); + } + hits.push(Hit { point: n, part: Part::SearchCorner(k), rect: r.expand(2.0), quad: None }); + app.auto.add(&format!("viewer.track.{n}.search.{k}"), r, "Search region corner"); + } + let fr = Rect::from_points(&fq); + let sr = Rect::from_points(&sq); + app.auto.add(&format!("viewer.track.{n}.feature"), fr, "Feature region"); + app.auto.add(&format!("viewer.track.{n}.search"), sr, "Search region"); + inside.push(Hit { point: n, part: Part::Point, rect: fr, quad: Some(fq) }); + inside.push(Hit { point: n, part: Part::Search, rect: sr, quad: Some(sq) }); + } + // Regions after handles; feature regions before search regions. + inside.sort_by_key(|h| h.part == Part::Search); + hits.extend(inside); + hits +} + +/// The hit under `pos`. +pub fn hit_at(hits: &[Hit], pos: Pos2) -> Option { + hits.iter() + .find(|h| match h.quad { + Some(q) => quad_contains(&q, pos), + None => h.rect.contains(pos), + }) + .copied() +} + +/// Start dragging `hit` (press at comp point `cpt`). +pub fn begin_drag(layer: &Layer, tracker: Uid, comp_time: Tick, l2c: &Mat3, hit: Hit, cpt: [f64; 2]) -> Option { + let (g, _) = layer.tracker(tracker)?; + let tp = g.track_points().nth(hit.point - 1)?; + let v = point_values(tp, layer.layer_time(comp_time)); + let inv = l2c.inverse()?; + let s = inv.apply(gv2(cpt[0], cpt[1])); + Some(Drag { + point: hit.point, + part: hit.part, + inv, + start: [s.x, s.y], + center: v.spec.center, + feature: v.spec.feature_size, + search_offset: v.spec.search_offset, + search: v.spec.search_size, + attach: v.attach_offset, + }) +} + +/// The `track.setPoint` parameters for the drag at comp point `cpt`. +pub fn drag_params(d: &Drag, cpt: [f64; 2]) -> serde_json::Value { + let q = d.inv.apply(gv2(cpt[0], cpt[1])); + let lp = [q.x, q.y]; + let dl = [lp[0] - d.start[0], lp[1] - d.start[1]]; + match d.part { + Part::Point => json!({"point": d.point, "center": [d.center[0] + dl[0], d.center[1] + dl[1]]}), + Part::Search => json!({"point": d.point, "searchOffset": [d.search_offset[0] + dl[0], d.search_offset[1] + dl[1]]}), + Part::Attach => json!({"point": d.point, "attachOffset": [d.attach[0] + dl[0], d.attach[1] + dl[1]]}), + Part::FeatureCorner(_) => { + let size = [(2.0 * (lp[0] - d.center[0]).abs()).max(4.0), (2.0 * (lp[1] - d.center[1]).abs()).max(4.0)]; + json!({"point": d.point, "featureSize": size}) + } + Part::SearchCorner(_) => { + let sc = [d.center[0] + d.search_offset[0], d.center[1] + d.search_offset[1]]; + let size = [(2.0 * (lp[0] - sc[0]).abs()).max(4.0), (2.0 * (lp[1] - sc[1]).abs()).max(4.0)]; + json!({"point": d.point, "searchSize": size}) + } + } +} + +/// The tracked layer and tracker the viewer shows widgets for. +pub fn viewer_track(app: &EffectcraftApp) -> Option<(LayerId, Uid)> { + current(app) +} diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index c0a927bc..f1962077 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -119,6 +119,12 @@ enum Gesture { inv: Mat3, press: Pos2, }, + /// Dragging a track point's feature/search region, a corner or its attach point. + Track { + layer: LayerId, + tracker: u64, + drag: super::tracker::Drag, + }, } /// Pen tool path in progress: (layer, mask uid). @@ -502,6 +508,19 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } } + // Track points of the current track (Tracker panel) on its layer, while that layer is selected. + let mut track_hits = vec![]; + let track_cur = super::tracker::viewer_track(app); + if app.ui.viewer.show_layer_controls + && let Some((tl, tu)) = track_cur + && selected.contains(&tl) + && let Some(layer) = comp.layer(tl) + && layer.is_active_at(time) + { + let (m, _) = l2c(&ectx, layer); + track_hits = super::tracker::draw_overlay(app, &painter, &map, &m, layer, tu, time); + } + // Cameras and lights (wireframes) and the 3D view name. if comp.has_3d() { draw_rigs(app, &painter, &map, &ectx, &selected); @@ -530,7 +549,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { Tool::Dolly => egui::CursorIcon::ResizeVertical, t if t.is_shape() || t == Tool::Pen => egui::CursorIcon::Crosshair, _ => { - if handle_hits.iter().any(|(_, _, h)| h.distance(hp) < HANDLE) { + if super::tracker::hit_at(&track_hits, hp).is_some() { + egui::CursorIcon::Move + } else if handle_hits.iter().any(|(_, _, h)| h.distance(hp) < HANDLE) { egui::CursorIcon::ResizeNwSe } else { egui::CursorIcon::Default @@ -610,6 +631,14 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { }), t if t.is_shape() => Some(Gesture::Create { tool: t, start: cpt }), Tool::Orbit | Tool::PanCamera | Tool::Dolly => Some(Gesture::Camera { tool }), + Tool::Selection if super::tracker::hit_at(&track_hits, press).is_some() => { + let hit = super::tracker::hit_at(&track_hits, press); + track_cur.zip(hit).and_then(|((tl, tu), hit)| { + let layer = comp.layer(tl)?; + let (m, _) = l2c(&ectx, layer); + super::tracker::begin_drag(layer, tu, time, &m, hit, map.to_comp(press)).map(|drag| Gesture::Track { layer: tl, tracker: tu, drag }) + }) + } Tool::Selection if vertex_at(press, true).is_some() => vertex_at(press, true).map(|h| Gesture::Tangent { layer: h.layer, mask: h.mask, @@ -793,6 +822,13 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { .execute("mask.setVertex", json!({"layer": layer.0, "mask": mask, "index": index, "out": t, "in": [-t[0], -t[1]], "merge": merge})); } } + Gesture::Track { layer, tracker, drag } => { + let mut q = super::tracker::drag_params(&drag, cpt); + q["layer"] = json!(layer.0); + q["tracker"] = json!(tracker); + q["merge"] = json!(merge); + let _ = app.session.execute("track.setPoint", q); + } Gesture::Marquee { start } => { let r = Rect::from_two_pos(start, pos); painter.rect_filled(r, 0.0, t.accent.gamma_multiply(0.12)); diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index 02e11b8a..f7bc7432 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -412,6 +412,9 @@ pub struct UiState { pub start_screen: bool, /// Composition ▸ Preview ▸ Cache Frames When Idle. pub cache_when_idle: bool, + /// Tracker panel ▸ Motion Source chosen without a tracker yet (layer id). + #[serde(default)] + pub tracker_source: Option, } impl Default for UiState { @@ -452,6 +455,7 @@ impl Default for UiState { preview_audio: true, start_screen: false, cache_when_idle: false, + tracker_source: None, } } } diff --git a/crates/ui-egui/tests/ui_tracker.rs b/crates/ui-egui/tests/ui_tracker.rs new file mode 100644 index 00000000..de8311e2 --- /dev/null +++ b/crates/ui-egui/tests/ui_tracker.rs @@ -0,0 +1,117 @@ +//! Headless checks for the Tracker panel and the viewer's track point widgets (egui_kittest, UI +//! logic only: no GPU needed). + +use effectcraft_engine::Session; +use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::dock::PanelKind; +use egui_kittest::Harness; +use serde_json::json; + +fn app() -> EffectcraftApp { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Track", "width": 640, "height": 360, "duration": 2})).unwrap(); + s.execute("layer.newSolid", json!({"name": "Plate", "color": "#406080"})).unwrap(); + s.execute("layer.newSolid", json!({"name": "Target", "width": 80, "height": 80})).unwrap(); + s.execute("layer.select", json!({"layers": ["Plate"]})).unwrap(); + EffectcraftApp::new(s) +} + +fn ids(h: &Harness<'_, EffectcraftApp>) -> Vec { + h.state().auto.previous.iter().chain(h.state().auto.elements.iter()).map(|e| e.id.clone()).collect() +} + +fn rect(h: &Harness<'_, EffectcraftApp>, id: &str) -> egui::Rect { + let e = h.state().auto.find(id).unwrap_or_else(|| panic!("no {id}")); + egui::Rect::from_min_size(egui::pos2(e.rect[0], e.rect[1]), egui::vec2(e.rect[2], e.rect[3])) +} + +#[test] +fn tracker_panel_and_track_point_widgets() { + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).build_eframe(|_| app()); + h.state_mut().show_panel(PanelKind::Tracker); + h.run_steps(3); + let all = ids(&h); + for want in [ + "tracker.trackCamera", + "tracker.warpStabilizer", + "tracker.trackMotion", + "tracker.stabilizeMotion", + "tracker.motionSource", + "tracker.currentTrack", + "tracker.trackType", + "tracker.position", + "tracker.rotation", + "tracker.scale", + "tracker.motionTarget", + "tracker.editTarget", + "tracker.options", + "tracker.analyze.frameBackward", + "tracker.analyze.backward", + "tracker.analyze.forward", + "tracker.analyze.frameForward", + "tracker.reset", + "tracker.apply", + ] { + assert!(all.iter().any(|i| i == want), "missing {want}"); + } + // Track Motion from the panel. + let r = rect(&h, "tracker.trackMotion"); + h.input_mut().events.push(egui::Event::PointerMoved(r.center())); + h.input_mut().events.push(egui::Event::PointerButton { + pos: r.center(), + button: egui::PointerButton::Primary, + pressed: true, + modifiers: Default::default(), + }); + h.input_mut().events.push(egui::Event::PointerButton { + pos: r.center(), + button: egui::PointerButton::Primary, + pressed: false, + modifiers: Default::default(), + }); + h.run_steps(3); + let st = h.state_mut().session.execute("track.status", json!({})).unwrap(); + assert_eq!(st["tracker"]["name"], "Tracker 1", "{st}"); + h.state_mut().session.execute("track.setPoint", json!({"point": 1, "featureSize": [60, 60], "searchSize": [120, 120]})).unwrap(); + h.state_mut().show_panel(PanelKind::Composition); + h.run_steps(3); + let all = ids(&h); + for want in ["viewer.track.1.feature", "viewer.track.1.search", "viewer.track.1.attach", "viewer.track.1.feature.0", "viewer.track.1.search.2"] { + assert!(all.iter().any(|i| i == want), "missing {want}"); + } + // Drag the feature region: the point moves by the drag in layer pixels (one undo step). + let c0 = st["tracker"]["points"][0]["center"].clone(); + let fr = rect(&h, "viewer.track.1.feature"); + let zoom = { + let a = rect(&h, "viewer.comp"); + a.width() / 640.0 + }; + // Inside the feature region, away from the attach point crosshair (which sits on the centre). + let from = fr.center() + egui::vec2(15.0, 0.0); + let to = from + egui::vec2(40.0, 20.0); + h.input_mut().events.push(egui::Event::PointerMoved(from)); + h.input_mut().events.push(egui::Event::PointerButton { pos: from, button: egui::PointerButton::Primary, pressed: true, modifiers: Default::default() }); + h.step(); + for i in 1..=8 { + h.input_mut().events.push(egui::Event::PointerMoved(from + (to - from) * (i as f32 / 8.0))); + h.step(); + } + h.input_mut().events.push(egui::Event::PointerButton { pos: to, button: egui::PointerButton::Primary, pressed: false, modifiers: Default::default() }); + h.run_steps(2); + let st = h.state_mut().session.execute("track.status", json!({})).unwrap(); + let c1 = &st["tracker"]["points"][0]["center"]; + let dx = c1[0].as_f64().unwrap() - c0[0].as_f64().unwrap(); + let dy = c1[1].as_f64().unwrap() - c0[1].as_f64().unwrap(); + assert!((dx - 40.0 / zoom as f64).abs() < 1.0 && (dy - 20.0 / zoom as f64).abs() < 1.0, "moved by ({dx}, {dy}) at zoom {zoom}"); + let undo = h.state().session.history.undo.iter().filter(|(l, _)| l == "Move Track Point").count(); + assert_eq!(undo, 2, "one undo step per drag"); + // Options and Edit Target dialogs. + let ctx = h.ctx.clone(); + effectcraft_ui_egui::menus::invoke(h.state_mut(), &ctx, "track.optionsDialog", json!({})).unwrap(); + assert_eq!(h.state().dialog, Some(effectcraft_ui_egui::Dialog::TrackOptions)); + h.step(); + assert!(ids(&h).iter().any(|i| i == "dialog.trackOptions.ok")); + h.state_mut().dialog = None; + effectcraft_ui_egui::menus::invoke(h.state_mut(), &ctx, "track.editTargetDialog", json!({})).unwrap(); + assert_eq!(h.state().dialog, Some(effectcraft_ui_egui::Dialog::TrackTarget)); +} diff --git a/docs/agents.md b/docs/agents.md index 54a5ff38..6b1badb0 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -77,6 +77,29 @@ instead of being ignored. 4. `execute_command {"command":"effect.apply","params":{"layer":2,"effect":"Gaussian Blur"}}`, then `set_property {"layer":2,"path":"effects/#1/blurriness","value":8}` 5. `render_frame {"time":1.0}` lets you look at the result, and `save_project {"path":"intro.ecproj"}` saves it. +### Motion tracking + +Trackers live on the tracked layer (`Motion Trackers ▸ Tracker n ▸ Track Point n`) and are driven +with `track.*` commands. `track.analyze` runs in the background in the app (poll `track.status`); +pass `"wait": true` to block until it finishes, which is what the CLI and headless MCP want. + +1. `execute_command {"command":"track.motion","params":{"layer":"clip.mov"}}` creates a Transform + tracker (Motion Target: the layer above). Use `track.stabilize` for Stabilize, or `track.new` with + `kind` `transform|stabilize|affine|perspective|raw` and `rotation`/`scale`. +2. `execute_command {"command":"track.setPoint","params":{"point":1,"center":[812,440],"featureSize":[40,40],"searchSize":[96,96]}}` +3. `execute_command {"command":"track.analyze","params":{"direction":"forward","wait":true}}` keys Feature + Center, Confidence and Attach Point on every frame from the current time to the layer's end. +4. `execute_command {"command":"track.apply","params":{"dimensions":"xy"}}` keys the target's Position + (and Rotation/Scale), the layer's own Anchor Point and Position for Stabilize, or a Corner Pin + effect for `affine`/`perspective` tracks. `track.options` sets the channel, blur/enhance, + adapt-feature and "If Confidence is Below" behaviour; `track.status` reports the track points. + +```sh +effectcraft-cli run clip.ecproj track.motion '{"layer":"#2"}' \ + track.setPoint '{"point":1,"center":[812,440]}' \ + track.analyze '{"wait":true}' track.apply '{}' --save +``` + ## CLI Each invocation runs a headless engine with no window. It opens the demo project unless you pass diff --git a/docs/architecture.md b/docs/architecture.md index 4baa4c79..59d9ef4a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -22,6 +22,7 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L2 | `project` | The document: items, compositions, layers, the property tree, render queue model, `.ecproj` serde | | L2 | `text` | Fonts, shaping, layout, per-glyph geometry, text animators and selectors | | L2 | `effects` | The effect registry (241 effects) and their CPU implementations | +| L2 | `track` | Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation, Lucas–Kanade sub-pixel refinement), confidence, homography/affine/similarity solves | | L3 | `render` | Evaluation and compositing: sources, masks, effects, transforms, 3D, motion blur, mattes, blending, layer cache, audio mixdown | | L3 | `media` | Footage decoding (FilmCraft's pure-Rust codecs), image sequences, frame cache | | L3 | `expr` | The expression engine (JavaScript via boa) with the After Effects object model | @@ -92,6 +93,15 @@ layer's Blending Options and kept in step after each edit. Half, Third and Quarter resolution render proportionally fewer pixels end to end. +**Motion tracking** (Animation ▸ Track Motion / Stabilize Motion, Window ▸ Tracker): a tracker +is a `Tracker` group under the layer's Motion Trackers group, with Track Point groups (Feature +Center, Feature Size, Search Offset, Search Size, Confidence, Attach Point, Attach Point Offset) +keyframed per analysed frame. `track.analyze` renders the layer's source frames +(`Renderer::layer_source`) on a background thread (`engine::tracking`, polled like the render +queue; blocking with `wait`), and `effectcraft-track` matches each point in parallel. One undo step +covers an analysis. `track.apply` keys the target's Position/Rotation/Scale, the tracked layer's +Anchor Point and Position (Stabilize), or a Corner Pin effect (Parallel / Perspective). + ## 5. Expressions Expressions are JavaScript, run by boa, with After Effects' object model (`thisComp`, `thisLayer`, diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 81b61a88..323b9ca7 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -26,6 +26,7 @@ const LAYERS: &[(&str, u8)] = &[ ("project", 2), ("text", 2), ("effects", 2), + ("track", 2), ("render", 3), ("media", 3), ("expr", 3),