Merge branch 'worktree-agent-a128c7366e475da55'
This commit is contained in:
Brandon Thomas
commit
7523898705
|
|
@ -1197,8 +1197,10 @@ dependencies = [
|
|||
"effectcraft-render",
|
||||
"effectcraft-text",
|
||||
"effectcraft-time",
|
||||
"effectcraft-track",
|
||||
"image",
|
||||
"log",
|
||||
"rayon",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.21",
|
||||
|
|
@ -1368,6 +1370,15 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "effectcraft-track"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"effectcraft-raster",
|
||||
"rayon",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "effectcraft-ui-egui"
|
||||
version = "0.1.0"
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ effectcraft-keyframe = { path = "crates/keyframe" }
|
|||
effectcraft-project = { path = "crates/project" }
|
||||
effectcraft-text = { path = "crates/text" }
|
||||
effectcraft-effects = { path = "crates/effects" }
|
||||
effectcraft-track = { path = "crates/track" }
|
||||
effectcraft-render = { path = "crates/render" }
|
||||
effectcraft-media = { path = "crates/media" }
|
||||
effectcraft-export = { path = "crates/export" }
|
||||
|
|
@ -109,6 +110,9 @@ opt-level = 3
|
|||
[profile.dev.package.effectcraft-effects]
|
||||
opt-level = 3
|
||||
|
||||
[profile.dev.package.effectcraft-track]
|
||||
opt-level = 3
|
||||
|
||||
[profile.dev.package.effectcraft-path]
|
||||
opt-level = 3
|
||||
|
||||
|
|
|
|||
|
|
@ -20,9 +20,11 @@ effectcraft-path = { workspace = true }
|
|||
effectcraft-text = { workspace = true }
|
||||
effectcraft-effects = { workspace = true }
|
||||
effectcraft-render = { workspace = true }
|
||||
effectcraft-track = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
log = { workspace = true }
|
||||
rayon = { workspace = true }
|
||||
image = { workspace = true }
|
||||
web-time = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ mod styles;
|
|||
mod text_anim;
|
||||
mod three_d;
|
||||
mod time;
|
||||
mod track;
|
||||
mod view;
|
||||
|
||||
use std::sync::OnceLock;
|
||||
|
|
@ -102,6 +103,7 @@ pub fn command_specs() -> &'static [CommandSpec] {
|
|||
v.extend(file_more::specs());
|
||||
v.extend(comp_more::specs());
|
||||
v.extend(frontend::specs());
|
||||
v.extend(track::specs());
|
||||
v.extend(stubs::specs());
|
||||
v
|
||||
})
|
||||
|
|
|
|||
|
|
@ -83,9 +83,7 @@ pub fn specs() -> Vec<CommandSpec> {
|
|||
stub!("text.animatorFontAxes", "Variable Font Axes", ["Animation", "Animate Text"], "{}"),
|
||||
stub!("track.camera", "Track Camera", ["Animation"], "{}"),
|
||||
stub!("track.warpStabilizer", "Warp Stabilizer VFX", ["Animation"], "{}"),
|
||||
stub!("track.motion", "Track Motion", ["Animation"], "{}"),
|
||||
stub!("track.mask", "Track Mask", ["Animation"], "{}"),
|
||||
stub!("track.property", "Track this Property", ["Animation"], "{}"),
|
||||
// Workspaces and panels that don't exist yet.
|
||||
stub!("window.saveWorkspace", "Save Changes to this Workspace", ["Window", "Workspace"], "{}"),
|
||||
stub!("window.saveWorkspaceAs", "Save as New Workspace...", ["Window", "Workspace"], "{name}"),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,584 @@
|
|||
//! Motion tracking commands (Animation ▸ Track Motion / Stabilize Motion / Track this Property and
|
||||
//! the Tracker panel): create trackers, set type / target / options, move track points, analyze
|
||||
//! (background job or blocking), apply, reset.
|
||||
//!
|
||||
//! A tracker is addressed by `layer` (the tracked layer, a.k.a. Motion Source) and `tracker`
|
||||
//! (group uid, name or 1-based index); both default to the Tracker panel's Current Track.
|
||||
|
||||
use effectcraft_project::build::Ids;
|
||||
use effectcraft_project::tracking::{self, LowConfidence, TrackChannel, TrackKind, TrackerSettings};
|
||||
use effectcraft_project::{ItemId, Layer, LayerId, Node, PropGroup, Uid, Value};
|
||||
use effectcraft_time::Tick;
|
||||
use serde_json::json;
|
||||
|
||||
use super::{CommandSpec, b_p, bad, comp_id, f_p, frontend, has_comp, layer_mut, layer_p, resolve_layer, str_p};
|
||||
use crate::tracking::{Dims, Direction, Work, point_values};
|
||||
use crate::{EngineError, Result, Session, cmd, query};
|
||||
|
||||
type V = serde_json::Value;
|
||||
|
||||
pub fn specs() -> Vec<CommandSpec> {
|
||||
vec![
|
||||
cmd!("track.motion", "Track Motion", ["Animation"], None, "{layer?}", can_track, |s, p| new_with(s, p, TrackKind::Transform, "track.motion")),
|
||||
cmd!("track.stabilize", "Stabilize Motion", ["Animation"], None, "{layer?}", can_track, |s, p| new_with(s, p, TrackKind::Stabilize, "track.stabilize")),
|
||||
cmd!(
|
||||
"track.new",
|
||||
"New Tracker",
|
||||
[],
|
||||
None,
|
||||
"{layer?, kind?: transform|stabilize|affine|perspective|raw, position?, rotation?, scale?, target?: layer}",
|
||||
can_track,
|
||||
new_tracker
|
||||
),
|
||||
cmd!("track.property", "Track this Property", ["Animation"], None, "{layer?, source?: layer}", has_prop_selected, track_property),
|
||||
cmd!("track.select", "Current Track", [], None, "{layer?, tracker?: uid|name|#n}", has_comp, select),
|
||||
cmd!(
|
||||
"track.setType",
|
||||
"Track Type",
|
||||
[],
|
||||
None,
|
||||
"{layer?, tracker?, kind?: transform|stabilize|affine|perspective|raw, position?, rotation?, scale?}",
|
||||
has_track,
|
||||
set_type
|
||||
),
|
||||
cmd!("track.setTarget", "Edit Target", [], None, "{layer?, tracker?, target: layer|null}", has_track, set_target),
|
||||
cmd!(
|
||||
"track.options",
|
||||
"Motion Tracker Options",
|
||||
[],
|
||||
None,
|
||||
"{layer?, tracker?, name?, channel?: rgb|luminance|saturation, blur? (px, 0 = off), enhance?, subpixel?, adaptEveryFrame?, threshold? (%), action?: continue|stop|extrapolate|adapt, trackShape?}",
|
||||
has_track,
|
||||
options
|
||||
),
|
||||
cmd!(
|
||||
"track.setPoint",
|
||||
"Move Track Point",
|
||||
[],
|
||||
None,
|
||||
"{layer?, tracker?, point: n (1-based), center?: [x,y], featureSize?: [w,h], searchOffset?: [x,y], searchSize?: [w,h], attachOffset?: [x,y], move?: [dx,dy], time? (s)}",
|
||||
has_track,
|
||||
set_point
|
||||
),
|
||||
cmd!(
|
||||
"track.analyze",
|
||||
"Analyze",
|
||||
[],
|
||||
None,
|
||||
"{layer?, tracker?, direction?: forward|backward|frameForward|frameBackward, start? (s), end? (s), wait?: block until done}",
|
||||
has_track,
|
||||
analyze
|
||||
),
|
||||
cmd!("track.stop", "Stop Analysis", [], None, "{}", is_tracking, |s, _| Ok(json!({"stopped": s.stop_track()}))),
|
||||
cmd!("track.apply", "Apply", [], None, "{layer?, tracker?, dimensions?: xy|x|y}", has_track, apply),
|
||||
cmd!("track.reset", "Reset", [], None, "{layer?, tracker?}", has_track, reset),
|
||||
cmd!("track.delete", "Delete Tracker", [], None, "{layer?, tracker?}", has_track, delete),
|
||||
cmd!("track.editTargetDialog", "Edit Target...", [], None, "{}", has_track, |s, p| frontend(s, "track.editTargetDialog", p)),
|
||||
cmd!("track.optionsDialog", "Options...", [], None, "{}", has_track, |s, p| frontend(s, "track.optionsDialog", p)),
|
||||
query!("track.status", "Tracker Status", "{layer?, tracker?}", status),
|
||||
]
|
||||
}
|
||||
|
||||
// ---------- enablement ----------
|
||||
|
||||
fn trackable(l: &Layer) -> bool {
|
||||
l.has_video() && !matches!(l.source, effectcraft_project::LayerSource::Text | effectcraft_project::LayerSource::Shape)
|
||||
}
|
||||
|
||||
fn can_track(s: &Session) -> std::result::Result<(), String> {
|
||||
has_comp(s)?;
|
||||
let c = s.active_comp().ok_or("no composition is open")?;
|
||||
match s.state.selected_layers.first().and_then(|l| c.layer(*l)) {
|
||||
Some(l) if trackable(l) => Ok(()),
|
||||
Some(_) => Err("select a footage, composition or solid layer to track".into()),
|
||||
None => Err("select a layer to track".into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn has_prop_selected(s: &Session) -> std::result::Result<(), String> {
|
||||
has_comp(s)?;
|
||||
if s.state.selected_props.is_empty() { Err("select a property to track".into()) } else { Ok(()) }
|
||||
}
|
||||
|
||||
fn has_track(s: &Session) -> std::result::Result<(), String> {
|
||||
current(s, &json!({}), "track").map(|_| ()).map_err(|e| e.to_string().replace("invalid parameters for `track`: ", ""))
|
||||
}
|
||||
|
||||
fn is_tracking(s: &Session) -> std::result::Result<(), String> {
|
||||
if s.is_tracking() { Ok(()) } else { Err("no track analysis is running".into()) }
|
||||
}
|
||||
|
||||
// ---------- addressing ----------
|
||||
|
||||
/// (comp, tracked layer, tracker uid) from `layer` / `tracker` or the Current Track.
|
||||
pub(crate) fn current(s: &Session, p: &V, cmd: &str) -> Result<(ItemId, LayerId, Uid)> {
|
||||
let cid = comp_id(s, p)?;
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let lid = match p.get("layer") {
|
||||
Some(v) => resolve_layer(comp, v).ok_or_else(|| bad(cmd, format!("no layer {v}")))?,
|
||||
None => match s.state.current_track {
|
||||
Some((l, _)) if comp.layer(l).is_some() => l,
|
||||
_ => s
|
||||
.state
|
||||
.selected_layers
|
||||
.iter()
|
||||
.copied()
|
||||
.find(|l| comp.layer(*l).is_some_and(|l| l.trackers().next().is_some()))
|
||||
.ok_or_else(|| bad(cmd, "no current track: use Track Motion first"))?,
|
||||
},
|
||||
};
|
||||
let layer = comp.layer(lid).ok_or_else(|| bad(cmd, "no layer"))?;
|
||||
let uid = match p.get("tracker") {
|
||||
Some(V::Number(n)) => {
|
||||
let n = n.as_u64().unwrap_or(0);
|
||||
layer.tracker(n).map(|(g, _)| g.uid).or_else(|| layer.trackers().nth((n as usize).saturating_sub(1)).map(|(g, _)| g.uid))
|
||||
}
|
||||
Some(V::String(name)) => match name.strip_prefix('#').and_then(|i| i.parse::<usize>().ok()) {
|
||||
Some(i) => layer.trackers().nth(i.saturating_sub(1)).map(|(g, _)| g.uid),
|
||||
None => layer.trackers().find(|(g, _)| &g.name == name).map(|(g, _)| g.uid),
|
||||
},
|
||||
_ => match s.state.current_track {
|
||||
Some((l, t)) if l == lid && layer.tracker(t).is_some() => Some(t),
|
||||
_ => layer.trackers().next().map(|(g, _)| g.uid),
|
||||
},
|
||||
}
|
||||
.ok_or_else(|| bad(cmd, format!("layer `{}` has no such tracker", layer.name)))?;
|
||||
Ok((cid, lid, uid))
|
||||
}
|
||||
|
||||
fn tracker_mut(proj: &mut effectcraft_project::Project, cid: ItemId, lid: LayerId, uid: Uid) -> Result<&mut PropGroup> {
|
||||
layer_mut(proj, cid, lid)?.props.find_group_mut(uid).ok_or_else(|| EngineError::Other("tracker gone".into()))
|
||||
}
|
||||
|
||||
fn layer_size(s: &Session, cid: ItemId, l: &Layer) -> [f64; 2] {
|
||||
let (w, h) = effectcraft_render::source_size(&s.project, l);
|
||||
if w == 0 {
|
||||
let c = s.project.comp(cid);
|
||||
c.map(|c| [c.width as f64, c.height as f64]).unwrap_or([1920.0, 1080.0])
|
||||
} else {
|
||||
[w as f64, h as f64]
|
||||
}
|
||||
}
|
||||
|
||||
/// The layer directly above `lid` (the default Motion Target).
|
||||
fn layer_above(comp: &effectcraft_project::Comp, lid: LayerId) -> Option<LayerId> {
|
||||
let i = comp.layers.iter().position(|l| l.id == lid)?;
|
||||
i.checked_sub(1).map(|i| comp.layers[i].id)
|
||||
}
|
||||
|
||||
fn kind_p(p: &V, cmd: &str) -> Result<Option<TrackKind>> {
|
||||
match str_p(p, "kind") {
|
||||
Some(k) => TrackKind::from_name(k).map(Some).ok_or_else(|| bad(cmd, format!("unknown kind `{k}` (transform|stabilize|affine|perspective|raw)"))),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- create ----------
|
||||
|
||||
fn new_with(s: &mut Session, p: &V, kind: TrackKind, cmd: &str) -> Result<V> {
|
||||
let mut q = p.clone();
|
||||
q["kind"] = json!(kind.id());
|
||||
create(s, &q, cmd)
|
||||
}
|
||||
|
||||
fn new_tracker(s: &mut Session, p: &V) -> Result<V> {
|
||||
create(s, p, "track.new")
|
||||
}
|
||||
|
||||
fn create(s: &mut Session, p: &V, cmd: &str) -> Result<V> {
|
||||
let (cid, lid) = layer_p(s, p, cmd)?;
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let layer = comp.layer(lid).ok_or(EngineError::NoComp)?;
|
||||
if !trackable(layer) {
|
||||
return Err(bad(cmd, format!("layer `{}` has no pixels to track", layer.name)));
|
||||
}
|
||||
let kind = kind_p(p, cmd)?.unwrap_or_default();
|
||||
let mut st = TrackerSettings::new(kind);
|
||||
st.position = b_p(p, "position").unwrap_or(true);
|
||||
st.rotation = b_p(p, "rotation").unwrap_or(false);
|
||||
st.scale = b_p(p, "scale").unwrap_or(false);
|
||||
// Motion Target: the layer above the tracked layer (Stabilize applies to the layer itself).
|
||||
st.target = match p.get("target") {
|
||||
Some(V::Null) => None,
|
||||
Some(v) => Some(resolve_layer(comp, v).ok_or_else(|| bad(cmd, format!("no target layer {v}")))?),
|
||||
None if kind == TrackKind::Stabilize => Some(lid),
|
||||
None => layer_above(comp, lid),
|
||||
};
|
||||
let size = layer_size(s, cid, layer);
|
||||
let n_existing = layer.trackers().count();
|
||||
let name = format!("Tracker {}", n_existing + 1);
|
||||
let pts = tracking::default_points(kind, st.point_count(), size);
|
||||
let uid = s.edit(&format!("New {}", name), None, |proj, state| {
|
||||
let mut next = proj.next_id;
|
||||
let l = layer_mut(proj, cid, lid)?;
|
||||
let mut ids = Ids(&mut next);
|
||||
let g = tracking::tracker(&mut ids, &name, st, &pts);
|
||||
let uid = g.uid;
|
||||
l.motion_trackers_mut(&mut ids).children.push(g.into());
|
||||
proj.next_id = next;
|
||||
state.current_track = Some((lid, uid));
|
||||
state.selected_layers = vec![lid];
|
||||
Ok(uid)
|
||||
})?;
|
||||
let _ = frontend(s, "window.panel", &json!({"panel": "tracker"}));
|
||||
Ok(json!({"tracker": uid, "name": name, "layer": lid.0, "points": pts.len()}))
|
||||
}
|
||||
|
||||
/// Track this Property: track another layer and drive the selected property's layer.
|
||||
fn track_property(s: &mut Session, p: &V) -> Result<V> {
|
||||
let cid = comp_id(s, p)?;
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let target = match p.get("layer") {
|
||||
Some(v) => resolve_layer(comp, v).ok_or_else(|| bad("track.property", format!("no layer {v}")))?,
|
||||
None => s.state.selected_props.first().map(|(l, _)| *l).ok_or_else(|| bad("track.property", "select a property first"))?,
|
||||
};
|
||||
let source = match p.get("source") {
|
||||
Some(v) => resolve_layer(comp, v).ok_or_else(|| bad("track.property", format!("no layer {v}")))?,
|
||||
None => {
|
||||
let ti = comp.layers.iter().position(|l| l.id == target).unwrap_or(0);
|
||||
comp.layers
|
||||
.iter()
|
||||
.skip(ti + 1)
|
||||
.chain(comp.layers.iter().take(ti))
|
||||
.find(|l| l.id != target && trackable(l))
|
||||
.map(|l| l.id)
|
||||
.ok_or_else(|| bad("track.property", "no layer to track"))?
|
||||
}
|
||||
};
|
||||
create(s, &json!({"layer": source.0, "kind": "transform", "target": target.0, "comp": cid.0}), "track.property")
|
||||
}
|
||||
|
||||
// ---------- settings ----------
|
||||
|
||||
fn select(s: &mut Session, p: &V) -> Result<V> {
|
||||
let (_, lid, uid) = current(s, p, "track.select")?;
|
||||
s.state.current_track = Some((lid, uid));
|
||||
s.bump();
|
||||
Ok(json!({"layer": lid.0, "tracker": uid}))
|
||||
}
|
||||
|
||||
fn set_type(s: &mut Session, p: &V) -> Result<V> {
|
||||
let (cid, lid, uid) = current(s, p, "track.setType")?;
|
||||
let kind = kind_p(p, "track.setType")?;
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let layer = comp.layer(lid).ok_or(EngineError::NoComp)?;
|
||||
let size = layer_size(s, cid, layer);
|
||||
let above = layer_above(comp, lid);
|
||||
let n = s.edit("Track Type", None, |proj, _| {
|
||||
let mut next = proj.next_id;
|
||||
let g = tracker_mut(proj, cid, lid, uid)?;
|
||||
let st = g.tracker_settings_mut().ok_or_else(|| bad("track.setType", "not a tracker"))?;
|
||||
if let Some(k) = kind {
|
||||
if k == TrackKind::Stabilize {
|
||||
st.target = Some(lid);
|
||||
} else if st.kind == TrackKind::Stabilize && st.target == Some(lid) {
|
||||
st.target = above;
|
||||
}
|
||||
st.kind = k;
|
||||
}
|
||||
for (k, f) in [("position", &mut st.position), ("rotation", &mut st.rotation), ("scale", &mut st.scale)] {
|
||||
if let Some(v) = b_p(p, k) {
|
||||
*f = v;
|
||||
}
|
||||
}
|
||||
let (kind, n) = (st.kind, st.point_count());
|
||||
// Add or remove Track Points to match the type.
|
||||
let have = g.track_points().count();
|
||||
if have > n {
|
||||
let mut seen = 0;
|
||||
g.children.retain(|c| match c {
|
||||
Node::Group(tp) if tp.match_id == tracking::TRACK_POINT => {
|
||||
seen += 1;
|
||||
seen <= n
|
||||
}
|
||||
_ => true,
|
||||
});
|
||||
} else if have < n {
|
||||
let defaults = tracking::default_points(kind, n, size);
|
||||
let mut ids = Ids(&mut next);
|
||||
for (i, d) in defaults.iter().enumerate().skip(have) {
|
||||
g.children.push(tracking::track_point(&mut ids, &format!("Track Point {}", i + 1), d).into());
|
||||
}
|
||||
}
|
||||
proj.next_id = next;
|
||||
Ok(n)
|
||||
})?;
|
||||
Ok(json!({"points": n}))
|
||||
}
|
||||
|
||||
fn set_target(s: &mut Session, p: &V) -> Result<V> {
|
||||
let (cid, lid, uid) = current(s, p, "track.setTarget")?;
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let target = match p.get("target") {
|
||||
None | Some(V::Null) => None,
|
||||
Some(v) => Some(resolve_layer(comp, v).ok_or_else(|| bad("track.setTarget", format!("no layer {v}")))?),
|
||||
};
|
||||
s.edit("Edit Target", None, |proj, _| {
|
||||
let st = tracker_mut(proj, cid, lid, uid)?.tracker_settings_mut().ok_or_else(|| bad("track.setTarget", "not a tracker"))?;
|
||||
if st.kind == TrackKind::Stabilize && target.is_some_and(|t| t != lid) {
|
||||
return Err(bad("track.setTarget", "Stabilize applies to the tracked layer"));
|
||||
}
|
||||
st.target = target;
|
||||
Ok(())
|
||||
})?;
|
||||
Ok(json!({"target": target.map(|t| t.0)}))
|
||||
}
|
||||
|
||||
fn options(s: &mut Session, p: &V) -> Result<V> {
|
||||
let (cid, lid, uid) = current(s, p, "track.options")?;
|
||||
let channel = match str_p(p, "channel").map(|c| c.to_ascii_lowercase()) {
|
||||
None => None,
|
||||
Some(c) => Some(match c.as_str() {
|
||||
"rgb" => TrackChannel::Rgb,
|
||||
"luminance" | "luma" => TrackChannel::Luminance,
|
||||
"saturation" => TrackChannel::Saturation,
|
||||
_ => return Err(bad("track.options", "channel: rgb|luminance|saturation")),
|
||||
}),
|
||||
};
|
||||
let action = match str_p(p, "action") {
|
||||
None => None,
|
||||
Some(a) => Some(match effectcraft_track::ConfidenceAction::from_name(a) {
|
||||
Some(effectcraft_track::ConfidenceAction::Continue) => LowConfidence::Continue,
|
||||
Some(effectcraft_track::ConfidenceAction::Stop) => LowConfidence::Stop,
|
||||
Some(effectcraft_track::ConfidenceAction::Extrapolate) => LowConfidence::Extrapolate,
|
||||
Some(effectcraft_track::ConfidenceAction::Adapt) => LowConfidence::Adapt,
|
||||
None => return Err(bad("track.options", "action: continue|stop|extrapolate|adapt")),
|
||||
}),
|
||||
};
|
||||
let r = s.edit("Motion Tracker Options", None, |proj, _| {
|
||||
let g = tracker_mut(proj, cid, lid, uid)?;
|
||||
if let Some(n) = str_p(p, "name") {
|
||||
g.name = n.to_string();
|
||||
}
|
||||
let o = &mut g.tracker_settings_mut().ok_or_else(|| bad("track.options", "not a tracker"))?.options;
|
||||
if let Some(c) = channel {
|
||||
o.channel = c;
|
||||
}
|
||||
if let Some(b) = f_p(p, "blur") {
|
||||
o.blur = b.clamp(0.0, 100.0);
|
||||
}
|
||||
if let Some(v) = b_p(p, "enhance") {
|
||||
o.enhance = v;
|
||||
}
|
||||
if let Some(v) = b_p(p, "subpixel") {
|
||||
o.subpixel = v;
|
||||
}
|
||||
if let Some(v) = b_p(p, "adaptEveryFrame") {
|
||||
o.adapt_every_frame = v;
|
||||
}
|
||||
if let Some(v) = f_p(p, "threshold") {
|
||||
o.threshold = v.clamp(0.0, 100.0);
|
||||
}
|
||||
if let Some(a) = action {
|
||||
o.action = a;
|
||||
}
|
||||
if let Some(v) = b_p(p, "trackShape") {
|
||||
o.track_shape = v;
|
||||
}
|
||||
Ok(serde_json::to_value(&*o).unwrap_or(V::Null))
|
||||
})?;
|
||||
Ok(r)
|
||||
}
|
||||
|
||||
fn vec2_p(p: &V, k: &str) -> Option<[f64; 2]> {
|
||||
let a = p.get(k)?.as_array()?;
|
||||
Some([a.first()?.as_f64()?, a.get(1)?.as_f64()?])
|
||||
}
|
||||
|
||||
fn set_point(s: &mut Session, p: &V) -> Result<V> {
|
||||
let (cid, lid, uid) = current(s, p, "track.setPoint")?;
|
||||
let n = p.get("point").and_then(V::as_u64).unwrap_or(1).max(1) as usize;
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let layer = comp.layer(lid).ok_or(EngineError::NoComp)?;
|
||||
let t = f_p(p, "time").map(Tick::from_seconds_f64).map(|t| comp.frame_rate.snap_nearest(t)).unwrap_or_else(|| s.time());
|
||||
let lt = layer.layer_time(t);
|
||||
let merge = super::merge_p(p).map(str::to_string);
|
||||
s.edit("Move Track Point", merge.as_deref(), |proj, _| {
|
||||
let g = tracker_mut(proj, cid, lid, uid)?;
|
||||
let tp_uid = g.track_points().nth(n - 1).map(|g| g.uid).ok_or_else(|| bad("track.setPoint", format!("no track point {n}")))?;
|
||||
let tp = g.find_group_mut(tp_uid).ok_or_else(|| bad("track.setPoint", "no track point"))?;
|
||||
let cur = point_values(tp, lt);
|
||||
let mut center = vec2_p(p, "center").unwrap_or(cur.spec.center);
|
||||
if let Some(d) = vec2_p(p, "move") {
|
||||
center = [center[0] + d[0], center[1] + d[1]];
|
||||
}
|
||||
let set = |tp: &mut PropGroup, m: &str, v: [f64; 2]| {
|
||||
if let Some(pr) = tp.get_mut(m) {
|
||||
pr.set_value_at(lt, Value::Vec2(v));
|
||||
}
|
||||
};
|
||||
set(tp, "featureCenter", center);
|
||||
for (k, m) in [("featureSize", "featureSize"), ("searchOffset", "searchOffset"), ("searchSize", "searchSize"), ("attachOffset", "attachPointOffset")] {
|
||||
if let Some(v) = vec2_p(p, k) {
|
||||
let v = if k.ends_with("Size") { [v[0].max(4.0), v[1].max(4.0)] } else { v };
|
||||
set(tp, m, v);
|
||||
}
|
||||
}
|
||||
let off = tp.get("attachPointOffset").map(|pr| pr.value_at(lt).as_vec2()).unwrap_or([0.0; 2]);
|
||||
set(tp, "attachPoint", [center[0] + off[0], center[1] + off[1]]);
|
||||
Ok(())
|
||||
})?;
|
||||
Ok(V::Null)
|
||||
}
|
||||
|
||||
// ---------- analyze / apply / reset ----------
|
||||
|
||||
fn analyze(s: &mut Session, p: &V) -> Result<V> {
|
||||
let (cid, lid, uid) = current(s, p, "track.analyze")?;
|
||||
let dir = match str_p(p, "direction") {
|
||||
Some(d) => Direction::from_name(d).ok_or_else(|| bad("track.analyze", "direction: forward|backward|frameForward|frameBackward"))?,
|
||||
None => Direction::Forward,
|
||||
};
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let layer = comp.layer(lid).ok_or(EngineError::NoComp)?;
|
||||
let (tg, settings) = layer.tracker(uid).ok_or_else(|| bad("track.analyze", "no tracker"))?;
|
||||
let fd = comp.frame_duration();
|
||||
let now = comp.frame_rate.snap_nearest(if s.state.active_comp == Some(cid) { s.time() } else { Tick::ZERO });
|
||||
let start = f_p(p, "start").map(|v| comp.frame_rate.snap_nearest(Tick::from_seconds_f64(v)));
|
||||
let end = f_p(p, "end").map(|v| comp.frame_rate.snap_nearest(Tick::from_seconds_f64(v)));
|
||||
let lo = layer.in_point.max(Tick::ZERO);
|
||||
let hi = layer.out_point.min(comp.duration) - fd;
|
||||
let mut times = vec![];
|
||||
match dir {
|
||||
Direction::Forward | Direction::FrameForward => {
|
||||
let mut t = start.unwrap_or(now).clamp(lo, hi);
|
||||
let stop = if dir == Direction::FrameForward { t + fd } else { end.unwrap_or(hi).min(hi) };
|
||||
while t <= stop && t <= hi {
|
||||
times.push(t);
|
||||
t += fd;
|
||||
}
|
||||
}
|
||||
Direction::Backward | Direction::FrameBackward => {
|
||||
let mut t = end.unwrap_or(now).clamp(lo, hi);
|
||||
let stop = if dir == Direction::FrameBackward { t - fd } else { start.unwrap_or(lo).max(lo) };
|
||||
while t >= stop && t >= lo {
|
||||
times.push(t);
|
||||
t -= fd;
|
||||
}
|
||||
}
|
||||
}
|
||||
if times.len() < 2 {
|
||||
return Err(bad("track.analyze", "nothing to analyze in that direction (at the layer's end)"));
|
||||
}
|
||||
let lt0 = layer.layer_time(times[0]);
|
||||
let points: Vec<_> = tg.track_points().map(|g| point_values(g, lt0)).collect();
|
||||
if points.is_empty() {
|
||||
return Err(bad("track.analyze", "the tracker has no track points"));
|
||||
}
|
||||
let work = Work {
|
||||
project: s.project.clone(),
|
||||
footage: s.footage.clone(),
|
||||
expr: s.expr.clone(),
|
||||
cache: s.layer_cache.clone(),
|
||||
comp: cid,
|
||||
layer: lid,
|
||||
settings: settings.clone(),
|
||||
points,
|
||||
times: times.clone(),
|
||||
};
|
||||
s.state.current_track = Some((lid, uid));
|
||||
let wait = b_p(p, "wait").unwrap_or(false);
|
||||
s.start_track(work, uid, dir, wait).map_err(EngineError::Other)?;
|
||||
let prog = s.track_progress();
|
||||
Ok(json!({
|
||||
"frames": times.len() - 1,
|
||||
"start": times[0].seconds(),
|
||||
"end": times.last().map(|t| t.seconds()),
|
||||
"running": s.is_tracking(),
|
||||
"progress": prog,
|
||||
}))
|
||||
}
|
||||
|
||||
fn apply(s: &mut Session, p: &V) -> Result<V> {
|
||||
let (cid, lid, uid) = current(s, p, "track.apply")?;
|
||||
let dims = match str_p(p, "dimensions") {
|
||||
Some(d) => Dims::from_name(d).ok_or_else(|| bad("track.apply", "dimensions: xy|x|y"))?,
|
||||
None => Dims::XY,
|
||||
};
|
||||
if s.is_tracking() {
|
||||
return Err(bad("track.apply", "wait for the analysis to finish"));
|
||||
}
|
||||
let n = s.edit("Apply Track", None, |proj, _| crate::tracking::apply(proj, cid, lid, uid, dims).map_err(|e| bad("track.apply", e)))?;
|
||||
Ok(json!({"frames": n}))
|
||||
}
|
||||
|
||||
fn reset(s: &mut Session, p: &V) -> Result<V> {
|
||||
let (cid, lid, uid) = current(s, p, "track.reset")?;
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let layer = comp.layer(lid).ok_or(EngineError::NoComp)?;
|
||||
let size = layer_size(s, cid, layer);
|
||||
s.edit("Reset Track", None, |proj, _| {
|
||||
let g = tracker_mut(proj, cid, lid, uid)?;
|
||||
let st = g.tracker_settings().cloned().unwrap_or_default();
|
||||
let n = g.track_points().count();
|
||||
let defaults = tracking::default_points(st.kind, n, size);
|
||||
let mut k = 0;
|
||||
for c in &mut g.children {
|
||||
let Node::Group(tp) = c else { continue };
|
||||
if tp.match_id != tracking::TRACK_POINT {
|
||||
continue;
|
||||
}
|
||||
let d = defaults.get(k).copied().unwrap_or(defaults[0]);
|
||||
k += 1;
|
||||
for (m, v) in [
|
||||
("featureCenter", Value::Vec2(d.center)),
|
||||
("featureSize", Value::Vec2(d.feature_size)),
|
||||
("searchOffset", Value::Vec2(d.search_offset)),
|
||||
("searchSize", Value::Vec2(d.search_size)),
|
||||
("confidence", Value::Scalar(0.0)),
|
||||
("attachPoint", Value::Vec2(d.center)),
|
||||
("attachPointOffset", Value::Vec2(d.attach_offset)),
|
||||
] {
|
||||
if let Some(pr) = tp.get_mut(m) {
|
||||
pr.keys.clear();
|
||||
pr.value = v;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
Ok(V::Null)
|
||||
}
|
||||
|
||||
fn delete(s: &mut Session, p: &V) -> Result<V> {
|
||||
let (cid, lid, uid) = current(s, p, "track.delete")?;
|
||||
s.edit("Delete Tracker", None, |proj, st| {
|
||||
let l = layer_mut(proj, cid, lid)?;
|
||||
if let Some(mt) = l.props.sub_mut(tracking::MOTION_TRACKERS) {
|
||||
mt.children.retain(|c| c.uid() != uid);
|
||||
if mt.children.is_empty() {
|
||||
l.props.children.retain(|c| c.match_id() != tracking::MOTION_TRACKERS);
|
||||
}
|
||||
}
|
||||
if st.current_track == Some((lid, uid)) {
|
||||
st.current_track = None;
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
Ok(V::Null)
|
||||
}
|
||||
|
||||
fn status(s: &mut Session, p: &V) -> Result<V> {
|
||||
s.poll_track();
|
||||
let progress = s.track_progress();
|
||||
let cur = current(s, p, "track.status").ok();
|
||||
let mut out = json!({"running": s.is_tracking(), "progress": progress});
|
||||
if let Some((cid, lid, uid)) = cur
|
||||
&& let Some(layer) = s.project.comp(cid).and_then(|c| c.layer(lid))
|
||||
&& let Some((g, st)) = layer.tracker(uid)
|
||||
{
|
||||
let points: Vec<V> = g
|
||||
.track_points()
|
||||
.map(|tp| {
|
||||
let keys = tp.get("featureCenter").map(|p| p.keys.len()).unwrap_or(0);
|
||||
let v = point_values(tp, layer.layer_time(s.time()));
|
||||
json!({"name": tp.name, "uid": tp.uid, "center": v.spec.center, "featureSize": v.spec.feature_size, "searchOffset": v.spec.search_offset, "searchSize": v.spec.search_size, "attachOffset": v.attach_offset, "keys": keys})
|
||||
})
|
||||
.collect();
|
||||
let trackers: Vec<V> = layer.trackers().map(|(g, _)| json!({"uid": g.uid, "name": g.name})).collect();
|
||||
out["layer"] = json!(lid.0);
|
||||
out["tracker"] = json!({"uid": uid, "name": g.name, "settings": st, "points": points});
|
||||
out["trackers"] = json!(trackers);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
|
@ -13,6 +13,7 @@ pub mod demo;
|
|||
pub mod links;
|
||||
pub mod menus;
|
||||
pub mod render_queue;
|
||||
pub mod tracking;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
|
|
@ -157,6 +158,9 @@ pub struct EditorState {
|
|||
/// Viewer 3D view per comp (Active Camera / Front / … / Custom View 3 and edited view cameras).
|
||||
#[serde(default)]
|
||||
pub views3d: std::collections::BTreeMap<ItemId, effectcraft_render::three_d::Views3D>,
|
||||
/// Tracker panel ▸ Current Track: (tracked layer, tracker group uid) in the active comp.
|
||||
#[serde(default)]
|
||||
pub current_track: Option<(LayerId, Uid)>,
|
||||
}
|
||||
|
||||
/// What Copy with Property Links / Copy Expression Only put on the clipboard.
|
||||
|
|
@ -207,6 +211,8 @@ pub struct Session {
|
|||
pub exporter: Option<Arc<dyn Exporter>>,
|
||||
/// The running (or finished, not yet polled) render.
|
||||
pub render_job: Option<render_queue::RenderJob>,
|
||||
/// The running (or finished, not yet polled) track analysis.
|
||||
pub track_job: Option<tracking::TrackJob>,
|
||||
pub events: Vec<Event>,
|
||||
/// Commands executed: (id, params).
|
||||
pub journal: Vec<(String, Value)>,
|
||||
|
|
@ -230,6 +236,7 @@ impl Default for Session {
|
|||
importer: None,
|
||||
exporter: None,
|
||||
render_job: None,
|
||||
track_job: None,
|
||||
events: vec![],
|
||||
journal: vec![],
|
||||
layer_cache: Arc::new(LayerCache::default()),
|
||||
|
|
@ -336,6 +343,11 @@ impl Session {
|
|||
self.state.selected_keys.retain(|k| ids.contains(&k.layer));
|
||||
self.state.selected_vertices.retain(|v| ids.contains(&v.layer));
|
||||
}
|
||||
if let Some((l, t)) = self.state.current_track
|
||||
&& self.active_comp().and_then(|c| c.layer(l)).and_then(|l| l.tracker(t)).is_none()
|
||||
{
|
||||
self.state.current_track = None;
|
||||
}
|
||||
self.state.project_selection.retain(|i| p.item(*i).is_some());
|
||||
}
|
||||
|
||||
|
|
@ -380,6 +392,7 @@ impl Session {
|
|||
self.state.selected_props.clear();
|
||||
self.state.selected_keys.clear();
|
||||
self.state.selected_vertices.clear();
|
||||
self.state.current_track = None;
|
||||
}
|
||||
self.events.push(Event::OpenComp(id));
|
||||
}
|
||||
|
|
@ -437,6 +450,8 @@ impl Session {
|
|||
job.wait();
|
||||
}
|
||||
self.render_job = None;
|
||||
self.stop_track();
|
||||
self.track_job = None;
|
||||
p.fix_next_id();
|
||||
self.project = Arc::new(p);
|
||||
self.history = History::default();
|
||||
|
|
@ -467,6 +482,8 @@ mod tests_styles;
|
|||
mod tests_text;
|
||||
#[cfg(test)]
|
||||
mod tests_timeline;
|
||||
#[cfg(test)]
|
||||
mod tests_track;
|
||||
|
||||
/// Font families available to text layers (bundled + scanned system fonts).
|
||||
/// Text animation presets: (id, name), for menus.
|
||||
|
|
|
|||
|
|
@ -788,6 +788,7 @@ Animation
|
|||
Track Camera | track.camera
|
||||
Warp Stabilizer VFX | track.warpStabilizer
|
||||
Track Motion | track.motion
|
||||
Stabilize Motion | track.stabilize
|
||||
Track Mask | track.mask
|
||||
Track this Property | track.property
|
||||
---
|
||||
|
|
|
|||
|
|
@ -0,0 +1,431 @@
|
|||
//! Motion tracking through the commands: synthetic footage generated in-test (a textured patch on
|
||||
//! a known path, or a whole frame under a known homography), analysis (blocking and background),
|
||||
//! apply (Transform, Stabilize, Corner Pin), undo/redo and serde.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use effectcraft_color::Label;
|
||||
use effectcraft_project::build;
|
||||
use effectcraft_project::tracking::TrackKind;
|
||||
use effectcraft_project::{Footage, FootageKind, ItemId, ItemKind, LayerId, LayerSource};
|
||||
use effectcraft_raster::Image;
|
||||
use effectcraft_render::{EvalCtx, FootageSource};
|
||||
use effectcraft_time::{FrameRate, Tick};
|
||||
use effectcraft_track::Homography;
|
||||
use serde_json::json;
|
||||
|
||||
use crate::Session;
|
||||
|
||||
const W: u32 = 320;
|
||||
const H: u32 = 240;
|
||||
const FPS: u32 = 25;
|
||||
|
||||
fn texture(x: f64, y: f64, seed: u32) -> f32 {
|
||||
let mut v = 0.0f64;
|
||||
let mut s = seed.wrapping_mul(2654435761).wrapping_add(12345);
|
||||
let mut rnd = || {
|
||||
s ^= s << 13;
|
||||
s ^= s >> 17;
|
||||
s ^= s << 5;
|
||||
(s % 10_000) as f64 / 10_000.0
|
||||
};
|
||||
for _ in 0..6 {
|
||||
let ang = rnd() * std::f64::consts::TAU;
|
||||
let period = 9.0 + rnd() * 25.0;
|
||||
let phase = rnd() * std::f64::consts::TAU;
|
||||
let k = std::f64::consts::TAU / period;
|
||||
v += (k * (x * ang.cos() + y * ang.sin()) + phase).sin() * (0.4 + rnd()) * 0.1;
|
||||
}
|
||||
(0.5 + v).clamp(0.0, 1.0) as f32
|
||||
}
|
||||
|
||||
/// Footage computed per frame (cached).
|
||||
struct Synth {
|
||||
make: Box<dyn Fn(u32) -> Image + Send + Sync>,
|
||||
cache: Mutex<HashMap<u32, Arc<Image>>>,
|
||||
}
|
||||
|
||||
impl FootageSource for Synth {
|
||||
fn frame(&self, _: ItemId, _: &Footage, t: Tick) -> Option<Arc<Image>> {
|
||||
let f = (t.seconds() * FPS as f64).round().max(0.0) as u32;
|
||||
let mut c = self.cache.lock().unwrap();
|
||||
Some(c.entry(f).or_insert_with(|| Arc::new((self.make)(f))).clone())
|
||||
}
|
||||
}
|
||||
|
||||
/// Patch pose on frame `f`: centre, rotation (degrees), scale.
|
||||
type Pose = fn(u32) -> ([f64; 2], f64, f64);
|
||||
|
||||
fn patch_frames(pose: Pose) -> impl Fn(u32) -> Image + Send + Sync {
|
||||
move |f| {
|
||||
let (c, rot, sc) = pose(f);
|
||||
let (sn, cs) = rot.to_radians().sin_cos();
|
||||
let mut img = Image::new(W, H);
|
||||
for y in 0..H {
|
||||
for x in 0..W {
|
||||
let p = [x as f64 + 0.5, y as f64 + 0.5];
|
||||
let d = [(p[0] - c[0]) / sc, (p[1] - c[1]) / sc];
|
||||
let l = [cs * d[0] + sn * d[1], -sn * d[0] + cs * d[1]];
|
||||
let v = if l[0].abs() <= 50.0 && l[1].abs() <= 50.0 { texture(l[0], l[1], 7) } else { 0.6 * texture(p[0] * 0.7, p[1] * 0.7, 3) };
|
||||
img.set(x, y, [v, v, v, 1.0]);
|
||||
}
|
||||
}
|
||||
img
|
||||
}
|
||||
}
|
||||
|
||||
/// A comp with the synthetic clip (layer `clip`) and a 100 × 100 solid above it (layer `solid`).
|
||||
fn setup(make: impl Fn(u32) -> Image + Send + Sync + 'static) -> (Session, LayerId, LayerId) {
|
||||
let mut s = Session::default();
|
||||
s.execute("comp.new", json!({"name": "Track", "width": W, "height": H, "frameRate": FPS, "duration": 1})).unwrap();
|
||||
let cid = s.active_comp_id().unwrap();
|
||||
let clip = s
|
||||
.edit("clip", None, |p, _| {
|
||||
let fr = FrameRate::new(FPS as i64, 1);
|
||||
let footage = Footage {
|
||||
path: "synthetic.mov".into(),
|
||||
kind: FootageKind::Video,
|
||||
width: W,
|
||||
height: H,
|
||||
pixel_aspect: 1.0,
|
||||
frame_rate: fr,
|
||||
native_rate: None,
|
||||
duration: Tick::from_seconds_f64(1.0),
|
||||
has_video: true,
|
||||
has_audio: false,
|
||||
alpha: Default::default(),
|
||||
premul_color: [0.0; 3],
|
||||
loop_count: 1,
|
||||
codec: String::new(),
|
||||
missing: false,
|
||||
sequence: vec![],
|
||||
};
|
||||
let item = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(footage));
|
||||
let comp = p.comp(cid).unwrap().clone();
|
||||
let l = build::layer(p, &comp, "Clip", LayerSource::Footage { item }, (W, H), None);
|
||||
let id = l.id;
|
||||
p.comp_mut(cid).unwrap().layers.insert(0, l);
|
||||
Ok(id)
|
||||
})
|
||||
.unwrap();
|
||||
s.footage = Arc::new(Synth { make: Box::new(make), cache: Mutex::default() });
|
||||
let solid = s.execute("layer.newSolid", json!({"width": 100, "height": 100, "color": "#ff0000"})).unwrap()["layer"].as_u64().unwrap();
|
||||
s.execute("layer.select", json!({"layers": [clip.0]})).unwrap();
|
||||
(s, clip, LayerId(solid))
|
||||
}
|
||||
|
||||
fn tracker_points(s: &Session, layer: LayerId) -> Vec<effectcraft_project::PropGroup> {
|
||||
let l = s.active_comp().unwrap().layer(layer).unwrap();
|
||||
l.trackers().next().unwrap().0.track_points().cloned().collect()
|
||||
}
|
||||
|
||||
fn frame_time(f: u32) -> Tick {
|
||||
FrameRate::new(FPS as i64, 1).tick_of(f as i64)
|
||||
}
|
||||
|
||||
fn dist(a: [f64; 2], b: [f64; 2]) -> f64 {
|
||||
((a[0] - b[0]).powi(2) + (a[1] - b[1]).powi(2)).sqrt()
|
||||
}
|
||||
|
||||
fn translate_pose(f: u32) -> ([f64; 2], f64, f64) {
|
||||
let t = f as f64;
|
||||
([150.0 + 2.37 * t + 4.0 * (t * 0.4).sin(), 110.0 + 1.13 * t], 0.0, 1.0)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn track_motion_analyze_apply_transform_undo_redo() {
|
||||
let (mut s, clip, solid) = setup(patch_frames(translate_pose));
|
||||
// The menu entry and the Tracker panel's Track Motion button.
|
||||
assert!(s.is_enabled("track.motion"));
|
||||
let r = s.execute("track.motion", json!({})).unwrap();
|
||||
assert_eq!(r["points"], 1);
|
||||
let st = s.execute("track.status", json!({})).unwrap();
|
||||
assert_eq!(st["tracker"]["settings"]["target"], json!(solid.0), "{st}");
|
||||
// Put the track point on the patch and analyze to the end (blocking).
|
||||
s.execute("track.setPoint", json!({"point": 1, "center": translate_pose(0).0, "featureSize": [36, 36], "searchSize": [80, 80]})).unwrap();
|
||||
let r = s.execute("track.analyze", json!({"direction": "forward", "wait": true})).unwrap();
|
||||
assert_eq!(r["frames"], 24);
|
||||
s.poll_track();
|
||||
assert!(s.track_job.is_none());
|
||||
let tp = &tracker_points(&s, clip)[0];
|
||||
let fc = tp.get("featureCenter").unwrap();
|
||||
assert_eq!(fc.keys.len(), 25);
|
||||
let conf = tp.get("confidence").unwrap();
|
||||
let mut worst: f64 = 0.0;
|
||||
for f in 0..25 {
|
||||
let t = frame_time(f);
|
||||
worst = worst.max(dist(fc.value_at(t).as_vec2(), translate_pose(f).0));
|
||||
assert!(conf.value_at(t).as_f64() > 90.0);
|
||||
}
|
||||
assert!(worst < 0.25, "worst {worst}");
|
||||
// CTI moved to the last analysed frame.
|
||||
assert_eq!(s.time(), frame_time(24));
|
||||
// Apply: the solid's position follows the attach point.
|
||||
s.execute("track.apply", json!({})).unwrap();
|
||||
let pos = s.active_comp().unwrap().layer(solid).unwrap().props.prop("transform/position").unwrap().clone();
|
||||
assert_eq!(pos.keys.len(), 25);
|
||||
for f in [0, 7, 24] {
|
||||
let v = pos.value_at(frame_time(f)).as_vec2();
|
||||
assert!(dist(v, translate_pose(f).0) < 0.25, "frame {f}: {v:?}");
|
||||
}
|
||||
// Undo the apply, then the analysis; redo both.
|
||||
assert!(s.undo());
|
||||
assert!(s.active_comp().unwrap().layer(solid).unwrap().props.prop("transform/position").unwrap().keys.is_empty());
|
||||
assert!(s.undo());
|
||||
assert!(tracker_points(&s, clip)[0].get("featureCenter").unwrap().keys.is_empty());
|
||||
assert!(s.redo());
|
||||
assert_eq!(tracker_points(&s, clip)[0].get("featureCenter").unwrap().keys.len(), 25);
|
||||
assert!(s.redo());
|
||||
assert_eq!(s.active_comp().unwrap().layer(solid).unwrap().props.prop("transform/position").unwrap().keys.len(), 25);
|
||||
// X only keeps the solid's Y.
|
||||
s.undo();
|
||||
s.execute("track.apply", json!({"dimensions": "x"})).unwrap();
|
||||
let pos = s.active_comp().unwrap().layer(solid).unwrap().props.prop("transform/position").unwrap().clone();
|
||||
let v = pos.value_at(frame_time(10)).as_vec2();
|
||||
assert!((v[0] - translate_pose(10).0[0]).abs() < 0.25 && (v[1] - 120.0).abs() < 1e-9, "{v:?}");
|
||||
// Serde round trip keeps the tracker.
|
||||
let json = s.project.to_json();
|
||||
let back = effectcraft_project::Project::from_json(&json).unwrap();
|
||||
let l = back.comp(s.active_comp_id().unwrap()).unwrap().layer(clip).unwrap();
|
||||
let (g, st) = l.trackers().next().unwrap();
|
||||
assert_eq!(st.kind, TrackKind::Transform);
|
||||
assert_eq!(g.track_points().next().unwrap().get("featureCenter").unwrap().keys.len(), 25);
|
||||
assert_eq!(l.props.children[0].match_id(), "motionTrackers");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stabilize_keeps_the_feature_fixed() {
|
||||
let (mut s, clip, _) = setup(patch_frames(translate_pose));
|
||||
s.execute("track.stabilize", json!({})).unwrap();
|
||||
s.execute("track.setPoint", json!({"point": 1, "center": translate_pose(0).0, "featureSize": [36, 36], "searchSize": [80, 80]})).unwrap();
|
||||
s.execute("time.set", json!({"time": 0})).unwrap();
|
||||
s.execute("track.analyze", json!({"wait": true})).unwrap();
|
||||
s.execute("track.apply", json!({})).unwrap();
|
||||
let cid = s.active_comp_id().unwrap();
|
||||
let comp = s.project.comp(cid).unwrap().clone();
|
||||
let l = comp.layer(clip).unwrap();
|
||||
let at = |f: u32| {
|
||||
let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(f));
|
||||
let (m, _) = ctx.layer_to_comp(l);
|
||||
let c = translate_pose(f).0;
|
||||
let q = m.apply(effectcraft_geom::vec2(c[0], c[1]));
|
||||
[q.x, q.y]
|
||||
};
|
||||
let p0 = at(0);
|
||||
// The layer itself didn't move on the first frame.
|
||||
assert!(dist(p0, translate_pose(0).0) < 1e-6, "{p0:?}");
|
||||
let worst = (1..25).map(|f| dist(at(f), p0)).fold(0.0, f64::max);
|
||||
assert!(worst < 0.25, "feature drifts by {worst} px");
|
||||
}
|
||||
|
||||
fn rotating_pose(f: u32) -> ([f64; 2], f64, f64) {
|
||||
let t = f as f64;
|
||||
([160.0 + 1.2 * t, 120.0 - 0.7 * t], 1.5 * t, 1.0 + 0.008 * t)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transform_rotation_scale_and_background_job() {
|
||||
let (mut s, clip, solid) = setup(patch_frames(rotating_pose));
|
||||
s.execute("track.new", json!({"kind": "transform", "rotation": true, "scale": true})).unwrap();
|
||||
assert_eq!(tracker_points(&s, clip).len(), 2);
|
||||
let (c, _, _) = rotating_pose(0);
|
||||
s.execute("track.setPoint", json!({"point": 1, "center": [c[0] - 26.0, c[1]], "featureSize": [26, 26], "searchSize": [56, 56]})).unwrap();
|
||||
s.execute("track.setPoint", json!({"point": 2, "center": [c[0] + 26.0, c[1]], "featureSize": [26, 26], "searchSize": [56, 56]})).unwrap();
|
||||
// Background analysis with progress, polled like the UI does.
|
||||
let r = s.execute("track.analyze", json!({"direction": "forward", "end": 0.6})).unwrap();
|
||||
assert_eq!(r["frames"], 15);
|
||||
let t0 = std::time::Instant::now();
|
||||
while s.is_tracking() {
|
||||
assert!(t0.elapsed().as_secs() < 120);
|
||||
std::thread::sleep(std::time::Duration::from_millis(5));
|
||||
s.poll_track();
|
||||
}
|
||||
s.poll_track();
|
||||
let st = s.execute("track.status", json!({})).unwrap();
|
||||
assert_eq!(st["running"], false);
|
||||
assert_eq!(st["tracker"]["points"][0]["keys"], 16);
|
||||
s.execute("track.apply", json!({})).unwrap();
|
||||
let tr = s.active_comp().unwrap().layer(solid).unwrap().props.sub("transform").unwrap().clone();
|
||||
let rot = tr.get("rotation").unwrap();
|
||||
let sc = tr.get("scale").unwrap();
|
||||
for f in [5, 10, 15] {
|
||||
let (_, r, k) = rotating_pose(f);
|
||||
let got = rot.value_at(frame_time(f)).as_f64();
|
||||
assert!((got - r).abs() < 0.3, "frame {f}: rotation {got} vs {r}");
|
||||
let gs = sc.value_at(frame_time(f)).as_vec3()[0];
|
||||
assert!((gs - 100.0 * k).abs() < 0.5, "frame {f}: scale {gs} vs {}", 100.0 * k);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn perspective_corner_pin_follows_a_known_homography() {
|
||||
// Frame f shows the texture under H_f (identity → `end` over 12 frames).
|
||||
let end = Homography([[1.04, 0.05, 6.0], [-0.03, 0.97, 4.0], [0.00012, -0.00008, 1.0]]);
|
||||
let hf = move |f: u32| {
|
||||
let k = f as f64 / 12.0;
|
||||
let mut m = [[0.0; 3]; 3];
|
||||
for i in 0..3 {
|
||||
for j in 0..3 {
|
||||
let id = if i == j { 1.0 } else { 0.0 };
|
||||
m[i][j] = id + (end.0[i][j] - id) * k;
|
||||
}
|
||||
}
|
||||
Homography(m)
|
||||
};
|
||||
let inv = move |f: u32, p: [f64; 2]| {
|
||||
let h = hf(f);
|
||||
// Solve H x = p by mapping four points back (exact for a homography).
|
||||
let src = [[0.0, 0.0], [W as f64, 0.0], [0.0, H as f64], [W as f64, H as f64]];
|
||||
let dst = src.map(|q| h.apply(q));
|
||||
Homography::from_points(dst, src).unwrap().apply(p)
|
||||
};
|
||||
let make = move |f: u32| {
|
||||
let h = hf(f);
|
||||
let src = [[0.0, 0.0], [W as f64, 0.0], [0.0, H as f64], [W as f64, H as f64]];
|
||||
let back = Homography::from_points(src.map(|q| h.apply(q)), src).unwrap();
|
||||
let _ = inv;
|
||||
let mut img = Image::new(W, H);
|
||||
for y in 0..H {
|
||||
for x in 0..W {
|
||||
let q = back.apply([x as f64 + 0.5, y as f64 + 0.5]);
|
||||
let v = texture(q[0], q[1], 11);
|
||||
img.set(x, y, [v, v, v, 1.0]);
|
||||
}
|
||||
}
|
||||
img
|
||||
};
|
||||
let (mut s, clip, solid) = setup(make);
|
||||
s.execute("track.new", json!({"kind": "perspective"})).unwrap();
|
||||
let corners = [[90.0, 70.0], [230.0, 70.0], [90.0, 170.0], [230.0, 170.0]];
|
||||
for (i, c) in corners.iter().enumerate() {
|
||||
s.execute("track.setPoint", json!({"point": i + 1, "center": c, "featureSize": [30, 30], "searchSize": [60, 60]})).unwrap();
|
||||
}
|
||||
s.execute("track.analyze", json!({"wait": true, "end": 12.0 / FPS as f64})).unwrap();
|
||||
let pts = tracker_points(&s, clip);
|
||||
for (i, tp) in pts.iter().enumerate() {
|
||||
let got = tp.get("featureCenter").unwrap().value_at(frame_time(12)).as_vec2();
|
||||
assert!(dist(got, end.apply(corners[i])) < 0.3, "point {i}: {got:?} vs {:?}", end.apply(corners[i]));
|
||||
}
|
||||
s.execute("track.apply", json!({})).unwrap();
|
||||
let l = s.active_comp().unwrap().layer(solid).unwrap().clone();
|
||||
let fx = l.effects().unwrap().groups().next().unwrap().clone();
|
||||
assert_eq!(fx.match_id, "ec.distort.cornerpin");
|
||||
// The solid is 100 × 100, centred: comp = layer + (110, 70).
|
||||
for (m, i) in [("ul", 0), ("ur", 1), ("ll", 2), ("lr", 3)] {
|
||||
let p = fx.get(m).unwrap();
|
||||
assert_eq!(p.keys.len(), 13);
|
||||
for f in [6, 12] {
|
||||
let want = hf(f).apply(corners[i]);
|
||||
let got = p.value_at(frame_time(f)).as_vec2();
|
||||
assert!(dist([got[0] + 110.0, got[1] + 70.0], want) < 0.35, "{m} frame {f}: {got:?} vs {want:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backward_frame_steps_options_and_reset() {
|
||||
let (mut s, clip, _) = setup(patch_frames(translate_pose));
|
||||
s.execute("track.motion", json!({})).unwrap();
|
||||
s.execute("time.set", json!({"time": 10.0 / FPS as f64})).unwrap();
|
||||
s.execute("track.setPoint", json!({"point": 1, "center": translate_pose(10).0, "featureSize": [36, 36], "searchSize": [80, 80]})).unwrap();
|
||||
s.execute("track.options", json!({"channel": "rgb", "blur": 2, "threshold": 70, "action": "extrapolate", "name": "Patch"})).unwrap();
|
||||
let st = s.execute("track.status", json!({})).unwrap();
|
||||
assert_eq!(st["tracker"]["name"], "Patch");
|
||||
assert_eq!(st["tracker"]["settings"]["options"]["channel"], "rgb");
|
||||
s.execute("track.analyze", json!({"direction": "frameBackward", "wait": true})).unwrap();
|
||||
s.execute("track.analyze", json!({"direction": "backward", "wait": true})).unwrap();
|
||||
assert_eq!(s.time(), Tick::ZERO);
|
||||
s.execute("track.analyze", json!({"direction": "frameForward", "wait": true, "start": 10.0 / FPS as f64})).unwrap();
|
||||
let fc = tracker_points(&s, clip)[0].get("featureCenter").unwrap().clone();
|
||||
assert_eq!(fc.keys.len(), 12);
|
||||
for f in 0..12 {
|
||||
assert!(dist(fc.value_at(frame_time(f)).as_vec2(), translate_pose(f).0) < 0.25, "frame {f}");
|
||||
}
|
||||
// Unknown params are rejected for agents.
|
||||
assert!(s.execute_checked("track.analyze", json!({"dir": "forward"})).is_err());
|
||||
s.execute("track.setType", json!({"kind": "perspective"})).unwrap();
|
||||
assert_eq!(tracker_points(&s, clip).len(), 4);
|
||||
s.execute("track.setType", json!({"kind": "transform", "rotation": false})).unwrap();
|
||||
assert_eq!(tracker_points(&s, clip).len(), 1);
|
||||
s.execute("track.reset", json!({})).unwrap();
|
||||
assert!(tracker_points(&s, clip)[0].get("featureCenter").unwrap().keys.is_empty());
|
||||
s.execute("track.delete", json!({})).unwrap();
|
||||
assert!(s.active_comp().unwrap().layer(clip).unwrap().motion_trackers().is_none());
|
||||
assert!(s.execute("track.analyze", json!({})).is_err());
|
||||
}
|
||||
|
||||
/// End-to-end analysis speed at 1080p for one track point: footage frames through the renderer's
|
||||
/// layer source and the tracker (frames are prepared before timing).
|
||||
#[test]
|
||||
fn analyze_1080p_single_point_speed() {
|
||||
let (w, h) = (1920u32, 1080u32);
|
||||
let mut base = Image::new(w + 64, h + 64);
|
||||
for y in 0..base.height {
|
||||
for x in 0..base.width {
|
||||
let v = 0.5 + 0.25 * ((x as f32 * 0.21).sin() * (y as f32 * 0.13).cos()) + 0.15 * ((x + 2 * y) as f32 * 0.07).sin();
|
||||
base.set(x, y, [v, v, v, 1.0]);
|
||||
}
|
||||
}
|
||||
let frames: Vec<Arc<Image>> = (0..31u32)
|
||||
.map(|f| {
|
||||
let (dx, dy) = (f as i64, (f / 2) as i64);
|
||||
let mut img = Image::new(w, h);
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
img.set(x, y, base.get(x as i64 + 32 - dx, y as i64 + 32 - dy));
|
||||
}
|
||||
}
|
||||
Arc::new(img)
|
||||
})
|
||||
.collect();
|
||||
struct Pre(Vec<Arc<Image>>);
|
||||
impl FootageSource for Pre {
|
||||
fn frame(&self, _: ItemId, _: &Footage, t: Tick) -> Option<Arc<Image>> {
|
||||
let f = (t.seconds() * FPS as f64).round().max(0.0) as usize;
|
||||
self.0.get(f.min(self.0.len() - 1)).cloned()
|
||||
}
|
||||
}
|
||||
let mut s = Session::default();
|
||||
s.execute("comp.new", json!({"name": "HD", "width": w, "height": h, "frameRate": FPS, "duration": 2})).unwrap();
|
||||
let cid = s.active_comp_id().unwrap();
|
||||
let clip = s
|
||||
.edit("clip", None, |p, _| {
|
||||
let footage = Footage {
|
||||
path: "hd.mov".into(),
|
||||
kind: FootageKind::Video,
|
||||
width: w,
|
||||
height: h,
|
||||
pixel_aspect: 1.0,
|
||||
frame_rate: FrameRate::new(FPS as i64, 1),
|
||||
native_rate: None,
|
||||
duration: Tick::from_seconds_f64(2.0),
|
||||
has_video: true,
|
||||
has_audio: false,
|
||||
alpha: Default::default(),
|
||||
premul_color: [0.0; 3],
|
||||
loop_count: 1,
|
||||
codec: String::new(),
|
||||
missing: false,
|
||||
sequence: vec![],
|
||||
};
|
||||
let item = p.add_item("hd", Label::Aqua, None, ItemKind::Footage(footage));
|
||||
let comp = p.comp(cid).unwrap().clone();
|
||||
let l = build::layer(p, &comp, "HD", LayerSource::Footage { item }, (w, h), None);
|
||||
let id = l.id;
|
||||
p.comp_mut(cid).unwrap().layers.insert(0, l);
|
||||
Ok(id)
|
||||
})
|
||||
.unwrap();
|
||||
s.footage = Arc::new(Pre(frames));
|
||||
s.execute("track.motion", json!({"layer": clip.0})).unwrap();
|
||||
s.execute("track.setPoint", json!({"point": 1, "center": [960, 540], "featureSize": [48, 48], "searchSize": [128, 128]})).unwrap();
|
||||
let t0 = std::time::Instant::now();
|
||||
s.execute("track.analyze", json!({"wait": true, "end": 30.0 / FPS as f64})).unwrap();
|
||||
let secs = t0.elapsed().as_secs_f64();
|
||||
let fps = 30.0 / secs;
|
||||
eprintln!("1080p single-point analysis: {fps:.0} frames/s end to end ({:.1} ms/frame)", secs * 1e3 / 30.0);
|
||||
let fc = tracker_points(&s, clip)[0].get("featureCenter").unwrap().clone();
|
||||
assert!(dist(fc.value_at(frame_time(30)).as_vec2(), [990.0, 555.0]) < 0.25);
|
||||
assert!(fps > 30.0, "{fps} frames/s");
|
||||
}
|
||||
|
|
@ -0,0 +1,607 @@
|
|||
//! Motion tracking runtime: the analysis job (background thread with progress and cancel, or
|
||||
//! blocking for the CLI / agents / wasm), writing track results into the tracker's Track Point
|
||||
//! properties, and applying tracks (Transform, Stabilize, Corner Pin) to layers.
|
||||
//!
|
||||
//! The algorithms are in `effectcraft-track`; the property layout in
|
||||
//! `effectcraft_project::tracking`.
|
||||
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use effectcraft_geom::{Mat3, vec2};
|
||||
use effectcraft_project::tracking::{LowConfidence, TrackChannel, TrackKind, TrackerOptions, TrackerSettings};
|
||||
use effectcraft_project::{Comp, ItemId, Keyframe, Layer, LayerId, Project, PropGroup, Uid, Value};
|
||||
use effectcraft_raster::Image;
|
||||
use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer};
|
||||
use effectcraft_time::Tick;
|
||||
use effectcraft_track as trk;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{Event, Session};
|
||||
|
||||
/// Tracker panel ▸ Analyze buttons.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Direction {
|
||||
#[default]
|
||||
Forward,
|
||||
Backward,
|
||||
FrameForward,
|
||||
FrameBackward,
|
||||
}
|
||||
|
||||
impl Direction {
|
||||
pub fn from_name(s: &str) -> Option<Direction> {
|
||||
let n = s.to_ascii_lowercase().replace([' ', '_', '-'], "");
|
||||
Some(match n.as_str() {
|
||||
"forward" | "fwd" => Direction::Forward,
|
||||
"backward" | "back" | "bwd" => Direction::Backward,
|
||||
"frameforward" | "oneframeforward" | "1frameforward" => Direction::FrameForward,
|
||||
"framebackward" | "oneframebackward" | "1framebackward" => Direction::FrameBackward,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
pub fn forward(self) -> bool {
|
||||
matches!(self, Direction::Forward | Direction::FrameForward)
|
||||
}
|
||||
}
|
||||
|
||||
/// Track options for the algorithms.
|
||||
pub fn track_options(o: &TrackerOptions) -> trk::TrackOptions {
|
||||
trk::TrackOptions {
|
||||
channel: match o.channel {
|
||||
TrackChannel::Rgb => trk::Channel::Rgb,
|
||||
TrackChannel::Luminance => trk::Channel::Luminance,
|
||||
TrackChannel::Saturation => trk::Channel::Saturation,
|
||||
},
|
||||
blur: o.blur,
|
||||
enhance: o.enhance,
|
||||
subpixel: o.subpixel,
|
||||
adapt_every_frame: o.adapt_every_frame,
|
||||
threshold: o.threshold,
|
||||
action: match o.action {
|
||||
LowConfidence::Continue => trk::ConfidenceAction::Continue,
|
||||
LowConfidence::Stop => trk::ConfidenceAction::Stop,
|
||||
LowConfidence::Extrapolate => trk::ConfidenceAction::Extrapolate,
|
||||
LowConfidence::Adapt => trk::ConfidenceAction::Adapt,
|
||||
},
|
||||
track_shape: o.track_shape,
|
||||
}
|
||||
}
|
||||
|
||||
/// A track point's regions and attach offset read from its properties at a layer time.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct PointValues {
|
||||
pub uid: Uid,
|
||||
pub spec: trk::PointSpec,
|
||||
pub attach_offset: [f64; 2],
|
||||
}
|
||||
|
||||
pub fn point_values(tp: &PropGroup, lt: Tick) -> PointValues {
|
||||
let v = |m: &str, d: [f64; 2]| tp.get(m).map(|p| p.value_at(lt).as_vec2()).unwrap_or(d);
|
||||
PointValues {
|
||||
uid: tp.uid,
|
||||
spec: trk::PointSpec {
|
||||
center: v("featureCenter", [0.0; 2]),
|
||||
feature_size: v("featureSize", [32.0; 2]),
|
||||
search_offset: v("searchOffset", [0.0; 2]),
|
||||
search_size: v("searchSize", [64.0; 2]),
|
||||
},
|
||||
attach_offset: v("attachPointOffset", [0.0; 2]),
|
||||
}
|
||||
}
|
||||
|
||||
/// Attach points of a frame from the feature centres, given the start frame's centres and attach
|
||||
/// points: two-point transforms rotate and scale the offsets with the track, Parallel Corner Pin
|
||||
/// maps them through the 3-point affine, Perspective Corner Pin through the 4-point homography.
|
||||
pub fn attach_points(kind: TrackKind, rot_scale: bool, ref_c: &[[f64; 2]], ref_a: &[[f64; 2]], cur: &[[f64; 2]]) -> Vec<[f64; 2]> {
|
||||
let translate = || cur.iter().zip(ref_c.iter().zip(ref_a)).map(|(c, (r, a))| [c[0] + a[0] - r[0], c[1] + a[1] - r[1]]).collect::<Vec<_>>();
|
||||
let through = |h: Option<trk::Homography>| match h {
|
||||
Some(h) => ref_a.iter().map(|a| h.apply(*a)).collect(),
|
||||
None => translate(),
|
||||
};
|
||||
match kind {
|
||||
TrackKind::Perspective if cur.len() >= 4 => {
|
||||
let h = trk::Homography::from_points([ref_c[0], ref_c[1], ref_c[2], ref_c[3]], [cur[0], cur[1], cur[2], cur[3]]);
|
||||
through(h)
|
||||
}
|
||||
TrackKind::Affine if cur.len() >= 3 => through(trk::Homography::affine_from_points([ref_c[0], ref_c[1], ref_c[2]], [cur[0], cur[1], cur[2]])),
|
||||
_ if rot_scale && cur.len() >= 2 => {
|
||||
let Some(s) = trk::Homography::similarity_from_points([ref_c[0], ref_c[1]], [cur[0], cur[1]]) else { return translate() };
|
||||
let m = s.0;
|
||||
cur.iter()
|
||||
.zip(ref_c.iter().zip(ref_a))
|
||||
.map(|(c, (r, a))| {
|
||||
let d = [a[0] - r[0], a[1] - r[1]];
|
||||
[c[0] + m[0][0] * d[0] + m[0][1] * d[1], c[1] + m[1][0] * d[0] + m[1][1] * d[1]]
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
_ => translate(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Live analysis progress (serde for agents / the control channel).
|
||||
#[derive(Clone, Debug, Default, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TrackProgress {
|
||||
pub done: u64,
|
||||
pub total: u64,
|
||||
pub elapsed: f64,
|
||||
/// Analysed frames per second.
|
||||
pub fps: f64,
|
||||
pub finished: bool,
|
||||
pub cancelled: bool,
|
||||
/// Stopped by "If Confidence is Below … ▸ Stop Tracking".
|
||||
pub low_confidence_stop: bool,
|
||||
pub error: Option<String>,
|
||||
/// Comp time (seconds) of the last analysed frame.
|
||||
pub time: f64,
|
||||
}
|
||||
|
||||
/// One analysed frame.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct FrameResult {
|
||||
pub comp_time: Tick,
|
||||
pub layer_time: Tick,
|
||||
/// Per point: (point group uid, result, attach point).
|
||||
pub points: Vec<(Uid, trk::PointResult, [f64; 2])>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct TrackShared {
|
||||
pub cancel: AtomicBool,
|
||||
pub state: Mutex<TrackProgress>,
|
||||
pub frames: Mutex<Vec<FrameResult>>,
|
||||
}
|
||||
|
||||
fn lock<T>(m: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
|
||||
m.lock().unwrap_or_else(|e| e.into_inner())
|
||||
}
|
||||
|
||||
/// A running (or finished, not yet polled) analysis.
|
||||
pub struct TrackJob {
|
||||
pub shared: Arc<TrackShared>,
|
||||
thread: Option<std::thread::JoinHandle<()>>,
|
||||
pub comp: ItemId,
|
||||
pub layer: LayerId,
|
||||
pub tracker: Uid,
|
||||
pub direction: Direction,
|
||||
}
|
||||
|
||||
impl TrackJob {
|
||||
pub fn is_finished(&self) -> bool {
|
||||
lock(&self.shared.state).finished
|
||||
}
|
||||
pub fn progress(&self) -> TrackProgress {
|
||||
lock(&self.shared.state).clone()
|
||||
}
|
||||
pub fn cancel(&self) {
|
||||
self.shared.cancel.store(true, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/// What the worker needs, detached from the session.
|
||||
pub(crate) struct Work {
|
||||
pub project: Arc<Project>,
|
||||
pub footage: Arc<dyn FootageSource>,
|
||||
pub expr: Option<Arc<dyn ExprHost>>,
|
||||
pub cache: Arc<LayerCache>,
|
||||
pub comp: ItemId,
|
||||
pub layer: LayerId,
|
||||
pub settings: TrackerSettings,
|
||||
pub points: Vec<PointValues>,
|
||||
/// Comp times to analyse; the first is the start frame.
|
||||
pub times: Vec<Tick>,
|
||||
}
|
||||
|
||||
/// The layer's source frame at comp time `t` at 100 % (layer pixels + offset = image pixels).
|
||||
/// Footage at its native size is used straight from the footage source (no copy).
|
||||
fn source_frame(
|
||||
r: &Renderer,
|
||||
project: &Project,
|
||||
cid: ItemId,
|
||||
comp: &Comp,
|
||||
layer: &Layer,
|
||||
t: Tick,
|
||||
expr: Option<&dyn ExprHost>,
|
||||
) -> Option<(Arc<Image>, [f64; 2])> {
|
||||
let ctx = EvalCtx { project, comp_id: cid, comp, time: t, expr };
|
||||
if let effectcraft_project::LayerSource::Footage { item } = &layer.source
|
||||
&& let Some(effectcraft_project::ItemKind::Footage(f)) = project.item(*item).map(|i| &i.kind)
|
||||
&& f.has_video
|
||||
&& let Some(img) = r.footage.frame(*item, f, ctx.source_time(layer))
|
||||
&& img.width == f.width
|
||||
&& img.height == f.height
|
||||
{
|
||||
return Some((img, [0.0; 2]));
|
||||
}
|
||||
let buf = r.layer_source(&ctx, layer)?;
|
||||
if (buf.scale - 1.0).abs() > 1e-9 && buf.scale > 0.0 {
|
||||
// Footage decoded at another size: bring it to layer pixels.
|
||||
let w = (buf.img.width as f64 / buf.scale).round().max(1.0) as u32;
|
||||
let h = (buf.img.height as f64 / buf.scale).round().max(1.0) as u32;
|
||||
let off = [buf.offset[0] / buf.scale, buf.offset[1] / buf.scale];
|
||||
return Some((Arc::new(effectcraft_raster::resample(&buf.img, w, h)), off));
|
||||
}
|
||||
Some((Arc::new(buf.img), buf.offset))
|
||||
}
|
||||
|
||||
pub(crate) fn run_work(w: Work, shared: &TrackShared) {
|
||||
let t0 = web_time::Instant::now();
|
||||
let fail = |msg: &str| {
|
||||
let mut s = lock(&shared.state);
|
||||
s.error = Some(msg.to_string());
|
||||
s.finished = true;
|
||||
};
|
||||
let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") };
|
||||
let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") };
|
||||
let mut r = Renderer::new(&w.project, w.footage.as_ref(), RenderOpts { scale: 1.0, ..Default::default() });
|
||||
r.expr = w.expr.as_deref();
|
||||
r.cache = Some(&w.cache);
|
||||
let r = &r;
|
||||
let frame_at = |t: Tick| source_frame(r, &w.project, w.comp, comp, layer, t, w.expr.as_deref());
|
||||
{
|
||||
let mut s = lock(&shared.state);
|
||||
s.total = w.times.len().saturating_sub(1) as u64;
|
||||
}
|
||||
let Some(first) = frame_at(w.times[0]) else { return fail("the layer has no pixels to track") };
|
||||
let specs: Vec<trk::PointSpec> = w.points.iter().map(|p| p.spec).collect();
|
||||
let mut tracker = trk::Tracker::new(track_options(&w.settings.options), &specs, &trk::Frame { img: &first.0, offset: first.1 });
|
||||
let ref_c: Vec<[f64; 2]> = specs.iter().map(|s| s.center).collect();
|
||||
let ref_a: Vec<[f64; 2]> = w.points.iter().map(|p| [p.spec.center[0] + p.attach_offset[0], p.spec.center[1] + p.attach_offset[1]]).collect();
|
||||
let rot_scale = w.settings.rotation || w.settings.scale;
|
||||
// The start frame keeps its regions and gets keys too.
|
||||
lock(&shared.frames).push(FrameResult {
|
||||
comp_time: w.times[0],
|
||||
layer_time: layer.layer_time(w.times[0]),
|
||||
points: w
|
||||
.points
|
||||
.iter()
|
||||
.zip(&ref_a)
|
||||
.map(|(p, a)| (p.uid, trk::PointResult { center: p.spec.center, confidence: 100.0, rotation: 0.0, scale: 1.0, status: trk::Status::Tracked }, *a))
|
||||
.collect(),
|
||||
});
|
||||
let mut next = if w.times.len() > 1 { frame_at(w.times[1]) } else { None };
|
||||
for k in 1..w.times.len() {
|
||||
if shared.cancel.load(Ordering::Relaxed) {
|
||||
lock(&shared.state).cancelled = true;
|
||||
break;
|
||||
}
|
||||
let Some(cur) = next.take() else {
|
||||
lock(&shared.state).error = Some(format!("no frame at {:.3} s", w.times[k].seconds()));
|
||||
break;
|
||||
};
|
||||
// Track this frame while the next one renders.
|
||||
let (res, nf) =
|
||||
rayon::join(|| tracker.step(&trk::Frame { img: &cur.0, offset: cur.1 }), || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None });
|
||||
next = nf;
|
||||
if res.iter().any(|p| p.status == trk::Status::Stopped) {
|
||||
lock(&shared.state).low_confidence_stop = true;
|
||||
break;
|
||||
}
|
||||
let centers: Vec<[f64; 2]> = res.iter().map(|p| p.center).collect();
|
||||
let attach = attach_points(w.settings.kind, rot_scale, &ref_c, &ref_a, ¢ers);
|
||||
lock(&shared.frames).push(FrameResult {
|
||||
comp_time: w.times[k],
|
||||
layer_time: layer.layer_time(w.times[k]),
|
||||
points: w.points.iter().zip(res).zip(attach).map(|((p, r), a)| (p.uid, r, a)).collect(),
|
||||
});
|
||||
let mut s = lock(&shared.state);
|
||||
s.done = k as u64;
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
s.fps = if s.elapsed > 0.0 { k as f64 / s.elapsed } else { 0.0 };
|
||||
s.time = w.times[k].seconds();
|
||||
}
|
||||
let mut s = lock(&shared.state);
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
s.finished = true;
|
||||
}
|
||||
|
||||
/// Set (or add) a key at layer time `t`.
|
||||
pub(crate) fn key(g: &mut PropGroup, m: &str, t: Tick, v: Value) {
|
||||
if let Some(p) = g.get_mut(m) {
|
||||
effectcraft_keyframe::set_key(&mut p.keys, Keyframe::new(t, v));
|
||||
}
|
||||
}
|
||||
|
||||
/// Write analysed frames into the tracker's points.
|
||||
fn write_frames(p: &mut Project, job: &TrackJob, frames: &[FrameResult]) {
|
||||
let Some(comp) = p.comp_mut(job.comp) else { return };
|
||||
let Some(layer) = comp.layer_mut(job.layer) else { return };
|
||||
for f in frames {
|
||||
for (uid, r, a) in &f.points {
|
||||
let Some(tp) = layer.props.find_group_mut(*uid) else { continue };
|
||||
key(tp, "featureCenter", f.layer_time, Value::Vec2(r.center));
|
||||
key(tp, "confidence", f.layer_time, Value::Scalar(r.confidence));
|
||||
key(tp, "attachPoint", f.layer_time, Value::Vec2(*a));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Session {
|
||||
/// Whether a track analysis is running.
|
||||
pub fn is_tracking(&self) -> bool {
|
||||
self.track_job.as_ref().is_some_and(|j| !j.is_finished())
|
||||
}
|
||||
|
||||
/// Live progress of the running (or just finished) analysis.
|
||||
pub fn track_progress(&self) -> Option<TrackProgress> {
|
||||
self.track_job.as_ref().map(|j| j.progress())
|
||||
}
|
||||
|
||||
/// Cancel the running analysis (results so far are kept).
|
||||
pub fn stop_track(&mut self) -> bool {
|
||||
match &self.track_job {
|
||||
Some(j) if !j.is_finished() => {
|
||||
j.cancel();
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Start analysing. `wait`: block until done (wasm32 always does). One undo step covers the
|
||||
/// whole analysis.
|
||||
pub(crate) fn start_track(&mut self, work: Work, tracker: Uid, direction: Direction, wait: bool) -> Result<(), String> {
|
||||
if self.is_tracking() {
|
||||
return Err("a track analysis is already running".into());
|
||||
}
|
||||
self.poll_track();
|
||||
let wait = wait || cfg!(target_arch = "wasm32");
|
||||
self.history.undo.push(("Analyze Track".into(), self.project.clone()));
|
||||
self.history.redo.clear();
|
||||
self.history.merge_key = None;
|
||||
let shared = Arc::new(TrackShared::default());
|
||||
let (comp, layer) = (work.comp, work.layer);
|
||||
let thread = if wait {
|
||||
run_work(work, &shared);
|
||||
None
|
||||
} else {
|
||||
let sh = shared.clone();
|
||||
Some(std::thread::Builder::new().name("track-analyze".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?)
|
||||
};
|
||||
self.track_job = Some(TrackJob { shared, thread, comp, layer, tracker, direction });
|
||||
self.poll_track();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Write new results into the project and move the CTI to the last analysed frame; drop the
|
||||
/// job once it finished. Frontends call this every frame while tracking.
|
||||
pub fn poll_track(&mut self) -> bool {
|
||||
let Some(job) = &self.track_job else { return false };
|
||||
let frames = std::mem::take(&mut *lock(&job.shared.frames));
|
||||
let finished = job.is_finished();
|
||||
let changed = !frames.is_empty();
|
||||
if changed {
|
||||
write_frames(Arc::make_mut(&mut self.project), job, &frames);
|
||||
if self.state.active_comp == Some(job.comp)
|
||||
&& let Some(last) = frames.last()
|
||||
{
|
||||
self.state.times.insert(job.comp, last.comp_time);
|
||||
}
|
||||
self.bump();
|
||||
}
|
||||
if finished {
|
||||
let mut job = self.track_job.take().expect("job");
|
||||
if let Some(t) = job.thread.take() {
|
||||
let _ = t.join();
|
||||
}
|
||||
let st = job.progress();
|
||||
let msg = match (&st.error, st.cancelled, st.low_confidence_stop) {
|
||||
(Some(e), _, _) => format!("Tracking stopped: {e}"),
|
||||
(None, true, _) => format!("Tracking stopped after {} frame(s)", st.done),
|
||||
(None, false, true) => format!("Tracking stopped: confidence below threshold after {} frame(s)", st.done),
|
||||
_ => format!("Tracked {} frame(s) in {:.1} s", st.done, st.elapsed),
|
||||
};
|
||||
self.events.push(Event::Toast { message: msg, error: st.error.is_some() });
|
||||
}
|
||||
changed || finished
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- apply
|
||||
|
||||
/// Apply Dimensions.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum Dims {
|
||||
#[default]
|
||||
XY,
|
||||
X,
|
||||
Y,
|
||||
}
|
||||
|
||||
impl Dims {
|
||||
pub fn from_name(s: &str) -> Option<Dims> {
|
||||
match s.to_ascii_lowercase().replace([' ', '_', '-'], "").as_str() {
|
||||
"xy" | "xandy" | "both" => Some(Dims::XY),
|
||||
"x" | "xonly" => Some(Dims::X),
|
||||
"y" | "yonly" => Some(Dims::Y),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
fn mix(self, old: [f64; 3], new: [f64; 2]) -> [f64; 3] {
|
||||
match self {
|
||||
Dims::XY => [new[0], new[1], old[2]],
|
||||
Dims::X => [new[0], old[1], old[2]],
|
||||
Dims::Y => [old[0], new[1], old[2]],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_m(m: &Mat3, p: [f64; 2]) -> [f64; 2] {
|
||||
let q = m.apply(vec2(p[0], p[1]));
|
||||
[q.x, q.y]
|
||||
}
|
||||
|
||||
/// Parent space of a layer (comp space when unparented), as a 2D matrix from comp space.
|
||||
fn comp_to_parent(ctx: &EvalCtx, layer: &Layer) -> Mat3 {
|
||||
match layer.parent.and_then(|p| ctx.comp.layer(p)) {
|
||||
Some(p) => ctx.layer_to_comp(p).0.inverse().unwrap_or(Mat3::IDENTITY),
|
||||
None => Mat3::IDENTITY,
|
||||
}
|
||||
}
|
||||
|
||||
/// Per analysed time: (source layer time, attach points in source layer space).
|
||||
pub fn attach_track(tracker: &PropGroup) -> Vec<(Tick, Vec<[f64; 2]>)> {
|
||||
let pts: Vec<&PropGroup> = tracker.track_points().collect();
|
||||
let Some(first) = pts.first().and_then(|g| g.get("attachPoint")) else { return vec![] };
|
||||
first.keys.iter().map(|k| (k.time, pts.iter().map(|g| g.get("attachPoint").map(|p| p.value_at(k.time).as_vec2()).unwrap_or([0.0; 2])).collect())).collect()
|
||||
}
|
||||
|
||||
/// Apply a tracker (Tracker panel ▸ Apply). Returns the number of keyframed frames.
|
||||
pub fn apply(p: &mut Project, cid: ItemId, src: LayerId, tracker: Uid, dims: Dims) -> Result<usize, String> {
|
||||
let comp = p.comp(cid).ok_or("no composition")?.clone();
|
||||
let source = comp.layer(src).ok_or("no source layer")?.clone();
|
||||
let (tg, settings) = source.tracker(tracker).ok_or("no such tracker")?;
|
||||
let settings = settings.clone();
|
||||
let track = attach_track(tg);
|
||||
if track.is_empty() {
|
||||
return Err("analyze the track first".into());
|
||||
}
|
||||
let target_id = match settings.kind {
|
||||
TrackKind::Stabilize => src,
|
||||
TrackKind::Raw => return Err("a Raw track has nothing to apply".into()),
|
||||
_ => settings.target.ok_or("set a Motion Target first (Edit Target…)")?,
|
||||
};
|
||||
let target = comp.layer(target_id).ok_or("the motion target layer is gone")?.clone();
|
||||
let ctx0 = EvalCtx::new(p, cid, &comp, Tick::ZERO);
|
||||
let t_first = source.comp_time(track[0].0);
|
||||
let rot_scale = settings.point_count() >= 2 && track[0].1.len() >= 2;
|
||||
let (ang0, len0) = if rot_scale { effectcraft_track::angle_and_length(track[0].1[0], track[0].1[1]) } else { (0.0, 1.0) };
|
||||
// (target layer time, values) to write.
|
||||
let mut pos_keys = vec![];
|
||||
let mut anchor_keys = vec![];
|
||||
let mut rot_keys = vec![];
|
||||
let mut scale_keys = vec![];
|
||||
let mut corner_keys = vec![];
|
||||
let ttr = target.transform().ok_or("the target has no transform")?;
|
||||
let c0 = ctx0.at(t_first);
|
||||
let rot0 = c0.f(&target, ttr, "rotation", 0.0);
|
||||
let scale0 = c0.v3(&target, ttr, "scale", [100.0; 3]);
|
||||
let mut prev_ang = 0.0;
|
||||
// Stabilize: the feature stays where it was on the first tracked frame.
|
||||
let stab_pos = {
|
||||
let (m, _) = c0.layer_to_comp(&source);
|
||||
apply_m(&comp_to_parent(&c0, &source), apply_m(&m, track[0].1[0]))
|
||||
};
|
||||
for (lt, pts) in &track {
|
||||
let ct = source.comp_time(*lt);
|
||||
let ctx = ctx0.at(ct);
|
||||
let tt = target.layer_time(ct);
|
||||
let (ang, len) = if rot_scale { effectcraft_track::angle_and_length(pts[0], pts[1]) } else { (ang0, len0) };
|
||||
let dang = effectcraft_track::unwrap_degrees(prev_ang, ang - ang0);
|
||||
prev_ang = dang;
|
||||
let ratio = if len0 > 1e-9 { len / len0 } else { 1.0 };
|
||||
match settings.kind {
|
||||
TrackKind::Stabilize => {
|
||||
if settings.position {
|
||||
let old = ctx.v3(&target, ttr, "anchor", [0.0; 3]);
|
||||
anchor_keys.push((tt, dims.mix(old, pts[0])));
|
||||
let oldp = ctx.position(&target, ttr);
|
||||
pos_keys.push((tt, dims.mix(oldp, stab_pos)));
|
||||
}
|
||||
if settings.rotation && rot_scale {
|
||||
rot_keys.push((tt, rot0 - dang));
|
||||
}
|
||||
if settings.scale && rot_scale {
|
||||
scale_keys.push((tt, [scale0[0] / ratio, scale0[1] / ratio, scale0[2]]));
|
||||
}
|
||||
}
|
||||
TrackKind::Transform => {
|
||||
let (m, _) = ctx.layer_to_comp(&source);
|
||||
if settings.position {
|
||||
let c = apply_m(&m, pts[0]);
|
||||
let pp = apply_m(&comp_to_parent(&ctx, &target), c);
|
||||
pos_keys.push((tt, dims.mix(ctx.position(&target, ttr), pp)));
|
||||
}
|
||||
if settings.rotation && rot_scale {
|
||||
rot_keys.push((tt, rot0 + dang));
|
||||
}
|
||||
if settings.scale && rot_scale {
|
||||
scale_keys.push((tt, [scale0[0] * ratio, scale0[1] * ratio, scale0[2]]));
|
||||
}
|
||||
}
|
||||
TrackKind::Affine | TrackKind::Perspective => {
|
||||
let (ms, _) = ctx.layer_to_comp(&source);
|
||||
let (mt, _) = ctx.layer_to_comp(&target);
|
||||
let inv = mt.inverse().ok_or("the target layer's transform is not invertible")?;
|
||||
let n = pts.len();
|
||||
if n < 3 {
|
||||
return Err("corner pin tracks need 3 or 4 track points".into());
|
||||
}
|
||||
let lr =
|
||||
if settings.kind == TrackKind::Affine || n < 4 { [pts[1][0] + pts[2][0] - pts[0][0], pts[1][1] + pts[2][1] - pts[0][1]] } else { pts[3] };
|
||||
// Track Points 1–4: upper left, upper right, lower left, lower right.
|
||||
let map = |q: [f64; 2]| apply_m(&inv, apply_m(&ms, q));
|
||||
corner_keys.push((tt, [map(pts[0]), map(pts[1]), map(pts[2]), map(lr)]));
|
||||
}
|
||||
TrackKind::Raw => {}
|
||||
}
|
||||
}
|
||||
let n = track.len();
|
||||
let tsize = effectcraft_render::source_size(p, &target);
|
||||
let mut ids_next = p.next_id;
|
||||
let layer = p.comp_mut(cid).and_then(|c| c.layer_mut(target_id)).ok_or("no target layer")?;
|
||||
if !corner_keys.is_empty() {
|
||||
let spec = effectcraft_effects::find("ec.distort.cornerpin").ok_or("Corner Pin is not available")?;
|
||||
let size = tsize;
|
||||
let size = if size.0 == 0 { [comp.width as f64, comp.height as f64] } else { [size.0 as f64, size.1 as f64] };
|
||||
let fx = layer.props.sub_mut("effects").ok_or("the target layer can't have effects")?;
|
||||
let same = fx.groups().filter(|g| g.match_id == spec.id).count();
|
||||
let name = if same == 0 { spec.name.to_string() } else { format!("{} {}", spec.name, same + 1) };
|
||||
let mut g = effectcraft_effects::instantiate(spec, &mut effectcraft_project::build::Ids(&mut ids_next), &name, size);
|
||||
for (t, c) in &corner_keys {
|
||||
for (m, v) in ["ul", "ur", "ll", "lr"].iter().zip(c) {
|
||||
key(&mut g, m, *t, Value::Vec2(*v));
|
||||
}
|
||||
}
|
||||
fx.children.push(g.into());
|
||||
}
|
||||
let tr = layer.transform_mut().ok_or("no transform")?;
|
||||
for (t, v) in anchor_keys {
|
||||
key(tr, "anchor", t, Value::Vec3(v));
|
||||
}
|
||||
let separated = tr.get("positionX").is_some();
|
||||
for (t, v) in pos_keys {
|
||||
if separated {
|
||||
key(tr, "positionX", t, Value::Scalar(v[0]));
|
||||
key(tr, "positionY", t, Value::Scalar(v[1]));
|
||||
} else {
|
||||
key(tr, "position", t, Value::Vec3(v));
|
||||
}
|
||||
}
|
||||
for (t, v) in rot_keys {
|
||||
key(tr, "rotation", t, Value::Scalar(v));
|
||||
}
|
||||
for (t, v) in scale_keys {
|
||||
key(tr, "scale", t, Value::Vec3(v));
|
||||
}
|
||||
p.next_id = ids_next;
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn attach_points_follow_the_track() {
|
||||
let rc = [[0.0, 0.0], [10.0, 0.0]];
|
||||
let ra = [[0.0, 5.0], [10.0, 5.0]];
|
||||
// Rotated 90° and doubled.
|
||||
let cur = [[100.0, 100.0], [100.0, 120.0]];
|
||||
let a = attach_points(TrackKind::Transform, true, &rc, &ra, &cur);
|
||||
assert!((a[0][0] - 90.0).abs() < 1e-9 && (a[0][1] - 100.0).abs() < 1e-9, "{a:?}");
|
||||
let a = attach_points(TrackKind::Transform, false, &rc, &ra, &cur);
|
||||
assert_eq!(a[0], [100.0, 105.0]);
|
||||
let h = trk::Homography([[1.1, 0.1, 5.0], [0.05, 0.9, -3.0], [0.001, 0.0005, 1.0]]);
|
||||
let rc4 = [[0.0, 0.0], [100.0, 0.0], [0.0, 100.0], [100.0, 100.0]];
|
||||
let ra4 = [[10.0, 10.0], [90.0, 10.0], [10.0, 90.0], [90.0, 90.0]];
|
||||
let cur4: Vec<[f64; 2]> = rc4.iter().map(|p| h.apply(*p)).collect();
|
||||
let a = attach_points(TrackKind::Perspective, false, &rc4, &ra4, &cur4);
|
||||
for (got, want) in a.iter().zip(ra4.iter().map(|p| h.apply(*p))) {
|
||||
assert!((got[0] - want[0]).abs() < 1e-6 && (got[1] - want[1]).abs() < 1e-6);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -8,6 +8,7 @@ pub mod build;
|
|||
pub mod props;
|
||||
pub mod render_queue;
|
||||
pub mod styles;
|
||||
pub mod tracking;
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
|
|
|
|||
|
|
@ -170,6 +170,10 @@ pub enum GroupKind {
|
|||
/// Effect spec id, e.g. `ec.blur.gaussian`.
|
||||
effect: String,
|
||||
},
|
||||
/// A motion tracker (Motion Trackers ▸ Tracker n) with its Tracker panel settings.
|
||||
Tracker {
|
||||
settings: Box<crate::tracking::TrackerSettings>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,288 @@
|
|||
//! Motion trackers in the property tree, laid out like After Effects':
|
||||
//!
|
||||
//! ```text
|
||||
//! Motion Trackers (match `motionTrackers`, first group of the layer)
|
||||
//! Tracker 1 (GroupKind::Tracker: type, target, options)
|
||||
//! Track Point 1 (match `trackPoint`)
|
||||
//! Feature Center, Feature Size, Search Offset, Search Size,
|
||||
//! Confidence, Attach Point, Attach Point Offset
|
||||
//! ```
|
||||
//!
|
||||
//! Feature Center, Confidence and Attach Point get one keyframe per analysed frame (layer time).
|
||||
//! The tracking algorithms live in `effectcraft-track`; applying tracks lives in the engine.
|
||||
|
||||
use effectcraft_keyframe::Value;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::build::Ids;
|
||||
use crate::props::{GroupKind, ParamUi, PropGroup};
|
||||
use crate::{Layer, LayerId, Uid};
|
||||
|
||||
/// Tracker panel ▸ Track Type.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum TrackKind {
|
||||
Stabilize,
|
||||
#[default]
|
||||
Transform,
|
||||
/// Parallel (affine) corner pin: three points, the fourth corner completes the parallelogram.
|
||||
#[serde(alias = "parallel", alias = "parallelCornerPin")]
|
||||
Affine,
|
||||
/// Perspective corner pin: four points.
|
||||
#[serde(alias = "perspectiveCornerPin")]
|
||||
Perspective,
|
||||
/// Raw: track data only (nothing to apply).
|
||||
Raw,
|
||||
}
|
||||
|
||||
impl TrackKind {
|
||||
pub const ALL: [TrackKind; 5] = [TrackKind::Stabilize, TrackKind::Transform, TrackKind::Affine, TrackKind::Perspective, TrackKind::Raw];
|
||||
pub fn label(self) -> &'static str {
|
||||
match self {
|
||||
TrackKind::Stabilize => "Stabilize",
|
||||
TrackKind::Transform => "Transform",
|
||||
TrackKind::Affine => "Parallel Corner Pin",
|
||||
TrackKind::Perspective => "Perspective Corner Pin",
|
||||
TrackKind::Raw => "Raw",
|
||||
}
|
||||
}
|
||||
pub fn id(self) -> &'static str {
|
||||
match self {
|
||||
TrackKind::Stabilize => "stabilize",
|
||||
TrackKind::Transform => "transform",
|
||||
TrackKind::Affine => "affine",
|
||||
TrackKind::Perspective => "perspective",
|
||||
TrackKind::Raw => "raw",
|
||||
}
|
||||
}
|
||||
pub fn from_name(s: &str) -> Option<TrackKind> {
|
||||
let n = s.to_ascii_lowercase().replace([' ', '_', '-'], "");
|
||||
match n.as_str() {
|
||||
"parallel" | "parallelcornerpin" | "affinecornerpin" => return Some(TrackKind::Affine),
|
||||
"perspectivecornerpin" => return Some(TrackKind::Perspective),
|
||||
_ => {}
|
||||
}
|
||||
TrackKind::ALL.into_iter().find(|k| k.id() == n || k.label().to_ascii_lowercase().replace(' ', "") == n)
|
||||
}
|
||||
pub fn is_corner_pin(self) -> bool {
|
||||
matches!(self, TrackKind::Affine | TrackKind::Perspective)
|
||||
}
|
||||
/// Number of track points for this type (Transform/Stabilize: 2 with rotation or scale).
|
||||
pub fn point_count(self, rotation_or_scale: bool) -> usize {
|
||||
match self {
|
||||
TrackKind::Affine => 3,
|
||||
TrackKind::Perspective => 4,
|
||||
_ if rotation_or_scale => 2,
|
||||
_ => 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Motion Tracker Options ▸ Channel.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum TrackChannel {
|
||||
Rgb,
|
||||
#[default]
|
||||
Luminance,
|
||||
Saturation,
|
||||
}
|
||||
|
||||
/// Motion Tracker Options ▸ "If Confidence is Below … %".
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum LowConfidence {
|
||||
Continue,
|
||||
Stop,
|
||||
Extrapolate,
|
||||
#[default]
|
||||
Adapt,
|
||||
}
|
||||
|
||||
/// Motion Tracker Options.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct TrackerOptions {
|
||||
pub channel: TrackChannel,
|
||||
/// Process Before Match ▸ Blur radius in pixels (0 = off).
|
||||
pub blur: f64,
|
||||
/// Process Before Match ▸ Enhance.
|
||||
pub enhance: bool,
|
||||
pub subpixel: bool,
|
||||
/// Adapt Feature on Every Frame.
|
||||
pub adapt_every_frame: bool,
|
||||
/// Confidence threshold, percent.
|
||||
pub threshold: f64,
|
||||
pub action: LowConfidence,
|
||||
/// Follow each feature's rotation and scale while matching (EffectCraft extension).
|
||||
pub track_shape: bool,
|
||||
}
|
||||
|
||||
impl Default for TrackerOptions {
|
||||
fn default() -> Self {
|
||||
TrackerOptions {
|
||||
channel: TrackChannel::Luminance,
|
||||
blur: 0.0,
|
||||
enhance: false,
|
||||
subpixel: true,
|
||||
adapt_every_frame: false,
|
||||
threshold: 80.0,
|
||||
action: LowConfidence::Adapt,
|
||||
track_shape: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A tracker's non-animatable settings (Tracker panel).
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct TrackerSettings {
|
||||
pub kind: TrackKind,
|
||||
pub position: bool,
|
||||
pub rotation: bool,
|
||||
pub scale: bool,
|
||||
/// Motion Target (None: no target; Stabilize always applies to the tracked layer).
|
||||
pub target: Option<LayerId>,
|
||||
pub options: TrackerOptions,
|
||||
}
|
||||
|
||||
impl TrackerSettings {
|
||||
pub fn new(kind: TrackKind) -> TrackerSettings {
|
||||
TrackerSettings { kind, position: true, ..Default::default() }
|
||||
}
|
||||
pub fn point_count(&self) -> usize {
|
||||
self.kind.point_count(self.rotation || self.scale)
|
||||
}
|
||||
}
|
||||
|
||||
/// Initial regions of a track point (layer pixels).
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct PointInit {
|
||||
pub center: [f64; 2],
|
||||
pub feature_size: [f64; 2],
|
||||
pub search_offset: [f64; 2],
|
||||
pub search_size: [f64; 2],
|
||||
pub attach_offset: [f64; 2],
|
||||
}
|
||||
|
||||
pub const MOTION_TRACKERS: &str = "motionTrackers";
|
||||
pub const TRACK_POINT: &str = "trackPoint";
|
||||
|
||||
/// The (empty) Motion Trackers group.
|
||||
pub fn motion_trackers(ids: &mut Ids) -> PropGroup {
|
||||
ids.group(MOTION_TRACKERS, "Motion Trackers")
|
||||
}
|
||||
|
||||
/// One Track Point group.
|
||||
pub fn track_point(ids: &mut Ids, name: &str, p: &PointInit) -> PropGroup {
|
||||
let pt = |ids: &mut Ids, m: &str, n: &str, v: [f64; 2]| ids.prop(m, n, Value::Vec2(v)).with_ui(ParamUi::Point).spatial();
|
||||
let vec = |ids: &mut Ids, m: &str, n: &str, v: [f64; 2]| ids.prop(m, n, Value::Vec2(v)).with_ui(ParamUi::Point);
|
||||
let attach = [p.center[0] + p.attach_offset[0], p.center[1] + p.attach_offset[1]];
|
||||
let fc = pt(ids, "featureCenter", "Feature Center", p.center);
|
||||
let fs = vec(ids, "featureSize", "Feature Size", p.feature_size);
|
||||
let so = vec(ids, "searchOffset", "Search Offset", p.search_offset);
|
||||
let ss = vec(ids, "searchSize", "Search Size", p.search_size);
|
||||
let cf = ids.prop("confidence", "Confidence", Value::Scalar(0.0)).with_ui(ParamUi::Percent);
|
||||
let ap = pt(ids, "attachPoint", "Attach Point", attach);
|
||||
let ao = vec(ids, "attachPointOffset", "Attach Point Offset", p.attach_offset);
|
||||
let mut g = ids.group(TRACK_POINT, name).with(fc).with(fs).with(so).with(ss).with(cf).with(ap).with(ao);
|
||||
g.kind = GroupKind::Indexed;
|
||||
g
|
||||
}
|
||||
|
||||
/// A Tracker group with its points.
|
||||
pub fn tracker(ids: &mut Ids, name: &str, settings: TrackerSettings, points: &[PointInit]) -> PropGroup {
|
||||
let mut g = ids.group("tracker", name);
|
||||
g.kind = GroupKind::Tracker { settings: Box::new(settings) };
|
||||
for (i, p) in points.iter().enumerate() {
|
||||
let tp = track_point(ids, &format!("Track Point {}", i + 1), p);
|
||||
g.children.push(tp.into());
|
||||
}
|
||||
g
|
||||
}
|
||||
|
||||
/// Default track point layout for a tracker type, centred on a layer of size `size`.
|
||||
pub fn default_points(kind: TrackKind, n: usize, size: [f64; 2]) -> Vec<PointInit> {
|
||||
let m = size[0].min(size[1]).max(1.0);
|
||||
let f = (m * 0.04).clamp(16.0, 80.0).round();
|
||||
let s = (f * 2.0).round();
|
||||
let c = [size[0] / 2.0, size[1] / 2.0];
|
||||
let mk = |p: [f64; 2]| PointInit { center: p, feature_size: [f, f], search_offset: [0.0; 2], search_size: [s, s], attach_offset: [0.0; 2] };
|
||||
let (dx, dy) = ((size[0] * 0.18).max(2.0 * s), (size[1] * 0.18).max(2.0 * s));
|
||||
match (kind, n) {
|
||||
(TrackKind::Perspective, _) | (_, 4) => {
|
||||
vec![mk([c[0] - dx, c[1] - dy]), mk([c[0] + dx, c[1] - dy]), mk([c[0] - dx, c[1] + dy]), mk([c[0] + dx, c[1] + dy])]
|
||||
}
|
||||
(TrackKind::Affine, _) | (_, 3) => vec![mk([c[0] - dx, c[1] - dy]), mk([c[0] + dx, c[1] - dy]), mk([c[0] - dx, c[1] + dy])],
|
||||
(_, 2) => vec![mk([c[0] - dx, c[1]]), mk([c[0] + dx, c[1]])],
|
||||
_ => vec![mk(c)],
|
||||
}
|
||||
}
|
||||
|
||||
impl Layer {
|
||||
/// The Motion Trackers group, if the layer has one.
|
||||
pub fn motion_trackers(&self) -> Option<&PropGroup> {
|
||||
self.props.sub(MOTION_TRACKERS)
|
||||
}
|
||||
/// Trackers of the layer (groups with [`GroupKind::Tracker`]).
|
||||
pub fn trackers(&self) -> impl Iterator<Item = (&PropGroup, &TrackerSettings)> {
|
||||
self.motion_trackers().into_iter().flat_map(|g| g.groups()).filter_map(|g| match &g.kind {
|
||||
GroupKind::Tracker { settings } => Some((g, settings.as_ref())),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
pub fn tracker(&self, uid: Uid) -> Option<(&PropGroup, &TrackerSettings)> {
|
||||
self.trackers().find(|(g, _)| g.uid == uid)
|
||||
}
|
||||
/// The Motion Trackers group, created (first in the layer's tree, like AE) when missing.
|
||||
pub fn motion_trackers_mut(&mut self, ids: &mut Ids) -> &mut PropGroup {
|
||||
if self.props.sub(MOTION_TRACKERS).is_none() {
|
||||
self.props.children.insert(0, motion_trackers(ids).into());
|
||||
}
|
||||
self.props.sub_mut(MOTION_TRACKERS).expect("motion trackers group")
|
||||
}
|
||||
}
|
||||
|
||||
impl PropGroup {
|
||||
/// Tracker settings of a tracker group.
|
||||
pub fn tracker_settings(&self) -> Option<&TrackerSettings> {
|
||||
match &self.kind {
|
||||
GroupKind::Tracker { settings } => Some(settings),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
pub fn tracker_settings_mut(&mut self) -> Option<&mut TrackerSettings> {
|
||||
match &mut self.kind {
|
||||
GroupKind::Tracker { settings } => Some(settings),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
/// Track Point groups of a tracker group, in order.
|
||||
pub fn track_points(&self) -> impl Iterator<Item = &PropGroup> {
|
||||
self.groups().filter(|g| g.match_id == TRACK_POINT)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn tracker_tree_and_serde() {
|
||||
let mut n = 1;
|
||||
let mut ids = Ids(&mut n);
|
||||
let pts = default_points(TrackKind::Perspective, 4, [1920.0, 1080.0]);
|
||||
let g = tracker(&mut ids, "Tracker 1", TrackerSettings::new(TrackKind::Perspective), &pts);
|
||||
assert_eq!(g.track_points().count(), 4);
|
||||
let tp = g.track_points().next().unwrap();
|
||||
let names: Vec<&str> = tp.props().map(|p| p.name.as_str()).collect();
|
||||
assert_eq!(names, ["Feature Center", "Feature Size", "Search Offset", "Search Size", "Confidence", "Attach Point", "Attach Point Offset"]);
|
||||
let json = serde_json::to_string(&g).unwrap();
|
||||
let back: PropGroup = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(back, g);
|
||||
assert_eq!(back.tracker_settings().unwrap().kind, TrackKind::Perspective);
|
||||
assert_eq!(TrackKind::from_name("Parallel Corner Pin"), Some(TrackKind::Affine));
|
||||
assert_eq!(TrackKind::from_name("perspective"), Some(TrackKind::Perspective));
|
||||
assert_eq!(TrackerSettings { rotation: true, ..TrackerSettings::new(TrackKind::Transform) }.point_count(), 2);
|
||||
}
|
||||
}
|
||||
|
|
@ -341,6 +341,13 @@ impl<'a> Renderer<'a> {
|
|||
Some(flat)
|
||||
}
|
||||
|
||||
/// A layer's source pixels at the context time, before masks, effects and the transform
|
||||
/// (what the motion tracker analyses, like After Effects' Layer panel). `None` for layers
|
||||
/// without pixels.
|
||||
pub fn layer_source(&self, ctx: &EvalCtx, layer: &Layer) -> Option<Buf> {
|
||||
self.source(ctx, layer)
|
||||
}
|
||||
|
||||
/// Layer pixels after source → masks → effects (no layer styles).
|
||||
pub fn content_buf(&self, ctx: &EvalCtx, layer: &Layer) -> Option<Arc<Buf>> {
|
||||
self.content_buf_timed(ctx, layer, None).map(|(b, _)| b)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
[package]
|
||||
name = "effectcraft-track"
|
||||
description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, and corner-pin / transform solves"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
effectcraft-raster = { workspace = true }
|
||||
rayon = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
|
|
@ -0,0 +1,639 @@
|
|||
//! Motion tracking, clean-room from textbook methods:
|
||||
//!
|
||||
//! - each **track point** has a *feature region* (the pattern to follow) and a *search region*
|
||||
//! (where to look for it in the next frame), as in After Effects' Tracker;
|
||||
//! - the pattern is located by **normalized cross-correlation** (Lewis, "Fast Normalized
|
||||
//! Cross-Correlation", 1995) searched **coarse to fine over an image pyramid**;
|
||||
//! - the integer match is refined to **sub-pixel** precision (and, optionally, the feature's
|
||||
//! rotation and scale) by **Lucas–Kanade** Gauss–Newton image alignment (Lucas & Kanade 1981;
|
||||
//! Baker & Matthews, "Lucas-Kanade 20 Years On", 2004) with a gain/bias photometric model;
|
||||
//! - **confidence** is the correlation of the template with the matched region, in percent;
|
||||
//! when it drops below a threshold the tracker continues, stops, extrapolates the motion or
|
||||
//! adapts the feature (re-samples the template), like AE's Motion Tracker Options.
|
||||
//!
|
||||
//! Only the pixels around each search region are converted and pre-processed, so tracking cost is
|
||||
//! independent of the frame size. Points are tracked in parallel and the coarse correlation search
|
||||
//! is parallel over candidate rows (rayon).
|
||||
//!
|
||||
//! Coordinates are *layer* pixels: the frame image's pixel `(i, j)` covers `[i, i+1) × [j, j+1)`
|
||||
//! after subtracting the frame's `offset` (see [`Frame`]).
|
||||
|
||||
pub mod plane;
|
||||
pub mod solve;
|
||||
|
||||
use effectcraft_raster::Image;
|
||||
use rayon::prelude::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub use plane::{Channel, Plane};
|
||||
pub use solve::{Homography, angle_and_length, unwrap_degrees};
|
||||
|
||||
/// What to do when a frame's confidence is below the threshold.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum ConfidenceAction {
|
||||
/// Keep the match anyway.
|
||||
Continue,
|
||||
/// Stop the analysis before this frame.
|
||||
Stop,
|
||||
/// Ignore the match and continue the previous motion.
|
||||
Extrapolate,
|
||||
/// Keep the match and re-sample the feature from this frame.
|
||||
#[default]
|
||||
Adapt,
|
||||
}
|
||||
|
||||
impl ConfidenceAction {
|
||||
pub const ALL: [ConfidenceAction; 4] = [ConfidenceAction::Continue, ConfidenceAction::Stop, ConfidenceAction::Extrapolate, ConfidenceAction::Adapt];
|
||||
pub fn label(self) -> &'static str {
|
||||
match self {
|
||||
ConfidenceAction::Continue => "Continue Tracking",
|
||||
ConfidenceAction::Stop => "Stop Tracking",
|
||||
ConfidenceAction::Extrapolate => "Extrapolate Motion",
|
||||
ConfidenceAction::Adapt => "Adapt Feature",
|
||||
}
|
||||
}
|
||||
pub fn from_name(s: &str) -> Option<ConfidenceAction> {
|
||||
let n = s.to_ascii_lowercase().replace([' ', '_', '-'], "");
|
||||
ConfidenceAction::ALL.into_iter().find(|a| {
|
||||
format!("{a:?}").to_ascii_lowercase() == n
|
||||
|| a.label().to_ascii_lowercase().replace(' ', "") == n
|
||||
|| a.label().to_ascii_lowercase().replace(' ', "").starts_with(&n)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Motion Tracker Options that affect matching.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct TrackOptions {
|
||||
pub channel: Channel,
|
||||
/// Pre-process blur radius in pixels (0 = off).
|
||||
pub blur: f64,
|
||||
/// Pre-process edge enhancement.
|
||||
pub enhance: bool,
|
||||
/// Sub-pixel positioning (else whole-pixel matches).
|
||||
pub subpixel: bool,
|
||||
/// Re-sample the feature on every frame.
|
||||
pub adapt_every_frame: bool,
|
||||
/// Confidence threshold in percent for `action`.
|
||||
pub threshold: f64,
|
||||
pub action: ConfidenceAction,
|
||||
/// Also estimate each feature's rotation and scale while matching (follows rotating or
|
||||
/// zooming features without adapting).
|
||||
pub track_shape: bool,
|
||||
}
|
||||
|
||||
impl Default for TrackOptions {
|
||||
fn default() -> Self {
|
||||
TrackOptions {
|
||||
channel: Channel::Luminance,
|
||||
blur: 0.0,
|
||||
enhance: false,
|
||||
subpixel: true,
|
||||
adapt_every_frame: false,
|
||||
threshold: 80.0,
|
||||
action: ConfidenceAction::Adapt,
|
||||
track_shape: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One track point's regions (layer pixels).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct PointSpec {
|
||||
pub center: [f64; 2],
|
||||
/// Feature region width and height.
|
||||
pub feature_size: [f64; 2],
|
||||
/// Search region centre relative to the feature centre.
|
||||
pub search_offset: [f64; 2],
|
||||
/// Search region width and height.
|
||||
pub search_size: [f64; 2],
|
||||
}
|
||||
|
||||
/// A frame to track in: layer pixels map to image pixels by adding `offset`.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct Frame<'a> {
|
||||
pub img: &'a Image,
|
||||
pub offset: [f64; 2],
|
||||
}
|
||||
|
||||
impl<'a> Frame<'a> {
|
||||
pub fn new(img: &'a Image) -> Frame<'a> {
|
||||
Frame { img, offset: [0.0; 2] }
|
||||
}
|
||||
}
|
||||
|
||||
/// How a point's position on a frame was obtained.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Status {
|
||||
Tracked,
|
||||
/// Tracked, and the feature was re-sampled from this frame.
|
||||
Adapted,
|
||||
/// Low confidence: the previous motion was continued.
|
||||
Extrapolated,
|
||||
/// Low confidence with the Stop action: no result for this frame.
|
||||
Stopped,
|
||||
}
|
||||
|
||||
/// A point's result on one frame.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct PointResult {
|
||||
/// Feature centre (layer pixels).
|
||||
pub center: [f64; 2],
|
||||
/// Match confidence, 0–100 %.
|
||||
pub confidence: f64,
|
||||
/// Feature rotation (degrees) and scale (1 = start) relative to the start frame (when
|
||||
/// `track_shape` is on).
|
||||
pub rotation: f64,
|
||||
pub scale: f64,
|
||||
pub status: Status,
|
||||
}
|
||||
|
||||
/// A template sampled on a unit grid around the feature centre at one pyramid level.
|
||||
#[derive(Clone, Debug)]
|
||||
struct Patch {
|
||||
r: [i32; 2],
|
||||
ch: usize,
|
||||
raw: Vec<f32>,
|
||||
/// Zero-mean values and their sum of squares.
|
||||
zm: Vec<f32>,
|
||||
ss: f64,
|
||||
}
|
||||
|
||||
impl Patch {
|
||||
fn sample(plane: &Plane, c: [f64; 2], a: [f64; 2], r: [i32; 2]) -> Patch {
|
||||
let ch = plane.ch;
|
||||
let mut raw = Vec::with_capacity(((2 * r[0] + 1) * (2 * r[1] + 1)) as usize * ch);
|
||||
let mut v = [0.0f32; 3];
|
||||
for gy in -r[1]..=r[1] {
|
||||
for gx in -r[0]..=r[0] {
|
||||
let (ux, uy) = (gx as f64, gy as f64);
|
||||
plane.sample(c[0] + a[0] * ux - a[1] * uy, c[1] + a[1] * ux + a[0] * uy, &mut v);
|
||||
raw.extend_from_slice(&v[..ch]);
|
||||
}
|
||||
}
|
||||
let mean = raw.iter().map(|x| *x as f64).sum::<f64>() / raw.len().max(1) as f64;
|
||||
let zm: Vec<f32> = raw.iter().map(|x| (*x as f64 - mean) as f32).collect();
|
||||
let ss = zm.iter().map(|x| (*x as f64) * (*x as f64)).sum();
|
||||
Patch { r, ch, raw, zm, ss }
|
||||
}
|
||||
}
|
||||
|
||||
/// Normalized cross-correlation of `patch` with `plane` sampled at `c + A u` (A = similarity
|
||||
/// `[[a0, -a1], [a1, a0]]`). Pure translations share one set of bilinear weights.
|
||||
fn ncc(plane: &Plane, p: &Patch, c: [f64; 2], a: [f64; 2]) -> f64 {
|
||||
if p.ss <= 1e-12 {
|
||||
return 0.0;
|
||||
}
|
||||
let ch = p.ch;
|
||||
let (mut s, mut s2, mut st) = (0.0f64, 0.0f64, 0.0f64);
|
||||
let identity = (a[0] - 1.0).abs() < 1e-9 && a[1].abs() < 1e-9;
|
||||
let fx = c[0] - 0.5;
|
||||
let fy = c[1] - 0.5;
|
||||
let (bx, by) = (fx.floor(), fy.floor());
|
||||
let inside = bx as i64 - p.r[0] as i64 >= 0
|
||||
&& by as i64 - p.r[1] as i64 >= 0
|
||||
&& (bx as i64 + p.r[0] as i64 + 1) < plane.w as i64
|
||||
&& (by as i64 + p.r[1] as i64 + 1) < plane.h as i64;
|
||||
if identity && inside {
|
||||
let (tx, ty) = ((fx - bx) as f32, (fy - by) as f32);
|
||||
let (w00, w10, w01, w11) = ((1.0 - tx) * (1.0 - ty), tx * (1.0 - ty), (1.0 - tx) * ty, tx * ty);
|
||||
let row = plane.w * ch;
|
||||
let mut k = 0;
|
||||
for gy in -p.r[1]..=p.r[1] {
|
||||
let y = (by as i64 + gy as i64) as usize;
|
||||
let base = y * row;
|
||||
for gx in -p.r[0]..=p.r[0] {
|
||||
let x = (bx as i64 + gx as i64) as usize;
|
||||
let i00 = base + x * ch;
|
||||
for cc in 0..ch {
|
||||
let v =
|
||||
w00 * plane.data[i00 + cc] + w10 * plane.data[i00 + ch + cc] + w01 * plane.data[i00 + row + cc] + w11 * plane.data[i00 + row + ch + cc];
|
||||
let v = v as f64;
|
||||
s += v;
|
||||
s2 += v * v;
|
||||
st += v * p.zm[k] as f64;
|
||||
k += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let mut v = [0.0f32; 3];
|
||||
let mut k = 0;
|
||||
for gy in -p.r[1]..=p.r[1] {
|
||||
for gx in -p.r[0]..=p.r[0] {
|
||||
let (ux, uy) = (gx as f64, gy as f64);
|
||||
plane.sample(c[0] + a[0] * ux - a[1] * uy, c[1] + a[1] * ux + a[0] * uy, &mut v);
|
||||
for cc in 0..ch {
|
||||
let x = v[cc] as f64;
|
||||
s += x;
|
||||
s2 += x * x;
|
||||
st += x * p.zm[k] as f64;
|
||||
k += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let n = p.zm.len() as f64;
|
||||
let var = s2 - s * s / n;
|
||||
if var <= 1e-12 {
|
||||
return 0.0;
|
||||
}
|
||||
(st / (var * p.ss).sqrt()).clamp(-1.0, 1.0)
|
||||
}
|
||||
|
||||
/// The pre-processed pixels around one point: a pyramid of a crop. Plane coordinates are image
|
||||
/// pixels minus `origin`, divided by `2^level`.
|
||||
struct Crop {
|
||||
origin: [f64; 2],
|
||||
levels: Vec<Plane>,
|
||||
grad0: (Plane, Plane),
|
||||
}
|
||||
|
||||
impl Crop {
|
||||
/// Crop image pixels `[c - ext, c + ext]` (image pixel coordinates), pre-processed.
|
||||
fn new(frame: &Frame, opts: &TrackOptions, c: [f64; 2], ext: [f64; 2], levels: usize) -> Crop {
|
||||
let sigma = opts.blur.max(0.0) / 2.0;
|
||||
let margin = (sigma * 3.0).ceil() + 3.0 + if opts.enhance { 6.0 } else { 0.0 } + (1 << levels) as f64;
|
||||
let x0 = (c[0] - ext[0] - margin).floor();
|
||||
let y0 = (c[1] - ext[1] - margin).floor();
|
||||
let w = ((2.0 * (ext[0] + margin)).ceil() as usize + 2).max(4);
|
||||
let h = ((2.0 * (ext[1] + margin)).ceil() as usize + 2).max(4);
|
||||
let mut p = Plane::from_image(frame.img, opts.channel, x0 as i64, y0 as i64, w, h);
|
||||
if sigma > 0.05 {
|
||||
p = p.gaussian(sigma);
|
||||
}
|
||||
if opts.enhance {
|
||||
p = p.enhance();
|
||||
}
|
||||
let grad0 = p.gradients();
|
||||
let mut lv = vec![p];
|
||||
for _ in 0..levels {
|
||||
let d = lv.last().map(Plane::downsample).unwrap_or_default();
|
||||
lv.push(d);
|
||||
}
|
||||
Crop { origin: [x0, y0], levels: lv, grad0 }
|
||||
}
|
||||
fn to_level(&self, p: [f64; 2], k: usize) -> [f64; 2] {
|
||||
let s = (1u32 << k) as f64;
|
||||
[(p[0] - self.origin[0]) / s, (p[1] - self.origin[1]) / s]
|
||||
}
|
||||
fn level_to_image(&self, p: [f64; 2], k: usize) -> [f64; 2] {
|
||||
let s = (1u32 << k) as f64;
|
||||
[p[0] * s + self.origin[0], p[1] * s + self.origin[1]]
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-point tracking state.
|
||||
#[derive(Clone, Debug)]
|
||||
struct PointState {
|
||||
spec: PointSpec,
|
||||
/// Current centre in image pixels.
|
||||
center: [f64; 2],
|
||||
/// Current similarity `[s cos θ, s sin θ]` relative to the template's frame.
|
||||
shape: [f64; 2],
|
||||
velocity: [f64; 2],
|
||||
templates: Vec<Patch>,
|
||||
levels: usize,
|
||||
}
|
||||
|
||||
fn half(v: [f64; 2]) -> [f64; 2] {
|
||||
[(v[0] / 2.0).max(1.0), (v[1] / 2.0).max(1.0)]
|
||||
}
|
||||
|
||||
impl PointState {
|
||||
fn feature_r(&self) -> [i32; 2] {
|
||||
let h = half(self.spec.feature_size);
|
||||
[h[0].round().max(2.0) as i32, h[1].round().max(2.0) as i32]
|
||||
}
|
||||
fn level_r(&self, k: usize) -> [i32; 2] {
|
||||
let r = self.feature_r();
|
||||
let s = (1 << k) as f64;
|
||||
[((r[0] as f64 / s).ceil() as i32).max(2), ((r[1] as f64 / s).ceil() as i32).max(2)]
|
||||
}
|
||||
/// Search range around the predicted centre (pixels, per axis).
|
||||
fn range(&self) -> [f64; 2] {
|
||||
let f = half(self.spec.feature_size);
|
||||
let s = half(self.spec.search_size);
|
||||
[(s[0] - f[0]).max(2.0), (s[1] - f[1]).max(2.0)]
|
||||
}
|
||||
fn extent(&self) -> f64 {
|
||||
let r = self.feature_r();
|
||||
let s = (self.shape[0] * self.shape[0] + self.shape[1] * self.shape[1]).sqrt().max(1.0);
|
||||
((r[0] * r[0] + r[1] * r[1]) as f64).sqrt() * s + 2.0
|
||||
}
|
||||
|
||||
fn new(spec: PointSpec, frame: &Frame, opts: &TrackOptions) -> PointState {
|
||||
let c = [spec.center[0] + frame.offset[0], spec.center[1] + frame.offset[1]];
|
||||
let mut st = PointState { spec, center: c, shape: [1.0, 0.0], velocity: [0.0; 2], templates: vec![], levels: 0 };
|
||||
let range = st.range();
|
||||
let fr = st.feature_r();
|
||||
let mut l = 0;
|
||||
while l < 4 && range[0].min(range[1]) / (2u32 << l) as f64 >= 3.0 && (fr[0].min(fr[1]) as f64) / (2u32 << l) as f64 >= 3.0 {
|
||||
l += 1;
|
||||
}
|
||||
st.levels = l;
|
||||
let e = st.extent();
|
||||
let crop = Crop::new(frame, opts, c, [e, e], l);
|
||||
st.resample(&crop);
|
||||
st
|
||||
}
|
||||
|
||||
/// Re-sample the templates from `crop` at the current centre and shape.
|
||||
fn resample(&mut self, crop: &Crop) {
|
||||
self.templates = (0..=self.levels).map(|k| Patch::sample(&crop.levels[k], crop.to_level(self.center, k), self.shape, self.level_r(k))).collect();
|
||||
}
|
||||
|
||||
fn step(&mut self, frame: &Frame, opts: &TrackOptions) -> PointResult {
|
||||
let prev = self.center;
|
||||
let predicted = [prev[0] + self.velocity[0], prev[1] + self.velocity[1]];
|
||||
let sc = [predicted[0] + self.spec.search_offset[0], predicted[1] + self.spec.search_offset[1]];
|
||||
let range = self.range();
|
||||
let e = self.extent();
|
||||
let crop = Crop::new(frame, opts, sc, [range[0] + e, range[1] + e], self.levels);
|
||||
let top = self.levels;
|
||||
// Coarse: exhaustive correlation over the search range at the top level.
|
||||
let s = (1u32 << top) as f64;
|
||||
let ctop = crop.to_level(sc, top);
|
||||
let (rx, ry) = ((range[0] / s).ceil() as i32, (range[1] / s).ceil() as i32);
|
||||
let plane = &crop.levels[top];
|
||||
let tpl = &self.templates[top];
|
||||
let shape = self.shape;
|
||||
let mut cands: Vec<(f64, [f64; 2])> = (-ry..=ry)
|
||||
.into_par_iter()
|
||||
.flat_map_iter(|j| {
|
||||
(-rx..=rx).map(move |i| {
|
||||
let c = [ctop[0] + i as f64, ctop[1] + j as f64];
|
||||
(ncc(plane, tpl, c, shape), c)
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
// Best score first; ties (flat or repetitive regions) go to the candidate nearest the
|
||||
// prediction so featureless areas don't drift.
|
||||
let cp = crop.to_level(predicted, top);
|
||||
let d2 = |c: &[f64; 2]| (c[0] - cp[0]).powi(2) + (c[1] - cp[1]).powi(2);
|
||||
cands.sort_by(|a, b| if (a.0 - b.0).abs() < 1e-9 { d2(&a.1).total_cmp(&d2(&b.1)) } else { b.0.total_cmp(&a.0) });
|
||||
let mut seeds: Vec<[f64; 2]> = vec![];
|
||||
for (_, c) in &cands {
|
||||
if seeds.iter().all(|q| (q[0] - c[0]).abs() > 1.5 || (q[1] - c[1]).abs() > 1.5) {
|
||||
seeds.push(*c);
|
||||
}
|
||||
if seeds.len() == 3 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Fine: refine each seed down the pyramid in a 5 × 5 neighbourhood.
|
||||
let mut best = (f64::MIN, sc);
|
||||
for seed in seeds {
|
||||
let mut c = seed;
|
||||
let mut score = f64::MIN;
|
||||
for k in (0..top).rev() {
|
||||
c = [c[0] * 2.0, c[1] * 2.0];
|
||||
let (pl, tp) = (&crop.levels[k], &self.templates[k]);
|
||||
// Start from the projected match; neighbours must beat it (no drift on ties).
|
||||
let mut lb = (ncc(pl, tp, c, shape), c);
|
||||
for j in -2..=2 {
|
||||
for i in -2..=2 {
|
||||
if i == 0 && j == 0 {
|
||||
continue;
|
||||
}
|
||||
let q = [c[0] + i as f64, c[1] + j as f64];
|
||||
let v = ncc(pl, tp, q, shape);
|
||||
if v > lb.0 + 1e-9 {
|
||||
lb = (v, q);
|
||||
}
|
||||
}
|
||||
}
|
||||
(score, c) = lb;
|
||||
}
|
||||
if top == 0 {
|
||||
score = ncc(&crop.levels[0], &self.templates[0], c, shape);
|
||||
}
|
||||
if score > best.0 + 1e-9 {
|
||||
best = (score, crop.level_to_image(c, 0));
|
||||
}
|
||||
}
|
||||
let c0 = crop.to_level(best.1, 0);
|
||||
// Sub-pixel (and shape) refinement.
|
||||
let (mut c, mut a) = (c0, shape);
|
||||
if opts.subpixel {
|
||||
let mut ok = false;
|
||||
if opts.track_shape
|
||||
&& let Some((cr, ar)) = lk_refine(&crop, &self.templates[0], c0, shape, true)
|
||||
{
|
||||
(c, a) = (cr, ar);
|
||||
ok = true;
|
||||
}
|
||||
if !ok {
|
||||
if let Some((cr, _)) = lk_refine(&crop, &self.templates[0], c0, shape, false) {
|
||||
c = cr;
|
||||
} else {
|
||||
c = parabolic(&crop.levels[0], &self.templates[0], c0, shape);
|
||||
}
|
||||
}
|
||||
}
|
||||
// A featureless template can't be matched: stay put.
|
||||
if self.templates[0].ss < 1e-6 {
|
||||
(c, a) = (crop.to_level(predicted, 0), shape);
|
||||
}
|
||||
let mut conf = ncc(&crop.levels[0], &self.templates[0], c, a);
|
||||
// Keep the coarse match if refinement made it worse.
|
||||
let coarse = ncc(&crop.levels[0], &self.templates[0], c0, shape);
|
||||
if coarse > conf + 0.02 {
|
||||
(c, a, conf) = (c0, shape, coarse);
|
||||
}
|
||||
let confidence = (conf.max(0.0) * 100.0).min(100.0);
|
||||
let found = crop.level_to_image(c, 0);
|
||||
let low = confidence < opts.threshold;
|
||||
let mut status = Status::Tracked;
|
||||
if low {
|
||||
match opts.action {
|
||||
ConfidenceAction::Stop => {
|
||||
return self.result(prev, frame, confidence, Status::Stopped);
|
||||
}
|
||||
ConfidenceAction::Extrapolate => {
|
||||
self.center = predicted;
|
||||
return self.result(predicted, frame, confidence, Status::Extrapolated);
|
||||
}
|
||||
ConfidenceAction::Continue => {}
|
||||
ConfidenceAction::Adapt => status = Status::Adapted,
|
||||
}
|
||||
}
|
||||
// Only confident matches predict the next search position.
|
||||
self.velocity = if low { [0.0; 2] } else { [found[0] - prev[0], found[1] - prev[1]] };
|
||||
self.center = found;
|
||||
self.shape = a;
|
||||
if opts.adapt_every_frame || status == Status::Adapted {
|
||||
status = Status::Adapted;
|
||||
self.resample(&crop);
|
||||
}
|
||||
self.result(found, frame, confidence, status)
|
||||
}
|
||||
|
||||
fn result(&self, c: [f64; 2], frame: &Frame, confidence: f64, status: Status) -> PointResult {
|
||||
PointResult {
|
||||
center: [c[0] - frame.offset[0], c[1] - frame.offset[1]],
|
||||
confidence,
|
||||
rotation: self.shape[1].atan2(self.shape[0]).to_degrees(),
|
||||
scale: (self.shape[0] * self.shape[0] + self.shape[1] * self.shape[1]).sqrt(),
|
||||
status,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Sub-pixel peak from 1-D parabola fits of the correlation around `c`.
|
||||
fn parabolic(plane: &Plane, p: &Patch, c: [f64; 2], a: [f64; 2]) -> [f64; 2] {
|
||||
let f = |dx: f64, dy: f64| ncc(plane, p, [c[0] + dx, c[1] + dy], a);
|
||||
let m = f(0.0, 0.0);
|
||||
let off = |l: f64, r: f64| {
|
||||
let d = l - 2.0 * m + r;
|
||||
if d.abs() < 1e-12 { 0.0 } else { (0.5 * (l - r) / d).clamp(-0.5, 0.5) }
|
||||
};
|
||||
[c[0] + off(f(-1.0, 0.0), f(1.0, 0.0)), c[1] + off(f(0.0, -1.0), f(0.0, 1.0))]
|
||||
}
|
||||
|
||||
/// Forward-additive Lucas–Kanade (Gauss–Newton) alignment of the level-0 template with gain and
|
||||
/// bias: minimise Σ (I(c + A u) − g T(u) − b)² over the centre `c` and, with `shape`, the
|
||||
/// similarity `A`. Returns `None` when it diverges.
|
||||
fn lk_refine(crop: &Crop, p: &Patch, c0: [f64; 2], a0: [f64; 2], shape: bool) -> Option<([f64; 2], [f64; 2])> {
|
||||
let plane = &crop.levels[0];
|
||||
let (gxp, gyp) = &crop.grad0;
|
||||
let ch = p.ch;
|
||||
let (mut c, mut a) = (c0, a0);
|
||||
let mut g = 1.0f64;
|
||||
let mut b;
|
||||
// Initial photometric estimate from means/variances.
|
||||
{
|
||||
let mut v = [0.0f32; 3];
|
||||
let (mut si, mut si2, mut st, mut st2, mut n) = (0.0, 0.0, 0.0, 0.0, 0.0);
|
||||
let mut k = 0;
|
||||
for gy in -p.r[1]..=p.r[1] {
|
||||
for gx in -p.r[0]..=p.r[0] {
|
||||
let (ux, uy) = (gx as f64, gy as f64);
|
||||
plane.sample(c[0] + a[0] * ux - a[1] * uy, c[1] + a[1] * ux + a[0] * uy, &mut v);
|
||||
for cc in 0..ch {
|
||||
let (i, t) = (v[cc] as f64, p.raw[k] as f64);
|
||||
si += i;
|
||||
si2 += i * i;
|
||||
st += t;
|
||||
st2 += t * t;
|
||||
n += 1.0;
|
||||
k += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
let (mi, mt) = (si / n, st / n);
|
||||
let (vi, vt) = ((si2 / n - mi * mi).max(0.0), (st2 / n - mt * mt).max(0.0));
|
||||
if vt > 1e-12 && vi > 1e-12 {
|
||||
g = (vi / vt).sqrt();
|
||||
}
|
||||
b = mi - g * mt;
|
||||
}
|
||||
let np = if shape { 6 } else { 4 };
|
||||
let (mut v, mut vx, mut vy) = ([0.0f32; 3], [0.0f32; 3], [0.0f32; 3]);
|
||||
for _ in 0..30 {
|
||||
let mut h = [[0.0f64; 6]; 6];
|
||||
let mut rhs = [0.0f64; 6];
|
||||
let mut k = 0;
|
||||
for gy in -p.r[1]..=p.r[1] {
|
||||
for gx in -p.r[0]..=p.r[0] {
|
||||
let (ux, uy) = (gx as f64, gy as f64);
|
||||
let q = [c[0] + a[0] * ux - a[1] * uy, c[1] + a[1] * ux + a[0] * uy];
|
||||
plane.sample(q[0], q[1], &mut v);
|
||||
gxp.sample(q[0], q[1], &mut vx);
|
||||
gyp.sample(q[0], q[1], &mut vy);
|
||||
for cc in 0..ch {
|
||||
let t = p.raw[k] as f64;
|
||||
k += 1;
|
||||
let (ix, iy) = (vx[cc] as f64, vy[cc] as f64);
|
||||
let r = v[cc] as f64 - g * t - b;
|
||||
let j: [f64; 6] = if shape { [ix, iy, ix * ux + iy * uy, -ix * uy + iy * ux, -t, -1.0] } else { [ix, iy, -t, -1.0, 0.0, 0.0] };
|
||||
for m in 0..np {
|
||||
rhs[m] -= j[m] * r;
|
||||
for n in m..np {
|
||||
h[m][n] += j[m] * j[n];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for m in 0..np {
|
||||
for n in 0..m {
|
||||
h[m][n] = h[n][m];
|
||||
}
|
||||
h[m][m] *= 1.0 + 1e-6;
|
||||
h[m][m] += 1e-9;
|
||||
}
|
||||
let d = if shape {
|
||||
solve::solve_linear(h, rhs)?
|
||||
} else {
|
||||
let hh = [
|
||||
[h[0][0], h[0][1], h[0][2], h[0][3]],
|
||||
[h[1][0], h[1][1], h[1][2], h[1][3]],
|
||||
[h[2][0], h[2][1], h[2][2], h[2][3]],
|
||||
[h[3][0], h[3][1], h[3][2], h[3][3]],
|
||||
];
|
||||
let x = solve::solve_linear(hh, [rhs[0], rhs[1], rhs[2], rhs[3]])?;
|
||||
[x[0], x[1], 0.0, 0.0, x[2], x[3]]
|
||||
};
|
||||
// Limit a single step (Gauss-Newton overshoot on poorly textured features).
|
||||
let step = (d[0] * d[0] + d[1] * d[1]).sqrt();
|
||||
let k = if step > 1.0 { 1.0 / step } else { 1.0 };
|
||||
c[0] += d[0] * k;
|
||||
c[1] += d[1] * k;
|
||||
if shape {
|
||||
a[0] += d[2] * k;
|
||||
a[1] += d[3] * k;
|
||||
}
|
||||
g += d[4] * k;
|
||||
b += d[5] * k;
|
||||
if !(c[0].is_finite() && c[1].is_finite() && a[0].is_finite() && a[1].is_finite()) {
|
||||
return None;
|
||||
}
|
||||
if step < 1e-4 && (!shape || (d[2].abs() + d[3].abs()) < 1e-6) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let moved = ((c[0] - c0[0]).powi(2) + (c[1] - c0[1]).powi(2)).sqrt();
|
||||
let s0 = (a0[0] * a0[0] + a0[1] * a0[1]).sqrt();
|
||||
let s1 = (a[0] * a[0] + a[1] * a[1]).sqrt();
|
||||
if moved > 2.5 || !(0.75..=1.33).contains(&(s1 / s0.max(1e-9))) || g <= 0.0 {
|
||||
return None;
|
||||
}
|
||||
Some((c, a))
|
||||
}
|
||||
|
||||
/// Tracks a set of points frame by frame (the points of one tracker).
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Tracker {
|
||||
pub opts: TrackOptions,
|
||||
points: Vec<PointState>,
|
||||
}
|
||||
|
||||
impl Tracker {
|
||||
/// Start tracking `points` as they appear on `frame` (the start frame).
|
||||
pub fn new(opts: TrackOptions, points: &[PointSpec], frame: &Frame) -> Tracker {
|
||||
let pts = points.par_iter().map(|s| PointState::new(*s, frame, &opts)).collect();
|
||||
Tracker { opts, points: pts }
|
||||
}
|
||||
|
||||
/// Track every point into the next frame (in either time direction).
|
||||
pub fn step(&mut self, frame: &Frame) -> Vec<PointResult> {
|
||||
let opts = &self.opts;
|
||||
self.points.par_iter_mut().map(|p| p.step(frame, opts)).collect()
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.points.len()
|
||||
}
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.points.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
|
@ -0,0 +1,243 @@
|
|||
//! Single- or multi-channel float planes: the tracker's working images (a crop of the frame in
|
||||
//! the matching channel, pre-processed), their pyramids and bilinear sampling.
|
||||
//!
|
||||
//! Continuous coordinates follow the raster crate: pixel `i` covers `[i, i + 1)` and its centre is
|
||||
//! at `i + 0.5`. Downsampling by 2 keeps that convention (level-1 coordinate = level-0 / 2).
|
||||
|
||||
use effectcraft_raster::Image;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// What the tracker compares (Motion Tracker Options ▸ Channel).
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum Channel {
|
||||
/// All three colour channels.
|
||||
Rgb,
|
||||
/// Rec. 709 luma.
|
||||
#[default]
|
||||
Luminance,
|
||||
/// HSV saturation.
|
||||
Saturation,
|
||||
}
|
||||
|
||||
impl Channel {
|
||||
pub const ALL: [Channel; 3] = [Channel::Rgb, Channel::Luminance, Channel::Saturation];
|
||||
pub fn label(self) -> &'static str {
|
||||
match self {
|
||||
Channel::Rgb => "RGB",
|
||||
Channel::Luminance => "Luminance",
|
||||
Channel::Saturation => "Saturation",
|
||||
}
|
||||
}
|
||||
pub fn from_name(s: &str) -> Option<Channel> {
|
||||
Channel::ALL.into_iter().find(|c| c.label().eq_ignore_ascii_case(s) || format!("{c:?}").eq_ignore_ascii_case(s))
|
||||
}
|
||||
pub fn channels(self) -> usize {
|
||||
if self == Channel::Rgb { 3 } else { 1 }
|
||||
}
|
||||
}
|
||||
|
||||
/// An interleaved float plane.
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct Plane {
|
||||
pub w: usize,
|
||||
pub h: usize,
|
||||
pub ch: usize,
|
||||
pub data: Vec<f32>,
|
||||
}
|
||||
|
||||
impl Plane {
|
||||
pub fn new(w: usize, h: usize, ch: usize) -> Plane {
|
||||
Plane { w, h, ch, data: vec![0.0; w * h * ch] }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn idx(&self, x: usize, y: usize) -> usize {
|
||||
(y * self.w + x) * self.ch
|
||||
}
|
||||
|
||||
/// Value at integer pixel `(x, y)` with edges repeated.
|
||||
#[inline]
|
||||
pub fn at(&self, x: isize, y: isize, c: usize) -> f32 {
|
||||
let x = x.clamp(0, self.w as isize - 1) as usize;
|
||||
let y = y.clamp(0, self.h as isize - 1) as usize;
|
||||
self.data[self.idx(x, y) + c]
|
||||
}
|
||||
|
||||
/// Bilinear sample at continuous coordinates (edges repeated) into `out[..ch]`.
|
||||
#[inline]
|
||||
pub fn sample(&self, x: f64, y: f64, out: &mut [f32]) {
|
||||
let fx = x - 0.5;
|
||||
let fy = y - 0.5;
|
||||
let x0 = fx.floor();
|
||||
let y0 = fy.floor();
|
||||
let tx = (fx - x0) as f32;
|
||||
let ty = (fy - y0) as f32;
|
||||
let (x0, y0) = (x0 as isize, y0 as isize);
|
||||
let inside = x0 >= 0 && y0 >= 0 && (x0 as usize) + 1 < self.w && (y0 as usize) + 1 < self.h;
|
||||
if inside {
|
||||
let i00 = self.idx(x0 as usize, y0 as usize);
|
||||
let i10 = i00 + self.ch;
|
||||
let i01 = i00 + self.w * self.ch;
|
||||
let i11 = i01 + self.ch;
|
||||
for c in 0..self.ch {
|
||||
let top = self.data[i00 + c] + (self.data[i10 + c] - self.data[i00 + c]) * tx;
|
||||
let bot = self.data[i01 + c] + (self.data[i11 + c] - self.data[i01 + c]) * tx;
|
||||
out[c] = top + (bot - top) * ty;
|
||||
}
|
||||
} else {
|
||||
for c in 0..self.ch {
|
||||
let a = self.at(x0, y0, c);
|
||||
let b = self.at(x0 + 1, y0, c);
|
||||
let cc = self.at(x0, y0 + 1, c);
|
||||
let d = self.at(x0 + 1, y0 + 1, c);
|
||||
let top = a + (b - a) * tx;
|
||||
let bot = cc + (d - cc) * tx;
|
||||
out[c] = top + (bot - top) * ty;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Crop `w × h` pixels at `(x0, y0)` of `img` in the given channel (edges repeated outside).
|
||||
pub fn from_image(img: &Image, channel: Channel, x0: i64, y0: i64, w: usize, h: usize) -> Plane {
|
||||
let ch = channel.channels();
|
||||
let mut p = Plane::new(w, h, ch);
|
||||
if img.is_empty() {
|
||||
return p;
|
||||
}
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
let px = img.get_clamped(x0 + x as i64, y0 + y as i64);
|
||||
let i = (y * w + x) * ch;
|
||||
match channel {
|
||||
Channel::Rgb => p.data[i..i + 3].copy_from_slice(&px[..3]),
|
||||
Channel::Luminance => p.data[i] = 0.2126 * px[0] + 0.7152 * px[1] + 0.0722 * px[2],
|
||||
Channel::Saturation => {
|
||||
let mx = px[0].max(px[1]).max(px[2]);
|
||||
let mn = px[0].min(px[1]).min(px[2]);
|
||||
p.data[i] = if mx > 1e-6 { (mx - mn) / mx } else { 0.0 };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
p
|
||||
}
|
||||
|
||||
/// Separable Gaussian blur (edges repeated).
|
||||
pub fn gaussian(&self, sigma: f64) -> Plane {
|
||||
if sigma <= 0.05 {
|
||||
return self.clone();
|
||||
}
|
||||
let r = (sigma * 3.0).ceil() as isize;
|
||||
let mut k: Vec<f32> = (-r..=r).map(|i| (-(i * i) as f64 / (2.0 * sigma * sigma)).exp() as f32).collect();
|
||||
let s: f32 = k.iter().sum();
|
||||
k.iter_mut().for_each(|v| *v /= s);
|
||||
let mut tmp = Plane::new(self.w, self.h, self.ch);
|
||||
for y in 0..self.h {
|
||||
for x in 0..self.w {
|
||||
for c in 0..self.ch {
|
||||
let mut acc = 0.0;
|
||||
for (j, kv) in k.iter().enumerate() {
|
||||
acc += kv * self.at(x as isize + j as isize - r, y as isize, c);
|
||||
}
|
||||
let i = tmp.idx(x, y) + c;
|
||||
tmp.data[i] = acc;
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut out = Plane::new(self.w, self.h, self.ch);
|
||||
for y in 0..self.h {
|
||||
for x in 0..self.w {
|
||||
for c in 0..self.ch {
|
||||
let mut acc = 0.0;
|
||||
for (j, kv) in k.iter().enumerate() {
|
||||
acc += kv * tmp.at(x as isize, y as isize + j as isize - r, c);
|
||||
}
|
||||
let i = out.idx(x, y) + c;
|
||||
out.data[i] = acc;
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Edge enhancement (unsharp mask): `x + 2 (x - blur(x))`.
|
||||
pub fn enhance(&self) -> Plane {
|
||||
let b = self.gaussian(1.5);
|
||||
let mut out = self.clone();
|
||||
for (o, (x, y)) in out.data.iter_mut().zip(self.data.iter().zip(&b.data)) {
|
||||
*o = x + 2.0 * (x - y);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Half resolution (2 × 2 box average; odd sizes round up with the edge repeated).
|
||||
pub fn downsample(&self) -> Plane {
|
||||
let w = self.w.div_ceil(2).max(1);
|
||||
let h = self.h.div_ceil(2).max(1);
|
||||
let mut out = Plane::new(w, h, self.ch);
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
for c in 0..self.ch {
|
||||
let (sx, sy) = (2 * x as isize, 2 * y as isize);
|
||||
let v = self.at(sx, sy, c) + self.at(sx + 1, sy, c) + self.at(sx, sy + 1, c) + self.at(sx + 1, sy + 1, c);
|
||||
let i = out.idx(x, y) + c;
|
||||
out.data[i] = v * 0.25;
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Central-difference gradients `(d/dx, d/dy)`.
|
||||
pub fn gradients(&self) -> (Plane, Plane) {
|
||||
let mut gx = Plane::new(self.w, self.h, self.ch);
|
||||
let mut gy = Plane::new(self.w, self.h, self.ch);
|
||||
for y in 0..self.h {
|
||||
for x in 0..self.w {
|
||||
for c in 0..self.ch {
|
||||
let (xi, yi) = (x as isize, y as isize);
|
||||
let i = self.idx(x, y) + c;
|
||||
gx.data[i] = 0.5 * (self.at(xi + 1, yi, c) - self.at(xi - 1, yi, c));
|
||||
gy.data[i] = 0.5 * (self.at(xi, yi + 1, c) - self.at(xi, yi - 1, c));
|
||||
}
|
||||
}
|
||||
}
|
||||
(gx, gy)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn sampling_and_pyramid_share_coordinates() {
|
||||
let mut p = Plane::new(8, 8, 1);
|
||||
for y in 0..8 {
|
||||
for x in 0..8 {
|
||||
p.data[y * 8 + x] = x as f32 + 10.0 * y as f32;
|
||||
}
|
||||
}
|
||||
let mut o = [0.0];
|
||||
p.sample(2.5, 3.5, &mut o);
|
||||
assert_eq!(o[0], 32.0);
|
||||
p.sample(3.0, 3.5, &mut o);
|
||||
assert!((o[0] - 32.5).abs() < 1e-6);
|
||||
// A linear ramp keeps its value at matching coordinates on the next level.
|
||||
let d = p.downsample();
|
||||
d.sample(1.5, 1.5, &mut o);
|
||||
let mut o0 = [0.0];
|
||||
p.sample(3.0, 3.0, &mut o0);
|
||||
assert!((o[0] - o0[0]).abs() < 1e-5, "{} vs {}", o[0], o0[0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn channels() {
|
||||
let img = Image::filled(2, 2, [1.0, 0.5, 0.0, 1.0]);
|
||||
assert_eq!(Plane::from_image(&img, Channel::Rgb, 0, 0, 2, 2).data[..3], [1.0, 0.5, 0.0]);
|
||||
assert!((Plane::from_image(&img, Channel::Luminance, 0, 0, 2, 2).data[0] - (0.2126 + 0.3576)).abs() < 1e-6);
|
||||
assert_eq!(Plane::from_image(&img, Channel::Saturation, 0, 0, 2, 2).data[0], 1.0);
|
||||
assert_eq!(Channel::from_name("luminance"), Some(Channel::Luminance));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,176 @@
|
|||
//! Small geometric solves for applying tracks: a 2-point similarity (Transform track rotation and
|
||||
//! scale), a 3-point affine (Affine Corner Pin) and a 4-point homography (Perspective Corner Pin,
|
||||
//! the direct linear transform with `h33 = 1`, Hartley & Zisserman §4.1).
|
||||
|
||||
/// Solve `a x = b` (n × n, row-major) by Gaussian elimination with partial pivoting.
|
||||
pub fn solve_linear<const N: usize>(mut a: [[f64; N]; N], mut b: [f64; N]) -> Option<[f64; N]> {
|
||||
for col in 0..N {
|
||||
let piv = (col..N).max_by(|&i, &j| a[i][col].abs().total_cmp(&a[j][col].abs()))?;
|
||||
if a[piv][col].abs() < 1e-12 {
|
||||
return None;
|
||||
}
|
||||
a.swap(col, piv);
|
||||
b.swap(col, piv);
|
||||
for r in col + 1..N {
|
||||
let f = a[r][col] / a[col][col];
|
||||
if f == 0.0 {
|
||||
continue;
|
||||
}
|
||||
for c in col..N {
|
||||
a[r][c] -= f * a[col][c];
|
||||
}
|
||||
b[r] -= f * b[col];
|
||||
}
|
||||
}
|
||||
let mut x = [0.0; N];
|
||||
for r in (0..N).rev() {
|
||||
let mut s = b[r];
|
||||
for c in r + 1..N {
|
||||
s -= a[r][c] * x[c];
|
||||
}
|
||||
x[r] = s / a[r][r];
|
||||
}
|
||||
x.iter().all(|v| v.is_finite()).then_some(x)
|
||||
}
|
||||
|
||||
/// A 3 × 3 projective transform (row-major), mapping `[x, y, 1]`.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Homography(pub [[f64; 3]; 3]);
|
||||
|
||||
impl Homography {
|
||||
pub const IDENTITY: Homography = Homography([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]);
|
||||
|
||||
pub fn apply(&self, p: [f64; 2]) -> [f64; 2] {
|
||||
let m = &self.0;
|
||||
let w = m[2][0] * p[0] + m[2][1] * p[1] + m[2][2];
|
||||
let w = if w.abs() < 1e-12 { 1e-12 } else { w };
|
||||
[(m[0][0] * p[0] + m[0][1] * p[1] + m[0][2]) / w, (m[1][0] * p[0] + m[1][1] * p[1] + m[1][2]) / w]
|
||||
}
|
||||
|
||||
/// The homography taking `src[i]` to `dst[i]` (four points, no three collinear).
|
||||
pub fn from_points(src: [[f64; 2]; 4], dst: [[f64; 2]; 4]) -> Option<Homography> {
|
||||
// Normalise both sets (centroid at 0, mean distance √2) for conditioning.
|
||||
let (ns, ts) = normalise(&src);
|
||||
let (nd, td) = normalise(&dst);
|
||||
let mut a = [[0.0; 8]; 8];
|
||||
let mut b = [0.0; 8];
|
||||
for i in 0..4 {
|
||||
let ([x, y], [u, v]) = (ns[i], nd[i]);
|
||||
a[2 * i] = [x, y, 1.0, 0.0, 0.0, 0.0, -u * x, -u * y];
|
||||
b[2 * i] = u;
|
||||
a[2 * i + 1] = [0.0, 0.0, 0.0, x, y, 1.0, -v * x, -v * y];
|
||||
b[2 * i + 1] = v;
|
||||
}
|
||||
let h = solve_linear(a, b)?;
|
||||
let hn = [[h[0], h[1], h[2]], [h[3], h[4], h[5]], [h[6], h[7], 1.0]];
|
||||
// H = Td⁻¹ · Hn · Ts
|
||||
let m = mul(inv_similarity(td), mul(hn, ts));
|
||||
let s = m[2][2];
|
||||
if s.abs() < 1e-12 {
|
||||
return None;
|
||||
}
|
||||
Some(Homography(m.map(|r| r.map(|v| v / s))))
|
||||
}
|
||||
|
||||
/// The affine map taking three points to three points.
|
||||
pub fn affine_from_points(src: [[f64; 2]; 3], dst: [[f64; 2]; 3]) -> Option<Homography> {
|
||||
let mut a = [[0.0; 3]; 3];
|
||||
for i in 0..3 {
|
||||
a[i] = [src[i][0], src[i][1], 1.0];
|
||||
}
|
||||
let x = solve_linear(a, [dst[0][0], dst[1][0], dst[2][0]])?;
|
||||
let y = solve_linear(a, [dst[0][1], dst[1][1], dst[2][1]])?;
|
||||
Some(Homography([x, y, [0.0, 0.0, 1.0]]))
|
||||
}
|
||||
|
||||
/// The similarity (rotation, uniform scale, translation) taking two points to two points.
|
||||
pub fn similarity_from_points(src: [[f64; 2]; 2], dst: [[f64; 2]; 2]) -> Option<Homography> {
|
||||
let (ds, dd) = (sub(src[1], src[0]), sub(dst[1], dst[0]));
|
||||
let n = ds[0] * ds[0] + ds[1] * ds[1];
|
||||
if n < 1e-12 {
|
||||
return None;
|
||||
}
|
||||
// dd = [[a, -b], [b, a]] ds
|
||||
let a = (dd[0] * ds[0] + dd[1] * ds[1]) / n;
|
||||
let b = (dd[1] * ds[0] - dd[0] * ds[1]) / n;
|
||||
let tx = dst[0][0] - (a * src[0][0] - b * src[0][1]);
|
||||
let ty = dst[0][1] - (b * src[0][0] + a * src[0][1]);
|
||||
Some(Homography([[a, -b, tx], [b, a, ty], [0.0, 0.0, 1.0]]))
|
||||
}
|
||||
}
|
||||
|
||||
/// Angle (degrees, y down = clockwise positive like After Effects' Rotation) and length of `b - a`.
|
||||
pub fn angle_and_length(a: [f64; 2], b: [f64; 2]) -> (f64, f64) {
|
||||
let d = sub(b, a);
|
||||
(d[1].atan2(d[0]).to_degrees(), (d[0] * d[0] + d[1] * d[1]).sqrt())
|
||||
}
|
||||
|
||||
/// `angle` moved by multiples of 360° to be closest to `prev` (continuous rotation curves).
|
||||
pub fn unwrap_degrees(prev: f64, angle: f64) -> f64 {
|
||||
angle + ((prev - angle) / 360.0).round() * 360.0
|
||||
}
|
||||
|
||||
fn sub(a: [f64; 2], b: [f64; 2]) -> [f64; 2] {
|
||||
[a[0] - b[0], a[1] - b[1]]
|
||||
}
|
||||
|
||||
fn mul(a: [[f64; 3]; 3], b: [[f64; 3]; 3]) -> [[f64; 3]; 3] {
|
||||
let mut o = [[0.0; 3]; 3];
|
||||
for i in 0..3 {
|
||||
for j in 0..3 {
|
||||
o[i][j] = (0..3).map(|k| a[i][k] * b[k][j]).sum();
|
||||
}
|
||||
}
|
||||
o
|
||||
}
|
||||
|
||||
fn normalise(p: &[[f64; 2]; 4]) -> ([[f64; 2]; 4], [[f64; 3]; 3]) {
|
||||
let c = [p.iter().map(|q| q[0]).sum::<f64>() / 4.0, p.iter().map(|q| q[1]).sum::<f64>() / 4.0];
|
||||
let d = p.iter().map(|q| ((q[0] - c[0]).powi(2) + (q[1] - c[1]).powi(2)).sqrt()).sum::<f64>() / 4.0;
|
||||
let s = if d > 1e-12 { std::f64::consts::SQRT_2 / d } else { 1.0 };
|
||||
let t = [[s, 0.0, -s * c[0]], [0.0, s, -s * c[1]], [0.0, 0.0, 1.0]];
|
||||
(p.map(|q| [s * (q[0] - c[0]), s * (q[1] - c[1])]), t)
|
||||
}
|
||||
|
||||
fn inv_similarity(t: [[f64; 3]; 3]) -> [[f64; 3]; 3] {
|
||||
let s = t[0][0];
|
||||
[[1.0 / s, 0.0, -t[0][2] / s], [0.0, 1.0 / s, -t[1][2] / s], [0.0, 0.0, 1.0]]
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn close(a: [f64; 2], b: [f64; 2], tol: f64) -> bool {
|
||||
(a[0] - b[0]).abs() < tol && (a[1] - b[1]).abs() < tol
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn homography_recovers_known_projection() {
|
||||
let h = Homography([[0.9, 0.12, 30.0], [-0.08, 1.1, -12.0], [0.0004, -0.0002, 1.0]]);
|
||||
let src = [[100.0, 100.0], [700.0, 120.0], [680.0, 500.0], [90.0, 520.0]];
|
||||
let dst = src.map(|p| h.apply(p));
|
||||
let got = Homography::from_points(src, dst).unwrap();
|
||||
for i in 0..3 {
|
||||
for j in 0..3 {
|
||||
assert!((got.0[i][j] - h.0[i][j]).abs() < 1e-6 * (1.0 + h.0[i][j].abs()), "{:?}", got.0);
|
||||
}
|
||||
}
|
||||
for p in [[0.0, 0.0], [400.0, 300.0], [1000.0, 50.0]] {
|
||||
assert!(close(got.apply(p), h.apply(p), 1e-6));
|
||||
}
|
||||
// Degenerate (collinear) input has no solution.
|
||||
assert!(Homography::from_points([[0.0, 0.0], [1.0, 1.0], [2.0, 2.0], [3.0, 3.0]], dst).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn affine_and_similarity() {
|
||||
let a = Homography::affine_from_points([[0.0, 0.0], [10.0, 0.0], [0.0, 10.0]], [[5.0, 5.0], [15.0, 7.0], [3.0, 15.0]]).unwrap();
|
||||
assert!(close(a.apply([10.0, 10.0]), [13.0, 17.0], 1e-9));
|
||||
let s = Homography::similarity_from_points([[0.0, 0.0], [10.0, 0.0]], [[1.0, 1.0], [1.0, 21.0]]).unwrap();
|
||||
assert!(close(s.apply([0.0, 10.0]), [-19.0, 1.0], 1e-9));
|
||||
let (ang, len) = angle_and_length([1.0, 1.0], [1.0, 21.0]);
|
||||
assert!((ang - 90.0).abs() < 1e-9 && (len - 20.0).abs() < 1e-9);
|
||||
assert!((unwrap_degrees(350.0, -5.0) - 355.0).abs() < 1e-9);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,256 @@
|
|||
//! Synthetic footage: a textured patch moving along a known path (with rotation and scale) over a
|
||||
//! textured background, with noise. Ground truth is exact, so recovered positions are compared
|
||||
//! against it directly.
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Smooth, band-limited texture (periods of 9–40 px).
|
||||
fn texture(x: f64, y: f64, seed: u32) -> [f32; 3] {
|
||||
let mut v = [0.0f64; 3];
|
||||
let mut s = seed.wrapping_mul(2654435761).wrapping_add(12345);
|
||||
let mut rnd = || {
|
||||
s ^= s << 13;
|
||||
s ^= s >> 17;
|
||||
s ^= s << 5;
|
||||
(s % 10_000) as f64 / 10_000.0
|
||||
};
|
||||
for _ in 0..7 {
|
||||
let ang = rnd() * std::f64::consts::TAU;
|
||||
let period = 9.0 + rnd() * 31.0;
|
||||
let phase = rnd() * std::f64::consts::TAU;
|
||||
let k = std::f64::consts::TAU / period;
|
||||
let w = (k * (x * ang.cos() + y * ang.sin()) + phase).sin();
|
||||
let c = [rnd(), rnd(), rnd()];
|
||||
for i in 0..3 {
|
||||
v[i] += w * (0.3 + c[i]) * 0.12;
|
||||
}
|
||||
}
|
||||
v.map(|q| (0.5 + q).clamp(0.0, 1.0) as f32)
|
||||
}
|
||||
|
||||
fn noise(x: u32, y: u32, f: u32) -> f32 {
|
||||
effectcraft_raster::hash_noise(x, y.wrapping_add(f.wrapping_mul(7919)), 99) - 0.5
|
||||
}
|
||||
|
||||
/// Patch pose on a frame: centre, rotation (degrees), scale.
|
||||
#[derive(Clone, Copy)]
|
||||
struct Pose {
|
||||
c: [f64; 2],
|
||||
rot: f64,
|
||||
scale: f64,
|
||||
}
|
||||
|
||||
impl Pose {
|
||||
/// Where the patch-local point `u` lands in the frame.
|
||||
fn map(&self, u: [f64; 2]) -> [f64; 2] {
|
||||
let (s, c) = self.rot.to_radians().sin_cos();
|
||||
[self.c[0] + self.scale * (c * u[0] - s * u[1]), self.c[1] + self.scale * (s * u[0] + c * u[1])]
|
||||
}
|
||||
fn local(&self, p: [f64; 2]) -> [f64; 2] {
|
||||
let (s, c) = self.rot.to_radians().sin_cos();
|
||||
let d = [(p[0] - self.c[0]) / self.scale, (p[1] - self.c[1]) / self.scale];
|
||||
[c * d[0] + s * d[1], -s * d[0] + c * d[1]]
|
||||
}
|
||||
}
|
||||
|
||||
/// Render a frame: background texture, the patch (half size `half`) on top, Gaussian-ish noise
|
||||
/// of amplitude `amp`, and an optional flat occluder rectangle.
|
||||
fn frame(w: u32, h: u32, pose: Pose, half: f64, amp: f32, f: u32, occluder: Option<[f64; 4]>) -> Image {
|
||||
let mut img = Image::new(w, h);
|
||||
let ss = [0.25, 0.75];
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
let mut acc = [0.0f32; 3];
|
||||
for sy in ss {
|
||||
for sx in ss {
|
||||
let p = [x as f64 + sx, y as f64 + sy];
|
||||
let l = pose.local(p);
|
||||
let t = if l[0].abs() <= half && l[1].abs() <= half { texture(l[0], l[1], 7) } else { texture(p[0] * 0.7, p[1] * 0.7, 3).map(|v| v * 0.6) };
|
||||
for i in 0..3 {
|
||||
acc[i] += t[i] * 0.25;
|
||||
}
|
||||
}
|
||||
}
|
||||
let n = noise(x, y, f) * amp;
|
||||
let mut px = [acc[0] + n, acc[1] + n, acc[2] + n, 1.0];
|
||||
if let Some([ox, oy, ow, oh]) = occluder {
|
||||
let (cx, cy) = (x as f64 + 0.5, y as f64 + 0.5);
|
||||
if cx >= ox && cx < ox + ow && cy >= oy && cy < oy + oh {
|
||||
px = [0.4, 0.4, 0.4, 1.0];
|
||||
}
|
||||
}
|
||||
img.set(x, y, px);
|
||||
}
|
||||
}
|
||||
img
|
||||
}
|
||||
|
||||
fn spec(c: [f64; 2], feature: f64, search: f64) -> PointSpec {
|
||||
PointSpec { center: c, feature_size: [feature, feature], search_offset: [0.0, 0.0], search_size: [search, search] }
|
||||
}
|
||||
|
||||
fn dist(a: [f64; 2], b: [f64; 2]) -> f64 {
|
||||
((a[0] - b[0]).powi(2) + (a[1] - b[1]).powi(2)).sqrt()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovers_translation_path_subpixel() {
|
||||
let path = |f: usize| {
|
||||
let t = f as f64;
|
||||
Pose { c: [100.0 + 4.37 * t + 6.0 * (t * 0.3).sin(), 90.0 + 2.71 * t - 3.0 * (t * 0.21).cos()], rot: 0.0, scale: 1.0 }
|
||||
};
|
||||
let f0 = frame(320, 240, path(0), 40.0, 0.02, 0, None);
|
||||
let mut tr = Tracker::new(TrackOptions::default(), &[spec(path(0).c, 31.0, 71.0)], &Frame::new(&f0));
|
||||
let mut worst: f64 = 0.0;
|
||||
for f in 1..24 {
|
||||
let img = frame(320, 240, path(f), 40.0, 0.02, f as u32, None);
|
||||
let r = tr.step(&Frame::new(&img))[0];
|
||||
let e = dist(r.center, path(f).c);
|
||||
worst = worst.max(e);
|
||||
assert!(r.confidence > 90.0, "frame {f}: confidence {}", r.confidence);
|
||||
assert_eq!(r.status, Status::Tracked);
|
||||
}
|
||||
eprintln!("worst error {worst:.4} px");
|
||||
assert!(worst < 0.25, "worst error {worst} px");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn layer_offset_and_channels() {
|
||||
let pose = |f: usize| Pose { c: [120.0 + 3.3 * f as f64, 100.0 - 1.7 * f as f64], rot: 0.0, scale: 1.0 };
|
||||
for channel in Channel::ALL {
|
||||
for (blur, enhance) in [(0.0, false), (3.0, false), (0.0, true)] {
|
||||
let opts = TrackOptions { channel, blur, enhance, ..Default::default() };
|
||||
let f0 = frame(240, 200, pose(0), 36.0, 0.01, 0, None);
|
||||
// Layer coordinates are image pixels minus the offset.
|
||||
let off = [10.0, -5.0];
|
||||
let c0 = [pose(0).c[0] - off[0], pose(0).c[1] - off[1]];
|
||||
let mut tr = Tracker::new(opts, &[spec(c0, 29.0, 61.0)], &Frame { img: &f0, offset: off });
|
||||
for f in 1..6 {
|
||||
let img = frame(240, 200, pose(f), 36.0, 0.01, f as u32, None);
|
||||
let r = tr.step(&Frame { img: &img, offset: off })[0];
|
||||
let want = [pose(f).c[0] - off[0], pose(f).c[1] - off[1]];
|
||||
assert!(dist(r.center, want) < 0.3, "{channel:?} blur {blur} enhance {enhance} frame {f}: {:?} vs {want:?}", r.center);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_points_recover_rotation_and_scale() {
|
||||
let pose = |f: usize| Pose { c: [160.0 + 1.5 * f as f64, 120.0 + 0.8 * f as f64], rot: 1.8 * f as f64, scale: 1.0 + 0.01 * f as f64 };
|
||||
let u = [[-26.0, 0.0], [26.0, 0.0]];
|
||||
let p0 = pose(0);
|
||||
let f0 = frame(320, 240, p0, 52.0, 0.01, 0, None);
|
||||
let mut tr = Tracker::new(TrackOptions::default(), &u.map(|q| spec(p0.map(q), 23.0, 51.0)), &Frame::new(&f0));
|
||||
let (a0, l0) = angle_and_length(p0.map(u[0]), p0.map(u[1]));
|
||||
let mut worst: f64 = 0.0;
|
||||
for f in 1..20 {
|
||||
let p = pose(f);
|
||||
let img = frame(320, 240, p, 52.0, 0.01, f as u32, None);
|
||||
let r = tr.step(&Frame::new(&img));
|
||||
for k in 0..2 {
|
||||
worst = worst.max(dist(r[k].center, p.map(u[k])));
|
||||
}
|
||||
let (a, l) = angle_and_length(r[0].center, r[1].center);
|
||||
assert!((unwrap_degrees(0.0, a - a0) - p.rot).abs() < 0.5, "frame {f}: rotation {} vs {}", a - a0, p.rot);
|
||||
assert!((l / l0 - p.scale).abs() < 0.01, "frame {f}: scale {} vs {}", l / l0, p.scale);
|
||||
// The per-feature shape estimate follows too.
|
||||
assert!((r[0].rotation - p.rot).abs() < 1.0, "frame {f}: feature rotation {}", r[0].rotation);
|
||||
assert!((r[0].scale - p.scale).abs() < 0.02, "frame {f}: feature scale {}", r[0].scale);
|
||||
}
|
||||
eprintln!("worst error {worst:.4} px");
|
||||
assert!(worst < 0.25, "worst error {worst} px");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn confidence_drops_on_occlusion_and_actions() {
|
||||
let pose = |f: usize| Pose { c: [110.0 + 2.0 * f as f64, 100.0], rot: 0.0, scale: 1.0 };
|
||||
let occ = |f: usize| (f == 5).then(|| [pose(f).c[0] - 12.0, pose(f).c[1] - 30.0, 40.0, 60.0]);
|
||||
let run = |action: ConfidenceAction| {
|
||||
let f0 = frame(240, 200, pose(0), 36.0, 0.01, 0, None);
|
||||
let opts = TrackOptions { action, ..Default::default() };
|
||||
let mut tr = Tracker::new(opts, &[spec(pose(0).c, 31.0, 61.0)], &Frame::new(&f0));
|
||||
(1..9)
|
||||
.map(|f| {
|
||||
let img = frame(240, 200, pose(f), 36.0, 0.01, f as u32, occ(f));
|
||||
tr.step(&Frame::new(&img))[0]
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let r = run(ConfidenceAction::Continue);
|
||||
assert!(r[3].confidence > 95.0, "{}", r[3].confidence);
|
||||
assert!(r[4].confidence < 70.0, "occluded frame confidence {}", r[4].confidence);
|
||||
let r = run(ConfidenceAction::Extrapolate);
|
||||
assert_eq!(r[4].status, Status::Extrapolated);
|
||||
assert!(dist(r[4].center, pose(5).c) < 0.5, "extrapolated {:?}", r[4].center);
|
||||
// Back on track after the occlusion.
|
||||
assert!(dist(r[6].center, pose(7).c) < 0.25 && r[6].status == Status::Tracked);
|
||||
let r = run(ConfidenceAction::Stop);
|
||||
assert_eq!(r[4].status, Status::Stopped);
|
||||
let r = run(ConfidenceAction::Adapt);
|
||||
assert_eq!(r[4].status, Status::Adapted);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backward_and_whole_pixel() {
|
||||
let pose = |f: usize| Pose { c: [100.0 + 3.25 * f as f64, 100.0 + 1.5 * f as f64], rot: 0.0, scale: 1.0 };
|
||||
let last = 6;
|
||||
let fl = frame(240, 200, pose(last), 36.0, 0.0, 0, None);
|
||||
let opts = TrackOptions { subpixel: false, ..Default::default() };
|
||||
let mut tr = Tracker::new(opts, &[spec(pose(last).c, 31.0, 61.0)], &Frame::new(&fl));
|
||||
for f in (0..last).rev() {
|
||||
let img = frame(240, 200, pose(f), 36.0, 0.0, 0, None);
|
||||
let r = tr.step(&Frame::new(&img))[0];
|
||||
let want = pose(f).c;
|
||||
// Whole-pixel displacements of the start position.
|
||||
let d = [r.center[0] - pose(last).c[0], r.center[1] - pose(last).c[1]];
|
||||
assert!((d[0] - d[0].round()).abs() < 1e-9 && (d[1] - d[1].round()).abs() < 1e-9);
|
||||
assert!(dist(r.center, want) < 0.75, "frame {f}: {:?} vs {want:?}", r.center);
|
||||
}
|
||||
}
|
||||
|
||||
/// 1080p, one point: tracking cost per frame (frames are generated outside the timing).
|
||||
#[test]
|
||||
fn performance_1080p_single_point() {
|
||||
let w = 1920;
|
||||
let h = 1080;
|
||||
// A cheap textured frame (the full synthetic renderer is slow at 1080p in debug builds).
|
||||
let make = |dx: f64, dy: f64| {
|
||||
let mut img = Image::new(w, h);
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
let (fx, fy) = (x as f64 + 0.5 - dx, y as f64 + 0.5 - dy);
|
||||
let v = 0.5 + 0.2 * (fx * 0.31).sin() * (fy * 0.17).cos() + 0.15 * ((fx + fy) * 0.11).sin() + 0.1 * ((fx * 0.05).sin() * (fy * 0.23).sin());
|
||||
img.set(x, y, [v as f32, v as f32, v as f32, 1.0]);
|
||||
}
|
||||
}
|
||||
img
|
||||
};
|
||||
let frames: Vec<Image> = (0..6).map(|f| make(2.3 * f as f64, 1.1 * f as f64)).collect();
|
||||
let c0 = [960.0, 540.0];
|
||||
let s = PointSpec { center: c0, feature_size: [48.0, 48.0], search_offset: [0.0; 2], search_size: [128.0, 128.0] };
|
||||
let t0 = std::time::Instant::now();
|
||||
let mut tr = Tracker::new(TrackOptions::default(), &[s], &Frame::new(&frames[0]));
|
||||
let mut last = PointResult { center: c0, confidence: 0.0, rotation: 0.0, scale: 1.0, status: Status::Tracked };
|
||||
for f in frames.iter().skip(1) {
|
||||
last = tr.step(&Frame::new(f))[0];
|
||||
}
|
||||
let per = t0.elapsed().as_secs_f64() / 5.0;
|
||||
eprintln!("1080p single-point track: {:.2} ms/frame ({:.0} fps)", per * 1e3, 1.0 / per);
|
||||
assert!(dist(last.center, [c0[0] + 2.3 * 5.0, c0[1] + 1.1 * 5.0]) < 0.25, "{:?}", last.center);
|
||||
assert!(per < 0.1, "{per} s per frame");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn featureless_regions_stay_put() {
|
||||
let flat = Image::filled(200, 160, [0.3, 0.3, 0.3, 1.0]);
|
||||
for action in ConfidenceAction::ALL {
|
||||
let opts = TrackOptions { action, ..Default::default() };
|
||||
let mut tr = Tracker::new(opts, &[spec([100.0, 80.0], 21.0, 61.0)], &Frame::new(&flat));
|
||||
for _ in 0..10 {
|
||||
let r = tr.step(&Frame::new(&flat))[0];
|
||||
assert!(dist(r.center, [100.0, 80.0]) < 1e-9, "{action:?}: drifted to {:?}", r.center);
|
||||
assert!(r.confidence < 1.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -57,6 +57,12 @@ pub enum Dialog {
|
|||
KeyVelocity,
|
||||
KeyInterpolation,
|
||||
TimeStretch,
|
||||
/// Tracker ▸ Options…
|
||||
TrackOptions,
|
||||
/// Tracker ▸ Edit Target…
|
||||
TrackTarget,
|
||||
/// Tracker ▸ Apply (Transform / Stabilize): Apply Dimensions.
|
||||
TrackApply,
|
||||
}
|
||||
|
||||
/// Host hooks provided by the native app (file pickers etc.).
|
||||
|
|
@ -536,6 +542,10 @@ impl EffectcraftApp {
|
|||
self.session.poll_render();
|
||||
ctx.request_repaint_after(std::time::Duration::from_millis(100));
|
||||
}
|
||||
if self.session.track_job.is_some() {
|
||||
self.session.poll_track();
|
||||
ctx.request_repaint_after(std::time::Duration::from_millis(50));
|
||||
}
|
||||
self.handle_events(&ctx);
|
||||
if let Some(rx) = self.command_inbox.take() {
|
||||
while let Ok(id) = rx.try_recv() {
|
||||
|
|
|
|||
|
|
@ -446,6 +446,14 @@ pub fn frontend(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, p: Valu
|
|||
app.show_panel(PanelKind::Flowchart);
|
||||
Value::Null
|
||||
}
|
||||
"track.editTargetDialog" => {
|
||||
crate::panels::tracker::open_target(app)?;
|
||||
Value::Null
|
||||
}
|
||||
"track.optionsDialog" => {
|
||||
crate::panels::tracker::open_options(app)?;
|
||||
Value::Null
|
||||
}
|
||||
"layer.openLayer" => {
|
||||
app.show_panel(PanelKind::Layer);
|
||||
Value::Null
|
||||
|
|
|
|||
|
|
@ -31,6 +31,9 @@ pub struct DialogState {
|
|||
pub velocity: super::key_dialogs::VelocityDraft,
|
||||
pub interp: super::key_dialogs::InterpDraft,
|
||||
pub stretch: super::key_dialogs::StretchDraft,
|
||||
pub track_options: super::tracker::OptionsDraft,
|
||||
pub track_target: super::tracker::TargetDraft,
|
||||
pub track_apply: super::tracker::ApplyDraft,
|
||||
}
|
||||
|
||||
pub fn open_new_comp(app: &mut EffectcraftApp) {
|
||||
|
|
@ -108,6 +111,9 @@ pub fn show(app: &mut EffectcraftApp, ctx: &egui::Context) {
|
|||
Dialog::KeyVelocity => super::key_dialogs::velocity(app, ctx, &t),
|
||||
Dialog::KeyInterpolation => super::key_dialogs::interpolation(app, ctx, &t),
|
||||
Dialog::TimeStretch => super::key_dialogs::time_stretch(app, ctx, &t),
|
||||
Dialog::TrackOptions => super::tracker::options_dialog(app, ctx, &t),
|
||||
Dialog::TrackTarget => super::tracker::target_dialog(app, ctx, &t),
|
||||
Dialog::TrackApply => super::tracker::apply_dialog(app, ctx, &t),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ pub mod properties;
|
|||
pub mod render_queue;
|
||||
pub mod text_panels;
|
||||
pub mod timeline;
|
||||
pub mod tracker;
|
||||
pub mod viewer;
|
||||
pub mod waveform;
|
||||
|
||||
|
|
@ -70,6 +71,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: PanelKind, rect: Rec
|
|||
PanelKind::Markers => misc::markers(app, ui, rect),
|
||||
PanelKind::Wiggler => misc::wiggler(app, ui, rect),
|
||||
PanelKind::RenderQueue => render_queue::show(app, ui, rect),
|
||||
PanelKind::Tracker => tracker::show(app, ui, rect),
|
||||
other => misc::placeholder(app, ui, rect, other),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,722 @@
|
|||
//! Tracker panel (Window ▸ Tracker), its Motion Tracker Options / Motion Target / Apply Options
|
||||
//! dialogs, and the track point widgets drawn in the viewer (feature region, search region,
|
||||
//! attach point crosshair, track path) with their drag gestures.
|
||||
//!
|
||||
//! Everything goes through `track.*` commands, so agents can do the same.
|
||||
|
||||
use effectcraft_engine::geom::{Mat3, vec2 as gv2};
|
||||
use effectcraft_engine::project::tracking::{LowConfidence, TrackChannel, TrackKind, TrackerOptions};
|
||||
use effectcraft_engine::project::{Layer, LayerId, Uid};
|
||||
use effectcraft_engine::time::Tick;
|
||||
use effectcraft_engine::tracking::point_values;
|
||||
use egui::{Align2, Color32, Pos2, Rect, Sense, Stroke, StrokeKind, pos2, vec2};
|
||||
use serde_json::json;
|
||||
|
||||
use super::viewer::ViewerMap;
|
||||
use crate::theme::Tokens;
|
||||
use crate::{Dialog, EffectcraftApp, widgets};
|
||||
|
||||
const ROW: f32 = 26.0;
|
||||
|
||||
/// Motion Tracker Options dialog state.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct OptionsDraft {
|
||||
pub name: String,
|
||||
pub opts: TrackerOptions,
|
||||
pub blur_on: bool,
|
||||
}
|
||||
|
||||
/// Motion Target dialog state.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct TargetDraft {
|
||||
pub target: Option<u64>,
|
||||
}
|
||||
|
||||
/// Apply Options dialog state (0 = X and Y, 1 = X only, 2 = Y only).
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct ApplyDraft {
|
||||
pub dims: usize,
|
||||
}
|
||||
|
||||
/// The current track: (tracked layer, tracker uid), resolved like the engine does.
|
||||
fn current(app: &EffectcraftApp) -> Option<(LayerId, Uid)> {
|
||||
let comp = app.session.active_comp()?;
|
||||
if let Some((l, t)) = app.session.state.current_track
|
||||
&& comp.layer(l).and_then(|l| l.tracker(t)).is_some()
|
||||
{
|
||||
return Some((l, t));
|
||||
}
|
||||
let src = motion_source(app)?;
|
||||
let l = comp.layer(src)?;
|
||||
l.trackers().next().map(|(g, _)| (l.id, g.uid))
|
||||
}
|
||||
|
||||
fn source_id() -> egui::Id {
|
||||
egui::Id::new("tracker-motion-source")
|
||||
}
|
||||
|
||||
/// Motion Source: the current track's layer, the chosen source, or the selected layer.
|
||||
fn motion_source(app: &EffectcraftApp) -> Option<LayerId> {
|
||||
let comp = app.session.active_comp()?;
|
||||
if let Some((l, _)) = app.session.state.current_track
|
||||
&& comp.layer(l).is_some()
|
||||
{
|
||||
return Some(l);
|
||||
}
|
||||
let chosen: Option<u64> = app.ui.tracker_source;
|
||||
if let Some(l) = chosen.map(LayerId).filter(|l| comp.layer(*l).is_some()) {
|
||||
return Some(l);
|
||||
}
|
||||
app.session.state.selected_layers.first().copied().filter(|l| comp.layer(*l).is_some_and(trackable))
|
||||
}
|
||||
|
||||
fn trackable(l: &Layer) -> bool {
|
||||
use effectcraft_engine::project::LayerSource;
|
||||
l.has_video() && !matches!(l.source, LayerSource::Text | LayerSource::Shape)
|
||||
}
|
||||
|
||||
fn label(p: &egui::Painter, pos: Pos2, s: &str, t: &Tokens) {
|
||||
p.text(pos, Align2::LEFT_CENTER, s, Tokens::ui(12.0), t.text_dim);
|
||||
}
|
||||
|
||||
/// A button that can be disabled (drawn dim and inert).
|
||||
fn button(app: &mut EffectcraftApp, ui: &mut egui::Ui, r: Rect, text: &str, enabled: bool, id: &str) -> bool {
|
||||
let t = app.tokens;
|
||||
app.auto.add(id, r, text);
|
||||
if !enabled {
|
||||
ui.painter().rect_filled(r, r.height() / 2.0, Color32::from_rgb(0x2a, 0x2a, 0x2a));
|
||||
ui.painter().text(r.center(), Align2::CENTER_CENTER, text, Tokens::medium(12.0), t.text_faint);
|
||||
return false;
|
||||
}
|
||||
widgets::text_button(ui, r, text, false, &t, egui::Id::new(id)).clicked()
|
||||
}
|
||||
|
||||
fn run(app: &mut EffectcraftApp, ctx: &egui::Context, cmd: &str, p: serde_json::Value) {
|
||||
if let Err(e) = crate::menus::invoke(app, ctx, cmd, p) {
|
||||
app.ui.status = e;
|
||||
}
|
||||
}
|
||||
|
||||
/// Analyze button glyphs: ◀| ◀ ▶ |▶ (and ■ while analysing in that direction).
|
||||
fn analyze_glyph(p: &egui::Painter, r: Rect, forward: bool, frame: bool, stop: bool, col: Color32) {
|
||||
let c = r.center();
|
||||
if stop {
|
||||
p.rect_filled(Rect::from_center_size(c, vec2(9.0, 9.0)), 1.0, col);
|
||||
return;
|
||||
}
|
||||
let s = if forward { 1.0 } else { -1.0 };
|
||||
let tri = vec![pos2(c.x - 4.0 * s, c.y - 5.0), pos2(c.x + 4.0 * s, c.y), pos2(c.x - 4.0 * s, c.y + 5.0)];
|
||||
p.add(egui::Shape::convex_polygon(tri, col, Stroke::NONE));
|
||||
if frame {
|
||||
let x = c.x + 5.5 * s;
|
||||
p.line_segment([pos2(x, c.y - 5.0), pos2(x, c.y + 5.0)], Stroke::new(1.6, col));
|
||||
}
|
||||
}
|
||||
|
||||
pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
|
||||
let t = app.tokens;
|
||||
let ctx = ui.ctx().clone();
|
||||
if app.session.track_job.is_some() {
|
||||
app.session.poll_track();
|
||||
ctx.request_repaint_after(std::time::Duration::from_millis(50));
|
||||
}
|
||||
let p = ui.painter().with_clip_rect(rect);
|
||||
let x0 = rect.min.x + 10.0;
|
||||
let w = (rect.width() - 20.0).max(120.0);
|
||||
let half = (w - 6.0) / 2.0;
|
||||
let mut y = rect.min.y + 10.0;
|
||||
let has_comp = app.session.active_comp().is_some();
|
||||
let source = motion_source(app);
|
||||
let cur = current(app);
|
||||
let can_track = has_comp && source.is_some();
|
||||
|
||||
// Track Camera / Warp Stabilizer (not available) and Track Motion / Stabilize Motion.
|
||||
let r1 = Rect::from_min_size(pos2(x0, y), vec2(half, 22.0));
|
||||
let r2 = Rect::from_min_size(pos2(x0 + half + 6.0, y), vec2(half, 22.0));
|
||||
button(app, ui, r1, "Track Camera", false, "tracker.trackCamera");
|
||||
button(app, ui, r2, "Warp Stabilizer", false, "tracker.warpStabilizer");
|
||||
y += 28.0;
|
||||
let r1 = Rect::from_min_size(pos2(x0, y), vec2(half, 22.0));
|
||||
let r2 = Rect::from_min_size(pos2(x0 + half + 6.0, y), vec2(half, 22.0));
|
||||
let src_p = source.map(|l| json!({"layer": l.0})).unwrap_or_else(|| json!({}));
|
||||
if button(app, ui, r1, "Track Motion", can_track, "tracker.trackMotion") {
|
||||
run(app, &ctx, "track.motion", src_p.clone());
|
||||
}
|
||||
if button(app, ui, r2, "Stabilize Motion", can_track, "tracker.stabilizeMotion") {
|
||||
run(app, &ctx, "track.stabilize", src_p.clone());
|
||||
}
|
||||
y += 34.0;
|
||||
let Some(comp) = app.session.active_comp().cloned() else {
|
||||
p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, "Open a composition to track motion.", Tokens::ui(12.0), t.text_faint);
|
||||
return;
|
||||
};
|
||||
let field_x = x0 + 100.0;
|
||||
let field_w = (w - 100.0).max(80.0);
|
||||
|
||||
// Motion Source.
|
||||
label(&p, pos2(x0, y + 10.0), "Motion Source:", &t);
|
||||
let srcs: Vec<(LayerId, String)> =
|
||||
comp.layers.iter().enumerate().filter(|(_, l)| trackable(l)).map(|(i, l)| (l.id, format!("{}. {}", i + 1, l.name))).collect();
|
||||
let dd = Rect::from_min_size(pos2(field_x, y), vec2(field_w, 20.0));
|
||||
let name = source.and_then(|s| srcs.iter().find(|(l, _)| *l == s)).map(|(_, n)| n.clone()).unwrap_or_else(|| "None".into());
|
||||
let did = egui::Id::new("tracker-source-dd");
|
||||
if widgets::dropdown(ui, dd, &name, &t, did).clicked() {
|
||||
widgets::open_popup(ui, did);
|
||||
}
|
||||
app.auto.add("tracker.motionSource", dd, "Motion Source");
|
||||
let mut opts: Vec<String> = vec!["None".into()];
|
||||
opts.extend(srcs.iter().map(|(_, n)| n.clone()));
|
||||
let cur_i = source.and_then(|s| srcs.iter().position(|(l, _)| *l == s)).map(|i| i + 1).or(Some(0));
|
||||
if let Some(i) = widgets::popup_menu(ui, did, dd.left_bottom(), &opts, cur_i) {
|
||||
let chosen = i.checked_sub(1).and_then(|i| srcs.get(i)).map(|(l, _)| *l);
|
||||
app.ui.tracker_source = chosen.map(|l| l.0);
|
||||
ctx.data_mut(|d| d.remove::<u64>(source_id()));
|
||||
match chosen.and_then(|l| comp.layer(l)).and_then(|l| l.trackers().next().map(|(g, _)| (l.id, g.uid))) {
|
||||
Some((l, g)) => run(app, &ctx, "track.select", json!({"layer": l.0, "tracker": g})),
|
||||
None => app.session.state.current_track = None,
|
||||
}
|
||||
}
|
||||
y += ROW;
|
||||
|
||||
// Current Track.
|
||||
label(&p, pos2(x0, y + 10.0), "Current Track:", &t);
|
||||
let trackers: Vec<(Uid, String)> =
|
||||
source.and_then(|s| comp.layer(s)).map(|l| l.trackers().map(|(g, _)| (g.uid, g.name.clone())).collect()).unwrap_or_default();
|
||||
let dd = Rect::from_min_size(pos2(field_x, y), vec2(field_w, 20.0));
|
||||
let cname = cur.and_then(|(_, u)| trackers.iter().find(|(g, _)| *g == u)).map(|(_, n)| n.clone()).unwrap_or_else(|| "None".into());
|
||||
let did = egui::Id::new("tracker-current-dd");
|
||||
if widgets::dropdown(ui, dd, &cname, &t, did).clicked() && !trackers.is_empty() {
|
||||
widgets::open_popup(ui, did);
|
||||
}
|
||||
app.auto.add("tracker.currentTrack", dd, "Current Track");
|
||||
let names: Vec<String> = trackers.iter().map(|(_, n)| n.clone()).collect();
|
||||
let ci = cur.and_then(|(_, u)| trackers.iter().position(|(g, _)| *g == u));
|
||||
if let Some(i) = widgets::popup_menu(ui, did, dd.left_bottom(), &names, ci)
|
||||
&& let (Some(src), Some((uid, _))) = (source, trackers.get(i))
|
||||
{
|
||||
run(app, &ctx, "track.select", json!({"layer": src.0, "tracker": uid}));
|
||||
}
|
||||
y += ROW;
|
||||
|
||||
let settings = cur.and_then(|(l, u)| comp.layer(l).and_then(|l| l.tracker(u)).map(|(_, s)| s.clone()));
|
||||
let enabled = settings.is_some();
|
||||
let addr = cur.map(|(l, u)| json!({"layer": l.0, "tracker": u})).unwrap_or_else(|| json!({}));
|
||||
let with = |extra: serde_json::Value| {
|
||||
let mut v = addr.clone();
|
||||
if let (Some(o), Some(e)) = (v.as_object_mut(), extra.as_object()) {
|
||||
for (k, x) in e {
|
||||
o.insert(k.clone(), x.clone());
|
||||
}
|
||||
}
|
||||
v
|
||||
};
|
||||
|
||||
// Track Type.
|
||||
label(&p, pos2(x0, y + 10.0), "Track Type:", &t);
|
||||
let dd = Rect::from_min_size(pos2(field_x, y), vec2(field_w, 20.0));
|
||||
let kind = settings.as_ref().map(|s| s.kind).unwrap_or_default();
|
||||
let did = egui::Id::new("tracker-type-dd");
|
||||
if widgets::dropdown(ui, dd, if enabled { kind.label() } else { "" }, &t, did).clicked() && enabled {
|
||||
widgets::open_popup(ui, did);
|
||||
}
|
||||
app.auto.add("tracker.trackType", dd, "Track Type");
|
||||
let kinds: Vec<String> = TrackKind::ALL.iter().map(|k| k.label().to_string()).collect();
|
||||
if let Some(i) = widgets::popup_menu(ui, did, dd.left_bottom(), &kinds, TrackKind::ALL.iter().position(|k| *k == kind)) {
|
||||
run(app, &ctx, "track.setType", with(json!({"kind": TrackKind::ALL[i].id()})));
|
||||
}
|
||||
y += ROW;
|
||||
|
||||
// Position / Rotation / Scale.
|
||||
let transformish = matches!(kind, TrackKind::Transform | TrackKind::Stabilize) && enabled;
|
||||
let mut xx = x0;
|
||||
for (key, text, on) in [
|
||||
("position", "Position", settings.as_ref().is_some_and(|s| s.position)),
|
||||
("rotation", "Rotation", settings.as_ref().is_some_and(|s| s.rotation)),
|
||||
("scale", "Scale", settings.as_ref().is_some_and(|s| s.scale)),
|
||||
] {
|
||||
let cr = Rect::from_min_size(pos2(xx, y + 2.0), vec2(16.0, 16.0));
|
||||
let resp = widgets::checkbox(ui, cr, on && transformish, &t, egui::Id::new(("tracker-cb", key)));
|
||||
app.auto.add(&format!("tracker.{key}"), cr, text);
|
||||
p.text(pos2(cr.max.x + 4.0, cr.center().y), Align2::LEFT_CENTER, text, Tokens::ui(12.0), if transformish { t.text } else { t.text_faint });
|
||||
if resp.clicked() && transformish {
|
||||
run(app, &ctx, "track.setType", with(json!({ key: !on })));
|
||||
}
|
||||
xx += 84.0;
|
||||
}
|
||||
y += ROW;
|
||||
|
||||
// Motion Target.
|
||||
let target_name = settings
|
||||
.as_ref()
|
||||
.and_then(|s| s.target)
|
||||
.and_then(|l| comp.layer(l).map(|l| format!("{}. {}", comp.layers.iter().position(|x| x.id == l.id).unwrap_or(0) + 1, l.name)));
|
||||
label(&p, pos2(x0, y + 10.0), "Motion Target:", &t);
|
||||
let tr = Rect::from_min_size(pos2(field_x, y), vec2(field_w, 20.0));
|
||||
p.text(
|
||||
pos2(tr.min.x, tr.center().y),
|
||||
Align2::LEFT_CENTER,
|
||||
target_name.unwrap_or_else(|| if enabled && kind != TrackKind::Raw { "None".into() } else { String::new() }),
|
||||
Tokens::ui(12.0),
|
||||
t.text,
|
||||
);
|
||||
app.auto.add("tracker.motionTarget", tr, "Motion Target");
|
||||
y += ROW;
|
||||
let r1 = Rect::from_min_size(pos2(x0, y), vec2(half, 22.0));
|
||||
let r2 = Rect::from_min_size(pos2(x0 + half + 6.0, y), vec2(half, 22.0));
|
||||
if button(app, ui, r1, "Edit Target...", enabled && kind != TrackKind::Raw && kind != TrackKind::Stabilize, "tracker.editTarget") {
|
||||
run(app, &ctx, "track.editTargetDialog", json!({}));
|
||||
}
|
||||
if button(app, ui, r2, "Options...", enabled, "tracker.options") {
|
||||
run(app, &ctx, "track.optionsDialog", json!({}));
|
||||
}
|
||||
y += 32.0;
|
||||
|
||||
// Analyze.
|
||||
label(&p, pos2(x0, y + 11.0), "Analyze:", &t);
|
||||
let tracking = app.session.is_tracking();
|
||||
let running_dir = app.session.track_job.as_ref().map(|j| j.direction);
|
||||
let mut bx = field_x;
|
||||
for (dir, fwd, frame, id, tip) in [
|
||||
(effectcraft_engine::tracking::Direction::FrameBackward, false, true, "frameBackward", "Analyze 1 frame backward"),
|
||||
(effectcraft_engine::tracking::Direction::Backward, false, false, "backward", "Analyze backward"),
|
||||
(effectcraft_engine::tracking::Direction::Forward, true, false, "forward", "Analyze forward"),
|
||||
(effectcraft_engine::tracking::Direction::FrameForward, true, true, "frameForward", "Analyze 1 frame forward"),
|
||||
] {
|
||||
let r = Rect::from_min_size(pos2(bx, y), vec2(30.0, 22.0));
|
||||
let resp = ui.interact(r, egui::Id::new(("tracker-analyze", id)), Sense::click()).on_hover_text(tip);
|
||||
let stop = tracking && running_dir == Some(dir) && !frame;
|
||||
let on = enabled && (!tracking || stop);
|
||||
p.rect_filled(r, 4.0, if resp.hovered() && on { t.hover } else { t.field_bg });
|
||||
p.rect_stroke(r, 4.0, Stroke::new(1.0, t.field_border), StrokeKind::Inside);
|
||||
analyze_glyph(&p, r, fwd, frame, stop, if on { t.text } else { t.text_faint });
|
||||
app.auto.add(&format!("tracker.analyze.{id}"), r, tip);
|
||||
if resp.clicked() && on {
|
||||
if stop {
|
||||
run(app, &ctx, "track.stop", json!({}));
|
||||
} else {
|
||||
run(app, &ctx, "track.analyze", with(json!({ "direction": id })));
|
||||
}
|
||||
}
|
||||
bx += 34.0;
|
||||
}
|
||||
y += 32.0;
|
||||
|
||||
// Reset / Apply.
|
||||
let r1 = Rect::from_min_size(pos2(x0, y), vec2(half, 22.0));
|
||||
let r2 = Rect::from_min_size(pos2(x0 + half + 6.0, y), vec2(half, 22.0));
|
||||
if button(app, ui, r1, "Reset", enabled && !tracking, "tracker.reset") {
|
||||
run(app, &ctx, "track.reset", addr.clone());
|
||||
}
|
||||
if button(app, ui, r2, "Apply", enabled && !tracking && kind != TrackKind::Raw, "tracker.apply") {
|
||||
if matches!(kind, TrackKind::Transform | TrackKind::Stabilize) {
|
||||
app.dialog_state.track_apply = ApplyDraft::default();
|
||||
app.dialog = Some(Dialog::TrackApply);
|
||||
} else {
|
||||
run(app, &ctx, "track.apply", addr.clone());
|
||||
}
|
||||
}
|
||||
y += 32.0;
|
||||
|
||||
// Progress.
|
||||
if let Some(pr) = app.session.track_progress() {
|
||||
let bar = Rect::from_min_size(pos2(x0, y), vec2(w, 6.0));
|
||||
p.rect_filled(bar, 3.0, t.field_bg);
|
||||
let f = if pr.total > 0 { pr.done as f32 / pr.total as f32 } else { 0.0 };
|
||||
p.rect_filled(Rect::from_min_size(bar.min, vec2(bar.width() * f.clamp(0.0, 1.0), 6.0)), 3.0, t.accent);
|
||||
app.auto.add("tracker.progress", bar, "Analysis progress");
|
||||
p.text(
|
||||
pos2(x0, y + 18.0),
|
||||
Align2::LEFT_CENTER,
|
||||
format!("Analyzing {} / {} frames ({:.0} fps)", pr.done, pr.total, pr.fps),
|
||||
Tokens::ui(11.0),
|
||||
t.text_dim,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- dialogs
|
||||
|
||||
pub fn open_options(app: &mut EffectcraftApp) -> Result<(), String> {
|
||||
let (l, u) = current(app).ok_or("no current track")?;
|
||||
let comp = app.session.active_comp().ok_or("no composition")?;
|
||||
let (g, s) = comp.layer(l).and_then(|l| l.tracker(u)).ok_or("no tracker")?;
|
||||
app.dialog_state.track_options = OptionsDraft { name: g.name.clone(), opts: s.options.clone(), blur_on: s.options.blur > 0.0 };
|
||||
if app.dialog_state.track_options.opts.blur <= 0.0 {
|
||||
app.dialog_state.track_options.opts.blur = 2.0;
|
||||
}
|
||||
app.dialog = Some(Dialog::TrackOptions);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn open_target(app: &mut EffectcraftApp) -> Result<(), String> {
|
||||
let (l, u) = current(app).ok_or("no current track")?;
|
||||
let comp = app.session.active_comp().ok_or("no composition")?;
|
||||
let (_, s) = comp.layer(l).and_then(|l| l.tracker(u)).ok_or("no tracker")?;
|
||||
app.dialog_state.track_target = TargetDraft { target: s.target.map(|l| l.0) };
|
||||
app.dialog = Some(Dialog::TrackTarget);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ok_cancel(ui: &mut egui::Ui, app: &mut EffectcraftApp, t: &Tokens, prefix: &str) -> (bool, bool) {
|
||||
let (mut ok, mut cancel) = (false, false);
|
||||
ui.add_space(14.0);
|
||||
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
|
||||
let r = ui.add(egui::Button::new(egui::RichText::new(" OK ").color(Color32::WHITE)).fill(t.accent));
|
||||
app.auto.add(&format!("{prefix}.ok"), r.rect, "OK");
|
||||
ok = r.clicked();
|
||||
let r = ui.button("Cancel");
|
||||
app.auto.add(&format!("{prefix}.cancel"), r.rect, "Cancel");
|
||||
cancel = r.clicked();
|
||||
});
|
||||
(ok, cancel)
|
||||
}
|
||||
|
||||
pub fn options_dialog(app: &mut EffectcraftApp, ctx: &egui::Context, t: &Tokens) {
|
||||
let mut d = app.dialog_state.track_options.clone();
|
||||
let (mut ok, mut cancel) = (false, false);
|
||||
super::dialogs::modal(ctx, "Motion Tracker Options", vec2(460.0, 470.0), t, |ui| {
|
||||
ui.horizontal(|ui| {
|
||||
ui.label("Track Name:");
|
||||
let r = ui.text_edit_singleline(&mut d.name);
|
||||
app.auto.add("dialog.trackOptions.name", r.rect, "Track Name");
|
||||
});
|
||||
ui.add_space(8.0);
|
||||
ui.label(egui::RichText::new("Channel").font(Tokens::medium(12.5)));
|
||||
ui.horizontal(|ui| {
|
||||
for (c, l) in [(TrackChannel::Rgb, "RGB"), (TrackChannel::Luminance, "Luminance"), (TrackChannel::Saturation, "Saturation")] {
|
||||
let r = ui.radio_value(&mut d.opts.channel, c, l);
|
||||
app.auto.add(&format!("dialog.trackOptions.channel.{}", l.to_ascii_lowercase()), r.rect, l);
|
||||
}
|
||||
});
|
||||
ui.add_space(8.0);
|
||||
ui.label(egui::RichText::new("Process Before Match").font(Tokens::medium(12.5)));
|
||||
ui.horizontal(|ui| {
|
||||
let r = ui.checkbox(&mut d.blur_on, "Blur");
|
||||
app.auto.add("dialog.trackOptions.blurOn", r.rect, "Blur");
|
||||
let r = ui.add_enabled(d.blur_on, egui::DragValue::new(&mut d.opts.blur).speed(0.1).range(0.5..=50.0).suffix(" pixels"));
|
||||
app.auto.add("dialog.trackOptions.blur", r.rect, "Blur pixels");
|
||||
let r = ui.checkbox(&mut d.opts.enhance, "Enhance");
|
||||
app.auto.add("dialog.trackOptions.enhance", r.rect, "Enhance");
|
||||
});
|
||||
ui.add_space(8.0);
|
||||
let r = ui.checkbox(&mut d.opts.subpixel, "Subpixel Positioning");
|
||||
app.auto.add("dialog.trackOptions.subpixel", r.rect, "Subpixel Positioning");
|
||||
let r = ui.checkbox(&mut d.opts.adapt_every_frame, "Adapt Feature On Every Frame");
|
||||
app.auto.add("dialog.trackOptions.adaptEveryFrame", r.rect, "Adapt Feature On Every Frame");
|
||||
let r = ui.checkbox(&mut d.opts.track_shape, "Follow Feature Rotation and Scale");
|
||||
app.auto.add("dialog.trackOptions.trackShape", r.rect, "Follow Feature Rotation and Scale");
|
||||
ui.add_space(8.0);
|
||||
ui.horizontal(|ui| {
|
||||
let actions = [
|
||||
(LowConfidence::Continue, "Continue Tracking"),
|
||||
(LowConfidence::Stop, "Stop Tracking"),
|
||||
(LowConfidence::Extrapolate, "Extrapolate Motion"),
|
||||
(LowConfidence::Adapt, "Adapt Feature"),
|
||||
];
|
||||
let cur = actions.iter().find(|(a, _)| *a == d.opts.action).map(|(_, l)| *l).unwrap_or("Adapt Feature");
|
||||
let r = egui::ComboBox::from_id_salt("track-action")
|
||||
.selected_text(cur)
|
||||
.show_ui(ui, |ui| {
|
||||
for (a, l) in actions {
|
||||
ui.selectable_value(&mut d.opts.action, a, l);
|
||||
}
|
||||
})
|
||||
.response;
|
||||
app.auto.add("dialog.trackOptions.action", r.rect, "If Confidence is Below action");
|
||||
ui.label("If Confidence is Below");
|
||||
let r = ui.add(egui::DragValue::new(&mut d.opts.threshold).speed(0.5).range(0.0..=100.0).suffix(" %"));
|
||||
app.auto.add("dialog.trackOptions.threshold", r.rect, "Confidence threshold");
|
||||
});
|
||||
(ok, cancel) = ok_cancel(ui, app, t, "dialog.trackOptions");
|
||||
});
|
||||
app.dialog_state.track_options = d.clone();
|
||||
if ok || super::dialogs::ui_enter(ctx) {
|
||||
let o = &d.opts;
|
||||
let action = match o.action {
|
||||
LowConfidence::Continue => "continue",
|
||||
LowConfidence::Stop => "stop",
|
||||
LowConfidence::Extrapolate => "extrapolate",
|
||||
LowConfidence::Adapt => "adapt",
|
||||
};
|
||||
let channel = match o.channel {
|
||||
TrackChannel::Rgb => "rgb",
|
||||
TrackChannel::Luminance => "luminance",
|
||||
TrackChannel::Saturation => "saturation",
|
||||
};
|
||||
let p = json!({
|
||||
"name": d.name, "channel": channel, "blur": if d.blur_on { o.blur } else { 0.0 }, "enhance": o.enhance,
|
||||
"subpixel": o.subpixel, "adaptEveryFrame": o.adapt_every_frame, "threshold": o.threshold, "action": action, "trackShape": o.track_shape,
|
||||
});
|
||||
if let Err(e) = app.session.execute("track.options", p) {
|
||||
app.ui.status = e.to_string();
|
||||
}
|
||||
cancel = true;
|
||||
}
|
||||
if cancel {
|
||||
app.dialog = None;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn target_dialog(app: &mut EffectcraftApp, ctx: &egui::Context, t: &Tokens) {
|
||||
let mut d = app.dialog_state.track_target.clone();
|
||||
let (mut ok, mut cancel) = (false, false);
|
||||
let layers: Vec<(u64, String)> =
|
||||
app.session.active_comp().map(|c| c.layers.iter().enumerate().map(|(i, l)| (l.id.0, format!("{}. {}", i + 1, l.name))).collect()).unwrap_or_default();
|
||||
super::dialogs::modal(ctx, "Motion Target", vec2(400.0, 200.0), t, |ui| {
|
||||
ui.label("Apply Motion To:");
|
||||
ui.horizontal(|ui| {
|
||||
ui.label("Layer:");
|
||||
let cur = d.target.and_then(|id| layers.iter().find(|(l, _)| *l == id)).map(|(_, n)| n.clone()).unwrap_or_else(|| "None".into());
|
||||
let r = egui::ComboBox::from_id_salt("track-target")
|
||||
.selected_text(cur)
|
||||
.width(240.0)
|
||||
.show_ui(ui, |ui| {
|
||||
ui.selectable_value(&mut d.target, None, "None");
|
||||
for (id, n) in &layers {
|
||||
ui.selectable_value(&mut d.target, Some(*id), n);
|
||||
}
|
||||
})
|
||||
.response;
|
||||
app.auto.add("dialog.trackTarget.layer", r.rect, "Layer");
|
||||
});
|
||||
(ok, cancel) = ok_cancel(ui, app, t, "dialog.trackTarget");
|
||||
});
|
||||
app.dialog_state.track_target = d.clone();
|
||||
if ok || super::dialogs::ui_enter(ctx) {
|
||||
if let Err(e) = app.session.execute("track.setTarget", json!({"target": d.target})) {
|
||||
app.ui.status = e.to_string();
|
||||
}
|
||||
cancel = true;
|
||||
}
|
||||
if cancel {
|
||||
app.dialog = None;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn apply_dialog(app: &mut EffectcraftApp, ctx: &egui::Context, t: &Tokens) {
|
||||
let mut d = app.dialog_state.track_apply.clone();
|
||||
let (mut ok, mut cancel) = (false, false);
|
||||
super::dialogs::modal(ctx, "Motion Tracker Apply Options", vec2(380.0, 170.0), t, |ui| {
|
||||
ui.horizontal(|ui| {
|
||||
ui.label("Apply Dimensions:");
|
||||
let labels = ["X and Y", "X only", "Y only"];
|
||||
let r = egui::ComboBox::from_id_salt("track-apply-dims")
|
||||
.selected_text(labels[d.dims.min(2)])
|
||||
.show_ui(ui, |ui| {
|
||||
for (i, l) in labels.iter().enumerate() {
|
||||
ui.selectable_value(&mut d.dims, i, *l);
|
||||
}
|
||||
})
|
||||
.response;
|
||||
app.auto.add("dialog.trackApply.dimensions", r.rect, "Apply Dimensions");
|
||||
});
|
||||
(ok, cancel) = ok_cancel(ui, app, t, "dialog.trackApply");
|
||||
});
|
||||
app.dialog_state.track_apply = d.clone();
|
||||
if ok || super::dialogs::ui_enter(ctx) {
|
||||
let dims = ["xy", "x", "y"][d.dims.min(2)];
|
||||
if let Err(e) = app.session.execute("track.apply", json!({"dimensions": dims})) {
|
||||
app.ui.status = e.to_string();
|
||||
}
|
||||
cancel = true;
|
||||
}
|
||||
if cancel {
|
||||
app.dialog = None;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- viewer widgets
|
||||
|
||||
/// Which part of a track point a drag grabbed.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Part {
|
||||
/// Inside the feature region: move the whole point.
|
||||
Point,
|
||||
/// Inside the search region: move only the search region.
|
||||
Search,
|
||||
/// A feature region corner (0..4: UL, UR, LR, LL).
|
||||
FeatureCorner(usize),
|
||||
SearchCorner(usize),
|
||||
Attach,
|
||||
}
|
||||
|
||||
/// A track point widget hit area (screen space).
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Hit {
|
||||
pub point: usize,
|
||||
pub part: Part,
|
||||
pub rect: Rect,
|
||||
/// Screen quad for inside tests (feature / search region).
|
||||
pub quad: Option<[Pos2; 4]>,
|
||||
}
|
||||
|
||||
/// A track point drag in progress.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Drag {
|
||||
pub point: usize,
|
||||
pub part: Part,
|
||||
/// Comp → tracked-layer matrix and the layer-space press point.
|
||||
pub inv: Mat3,
|
||||
pub start: [f64; 2],
|
||||
pub center: [f64; 2],
|
||||
pub feature: [f64; 2],
|
||||
pub search_offset: [f64; 2],
|
||||
pub search: [f64; 2],
|
||||
pub attach: [f64; 2],
|
||||
}
|
||||
|
||||
const TRACK_COL: Color32 = Color32::from_rgb(0xf0, 0xc8, 0x3c);
|
||||
|
||||
fn quad_contains(q: &[Pos2; 4], p: Pos2) -> bool {
|
||||
let mut sign = 0.0f32;
|
||||
for i in 0..4 {
|
||||
let (a, b) = (q[i], q[(i + 1) % 4]);
|
||||
let c = (b.x - a.x) * (p.y - a.y) - (b.y - a.y) * (p.x - a.x);
|
||||
if c.abs() < 1e-6 {
|
||||
continue;
|
||||
}
|
||||
if sign == 0.0 {
|
||||
sign = c.signum();
|
||||
} else if c.signum() != sign {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Draw the current track's points (layer → comp matrix `m`) and return their hit areas, from the
|
||||
/// most to the least specific.
|
||||
pub fn draw_overlay(app: &mut EffectcraftApp, painter: &egui::Painter, map: &ViewerMap, m: &Mat3, layer: &Layer, tracker: Uid, comp_time: Tick) -> Vec<Hit> {
|
||||
let Some((g, _)) = layer.tracker(tracker) else { return vec![] };
|
||||
let lt = layer.layer_time(comp_time);
|
||||
let scr = |p: [f64; 2]| {
|
||||
let c = m.apply(gv2(p[0], p[1]));
|
||||
map.to_screen([c.x, c.y])
|
||||
};
|
||||
let mut hits = vec![];
|
||||
let mut inside = vec![];
|
||||
for (i, tp) in g.track_points().enumerate() {
|
||||
let v = point_values(tp, lt);
|
||||
let c = v.spec.center;
|
||||
let attach = tp
|
||||
.get("attachPoint")
|
||||
.filter(|p| !p.keys.is_empty())
|
||||
.map(|p| p.value_at(lt).as_vec2())
|
||||
.unwrap_or([c[0] + v.attach_offset[0], c[1] + v.attach_offset[1]]);
|
||||
let rect_q = |cx: [f64; 2], s: [f64; 2]| {
|
||||
let (hx, hy) = (s[0] / 2.0, s[1] / 2.0);
|
||||
[scr([cx[0] - hx, cx[1] - hy]), scr([cx[0] + hx, cx[1] - hy]), scr([cx[0] + hx, cx[1] + hy]), scr([cx[0] - hx, cx[1] + hy])]
|
||||
};
|
||||
// Track path (feature centre keys).
|
||||
if let Some(fc) = tp.get("featureCenter")
|
||||
&& fc.keys.len() > 1
|
||||
{
|
||||
let pts: Vec<Pos2> = fc.keys.iter().map(|k| scr(k.value.as_vec2())).collect();
|
||||
painter.add(egui::Shape::line(pts.clone(), Stroke::new(1.0, TRACK_COL.gamma_multiply(0.6))));
|
||||
for q in &pts {
|
||||
painter.circle_filled(*q, 1.6, TRACK_COL);
|
||||
}
|
||||
}
|
||||
let fq = rect_q(c, v.spec.feature_size);
|
||||
let sc = [c[0] + v.spec.search_offset[0], c[1] + v.spec.search_offset[1]];
|
||||
let sq = rect_q(sc, v.spec.search_size);
|
||||
painter.add(egui::Shape::closed_line(sq.to_vec(), Stroke::new(1.0, TRACK_COL.gamma_multiply(0.7))));
|
||||
painter.add(egui::Shape::closed_line(fq.to_vec(), Stroke::new(1.0, TRACK_COL)));
|
||||
painter.circle_filled(scr(c), 2.0, TRACK_COL);
|
||||
let a = scr(attach);
|
||||
for d in [vec2(7.0, 0.0), vec2(0.0, 7.0)] {
|
||||
painter.line_segment([a - d, a + d], Stroke::new(1.4, Color32::BLACK));
|
||||
painter.line_segment([a - d, a + d], Stroke::new(1.0, TRACK_COL));
|
||||
}
|
||||
painter.text(sq[0] + vec2(0.0, -3.0), Align2::LEFT_BOTTOM, &tp.name, Tokens::ui(10.5), TRACK_COL);
|
||||
// Corner handles only when the region is big enough on screen to see inside it.
|
||||
let fbig = fq[0].distance(fq[2]) > 28.0;
|
||||
let sbig = sq[0].distance(sq[2]) > 28.0;
|
||||
let n = i + 1;
|
||||
let hr = Rect::from_center_size(a, vec2(12.0, 12.0));
|
||||
hits.push(Hit { point: n, part: Part::Attach, rect: hr, quad: None });
|
||||
app.auto.add(&format!("viewer.track.{n}.attach"), hr, "Attach Point");
|
||||
for (k, q) in fq.iter().enumerate() {
|
||||
let r = Rect::from_center_size(*q, vec2(5.0, 5.0));
|
||||
if fbig {
|
||||
painter.rect_filled(r, 0.0, TRACK_COL);
|
||||
}
|
||||
hits.push(Hit { point: n, part: Part::FeatureCorner(k), rect: r.expand(2.0), quad: None });
|
||||
app.auto.add(&format!("viewer.track.{n}.feature.{k}"), r, "Feature region corner");
|
||||
}
|
||||
for (k, q) in sq.iter().enumerate() {
|
||||
let r = Rect::from_center_size(*q, vec2(5.0, 5.0));
|
||||
if sbig {
|
||||
painter.rect_stroke(r, 0.0, Stroke::new(1.0, TRACK_COL), StrokeKind::Middle);
|
||||
}
|
||||
hits.push(Hit { point: n, part: Part::SearchCorner(k), rect: r.expand(2.0), quad: None });
|
||||
app.auto.add(&format!("viewer.track.{n}.search.{k}"), r, "Search region corner");
|
||||
}
|
||||
let fr = Rect::from_points(&fq);
|
||||
let sr = Rect::from_points(&sq);
|
||||
app.auto.add(&format!("viewer.track.{n}.feature"), fr, "Feature region");
|
||||
app.auto.add(&format!("viewer.track.{n}.search"), sr, "Search region");
|
||||
inside.push(Hit { point: n, part: Part::Point, rect: fr, quad: Some(fq) });
|
||||
inside.push(Hit { point: n, part: Part::Search, rect: sr, quad: Some(sq) });
|
||||
}
|
||||
// Regions after handles; feature regions before search regions.
|
||||
inside.sort_by_key(|h| h.part == Part::Search);
|
||||
hits.extend(inside);
|
||||
hits
|
||||
}
|
||||
|
||||
/// The hit under `pos`.
|
||||
pub fn hit_at(hits: &[Hit], pos: Pos2) -> Option<Hit> {
|
||||
hits.iter()
|
||||
.find(|h| match h.quad {
|
||||
Some(q) => quad_contains(&q, pos),
|
||||
None => h.rect.contains(pos),
|
||||
})
|
||||
.copied()
|
||||
}
|
||||
|
||||
/// Start dragging `hit` (press at comp point `cpt`).
|
||||
pub fn begin_drag(layer: &Layer, tracker: Uid, comp_time: Tick, l2c: &Mat3, hit: Hit, cpt: [f64; 2]) -> Option<Drag> {
|
||||
let (g, _) = layer.tracker(tracker)?;
|
||||
let tp = g.track_points().nth(hit.point - 1)?;
|
||||
let v = point_values(tp, layer.layer_time(comp_time));
|
||||
let inv = l2c.inverse()?;
|
||||
let s = inv.apply(gv2(cpt[0], cpt[1]));
|
||||
Some(Drag {
|
||||
point: hit.point,
|
||||
part: hit.part,
|
||||
inv,
|
||||
start: [s.x, s.y],
|
||||
center: v.spec.center,
|
||||
feature: v.spec.feature_size,
|
||||
search_offset: v.spec.search_offset,
|
||||
search: v.spec.search_size,
|
||||
attach: v.attach_offset,
|
||||
})
|
||||
}
|
||||
|
||||
/// The `track.setPoint` parameters for the drag at comp point `cpt`.
|
||||
pub fn drag_params(d: &Drag, cpt: [f64; 2]) -> serde_json::Value {
|
||||
let q = d.inv.apply(gv2(cpt[0], cpt[1]));
|
||||
let lp = [q.x, q.y];
|
||||
let dl = [lp[0] - d.start[0], lp[1] - d.start[1]];
|
||||
match d.part {
|
||||
Part::Point => json!({"point": d.point, "center": [d.center[0] + dl[0], d.center[1] + dl[1]]}),
|
||||
Part::Search => json!({"point": d.point, "searchOffset": [d.search_offset[0] + dl[0], d.search_offset[1] + dl[1]]}),
|
||||
Part::Attach => json!({"point": d.point, "attachOffset": [d.attach[0] + dl[0], d.attach[1] + dl[1]]}),
|
||||
Part::FeatureCorner(_) => {
|
||||
let size = [(2.0 * (lp[0] - d.center[0]).abs()).max(4.0), (2.0 * (lp[1] - d.center[1]).abs()).max(4.0)];
|
||||
json!({"point": d.point, "featureSize": size})
|
||||
}
|
||||
Part::SearchCorner(_) => {
|
||||
let sc = [d.center[0] + d.search_offset[0], d.center[1] + d.search_offset[1]];
|
||||
let size = [(2.0 * (lp[0] - sc[0]).abs()).max(4.0), (2.0 * (lp[1] - sc[1]).abs()).max(4.0)];
|
||||
json!({"point": d.point, "searchSize": size})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The tracked layer and tracker the viewer shows widgets for.
|
||||
pub fn viewer_track(app: &EffectcraftApp) -> Option<(LayerId, Uid)> {
|
||||
current(app)
|
||||
}
|
||||
|
|
@ -119,6 +119,12 @@ enum Gesture {
|
|||
inv: Mat3,
|
||||
press: Pos2,
|
||||
},
|
||||
/// Dragging a track point's feature/search region, a corner or its attach point.
|
||||
Track {
|
||||
layer: LayerId,
|
||||
tracker: u64,
|
||||
drag: super::tracker::Drag,
|
||||
},
|
||||
}
|
||||
|
||||
/// Pen tool path in progress: (layer, mask uid).
|
||||
|
|
@ -502,6 +508,19 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
|
|||
}
|
||||
}
|
||||
|
||||
// Track points of the current track (Tracker panel) on its layer, while that layer is selected.
|
||||
let mut track_hits = vec![];
|
||||
let track_cur = super::tracker::viewer_track(app);
|
||||
if app.ui.viewer.show_layer_controls
|
||||
&& let Some((tl, tu)) = track_cur
|
||||
&& selected.contains(&tl)
|
||||
&& let Some(layer) = comp.layer(tl)
|
||||
&& layer.is_active_at(time)
|
||||
{
|
||||
let (m, _) = l2c(&ectx, layer);
|
||||
track_hits = super::tracker::draw_overlay(app, &painter, &map, &m, layer, tu, time);
|
||||
}
|
||||
|
||||
// Cameras and lights (wireframes) and the 3D view name.
|
||||
if comp.has_3d() {
|
||||
draw_rigs(app, &painter, &map, &ectx, &selected);
|
||||
|
|
@ -530,7 +549,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
|
|||
Tool::Dolly => egui::CursorIcon::ResizeVertical,
|
||||
t if t.is_shape() || t == Tool::Pen => egui::CursorIcon::Crosshair,
|
||||
_ => {
|
||||
if handle_hits.iter().any(|(_, _, h)| h.distance(hp) < HANDLE) {
|
||||
if super::tracker::hit_at(&track_hits, hp).is_some() {
|
||||
egui::CursorIcon::Move
|
||||
} else if handle_hits.iter().any(|(_, _, h)| h.distance(hp) < HANDLE) {
|
||||
egui::CursorIcon::ResizeNwSe
|
||||
} else {
|
||||
egui::CursorIcon::Default
|
||||
|
|
@ -610,6 +631,14 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
|
|||
}),
|
||||
t if t.is_shape() => Some(Gesture::Create { tool: t, start: cpt }),
|
||||
Tool::Orbit | Tool::PanCamera | Tool::Dolly => Some(Gesture::Camera { tool }),
|
||||
Tool::Selection if super::tracker::hit_at(&track_hits, press).is_some() => {
|
||||
let hit = super::tracker::hit_at(&track_hits, press);
|
||||
track_cur.zip(hit).and_then(|((tl, tu), hit)| {
|
||||
let layer = comp.layer(tl)?;
|
||||
let (m, _) = l2c(&ectx, layer);
|
||||
super::tracker::begin_drag(layer, tu, time, &m, hit, map.to_comp(press)).map(|drag| Gesture::Track { layer: tl, tracker: tu, drag })
|
||||
})
|
||||
}
|
||||
Tool::Selection if vertex_at(press, true).is_some() => vertex_at(press, true).map(|h| Gesture::Tangent {
|
||||
layer: h.layer,
|
||||
mask: h.mask,
|
||||
|
|
@ -793,6 +822,13 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) {
|
|||
.execute("mask.setVertex", json!({"layer": layer.0, "mask": mask, "index": index, "out": t, "in": [-t[0], -t[1]], "merge": merge}));
|
||||
}
|
||||
}
|
||||
Gesture::Track { layer, tracker, drag } => {
|
||||
let mut q = super::tracker::drag_params(&drag, cpt);
|
||||
q["layer"] = json!(layer.0);
|
||||
q["tracker"] = json!(tracker);
|
||||
q["merge"] = json!(merge);
|
||||
let _ = app.session.execute("track.setPoint", q);
|
||||
}
|
||||
Gesture::Marquee { start } => {
|
||||
let r = Rect::from_two_pos(start, pos);
|
||||
painter.rect_filled(r, 0.0, t.accent.gamma_multiply(0.12));
|
||||
|
|
|
|||
|
|
@ -412,6 +412,9 @@ pub struct UiState {
|
|||
pub start_screen: bool,
|
||||
/// Composition ▸ Preview ▸ Cache Frames When Idle.
|
||||
pub cache_when_idle: bool,
|
||||
/// Tracker panel ▸ Motion Source chosen without a tracker yet (layer id).
|
||||
#[serde(default)]
|
||||
pub tracker_source: Option<u64>,
|
||||
}
|
||||
|
||||
impl Default for UiState {
|
||||
|
|
@ -452,6 +455,7 @@ impl Default for UiState {
|
|||
preview_audio: true,
|
||||
start_screen: false,
|
||||
cache_when_idle: false,
|
||||
tracker_source: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,117 @@
|
|||
//! Headless checks for the Tracker panel and the viewer's track point widgets (egui_kittest, UI
|
||||
//! logic only: no GPU needed).
|
||||
|
||||
use effectcraft_engine::Session;
|
||||
use effectcraft_ui_egui::EffectcraftApp;
|
||||
use effectcraft_ui_egui::dock::PanelKind;
|
||||
use egui_kittest::Harness;
|
||||
use serde_json::json;
|
||||
|
||||
fn app() -> EffectcraftApp {
|
||||
let mut s = Session::default();
|
||||
s.execute("comp.new", json!({"name": "Track", "width": 640, "height": 360, "duration": 2})).unwrap();
|
||||
s.execute("layer.newSolid", json!({"name": "Plate", "color": "#406080"})).unwrap();
|
||||
s.execute("layer.newSolid", json!({"name": "Target", "width": 80, "height": 80})).unwrap();
|
||||
s.execute("layer.select", json!({"layers": ["Plate"]})).unwrap();
|
||||
EffectcraftApp::new(s)
|
||||
}
|
||||
|
||||
fn ids(h: &Harness<'_, EffectcraftApp>) -> Vec<String> {
|
||||
h.state().auto.previous.iter().chain(h.state().auto.elements.iter()).map(|e| e.id.clone()).collect()
|
||||
}
|
||||
|
||||
fn rect(h: &Harness<'_, EffectcraftApp>, id: &str) -> egui::Rect {
|
||||
let e = h.state().auto.find(id).unwrap_or_else(|| panic!("no {id}"));
|
||||
egui::Rect::from_min_size(egui::pos2(e.rect[0], e.rect[1]), egui::vec2(e.rect[2], e.rect[3]))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tracker_panel_and_track_point_widgets() {
|
||||
let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).build_eframe(|_| app());
|
||||
h.state_mut().show_panel(PanelKind::Tracker);
|
||||
h.run_steps(3);
|
||||
let all = ids(&h);
|
||||
for want in [
|
||||
"tracker.trackCamera",
|
||||
"tracker.warpStabilizer",
|
||||
"tracker.trackMotion",
|
||||
"tracker.stabilizeMotion",
|
||||
"tracker.motionSource",
|
||||
"tracker.currentTrack",
|
||||
"tracker.trackType",
|
||||
"tracker.position",
|
||||
"tracker.rotation",
|
||||
"tracker.scale",
|
||||
"tracker.motionTarget",
|
||||
"tracker.editTarget",
|
||||
"tracker.options",
|
||||
"tracker.analyze.frameBackward",
|
||||
"tracker.analyze.backward",
|
||||
"tracker.analyze.forward",
|
||||
"tracker.analyze.frameForward",
|
||||
"tracker.reset",
|
||||
"tracker.apply",
|
||||
] {
|
||||
assert!(all.iter().any(|i| i == want), "missing {want}");
|
||||
}
|
||||
// Track Motion from the panel.
|
||||
let r = rect(&h, "tracker.trackMotion");
|
||||
h.input_mut().events.push(egui::Event::PointerMoved(r.center()));
|
||||
h.input_mut().events.push(egui::Event::PointerButton {
|
||||
pos: r.center(),
|
||||
button: egui::PointerButton::Primary,
|
||||
pressed: true,
|
||||
modifiers: Default::default(),
|
||||
});
|
||||
h.input_mut().events.push(egui::Event::PointerButton {
|
||||
pos: r.center(),
|
||||
button: egui::PointerButton::Primary,
|
||||
pressed: false,
|
||||
modifiers: Default::default(),
|
||||
});
|
||||
h.run_steps(3);
|
||||
let st = h.state_mut().session.execute("track.status", json!({})).unwrap();
|
||||
assert_eq!(st["tracker"]["name"], "Tracker 1", "{st}");
|
||||
h.state_mut().session.execute("track.setPoint", json!({"point": 1, "featureSize": [60, 60], "searchSize": [120, 120]})).unwrap();
|
||||
h.state_mut().show_panel(PanelKind::Composition);
|
||||
h.run_steps(3);
|
||||
let all = ids(&h);
|
||||
for want in ["viewer.track.1.feature", "viewer.track.1.search", "viewer.track.1.attach", "viewer.track.1.feature.0", "viewer.track.1.search.2"] {
|
||||
assert!(all.iter().any(|i| i == want), "missing {want}");
|
||||
}
|
||||
// Drag the feature region: the point moves by the drag in layer pixels (one undo step).
|
||||
let c0 = st["tracker"]["points"][0]["center"].clone();
|
||||
let fr = rect(&h, "viewer.track.1.feature");
|
||||
let zoom = {
|
||||
let a = rect(&h, "viewer.comp");
|
||||
a.width() / 640.0
|
||||
};
|
||||
// Inside the feature region, away from the attach point crosshair (which sits on the centre).
|
||||
let from = fr.center() + egui::vec2(15.0, 0.0);
|
||||
let to = from + egui::vec2(40.0, 20.0);
|
||||
h.input_mut().events.push(egui::Event::PointerMoved(from));
|
||||
h.input_mut().events.push(egui::Event::PointerButton { pos: from, button: egui::PointerButton::Primary, pressed: true, modifiers: Default::default() });
|
||||
h.step();
|
||||
for i in 1..=8 {
|
||||
h.input_mut().events.push(egui::Event::PointerMoved(from + (to - from) * (i as f32 / 8.0)));
|
||||
h.step();
|
||||
}
|
||||
h.input_mut().events.push(egui::Event::PointerButton { pos: to, button: egui::PointerButton::Primary, pressed: false, modifiers: Default::default() });
|
||||
h.run_steps(2);
|
||||
let st = h.state_mut().session.execute("track.status", json!({})).unwrap();
|
||||
let c1 = &st["tracker"]["points"][0]["center"];
|
||||
let dx = c1[0].as_f64().unwrap() - c0[0].as_f64().unwrap();
|
||||
let dy = c1[1].as_f64().unwrap() - c0[1].as_f64().unwrap();
|
||||
assert!((dx - 40.0 / zoom as f64).abs() < 1.0 && (dy - 20.0 / zoom as f64).abs() < 1.0, "moved by ({dx}, {dy}) at zoom {zoom}");
|
||||
let undo = h.state().session.history.undo.iter().filter(|(l, _)| l == "Move Track Point").count();
|
||||
assert_eq!(undo, 2, "one undo step per drag");
|
||||
// Options and Edit Target dialogs.
|
||||
let ctx = h.ctx.clone();
|
||||
effectcraft_ui_egui::menus::invoke(h.state_mut(), &ctx, "track.optionsDialog", json!({})).unwrap();
|
||||
assert_eq!(h.state().dialog, Some(effectcraft_ui_egui::Dialog::TrackOptions));
|
||||
h.step();
|
||||
assert!(ids(&h).iter().any(|i| i == "dialog.trackOptions.ok"));
|
||||
h.state_mut().dialog = None;
|
||||
effectcraft_ui_egui::menus::invoke(h.state_mut(), &ctx, "track.editTargetDialog", json!({})).unwrap();
|
||||
assert_eq!(h.state().dialog, Some(effectcraft_ui_egui::Dialog::TrackTarget));
|
||||
}
|
||||
|
|
@ -77,6 +77,29 @@ instead of being ignored.
|
|||
4. `execute_command {"command":"effect.apply","params":{"layer":2,"effect":"Gaussian Blur"}}`, then `set_property {"layer":2,"path":"effects/#1/blurriness","value":8}`
|
||||
5. `render_frame {"time":1.0}` lets you look at the result, and `save_project {"path":"intro.ecproj"}` saves it.
|
||||
|
||||
### Motion tracking
|
||||
|
||||
Trackers live on the tracked layer (`Motion Trackers ▸ Tracker n ▸ Track Point n`) and are driven
|
||||
with `track.*` commands. `track.analyze` runs in the background in the app (poll `track.status`);
|
||||
pass `"wait": true` to block until it finishes, which is what the CLI and headless MCP want.
|
||||
|
||||
1. `execute_command {"command":"track.motion","params":{"layer":"clip.mov"}}` creates a Transform
|
||||
tracker (Motion Target: the layer above). Use `track.stabilize` for Stabilize, or `track.new` with
|
||||
`kind` `transform|stabilize|affine|perspective|raw` and `rotation`/`scale`.
|
||||
2. `execute_command {"command":"track.setPoint","params":{"point":1,"center":[812,440],"featureSize":[40,40],"searchSize":[96,96]}}`
|
||||
3. `execute_command {"command":"track.analyze","params":{"direction":"forward","wait":true}}` keys Feature
|
||||
Center, Confidence and Attach Point on every frame from the current time to the layer's end.
|
||||
4. `execute_command {"command":"track.apply","params":{"dimensions":"xy"}}` keys the target's Position
|
||||
(and Rotation/Scale), the layer's own Anchor Point and Position for Stabilize, or a Corner Pin
|
||||
effect for `affine`/`perspective` tracks. `track.options` sets the channel, blur/enhance,
|
||||
adapt-feature and "If Confidence is Below" behaviour; `track.status` reports the track points.
|
||||
|
||||
```sh
|
||||
effectcraft-cli run clip.ecproj track.motion '{"layer":"#2"}' \
|
||||
track.setPoint '{"point":1,"center":[812,440]}' \
|
||||
track.analyze '{"wait":true}' track.apply '{}' --save
|
||||
```
|
||||
|
||||
## CLI
|
||||
|
||||
Each invocation runs a headless engine with no window. It opens the demo project unless you pass
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for
|
|||
| L2 | `project` | The document: items, compositions, layers, the property tree, render queue model, `.ecproj` serde |
|
||||
| L2 | `text` | Fonts, shaping, layout, per-glyph geometry, text animators and selectors |
|
||||
| L2 | `effects` | The effect registry (241 effects) and their CPU implementations |
|
||||
| L2 | `track` | Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation, Lucas–Kanade sub-pixel refinement), confidence, homography/affine/similarity solves |
|
||||
| L3 | `render` | Evaluation and compositing: sources, masks, effects, transforms, 3D, motion blur, mattes, blending, layer cache, audio mixdown |
|
||||
| L3 | `media` | Footage decoding (FilmCraft's pure-Rust codecs), image sequences, frame cache |
|
||||
| L3 | `expr` | The expression engine (JavaScript via boa) with the After Effects object model |
|
||||
|
|
@ -92,6 +93,15 @@ layer's Blending Options and kept in step after each edit.
|
|||
|
||||
Half, Third and Quarter resolution render proportionally fewer pixels end to end.
|
||||
|
||||
**Motion tracking** (Animation ▸ Track Motion / Stabilize Motion, Window ▸ Tracker): a tracker
|
||||
is a `Tracker` group under the layer's Motion Trackers group, with Track Point groups (Feature
|
||||
Center, Feature Size, Search Offset, Search Size, Confidence, Attach Point, Attach Point Offset)
|
||||
keyframed per analysed frame. `track.analyze` renders the layer's source frames
|
||||
(`Renderer::layer_source`) on a background thread (`engine::tracking`, polled like the render
|
||||
queue; blocking with `wait`), and `effectcraft-track` matches each point in parallel. One undo step
|
||||
covers an analysis. `track.apply` keys the target's Position/Rotation/Scale, the tracked layer's
|
||||
Anchor Point and Position (Stabilize), or a Corner Pin effect (Parallel / Perspective).
|
||||
|
||||
## 5. Expressions
|
||||
|
||||
Expressions are JavaScript, run by boa, with After Effects' object model (`thisComp`, `thisLayer`,
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ const LAYERS: &[(&str, u8)] = &[
|
|||
("project", 2),
|
||||
("text", 2),
|
||||
("effects", 2),
|
||||
("track", 2),
|
||||
("render", 3),
|
||||
("media", 3),
|
||||
("expr", 3),
|
||||
|
|
|
|||
Loading…
Reference in New Issue