M13.3: face tracking methods, Face Track Points / Measurements, camera lens distortion, docs
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Brandon Thomas
parent
0bce3f2e49
commit
d209fa001f
|
|
@ -8,8 +8,12 @@
|
|||
//! Shot Type, Angle of View, Solve Method and deleted points it was solved with, so changing
|
||||
//! those re-solves without re-tracking). **Method Used** and **Average Error** report the solve.
|
||||
//!
|
||||
//! The effect passes its input through; with **Render Track Points** on it draws the solved
|
||||
//! points (as the viewer shows them) into the frame. The viewer overlay, target and the Create
|
||||
//! **Solve Lens Distortion** (Advanced; also on with Detailed Analysis) solves radial lens
|
||||
//! distortion `k1, k2` jointly with the camera; **Undistort Footage** then renders the input
|
||||
//! undistorted (the pinhole image the solved camera sees) so 3D layers line up with it.
|
||||
//!
|
||||
//! The effect passes its input through (undistorted when asked); with **Render Track Points**
|
||||
//! on it draws the solved points (as the viewer shows them) into the frame. The viewer overlay, target and the Create
|
||||
//! commands live in the engine and the UI.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
|
@ -19,6 +23,7 @@ use effectcraft_keyframe::Value;
|
|||
use effectcraft_project::ParamUi;
|
||||
use effectcraft_raster::{Image, Px};
|
||||
use effectcraft_track::camtrack::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, linalg, point_color};
|
||||
use rayon::prelude::*;
|
||||
|
||||
use crate::warp_stab::param;
|
||||
use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider};
|
||||
|
|
@ -58,6 +63,8 @@ pub fn specs() -> Vec<EffectSpec> {
|
|||
p(METHOD_USED, "Method Used", Value::Str(String::new()), ParamUi::Hidden),
|
||||
p(AVERAGE_ERROR, "Average Error", num(0.0), ParamUi::Hidden),
|
||||
p("advanced/detailedAnalysis", "Detailed Analysis", Value::Bool(false), ParamUi::Checkbox),
|
||||
p("advanced/lensDistortion", "Solve Lens Distortion", Value::Bool(false), ParamUi::Checkbox),
|
||||
p("advanced/undistort", "Undistort Footage", Value::Bool(false), ParamUi::Checkbox),
|
||||
p("advanced/autoDeletePoints", "Auto-delete Points Across Time", Value::Bool(true), ParamUi::Checkbox),
|
||||
p("advanced/hideWarningBanner", "Hide Warning Banner", Value::Bool(false), ParamUi::Checkbox),
|
||||
p(TRACKS, "Tracks", Value::Str(String::new()), ParamUi::Hidden),
|
||||
|
|
@ -104,13 +111,15 @@ pub fn settings(params: &Params) -> SolveSettings {
|
|||
hfov: f(params, "horizontalAngleOfView"),
|
||||
method: SolveMethod::ALL.get(e(params, "advanced/solveMethod") as usize).copied().unwrap_or_default(),
|
||||
deleted: del,
|
||||
lens_distortion: b(params, "advanced/lensDistortion") || b(params, "advanced/detailedAnalysis"),
|
||||
}
|
||||
}
|
||||
|
||||
/// What a solve depends on besides the tracks.
|
||||
pub fn solve_key(st: &SolveSettings) -> String {
|
||||
let hfov = if st.shot == ShotType::SpecifyAngle { format!("{:.4}", st.hfov) } else { String::new() };
|
||||
format!("{:?}|{hfov}|{:?}|{:?}", st.shot, st.method, st.deleted)
|
||||
let lens = if st.lens_distortion { "|lens" } else { "" };
|
||||
format!("{:?}|{hfov}|{:?}|{:?}{lens}", st.shot, st.method, st.deleted)
|
||||
}
|
||||
|
||||
fn fnv(bytes: &[u8]) -> u64 {
|
||||
|
|
@ -165,18 +174,49 @@ pub fn target_radius(size_pct: f64, z: f64, median_z: f64) -> f64 {
|
|||
(6.0 * size_pct / 100.0 * (median_z / z.max(1e-9))).clamp(1.5, 40.0)
|
||||
}
|
||||
|
||||
/// The solved points visible on frame `k`: (id, layer pixel position, camera depth).
|
||||
/// The solved points visible on frame `k`: (id, layer pixel position in the footage, camera
|
||||
/// depth).
|
||||
pub fn projected(solve: &CameraSolve, k: usize) -> Vec<(u32, [f64; 2], f64)> {
|
||||
projected_with(solve, k, false)
|
||||
}
|
||||
|
||||
/// [`projected`] into the footage, or (`undistorted`) into the undistorted image.
|
||||
pub fn projected_with(solve: &CameraSolve, k: usize, undistorted: bool) -> Vec<(u32, [f64; 2], f64)> {
|
||||
let Some(cam) = solve.frames.get(k) else { return vec![] };
|
||||
solve
|
||||
.visible(k)
|
||||
.filter_map(|p| {
|
||||
let c = cam.to_cam(p.pos);
|
||||
cam.project(solve.size, p.pos).map(|q| (p.id, q, c[2]))
|
||||
solve.project(k, p.pos, undistorted).map(|q| (p.id, q, c[2]))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Whether the instance renders its input undistorted (Undistort Footage with a solved
|
||||
/// distortion).
|
||||
pub fn undistorts(params: &Params, solve: &CameraSolve) -> bool {
|
||||
b(params, "advanced/undistort") && solve.distortion.is_some_and(|d| !d.is_none())
|
||||
}
|
||||
|
||||
/// The input resampled through the solved distortion: every output pixel shows the footage
|
||||
/// at its distorted position (transparent where that falls outside the layer).
|
||||
fn undistort(buf: &Buf, sol: &CameraSolve) -> Buf {
|
||||
let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 };
|
||||
let mut out = Image::new(buf.img.width, buf.img.height);
|
||||
let size = sol.size;
|
||||
out.rows_mut().for_each(|(y, row)| {
|
||||
for (x, px) in row.iter_mut().enumerate() {
|
||||
let p = [(x as f64 + 0.5 - buf.offset[0]) / sc, (y as f64 + 0.5 - buf.offset[1]) / sc];
|
||||
let q = sol.to_image(p);
|
||||
if q[0] < 0.0 || q[1] < 0.0 || q[0] > size[0] || q[1] > size[1] {
|
||||
continue;
|
||||
}
|
||||
*px = buf.img.sample_bilinear_clamped(q[0] * sc + buf.offset[0], q[1] * sc + buf.offset[1]);
|
||||
}
|
||||
});
|
||||
Buf { img: out, offset: buf.offset, scale: buf.scale }
|
||||
}
|
||||
|
||||
/// Median of the depths.
|
||||
pub fn median_depth(v: &[(u32, [f64; 2], f64)]) -> f64 {
|
||||
let mut z: Vec<f64> = v.iter().map(|x| x.2).collect();
|
||||
|
|
@ -188,15 +228,23 @@ pub fn median_depth(v: &[(u32, [f64; 2], f64)]) -> f64 {
|
|||
}
|
||||
|
||||
fn render(ctx: &EffectCtx, mut buf: Buf) -> Buf {
|
||||
if !b(ctx.params, "renderTrackPoints") {
|
||||
let points = b(ctx.params, "renderTrackPoints");
|
||||
if !points && !b(ctx.params, "advanced/undistort") {
|
||||
return buf;
|
||||
}
|
||||
let Some(sol) = solve(ctx.params) else { return buf };
|
||||
if (sol.size[0] - ctx.layer_size[0]).abs() > 0.5 || (sol.size[1] - ctx.layer_size[1]).abs() > 0.5 {
|
||||
return buf;
|
||||
}
|
||||
let und = undistorts(ctx.params, &sol);
|
||||
if und {
|
||||
buf = undistort(&buf, &sol);
|
||||
}
|
||||
if !points {
|
||||
return buf;
|
||||
}
|
||||
let Some(k) = sol.frame_at(ctx.time) else { return buf };
|
||||
let pts = projected(&sol, k);
|
||||
let pts = projected_with(&sol, k, und);
|
||||
let med = median_depth(&pts);
|
||||
let size = f(ctx.params, "trackPointSize");
|
||||
let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 };
|
||||
|
|
@ -287,5 +335,37 @@ mod tests {
|
|||
let st = settings(&Params::default());
|
||||
assert_eq!(st.shot, ShotType::FixedAngle);
|
||||
assert!(solve_key(&st).contains("FixedAngle"));
|
||||
assert!(!st.lens_distortion && !solve_key(&st).contains("lens"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn undistort_footage_remaps_through_the_solved_lens() {
|
||||
use effectcraft_track::camtrack::Distortion;
|
||||
let sol = CameraSolve {
|
||||
version: 1,
|
||||
size: [64.0, 48.0],
|
||||
start: 0.0,
|
||||
frame_duration: 1.0 / 30.0,
|
||||
frames: vec![SolvedFrame { rot: [0.0; 3], center: [0.0; 3], focal: 50.0, solved: true }],
|
||||
distortion: Some(Distortion { k1: -0.2, k2: 0.0, radius: 40.0 }),
|
||||
..Default::default()
|
||||
};
|
||||
let mut params = params_with(&sol, false);
|
||||
params.values.insert("advanced/undistort".into(), Value::Bool(true));
|
||||
assert!(undistorts(¶ms, &sol));
|
||||
// A horizontal ramp: undistortion keeps the centre and pulls edge content outwards.
|
||||
let mut img = Image::new(64, 48);
|
||||
for (i, px) in img.data.iter_mut().enumerate() {
|
||||
let x = (i % 64) as f32 / 64.0;
|
||||
*px = [x, x, x, 1.0];
|
||||
}
|
||||
let ctx = EffectCtx { params: ¶ms, time: 0.0, layer_size: [64.0, 48.0], seed: 0, adjustment: false, env: EffectEnv::default() };
|
||||
let out = render(&ctx, Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 });
|
||||
let at = |im: &Image, x: u32, y: u32| im.data[im.idx(x, y)][0];
|
||||
assert!((at(&out.img, 32, 24) - at(&img, 32, 24)).abs() < 0.02);
|
||||
// Barrel distortion compresses the edges: the ideal pixel at x = 56 shows footage nearer the centre.
|
||||
assert!(at(&out.img, 56, 24) < at(&img, 56, 24) - 0.01, "{} vs {}", at(&out.img, 56, 24), at(&img, 56, 24));
|
||||
let sp = sol.to_image([56.0, 24.0]);
|
||||
assert!(sp[0] < 56.0 && (sol.to_ideal(sp)[0] - 56.0).abs() < 1e-6);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,45 @@
|
|||
//! Face tracking results (Tracker panel ▸ Method ▸ Face Tracking (Detailed Features)):
|
||||
//!
|
||||
//! - **Face Track Points** holds one point control per facial landmark (eyebrows, eyes and
|
||||
//! pupils, nose, nostrils, mouth, chin and jaw), keyed on every tracked frame by the engine's
|
||||
//! mask tracker ([`effectcraft_track::face`]); expressions and other layers can follow them.
|
||||
//! - **Face Measurements** holds the measurements derived from those points (head position,
|
||||
//! scale and orientation, eye openness, eyebrow raise, mouth openness, width and offset), keyed
|
||||
//! by Extract & Copy Face Measurements (`track.extractFaceMeasurements`).
|
||||
//!
|
||||
//! Both pass their input through unchanged: they are data, like expression controls.
|
||||
|
||||
use effectcraft_keyframe::Value;
|
||||
use effectcraft_project::ParamUi;
|
||||
use effectcraft_track::face::{LANDMARKS, MEASUREMENTS};
|
||||
|
||||
use crate::{Buf, EffectCtx, EffectSpec, ParamSpec, num, p, slider};
|
||||
|
||||
/// Effect ids.
|
||||
pub const POINTS_ID: &str = "ec.utility.facetrackpoints";
|
||||
pub const MEASUREMENTS_ID: &str = "ec.utility.facemeasurements";
|
||||
|
||||
fn passthrough(_: &EffectCtx, b: Buf) -> Buf {
|
||||
b
|
||||
}
|
||||
|
||||
fn measurement_ui(id: &str) -> ParamUi {
|
||||
match id {
|
||||
"headPositionX" | "headPositionY" => slider(-100_000.0, 100_000.0, 0.0, 1920.0, 1),
|
||||
"headOrientationX" | "headOrientationY" | "headOrientationZ" => slider(-180.0, 180.0, -90.0, 90.0, 1),
|
||||
_ => slider(-100_000.0, 100_000.0, 0.0, 200.0, 1),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn specs() -> Vec<EffectSpec> {
|
||||
let points: Vec<ParamSpec> = LANDMARKS
|
||||
.iter()
|
||||
// Point defaults are fractions of the layer size: the canonical face in the middle.
|
||||
.map(|(id, name, q)| p(id, name, Value::Vec2([0.5 + 0.25 * q[0], 0.5 + 0.25 * q[1]]), ParamUi::Point))
|
||||
.collect();
|
||||
let measurements: Vec<ParamSpec> = MEASUREMENTS.iter().map(|(id, name)| p(id, name, num(0.0), measurement_ui(id))).collect();
|
||||
vec![
|
||||
EffectSpec { id: POINTS_ID, name: "Face Track Points", category: "Utility", params: points, render: passthrough, gpu: false, float: true },
|
||||
EffectSpec { id: MEASUREMENTS_ID, name: "Face Measurements", category: "Utility", params: measurements, render: passthrough, gpu: false, float: true },
|
||||
]
|
||||
}
|
||||
|
|
@ -22,6 +22,7 @@ mod distort;
|
|||
mod distort2;
|
||||
mod distort3;
|
||||
pub mod distort4;
|
||||
pub mod face_track;
|
||||
mod generate;
|
||||
mod generate2;
|
||||
mod generate3;
|
||||
|
|
@ -412,6 +413,7 @@ pub fn registry() -> &'static [EffectSpec] {
|
|||
v.extend(puppet::specs());
|
||||
v.extend(warp_stab::specs());
|
||||
v.extend(camera_tracker::specs());
|
||||
v.extend(face_track::specs());
|
||||
v.extend(roto::specs());
|
||||
v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name)));
|
||||
for s in v.iter_mut() {
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ pub fn specs() -> Vec<CommandSpec> {
|
|||
"Track Camera",
|
||||
["Animation"],
|
||||
None,
|
||||
"{layer?, shotType?: fixed|variable|specify, aov?: degrees, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}",
|
||||
"{layer?, shotType?: fixed|variable|specify, aov?: degrees, solveMethod?: auto|typical|flat|tripod, detailed?, lensDistortion?: solve k1/k2, undistort?: render undistorted, wait?}",
|
||||
can_track,
|
||||
track_camera
|
||||
),
|
||||
|
|
@ -140,6 +140,8 @@ fn track_camera(s: &mut Session, p: &Value) -> Result<Value> {
|
|||
return Err(bad("track.camera", "aov: 1..170 degrees"));
|
||||
}
|
||||
let detailed = b_p(p, "detailed");
|
||||
let lens = b_p(p, "lensDistortion");
|
||||
let undistort = b_p(p, "undistort");
|
||||
let existing = s.project.comp(cid).and_then(|c| c.layer(lid)).and_then(Layer::effects).and_then(|fx| fx.groups().find(|g| is_tracker(g)).map(|g| g.uid));
|
||||
let uid = match existing {
|
||||
Some(u) => u,
|
||||
|
|
@ -148,7 +150,7 @@ fn track_camera(s: &mut Session, p: &Value) -> Result<Value> {
|
|||
r["effects"].get(0).and_then(Value::as_u64).ok_or_else(|| EngineError::Other("the effect could not be applied".into()))?
|
||||
}
|
||||
};
|
||||
if shot.is_some() || method.is_some() || aov.is_some() || detailed.is_some() {
|
||||
if shot.is_some() || method.is_some() || aov.is_some() || detailed.is_some() || lens.is_some() || undistort.is_some() {
|
||||
s.edit("3D Camera Tracker Settings", None, |proj, _| {
|
||||
let g = proj
|
||||
.comp_mut(cid)
|
||||
|
|
@ -176,6 +178,12 @@ fn track_camera(s: &mut Session, p: &Value) -> Result<Value> {
|
|||
if let Some(d) = detailed {
|
||||
set("advanced/detailedAnalysis", KV::Bool(d));
|
||||
}
|
||||
if let Some(d) = lens {
|
||||
set("advanced/lensDistortion", KV::Bool(d));
|
||||
}
|
||||
if let Some(d) = undistort {
|
||||
set("advanced/undistort", KV::Bool(d));
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
}
|
||||
|
|
@ -222,6 +230,7 @@ fn status(s: &mut Session, p: &Value) -> Result<Value> {
|
|||
out["shotType"] = json!(settings.shot.label());
|
||||
out["solveMethod"] = json!(settings.method.label());
|
||||
out["deleted"] = json!(settings.deleted.len());
|
||||
out["lensDistortionSolved"] = json!(settings.lens_distortion);
|
||||
let solve = ct::solve(¶ms);
|
||||
out["solved"] = json!(solve.is_some());
|
||||
if let Some(sv) = &solve {
|
||||
|
|
@ -233,6 +242,11 @@ fn status(s: &mut Session, p: &Value) -> Result<Value> {
|
|||
out["focalLength"] = json!(f);
|
||||
out["horizontalAngleOfView"] = json!(sv.hfov(f));
|
||||
out["groundPlane"] = json!(sv.ground.is_some());
|
||||
out["lensDistortion"] = match &sv.distortion {
|
||||
Some(d) => json!({"k1": d.k1, "k2": d.k2, "radius": d.radius}),
|
||||
None => Value::Null,
|
||||
};
|
||||
out["undistort"] = json!(ct::undistorts(¶ms, sv));
|
||||
if let Some(pl) = placed(&s.project, cid, lid, uid) {
|
||||
let k = pl.frame_at(s.time_of(cid));
|
||||
let (pos, ori, zoom) = pl.camera(k);
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
//! Mask tracking (Layer ▸ Mask ▸ Track Mask, the Tracker panel in mask mode) and Smart Mask
|
||||
//! Interpolation (Window ▸ Mask Interpolation).
|
||||
//! Mask tracking (Layer ▸ Mask ▸ Track Mask, the Tracker panel in mask mode, including the Face
|
||||
//! Tracking methods and Extract & Copy Face Measurements) and Smart Mask Interpolation
|
||||
//! (Window ▸ Mask Interpolation).
|
||||
|
||||
use effectcraft_keyframe::{Keyframe, ShapePath, Value as KV};
|
||||
use effectcraft_path::interp::{self, AddVertices, InterpOpts, Matching};
|
||||
|
|
@ -20,7 +21,7 @@ pub fn specs() -> Vec<CommandSpec> {
|
|||
"Track Mask",
|
||||
["Animation"],
|
||||
None,
|
||||
"{layer?, mask?: uid|index|name, method?: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective, direction?: forward|backward|frameForward|frameBackward, start? (s), end? (s), wait?: block until done}",
|
||||
"{layer?, mask?: uid|index|name, method?: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective|faceOutline|faceDetailed, direction?: forward|backward|frameForward|frameBackward, start? (s), end? (s), wait?: block until done}",
|
||||
has_mask,
|
||||
track_mask
|
||||
),
|
||||
|
|
@ -29,10 +30,19 @@ pub fn specs() -> Vec<CommandSpec> {
|
|||
"Mask Tracking Method",
|
||||
[],
|
||||
None,
|
||||
"{method: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective}",
|
||||
"{method: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective|faceOutline|faceDetailed}",
|
||||
super::always,
|
||||
mask_method
|
||||
),
|
||||
cmd!(
|
||||
"track.extractFaceMeasurements",
|
||||
"Extract & Copy Face Measurements",
|
||||
[],
|
||||
None,
|
||||
"{layer?} → keys a Face Measurements effect from the layer's Face Track Points (one key per tracked frame) and copies those keys",
|
||||
has_face_points,
|
||||
extract_face
|
||||
),
|
||||
cmd!(
|
||||
"mask.interpolate",
|
||||
"Apply Mask Interpolation",
|
||||
|
|
@ -116,9 +126,12 @@ fn has_mask(s: &Session) -> std::result::Result<(), String> {
|
|||
|
||||
fn method_p(p: &Value, cmd: &str) -> Result<Option<MaskMethod>> {
|
||||
match str_p(p, "method") {
|
||||
Some(m) => MaskMethod::from_name(m)
|
||||
.map(Some)
|
||||
.ok_or_else(|| bad(cmd, format!("unknown method `{m}` (position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective)"))),
|
||||
Some(m) => MaskMethod::from_name(m).map(Some).ok_or_else(|| {
|
||||
bad(
|
||||
cmd,
|
||||
format!("unknown method `{m}` (position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective|faceOutline|faceDetailed)"),
|
||||
)
|
||||
}),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
|
@ -190,6 +203,112 @@ fn track_mask(s: &mut Session, p: &Value) -> Result<Value> {
|
|||
}))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- face measurements
|
||||
|
||||
fn face_points_layer(s: &Session, p: &Value) -> Option<(effectcraft_project::ItemId, LayerId)> {
|
||||
use effectcraft_effects::face_track::POINTS_ID;
|
||||
let has = |l: &Layer| l.effects().is_some_and(|fx| fx.groups().any(|g| g.match_id == POINTS_ID));
|
||||
if p.get("layer").is_some() {
|
||||
return layer_p(s, p, "track.extractFaceMeasurements").ok();
|
||||
}
|
||||
let cid = s.active_comp_id()?;
|
||||
let comp = s.project.comp(cid)?;
|
||||
let sel = s.state.selected_layers.iter().filter_map(|l| comp.layer(*l)).find(|l| has(l));
|
||||
sel.or_else(|| comp.layers.iter().find(|l| has(l))).map(|l| (cid, l.id))
|
||||
}
|
||||
|
||||
fn has_face_points(s: &Session) -> std::result::Result<(), String> {
|
||||
has_comp(s)?;
|
||||
face_points_layer(s, &json!({})).map(|_| ()).ok_or_else(|| "no Face Track Points: track a mask with Face Tracking (Detailed Features) first".into())
|
||||
}
|
||||
|
||||
/// Extract & Copy Face Measurements: measure the Face Track Points at every keyed time, key them
|
||||
/// into the layer's Face Measurements effect and put those keys on the keyframe clipboard.
|
||||
fn extract_face(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
use effectcraft_effects::face_track::{MEASUREMENTS_ID, POINTS_ID};
|
||||
use effectcraft_track::face::{LANDMARKS, MEASUREMENTS, N, face_frame, measure};
|
||||
const CMD: &str = "track.extractFaceMeasurements";
|
||||
let (cid, lid) = face_points_layer(s, p).ok_or_else(|| bad(CMD, "no Face Track Points: track a mask with Face Tracking (Detailed Features) first"))?;
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let layer = comp.layer(lid).ok_or(EngineError::NoComp)?;
|
||||
let g = layer.effects().and_then(|fx| fx.groups().find(|g| g.match_id == POINTS_ID)).ok_or_else(|| bad(CMD, "the layer has no Face Track Points"))?;
|
||||
let mut times: Vec<Tick> = LANDMARKS.iter().filter_map(|l| g.get(l.0)).flat_map(|pr| pr.keys.iter().map(|k| k.time)).collect();
|
||||
times.sort();
|
||||
times.dedup();
|
||||
if times.is_empty() {
|
||||
times.push(layer.layer_time(s.time_of(cid)));
|
||||
}
|
||||
let rows: Vec<(Tick, [f64; 14])> = {
|
||||
let pts_at = |t: Tick| {
|
||||
let mut l = [[0.0; 2]; N];
|
||||
for (i, (id, _, _)) in LANDMARKS.iter().enumerate() {
|
||||
if let Some(KV::Vec2(v)) = g.get(id).map(|pr| pr.value_at(t)) {
|
||||
l[i] = v;
|
||||
}
|
||||
}
|
||||
l
|
||||
};
|
||||
let reference = face_frame(&pts_at(times[0])).1;
|
||||
times.iter().map(|t| (*t, measure(&pts_at(*t), reference))).collect()
|
||||
};
|
||||
let comp_times: Vec<Tick> = times.iter().map(|t| layer.comp_time(*t)).collect();
|
||||
let fd = comp.frame_duration();
|
||||
let uid = s.edit("Extract Face Measurements", None, |proj, _| {
|
||||
let uid = crate::mask_track::face_effect(proj, cid, lid, MEASUREMENTS_ID).ok_or_else(|| bad(CMD, "Face Measurements is not available"))?;
|
||||
let g = proj.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)).ok_or(EngineError::NoComp)?;
|
||||
for (k, (id, _)) in MEASUREMENTS.iter().enumerate() {
|
||||
let Some(pr) = g.get_mut(id) else { continue };
|
||||
pr.keys.clear();
|
||||
for (t, m) in &rows {
|
||||
effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*t, KV::Scalar(m[k])));
|
||||
}
|
||||
}
|
||||
Ok(uid)
|
||||
})?;
|
||||
// Copy the keys (Edit ▸ Paste pastes them at the CTI, like copied keyframes).
|
||||
let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?;
|
||||
let layer = comp.layer(lid).ok_or(EngineError::NoComp)?;
|
||||
let mg = layer.props.find_group(uid).ok_or(EngineError::NoComp)?;
|
||||
let earliest = comp_times.iter().copied().min().unwrap_or(Tick::ZERO);
|
||||
let mut clip = vec![];
|
||||
for (id, _) in MEASUREMENTS {
|
||||
let Some(pr) = mg.get(id) else { continue };
|
||||
let Some(path) = layer.props.match_path_of(pr.uid) else { continue };
|
||||
let keys = pr
|
||||
.keys
|
||||
.iter()
|
||||
.map(|k| {
|
||||
let mut k = k.clone();
|
||||
k.time = layer.comp_time(k.time) - earliest;
|
||||
k
|
||||
})
|
||||
.collect();
|
||||
clip.push(crate::KeyClip { layer: lid, path, keys });
|
||||
}
|
||||
s.state.key_clipboard = clip;
|
||||
s.state.clip_is_keys = true;
|
||||
// The same data as text (tab-separated, one row per frame) for other applications.
|
||||
let mut text = String::from("EffectCraft Face Measurements\n\nFrame\tTime");
|
||||
for (_, name) in MEASUREMENTS {
|
||||
text.push('\t');
|
||||
text.push_str(name);
|
||||
}
|
||||
text.push('\n');
|
||||
let mut table = vec![];
|
||||
for ((_, m), ct) in rows.iter().zip(&comp_times) {
|
||||
let frame = if fd > Tick::ZERO { (ct.seconds() / fd.seconds()).round() as i64 } else { 0 };
|
||||
text.push_str(&format!("{frame}\t{:.4}", ct.seconds()));
|
||||
for v in m {
|
||||
text.push_str(&format!("\t{v:.3}"));
|
||||
}
|
||||
text.push('\n');
|
||||
let obj: serde_json::Map<String, Value> = MEASUREMENTS.iter().zip(m).map(|((id, _), v)| (id.to_string(), json!(v))).collect();
|
||||
table.push(json!({"time": ct.seconds(), "frame": frame, "values": obj}));
|
||||
}
|
||||
s.bump();
|
||||
Ok(json!({"layer": lid.0, "effect": uid, "frames": rows.len(), "measurements": table, "clipboard": text}))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- mask interpolation
|
||||
|
||||
/// Mask Interpolation panel options (kept in the editor state, serde for agents).
|
||||
|
|
|
|||
|
|
@ -753,6 +753,8 @@ mod tests_effects;
|
|||
#[cfg(test)]
|
||||
mod tests_essential;
|
||||
#[cfg(test)]
|
||||
mod tests_face;
|
||||
#[cfg(test)]
|
||||
mod tests_fidelity;
|
||||
#[cfg(test)]
|
||||
mod tests_lottie;
|
||||
|
|
|
|||
|
|
@ -6,6 +6,12 @@
|
|||
//! algorithm is `effectcraft_track::mask` (KLT features inside the mask + a RANSAC fit of the
|
||||
//! chosen motion model), and every frame's motion is applied to the mask's vertices and
|
||||
//! tangents, so the shape's Bezier structure is kept.
|
||||
//!
|
||||
//! The two **Face Tracking** methods use `effectcraft_track::face` instead: the mask (drawn
|
||||
//! around a face) seeds the tracker, and every frame's face outline becomes the Mask Path
|
||||
//! (*Outline Only*); *Detailed Features* also keys the facial landmarks into a **Face Track
|
||||
//! Points** effect on the layer (created on first use), from which Extract & Copy Face
|
||||
//! Measurements (`track.extractFaceMeasurements`) derives a keyed **Face Measurements** effect.
|
||||
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
|
@ -31,11 +37,22 @@ pub enum MaskMethod {
|
|||
PositionScaleRotation,
|
||||
PositionScaleRotationSkew,
|
||||
Perspective,
|
||||
/// Face Tracking (Outline Only).
|
||||
FaceOutline,
|
||||
/// Face Tracking (Detailed Features).
|
||||
FaceDetailed,
|
||||
}
|
||||
|
||||
impl MaskMethod {
|
||||
pub const ALL: [MaskMethod; 5] =
|
||||
[MaskMethod::Position, MaskMethod::PositionScale, MaskMethod::PositionScaleRotation, MaskMethod::PositionScaleRotationSkew, MaskMethod::Perspective];
|
||||
pub const ALL: [MaskMethod; 7] = [
|
||||
MaskMethod::Position,
|
||||
MaskMethod::PositionScale,
|
||||
MaskMethod::PositionScaleRotation,
|
||||
MaskMethod::PositionScaleRotationSkew,
|
||||
MaskMethod::Perspective,
|
||||
MaskMethod::FaceOutline,
|
||||
MaskMethod::FaceDetailed,
|
||||
];
|
||||
pub fn label(self) -> &'static str {
|
||||
match self {
|
||||
MaskMethod::Position => "Position",
|
||||
|
|
@ -43,8 +60,14 @@ impl MaskMethod {
|
|||
MaskMethod::PositionScaleRotation => "Position, Scale & Rotation",
|
||||
MaskMethod::PositionScaleRotationSkew => "Position, Scale, Rotation & Skew",
|
||||
MaskMethod::Perspective => "Perspective",
|
||||
MaskMethod::FaceOutline => "Face Tracking (Outline Only)",
|
||||
MaskMethod::FaceDetailed => "Face Tracking (Detailed Features)",
|
||||
}
|
||||
}
|
||||
/// One of the Face Tracking methods.
|
||||
pub fn is_face(self) -> bool {
|
||||
matches!(self, MaskMethod::FaceOutline | MaskMethod::FaceDetailed)
|
||||
}
|
||||
pub fn id(self) -> &'static str {
|
||||
match self {
|
||||
MaskMethod::Position => "position",
|
||||
|
|
@ -52,6 +75,8 @@ impl MaskMethod {
|
|||
MaskMethod::PositionScaleRotation => "positionScaleRotation",
|
||||
MaskMethod::PositionScaleRotationSkew => "positionScaleRotationSkew",
|
||||
MaskMethod::Perspective => "perspective",
|
||||
MaskMethod::FaceOutline => "faceOutline",
|
||||
MaskMethod::FaceDetailed => "faceDetailed",
|
||||
}
|
||||
}
|
||||
pub fn from_name(s: &str) -> Option<MaskMethod> {
|
||||
|
|
@ -67,6 +92,7 @@ impl MaskMethod {
|
|||
MaskMethod::PositionScaleRotation => Model::Similarity,
|
||||
MaskMethod::PositionScaleRotationSkew => Model::Affine,
|
||||
MaskMethod::Perspective => Model::Homography,
|
||||
MaskMethod::FaceOutline | MaskMethod::FaceDetailed => Model::Similarity,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -109,8 +135,29 @@ pub fn transform_path(p: &ShapePath, h: &trk::Homography) -> ShapePath {
|
|||
}
|
||||
}
|
||||
|
||||
/// One tracked frame: (layer time, comp time, mask path).
|
||||
type MaskFrame = (Tick, Tick, ShapePath);
|
||||
/// A closed, smooth mask path through a face outline polygon: `n` vertices with Catmull-Rom
|
||||
/// tangents. `like` lends its feather points.
|
||||
pub fn outline_path(poly: &[[f64; 2]], n: usize, like: &ShapePath) -> ShapePath {
|
||||
let m = poly.len();
|
||||
let n = n.clamp(3, m.max(3));
|
||||
let v: Vec<[f64; 2]> = (0..n).map(|i| poly[(i * m / n) % m.max(1)]).collect();
|
||||
let tan: Vec<[f64; 2]> = (0..n)
|
||||
.map(|i| {
|
||||
let (a, b) = (v[(i + n - 1) % n], v[(i + 1) % n]);
|
||||
[(b[0] - a[0]) / 6.0, (b[1] - a[1]) / 6.0]
|
||||
})
|
||||
.collect();
|
||||
ShapePath {
|
||||
vertices: v,
|
||||
in_tangents: tan.iter().map(|t| [-t[0], -t[1]]).collect(),
|
||||
out_tangents: tan,
|
||||
closed: true,
|
||||
feather: if like.len() == n { like.feather.clone() } else { vec![] },
|
||||
}
|
||||
}
|
||||
|
||||
/// One tracked frame: (layer time, comp time, mask path, face landmarks for Detailed Features).
|
||||
type MaskFrame = (Tick, Tick, ShapePath, Option<[[f64; 2]; trk::face::N]>);
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct MaskTrackShared {
|
||||
|
|
@ -169,9 +216,12 @@ fn run_work(w: MaskWork, shared: &MaskTrackShared) {
|
|||
let frame_at = |t: Tick| source_frame(r, &w.project, w.comp, comp, layer, t, w.expr.as_deref());
|
||||
lock(&shared.state).total = w.times.len().saturating_sub(1) as u64;
|
||||
let Some(first) = frame_at(w.times[0]) else { return fail("the layer has no pixels to track") };
|
||||
if w.method.is_face() {
|
||||
return run_face(&w, shared, first, &frame_at, layer, t0);
|
||||
}
|
||||
let mut tracker = trk::mask::MaskTracker::new(w.method.model(), flatten(&w.path, 8), &trk::Frame { img: &first.0, offset: first.1 });
|
||||
let mut path = w.path.clone();
|
||||
lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path.clone()));
|
||||
lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path.clone(), None));
|
||||
let mut next = if w.times.len() > 1 { frame_at(w.times[1]) } else { None };
|
||||
for k in 1..w.times.len() {
|
||||
if shared.cancel.load(Ordering::Relaxed) {
|
||||
|
|
@ -190,7 +240,7 @@ fn run_work(w: MaskWork, shared: &MaskTrackShared) {
|
|||
break;
|
||||
};
|
||||
path = transform_path(&path, &step.motion);
|
||||
lock(&shared.frames).push((layer.layer_time(w.times[k]), w.times[k], path.clone()));
|
||||
lock(&shared.frames).push((layer.layer_time(w.times[k]), w.times[k], path.clone(), None));
|
||||
let mut s = lock(&shared.state);
|
||||
s.done = k as u64;
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
|
|
@ -203,12 +253,99 @@ fn run_work(w: MaskWork, shared: &MaskTrackShared) {
|
|||
s.finished = true;
|
||||
}
|
||||
|
||||
/// Face tracking: the face outline keys the mask; Detailed Features adds the landmarks.
|
||||
fn run_face(
|
||||
w: &MaskWork,
|
||||
shared: &MaskTrackShared,
|
||||
first: (Arc<effectcraft_raster::Image>, [f64; 2]),
|
||||
frame_at: &(dyn Fn(Tick) -> Option<(Arc<effectcraft_raster::Image>, [f64; 2])> + Sync),
|
||||
layer: &effectcraft_project::Layer,
|
||||
t0: web_time::Instant,
|
||||
) {
|
||||
let detailed = w.method == MaskMethod::FaceDetailed;
|
||||
let b = trk::mask::bounds(&flatten(&w.path, 8));
|
||||
let Some((mut tracker, fit)) = trk::face::FaceTracker::new(&trk::Frame { img: &first.0, offset: first.1 }, b) else {
|
||||
let mut s = lock(&shared.state);
|
||||
s.error = Some("no face found inside the mask (draw the mask around a face)".into());
|
||||
s.finished = true;
|
||||
return;
|
||||
};
|
||||
let key = |fit: &trk::face::FaceFit| (outline_path(&fit.outline, 16, &w.path), detailed.then_some(fit.landmarks));
|
||||
let (path, pts) = key(&fit);
|
||||
lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path, pts));
|
||||
let mut next = if w.times.len() > 1 { frame_at(w.times[1]) } else { None };
|
||||
for k in 1..w.times.len() {
|
||||
if shared.cancel.load(Ordering::Relaxed) {
|
||||
lock(&shared.state).cancelled = true;
|
||||
break;
|
||||
}
|
||||
let Some(cur) = next.take() else {
|
||||
lock(&shared.state).error = Some(format!("no frame at {:.3} s", w.times[k].seconds()));
|
||||
break;
|
||||
};
|
||||
let (res, nf) =
|
||||
rayon::join(|| tracker.step(&trk::Frame { img: &cur.0, offset: cur.1 }), || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None });
|
||||
next = nf;
|
||||
let Some(fit) = res else {
|
||||
lock(&shared.state).error = Some(format!("lost the face at {:.3} s", w.times[k].seconds()));
|
||||
break;
|
||||
};
|
||||
let (path, pts) = key(&fit);
|
||||
lock(&shared.frames).push((layer.layer_time(w.times[k]), w.times[k], path, pts));
|
||||
let mut s = lock(&shared.state);
|
||||
s.done = k as u64;
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
s.fps = if s.elapsed > 0.0 { k as f64 / s.elapsed } else { 0.0 };
|
||||
s.time = w.times[k].seconds();
|
||||
crate::offload::report(s.done, s.total);
|
||||
}
|
||||
let mut s = lock(&shared.state);
|
||||
s.elapsed = t0.elapsed().as_secs_f64();
|
||||
s.finished = true;
|
||||
}
|
||||
|
||||
/// The layer's Face Track Points effect (created when missing). Returns its uid.
|
||||
pub(crate) fn face_points_group(p: &mut Project, comp: ItemId, layer: LayerId) -> Option<Uid> {
|
||||
face_effect(p, comp, layer, effectcraft_effects::face_track::POINTS_ID)
|
||||
}
|
||||
|
||||
/// The layer's effect `id`, applied when missing (data effects of face tracking).
|
||||
pub(crate) fn face_effect(p: &mut Project, comp: ItemId, layer: LayerId, id: &str) -> Option<Uid> {
|
||||
let spec = effectcraft_effects::find(id)?;
|
||||
let l = p.comp(comp)?.layer(layer)?;
|
||||
if let Some(g) = l.effects().and_then(|fx| fx.groups().find(|g| g.match_id == spec.id)) {
|
||||
return Some(g.uid);
|
||||
}
|
||||
let (w, h) = effectcraft_render::source_size(p, l);
|
||||
let size = if w == 0 { p.comp(comp).map(|c| [c.width as f64, c.height as f64]).unwrap_or([1920.0, 1080.0]) } else { [w as f64, h as f64] };
|
||||
let mut next = p.next_id;
|
||||
let g = effectcraft_effects::instantiate(spec, &mut effectcraft_project::build::Ids(&mut next), spec.name, size);
|
||||
let uid = g.uid;
|
||||
let fx = p.comp_mut(comp)?.layer_mut(layer)?.props.sub_mut("effects")?;
|
||||
fx.children.push(g.into());
|
||||
p.next_id = next;
|
||||
Some(uid)
|
||||
}
|
||||
|
||||
fn write_frames(p: &mut Project, job: &MaskTrackJob, frames: &[MaskFrame]) {
|
||||
let points = if frames.iter().any(|f| f.3.is_some()) { face_points_group(p, job.comp, job.layer) } else { None };
|
||||
let Some(layer) = p.comp_mut(job.comp).and_then(|c| c.layer_mut(job.layer)) else { return };
|
||||
let Some(g) = layer.props.find_group_mut(job.mask) else { return };
|
||||
let Some(pr) = g.get_mut("path") else { return };
|
||||
for (lt, _, path) in frames {
|
||||
effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*lt, Value::Path(path.clone())));
|
||||
if let Some(g) = layer.props.find_group_mut(job.mask)
|
||||
&& let Some(pr) = g.get_mut("path")
|
||||
{
|
||||
for (lt, _, path, _) in frames {
|
||||
effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*lt, Value::Path(path.clone())));
|
||||
}
|
||||
}
|
||||
if let Some(g) = points.and_then(|u| layer.props.find_group_mut(u)) {
|
||||
for (lt, _, _, pts) in frames {
|
||||
let Some(pts) = pts else { continue };
|
||||
for (i, (id, _, _)) in trk::face::LANDMARKS.iter().enumerate() {
|
||||
if let Some(pr) = g.get_mut(id) {
|
||||
effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*lt, Value::Vec2(pts[i])));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -242,7 +379,7 @@ impl Session {
|
|||
return Err("a track analysis is already running".into());
|
||||
}
|
||||
self.poll_mask_track();
|
||||
if !wait && self.offloads() {
|
||||
if !wait && self.offloads() && !work.method.is_face() {
|
||||
let MaskWork { comp, layer, path, method, times, .. } = work;
|
||||
return self.offload_analysis(crate::offload::WorkerJob::MaskTrack { comp, layer, mask, direction, path, method, times });
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,106 @@
|
|||
//! Face tracking end to end through the commands, on synthetic face footage generated in-test
|
||||
//! (a moving, rotating face that opens its mouth and blinks).
|
||||
|
||||
use effectcraft_keyframe::{ShapePath, Value as KV};
|
||||
use effectcraft_track::face::{FaceParams, LANDMARKS, MEASUREMENTS, render_synthetic, synth_landmarks};
|
||||
use serde_json::json;
|
||||
|
||||
use crate::mask_track::MaskMethod;
|
||||
use crate::tests_track::{H, W, frame_time, setup};
|
||||
|
||||
fn face(f: u32) -> FaceParams {
|
||||
let t = f as f64;
|
||||
FaceParams {
|
||||
center: [160.0 + 25.0 * (t * 0.2).sin(), 120.0 + 8.0 * (t * 0.15).cos()],
|
||||
height: 70.0 * (1.0 + 0.05 * (t * 0.1).sin()),
|
||||
roll: 12.0 * (t * 0.18).sin(),
|
||||
yaw: 10.0 * (t * 0.12).sin(),
|
||||
mouth_open: 0.5 + 0.45 * (t * 0.3).sin(),
|
||||
eye_open: [0.85, 0.85],
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn face_tracking_keys_outline_points_and_measurements() {
|
||||
let (mut s, clip, _) = setup(|f| render_synthetic(W, H, &face(f), 5));
|
||||
assert_eq!(MaskMethod::from_name("faceDetailed"), Some(MaskMethod::FaceDetailed));
|
||||
assert_eq!(MaskMethod::from_name("Face Tracking (Outline Only)"), Some(MaskMethod::FaceOutline));
|
||||
// A rough mask around the face.
|
||||
let c = face(0).center;
|
||||
let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect(c, 120.0, 160.0).vertices, "closed": true})).unwrap();
|
||||
let mask = r["mask"].as_u64().unwrap();
|
||||
s.execute("time.set", json!({"time": 0})).unwrap();
|
||||
let r = s.execute_checked("track.mask", json!({"layer": clip.0, "mask": mask, "method": "faceDetailed", "direction": "forward", "wait": true})).unwrap();
|
||||
assert_eq!(r["frames"], 24, "{r}");
|
||||
assert_eq!(s.state.mask_track_method, MaskMethod::FaceDetailed);
|
||||
let comp = s.active_comp().unwrap();
|
||||
let layer = comp.layer(clip).unwrap();
|
||||
// The mask follows the face outline.
|
||||
let path = layer.props.find_group(mask).unwrap().get("path").unwrap();
|
||||
assert_eq!(path.keys.len(), 25);
|
||||
for f in [0, 12, 24] {
|
||||
let KV::Path(p) = path.value_at(frame_time(f)) else { panic!() };
|
||||
let fp = face(f);
|
||||
let (sn, cs) = fp.roll.to_radians().sin_cos();
|
||||
for v in &p.vertices {
|
||||
let d = [(v[0] - fp.center[0]) / fp.height, (v[1] - fp.center[1]) / fp.height];
|
||||
let (u, w) = (cs * d[0] + sn * d[1], -sn * d[0] + cs * d[1]);
|
||||
let r = ((u / 0.75).powi(2) + w * w).sqrt();
|
||||
assert!((r - 1.0).abs() < 0.06, "frame {f}: vertex {v:?} at {r}");
|
||||
}
|
||||
}
|
||||
// Face Track Points: one keyed point per landmark, on the face.
|
||||
let fx = layer.effects().unwrap();
|
||||
let pts = fx.groups().find(|g| g.match_id == effectcraft_effects::face_track::POINTS_ID).expect("Face Track Points");
|
||||
let mut worst: f64 = 0.0;
|
||||
for f in [0, 6, 12, 18, 24] {
|
||||
let truth = synth_landmarks(&face(f));
|
||||
for (i, (id, _, _)) in LANDMARKS.iter().enumerate() {
|
||||
let pr = pts.get(id).unwrap();
|
||||
assert_eq!(pr.keys.len(), 25, "{id}");
|
||||
let KV::Vec2(v) = pr.value_at(frame_time(f)) else { panic!() };
|
||||
worst = worst.max((v[0] - truth[i][0]).hypot(v[1] - truth[i][1]) / face(f).height);
|
||||
}
|
||||
}
|
||||
assert!(worst < 0.1, "worst landmark error {worst} face heights");
|
||||
// Extract & Copy Face Measurements.
|
||||
assert!(s.is_enabled("track.extractFaceMeasurements"));
|
||||
let r = s.execute_checked("track.extractFaceMeasurements", json!({"layer": clip.0})).unwrap();
|
||||
assert_eq!(r["frames"], 25);
|
||||
assert!(r["clipboard"].as_str().unwrap().contains("Mouth Openness"));
|
||||
assert_eq!(s.state.key_clipboard.len(), MEASUREMENTS.len());
|
||||
assert!(s.state.clip_is_keys);
|
||||
let layer = s.active_comp().unwrap().layer(clip).unwrap();
|
||||
let m = layer.effects().unwrap().groups().find(|g| g.match_id == effectcraft_effects::face_track::MEASUREMENTS_ID).expect("Face Measurements");
|
||||
let mo = m.get("mouthOpenness").unwrap();
|
||||
assert_eq!(mo.keys.len(), 25);
|
||||
// Mouth openness follows the footage: widest open where the generator opens it most.
|
||||
let open = |f: u32| mo.value_at(frame_time(f)).as_f64();
|
||||
let (fmax, fmin) = (
|
||||
(0..25).max_by(|a, b| face(*a).mouth_open.total_cmp(&face(*b).mouth_open)).unwrap(),
|
||||
(0..25).min_by(|a, b| face(*a).mouth_open.total_cmp(&face(*b).mouth_open)).unwrap(),
|
||||
);
|
||||
assert!(open(fmax) > open(fmin) + 20.0, "{} vs {}", open(fmax), open(fmin));
|
||||
let roll = m.get("headOrientationZ").unwrap();
|
||||
for f in [0, 10, 20] {
|
||||
assert!((roll.value_at(frame_time(f)).as_f64() - face(f).roll).abs() < 3.0);
|
||||
}
|
||||
// One undo step for the extraction.
|
||||
assert!(s.undo());
|
||||
assert!(!s.active_comp().unwrap().layer(clip).unwrap().effects().unwrap().groups().any(|g| g.match_id == effectcraft_effects::face_track::MEASUREMENTS_ID));
|
||||
// Outline Only keys just the mask.
|
||||
let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect(c, 120.0, 160.0).vertices, "closed": true})).unwrap();
|
||||
let m2 = r["mask"].as_u64().unwrap();
|
||||
s.execute("time.set", json!({"time": 0})).unwrap();
|
||||
s.execute_checked("track.mask", json!({"layer": clip.0, "mask": m2, "method": "faceOutline", "direction": "frameForward", "wait": true})).unwrap();
|
||||
let layer = s.active_comp().unwrap().layer(clip).unwrap();
|
||||
assert_eq!(layer.props.find_group(m2).unwrap().get("path").unwrap().keys.len(), 2);
|
||||
// A mask without a face reports it.
|
||||
let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect([20.0, 20.0], 30.0, 30.0).vertices, "closed": true})).unwrap();
|
||||
let m3 = r["mask"].as_u64().unwrap();
|
||||
s.execute("time.set", json!({"time": 0})).unwrap();
|
||||
let _ = s.execute("track.mask", json!({"layer": clip.0, "mask": m3, "method": "faceOutline", "direction": "forward", "wait": true}));
|
||||
let layer = s.active_comp().unwrap().layer(clip).unwrap();
|
||||
assert!(layer.props.find_group(m3).unwrap().get("path").unwrap().keys.is_empty());
|
||||
}
|
||||
|
|
@ -10,6 +10,7 @@
|
|||
|
||||
use rayon::prelude::*;
|
||||
|
||||
use super::Distortion;
|
||||
use super::geometry::Pose;
|
||||
use super::linalg::*;
|
||||
|
||||
|
|
@ -19,6 +20,8 @@ pub struct BaCam {
|
|||
pub pose: Pose,
|
||||
/// Focal length (pixels).
|
||||
pub f: f64,
|
||||
/// Lens distortion (shared by all cameras; adjusted when [`BaOpts::distortion`]).
|
||||
pub dist: Distortion,
|
||||
}
|
||||
|
||||
/// One observation: camera `cam` sees point `pt` at `uv` (pixels relative to the principal
|
||||
|
|
@ -50,11 +53,13 @@ pub struct BaOpts {
|
|||
/// Huber threshold (pixels).
|
||||
pub huber: f64,
|
||||
pub max_iter: usize,
|
||||
/// Adjust the shared radial distortion `(k1, k2)` too.
|
||||
pub distortion: bool,
|
||||
}
|
||||
|
||||
impl Default for BaOpts {
|
||||
fn default() -> Self {
|
||||
BaOpts { focal: FocalMode::Fixed, fix_points: false, fix_centers: false, fixed: vec![], huber: 2.0, max_iter: 30 }
|
||||
BaOpts { focal: FocalMode::Fixed, fix_points: false, fix_centers: false, fixed: vec![], huber: 2.0, max_iter: 30, distortion: false }
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -77,7 +82,8 @@ pub fn residual(c: &BaCam, x: V3, uv: [f64; 2]) -> Option<[f64; 2]> {
|
|||
if p[2] <= 1e-9 {
|
||||
return None;
|
||||
}
|
||||
Some([c.f * p[0] / p[2] - uv[0], c.f * p[1] / p[2] - uv[1]])
|
||||
let q = c.dist.distort([c.f * p[0] / p[2], c.f * p[1] / p[2]]);
|
||||
Some([q[0] - uv[0], q[1] - uv[1]])
|
||||
}
|
||||
|
||||
/// Total robust cost.
|
||||
|
|
@ -108,6 +114,8 @@ struct Layout {
|
|||
fidx: Vec<usize>,
|
||||
/// Index of the shared focal parameter.
|
||||
shared: Option<usize>,
|
||||
/// Index of `k1` (then `k2`) when the distortion is adjusted.
|
||||
dist: Option<usize>,
|
||||
n: usize,
|
||||
rot_only: bool,
|
||||
}
|
||||
|
|
@ -132,7 +140,11 @@ fn layout(ncam: usize, o: &BaOpts) -> Layout {
|
|||
n += 1;
|
||||
n - 1
|
||||
});
|
||||
Layout { start, fidx, shared, n, rot_only: o.fix_centers }
|
||||
let dist = o.distortion.then(|| {
|
||||
n += 2;
|
||||
n - 2
|
||||
});
|
||||
Layout { start, fidx, shared, dist, n, rot_only: o.fix_centers }
|
||||
}
|
||||
|
||||
/// Camera-side Jacobian entries of one observation: (parameter index, d r / d param).
|
||||
|
|
@ -204,12 +216,21 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) ->
|
|||
continue;
|
||||
}
|
||||
let iz = 1.0 / xc[2];
|
||||
let p = [c.f * xc[0] * iz, c.f * xc[1] * iz];
|
||||
let r = [p[0] - ob.uv[0], p[1] - ob.uv[1]];
|
||||
let p0 = [c.f * xc[0] * iz, c.f * xc[1] * iz];
|
||||
let pd = c.dist.distort(p0);
|
||||
let r = [pd[0] - ob.uv[0], pd[1] - ob.uv[1]];
|
||||
let e = (r[0] * r[0] + r[1] * r[1]).sqrt();
|
||||
let w = if e <= o.huber { 1.0 } else { o.huber / e };
|
||||
let a = [[c.f * iz, 0.0, -c.f * xc[0] * iz * iz], [0.0, c.f * iz, -c.f * xc[1] * iz * iz]];
|
||||
let a0 = [[c.f * iz, 0.0, -c.f * xc[0] * iz * iz], [0.0, c.f * iz, -c.f * xc[1] * iz * iz]];
|
||||
// Through the distortion: d r = D d p.
|
||||
let (dj, dk) = c.dist.jacobian(p0);
|
||||
let a = [0, 1].map(|i| [0, 1, 2].map(|k| dj[i][0] * a0[0][k] + dj[i][1] * a0[1][k]));
|
||||
let p = [dj[0][0] * p0[0] + dj[0][1] * p0[1], dj[1][0] * p0[0] + dj[1][1] * p0[1]];
|
||||
cam_jac(&l, ci, c, xc, &a, p, &mut jc);
|
||||
if let Some(g) = l.dist {
|
||||
jc.push((g, [dk[0][0], dk[1][0]]));
|
||||
jc.push((g + 1, [dk[0][1], dk[1][1]]));
|
||||
}
|
||||
// Point Jacobian: A R.
|
||||
let jx = if o.fix_points {
|
||||
[[0.0; 3]; 2]
|
||||
|
|
@ -236,7 +257,8 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) ->
|
|||
[w * (ja[0] * jx[0][0] + ja[1] * jx[1][0]), w * (ja[0] * jx[0][1] + ja[1] * jx[1][1]), w * (ja[0] * jx[0][2] + ja[1] * jx[1][2])];
|
||||
// Each observation of a point is on another camera: only the shared focal
|
||||
// length repeats.
|
||||
let found = if Some(*ia) == l.shared { ps.w.iter_mut().find(|e| e.0 == *ia) } else { None };
|
||||
let repeats = Some(*ia) == l.shared || l.dist.is_some_and(|g| *ia == g || *ia == g + 1);
|
||||
let found = if repeats { ps.w.iter_mut().find(|e| e.0 == *ia) } else { None };
|
||||
match found {
|
||||
Some(e) => e.1 = add(e.1, wv),
|
||||
None => ps.w.push((*ia, wv)),
|
||||
|
|
@ -385,6 +407,10 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) ->
|
|||
if let Some(g) = l.shared {
|
||||
c.f *= dc[g].clamp(-0.5, 0.5).exp();
|
||||
}
|
||||
if let Some(g) = l.dist {
|
||||
c.dist.k1 += dc[g].clamp(-0.2, 0.2);
|
||||
c.dist.k2 += dc[g + 1].clamp(-0.2, 0.2);
|
||||
}
|
||||
}
|
||||
let mut np: Vec<V3> = pts.to_vec();
|
||||
if !o.fix_points {
|
||||
|
|
@ -445,7 +471,11 @@ mod tests {
|
|||
let truth: Vec<V3> = (0..80).map(|_| [u() * 6.0, u() * 4.0, 6.0 + u() * 4.0]).collect();
|
||||
let f = 900.0;
|
||||
let cams_true: Vec<BaCam> = (0..6)
|
||||
.map(|i| BaCam { pose: Pose { r: rodrigues([0.0, -0.05 * i as f64, 0.01 * i as f64]), c: [0.3 * i as f64, 0.05 * i as f64, 0.0] }, f })
|
||||
.map(|i| BaCam {
|
||||
pose: Pose { r: rodrigues([0.0, -0.05 * i as f64, 0.01 * i as f64]), c: [0.3 * i as f64, 0.05 * i as f64, 0.0] },
|
||||
f,
|
||||
dist: Distortion::NONE,
|
||||
})
|
||||
.collect();
|
||||
let mut obs = vec![];
|
||||
for (ci, c) in cams_true.iter().enumerate() {
|
||||
|
|
|
|||
|
|
@ -28,8 +28,15 @@
|
|||
//! tripod solution when it explains the tracks as well as the general one (a quick rotation fit
|
||||
//! at the general solution's focal length skips the tripod search when it clearly fails).
|
||||
//!
|
||||
//! The camera model has square pixels, the principal point at the layer centre and no lens
|
||||
//! distortion. Camera space is x right, y down, z forward (After Effects' convention); a camera
|
||||
//! 5. **Lens distortion** (Solve Lens Distortion, or Detailed Analysis): radial distortion
|
||||
//! `p_d = p (1 + k1 ρ² + k2 ρ⁴)` (ρ = distance from the principal point over half the frame
|
||||
//! diagonal; Brown's model without the tangential terms) is solved jointly with the cameras
|
||||
//! and points in a final bundle adjustment on the raw tracks; the whole solve is then repeated
|
||||
//! on tracks undistorted with that estimate (the linear initialisations assume a pinhole
|
||||
//! camera) and adjusted jointly again. The solve stores `k1, k2`; the effect can undistort
|
||||
//! its output so composited 3D layers line up.
|
||||
//!
|
||||
//! The camera model has square pixels and the principal point at the layer centre. Camera space is x right, y down, z forward (After Effects' convention); a camera
|
||||
//! is `(R, C, f)` with `X_cam = R (X − C)` and pixel `u = f x / z + w / 2`.
|
||||
//!
|
||||
//! The solve is stored in a canonical frame: the first solved frame's camera is at the origin
|
||||
|
|
@ -105,6 +112,69 @@ impl CameraTracks {
|
|||
}
|
||||
}
|
||||
|
||||
/// Radial lens distortion around the principal point (see the module docs).
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", default)]
|
||||
pub struct Distortion {
|
||||
pub k1: f64,
|
||||
pub k2: f64,
|
||||
/// Normalisation radius ρ = 1 (pixels): half the frame diagonal; 0 = no distortion.
|
||||
pub radius: f64,
|
||||
}
|
||||
|
||||
impl Distortion {
|
||||
pub const NONE: Distortion = Distortion { k1: 0.0, k2: 0.0, radius: 0.0 };
|
||||
|
||||
/// No distortion for frames of `size` (the radius set, coefficients zero).
|
||||
pub fn for_size(size: [f64; 2]) -> Distortion {
|
||||
Distortion { k1: 0.0, k2: 0.0, radius: 0.5 * size[0].hypot(size[1]) }
|
||||
}
|
||||
pub fn is_none(&self) -> bool {
|
||||
self.radius <= 0.0 || (self.k1 == 0.0 && self.k2 == 0.0)
|
||||
}
|
||||
fn rho2(&self, p: [f64; 2]) -> f64 {
|
||||
(p[0] * p[0] + p[1] * p[1]) / (self.radius * self.radius)
|
||||
}
|
||||
/// Ideal → distorted (pixels relative to the principal point).
|
||||
pub fn distort(&self, p: [f64; 2]) -> [f64; 2] {
|
||||
if self.radius <= 0.0 {
|
||||
return p;
|
||||
}
|
||||
let r2 = self.rho2(p);
|
||||
let d = 1.0 + self.k1 * r2 + self.k2 * r2 * r2;
|
||||
[p[0] * d, p[1] * d]
|
||||
}
|
||||
/// Distorted → ideal (fixed-point iteration).
|
||||
pub fn undistort(&self, q: [f64; 2]) -> [f64; 2] {
|
||||
if self.is_none() {
|
||||
return q;
|
||||
}
|
||||
let mut p = q;
|
||||
for _ in 0..30 {
|
||||
let r2 = self.rho2(p);
|
||||
let d = 1.0 + self.k1 * r2 + self.k2 * r2 * r2;
|
||||
if d.abs() < 1e-6 {
|
||||
break;
|
||||
}
|
||||
p = [q[0] / d, q[1] / d];
|
||||
}
|
||||
p
|
||||
}
|
||||
/// `d distort / d p` (2 × 2, row-major) and `d distort / d (k1, k2)` (rows x, y).
|
||||
pub fn jacobian(&self, p: [f64; 2]) -> ([[f64; 2]; 2], [[f64; 2]; 2]) {
|
||||
if self.radius <= 0.0 {
|
||||
return ([[1.0, 0.0], [0.0, 1.0]], [[0.0; 2]; 2]);
|
||||
}
|
||||
let r2 = self.rho2(p);
|
||||
let d = 1.0 + self.k1 * r2 + self.k2 * r2 * r2;
|
||||
// d(r2)/dp = 2 p / R².
|
||||
let dd = (self.k1 + 2.0 * self.k2 * r2) * 2.0 / (self.radius * self.radius);
|
||||
let j = [[d + p[0] * dd * p[0], p[0] * dd * p[1]], [p[1] * dd * p[0], d + p[1] * dd * p[1]]];
|
||||
let k = [[p[0] * r2, p[0] * r2 * r2], [p[1] * r2, p[1] * r2 * r2]];
|
||||
(j, k)
|
||||
}
|
||||
}
|
||||
|
||||
/// 3D Camera Tracker ▸ Shot Type.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
|
|
@ -178,6 +248,8 @@ pub struct SolveSettings {
|
|||
pub method: SolveMethod,
|
||||
/// Track ids the user deleted.
|
||||
pub deleted: Vec<u32>,
|
||||
/// Solve radial lens distortion jointly with the cameras.
|
||||
pub lens_distortion: bool,
|
||||
}
|
||||
|
||||
/// The solved camera of one frame.
|
||||
|
|
@ -261,6 +333,9 @@ pub struct CameraSolve {
|
|||
pub points: Vec<SolvedPoint>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ground: Option<Ground>,
|
||||
/// The solved lens distortion (Solve Lens Distortion).
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub distortion: Option<Distortion>,
|
||||
}
|
||||
|
||||
impl CameraSolve {
|
||||
|
|
@ -355,6 +430,36 @@ impl CameraSolve {
|
|||
}
|
||||
}
|
||||
|
||||
/// An ideal (pinhole) image point → where it appears in the footage (layer pixels), through
|
||||
/// the solved lens distortion.
|
||||
pub fn to_image(&self, p: [f64; 2]) -> [f64; 2] {
|
||||
match &self.distortion {
|
||||
Some(d) if !d.is_none() => {
|
||||
let c = [self.size[0] * 0.5, self.size[1] * 0.5];
|
||||
let q = d.distort([p[0] - c[0], p[1] - c[1]]);
|
||||
[q[0] + c[0], q[1] + c[1]]
|
||||
}
|
||||
_ => p,
|
||||
}
|
||||
}
|
||||
/// A footage point → its ideal (undistorted) position.
|
||||
pub fn to_ideal(&self, p: [f64; 2]) -> [f64; 2] {
|
||||
match &self.distortion {
|
||||
Some(d) if !d.is_none() => {
|
||||
let c = [self.size[0] * 0.5, self.size[1] * 0.5];
|
||||
let q = d.undistort([p[0] - c[0], p[1] - c[1]]);
|
||||
[q[0] + c[0], q[1] + c[1]]
|
||||
}
|
||||
_ => p,
|
||||
}
|
||||
}
|
||||
/// Project world point `x` on frame `k`: through the lens distortion into the footage, or
|
||||
/// (`undistorted`) into the undistorted image.
|
||||
pub fn project(&self, k: usize, x: V3, undistorted: bool) -> Option<[f64; 2]> {
|
||||
let q = self.frames.get(k)?.project(self.size, x)?;
|
||||
Some(if undistorted { q } else { self.to_image(q) })
|
||||
}
|
||||
|
||||
/// The first solved frame.
|
||||
pub fn first_solved(&self) -> Option<usize> {
|
||||
self.frames.iter().position(|c| c.solved)
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ use rayon::prelude::*;
|
|||
use super::bundle::{self, BaCam, BaObs, BaOpts, FocalMode};
|
||||
use super::geometry::{self, Pose};
|
||||
use super::linalg::*;
|
||||
use super::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, SolvedFrame, SolvedPoint};
|
||||
use super::{CameraSolve, CameraTracks, Distortion, ShotType, SolveMethod, SolveSettings, SolvedFrame, SolvedPoint};
|
||||
use crate::fit::{Model, ransac};
|
||||
|
||||
fn debug() -> bool {
|
||||
|
|
@ -142,6 +142,8 @@ struct Cfg {
|
|||
focal: FocalMode,
|
||||
/// Registration estimates a focal length per frame.
|
||||
per_frame_f: bool,
|
||||
/// The global adjustments also adjust the lens distortion.
|
||||
distortion: bool,
|
||||
}
|
||||
|
||||
const F_PRIOR: f64 = 30.0;
|
||||
|
|
@ -183,7 +185,7 @@ impl Recon {
|
|||
&mut cams,
|
||||
&mut pts,
|
||||
&obs,
|
||||
&BaOpts { focal, fix_points: false, fix_centers: cfg.kind == Kind::Tripod, fixed, huber: d.huber(), max_iter: iters },
|
||||
&BaOpts { focal, fix_points: false, fix_centers: cfg.kind == Kind::Tripod, fixed, huber: d.huber(), max_iter: iters, distortion: cfg.distortion },
|
||||
);
|
||||
for (k, c) in frames.iter().zip(&cams) {
|
||||
self.cams.insert(*k, *c);
|
||||
|
|
@ -289,6 +291,7 @@ impl Recon {
|
|||
fixed: vec![false],
|
||||
huber,
|
||||
max_iter: 40,
|
||||
distortion: false,
|
||||
};
|
||||
let mut best: Option<(usize, BaCam)> = None;
|
||||
// Robust start (large Huber threshold), then tighten; from the prediction and, failing
|
||||
|
|
@ -371,7 +374,11 @@ fn predict(r: &Recon, k: usize) -> Option<BaCam> {
|
|||
match (before, after) {
|
||||
(Some((ka, a)), Some((kb, b))) => {
|
||||
let t = (k - ka) as f64 / (kb - ka) as f64;
|
||||
Some(BaCam { pose: Pose { r: slerp(&a.pose.r, &b.pose.r, t), c: add(scale(a.pose.c, 1.0 - t), scale(b.pose.c, t)) }, f: a.f * (b.f / a.f).powf(t) })
|
||||
Some(BaCam {
|
||||
pose: Pose { r: slerp(&a.pose.r, &b.pose.r, t), c: add(scale(a.pose.c, 1.0 - t), scale(b.pose.c, t)) },
|
||||
f: a.f * (b.f / a.f).powf(t),
|
||||
dist: a.dist,
|
||||
})
|
||||
}
|
||||
(Some((ka, a)), None) => {
|
||||
// Constant velocity from the two previous cameras.
|
||||
|
|
@ -381,7 +388,7 @@ fn predict(r: &Recon, k: usize) -> Option<BaCam> {
|
|||
let dr = log_rot(&mm(&a.pose.r, &mt(&p.pose.r)));
|
||||
let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &a.pose.r);
|
||||
let c = add(a.pose.c, scale(sub(a.pose.c, p.pose.c), t.min(2.0)));
|
||||
Some(BaCam { pose: Pose { r: rr, c }, f: a.f })
|
||||
Some(BaCam { pose: Pose { r: rr, c }, f: a.f, dist: a.dist })
|
||||
}
|
||||
None => Some(a),
|
||||
}
|
||||
|
|
@ -392,7 +399,7 @@ fn predict(r: &Recon, k: usize) -> Option<BaCam> {
|
|||
let dr = log_rot(&mm(&b.pose.r, &mt(&nx.pose.r)));
|
||||
let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &b.pose.r);
|
||||
let c = add(b.pose.c, scale(sub(b.pose.c, nx.pose.c), t.min(2.0)));
|
||||
Some(BaCam { pose: Pose { r: rr, c }, f: b.f })
|
||||
Some(BaCam { pose: Pose { r: rr, c }, f: b.f, dist: b.dist })
|
||||
}
|
||||
None => Some(b),
|
||||
},
|
||||
|
|
@ -459,8 +466,8 @@ fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) ->
|
|||
fn grow(d: &Data, kfs: &[usize], j: usize, init: &(Pose, HashMap<u32, V3>), f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option<Recon> {
|
||||
let (a, b) = (kfs[0], kfs[j]);
|
||||
let mut r = Recon::default();
|
||||
r.cams.insert(a, BaCam { pose: Pose::IDENTITY, f });
|
||||
r.cams.insert(b, BaCam { pose: init.0, f });
|
||||
r.cams.insert(a, BaCam { pose: Pose::IDENTITY, f, dist: Distortion::NONE });
|
||||
r.cams.insert(b, BaCam { pose: init.0, f, dist: Distortion::NONE });
|
||||
r.pts = init.1.clone();
|
||||
let inner = |r: &mut Recon, frames: &[usize]| {
|
||||
let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed };
|
||||
|
|
@ -495,7 +502,7 @@ fn grow(d: &Data, kfs: &[usize], j: usize, init: &(Pose, HashMap<u32, V3>), f: f
|
|||
/// Tripod reconstruction over keyframes `kfs` at focal length `f`.
|
||||
fn grow_tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option<Recon> {
|
||||
let mut r = Recon::default();
|
||||
r.cams.insert(kfs[0], BaCam { pose: Pose::IDENTITY, f });
|
||||
r.cams.insert(kfs[0], BaCam { pose: Pose::IDENTITY, f, dist: Distortion::NONE });
|
||||
let ray = |p: [f64; 2], f: f64| normalize([p[0] / f, p[1] / f, 1.0]);
|
||||
for w in kfs.windows(2) {
|
||||
if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) {
|
||||
|
|
@ -519,7 +526,7 @@ fn grow_tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&Atomi
|
|||
rel = geometry::procrustes(&ia, &ib)?;
|
||||
}
|
||||
let pa = r.cams[&a].pose.r;
|
||||
r.cams.insert(b, BaCam { pose: Pose { r: mm(&rel, &pa), c: [0.0; 3] }, f });
|
||||
r.cams.insert(b, BaCam { pose: Pose { r: mm(&rel, &pa), c: [0.0; 3] }, f, dist: Distortion::NONE });
|
||||
}
|
||||
r.triangulate(d, Kind::Tripod, 0.0, 20.0);
|
||||
let frames: Vec<usize> = r.cams.keys().copied().collect();
|
||||
|
|
@ -640,8 +647,101 @@ fn search_focal(w: f64, eval: &(dyn Fn(f64) -> f64 + Sync)) -> f64 {
|
|||
if best.1 <= errs[i] { best.0.exp() } else { fs[i] }
|
||||
}
|
||||
|
||||
/// A solve before its canonical output.
|
||||
struct Core {
|
||||
d: Data,
|
||||
r: Recon,
|
||||
method: SolveMethod,
|
||||
kind: Kind,
|
||||
cfg: Cfg,
|
||||
focal: FocalMode,
|
||||
kfs: Vec<usize>,
|
||||
}
|
||||
|
||||
/// `tracks` with every position undistorted by `dist`.
|
||||
fn undistorted(tracks: &CameraTracks, dist: &Distortion) -> CameraTracks {
|
||||
let c = [tracks.size[0] * 0.5, tracks.size[1] * 0.5];
|
||||
let mut t = tracks.clone();
|
||||
for tr in t.tracks.iter_mut() {
|
||||
for p in tr.pts.iter_mut() {
|
||||
let q = dist.undistort([p[0] as f64 - c[0], p[1] as f64 - c[1]]);
|
||||
*p = [(q[0] + c[0]) as f32, (q[1] + c[1]) as f32];
|
||||
}
|
||||
}
|
||||
t
|
||||
}
|
||||
|
||||
/// Solve the camera of a tracked clip.
|
||||
pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBool>) -> Result<CameraSolve, SolveError> {
|
||||
if !s.lens_distortion {
|
||||
let c = solve_core(tracks, s, cancel)?;
|
||||
return Ok(finish(&c.d, tracks, s, c.r, c.method, c.kind));
|
||||
}
|
||||
// Lens distortion: solve, adjust (k1, k2) jointly on the raw tracks, then solve again on
|
||||
// tracks undistorted with the estimate and adjust jointly once more.
|
||||
let mut dist = Distortion::for_size(tracks.size);
|
||||
let mut out = None;
|
||||
for pass in 0..2 {
|
||||
let und = (pass > 0).then(|| undistorted(tracks, &dist));
|
||||
let c = solve_core(und.as_ref().unwrap_or(tracks), s, cancel)?;
|
||||
let d = Data::new(tracks, &s.deleted);
|
||||
d.set_noise(c.d.noise());
|
||||
let mut r = c.r;
|
||||
for cam in r.cams.values_mut() {
|
||||
cam.dist = dist;
|
||||
}
|
||||
let cfg = Cfg { distortion: true, ..c.cfg };
|
||||
let all: Vec<usize> = r.cams.keys().copied().collect();
|
||||
let frames: Vec<usize> = if all.len() <= 160 { all.clone() } else { c.kfs.iter().copied().filter(|k| r.cams.contains_key(k)).collect() };
|
||||
if frames.len() < 2 {
|
||||
return err("the camera could not be solved (not enough parallax or features)");
|
||||
}
|
||||
// The pinhole solve's outliers include the distorted frame edges: start from all
|
||||
// observations (the Huber loss keeps real outliers in check), then reject again.
|
||||
r.outliers.clear();
|
||||
r.adjust(&d, &frames, &cfg, c.focal, 30);
|
||||
r.outliers.clear();
|
||||
r.reject(&d);
|
||||
r.adjust(&d, &frames, &cfg, c.focal, 30);
|
||||
dist = r.cams[&frames[0]].dist;
|
||||
for cam in r.cams.values_mut() {
|
||||
cam.dist = dist;
|
||||
}
|
||||
if frames.len() < all.len() {
|
||||
// Refine the other frames against the adjusted points and distortion.
|
||||
let base = r.clone();
|
||||
let rcfg = Cfg { distortion: false, ..cfg };
|
||||
let refined: Vec<(usize, Option<BaCam>)> = all
|
||||
.par_iter()
|
||||
.filter(|k| !frames.contains(k))
|
||||
.map(|k| {
|
||||
let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() };
|
||||
let start = local.cams[k];
|
||||
let ok = local.register(&d, *k, start, &rcfg);
|
||||
(*k, if ok { local.cams.get(k).copied() } else { None })
|
||||
})
|
||||
.collect();
|
||||
for (k, cam) in refined {
|
||||
if let Some(cam) = cam {
|
||||
r.cams.insert(k, cam);
|
||||
}
|
||||
}
|
||||
r.reject(&d);
|
||||
}
|
||||
if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) {
|
||||
return err("cancelled");
|
||||
}
|
||||
if debug() {
|
||||
eprintln!("lens distortion pass {pass}: k1 {:.5} k2 {:.5}", dist.k1, dist.k2);
|
||||
}
|
||||
let mut sol = finish(&d, tracks, s, r, c.method, c.kind);
|
||||
sol.distortion = Some(dist);
|
||||
out = Some(sol);
|
||||
}
|
||||
out.ok_or_else(|| SolveError("the camera could not be solved".into()))
|
||||
}
|
||||
|
||||
fn solve_core(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBool>) -> Result<Core, SolveError> {
|
||||
let d = Data::new(tracks, &s.deleted);
|
||||
if d.n < 2 {
|
||||
return err("the clip must be at least two frames long");
|
||||
|
|
@ -653,7 +753,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo
|
|||
let cancelled = || cancel.is_some_and(|c| c.load(Ordering::Relaxed));
|
||||
let fixed_f = (s.shot == ShotType::SpecifyAngle).then(|| focal_of(d.w, if s.hfov > 0.0 { s.hfov } else { 40.0 }));
|
||||
let variable = s.shot == ShotType::VariableZoom;
|
||||
let gcfg = Cfg { kind: Kind::General, focal: FocalMode::Fixed, per_frame_f: false };
|
||||
let gcfg = Cfg { kind: Kind::General, focal: FocalMode::Fixed, per_frame_f: false, distortion: false };
|
||||
let tcfg = Cfg { kind: Kind::Tripod, ..gcfg };
|
||||
// The focal-length search runs on the first keyframes only.
|
||||
let mini: Vec<usize> = kfs.iter().copied().take(10).collect();
|
||||
|
|
@ -738,7 +838,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo
|
|||
}
|
||||
let (att, _) = results.swap_remove(pick);
|
||||
let kind = if att.method == SolveMethod::TripodPan { Kind::Tripod } else { Kind::General };
|
||||
let cfg = Cfg { kind, focal: FocalMode::Fixed, per_frame_f: variable };
|
||||
let cfg = Cfg { kind, focal: FocalMode::Fixed, per_frame_f: variable, distortion: false };
|
||||
let mut r = att.recon;
|
||||
let focal = match (fixed_f, variable) {
|
||||
(Some(_), _) => FocalMode::Fixed,
|
||||
|
|
@ -845,7 +945,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo
|
|||
(m, _) => m,
|
||||
};
|
||||
lap("final adjustment");
|
||||
Ok(finish(&d, tracks, s, r, method, kind))
|
||||
Ok(Core { d, r, method, kind, cfg, focal, kfs: kf_frames })
|
||||
}
|
||||
|
||||
/// Thickness of a point cloud: √(smallest / middle eigenvalue of its covariance).
|
||||
|
|
@ -949,5 +1049,6 @@ fn finish(d: &Data, tracks: &CameraTracks, s: &SolveSettings, r: Recon, method:
|
|||
frames,
|
||||
points,
|
||||
ground: None,
|
||||
distortion: None,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ fn path_error(s: &CameraSolve, t: &Truth) -> (f64, f64) {
|
|||
}
|
||||
|
||||
fn settings(shot: ShotType, method: SolveMethod) -> SolveSettings {
|
||||
SolveSettings { shot, hfov: 0.0, method, deleted: vec![] }
|
||||
SolveSettings { shot, hfov: 0.0, method, deleted: vec![], lens_distortion: false }
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -149,7 +149,8 @@ fn specified_angle_of_view_is_kept() {
|
|||
let pts = random_points(300, 5, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]);
|
||||
let tr = tracks_of(&pts, &t, 6, true);
|
||||
let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees();
|
||||
let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: vec![] }, None).unwrap();
|
||||
let s =
|
||||
solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: vec![], lens_distortion: false }, None).unwrap();
|
||||
assert!((s.focal() - f).abs() < 1e-6);
|
||||
assert!(s.average_error < 0.5);
|
||||
let (pe, _) = path_error(&s, &t);
|
||||
|
|
@ -226,7 +227,9 @@ fn mostly_flat_scene_and_ground_plane() {
|
|||
}
|
||||
// Ground plane from the solved (coplanar) points: every point lies on it.
|
||||
let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees();
|
||||
let mut s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::MostlyFlat, deleted: vec![] }, None).expect("solve");
|
||||
let mut s =
|
||||
solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::MostlyFlat, deleted: vec![], lens_distortion: false }, None)
|
||||
.expect("solve");
|
||||
let ids: Vec<u32> = s.visible(0).map(|p| p.id).take(12).collect();
|
||||
let g = s.ground_from(&ids, 0).expect("ground");
|
||||
let spread = s.points.iter().map(|p| dot(sub(p.pos, g.origin), g.normal).abs()).fold(0.0, f64::max);
|
||||
|
|
@ -251,7 +254,9 @@ fn deleted_tracks_are_ignored_and_cancel_stops() {
|
|||
let tr = tracks_of(&pts, &t, 14, true);
|
||||
let deleted: Vec<u32> = (0..40).collect();
|
||||
let hfov = 2.0 * (W * 0.5 / 1000.0f64).atan().to_degrees();
|
||||
let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: deleted.clone() }, None).unwrap();
|
||||
let s =
|
||||
solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: deleted.clone(), lens_distortion: false }, None)
|
||||
.unwrap();
|
||||
assert!(s.points.iter().all(|p| !deleted.contains(&p.id)));
|
||||
let stop = std::sync::atomic::AtomicBool::new(true);
|
||||
let r = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), Some(&stop));
|
||||
|
|
@ -436,3 +441,55 @@ fn solve_saved_tracks() {
|
|||
t0.elapsed()
|
||||
);
|
||||
}
|
||||
|
||||
/// Pass every track position through a lens distortion (around the frame centre).
|
||||
fn distort_tracks(t: &CameraTracks, d: &Distortion) -> CameraTracks {
|
||||
let mut t = t.clone();
|
||||
for tr in t.tracks.iter_mut() {
|
||||
for p in tr.pts.iter_mut() {
|
||||
let q = d.distort([p[0] as f64 - W / 2.0, p[1] as f64 - H / 2.0]);
|
||||
*p = [(q[0] + W / 2.0) as f32, (q[1] + H / 2.0) as f32];
|
||||
}
|
||||
}
|
||||
t
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lens_distortion_is_recovered_in_the_bundle_adjustment() {
|
||||
let f = 1000.0;
|
||||
let t = moving_camera(36, |_| f);
|
||||
let pts = random_points(450, 7, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]);
|
||||
let truth = Distortion { k1: -0.12, k2: 0.03, radius: 0.5 * W.hypot(H) };
|
||||
let tr = distort_tracks(&tracks_of(&pts, &t, 8, true), &truth);
|
||||
let plain = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), None).expect("pinhole solve");
|
||||
let lens = solve(&tr, &SolveSettings { lens_distortion: true, ..settings(ShotType::FixedAngle, SolveMethod::Typical) }, None).expect("lens solve");
|
||||
let d = lens.distortion.expect("distortion solved");
|
||||
eprintln!(
|
||||
"pinhole error {:.3} px (focal {:.1}); with distortion {:.3} px (focal {:.1}), k1 {:.4} k2 {:.4}",
|
||||
plain.average_error,
|
||||
plain.focal(),
|
||||
lens.average_error,
|
||||
lens.focal(),
|
||||
d.k1,
|
||||
d.k2
|
||||
);
|
||||
assert!(plain.distortion.is_none());
|
||||
assert!((d.k1 - truth.k1).abs() < 0.005, "k1 {}", d.k1);
|
||||
assert!((d.k2 - truth.k2).abs() < 0.01, "k2 {}", d.k2);
|
||||
// The distortion model explains the tracks: about the noise level, far below the pinhole fit.
|
||||
assert!(lens.average_error < 0.3, "{}", lens.average_error);
|
||||
assert!(lens.average_error < 0.5 * plain.average_error, "{} vs {}", lens.average_error, plain.average_error);
|
||||
assert!((lens.focal() - f).abs() < 0.02 * f, "focal {}", lens.focal());
|
||||
let (path, rot) = path_error(&lens, &t);
|
||||
assert!(path < 0.01 && rot < 0.3, "path {path} rot {rot}");
|
||||
// Projection through the lens lands on the (distorted) tracks; undistorting inverts it.
|
||||
let p = [100.0, 80.0];
|
||||
let q = lens.to_image(p);
|
||||
let back = lens.to_ideal(q);
|
||||
assert!((back[0] - p[0]).abs() < 1e-6 && (back[1] - p[1]).abs() < 1e-6);
|
||||
let sp = &lens.points[0];
|
||||
let k = sp.first as usize;
|
||||
let obs = tr.track(sp.id).and_then(|t| t.at(k as u32)).expect("observed");
|
||||
let proj = lens.project(k, sp.pos, false).expect("in front");
|
||||
assert!((proj[0] - obs[0]).hypot(proj[1] - obs[1]) < 2.0, "{proj:?} vs {obs:?}");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -624,6 +624,11 @@ impl FaceTracker {
|
|||
};
|
||||
let mut t = FaceTracker { skin, prev: guess, expect: canon };
|
||||
let fit = t.fit(frame)?;
|
||||
// A face shows at least two of: left eye, right eye, mouth.
|
||||
let found = [EYE[0][4], EYE[1][4], MOUTH[0]].iter().filter(|i| fit.detected[**i]).count();
|
||||
if found < 2 {
|
||||
return None;
|
||||
}
|
||||
t.accept(&fit);
|
||||
Some((t, fit))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,6 +38,8 @@ struct Active {
|
|||
show_3d: bool,
|
||||
point_size: f64,
|
||||
target_size: f64,
|
||||
/// Undistort Footage is on: points project without the lens distortion.
|
||||
undistort: bool,
|
||||
/// Layer pixels → comp pixels.
|
||||
l2c: Mat3,
|
||||
frame: usize,
|
||||
|
|
@ -66,6 +68,7 @@ fn active(s: &Session, comp: &Comp, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Laye
|
|||
let t = tracks.as_ref()?;
|
||||
(t.frame_duration > 0.0).then(|| ((lt - t.start) / t.frame_duration).round().clamp(0.0, t.frames.saturating_sub(1) as f64) as usize)
|
||||
});
|
||||
let undistort = solve.as_ref().is_some_and(|sv| ct::undistorts(¶ms, sv));
|
||||
return Some(Active {
|
||||
layer: *lid,
|
||||
uid: *uid,
|
||||
|
|
@ -74,6 +77,7 @@ fn active(s: &Session, comp: &Comp, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Laye
|
|||
show_3d: val("showTrackPoints").map(|v| v.as_enum()).unwrap_or(1) == 1,
|
||||
point_size: val("trackPointSize").map(|v| v.as_f64()).unwrap_or(100.0),
|
||||
target_size: val("targetSize").map(|v| v.as_f64()).unwrap_or(100.0),
|
||||
undistort,
|
||||
l2c: l2c(ectx, l),
|
||||
frame: frame.unwrap_or(0),
|
||||
});
|
||||
|
|
@ -98,18 +102,20 @@ fn l2c_scale(m: &Mat3) -> f32 {
|
|||
|
||||
/// Draw the target disc (concentric rings in perspective) of `t` on frame `k`.
|
||||
fn draw_target(p: &egui::Painter, a: &Active, map: &ViewerMap, sv: &CameraSolve, t: &Target) {
|
||||
let Some(cam) = sv.frames.get(a.frame) else { return };
|
||||
if sv.frames.get(a.frame).is_none() {
|
||||
return;
|
||||
}
|
||||
let r = t.size * 0.5 * a.target_size / 100.0;
|
||||
for (i, frac) in [1.0, 0.66, 0.33].into_iter().enumerate() {
|
||||
let ring = ct::target_circle(t.center, t.normal, r * frac, 48);
|
||||
let pts: Vec<Pos2> = ring.iter().filter_map(|x| cam.project(sv.size, *x)).map(|q| to_screen(a, map, q)).collect();
|
||||
let pts: Vec<Pos2> = ring.iter().filter_map(|x| sv.project(a.frame, *x, a.undistort)).map(|q| to_screen(a, map, q)).collect();
|
||||
if pts.len() < 3 {
|
||||
return;
|
||||
}
|
||||
let fill = if i % 2 == 0 { Color32::from_rgba_unmultiplied(0xd0, 0x30, 0x30, 70) } else { Color32::from_rgba_unmultiplied(0xff, 0xff, 0xff, 50) };
|
||||
p.add(egui::Shape::convex_polygon(pts.clone(), fill, Stroke::new(1.0, Color32::from_rgb(0xe0, 0x40, 0x40))));
|
||||
}
|
||||
if let Some(c) = cam.project(sv.size, t.center) {
|
||||
if let Some(c) = sv.project(a.frame, t.center, a.undistort) {
|
||||
let c = to_screen(a, map, c);
|
||||
p.circle_filled(c, 3.0, Color32::from_rgb(0xff, 0xd0, 0x40));
|
||||
}
|
||||
|
|
@ -168,7 +174,7 @@ pub fn viewer_hook(
|
|||
let mut hits: Vec<Hit> = vec![];
|
||||
match (&a.solve, a.show_3d) {
|
||||
(Some(sv), true) => {
|
||||
let pts = ct::projected(sv, a.frame);
|
||||
let pts = ct::projected_with(sv, a.frame, a.undistort);
|
||||
let med = ct::median_depth(&pts);
|
||||
for (id, q, z) in pts {
|
||||
let sp = to_screen(&a, map, q);
|
||||
|
|
|
|||
|
|
@ -404,6 +404,27 @@ fn mask_mode(app: &mut EffectcraftApp, ui: &mut egui::Ui, ctx: &egui::Context, p
|
|||
bx += 34.0;
|
||||
}
|
||||
y += 34.0;
|
||||
// Face Tracking (Detailed Features): Extract & Copy Face Measurements.
|
||||
let has_points = app
|
||||
.session
|
||||
.active_comp()
|
||||
.and_then(|c| c.layer(layer))
|
||||
.and_then(|l| l.effects())
|
||||
.is_some_and(|fx| fx.groups().any(|g| g.match_id == effectcraft_engine::effects::face_track::POINTS_ID));
|
||||
if method == MaskMethod::FaceDetailed || has_points {
|
||||
let r = Rect::from_min_size(pos2(x0, y), vec2(w.min(260.0), 24.0));
|
||||
if button(app, ui, r, "Extract & Copy Face Measurements", has_points && !running, "tracker.mask.extractFace") {
|
||||
match crate::menus::invoke(app, ctx, "track.extractFaceMeasurements", json!({"layer": layer.0})) {
|
||||
Ok(v) => {
|
||||
if let Some(text) = v.get("clipboard").and_then(|t| t.as_str()) {
|
||||
ctx.copy_text(text.to_string());
|
||||
}
|
||||
}
|
||||
Err(e) => app.ui.status = e,
|
||||
}
|
||||
}
|
||||
y += 32.0;
|
||||
}
|
||||
if let Some(pr) = app.session.mask_track_progress() {
|
||||
let bar = Rect::from_min_size(pos2(x0, y), vec2(w, 6.0));
|
||||
p.rect_filled(bar, 3.0, t.field_bg);
|
||||
|
|
@ -419,7 +440,12 @@ fn mask_mode(app: &mut EffectcraftApp, ui: &mut egui::Ui, ctx: &egui::Context, p
|
|||
);
|
||||
ctx.request_repaint_after(std::time::Duration::from_millis(50));
|
||||
} else {
|
||||
p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, "Tracks the pixels inside the mask and keys its Mask Path.", Tokens::ui(11.0), t.text_faint);
|
||||
let hint = if method.is_face() {
|
||||
"Finds the face inside the mask and keys its outline (and, with Detailed Features, Face Track Points)."
|
||||
} else {
|
||||
"Tracks the pixels inside the mask and keys its Mask Path."
|
||||
};
|
||||
p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, hint, Tokens::ui(11.0), t.text_faint);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -105,11 +105,18 @@ effectcraft-cli run clip.ecproj track.motion '{"layer":"#2"}' \
|
|||
|
||||
`track.mask` follows the pixels inside a mask and keys its Mask Path on every frame (vertices and
|
||||
tangents move with the fitted motion). `method` is `position`, `positionScale`,
|
||||
`positionScaleRotation` (the default, also kept as the Tracker panel's Method), `positionScaleRotationSkew`
|
||||
or `perspective`; `direction` is `forward|backward|frameForward|frameBackward` from the current
|
||||
`positionScaleRotation` (the default, also kept as the Tracker panel's Method), `positionScaleRotationSkew`,
|
||||
`perspective`, `faceOutline` (Face Tracking (Outline Only): the mask, drawn around a face, becomes
|
||||
the face outline on every frame) or `faceDetailed` (Face Tracking (Detailed Features): also keys a
|
||||
Face Track Points effect with one point per landmark — eyebrows, eyes, pupils, nose tip,
|
||||
nostrils, mouth corners and lips, chin, jaw); `direction` is `forward|backward|frameForward|frameBackward` from the current
|
||||
time (or `start` / `end` in seconds). Like `track.analyze` it runs in the background in the app
|
||||
(progress in `track.status` under `mask`, `track.stop` cancels and keeps what was tracked) and is
|
||||
one undo step; `"wait": true` blocks.
|
||||
one undo step; `"wait": true` blocks. `track.extractFaceMeasurements {layer?}` (Extract & Copy Face
|
||||
Measurements) keys a Face Measurements effect from the Face Track Points (head position, scale and
|
||||
orientation X/Y/Z, eye openness, eyebrow distance from eye, mouth openness, width and offset), puts
|
||||
those keys on the keyframe clipboard and returns them per frame plus a tab-separated `clipboard`
|
||||
text.
|
||||
|
||||
`mask.interpolate` (Window ▸ Mask Interpolation ▸ Apply) adds in-between Mask Path keys between
|
||||
each pair of selected Mask Path keys (or the existing keys at `times`, in seconds), giving both
|
||||
|
|
@ -138,7 +145,11 @@ the app re-analyses automatically (headless: run `warp.analyze` again). Settings
|
|||
properties under the instance's groups, e.g. `effects/#1/stabilization/result` (0 Smooth Motion,
|
||||
1 No Motion), `effects/#1/stabilization/smoothness`, `effects/#1/stabilization/method`,
|
||||
`effects/#1/borders/framing` (0 Stabilize Only … 3 Stabilize, Synthesize Edges),
|
||||
`effects/#1/borders/autoScale/maximumScale`, `effects/#1/advanced/showTrackPoints`.
|
||||
`effects/#1/borders/autoScale/maximumScale`, `effects/#1/advanced/showTrackPoints`. Method 3
|
||||
(Subspace Warp, the default) bends a mesh per frame to the smoothed feature trajectories, so
|
||||
scenes with depth stabilise where one perspective transform cannot;
|
||||
`effects/#1/advanced/rollingShutterRipple` (0 Automatic Reduction, 1 Enhanced Reduction: a finer,
|
||||
softer mesh) tunes it.
|
||||
|
||||
```sh
|
||||
effectcraft-cli run shaky.ecproj track.warpStabilizer '{"layer":"#1","wait":true}' \
|
||||
|
|
@ -150,13 +161,17 @@ effectcraft-cli exec warp.status '{"layer":"#1"}' shaky.ecproj --json
|
|||
### 3D Camera Tracker
|
||||
|
||||
Animation ▸ Track Camera and the Tracker panel's button (`track.camera {layer?, shotType?:
|
||||
fixed|variable|specify, aov?, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}`) apply
|
||||
fixed|variable|specify, aov?, solveMethod?: auto|typical|flat|tripod, detailed?, lensDistortion?,
|
||||
undistort?, wait?}`) apply
|
||||
Effect ▸ Perspective ▸ 3D Camera Tracker (or reuse the layer's) and analyse it in the background:
|
||||
"Analyzing in background (step 1 of 2)" tracks features, "Solving camera" solves.
|
||||
`camera.analyze {layer?, effect?, wait?}` re-runs it, `camera.cancel` stops it without writing
|
||||
anything. `camera.solveStatus` reports `progress`/`banner`, `analyzed`, `solved`, `methodUsed`,
|
||||
`averageError` (pixels), `focalLength`, `horizontalAngleOfView`, the current frame's `camera`
|
||||
(position, orientation, zoom) and `groundPlane`. `camera.points {time?}` lists the solved points
|
||||
(position, orientation, zoom), `groundPlane` and the solved `lensDistortion` (`k1`, `k2`, normalised by
|
||||
half the frame diagonal) when Solve Lens Distortion (`effects/#1/advanced/lensDistortion`, also on
|
||||
with Detailed Analysis) is on; `effects/#1/advanced/undistort` renders the footage undistorted so
|
||||
the solved camera's 3D layers line up. `camera.points {time?}` lists the solved points
|
||||
visible now with their `id`, `comp` position, `depth`, `world` position and `error`.
|
||||
|
||||
Select points with `camera.selectPoints {points, add?, toggle?}` (the viewer's click / Shift-click /
|
||||
|
|
|
|||
|
|
@ -228,6 +228,13 @@ tracks them with pyramidal Lucas–Kanade and fits the chosen model (position
|
|||
RANSAC; the motion moves the Mask Path's vertices and tangents and is keyed per frame. **Mask
|
||||
Interpolation** (`mask.interpolate`) uses `effectcraft-path`'s smart interpolation (arc-length
|
||||
vertex insertion, shape-context matching, rigid in-betweens) to key in-between shapes.
|
||||
The two **Face Tracking** methods (`effectcraft_track::face`) fit a face inside the mask instead:
|
||||
a skin colour model from the first frame, the face outline along rays from the skin component's
|
||||
centre (an area-moment ellipse gives centre, size and roll), facial features as the non-skin
|
||||
components inside it, and an active-shape point distribution model trained on synthetic face
|
||||
shapes (our own generator, no external weights) that fills and checks the landmarks. The outline
|
||||
keys the Mask Path; Detailed Features keys a Face Track Points effect, and
|
||||
`track.extractFaceMeasurements` derives a keyed Face Measurements effect (and copies its keys).
|
||||
|
||||
**Warp Stabilizer** (`effects::warp_stab`, `engine::warp`): `warp.analyze` renders the layer's
|
||||
input to the effect (`Renderer::layer_input`: source, masks and the effects above it) for every
|
||||
|
|
@ -236,7 +243,10 @@ translation / similarity / homography fits) as JSON in the effect's hidden Analy
|
|||
with a key of the layer's source, In/Out, start, stretch and Time Remap. `Session::edit` clears
|
||||
analyses whose key no longer matches and queues them; the desktop app re-analyses them in the
|
||||
background. Rendering derives the stabilization plan (smoothed camera path or No Motion, framing,
|
||||
auto-scale; cached per analysis and settings) and warps each frame; Synthesize Edges fills the
|
||||
auto-scale; cached per analysis and settings) and warps each frame. Subspace Warp also stores long
|
||||
feature trajectories in the analysis, smooths them in a low-rank subspace (Liu et al. 2011) and
|
||||
warps each frame with a content-preserving mesh (Liu et al. 2009) fitted to the smoothed
|
||||
positions, rendered as a per-pixel mesh lookup; Synthesize Edges fills the
|
||||
borders from neighbouring frames read with `EffectHost::self_at`. The effect lives in the
|
||||
effects crate, hence the `effects → track` edge.
|
||||
|
||||
|
|
@ -267,7 +277,10 @@ the essential matrix (normalised 8-point in RANSAC) or a planar homography decom
|
|||
resection and triangulation; a sparse Levenberg–Marquardt bundle adjustment (Schur complement over
|
||||
the points, Huber loss) with the focal length fixed, shared, or per frame; a log-spaced and
|
||||
golden-section focal search for Fixed Angle of View / Variable Zoom; a rotation-only model for tripod
|
||||
pans, chosen by Auto Detect when it explains the tracks as well. Tracks and solve are stored as JSON
|
||||
pans, chosen by Auto Detect when it explains the tracks as well; with Solve Lens Distortion (or
|
||||
Detailed Analysis) radial distortion `k1, k2` is adjusted jointly in a final bundle adjustment on
|
||||
the raw tracks and the solve repeated on tracks undistorted with that estimate; Undistort Footage
|
||||
renders the input through the inverse model. Tracks and solve are stored as JSON
|
||||
in hidden effect parameters with keys: changing the layer's frames clears both, changing Shot Type,
|
||||
Angle of View, Solve Method or deleting points only re-solves. The solve's canonical frame maps to
|
||||
comp space so the first frame's camera is the default comp camera (or so a chosen ground plane is
|
||||
|
|
|
|||
|
|
@ -3,11 +3,11 @@
|
|||
<!-- Generated from the effect registry (crates/effects/src/catalog.rs); do not edit by hand.
|
||||
Regenerate: UPDATE_DOCS=1 cargo test -p effectcraft-effects --lib effects_doc_is_current -->
|
||||
|
||||
EffectCraft ships 304 effects with 2446 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over.
|
||||
EffectCraft ships 306 effects with 2488 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over.
|
||||
|
||||
- **GPU**: 59 effects also run on the GPU compositor with identical results.
|
||||
- **32**: 304 effects process 32-bit float (HDR, overbright) pixels without clamping.
|
||||
- **Status**: 262 implemented in full, 42 partial (what is missing is listed).
|
||||
- **32**: 306 effects process 32-bit float (HDR, overbright) pixels without clamping.
|
||||
- **Status**: 264 implemented in full, 42 partial (what is missing is listed).
|
||||
|
||||
Effects are addressed by id (`ec.<category>.<name>`) or display name in commands, scripts and the control channel, and every parameter by its id (`effects/#1/blurriness`).
|
||||
|
||||
|
|
@ -302,7 +302,7 @@ Effects are addressed by id (`ec.<category>.<name>`) or display name in commands
|
|||
|
||||
| Effect | Id | Params | GPU | 32 | Status |
|
||||
|---|---|---:|:-:|:-:|---|
|
||||
| 3D Camera Tracker | `ec.perspective.cameratracker` | 10 | | 32 | Implemented |
|
||||
| 3D Camera Tracker | `ec.perspective.cameratracker` | 12 | | 32 | Implemented |
|
||||
| 3D Glasses | `ec.perspective.3dglasses` | 8 | | 32 | Implemented |
|
||||
| Bevel Alpha | `ec.perspective.bevelalpha` | 4 | | 32 | Implemented |
|
||||
| Bevel Edges | `ec.perspective.beveledges` | 4 | | 32 | Implemented |
|
||||
|
|
@ -416,6 +416,8 @@ Effects are addressed by id (`ec.<category>.<name>`) or display name in commands
|
|||
| CC Overbrights | `ec.utility.ccoverbrights` | 2 | | 32 | Implemented |
|
||||
| Cineon Converter | `ec.utility.cineon` | 7 | | 32 | Implemented |
|
||||
| Color Profile Converter | `ec.utility.colorprofileconverter` | 7 | | 32 | Implemented |
|
||||
| Face Measurements | `ec.utility.facemeasurements` | 14 | | 32 | Implemented |
|
||||
| Face Track Points | `ec.utility.facetrackpoints` | 26 | | 32 | Implemented |
|
||||
| Grow Bounds | `ec.utility.growbounds` | 1 | | 32 | Implemented |
|
||||
| HDR Compander | `ec.utility.hdrcompander` | 3 | | 32 | Implemented |
|
||||
| HDR Highlight Compression | `ec.utility.hdrcompression` | 1 | | 32 | Implemented |
|
||||
|
|
|
|||
|
|
@ -17,13 +17,13 @@ How close EffectCraft is to After Effects 2026, feature by feature, and how much
|
|||
|
||||
By area: Layers 98%, Output 95%, Compositions 95%, Automation 96%, Paint 95%, Text 95%, Import 96%,
|
||||
Animation 94%, Masks 94%, Preview 94%, Interface 93%, Shapes 93%, 3D 91%, Audio 90%, Project 90%,
|
||||
Tracking 88%, Effects 85%, Web 65%.
|
||||
Tracking 96%, Effects 85%, Web 65%.
|
||||
|
||||
What is left, in priority order: the web app's depth (threads, storage, audio, non-blocking
|
||||
renders); stroke taper/wave and multi-segment dashes; variable mask feather points; camera iris/bokeh and focus-link commands; Render Queue field render, crop/resize and templates;
|
||||
approximated effects (Subspace Warp, Key Cleaner) and Liquify's viewer brush; OpenType features and variable
|
||||
approximated effects (Key Cleaner) and Liquify's viewer brush; OpenType features and variable
|
||||
font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; ScriptUI;
|
||||
Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; face tracking; and
|
||||
Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; and
|
||||
the "better than After Effects" items (a plug-in API, branching history; GPU particles landed in M12.7).
|
||||
|
||||
The sections below are the original audit (morning of 2 October, ≈ 64%) and its updates, kept for
|
||||
|
|
@ -124,7 +124,7 @@ the disk cache in the formats wave; the 3D Camera Tracker in M12.6.)
|
|||
| Project | ≈ 68% | 5.5 | auto-save, folder moves, OCIO displays beyond the built-in tone map (Color Engine with OCIO/ACES working spaces, HDR compand/tone mapping, Rec. 2100 PQ/HLG output, Feet + Frames, display colour management, Simulate Output and the locked viewer landed in M7.7; proxies and Interpret Footage fields / pixel aspect / alpha guess landed: PRJ-8, PRJ-3) |
|
||||
| Masks & roto | 74% | 5.0 | Roto Brush's learned (3.0) segmentation model (variable-width mask feather points with the Mask Feather tool landed in M13.5; mask tracking and Mask Interpolation landed in M6.6; Roto Brush & Refine Edge with graph-cut segmentation, flow propagation, edge matting, decontamination and Freeze in M6.7) |
|
||||
| Preview | 66% | 4.0 | GPU bokeh depth of field, wireframes and Advanced 3D compositing (Classic 3D runs and adjustment layers composite on the GPU since M12.7; persistent disk cache with the blue cache bar landed; region of interest, snapshots, exposure and Fast Previews landed in M0.13) |
|
||||
| Tracking | ≈ 85% | 2.5 | face tracking, Subspace Warp's mesh warp, lens distortion in the camera solve (Rolling Shutter Repair landed in M9.11; point tracker, mask tracking, Warp Stabilizer and the 3D Camera Tracker landed in M6.x / M12.5 / M12.6) |
|
||||
| Tracking | ≈ 96% | 0.3 | face tracking is a classical (skin model + feature components + shape model) fitter, not a learned detector: profile views and occluded faces are weak; Rolling Shutter Ripple is approximated by Subspace Warp's mesh density (face tracking (Outline Only / Detailed Features with Face Track Points and Extract & Copy Face Measurements), Subspace Warp's content-preserving mesh warp on subspace-smoothed trajectories, and radial lens distortion (k1, k2) in the camera bundle adjustment with Undistort Footage landed in M13.3; Rolling Shutter Repair in M9.11; point tracker, mask tracking, Warp Stabilizer and the 3D Camera Tracker in M6.x / M12.5 / M12.6) |
|
||||
| Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser |
|
||||
|
||||
## Effects still missing
|
||||
|
|
|
|||
Loading…
Reference in New Issue