diff --git a/crates/effects/src/camera_tracker.rs b/crates/effects/src/camera_tracker.rs index dd97b585..ff4e51fe 100644 --- a/crates/effects/src/camera_tracker.rs +++ b/crates/effects/src/camera_tracker.rs @@ -8,8 +8,12 @@ //! Shot Type, Angle of View, Solve Method and deleted points it was solved with, so changing //! those re-solves without re-tracking). **Method Used** and **Average Error** report the solve. //! -//! The effect passes its input through; with **Render Track Points** on it draws the solved -//! points (as the viewer shows them) into the frame. The viewer overlay, target and the Create +//! **Solve Lens Distortion** (Advanced; also on with Detailed Analysis) solves radial lens +//! distortion `k1, k2` jointly with the camera; **Undistort Footage** then renders the input +//! undistorted (the pinhole image the solved camera sees) so 3D layers line up with it. +//! +//! The effect passes its input through (undistorted when asked); with **Render Track Points** +//! on it draws the solved points (as the viewer shows them) into the frame. The viewer overlay, target and the Create //! commands live in the engine and the UI. use std::collections::HashMap; @@ -19,6 +23,7 @@ use effectcraft_keyframe::Value; use effectcraft_project::ParamUi; use effectcraft_raster::{Image, Px}; use effectcraft_track::camtrack::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, linalg, point_color}; +use rayon::prelude::*; use crate::warp_stab::param; use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; @@ -58,6 +63,8 @@ pub fn specs() -> Vec { p(METHOD_USED, "Method Used", Value::Str(String::new()), ParamUi::Hidden), p(AVERAGE_ERROR, "Average Error", num(0.0), ParamUi::Hidden), p("advanced/detailedAnalysis", "Detailed Analysis", Value::Bool(false), ParamUi::Checkbox), + p("advanced/lensDistortion", "Solve Lens Distortion", Value::Bool(false), ParamUi::Checkbox), + p("advanced/undistort", "Undistort Footage", Value::Bool(false), ParamUi::Checkbox), p("advanced/autoDeletePoints", "Auto-delete Points Across Time", Value::Bool(true), ParamUi::Checkbox), p("advanced/hideWarningBanner", "Hide Warning Banner", Value::Bool(false), ParamUi::Checkbox), p(TRACKS, "Tracks", Value::Str(String::new()), ParamUi::Hidden), @@ -104,13 +111,15 @@ pub fn settings(params: &Params) -> SolveSettings { hfov: f(params, "horizontalAngleOfView"), method: SolveMethod::ALL.get(e(params, "advanced/solveMethod") as usize).copied().unwrap_or_default(), deleted: del, + lens_distortion: b(params, "advanced/lensDistortion") || b(params, "advanced/detailedAnalysis"), } } /// What a solve depends on besides the tracks. pub fn solve_key(st: &SolveSettings) -> String { let hfov = if st.shot == ShotType::SpecifyAngle { format!("{:.4}", st.hfov) } else { String::new() }; - format!("{:?}|{hfov}|{:?}|{:?}", st.shot, st.method, st.deleted) + let lens = if st.lens_distortion { "|lens" } else { "" }; + format!("{:?}|{hfov}|{:?}|{:?}{lens}", st.shot, st.method, st.deleted) } fn fnv(bytes: &[u8]) -> u64 { @@ -165,18 +174,49 @@ pub fn target_radius(size_pct: f64, z: f64, median_z: f64) -> f64 { (6.0 * size_pct / 100.0 * (median_z / z.max(1e-9))).clamp(1.5, 40.0) } -/// The solved points visible on frame `k`: (id, layer pixel position, camera depth). +/// The solved points visible on frame `k`: (id, layer pixel position in the footage, camera +/// depth). pub fn projected(solve: &CameraSolve, k: usize) -> Vec<(u32, [f64; 2], f64)> { + projected_with(solve, k, false) +} + +/// [`projected`] into the footage, or (`undistorted`) into the undistorted image. +pub fn projected_with(solve: &CameraSolve, k: usize, undistorted: bool) -> Vec<(u32, [f64; 2], f64)> { let Some(cam) = solve.frames.get(k) else { return vec![] }; solve .visible(k) .filter_map(|p| { let c = cam.to_cam(p.pos); - cam.project(solve.size, p.pos).map(|q| (p.id, q, c[2])) + solve.project(k, p.pos, undistorted).map(|q| (p.id, q, c[2])) }) .collect() } +/// Whether the instance renders its input undistorted (Undistort Footage with a solved +/// distortion). +pub fn undistorts(params: &Params, solve: &CameraSolve) -> bool { + b(params, "advanced/undistort") && solve.distortion.is_some_and(|d| !d.is_none()) +} + +/// The input resampled through the solved distortion: every output pixel shows the footage +/// at its distorted position (transparent where that falls outside the layer). +fn undistort(buf: &Buf, sol: &CameraSolve) -> Buf { + let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + let mut out = Image::new(buf.img.width, buf.img.height); + let size = sol.size; + out.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let p = [(x as f64 + 0.5 - buf.offset[0]) / sc, (y as f64 + 0.5 - buf.offset[1]) / sc]; + let q = sol.to_image(p); + if q[0] < 0.0 || q[1] < 0.0 || q[0] > size[0] || q[1] > size[1] { + continue; + } + *px = buf.img.sample_bilinear_clamped(q[0] * sc + buf.offset[0], q[1] * sc + buf.offset[1]); + } + }); + Buf { img: out, offset: buf.offset, scale: buf.scale } +} + /// Median of the depths. pub fn median_depth(v: &[(u32, [f64; 2], f64)]) -> f64 { let mut z: Vec = v.iter().map(|x| x.2).collect(); @@ -188,15 +228,23 @@ pub fn median_depth(v: &[(u32, [f64; 2], f64)]) -> f64 { } fn render(ctx: &EffectCtx, mut buf: Buf) -> Buf { - if !b(ctx.params, "renderTrackPoints") { + let points = b(ctx.params, "renderTrackPoints"); + if !points && !b(ctx.params, "advanced/undistort") { return buf; } let Some(sol) = solve(ctx.params) else { return buf }; if (sol.size[0] - ctx.layer_size[0]).abs() > 0.5 || (sol.size[1] - ctx.layer_size[1]).abs() > 0.5 { return buf; } + let und = undistorts(ctx.params, &sol); + if und { + buf = undistort(&buf, &sol); + } + if !points { + return buf; + } let Some(k) = sol.frame_at(ctx.time) else { return buf }; - let pts = projected(&sol, k); + let pts = projected_with(&sol, k, und); let med = median_depth(&pts); let size = f(ctx.params, "trackPointSize"); let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; @@ -287,5 +335,37 @@ mod tests { let st = settings(&Params::default()); assert_eq!(st.shot, ShotType::FixedAngle); assert!(solve_key(&st).contains("FixedAngle")); + assert!(!st.lens_distortion && !solve_key(&st).contains("lens")); + } + + #[test] + fn undistort_footage_remaps_through_the_solved_lens() { + use effectcraft_track::camtrack::Distortion; + let sol = CameraSolve { + version: 1, + size: [64.0, 48.0], + start: 0.0, + frame_duration: 1.0 / 30.0, + frames: vec![SolvedFrame { rot: [0.0; 3], center: [0.0; 3], focal: 50.0, solved: true }], + distortion: Some(Distortion { k1: -0.2, k2: 0.0, radius: 40.0 }), + ..Default::default() + }; + let mut params = params_with(&sol, false); + params.values.insert("advanced/undistort".into(), Value::Bool(true)); + assert!(undistorts(¶ms, &sol)); + // A horizontal ramp: undistortion keeps the centre and pulls edge content outwards. + let mut img = Image::new(64, 48); + for (i, px) in img.data.iter_mut().enumerate() { + let x = (i % 64) as f32 / 64.0; + *px = [x, x, x, 1.0]; + } + let ctx = EffectCtx { params: ¶ms, time: 0.0, layer_size: [64.0, 48.0], seed: 0, adjustment: false, env: EffectEnv::default() }; + let out = render(&ctx, Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 }); + let at = |im: &Image, x: u32, y: u32| im.data[im.idx(x, y)][0]; + assert!((at(&out.img, 32, 24) - at(&img, 32, 24)).abs() < 0.02); + // Barrel distortion compresses the edges: the ideal pixel at x = 56 shows footage nearer the centre. + assert!(at(&out.img, 56, 24) < at(&img, 56, 24) - 0.01, "{} vs {}", at(&out.img, 56, 24), at(&img, 56, 24)); + let sp = sol.to_image([56.0, 24.0]); + assert!(sp[0] < 56.0 && (sol.to_ideal(sp)[0] - 56.0).abs() < 1e-6); } } diff --git a/crates/effects/src/face_track.rs b/crates/effects/src/face_track.rs new file mode 100644 index 00000000..7b925165 --- /dev/null +++ b/crates/effects/src/face_track.rs @@ -0,0 +1,45 @@ +//! Face tracking results (Tracker panel ▸ Method ▸ Face Tracking (Detailed Features)): +//! +//! - **Face Track Points** holds one point control per facial landmark (eyebrows, eyes and +//! pupils, nose, nostrils, mouth, chin and jaw), keyed on every tracked frame by the engine's +//! mask tracker ([`effectcraft_track::face`]); expressions and other layers can follow them. +//! - **Face Measurements** holds the measurements derived from those points (head position, +//! scale and orientation, eye openness, eyebrow raise, mouth openness, width and offset), keyed +//! by Extract & Copy Face Measurements (`track.extractFaceMeasurements`). +//! +//! Both pass their input through unchanged: they are data, like expression controls. + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_track::face::{LANDMARKS, MEASUREMENTS}; + +use crate::{Buf, EffectCtx, EffectSpec, ParamSpec, num, p, slider}; + +/// Effect ids. +pub const POINTS_ID: &str = "ec.utility.facetrackpoints"; +pub const MEASUREMENTS_ID: &str = "ec.utility.facemeasurements"; + +fn passthrough(_: &EffectCtx, b: Buf) -> Buf { + b +} + +fn measurement_ui(id: &str) -> ParamUi { + match id { + "headPositionX" | "headPositionY" => slider(-100_000.0, 100_000.0, 0.0, 1920.0, 1), + "headOrientationX" | "headOrientationY" | "headOrientationZ" => slider(-180.0, 180.0, -90.0, 90.0, 1), + _ => slider(-100_000.0, 100_000.0, 0.0, 200.0, 1), + } +} + +pub fn specs() -> Vec { + let points: Vec = LANDMARKS + .iter() + // Point defaults are fractions of the layer size: the canonical face in the middle. + .map(|(id, name, q)| p(id, name, Value::Vec2([0.5 + 0.25 * q[0], 0.5 + 0.25 * q[1]]), ParamUi::Point)) + .collect(); + let measurements: Vec = MEASUREMENTS.iter().map(|(id, name)| p(id, name, num(0.0), measurement_ui(id))).collect(); + vec![ + EffectSpec { id: POINTS_ID, name: "Face Track Points", category: "Utility", params: points, render: passthrough, gpu: false, float: true }, + EffectSpec { id: MEASUREMENTS_ID, name: "Face Measurements", category: "Utility", params: measurements, render: passthrough, gpu: false, float: true }, + ] +} diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index c4a47508..febdd9da 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -22,6 +22,7 @@ mod distort; mod distort2; mod distort3; pub mod distort4; +pub mod face_track; mod generate; mod generate2; mod generate3; @@ -412,6 +413,7 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(puppet::specs()); v.extend(warp_stab::specs()); v.extend(camera_tracker::specs()); + v.extend(face_track::specs()); v.extend(roto::specs()); v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); for s in v.iter_mut() { diff --git a/crates/engine/src/commands/camera_cmds.rs b/crates/engine/src/commands/camera_cmds.rs index 5d454aed..68cd14fb 100644 --- a/crates/engine/src/commands/camera_cmds.rs +++ b/crates/engine/src/commands/camera_cmds.rs @@ -23,7 +23,7 @@ pub fn specs() -> Vec { "Track Camera", ["Animation"], None, - "{layer?, shotType?: fixed|variable|specify, aov?: degrees, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}", + "{layer?, shotType?: fixed|variable|specify, aov?: degrees, solveMethod?: auto|typical|flat|tripod, detailed?, lensDistortion?: solve k1/k2, undistort?: render undistorted, wait?}", can_track, track_camera ), @@ -140,6 +140,8 @@ fn track_camera(s: &mut Session, p: &Value) -> Result { return Err(bad("track.camera", "aov: 1..170 degrees")); } let detailed = b_p(p, "detailed"); + let lens = b_p(p, "lensDistortion"); + let undistort = b_p(p, "undistort"); let existing = s.project.comp(cid).and_then(|c| c.layer(lid)).and_then(Layer::effects).and_then(|fx| fx.groups().find(|g| is_tracker(g)).map(|g| g.uid)); let uid = match existing { Some(u) => u, @@ -148,7 +150,7 @@ fn track_camera(s: &mut Session, p: &Value) -> Result { r["effects"].get(0).and_then(Value::as_u64).ok_or_else(|| EngineError::Other("the effect could not be applied".into()))? } }; - if shot.is_some() || method.is_some() || aov.is_some() || detailed.is_some() { + if shot.is_some() || method.is_some() || aov.is_some() || detailed.is_some() || lens.is_some() || undistort.is_some() { s.edit("3D Camera Tracker Settings", None, |proj, _| { let g = proj .comp_mut(cid) @@ -176,6 +178,12 @@ fn track_camera(s: &mut Session, p: &Value) -> Result { if let Some(d) = detailed { set("advanced/detailedAnalysis", KV::Bool(d)); } + if let Some(d) = lens { + set("advanced/lensDistortion", KV::Bool(d)); + } + if let Some(d) = undistort { + set("advanced/undistort", KV::Bool(d)); + } Ok(()) })?; } @@ -222,6 +230,7 @@ fn status(s: &mut Session, p: &Value) -> Result { out["shotType"] = json!(settings.shot.label()); out["solveMethod"] = json!(settings.method.label()); out["deleted"] = json!(settings.deleted.len()); + out["lensDistortionSolved"] = json!(settings.lens_distortion); let solve = ct::solve(¶ms); out["solved"] = json!(solve.is_some()); if let Some(sv) = &solve { @@ -233,6 +242,11 @@ fn status(s: &mut Session, p: &Value) -> Result { out["focalLength"] = json!(f); out["horizontalAngleOfView"] = json!(sv.hfov(f)); out["groundPlane"] = json!(sv.ground.is_some()); + out["lensDistortion"] = match &sv.distortion { + Some(d) => json!({"k1": d.k1, "k2": d.k2, "radius": d.radius}), + None => Value::Null, + }; + out["undistort"] = json!(ct::undistorts(¶ms, sv)); if let Some(pl) = placed(&s.project, cid, lid, uid) { let k = pl.frame_at(s.time_of(cid)); let (pos, ori, zoom) = pl.camera(k); diff --git a/crates/engine/src/commands/mask_interp.rs b/crates/engine/src/commands/mask_interp.rs index 5df40c6d..a4548944 100644 --- a/crates/engine/src/commands/mask_interp.rs +++ b/crates/engine/src/commands/mask_interp.rs @@ -1,5 +1,6 @@ -//! Mask tracking (Layer ▸ Mask ▸ Track Mask, the Tracker panel in mask mode) and Smart Mask -//! Interpolation (Window ▸ Mask Interpolation). +//! Mask tracking (Layer ▸ Mask ▸ Track Mask, the Tracker panel in mask mode, including the Face +//! Tracking methods and Extract & Copy Face Measurements) and Smart Mask Interpolation +//! (Window ▸ Mask Interpolation). use effectcraft_keyframe::{Keyframe, ShapePath, Value as KV}; use effectcraft_path::interp::{self, AddVertices, InterpOpts, Matching}; @@ -20,7 +21,7 @@ pub fn specs() -> Vec { "Track Mask", ["Animation"], None, - "{layer?, mask?: uid|index|name, method?: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective, direction?: forward|backward|frameForward|frameBackward, start? (s), end? (s), wait?: block until done}", + "{layer?, mask?: uid|index|name, method?: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective|faceOutline|faceDetailed, direction?: forward|backward|frameForward|frameBackward, start? (s), end? (s), wait?: block until done}", has_mask, track_mask ), @@ -29,10 +30,19 @@ pub fn specs() -> Vec { "Mask Tracking Method", [], None, - "{method: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective}", + "{method: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective|faceOutline|faceDetailed}", super::always, mask_method ), + cmd!( + "track.extractFaceMeasurements", + "Extract & Copy Face Measurements", + [], + None, + "{layer?} → keys a Face Measurements effect from the layer's Face Track Points (one key per tracked frame) and copies those keys", + has_face_points, + extract_face + ), cmd!( "mask.interpolate", "Apply Mask Interpolation", @@ -116,9 +126,12 @@ fn has_mask(s: &Session) -> std::result::Result<(), String> { fn method_p(p: &Value, cmd: &str) -> Result> { match str_p(p, "method") { - Some(m) => MaskMethod::from_name(m) - .map(Some) - .ok_or_else(|| bad(cmd, format!("unknown method `{m}` (position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective)"))), + Some(m) => MaskMethod::from_name(m).map(Some).ok_or_else(|| { + bad( + cmd, + format!("unknown method `{m}` (position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective|faceOutline|faceDetailed)"), + ) + }), None => Ok(None), } } @@ -190,6 +203,112 @@ fn track_mask(s: &mut Session, p: &Value) -> Result { })) } +// ---------------------------------------------------------------- face measurements + +fn face_points_layer(s: &Session, p: &Value) -> Option<(effectcraft_project::ItemId, LayerId)> { + use effectcraft_effects::face_track::POINTS_ID; + let has = |l: &Layer| l.effects().is_some_and(|fx| fx.groups().any(|g| g.match_id == POINTS_ID)); + if p.get("layer").is_some() { + return layer_p(s, p, "track.extractFaceMeasurements").ok(); + } + let cid = s.active_comp_id()?; + let comp = s.project.comp(cid)?; + let sel = s.state.selected_layers.iter().filter_map(|l| comp.layer(*l)).find(|l| has(l)); + sel.or_else(|| comp.layers.iter().find(|l| has(l))).map(|l| (cid, l.id)) +} + +fn has_face_points(s: &Session) -> std::result::Result<(), String> { + has_comp(s)?; + face_points_layer(s, &json!({})).map(|_| ()).ok_or_else(|| "no Face Track Points: track a mask with Face Tracking (Detailed Features) first".into()) +} + +/// Extract & Copy Face Measurements: measure the Face Track Points at every keyed time, key them +/// into the layer's Face Measurements effect and put those keys on the keyframe clipboard. +fn extract_face(s: &mut Session, p: &Value) -> Result { + use effectcraft_effects::face_track::{MEASUREMENTS_ID, POINTS_ID}; + use effectcraft_track::face::{LANDMARKS, MEASUREMENTS, N, face_frame, measure}; + const CMD: &str = "track.extractFaceMeasurements"; + let (cid, lid) = face_points_layer(s, p).ok_or_else(|| bad(CMD, "no Face Track Points: track a mask with Face Tracking (Detailed Features) first"))?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let g = layer.effects().and_then(|fx| fx.groups().find(|g| g.match_id == POINTS_ID)).ok_or_else(|| bad(CMD, "the layer has no Face Track Points"))?; + let mut times: Vec = LANDMARKS.iter().filter_map(|l| g.get(l.0)).flat_map(|pr| pr.keys.iter().map(|k| k.time)).collect(); + times.sort(); + times.dedup(); + if times.is_empty() { + times.push(layer.layer_time(s.time_of(cid))); + } + let rows: Vec<(Tick, [f64; 14])> = { + let pts_at = |t: Tick| { + let mut l = [[0.0; 2]; N]; + for (i, (id, _, _)) in LANDMARKS.iter().enumerate() { + if let Some(KV::Vec2(v)) = g.get(id).map(|pr| pr.value_at(t)) { + l[i] = v; + } + } + l + }; + let reference = face_frame(&pts_at(times[0])).1; + times.iter().map(|t| (*t, measure(&pts_at(*t), reference))).collect() + }; + let comp_times: Vec = times.iter().map(|t| layer.comp_time(*t)).collect(); + let fd = comp.frame_duration(); + let uid = s.edit("Extract Face Measurements", None, |proj, _| { + let uid = crate::mask_track::face_effect(proj, cid, lid, MEASUREMENTS_ID).ok_or_else(|| bad(CMD, "Face Measurements is not available"))?; + let g = proj.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)).ok_or(EngineError::NoComp)?; + for (k, (id, _)) in MEASUREMENTS.iter().enumerate() { + let Some(pr) = g.get_mut(id) else { continue }; + pr.keys.clear(); + for (t, m) in &rows { + effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*t, KV::Scalar(m[k]))); + } + } + Ok(uid) + })?; + // Copy the keys (Edit ▸ Paste pastes them at the CTI, like copied keyframes). + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let mg = layer.props.find_group(uid).ok_or(EngineError::NoComp)?; + let earliest = comp_times.iter().copied().min().unwrap_or(Tick::ZERO); + let mut clip = vec![]; + for (id, _) in MEASUREMENTS { + let Some(pr) = mg.get(id) else { continue }; + let Some(path) = layer.props.match_path_of(pr.uid) else { continue }; + let keys = pr + .keys + .iter() + .map(|k| { + let mut k = k.clone(); + k.time = layer.comp_time(k.time) - earliest; + k + }) + .collect(); + clip.push(crate::KeyClip { layer: lid, path, keys }); + } + s.state.key_clipboard = clip; + s.state.clip_is_keys = true; + // The same data as text (tab-separated, one row per frame) for other applications. + let mut text = String::from("EffectCraft Face Measurements\n\nFrame\tTime"); + for (_, name) in MEASUREMENTS { + text.push('\t'); + text.push_str(name); + } + text.push('\n'); + let mut table = vec![]; + for ((_, m), ct) in rows.iter().zip(&comp_times) { + let frame = if fd > Tick::ZERO { (ct.seconds() / fd.seconds()).round() as i64 } else { 0 }; + text.push_str(&format!("{frame}\t{:.4}", ct.seconds())); + for v in m { + text.push_str(&format!("\t{v:.3}")); + } + text.push('\n'); + let obj: serde_json::Map = MEASUREMENTS.iter().zip(m).map(|((id, _), v)| (id.to_string(), json!(v))).collect(); + table.push(json!({"time": ct.seconds(), "frame": frame, "values": obj})); + } + s.bump(); + Ok(json!({"layer": lid.0, "effect": uid, "frames": rows.len(), "measurements": table, "clipboard": text})) +} + // ---------------------------------------------------------------- mask interpolation /// Mask Interpolation panel options (kept in the editor state, serde for agents). diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 057c7d10..538e8844 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -753,6 +753,8 @@ mod tests_effects; #[cfg(test)] mod tests_essential; #[cfg(test)] +mod tests_face; +#[cfg(test)] mod tests_fidelity; #[cfg(test)] mod tests_lottie; diff --git a/crates/engine/src/mask_track.rs b/crates/engine/src/mask_track.rs index 00315198..a5741052 100644 --- a/crates/engine/src/mask_track.rs +++ b/crates/engine/src/mask_track.rs @@ -6,6 +6,12 @@ //! algorithm is `effectcraft_track::mask` (KLT features inside the mask + a RANSAC fit of the //! chosen motion model), and every frame's motion is applied to the mask's vertices and //! tangents, so the shape's Bezier structure is kept. +//! +//! The two **Face Tracking** methods use `effectcraft_track::face` instead: the mask (drawn +//! around a face) seeds the tracker, and every frame's face outline becomes the Mask Path +//! (*Outline Only*); *Detailed Features* also keys the facial landmarks into a **Face Track +//! Points** effect on the layer (created on first use), from which Extract & Copy Face +//! Measurements (`track.extractFaceMeasurements`) derives a keyed **Face Measurements** effect. use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; @@ -31,11 +37,22 @@ pub enum MaskMethod { PositionScaleRotation, PositionScaleRotationSkew, Perspective, + /// Face Tracking (Outline Only). + FaceOutline, + /// Face Tracking (Detailed Features). + FaceDetailed, } impl MaskMethod { - pub const ALL: [MaskMethod; 5] = - [MaskMethod::Position, MaskMethod::PositionScale, MaskMethod::PositionScaleRotation, MaskMethod::PositionScaleRotationSkew, MaskMethod::Perspective]; + pub const ALL: [MaskMethod; 7] = [ + MaskMethod::Position, + MaskMethod::PositionScale, + MaskMethod::PositionScaleRotation, + MaskMethod::PositionScaleRotationSkew, + MaskMethod::Perspective, + MaskMethod::FaceOutline, + MaskMethod::FaceDetailed, + ]; pub fn label(self) -> &'static str { match self { MaskMethod::Position => "Position", @@ -43,8 +60,14 @@ impl MaskMethod { MaskMethod::PositionScaleRotation => "Position, Scale & Rotation", MaskMethod::PositionScaleRotationSkew => "Position, Scale, Rotation & Skew", MaskMethod::Perspective => "Perspective", + MaskMethod::FaceOutline => "Face Tracking (Outline Only)", + MaskMethod::FaceDetailed => "Face Tracking (Detailed Features)", } } + /// One of the Face Tracking methods. + pub fn is_face(self) -> bool { + matches!(self, MaskMethod::FaceOutline | MaskMethod::FaceDetailed) + } pub fn id(self) -> &'static str { match self { MaskMethod::Position => "position", @@ -52,6 +75,8 @@ impl MaskMethod { MaskMethod::PositionScaleRotation => "positionScaleRotation", MaskMethod::PositionScaleRotationSkew => "positionScaleRotationSkew", MaskMethod::Perspective => "perspective", + MaskMethod::FaceOutline => "faceOutline", + MaskMethod::FaceDetailed => "faceDetailed", } } pub fn from_name(s: &str) -> Option { @@ -67,6 +92,7 @@ impl MaskMethod { MaskMethod::PositionScaleRotation => Model::Similarity, MaskMethod::PositionScaleRotationSkew => Model::Affine, MaskMethod::Perspective => Model::Homography, + MaskMethod::FaceOutline | MaskMethod::FaceDetailed => Model::Similarity, } } } @@ -109,8 +135,29 @@ pub fn transform_path(p: &ShapePath, h: &trk::Homography) -> ShapePath { } } -/// One tracked frame: (layer time, comp time, mask path). -type MaskFrame = (Tick, Tick, ShapePath); +/// A closed, smooth mask path through a face outline polygon: `n` vertices with Catmull-Rom +/// tangents. `like` lends its feather points. +pub fn outline_path(poly: &[[f64; 2]], n: usize, like: &ShapePath) -> ShapePath { + let m = poly.len(); + let n = n.clamp(3, m.max(3)); + let v: Vec<[f64; 2]> = (0..n).map(|i| poly[(i * m / n) % m.max(1)]).collect(); + let tan: Vec<[f64; 2]> = (0..n) + .map(|i| { + let (a, b) = (v[(i + n - 1) % n], v[(i + 1) % n]); + [(b[0] - a[0]) / 6.0, (b[1] - a[1]) / 6.0] + }) + .collect(); + ShapePath { + vertices: v, + in_tangents: tan.iter().map(|t| [-t[0], -t[1]]).collect(), + out_tangents: tan, + closed: true, + feather: if like.len() == n { like.feather.clone() } else { vec![] }, + } +} + +/// One tracked frame: (layer time, comp time, mask path, face landmarks for Detailed Features). +type MaskFrame = (Tick, Tick, ShapePath, Option<[[f64; 2]; trk::face::N]>); #[derive(Default)] pub struct MaskTrackShared { @@ -169,9 +216,12 @@ fn run_work(w: MaskWork, shared: &MaskTrackShared) { let frame_at = |t: Tick| source_frame(r, &w.project, w.comp, comp, layer, t, w.expr.as_deref()); lock(&shared.state).total = w.times.len().saturating_sub(1) as u64; let Some(first) = frame_at(w.times[0]) else { return fail("the layer has no pixels to track") }; + if w.method.is_face() { + return run_face(&w, shared, first, &frame_at, layer, t0); + } let mut tracker = trk::mask::MaskTracker::new(w.method.model(), flatten(&w.path, 8), &trk::Frame { img: &first.0, offset: first.1 }); let mut path = w.path.clone(); - lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path.clone())); + lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path.clone(), None)); let mut next = if w.times.len() > 1 { frame_at(w.times[1]) } else { None }; for k in 1..w.times.len() { if shared.cancel.load(Ordering::Relaxed) { @@ -190,7 +240,7 @@ fn run_work(w: MaskWork, shared: &MaskTrackShared) { break; }; path = transform_path(&path, &step.motion); - lock(&shared.frames).push((layer.layer_time(w.times[k]), w.times[k], path.clone())); + lock(&shared.frames).push((layer.layer_time(w.times[k]), w.times[k], path.clone(), None)); let mut s = lock(&shared.state); s.done = k as u64; s.elapsed = t0.elapsed().as_secs_f64(); @@ -203,12 +253,99 @@ fn run_work(w: MaskWork, shared: &MaskTrackShared) { s.finished = true; } +/// Face tracking: the face outline keys the mask; Detailed Features adds the landmarks. +fn run_face( + w: &MaskWork, + shared: &MaskTrackShared, + first: (Arc, [f64; 2]), + frame_at: &(dyn Fn(Tick) -> Option<(Arc, [f64; 2])> + Sync), + layer: &effectcraft_project::Layer, + t0: web_time::Instant, +) { + let detailed = w.method == MaskMethod::FaceDetailed; + let b = trk::mask::bounds(&flatten(&w.path, 8)); + let Some((mut tracker, fit)) = trk::face::FaceTracker::new(&trk::Frame { img: &first.0, offset: first.1 }, b) else { + let mut s = lock(&shared.state); + s.error = Some("no face found inside the mask (draw the mask around a face)".into()); + s.finished = true; + return; + }; + let key = |fit: &trk::face::FaceFit| (outline_path(&fit.outline, 16, &w.path), detailed.then_some(fit.landmarks)); + let (path, pts) = key(&fit); + lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path, pts)); + let mut next = if w.times.len() > 1 { frame_at(w.times[1]) } else { None }; + for k in 1..w.times.len() { + if shared.cancel.load(Ordering::Relaxed) { + lock(&shared.state).cancelled = true; + break; + } + let Some(cur) = next.take() else { + lock(&shared.state).error = Some(format!("no frame at {:.3} s", w.times[k].seconds())); + break; + }; + let (res, nf) = + rayon::join(|| tracker.step(&trk::Frame { img: &cur.0, offset: cur.1 }), || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None }); + next = nf; + let Some(fit) = res else { + lock(&shared.state).error = Some(format!("lost the face at {:.3} s", w.times[k].seconds())); + break; + }; + let (path, pts) = key(&fit); + lock(&shared.frames).push((layer.layer_time(w.times[k]), w.times[k], path, pts)); + let mut s = lock(&shared.state); + s.done = k as u64; + s.elapsed = t0.elapsed().as_secs_f64(); + s.fps = if s.elapsed > 0.0 { k as f64 / s.elapsed } else { 0.0 }; + s.time = w.times[k].seconds(); + crate::offload::report(s.done, s.total); + } + let mut s = lock(&shared.state); + s.elapsed = t0.elapsed().as_secs_f64(); + s.finished = true; +} + +/// The layer's Face Track Points effect (created when missing). Returns its uid. +pub(crate) fn face_points_group(p: &mut Project, comp: ItemId, layer: LayerId) -> Option { + face_effect(p, comp, layer, effectcraft_effects::face_track::POINTS_ID) +} + +/// The layer's effect `id`, applied when missing (data effects of face tracking). +pub(crate) fn face_effect(p: &mut Project, comp: ItemId, layer: LayerId, id: &str) -> Option { + let spec = effectcraft_effects::find(id)?; + let l = p.comp(comp)?.layer(layer)?; + if let Some(g) = l.effects().and_then(|fx| fx.groups().find(|g| g.match_id == spec.id)) { + return Some(g.uid); + } + let (w, h) = effectcraft_render::source_size(p, l); + let size = if w == 0 { p.comp(comp).map(|c| [c.width as f64, c.height as f64]).unwrap_or([1920.0, 1080.0]) } else { [w as f64, h as f64] }; + let mut next = p.next_id; + let g = effectcraft_effects::instantiate(spec, &mut effectcraft_project::build::Ids(&mut next), spec.name, size); + let uid = g.uid; + let fx = p.comp_mut(comp)?.layer_mut(layer)?.props.sub_mut("effects")?; + fx.children.push(g.into()); + p.next_id = next; + Some(uid) +} + fn write_frames(p: &mut Project, job: &MaskTrackJob, frames: &[MaskFrame]) { + let points = if frames.iter().any(|f| f.3.is_some()) { face_points_group(p, job.comp, job.layer) } else { None }; let Some(layer) = p.comp_mut(job.comp).and_then(|c| c.layer_mut(job.layer)) else { return }; - let Some(g) = layer.props.find_group_mut(job.mask) else { return }; - let Some(pr) = g.get_mut("path") else { return }; - for (lt, _, path) in frames { - effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*lt, Value::Path(path.clone()))); + if let Some(g) = layer.props.find_group_mut(job.mask) + && let Some(pr) = g.get_mut("path") + { + for (lt, _, path, _) in frames { + effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*lt, Value::Path(path.clone()))); + } + } + if let Some(g) = points.and_then(|u| layer.props.find_group_mut(u)) { + for (lt, _, _, pts) in frames { + let Some(pts) = pts else { continue }; + for (i, (id, _, _)) in trk::face::LANDMARKS.iter().enumerate() { + if let Some(pr) = g.get_mut(id) { + effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*lt, Value::Vec2(pts[i]))); + } + } + } } } @@ -242,7 +379,7 @@ impl Session { return Err("a track analysis is already running".into()); } self.poll_mask_track(); - if !wait && self.offloads() { + if !wait && self.offloads() && !work.method.is_face() { let MaskWork { comp, layer, path, method, times, .. } = work; return self.offload_analysis(crate::offload::WorkerJob::MaskTrack { comp, layer, mask, direction, path, method, times }); } diff --git a/crates/engine/src/tests_face.rs b/crates/engine/src/tests_face.rs new file mode 100644 index 00000000..dad97935 --- /dev/null +++ b/crates/engine/src/tests_face.rs @@ -0,0 +1,106 @@ +//! Face tracking end to end through the commands, on synthetic face footage generated in-test +//! (a moving, rotating face that opens its mouth and blinks). + +use effectcraft_keyframe::{ShapePath, Value as KV}; +use effectcraft_track::face::{FaceParams, LANDMARKS, MEASUREMENTS, render_synthetic, synth_landmarks}; +use serde_json::json; + +use crate::mask_track::MaskMethod; +use crate::tests_track::{H, W, frame_time, setup}; + +fn face(f: u32) -> FaceParams { + let t = f as f64; + FaceParams { + center: [160.0 + 25.0 * (t * 0.2).sin(), 120.0 + 8.0 * (t * 0.15).cos()], + height: 70.0 * (1.0 + 0.05 * (t * 0.1).sin()), + roll: 12.0 * (t * 0.18).sin(), + yaw: 10.0 * (t * 0.12).sin(), + mouth_open: 0.5 + 0.45 * (t * 0.3).sin(), + eye_open: [0.85, 0.85], + ..Default::default() + } +} + +#[test] +fn face_tracking_keys_outline_points_and_measurements() { + let (mut s, clip, _) = setup(|f| render_synthetic(W, H, &face(f), 5)); + assert_eq!(MaskMethod::from_name("faceDetailed"), Some(MaskMethod::FaceDetailed)); + assert_eq!(MaskMethod::from_name("Face Tracking (Outline Only)"), Some(MaskMethod::FaceOutline)); + // A rough mask around the face. + let c = face(0).center; + let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect(c, 120.0, 160.0).vertices, "closed": true})).unwrap(); + let mask = r["mask"].as_u64().unwrap(); + s.execute("time.set", json!({"time": 0})).unwrap(); + let r = s.execute_checked("track.mask", json!({"layer": clip.0, "mask": mask, "method": "faceDetailed", "direction": "forward", "wait": true})).unwrap(); + assert_eq!(r["frames"], 24, "{r}"); + assert_eq!(s.state.mask_track_method, MaskMethod::FaceDetailed); + let comp = s.active_comp().unwrap(); + let layer = comp.layer(clip).unwrap(); + // The mask follows the face outline. + let path = layer.props.find_group(mask).unwrap().get("path").unwrap(); + assert_eq!(path.keys.len(), 25); + for f in [0, 12, 24] { + let KV::Path(p) = path.value_at(frame_time(f)) else { panic!() }; + let fp = face(f); + let (sn, cs) = fp.roll.to_radians().sin_cos(); + for v in &p.vertices { + let d = [(v[0] - fp.center[0]) / fp.height, (v[1] - fp.center[1]) / fp.height]; + let (u, w) = (cs * d[0] + sn * d[1], -sn * d[0] + cs * d[1]); + let r = ((u / 0.75).powi(2) + w * w).sqrt(); + assert!((r - 1.0).abs() < 0.06, "frame {f}: vertex {v:?} at {r}"); + } + } + // Face Track Points: one keyed point per landmark, on the face. + let fx = layer.effects().unwrap(); + let pts = fx.groups().find(|g| g.match_id == effectcraft_effects::face_track::POINTS_ID).expect("Face Track Points"); + let mut worst: f64 = 0.0; + for f in [0, 6, 12, 18, 24] { + let truth = synth_landmarks(&face(f)); + for (i, (id, _, _)) in LANDMARKS.iter().enumerate() { + let pr = pts.get(id).unwrap(); + assert_eq!(pr.keys.len(), 25, "{id}"); + let KV::Vec2(v) = pr.value_at(frame_time(f)) else { panic!() }; + worst = worst.max((v[0] - truth[i][0]).hypot(v[1] - truth[i][1]) / face(f).height); + } + } + assert!(worst < 0.1, "worst landmark error {worst} face heights"); + // Extract & Copy Face Measurements. + assert!(s.is_enabled("track.extractFaceMeasurements")); + let r = s.execute_checked("track.extractFaceMeasurements", json!({"layer": clip.0})).unwrap(); + assert_eq!(r["frames"], 25); + assert!(r["clipboard"].as_str().unwrap().contains("Mouth Openness")); + assert_eq!(s.state.key_clipboard.len(), MEASUREMENTS.len()); + assert!(s.state.clip_is_keys); + let layer = s.active_comp().unwrap().layer(clip).unwrap(); + let m = layer.effects().unwrap().groups().find(|g| g.match_id == effectcraft_effects::face_track::MEASUREMENTS_ID).expect("Face Measurements"); + let mo = m.get("mouthOpenness").unwrap(); + assert_eq!(mo.keys.len(), 25); + // Mouth openness follows the footage: widest open where the generator opens it most. + let open = |f: u32| mo.value_at(frame_time(f)).as_f64(); + let (fmax, fmin) = ( + (0..25).max_by(|a, b| face(*a).mouth_open.total_cmp(&face(*b).mouth_open)).unwrap(), + (0..25).min_by(|a, b| face(*a).mouth_open.total_cmp(&face(*b).mouth_open)).unwrap(), + ); + assert!(open(fmax) > open(fmin) + 20.0, "{} vs {}", open(fmax), open(fmin)); + let roll = m.get("headOrientationZ").unwrap(); + for f in [0, 10, 20] { + assert!((roll.value_at(frame_time(f)).as_f64() - face(f).roll).abs() < 3.0); + } + // One undo step for the extraction. + assert!(s.undo()); + assert!(!s.active_comp().unwrap().layer(clip).unwrap().effects().unwrap().groups().any(|g| g.match_id == effectcraft_effects::face_track::MEASUREMENTS_ID)); + // Outline Only keys just the mask. + let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect(c, 120.0, 160.0).vertices, "closed": true})).unwrap(); + let m2 = r["mask"].as_u64().unwrap(); + s.execute("time.set", json!({"time": 0})).unwrap(); + s.execute_checked("track.mask", json!({"layer": clip.0, "mask": m2, "method": "faceOutline", "direction": "frameForward", "wait": true})).unwrap(); + let layer = s.active_comp().unwrap().layer(clip).unwrap(); + assert_eq!(layer.props.find_group(m2).unwrap().get("path").unwrap().keys.len(), 2); + // A mask without a face reports it. + let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect([20.0, 20.0], 30.0, 30.0).vertices, "closed": true})).unwrap(); + let m3 = r["mask"].as_u64().unwrap(); + s.execute("time.set", json!({"time": 0})).unwrap(); + let _ = s.execute("track.mask", json!({"layer": clip.0, "mask": m3, "method": "faceOutline", "direction": "forward", "wait": true})); + let layer = s.active_comp().unwrap().layer(clip).unwrap(); + assert!(layer.props.find_group(m3).unwrap().get("path").unwrap().keys.is_empty()); +} diff --git a/crates/track/src/camtrack/bundle.rs b/crates/track/src/camtrack/bundle.rs index 72cb2f17..077f04b7 100644 --- a/crates/track/src/camtrack/bundle.rs +++ b/crates/track/src/camtrack/bundle.rs @@ -10,6 +10,7 @@ use rayon::prelude::*; +use super::Distortion; use super::geometry::Pose; use super::linalg::*; @@ -19,6 +20,8 @@ pub struct BaCam { pub pose: Pose, /// Focal length (pixels). pub f: f64, + /// Lens distortion (shared by all cameras; adjusted when [`BaOpts::distortion`]). + pub dist: Distortion, } /// One observation: camera `cam` sees point `pt` at `uv` (pixels relative to the principal @@ -50,11 +53,13 @@ pub struct BaOpts { /// Huber threshold (pixels). pub huber: f64, pub max_iter: usize, + /// Adjust the shared radial distortion `(k1, k2)` too. + pub distortion: bool, } impl Default for BaOpts { fn default() -> Self { - BaOpts { focal: FocalMode::Fixed, fix_points: false, fix_centers: false, fixed: vec![], huber: 2.0, max_iter: 30 } + BaOpts { focal: FocalMode::Fixed, fix_points: false, fix_centers: false, fixed: vec![], huber: 2.0, max_iter: 30, distortion: false } } } @@ -77,7 +82,8 @@ pub fn residual(c: &BaCam, x: V3, uv: [f64; 2]) -> Option<[f64; 2]> { if p[2] <= 1e-9 { return None; } - Some([c.f * p[0] / p[2] - uv[0], c.f * p[1] / p[2] - uv[1]]) + let q = c.dist.distort([c.f * p[0] / p[2], c.f * p[1] / p[2]]); + Some([q[0] - uv[0], q[1] - uv[1]]) } /// Total robust cost. @@ -108,6 +114,8 @@ struct Layout { fidx: Vec, /// Index of the shared focal parameter. shared: Option, + /// Index of `k1` (then `k2`) when the distortion is adjusted. + dist: Option, n: usize, rot_only: bool, } @@ -132,7 +140,11 @@ fn layout(ncam: usize, o: &BaOpts) -> Layout { n += 1; n - 1 }); - Layout { start, fidx, shared, n, rot_only: o.fix_centers } + let dist = o.distortion.then(|| { + n += 2; + n - 2 + }); + Layout { start, fidx, shared, dist, n, rot_only: o.fix_centers } } /// Camera-side Jacobian entries of one observation: (parameter index, d r / d param). @@ -204,12 +216,21 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> continue; } let iz = 1.0 / xc[2]; - let p = [c.f * xc[0] * iz, c.f * xc[1] * iz]; - let r = [p[0] - ob.uv[0], p[1] - ob.uv[1]]; + let p0 = [c.f * xc[0] * iz, c.f * xc[1] * iz]; + let pd = c.dist.distort(p0); + let r = [pd[0] - ob.uv[0], pd[1] - ob.uv[1]]; let e = (r[0] * r[0] + r[1] * r[1]).sqrt(); let w = if e <= o.huber { 1.0 } else { o.huber / e }; - let a = [[c.f * iz, 0.0, -c.f * xc[0] * iz * iz], [0.0, c.f * iz, -c.f * xc[1] * iz * iz]]; + let a0 = [[c.f * iz, 0.0, -c.f * xc[0] * iz * iz], [0.0, c.f * iz, -c.f * xc[1] * iz * iz]]; + // Through the distortion: d r = D d p. + let (dj, dk) = c.dist.jacobian(p0); + let a = [0, 1].map(|i| [0, 1, 2].map(|k| dj[i][0] * a0[0][k] + dj[i][1] * a0[1][k])); + let p = [dj[0][0] * p0[0] + dj[0][1] * p0[1], dj[1][0] * p0[0] + dj[1][1] * p0[1]]; cam_jac(&l, ci, c, xc, &a, p, &mut jc); + if let Some(g) = l.dist { + jc.push((g, [dk[0][0], dk[1][0]])); + jc.push((g + 1, [dk[0][1], dk[1][1]])); + } // Point Jacobian: A R. let jx = if o.fix_points { [[0.0; 3]; 2] @@ -236,7 +257,8 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> [w * (ja[0] * jx[0][0] + ja[1] * jx[1][0]), w * (ja[0] * jx[0][1] + ja[1] * jx[1][1]), w * (ja[0] * jx[0][2] + ja[1] * jx[1][2])]; // Each observation of a point is on another camera: only the shared focal // length repeats. - let found = if Some(*ia) == l.shared { ps.w.iter_mut().find(|e| e.0 == *ia) } else { None }; + let repeats = Some(*ia) == l.shared || l.dist.is_some_and(|g| *ia == g || *ia == g + 1); + let found = if repeats { ps.w.iter_mut().find(|e| e.0 == *ia) } else { None }; match found { Some(e) => e.1 = add(e.1, wv), None => ps.w.push((*ia, wv)), @@ -385,6 +407,10 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> if let Some(g) = l.shared { c.f *= dc[g].clamp(-0.5, 0.5).exp(); } + if let Some(g) = l.dist { + c.dist.k1 += dc[g].clamp(-0.2, 0.2); + c.dist.k2 += dc[g + 1].clamp(-0.2, 0.2); + } } let mut np: Vec = pts.to_vec(); if !o.fix_points { @@ -445,7 +471,11 @@ mod tests { let truth: Vec = (0..80).map(|_| [u() * 6.0, u() * 4.0, 6.0 + u() * 4.0]).collect(); let f = 900.0; let cams_true: Vec = (0..6) - .map(|i| BaCam { pose: Pose { r: rodrigues([0.0, -0.05 * i as f64, 0.01 * i as f64]), c: [0.3 * i as f64, 0.05 * i as f64, 0.0] }, f }) + .map(|i| BaCam { + pose: Pose { r: rodrigues([0.0, -0.05 * i as f64, 0.01 * i as f64]), c: [0.3 * i as f64, 0.05 * i as f64, 0.0] }, + f, + dist: Distortion::NONE, + }) .collect(); let mut obs = vec![]; for (ci, c) in cams_true.iter().enumerate() { diff --git a/crates/track/src/camtrack/mod.rs b/crates/track/src/camtrack/mod.rs index 76c0637f..e8cd28da 100644 --- a/crates/track/src/camtrack/mod.rs +++ b/crates/track/src/camtrack/mod.rs @@ -28,8 +28,15 @@ //! tripod solution when it explains the tracks as well as the general one (a quick rotation fit //! at the general solution's focal length skips the tripod search when it clearly fails). //! -//! The camera model has square pixels, the principal point at the layer centre and no lens -//! distortion. Camera space is x right, y down, z forward (After Effects' convention); a camera +//! 5. **Lens distortion** (Solve Lens Distortion, or Detailed Analysis): radial distortion +//! `p_d = p (1 + k1 ρ² + k2 ρ⁴)` (ρ = distance from the principal point over half the frame +//! diagonal; Brown's model without the tangential terms) is solved jointly with the cameras +//! and points in a final bundle adjustment on the raw tracks; the whole solve is then repeated +//! on tracks undistorted with that estimate (the linear initialisations assume a pinhole +//! camera) and adjusted jointly again. The solve stores `k1, k2`; the effect can undistort +//! its output so composited 3D layers line up. +//! +//! The camera model has square pixels and the principal point at the layer centre. Camera space is x right, y down, z forward (After Effects' convention); a camera //! is `(R, C, f)` with `X_cam = R (X − C)` and pixel `u = f x / z + w / 2`. //! //! The solve is stored in a canonical frame: the first solved frame's camera is at the origin @@ -105,6 +112,69 @@ impl CameraTracks { } } +/// Radial lens distortion around the principal point (see the module docs). +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct Distortion { + pub k1: f64, + pub k2: f64, + /// Normalisation radius ρ = 1 (pixels): half the frame diagonal; 0 = no distortion. + pub radius: f64, +} + +impl Distortion { + pub const NONE: Distortion = Distortion { k1: 0.0, k2: 0.0, radius: 0.0 }; + + /// No distortion for frames of `size` (the radius set, coefficients zero). + pub fn for_size(size: [f64; 2]) -> Distortion { + Distortion { k1: 0.0, k2: 0.0, radius: 0.5 * size[0].hypot(size[1]) } + } + pub fn is_none(&self) -> bool { + self.radius <= 0.0 || (self.k1 == 0.0 && self.k2 == 0.0) + } + fn rho2(&self, p: [f64; 2]) -> f64 { + (p[0] * p[0] + p[1] * p[1]) / (self.radius * self.radius) + } + /// Ideal → distorted (pixels relative to the principal point). + pub fn distort(&self, p: [f64; 2]) -> [f64; 2] { + if self.radius <= 0.0 { + return p; + } + let r2 = self.rho2(p); + let d = 1.0 + self.k1 * r2 + self.k2 * r2 * r2; + [p[0] * d, p[1] * d] + } + /// Distorted → ideal (fixed-point iteration). + pub fn undistort(&self, q: [f64; 2]) -> [f64; 2] { + if self.is_none() { + return q; + } + let mut p = q; + for _ in 0..30 { + let r2 = self.rho2(p); + let d = 1.0 + self.k1 * r2 + self.k2 * r2 * r2; + if d.abs() < 1e-6 { + break; + } + p = [q[0] / d, q[1] / d]; + } + p + } + /// `d distort / d p` (2 × 2, row-major) and `d distort / d (k1, k2)` (rows x, y). + pub fn jacobian(&self, p: [f64; 2]) -> ([[f64; 2]; 2], [[f64; 2]; 2]) { + if self.radius <= 0.0 { + return ([[1.0, 0.0], [0.0, 1.0]], [[0.0; 2]; 2]); + } + let r2 = self.rho2(p); + let d = 1.0 + self.k1 * r2 + self.k2 * r2 * r2; + // d(r2)/dp = 2 p / R². + let dd = (self.k1 + 2.0 * self.k2 * r2) * 2.0 / (self.radius * self.radius); + let j = [[d + p[0] * dd * p[0], p[0] * dd * p[1]], [p[1] * dd * p[0], d + p[1] * dd * p[1]]]; + let k = [[p[0] * r2, p[0] * r2 * r2], [p[1] * r2, p[1] * r2 * r2]]; + (j, k) + } +} + /// 3D Camera Tracker ▸ Shot Type. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -178,6 +248,8 @@ pub struct SolveSettings { pub method: SolveMethod, /// Track ids the user deleted. pub deleted: Vec, + /// Solve radial lens distortion jointly with the cameras. + pub lens_distortion: bool, } /// The solved camera of one frame. @@ -261,6 +333,9 @@ pub struct CameraSolve { pub points: Vec, #[serde(skip_serializing_if = "Option::is_none")] pub ground: Option, + /// The solved lens distortion (Solve Lens Distortion). + #[serde(skip_serializing_if = "Option::is_none")] + pub distortion: Option, } impl CameraSolve { @@ -355,6 +430,36 @@ impl CameraSolve { } } + /// An ideal (pinhole) image point → where it appears in the footage (layer pixels), through + /// the solved lens distortion. + pub fn to_image(&self, p: [f64; 2]) -> [f64; 2] { + match &self.distortion { + Some(d) if !d.is_none() => { + let c = [self.size[0] * 0.5, self.size[1] * 0.5]; + let q = d.distort([p[0] - c[0], p[1] - c[1]]); + [q[0] + c[0], q[1] + c[1]] + } + _ => p, + } + } + /// A footage point → its ideal (undistorted) position. + pub fn to_ideal(&self, p: [f64; 2]) -> [f64; 2] { + match &self.distortion { + Some(d) if !d.is_none() => { + let c = [self.size[0] * 0.5, self.size[1] * 0.5]; + let q = d.undistort([p[0] - c[0], p[1] - c[1]]); + [q[0] + c[0], q[1] + c[1]] + } + _ => p, + } + } + /// Project world point `x` on frame `k`: through the lens distortion into the footage, or + /// (`undistorted`) into the undistorted image. + pub fn project(&self, k: usize, x: V3, undistorted: bool) -> Option<[f64; 2]> { + let q = self.frames.get(k)?.project(self.size, x)?; + Some(if undistorted { q } else { self.to_image(q) }) + } + /// The first solved frame. pub fn first_solved(&self) -> Option { self.frames.iter().position(|c| c.solved) diff --git a/crates/track/src/camtrack/solver.rs b/crates/track/src/camtrack/solver.rs index 85edd2bd..5ed7b81b 100644 --- a/crates/track/src/camtrack/solver.rs +++ b/crates/track/src/camtrack/solver.rs @@ -8,7 +8,7 @@ use rayon::prelude::*; use super::bundle::{self, BaCam, BaObs, BaOpts, FocalMode}; use super::geometry::{self, Pose}; use super::linalg::*; -use super::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, SolvedFrame, SolvedPoint}; +use super::{CameraSolve, CameraTracks, Distortion, ShotType, SolveMethod, SolveSettings, SolvedFrame, SolvedPoint}; use crate::fit::{Model, ransac}; fn debug() -> bool { @@ -142,6 +142,8 @@ struct Cfg { focal: FocalMode, /// Registration estimates a focal length per frame. per_frame_f: bool, + /// The global adjustments also adjust the lens distortion. + distortion: bool, } const F_PRIOR: f64 = 30.0; @@ -183,7 +185,7 @@ impl Recon { &mut cams, &mut pts, &obs, - &BaOpts { focal, fix_points: false, fix_centers: cfg.kind == Kind::Tripod, fixed, huber: d.huber(), max_iter: iters }, + &BaOpts { focal, fix_points: false, fix_centers: cfg.kind == Kind::Tripod, fixed, huber: d.huber(), max_iter: iters, distortion: cfg.distortion }, ); for (k, c) in frames.iter().zip(&cams) { self.cams.insert(*k, *c); @@ -289,6 +291,7 @@ impl Recon { fixed: vec![false], huber, max_iter: 40, + distortion: false, }; let mut best: Option<(usize, BaCam)> = None; // Robust start (large Huber threshold), then tighten; from the prediction and, failing @@ -371,7 +374,11 @@ fn predict(r: &Recon, k: usize) -> Option { match (before, after) { (Some((ka, a)), Some((kb, b))) => { let t = (k - ka) as f64 / (kb - ka) as f64; - Some(BaCam { pose: Pose { r: slerp(&a.pose.r, &b.pose.r, t), c: add(scale(a.pose.c, 1.0 - t), scale(b.pose.c, t)) }, f: a.f * (b.f / a.f).powf(t) }) + Some(BaCam { + pose: Pose { r: slerp(&a.pose.r, &b.pose.r, t), c: add(scale(a.pose.c, 1.0 - t), scale(b.pose.c, t)) }, + f: a.f * (b.f / a.f).powf(t), + dist: a.dist, + }) } (Some((ka, a)), None) => { // Constant velocity from the two previous cameras. @@ -381,7 +388,7 @@ fn predict(r: &Recon, k: usize) -> Option { let dr = log_rot(&mm(&a.pose.r, &mt(&p.pose.r))); let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &a.pose.r); let c = add(a.pose.c, scale(sub(a.pose.c, p.pose.c), t.min(2.0))); - Some(BaCam { pose: Pose { r: rr, c }, f: a.f }) + Some(BaCam { pose: Pose { r: rr, c }, f: a.f, dist: a.dist }) } None => Some(a), } @@ -392,7 +399,7 @@ fn predict(r: &Recon, k: usize) -> Option { let dr = log_rot(&mm(&b.pose.r, &mt(&nx.pose.r))); let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &b.pose.r); let c = add(b.pose.c, scale(sub(b.pose.c, nx.pose.c), t.min(2.0))); - Some(BaCam { pose: Pose { r: rr, c }, f: b.f }) + Some(BaCam { pose: Pose { r: rr, c }, f: b.f, dist: b.dist }) } None => Some(b), }, @@ -459,8 +466,8 @@ fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> fn grow(d: &Data, kfs: &[usize], j: usize, init: &(Pose, HashMap), f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { let (a, b) = (kfs[0], kfs[j]); let mut r = Recon::default(); - r.cams.insert(a, BaCam { pose: Pose::IDENTITY, f }); - r.cams.insert(b, BaCam { pose: init.0, f }); + r.cams.insert(a, BaCam { pose: Pose::IDENTITY, f, dist: Distortion::NONE }); + r.cams.insert(b, BaCam { pose: init.0, f, dist: Distortion::NONE }); r.pts = init.1.clone(); let inner = |r: &mut Recon, frames: &[usize]| { let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed }; @@ -495,7 +502,7 @@ fn grow(d: &Data, kfs: &[usize], j: usize, init: &(Pose, HashMap), f: f /// Tripod reconstruction over keyframes `kfs` at focal length `f`. fn grow_tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { let mut r = Recon::default(); - r.cams.insert(kfs[0], BaCam { pose: Pose::IDENTITY, f }); + r.cams.insert(kfs[0], BaCam { pose: Pose::IDENTITY, f, dist: Distortion::NONE }); let ray = |p: [f64; 2], f: f64| normalize([p[0] / f, p[1] / f, 1.0]); for w in kfs.windows(2) { if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { @@ -519,7 +526,7 @@ fn grow_tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&Atomi rel = geometry::procrustes(&ia, &ib)?; } let pa = r.cams[&a].pose.r; - r.cams.insert(b, BaCam { pose: Pose { r: mm(&rel, &pa), c: [0.0; 3] }, f }); + r.cams.insert(b, BaCam { pose: Pose { r: mm(&rel, &pa), c: [0.0; 3] }, f, dist: Distortion::NONE }); } r.triangulate(d, Kind::Tripod, 0.0, 20.0); let frames: Vec = r.cams.keys().copied().collect(); @@ -640,8 +647,101 @@ fn search_focal(w: f64, eval: &(dyn Fn(f64) -> f64 + Sync)) -> f64 { if best.1 <= errs[i] { best.0.exp() } else { fs[i] } } +/// A solve before its canonical output. +struct Core { + d: Data, + r: Recon, + method: SolveMethod, + kind: Kind, + cfg: Cfg, + focal: FocalMode, + kfs: Vec, +} + +/// `tracks` with every position undistorted by `dist`. +fn undistorted(tracks: &CameraTracks, dist: &Distortion) -> CameraTracks { + let c = [tracks.size[0] * 0.5, tracks.size[1] * 0.5]; + let mut t = tracks.clone(); + for tr in t.tracks.iter_mut() { + for p in tr.pts.iter_mut() { + let q = dist.undistort([p[0] as f64 - c[0], p[1] as f64 - c[1]]); + *p = [(q[0] + c[0]) as f32, (q[1] + c[1]) as f32]; + } + } + t +} + /// Solve the camera of a tracked clip. pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBool>) -> Result { + if !s.lens_distortion { + let c = solve_core(tracks, s, cancel)?; + return Ok(finish(&c.d, tracks, s, c.r, c.method, c.kind)); + } + // Lens distortion: solve, adjust (k1, k2) jointly on the raw tracks, then solve again on + // tracks undistorted with the estimate and adjust jointly once more. + let mut dist = Distortion::for_size(tracks.size); + let mut out = None; + for pass in 0..2 { + let und = (pass > 0).then(|| undistorted(tracks, &dist)); + let c = solve_core(und.as_ref().unwrap_or(tracks), s, cancel)?; + let d = Data::new(tracks, &s.deleted); + d.set_noise(c.d.noise()); + let mut r = c.r; + for cam in r.cams.values_mut() { + cam.dist = dist; + } + let cfg = Cfg { distortion: true, ..c.cfg }; + let all: Vec = r.cams.keys().copied().collect(); + let frames: Vec = if all.len() <= 160 { all.clone() } else { c.kfs.iter().copied().filter(|k| r.cams.contains_key(k)).collect() }; + if frames.len() < 2 { + return err("the camera could not be solved (not enough parallax or features)"); + } + // The pinhole solve's outliers include the distorted frame edges: start from all + // observations (the Huber loss keeps real outliers in check), then reject again. + r.outliers.clear(); + r.adjust(&d, &frames, &cfg, c.focal, 30); + r.outliers.clear(); + r.reject(&d); + r.adjust(&d, &frames, &cfg, c.focal, 30); + dist = r.cams[&frames[0]].dist; + for cam in r.cams.values_mut() { + cam.dist = dist; + } + if frames.len() < all.len() { + // Refine the other frames against the adjusted points and distortion. + let base = r.clone(); + let rcfg = Cfg { distortion: false, ..cfg }; + let refined: Vec<(usize, Option)> = all + .par_iter() + .filter(|k| !frames.contains(k)) + .map(|k| { + let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() }; + let start = local.cams[k]; + let ok = local.register(&d, *k, start, &rcfg); + (*k, if ok { local.cams.get(k).copied() } else { None }) + }) + .collect(); + for (k, cam) in refined { + if let Some(cam) = cam { + r.cams.insert(k, cam); + } + } + r.reject(&d); + } + if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return err("cancelled"); + } + if debug() { + eprintln!("lens distortion pass {pass}: k1 {:.5} k2 {:.5}", dist.k1, dist.k2); + } + let mut sol = finish(&d, tracks, s, r, c.method, c.kind); + sol.distortion = Some(dist); + out = Some(sol); + } + out.ok_or_else(|| SolveError("the camera could not be solved".into())) +} + +fn solve_core(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBool>) -> Result { let d = Data::new(tracks, &s.deleted); if d.n < 2 { return err("the clip must be at least two frames long"); @@ -653,7 +753,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo let cancelled = || cancel.is_some_and(|c| c.load(Ordering::Relaxed)); let fixed_f = (s.shot == ShotType::SpecifyAngle).then(|| focal_of(d.w, if s.hfov > 0.0 { s.hfov } else { 40.0 })); let variable = s.shot == ShotType::VariableZoom; - let gcfg = Cfg { kind: Kind::General, focal: FocalMode::Fixed, per_frame_f: false }; + let gcfg = Cfg { kind: Kind::General, focal: FocalMode::Fixed, per_frame_f: false, distortion: false }; let tcfg = Cfg { kind: Kind::Tripod, ..gcfg }; // The focal-length search runs on the first keyframes only. let mini: Vec = kfs.iter().copied().take(10).collect(); @@ -738,7 +838,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo } let (att, _) = results.swap_remove(pick); let kind = if att.method == SolveMethod::TripodPan { Kind::Tripod } else { Kind::General }; - let cfg = Cfg { kind, focal: FocalMode::Fixed, per_frame_f: variable }; + let cfg = Cfg { kind, focal: FocalMode::Fixed, per_frame_f: variable, distortion: false }; let mut r = att.recon; let focal = match (fixed_f, variable) { (Some(_), _) => FocalMode::Fixed, @@ -845,7 +945,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo (m, _) => m, }; lap("final adjustment"); - Ok(finish(&d, tracks, s, r, method, kind)) + Ok(Core { d, r, method, kind, cfg, focal, kfs: kf_frames }) } /// Thickness of a point cloud: √(smallest / middle eigenvalue of its covariance). @@ -949,5 +1049,6 @@ fn finish(d: &Data, tracks: &CameraTracks, s: &SolveSettings, r: Recon, method: frames, points, ground: None, + distortion: None, } } diff --git a/crates/track/src/camtrack/tests.rs b/crates/track/src/camtrack/tests.rs index 4b5a8346..ff92cfe4 100644 --- a/crates/track/src/camtrack/tests.rs +++ b/crates/track/src/camtrack/tests.rs @@ -105,7 +105,7 @@ fn path_error(s: &CameraSolve, t: &Truth) -> (f64, f64) { } fn settings(shot: ShotType, method: SolveMethod) -> SolveSettings { - SolveSettings { shot, hfov: 0.0, method, deleted: vec![] } + SolveSettings { shot, hfov: 0.0, method, deleted: vec![], lens_distortion: false } } #[test] @@ -149,7 +149,8 @@ fn specified_angle_of_view_is_kept() { let pts = random_points(300, 5, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); let tr = tracks_of(&pts, &t, 6, true); let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); - let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: vec![] }, None).unwrap(); + let s = + solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: vec![], lens_distortion: false }, None).unwrap(); assert!((s.focal() - f).abs() < 1e-6); assert!(s.average_error < 0.5); let (pe, _) = path_error(&s, &t); @@ -226,7 +227,9 @@ fn mostly_flat_scene_and_ground_plane() { } // Ground plane from the solved (coplanar) points: every point lies on it. let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); - let mut s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::MostlyFlat, deleted: vec![] }, None).expect("solve"); + let mut s = + solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::MostlyFlat, deleted: vec![], lens_distortion: false }, None) + .expect("solve"); let ids: Vec = s.visible(0).map(|p| p.id).take(12).collect(); let g = s.ground_from(&ids, 0).expect("ground"); let spread = s.points.iter().map(|p| dot(sub(p.pos, g.origin), g.normal).abs()).fold(0.0, f64::max); @@ -251,7 +254,9 @@ fn deleted_tracks_are_ignored_and_cancel_stops() { let tr = tracks_of(&pts, &t, 14, true); let deleted: Vec = (0..40).collect(); let hfov = 2.0 * (W * 0.5 / 1000.0f64).atan().to_degrees(); - let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: deleted.clone() }, None).unwrap(); + let s = + solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: deleted.clone(), lens_distortion: false }, None) + .unwrap(); assert!(s.points.iter().all(|p| !deleted.contains(&p.id))); let stop = std::sync::atomic::AtomicBool::new(true); let r = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), Some(&stop)); @@ -436,3 +441,55 @@ fn solve_saved_tracks() { t0.elapsed() ); } + +/// Pass every track position through a lens distortion (around the frame centre). +fn distort_tracks(t: &CameraTracks, d: &Distortion) -> CameraTracks { + let mut t = t.clone(); + for tr in t.tracks.iter_mut() { + for p in tr.pts.iter_mut() { + let q = d.distort([p[0] as f64 - W / 2.0, p[1] as f64 - H / 2.0]); + *p = [(q[0] + W / 2.0) as f32, (q[1] + H / 2.0) as f32]; + } + } + t +} + +#[test] +fn lens_distortion_is_recovered_in_the_bundle_adjustment() { + let f = 1000.0; + let t = moving_camera(36, |_| f); + let pts = random_points(450, 7, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]); + let truth = Distortion { k1: -0.12, k2: 0.03, radius: 0.5 * W.hypot(H) }; + let tr = distort_tracks(&tracks_of(&pts, &t, 8, true), &truth); + let plain = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), None).expect("pinhole solve"); + let lens = solve(&tr, &SolveSettings { lens_distortion: true, ..settings(ShotType::FixedAngle, SolveMethod::Typical) }, None).expect("lens solve"); + let d = lens.distortion.expect("distortion solved"); + eprintln!( + "pinhole error {:.3} px (focal {:.1}); with distortion {:.3} px (focal {:.1}), k1 {:.4} k2 {:.4}", + plain.average_error, + plain.focal(), + lens.average_error, + lens.focal(), + d.k1, + d.k2 + ); + assert!(plain.distortion.is_none()); + assert!((d.k1 - truth.k1).abs() < 0.005, "k1 {}", d.k1); + assert!((d.k2 - truth.k2).abs() < 0.01, "k2 {}", d.k2); + // The distortion model explains the tracks: about the noise level, far below the pinhole fit. + assert!(lens.average_error < 0.3, "{}", lens.average_error); + assert!(lens.average_error < 0.5 * plain.average_error, "{} vs {}", lens.average_error, plain.average_error); + assert!((lens.focal() - f).abs() < 0.02 * f, "focal {}", lens.focal()); + let (path, rot) = path_error(&lens, &t); + assert!(path < 0.01 && rot < 0.3, "path {path} rot {rot}"); + // Projection through the lens lands on the (distorted) tracks; undistorting inverts it. + let p = [100.0, 80.0]; + let q = lens.to_image(p); + let back = lens.to_ideal(q); + assert!((back[0] - p[0]).abs() < 1e-6 && (back[1] - p[1]).abs() < 1e-6); + let sp = &lens.points[0]; + let k = sp.first as usize; + let obs = tr.track(sp.id).and_then(|t| t.at(k as u32)).expect("observed"); + let proj = lens.project(k, sp.pos, false).expect("in front"); + assert!((proj[0] - obs[0]).hypot(proj[1] - obs[1]) < 2.0, "{proj:?} vs {obs:?}"); +} diff --git a/crates/track/src/face.rs b/crates/track/src/face.rs index 78344a14..62073d0a 100644 --- a/crates/track/src/face.rs +++ b/crates/track/src/face.rs @@ -624,6 +624,11 @@ impl FaceTracker { }; let mut t = FaceTracker { skin, prev: guess, expect: canon }; let fit = t.fit(frame)?; + // A face shows at least two of: left eye, right eye, mouth. + let found = [EYE[0][4], EYE[1][4], MOUTH[0]].iter().filter(|i| fit.detected[**i]).count(); + if found < 2 { + return None; + } t.accept(&fit); Some((t, fit)) } diff --git a/crates/ui-egui/src/panels/camera_tracker_ui.rs b/crates/ui-egui/src/panels/camera_tracker_ui.rs index 41fb7204..c9c40f05 100644 --- a/crates/ui-egui/src/panels/camera_tracker_ui.rs +++ b/crates/ui-egui/src/panels/camera_tracker_ui.rs @@ -38,6 +38,8 @@ struct Active { show_3d: bool, point_size: f64, target_size: f64, + /// Undistort Footage is on: points project without the lens distortion. + undistort: bool, /// Layer pixels → comp pixels. l2c: Mat3, frame: usize, @@ -66,6 +68,7 @@ fn active(s: &Session, comp: &Comp, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Laye let t = tracks.as_ref()?; (t.frame_duration > 0.0).then(|| ((lt - t.start) / t.frame_duration).round().clamp(0.0, t.frames.saturating_sub(1) as f64) as usize) }); + let undistort = solve.as_ref().is_some_and(|sv| ct::undistorts(¶ms, sv)); return Some(Active { layer: *lid, uid: *uid, @@ -74,6 +77,7 @@ fn active(s: &Session, comp: &Comp, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Laye show_3d: val("showTrackPoints").map(|v| v.as_enum()).unwrap_or(1) == 1, point_size: val("trackPointSize").map(|v| v.as_f64()).unwrap_or(100.0), target_size: val("targetSize").map(|v| v.as_f64()).unwrap_or(100.0), + undistort, l2c: l2c(ectx, l), frame: frame.unwrap_or(0), }); @@ -98,18 +102,20 @@ fn l2c_scale(m: &Mat3) -> f32 { /// Draw the target disc (concentric rings in perspective) of `t` on frame `k`. fn draw_target(p: &egui::Painter, a: &Active, map: &ViewerMap, sv: &CameraSolve, t: &Target) { - let Some(cam) = sv.frames.get(a.frame) else { return }; + if sv.frames.get(a.frame).is_none() { + return; + } let r = t.size * 0.5 * a.target_size / 100.0; for (i, frac) in [1.0, 0.66, 0.33].into_iter().enumerate() { let ring = ct::target_circle(t.center, t.normal, r * frac, 48); - let pts: Vec = ring.iter().filter_map(|x| cam.project(sv.size, *x)).map(|q| to_screen(a, map, q)).collect(); + let pts: Vec = ring.iter().filter_map(|x| sv.project(a.frame, *x, a.undistort)).map(|q| to_screen(a, map, q)).collect(); if pts.len() < 3 { return; } let fill = if i % 2 == 0 { Color32::from_rgba_unmultiplied(0xd0, 0x30, 0x30, 70) } else { Color32::from_rgba_unmultiplied(0xff, 0xff, 0xff, 50) }; p.add(egui::Shape::convex_polygon(pts.clone(), fill, Stroke::new(1.0, Color32::from_rgb(0xe0, 0x40, 0x40)))); } - if let Some(c) = cam.project(sv.size, t.center) { + if let Some(c) = sv.project(a.frame, t.center, a.undistort) { let c = to_screen(a, map, c); p.circle_filled(c, 3.0, Color32::from_rgb(0xff, 0xd0, 0x40)); } @@ -168,7 +174,7 @@ pub fn viewer_hook( let mut hits: Vec = vec![]; match (&a.solve, a.show_3d) { (Some(sv), true) => { - let pts = ct::projected(sv, a.frame); + let pts = ct::projected_with(sv, a.frame, a.undistort); let med = ct::median_depth(&pts); for (id, q, z) in pts { let sp = to_screen(&a, map, q); diff --git a/crates/ui-egui/src/panels/tracker.rs b/crates/ui-egui/src/panels/tracker.rs index 364e114e..02c59636 100644 --- a/crates/ui-egui/src/panels/tracker.rs +++ b/crates/ui-egui/src/panels/tracker.rs @@ -404,6 +404,27 @@ fn mask_mode(app: &mut EffectcraftApp, ui: &mut egui::Ui, ctx: &egui::Context, p bx += 34.0; } y += 34.0; + // Face Tracking (Detailed Features): Extract & Copy Face Measurements. + let has_points = app + .session + .active_comp() + .and_then(|c| c.layer(layer)) + .and_then(|l| l.effects()) + .is_some_and(|fx| fx.groups().any(|g| g.match_id == effectcraft_engine::effects::face_track::POINTS_ID)); + if method == MaskMethod::FaceDetailed || has_points { + let r = Rect::from_min_size(pos2(x0, y), vec2(w.min(260.0), 24.0)); + if button(app, ui, r, "Extract & Copy Face Measurements", has_points && !running, "tracker.mask.extractFace") { + match crate::menus::invoke(app, ctx, "track.extractFaceMeasurements", json!({"layer": layer.0})) { + Ok(v) => { + if let Some(text) = v.get("clipboard").and_then(|t| t.as_str()) { + ctx.copy_text(text.to_string()); + } + } + Err(e) => app.ui.status = e, + } + } + y += 32.0; + } if let Some(pr) = app.session.mask_track_progress() { let bar = Rect::from_min_size(pos2(x0, y), vec2(w, 6.0)); p.rect_filled(bar, 3.0, t.field_bg); @@ -419,7 +440,12 @@ fn mask_mode(app: &mut EffectcraftApp, ui: &mut egui::Ui, ctx: &egui::Context, p ); ctx.request_repaint_after(std::time::Duration::from_millis(50)); } else { - p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, "Tracks the pixels inside the mask and keys its Mask Path.", Tokens::ui(11.0), t.text_faint); + let hint = if method.is_face() { + "Finds the face inside the mask and keys its outline (and, with Detailed Features, Face Track Points)." + } else { + "Tracks the pixels inside the mask and keys its Mask Path." + }; + p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, hint, Tokens::ui(11.0), t.text_faint); } } diff --git a/docs/agents.md b/docs/agents.md index 8689b9da..c25aa399 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -105,11 +105,18 @@ effectcraft-cli run clip.ecproj track.motion '{"layer":"#2"}' \ `track.mask` follows the pixels inside a mask and keys its Mask Path on every frame (vertices and tangents move with the fitted motion). `method` is `position`, `positionScale`, -`positionScaleRotation` (the default, also kept as the Tracker panel's Method), `positionScaleRotationSkew` -or `perspective`; `direction` is `forward|backward|frameForward|frameBackward` from the current +`positionScaleRotation` (the default, also kept as the Tracker panel's Method), `positionScaleRotationSkew`, +`perspective`, `faceOutline` (Face Tracking (Outline Only): the mask, drawn around a face, becomes +the face outline on every frame) or `faceDetailed` (Face Tracking (Detailed Features): also keys a +Face Track Points effect with one point per landmark — eyebrows, eyes, pupils, nose tip, +nostrils, mouth corners and lips, chin, jaw); `direction` is `forward|backward|frameForward|frameBackward` from the current time (or `start` / `end` in seconds). Like `track.analyze` it runs in the background in the app (progress in `track.status` under `mask`, `track.stop` cancels and keeps what was tracked) and is -one undo step; `"wait": true` blocks. +one undo step; `"wait": true` blocks. `track.extractFaceMeasurements {layer?}` (Extract & Copy Face +Measurements) keys a Face Measurements effect from the Face Track Points (head position, scale and +orientation X/Y/Z, eye openness, eyebrow distance from eye, mouth openness, width and offset), puts +those keys on the keyframe clipboard and returns them per frame plus a tab-separated `clipboard` +text. `mask.interpolate` (Window ▸ Mask Interpolation ▸ Apply) adds in-between Mask Path keys between each pair of selected Mask Path keys (or the existing keys at `times`, in seconds), giving both @@ -138,7 +145,11 @@ the app re-analyses automatically (headless: run `warp.analyze` again). Settings properties under the instance's groups, e.g. `effects/#1/stabilization/result` (0 Smooth Motion, 1 No Motion), `effects/#1/stabilization/smoothness`, `effects/#1/stabilization/method`, `effects/#1/borders/framing` (0 Stabilize Only … 3 Stabilize, Synthesize Edges), -`effects/#1/borders/autoScale/maximumScale`, `effects/#1/advanced/showTrackPoints`. +`effects/#1/borders/autoScale/maximumScale`, `effects/#1/advanced/showTrackPoints`. Method 3 +(Subspace Warp, the default) bends a mesh per frame to the smoothed feature trajectories, so +scenes with depth stabilise where one perspective transform cannot; +`effects/#1/advanced/rollingShutterRipple` (0 Automatic Reduction, 1 Enhanced Reduction: a finer, +softer mesh) tunes it. ```sh effectcraft-cli run shaky.ecproj track.warpStabilizer '{"layer":"#1","wait":true}' \ @@ -150,13 +161,17 @@ effectcraft-cli exec warp.status '{"layer":"#1"}' shaky.ecproj --json ### 3D Camera Tracker Animation ▸ Track Camera and the Tracker panel's button (`track.camera {layer?, shotType?: -fixed|variable|specify, aov?, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}`) apply +fixed|variable|specify, aov?, solveMethod?: auto|typical|flat|tripod, detailed?, lensDistortion?, +undistort?, wait?}`) apply Effect ▸ Perspective ▸ 3D Camera Tracker (or reuse the layer's) and analyse it in the background: "Analyzing in background (step 1 of 2)" tracks features, "Solving camera" solves. `camera.analyze {layer?, effect?, wait?}` re-runs it, `camera.cancel` stops it without writing anything. `camera.solveStatus` reports `progress`/`banner`, `analyzed`, `solved`, `methodUsed`, `averageError` (pixels), `focalLength`, `horizontalAngleOfView`, the current frame's `camera` -(position, orientation, zoom) and `groundPlane`. `camera.points {time?}` lists the solved points +(position, orientation, zoom), `groundPlane` and the solved `lensDistortion` (`k1`, `k2`, normalised by +half the frame diagonal) when Solve Lens Distortion (`effects/#1/advanced/lensDistortion`, also on +with Detailed Analysis) is on; `effects/#1/advanced/undistort` renders the footage undistorted so +the solved camera's 3D layers line up. `camera.points {time?}` lists the solved points visible now with their `id`, `comp` position, `depth`, `world` position and `error`. Select points with `camera.selectPoints {points, add?, toggle?}` (the viewer's click / Shift-click / diff --git a/docs/architecture.md b/docs/architecture.md index a54703b7..f54c2b9c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -228,6 +228,13 @@ tracks them with pyramidal Lucas–Kanade and fits the chosen model (position RANSAC; the motion moves the Mask Path's vertices and tangents and is keyed per frame. **Mask Interpolation** (`mask.interpolate`) uses `effectcraft-path`'s smart interpolation (arc-length vertex insertion, shape-context matching, rigid in-betweens) to key in-between shapes. +The two **Face Tracking** methods (`effectcraft_track::face`) fit a face inside the mask instead: +a skin colour model from the first frame, the face outline along rays from the skin component's +centre (an area-moment ellipse gives centre, size and roll), facial features as the non-skin +components inside it, and an active-shape point distribution model trained on synthetic face +shapes (our own generator, no external weights) that fills and checks the landmarks. The outline +keys the Mask Path; Detailed Features keys a Face Track Points effect, and +`track.extractFaceMeasurements` derives a keyed Face Measurements effect (and copies its keys). **Warp Stabilizer** (`effects::warp_stab`, `engine::warp`): `warp.analyze` renders the layer's input to the effect (`Renderer::layer_input`: source, masks and the effects above it) for every @@ -236,7 +243,10 @@ translation / similarity / homography fits) as JSON in the effect's hidden Analy with a key of the layer's source, In/Out, start, stretch and Time Remap. `Session::edit` clears analyses whose key no longer matches and queues them; the desktop app re-analyses them in the background. Rendering derives the stabilization plan (smoothed camera path or No Motion, framing, -auto-scale; cached per analysis and settings) and warps each frame; Synthesize Edges fills the +auto-scale; cached per analysis and settings) and warps each frame. Subspace Warp also stores long +feature trajectories in the analysis, smooths them in a low-rank subspace (Liu et al. 2011) and +warps each frame with a content-preserving mesh (Liu et al. 2009) fitted to the smoothed +positions, rendered as a per-pixel mesh lookup; Synthesize Edges fills the borders from neighbouring frames read with `EffectHost::self_at`. The effect lives in the effects crate, hence the `effects → track` edge. @@ -267,7 +277,10 @@ the essential matrix (normalised 8-point in RANSAC) or a planar homography decom resection and triangulation; a sparse Levenberg–Marquardt bundle adjustment (Schur complement over the points, Huber loss) with the focal length fixed, shared, or per frame; a log-spaced and golden-section focal search for Fixed Angle of View / Variable Zoom; a rotation-only model for tripod -pans, chosen by Auto Detect when it explains the tracks as well. Tracks and solve are stored as JSON +pans, chosen by Auto Detect when it explains the tracks as well; with Solve Lens Distortion (or +Detailed Analysis) radial distortion `k1, k2` is adjusted jointly in a final bundle adjustment on +the raw tracks and the solve repeated on tracks undistorted with that estimate; Undistort Footage +renders the input through the inverse model. Tracks and solve are stored as JSON in hidden effect parameters with keys: changing the layer's frames clears both, changing Shot Type, Angle of View, Solve Method or deleting points only re-solves. The solve's canonical frame maps to comp space so the first frame's camera is the default comp camera (or so a chosen ground plane is diff --git a/docs/effects.md b/docs/effects.md index 84735286..fd29541b 100644 --- a/docs/effects.md +++ b/docs/effects.md @@ -3,11 +3,11 @@ -EffectCraft ships 304 effects with 2446 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. +EffectCraft ships 306 effects with 2488 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. - **GPU**: 59 effects also run on the GPU compositor with identical results. -- **32**: 304 effects process 32-bit float (HDR, overbright) pixels without clamping. -- **Status**: 262 implemented in full, 42 partial (what is missing is listed). +- **32**: 306 effects process 32-bit float (HDR, overbright) pixels without clamping. +- **Status**: 264 implemented in full, 42 partial (what is missing is listed). Effects are addressed by id (`ec..`) or display name in commands, scripts and the control channel, and every parameter by its id (`effects/#1/blurriness`). @@ -302,7 +302,7 @@ Effects are addressed by id (`ec..`) or display name in commands | Effect | Id | Params | GPU | 32 | Status | |---|---|---:|:-:|:-:|---| -| 3D Camera Tracker | `ec.perspective.cameratracker` | 10 | | 32 | Implemented | +| 3D Camera Tracker | `ec.perspective.cameratracker` | 12 | | 32 | Implemented | | 3D Glasses | `ec.perspective.3dglasses` | 8 | | 32 | Implemented | | Bevel Alpha | `ec.perspective.bevelalpha` | 4 | | 32 | Implemented | | Bevel Edges | `ec.perspective.beveledges` | 4 | | 32 | Implemented | @@ -416,6 +416,8 @@ Effects are addressed by id (`ec..`) or display name in commands | CC Overbrights | `ec.utility.ccoverbrights` | 2 | | 32 | Implemented | | Cineon Converter | `ec.utility.cineon` | 7 | | 32 | Implemented | | Color Profile Converter | `ec.utility.colorprofileconverter` | 7 | | 32 | Implemented | +| Face Measurements | `ec.utility.facemeasurements` | 14 | | 32 | Implemented | +| Face Track Points | `ec.utility.facetrackpoints` | 26 | | 32 | Implemented | | Grow Bounds | `ec.utility.growbounds` | 1 | | 32 | Implemented | | HDR Compander | `ec.utility.hdrcompander` | 3 | | 32 | Implemented | | HDR Highlight Compression | `ec.utility.hdrcompression` | 1 | | 32 | Implemented | diff --git a/docs/parity.md b/docs/parity.md index 4437bece..2d03442b 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -17,13 +17,13 @@ How close EffectCraft is to After Effects 2026, feature by feature, and how much By area: Layers 98%, Output 95%, Compositions 95%, Automation 96%, Paint 95%, Text 95%, Import 96%, Animation 94%, Masks 94%, Preview 94%, Interface 93%, Shapes 93%, 3D 91%, Audio 90%, Project 90%, -Tracking 88%, Effects 85%, Web 65%. +Tracking 96%, Effects 85%, Web 65%. What is left, in priority order: the web app's depth (threads, storage, audio, non-blocking renders); stroke taper/wave and multi-segment dashes; variable mask feather points; camera iris/bokeh and focus-link commands; Render Queue field render, crop/resize and templates; -approximated effects (Subspace Warp, Key Cleaner) and Liquify's viewer brush; OpenType features and variable +approximated effects (Key Cleaner) and Liquify's viewer brush; OpenType features and variable font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; ScriptUI; -Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; face tracking; and +Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; and the "better than After Effects" items (a plug-in API, branching history; GPU particles landed in M12.7). The sections below are the original audit (morning of 2 October, ≈ 64%) and its updates, kept for @@ -124,7 +124,7 @@ the disk cache in the formats wave; the 3D Camera Tracker in M12.6.) | Project | ≈ 68% | 5.5 | auto-save, folder moves, OCIO displays beyond the built-in tone map (Color Engine with OCIO/ACES working spaces, HDR compand/tone mapping, Rec. 2100 PQ/HLG output, Feet + Frames, display colour management, Simulate Output and the locked viewer landed in M7.7; proxies and Interpret Footage fields / pixel aspect / alpha guess landed: PRJ-8, PRJ-3) | | Masks & roto | 74% | 5.0 | Roto Brush's learned (3.0) segmentation model (variable-width mask feather points with the Mask Feather tool landed in M13.5; mask tracking and Mask Interpolation landed in M6.6; Roto Brush & Refine Edge with graph-cut segmentation, flow propagation, edge matting, decontamination and Freeze in M6.7) | | Preview | 66% | 4.0 | GPU bokeh depth of field, wireframes and Advanced 3D compositing (Classic 3D runs and adjustment layers composite on the GPU since M12.7; persistent disk cache with the blue cache bar landed; region of interest, snapshots, exposure and Fast Previews landed in M0.13) | -| Tracking | ≈ 85% | 2.5 | face tracking, Subspace Warp's mesh warp, lens distortion in the camera solve (Rolling Shutter Repair landed in M9.11; point tracker, mask tracking, Warp Stabilizer and the 3D Camera Tracker landed in M6.x / M12.5 / M12.6) | +| Tracking | ≈ 96% | 0.3 | face tracking is a classical (skin model + feature components + shape model) fitter, not a learned detector: profile views and occluded faces are weak; Rolling Shutter Ripple is approximated by Subspace Warp's mesh density (face tracking (Outline Only / Detailed Features with Face Track Points and Extract & Copy Face Measurements), Subspace Warp's content-preserving mesh warp on subspace-smoothed trajectories, and radial lens distortion (k1, k2) in the camera bundle adjustment with Undistort Footage landed in M13.3; Rolling Shutter Repair in M9.11; point tracker, mask tracking, Warp Stabilizer and the 3D Camera Tracker in M6.x / M12.5 / M12.6) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | ## Effects still missing