From bab4f88c22e525ac69474a8293b155fe70fccab4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 21:44:51 -0400 Subject: [PATCH 1/2] M12.6: camera tracker solve Structure from motion for the 3D Camera Tracker in effectcraft-track (camtrack): long KLT feature tracks with re-detection, parallax keyframes, two-view initialisation (normalised 8-point essential matrix in RANSAC or planar homography decomposition), incremental resection + triangulation, sparse Levenberg-Marquardt bundle adjustment with the Schur complement and Huber loss, focal length search (Fixed / Variable Zoom / Specify Angle of View), tripod pan solve, Auto Detect, ground plane / target / world mapping. Synthetic-scene tests (path, focal, error, tripod, flat, rendered frames). Co-Authored-By: Claude Opus 5.5 --- crates/track/Cargo.toml | 2 +- crates/track/src/camtrack/bundle.rs | 416 ++++++++++++ crates/track/src/camtrack/features.rs | 115 ++++ crates/track/src/camtrack/geometry.rs | 432 +++++++++++++ crates/track/src/camtrack/linalg.rs | 476 ++++++++++++++ crates/track/src/camtrack/mod.rs | 376 +++++++++++ crates/track/src/camtrack/solver.rs | 894 ++++++++++++++++++++++++++ crates/track/src/camtrack/tests.rs | 360 +++++++++++ crates/track/src/lib.rs | 1 + 9 files changed, 3071 insertions(+), 1 deletion(-) create mode 100644 crates/track/src/camtrack/bundle.rs create mode 100644 crates/track/src/camtrack/features.rs create mode 100644 crates/track/src/camtrack/geometry.rs create mode 100644 crates/track/src/camtrack/linalg.rs create mode 100644 crates/track/src/camtrack/mod.rs create mode 100644 crates/track/src/camtrack/solver.rs create mode 100644 crates/track/src/camtrack/tests.rs diff --git a/crates/track/Cargo.toml b/crates/track/Cargo.toml index ba449c92..9bd46e26 100644 --- a/crates/track/Cargo.toml +++ b/crates/track/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-track" -description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, and corner-pin / transform solves" +description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, corner-pin / transform solves, stabilization and 3D camera solving (structure from motion)" version.workspace = true edition.workspace = true license.workspace = true diff --git a/crates/track/src/camtrack/bundle.rs b/crates/track/src/camtrack/bundle.rs new file mode 100644 index 00000000..84e8a5ad --- /dev/null +++ b/crates/track/src/camtrack/bundle.rs @@ -0,0 +1,416 @@ +//! Sparse bundle adjustment: Levenberg–Marquardt over cameras and points with the **Schur +//! complement** eliminating the points (Triggs, McLauchlan, Hartley & Fitzgibbon, "Bundle +//! Adjustment — A Modern Synthesis", 2000, §6; H&Z Appendix 6). Residuals are reprojection +//! errors in pixels with a **Huber** robust loss (applied by iteratively reweighted least +//! squares, Triggs et al. §3.3). +//! +//! Cameras are `(R, C, f)`; rotations are updated multiplicatively (`R ← exp([δ]×) R`), focal +//! lengths in log space. The focal length is held fixed, shared by all cameras, or per camera +//! (with a smoothness prior between consecutive cameras). Tripod solves hold the centres fixed. + +use rayon::prelude::*; + +use super::geometry::Pose; +use super::linalg::*; + +/// A camera in the adjustment. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct BaCam { + pub pose: Pose, + /// Focal length (pixels). + pub f: f64, +} + +/// One observation: camera `cam` sees point `pt` at `uv` (pixels relative to the principal +/// point). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct BaObs { + pub cam: u32, + pub pt: u32, + pub uv: [f64; 2], +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum FocalMode { + Fixed, + Shared, + /// Per camera, with prior weight `k` on the log-focal difference of consecutive cameras. + PerCamera(f64), +} + +#[derive(Clone, Debug)] +pub struct BaOpts { + pub focal: FocalMode, + /// Motion only: the points stay fixed. + pub fix_points: bool, + /// The camera centres stay fixed (tripod). + pub fix_centers: bool, + /// Cameras that stay fixed (gauge), by index; missing entries are free. + pub fixed: Vec, + /// Huber threshold (pixels). + pub huber: f64, + pub max_iter: usize, +} + +impl Default for BaOpts { + fn default() -> Self { + BaOpts { focal: FocalMode::Fixed, fix_points: false, fix_centers: false, fixed: vec![], huber: 2.0, max_iter: 30 } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct BaReport { + pub initial_cost: f64, + pub cost: f64, + pub iterations: usize, +} + +const BEHIND: f64 = 1e6; + +fn huber(e2: f64, k: f64) -> f64 { + if e2 <= k * k { e2 } else { 2.0 * k * e2.sqrt() - k * k } +} + +/// Residual (pixels) of one observation, `None` behind the camera. +pub fn residual(c: &BaCam, x: V3, uv: [f64; 2]) -> Option<[f64; 2]> { + let p = c.pose.to_cam(x); + if p[2] <= 1e-9 { + return None; + } + Some([c.f * p[0] / p[2] - uv[0], c.f * p[1] / p[2] - uv[1]]) +} + +/// Total robust cost. +pub fn cost(cams: &[BaCam], pts: &[V3], obs: &[BaObs], o: &BaOpts) -> f64 { + let c: f64 = obs + .par_iter() + .map(|ob| match residual(&cams[ob.cam as usize], pts[ob.pt as usize], ob.uv) { + Some(r) => huber(r[0] * r[0] + r[1] * r[1], o.huber), + None => BEHIND, + }) + .sum(); + c + prior_cost(cams, o) +} + +fn prior_cost(cams: &[BaCam], o: &BaOpts) -> f64 { + match o.focal { + FocalMode::PerCamera(k) => cams.windows(2).map(|w| (k * (w[0].f.ln() - w[1].f.ln())).powi(2)).sum(), + _ => 0.0, + } +} + +/// Parameter layout. +struct Layout { + /// First pose parameter of each camera (`usize::MAX` when its pose is fixed). + start: Vec, + /// Per-camera focal parameter (`usize::MAX` when none). Gauge-fixed cameras keep a free + /// focal length. + fidx: Vec, + /// Index of the shared focal parameter. + shared: Option, + n: usize, + rot_only: bool, +} + +fn layout(ncam: usize, o: &BaOpts) -> Layout { + let per_cam_f = matches!(o.focal, FocalMode::PerCamera(_)); + let d = if o.fix_centers { 3 } else { 6 }; + let mut start = vec![usize::MAX; ncam]; + let mut fidx = vec![usize::MAX; ncam]; + let mut n = 0; + for i in 0..ncam { + if !o.fixed.get(i).copied().unwrap_or(false) { + start[i] = n; + n += d; + } + if per_cam_f { + fidx[i] = n; + n += 1; + } + } + let shared = matches!(o.focal, FocalMode::Shared).then(|| { + n += 1; + n - 1 + }); + Layout { start, fidx, shared, n, rot_only: o.fix_centers } +} + +/// Camera-side Jacobian entries of one observation: (parameter index, d r / d param). +fn cam_jac(l: &Layout, ci: usize, c: &BaCam, xc: V3, a: &[[f64; 3]; 2], p: [f64; 2], out: &mut Vec<(usize, [f64; 2])>) { + out.clear(); + let s = l.start[ci]; + if s != usize::MAX { + // dXc/dδ = −[Xc]× + let sk = skew(xc); + for k in 0..3 { + let g = [-sk[0][k], -sk[1][k], -sk[2][k]]; + out.push((s + k, [dot(a[0], g), dot(a[1], g)])); + } + if !l.rot_only { + // dXc/dC = −R + for k in 0..3 { + let g = [-c.pose.r[0][k], -c.pose.r[1][k], -c.pose.r[2][k]]; + out.push((s + 3 + k, [dot(a[0], g), dot(a[1], g)])); + } + } + } + if l.fidx[ci] != usize::MAX { + out.push((l.fidx[ci], p)); + } + if let Some(g) = l.shared { + out.push((g, p)); + } +} + +/// Run Levenberg–Marquardt. `cams` and `pts` are updated in place. +pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> BaReport { + let l = layout(cams.len(), o); + let npts = pts.len(); + // Observations by point. + let mut by_pt: Vec> = vec![vec![]; npts]; + for (i, ob) in obs.iter().enumerate() { + by_pt[ob.pt as usize].push(i as u32); + } + let mut cur = cost(cams, pts, obs, o); + let initial = cur; + let mut lambda = 1e-4; + let mut iters = 0; + if l.n == 0 && o.fix_points { + return BaReport { initial_cost: initial, cost: cur, iterations: 0 }; + } + let n = l.n; + let parallel = n <= 700; + for _ in 0..o.max_iter { + iters += 1; + // ---- Normal equations (Gauss–Newton with IRLS weights). + // Per point: V (3×3), bp (3), W entries (cam-side index → 3). + struct PtSys { + v: [[f64; 3]; 3], + bp: V3, + w: Vec<(usize, V3)>, + } + let build_point = |j: usize, s_acc: &mut Vec, bc: &mut Vec| -> PtSys { + let mut ps = PtSys { v: [[0.0; 3]; 3], bp: [0.0; 3], w: vec![] }; + let mut jc: Vec<(usize, [f64; 2])> = Vec::with_capacity(8); + for &oi in &by_pt[j] { + let ob = &obs[oi as usize]; + let ci = ob.cam as usize; + let c = &cams[ci]; + let xc = c.pose.to_cam(pts[j]); + if xc[2] <= 1e-9 { + continue; + } + let iz = 1.0 / xc[2]; + let p = [c.f * xc[0] * iz, c.f * xc[1] * iz]; + let r = [p[0] - ob.uv[0], p[1] - ob.uv[1]]; + let e = (r[0] * r[0] + r[1] * r[1]).sqrt(); + let w = if e <= o.huber { 1.0 } else { o.huber / e }; + let a = [[c.f * iz, 0.0, -c.f * xc[0] * iz * iz], [0.0, c.f * iz, -c.f * xc[1] * iz * iz]]; + cam_jac(&l, ci, c, xc, &a, p, &mut jc); + // Point Jacobian: A R. + let jx = if o.fix_points { + [[0.0; 3]; 2] + } else { + let r0 = mtv(&c.pose.r, a[0]); + let r1 = mtv(&c.pose.r, a[1]); + [r0, r1] + }; + for (ia, ja) in &jc { + bc[*ia] -= w * (ja[0] * r[0] + ja[1] * r[1]); + for (ib, jb) in &jc { + s_acc[ia * n + ib] += w * (ja[0] * jb[0] + ja[1] * jb[1]); + } + } + if !o.fix_points { + for a1 in 0..3 { + ps.bp[a1] -= w * (jx[0][a1] * r[0] + jx[1][a1] * r[1]); + for b1 in 0..3 { + ps.v[a1][b1] += w * (jx[0][a1] * jx[0][b1] + jx[1][a1] * jx[1][b1]); + } + } + for (ia, ja) in &jc { + let wv = + [w * (ja[0] * jx[0][0] + ja[1] * jx[1][0]), w * (ja[0] * jx[0][1] + ja[1] * jx[1][1]), w * (ja[0] * jx[0][2] + ja[1] * jx[1][2])]; + match ps.w.iter_mut().find(|e| e.0 == *ia) { + Some(e) => e.1 = add(e.1, wv), + None => ps.w.push((*ia, wv)), + } + } + } + } + ps + }; + let (mut s_mat, mut bc, systems): (Vec, Vec, Vec) = if parallel && npts > 64 { + let chunks = rayon::current_num_threads().clamp(1, 16); + let per = npts.div_ceil(chunks); + let parts: Vec<(Vec, Vec, Vec)> = (0..chunks) + .into_par_iter() + .map(|t| { + let (mut s, mut b) = (vec![0.0; n * n], vec![0.0; n]); + let v = (t * per..((t + 1) * per).min(npts)).map(|j| build_point(j, &mut s, &mut b)).collect(); + (s, b, v) + }) + .collect(); + let mut s = vec![0.0; n * n]; + let mut b = vec![0.0; n]; + let mut sys = Vec::with_capacity(npts); + for (ps, pb, pv) in parts { + for (x, y) in s.iter_mut().zip(ps) { + *x += y; + } + for (x, y) in b.iter_mut().zip(pb) { + *x += y; + } + sys.extend(pv); + } + (s, b, sys) + } else { + let mut s = vec![0.0; n * n]; + let mut b = vec![0.0; n]; + let sys = (0..npts).map(|j| build_point(j, &mut s, &mut b)).collect(); + (s, b, sys) + }; + // Focal prior. + if let FocalMode::PerCamera(k) = o.focal { + for i in 0..cams.len().saturating_sub(1) { + let r = k * (cams[i].f.ln() - cams[i + 1].f.ln()); + let (a, b) = (l.fidx[i], l.fidx[i + 1]); + let ia = (a != usize::MAX).then_some(a); + let ib = (b != usize::MAX).then_some(b); + for (idx, g) in [(ia, k), (ib, -k)] { + let Some(x) = idx else { continue }; + bc[x] -= g * r; + for (idy, h) in [(ia, k), (ib, -k)] { + if let Some(y) = idy { + s_mat[x * n + y] += g * h; + } + } + } + } + } + let base_s = s_mat.clone(); + let base_bc = bc.clone(); + let mean_diag = ((0..n).map(|i| base_s[i * n + i]).sum::() / n.max(1) as f64).max(1e-12); + // ---- Try steps with increasing damping. + let mut accepted = false; + for _try in 0..12 { + let mut s = base_s.clone(); + let mut rhs = base_bc.clone(); + for i in 0..n { + let d = s[i * n + i]; + s[i * n + i] = d + lambda * d.max(1e-6 * mean_diag); + } + // Schur complement over the points. + let mut vinv: Vec> = vec![None; npts]; + if !o.fix_points { + for (j, ps) in systems.iter().enumerate() { + let mut v = ps.v; + let vd = (v[0][0] + v[1][1] + v[2][2]) / 3.0; + for k in 0..3 { + v[k][k] += lambda * v[k][k].max(1e-6 * vd.max(1e-12)) + 1e-12; + } + let Some(vi) = inv3(&v) else { continue }; + vinv[j] = Some(vi); + let vb = mv(&vi, ps.bp); + let wv: Vec = ps.w.iter().map(|(_, w)| mv(&vi, *w)).collect(); + for (a, (ia, wa)) in ps.w.iter().enumerate() { + rhs[*ia] -= dot(*wa, vb); + for (ib, wb) in &ps.w { + s[ia * n + ib] -= dot(wv[a], *wb); + } + } + } + } + let mut dc = rhs; + if n > 0 && !cholesky_solve(&mut s, n, &mut dc) { + lambda *= 10.0; + continue; + } + // Apply. + let mut nc: Vec = cams.to_vec(); + for (i, c) in nc.iter_mut().enumerate() { + let st = l.start[i]; + if st != usize::MAX { + let dr = [dc[st], dc[st + 1], dc[st + 2]]; + c.pose.r = mm(&rodrigues(dr), &c.pose.r); + if !l.rot_only { + c.pose.c = add(c.pose.c, [dc[st + 3], dc[st + 4], dc[st + 5]]); + } + } + if l.fidx[i] != usize::MAX { + c.f *= dc[l.fidx[i]].clamp(-0.5, 0.5).exp(); + } + if let Some(g) = l.shared { + c.f *= dc[g].clamp(-0.5, 0.5).exp(); + } + } + let mut np: Vec = pts.to_vec(); + if !o.fix_points { + np.par_iter_mut().enumerate().for_each(|(j, x)| { + let Some(vi) = &vinv[j] else { return }; + let ps = &systems[j]; + let mut b = ps.bp; + for (ia, w) in &ps.w { + b = sub(b, scale(*w, dc[*ia])); + } + *x = add(*x, mv(vi, b)); + }); + } + let c2 = cost(&nc, &np, obs, o); + if c2.is_finite() && c2 < cur { + cams.copy_from_slice(&nc); + pts.copy_from_slice(&np); + let gain = cur - c2; + cur = c2; + lambda = (lambda / 3.0).max(1e-9); + accepted = gain > 1e-10 * cur.max(1e-12); + if !accepted { + // Converged (tiny improvement). + return BaReport { initial_cost: initial, cost: cur, iterations: iters }; + } + break; + } + lambda *= 4.0; + } + if !accepted { + break; + } + } + BaReport { initial_cost: initial, cost: cur, iterations: iters } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::camtrack::geometry::Rng; + + #[test] + fn bundle_adjustment_converges_from_noise() { + let mut rng = Rng(99); + let mut u = || (rng.next_u64() % 100_000) as f64 / 100_000.0 - 0.5; + let truth: Vec = (0..80).map(|_| [u() * 6.0, u() * 4.0, 6.0 + u() * 4.0]).collect(); + let f = 900.0; + let cams_true: Vec = (0..6) + .map(|i| BaCam { pose: Pose { r: rodrigues([0.0, -0.05 * i as f64, 0.01 * i as f64]), c: [0.3 * i as f64, 0.05 * i as f64, 0.0] }, f }) + .collect(); + let mut obs = vec![]; + for (ci, c) in cams_true.iter().enumerate() { + for (pi, x) in truth.iter().enumerate() { + let r = residual(c, *x, [0.0, 0.0]).unwrap(); + obs.push(BaObs { cam: ci as u32, pt: pi as u32, uv: r }); + } + } + let mut cams = cams_true.clone(); + for (i, c) in cams.iter_mut().enumerate().skip(2) { + c.pose.r = mm(&rodrigues([0.01, -0.01 * i as f64 * 0.3, 0.005]), &c.pose.r); + c.pose.c = add(c.pose.c, [0.02, -0.03, 0.01]); + } + let mut pts: Vec = truth.iter().map(|x| add(*x, [u() * 0.1, u() * 0.1, u() * 0.2])).collect(); + let mut fixed = vec![false; 6]; + fixed[0] = true; + fixed[1] = true; + let rep = adjust(&mut cams, &mut pts, &obs, &BaOpts { fixed, max_iter: 50, ..Default::default() }); + let rms = (rep.cost / obs.len() as f64).sqrt(); + assert!(rms < 1e-4, "{rep:?}"); + } +} diff --git a/crates/track/src/camtrack/features.rs b/crates/track/src/camtrack/features.rs new file mode 100644 index 00000000..311ec711 --- /dev/null +++ b/crates/track/src/camtrack/features.rs @@ -0,0 +1,115 @@ +//! Step 1 of the camera analysis: long feature tracks through the clip. +//! +//! Every frame, the active features are followed from the previous frame with pyramidal +//! Lucas–Kanade and a forward–backward check ([`crate::klt::track`]); a feature that fails ends +//! its track. When fewer than ~85 % of the wanted features remain, new Shi–Tomasi corners are +//! detected away from the active ones and start new tracks. Tracks shorter than three frames are +//! dropped at the end. + +use crate::Frame; +use crate::klt::{CornerOpts, GrayPyramid, LkOpts, analysis_factor, good_features, track}; + +use super::{CameraTracks, Track2D}; + +/// Analysis options. +#[derive(Clone, Copy, Debug, Default)] +pub struct AnalyzeOpts { + /// Detailed Analysis: twice the resolution and more features. + pub detailed: bool, +} + +/// Incremental feature tracking: push the clip's frames in order. +pub struct TrackAnalyzer { + opts: AnalyzeOpts, + prev: Option, + /// (track index, level-0 plane position) of the live features. + active: Vec<(usize, [f64; 2])>, + tracks: Vec, + frame: u32, + size: [f64; 2], + factor: f64, +} + +impl TrackAnalyzer { + pub fn new(size: [f64; 2], opts: AnalyzeOpts) -> TrackAnalyzer { + TrackAnalyzer { opts, prev: None, active: vec![], tracks: vec![], frame: 0, size, factor: 1.0 } + } + + pub fn frames(&self) -> u32 { + self.frame + } + + /// Features currently followed. + pub fn active(&self) -> usize { + self.active.len() + } + + fn params(&self) -> (u32, usize) { + if self.opts.detailed { (1280, 900) } else { (720, 500) } + } + + /// Analyse the next frame. + pub fn push(&mut self, frame: &Frame) { + let (max_side, want) = self.params(); + let factor = analysis_factor(frame.img.width, frame.img.height, max_side); + self.factor = factor as f64; + let pyr = GrayPyramid::from_image(frame.img, frame.offset, factor, 4); + let k = self.frame; + match &self.prev { + Some(prev) if prev.width() == pyr.width() && prev.height() == pyr.height() && prev.offset == pyr.offset => { + let pts: Vec<[f64; 2]> = self.active.iter().map(|a| a.1).collect(); + let q = track(prev, &pyr, &pts, None, &LkOpts { radius: 6, iterations: 30, epsilon: 0.005, fb_max: 0.5, ..Default::default() }); + let mut next = Vec::with_capacity(self.active.len()); + for ((ti, _), q) in self.active.iter().zip(q) { + if let Some(q) = q { + let l = pyr.to_layer(q); + self.tracks[*ti].pts.push([l[0] as f32, l[1] as f32]); + next.push((*ti, q)); + } + } + self.active = next; + } + _ => self.active.clear(), + } + // Re-detect where there are no features. + if (self.active.len() as f64) < want as f64 * 0.85 { + let md = (pyr.width().max(pyr.height()) as f64 / (want as f64).sqrt() * 0.6).clamp(5.0, 40.0); + let cell = md; + let (gw, gh) = ((pyr.width() as f64 / cell).ceil() as usize + 1, (pyr.height() as f64 / cell).ceil() as usize + 1); + let mut occ = vec![false; gw * gh]; + for (_, p) in &self.active { + let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize); + for yy in cy.saturating_sub(1)..=(cy + 1).min(gh - 1) { + for xx in cx.saturating_sub(1)..=(cx + 1).min(gw - 1) { + occ[yy * gw + xx] = true; + } + } + } + let free = |p: [f64; 2]| { + let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize); + !occ.get(cy * gw + cx).copied().unwrap_or(true) + }; + let opts = CornerOpts { max_features: want - self.active.len().min(want), min_distance: md, quality: 0.005, window: 2, border: 8 }; + for p in good_features(&pyr, &opts, Some(&free)) { + let l = pyr.to_layer(p); + let id = self.tracks.len() as u32; + self.tracks.push(Track2D { id, start: k, pts: vec![[l[0] as f32, l[1] as f32]] }); + self.active.push((id as usize, p)); + } + } + self.prev = Some(pyr); + self.frame += 1; + } + + pub fn finish(self, start: f64, frame_duration: f64) -> CameraTracks { + let mut tracks: Vec = self.tracks.into_iter().filter(|t| t.pts.len() >= 3).collect(); + // Round to 1/100 pixel (smaller JSON) and renumber densely. + for (i, t) in tracks.iter_mut().enumerate() { + t.id = i as u32; + for p in t.pts.iter_mut() { + *p = [(p[0] * 100.0).round() / 100.0, (p[1] * 100.0).round() / 100.0]; + } + } + CameraTracks { version: 1, size: self.size, start, frame_duration, frames: self.frame, factor: self.factor, detailed: self.opts.detailed, tracks } + } +} diff --git a/crates/track/src/camtrack/geometry.rs b/crates/track/src/camtrack/geometry.rs new file mode 100644 index 00000000..6956ac2b --- /dev/null +++ b/crates/track/src/camtrack/geometry.rs @@ -0,0 +1,432 @@ +//! Two-view and multi-view geometry for the camera solver, from Hartley & Zisserman (H&Z), +//! *Multiple View Geometry in Computer Vision* (2nd ed.) and Ma et al., *An Invitation to 3-D +//! Vision* (2004). +//! +//! All image points here are **normalised** (calibrated) coordinates `x = (u − c) / f`. +//! A camera is `(R, C)` with `X_cam = R (X − C)`. + +use super::linalg::*; + +/// A deterministic xorshift generator for RANSAC samples. +pub struct Rng(pub u64); +impl Rng { + pub fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + pub fn below(&mut self, n: usize) -> usize { + (self.next_u64() % n.max(1) as u64) as usize + } + /// `k` distinct indices below `n`. + pub fn sample(&mut self, n: usize, k: usize, out: &mut Vec) { + out.clear(); + let mut guard = 0; + while out.len() < k && guard < 1000 { + guard += 1; + let c = self.below(n); + if !out.contains(&c) { + out.push(c); + } + } + } +} + +/// Hartley normalisation (H&Z §4.4.4): centroid to the origin, mean distance √2. +fn normalizer(p: &[[f64; 2]]) -> M3 { + let n = p.len().max(1) as f64; + let c = [p.iter().map(|q| q[0]).sum::() / n, p.iter().map(|q| q[1]).sum::() / n]; + let d = p.iter().map(|q| (q[0] - c[0]).hypot(q[1] - c[1])).sum::() / n; + let s = if d > 1e-12 { std::f64::consts::SQRT_2 / d } else { 1.0 }; + [[s, 0.0, -s * c[0]], [0.0, s, -s * c[1]], [0.0, 0.0, 1.0]] +} + +fn h2(m: &M3, p: [f64; 2]) -> V3 { + mv(m, [p[0], p[1], 1.0]) +} + +/// The essential matrix from ≥ 8 correspondences (`x2ᵀ E x1 = 0`) by the normalised 8-point +/// algorithm (H&Z Alg. 11.1), with the essential constraint (two equal singular values, one +/// zero) enforced (H&Z §9.6.1 / Result 9.18). +pub fn essential_8pt(x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Option { + let n = x1.len().min(x2.len()); + if n < 8 { + return None; + } + let (t1, t2) = (normalizer(&x1[..n]), normalizer(&x2[..n])); + let mut a = Vec::with_capacity(n * 9); + for i in 0..n { + let p = h2(&t1, x1[i]); + let q = h2(&t2, x2[i]); + a.extend_from_slice(&[q[0] * p[0], q[0] * p[1], q[0], q[1] * p[0], q[1] * p[1], q[1], p[0], p[1], 1.0]); + } + let e = null_vector(&a, n, 9); + let en = [[e[0], e[1], e[2]], [e[3], e[4], e[5]], [e[6], e[7], e[8]]]; + // Denormalise: E = T2ᵀ Ê T1. + let e = mm(&mt(&t2), &mm(&en, &t1)); + let (u, s, v) = svd3(&e); + let m = (s[0] + s[1]) * 0.5; + if m <= 0.0 || !m.is_finite() { + return None; + } + let d = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]]; + Some(mm(&u, &mm(&d, &mt(&v)))) +} + +/// Squared Sampson distance of a correspondence to `x2ᵀ E x1 = 0` (H&Z §11.4.3), in the units of +/// the coordinates. +pub fn sampson2(e: &M3, p: [f64; 2], q: [f64; 2]) -> f64 { + let x1 = [p[0], p[1], 1.0]; + let x2 = [q[0], q[1], 1.0]; + let ex1 = mv(e, x1); + let etx2 = mtv(e, x2); + let num = dot(x2, ex1); + let den = ex1[0] * ex1[0] + ex1[1] * ex1[1] + etx2[0] * etx2[0] + etx2[1] * etx2[1]; + if den < 1e-300 { f64::INFINITY } else { num * num / den } +} + +/// RANSAC essential matrix (8-point samples, Sampson distance below `thr`), refitted on the +/// consensus set. Returns the matrix and the inlier mask. +pub fn ransac_essential(x1: &[[f64; 2]], x2: &[[f64; 2]], thr: f64, iterations: usize, seed: u64) -> Option<(M3, Vec)> { + let n = x1.len().min(x2.len()); + if n < 8 { + return None; + } + let t2 = thr * thr; + let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1); + let mut best: Option<(usize, f64, M3)> = None; + let mut idx = vec![]; + let mut max_iter = iterations; + let mut it = 0; + let (mut s1, mut s2) = (vec![[0.0; 2]; 8], vec![[0.0; 2]; 8]); + while it < max_iter { + it += 1; + rng.sample(n, 8, &mut idx); + if idx.len() < 8 { + break; + } + for (k, &i) in idx.iter().enumerate() { + s1[k] = x1[i]; + s2[k] = x2[i]; + } + let Some(e) = essential_8pt(&s1, &s2) else { continue }; + let (mut c, mut err) = (0, 0.0); + for i in 0..n { + let d = sampson2(&e, x1[i], x2[i]); + if d < t2 { + c += 1; + err += d; + } + } + if best.as_ref().is_none_or(|b| c > b.0 || (c == b.0 && err < b.1)) { + best = Some((c, err, e)); + // Adaptive iteration count for 99 % confidence. + let w = c as f64 / n as f64; + let p_good = w.powi(8); + if p_good > 1e-9 { + let need = ((1.0 - 0.99f64).ln() / (1.0 - p_good).max(1e-12).ln()).ceil(); + if need.is_finite() { + max_iter = max_iter.min((need as usize).max(30)); + } + } + } + } + let (_, _, mut e) = best?; + // Refit on the inliers (twice: the consensus may grow). + let mut inl = vec![false; n]; + for _ in 0..2 { + for i in 0..n { + inl[i] = sampson2(&e, x1[i], x2[i]) < t2; + } + let (a, b): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..n).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip(); + match essential_8pt(&a, &b) { + Some(e2) => e = e2, + None => break, + } + } + for i in 0..n { + inl[i] = sampson2(&e, x1[i], x2[i]) < t2; + } + Some((e, inl)) +} + +/// The four `(R, t)` with `x2 ~ R x1 + t` (|t| = 1) of an essential matrix (H&Z Result 9.19). +pub fn decompose_essential(e: &M3) -> [(M3, V3); 4] { + let (mut u, _, mut v) = svd3(e); + if det3(&u) < 0.0 { + for row in u.iter_mut() { + row[2] = -row[2]; + } + } + if det3(&v) < 0.0 { + for row in v.iter_mut() { + row[2] = -row[2]; + } + } + let w = [[0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]]; + let r1 = mm(&u, &mm(&w, &mt(&v))); + let r2 = mm(&u, &mm(&mt(&w), &mt(&v))); + let t = col(&u, 2); + [(r1, t), (r1, scale(t, -1.0)), (r2, t), (r2, scale(t, -1.0))] +} + +/// A camera `(R, C)`. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Pose { + pub r: M3, + pub c: V3, +} + +impl Pose { + pub const IDENTITY: Pose = Pose { r: I3, c: [0.0; 3] }; + /// From `x_cam = R X + t`. + pub fn from_rt(r: M3, t: V3) -> Pose { + Pose { r, c: scale(mtv(&r, t), -1.0) } + } + pub fn t(&self) -> V3 { + scale(mv(&self.r, self.c), -1.0) + } + pub fn to_cam(&self, x: V3) -> V3 { + mv(&self.r, sub(x, self.c)) + } +} + +/// Linear (DLT) triangulation from normalised observations (H&Z §12.2), minimising the +/// algebraic error over all views. +pub fn triangulate(views: &[(Pose, [f64; 2])]) -> Option { + if views.len() < 2 { + return None; + } + let mut ata = [0.0f64; 16]; + for (p, x) in views { + let t = p.t(); + let rows = [[p.r[0][0], p.r[0][1], p.r[0][2], t[0]], [p.r[1][0], p.r[1][1], p.r[1][2], t[1]], [p.r[2][0], p.r[2][1], p.r[2][2], t[2]]]; + let a0: Vec = (0..4).map(|j| x[0] * rows[2][j] - rows[0][j]).collect(); + let a1: Vec = (0..4).map(|j| x[1] * rows[2][j] - rows[1][j]).collect(); + for a in [a0, a1] { + let nrm = a.iter().map(|v| v * v).sum::().sqrt().max(1e-300); + for i in 0..4 { + for j in 0..4 { + ata[i * 4 + j] += a[i] * a[j] / (nrm * nrm); + } + } + } + } + let (_, vecs) = sym_eigen(&ata, 4); + let h = [vecs[0], vecs[4], vecs[8], vecs[12]]; + if h[3].abs() < 1e-12 { + return None; + } + let x = [h[0] / h[3], h[1] / h[3], h[2] / h[3]]; + x.iter().all(|v| v.is_finite()).then_some(x) +} + +/// Angle (radians) between the rays from two camera centres to `x`. +pub fn ray_angle(a: V3, b: V3, x: V3) -> f64 { + let (u, v) = (normalize(sub(x, a)), normalize(sub(x, b))); + dot(u, v).clamp(-1.0, 1.0).acos() +} + +/// Squared reprojection error (normalised units) of `x` in `pose` against observation `o` +/// (`None` behind the camera). +pub fn reproj2(pose: &Pose, x: V3, o: [f64; 2]) -> Option { + let c = pose.to_cam(x); + if c[2] <= 1e-9 { + return None; + } + let (u, v) = (c[0] / c[2] - o[0], c[1] / c[2] - o[1]); + Some(u * u + v * v) +} + +/// Pick the essential decomposition with the most points in front of both cameras; returns +/// the second camera (the first is the identity) and that count. +pub fn select_pose(cands: &[(M3, V3)], x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Option<(Pose, usize)> { + let mut best: Option<(Pose, usize)> = None; + for (r, t) in cands { + let p2 = Pose::from_rt(*r, *t); + let mut c = 0; + for i in 0..x1.len() { + if let Some(x) = triangulate(&[(Pose::IDENTITY, x1[i]), (p2, x2[i])]) + && x[2] > 0.0 + && p2.to_cam(x)[2] > 0.0 + { + c += 1; + } + } + if best.as_ref().is_none_or(|b| c > b.1) { + best = Some((p2, c)); + } + } + best +} + +/// Decompose a calibrated planar homography `H ~ R + t nᵀ / d` (normalised coordinates, +/// `x2 ~ H x1`) into its motions `(R, t/d, n)` (Ma et al. §5.3.3). Up to four solutions; those +/// that put the plane behind the first camera are dropped. +pub fn decompose_homography(h: &M3, x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Vec<(M3, V3, V3)> { + let (_, s, _) = svd3(h); + if s[1] <= 1e-12 { + return vec![]; + } + let mut hn = mscale(h, 1.0 / s[1]); + // Sign: x2ᵀ H x1 > 0 for the correspondences (positive depth). + let pos = x1.iter().zip(x2).filter(|(p, q)| dot([q[0], q[1], 1.0], mv(&hn, [p[0], p[1], 1.0])) > 0.0).count(); + if pos * 2 < x1.len() { + hn = mscale(&hn, -1.0); + } + let hth = mm(&mt(&hn), &hn); + let a: Vec = hth.iter().flatten().copied().collect(); + let (vals, vecs) = sym_eigen(&a, 3); + // Descending: σ1² ≥ σ2² = 1 ≥ σ3². + let v1 = [vecs[2], vecs[5], vecs[8]]; + let v2 = [vecs[1], vecs[4], vecs[7]]; + let v3 = [vecs[0], vecs[3], vecs[6]]; + let (s1, s3) = (vals[2].max(1.0), vals[0].clamp(0.0, 1.0)); + let mut out = vec![]; + if s1 - s3 < 1e-10 { + // H is a rotation: no translation (pure rotation or plane at infinity). + return vec![(nearest_rotation(&hn), [0.0; 3], [0.0, 0.0, 1.0])]; + } + let k = (s1 - s3).sqrt(); + let (a1, a3) = ((1.0 - s3).sqrt(), (s1 - 1.0).sqrt()); + let u1 = scale(add(scale(v1, a1), scale(v3, a3)), 1.0 / k); + let u2 = scale(sub(scale(v1, a1), scale(v3, a3)), 1.0 / k); + for u in [u1, u2] { + let uu = from_cols(v2, u, cross(v2, u)); + let hv2 = mv(&hn, v2); + let hu = mv(&hn, u); + let ww = from_cols(hv2, hu, cross(hv2, hu)); + let r = mm(&ww, &mt(&uu)); + let n = cross(v2, u); + let t = mv(&madd(&hn, &r, -1.0), n); + for sgn in [1.0, -1.0] { + let (n2, t2) = (scale(n, sgn), scale(t, sgn)); + // The plane must be in front of the first camera: nᵀ x1 > 0 (with n pointing away). + let front = x1.iter().filter(|p| dot(n2, [p[0], p[1], 1.0]) > 0.0).count(); + if front * 2 >= x1.len() { + out.push((r, t2, n2)); + } + } + } + out +} + +/// Least-squares plane through points: (centroid, unit normal, RMS in-plane spread). +pub fn fit_plane(p: &[V3]) -> Option<(V3, V3, f64)> { + if p.len() < 3 { + return None; + } + let k = 1.0 / p.len() as f64; + let c = scale(p.iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [0.0; 9]; + for q in p { + let d = sub(*q, c); + for i in 0..3 { + for j in 0..3 { + cov[i * 3 + j] += d[i] * d[j] * k; + } + } + } + let (vals, vecs) = sym_eigen(&cov, 3); + if vals[1] <= 1e-18 { + return None; + } + let n = normalize([vecs[0], vecs[3], vecs[6]]); + Some((c, n, (vals[1] + vals[2]).max(0.0).sqrt())) +} + +/// The rotation `R` minimising `Σ |b − R a|²` over unit rays (orthogonal Procrustes / Kabsch). +pub fn procrustes(a: &[V3], b: &[V3]) -> Option { + if a.len() < 2 { + return None; + } + let mut m = [[0.0; 3]; 3]; + for (x, y) in a.iter().zip(b) { + m = madd(&m, &outer(*y, *x), 1.0); + } + let (u, _, v) = svd3(&m); + let mut r = mm(&u, &mt(&v)); + if det3(&r) < 0.0 { + let mut u2 = u; + for row in u2.iter_mut() { + row[2] = -row[2]; + } + r = mm(&u2, &mt(&v)); + } + Some(r) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scene() -> (Vec, Pose) { + let mut rng = Rng(12345); + let pts: Vec = (0..60) + .map(|_| { + let f = |r: &mut Rng| (r.next_u64() % 10_000) as f64 / 10_000.0 - 0.5; + [f(&mut rng) * 4.0, f(&mut rng) * 3.0, 5.0 + f(&mut rng) * 3.0] + }) + .collect(); + let p2 = Pose { r: rodrigues([0.02, -0.15, 0.03]), c: [0.6, 0.1, -0.2] }; + (pts, p2) + } + + fn proj(p: &Pose, x: V3) -> [f64; 2] { + let c = p.to_cam(x); + [c[0] / c[2], c[1] / c[2]] + } + + #[test] + fn essential_recovers_relative_pose() { + let (pts, p2) = scene(); + let x1: Vec<[f64; 2]> = pts.iter().map(|x| proj(&Pose::IDENTITY, *x)).collect(); + let mut x2: Vec<[f64; 2]> = pts.iter().map(|x| proj(&p2, *x)).collect(); + // Outliers. + x2[3] = [0.3, -0.2]; + x2[17] = [-0.1, 0.25]; + let (e, inl) = ransac_essential(&x1, &x2, 1e-3, 500, 7).unwrap(); + assert!(!inl[3] && !inl[17]); + assert!(inl.iter().filter(|b| **b).count() >= 57); + let (pose, n) = select_pose(&decompose_essential(&e), &x1, &x2).unwrap(); + assert!(n >= 57); + let r_err = norm(log_rot(&mm(&pose.r, &mt(&p2.r)))); + assert!(r_err < 1e-6, "{r_err}"); + let dir = normalize(p2.c); + assert!(norm(sub(normalize(pose.c), dir)) < 1e-5, "{:?} {:?}", pose.c, dir); + } + + #[test] + fn homography_decomposition_contains_the_motion() { + // Plane z = 4 (normal (0,0,1) in camera 1), seen after a motion. + let p2 = Pose { r: rodrigues([0.05, 0.2, -0.04]), c: [0.8, -0.3, 0.4] }; + let pts: Vec = (0..30).map(|i| [((i * 37) % 17) as f64 / 8.0 - 1.0, ((i * 11) % 13) as f64 / 6.0 - 1.0, 4.0]).collect(); + let x1: Vec<[f64; 2]> = pts.iter().map(|x| proj(&Pose::IDENTITY, *x)).collect(); + let x2: Vec<[f64; 2]> = pts.iter().map(|x| proj(&p2, *x)).collect(); + // H = R + t nᵀ / d with t = −R C, n = (0,0,1), d = 4. + let t = p2.t(); + let h = madd(&p2.r, &outer(t, [0.0, 0.0, 0.25]), 1.0); + let sols = decompose_homography(&mscale(&h, 3.7), &x1, &x2); + assert!(!sols.is_empty()); + let ok = sols + .iter() + .any(|(r, td, n)| norm(log_rot(&mm(r, &mt(&p2.r)))) < 1e-6 && norm(sub(*td, scale(t, 0.25))) < 1e-6 && norm(sub(*n, [0.0, 0.0, 1.0])) < 1e-6); + assert!(ok, "{sols:?}"); + } + + #[test] + fn triangulation_and_plane() { + let (pts, p2) = scene(); + for x in &pts[..5] { + let y = triangulate(&[(Pose::IDENTITY, proj(&Pose::IDENTITY, *x)), (p2, proj(&p2, *x))]).unwrap(); + assert!(norm(sub(*x, y)) < 1e-8); + } + let plane: Vec = (0..10).map(|i| [i as f64, (i * i % 7) as f64, 2.0 * i as f64 + 1.0]).collect(); + let (_, n, _) = fit_plane(&plane).unwrap(); + // All points satisfy z = 2x + 1: normal ∝ (2, 0, −1). + assert!(norm(cross(n, normalize([2.0, 0.0, -1.0]))) < 1e-9); + } +} diff --git a/crates/track/src/camtrack/linalg.rs b/crates/track/src/camtrack/linalg.rs new file mode 100644 index 00000000..a7067f26 --- /dev/null +++ b/crates/track/src/camtrack/linalg.rs @@ -0,0 +1,476 @@ +//! Small dense linear algebra for the camera solver, written from the textbook methods: +//! +//! - 3-vectors / 3 × 3 matrices and rotations (Rodrigues' formula and its inverse); +//! - the **one-sided Jacobi SVD** (Hestenes 1958; Golub & Van Loan, *Matrix Computations*, +//! §8.6.4), accurate for the small, badly scaled systems of the 8-point algorithm; +//! - the **cyclic Jacobi eigenvalue method** for symmetric matrices (Golub & Van Loan §8.5); +//! - **Cholesky** factorisation for the bundle adjustment's normal equations; +//! - the closed-form **similarity alignment** of two point sets (Umeyama, "Least-squares +//! estimation of transformation parameters between two point patterns", PAMI 1991). + +pub type V3 = [f64; 3]; +pub type M3 = [[f64; 3]; 3]; + +pub const I3: M3 = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]; + +pub fn add(a: V3, b: V3) -> V3 { + [a[0] + b[0], a[1] + b[1], a[2] + b[2]] +} +pub fn sub(a: V3, b: V3) -> V3 { + [a[0] - b[0], a[1] - b[1], a[2] - b[2]] +} +pub fn scale(a: V3, k: f64) -> V3 { + [a[0] * k, a[1] * k, a[2] * k] +} +pub fn dot(a: V3, b: V3) -> f64 { + a[0] * b[0] + a[1] * b[1] + a[2] * b[2] +} +pub fn cross(a: V3, b: V3) -> V3 { + [a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]] +} +pub fn norm(a: V3) -> f64 { + dot(a, a).sqrt() +} +pub fn normalize(a: V3) -> V3 { + let n = norm(a); + if n > 0.0 { scale(a, 1.0 / n) } else { a } +} +pub fn mv(m: &M3, v: V3) -> V3 { + [dot(m[0], v), dot(m[1], v), dot(m[2], v)] +} +/// `mᵀ v`. +pub fn mtv(m: &M3, v: V3) -> V3 { + [m[0][0] * v[0] + m[1][0] * v[1] + m[2][0] * v[2], m[0][1] * v[0] + m[1][1] * v[1] + m[2][1] * v[2], m[0][2] * v[0] + m[1][2] * v[1] + m[2][2] * v[2]] +} +pub fn mm(a: &M3, b: &M3) -> M3 { + let mut o = [[0.0; 3]; 3]; + for (i, row) in o.iter_mut().enumerate() { + for (j, v) in row.iter_mut().enumerate() { + *v = a[i][0] * b[0][j] + a[i][1] * b[1][j] + a[i][2] * b[2][j]; + } + } + o +} +pub fn mt(a: &M3) -> M3 { + [[a[0][0], a[1][0], a[2][0]], [a[0][1], a[1][1], a[2][1]], [a[0][2], a[1][2], a[2][2]]] +} +pub fn madd(a: &M3, b: &M3, k: f64) -> M3 { + let mut o = *a; + for i in 0..3 { + for j in 0..3 { + o[i][j] += k * b[i][j]; + } + } + o +} +pub fn mscale(a: &M3, k: f64) -> M3 { + madd(&[[0.0; 3]; 3], a, k) +} +pub fn det3(m: &M3) -> f64 { + m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0]) + m[0][2] * (m[1][0] * m[2][1] - m[1][1] * m[2][0]) +} +pub fn skew(v: V3) -> M3 { + [[0.0, -v[2], v[1]], [v[2], 0.0, -v[0]], [-v[1], v[0], 0.0]] +} +pub fn outer(a: V3, b: V3) -> M3 { + [[a[0] * b[0], a[0] * b[1], a[0] * b[2]], [a[1] * b[0], a[1] * b[1], a[1] * b[2]], [a[2] * b[0], a[2] * b[1], a[2] * b[2]]] +} +pub fn col(m: &M3, j: usize) -> V3 { + [m[0][j], m[1][j], m[2][j]] +} +pub fn from_cols(a: V3, b: V3, c: V3) -> M3 { + [[a[0], b[0], c[0]], [a[1], b[1], c[1]], [a[2], b[2], c[2]]] +} +pub fn inv3(m: &M3) -> Option { + let d = det3(m); + if d.abs() < 1e-300 { + return None; + } + let c = |r0: usize, c0: usize, r1: usize, c1: usize| m[r0][c0] * m[r1][c1] - m[r0][c1] * m[r1][c0]; + let adj = [[c(1, 1, 2, 2), -c(0, 1, 2, 2), c(0, 1, 1, 2)], [-c(1, 0, 2, 2), c(0, 0, 2, 2), -c(0, 0, 1, 2)], [c(1, 0, 2, 1), -c(0, 0, 2, 1), c(0, 0, 1, 1)]]; + Some(mscale(&adj, 1.0 / d)) +} + +/// Rotation matrix of an axis-angle vector (Rodrigues' formula). +pub fn rodrigues(w: V3) -> M3 { + let th = norm(w); + let k = skew(w); + if th < 1e-12 { + return madd(&I3, &k, 1.0); + } + let a = th.sin() / th; + let b = (1.0 - th.cos()) / (th * th); + madd(&madd(&I3, &k, a), &mm(&k, &k), b) +} + +/// Axis-angle vector of a rotation matrix (inverse of [`rodrigues`]). +pub fn log_rot(r: &M3) -> V3 { + let c = ((r[0][0] + r[1][1] + r[2][2] - 1.0) * 0.5).clamp(-1.0, 1.0); + let th = c.acos(); + let v = [r[2][1] - r[1][2], r[0][2] - r[2][0], r[1][0] - r[0][1]]; + if th < 1e-9 { + return scale(v, 0.5); + } + if std::f64::consts::PI - th < 1e-6 { + // Near π: the axis is the dominant column of R + I. + let b = madd(r, &I3, 1.0); + let j = (0..3).max_by(|&a, &bb| b[a][a].total_cmp(&b[bb][bb])).unwrap_or(0); + let axis = normalize(col(&b, j)); + return scale(axis, th); + } + scale(v, th / (2.0 * th.sin())) +} + +/// The rotation closest to `m` (Frobenius norm), via the SVD. +pub fn nearest_rotation(m: &M3) -> M3 { + let (u, _, v) = svd3(m); + let mut r = mm(&u, &mt(&v)); + if det3(&r) < 0.0 { + let mut u2 = u; + for row in u2.iter_mut() { + row[2] = -row[2]; + } + r = mm(&u2, &mt(&v)); + } + r +} + +/// Spherical interpolation between two rotations. +pub fn slerp(a: &M3, b: &M3, t: f64) -> M3 { + let d = log_rot(&mm(b, &mt(a))); + mm(&rodrigues(scale(d, t)), a) +} + +/// Thin SVD `A = U diag(s) Vᵀ` of an `m × n` row-major matrix (one-sided Jacobi). `U` is +/// `max(m, n) × n` (rows beyond `m` belong to zero padding), `s` is sorted descending and `V` +/// is `n × n` (columns are the right singular vectors). +pub fn svd(a: &[f64], m: usize, n: usize) -> (Vec, Vec, Vec) { + let rows = m.max(n); + let mut u = vec![0.0; rows * n]; + u[..m * n].copy_from_slice(&a[..m * n]); + let mut v = vec![0.0; n * n]; + for i in 0..n { + v[i * n + i] = 1.0; + } + for _sweep in 0..60 { + let mut off = 0.0f64; + for p in 0..n { + for q in p + 1..n { + let (mut alpha, mut beta, mut gamma) = (0.0, 0.0, 0.0); + for r in 0..rows { + let (x, y) = (u[r * n + p], u[r * n + q]); + alpha += x * x; + beta += y * y; + gamma += x * y; + } + if gamma == 0.0 { + continue; + } + let c0 = gamma.abs() / (alpha * beta).sqrt().max(1e-300); + off = off.max(c0); + if c0 < 1e-15 { + continue; + } + let zeta = (beta - alpha) / (2.0 * gamma); + let t = zeta.signum() / (zeta.abs() + (1.0 + zeta * zeta).sqrt()); + let t = if zeta == 0.0 { 1.0 } else { t }; + let c = 1.0 / (1.0 + t * t).sqrt(); + let s = c * t; + for r in 0..rows { + let (x, y) = (u[r * n + p], u[r * n + q]); + u[r * n + p] = c * x - s * y; + u[r * n + q] = s * x + c * y; + } + for r in 0..n { + let (x, y) = (v[r * n + p], v[r * n + q]); + v[r * n + p] = c * x - s * y; + v[r * n + q] = s * x + c * y; + } + } + } + if off < 1e-15 { + break; + } + } + let mut s: Vec = (0..n).map(|j| (0..rows).map(|r| u[r * n + j] * u[r * n + j]).sum::().sqrt()).collect(); + for (j, sj) in s.iter().enumerate() { + if *sj > 1e-300 { + for r in 0..rows { + u[r * n + j] /= sj; + } + } + } + // Sort descending. + let mut order: Vec = (0..n).collect(); + order.sort_by(|&a, &b| s[b].total_cmp(&s[a])); + let (mut u2, mut v2) = (vec![0.0; rows * n], vec![0.0; n * n]); + for (k, &j) in order.iter().enumerate() { + for r in 0..rows { + u2[r * n + k] = u[r * n + j]; + } + for r in 0..n { + v2[r * n + k] = v[r * n + j]; + } + } + s = order.iter().map(|&j| s[j]).collect(); + (u2, s, v2) +} + +/// SVD of a 3 × 3 matrix: (U, singular values descending, V) with `m = U diag(s) Vᵀ`. +pub fn svd3(m: &M3) -> (M3, V3, M3) { + let a: Vec = m.iter().flatten().copied().collect(); + let (u, s, v) = svd(&a, 3, 3); + let g = |x: &[f64]| [[x[0], x[1], x[2]], [x[3], x[4], x[5]], [x[6], x[7], x[8]]]; + let mut um = g(&u); + // Columns of U for zero singular values are undefined: complete them to an orthonormal basis. + if s[2] < 1e-300 { + let c = cross(col(&um, 0), col(&um, 1)); + for (r, row) in um.iter_mut().enumerate() { + row[2] = c[r]; + } + } + (um, [s[0], s[1], s[2]], g(&v)) +} + +/// The unit vector `x` minimising `|A x|` (`m × n` row-major): the right singular vector of +/// the smallest singular value. +pub fn null_vector(a: &[f64], m: usize, n: usize) -> Vec { + let (_, _, v) = svd(a, m, n); + (0..n).map(|r| v[r * n + n - 1]).collect() +} + +/// Eigen-decomposition of a symmetric `n × n` matrix (cyclic Jacobi): eigenvalues ascending +/// and the matching eigenvectors as the columns of the returned row-major matrix. +pub fn sym_eigen(a: &[f64], n: usize) -> (Vec, Vec) { + let mut a = a.to_vec(); + let mut v = vec![0.0; n * n]; + for i in 0..n { + v[i * n + i] = 1.0; + } + for _ in 0..100 { + let mut off = 0.0; + for p in 0..n { + for q in p + 1..n { + off += a[p * n + q] * a[p * n + q]; + } + } + if off < 1e-30 { + break; + } + for p in 0..n { + for q in p + 1..n { + let apq = a[p * n + q]; + if apq.abs() < 1e-300 { + continue; + } + let theta = (a[q * n + q] - a[p * n + p]) / (2.0 * apq); + let t = theta.signum() / (theta.abs() + (theta * theta + 1.0).sqrt()); + let t = if theta == 0.0 { 1.0 } else { t }; + let c = 1.0 / (t * t + 1.0).sqrt(); + let s = t * c; + for k in 0..n { + let (akp, akq) = (a[k * n + p], a[k * n + q]); + a[k * n + p] = c * akp - s * akq; + a[k * n + q] = s * akp + c * akq; + } + for k in 0..n { + let (apk, aqk) = (a[p * n + k], a[q * n + k]); + a[p * n + k] = c * apk - s * aqk; + a[q * n + k] = s * apk + c * aqk; + } + for k in 0..n { + let (vkp, vkq) = (v[k * n + p], v[k * n + q]); + v[k * n + p] = c * vkp - s * vkq; + v[k * n + q] = s * vkp + c * vkq; + } + } + } + } + let mut order: Vec = (0..n).collect(); + order.sort_by(|&x, &y| a[x * n + x].total_cmp(&a[y * n + y])); + let vals = order.iter().map(|&j| a[j * n + j]).collect(); + let mut vecs = vec![0.0; n * n]; + for (k, &j) in order.iter().enumerate() { + for r in 0..n { + vecs[r * n + k] = v[r * n + j]; + } + } + (vals, vecs) +} + +/// Solve `A x = b` for a symmetric positive definite `A` (`n × n`, row-major) in place by +/// Cholesky factorisation. Returns false when `A` is not positive definite. +pub fn cholesky_solve(a: &mut [f64], n: usize, b: &mut [f64]) -> bool { + for j in 0..n { + let mut d = a[j * n + j]; + for k in 0..j { + d -= a[j * n + k] * a[j * n + k]; + } + if d <= 0.0 || !d.is_finite() { + return false; + } + let d = d.sqrt(); + a[j * n + j] = d; + let (head, tail) = a.split_at_mut((j + 1) * n); + let rowj = &head[j * n..j * n + j]; + // Rows below j are independent. + use rayon::prelude::*; + let work = |row: &mut [f64]| { + let s: f64 = row[j] - row[..j].iter().zip(rowj).map(|(a, b)| a * b).sum::(); + row[j] = s / d; + }; + if n - j > 192 { + tail.par_chunks_mut(n).for_each(work); + } else { + tail.chunks_mut(n).for_each(work); + } + } + // Forward: L y = b. + for i in 0..n { + let mut s = b[i]; + for k in 0..i { + s -= a[i * n + k] * b[k]; + } + b[i] = s / a[i * n + i]; + } + // Back: Lᵀ x = y. + for i in (0..n).rev() { + let mut s = b[i]; + for k in i + 1..n { + s -= a[k * n + i] * b[k]; + } + b[i] = s / a[i * n + i]; + } + b.iter().all(|v| v.is_finite()) +} + +/// A similarity transform `x ↦ s R x + t`. +#[derive(Clone, Copy, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct Similarity { + pub s: f64, + pub r: M3, + pub t: V3, +} + +impl Default for Similarity { + fn default() -> Self { + Similarity { s: 1.0, r: I3, t: [0.0; 3] } + } +} + +impl Similarity { + pub fn apply(&self, x: V3) -> V3 { + add(scale(mv(&self.r, x), self.s), self.t) + } +} + +/// The similarity minimising `Σ |dst − (s R src + t)|²` (Umeyama 1991). +pub fn umeyama(src: &[V3], dst: &[V3]) -> Option { + let n = src.len().min(dst.len()); + if n < 3 { + return None; + } + let k = 1.0 / n as f64; + let ms = scale(src[..n].iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let md = scale(dst[..n].iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [[0.0; 3]; 3]; + let mut var = 0.0; + for i in 0..n { + let (a, b) = (sub(src[i], ms), sub(dst[i], md)); + cov = madd(&cov, &outer(b, a), k); + var += dot(a, a) * k; + } + if var < 1e-300 { + return None; + } + let (u, d, v) = svd3(&cov); + let mut sgn = [1.0, 1.0, 1.0]; + if det3(&u) * det3(&v) < 0.0 { + sgn[2] = -1.0; + } + let ud = from_cols(scale(col(&u, 0), sgn[0]), scale(col(&u, 1), sgn[1]), scale(col(&u, 2), sgn[2])); + let r = mm(&ud, &mt(&v)); + let s = (d[0] * sgn[0] + d[1] * sgn[1] + d[2] * sgn[2]) / var; + let t = sub(md, scale(mv(&r, ms), s)); + Some(Similarity { s, r, t }) +} + +/// Euler angles (degrees) `[x, y, z]` with `m = Rz(z) Ry(y) Rx(x)` (After Effects' Orientation +/// order: X, then Y, then Z applied to the vector). +pub fn euler_xyz(m: &M3) -> V3 { + let sy = (-m[2][0]).clamp(-1.0, 1.0); + let y = sy.asin(); + let (x, z) = if sy.abs() < 0.999_999 { (m[2][1].atan2(m[2][2]), m[1][0].atan2(m[0][0])) } else { ((-m[1][2]).atan2(m[1][1]), 0.0) }; + [x.to_degrees(), y.to_degrees(), z.to_degrees()] +} + +/// `Rz(z) Ry(y) Rx(x)` from degrees. +pub fn from_euler_xyz(e: V3) -> M3 { + let (sx, cx) = e[0].to_radians().sin_cos(); + let (sy, cy) = e[1].to_radians().sin_cos(); + let (sz, cz) = e[2].to_radians().sin_cos(); + let rx = [[1.0, 0.0, 0.0], [0.0, cx, -sx], [0.0, sx, cx]]; + let ry = [[cy, 0.0, sy], [0.0, 1.0, 0.0], [-sy, 0.0, cy]]; + let rz = [[cz, -sz, 0.0], [sz, cz, 0.0], [0.0, 0.0, 1.0]]; + mm(&rz, &mm(&ry, &rx)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn svd_reconstructs() { + let a = [3.0, 1.0, 2.0, -1.0, 4.0, 0.5, 2.0, 2.0, 7.0, 0.1, -3.0, 1.0]; + let (u, s, v) = svd(&a, 4, 3); + for r in 0..4 { + for c in 0..3 { + let x: f64 = (0..3).map(|k| u[r * 3 + k] * s[k] * v[c * 3 + k]).sum(); + assert!((x - a[r * 3 + c]).abs() < 1e-10); + } + } + assert!(s[0] >= s[1] && s[1] >= s[2]); + } + + #[test] + fn rotations_round_trip() { + let w = [0.3, -1.2, 0.7]; + let r = rodrigues(w); + let w2 = log_rot(&r); + assert!(norm(sub(w, w2)) < 1e-12); + let e = euler_xyz(&r); + let r2 = from_euler_xyz(e); + for i in 0..3 { + for j in 0..3 { + assert!((r[i][j] - r2[i][j]).abs() < 1e-12); + } + } + assert!((det3(&r) - 1.0).abs() < 1e-12); + } + + #[test] + fn eigen_and_cholesky() { + let a = [4.0, 1.0, 0.5, 1.0, 3.0, 0.2, 0.5, 0.2, 2.0]; + let (vals, vecs) = sym_eigen(&a, 3); + for k in 0..3 { + let v = [vecs[k], vecs[3 + k], vecs[6 + k]]; + let av = [dot([a[0], a[1], a[2]], v), dot([a[3], a[4], a[5]], v), dot([a[6], a[7], a[8]], v)]; + assert!(norm(sub(av, scale(v, vals[k]))) < 1e-10); + } + let mut m = a; + let mut b = [1.0, 2.0, 3.0]; + assert!(cholesky_solve(&mut m, 3, &mut b)); + let r = [dot([a[0], a[1], a[2]], b), dot([a[3], a[4], a[5]], b), dot([a[6], a[7], a[8]], b)]; + assert!(norm(sub(r, [1.0, 2.0, 3.0])) < 1e-12); + } + + #[test] + fn umeyama_recovers_a_similarity() { + let t = Similarity { s: 2.5, r: rodrigues([0.1, 0.4, -0.3]), t: [1.0, -2.0, 5.0] }; + let src = [[0.0, 0.0, 0.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [1.0, 2.0, 3.0]]; + let dst: Vec = src.iter().map(|p| t.apply(*p)).collect(); + let e = umeyama(&src, &dst).unwrap(); + assert!((e.s - 2.5).abs() < 1e-10); + assert!(norm(sub(e.t, t.t)) < 1e-9); + } +} diff --git a/crates/track/src/camtrack/mod.rs b/crates/track/src/camtrack/mod.rs new file mode 100644 index 00000000..056d7ad0 --- /dev/null +++ b/crates/track/src/camtrack/mod.rs @@ -0,0 +1,376 @@ +//! The 3D Camera Tracker's analysis: structure from motion on a single clip, clean-room from the +//! textbook pipeline (Hartley & Zisserman, *Multiple View Geometry in Computer Vision*, 2nd ed., +//! 2004; Triggs et al., "Bundle Adjustment — A Modern Synthesis", 2000): +//! +//! 1. **Feature tracks** ([`features::TrackAnalyzer`]): Shi–Tomasi corners followed through the +//! clip with pyramidal Lucas–Kanade and a forward–backward check ([`crate::klt`]); lost +//! features end their track and new ones are detected where the frame has none, so tracks +//! are as long as possible. +//! 2. **Solve** ([`solve`]): keyframes are selected by parallax; a two-view reconstruction +//! initialises the scene from either the **essential matrix** (normalised 8-point algorithm +//! inside RANSAC, H&Z §11.2 and §9.6) or, for mostly flat scenes, a **homography +//! decomposition** (Ma, Soatto, Košecká & Sastry, *An Invitation to 3-D Vision*, 2004, §5.3); +//! the other keyframes are added incrementally (**resection** by robust pose refinement, +//! **triangulation** of new tracks by the linear method of H&Z §12.2); a sparse +//! **Levenberg–Marquardt bundle adjustment** with the Schur complement over the points +//! ([`bundle`]) refines everything; the frames between keyframes are resected last. +//! 3. **Focal length**: *Specify Angle of View* fixes it; *Fixed Angle of View* searches it +//! (the reprojection error of the keyframe reconstruction over a log-spaced range of focal +//! lengths, refined by golden-section search) and then adjusts it in the bundle adjustment +//! as one shared parameter; *Variable Zoom* starts from the same estimate and adjusts a focal +//! length per frame, with a weak smoothness prior between neighbouring frames. +//! 4. **Tripod pan** shots (no camera translation) are solved as a pure rotation: frame-to-frame +//! rotations from the tracked rays (orthogonal Procrustes) refined by the same bundle +//! adjustment with the camera centres fixed. *Auto Detect* solves both ways and keeps the +//! tripod solution when it explains the tracks as well as the general one. +//! +//! The camera model has square pixels, the principal point at the layer centre and no lens +//! distortion. Camera space is x right, y down, z forward (After Effects' convention); a camera +//! is `(R, C, f)` with `X_cam = R (X − C)` and pixel `u = f x / z + w / 2`. +//! +//! The solve is stored in a canonical frame: the first solved frame's camera is at the origin +//! looking along +z, scaled so the median depth of the points it sees is 1. [`CameraSolve::world`] +//! maps it into composition space for a camera zoom, optionally re-based on a ground plane. + +pub mod bundle; +pub mod features; +pub mod geometry; +pub mod linalg; +mod solver; +#[cfg(test)] +mod tests; + +use serde::{Deserialize, Serialize}; + +pub use features::{AnalyzeOpts, TrackAnalyzer}; +use linalg::{M3, Similarity, V3}; +pub use solver::{SolveError, solve}; + +/// One feature track: positions (layer pixels) on consecutive frames from `start`. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Track2D { + pub id: u32, + pub start: u32, + pub pts: Vec<[f32; 2]>, +} + +impl Track2D { + pub fn end(&self) -> u32 { + self.start + self.pts.len() as u32 + } + /// Position on frame `k` when the track covers it. + pub fn at(&self, k: u32) -> Option<[f64; 2]> { + let i = k.checked_sub(self.start)? as usize; + self.pts.get(i).map(|p| [p[0] as f64, p[1] as f64]) + } +} + +/// The tracked features of a clip (step 1 of the analysis). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct CameraTracks { + pub version: u32, + /// Layer size (pixels). + pub size: [f64; 2], + /// Layer time (seconds) of frame 0, and the frame duration. + pub start: f64, + pub frame_duration: f64, + pub frames: u32, + /// Analysis downsampling factor (image pixels per analysis pixel): the tracking noise scale. + pub factor: f64, + pub detailed: bool, + pub tracks: Vec, +} + +impl CameraTracks { + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } + pub fn from_json(s: &str) -> Option { + if s.trim().is_empty() { + return None; + } + serde_json::from_str(s).ok() + } + pub fn is_empty(&self) -> bool { + self.frames == 0 + } + pub fn track(&self, id: u32) -> Option<&Track2D> { + self.tracks.binary_search_by_key(&id, |t| t.id).ok().map(|i| &self.tracks[i]) + } +} + +/// 3D Camera Tracker ▸ Shot Type. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ShotType { + #[default] + FixedAngle, + VariableZoom, + SpecifyAngle, +} + +impl ShotType { + pub const ALL: [ShotType; 3] = [ShotType::FixedAngle, ShotType::VariableZoom, ShotType::SpecifyAngle]; + pub fn label(self) -> &'static str { + match self { + ShotType::FixedAngle => "Fixed Angle of View", + ShotType::VariableZoom => "Variable Zoom", + ShotType::SpecifyAngle => "Specify Angle of View", + } + } + pub fn from_name(s: &str) -> Option { + let k = s.to_ascii_lowercase().replace([' ', '-', '_'], ""); + match k.as_str() { + "fixed" | "fixedangle" | "fixedangleofview" => Some(ShotType::FixedAngle), + "variable" | "variablezoom" | "zoom" => Some(ShotType::VariableZoom), + "specify" | "specifyangle" | "specifyangleofview" => Some(ShotType::SpecifyAngle), + _ => None, + } + } +} + +/// 3D Camera Tracker ▸ Advanced ▸ Solve Method (and Method Used). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum SolveMethod { + #[default] + Auto, + Typical, + MostlyFlat, + TripodPan, +} + +impl SolveMethod { + pub const ALL: [SolveMethod; 4] = [SolveMethod::Auto, SolveMethod::Typical, SolveMethod::MostlyFlat, SolveMethod::TripodPan]; + pub fn label(self) -> &'static str { + match self { + SolveMethod::Auto => "Auto Detect", + SolveMethod::Typical => "Typical", + SolveMethod::MostlyFlat => "Mostly Flat Scene", + SolveMethod::TripodPan => "Tripod Pan", + } + } + pub fn from_name(s: &str) -> Option { + let k = s.to_ascii_lowercase().replace([' ', '-', '_'], ""); + match k.as_str() { + "auto" | "autodetect" => Some(SolveMethod::Auto), + "typical" => Some(SolveMethod::Typical), + "flat" | "mostlyflat" | "mostlyflatscene" => Some(SolveMethod::MostlyFlat), + "tripod" | "tripodpan" => Some(SolveMethod::TripodPan), + _ => None, + } + } +} + +/// What a solve is made with. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolveSettings { + pub shot: ShotType, + /// Horizontal angle of view (degrees) for [`ShotType::SpecifyAngle`]. + pub hfov: f64, + pub method: SolveMethod, + /// Track ids the user deleted. + pub deleted: Vec, +} + +/// The solved camera of one frame. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolvedFrame { + /// World → camera rotation (axis-angle). + pub rot: V3, + /// Camera centre. + pub center: V3, + /// Focal length (layer pixels). + pub focal: f64, + pub solved: bool, +} + +impl SolvedFrame { + pub fn r(&self) -> M3 { + linalg::rodrigues(self.rot) + } + /// Camera-space position of a world point. + pub fn to_cam(&self, x: V3) -> V3 { + linalg::mv(&self.r(), linalg::sub(x, self.center)) + } + /// Project a world point to layer pixels (`None` behind the camera). + pub fn project(&self, size: [f64; 2], x: V3) -> Option<[f64; 2]> { + let c = self.to_cam(x); + (c[2] > 1e-9).then(|| [self.focal * c[0] / c[2] + size[0] * 0.5, self.focal * c[1] / c[2] + size[1] * 0.5]) + } +} + +/// A solved 3D track point. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolvedPoint { + /// The feature track it was made from. + pub id: u32, + pub pos: V3, + /// Average reprojection error (pixels). + pub error: f32, + /// Frames `[first, last]` the point was tracked on. + pub first: u32, + pub last: u32, +} + +/// A ground plane and origin (canonical solve coordinates). +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct Ground { + pub origin: V3, + /// Unit normal on the cameras' side ("up"). + pub normal: V3, + /// Unit in-plane direction that becomes world +x. + pub x_axis: V3, +} + +/// A plane through solved points: After Effects' target. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct Target { + pub center: V3, + /// Unit normal facing the camera. + pub normal: V3, + /// Diameter in canonical units. + pub size: f64, +} + +/// The camera solve (step 2 of the analysis). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct CameraSolve { + pub version: u32, + pub size: [f64; 2], + pub start: f64, + pub frame_duration: f64, + pub shot: ShotType, + /// The model that was used (Auto Detect resolves to one of the others). + pub method_used: SolveMethod, + /// Average reprojection error (pixels) over all solved observations. + pub average_error: f64, + pub frames: Vec, + pub points: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub ground: Option, +} + +impl CameraSolve { + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } + pub fn from_json(s: &str) -> Option { + if s.trim().is_empty() { + return None; + } + serde_json::from_str(s).ok() + } + pub fn is_empty(&self) -> bool { + self.frames.is_empty() + } + /// Frame index at a layer time (held at the ends). + pub fn frame_at(&self, layer_time: f64) -> Option { + if self.frames.is_empty() || self.frame_duration <= 0.0 { + return None; + } + let i = ((layer_time - self.start) / self.frame_duration).round(); + Some(i.clamp(0.0, (self.frames.len() - 1) as f64) as usize) + } + pub fn point(&self, id: u32) -> Option<&SolvedPoint> { + self.points.iter().find(|p| p.id == id) + } + /// Points tracked on frame `k`. + pub fn visible(&self, k: usize) -> impl Iterator { + let k = k as u32; + self.points.iter().filter(move |p| p.first <= k && k <= p.last) + } + /// Median focal length of the solved frames (layer pixels). + pub fn focal(&self) -> f64 { + let mut f: Vec = self.frames.iter().filter(|c| c.solved).map(|c| c.focal).collect(); + if f.is_empty() { + return 0.0; + } + f.sort_by(f64::total_cmp); + f[f.len() / 2] + } + /// Horizontal angle of view (degrees) of focal length `f`. + pub fn hfov(&self, f: f64) -> f64 { + 2.0 * (self.size[0] * 0.5 / f.max(1e-9)).atan().to_degrees() + } + + /// The plane through points `ids` seen from frame `k` (needs three points; with more the + /// plane is a least-squares fit). + pub fn target(&self, ids: &[u32], k: usize) -> Option { + let pts: Vec = ids.iter().filter_map(|i| self.point(*i)).map(|p| p.pos).collect(); + let cam = self.frames.get(k).filter(|c| c.solved).or_else(|| self.frames.iter().find(|c| c.solved))?; + geometry::fit_plane(&pts).map(|(c, mut n, spread)| { + if linalg::dot(n, linalg::sub(cam.center, c)) < 0.0 { + n = linalg::scale(n, -1.0); + } + // After Effects sizes the target from the camera distance when the points are close. + let depth = linalg::norm(linalg::sub(cam.center, c)); + Target { center: c, normal: n, size: (spread * 2.0).max(depth * 0.12) } + }) + } + + /// A ground plane through `ids` (origin at their centroid, normal towards frame `k`'s + /// camera, +x along that camera's right vector projected on the plane). + pub fn ground_from(&self, ids: &[u32], k: usize) -> Option { + let t = self.target(ids, k)?; + let cam = self.frames.get(k).filter(|c| c.solved).or_else(|| self.frames.iter().find(|c| c.solved))?; + let right = linalg::mtv(&cam.r(), [1.0, 0.0, 0.0]); + let mut x = linalg::sub(right, linalg::scale(t.normal, linalg::dot(right, t.normal))); + if linalg::norm(x) < 1e-6 { + let fwd = linalg::mtv(&cam.r(), [0.0, 0.0, 1.0]); + x = linalg::cross(fwd, t.normal); + } + Some(Ground { origin: t.center, normal: t.normal, x_axis: linalg::normalize(x) }) + } + + /// Canonical solve coordinates → composition world, for a camera whose zoom on the first + /// solved frame is `zoom0` (comp pixels) in a `comp` sized composition. Without a ground + /// plane, the first solved frame's camera becomes After Effects' default camera (at + /// `(w/2, h/2, −zoom)` looking down +z, so points at the median depth land near z = 0). With + /// one, the ground plane becomes the X-Z plane through the origin, normal along −y (up). + pub fn world(&self, comp: [f64; 2], zoom0: f64) -> Similarity { + match &self.ground { + Some(g) => { + // Rows: canonical directions that map to world x, y, z. + let y = linalg::scale(g.normal, -1.0); + let x = g.x_axis; + let z = linalg::cross(x, y); + let r = [x, y, z]; + let t = linalg::scale(linalg::mv(&r, g.origin), -zoom0); + Similarity { s: zoom0, r, t } + } + None => Similarity { s: zoom0, r: linalg::I3, t: [comp[0] * 0.5, comp[1] * 0.5, -zoom0] }, + } + } + + /// The first solved frame. + pub fn first_solved(&self) -> Option { + self.frames.iter().position(|c| c.solved) + } +} + +/// Deterministic colour of a track point (After Effects draws each target in its own colour). +pub fn point_color(id: u32) -> [f32; 3] { + let mut h = id.wrapping_mul(0x9E37_79B9) ^ 0x5bd1_e995; + h ^= h >> 15; + let hue = (h % 360) as f32 / 60.0; + let x = 1.0 - (hue % 2.0 - 1.0).abs(); + let (r, g, b) = match hue as u32 { + 0 => (1.0, x, 0.0), + 1 => (x, 1.0, 0.0), + 2 => (0.0, 1.0, x), + 3 => (0.0, x, 1.0), + 4 => (x, 0.0, 1.0), + _ => (1.0, 0.0, x), + }; + // Bright, slightly desaturated. + [0.25 + 0.75 * r, 0.25 + 0.75 * g, 0.25 + 0.75 * b] +} diff --git a/crates/track/src/camtrack/solver.rs b/crates/track/src/camtrack/solver.rs new file mode 100644 index 00000000..da3a9290 --- /dev/null +++ b/crates/track/src/camtrack/solver.rs @@ -0,0 +1,894 @@ +//! The camera solve pipeline (see the module docs of [`super`]). + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::atomic::{AtomicBool, Ordering}; + +use rayon::prelude::*; + +use super::bundle::{self, BaCam, BaObs, BaOpts, FocalMode}; +use super::geometry::{self, Pose}; +use super::linalg::*; +use super::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, SolvedFrame, SolvedPoint}; +use crate::fit::{Model, ransac}; + +fn debug() -> bool { + std::env::var_os("EC_CAMTRACK_DEBUG").is_some() +} + +/// Why a solve failed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SolveError(pub String); + +impl std::fmt::Display for SolveError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +fn err(m: &str) -> Result { + Err(SolveError(m.to_string())) +} + +/// Tracks in solver form: pixels relative to the principal point. +struct Data { + n: usize, + w: f64, + /// (track id, first frame, positions). + tracks: Vec<(u32, usize, Vec<[f64; 2]>)>, + /// Per frame: (track index, position). + by_frame: Vec>, + /// Tracking noise scale (pixels). + noise: f64, +} + +impl Data { + fn new(t: &CameraTracks, deleted: &[u32]) -> Data { + let n = t.frames as usize; + let c = [t.size[0] * 0.5, t.size[1] * 0.5]; + let del: HashSet = deleted.iter().copied().collect(); + let mut tracks = vec![]; + let mut by_frame = vec![vec![]; n]; + for tr in t.tracks.iter().filter(|tr| !del.contains(&tr.id) && tr.pts.len() >= 3) { + let ti = tracks.len() as u32; + let pts: Vec<[f64; 2]> = tr.pts.iter().map(|p| [p[0] as f64 - c[0], p[1] as f64 - c[1]]).collect(); + for (k, p) in pts.iter().enumerate() { + if let Some(v) = by_frame.get_mut(tr.start as usize + k) { + v.push((ti, *p)); + } + } + tracks.push((tr.id, tr.start as usize, pts)); + } + Data { n, w: t.size[0], tracks, by_frame, noise: t.factor.max(1.0) } + } + fn at(&self, ti: u32, k: usize) -> Option<[f64; 2]> { + let (_, s, p) = &self.tracks[ti as usize]; + k.checked_sub(*s).and_then(|i| p.get(i).copied()) + } + fn inlier_thr(&self) -> f64 { + 4.0 * self.noise + } + fn huber(&self) -> f64 { + 1.0 * self.noise + } +} + +/// Parallax-based keyframes: a new keyframe when the median feature motion since the last +/// exceeds 3 % of the width, when fewer than 60 % of its features survive, or before the shared +/// features drop below 24. The first and last frames with features are keyframes. +fn keyframes(d: &Data) -> Vec { + let Some(first) = (0..d.n).find(|k| d.by_frame[*k].len() >= 8) else { return vec![] }; + let mut kf = vec![first]; + let mut last = first; + let mut j = first + 1; + while j < d.n { + let base = &d.by_frame[last]; + let mut disp = vec![]; + for (ti, p) in base { + if let Some(q) = d.at(*ti, j) { + disp.push((q[0] - p[0]).hypot(q[1] - p[1])); + } + } + if disp.len() < 24 && j > last + 1 { + // Too few shared features: the previous frame becomes a keyframe. + last = j - 1; + kf.push(last); + continue; + } + disp.sort_by(f64::total_cmp); + let med = disp.get(disp.len() / 2).copied().unwrap_or(0.0); + let ratio = disp.len() as f64 / base.len().max(1) as f64; + if med > 0.03 * d.w || ratio < 0.6 || disp.len() < 24 { + last = j; + kf.push(j); + } + j += 1; + } + if let Some(lastf) = (0..d.n).rev().find(|k| d.by_frame[*k].len() >= 8) + && *kf.last().expect("one") < lastf + { + kf.push(lastf); + } + kf +} + +/// A (partial) reconstruction. +#[derive(Clone, Default)] +struct Recon { + cams: BTreeMap, + pts: HashMap, + /// Rejected observations (track index, frame). + outliers: HashSet<(u32, usize)>, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Kind { + General, + Tripod, +} + +#[derive(Clone, Copy, PartialEq, Debug)] +struct Cfg { + kind: Kind, + /// Focal mode of the global adjustments. + focal: FocalMode, + /// Registration estimates a focal length per frame. + per_frame_f: bool, +} + +const F_PRIOR: f64 = 30.0; + +impl Recon { + /// Observations among `frames` (inliers, with a 3D point). + fn observations(&self, d: &Data, frames: &[usize]) -> (Vec, Vec, Vec, Vec) { + let cams: Vec = frames.iter().map(|k| self.cams[k]).collect(); + let mut pidx: HashMap = HashMap::new(); + let mut pts = vec![]; + let mut ids = vec![]; + let mut obs = vec![]; + for (ci, k) in frames.iter().enumerate() { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + let pi = *pidx.entry(*ti).or_insert_with(|| { + pts.push(*x); + ids.push(*ti); + (pts.len() - 1) as u32 + }); + obs.push(BaObs { cam: ci as u32, pt: pi, uv: *uv }); + } + } + (cams, pts, obs, ids) + } + + /// Bundle-adjust the cameras of `frames` (the first stays fixed) and their points. + fn adjust(&mut self, d: &Data, frames: &[usize], cfg: &Cfg, focal: FocalMode, iters: usize) -> f64 { + let (mut cams, mut pts, obs, ids) = self.observations(d, frames); + if obs.is_empty() { + return f64::INFINITY; + } + let mut fixed = vec![false; cams.len()]; + fixed[0] = true; + let rep = bundle::adjust( + &mut cams, + &mut pts, + &obs, + &BaOpts { focal, fix_points: false, fix_centers: cfg.kind == Kind::Tripod, fixed, huber: d.huber(), max_iter: iters }, + ); + for (k, c) in frames.iter().zip(&cams) { + self.cams.insert(*k, *c); + } + for (ti, x) in ids.iter().zip(&pts) { + self.pts.insert(*ti, *x); + } + rep.cost + } + + /// Mark observations above the inlier threshold as outliers; drop points left with fewer + /// than two observations among the registered frames. + fn reject(&mut self, d: &Data) { + let thr2 = d.inlier_thr().powi(2); + let mut count: HashMap = HashMap::new(); + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + match bundle::residual(c, *x, *uv) { + Some(r) if r[0] * r[0] + r[1] * r[1] <= thr2 => *count.entry(*ti).or_default() += 1, + _ => { + self.outliers.insert((*ti, *k)); + } + } + } + } + self.pts.retain(|ti, _| count.get(ti).copied().unwrap_or(0) >= 2); + } + + /// Triangulate tracks seen on ≥ 2 registered frames that have no point yet. + fn triangulate(&mut self, d: &Data, kind: Kind, min_angle_deg: f64, check: f64) { + let mut seen: HashMap> = HashMap::new(); + for k in self.cams.keys() { + for (ti, _) in &d.by_frame[*k] { + if !self.pts.contains_key(ti) { + seen.entry(*ti).or_default().push(*k); + } + } + } + let thr2 = (d.inlier_thr() * check).powi(2); + let new: Vec<(u32, V3)> = seen + .par_iter() + .filter(|(_, ks)| ks.len() >= 2) + .filter_map(|(ti, ks)| { + let views: Vec<(Pose, [f64; 2], &BaCam)> = ks + .iter() + .map(|k| { + let c = &self.cams[k]; + let uv = d.at(*ti, *k).expect("seen"); + (c.pose, [uv[0] / c.f, uv[1] / c.f], c) + }) + .collect(); + let x = match kind { + Kind::Tripod => normalize(views.iter().fold([0.0; 3], |a, (p, o, _)| add(a, normalize(mtv(&p.r, [o[0], o[1], 1.0]))))), + Kind::General => { + let v: Vec<(Pose, [f64; 2])> = views.iter().map(|(p, o, _)| (*p, *o)).collect(); + let x = geometry::triangulate(&v)?; + // Enough parallax between the most separated views. + let (a, b) = (views[0].0.c, views[views.len() - 1].0.c); + if geometry::ray_angle(a, b, x).to_degrees() < min_angle_deg { + return None; + } + x + } + }; + for (k, (_, _, c)) in ks.iter().zip(&views) { + let uv = d.at(*ti, *k).expect("seen"); + let r = bundle::residual(c, x, uv)?; + if r[0] * r[0] + r[1] * r[1] > thr2 { + return None; + } + } + Some((*ti, x)) + }) + .collect(); + self.pts.extend(new); + } + + /// Resect frame `k` from the existing points starting at `init` (pose, and the focal + /// length when `per_frame_f`). Returns false when too few points agree. + fn register(&mut self, d: &Data, k: usize, init: BaCam, cfg: &Cfg) -> bool { + let mut pts = vec![]; + let mut obs = vec![]; + let mut ids = vec![]; + for (ti, uv) in &d.by_frame[k] { + if let Some(x) = self.pts.get(ti) { + obs.push(BaObs { cam: 0, pt: pts.len() as u32, uv: *uv }); + pts.push(*x); + ids.push(*ti); + } + } + if obs.len() < 6 { + return false; + } + let thr2 = d.inlier_thr().powi(2); + let opts = |huber: f64| BaOpts { + focal: if cfg.per_frame_f { FocalMode::PerCamera(0.0) } else { FocalMode::Fixed }, + fix_points: true, + fix_centers: cfg.kind == Kind::Tripod, + fixed: vec![false], + huber, + max_iter: 40, + }; + let mut best: Option<(usize, BaCam)> = None; + // Robust start (large Huber threshold), then tighten; from the prediction and, failing + // that, from the nearest registered camera. + let nearest = self.cams.iter().min_by_key(|(c, _)| c.abs_diff(k)).map(|(_, c)| *c); + for start in std::iter::once(init).chain(nearest) { + if best.as_ref().is_some_and(|b| b.0 as f64 >= 0.8 * obs.len() as f64) { + break; + } + let mut cams = vec![start]; + bundle::adjust(&mut cams, &mut pts.clone(), &obs, &opts(d.inlier_thr() * 4.0)); + bundle::adjust(&mut cams, &mut pts.clone(), &obs, &opts(d.huber())); + let inl = obs.iter().filter(|o| bundle::residual(&cams[0], pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2)).count(); + if best.as_ref().is_none_or(|b| inl > b.0) { + best = Some((inl, cams[0])); + } + } + let Some((inl, cam)) = best else { return false }; + if inl < 6 || (inl as f64) < 0.5 * obs.len() as f64 { + return false; + } + // Refine on the inliers only. + let keep: Vec = + obs.iter().filter(|o| bundle::residual(&cam, pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2)).copied().collect(); + let mut cams = vec![cam]; + bundle::adjust(&mut cams, &mut pts.clone(), &keep, &opts(d.huber())); + for o in &obs { + let ok = bundle::residual(&cams[0], pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2); + if !ok { + self.outliers.insert((ids[o.pt as usize], k)); + } + } + self.cams.insert(k, cams[0]); + true + } + + /// How well the reconstruction explains every observation on its frames: the mean of the + /// reprojection error truncated at the inlier threshold, an observation without a point + /// counting as the threshold. + fn score(&self, d: &Data) -> f64 { + let thr = d.inlier_thr(); + let mut s = 0.0; + let mut n = 0usize; + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + n += 1; + s += match self.pts.get(ti).and_then(|x| bundle::residual(c, *x, *uv)) { + Some(r) => r[0].hypot(r[1]).min(thr), + None => thr, + }; + } + } + if n > 0 { s / n as f64 } else { f64::INFINITY } + } + + /// Average reprojection error (pixels) and observation count over the inliers. + fn error(&self, d: &Data) -> (f64, usize) { + let mut s = 0.0; + let mut n = 0; + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + if let Some(r) = bundle::residual(c, *x, *uv) { + s += r[0].hypot(r[1]); + n += 1; + } + } + } + (if n > 0 { s / n as f64 } else { f64::INFINITY }, n) + } +} + +/// Interpolated / extrapolated starting camera for frame `k` from the registered ones. +fn predict(r: &Recon, k: usize) -> Option { + let before = r.cams.range(..k).next_back().map(|(a, b)| (*a, *b)); + let after = r.cams.range(k + 1..).next().map(|(a, b)| (*a, *b)); + match (before, after) { + (Some((ka, a)), Some((kb, b))) => { + let t = (k - ka) as f64 / (kb - ka) as f64; + Some(BaCam { pose: Pose { r: slerp(&a.pose.r, &b.pose.r, t), c: add(scale(a.pose.c, 1.0 - t), scale(b.pose.c, t)) }, f: a.f * (b.f / a.f).powf(t) }) + } + (Some((ka, a)), None) => { + // Constant velocity from the two previous cameras. + match r.cams.range(..ka).next_back() { + Some((kp, p)) => { + let t = (k - ka) as f64 / (ka - kp) as f64; + let dr = log_rot(&mm(&a.pose.r, &mt(&p.pose.r))); + let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &a.pose.r); + let c = add(a.pose.c, scale(sub(a.pose.c, p.pose.c), t.min(2.0))); + Some(BaCam { pose: Pose { r: rr, c }, f: a.f }) + } + None => Some(a), + } + } + (None, Some((kb, b))) => match r.cams.range(kb + 1..).next() { + Some((kn, nx)) => { + let t = (kb - k) as f64 / (kn - kb) as f64; + let dr = log_rot(&mm(&b.pose.r, &mt(&nx.pose.r))); + let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &b.pose.r); + let c = add(b.pose.c, scale(sub(b.pose.c, nx.pose.c), t.min(2.0))); + Some(BaCam { pose: Pose { r: rr, c }, f: b.f }) + } + None => Some(b), + }, + (None, None) => None, + } +} + +/// Shared tracks of frames `a` and `b`: (track index, pixels in a, pixels in b). +fn shared(d: &Data, a: usize, b: usize) -> Vec<(u32, [f64; 2], [f64; 2])> { + d.by_frame[a].iter().filter_map(|(ti, p)| d.at(*ti, b).map(|q| (*ti, *p, q))).collect() +} + +/// Two-view initialisations of frames `a`, `b` at focal length `f`: the second camera and the +/// triangulated points (the first camera is the identity). +fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> Vec<(Pose, HashMap)> { + let sh = shared(d, a, b); + if sh.len() < 12 { + return vec![]; + } + let x1: Vec<[f64; 2]> = sh.iter().map(|s| [s.1[0] / f, s.1[1] / f]).collect(); + let x2: Vec<[f64; 2]> = sh.iter().map(|s| [s.2[0] / f, s.2[1] / f]).collect(); + let thr = 2.0 * d.noise / f; + let mut cands: Vec = vec![]; + if use_e && let Some((e, inl)) = geometry::ransac_essential(&x1, &x2, thr, 400, (a * 7919 + b) as u64) { + let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip(); + if let Some((pose, _)) = geometry::select_pose(&geometry::decompose_essential(&e), &p1, &p2) { + cands.push(pose); + } + } + if use_h { + let (s1, s2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = sh.iter().map(|s| (s.1, s.2)).unzip(); + if let Some(fit) = ransac(Model::Homography, &s1, &s2, 2.0 * d.noise, 500, (a * 31 + b) as u64) { + let h = fit.h.0; + // Normalised: K⁻¹ H K with K = diag(f, f, 1). + let hn = [[h[0][0], h[0][1], h[0][2] / f], [h[1][0], h[1][1], h[1][2] / f], [h[2][0] * f, h[2][1] * f, h[2][2]]]; + let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| fit.inliers[*i]).map(|i| (x1[i], x2[i])).unzip(); + for (r, td, _) in geometry::decompose_homography(&hn, &p1, &p2) { + if norm(td) > 1e-6 { + cands.push(Pose::from_rt(r, normalize(td))); + } + } + } + } + let thr2 = (d.inlier_thr() / f).powi(2); + cands + .into_iter() + .filter_map(|pose| { + let mut pts = HashMap::new(); + for (i, s) in sh.iter().enumerate() { + let Some(x) = geometry::triangulate(&[(Pose::IDENTITY, x1[i]), (pose, x2[i])]) else { continue }; + let ok = geometry::reproj2(&Pose::IDENTITY, x, x1[i]).is_some_and(|e| e < thr2) + && geometry::reproj2(&pose, x, x2[i]).is_some_and(|e| e < thr2) + && geometry::ray_angle([0.0; 3], pose.c, x).to_degrees() > 0.3; + if ok { + pts.insert(s.0, x); + } + } + (pts.len() >= 10 && pts.len() * 3 >= sh.len()).then_some((pose, pts)) + }) + .collect() +} + +/// Grow a reconstruction over keyframes `kfs` from an initial pair (`kfs[0]`, `kfs[j]`). +fn grow(d: &Data, kfs: &[usize], j: usize, init: &(Pose, HashMap), f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let (a, b) = (kfs[0], kfs[j]); + let mut r = Recon::default(); + r.cams.insert(a, BaCam { pose: Pose::IDENTITY, f }); + r.cams.insert(b, BaCam { pose: init.0, f }); + r.pts = init.1.clone(); + let inner = |r: &mut Recon, frames: &[usize]| { + let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed }; + r.adjust(d, frames, cfg, focal, 10); + r.reject(d); + }; + inner(&mut r, &[a, b]); + let order: Vec = kfs[1..j].iter().chain(&kfs[j + 1..]).copied().collect(); + let mut done = 0; + for k in order { + if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return None; + } + let start = predict(&r, k)?; + if !r.register(d, k, start, cfg) { + // A failed keyframe ends the reconstruction (a later frame is too far from the known + // points); the caller resects the rest from what is known. + break; + } + r.triangulate(d, cfg.kind, 1.0, 1.0); + done += 1; + let frames: Vec = r.cams.keys().copied().collect(); + if done % 3 == 0 || frames.len() <= 6 { + inner(&mut r, &frames); + } + } + let frames: Vec = r.cams.keys().copied().collect(); + inner(&mut r, &frames); + (r.cams.len() >= 2 && r.pts.len() >= 8).then_some(r) +} + +/// Tripod reconstruction over keyframes `kfs` at focal length `f`. +fn grow_tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let mut r = Recon::default(); + r.cams.insert(kfs[0], BaCam { pose: Pose::IDENTITY, f }); + let ray = |p: [f64; 2], f: f64| normalize([p[0] / f, p[1] / f, 1.0]); + for w in kfs.windows(2) { + if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return None; + } + let (a, b) = (w[0], w[1]); + let sh = shared(d, a, b); + if sh.len() < 6 { + break; + } + let ra: Vec = sh.iter().map(|s| ray(s.1, f)).collect(); + let rb: Vec = sh.iter().map(|s| ray(s.2, f)).collect(); + let mut rel = geometry::procrustes(&ra, &rb)?; + // One round of outlier trimming. + let errs: Vec = ra.iter().zip(&rb).map(|(x, y)| norm(sub(*y, mv(&rel, *x)))).collect(); + let mut sorted = errs.clone(); + sorted.sort_by(f64::total_cmp); + let lim = (sorted[sorted.len() / 2] * 3.0).max(d.inlier_thr() / f); + let (ia, ib): (Vec, Vec) = ra.iter().zip(&rb).zip(&errs).filter(|(_, e)| **e <= lim).map(|((x, y), _)| (*x, *y)).unzip(); + if ia.len() >= 4 { + rel = geometry::procrustes(&ia, &ib)?; + } + let pa = r.cams[&a].pose.r; + r.cams.insert(b, BaCam { pose: Pose { r: mm(&rel, &pa), c: [0.0; 3] }, f }); + } + r.triangulate(d, Kind::Tripod, 0.0, 20.0); + let frames: Vec = r.cams.keys().copied().collect(); + let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed }; + r.adjust(d, &frames, cfg, focal, 15); + r.reject(d); + r.triangulate(d, Kind::Tripod, 0.0, 1.0); + r.adjust(d, &frames, cfg, focal, 15); + (r.cams.len() >= 2 && r.pts.len() >= 8).then_some(r) +} + +/// Focal length (pixels) of a horizontal angle of view. +fn focal_of(w: f64, hfov: f64) -> f64 { + w * 0.5 / (hfov.clamp(1.0, 170.0).to_radians() * 0.5).tan() +} + +struct Attempt { + recon: Recon, + err: f64, + method: SolveMethod, +} + +/// Choose the initial pair: the first keyframe and the later keyframe (among the next few) +/// that keeps the most points while having enough parallax. +fn best_init(d: &Data, kfs: &[usize], f: f64, method: SolveMethod) -> Option<(usize, (Pose, HashMap), bool)> { + let (use_e, use_h) = match method { + SolveMethod::Typical => (true, false), + SolveMethod::MostlyFlat => (false, true), + _ => (true, true), + }; + let mut best: Option<(f64, usize, (Pose, HashMap), bool)> = None; + for j in 1..kfs.len().min(6) { + let cands = init_pair(d, kfs[0], kfs[j], f, use_e, false); + let hc = if use_h { init_pair(d, kfs[0], kfs[j], f, false, true) } else { vec![] }; + for (c, is_h) in cands.into_iter().map(|c| (c, false)).chain(hc.into_iter().map(|c| (c, true))) { + // Score: points × median triangulation angle (capped). + let mut ang: Vec = c.1.values().map(|x| geometry::ray_angle([0.0; 3], c.0.c, *x).to_degrees()).collect(); + ang.sort_by(f64::total_cmp); + let med = ang.get(ang.len() / 2).copied().unwrap_or(0.0); + let score = c.1.len() as f64 * med.min(4.0); + if best.as_ref().is_none_or(|b| score > b.0) { + best = Some((score, j, c, is_h)); + } + } + } + best.map(|b| (b.1, b.2, b.3)) +} + +/// All initial candidates for the pair chosen by [`best_init`] (E and H decompositions). +fn init_candidates(d: &Data, kfs: &[usize], f: f64, method: SolveMethod) -> Vec<(usize, (Pose, HashMap), bool)> { + let Some((j, first, is_h)) = best_init(d, kfs, f, method) else { return vec![] }; + let mut out = vec![(j, first, is_h)]; + if matches!(method, SolveMethod::Auto | SolveMethod::MostlyFlat) { + for c in init_pair(d, kfs[0], kfs[j], f, false, true) { + if out.iter().all(|o| norm(log_rot(&mm(&o.1.0.r, &mt(&c.0.r)))) > 1e-3 || norm(sub(o.1.0.c, c.0.c)) > 1e-3) { + out.push((j, c, true)); + } + } + } + out +} + +/// A general (translating camera) reconstruction of keyframes `kfs` at focal length `f`. +fn general(d: &Data, kfs: &[usize], f: f64, method: SolveMethod, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let cands = init_candidates(d, kfs, f, method); + let res: Vec = cands + .par_iter() + .filter_map(|(j, init, is_h)| { + let r = grow(d, kfs, *j, init, f, cfg, cancel)?; + let (_, n) = r.error(d); + // Prefer reconstructions covering more keyframes. + let cover = r.cams.len() as f64 / kfs.len() as f64; + (n > 0).then(|| Attempt { err: r.score(d) / cover.max(0.05), recon: r, method: if *is_h { SolveMethod::MostlyFlat } else { SolveMethod::Typical } }) + }) + .collect(); + res.into_iter().min_by(|a, b| a.err.total_cmp(&b.err)) +} + +fn tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let r = grow_tripod(d, kfs, f, cfg, cancel)?; + let cover = r.cams.len() as f64 / kfs.len() as f64; + Some(Attempt { err: r.score(d) / cover.max(0.05), recon: r, method: SolveMethod::TripodPan }) +} + +/// Search the focal length minimising `eval(f)` (a reprojection error): a log-spaced scan of +/// horizontal angles of view, then golden-section refinement around the best. +fn search_focal(w: f64, eval: &(dyn Fn(f64) -> f64 + Sync)) -> f64 { + let fovs = [8.0, 12.0, 16.0, 20.0, 25.0, 30.0, 36.0, 43.0, 51.0, 60.0, 70.0, 82.0, 96.0, 112.0]; + let fs: Vec = fovs.iter().map(|a| focal_of(w, *a)).collect(); + let errs: Vec = fs.par_iter().map(|f| eval(*f)).collect(); + if debug() { + eprintln!("focal scan: {:?}", fs.iter().zip(&errs).map(|(f, e)| format!("{f:.0}:{e:.3}")).collect::>()); + } + let i = (0..fs.len()).min_by(|a, b| errs[*a].total_cmp(&errs[*b])).unwrap_or(5); + let (mut lo, mut hi) = (fs[(i + 1).min(fs.len() - 1)].ln(), fs[i.saturating_sub(1)].ln()); + if lo > hi { + std::mem::swap(&mut lo, &mut hi); + } + let g = 0.618_033_988_75; + let (mut x1, mut x2) = (hi - g * (hi - lo), lo + g * (hi - lo)); + let (mut e1, mut e2) = rayon::join(|| eval(x1.exp()), || eval(x2.exp())); + for _ in 0..7 { + if e1 < e2 { + hi = x2; + x2 = x1; + e2 = e1; + x1 = hi - g * (hi - lo); + e1 = eval(x1.exp()); + } else { + lo = x1; + x1 = x2; + e1 = e2; + x2 = lo + g * (hi - lo); + e2 = eval(x2.exp()); + } + } + let best = if e1 < e2 { (x1, e1) } else { (x2, e2) }; + if best.1 <= errs[i] { best.0.exp() } else { fs[i] } +} + +/// Solve the camera of a tracked clip. +pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBool>) -> Result { + let d = Data::new(tracks, &s.deleted); + if d.n < 2 { + return err("the clip must be at least two frames long"); + } + let kfs = keyframes(&d); + if kfs.len() < 2 { + return err("not enough features were tracked to solve the camera"); + } + let cancelled = || cancel.is_some_and(|c| c.load(Ordering::Relaxed)); + let fixed_f = (s.shot == ShotType::SpecifyAngle).then(|| focal_of(d.w, if s.hfov > 0.0 { s.hfov } else { 40.0 })); + let variable = s.shot == ShotType::VariableZoom; + let gcfg = Cfg { kind: Kind::General, focal: FocalMode::Fixed, per_frame_f: false }; + let tcfg = Cfg { kind: Kind::Tripod, ..gcfg }; + // The focal-length search runs on the first keyframes only. + let mini: Vec = kfs.iter().copied().take(10).collect(); + let try_general = matches!(s.method, SolveMethod::Auto | SolveMethod::Typical | SolveMethod::MostlyFlat); + let try_tripod = matches!(s.method, SolveMethod::Auto | SolveMethod::TripodPan); + + let mut results: Vec<(Attempt, f64)> = vec![]; + if try_general { + let f = match fixed_f { + Some(f) => f, + None => { + search_focal(d.w, &|f| general(&d, &mini, f, s.method, &Cfg { per_frame_f: variable, ..gcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)) + } + }; + if cancelled() { + return err("cancelled"); + } + if let Some(a) = general(&d, &kfs, f, s.method, &Cfg { per_frame_f: variable, ..gcfg }, cancel) { + results.push((a, f)); + } + } + if try_tripod && !cancelled() { + let f = match fixed_f { + Some(f) => f, + None => search_focal(d.w, &|f| tripod(&d, &mini, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)), + }; + if let Some(a) = tripod(&d, &kfs, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel) { + results.push((a, f)); + } + } + if cancelled() { + return err("cancelled"); + } + // Auto Detect keeps the tripod model when it explains the tracks about as well. + let pick = { + let g = results.iter().position(|r| r.0.method != SolveMethod::TripodPan); + let t = results.iter().position(|r| r.0.method == SolveMethod::TripodPan); + match (g, t) { + (Some(g), Some(t)) => { + let (eg, et) = (results[g].0.err, results[t].0.err); + // A general solve whose camera hardly moves relative to the scene depth is a pan. + let rg = &results[g].0.recon; + let cs: Vec = rg.cams.values().map(|c| c.pose.c).collect(); + let base = cs.iter().flat_map(|a| cs.iter().map(move |b| norm(sub(*a, *b)))).fold(0.0, f64::max); + let c0 = rg.cams.values().next().expect("camera"); + let mut depth: Vec = rg.pts.values().map(|x| norm(sub(*x, c0.pose.c))).collect(); + depth.sort_by(f64::total_cmp); + let med = depth.get(depth.len() / 2).copied().unwrap_or(1.0); + if debug() { + eprintln!("auto: general {eg:.4} tripod {et:.4} baseline/depth {:.4}", base / med); + } + if et <= (eg * 1.25).max(eg + 0.05 * d.noise) || base / med < 0.01 { t } else { g } + } + (Some(g), None) => g, + (None, Some(t)) => t, + (None, None) => return err("the camera could not be solved (not enough parallax or features)"), + } + }; + if debug() { + for (a, f) in &results { + eprintln!("candidate {:?}: err {:.4} f {f:.1} cams {} pts {}", a.method, a.err, a.recon.cams.len(), a.recon.pts.len()); + } + } + let (att, _) = results.swap_remove(pick); + let kind = if att.method == SolveMethod::TripodPan { Kind::Tripod } else { Kind::General }; + let cfg = Cfg { kind, focal: FocalMode::Fixed, per_frame_f: variable }; + let mut r = att.recon; + let focal = match (fixed_f, variable) { + (Some(_), _) => FocalMode::Fixed, + (None, true) => FocalMode::PerCamera(F_PRIOR), + (None, false) => FocalMode::Shared, + }; + // Keyframes: adjust with the focal length free, reject, re-triangulate, adjust again. + let kf_frames: Vec = r.cams.keys().copied().collect(); + r.adjust(&d, &kf_frames, &cfg, focal, 40); + r.reject(&d); + r.triangulate(&d, kind, 1.0, 1.0); + r.adjust(&d, &kf_frames, &cfg, focal, 40); + if cancelled() { + return err("cancelled"); + } + // Every other frame: resect from the points (nearest registered camera first). + let mut todo: Vec = (0..d.n).filter(|k| !r.cams.contains_key(k)).collect(); + todo.sort_by_key(|k| r.cams.keys().map(|c| c.abs_diff(*k)).min().unwrap_or(usize::MAX)); + let base = r.clone(); + let resected: Vec<(usize, Option)> = todo + .par_iter() + .map(|k| { + let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() }; + let ok = predict(&local, *k).is_some_and(|start| local.register(&d, *k, start, &cfg)); + (*k, if ok { local.cams.get(k).copied() } else { None }) + }) + .collect(); + for (k, c) in resected { + if let Some(c) = c { + r.cams.insert(k, c); + } + } + if cancelled() { + return err("cancelled"); + } + // Points for the tracks that only lived between keyframes, then the final adjustment over + // every frame (when that stays affordable). + r.triangulate(&d, kind, 1.0, 1.0); + let all: Vec = r.cams.keys().copied().collect(); + r.reject(&d); + if all.len() <= 160 { + r.adjust(&d, &all, &cfg, focal, 25); + r.reject(&d); + } else { + // Points only (cameras fixed) would be cheap but the keyframe solution is already + // adjusted; refine each non-keyframe once more against the final points. + let base = r.clone(); + let refined: Vec<(usize, Option)> = all + .par_iter() + .filter(|k| !kf_frames.contains(k)) + .map(|k| { + let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() }; + let start = local.cams[k]; + let ok = local.register(&d, *k, start, &cfg); + (*k, if ok { local.cams.get(k).copied() } else { None }) + }) + .collect(); + for (k, c) in refined.into_iter() { + if let Some(c) = c { + r.cams.insert(k, c); + } + } + r.reject(&d); + } + // Method Used: the chosen model; Auto Detect calls a general solve "Mostly Flat Scene" when + // its points are close to one plane. + let method = match (s.method, att.method) { + (_, SolveMethod::TripodPan) => SolveMethod::TripodPan, + (SolveMethod::Auto, _) => { + let p: Vec = r.pts.values().copied().collect(); + let flat = cov_ratio(&p) < 0.08; + if flat { SolveMethod::MostlyFlat } else { SolveMethod::Typical } + } + (m, _) => m, + }; + Ok(finish(&d, tracks, s, r, method, kind)) +} + +/// Thickness of a point cloud: √(smallest / middle eigenvalue of its covariance). +fn cov_ratio(p: &[V3]) -> f64 { + if p.len() < 3 { + return 1.0; + } + let k = 1.0 / p.len() as f64; + let c = scale(p.iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [0.0; 9]; + for q in p { + let d = sub(*q, c); + for i in 0..3 { + for j in 0..3 { + cov[i * 3 + j] += d[i] * d[j] * k; + } + } + } + let (vals, _) = sym_eigen(&cov, 3); + (vals[0].max(0.0) / vals[1].max(1e-300)).sqrt() +} + +/// Canonical frame, per-frame output (unsolved frames interpolated), points and errors. +fn finish(d: &Data, tracks: &CameraTracks, s: &SolveSettings, r: Recon, method: SolveMethod, kind: Kind) -> CameraSolve { + let (k0, c0) = r.cams.iter().next().map(|(k, c)| (*k, *c)).expect("solved"); + // Scale: median depth of the points seen by the first solved camera is 1. + let mut depths: Vec = d.by_frame[k0].iter().filter_map(|(ti, _)| r.pts.get(ti)).map(|x| c0.pose.to_cam(*x)[2]).filter(|z| *z > 0.0).collect(); + depths.sort_by(f64::total_cmp); + let sc = match kind { + Kind::Tripod => 1.0, + Kind::General => depths.get(depths.len() / 2).copied().filter(|v| *v > 1e-12).unwrap_or(1.0), + }; + let r0 = c0.pose.r; + let cc = c0.pose.c; + let to_world = |x: V3| scale(mv(&r0, sub(x, cc)), 1.0 / sc); + let mut frames = vec![SolvedFrame::default(); d.n]; + for (k, c) in &r.cams { + let rr = mm(&c.pose.r, &mt(&r0)); + frames[*k] = SolvedFrame { rot: log_rot(&rr), center: to_world(c.pose.c), focal: c.f, solved: true }; + } + // Hold / interpolate unsolved frames so the camera path stays continuous. + let solved: Vec = (0..d.n).filter(|k| frames[*k].solved).collect(); + for k in 0..d.n { + if frames[k].solved { + continue; + } + let a = solved.iter().rev().find(|x| **x < k).copied(); + let b = solved.iter().find(|x| **x > k).copied(); + frames[k] = match (a, b) { + (Some(a), Some(b)) => { + let t = (k - a) as f64 / (b - a) as f64; + let (fa, fb) = (frames[a], frames[b]); + let rr = slerp(&fa.r(), &fb.r(), t); + SolvedFrame { + rot: log_rot(&rr), + center: add(scale(fa.center, 1.0 - t), scale(fb.center, t)), + focal: fa.focal + (fb.focal - fa.focal) * t, + solved: false, + } + } + (Some(a), None) => SolvedFrame { solved: false, ..frames[a] }, + (None, Some(b)) => SolvedFrame { solved: false, ..frames[b] }, + (None, None) => frames[k], + }; + } + let mut points = vec![]; + let (mut esum, mut en) = (0.0, 0usize); + let mut ids: Vec<&u32> = r.pts.keys().collect(); + ids.sort(); + for ti in ids { + let x = r.pts[ti]; + let (id, start, pts) = &d.tracks[*ti as usize]; + let (mut s1, mut n1) = (0.0, 0); + for (i, uv) in pts.iter().enumerate() { + let k = start + i; + if r.outliers.contains(&(*ti, k)) { + continue; + } + if let Some(c) = r.cams.get(&k) + && let Some(res) = bundle::residual(c, x, *uv) + { + s1 += res[0].hypot(res[1]); + n1 += 1; + } + } + if n1 < 2 { + continue; + } + esum += s1; + en += n1; + points.push(SolvedPoint { id: *id, pos: to_world(x), error: (s1 / n1 as f64) as f32, first: *start as u32, last: (start + pts.len() - 1) as u32 }); + } + CameraSolve { + version: 1, + size: tracks.size, + start: tracks.start, + frame_duration: tracks.frame_duration, + shot: s.shot, + method_used: method, + average_error: if en > 0 { esum / en as f64 } else { 0.0 }, + frames, + points, + ground: None, + } +} diff --git a/crates/track/src/camtrack/tests.rs b/crates/track/src/camtrack/tests.rs new file mode 100644 index 00000000..258161f9 --- /dev/null +++ b/crates/track/src/camtrack/tests.rs @@ -0,0 +1,360 @@ +//! Synthetic scenes: random 3D points (or a plane) seen by a known moving camera with a known +//! focal length, projected directly (with sub-pixel noise) into feature tracks. + +use super::geometry::Rng; +use super::linalg::*; +use super::*; + +const W: f64 = 1280.0; +const H: f64 = 720.0; + +struct Truth { + /// Per frame: (R world→cam, centre, focal). + cams: Vec<(M3, V3, f64)>, +} + +fn noise(rng: &mut Rng) -> f64 { + // Sum of uniforms ≈ Gaussian, σ ≈ 0.1 px. + let u: f64 = (0..4).map(|_| (rng.next_u64() % 10_000) as f64 / 10_000.0 - 0.5).sum(); + u * 0.17 +} + +/// Project `pts` through the cameras into tracks (a track ends when its point leaves the frame; +/// re-entering starts a new track). +fn tracks_of(pts: &[V3], t: &Truth, seed: u64, noise_on: bool) -> CameraTracks { + let mut rng = Rng(seed); + let mut tracks: Vec = vec![]; + for x in pts { + let mut cur: Option = None; + for (k, (r, c, f)) in t.cams.iter().enumerate() { + let p = mv(r, sub(*x, *c)); + let uv = (p[2] > 0.1).then(|| [f * p[0] / p[2] + W / 2.0, f * p[1] / p[2] + H / 2.0]); + match uv.filter(|u| u[0] >= 2.0 && u[1] >= 2.0 && u[0] < W - 2.0 && u[1] < H - 2.0) { + Some(u) => { + let (nx, ny) = if noise_on { (noise(&mut rng), noise(&mut rng)) } else { (0.0, 0.0) }; + let q = [(u[0] + nx) as f32, (u[1] + ny) as f32]; + match &mut cur { + Some(tr) => tr.pts.push(q), + None => cur = Some(Track2D { id: 0, start: k as u32, pts: vec![q] }), + } + } + None => { + if let Some(tr) = cur.take() { + tracks.push(tr); + } + } + } + } + if let Some(tr) = cur.take() { + tracks.push(tr); + } + } + tracks.retain(|t| t.pts.len() >= 3); + for (i, tr) in tracks.iter_mut().enumerate() { + tr.id = i as u32; + } + CameraTracks { version: 1, size: [W, H], start: 0.0, frame_duration: 1.0 / 30.0, frames: t.cams.len() as u32, factor: 1.0, detailed: false, tracks } +} + +fn random_points(n: usize, seed: u64, center: V3, ext: V3) -> Vec { + let mut rng = Rng(seed); + (0..n) + .map(|_| { + let mut u = || (rng.next_u64() % 100_000) as f64 / 100_000.0 - 0.5; + [center[0] + u() * ext[0], center[1] + u() * ext[1], center[2] + u() * ext[2]] + }) + .collect() +} + +/// A camera dollying sideways and forward while panning to keep the scene in view. +fn moving_camera(frames: usize, f: impl Fn(usize) -> f64) -> Truth { + let cams = (0..frames) + .map(|k| { + let t = k as f64 / (frames - 1) as f64; + let c = [-3.0 + 6.0 * t, -0.6 + 0.8 * t + 0.5 * (std::f64::consts::PI * t).sin(), -1.0 + 2.0 * t - 1.5 * (t - 0.5) * (t - 0.5)]; + // Look at the scene centre (0, 0, 10). + let fwd = normalize(sub([0.3 * t, 0.2, 10.0], c)); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + let down = cross(fwd, right); + let r = [right, down, fwd]; + // A little roll. + let roll = rodrigues([0.0, 0.0, 0.05 * (t * 6.0).sin()]); + (mm(&roll, &r), c, f(k)) + }) + .collect(); + Truth { cams } +} + +/// Align the solved centres to the true ones (similarity) and return the RMS path error +/// relative to the path length, plus the worst rotation error (degrees). +fn path_error(s: &CameraSolve, t: &Truth) -> (f64, f64) { + let solved: Vec = (0..s.frames.len()).filter(|k| s.frames[*k].solved).collect(); + let src: Vec = solved.iter().map(|k| s.frames[*k].center).collect(); + let dst: Vec = solved.iter().map(|k| t.cams[*k].1).collect(); + let sim = umeyama(&src, &dst).expect("alignment"); + let len = norm(sub(dst[0], dst[dst.len() - 1])).max(1e-9); + let rms = (src.iter().zip(&dst).map(|(a, b)| dot(sub(sim.apply(*a), *b), sub(sim.apply(*a), *b))).sum::() / src.len() as f64).sqrt(); + // Rotation: R_true ≈ R_solved · simᵀ. + let mut worst: f64 = 0.0; + for k in &solved { + let rs = mm(&s.frames[*k].r(), &mt(&sim.r)); + let e = norm(log_rot(&mm(&rs, &mt(&t.cams[*k].0)))).to_degrees(); + worst = worst.max(e); + } + (rms / len, worst) +} + +fn settings(shot: ShotType, method: SolveMethod) -> SolveSettings { + SolveSettings { shot, hfov: 0.0, method, deleted: vec![] } +} + +#[test] +fn fixed_angle_solve_recovers_path_and_focal() { + let f = 1100.0; + let t = moving_camera(40, |_| f); + let pts = random_points(400, 1, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 2, true); + let t0 = std::time::Instant::now(); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), None).expect("solve"); + eprintln!("fixed: {:?} err {:.3} px focal {:.1} in {:?}", s.method_used, s.average_error, s.focal(), t0.elapsed()); + assert!(s.frames.iter().all(|c| c.solved)); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.02, "focal {}", s.focal()); + let (pe, re) = path_error(&s, &t); + assert!(pe < 0.01, "path {pe}"); + assert!(re < 0.5, "rotation {re}"); + // Canonical frame: first camera at the origin. + assert!(norm(s.frames[0].center) < 1e-9 && norm(s.frames[0].rot) < 1e-9); + // Serde round trip. + let j = s.to_json(); + assert_eq!(CameraSolve::from_json(&j).unwrap().points.len(), s.points.len()); +} + +#[test] +fn auto_detect_picks_a_general_model_for_a_translating_camera() { + let f = 900.0; + let t = moving_camera(30, |_| f); + let pts = random_points(300, 3, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 4, true); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + assert_eq!(s.method_used, SolveMethod::Typical); + assert!(s.average_error < 0.5); + assert!((s.focal() - f).abs() / f < 0.02, "focal {}", s.focal()); +} + +#[test] +fn specified_angle_of_view_is_kept() { + let f = 1000.0; + let t = moving_camera(25, |_| f); + let pts = random_points(300, 5, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 6, true); + let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); + let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: vec![] }, None).unwrap(); + assert!((s.focal() - f).abs() < 1e-6); + assert!(s.average_error < 0.5); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.01, "{pe}"); +} + +#[test] +fn variable_zoom_recovers_per_frame_focal() { + let t = moving_camera(40, |k| 950.0 + 6.0 * k as f64); + let pts = random_points(450, 7, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]); + let tr = tracks_of(&pts, &t, 8, true); + let s = solve(&tr, &settings(ShotType::VariableZoom, SolveMethod::Typical), None).expect("solve"); + eprintln!("variable: err {:.3}", s.average_error); + assert!(s.average_error < 0.5); + for (k, c) in s.frames.iter().enumerate() { + let f = t.cams[k].2; + assert!((c.focal - f).abs() / f < 0.02, "frame {k}: {} vs {f}", c.focal); + } +} + +#[test] +fn tripod_pan_solve() { + let f = 1200.0; + let cams = (0..40) + .map(|k| { + let a = -0.4 + 0.8 * k as f64 / 39.0; + (mm(&rodrigues([0.0, 0.0, 0.02]), &mm(&rodrigues([0.05 * a, 0.0, 0.0]), &rodrigues([0.0, a, 0.0]))), [0.0; 3], f) + }) + .collect(); + let t = Truth { cams }; + // Points all around at varying distances (rotation only: depth is unobservable). + let pts = random_points(600, 9, [0.0, 0.0, 0.0], [60.0, 20.0, 60.0]).into_iter().filter(|p| p[2] > 3.0).collect::>(); + let tr = tracks_of(&pts, &t, 10, true); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("tripod: {:?} err {:.3} focal {:.1}", s.method_used, s.average_error, s.focal()); + assert_eq!(s.method_used, SolveMethod::TripodPan); + assert!(s.average_error < 0.5); + assert!((s.focal() - f).abs() / f < 0.02, "{}", s.focal()); + for (k, c) in s.frames.iter().enumerate() { + assert!(norm(c.center) < 1e-9); + // Relative rotation to frame 0 matches. + let rel_s = mm(&c.r(), &mt(&s.frames[0].r())); + let rel_t = mm(&t.cams[k].0, &mt(&t.cams[0].0)); + assert!(norm(log_rot(&mm(&rel_s, &mt(&rel_t)))).to_degrees() < 0.2); + } +} + +#[test] +fn mostly_flat_scene_and_ground_plane() { + let f = 1000.0; + // A textured floor y = 2 (below the camera), seen by a camera moving forward and sideways. + let cams = (0..40) + .map(|k| { + let t = k as f64 / 39.0; + let c = [-2.0 + 4.0 * t, -0.3 * (std::f64::consts::PI * t).sin(), 2.0 * t]; + let fwd = normalize([0.1 - 0.2 * t, 0.45, 1.0]); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + let down = cross(fwd, right); + ([right, down, fwd], c, f) + }) + .collect(); + let t = Truth { cams }; + let mut rng = Rng(11); + let pts: Vec = (0..500).map(|_| [((rng.next_u64() % 2000) as f64 / 100.0) - 10.0, 2.0, 2.0 + (rng.next_u64() % 2000) as f64 / 100.0]).collect(); + let tr = tracks_of(&pts, &t, 12, true); + for method in [SolveMethod::MostlyFlat, SolveMethod::Auto] { + let s = solve(&tr, &settings(ShotType::FixedAngle, method), None).expect("solve"); + eprintln!("flat {method:?}: {:?} err {:.3} focal {:.1}", s.method_used, s.average_error, s.focal()); + assert_eq!(s.method_used, SolveMethod::MostlyFlat); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.02, "{}", s.focal()); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.01, "{pe}"); + } + // Ground plane from the solved (coplanar) points: every point lies on it. + let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); + let mut s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::MostlyFlat, deleted: vec![] }, None).expect("solve"); + let ids: Vec = s.visible(0).map(|p| p.id).take(12).collect(); + let g = s.ground_from(&ids, 0).expect("ground"); + let spread = s.points.iter().map(|p| dot(sub(p.pos, g.origin), g.normal).abs()).fold(0.0, f64::max); + let depth = norm(sub(s.frames[0].center, g.origin)); + assert!(spread < 0.01 * depth, "{spread}"); + // Up points towards the camera. + assert!(dot(sub(s.frames[0].center, g.origin), g.normal) > 0.0); + s.ground = Some(g); + let w = s.world([W, H], 1000.0); + // In the world, ground points have y ≈ 0 and the cameras are above (y < 0). + for p in s.points.iter().take(20) { + assert!(w.apply(p.pos)[1].abs() < 0.01 * 1000.0 * depth, "{:?}", w.apply(p.pos)); + } + assert!(w.apply(s.frames[0].center)[1] < 0.0); + assert!(norm(w.apply(g.origin)) < 1e-6); +} + +#[test] +fn deleted_tracks_are_ignored_and_cancel_stops() { + let t = moving_camera(20, |_| 1000.0); + let pts = random_points(250, 13, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 14, true); + let deleted: Vec = (0..40).collect(); + let hfov = 2.0 * (W * 0.5 / 1000.0f64).atan().to_degrees(); + let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: deleted.clone() }, None).unwrap(); + assert!(s.points.iter().all(|p| !deleted.contains(&p.id))); + let stop = std::sync::atomic::AtomicBool::new(true); + let r = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), Some(&stop)); + assert!(r.is_err()); +} + +/// End to end: rendered frames of a textured scene through the KLT feature tracker. +#[test] +fn rendered_scene_end_to_end() { + use effectcraft_raster::Image; + let (w, h) = (480u32, 270u32); + let f = 420.0; + // Three textured planes at different depths (a floor, a back wall and a box face). + fn tex(u: f64, v: f64) -> f32 { + fn hsh(i: i64, j: i64) -> f64 { + let mut x = (i.wrapping_mul(73856093) ^ j.wrapping_mul(19349663)) as u64; + x ^= x >> 13; + x = x.wrapping_mul(0x5bd1e995); + x ^= x >> 15; + (x % 1000) as f64 / 1000.0 + } + fn noise(x: f64, y: f64) -> f64 { + let (i, j) = (x.floor(), y.floor()); + let (tx, ty) = (x - i, y - j); + let (sx, sy) = (tx * tx * (3.0 - 2.0 * tx), ty * ty * (3.0 - 2.0 * ty)); + let (i, j) = (i as i64, j as i64); + let a = hsh(i, j) + (hsh(i + 1, j) - hsh(i, j)) * sx; + let b = hsh(i, j + 1) + (hsh(i + 1, j + 1) - hsh(i, j + 1)) * sx; + a + (b - a) * sy + } + (0.15 + 0.5 * noise(u * 4.0, v * 4.0) + 0.3 * noise(u * 11.0 + 7.0, v * 11.0 + 3.0)) as f32 + } + let frames = 24; + let t = Truth { + cams: (0..frames) + .map(|k| { + let s = k as f64 / (frames - 1) as f64; + let c = [-1.0 + 2.0 * s, -0.2, 0.5 * s]; + let fwd = normalize(sub([0.0, 0.3, 8.0], c)); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + ([right, cross(fwd, right), fwd], c, f) + }) + .collect(), + }; + // Planes: (point, normal, u axis, v axis). + let planes: [(V3, V3, V3, V3); 3] = [ + ([0.0, 1.5, 0.0], [0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]), + ([0.0, 0.0, 10.0], [0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0]), + ([-1.0, 0.0, 6.0], [0.3, 0.0, -1.0], [1.0, 0.0, 0.3], [0.0, 1.0, 0.0]), + ]; + let render = |k: usize| { + let (r, c, f) = t.cams[k]; + let mut img = Image::new(w, h); + for y in 0..h { + for x in 0..w { + let d = mtv(&r, [(x as f64 + 0.5 - w as f64 / 2.0) / f, (y as f64 + 0.5 - h as f64 / 2.0) / f, 1.0]); + let mut best = (f64::INFINITY, 0.0f32); + for (i, (p0, n, ua, va)) in planes.iter().enumerate() { + let den = dot(*n, d); + if den.abs() < 1e-9 { + continue; + } + let tt = dot(*n, sub(*p0, c)) / den; + let hit = add(c, scale(d, tt)); + // The box face is limited in extent. + if i == 2 && (dot(sub(hit, *p0), *ua).abs() > 1.2 || dot(sub(hit, *p0), *va).abs() > 1.0) { + continue; + } + if tt > 0.1 && tt < best.0 { + best = (tt, tex(dot(sub(hit, *p0), *ua) + 10.0 * i as f64, dot(sub(hit, *p0), *va))); + } + } + let v = best.1; + img.set(x, y, [v, v, v, 1.0]); + } + } + img + }; + let mut an = TrackAnalyzer::new([w as f64, h as f64], AnalyzeOpts::default()); + for k in 0..frames { + let img = render(k); + an.push(&crate::Frame::new(&img)); + } + let tr = an.finish(0.0, 1.0 / 24.0); + assert!(tr.tracks.len() > 100, "{}", tr.tracks.len()); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("rendered: {:?} err {:.3} focal {:.1} points {}", s.method_used, s.average_error, s.focal(), s.points.len()); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.05, "{}", s.focal()); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.03, "{pe}"); +} + +/// A long clip (300 frames, ~900 points): `cargo test -p effectcraft-track --release -- --ignored`. +#[test] +#[ignore] +fn perf_long_clip() { + let f = 1100.0; + let t = moving_camera(300, |_| f); + let pts = random_points(900, 21, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]); + let tr = tracks_of(&pts, &t, 22, true); + let t0 = std::time::Instant::now(); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("300 frames, {} tracks: {:?} err {:.3} focal {:.1} in {:?}", tr.tracks.len(), s.method_used, s.average_error, s.focal(), t0.elapsed()); + assert!(s.average_error < 0.5); +} diff --git a/crates/track/src/lib.rs b/crates/track/src/lib.rs index d7b239f5..bd4e21dc 100644 --- a/crates/track/src/lib.rs +++ b/crates/track/src/lib.rs @@ -18,6 +18,7 @@ //! Coordinates are *layer* pixels: the frame image's pixel `(i, j)` covers `[i, i+1) × [j, j+1)` //! after subtracting the frame's `offset` (see [`Frame`]). +pub mod camtrack; pub mod fit; pub mod klt; pub mod mask; From cba8d7e737db9438c9b0e46fd882875df5e7946a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 23:02:39 -0400 Subject: [PATCH 2/2] M12.6: 3D Camera Tracker effect + UI Effect > Perspective > 3D Camera Tracker (Shot Type, Horizontal Angle of View, Show/Render Track Points, Track Point Size, Target Size, Advanced: Solve Method, Method Used, Average Error, Detailed Analysis, Auto-delete Points Across Time, Hide Warning Banner); tracks and solve stored in the effect with keys (frame edits re-track, setting / deleted-point edits only re-solve). Background job (Analyzing in background (step 1 of 2) / Solving camera banner, cancel, one undo step). Commands: track.camera (replaces the stub), camera.analyze/cancel/solveStatus/points/selectPoints/ createFromSolve/create/setGroundPlane/deletePoints; the one-node 3D Tracker Camera keyed per frame and text / solids / nulls / shadow catcher + light on the target plane, Create Multiple. Viewer: depth-sized coloured targets, hover target plane, click / shift / marquee selection, AE right-click menu, Delete; Effect Controls Analyze / Cancel / Create Camera; Tracker panel Track Camera. Solver: adaptive noise thresholds, denser keyframes, quick tripod check, faster Cholesky and parallel Schur complement. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + ROADMAP.md | 4 +- crates/effects/src/camera_tracker.rs | 291 ++++++++ crates/effects/src/lib.rs | 4 + crates/engine/src/camera_track.rs | 640 ++++++++++++++++++ crates/engine/src/commands/camera_cmds.rs | 574 ++++++++++++++++ crates/engine/src/commands/mod.rs | 2 + crates/engine/src/commands/stubs.rs | 1 - crates/engine/src/lib.rs | 22 + crates/engine/src/tests_camera_track.rs | 307 +++++++++ crates/engine/src/tests_stubs.rs | 2 +- crates/track/Cargo.toml | 1 + crates/track/src/camtrack/bundle.rs | 90 ++- crates/track/src/camtrack/features.rs | 2 +- crates/track/src/camtrack/linalg.rs | 30 +- crates/track/src/camtrack/mod.rs | 8 +- crates/track/src/camtrack/solver.rs | 83 ++- crates/track/src/camtrack/tests.rs | 78 +++ crates/ui-egui/src/lib.rs | 5 + .../ui-egui/src/panels/camera_tracker_ui.rs | 438 ++++++++++++ crates/ui-egui/src/panels/effect_controls.rs | 2 + crates/ui-egui/src/panels/mod.rs | 1 + crates/ui-egui/src/panels/tracker.rs | 7 +- crates/ui-egui/src/panels/viewer.rs | 3 + crates/ui-egui/tests/ui_camera_tracker.rs | 175 +++++ docs/agents.md | 28 + docs/architecture.md | 16 + docs/parity.md | 10 +- 28 files changed, 2771 insertions(+), 54 deletions(-) create mode 100644 crates/effects/src/camera_tracker.rs create mode 100644 crates/engine/src/camera_track.rs create mode 100644 crates/engine/src/commands/camera_cmds.rs create mode 100644 crates/engine/src/tests_camera_track.rs create mode 100644 crates/ui-egui/src/panels/camera_tracker_ui.rs create mode 100644 crates/ui-egui/tests/ui_camera_tracker.rs diff --git a/Cargo.lock b/Cargo.lock index 46f34e63..ce792ecb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1420,6 +1420,7 @@ dependencies = [ "rayon", "serde", "serde_json", + "web-time", ] [[package]] diff --git a/ROADMAP.md b/ROADMAP.md index a3ed979c..7780c164 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -28,10 +28,10 @@ once. Measured feature by feature in [docs/parity.md](docs/parity.md) (end of 2 October 2026): - **≈ 79% of After Effects' features, weighted by importance** (87% of the essentials, 67% of - professional daily-use features, 21% of the long tail); 257 of 298 effects, 16 of them on the GPU. + professional daily-use features, 21% of the long tail); 258 of 298 effects, 16 of them on the GPU. - **≈ 100 agent-hours of work remain by the audit's estimate.** With five Claude Opus 5.5 agents in parallel that is **≈ 8–10 hours of wall-clock time at the pace measured so far, up to ≈ 25 hours** by the audit's conservative per-feature figures. -- Hardest remaining pieces: Roto Brush, the 3D camera tracker, Advanced 3D, a scripting object model. +- Hardest remaining pieces: Roto Brush, Advanced 3D, a scripting object model (the 3D Camera Tracker landed). Come tell us what matters most to you on [Discord](https://discord.gg/artcraft). diff --git a/crates/effects/src/camera_tracker.rs b/crates/effects/src/camera_tracker.rs new file mode 100644 index 00000000..dd97b585 --- /dev/null +++ b/crates/effects/src/camera_tracker.rs @@ -0,0 +1,291 @@ +//! 3D Camera Tracker (Effect ▸ Perspective ▸ 3D Camera Tracker; Animation ▸ Track Camera). +//! +//! The analysis runs as a background job in the engine (`camera.analyze`): step 1 tracks +//! features through the clip ([`effectcraft_track::camtrack::TrackAnalyzer`]), step 2 solves +//! the camera ([`effectcraft_track::camtrack::solve`]). Both results are stored as JSON in the +//! instance's hidden parameters — **Tracks** (with **Tracks Key**: what the frames were made +//! from, so source / In-Out / time edits invalidate them) and **Solve** (with **Solve Key**: the +//! Shot Type, Angle of View, Solve Method and deleted points it was solved with, so changing +//! those re-solves without re-tracking). **Method Used** and **Average Error** report the solve. +//! +//! The effect passes its input through; with **Render Track Points** on it draws the solved +//! points (as the viewer shows them) into the frame. The viewer overlay, target and the Create +//! commands live in the engine and the UI. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::{Image, Px}; +use effectcraft_track::camtrack::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, linalg, point_color}; + +use crate::warp_stab::param; +use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; + +/// The effect's id. +pub const ID: &str = "ec.perspective.cameratracker"; + +/// Hidden parameters (see the module docs). +pub const TRACKS: &str = "tracks"; +pub const TRACKS_KEY: &str = "tracksKey"; +pub const SOLVE: &str = "solve"; +pub const SOLVE_KEY: &str = "solveKey"; +/// Deleted track point ids (JSON array). +pub const DELETED: &str = "deleted"; +/// The "3D Tracker Camera" layer the instance created (layer id, 0 = none). +pub const CAMERA: &str = "camera"; +pub const METHOD_USED: &str = "advanced/methodUsed"; +pub const AVERAGE_ERROR: &str = "advanced/averageError"; + +pub const SHOT_TYPES: [&str; 3] = ["Fixed Angle of View", "Variable Zoom", "Specify Angle of View"]; +pub const SHOW_POINTS: [&str; 2] = ["2D Source", "3D Solved"]; +pub const SOLVE_METHODS: [&str; 4] = ["Auto Detect", "Typical", "Mostly Flat Scene", "Tripod Pan"]; + +pub fn specs() -> Vec { + vec![EffectSpec { + id: ID, + name: "3D Camera Tracker", + category: "Perspective", + params: vec![ + p("shotType", "Shot Type", Value::Enum(0), popup(&SHOT_TYPES)), + p("horizontalAngleOfView", "Horizontal Angle of View", num(40.0), slider(1.0, 170.0, 10.0, 120.0, 1)), + p("showTrackPoints", "Show Track Points", Value::Enum(1), popup(&SHOW_POINTS)), + p("renderTrackPoints", "Render Track Points", Value::Bool(false), ParamUi::Checkbox), + p("trackPointSize", "Track Point Size", num(100.0), slider(0.0, 1000.0, 0.0, 300.0, 0)), + p("targetSize", "Target Size", num(100.0), slider(1.0, 1000.0, 10.0, 300.0, 0)), + p("advanced/solveMethod", "Solve Method", Value::Enum(0), popup(&SOLVE_METHODS)), + p(METHOD_USED, "Method Used", Value::Str(String::new()), ParamUi::Hidden), + p(AVERAGE_ERROR, "Average Error", num(0.0), ParamUi::Hidden), + p("advanced/detailedAnalysis", "Detailed Analysis", Value::Bool(false), ParamUi::Checkbox), + p("advanced/autoDeletePoints", "Auto-delete Points Across Time", Value::Bool(true), ParamUi::Checkbox), + p("advanced/hideWarningBanner", "Hide Warning Banner", Value::Bool(false), ParamUi::Checkbox), + p(TRACKS, "Tracks", Value::Str(String::new()), ParamUi::Hidden), + p(TRACKS_KEY, "Tracks Key", Value::Str(String::new()), ParamUi::Hidden), + p(SOLVE, "Solve", Value::Str(String::new()), ParamUi::Hidden), + p(SOLVE_KEY, "Solve Key", Value::Str(String::new()), ParamUi::Hidden), + p(DELETED, "Deleted Points", Value::Str(String::new()), ParamUi::Hidden), + p(CAMERA, "Camera Layer", num(0.0), ParamUi::Hidden), + ], + render, + gpu: false, + float: true, + }] +} + +fn f(params: &Params, id: &str) -> f64 { + param(params, id).map(Value::as_f64).unwrap_or(0.0) +} +fn e(params: &Params, id: &str) -> u32 { + param(params, id).map(Value::as_enum).unwrap_or(0) +} +fn b(params: &Params, id: &str) -> bool { + param(params, id).map(Value::as_bool).unwrap_or(false) +} +fn s<'a>(params: &'a Params, id: &str) -> &'a str { + match param(params, id) { + Some(Value::Str(s)) => s, + _ => "", + } +} + +/// Deleted point ids of an instance (stored as a JSON array of numbers). +pub fn deleted(params: &Params) -> Vec { + s(params, DELETED).trim().trim_start_matches('[').trim_end_matches(']').split(',').filter_map(|v| v.trim().parse().ok()).collect() +} + +/// The solve settings of an instance. +pub fn settings(params: &Params) -> SolveSettings { + let mut del = deleted(params); + del.sort_unstable(); + del.dedup(); + SolveSettings { + shot: ShotType::ALL.get(e(params, "shotType") as usize).copied().unwrap_or_default(), + hfov: f(params, "horizontalAngleOfView"), + method: SolveMethod::ALL.get(e(params, "advanced/solveMethod") as usize).copied().unwrap_or_default(), + deleted: del, + } +} + +/// What a solve depends on besides the tracks. +pub fn solve_key(st: &SolveSettings) -> String { + let hfov = if st.shot == ShotType::SpecifyAngle { format!("{:.4}", st.hfov) } else { String::new() }; + format!("{:?}|{hfov}|{:?}|{:?}", st.shot, st.method, st.deleted) +} + +fn fnv(bytes: &[u8]) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for b in bytes { + h ^= *b as u64; + h = h.wrapping_mul(0x0100_0000_01b3); + } + h +} + +type Cache = Mutex>>; + +fn cached(c: &'static OnceLock>, json: &str, parse: impl Fn(&str) -> Option) -> Option> { + let key = fnv(json.as_bytes()); + let c = c.get_or_init(Default::default); + if let Some(a) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&key) { + return Some(a.clone()); + } + let a = Arc::new(parse(json)?); + let mut m = c.lock().unwrap_or_else(|e| e.into_inner()); + if m.len() > 8 { + m.clear(); + } + m.insert(key, a.clone()); + Some(a) +} + +/// The stored feature tracks of an instance (`None` when not analysed). +pub fn tracks(params: &Params) -> Option> { + static C: OnceLock> = OnceLock::new(); + let j = s(params, TRACKS); + if j.is_empty() { + return None; + } + cached(&C, j, CameraTracks::from_json).filter(|t| !t.is_empty()) +} + +/// The stored camera solve of an instance (`None` when not solved). +pub fn solve(params: &Params) -> Option> { + static C: OnceLock> = OnceLock::new(); + let j = s(params, SOLVE); + if j.is_empty() { + return None; + } + cached(&C, j, CameraSolve::from_json).filter(|t| !t.is_empty()) +} + +/// Screen size (layer pixels) of a target drawn for a point at depth `z`, relative to the +/// median depth of the frame's points. +pub fn target_radius(size_pct: f64, z: f64, median_z: f64) -> f64 { + (6.0 * size_pct / 100.0 * (median_z / z.max(1e-9))).clamp(1.5, 40.0) +} + +/// The solved points visible on frame `k`: (id, layer pixel position, camera depth). +pub fn projected(solve: &CameraSolve, k: usize) -> Vec<(u32, [f64; 2], f64)> { + let Some(cam) = solve.frames.get(k) else { return vec![] }; + solve + .visible(k) + .filter_map(|p| { + let c = cam.to_cam(p.pos); + cam.project(solve.size, p.pos).map(|q| (p.id, q, c[2])) + }) + .collect() +} + +/// Median of the depths. +pub fn median_depth(v: &[(u32, [f64; 2], f64)]) -> f64 { + let mut z: Vec = v.iter().map(|x| x.2).collect(); + if z.is_empty() { + return 1.0; + } + z.sort_by(f64::total_cmp); + z[z.len() / 2] +} + +fn render(ctx: &EffectCtx, mut buf: Buf) -> Buf { + if !b(ctx.params, "renderTrackPoints") { + return buf; + } + let Some(sol) = solve(ctx.params) else { return buf }; + if (sol.size[0] - ctx.layer_size[0]).abs() > 0.5 || (sol.size[1] - ctx.layer_size[1]).abs() > 0.5 { + return buf; + } + let Some(k) = sol.frame_at(ctx.time) else { return buf }; + let pts = projected(&sol, k); + let med = median_depth(&pts); + let size = f(ctx.params, "trackPointSize"); + let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + for (id, q, z) in pts { + let c = point_color(id); + let r = target_radius(size, z, med) * sc; + let at = [q[0] * sc + buf.offset[0], q[1] * sc + buf.offset[1]]; + ring(&mut buf.img, at, r, (r * 0.25).max(1.0), [c[0], c[1], c[2], 1.0]); + disc(&mut buf.img, at, (r * 0.2).max(0.8), [c[0], c[1], c[2], 1.0]); + } + buf +} + +fn lerp_px(a: Px, b: Px, t: f32) -> Px { + [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t, a[2] + (b[2] - a[2]) * t, a[3] + (b[3] - a[3]) * t] +} + +fn cover(img: &mut Image, c: [f64; 2], r_out: f64, cov: impl Fn(f64) -> f64, col: Px) { + let (x0, x1) = ((c[0] - r_out - 1.0).floor().max(0.0) as i64, (c[0] + r_out + 1.0).ceil() as i64); + let (y0, y1) = ((c[1] - r_out - 1.0).floor().max(0.0) as i64, (c[1] + r_out + 1.0).ceil() as i64); + for y in y0..y1.min(img.height as i64) { + for x in x0..x1.min(img.width as i64) { + let d = (x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1]); + let a = cov(d).clamp(0.0, 1.0) as f32; + if a > 0.0 { + let i = img.idx(x as u32, y as u32); + img.data[i] = lerp_px(img.data[i], col, a); + } + } + } +} + +fn disc(img: &mut Image, c: [f64; 2], r: f64, col: Px) { + cover(img, c, r, |d| r + 0.5 - d, col); +} + +fn ring(img: &mut Image, c: [f64; 2], r: f64, w: f64, col: Px) { + cover(img, c, r + w, |d| (w * 0.5 + 0.5 - (d - r).abs()).min(1.0), col); +} + +/// Points of a target disc (canonical coordinates) for drawing: `n` points on the circle of +/// radius `size / 2` in the plane through `center` with normal `normal`. +pub fn target_circle(center: linalg::V3, normal: linalg::V3, radius: f64, n: usize) -> Vec { + let a = if normal[0].abs() < 0.9 { [1.0, 0.0, 0.0] } else { [0.0, 1.0, 0.0] }; + let u = linalg::normalize(linalg::cross(normal, a)); + let v = linalg::cross(normal, u); + (0..n) + .map(|i| { + let t = i as f64 / n as f64 * std::f64::consts::TAU; + linalg::add(center, linalg::add(linalg::scale(u, radius * t.cos()), linalg::scale(v, radius * t.sin()))) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::EffectEnv; + use effectcraft_track::camtrack::{SolvedFrame, SolvedPoint}; + + fn params_with(solve: &CameraSolve, render: bool) -> Params { + let mut p = Params::default(); + p.values.insert(SOLVE.into(), Value::Str(solve.to_json())); + p.values.insert("renderTrackPoints".into(), Value::Bool(render)); + p.values.insert("trackPointSize".into(), Value::Scalar(100.0)); + p + } + + #[test] + fn render_track_points_draws_targets_and_passes_through_otherwise() { + let sol = CameraSolve { + version: 1, + size: [64.0, 48.0], + start: 0.0, + frame_duration: 1.0 / 30.0, + frames: vec![SolvedFrame { rot: [0.0; 3], center: [0.0; 3], focal: 50.0, solved: true }], + points: vec![SolvedPoint { id: 3, pos: [0.0, 0.0, 1.0], error: 0.1, first: 0, last: 0 }], + ..Default::default() + }; + let img = Image::new(64, 48); + for on in [false, true] { + let params = params_with(&sol, on); + let ctx = EffectCtx { params: ¶ms, time: 0.0, layer_size: [64.0, 48.0], seed: 0, adjustment: false, env: EffectEnv::default() }; + let out = render(&ctx, Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 }); + let px = out.img.data[out.img.idx(32, 24)]; + assert_eq!(px[3] > 0.5, on); + } + let st = settings(&Params::default()); + assert_eq!(st.shot, ShotType::FixedAngle); + assert!(solve_key(&st).contains("FixedAngle")); + } +} diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 6d11e19a..9c2aefb5 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -8,6 +8,7 @@ pub mod audio_fx; mod blur2; +pub mod camera_tracker; mod channel; mod channel2; mod color2; @@ -354,6 +355,7 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(paint::specs()); v.extend(puppet::specs()); v.extend(warp_stab::specs()); + v.extend(camera_tracker::specs()); v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); for s in v.iter_mut() { if GPU_EFFECTS.contains(&s.id) { @@ -494,6 +496,8 @@ pub const TIME_DEPENDENT: &[&str] = &[ "ec.time.pixelmotionblur", // Each frame gets its own stabilizing warp. warp_stab::ID, + // Render Track Points draws each frame's solved points. + camera_tracker::ID, ]; /// See [`TIME_DEPENDENT`]. diff --git a/crates/engine/src/camera_track.rs b/crates/engine/src/camera_track.rs new file mode 100644 index 00000000..1b984aac --- /dev/null +++ b/crates/engine/src/camera_track.rs @@ -0,0 +1,640 @@ +//! 3D Camera Tracker runtime: the analysis job (`camera.analyze`, `track.camera`), keeping +//! analyses valid, and turning a solve into composition layers (`camera.createFromSolve`). +//! +//! The job renders the layer's input to the effect (`Renderer::layer_input`) for every frame +//! between the layer's In and Out points on a background thread and tracks features through +//! them (step 1 of 2, "Analyzing in background"), then solves the camera (step 2, "Solving +//! camera"). When only the solve settings changed (Shot Type, Angle of View, Solve Method, +//! deleted points) the stored tracks are re-solved without re-tracking. Results are written into +//! the effect's hidden parameters as one undo step; edits that change the layer's frames clear +//! them and queue the effect again ([`invalidate`], run after every edit), like After Effects. +//! +//! Composition space: the solve's canonical frame maps to comp world by +//! [`CameraSolve::world`] with the zoom of the first solved frame (focal length × the layer's +//! scale in the comp), so the "3D Tracker Camera" reproduces the footage's view and created +//! layers sit on the selected points. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use effectcraft_effects::camera_tracker::{self as ct, AVERAGE_ERROR, METHOD_USED, SOLVE, SOLVE_KEY, TRACKS, TRACKS_KEY}; +use effectcraft_keyframe::{Keyframe, Value}; +use effectcraft_project::{AutoOrient, GroupKind, ItemId, ItemKind, Layer, LayerId, Project, PropGroup, Uid}; +use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; +use effectcraft_time::Tick; +use effectcraft_track::camtrack::linalg::{self, Similarity, V3}; +use effectcraft_track::camtrack::{AnalyzeOpts, CameraSolve, CameraTracks, SolveSettings, Target, TrackAnalyzer}; +use serde::Serialize; + +use crate::tracking::lock; +use crate::warp::{analysis_times, signature}; +use crate::{Event, Session}; + +/// Live analysis progress (serde for agents / the control channel). +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CameraProgress { + /// 1 = tracking features, 2 = solving the camera. + pub step: u8, + pub done: u64, + pub total: u64, + pub elapsed: f64, + pub fps: f64, + pub finished: bool, + pub cancelled: bool, + pub error: Option, + /// Re-solving stored tracks (no tracking step). + pub solve_only: bool, +} + +impl CameraProgress { + /// The viewer banner text, as After Effects words it. + pub fn banner(&self) -> String { + match self.step { + 2 => "Solving camera".into(), + _ => { + let pc = (100 * self.done).checked_div(self.total).unwrap_or(0); + format!("Analyzing in background (step 1 of 2): {pc}%") + } + } + } +} + +struct CamResult { + /// New tracks (None for a re-solve). + tracks: Option, + solve: Result, +} + +#[derive(Default)] +pub struct CameraShared { + pub cancel: AtomicBool, + pub state: Mutex, + result: Mutex>, +} + +/// A running (or finished, not yet polled) camera analysis. +pub struct CameraJob { + pub shared: Arc, + thread: Option>, + pub comp: ItemId, + pub layer: LayerId, + pub effect: Uid, + /// The layer signature the tracks are made from, and the solve settings key. + key: String, + solve_key: String, +} + +impl CameraJob { + pub fn is_finished(&self) -> bool { + lock(&self.shared.state).finished + } + pub fn progress(&self) -> CameraProgress { + lock(&self.shared.state).clone() + } +} + +struct CamWork { + project: Arc, + footage: Arc, + expr: Option>, + cache: Arc, + comp: ItemId, + layer: LayerId, + effect_index: usize, + size: [f64; 2], + detailed: bool, + times: Vec, + /// Stored tracks to re-solve (skips step 1). + tracks: Option>, + settings: SolveSettings, +} + +pub(crate) fn is_tracker(g: &PropGroup) -> bool { + matches!(&g.kind, GroupKind::Effect { effect } if effect == ct::ID) +} + +fn str_of(g: &PropGroup, m: &str) -> String { + match g.prop(m).map(|p| &p.value) { + Some(Value::Str(s)) => s.clone(), + _ => String::new(), + } +} + +/// What a tracker's tracks are made from: the layer's frames and Detailed Analysis. +fn tracks_key(project: &Project, cid: ItemId, l: &Layer, g: &PropGroup) -> String { + let detailed = effectcraft_effects::warp_stab::param(&static_params(g), "advanced/detailedAnalysis").is_some_and(Value::as_bool); + format!("{}{}", signature(project, cid, l), if detailed { "+detailed" } else { "" }) +} + +/// Parameters of an instance from static values (the tracker's settings are not animated). +pub fn static_params(g: &PropGroup) -> effectcraft_effects::Params { + effectcraft_effects::flatten_params(g, &mut |p| p.value.clone()) +} + +fn clear(g: &mut PropGroup, all: bool) { + let mut ids = vec![SOLVE, SOLVE_KEY, METHOD_USED]; + if all { + ids.extend([TRACKS, TRACKS_KEY]); + } + for m in ids { + if let Some(p) = g.prop_mut(m) { + p.value = Value::Str(String::new()); + p.keys.clear(); + } + } + if let Some(p) = g.prop_mut(AVERAGE_ERROR) { + p.value = Value::Scalar(0.0); + } +} + +/// Clear analyses that no longer match, after an edit: tracks made from other frames (and their +/// solve), and solves made with other settings. Returns the trackers that need (re-)analysis: +/// cleared ones and newly applied, unanalysed ones. +pub(crate) fn invalidate(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> { + let mut todo = vec![]; + let mut clears = vec![]; + for (cid, it) in &after.items { + let ItemKind::Comp(c) = &it.kind else { continue }; + for l in &c.layers { + let Some(fx) = l.effects() else { continue }; + if !fx.groups().any(is_tracker) { + continue; + } + let old_layer = before.comp(*cid).and_then(|bc| bc.layer(l.id)); + for g in fx.groups().filter(|g| is_tracker(g)) { + let tracked = !str_of(g, TRACKS).is_empty(); + if tracked && str_of(g, TRACKS_KEY) != tracks_key(after, *cid, l, g) { + clears.push((*cid, l.id, g.uid, true)); + todo.push((*cid, l.id, g.uid)); + } else if tracked && str_of(g, SOLVE_KEY) != ct::solve_key(&ct::settings(&static_params(g))) { + clears.push((*cid, l.id, g.uid, false)); + todo.push((*cid, l.id, g.uid)); + } else if !tracked && old_layer.and_then(|ol| ol.props.find_group(g.uid)).is_none() { + todo.push((*cid, l.id, g.uid)); + } + } + } + } + for (cid, lid, uid, all) in clears { + if let Some(g) = after.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)) { + clear(g, all); + } + } + todo +} + +fn input_frame(r: &Renderer, ctx: &EvalCtx, layer: &Layer, effect_index: usize) -> Option<(Arc, [f64; 2])> { + let buf = r.layer_input(ctx, layer, effect_index)?; + if (buf.scale - 1.0).abs() > 1e-9 && buf.scale > 0.0 { + let w = (buf.img.width as f64 / buf.scale).round().max(1.0) as u32; + let h = (buf.img.height as f64 / buf.scale).round().max(1.0) as u32; + let off = [buf.offset[0] / buf.scale, buf.offset[1] / buf.scale]; + return Some((Arc::new(effectcraft_raster::resample(&buf.img, w, h)), off)); + } + Some((Arc::new(buf.img.clone()), buf.offset)) +} + +fn run_work(w: CamWork, shared: &CameraShared) { + let t0 = web_time::Instant::now(); + let fail = |msg: &str| { + let mut s = lock(&shared.state); + s.error = Some(msg.to_string()); + s.finished = true; + }; + let cancelled = || { + let mut s = lock(&shared.state); + s.cancelled = true; + s.finished = true; + }; + let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") }; + let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") }; + let lt0 = layer.layer_time(w.times[0]).seconds(); + let fd = if w.times.len() > 1 { layer.layer_time(w.times[1]).seconds() - lt0 } else { comp.frame_duration().seconds() }; + let (tracks, fresh) = match w.tracks.clone() { + Some(t) => (t, false), + None => { + let mut r = Renderer::new(&w.project, w.footage.as_ref(), RenderOpts { scale: 1.0, ..Default::default() }); + r.expr = w.expr.as_deref(); + r.cache = Some(&w.cache); + let r = &r; + let ctx0 = EvalCtx::new(&w.project, w.comp, comp, Tick::ZERO); + let ctx0 = EvalCtx { expr: w.expr.as_deref(), ..ctx0 }; + let frame_at = |t: Tick| input_frame(r, &ctx0.at(t), layer, w.effect_index); + { + let mut s = lock(&shared.state); + s.step = 1; + s.total = w.times.len() as u64; + } + let mut an = TrackAnalyzer::new(w.size, AnalyzeOpts { detailed: w.detailed }); + let mut next = frame_at(w.times[0]); + for k in 0..w.times.len() { + if shared.cancel.load(Ordering::Relaxed) { + return cancelled(); + } + let Some(cur) = next.take() else { return fail(&format!("no frame at {:.3} s", w.times[k].seconds())) }; + let ((), nf) = rayon::join( + || an.push(&effectcraft_track::Frame { img: &cur.0, offset: cur.1 }), + || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None }, + ); + next = nf; + let mut s = lock(&shared.state); + s.done = k as u64 + 1; + s.elapsed = t0.elapsed().as_secs_f64(); + s.fps = if s.elapsed > 0.0 { s.done as f64 / s.elapsed } else { 0.0 }; + } + (Arc::new(an.finish(lt0, fd)), true) + } + }; + lock(&shared.state).step = 2; + let solve = effectcraft_track::camtrack::solve(&tracks, &w.settings, Some(&shared.cancel)).map_err(|e| e.0); + if shared.cancel.load(Ordering::Relaxed) { + return cancelled(); + } + *lock(&shared.result) = Some(CamResult { tracks: fresh.then(|| (*tracks).clone()), solve }); + let mut s = lock(&shared.state); + s.elapsed = t0.elapsed().as_secs_f64(); + s.finished = true; +} + +impl Session { + pub fn is_camera_analyzing(&self) -> bool { + self.camera_job.as_ref().is_some_and(|j| !j.is_finished()) + } + + /// Progress of the running (or just finished) analysis. + pub fn camera_progress(&self) -> Option { + self.camera_job.as_ref().map(|j| j.progress()) + } + + /// The tracker being analysed: (comp, layer, effect uid). + pub fn camera_target(&self) -> Option<(ItemId, LayerId, Uid)> { + self.camera_job.as_ref().map(|j| (j.comp, j.layer, j.effect)) + } + + /// Cancel the running analysis (nothing is written). + pub fn stop_camera(&mut self) -> bool { + match &self.camera_job { + Some(j) if !j.is_finished() => { + j.shared.cancel.store(true, Ordering::Relaxed); + true + } + _ => false, + } + } + + /// Start analysing 3D Camera Tracker `effect` on `layer` (re-solving the stored tracks when + /// they are still valid). `wait`: block until done (wasm32 always does). Returns the number + /// of frames. + pub fn start_camera(&mut self, comp: ItemId, layer: LayerId, effect: Uid, wait: bool) -> Result { + if self.is_camera_analyzing() { + return Err("a 3D Camera Tracker analysis is already running".into()); + } + self.poll_camera_job(); + self.camera_pending.retain(|x| *x != (comp, layer, effect)); + let c = self.project.comp(comp).ok_or("no composition")?; + let l = c.layer(layer).ok_or("no layer")?; + let fx = l.effects().ok_or("the layer has no effects")?; + let index = fx.groups().position(|g| g.uid == effect).ok_or("no such effect")?; + let g = fx.groups().nth(index).filter(|g| is_tracker(g)).ok_or("not a 3D Camera Tracker")?; + if l.switches.adjustment { + return Err("3D Camera Tracker needs footage, not an adjustment layer".into()); + } + let times = analysis_times(c, l); + if times.len() < 2 { + return Err("the layer must be at least two frames long".into()); + } + let (w, h) = effectcraft_render::source_size(&self.project, l); + let size = if w == 0 { [c.width as f64, c.height as f64] } else { [w as f64, h as f64] }; + let params = static_params(g); + let key = tracks_key(&self.project, comp, l, g); + let settings = ct::settings(¶ms); + let solve_key = ct::solve_key(&settings); + let tracks = ct::tracks(¶ms).filter(|_| str_of(g, TRACKS_KEY) == key); + let n = times.len(); + let work = CamWork { + project: self.project.clone(), + footage: self.footage.clone(), + expr: self.expr.clone(), + cache: self.layer_cache.clone(), + comp, + layer, + effect_index: index, + size, + detailed: params.values.iter().any(|(k, v)| k.ends_with("detailedAnalysis") && v.as_bool()), + times, + tracks: tracks.clone(), + settings, + }; + let wait = wait || cfg!(target_arch = "wasm32"); + let shared = Arc::new(CameraShared::default()); + { + let mut st = lock(&shared.state); + st.solve_only = tracks.is_some(); + st.step = if tracks.is_some() { 2 } else { 1 }; + st.total = n as u64; + } + let thread = if wait { + run_work(work, &shared); + None + } else { + let sh = shared.clone(); + Some(std::thread::Builder::new().name("camera-track".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?) + }; + self.camera_job = Some(CameraJob { shared, thread, comp, layer, effect, key, solve_key }); + self.events.push(Event::ProjectChanged { revision: self.revision }); + self.poll_camera_job(); + Ok(n) + } + + /// Write a finished analysis into its effect (one undo step) and drop the job. + fn poll_camera_job(&mut self) -> bool { + let Some(job) = &self.camera_job else { return false }; + if !job.is_finished() { + return false; + } + let mut job = self.camera_job.take().expect("job"); + if let Some(t) = job.thread.take() { + let _ = t.join(); + } + let st = job.progress(); + let result = lock(&job.shared.result).take(); + match (result, &st.error) { + (Some(res), None) if !st.cancelled => { + let (cid, lid, uid) = (job.comp, job.layer, job.effect); + let (key, skey) = (job.key.clone(), job.solve_key.clone()); + let tracks_json = res.tracks.as_ref().map(CameraTracks::to_json); + let (solve_json, method, avg, msg, failed) = match &res.solve { + Ok(s) => ( + s.to_json(), + s.method_used.label().to_string(), + s.average_error, + format!( + "3D Camera Tracker: solved {} frame(s), {} points, average error {:.2} px ({:.1} s)", + s.frames.len(), + s.points.len(), + s.average_error, + st.elapsed + ), + false, + ), + Err(e) => (String::new(), String::new(), 0.0, format!("3D Camera Tracker: {e}"), true), + }; + let r = self.edit(if st.solve_only { "3D Camera Tracker Solve" } else { "3D Camera Tracker Analysis" }, None, |p, _| { + let g = p + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .ok_or_else(|| crate::EngineError::Other("the 3D Camera Tracker was removed during analysis".into()))?; + let mut set = |m: &str, v: Value| { + if let Some(pr) = g.prop_mut(m) { + pr.value = v; + } + }; + if let Some(t) = &tracks_json { + set(TRACKS, Value::Str(t.clone())); + set(TRACKS_KEY, Value::Str(key.clone())); + } + set(SOLVE, Value::Str(solve_json.clone())); + set(SOLVE_KEY, Value::Str(skey.clone())); + set(METHOD_USED, Value::Str(method.clone())); + set(AVERAGE_ERROR, Value::Scalar(avg)); + Ok(()) + }); + let (msg, failed) = match r { + Ok(()) => (msg, failed), + Err(e) => (format!("3D Camera Tracker: {e}"), true), + }; + self.events.push(Event::Toast { message: msg, error: failed }); + } + _ => { + let msg = match &st.error { + Some(e) => format!("3D Camera Tracker analysis failed: {e}"), + None => "3D Camera Tracker analysis cancelled".into(), + }; + self.events.push(Event::Toast { message: msg, error: st.error.is_some() }); + self.bump(); + } + } + true + } + + /// Frontends call this every frame: finishes analyses and (with `auto`) starts queued + /// (re-)analyses in the background, like After Effects. + pub fn poll_camera(&mut self, auto: bool) -> bool { + let changed = self.poll_camera_job(); + if auto && self.camera_job.is_none() { + while let Some((c, l, e)) = self.camera_pending.first().copied() { + self.camera_pending.remove(0); + if self.start_camera(c, l, e, false).is_ok() { + return true; + } + } + } + changed + } +} + +// ---------------------------------------------------------------- solve → composition + +/// A tracker's solve placed in its composition. +pub struct Placed { + pub solve: Arc, + /// Canonical → comp world. + pub world: Similarity, + /// Comp pixels per layer pixel. + pub scale: f64, + /// Comp times of the analysed frames. + pub times: Vec, + pub comp_size: [f64; 2], +} + +impl Placed { + /// Comp-world position of a canonical point. + pub fn to_world(&self, x: V3) -> V3 { + self.world.apply(x) + } + /// Canonical position of a comp-world point. + pub fn from_world(&self, x: V3) -> V3 { + let r = &self.world; + linalg::scale(linalg::mtv(&r.r, linalg::sub(x, r.t)), 1.0 / r.s) + } + /// The 3D Tracker Camera on frame `k`: (position, orientation degrees, zoom). + pub fn camera(&self, k: usize) -> (V3, V3, f64) { + let c = &self.solve.frames[k]; + let m = linalg::mm(&self.world.r, &linalg::mt(&c.r())); + (self.to_world(c.center), linalg::euler_xyz(&m), c.focal * self.scale) + } + /// Analysed frame at comp time `t`. + pub fn frame_at(&self, t: Tick) -> usize { + self.times.iter().enumerate().min_by_key(|(_, x)| (x.0 - t.0).abs()).map(|(i, _)| i).unwrap_or(0).min(self.solve.frames.len().saturating_sub(1)) + } +} + +/// Comp pixels per layer pixel of `layer` (its 2D transform at its In point). +fn layer_scale(project: &Project, cid: ItemId, layer: &Layer) -> f64 { + let Some(comp) = project.comp(cid) else { return 1.0 }; + let ctx = EvalCtx::new(project, cid, comp, layer.in_point.max(Tick::ZERO)); + let m = ctx.world_matrix(layer).0; + let s = m[0][0].hypot(m[1][0]); + if s.is_finite() && s > 1e-9 { s } else { 1.0 } +} + +/// The solve of tracker `uid` on `lid`, placed in the comp. +pub fn placed(project: &Project, cid: ItemId, lid: LayerId, uid: Uid) -> Option { + let comp = project.comp(cid)?; + let layer = comp.layer(lid)?; + let g = layer.props.find_group(uid).filter(|g| is_tracker(g))?; + let solve = ct::solve(&static_params(g))?; + let scale = layer_scale(project, cid, layer); + let k0 = solve.first_solved()?; + let comp_size = [comp.width as f64, comp.height as f64]; + let world = solve.world(comp_size, solve.frames[k0].focal * scale); + let mut times = analysis_times(comp, layer); + times.truncate(solve.frames.len()); + Some(Placed { solve, world, scale, times, comp_size }) +} + +/// What to create from a target (`camera.createFromSolve`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CreateKind { + Text, + Solid, + Null, + ShadowCatcher, + /// Only the camera. + Camera, +} + +impl CreateKind { + pub fn from_name(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '-', '_'], "").as_str() { + "text" => Some(CreateKind::Text), + "solid" => Some(CreateKind::Solid), + "null" => Some(CreateKind::Null), + "shadowcatcher" | "shadow" => Some(CreateKind::ShadowCatcher), + "camera" => Some(CreateKind::Camera), + _ => None, + } + } +} + +/// Rotation (layer → world) of a layer lying on a plane with unit `normal` (towards the camera), +/// its x axis along `right` projected on the plane: the layer's front (−z) faces the camera. +pub fn plane_orientation(normal: V3, right: V3) -> V3 { + let z = linalg::scale(normal, -1.0); + let mut x = linalg::sub(right, linalg::scale(z, linalg::dot(right, z))); + if linalg::norm(x) < 1e-9 { + x = if z[0].abs() < 0.9 { [1.0, 0.0, 0.0] } else { [0.0, 1.0, 0.0] }; + x = linalg::sub(x, linalg::scale(z, linalg::dot(x, z))); + } + let x = linalg::normalize(x); + let y = linalg::cross(z, x); + let m = linalg::from_cols(x, y, z); + norm_angles(linalg::euler_xyz(&m)) +} + +fn norm_angles(a: V3) -> V3 { + a.map(|v| { + let r = v.rem_euclid(360.0); + if (r - 360.0).abs() < 1e-9 { 0.0 } else { r } + }) +} + +/// Key `path` on `l` at every `(time, value)`. +pub(crate) fn key_all(l: &mut Layer, path: &str, keys: &[(Tick, Value)]) { + if let Some(pr) = l.props.prop_mut(path) { + pr.keys = keys.iter().map(|(t, v)| Keyframe::new(*t, v.clone())).collect(); + if let Some((_, v)) = keys.first() { + pr.value = v.clone(); + } + } +} + +/// Write the solved camera path into camera layer `l` (one-node, keyed on every analysed frame; +/// Zoom keyed for Variable Zoom). +pub(crate) fn write_camera(l: &mut Layer, pl: &Placed) { + l.auto_orient = AutoOrient::Off; + let n = pl.times.len().min(pl.solve.frames.len()); + let mut pos = Vec::with_capacity(n); + let mut ori = Vec::with_capacity(n); + let mut zoom = Vec::with_capacity(n); + let mut prev: Option = None; + for k in 0..n { + let (p, o, z) = pl.camera(k); + // Unwrap the angles so per-frame keys interpolate the short way. + let o = match prev { + None => norm_angles(o), + Some(q) => [0, 1, 2].map(|i| q[i] + (o[i] - q[i] + 180.0).rem_euclid(360.0) - 180.0), + }; + prev = Some(o); + let t = l.layer_time(pl.times[k]); + pos.push((t, Value::Vec3(p))); + ori.push((t, Value::Vec3(o))); + zoom.push((t, Value::Scalar(z))); + } + key_all(l, "transform/position", &pos); + key_all(l, "transform/orientation", &ori); + let variable = pl.solve.frames.iter().any(|f| (f.focal - pl.solve.frames[0].focal).abs() > 1e-6 * f.focal); + if variable { + key_all(l, "cameraOptions/zoom", &zoom); + } else if let Some((_, z)) = zoom.first() + && let Some(pr) = l.props.prop_mut("cameraOptions/zoom") + { + pr.keys.clear(); + pr.value = z.clone(); + } + if let Some((_, z)) = zoom.first() + && let Some(pr) = l.props.prop_mut("cameraOptions/focusDistance") + { + pr.value = z.clone(); + } + // A one-node camera has no Point of Interest. + if let Some(tr) = l.props.sub_mut("transform") { + tr.children.retain(|c| !matches!(c, effectcraft_project::Node::Prop(p) if p.match_id == "poi")); + } + for rot in ["transform/rotationX", "transform/rotationY", "transform/rotation"] { + if let Some(pr) = l.props.prop_mut(rot) { + pr.keys.clear(); + pr.value = Value::Scalar(0.0); + } + } +} + +/// The target for `ids` on frame `k`: when fewer than three points are given, the nearest +/// solved neighbours (in 3D) complete the plane. +pub fn target_for(solve: &CameraSolve, ids: &[u32], k: usize) -> Option { + let mut use_ids: Vec = ids.iter().copied().filter(|i| solve.point(*i).is_some()).collect(); + if use_ids.is_empty() { + return None; + } + if use_ids.len() < 3 { + let c = solve.point(use_ids[0])?.pos; + let mut others: Vec<(f64, u32)> = solve.visible(k).filter(|p| !use_ids.contains(&p.id)).map(|p| (linalg::norm(linalg::sub(p.pos, c)), p.id)).collect(); + others.sort_by(|a, b| a.0.total_cmp(&b.0)); + for (_, id) in others.into_iter().take(6usize.saturating_sub(use_ids.len())) { + use_ids.push(id); + } + let mut t = solve.target(&use_ids, k)?; + // The target sits on the chosen point(s). + let pts: Vec = ids.iter().filter_map(|i| solve.point(*i)).map(|p| p.pos).collect(); + t.center = linalg::scale(pts.iter().fold([0.0; 3], |a, b| linalg::add(a, *b)), 1.0 / pts.len() as f64); + return Some(t); + } + solve.target(&use_ids, k) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plane_orientation_faces_the_normal() { + let n = linalg::normalize([0.2, -0.9, -0.3]); + let o = plane_orientation(n, [1.0, 0.0, 0.0]); + let m = linalg::from_euler_xyz(o); + // Layer −z (its front) is the normal. + let front = linalg::mv(&m, [0.0, 0.0, -1.0]); + assert!(linalg::norm(linalg::sub(front, n)) < 1e-9); + } +} diff --git a/crates/engine/src/commands/camera_cmds.rs b/crates/engine/src/commands/camera_cmds.rs new file mode 100644 index 00000000..5d454aed --- /dev/null +++ b/crates/engine/src/commands/camera_cmds.rs @@ -0,0 +1,574 @@ +//! 3D Camera Tracker commands: Animation ▸ Track Camera (`track.camera`), the effect's Analyze / +//! Cancel / Create Camera buttons, the viewer's track point selection and right-click menu +//! (Set Ground Plane and Origin, Create Text / Solid / Null / Shadow Catcher … and Camera, +//! Create Multiple …, Delete Selected Points) and the status / points queries. + +use effectcraft_color::Label; +use effectcraft_effects::camera_tracker::{self as ct, CAMERA, DELETED, SOLVE}; +use effectcraft_keyframe::{Justify, TextDoc, Value as KV}; +use effectcraft_project::build; +use effectcraft_project::{ItemId, ItemKind, Layer, LayerId, LayerSource, LightKind, Project, Solid, Uid}; +use effectcraft_track::camtrack::linalg::{self, V3}; +use effectcraft_track::camtrack::{ShotType, SolveMethod, Target, point_color}; +use serde_json::{Value, json}; + +use super::{CommandSpec, b_p, bad, f_p, has_comp, layer_p, str_p}; +use crate::camera_track::{CreateKind, Placed, is_tracker, placed, plane_orientation, static_params, target_for, write_camera}; +use crate::{EngineError, Result, Session, cmd, query}; + +pub fn specs() -> Vec { + vec![ + cmd!( + "track.camera", + "Track Camera", + ["Animation"], + None, + "{layer?, shotType?: fixed|variable|specify, aov?: degrees, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}", + can_track, + track_camera + ), + cmd!("camera.analyze", "Analyze", [], None, "{layer?, effect?: uid|name|index, wait?: block until done}", has_comp, analyze), + cmd!("camera.cancel", "Cancel", [], None, "{}", is_analyzing, |s, _| Ok(json!({"stopped": s.stop_camera()}))), + query!("camera.solveStatus", "3D Camera Tracker Status", "{layer?, effect?}", status), + query!("camera.points", "3D Camera Tracker Points", "{layer?, effect?, time?: comp seconds} → visible solved points", points), + cmd!("camera.selectPoints", "Select Track Points", [], None, "{points: [id], add?, toggle?}", has_comp, select_points), + cmd!( + "camera.createFromSolve", + "Create from Camera Solve", + [], + None, + "{kind: text|solid|null|shadowCatcher|camera, points?: [id] (default: selected), target?: {center, normal, size?} (comp world), multiple?, layer?, effect?}", + has_comp, + create_from_solve + ), + cmd!("camera.create", "Create Camera", [], None, "{layer?, effect?}", has_comp, |s, p| create_from_solve(s, &with_kind(p, "camera"))), + cmd!("camera.setGroundPlane", "Set Ground Plane and Origin", [], None, "{points?: [id] (default: selected), layer?, effect?}", has_comp, set_ground), + cmd!("camera.deletePoints", "Delete Selected Points", [], None, "{points?: [id] (default: selected), layer?, effect?, wait?}", has_comp, delete_points), + ] +} + +fn with_kind(p: &Value, kind: &str) -> Value { + let mut q = p.clone(); + if !q.is_object() { + q = json!({}); + } + q["kind"] = json!(kind); + q +} + +fn can_track(s: &Session) -> std::result::Result<(), String> { + has_comp(s)?; + let c = s.active_comp().ok_or("no composition is open")?; + match s.state.selected_layers.first().and_then(|l| c.layer(*l)) { + Some(l) if l.has_video() && !l.switches.adjustment && !l.is_camera() && !l.is_light() => Ok(()), + Some(_) => Err("Track Camera needs a footage or composition layer".into()), + None => Err("select a layer to track".into()), + } +} + +fn is_analyzing(s: &Session) -> std::result::Result<(), String> { + if s.is_camera_analyzing() { Ok(()) } else { Err("no 3D Camera Tracker analysis is running".into()) } +} + +/// (comp, layer, effect uid) of a 3D Camera Tracker: `layer` / `effect` (uid, name or 1-based +/// index), else a selected one, else the first in the comp. +fn ct_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> { + let cid = super::comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let candidates: Vec = match p.get("layer") { + Some(v) => vec![super::resolve_layer(comp, v).ok_or_else(|| bad(cmd, format!("no layer {v}")))?], + None => { + let mut v: Vec = s.state.selected_props.iter().map(|(l, _)| *l).collect(); + v.extend(s.state.selected_layers.iter().copied()); + v.extend(comp.layers.iter().map(|l| l.id)); + v + } + }; + for lid in candidates { + let Some(fx) = comp.layer(lid).and_then(Layer::effects) else { continue }; + let g = match p.get("effect") { + Some(Value::Number(n)) => { + let n = n.as_u64().unwrap_or(0); + fx.groups().find(|g| g.uid == n).or_else(|| fx.groups().nth((n as usize).saturating_sub(1))) + } + Some(Value::String(name)) => fx.groups().find(|g| &g.name == name), + _ => s + .state + .selected_props + .iter() + .find_map(|(l, u)| (*l == lid).then(|| fx.groups().find(|g| g.uid == *u && is_tracker(g))).flatten()) + .or_else(|| fx.groups().find(|g| is_tracker(g))), + }; + if let Some(g) = g.filter(|g| is_tracker(g)) { + return Ok((cid, lid, g.uid)); + } + if p.get("layer").is_some() { + break; + } + } + Err(bad(cmd, "no 3D Camera Tracker found (apply Animation ▸ Track Camera first)")) +} + +fn shot_p(p: &Value) -> Result> { + match p.get("shotType") { + None => Ok(None), + Some(Value::Number(n)) => ShotType::ALL.get(n.as_u64().unwrap_or(99) as usize).copied().map(Some).ok_or_else(|| bad("track.camera", "shotType: 0..2")), + Some(Value::String(v)) => ShotType::from_name(v).map(Some).ok_or_else(|| bad("track.camera", "shotType: fixed|variable|specify")), + Some(_) => Err(bad("track.camera", "shotType: fixed|variable|specify")), + } +} + +fn method_p(p: &Value) -> Result> { + match p.get("solveMethod") { + None => Ok(None), + Some(Value::Number(n)) => { + SolveMethod::ALL.get(n.as_u64().unwrap_or(99) as usize).copied().map(Some).ok_or_else(|| bad("track.camera", "solveMethod: 0..3")) + } + Some(Value::String(v)) => SolveMethod::from_name(v).map(Some).ok_or_else(|| bad("track.camera", "solveMethod: auto|typical|flat|tripod")), + Some(_) => Err(bad("track.camera", "solveMethod: auto|typical|flat|tripod")), + } +} + +/// Animation ▸ Track Camera / the Tracker panel button: apply the 3D Camera Tracker to the layer +/// (reusing one it already has), set the given options and start the analysis. +fn track_camera(s: &mut Session, p: &Value) -> Result { + let (cid, lid) = layer_p(s, p, "track.camera")?; + let shot = shot_p(p)?; + let method = method_p(p)?; + let aov = f_p(p, "aov").or_else(|| f_p(p, "angleOfView")); + if aov.is_some_and(|a| !(1.0..=170.0).contains(&a)) { + return Err(bad("track.camera", "aov: 1..170 degrees")); + } + let detailed = b_p(p, "detailed"); + let existing = s.project.comp(cid).and_then(|c| c.layer(lid)).and_then(Layer::effects).and_then(|fx| fx.groups().find(|g| is_tracker(g)).map(|g| g.uid)); + let uid = match existing { + Some(u) => u, + None => { + let r = s.execute("effect.apply", json!({"effect": ct::ID, "layers": [lid.0], "comp": cid.0}))?; + r["effects"].get(0).and_then(Value::as_u64).ok_or_else(|| EngineError::Other("the effect could not be applied".into()))? + } + }; + if shot.is_some() || method.is_some() || aov.is_some() || detailed.is_some() { + s.edit("3D Camera Tracker Settings", None, |proj, _| { + let g = proj + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?; + let mut set = |m: &str, v: KV| { + if let Some(pr) = g.prop_mut(m) { + pr.keys.clear(); + pr.value = v; + } + }; + if let Some(sh) = shot { + set("shotType", KV::Enum(ShotType::ALL.iter().position(|x| *x == sh).unwrap_or(0) as u32)); + } + if let Some(a) = aov { + set("horizontalAngleOfView", KV::Scalar(a)); + if shot.is_none() { + set("shotType", KV::Enum(2)); + } + } + if let Some(m) = method { + set("advanced/solveMethod", KV::Enum(SolveMethod::ALL.iter().position(|x| *x == m).unwrap_or(0) as u32)); + } + if let Some(d) = detailed { + set("advanced/detailedAnalysis", KV::Bool(d)); + } + Ok(()) + })?; + } + // Select the effect so the viewer shows its track points. + s.state.selected_props = vec![(lid, uid)]; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?; + let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_camera_analyzing(), "progress": s.camera_progress()}); + if wait { + out["status"] = status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?; + } + Ok(out) +} + +fn analyze(s: &mut Session, p: &Value) -> Result { + let (cid, lid, uid) = ct_p(s, p, "camera.analyze")?; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?; + let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_camera_analyzing(), "progress": s.camera_progress()}); + if wait { + out["status"] = status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?; + } + Ok(out) +} + +fn status(s: &mut Session, p: &Value) -> Result { + s.poll_camera(false); + let mut out = json!({"running": s.is_camera_analyzing(), "progress": s.camera_progress(), "banner": s.camera_progress().map(|p| p.banner())}); + let Ok((cid, lid, uid)) = ct_p(s, p, "camera.solveStatus") else { return Ok(out) }; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let g = layer.props.find_group(uid).ok_or(EngineError::NoComp)?; + let params = static_params(g); + out["layer"] = json!(lid.0); + out["effect"] = json!(uid); + out["pending"] = json!(s.camera_pending.contains(&(cid, lid, uid))); + let tracks = ct::tracks(¶ms); + out["analyzed"] = json!(tracks.is_some()); + if let Some(t) = &tracks { + out["tracks"] = json!(t.tracks.len()); + out["frames"] = json!(t.frames); + } + let settings = ct::settings(¶ms); + out["shotType"] = json!(settings.shot.label()); + out["solveMethod"] = json!(settings.method.label()); + out["deleted"] = json!(settings.deleted.len()); + let solve = ct::solve(¶ms); + out["solved"] = json!(solve.is_some()); + if let Some(sv) = &solve { + let f = sv.focal(); + out["methodUsed"] = json!(sv.method_used.label()); + out["averageError"] = json!(sv.average_error); + out["points"] = json!(sv.points.len()); + out["solvedFrames"] = json!(sv.frames.iter().filter(|c| c.solved).count()); + out["focalLength"] = json!(f); + out["horizontalAngleOfView"] = json!(sv.hfov(f)); + out["groundPlane"] = json!(sv.ground.is_some()); + if let Some(pl) = placed(&s.project, cid, lid, uid) { + let k = pl.frame_at(s.time_of(cid)); + let (pos, ori, zoom) = pl.camera(k); + out["frame"] = json!(k); + out["camera"] = json!({"position": pos, "orientation": ori, "zoom": zoom}); + out["visible"] = json!(sv.visible(k).count()); + } + } + let cam = params.get(CAMERA).map(|v| v.as_f64() as u64).unwrap_or(0); + out["cameraLayer"] = if cam > 0 && comp.layer(LayerId(cam)).is_some_and(Layer::is_camera) { json!(cam) } else { Value::Null }; + out["selected"] = json!(s.state.camera_points); + Ok(out) +} + +/// The visible solved points at the current (or given) time, with their comp positions. +fn points(s: &mut Session, p: &Value) -> Result { + let (cid, lid, uid) = ct_p(s, p, "camera.points")?; + let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad("camera.points", "the camera is not solved yet"))?; + let t = f_p(p, "time").map(effectcraft_time::Tick::from_seconds_f64).unwrap_or_else(|| s.time_of(cid)); + let k = pl.frame_at(t); + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let ctx = effectcraft_render::EvalCtx::new(&s.project, cid, comp, t); + let m = ctx.world_matrix(layer); + let list: Vec = ct::projected(&pl.solve, k) + .into_iter() + .map(|(id, q, z)| { + let c = m.apply(effectcraft_geom::vec3(q[0], q[1], 0.0)); + let sp = pl.solve.point(id).copied().unwrap_or_default(); + let col = point_color(id); + json!({"id": id, "comp": [c.x, c.y], "layer": q, "depth": z, "world": pl.to_world(sp.pos), "error": sp.error, "color": col, "selected": s.state.camera_points.contains(&id)}) + }) + .collect(); + Ok(json!({"frame": k, "points": list})) +} + +fn ids_p(s: &Session, p: &Value) -> Vec { + match p.get("points") { + Some(Value::Array(a)) => a.iter().filter_map(Value::as_u64).map(|v| v as u32).collect(), + Some(Value::Number(n)) => n.as_u64().map(|v| vec![v as u32]).unwrap_or_default(), + _ => s.state.camera_points.clone(), + } +} + +fn select_points(s: &mut Session, p: &Value) -> Result { + let ids: Vec = match p.get("points") { + Some(Value::Array(a)) => a.iter().filter_map(Value::as_u64).map(|v| v as u32).collect(), + _ => vec![], + }; + if b_p(p, "toggle").unwrap_or(false) { + for id in ids { + match s.state.camera_points.iter().position(|x| *x == id) { + Some(i) => { + s.state.camera_points.remove(i); + } + None => s.state.camera_points.push(id), + } + } + } else if b_p(p, "add").unwrap_or(false) { + for id in ids { + if !s.state.camera_points.contains(&id) { + s.state.camera_points.push(id); + } + } + } else { + s.state.camera_points = ids; + } + s.bump(); + Ok(json!({"selected": s.state.camera_points})) +} + +/// The target of a create / ground command in canonical solve coordinates. +fn target_p(s: &Session, p: &Value, pl: &Placed, k: usize, cmd: &str) -> Result { + if let Some(t) = p.get("target") { + let v3 = |k: &str| -> Option { + let a = t.get(k)?.as_array()?; + Some([a.first()?.as_f64()?, a.get(1)?.as_f64()?, a.get(2).and_then(Value::as_f64).unwrap_or(0.0)]) + }; + let (c, n) = (v3("center").ok_or_else(|| bad(cmd, "target.center: [x, y, z]"))?, v3("normal").unwrap_or([0.0, -1.0, 0.0])); + let size = t.get("size").and_then(Value::as_f64).unwrap_or(100.0); + let nc = linalg::normalize(linalg::mtv(&pl.world.r, n)); + return Ok(Target { center: pl.from_world(c), normal: nc, size: size / pl.world.s }); + } + let ids = ids_p(s, p); + if ids.is_empty() { + return Err(bad(cmd, "select track points (or give `points` / `target`)")); + } + target_for(&pl.solve, &ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane")) +} + +fn next_name(c: &effectcraft_project::Comp, base: &str) -> String { + let n = (1..).find(|i| !c.layers.iter().any(|l| l.name == format!("{base} {i}"))).unwrap_or(1); + format!("{base} {n}") +} + +fn place(l: &mut Layer, pos: V3, ori: V3, src: &Layer) { + l.switches.three_d = true; + l.in_point = src.in_point; + l.out_point = src.out_point; + if let Some(pr) = l.props.prop_mut("transform/position") { + pr.value = KV::Vec3(pos); + } + if let Some(pr) = l.props.prop_mut("transform/orientation") { + pr.value = KV::Vec3(ori); + } +} + +/// Insert `l` above the tracked layer. +fn insert_above(proj: &mut Project, cid: ItemId, above: LayerId, l: Layer) -> Result { + let comp = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let at = comp.layers.iter().position(|x| x.id == above).unwrap_or(0); + let id = l.id; + comp.layers.insert(at, l); + Ok(id) +} + +fn add_solid(proj: &mut Project, comp: &effectcraft_project::Comp, name: &str, size: u32, color: [f32; 3]) -> Layer { + let folder = proj.folder_named("Solids").unwrap_or_else(|| proj.add_item("Solids", Label::Yellow, None, ItemKind::Folder)); + let sid = proj.add_item(name, Label::Red, Some(folder), ItemKind::Solid(Solid { color, width: size, height: size, pixel_aspect: 1.0 })); + build::layer(proj, comp, name, LayerSource::Solid { item: sid }, (size, size), None) +} + +/// Create Text / Solid / Null / Shadow Catcher (and Light) on the target, plus the 3D Tracker +/// Camera when the tracker has none yet — one undo step. +fn create_from_solve(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.createFromSolve"; + let (cid, lid, uid) = ct_p(s, p, cmd)?; + let kind = CreateKind::from_name(str_p(p, "kind").unwrap_or("null")).ok_or_else(|| bad(cmd, "kind: text|solid|null|shadowCatcher|camera"))?; + let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad(cmd, "the camera is not solved yet (analysis still running or failed)"))?; + let k = pl.frame_at(s.time_of(cid)); + let multiple = b_p(p, "multiple").unwrap_or(false); + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let src = comp.layer(lid).ok_or(EngineError::NoComp)?.clone(); + let g = src.props.find_group(uid).ok_or(EngineError::NoComp)?; + let params = static_params(g); + let size_pct = effectcraft_effects::warp_stab::param(¶ms, "targetSize").map(|v| v.as_f64()).unwrap_or(100.0) / 100.0; + let existing_cam = params.get(CAMERA).map(|v| v.as_f64() as u64).filter(|id| *id > 0 && comp.layer(LayerId(*id)).is_some_and(Layer::is_camera)); + // Targets: one, or one per selected point (Create Multiple …) sharing the selection's plane. + let targets: Vec = if kind == CreateKind::Camera { + vec![] + } else { + let t = target_p(s, p, &pl, k, cmd)?; + if multiple { + let ids = ids_p(s, p); + ids.iter().filter_map(|i| pl.solve.point(*i)).map(|sp| Target { center: sp.pos, ..t }).collect() + } else { + vec![t] + } + }; + // Camera right vector on this frame (world), to align created layers with the view. + let cam = &pl.solve.frames[k]; + let right = linalg::mv(&pl.world.r, linalg::mtv(&cam.r(), [1.0, 0.0, 0.0])); + let label = match (kind, multiple) { + (CreateKind::Text, false) => "Create Text and Camera", + (CreateKind::Solid, false) => "Create Solid and Camera", + (CreateKind::Null, false) => "Create Null and Camera", + (CreateKind::ShadowCatcher, _) => "Create Shadow Catcher, Camera and Light", + (CreateKind::Text, true) => "Create Multiple Text Layers and Camera", + (CreateKind::Solid, true) => "Create Multiple Solids and Camera", + (CreateKind::Null, true) => "Create Multiple Nulls and Camera", + (CreateKind::Camera, _) => "Create Camera", + }; + let (camera, created) = s.edit(label, None, |proj, st| { + let mut created = vec![]; + // The camera: reuse the tracker's, else create "3D Tracker Camera" at the top. + let camera = match existing_cam { + Some(id) if kind != CreateKind::Camera => LayerId(id), + Some(id) => { + let l = proj.comp_mut(cid).and_then(|c| c.layer_mut(LayerId(id))).ok_or(EngineError::NoComp)?; + write_camera(l, &pl); + LayerId(id) + } + None => { + let mut l = build::layer(proj, &comp, "3D Tracker Camera", LayerSource::Camera, (comp.width, comp.height), None); + l.in_point = src.in_point; + l.out_point = src.out_point; + write_camera(&mut l, &pl); + let id = l.id; + proj.comp_mut(cid).ok_or(EngineError::NoComp)?.layers.insert(0, l); + if let Some(pr) = proj.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)).and_then(|g| g.prop_mut(CAMERA)) { + pr.value = KV::Scalar(id.0 as f64); + } + id + } + }; + for t in &targets { + let center = pl.to_world(t.center); + let normal = linalg::normalize(linalg::mv(&pl.world.r, t.normal)); + let ori = plane_orientation(normal, right); + let size = (t.size * pl.world.s * size_pct).max(1.0); + let cur = proj.comp(cid).ok_or(EngineError::NoComp)?.clone(); + match kind { + CreateKind::Text => { + let mut l = build::layer(proj, &cur, "Text", LayerSource::Text, (comp.width, comp.height), None); + let doc = TextDoc { text: "Text".into(), justify: Justify::Center, size: (size * 0.35).clamp(4.0, 2000.0), ..Default::default() }; + if let Some(pr) = l.props.prop_mut("text/sourceText") { + pr.value = KV::Text(Box::new(doc)); + } + place(&mut l, center, ori, &src); + created.push(insert_above(proj, cid, lid, l)?); + } + CreateKind::Solid => { + let name = next_name(&cur, "Track Solid"); + let c = point_color((created.len() as u32).wrapping_add(7)); + let mut l = add_solid(proj, &cur, &name, size.round().max(1.0) as u32, c); + place(&mut l, center, ori, &src); + created.push(insert_above(proj, cid, lid, l)?); + } + CreateKind::Null => { + let name = next_name(&cur, "Track Null"); + let mut l = build::layer(proj, &cur, &name, LayerSource::Null, (comp.width, comp.height), None); + place(&mut l, center, ori, &src); + if let Some(pr) = l.props.prop_mut("transform/scale") { + pr.value = KV::Vec3([size, size, size]); + } + created.push(insert_above(proj, cid, lid, l)?); + } + CreateKind::ShadowCatcher => { + let name = next_name(&cur, "Shadow Catcher"); + let mut l = add_solid(proj, &cur, &name, (size * 4.0).round().max(1.0) as u32, [1.0, 1.0, 1.0]); + place(&mut l, center, ori, &src); + if let Some(pr) = l.props.prop_mut("materialOptions/acceptsShadows") { + pr.value = KV::Enum(2); + } + if let Some(pr) = l.props.prop_mut("materialOptions/castsShadows") { + pr.value = KV::Enum(0); + } + let sc = created.len(); + created.push(insert_above(proj, cid, lid, l)?); + let cur = proj.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let lname = next_name(&cur, "Light"); + let mut light = build::layer(proj, &cur, &lname, LayerSource::Light { kind: LightKind::Point }, (comp.width, comp.height), None); + light.in_point = src.in_point; + light.out_point = src.out_point; + let lp = linalg::add(center, linalg::add(linalg::scale(normal, size * 2.0), linalg::scale(right, size))); + if let Some(pr) = light.props.prop_mut("transform/position") { + pr.value = KV::Vec3(lp); + } + if let Some(pr) = light.props.prop_mut("lightOptions/castsShadows") { + pr.value = KV::Bool(true); + } + let _ = sc; + created.push(insert_above(proj, cid, created[created.len() - 1], light)?); + } + CreateKind::Camera => {} + } + } + st.selected_layers = if created.is_empty() { vec![camera] } else { created.clone() }; + st.selected_props.clear(); + Ok((camera, created)) + })?; + let t0 = targets.first().map( + |t| json!({"center": pl.to_world(t.center), "normal": linalg::normalize(linalg::mv(&pl.world.r, t.normal)), "size": t.size * pl.world.s * size_pct}), + ); + Ok(json!({"camera": camera.0, "layers": created.iter().map(|l| l.0).collect::>(), "target": t0})) +} + +fn set_ground(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.setGroundPlane"; + let (cid, lid, uid) = ct_p(s, p, cmd)?; + let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad(cmd, "the camera is not solved yet"))?; + let k = pl.frame_at(s.time_of(cid)); + let ids = ids_p(s, p); + if ids.is_empty() { + return Err(bad(cmd, "select track points (or give `points`)")); + } + let t = target_for(&pl.solve, &ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane"))?; + let mut sv = (*pl.solve).clone(); + let mut g = sv.ground_from(&ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane"))?; + g.origin = t.center; + sv.ground = Some(g); + let json = sv.to_json(); + s.edit("Set Ground Plane and Origin", None, |proj, _| { + let pr = proj + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .and_then(|g| g.prop_mut(SOLVE)) + .ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?; + pr.value = KV::Str(json); + Ok(()) + })?; + Ok(json!({"origin": g.origin, "normal": g.normal})) +} + +/// Delete points (and, with Auto-delete Points Across Time, the points of other tracks on the +/// same 3D feature), then re-solve. +fn delete_points(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.deletePoints"; + let (cid, lid, uid) = ct_p(s, p, cmd)?; + let ids = ids_p(s, p); + if ids.is_empty() { + return Err(bad(cmd, "select track points (or give `points`)")); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let g = comp.layer(lid).and_then(|l| l.props.find_group(uid)).ok_or(EngineError::NoComp)?; + let params = static_params(g); + let mut del = ct::deleted(¶ms); + let auto = effectcraft_effects::warp_stab::param(¶ms, "advanced/autoDeletePoints").is_some_and(KV::as_bool); + let mut add = ids.clone(); + if auto && let Some(sv) = ct::solve(¶ms) { + // Same feature tracked again at other times: solved within 1 % of its depth. + let cam0 = sv.first_solved().map(|k| sv.frames[k].center).unwrap_or([0.0; 3]); + for id in &ids { + let Some(pt) = sv.point(*id) else { continue }; + let r = linalg::norm(linalg::sub(pt.pos, cam0)) * 0.01; + for q in &sv.points { + let disjoint = q.last < pt.first || q.first > pt.last; + if q.id != *id && disjoint && linalg::norm(linalg::sub(q.pos, pt.pos)) < r { + add.push(q.id); + } + } + } + } + for id in add { + if !del.contains(&id) { + del.push(id); + } + } + del.sort_unstable(); + let json = serde_json::to_string(&del).unwrap_or_default(); + s.edit("Delete Track Points", None, |proj, _| { + let pr = proj + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .and_then(|g| g.prop_mut(DELETED)) + .ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?; + pr.value = KV::Str(json); + Ok(()) + })?; + s.state.camera_points.retain(|x| !del.contains(x)); + // Re-solve now (After Effects re-solves after deleting points). + let wait = b_p(p, "wait").unwrap_or(false); + if !s.is_camera_analyzing() { + s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?; + } + Ok(json!({"deleted": del, "running": s.is_camera_analyzing()})) +} diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 4537752f..04034281 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -4,6 +4,7 @@ mod anim; pub mod anim_tools; mod animation; +mod camera_cmds; mod comp; mod comp_more; mod edit; @@ -129,6 +130,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(track::specs()); v.extend(mask_interp::specs()); v.extend(warp_cmds::specs()); + v.extend(camera_cmds::specs()); v.extend(paint::specs()); v.extend(puppet::specs()); v.extend(settings::specs()); diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index f670ffd7..74de3aec 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -59,7 +59,6 @@ pub fn specs() -> Vec { stub!("keys.audioToKeyframes", "Convert Audio to Keyframes", ["Animation", "Keyframe Assistant"], "{}"), stub!("keys.rpfCameraImport", "RPF Camera Import", ["Animation", "Keyframe Assistant"], "{}"), stub!("text.animatorFontAxes", "Variable Font Axes", ["Animation", "Animate Text"], "{}"), - stub!("track.camera", "Track Camera", ["Animation"], "{}"), // Workspaces and panels that don't exist yet. stub!("window.unavailablePanel", "Panel", [], "{panel}"), ] diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index ba34488a..fbd2b391 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -9,6 +9,7 @@ //! (compositions are `Arc`s, so untouched comps are shared). pub mod autosave; +pub mod camera_track; pub mod commands; pub mod config; pub mod demo; @@ -41,6 +42,7 @@ pub use effectcraft_project as project; pub use effectcraft_render as render; pub use effectcraft_text as text; pub use effectcraft_time as time; +pub use effectcraft_track as track; pub use render_queue::{ExportJob, ExportResult, Exporter, JobState}; #[derive(Debug, thiserror::Error)] @@ -170,6 +172,9 @@ pub struct EditorState { /// Tracker panel ▸ Method in mask mode (Track Mask). #[serde(default)] pub mask_track_method: mask_track::MaskMethod, + /// 3D Camera Tracker: the selected track points (ids) in the viewer. + #[serde(default)] + pub camera_points: Vec, /// Mask Interpolation panel options. #[serde(default)] pub mask_interp: commands::mask_interp::MaskInterpOptions, @@ -263,6 +268,10 @@ pub struct Session { pub warp_job: Option, /// Warp Stabilizers waiting for (re-)analysis: (comp, layer, effect uid). pub warp_pending: Vec<(ItemId, LayerId, Uid)>, + /// The running (or finished, not yet polled) 3D Camera Tracker analysis. + pub camera_job: Option, + /// 3D Camera Trackers waiting for (re-)analysis: (comp, layer, effect uid). + pub camera_pending: Vec<(ItemId, LayerId, Uid)>, pub events: Vec, /// Commands executed: (id, params). pub journal: Vec<(String, Value)>, @@ -308,6 +317,8 @@ impl Default for Session { mask_job: None, warp_job: None, warp_pending: vec![], + camera_job: None, + camera_pending: vec![], events: vec![], journal: vec![], layer_cache: Arc::new(LayerCache::default()), @@ -374,6 +385,12 @@ impl Session { self.warp_pending.push(w); } } + // So are 3D Camera Tracker tracks (and solves made with other settings). + for w in camera_track::invalidate(&before, &mut p) { + if !self.camera_pending.contains(&w) { + self.camera_pending.push(w); + } + } let same = merge.is_some() && merge.map(str::to_string) == self.history.merge_key; if !same { self.history.undo.push((label.to_string(), before)); @@ -551,6 +568,9 @@ impl Session { self.stop_warp(); self.warp_job = None; self.warp_pending.clear(); + self.stop_camera(); + self.camera_job = None; + self.camera_pending.clear(); p.fix_next_id(); self.project = Arc::new(p); self.history = History::default(); @@ -574,6 +594,8 @@ mod tests_3d; #[cfg(test)] mod tests_anim_tools; #[cfg(test)] +mod tests_camera_track; +#[cfg(test)] mod tests_effects; #[cfg(test)] mod tests_fidelity; diff --git a/crates/engine/src/tests_camera_track.rs b/crates/engine/src/tests_camera_track.rs new file mode 100644 index 00000000..5bd5539c --- /dev/null +++ b/crates/engine/src/tests_camera_track.rs @@ -0,0 +1,307 @@ +//! The 3D Camera Tracker end to end through the commands, on synthetic footage rendered in-test: +//! textured planes (a floor, a back wall and a box face) seen through a known moving camera. + +use effectcraft_effects::camera_tracker as ct; +use effectcraft_geom::vec3; +use effectcraft_project::LayerId; +use effectcraft_raster::Image; +use effectcraft_render::EvalCtx; +use effectcraft_render::three_d::camera::active_camera; +use effectcraft_track::camtrack::linalg::{self, M3, V3}; +use serde_json::{Value, json}; + +use crate::Session; +use crate::tests_track::{H, W, frame_time, setup, texture}; + +const F: f64 = 260.0; + +/// The true camera on frame `f`: (world → camera rotation, centre). +fn true_camera(f: u32) -> (M3, V3) { + let t = f as f64 / 24.0; + let c = [-1.0 + 2.0 * t, -0.3 + 0.2 * (t * 3.0).sin(), 0.5 * t]; + let fwd = linalg::normalize(linalg::sub([0.2 * t, 0.4, 8.0], c)); + let right = linalg::normalize(linalg::cross([0.0, 1.0, 0.0], fwd)); + ([right, linalg::cross(fwd, right), fwd], c) +} + +/// Planes: (point, normal, u axis, v axis, half extent or 0 = infinite). +const PLANES: [(V3, V3, V3, V3, f64); 3] = [ + ([0.0, 1.5, 0.0], [0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0], 0.0), + ([0.0, 0.0, 10.0], [0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0], 0.0), + ([-1.2, 0.2, 6.0], [0.3, 0.0, -1.0], [0.957_826, 0.0, 0.287_348], [0.0, 1.0, 0.0], 1.1), +]; + +fn scene_frame(f: u32) -> Image { + let (r, c) = true_camera(f); + let mut img = Image::new(W, H); + for y in 0..H { + for x in 0..W { + let d = linalg::mtv(&r, [(x as f64 + 0.5 - W as f64 / 2.0) / F, (y as f64 + 0.5 - H as f64 / 2.0) / F, 1.0]); + let mut best = (f64::INFINITY, 0.2f32); + for (i, (p0, n, ua, va, ext)) in PLANES.iter().enumerate() { + let den = linalg::dot(*n, d); + if den.abs() < 1e-9 { + continue; + } + let tt = linalg::dot(*n, linalg::sub(*p0, c)) / den; + if tt <= 0.1 || tt >= best.0 { + continue; + } + let hit = linalg::add(c, linalg::scale(d, tt)); + let (u, v) = (linalg::dot(linalg::sub(hit, *p0), *ua), linalg::dot(linalg::sub(hit, *p0), *va)); + if *ext > 0.0 && (u.abs() > *ext || v.abs() > *ext) { + continue; + } + best = (tt, texture(u * 40.0, v * 40.0, 11 + i as u32)); + } + img.set(x, y, [best.1, best.1, best.1, 1.0]); + } + } + img +} + +/// The comp with the clip tracked (blocking) with `params`. +fn tracked(params: Value) -> (Session, LayerId) { + let (mut s, clip, _) = setup(scene_frame); + s.execute("layer.select", json!({"layers": [clip.0]})).unwrap(); + assert!(s.is_enabled("track.camera")); + let mut p = params; + p["wait"] = json!(true); + let r = s.execute_checked("track.camera", p).unwrap(); + assert_eq!(r["frames"], 25, "{r}"); + assert!(s.camera_job.is_none()); + (s, clip) +} + +fn status(s: &mut Session) -> Value { + s.execute("camera.solveStatus", json!({})).unwrap() +} + +#[test] +fn track_camera_solves_and_creates_layers_on_the_plane() { + let (mut s, clip) = tracked(json!({})); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + let err = st["averageError"].as_f64().unwrap(); + assert!(err < 0.5, "average error {err}"); + let f = st["focalLength"].as_f64().unwrap(); + assert!((f - F).abs() / F < 0.05, "focal {f}"); + assert_ne!(st["methodUsed"], "Tripod Pan"); + // Method Used / Average Error are reported in the effect. + let g = s.active_comp().unwrap().layer(clip).unwrap().effects().unwrap().groups().next().unwrap().clone(); + assert!(matches!(g.prop(ct::METHOD_USED).map(|p| &p.value), Some(effectcraft_keyframe::Value::Str(m)) if !m.is_empty())); + + // Pick floor points on frame 12 (their 3D positions lie on y = 1.5 in the true scene: the + // ones seen below the horizon at the bottom of the frame). + s.execute("time.set", json!({"time": frame_time(12).seconds()})).unwrap(); + let pts = s.execute("camera.points", json!({})).unwrap(); + let list = pts["points"].as_array().unwrap(); + assert!(list.len() > 30, "{}", list.len()); + let floor: Vec = list.iter().filter(|p| p["comp"][1].as_f64().unwrap() > H as f64 * 0.8).map(|p| p["id"].as_u64().unwrap()).take(8).collect(); + assert!(floor.len() >= 3, "{floor:?}"); + s.execute("camera.selectPoints", json!({"points": floor})).unwrap(); + assert_eq!(s.state.camera_points.len(), floor.len()); + + // Create Solid and Camera: one undo step, camera + solid. + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + let cam = LayerId(r["camera"].as_u64().unwrap()); + let solid = LayerId(r["layers"][0].as_u64().unwrap()); + let comp = s.active_comp().unwrap().clone(); + assert!(comp.layer(cam).unwrap().is_camera()); + assert_eq!(comp.layer(cam).unwrap().name, "3D Tracker Camera"); + assert_eq!(comp.layers[0].id, cam); + let sl = comp.layer(solid).unwrap(); + assert!(sl.switches.three_d); + assert!(sl.name.starts_with("Track Solid")); + // The camera is keyed on every frame. + assert_eq!(comp.layer(cam).unwrap().props.prop("transform/position").unwrap().keys.len(), 25); + + // Every solved point projects through the created camera exactly where the solve puts it, + // and onto its tracked position (the footage fills the comp, so layer pixels are comp + // pixels): within 1 px for 95 % of the observations (the rest are the solve's outliers: + // features on occlusion edges), 0.5 px on average. + let cid = s.active_comp_id().unwrap(); + let uid = g.uid; + let pl = crate::camera_track::placed(&s.project, cid, clip, uid).unwrap(); + let tracks = ct::tracks(&crate::camera_track::static_params(&g)).unwrap(); + let mut errs = vec![]; + for k in [0u32, 6, 12, 18, 24] { + let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(k)); + let camst = active_camera(&ctx); + for sp in pl.solve.visible(k as usize) { + let Some(obs) = tracks.track(sp.id).and_then(|t| t.at(k)) else { continue }; + let w = pl.to_world(sp.pos); + let q = camst.project(W as f64, H as f64, vec3(w[0], w[1], w[2])).unwrap(); + let own = pl.solve.frames[k as usize].project(pl.solve.size, sp.pos).unwrap(); + assert!((q.x - own[0]).hypot(q.y - own[1]) < 0.01, "camera layer disagrees with the solve"); + errs.push((q.x - obs[0]).hypot(q.y - obs[1])); + } + } + assert!(errs.len() > 100, "{}", errs.len()); + errs.sort_by(f64::total_cmp); + let p95 = errs[errs.len() * 95 / 100]; + let mean = errs.iter().sum::() / errs.len() as f64; + assert!(p95 < 1.0 && mean < 0.5, "p95 {p95} px, mean {mean} px"); + + // The solid lies on the selected points' plane. + let tr = sl.props.sub("transform").unwrap(); + let pos = tr.get("position").unwrap().value.as_vec3(); + let ori = tr.get("orientation").unwrap().value.as_vec3(); + let normal = linalg::mv(&linalg::from_euler_xyz(ori), [0.0, 0.0, -1.0]); + let size = r["target"]["size"].as_f64().unwrap(); + for id in &s.state.camera_points { + let w = pl.to_world(pl.solve.point(*id).unwrap().pos); + let d = linalg::dot(linalg::sub(w, pos), normal).abs(); + assert!(d < 0.05 * size.max(1.0) + 1.0, "point {id} is {d} px off the plane"); + } + // The solid faces the camera. + let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(12)); + let eye = active_camera(&ctx).eye; + assert!(linalg::dot(normal, linalg::sub([eye.x, eye.y, eye.z], pos)) > 0.0); + + // Undo removes both layers; redo brings them back. + let before = comp.layers.len(); + assert!(s.undo()); + assert_eq!(s.active_comp().unwrap().layers.len(), before - 2); + assert!(s.redo()); + assert_eq!(s.active_comp().unwrap().layers.len(), before); + + // A second create reuses the camera: Create Multiple Nulls and Camera. + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "null", "multiple": true})).unwrap(); + assert_eq!(r["camera"].as_u64().unwrap(), cam.0); + assert_eq!(r["layers"].as_array().unwrap().len(), floor.len()); + // Text and Shadow Catcher (+ light). + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "text"})).unwrap(); + assert_eq!(r["layers"].as_array().unwrap().len(), 1); + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "shadowCatcher"})).unwrap(); + let ids: Vec = r["layers"].as_array().unwrap().iter().map(|v| LayerId(v.as_u64().unwrap())).collect(); + let comp = s.active_comp().unwrap(); + assert!(comp.layer(ids[0]).unwrap().name.starts_with("Shadow Catcher")); + assert_eq!(comp.layer(ids[0]).unwrap().props.prop("materialOptions/acceptsShadows").unwrap().value.as_enum(), 2); + assert!(comp.layer(ids[1]).unwrap().is_light()); + assert!(s.execute_checked("camera.createFromSolve", json!({"kind": "teapot"})).is_err()); + + // Set Ground Plane and Origin: the target becomes the origin; a null created there sits at + // (0, 0, 0) lying on the X-Z plane (its front facing up, −y). + s.execute_checked("camera.setGroundPlane", json!({})).unwrap(); + assert_eq!(status(&mut s)["groundPlane"], true); + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "null"})).unwrap(); + let null = LayerId(r["layers"][0].as_u64().unwrap()); + let tr = s.active_comp().unwrap().layer(null).unwrap().props.sub("transform").unwrap().clone(); + let pos = tr.get("position").unwrap().value.as_vec3(); + assert!(linalg::norm(pos) < 1e-6, "{pos:?}"); + let up = linalg::mv(&linalg::from_euler_xyz(tr.get("orientation").unwrap().value.as_vec3()), [0.0, 0.0, -1.0]); + assert!(linalg::norm(linalg::sub(up, [0.0, -1.0, 0.0])) < 1e-6, "{up:?}"); + + // An explicit target (comp world) places the layer there, its front facing the normal. + let r = s + .execute_checked("camera.createFromSolve", json!({"kind": "solid", "target": {"center": [10.0, 0.0, 20.0], "normal": [0.0, 0.0, -1.0], "size": 80.0}})) + .unwrap(); + let l = s.active_comp().unwrap().layer(LayerId(r["layers"][0].as_u64().unwrap())).unwrap().clone(); + let tr = l.props.sub("transform").unwrap(); + let pos = tr.get("position").unwrap().value.as_vec3(); + assert!(linalg::norm(linalg::sub(pos, [10.0, 0.0, 20.0])) < 1e-6, "{pos:?}"); + let front = linalg::mv(&linalg::from_euler_xyz(tr.get("orientation").unwrap().value.as_vec3()), [0.0, 0.0, -1.0]); + assert!(linalg::norm(linalg::sub(front, [0.0, 0.0, -1.0])) < 1e-6, "{front:?}"); + + // Serde keeps the analysis. + let back = effectcraft_project::Project::from_json(&s.project.to_json()).unwrap(); + let g2 = back.comp(cid).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap(); + let sv = ct::solve(&crate::camera_track::static_params(g2)).unwrap(); + assert!(sv.ground.is_some()); + assert_eq!(sv.points.len(), pl.solve.points.len()); +} + +#[test] +fn invalidation_resolve_cancel_and_undo() { + let (mut s, clip) = tracked(json!({"shotType": "specify", "aov": 2.0 * (W as f64 / 2.0 / F).atan().to_degrees()})); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + assert_eq!(st["shotType"], "Specify Angle of View"); + assert!((st["focalLength"].as_f64().unwrap() - F).abs() < 0.01 * F); + let cid = s.active_comp_id().unwrap(); + let uid = st["effect"].as_u64().unwrap(); + // Undo removes the analysis (one step); redo restores it. + assert!(s.undo()); + assert_eq!(status(&mut s)["analyzed"], false); + assert!(s.redo()); + assert_eq!(status(&mut s)["solved"], true); + + // Changing the Solve Method keeps the tracks and only re-solves. + s.execute_checked("prop.set", json!({"layer": clip.0, "path": "effects/#1/advanced/solveMethod", "value": 1})).unwrap(); + let st = status(&mut s); + assert_eq!(st["analyzed"], true); + assert_eq!(st["solved"], false); + assert_eq!(st["pending"], true); + s.execute_checked("camera.analyze", json!({"wait": true})).unwrap(); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + assert_eq!(st["methodUsed"], "Typical"); + + // Deleting points re-solves without them. + let pts = s.execute("camera.points", json!({})).unwrap(); + let del: Vec = pts["points"].as_array().unwrap().iter().take(3).map(|p| p["id"].as_u64().unwrap()).collect(); + s.execute_checked("camera.deletePoints", json!({"points": del, "wait": true})).unwrap(); + s.poll_camera(false); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + assert!(st["deleted"].as_u64().unwrap() >= 3); + let g = s.project.comp(cid).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().clone(); + let sv = ct::solve(&crate::camera_track::static_params(&g)).unwrap(); + assert!(sv.points.iter().all(|p| !del.contains(&(p.id as u64)))); + + // Trimming the layer changes its frames: tracks and solve are cleared and queued. + s.edit("trim", None, |p, _| { + p.comp_mut(cid).unwrap().layer_mut(clip).unwrap().in_point = frame_time(2); + Ok(()) + }) + .unwrap(); + let st = status(&mut s); + assert_eq!(st["analyzed"], false); + assert_eq!(st["pending"], true); + + // A background analysis can be cancelled: nothing is written. + s.execute_checked("camera.analyze", json!({})).unwrap(); + assert!(s.is_camera_analyzing() || s.camera_job.is_some()); + let banner = status(&mut s)["banner"].as_str().map(str::to_string); + if s.is_camera_analyzing() { + assert!(banner.unwrap().starts_with("Analyzing in background (step 1 of 2)")); + } + s.execute("camera.cancel", json!({})).ok(); + let t0 = std::time::Instant::now(); + while s.camera_job.is_some() { + assert!(t0.elapsed().as_secs() < 120); + std::thread::sleep(std::time::Duration::from_millis(5)); + s.poll_camera(false); + } + // (When the job finished before the cancel landed it wrote a valid analysis.) + let st = status(&mut s); + assert_eq!(st["running"], false); + assert!(s.execute_checked("track.camera", json!({"layer": clip.0, "shotType": "bogus"})).is_err()); + assert!(s.execute_checked("track.camera", json!({"layer": clip.0, "aov": 500})).is_err()); +} + +#[test] +fn effect_parameters_match_after_effects() { + let spec = effectcraft_effects::registry().iter().find(|e| e.id == ct::ID).unwrap(); + assert_eq!(spec.name, "3D Camera Tracker"); + assert_eq!(spec.category, "Perspective"); + let names: Vec<&str> = spec.params.iter().filter(|p| !matches!(p.ui, effectcraft_project::ParamUi::Hidden)).map(|p| p.name).collect(); + for n in [ + "Shot Type", + "Horizontal Angle of View", + "Show Track Points", + "Render Track Points", + "Track Point Size", + "Target Size", + "Solve Method", + "Detailed Analysis", + "Auto-delete Points Across Time", + "Hide Warning Banner", + ] { + assert!(names.contains(&n), "{n}"); + } + // The Animation menu entry is a real command now. + let c = crate::find_command("track.camera").unwrap(); + assert_eq!(c.label, "Track Camera"); +} diff --git a/crates/engine/src/tests_stubs.rs b/crates/engine/src/tests_stubs.rs index 141858ca..bde98add 100644 --- a/crates/engine/src/tests_stubs.rs +++ b/crates/engine/src/tests_stubs.rs @@ -331,7 +331,7 @@ fn implemented_stubs_are_enabled_and_workspaces_are_frontend() { assert!(s.drain_events().iter().any(|e| matches!(e, Event::Frontend { command, .. } if command == id))); } // What remains a stub is still disabled. - for id in ["view.displayColorManagement", "file.createProxy", "track.camera"] { + for id in ["view.displayColorManagement", "file.createProxy"] { assert!(!s.is_enabled(id), "{id}"); } } diff --git a/crates/track/Cargo.toml b/crates/track/Cargo.toml index 9bd46e26..4800e06e 100644 --- a/crates/track/Cargo.toml +++ b/crates/track/Cargo.toml @@ -14,3 +14,4 @@ effectcraft-raster = { workspace = true } rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +web-time = { workspace = true } diff --git a/crates/track/src/camtrack/bundle.rs b/crates/track/src/camtrack/bundle.rs index 84e8a5ad..72cb2f17 100644 --- a/crates/track/src/camtrack/bundle.rs +++ b/crates/track/src/camtrack/bundle.rs @@ -173,6 +173,9 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> } let mut cur = cost(cams, pts, obs, o); let initial = cur; + let t0 = web_time::Instant::now(); + let mut tries = 0usize; + let dbg = std::env::var_os("EC_CAMTRACK_DEBUG").is_some() && l.n > 200; let mut lambda = 1e-4; let mut iters = 0; if l.n == 0 && o.fix_points { @@ -231,7 +234,10 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> for (ia, ja) in &jc { let wv = [w * (ja[0] * jx[0][0] + ja[1] * jx[1][0]), w * (ja[0] * jx[0][1] + ja[1] * jx[1][1]), w * (ja[0] * jx[0][2] + ja[1] * jx[1][2])]; - match ps.w.iter_mut().find(|e| e.0 == *ia) { + // Each observation of a point is on another camera: only the shared focal + // length repeats. + let found = if Some(*ia) == l.shared { ps.w.iter_mut().find(|e| e.0 == *ia) } else { None }; + match found { Some(e) => e.1 = add(e.1, wv), None => ps.w.push((*ia, wv)), } @@ -294,31 +300,67 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> // ---- Try steps with increasing damping. let mut accepted = false; for _try in 0..12 { + tries += 1; let mut s = base_s.clone(); let mut rhs = base_bc.clone(); for i in 0..n { let d = s[i * n + i]; s[i * n + i] = d + lambda * d.max(1e-6 * mean_diag); } - // Schur complement over the points. - let mut vinv: Vec> = vec![None; npts]; + // Schur complement over the points: S −= W V⁻¹ Wᵀ, rhs −= W V⁻¹ bp (in parallel + // chunks of points when the reduced system is small enough to copy per thread). + let vinv: Vec> = if o.fix_points { + vec![None; npts] + } else { + systems + .par_iter() + .map(|ps| { + let mut v = ps.v; + let vd = (v[0][0] + v[1][1] + v[2][2]) / 3.0; + for k in 0..3 { + v[k][k] += lambda * v[k][k].max(1e-6 * vd.max(1e-12)) + 1e-12; + } + inv3(&v) + }) + .collect() + }; if !o.fix_points { - for (j, ps) in systems.iter().enumerate() { - let mut v = ps.v; - let vd = (v[0][0] + v[1][1] + v[2][2]) / 3.0; - for k in 0..3 { - v[k][k] += lambda * v[k][k].max(1e-6 * vd.max(1e-12)) + 1e-12; - } - let Some(vi) = inv3(&v) else { continue }; - vinv[j] = Some(vi); - let vb = mv(&vi, ps.bp); - let wv: Vec = ps.w.iter().map(|(_, w)| mv(&vi, *w)).collect(); - for (a, (ia, wa)) in ps.w.iter().enumerate() { - rhs[*ia] -= dot(*wa, vb); - for (ib, wb) in &ps.w { - s[ia * n + ib] -= dot(wv[a], *wb); + let reduce = |range: std::ops::Range, s: &mut [f64], rhs: &mut [f64]| { + for j in range { + let (Some(vi), ps) = (&vinv[j], &systems[j]) else { continue }; + let vb = mv(vi, ps.bp); + let wv: Vec = ps.w.iter().map(|(_, w)| mv(vi, *w)).collect(); + for (a, (ia, wa)) in ps.w.iter().enumerate() { + rhs[*ia] -= dot(*wa, vb); + for (ib, wb) in &ps.w[a..] { + let d = dot(wv[a], *wb); + s[ia * n + ib] -= d; + if ia != ib { + s[ib * n + ia] -= d; + } + } } } + }; + let chunks = rayon::current_num_threads().clamp(1, 8); + if n <= 1200 && npts >= 64 && chunks > 1 { + let per = npts.div_ceil(chunks); + let parts: Vec<(Vec, Vec)> = (0..chunks) + .into_par_iter() + .map(|t| { + let (mut ds, mut dr) = (vec![0.0; n * n], vec![0.0; n]); + reduce(t * per..((t + 1) * per).min(npts), &mut ds, &mut dr); + (ds, dr) + }) + .collect(); + for (ds, dr) in parts { + s.par_iter_mut().zip(ds.par_iter()).for_each(|(x, y)| *x += y); + for (x, y) in rhs.iter_mut().zip(dr) { + *x += y; + } + } + } else { + reduce(0..npts, &mut s, &mut rhs); } } let mut dc = rhs; @@ -363,9 +405,18 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> let gain = cur - c2; cur = c2; lambda = (lambda / 3.0).max(1e-9); - accepted = gain > 1e-10 * cur.max(1e-12); + accepted = gain > 1e-6 * cur.max(1e-12); if !accepted { // Converged (tiny improvement). + if dbg { + eprintln!( + "ba n={} pts={} obs={} iters={iters} tries={tries} {:.2}s cost {initial:.1} -> {cur:.1}", + l.n, + npts, + obs.len(), + t0.elapsed().as_secs_f64() + ); + } return BaReport { initial_cost: initial, cost: cur, iterations: iters }; } break; @@ -376,6 +427,9 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> break; } } + if dbg { + eprintln!("ba n={} pts={} obs={} iters={iters} tries={tries} {:.2}s cost {initial:.1} -> {cur:.1}", l.n, npts, obs.len(), t0.elapsed().as_secs_f64()); + } BaReport { initial_cost: initial, cost: cur, iterations: iters } } diff --git a/crates/track/src/camtrack/features.rs b/crates/track/src/camtrack/features.rs index 311ec711..30b019e5 100644 --- a/crates/track/src/camtrack/features.rs +++ b/crates/track/src/camtrack/features.rs @@ -89,7 +89,7 @@ impl TrackAnalyzer { let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize); !occ.get(cy * gw + cx).copied().unwrap_or(true) }; - let opts = CornerOpts { max_features: want - self.active.len().min(want), min_distance: md, quality: 0.005, window: 2, border: 8 }; + let opts = CornerOpts { max_features: want - self.active.len().min(want), min_distance: md, quality: 0.001, window: 2, border: 8 }; for p in good_features(&pyr, &opts, Some(&free)) { let l = pyr.to_layer(p); let id = self.tracks.len() as u32; diff --git a/crates/track/src/camtrack/linalg.rs b/crates/track/src/camtrack/linalg.rs index a7067f26..b719cc8e 100644 --- a/crates/track/src/camtrack/linalg.rs +++ b/crates/track/src/camtrack/linalg.rs @@ -298,6 +298,23 @@ pub fn sym_eigen(a: &[f64], n: usize) -> (Vec, Vec) { (vals, vecs) } +/// Dot product, four lanes (lets the compiler vectorise). +fn dot_n(a: &[f64], b: &[f64]) -> f64 { + let mut acc = [0.0f64; 4]; + let (ca, cb) = (a.chunks_exact(4), b.chunks_exact(4)); + let (ra, rb) = (ca.remainder(), cb.remainder()); + for (x, y) in ca.zip(cb) { + for k in 0..4 { + acc[k] += x[k] * y[k]; + } + } + let mut s = acc[0] + acc[1] + acc[2] + acc[3]; + for (x, y) in ra.iter().zip(rb) { + s += x * y; + } + s +} + /// Solve `A x = b` for a symmetric positive definite `A` (`n × n`, row-major) in place by /// Cholesky factorisation. Returns false when `A` is not positive definite. pub fn cholesky_solve(a: &mut [f64], n: usize, b: &mut [f64]) -> bool { @@ -313,16 +330,9 @@ pub fn cholesky_solve(a: &mut [f64], n: usize, b: &mut [f64]) -> bool { a[j * n + j] = d; let (head, tail) = a.split_at_mut((j + 1) * n); let rowj = &head[j * n..j * n + j]; - // Rows below j are independent. - use rayon::prelude::*; - let work = |row: &mut [f64]| { - let s: f64 = row[j] - row[..j].iter().zip(rowj).map(|(a, b)| a * b).sum::(); - row[j] = s / d; - }; - if n - j > 192 { - tail.par_chunks_mut(n).for_each(work); - } else { - tail.chunks_mut(n).for_each(work); + // Rows below j (sequential: per-column parallel dispatch costs more than it saves). + for row in tail.chunks_mut(n) { + row[j] = (row[j] - dot_n(&row[..j], rowj)) / d; } } // Forward: L y = b. diff --git a/crates/track/src/camtrack/mod.rs b/crates/track/src/camtrack/mod.rs index 056d7ad0..76c0637f 100644 --- a/crates/track/src/camtrack/mod.rs +++ b/crates/track/src/camtrack/mod.rs @@ -13,7 +13,10 @@ //! the other keyframes are added incrementally (**resection** by robust pose refinement, //! **triangulation** of new tracks by the linear method of H&Z §12.2); a sparse //! **Levenberg–Marquardt bundle adjustment** with the Schur complement over the points -//! ([`bundle`]) refines everything; the frames between keyframes are resected last. +//! ([`bundle`]) refines everything; the frames between keyframes are resected last. Inlier +//! thresholds start from the analysis resolution and are re-estimated from the residuals +//! (median absolute residual) once the keyframes are solved, so features on moving objects, +//! occlusion edges and screen-locked overlays drop out. //! 3. **Focal length**: *Specify Angle of View* fixes it; *Fixed Angle of View* searches it //! (the reprojection error of the keyframe reconstruction over a log-spaced range of focal //! lengths, refined by golden-section search) and then adjusts it in the bundle adjustment @@ -22,7 +25,8 @@ //! 4. **Tripod pan** shots (no camera translation) are solved as a pure rotation: frame-to-frame //! rotations from the tracked rays (orthogonal Procrustes) refined by the same bundle //! adjustment with the camera centres fixed. *Auto Detect* solves both ways and keeps the -//! tripod solution when it explains the tracks as well as the general one. +//! tripod solution when it explains the tracks as well as the general one (a quick rotation fit +//! at the general solution's focal length skips the tripod search when it clearly fails). //! //! The camera model has square pixels, the principal point at the layer centre and no lens //! distortion. Camera space is x right, y down, z forward (After Effects' convention); a camera diff --git a/crates/track/src/camtrack/solver.rs b/crates/track/src/camtrack/solver.rs index da3a9290..85edd2bd 100644 --- a/crates/track/src/camtrack/solver.rs +++ b/crates/track/src/camtrack/solver.rs @@ -37,8 +37,9 @@ struct Data { tracks: Vec<(u32, usize, Vec<[f64; 2]>)>, /// Per frame: (track index, position). by_frame: Vec>, - /// Tracking noise scale (pixels). - noise: f64, + /// Tracking noise scale (pixels): starts from the analysis resolution and is re-estimated + /// from the residuals once the keyframes are solved. + noise_bits: std::sync::atomic::AtomicU64, } impl Data { @@ -58,23 +59,31 @@ impl Data { } tracks.push((tr.id, tr.start as usize, pts)); } - Data { n, w: t.size[0], tracks, by_frame, noise: t.factor.max(1.0) } + let noise = (0.5 * t.factor).max(0.5); + Data { n, w: t.size[0], tracks, by_frame, noise_bits: std::sync::atomic::AtomicU64::new(noise.to_bits()) } } fn at(&self, ti: u32, k: usize) -> Option<[f64; 2]> { let (_, s, p) = &self.tracks[ti as usize]; k.checked_sub(*s).and_then(|i| p.get(i).copied()) } + fn noise(&self) -> f64 { + f64::from_bits(self.noise_bits.load(Ordering::Relaxed)) + } + fn set_noise(&self, v: f64) { + self.noise_bits.store(v.to_bits(), Ordering::Relaxed); + } fn inlier_thr(&self) -> f64 { - 4.0 * self.noise + 4.0 * self.noise() } fn huber(&self) -> f64 { - 1.0 * self.noise + self.noise() } } /// Parallax-based keyframes: a new keyframe when the median feature motion since the last -/// exceeds 3 % of the width, when fewer than 60 % of its features survive, or before the shared -/// features drop below 24. The first and last frames with features are keyframes. +/// exceeds 1.5 % of the width, when fewer than 70 % of its features survive, after 12 frames, or +/// before the shared features drop below 24. The first and last frames with features are +/// keyframes. fn keyframes(d: &Data) -> Vec { let Some(first) = (0..d.n).find(|k| d.by_frame[*k].len() >= 8) else { return vec![] }; let mut kf = vec![first]; @@ -97,7 +106,7 @@ fn keyframes(d: &Data) -> Vec { disp.sort_by(f64::total_cmp); let med = disp.get(disp.len() / 2).copied().unwrap_or(0.0); let ratio = disp.len() as f64 / base.len().max(1) as f64; - if med > 0.03 * d.w || ratio < 0.6 || disp.len() < 24 { + if med > 0.015 * d.w || ratio < 0.7 || disp.len() < 24 || j - last >= 12 { last = j; kf.push(j); } @@ -405,7 +414,7 @@ fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> } let x1: Vec<[f64; 2]> = sh.iter().map(|s| [s.1[0] / f, s.1[1] / f]).collect(); let x2: Vec<[f64; 2]> = sh.iter().map(|s| [s.2[0] / f, s.2[1] / f]).collect(); - let thr = 2.0 * d.noise / f; + let thr = 2.0 * d.noise() / f; let mut cands: Vec = vec![]; if use_e && let Some((e, inl)) = geometry::ransac_essential(&x1, &x2, thr, 400, (a * 7919 + b) as u64) { let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip(); @@ -415,7 +424,7 @@ fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> } if use_h { let (s1, s2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = sh.iter().map(|s| (s.1, s.2)).unzip(); - if let Some(fit) = ransac(Model::Homography, &s1, &s2, 2.0 * d.noise, 500, (a * 31 + b) as u64) { + if let Some(fit) = ransac(Model::Homography, &s1, &s2, 2.0 * d.noise(), 500, (a * 31 + b) as u64) { let h = fit.h.0; // Normalised: K⁻¹ H K with K = diag(f, f, 1). let hn = [[h[0][0], h[0][1], h[0][2] / f], [h[1][0], h[1][1], h[1][2] / f], [h[2][0] * f, h[2][1] * f, h[2][2]]]; @@ -651,6 +660,12 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo let try_general = matches!(s.method, SolveMethod::Auto | SolveMethod::Typical | SolveMethod::MostlyFlat); let try_tripod = matches!(s.method, SolveMethod::Auto | SolveMethod::TripodPan); + let t0 = web_time::Instant::now(); + let lap = |what: &str| { + if debug() { + eprintln!("{what}: {:.2} s", t0.elapsed().as_secs_f64()); + } + }; let mut results: Vec<(Attempt, f64)> = vec![]; if try_general { let f = match fixed_f { @@ -666,7 +681,20 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo results.push((a, f)); } } - if try_tripod && !cancelled() { + lap("general"); + // Auto Detect: a quick tripod check at the general solve's focal length; when a rotation + // clearly cannot explain the tracks, the tripod search is skipped. + let tripod_plausible = match (s.method, results.first()) { + (SolveMethod::Auto, Some((g, f))) => { + let quick = tripod(&d, &mini, *f, &tcfg, cancel).map(|a| a.err).unwrap_or(f64::INFINITY); + if debug() { + eprintln!("quick tripod check: {quick:.3} vs general {:.3}", g.err); + } + quick <= 3.0 * g.err + d.noise() + } + _ => true, + }; + if try_tripod && tripod_plausible && !cancelled() { let f = match fixed_f { Some(f) => f, None => search_focal(d.w, &|f| tripod(&d, &mini, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)), @@ -696,7 +724,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo if debug() { eprintln!("auto: general {eg:.4} tripod {et:.4} baseline/depth {:.4}", base / med); } - if et <= (eg * 1.25).max(eg + 0.05 * d.noise) || base / med < 0.01 { t } else { g } + if et <= (eg * 1.25).max(eg + 0.05 * d.noise()) || base / med < 0.01 { t } else { g } } (Some(g), None) => g, (None, Some(t)) => t, @@ -717,15 +745,44 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo (None, true) => FocalMode::PerCamera(F_PRIOR), (None, false) => FocalMode::Shared, }; + lap("tripod"); // Keyframes: adjust with the focal length free, reject, re-triangulate, adjust again. let kf_frames: Vec = r.cams.keys().copied().collect(); r.adjust(&d, &kf_frames, &cfg, focal, 40); r.reject(&d); + // Re-estimate the tracking noise from the inlier residuals (robustly, from their median: + // for 2D Gaussian noise of deviation σ the median distance is 1.1774 σ), then tighten the + // thresholds: observations of moving objects and occlusion edges become outliers. + let mut res: Vec = vec![]; + for (k, c) in &r.cams { + for (ti, uv) in &d.by_frame[*k] { + if r.outliers.contains(&(*ti, *k)) { + continue; + } + if let Some(e) = r.pts.get(ti).and_then(|x| bundle::residual(c, *x, *uv)) { + res.push(e[0].hypot(e[1])); + } + } + } + if res.len() > 20 { + res.sort_by(f64::total_cmp); + let sigma = res[res.len() / 2] / 1.1774; + let floor = 0.05 * tracks.factor.max(1.0); + d.set_noise((1.5 * sigma).clamp(floor, d.noise())); + if debug() { + eprintln!("noise: σ {sigma:.3} → {:.3}", d.noise()); + } + r.outliers.clear(); + r.reject(&d); + } r.triangulate(&d, kind, 1.0, 1.0); r.adjust(&d, &kf_frames, &cfg, focal, 40); + r.reject(&d); + r.adjust(&d, &kf_frames, &cfg, focal, 20); if cancelled() { return err("cancelled"); } + lap("keyframe adjustment"); // Every other frame: resect from the points (nearest registered camera first). let mut todo: Vec = (0..d.n).filter(|k| !r.cams.contains_key(k)).collect(); todo.sort_by_key(|k| r.cams.keys().map(|c| c.abs_diff(*k)).min().unwrap_or(usize::MAX)); @@ -746,6 +803,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo if cancelled() { return err("cancelled"); } + lap("resection"); // Points for the tracks that only lived between keyframes, then the final adjustment over // every frame (when that stays affordable). r.triangulate(&d, kind, 1.0, 1.0); @@ -786,6 +844,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo } (m, _) => m, }; + lap("final adjustment"); Ok(finish(&d, tracks, s, r, method, kind)) } diff --git a/crates/track/src/camtrack/tests.rs b/crates/track/src/camtrack/tests.rs index 258161f9..4b5a8346 100644 --- a/crates/track/src/camtrack/tests.rs +++ b/crates/track/src/camtrack/tests.rs @@ -358,3 +358,81 @@ fn perf_long_clip() { eprintln!("300 frames, {} tracks: {:?} err {:.3} focal {:.1} in {:?}", tr.tracks.len(), s.method_used, s.average_error, s.focal(), t0.elapsed()); assert!(s.average_error < 0.5); } + +/// Solve tracks saved from the app (`EC_CAMTRACK_FILE=tracks.json cargo test -p effectcraft-track +/// --release solve_saved_tracks -- --ignored --nocapture`): a debugging aid. +#[test] +#[ignore] +fn solve_saved_tracks() { + let Some(path) = std::env::var_os("EC_CAMTRACK_FILE") else { return }; + let mut tr = CameraTracks::from_json(&std::fs::read_to_string(path).unwrap()).unwrap(); + if let Some(maxy) = std::env::var("EC_CAMTRACK_MAXY").ok().and_then(|v| v.parse::().ok()) { + tr.tracks.retain(|t| t.pts.iter().all(|p| p[1] < maxy)); + } + let shot = if std::env::var_os("EC_CAMTRACK_VARIABLE").is_some() { ShotType::VariableZoom } else { ShotType::FixedAngle }; + let t0 = std::time::Instant::now(); + // With the true cameras (`[{pos, poi, zoom}]` per frame, two-node), how consistent are the + // tracks themselves? + if let Some(tp) = std::env::var_os("EC_CAMTRACK_TRUTH") { + let v: Vec = serde_json::from_str(&std::fs::read_to_string(tp).unwrap()).unwrap(); + let g = |x: &serde_json::Value| -> V3 { [x[0].as_f64().unwrap(), x[1].as_f64().unwrap(), x[2].as_f64().unwrap()] }; + let cams: Vec<(super::geometry::Pose, f64)> = v + .iter() + .map(|c| { + let (pos, poi) = (g(&c["pos"]), g(&c["poi"])); + let fwd = normalize(sub(poi, pos)); + let d0 = [0.0, 1.0, 0.0]; + let down = normalize(sub(d0, scale(fwd, dot(d0, fwd)))); + let right = cross(down, fwd); + (super::geometry::Pose { r: [right, down, fwd], c: pos }, c["zoom"].as_f64().unwrap()) + }) + .collect(); + let (w, h) = (tr.size[0], tr.size[1]); + let mut means = vec![]; + let mut drop_ids = vec![]; + for t in &tr.tracks { + let views: Vec<(super::geometry::Pose, [f64; 2])> = t + .pts + .iter() + .enumerate() + .map(|(i, p)| { + let (pose, z) = cams[t.start as usize + i]; + (pose, [(p[0] as f64 - w / 2.0) / z, (p[1] as f64 - h / 2.0) / z]) + }) + .collect(); + let Some(x) = super::geometry::triangulate(&views) else { continue }; + let e: Vec = views.iter().filter_map(|(p, o)| super::geometry::reproj2(p, x, *o).map(|e| e.sqrt() * cams[0].1)).collect(); + means.push((e.iter().sum::() / e.len().max(1) as f64, t.pts[0])); + if std::env::var_os("EC_CAMTRACK_TRUTH_FILTER").is_some() && means.last().unwrap().0 > 1.0 { + drop_ids.push(t.id); + } + } + tr.tracks.retain(|t| !drop_ids.contains(&t.id)); + means.sort_by(|a, b| a.0.total_cmp(&b.0)); + eprintln!( + "true-camera track errors (px): median {:.2}; quartiles {:?}", + means[means.len() / 2].0, + [means[means.len() / 4].0, means[means.len() * 3 / 4].0] + ); + for m in means.iter().step_by(6) { + eprintln!(" {:.2} at {:?}", m.0, m.1); + } + } + let mut st = settings(shot, SolveMethod::Auto); + if let Some(a) = std::env::var("EC_CAMTRACK_AOV").ok().and_then(|v| v.parse::().ok()) { + st.shot = ShotType::SpecifyAngle; + st.hfov = a; + } + let s = solve(&tr, &st, None).expect("solve"); + eprintln!( + "{} frames, {} tracks: {:?} err {:.3} focal {:.1} (hfov {:.1}) points {} in {:?}", + tr.frames, + tr.tracks.len(), + s.method_used, + s.average_error, + s.focal(), + s.hfov(s.focal()), + s.points.len(), + t0.elapsed() + ); +} diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 071c1948..d581ef35 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -727,6 +727,11 @@ impl EffectcraftApp { self.session.poll_warp(true); ctx.request_repaint_after(std::time::Duration::from_millis(100)); } + // 3D Camera Tracker: likewise (tracking, then solving). + if self.session.camera_job.is_some() || !self.session.camera_pending.is_empty() { + self.session.poll_camera(true); + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } self.apply_prefs(&ctx); self.handle_events(&ctx); self.tick_autosave(&ctx); diff --git a/crates/ui-egui/src/panels/camera_tracker_ui.rs b/crates/ui-egui/src/panels/camera_tracker_ui.rs new file mode 100644 index 00000000..41fb7204 --- /dev/null +++ b/crates/ui-egui/src/panels/camera_tracker_ui.rs @@ -0,0 +1,438 @@ +//! 3D Camera Tracker in the Composition viewer and Effect Controls: +//! +//! - while the effect is selected, the solved 3D track points are drawn as coloured targets +//! sized by depth (or the 2D source tracks with Show Track Points ▸ 2D Source); +//! - hovering between three points shows the target plane they span (a disc in perspective); +//! click a point (Shift adds / toggles) or drag a marquee to select points, click inside a +//! triangle to select its three points; the selection's target is drawn; +//! - right-click: Set Ground Plane and Origin, Create Text / Solid / Null / Shadow Catcher … +//! and Camera, Create Multiple …, Delete Selected Points; Delete removes the selection; +//! - the "Analyzing in background (step 1 of 2)" / "Solving camera" banner; +//! - the Effect Controls block with Analyze / Cancel / Create Camera, Method Used and Average +//! Error. +//! +//! Everything is dispatched to the `camera.*` commands. + +use std::sync::Arc; + +use effectcraft_engine::Session; +use effectcraft_engine::effects::camera_tracker as ct; +use effectcraft_engine::geom::{Mat3, vec2 as gv2}; +use effectcraft_engine::keyframe::Value; +use effectcraft_engine::project::{Comp, Layer, LayerId, PropGroup, Uid}; +use effectcraft_engine::render::EvalCtx; +use effectcraft_engine::track::camtrack::{CameraSolve, CameraTracks, Target, point_color}; +use egui::{Align2, Color32, Pos2, Rect, Sense, Stroke, pos2, vec2}; +use serde_json::json; + +use super::viewer::ViewerMap; +use crate::theme::Tokens; +use crate::{EffectcraftApp, widgets}; + +/// The tracker whose points the viewer shows. +struct Active { + layer: LayerId, + uid: Uid, + solve: Option>, + tracks: Option>, + show_3d: bool, + point_size: f64, + target_size: f64, + /// Layer pixels → comp pixels. + l2c: Mat3, + frame: usize, +} + +/// A drawn point: (id, screen position, screen radius). +type Hit = (u32, Pos2, f32); + +fn tracker_group(l: &Layer, uid: Uid) -> Option<&PropGroup> { + l.props.find_group(uid).filter(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == ct::ID)) +} + +/// The selected 3D Camera Tracker effect on an active layer (After Effects shows the points +/// while the effect is selected). +fn active(s: &Session, comp: &Comp, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3) -> Option { + let time = ectx.time; + for (lid, uid) in &s.state.selected_props { + let Some(l) = comp.layer(*lid).filter(|l| l.is_active_at(time)) else { continue }; + let Some(g) = tracker_group(l, *uid) else { continue }; + let params = effectcraft_engine::camera_track::static_params(g); + let val = |id: &str| effectcraft_engine::effects::warp_stab::param(¶ms, id).cloned(); + let solve = ct::solve(¶ms); + let tracks = ct::tracks(¶ms); + let lt = l.layer_time(time).seconds(); + let frame = solve.as_ref().and_then(|sv| sv.frame_at(lt)).or_else(|| { + let t = tracks.as_ref()?; + (t.frame_duration > 0.0).then(|| ((lt - t.start) / t.frame_duration).round().clamp(0.0, t.frames.saturating_sub(1) as f64) as usize) + }); + return Some(Active { + layer: *lid, + uid: *uid, + solve, + tracks, + show_3d: val("showTrackPoints").map(|v| v.as_enum()).unwrap_or(1) == 1, + point_size: val("trackPointSize").map(|v| v.as_f64()).unwrap_or(100.0), + target_size: val("targetSize").map(|v| v.as_f64()).unwrap_or(100.0), + l2c: l2c(ectx, l), + frame: frame.unwrap_or(0), + }); + } + None +} + +fn to_screen(a: &Active, map: &ViewerMap, q: [f64; 2]) -> Pos2 { + let c = a.l2c.apply(gv2(q[0], q[1])); + map.to_screen([c.x, c.y]) +} + +fn col(id: u32) -> Color32 { + let c = point_color(id); + Color32::from_rgb((c[0] * 255.0) as u8, (c[1] * 255.0) as u8, (c[2] * 255.0) as u8) +} + +/// Comp-pixel scale of the layer → comp mapping. +fn l2c_scale(m: &Mat3) -> f32 { + m.0[0][0].hypot(m.0[1][0]) as f32 +} + +/// Draw the target disc (concentric rings in perspective) of `t` on frame `k`. +fn draw_target(p: &egui::Painter, a: &Active, map: &ViewerMap, sv: &CameraSolve, t: &Target) { + let Some(cam) = sv.frames.get(a.frame) else { return }; + let r = t.size * 0.5 * a.target_size / 100.0; + for (i, frac) in [1.0, 0.66, 0.33].into_iter().enumerate() { + let ring = ct::target_circle(t.center, t.normal, r * frac, 48); + let pts: Vec = ring.iter().filter_map(|x| cam.project(sv.size, *x)).map(|q| to_screen(a, map, q)).collect(); + if pts.len() < 3 { + return; + } + let fill = if i % 2 == 0 { Color32::from_rgba_unmultiplied(0xd0, 0x30, 0x30, 70) } else { Color32::from_rgba_unmultiplied(0xff, 0xff, 0xff, 50) }; + p.add(egui::Shape::convex_polygon(pts.clone(), fill, Stroke::new(1.0, Color32::from_rgb(0xe0, 0x40, 0x40)))); + } + if let Some(c) = cam.project(sv.size, t.center) { + let c = to_screen(a, map, c); + p.circle_filled(c, 3.0, Color32::from_rgb(0xff, 0xd0, 0x40)); + } +} + +/// The three drawn points forming a triangle around `hp` (nearest first). +fn triangle_at(hits: &[Hit], hp: Pos2) -> Option<[u32; 3]> { + let mut near: Vec<&Hit> = hits.iter().collect(); + near.sort_by(|x, y| x.1.distance_sq(hp).total_cmp(&y.1.distance_sq(hp))); + let near: Vec<&Hit> = near.into_iter().take(8).collect(); + let inside = |a: Pos2, b: Pos2, c: Pos2| { + let s = |p: Pos2, q: Pos2| (q.x - p.x) * (hp.y - p.y) - (q.y - p.y) * (hp.x - p.x); + let (d1, d2, d3) = (s(a, b), s(b, c), s(c, a)); + !((d1 < 0.0 || d2 < 0.0 || d3 < 0.0) && (d1 > 0.0 || d2 > 0.0 || d3 > 0.0)) + }; + for i in 0..near.len() { + for j in i + 1..near.len() { + for k in j + 1..near.len() { + let (a, b, c) = (near[i], near[j], near[k]); + // Skip slivers. + let area = ((b.1 - a.1).x * (c.1 - a.1).y - (b.1 - a.1).y * (c.1 - a.1).x).abs(); + if area > 30.0 && inside(a.1, b.1, c.1) { + return Some([a.0, b.0, c.0]); + } + } + } + } + None +} + +fn run(app: &mut EffectcraftApp, ctx: &egui::Context, cmd: &str, p: serde_json::Value) { + if let Err(e) = crate::menus::invoke(app, ctx, cmd, p) { + app.ui.status = e; + } +} + +/// The viewer overlay and its interaction (call after the viewer's own gestures so it takes +/// the clicks while a 3D Camera Tracker is selected). +pub fn viewer_hook( + app: &mut EffectcraftApp, + ui: &mut egui::Ui, + painter: &egui::Painter, + map: &ViewerMap, + ectx: &EvalCtx, + l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3, +) { + let ctx = ui.ctx().clone(); + let comp = ectx.comp; + banner(app, painter, map, ectx, l2c); + if !app.ui.viewer.show_layer_controls { + return; + } + let Some(a) = active(&app.session, comp, ectx, l2c) else { return }; + let selected = app.session.state.camera_points.clone(); + let lscale = l2c_scale(&a.l2c) * map.zoom; + let mut hits: Vec = vec![]; + match (&a.solve, a.show_3d) { + (Some(sv), true) => { + let pts = ct::projected(sv, a.frame); + let med = ct::median_depth(&pts); + for (id, q, z) in pts { + let sp = to_screen(&a, map, q); + if !map.area.expand(20.0).contains(sp) { + continue; + } + let r = (ct::target_radius(a.point_size, z, med) as f32 * lscale).clamp(2.0, 30.0); + let c = col(id); + let sel = selected.contains(&id); + if sel { + painter.circle_filled(sp, r, c); + painter.circle_stroke(sp, r + 1.5, Stroke::new(1.5, Color32::WHITE)); + } else { + painter.circle_stroke(sp, r, Stroke::new((r * 0.3).clamp(1.0, 3.0), c)); + painter.circle_filled(sp, (r * 0.25).max(1.0), c); + } + hits.push((id, sp, r)); + } + } + _ => { + // 2D source tracks (also shown before the solve finishes). + if let Some(tr) = &a.tracks { + for t in &tr.tracks { + let Some(q) = t.at(a.frame as u32) else { continue }; + let sp = to_screen(&a, map, q); + let c = col(t.id); + let r = (3.0 * a.point_size as f32 / 100.0).clamp(1.5, 12.0); + painter.line_segment([sp - vec2(r, r), sp + vec2(r, r)], Stroke::new(1.2, c)); + painter.line_segment([sp - vec2(r, -r), sp + vec2(r, -r)], Stroke::new(1.2, c)); + hits.push((t.id, sp, r)); + } + } + } + } + let Some(sv) = a.solve.clone() else { return }; + if app.ui.tool != crate::state::Tool::Selection { + return; + } + // Interaction over the image. + let resp = ui.interact(map.area, egui::Id::new("viewer-camera-tracker"), Sense::click_and_drag()); + app.auto.add("viewer.cameraTracker", map.area, "3D Camera Tracker points"); + for (id, sp, r) in hits.iter().take(400) { + app.auto.add(&format!("viewer.cameraTracker.point.{id}"), Rect::from_center_size(*sp, vec2(r * 2.0 + 4.0, r * 2.0 + 4.0)), "track point"); + } + let hover = resp.hover_pos(); + let point_at = |p: Pos2| hits.iter().filter(|h| h.1.distance(p) <= h.2 + 3.0).min_by(|x, y| x.1.distance(p).total_cmp(&y.1.distance(p))).map(|h| h.0); + let tri = hover.filter(|p| point_at(*p).is_none()).and_then(|p| triangle_at(&hits, p)); + let mid = egui::Id::new("viewer-camera-tracker-marquee"); + let marquee: Option = ui.data(|d| d.get_temp(mid)); + // Hover target / the selection's target. + if marquee.is_none() { + if let Some(t3) = tri + && let Some(t) = effectcraft_engine::camera_track::target_for(&sv, &t3, a.frame) + { + let ps: Vec = t3.iter().filter_map(|i| hits.iter().find(|h| h.0 == *i)).map(|h| h.1).collect(); + if ps.len() == 3 { + painter.add(egui::Shape::closed_line(ps, Stroke::new(1.0, Color32::from_white_alpha(200)))); + } + draw_target(painter, &a, map, &sv, &t); + } else if !selected.is_empty() + && let Some(t) = effectcraft_engine::camera_track::target_for(&sv, &selected, a.frame) + { + draw_target(painter, &a, map, &sv, &t); + } + } + let mods = ui.input(|i| i.modifiers); + if resp.drag_started() + && let Some(p) = ui.input(|i| i.pointer.press_origin()) + { + ui.data_mut(|d| d.insert_temp(mid, p)); + } + if let (Some(start), Some(p)) = (marquee, resp.interact_pointer_pos()) { + let r = Rect::from_two_pos(start, p); + let t = app.tokens; + painter.rect_filled(r, 0.0, t.accent.gamma_multiply(0.12)); + painter.rect_stroke(r, 0.0, Stroke::new(1.0, t.accent), egui::StrokeKind::Middle); + if resp.drag_stopped() { + let ids: Vec = hits.iter().filter(|h| r.contains(h.1)).map(|h| h.0).collect(); + ui.data_mut(|d| d.remove::(mid)); + run(app, &ctx, "camera.selectPoints", json!({"points": ids, "add": mods.shift})); + } + } else if resp.drag_stopped() { + ui.data_mut(|d| d.remove::(mid)); + } + if resp.clicked() + && let Some(p) = resp.interact_pointer_pos() + { + match (point_at(p), triangle_at(&hits, p)) { + (Some(id), _) => run(app, &ctx, "camera.selectPoints", json!({"points": [id], "toggle": mods.shift})), + (None, Some(t3)) => run(app, &ctx, "camera.selectPoints", json!({"points": t3, "add": mods.shift})), + (None, None) => run(app, &ctx, "camera.selectPoints", json!({"points": []})), + } + } + // Right-click: the points under the cursor (selection, else the hovered triangle). + let cid = egui::Id::new("viewer-camera-tracker-menu-points"); + if resp.secondary_clicked() { + let pts: Vec = if !selected.is_empty() { + selected.clone() + } else if let Some(p) = resp.interact_pointer_pos() { + point_at(p).map(|i| vec![i]).or_else(|| triangle_at(&hits, p).map(|t| t.to_vec())).unwrap_or_default() + } else { + vec![] + }; + ui.data_mut(|d| d.insert_temp(cid, pts)); + } + let base = json!({"layer": a.layer.0, "effect": a.uid}); + resp.context_menu(|ui| { + let pts: Vec = ui.data(|d| d.get_temp(cid)).unwrap_or_default(); + let with = |kind: Option<&str>, multiple: bool| { + let mut q = base.clone(); + q["points"] = json!(pts); + if let Some(k) = kind { + q["kind"] = json!(k); + } + if multiple { + q["multiple"] = json!(true); + } + q + }; + let some = !pts.is_empty(); + let many = pts.len() > 1; + let mut pick: Option<(&str, serde_json::Value)> = None; + if ui.add_enabled(some, egui::Button::new("Set Ground Plane and Origin")).clicked() { + pick = Some(("camera.setGroundPlane", with(None, false))); + } + ui.separator(); + for (label, kind) in [ + ("Create Text and Camera", "text"), + ("Create Solid and Camera", "solid"), + ("Create Null and Camera", "null"), + ("Create Shadow Catcher, Camera and Light", "shadowCatcher"), + ] { + if ui.add_enabled(some, egui::Button::new(label)).clicked() { + pick = Some(("camera.createFromSolve", with(Some(kind), false))); + } + } + ui.separator(); + for (label, kind) in + [("Create Multiple Text Layers and Camera", "text"), ("Create Multiple Solids and Camera", "solid"), ("Create Multiple Nulls and Camera", "null")] + { + if ui.add_enabled(many, egui::Button::new(label)).clicked() { + pick = Some(("camera.createFromSolve", with(Some(kind), true))); + } + } + ui.separator(); + if ui.add_enabled(some, egui::Button::new("Delete Selected Points")).clicked() { + pick = Some(("camera.deletePoints", with(None, false))); + } + if let Some((cmd, q)) = pick { + ui.close(); + run(app, &ctx, cmd, q); + } + }); + if resp.hovered() + && !selected.is_empty() + && ui.input(|i| i.key_pressed(egui::Key::Delete) || i.key_pressed(egui::Key::Backspace)) + && !ctx.egui_wants_keyboard_input() + { + run(app, &ctx, "camera.deletePoints", json!({"layer": a.layer.0, "effect": a.uid, "points": selected})); + } +} + +/// "Analyzing in background (step 1 of 2)" / "Solving camera" over the analysed layer, and a +/// warning when the solve failed (unless Hide Warning Banner). +fn banner(app: &mut EffectcraftApp, painter: &egui::Painter, map: &ViewerMap, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3) { + let comp = ectx.comp; + let time = ectx.time; + let cid = ectx.comp_id; + let draw = |app: &mut EffectcraftApp, layer: &Layer, text: String, color: Color32, id: &str| { + let m = l2c(ectx, layer); + let (w, h) = effectcraft_engine::render::source_size(&app.session.project, layer); + let c = m.apply(gv2(w as f64 / 2.0, h as f64 / 2.0)); + let at = map.to_screen([c.x, c.y]); + let galley = painter.layout_no_wrap(text.clone(), Tokens::medium(13.0), Color32::WHITE); + let r = Rect::from_center_size(at, galley.size() + vec2(28.0, 14.0)).intersect(map.area); + painter.rect_filled(r, 3.0, color); + painter.galley(pos2(r.center().x - galley.size().x / 2.0, r.center().y - galley.size().y / 2.0), galley, Color32::WHITE); + app.auto.add(id, r, &text); + }; + if let Some((cc, cl, _)) = app.session.camera_target() + && cc == cid + && let Some(pr) = app.session.camera_progress() + && let Some(layer) = comp.layer(cl) + && layer.is_active_at(time) + { + draw(app, layer, pr.banner(), Color32::from_rgba_unmultiplied(0x1d, 0x4f, 0x9c, 0xe0), "viewer.cameraBanner"); + return; + } + // A failed solve: tracks without a solve while nothing is running or queued. + for l in comp.layers.iter().filter(|l| l.is_active_at(time)) { + let Some(fx) = l.effects() else { continue }; + for g in fx.groups().filter(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == ct::ID)) { + let params = effectcraft_engine::camera_track::static_params(g); + let hide = effectcraft_engine::effects::warp_stab::param(¶ms, "advanced/hideWarningBanner").is_some_and(Value::as_bool); + let solve_key = matches!(g.prop(ct::SOLVE_KEY).map(|p| &p.value), Some(Value::Str(s)) if !s.is_empty()); + let failed = solve_key && ct::tracks(¶ms).is_some() && ct::solve(¶ms).is_none(); + if failed && !hide && !app.session.camera_pending.iter().any(|(_, ll, u)| *ll == l.id && *u == g.uid) { + draw(app, l, "Unable to solve camera".into(), Color32::from_rgba_unmultiplied(0x9c, 0x2a, 0x1d, 0xe0), "viewer.cameraWarning"); + return; + } + } + } +} + +/// Effect Controls: Analyze / Cancel, Create Camera and the solve report. +#[allow(clippy::too_many_arguments)] +pub fn editor( + app: &mut EffectcraftApp, + ui: &mut egui::Ui, + p: &egui::Painter, + layer: &Layer, + g: &PropGroup, + r: Rect, + actions: &mut Vec<(String, serde_json::Value)>, +) { + let t = app.tokens; + let running = app.session.camera_target().is_some_and(|(_, l, u)| l == layer.id && u == g.uid) && app.session.is_camera_analyzing(); + let busy = app.session.is_camera_analyzing(); + let me = json!({"layer": layer.id.0, "effect": g.uid}); + let b1 = Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 4.0), vec2(84.0, 22.0)); + let b2 = Rect::from_min_size(pos2(b1.max.x + 8.0, b1.min.y), vec2(84.0, 22.0)); + let b3 = Rect::from_min_size(pos2(b2.max.x + 8.0, b1.min.y), vec2(110.0, 22.0)); + if widgets::text_button(ui, b1, "Analyze", false, &t, egui::Id::new(("ct-analyze", g.uid))).clicked() && !busy { + actions.push(("camera.analyze".into(), me.clone())); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.analyze", g.uid), b1, "Analyze"); + if running { + if widgets::text_button(ui, b2, "Cancel", false, &t, egui::Id::new(("ct-cancel", g.uid))).clicked() { + actions.push(("camera.cancel".into(), json!({}))); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.cancel", g.uid), b2, "Cancel"); + } + let params = effectcraft_engine::camera_track::static_params(g); + let solve = ct::solve(¶ms); + if solve.is_some() { + if widgets::text_button(ui, b3, "Create Camera", false, &t, egui::Id::new(("ct-camera", g.uid))).clicked() { + actions.push(("camera.create".into(), me.clone())); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.createCamera", g.uid), b3, "Create Camera"); + } + let status = match app.session.camera_progress() { + Some(pr) if running => pr.banner(), + _ => match &solve { + Some(sv) => format!("Method Used: {} Average Error: {:.2} pixels", sv.method_used.label(), sv.average_error), + None if ct::tracks(¶ms).is_some() && app.session.camera_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => { + "Waiting to solve".into() + } + None if ct::tracks(¶ms).is_some() => "Unable to solve camera: try another Solve Method or Shot Type".into(), + None if app.session.camera_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => "Waiting to analyze".into(), + None => "Not analyzed: click Analyze".into(), + }, + }; + p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim); + if let Some(sv) = &solve { + let f = sv.focal(); + p.text( + pos2(r.min.x + 24.0, r.min.y + 56.0), + Align2::LEFT_CENTER, + format!("{} solved points · Horizontal Angle of View {:.1}°", sv.points.len(), sv.hfov(f)), + Tokens::ui(11.5), + t.text_faint, + ); + } +} + +/// Height of [`editor`]. +pub const EDITOR_HEIGHT: f32 = 68.0; diff --git a/crates/ui-egui/src/panels/effect_controls.rs b/crates/ui-egui/src/panels/effect_controls.rs index 0a19d57f..11640d88 100644 --- a/crates/ui-egui/src/panels/effect_controls.rs +++ b/crates/ui-egui/src/panels/effect_controls.rs @@ -545,6 +545,7 @@ fn editor_height(effect: &str, width: f32) -> f32 { match effect { "ec.color.curves" => 34.0 + curves_size(width) + 26.0, effectcraft_engine::effects::warp_stab::ID => 50.0, + effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::EDITOR_HEIGHT, "ec.color.levels" | "ec.color.levelsic" => 34.0 + 80.0 + 58.0, "ec.color.autolevels" | "ec.color.autocontrast" | "ec.color.autocolor" => 24.0 + 80.0 + 12.0, _ => 0.0, @@ -920,6 +921,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { match effect.as_str() { "ec.color.curves" => curves_editor(app, ui, &bp, &layer, g, &ectx, er, &mut actions), effectcraft_engine::effects::warp_stab::ID => warp_editor(app, ui, &bp, &layer, g, er, &mut actions), + effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::editor(app, ui, &bp, &layer, g, er, &mut actions), "ec.color.levels" | "ec.color.levelsic" => levels_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions), _ => histogram_only(app, ui, &bp, g, er), } diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 3f737502..ead1fe99 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -1,6 +1,7 @@ //! Panel bodies. pub mod anim_tools; +pub mod camera_tracker_ui; pub mod comp_settings; pub mod dialogs; pub mod dialogs_3d; diff --git a/crates/ui-egui/src/panels/tracker.rs b/crates/ui-egui/src/panels/tracker.rs index d1f456b2..364e114e 100644 --- a/crates/ui-egui/src/panels/tracker.rs +++ b/crates/ui-egui/src/panels/tracker.rs @@ -130,11 +130,14 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let cur = current(app); let can_track = has_comp && source.is_some(); - // Track Camera / Warp Stabilizer (not available) and Track Motion / Stabilize Motion. + // Track Camera / Warp Stabilizer and Track Motion / Stabilize Motion. let r1 = Rect::from_min_size(pos2(x0, y), vec2(half, 22.0)); let r2 = Rect::from_min_size(pos2(x0 + half + 6.0, y), vec2(half, 22.0)); let src_p = source.map(|l| json!({"layer": l.0})).unwrap_or_else(|| json!({})); - button(app, ui, r1, "Track Camera", false, "tracker.trackCamera"); + let can_camera = app.session.is_enabled("track.camera") && !app.session.is_camera_analyzing(); + if button(app, ui, r1, "Track Camera", can_camera, "tracker.trackCamera") { + run(app, &ctx, "track.camera", json!({})); + } let can_warp = app.session.is_enabled("track.warpStabilizer") && !app.session.is_warp_analyzing(); if button(app, ui, r2, "Warp Stabilizer", can_warp, "tracker.warpStabilizer") { run(app, &ctx, "track.warpStabilizer", json!({})); diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index d847edb8..c9dd70db 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -1144,6 +1144,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { app.session.open_comp(*item); } } + // 3D Camera Tracker points, targets and their menu (on top of the viewer's gestures while + // the effect is selected), and its analysis banner. + super::camera_tracker_ui::viewer_hook(app, ui, &painter, &map, &ectx, &|c, l| l2c(c, l).0); // Effect point controls and crosshair/eyedropper picks (on top of the viewer's gestures). crate::panels::effect_controls::viewer_hook(app, ui, &painter, &map, &ectx, &|c, l| l2c(c, l).0); diff --git a/crates/ui-egui/tests/ui_camera_tracker.rs b/crates/ui-egui/tests/ui_camera_tracker.rs new file mode 100644 index 00000000..45374fba --- /dev/null +++ b/crates/ui-egui/tests/ui_camera_tracker.rs @@ -0,0 +1,175 @@ +//! Headless checks for the 3D Camera Tracker UI (egui_kittest, UI logic only): the viewer's track +//! points and selection, the Effect Controls buttons and the Tracker panel's Track Camera button. +//! The solve is a synthetic one written into the effect (the analysis itself is tested in the +//! engine and track crates). + +use effectcraft_engine::Session; +use effectcraft_engine::effects::camera_tracker as ct; +use effectcraft_engine::track::camtrack::{CameraSolve, ShotType, SolveMethod, SolvedFrame, SolvedPoint}; +use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::dock::PanelKind; +use egui_kittest::Harness; +use serde_json::json; + +const W: f64 = 640.0; +const H: f64 = 360.0; + +fn app() -> (EffectcraftApp, u64, u64) { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Shot", "width": W, "height": H, "duration": 2})).unwrap(); + let plate = s.execute("layer.newSolid", json!({"name": "Plate", "color": "#406080"})).unwrap()["layer"].as_u64().unwrap(); + let r = s.execute("effect.apply", json!({"effect": ct::ID, "layers": [plate]})).unwrap(); + let uid = r["effects"][0].as_u64().unwrap(); + // A synthetic solve: a static camera (focal 600) looking at a 6 × 4 grid of points on z = 1. + let frames = 60; + let solve = CameraSolve { + version: 1, + size: [W, H], + start: 0.0, + frame_duration: 1.0 / 30.0, + shot: ShotType::FixedAngle, + method_used: SolveMethod::Typical, + average_error: 0.21, + frames: vec![SolvedFrame { rot: [0.0; 3], center: [0.0; 3], focal: 600.0, solved: true }; frames], + points: (0..24) + .map(|i| SolvedPoint { + id: i, + pos: [((i % 6) as f64 - 2.5) * 0.15, ((i / 6) as f64 - 1.5) * 0.12, 1.0 + 0.01 * (i % 3) as f64], + error: 0.2, + first: 0, + last: frames as u32 - 1, + }) + .collect(), + ground: None, + }; + s.execute("prop.set", json!({"layer": plate, "path": "effects/#1/solve", "value": solve.to_json()})).unwrap(); + s.camera_pending.clear(); + s.state.selected_props = vec![(effectcraft_engine::project::LayerId(plate), uid)]; + s.execute("layer.select", json!({"layers": [plate]})).unwrap(); + s.state.selected_props = vec![(effectcraft_engine::project::LayerId(plate), uid)]; + (EffectcraftApp::new(s), plate, uid) +} + +fn ids(h: &Harness<'_, EffectcraftApp>) -> Vec { + h.state().auto.previous.iter().chain(h.state().auto.elements.iter()).map(|e| e.id.clone()).collect() +} + +fn rect(h: &Harness<'_, EffectcraftApp>, id: &str) -> egui::Rect { + let e = h.state().auto.find(id).unwrap_or_else(|| panic!("no {id}")); + egui::Rect::from_min_size(egui::pos2(e.rect[0], e.rect[1]), egui::vec2(e.rect[2], e.rect[3])) +} + +fn click(h: &mut Harness<'_, EffectcraftApp>, at: egui::Pos2, modifiers: egui::Modifiers) { + h.input_mut().events.push(egui::Event::ModifiersChanged(modifiers)); + h.input_mut().events.push(egui::Event::PointerMoved(at)); + h.input_mut().events.push(egui::Event::PointerButton { pos: at, button: egui::PointerButton::Primary, pressed: true, modifiers }); + h.input_mut().events.push(egui::Event::PointerButton { pos: at, button: egui::PointerButton::Primary, pressed: false, modifiers }); + h.run_steps(3); + h.input_mut().events.push(egui::Event::ModifiersChanged(Default::default())); + h.step(); +} + +#[test] +fn viewer_points_selection_and_effect_controls() { + let (a, plate, uid) = app(); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).build_eframe(|_| a); + h.state_mut().show_panel(PanelKind::Composition); + h.run_steps(4); + let all = ids(&h); + assert!(all.iter().any(|i| i == "viewer.cameraTracker"), "track point overlay registered"); + let mut pts: Vec<&String> = all.iter().filter(|i| i.starts_with("viewer.cameraTracker.point.")).collect(); + pts.sort(); + pts.dedup(); + let n = pts.len(); + assert_eq!(n, 24, "every visible point is drawn"); + // Click a point: it is selected; Shift-click another adds it. + let r = rect(&h, "viewer.cameraTracker.point.7"); + click(&mut h, r.center(), Default::default()); + assert_eq!(h.state().session.state.camera_points, vec![7]); + let r = rect(&h, "viewer.cameraTracker.point.9"); + click(&mut h, r.center(), egui::Modifiers::SHIFT); + assert_eq!(h.state().session.state.camera_points, vec![7, 9]); + // Marquee over the first row selects its six points. + let (a0, a5) = (rect(&h, "viewer.cameraTracker.point.0"), rect(&h, "viewer.cameraTracker.point.5")); + let from = a0.left_top() - egui::vec2(6.0, 6.0); + let to = a5.right_bottom() + egui::vec2(6.0, 6.0); + h.input_mut().events.push(egui::Event::PointerMoved(from)); + h.input_mut().events.push(egui::Event::PointerButton { pos: from, button: egui::PointerButton::Primary, pressed: true, modifiers: Default::default() }); + h.step(); + for i in 1..=8 { + h.input_mut().events.push(egui::Event::PointerMoved(from + (to - from) * (i as f32 / 8.0))); + h.step(); + } + h.input_mut().events.push(egui::Event::PointerButton { pos: to, button: egui::PointerButton::Primary, pressed: false, modifiers: Default::default() }); + h.run_steps(3); + let mut sel = h.state().session.state.camera_points.clone(); + sel.sort(); + assert_eq!(sel, vec![0, 1, 2, 3, 4, 5]); + // The right-click menu's Create Solid and Camera (via its command). + let ctx = h.ctx.clone(); + effectcraft_ui_egui::menus::invoke(h.state_mut(), &ctx, "camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + let comp = h.state().session.active_comp().unwrap(); + assert!(comp.layers.iter().any(|l| l.name == "3D Tracker Camera" && l.is_camera())); + assert!(comp.layers.iter().any(|l| l.name.starts_with("Track Solid"))); + + // Effect Controls: Analyze and Create Camera. + h.state_mut().session.execute("layer.select", json!({"layers": [plate]})).unwrap(); + h.state_mut().show_panel(PanelKind::EffectControls); + h.run_steps(4); + let all = ids(&h); + for want in [format!("effectControls.cameraTracker.{uid}.analyze"), format!("effectControls.cameraTracker.{uid}.createCamera")] { + assert!(all.contains(&want), "missing {want}"); + } + // Tracker panel: Track Camera is available for the selected layer. + h.state_mut().show_panel(PanelKind::Tracker); + h.run_steps(3); + assert!(ids(&h).iter().any(|i| i == "tracker.trackCamera")); + assert!(h.state().session.is_enabled("track.camera")); +} + +/// Step the UI until background frame renders have landed. +fn settle(h: &mut Harness<'_, EffectcraftApp>) { + for _ in 0..600 { + h.step(); + if h.state().frames.inflight() == 0 && h.state().frames.last_ms.lock().map(|v| *v > 0.0).unwrap_or(false) { + break; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + for _ in 0..4 { + h.step(); + } +} + +#[test] +#[ignore = "analyses the demo's 3D Showcase and renders with wgpu; run with --ignored (set EC_SNAPSHOT_DIR to keep PNGs)"] +fn snapshot_showcase_track() { + let dir = std::env::var("EC_SNAPSHOT_DIR").ok(); + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + s.execute("comp.new", json!({"name": "Camera Track", "width": 1920, "height": 1080, "duration": 4, "frameRate": 29.97})).unwrap(); + s.execute("layer.addItem", json!({"item": "3D Showcase"})).unwrap(); + s.execute("layer.select", json!({"layers": ["#1"]})).unwrap(); + s.execute("track.camera", json!({"layer": "#1", "wait": true})).unwrap(); + s.execute("time.set", json!({"time": 2.0})).unwrap(); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).with_pixels_per_point(1.0).wgpu().build_eframe(|_| EffectcraftApp::new(s)); + h.state_mut().show_panel(PanelKind::Composition); + settle(&mut h); + let pts: Vec = ids(&h).into_iter().filter(|i| i.starts_with("viewer.cameraTracker.point.")).collect(); + assert!(pts.len() > 10, "{}", pts.len()); + // Hover between points to show a target, then select three and show the selection's target. + let r = rect(&h, &pts[0]); + h.input_mut().events.push(egui::Event::PointerMoved(r.center() + egui::vec2(12.0, 12.0))); + h.run_steps(2); + if let Some(d) = &dir { + h.render().expect("render").save(format!("{d}/camera_tracker_hover.png")).unwrap(); + } + let sel: Vec = pts.iter().take(5).filter_map(|i| i.rsplit('.').next()?.parse().ok()).collect(); + h.state_mut().session.execute("camera.selectPoints", json!({"points": sel})).unwrap(); + h.state_mut().session.execute("camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + h.state_mut().session.execute("layer.select", json!({"layers": ["#3"]})).unwrap(); + settle(&mut h); + if let Some(d) = &dir { + h.render().expect("render").save(format!("{d}/camera_tracker_solid.png")).unwrap(); + } +} diff --git a/docs/agents.md b/docs/agents.md index 510bc356..bda72461 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -146,6 +146,34 @@ effectcraft-cli exec warp.analyze '{"layer":"#1","wait":true}' shaky.ecproj --sa effectcraft-cli exec warp.status '{"layer":"#1"}' shaky.ecproj --json ``` +### 3D Camera Tracker + +Animation ▸ Track Camera and the Tracker panel's button (`track.camera {layer?, shotType?: +fixed|variable|specify, aov?, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}`) apply +Effect ▸ Perspective ▸ 3D Camera Tracker (or reuse the layer's) and analyse it in the background: +"Analyzing in background (step 1 of 2)" tracks features, "Solving camera" solves. +`camera.analyze {layer?, effect?, wait?}` re-runs it, `camera.cancel` stops it without writing +anything. `camera.solveStatus` reports `progress`/`banner`, `analyzed`, `solved`, `methodUsed`, +`averageError` (pixels), `focalLength`, `horizontalAngleOfView`, the current frame's `camera` +(position, orientation, zoom) and `groundPlane`. `camera.points {time?}` lists the solved points +visible now with their `id`, `comp` position, `depth`, `world` position and `error`. + +Select points with `camera.selectPoints {points, add?, toggle?}` (the viewer's click / Shift-click / +marquee), then the right-click menu's commands: `camera.setGroundPlane {points?}` (Set Ground Plane +and Origin), `camera.createFromSolve {kind: text|solid|null|shadowCatcher, points? | target: +{center, normal, size?}, multiple?}` (Create Text / Solid / Null / Shadow Catcher, Camera and Light, +Create Multiple …), `camera.deletePoints {points?, wait?}` (re-solves; with Auto-delete Points +Across Time the same feature's other tracks go too) and `camera.create` (the Create Camera button). +Each is one undo step; the first create adds the one-node "3D Tracker Camera" keyed on every frame. +Changing the layer's frames clears the analysis; changing `effects/#1/shotType`, +`effects/#1/horizontalAngleOfView`, `effects/#1/advanced/solveMethod` re-solves the stored tracks. + +```sh +effectcraft-cli run shot.ecproj track.camera '{"layer":"#1","wait":true}' \ + camera.points '{}' --json +effectcraft-cli run shot.ecproj camera.createFromSolve '{"kind":"solid","points":[12,40,77]}' --save +``` + ### Text: styles and editing Source Text holds character style runs and per-paragraph settings. `layer.setText` changes the diff --git a/docs/architecture.md b/docs/architecture.md index 033ed6cb..f9f0586f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -176,6 +176,22 @@ auto-scale; cached per analysis and settings) and warps each frame; Synthesize E borders from neighbouring frames read with `EffectHost::self_at`. The effect lives in the effects crate, hence the `effects → track` edge. +**3D Camera Tracker** (`effects::camera_tracker`, `engine::camera_track`, `effectcraft-track`'s +`camtrack`): `camera.analyze` / `track.camera` render the layer's input to the effect on a +background thread and run structure from motion in two steps. Step 1 follows Shi–Tomasi features +through the clip with pyramidal Lucas–Kanade (forward–backward checked, re-detected where the frame +has none) into long 2D tracks. Step 2 solves the camera: parallax keyframes; a two-view start from +the essential matrix (normalised 8-point in RANSAC) or a planar homography decomposition; incremental +resection and triangulation; a sparse Levenberg–Marquardt bundle adjustment (Schur complement over +the points, Huber loss) with the focal length fixed, shared, or per frame; a log-spaced and +golden-section focal search for Fixed Angle of View / Variable Zoom; a rotation-only model for tripod +pans, chosen by Auto Detect when it explains the tracks as well. Tracks and solve are stored as JSON +in hidden effect parameters with keys: changing the layer's frames clears both, changing Shot Type, +Angle of View, Solve Method or deleting points only re-solves. The solve's canonical frame maps to +comp space so the first frame's camera is the default comp camera (or so a chosen ground plane is +the X-Z plane at the origin); `camera.createFromSolve` keys a one-node "3D Tracker Camera" on every +frame and places text, solids, nulls or a shadow catcher and light on the target plane. + ## 5. Expressions Expressions are JavaScript, run by boa, with After Effects' object model (`thisComp`, `thisLayer`, diff --git a/docs/parity.md b/docs/parity.md index d259fec2..25e5a080 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -15,7 +15,7 @@ rendered, does not count as done. | P1 (professional daily use) | ≈ 38% | | P2 (long tail) | ≈ 17% | | Features done / partial / missing | 18 / 60 / 14 of 92 | -| **Effects** | **257 of 298** After Effects 2026 effects by name (86%); ≈ 70% allowing for simplified implementations | +| **Effects** | **258 of 298** After Effects 2026 effects by name (87%); ≈ 70% allowing for simplified implementations | | Remaining work | ≈ **167 agent-hours** (139 h of features + ≈ 20% for 1:1 polish against After Effects) | | Wall-clock estimate | ≈ **42–50 hours** with five agents working in parallel and one integrating; ≈ 31–35 h for P0 + P1 only | @@ -73,7 +73,7 @@ library groundwork for mask tracking and Warp Stabilizer. | Remaining (audit-scale agent-hours) | ≈ 167 | ≈ 100 | | Wall clock, five parallel agents | 42–50 h | **≈ 8–10 h at today's measured pace; ≈ 25 h by the audit's conservative figures** | -What is left is concentrated in large systems: Roto Brush and Refine Edge, the 3D camera tracker, +What is left is concentrated in large systems: Roto Brush and Refine Edge, Advanced 3D (models, PBR, image-based lighting), the Warp Stabilizer and mask-tracking UI on top of the new tracking library, a JavaScript scripting object model, SVG/PSD import, WebM, a disk cache, the missing effect categories (3D Channel, Immersive Video, OCIO), native macOS menus, and @@ -93,13 +93,13 @@ wasm threads. | Animation | 65% | 10.4 | puppet, Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8) | | Text | ≈ 85% | 1.5 | vertical type is basic (upright characters, no tate-chu-yoko), no OpenType feature panel, no text on 3D bevels; per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API landed (M9.9–M9.10) | | Web | 60% | 2.5 | threads, browser storage, audio | -| 3D | 58% | 20.5 | Advanced 3D (models, PBR, image-based light), 3D camera tracker, multi-view layouts | +| 3D | 62% | 18.5 | Advanced 3D (models, PBR, image-based light), multi-view layouts (the 3D Camera Tracker landed in M12.6) | | Effects | 58% | 13.6 | Effect Controls widgets (angle dial, point crosshair, eyedropper, curves), GPU versions of more effects, 41 missing effects | | Interface | 55% | 10.4 | drag-to-dock and floating panels, preferences, native macOS menus (viewer rulers/snapping/channels/snapshots landed in M0.13) | | Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves | | Masks & roto | 52% | 10.0 | Roto Brush, variable-width mask feather points (mask tracking and Mask Interpolation landed in M6.6) | | Preview | 55% | 6.1 | GPU 3D and adjustment layers, disk cache (region of interest, snapshots, exposure and Fast Previews landed in M0.13) | -| Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking, Subspace Warp's mesh warp and rolling-shutter repair (point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) | +| Tracking | ≈ 85% | 2.5 | face tracking, Subspace Warp's mesh warp, rolling-shutter repair, lens distortion in the camera solve (point tracker, mask tracking, Warp Stabilizer and the 3D Camera Tracker landed in M6.x / M12.5 / M12.6) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | ## Effects still missing @@ -107,7 +107,7 @@ wasm threads. Immersive Video (all 12), 3D Channel (all 8, including Cryptomatte), Color Stabilizer and the five OCIO transforms, CC Flo Motion, Liquify, Rolling Shutter Repair, Compressor, Distortion and Gate (audio), Camera-Shake Deblur, CC Radial Blur, CC Hair, Particle Playground, -Keylight (our Screen Key stands in), Color Profile Converter and the 3D Camera Tracker. Boris FX +Keylight (our Screen Key stands in) and Color Profile Converter. Boris FX Mocha and Cineware are third-party and not counted. ## Highest-value gaps, in order