diff --git a/Cargo.lock b/Cargo.lock index 6a71540f..6108b643 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1504,6 +1504,7 @@ dependencies = [ "rayon", "serde", "serde_json", + "web-time", ] [[package]] diff --git a/ROADMAP.md b/ROADMAP.md index 55b05822..2bf7d8ba 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -28,10 +28,10 @@ once. Measured feature by feature in [docs/parity.md](docs/parity.md) (end of 2 October 2026): - **≈ 79% of After Effects' features, weighted by importance** (87% of the essentials, 67% of - professional daily-use features, 21% of the long tail); 257 of 298 effects, 16 of them on the GPU. + professional daily-use features, 21% of the long tail); 298 of 298 effects, 16 of them on the GPU. - **≈ 100 agent-hours of work remain by the audit's estimate.** With five Claude Opus 5.5 agents in parallel that is **≈ 8–10 hours of wall-clock time at the pace measured so far, up to ≈ 25 hours** by the audit's conservative per-feature figures. -- Hardest remaining pieces: the 3D camera tracker, Advanced 3D, a scripting object model (Roto Brush & Refine Edge landed in M6.7). +- Hardest remaining pieces: Advanced 3D, a scripting object model (Roto Brush & Refine Edge landed in M6.7, the 3D Camera Tracker in M12.6). Come tell us what matters most to you on [Discord](https://discord.gg/artcraft). diff --git a/crates/effects/src/camera_tracker.rs b/crates/effects/src/camera_tracker.rs new file mode 100644 index 00000000..dd97b585 --- /dev/null +++ b/crates/effects/src/camera_tracker.rs @@ -0,0 +1,291 @@ +//! 3D Camera Tracker (Effect ▸ Perspective ▸ 3D Camera Tracker; Animation ▸ Track Camera). +//! +//! The analysis runs as a background job in the engine (`camera.analyze`): step 1 tracks +//! features through the clip ([`effectcraft_track::camtrack::TrackAnalyzer`]), step 2 solves +//! the camera ([`effectcraft_track::camtrack::solve`]). Both results are stored as JSON in the +//! instance's hidden parameters — **Tracks** (with **Tracks Key**: what the frames were made +//! from, so source / In-Out / time edits invalidate them) and **Solve** (with **Solve Key**: the +//! Shot Type, Angle of View, Solve Method and deleted points it was solved with, so changing +//! those re-solves without re-tracking). **Method Used** and **Average Error** report the solve. +//! +//! The effect passes its input through; with **Render Track Points** on it draws the solved +//! points (as the viewer shows them) into the frame. The viewer overlay, target and the Create +//! commands live in the engine and the UI. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::{Image, Px}; +use effectcraft_track::camtrack::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, linalg, point_color}; + +use crate::warp_stab::param; +use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; + +/// The effect's id. +pub const ID: &str = "ec.perspective.cameratracker"; + +/// Hidden parameters (see the module docs). +pub const TRACKS: &str = "tracks"; +pub const TRACKS_KEY: &str = "tracksKey"; +pub const SOLVE: &str = "solve"; +pub const SOLVE_KEY: &str = "solveKey"; +/// Deleted track point ids (JSON array). +pub const DELETED: &str = "deleted"; +/// The "3D Tracker Camera" layer the instance created (layer id, 0 = none). +pub const CAMERA: &str = "camera"; +pub const METHOD_USED: &str = "advanced/methodUsed"; +pub const AVERAGE_ERROR: &str = "advanced/averageError"; + +pub const SHOT_TYPES: [&str; 3] = ["Fixed Angle of View", "Variable Zoom", "Specify Angle of View"]; +pub const SHOW_POINTS: [&str; 2] = ["2D Source", "3D Solved"]; +pub const SOLVE_METHODS: [&str; 4] = ["Auto Detect", "Typical", "Mostly Flat Scene", "Tripod Pan"]; + +pub fn specs() -> Vec { + vec![EffectSpec { + id: ID, + name: "3D Camera Tracker", + category: "Perspective", + params: vec![ + p("shotType", "Shot Type", Value::Enum(0), popup(&SHOT_TYPES)), + p("horizontalAngleOfView", "Horizontal Angle of View", num(40.0), slider(1.0, 170.0, 10.0, 120.0, 1)), + p("showTrackPoints", "Show Track Points", Value::Enum(1), popup(&SHOW_POINTS)), + p("renderTrackPoints", "Render Track Points", Value::Bool(false), ParamUi::Checkbox), + p("trackPointSize", "Track Point Size", num(100.0), slider(0.0, 1000.0, 0.0, 300.0, 0)), + p("targetSize", "Target Size", num(100.0), slider(1.0, 1000.0, 10.0, 300.0, 0)), + p("advanced/solveMethod", "Solve Method", Value::Enum(0), popup(&SOLVE_METHODS)), + p(METHOD_USED, "Method Used", Value::Str(String::new()), ParamUi::Hidden), + p(AVERAGE_ERROR, "Average Error", num(0.0), ParamUi::Hidden), + p("advanced/detailedAnalysis", "Detailed Analysis", Value::Bool(false), ParamUi::Checkbox), + p("advanced/autoDeletePoints", "Auto-delete Points Across Time", Value::Bool(true), ParamUi::Checkbox), + p("advanced/hideWarningBanner", "Hide Warning Banner", Value::Bool(false), ParamUi::Checkbox), + p(TRACKS, "Tracks", Value::Str(String::new()), ParamUi::Hidden), + p(TRACKS_KEY, "Tracks Key", Value::Str(String::new()), ParamUi::Hidden), + p(SOLVE, "Solve", Value::Str(String::new()), ParamUi::Hidden), + p(SOLVE_KEY, "Solve Key", Value::Str(String::new()), ParamUi::Hidden), + p(DELETED, "Deleted Points", Value::Str(String::new()), ParamUi::Hidden), + p(CAMERA, "Camera Layer", num(0.0), ParamUi::Hidden), + ], + render, + gpu: false, + float: true, + }] +} + +fn f(params: &Params, id: &str) -> f64 { + param(params, id).map(Value::as_f64).unwrap_or(0.0) +} +fn e(params: &Params, id: &str) -> u32 { + param(params, id).map(Value::as_enum).unwrap_or(0) +} +fn b(params: &Params, id: &str) -> bool { + param(params, id).map(Value::as_bool).unwrap_or(false) +} +fn s<'a>(params: &'a Params, id: &str) -> &'a str { + match param(params, id) { + Some(Value::Str(s)) => s, + _ => "", + } +} + +/// Deleted point ids of an instance (stored as a JSON array of numbers). +pub fn deleted(params: &Params) -> Vec { + s(params, DELETED).trim().trim_start_matches('[').trim_end_matches(']').split(',').filter_map(|v| v.trim().parse().ok()).collect() +} + +/// The solve settings of an instance. +pub fn settings(params: &Params) -> SolveSettings { + let mut del = deleted(params); + del.sort_unstable(); + del.dedup(); + SolveSettings { + shot: ShotType::ALL.get(e(params, "shotType") as usize).copied().unwrap_or_default(), + hfov: f(params, "horizontalAngleOfView"), + method: SolveMethod::ALL.get(e(params, "advanced/solveMethod") as usize).copied().unwrap_or_default(), + deleted: del, + } +} + +/// What a solve depends on besides the tracks. +pub fn solve_key(st: &SolveSettings) -> String { + let hfov = if st.shot == ShotType::SpecifyAngle { format!("{:.4}", st.hfov) } else { String::new() }; + format!("{:?}|{hfov}|{:?}|{:?}", st.shot, st.method, st.deleted) +} + +fn fnv(bytes: &[u8]) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for b in bytes { + h ^= *b as u64; + h = h.wrapping_mul(0x0100_0000_01b3); + } + h +} + +type Cache = Mutex>>; + +fn cached(c: &'static OnceLock>, json: &str, parse: impl Fn(&str) -> Option) -> Option> { + let key = fnv(json.as_bytes()); + let c = c.get_or_init(Default::default); + if let Some(a) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&key) { + return Some(a.clone()); + } + let a = Arc::new(parse(json)?); + let mut m = c.lock().unwrap_or_else(|e| e.into_inner()); + if m.len() > 8 { + m.clear(); + } + m.insert(key, a.clone()); + Some(a) +} + +/// The stored feature tracks of an instance (`None` when not analysed). +pub fn tracks(params: &Params) -> Option> { + static C: OnceLock> = OnceLock::new(); + let j = s(params, TRACKS); + if j.is_empty() { + return None; + } + cached(&C, j, CameraTracks::from_json).filter(|t| !t.is_empty()) +} + +/// The stored camera solve of an instance (`None` when not solved). +pub fn solve(params: &Params) -> Option> { + static C: OnceLock> = OnceLock::new(); + let j = s(params, SOLVE); + if j.is_empty() { + return None; + } + cached(&C, j, CameraSolve::from_json).filter(|t| !t.is_empty()) +} + +/// Screen size (layer pixels) of a target drawn for a point at depth `z`, relative to the +/// median depth of the frame's points. +pub fn target_radius(size_pct: f64, z: f64, median_z: f64) -> f64 { + (6.0 * size_pct / 100.0 * (median_z / z.max(1e-9))).clamp(1.5, 40.0) +} + +/// The solved points visible on frame `k`: (id, layer pixel position, camera depth). +pub fn projected(solve: &CameraSolve, k: usize) -> Vec<(u32, [f64; 2], f64)> { + let Some(cam) = solve.frames.get(k) else { return vec![] }; + solve + .visible(k) + .filter_map(|p| { + let c = cam.to_cam(p.pos); + cam.project(solve.size, p.pos).map(|q| (p.id, q, c[2])) + }) + .collect() +} + +/// Median of the depths. +pub fn median_depth(v: &[(u32, [f64; 2], f64)]) -> f64 { + let mut z: Vec = v.iter().map(|x| x.2).collect(); + if z.is_empty() { + return 1.0; + } + z.sort_by(f64::total_cmp); + z[z.len() / 2] +} + +fn render(ctx: &EffectCtx, mut buf: Buf) -> Buf { + if !b(ctx.params, "renderTrackPoints") { + return buf; + } + let Some(sol) = solve(ctx.params) else { return buf }; + if (sol.size[0] - ctx.layer_size[0]).abs() > 0.5 || (sol.size[1] - ctx.layer_size[1]).abs() > 0.5 { + return buf; + } + let Some(k) = sol.frame_at(ctx.time) else { return buf }; + let pts = projected(&sol, k); + let med = median_depth(&pts); + let size = f(ctx.params, "trackPointSize"); + let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + for (id, q, z) in pts { + let c = point_color(id); + let r = target_radius(size, z, med) * sc; + let at = [q[0] * sc + buf.offset[0], q[1] * sc + buf.offset[1]]; + ring(&mut buf.img, at, r, (r * 0.25).max(1.0), [c[0], c[1], c[2], 1.0]); + disc(&mut buf.img, at, (r * 0.2).max(0.8), [c[0], c[1], c[2], 1.0]); + } + buf +} + +fn lerp_px(a: Px, b: Px, t: f32) -> Px { + [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t, a[2] + (b[2] - a[2]) * t, a[3] + (b[3] - a[3]) * t] +} + +fn cover(img: &mut Image, c: [f64; 2], r_out: f64, cov: impl Fn(f64) -> f64, col: Px) { + let (x0, x1) = ((c[0] - r_out - 1.0).floor().max(0.0) as i64, (c[0] + r_out + 1.0).ceil() as i64); + let (y0, y1) = ((c[1] - r_out - 1.0).floor().max(0.0) as i64, (c[1] + r_out + 1.0).ceil() as i64); + for y in y0..y1.min(img.height as i64) { + for x in x0..x1.min(img.width as i64) { + let d = (x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1]); + let a = cov(d).clamp(0.0, 1.0) as f32; + if a > 0.0 { + let i = img.idx(x as u32, y as u32); + img.data[i] = lerp_px(img.data[i], col, a); + } + } + } +} + +fn disc(img: &mut Image, c: [f64; 2], r: f64, col: Px) { + cover(img, c, r, |d| r + 0.5 - d, col); +} + +fn ring(img: &mut Image, c: [f64; 2], r: f64, w: f64, col: Px) { + cover(img, c, r + w, |d| (w * 0.5 + 0.5 - (d - r).abs()).min(1.0), col); +} + +/// Points of a target disc (canonical coordinates) for drawing: `n` points on the circle of +/// radius `size / 2` in the plane through `center` with normal `normal`. +pub fn target_circle(center: linalg::V3, normal: linalg::V3, radius: f64, n: usize) -> Vec { + let a = if normal[0].abs() < 0.9 { [1.0, 0.0, 0.0] } else { [0.0, 1.0, 0.0] }; + let u = linalg::normalize(linalg::cross(normal, a)); + let v = linalg::cross(normal, u); + (0..n) + .map(|i| { + let t = i as f64 / n as f64 * std::f64::consts::TAU; + linalg::add(center, linalg::add(linalg::scale(u, radius * t.cos()), linalg::scale(v, radius * t.sin()))) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::EffectEnv; + use effectcraft_track::camtrack::{SolvedFrame, SolvedPoint}; + + fn params_with(solve: &CameraSolve, render: bool) -> Params { + let mut p = Params::default(); + p.values.insert(SOLVE.into(), Value::Str(solve.to_json())); + p.values.insert("renderTrackPoints".into(), Value::Bool(render)); + p.values.insert("trackPointSize".into(), Value::Scalar(100.0)); + p + } + + #[test] + fn render_track_points_draws_targets_and_passes_through_otherwise() { + let sol = CameraSolve { + version: 1, + size: [64.0, 48.0], + start: 0.0, + frame_duration: 1.0 / 30.0, + frames: vec![SolvedFrame { rot: [0.0; 3], center: [0.0; 3], focal: 50.0, solved: true }], + points: vec![SolvedPoint { id: 3, pos: [0.0, 0.0, 1.0], error: 0.1, first: 0, last: 0 }], + ..Default::default() + }; + let img = Image::new(64, 48); + for on in [false, true] { + let params = params_with(&sol, on); + let ctx = EffectCtx { params: ¶ms, time: 0.0, layer_size: [64.0, 48.0], seed: 0, adjustment: false, env: EffectEnv::default() }; + let out = render(&ctx, Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 }); + let px = out.img.data[out.img.idx(32, 24)]; + assert_eq!(px[3] > 0.5, on); + } + let st = settings(&Params::default()); + assert_eq!(st.shot, ShotType::FixedAngle); + assert!(solve_key(&st).contains("FixedAngle")); + } +} diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index c615df8d..6f47b01e 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -9,6 +9,7 @@ pub mod audio_fx; mod blur2; mod blur3; +pub mod camera_tracker; mod channel; mod channel2; mod channel3d; @@ -382,6 +383,7 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(paint::specs()); v.extend(puppet::specs()); v.extend(warp_stab::specs()); + v.extend(camera_tracker::specs()); v.extend(roto::specs()); v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); for s in v.iter_mut() { @@ -530,6 +532,8 @@ pub const TIME_DEPENDENT: &[&str] = &[ "ec.obsolete.mochashape", // Each frame gets its own stabilizing warp. warp_stab::ID, + // Render Track Points draws each frame's solved points. + camera_tracker::ID, // Each frame has its own segmentation. roto::ID, ]; diff --git a/crates/engine/src/camera_track.rs b/crates/engine/src/camera_track.rs new file mode 100644 index 00000000..1b984aac --- /dev/null +++ b/crates/engine/src/camera_track.rs @@ -0,0 +1,640 @@ +//! 3D Camera Tracker runtime: the analysis job (`camera.analyze`, `track.camera`), keeping +//! analyses valid, and turning a solve into composition layers (`camera.createFromSolve`). +//! +//! The job renders the layer's input to the effect (`Renderer::layer_input`) for every frame +//! between the layer's In and Out points on a background thread and tracks features through +//! them (step 1 of 2, "Analyzing in background"), then solves the camera (step 2, "Solving +//! camera"). When only the solve settings changed (Shot Type, Angle of View, Solve Method, +//! deleted points) the stored tracks are re-solved without re-tracking. Results are written into +//! the effect's hidden parameters as one undo step; edits that change the layer's frames clear +//! them and queue the effect again ([`invalidate`], run after every edit), like After Effects. +//! +//! Composition space: the solve's canonical frame maps to comp world by +//! [`CameraSolve::world`] with the zoom of the first solved frame (focal length × the layer's +//! scale in the comp), so the "3D Tracker Camera" reproduces the footage's view and created +//! layers sit on the selected points. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use effectcraft_effects::camera_tracker::{self as ct, AVERAGE_ERROR, METHOD_USED, SOLVE, SOLVE_KEY, TRACKS, TRACKS_KEY}; +use effectcraft_keyframe::{Keyframe, Value}; +use effectcraft_project::{AutoOrient, GroupKind, ItemId, ItemKind, Layer, LayerId, Project, PropGroup, Uid}; +use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; +use effectcraft_time::Tick; +use effectcraft_track::camtrack::linalg::{self, Similarity, V3}; +use effectcraft_track::camtrack::{AnalyzeOpts, CameraSolve, CameraTracks, SolveSettings, Target, TrackAnalyzer}; +use serde::Serialize; + +use crate::tracking::lock; +use crate::warp::{analysis_times, signature}; +use crate::{Event, Session}; + +/// Live analysis progress (serde for agents / the control channel). +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CameraProgress { + /// 1 = tracking features, 2 = solving the camera. + pub step: u8, + pub done: u64, + pub total: u64, + pub elapsed: f64, + pub fps: f64, + pub finished: bool, + pub cancelled: bool, + pub error: Option, + /// Re-solving stored tracks (no tracking step). + pub solve_only: bool, +} + +impl CameraProgress { + /// The viewer banner text, as After Effects words it. + pub fn banner(&self) -> String { + match self.step { + 2 => "Solving camera".into(), + _ => { + let pc = (100 * self.done).checked_div(self.total).unwrap_or(0); + format!("Analyzing in background (step 1 of 2): {pc}%") + } + } + } +} + +struct CamResult { + /// New tracks (None for a re-solve). + tracks: Option, + solve: Result, +} + +#[derive(Default)] +pub struct CameraShared { + pub cancel: AtomicBool, + pub state: Mutex, + result: Mutex>, +} + +/// A running (or finished, not yet polled) camera analysis. +pub struct CameraJob { + pub shared: Arc, + thread: Option>, + pub comp: ItemId, + pub layer: LayerId, + pub effect: Uid, + /// The layer signature the tracks are made from, and the solve settings key. + key: String, + solve_key: String, +} + +impl CameraJob { + pub fn is_finished(&self) -> bool { + lock(&self.shared.state).finished + } + pub fn progress(&self) -> CameraProgress { + lock(&self.shared.state).clone() + } +} + +struct CamWork { + project: Arc, + footage: Arc, + expr: Option>, + cache: Arc, + comp: ItemId, + layer: LayerId, + effect_index: usize, + size: [f64; 2], + detailed: bool, + times: Vec, + /// Stored tracks to re-solve (skips step 1). + tracks: Option>, + settings: SolveSettings, +} + +pub(crate) fn is_tracker(g: &PropGroup) -> bool { + matches!(&g.kind, GroupKind::Effect { effect } if effect == ct::ID) +} + +fn str_of(g: &PropGroup, m: &str) -> String { + match g.prop(m).map(|p| &p.value) { + Some(Value::Str(s)) => s.clone(), + _ => String::new(), + } +} + +/// What a tracker's tracks are made from: the layer's frames and Detailed Analysis. +fn tracks_key(project: &Project, cid: ItemId, l: &Layer, g: &PropGroup) -> String { + let detailed = effectcraft_effects::warp_stab::param(&static_params(g), "advanced/detailedAnalysis").is_some_and(Value::as_bool); + format!("{}{}", signature(project, cid, l), if detailed { "+detailed" } else { "" }) +} + +/// Parameters of an instance from static values (the tracker's settings are not animated). +pub fn static_params(g: &PropGroup) -> effectcraft_effects::Params { + effectcraft_effects::flatten_params(g, &mut |p| p.value.clone()) +} + +fn clear(g: &mut PropGroup, all: bool) { + let mut ids = vec![SOLVE, SOLVE_KEY, METHOD_USED]; + if all { + ids.extend([TRACKS, TRACKS_KEY]); + } + for m in ids { + if let Some(p) = g.prop_mut(m) { + p.value = Value::Str(String::new()); + p.keys.clear(); + } + } + if let Some(p) = g.prop_mut(AVERAGE_ERROR) { + p.value = Value::Scalar(0.0); + } +} + +/// Clear analyses that no longer match, after an edit: tracks made from other frames (and their +/// solve), and solves made with other settings. Returns the trackers that need (re-)analysis: +/// cleared ones and newly applied, unanalysed ones. +pub(crate) fn invalidate(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> { + let mut todo = vec![]; + let mut clears = vec![]; + for (cid, it) in &after.items { + let ItemKind::Comp(c) = &it.kind else { continue }; + for l in &c.layers { + let Some(fx) = l.effects() else { continue }; + if !fx.groups().any(is_tracker) { + continue; + } + let old_layer = before.comp(*cid).and_then(|bc| bc.layer(l.id)); + for g in fx.groups().filter(|g| is_tracker(g)) { + let tracked = !str_of(g, TRACKS).is_empty(); + if tracked && str_of(g, TRACKS_KEY) != tracks_key(after, *cid, l, g) { + clears.push((*cid, l.id, g.uid, true)); + todo.push((*cid, l.id, g.uid)); + } else if tracked && str_of(g, SOLVE_KEY) != ct::solve_key(&ct::settings(&static_params(g))) { + clears.push((*cid, l.id, g.uid, false)); + todo.push((*cid, l.id, g.uid)); + } else if !tracked && old_layer.and_then(|ol| ol.props.find_group(g.uid)).is_none() { + todo.push((*cid, l.id, g.uid)); + } + } + } + } + for (cid, lid, uid, all) in clears { + if let Some(g) = after.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)) { + clear(g, all); + } + } + todo +} + +fn input_frame(r: &Renderer, ctx: &EvalCtx, layer: &Layer, effect_index: usize) -> Option<(Arc, [f64; 2])> { + let buf = r.layer_input(ctx, layer, effect_index)?; + if (buf.scale - 1.0).abs() > 1e-9 && buf.scale > 0.0 { + let w = (buf.img.width as f64 / buf.scale).round().max(1.0) as u32; + let h = (buf.img.height as f64 / buf.scale).round().max(1.0) as u32; + let off = [buf.offset[0] / buf.scale, buf.offset[1] / buf.scale]; + return Some((Arc::new(effectcraft_raster::resample(&buf.img, w, h)), off)); + } + Some((Arc::new(buf.img.clone()), buf.offset)) +} + +fn run_work(w: CamWork, shared: &CameraShared) { + let t0 = web_time::Instant::now(); + let fail = |msg: &str| { + let mut s = lock(&shared.state); + s.error = Some(msg.to_string()); + s.finished = true; + }; + let cancelled = || { + let mut s = lock(&shared.state); + s.cancelled = true; + s.finished = true; + }; + let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") }; + let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") }; + let lt0 = layer.layer_time(w.times[0]).seconds(); + let fd = if w.times.len() > 1 { layer.layer_time(w.times[1]).seconds() - lt0 } else { comp.frame_duration().seconds() }; + let (tracks, fresh) = match w.tracks.clone() { + Some(t) => (t, false), + None => { + let mut r = Renderer::new(&w.project, w.footage.as_ref(), RenderOpts { scale: 1.0, ..Default::default() }); + r.expr = w.expr.as_deref(); + r.cache = Some(&w.cache); + let r = &r; + let ctx0 = EvalCtx::new(&w.project, w.comp, comp, Tick::ZERO); + let ctx0 = EvalCtx { expr: w.expr.as_deref(), ..ctx0 }; + let frame_at = |t: Tick| input_frame(r, &ctx0.at(t), layer, w.effect_index); + { + let mut s = lock(&shared.state); + s.step = 1; + s.total = w.times.len() as u64; + } + let mut an = TrackAnalyzer::new(w.size, AnalyzeOpts { detailed: w.detailed }); + let mut next = frame_at(w.times[0]); + for k in 0..w.times.len() { + if shared.cancel.load(Ordering::Relaxed) { + return cancelled(); + } + let Some(cur) = next.take() else { return fail(&format!("no frame at {:.3} s", w.times[k].seconds())) }; + let ((), nf) = rayon::join( + || an.push(&effectcraft_track::Frame { img: &cur.0, offset: cur.1 }), + || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None }, + ); + next = nf; + let mut s = lock(&shared.state); + s.done = k as u64 + 1; + s.elapsed = t0.elapsed().as_secs_f64(); + s.fps = if s.elapsed > 0.0 { s.done as f64 / s.elapsed } else { 0.0 }; + } + (Arc::new(an.finish(lt0, fd)), true) + } + }; + lock(&shared.state).step = 2; + let solve = effectcraft_track::camtrack::solve(&tracks, &w.settings, Some(&shared.cancel)).map_err(|e| e.0); + if shared.cancel.load(Ordering::Relaxed) { + return cancelled(); + } + *lock(&shared.result) = Some(CamResult { tracks: fresh.then(|| (*tracks).clone()), solve }); + let mut s = lock(&shared.state); + s.elapsed = t0.elapsed().as_secs_f64(); + s.finished = true; +} + +impl Session { + pub fn is_camera_analyzing(&self) -> bool { + self.camera_job.as_ref().is_some_and(|j| !j.is_finished()) + } + + /// Progress of the running (or just finished) analysis. + pub fn camera_progress(&self) -> Option { + self.camera_job.as_ref().map(|j| j.progress()) + } + + /// The tracker being analysed: (comp, layer, effect uid). + pub fn camera_target(&self) -> Option<(ItemId, LayerId, Uid)> { + self.camera_job.as_ref().map(|j| (j.comp, j.layer, j.effect)) + } + + /// Cancel the running analysis (nothing is written). + pub fn stop_camera(&mut self) -> bool { + match &self.camera_job { + Some(j) if !j.is_finished() => { + j.shared.cancel.store(true, Ordering::Relaxed); + true + } + _ => false, + } + } + + /// Start analysing 3D Camera Tracker `effect` on `layer` (re-solving the stored tracks when + /// they are still valid). `wait`: block until done (wasm32 always does). Returns the number + /// of frames. + pub fn start_camera(&mut self, comp: ItemId, layer: LayerId, effect: Uid, wait: bool) -> Result { + if self.is_camera_analyzing() { + return Err("a 3D Camera Tracker analysis is already running".into()); + } + self.poll_camera_job(); + self.camera_pending.retain(|x| *x != (comp, layer, effect)); + let c = self.project.comp(comp).ok_or("no composition")?; + let l = c.layer(layer).ok_or("no layer")?; + let fx = l.effects().ok_or("the layer has no effects")?; + let index = fx.groups().position(|g| g.uid == effect).ok_or("no such effect")?; + let g = fx.groups().nth(index).filter(|g| is_tracker(g)).ok_or("not a 3D Camera Tracker")?; + if l.switches.adjustment { + return Err("3D Camera Tracker needs footage, not an adjustment layer".into()); + } + let times = analysis_times(c, l); + if times.len() < 2 { + return Err("the layer must be at least two frames long".into()); + } + let (w, h) = effectcraft_render::source_size(&self.project, l); + let size = if w == 0 { [c.width as f64, c.height as f64] } else { [w as f64, h as f64] }; + let params = static_params(g); + let key = tracks_key(&self.project, comp, l, g); + let settings = ct::settings(¶ms); + let solve_key = ct::solve_key(&settings); + let tracks = ct::tracks(¶ms).filter(|_| str_of(g, TRACKS_KEY) == key); + let n = times.len(); + let work = CamWork { + project: self.project.clone(), + footage: self.footage.clone(), + expr: self.expr.clone(), + cache: self.layer_cache.clone(), + comp, + layer, + effect_index: index, + size, + detailed: params.values.iter().any(|(k, v)| k.ends_with("detailedAnalysis") && v.as_bool()), + times, + tracks: tracks.clone(), + settings, + }; + let wait = wait || cfg!(target_arch = "wasm32"); + let shared = Arc::new(CameraShared::default()); + { + let mut st = lock(&shared.state); + st.solve_only = tracks.is_some(); + st.step = if tracks.is_some() { 2 } else { 1 }; + st.total = n as u64; + } + let thread = if wait { + run_work(work, &shared); + None + } else { + let sh = shared.clone(); + Some(std::thread::Builder::new().name("camera-track".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?) + }; + self.camera_job = Some(CameraJob { shared, thread, comp, layer, effect, key, solve_key }); + self.events.push(Event::ProjectChanged { revision: self.revision }); + self.poll_camera_job(); + Ok(n) + } + + /// Write a finished analysis into its effect (one undo step) and drop the job. + fn poll_camera_job(&mut self) -> bool { + let Some(job) = &self.camera_job else { return false }; + if !job.is_finished() { + return false; + } + let mut job = self.camera_job.take().expect("job"); + if let Some(t) = job.thread.take() { + let _ = t.join(); + } + let st = job.progress(); + let result = lock(&job.shared.result).take(); + match (result, &st.error) { + (Some(res), None) if !st.cancelled => { + let (cid, lid, uid) = (job.comp, job.layer, job.effect); + let (key, skey) = (job.key.clone(), job.solve_key.clone()); + let tracks_json = res.tracks.as_ref().map(CameraTracks::to_json); + let (solve_json, method, avg, msg, failed) = match &res.solve { + Ok(s) => ( + s.to_json(), + s.method_used.label().to_string(), + s.average_error, + format!( + "3D Camera Tracker: solved {} frame(s), {} points, average error {:.2} px ({:.1} s)", + s.frames.len(), + s.points.len(), + s.average_error, + st.elapsed + ), + false, + ), + Err(e) => (String::new(), String::new(), 0.0, format!("3D Camera Tracker: {e}"), true), + }; + let r = self.edit(if st.solve_only { "3D Camera Tracker Solve" } else { "3D Camera Tracker Analysis" }, None, |p, _| { + let g = p + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .ok_or_else(|| crate::EngineError::Other("the 3D Camera Tracker was removed during analysis".into()))?; + let mut set = |m: &str, v: Value| { + if let Some(pr) = g.prop_mut(m) { + pr.value = v; + } + }; + if let Some(t) = &tracks_json { + set(TRACKS, Value::Str(t.clone())); + set(TRACKS_KEY, Value::Str(key.clone())); + } + set(SOLVE, Value::Str(solve_json.clone())); + set(SOLVE_KEY, Value::Str(skey.clone())); + set(METHOD_USED, Value::Str(method.clone())); + set(AVERAGE_ERROR, Value::Scalar(avg)); + Ok(()) + }); + let (msg, failed) = match r { + Ok(()) => (msg, failed), + Err(e) => (format!("3D Camera Tracker: {e}"), true), + }; + self.events.push(Event::Toast { message: msg, error: failed }); + } + _ => { + let msg = match &st.error { + Some(e) => format!("3D Camera Tracker analysis failed: {e}"), + None => "3D Camera Tracker analysis cancelled".into(), + }; + self.events.push(Event::Toast { message: msg, error: st.error.is_some() }); + self.bump(); + } + } + true + } + + /// Frontends call this every frame: finishes analyses and (with `auto`) starts queued + /// (re-)analyses in the background, like After Effects. + pub fn poll_camera(&mut self, auto: bool) -> bool { + let changed = self.poll_camera_job(); + if auto && self.camera_job.is_none() { + while let Some((c, l, e)) = self.camera_pending.first().copied() { + self.camera_pending.remove(0); + if self.start_camera(c, l, e, false).is_ok() { + return true; + } + } + } + changed + } +} + +// ---------------------------------------------------------------- solve → composition + +/// A tracker's solve placed in its composition. +pub struct Placed { + pub solve: Arc, + /// Canonical → comp world. + pub world: Similarity, + /// Comp pixels per layer pixel. + pub scale: f64, + /// Comp times of the analysed frames. + pub times: Vec, + pub comp_size: [f64; 2], +} + +impl Placed { + /// Comp-world position of a canonical point. + pub fn to_world(&self, x: V3) -> V3 { + self.world.apply(x) + } + /// Canonical position of a comp-world point. + pub fn from_world(&self, x: V3) -> V3 { + let r = &self.world; + linalg::scale(linalg::mtv(&r.r, linalg::sub(x, r.t)), 1.0 / r.s) + } + /// The 3D Tracker Camera on frame `k`: (position, orientation degrees, zoom). + pub fn camera(&self, k: usize) -> (V3, V3, f64) { + let c = &self.solve.frames[k]; + let m = linalg::mm(&self.world.r, &linalg::mt(&c.r())); + (self.to_world(c.center), linalg::euler_xyz(&m), c.focal * self.scale) + } + /// Analysed frame at comp time `t`. + pub fn frame_at(&self, t: Tick) -> usize { + self.times.iter().enumerate().min_by_key(|(_, x)| (x.0 - t.0).abs()).map(|(i, _)| i).unwrap_or(0).min(self.solve.frames.len().saturating_sub(1)) + } +} + +/// Comp pixels per layer pixel of `layer` (its 2D transform at its In point). +fn layer_scale(project: &Project, cid: ItemId, layer: &Layer) -> f64 { + let Some(comp) = project.comp(cid) else { return 1.0 }; + let ctx = EvalCtx::new(project, cid, comp, layer.in_point.max(Tick::ZERO)); + let m = ctx.world_matrix(layer).0; + let s = m[0][0].hypot(m[1][0]); + if s.is_finite() && s > 1e-9 { s } else { 1.0 } +} + +/// The solve of tracker `uid` on `lid`, placed in the comp. +pub fn placed(project: &Project, cid: ItemId, lid: LayerId, uid: Uid) -> Option { + let comp = project.comp(cid)?; + let layer = comp.layer(lid)?; + let g = layer.props.find_group(uid).filter(|g| is_tracker(g))?; + let solve = ct::solve(&static_params(g))?; + let scale = layer_scale(project, cid, layer); + let k0 = solve.first_solved()?; + let comp_size = [comp.width as f64, comp.height as f64]; + let world = solve.world(comp_size, solve.frames[k0].focal * scale); + let mut times = analysis_times(comp, layer); + times.truncate(solve.frames.len()); + Some(Placed { solve, world, scale, times, comp_size }) +} + +/// What to create from a target (`camera.createFromSolve`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CreateKind { + Text, + Solid, + Null, + ShadowCatcher, + /// Only the camera. + Camera, +} + +impl CreateKind { + pub fn from_name(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '-', '_'], "").as_str() { + "text" => Some(CreateKind::Text), + "solid" => Some(CreateKind::Solid), + "null" => Some(CreateKind::Null), + "shadowcatcher" | "shadow" => Some(CreateKind::ShadowCatcher), + "camera" => Some(CreateKind::Camera), + _ => None, + } + } +} + +/// Rotation (layer → world) of a layer lying on a plane with unit `normal` (towards the camera), +/// its x axis along `right` projected on the plane: the layer's front (−z) faces the camera. +pub fn plane_orientation(normal: V3, right: V3) -> V3 { + let z = linalg::scale(normal, -1.0); + let mut x = linalg::sub(right, linalg::scale(z, linalg::dot(right, z))); + if linalg::norm(x) < 1e-9 { + x = if z[0].abs() < 0.9 { [1.0, 0.0, 0.0] } else { [0.0, 1.0, 0.0] }; + x = linalg::sub(x, linalg::scale(z, linalg::dot(x, z))); + } + let x = linalg::normalize(x); + let y = linalg::cross(z, x); + let m = linalg::from_cols(x, y, z); + norm_angles(linalg::euler_xyz(&m)) +} + +fn norm_angles(a: V3) -> V3 { + a.map(|v| { + let r = v.rem_euclid(360.0); + if (r - 360.0).abs() < 1e-9 { 0.0 } else { r } + }) +} + +/// Key `path` on `l` at every `(time, value)`. +pub(crate) fn key_all(l: &mut Layer, path: &str, keys: &[(Tick, Value)]) { + if let Some(pr) = l.props.prop_mut(path) { + pr.keys = keys.iter().map(|(t, v)| Keyframe::new(*t, v.clone())).collect(); + if let Some((_, v)) = keys.first() { + pr.value = v.clone(); + } + } +} + +/// Write the solved camera path into camera layer `l` (one-node, keyed on every analysed frame; +/// Zoom keyed for Variable Zoom). +pub(crate) fn write_camera(l: &mut Layer, pl: &Placed) { + l.auto_orient = AutoOrient::Off; + let n = pl.times.len().min(pl.solve.frames.len()); + let mut pos = Vec::with_capacity(n); + let mut ori = Vec::with_capacity(n); + let mut zoom = Vec::with_capacity(n); + let mut prev: Option = None; + for k in 0..n { + let (p, o, z) = pl.camera(k); + // Unwrap the angles so per-frame keys interpolate the short way. + let o = match prev { + None => norm_angles(o), + Some(q) => [0, 1, 2].map(|i| q[i] + (o[i] - q[i] + 180.0).rem_euclid(360.0) - 180.0), + }; + prev = Some(o); + let t = l.layer_time(pl.times[k]); + pos.push((t, Value::Vec3(p))); + ori.push((t, Value::Vec3(o))); + zoom.push((t, Value::Scalar(z))); + } + key_all(l, "transform/position", &pos); + key_all(l, "transform/orientation", &ori); + let variable = pl.solve.frames.iter().any(|f| (f.focal - pl.solve.frames[0].focal).abs() > 1e-6 * f.focal); + if variable { + key_all(l, "cameraOptions/zoom", &zoom); + } else if let Some((_, z)) = zoom.first() + && let Some(pr) = l.props.prop_mut("cameraOptions/zoom") + { + pr.keys.clear(); + pr.value = z.clone(); + } + if let Some((_, z)) = zoom.first() + && let Some(pr) = l.props.prop_mut("cameraOptions/focusDistance") + { + pr.value = z.clone(); + } + // A one-node camera has no Point of Interest. + if let Some(tr) = l.props.sub_mut("transform") { + tr.children.retain(|c| !matches!(c, effectcraft_project::Node::Prop(p) if p.match_id == "poi")); + } + for rot in ["transform/rotationX", "transform/rotationY", "transform/rotation"] { + if let Some(pr) = l.props.prop_mut(rot) { + pr.keys.clear(); + pr.value = Value::Scalar(0.0); + } + } +} + +/// The target for `ids` on frame `k`: when fewer than three points are given, the nearest +/// solved neighbours (in 3D) complete the plane. +pub fn target_for(solve: &CameraSolve, ids: &[u32], k: usize) -> Option { + let mut use_ids: Vec = ids.iter().copied().filter(|i| solve.point(*i).is_some()).collect(); + if use_ids.is_empty() { + return None; + } + if use_ids.len() < 3 { + let c = solve.point(use_ids[0])?.pos; + let mut others: Vec<(f64, u32)> = solve.visible(k).filter(|p| !use_ids.contains(&p.id)).map(|p| (linalg::norm(linalg::sub(p.pos, c)), p.id)).collect(); + others.sort_by(|a, b| a.0.total_cmp(&b.0)); + for (_, id) in others.into_iter().take(6usize.saturating_sub(use_ids.len())) { + use_ids.push(id); + } + let mut t = solve.target(&use_ids, k)?; + // The target sits on the chosen point(s). + let pts: Vec = ids.iter().filter_map(|i| solve.point(*i)).map(|p| p.pos).collect(); + t.center = linalg::scale(pts.iter().fold([0.0; 3], |a, b| linalg::add(a, *b)), 1.0 / pts.len() as f64); + return Some(t); + } + solve.target(&use_ids, k) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plane_orientation_faces_the_normal() { + let n = linalg::normalize([0.2, -0.9, -0.3]); + let o = plane_orientation(n, [1.0, 0.0, 0.0]); + let m = linalg::from_euler_xyz(o); + // Layer −z (its front) is the normal. + let front = linalg::mv(&m, [0.0, 0.0, -1.0]); + assert!(linalg::norm(linalg::sub(front, n)) < 1e-9); + } +} diff --git a/crates/engine/src/commands/camera_cmds.rs b/crates/engine/src/commands/camera_cmds.rs new file mode 100644 index 00000000..5d454aed --- /dev/null +++ b/crates/engine/src/commands/camera_cmds.rs @@ -0,0 +1,574 @@ +//! 3D Camera Tracker commands: Animation ▸ Track Camera (`track.camera`), the effect's Analyze / +//! Cancel / Create Camera buttons, the viewer's track point selection and right-click menu +//! (Set Ground Plane and Origin, Create Text / Solid / Null / Shadow Catcher … and Camera, +//! Create Multiple …, Delete Selected Points) and the status / points queries. + +use effectcraft_color::Label; +use effectcraft_effects::camera_tracker::{self as ct, CAMERA, DELETED, SOLVE}; +use effectcraft_keyframe::{Justify, TextDoc, Value as KV}; +use effectcraft_project::build; +use effectcraft_project::{ItemId, ItemKind, Layer, LayerId, LayerSource, LightKind, Project, Solid, Uid}; +use effectcraft_track::camtrack::linalg::{self, V3}; +use effectcraft_track::camtrack::{ShotType, SolveMethod, Target, point_color}; +use serde_json::{Value, json}; + +use super::{CommandSpec, b_p, bad, f_p, has_comp, layer_p, str_p}; +use crate::camera_track::{CreateKind, Placed, is_tracker, placed, plane_orientation, static_params, target_for, write_camera}; +use crate::{EngineError, Result, Session, cmd, query}; + +pub fn specs() -> Vec { + vec![ + cmd!( + "track.camera", + "Track Camera", + ["Animation"], + None, + "{layer?, shotType?: fixed|variable|specify, aov?: degrees, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}", + can_track, + track_camera + ), + cmd!("camera.analyze", "Analyze", [], None, "{layer?, effect?: uid|name|index, wait?: block until done}", has_comp, analyze), + cmd!("camera.cancel", "Cancel", [], None, "{}", is_analyzing, |s, _| Ok(json!({"stopped": s.stop_camera()}))), + query!("camera.solveStatus", "3D Camera Tracker Status", "{layer?, effect?}", status), + query!("camera.points", "3D Camera Tracker Points", "{layer?, effect?, time?: comp seconds} → visible solved points", points), + cmd!("camera.selectPoints", "Select Track Points", [], None, "{points: [id], add?, toggle?}", has_comp, select_points), + cmd!( + "camera.createFromSolve", + "Create from Camera Solve", + [], + None, + "{kind: text|solid|null|shadowCatcher|camera, points?: [id] (default: selected), target?: {center, normal, size?} (comp world), multiple?, layer?, effect?}", + has_comp, + create_from_solve + ), + cmd!("camera.create", "Create Camera", [], None, "{layer?, effect?}", has_comp, |s, p| create_from_solve(s, &with_kind(p, "camera"))), + cmd!("camera.setGroundPlane", "Set Ground Plane and Origin", [], None, "{points?: [id] (default: selected), layer?, effect?}", has_comp, set_ground), + cmd!("camera.deletePoints", "Delete Selected Points", [], None, "{points?: [id] (default: selected), layer?, effect?, wait?}", has_comp, delete_points), + ] +} + +fn with_kind(p: &Value, kind: &str) -> Value { + let mut q = p.clone(); + if !q.is_object() { + q = json!({}); + } + q["kind"] = json!(kind); + q +} + +fn can_track(s: &Session) -> std::result::Result<(), String> { + has_comp(s)?; + let c = s.active_comp().ok_or("no composition is open")?; + match s.state.selected_layers.first().and_then(|l| c.layer(*l)) { + Some(l) if l.has_video() && !l.switches.adjustment && !l.is_camera() && !l.is_light() => Ok(()), + Some(_) => Err("Track Camera needs a footage or composition layer".into()), + None => Err("select a layer to track".into()), + } +} + +fn is_analyzing(s: &Session) -> std::result::Result<(), String> { + if s.is_camera_analyzing() { Ok(()) } else { Err("no 3D Camera Tracker analysis is running".into()) } +} + +/// (comp, layer, effect uid) of a 3D Camera Tracker: `layer` / `effect` (uid, name or 1-based +/// index), else a selected one, else the first in the comp. +fn ct_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> { + let cid = super::comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let candidates: Vec = match p.get("layer") { + Some(v) => vec![super::resolve_layer(comp, v).ok_or_else(|| bad(cmd, format!("no layer {v}")))?], + None => { + let mut v: Vec = s.state.selected_props.iter().map(|(l, _)| *l).collect(); + v.extend(s.state.selected_layers.iter().copied()); + v.extend(comp.layers.iter().map(|l| l.id)); + v + } + }; + for lid in candidates { + let Some(fx) = comp.layer(lid).and_then(Layer::effects) else { continue }; + let g = match p.get("effect") { + Some(Value::Number(n)) => { + let n = n.as_u64().unwrap_or(0); + fx.groups().find(|g| g.uid == n).or_else(|| fx.groups().nth((n as usize).saturating_sub(1))) + } + Some(Value::String(name)) => fx.groups().find(|g| &g.name == name), + _ => s + .state + .selected_props + .iter() + .find_map(|(l, u)| (*l == lid).then(|| fx.groups().find(|g| g.uid == *u && is_tracker(g))).flatten()) + .or_else(|| fx.groups().find(|g| is_tracker(g))), + }; + if let Some(g) = g.filter(|g| is_tracker(g)) { + return Ok((cid, lid, g.uid)); + } + if p.get("layer").is_some() { + break; + } + } + Err(bad(cmd, "no 3D Camera Tracker found (apply Animation ▸ Track Camera first)")) +} + +fn shot_p(p: &Value) -> Result> { + match p.get("shotType") { + None => Ok(None), + Some(Value::Number(n)) => ShotType::ALL.get(n.as_u64().unwrap_or(99) as usize).copied().map(Some).ok_or_else(|| bad("track.camera", "shotType: 0..2")), + Some(Value::String(v)) => ShotType::from_name(v).map(Some).ok_or_else(|| bad("track.camera", "shotType: fixed|variable|specify")), + Some(_) => Err(bad("track.camera", "shotType: fixed|variable|specify")), + } +} + +fn method_p(p: &Value) -> Result> { + match p.get("solveMethod") { + None => Ok(None), + Some(Value::Number(n)) => { + SolveMethod::ALL.get(n.as_u64().unwrap_or(99) as usize).copied().map(Some).ok_or_else(|| bad("track.camera", "solveMethod: 0..3")) + } + Some(Value::String(v)) => SolveMethod::from_name(v).map(Some).ok_or_else(|| bad("track.camera", "solveMethod: auto|typical|flat|tripod")), + Some(_) => Err(bad("track.camera", "solveMethod: auto|typical|flat|tripod")), + } +} + +/// Animation ▸ Track Camera / the Tracker panel button: apply the 3D Camera Tracker to the layer +/// (reusing one it already has), set the given options and start the analysis. +fn track_camera(s: &mut Session, p: &Value) -> Result { + let (cid, lid) = layer_p(s, p, "track.camera")?; + let shot = shot_p(p)?; + let method = method_p(p)?; + let aov = f_p(p, "aov").or_else(|| f_p(p, "angleOfView")); + if aov.is_some_and(|a| !(1.0..=170.0).contains(&a)) { + return Err(bad("track.camera", "aov: 1..170 degrees")); + } + let detailed = b_p(p, "detailed"); + let existing = s.project.comp(cid).and_then(|c| c.layer(lid)).and_then(Layer::effects).and_then(|fx| fx.groups().find(|g| is_tracker(g)).map(|g| g.uid)); + let uid = match existing { + Some(u) => u, + None => { + let r = s.execute("effect.apply", json!({"effect": ct::ID, "layers": [lid.0], "comp": cid.0}))?; + r["effects"].get(0).and_then(Value::as_u64).ok_or_else(|| EngineError::Other("the effect could not be applied".into()))? + } + }; + if shot.is_some() || method.is_some() || aov.is_some() || detailed.is_some() { + s.edit("3D Camera Tracker Settings", None, |proj, _| { + let g = proj + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?; + let mut set = |m: &str, v: KV| { + if let Some(pr) = g.prop_mut(m) { + pr.keys.clear(); + pr.value = v; + } + }; + if let Some(sh) = shot { + set("shotType", KV::Enum(ShotType::ALL.iter().position(|x| *x == sh).unwrap_or(0) as u32)); + } + if let Some(a) = aov { + set("horizontalAngleOfView", KV::Scalar(a)); + if shot.is_none() { + set("shotType", KV::Enum(2)); + } + } + if let Some(m) = method { + set("advanced/solveMethod", KV::Enum(SolveMethod::ALL.iter().position(|x| *x == m).unwrap_or(0) as u32)); + } + if let Some(d) = detailed { + set("advanced/detailedAnalysis", KV::Bool(d)); + } + Ok(()) + })?; + } + // Select the effect so the viewer shows its track points. + s.state.selected_props = vec![(lid, uid)]; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?; + let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_camera_analyzing(), "progress": s.camera_progress()}); + if wait { + out["status"] = status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?; + } + Ok(out) +} + +fn analyze(s: &mut Session, p: &Value) -> Result { + let (cid, lid, uid) = ct_p(s, p, "camera.analyze")?; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?; + let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_camera_analyzing(), "progress": s.camera_progress()}); + if wait { + out["status"] = status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?; + } + Ok(out) +} + +fn status(s: &mut Session, p: &Value) -> Result { + s.poll_camera(false); + let mut out = json!({"running": s.is_camera_analyzing(), "progress": s.camera_progress(), "banner": s.camera_progress().map(|p| p.banner())}); + let Ok((cid, lid, uid)) = ct_p(s, p, "camera.solveStatus") else { return Ok(out) }; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let g = layer.props.find_group(uid).ok_or(EngineError::NoComp)?; + let params = static_params(g); + out["layer"] = json!(lid.0); + out["effect"] = json!(uid); + out["pending"] = json!(s.camera_pending.contains(&(cid, lid, uid))); + let tracks = ct::tracks(¶ms); + out["analyzed"] = json!(tracks.is_some()); + if let Some(t) = &tracks { + out["tracks"] = json!(t.tracks.len()); + out["frames"] = json!(t.frames); + } + let settings = ct::settings(¶ms); + out["shotType"] = json!(settings.shot.label()); + out["solveMethod"] = json!(settings.method.label()); + out["deleted"] = json!(settings.deleted.len()); + let solve = ct::solve(¶ms); + out["solved"] = json!(solve.is_some()); + if let Some(sv) = &solve { + let f = sv.focal(); + out["methodUsed"] = json!(sv.method_used.label()); + out["averageError"] = json!(sv.average_error); + out["points"] = json!(sv.points.len()); + out["solvedFrames"] = json!(sv.frames.iter().filter(|c| c.solved).count()); + out["focalLength"] = json!(f); + out["horizontalAngleOfView"] = json!(sv.hfov(f)); + out["groundPlane"] = json!(sv.ground.is_some()); + if let Some(pl) = placed(&s.project, cid, lid, uid) { + let k = pl.frame_at(s.time_of(cid)); + let (pos, ori, zoom) = pl.camera(k); + out["frame"] = json!(k); + out["camera"] = json!({"position": pos, "orientation": ori, "zoom": zoom}); + out["visible"] = json!(sv.visible(k).count()); + } + } + let cam = params.get(CAMERA).map(|v| v.as_f64() as u64).unwrap_or(0); + out["cameraLayer"] = if cam > 0 && comp.layer(LayerId(cam)).is_some_and(Layer::is_camera) { json!(cam) } else { Value::Null }; + out["selected"] = json!(s.state.camera_points); + Ok(out) +} + +/// The visible solved points at the current (or given) time, with their comp positions. +fn points(s: &mut Session, p: &Value) -> Result { + let (cid, lid, uid) = ct_p(s, p, "camera.points")?; + let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad("camera.points", "the camera is not solved yet"))?; + let t = f_p(p, "time").map(effectcraft_time::Tick::from_seconds_f64).unwrap_or_else(|| s.time_of(cid)); + let k = pl.frame_at(t); + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let ctx = effectcraft_render::EvalCtx::new(&s.project, cid, comp, t); + let m = ctx.world_matrix(layer); + let list: Vec = ct::projected(&pl.solve, k) + .into_iter() + .map(|(id, q, z)| { + let c = m.apply(effectcraft_geom::vec3(q[0], q[1], 0.0)); + let sp = pl.solve.point(id).copied().unwrap_or_default(); + let col = point_color(id); + json!({"id": id, "comp": [c.x, c.y], "layer": q, "depth": z, "world": pl.to_world(sp.pos), "error": sp.error, "color": col, "selected": s.state.camera_points.contains(&id)}) + }) + .collect(); + Ok(json!({"frame": k, "points": list})) +} + +fn ids_p(s: &Session, p: &Value) -> Vec { + match p.get("points") { + Some(Value::Array(a)) => a.iter().filter_map(Value::as_u64).map(|v| v as u32).collect(), + Some(Value::Number(n)) => n.as_u64().map(|v| vec![v as u32]).unwrap_or_default(), + _ => s.state.camera_points.clone(), + } +} + +fn select_points(s: &mut Session, p: &Value) -> Result { + let ids: Vec = match p.get("points") { + Some(Value::Array(a)) => a.iter().filter_map(Value::as_u64).map(|v| v as u32).collect(), + _ => vec![], + }; + if b_p(p, "toggle").unwrap_or(false) { + for id in ids { + match s.state.camera_points.iter().position(|x| *x == id) { + Some(i) => { + s.state.camera_points.remove(i); + } + None => s.state.camera_points.push(id), + } + } + } else if b_p(p, "add").unwrap_or(false) { + for id in ids { + if !s.state.camera_points.contains(&id) { + s.state.camera_points.push(id); + } + } + } else { + s.state.camera_points = ids; + } + s.bump(); + Ok(json!({"selected": s.state.camera_points})) +} + +/// The target of a create / ground command in canonical solve coordinates. +fn target_p(s: &Session, p: &Value, pl: &Placed, k: usize, cmd: &str) -> Result { + if let Some(t) = p.get("target") { + let v3 = |k: &str| -> Option { + let a = t.get(k)?.as_array()?; + Some([a.first()?.as_f64()?, a.get(1)?.as_f64()?, a.get(2).and_then(Value::as_f64).unwrap_or(0.0)]) + }; + let (c, n) = (v3("center").ok_or_else(|| bad(cmd, "target.center: [x, y, z]"))?, v3("normal").unwrap_or([0.0, -1.0, 0.0])); + let size = t.get("size").and_then(Value::as_f64).unwrap_or(100.0); + let nc = linalg::normalize(linalg::mtv(&pl.world.r, n)); + return Ok(Target { center: pl.from_world(c), normal: nc, size: size / pl.world.s }); + } + let ids = ids_p(s, p); + if ids.is_empty() { + return Err(bad(cmd, "select track points (or give `points` / `target`)")); + } + target_for(&pl.solve, &ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane")) +} + +fn next_name(c: &effectcraft_project::Comp, base: &str) -> String { + let n = (1..).find(|i| !c.layers.iter().any(|l| l.name == format!("{base} {i}"))).unwrap_or(1); + format!("{base} {n}") +} + +fn place(l: &mut Layer, pos: V3, ori: V3, src: &Layer) { + l.switches.three_d = true; + l.in_point = src.in_point; + l.out_point = src.out_point; + if let Some(pr) = l.props.prop_mut("transform/position") { + pr.value = KV::Vec3(pos); + } + if let Some(pr) = l.props.prop_mut("transform/orientation") { + pr.value = KV::Vec3(ori); + } +} + +/// Insert `l` above the tracked layer. +fn insert_above(proj: &mut Project, cid: ItemId, above: LayerId, l: Layer) -> Result { + let comp = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let at = comp.layers.iter().position(|x| x.id == above).unwrap_or(0); + let id = l.id; + comp.layers.insert(at, l); + Ok(id) +} + +fn add_solid(proj: &mut Project, comp: &effectcraft_project::Comp, name: &str, size: u32, color: [f32; 3]) -> Layer { + let folder = proj.folder_named("Solids").unwrap_or_else(|| proj.add_item("Solids", Label::Yellow, None, ItemKind::Folder)); + let sid = proj.add_item(name, Label::Red, Some(folder), ItemKind::Solid(Solid { color, width: size, height: size, pixel_aspect: 1.0 })); + build::layer(proj, comp, name, LayerSource::Solid { item: sid }, (size, size), None) +} + +/// Create Text / Solid / Null / Shadow Catcher (and Light) on the target, plus the 3D Tracker +/// Camera when the tracker has none yet — one undo step. +fn create_from_solve(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.createFromSolve"; + let (cid, lid, uid) = ct_p(s, p, cmd)?; + let kind = CreateKind::from_name(str_p(p, "kind").unwrap_or("null")).ok_or_else(|| bad(cmd, "kind: text|solid|null|shadowCatcher|camera"))?; + let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad(cmd, "the camera is not solved yet (analysis still running or failed)"))?; + let k = pl.frame_at(s.time_of(cid)); + let multiple = b_p(p, "multiple").unwrap_or(false); + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let src = comp.layer(lid).ok_or(EngineError::NoComp)?.clone(); + let g = src.props.find_group(uid).ok_or(EngineError::NoComp)?; + let params = static_params(g); + let size_pct = effectcraft_effects::warp_stab::param(¶ms, "targetSize").map(|v| v.as_f64()).unwrap_or(100.0) / 100.0; + let existing_cam = params.get(CAMERA).map(|v| v.as_f64() as u64).filter(|id| *id > 0 && comp.layer(LayerId(*id)).is_some_and(Layer::is_camera)); + // Targets: one, or one per selected point (Create Multiple …) sharing the selection's plane. + let targets: Vec = if kind == CreateKind::Camera { + vec![] + } else { + let t = target_p(s, p, &pl, k, cmd)?; + if multiple { + let ids = ids_p(s, p); + ids.iter().filter_map(|i| pl.solve.point(*i)).map(|sp| Target { center: sp.pos, ..t }).collect() + } else { + vec![t] + } + }; + // Camera right vector on this frame (world), to align created layers with the view. + let cam = &pl.solve.frames[k]; + let right = linalg::mv(&pl.world.r, linalg::mtv(&cam.r(), [1.0, 0.0, 0.0])); + let label = match (kind, multiple) { + (CreateKind::Text, false) => "Create Text and Camera", + (CreateKind::Solid, false) => "Create Solid and Camera", + (CreateKind::Null, false) => "Create Null and Camera", + (CreateKind::ShadowCatcher, _) => "Create Shadow Catcher, Camera and Light", + (CreateKind::Text, true) => "Create Multiple Text Layers and Camera", + (CreateKind::Solid, true) => "Create Multiple Solids and Camera", + (CreateKind::Null, true) => "Create Multiple Nulls and Camera", + (CreateKind::Camera, _) => "Create Camera", + }; + let (camera, created) = s.edit(label, None, |proj, st| { + let mut created = vec![]; + // The camera: reuse the tracker's, else create "3D Tracker Camera" at the top. + let camera = match existing_cam { + Some(id) if kind != CreateKind::Camera => LayerId(id), + Some(id) => { + let l = proj.comp_mut(cid).and_then(|c| c.layer_mut(LayerId(id))).ok_or(EngineError::NoComp)?; + write_camera(l, &pl); + LayerId(id) + } + None => { + let mut l = build::layer(proj, &comp, "3D Tracker Camera", LayerSource::Camera, (comp.width, comp.height), None); + l.in_point = src.in_point; + l.out_point = src.out_point; + write_camera(&mut l, &pl); + let id = l.id; + proj.comp_mut(cid).ok_or(EngineError::NoComp)?.layers.insert(0, l); + if let Some(pr) = proj.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)).and_then(|g| g.prop_mut(CAMERA)) { + pr.value = KV::Scalar(id.0 as f64); + } + id + } + }; + for t in &targets { + let center = pl.to_world(t.center); + let normal = linalg::normalize(linalg::mv(&pl.world.r, t.normal)); + let ori = plane_orientation(normal, right); + let size = (t.size * pl.world.s * size_pct).max(1.0); + let cur = proj.comp(cid).ok_or(EngineError::NoComp)?.clone(); + match kind { + CreateKind::Text => { + let mut l = build::layer(proj, &cur, "Text", LayerSource::Text, (comp.width, comp.height), None); + let doc = TextDoc { text: "Text".into(), justify: Justify::Center, size: (size * 0.35).clamp(4.0, 2000.0), ..Default::default() }; + if let Some(pr) = l.props.prop_mut("text/sourceText") { + pr.value = KV::Text(Box::new(doc)); + } + place(&mut l, center, ori, &src); + created.push(insert_above(proj, cid, lid, l)?); + } + CreateKind::Solid => { + let name = next_name(&cur, "Track Solid"); + let c = point_color((created.len() as u32).wrapping_add(7)); + let mut l = add_solid(proj, &cur, &name, size.round().max(1.0) as u32, c); + place(&mut l, center, ori, &src); + created.push(insert_above(proj, cid, lid, l)?); + } + CreateKind::Null => { + let name = next_name(&cur, "Track Null"); + let mut l = build::layer(proj, &cur, &name, LayerSource::Null, (comp.width, comp.height), None); + place(&mut l, center, ori, &src); + if let Some(pr) = l.props.prop_mut("transform/scale") { + pr.value = KV::Vec3([size, size, size]); + } + created.push(insert_above(proj, cid, lid, l)?); + } + CreateKind::ShadowCatcher => { + let name = next_name(&cur, "Shadow Catcher"); + let mut l = add_solid(proj, &cur, &name, (size * 4.0).round().max(1.0) as u32, [1.0, 1.0, 1.0]); + place(&mut l, center, ori, &src); + if let Some(pr) = l.props.prop_mut("materialOptions/acceptsShadows") { + pr.value = KV::Enum(2); + } + if let Some(pr) = l.props.prop_mut("materialOptions/castsShadows") { + pr.value = KV::Enum(0); + } + let sc = created.len(); + created.push(insert_above(proj, cid, lid, l)?); + let cur = proj.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let lname = next_name(&cur, "Light"); + let mut light = build::layer(proj, &cur, &lname, LayerSource::Light { kind: LightKind::Point }, (comp.width, comp.height), None); + light.in_point = src.in_point; + light.out_point = src.out_point; + let lp = linalg::add(center, linalg::add(linalg::scale(normal, size * 2.0), linalg::scale(right, size))); + if let Some(pr) = light.props.prop_mut("transform/position") { + pr.value = KV::Vec3(lp); + } + if let Some(pr) = light.props.prop_mut("lightOptions/castsShadows") { + pr.value = KV::Bool(true); + } + let _ = sc; + created.push(insert_above(proj, cid, created[created.len() - 1], light)?); + } + CreateKind::Camera => {} + } + } + st.selected_layers = if created.is_empty() { vec![camera] } else { created.clone() }; + st.selected_props.clear(); + Ok((camera, created)) + })?; + let t0 = targets.first().map( + |t| json!({"center": pl.to_world(t.center), "normal": linalg::normalize(linalg::mv(&pl.world.r, t.normal)), "size": t.size * pl.world.s * size_pct}), + ); + Ok(json!({"camera": camera.0, "layers": created.iter().map(|l| l.0).collect::>(), "target": t0})) +} + +fn set_ground(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.setGroundPlane"; + let (cid, lid, uid) = ct_p(s, p, cmd)?; + let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad(cmd, "the camera is not solved yet"))?; + let k = pl.frame_at(s.time_of(cid)); + let ids = ids_p(s, p); + if ids.is_empty() { + return Err(bad(cmd, "select track points (or give `points`)")); + } + let t = target_for(&pl.solve, &ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane"))?; + let mut sv = (*pl.solve).clone(); + let mut g = sv.ground_from(&ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane"))?; + g.origin = t.center; + sv.ground = Some(g); + let json = sv.to_json(); + s.edit("Set Ground Plane and Origin", None, |proj, _| { + let pr = proj + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .and_then(|g| g.prop_mut(SOLVE)) + .ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?; + pr.value = KV::Str(json); + Ok(()) + })?; + Ok(json!({"origin": g.origin, "normal": g.normal})) +} + +/// Delete points (and, with Auto-delete Points Across Time, the points of other tracks on the +/// same 3D feature), then re-solve. +fn delete_points(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.deletePoints"; + let (cid, lid, uid) = ct_p(s, p, cmd)?; + let ids = ids_p(s, p); + if ids.is_empty() { + return Err(bad(cmd, "select track points (or give `points`)")); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let g = comp.layer(lid).and_then(|l| l.props.find_group(uid)).ok_or(EngineError::NoComp)?; + let params = static_params(g); + let mut del = ct::deleted(¶ms); + let auto = effectcraft_effects::warp_stab::param(¶ms, "advanced/autoDeletePoints").is_some_and(KV::as_bool); + let mut add = ids.clone(); + if auto && let Some(sv) = ct::solve(¶ms) { + // Same feature tracked again at other times: solved within 1 % of its depth. + let cam0 = sv.first_solved().map(|k| sv.frames[k].center).unwrap_or([0.0; 3]); + for id in &ids { + let Some(pt) = sv.point(*id) else { continue }; + let r = linalg::norm(linalg::sub(pt.pos, cam0)) * 0.01; + for q in &sv.points { + let disjoint = q.last < pt.first || q.first > pt.last; + if q.id != *id && disjoint && linalg::norm(linalg::sub(q.pos, pt.pos)) < r { + add.push(q.id); + } + } + } + } + for id in add { + if !del.contains(&id) { + del.push(id); + } + } + del.sort_unstable(); + let json = serde_json::to_string(&del).unwrap_or_default(); + s.edit("Delete Track Points", None, |proj, _| { + let pr = proj + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .and_then(|g| g.prop_mut(DELETED)) + .ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?; + pr.value = KV::Str(json); + Ok(()) + })?; + s.state.camera_points.retain(|x| !del.contains(x)); + // Re-solve now (After Effects re-solves after deleting points). + let wait = b_p(p, "wait").unwrap_or(false); + if !s.is_camera_analyzing() { + s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?; + } + Ok(json!({"deleted": del, "running": s.is_camera_analyzing()})) +} diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index cb55710b..2e2d2782 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -4,6 +4,7 @@ mod anim; pub mod anim_tools; mod animation; +mod camera_cmds; mod comp; mod comp_more; mod create; @@ -137,6 +138,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(track::specs()); v.extend(mask_interp::specs()); v.extend(warp_cmds::specs()); + v.extend(camera_cmds::specs()); v.extend(roto_cmds::specs()); v.extend(paint::specs()); v.extend(puppet::specs()); diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index 4cbd80c2..27cbc5d4 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -50,7 +50,6 @@ pub fn specs() -> Vec { stub!("keys.audioToKeyframes", "Convert Audio to Keyframes", ["Animation", "Keyframe Assistant"], "{}"), stub!("keys.rpfCameraImport", "RPF Camera Import", ["Animation", "Keyframe Assistant"], "{}"), stub!("text.animatorFontAxes", "Variable Font Axes", ["Animation", "Animate Text"], "{}"), - stub!("track.camera", "Track Camera", ["Animation"], "{}"), // Workspaces and panels that don't exist yet. stub!("window.unavailablePanel", "Panel", [], "{panel}"), ] diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 0da0b36d..a6dbe2c4 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -9,6 +9,7 @@ //! (compositions are `Arc`s, so untouched comps are shared). pub mod autosave; +pub mod camera_track; pub mod commands; pub mod config; pub mod demo; @@ -174,6 +175,9 @@ pub struct EditorState { /// Tracker panel ▸ Method in mask mode (Track Mask). #[serde(default)] pub mask_track_method: mask_track::MaskMethod, + /// 3D Camera Tracker: the selected track points (ids) in the viewer. + #[serde(default)] + pub camera_points: Vec, /// Mask Interpolation panel options. #[serde(default)] pub mask_interp: commands::mask_interp::MaskInterpOptions, @@ -270,6 +274,10 @@ pub struct Session { pub warp_job: Option, /// Warp Stabilizers waiting for (re-)analysis: (comp, layer, effect uid). pub warp_pending: Vec<(ItemId, LayerId, Uid)>, + /// The running (or finished, not yet polled) 3D Camera Tracker analysis. + pub camera_job: Option, + /// 3D Camera Trackers waiting for (re-)analysis: (comp, layer, effect uid). + pub camera_pending: Vec<(ItemId, LayerId, Uid)>, /// The running (or finished, not yet polled) Roto Brush propagation / Freeze. pub roto_job: Option, /// Roto Brush instances edited since their last propagation: (comp, layer, effect uid). @@ -338,6 +346,8 @@ impl Default for Session { mask_job: None, warp_job: None, warp_pending: vec![], + camera_job: None, + camera_pending: vec![], roto_job: None, roto_pending: vec![], events: vec![], @@ -408,6 +418,12 @@ impl Session { self.warp_pending.push(w); } } + // So are 3D Camera Tracker tracks (and solves made with other settings). + for w in camera_track::invalidate(&before, &mut p) { + if !self.camera_pending.contains(&w) { + self.camera_pending.push(w); + } + } // Roto Brush Input Keys follow the frames; edited instances propagate again. for r in roto::sync(&before, &mut p) { if !self.roto_pending.contains(&r) { @@ -594,6 +610,9 @@ impl Session { self.stop_warp(); self.warp_job = None; self.warp_pending.clear(); + self.stop_camera(); + self.camera_job = None; + self.camera_pending.clear(); self.stop_roto(); self.roto_job = None; self.roto_pending.clear(); @@ -620,6 +639,8 @@ mod tests_3d; #[cfg(test)] mod tests_anim_tools; #[cfg(test)] +mod tests_camera_track; +#[cfg(test)] mod tests_disk_cache; #[cfg(test)] mod tests_effects; diff --git a/crates/engine/src/tests_camera_track.rs b/crates/engine/src/tests_camera_track.rs new file mode 100644 index 00000000..5bd5539c --- /dev/null +++ b/crates/engine/src/tests_camera_track.rs @@ -0,0 +1,307 @@ +//! The 3D Camera Tracker end to end through the commands, on synthetic footage rendered in-test: +//! textured planes (a floor, a back wall and a box face) seen through a known moving camera. + +use effectcraft_effects::camera_tracker as ct; +use effectcraft_geom::vec3; +use effectcraft_project::LayerId; +use effectcraft_raster::Image; +use effectcraft_render::EvalCtx; +use effectcraft_render::three_d::camera::active_camera; +use effectcraft_track::camtrack::linalg::{self, M3, V3}; +use serde_json::{Value, json}; + +use crate::Session; +use crate::tests_track::{H, W, frame_time, setup, texture}; + +const F: f64 = 260.0; + +/// The true camera on frame `f`: (world → camera rotation, centre). +fn true_camera(f: u32) -> (M3, V3) { + let t = f as f64 / 24.0; + let c = [-1.0 + 2.0 * t, -0.3 + 0.2 * (t * 3.0).sin(), 0.5 * t]; + let fwd = linalg::normalize(linalg::sub([0.2 * t, 0.4, 8.0], c)); + let right = linalg::normalize(linalg::cross([0.0, 1.0, 0.0], fwd)); + ([right, linalg::cross(fwd, right), fwd], c) +} + +/// Planes: (point, normal, u axis, v axis, half extent or 0 = infinite). +const PLANES: [(V3, V3, V3, V3, f64); 3] = [ + ([0.0, 1.5, 0.0], [0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0], 0.0), + ([0.0, 0.0, 10.0], [0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0], 0.0), + ([-1.2, 0.2, 6.0], [0.3, 0.0, -1.0], [0.957_826, 0.0, 0.287_348], [0.0, 1.0, 0.0], 1.1), +]; + +fn scene_frame(f: u32) -> Image { + let (r, c) = true_camera(f); + let mut img = Image::new(W, H); + for y in 0..H { + for x in 0..W { + let d = linalg::mtv(&r, [(x as f64 + 0.5 - W as f64 / 2.0) / F, (y as f64 + 0.5 - H as f64 / 2.0) / F, 1.0]); + let mut best = (f64::INFINITY, 0.2f32); + for (i, (p0, n, ua, va, ext)) in PLANES.iter().enumerate() { + let den = linalg::dot(*n, d); + if den.abs() < 1e-9 { + continue; + } + let tt = linalg::dot(*n, linalg::sub(*p0, c)) / den; + if tt <= 0.1 || tt >= best.0 { + continue; + } + let hit = linalg::add(c, linalg::scale(d, tt)); + let (u, v) = (linalg::dot(linalg::sub(hit, *p0), *ua), linalg::dot(linalg::sub(hit, *p0), *va)); + if *ext > 0.0 && (u.abs() > *ext || v.abs() > *ext) { + continue; + } + best = (tt, texture(u * 40.0, v * 40.0, 11 + i as u32)); + } + img.set(x, y, [best.1, best.1, best.1, 1.0]); + } + } + img +} + +/// The comp with the clip tracked (blocking) with `params`. +fn tracked(params: Value) -> (Session, LayerId) { + let (mut s, clip, _) = setup(scene_frame); + s.execute("layer.select", json!({"layers": [clip.0]})).unwrap(); + assert!(s.is_enabled("track.camera")); + let mut p = params; + p["wait"] = json!(true); + let r = s.execute_checked("track.camera", p).unwrap(); + assert_eq!(r["frames"], 25, "{r}"); + assert!(s.camera_job.is_none()); + (s, clip) +} + +fn status(s: &mut Session) -> Value { + s.execute("camera.solveStatus", json!({})).unwrap() +} + +#[test] +fn track_camera_solves_and_creates_layers_on_the_plane() { + let (mut s, clip) = tracked(json!({})); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + let err = st["averageError"].as_f64().unwrap(); + assert!(err < 0.5, "average error {err}"); + let f = st["focalLength"].as_f64().unwrap(); + assert!((f - F).abs() / F < 0.05, "focal {f}"); + assert_ne!(st["methodUsed"], "Tripod Pan"); + // Method Used / Average Error are reported in the effect. + let g = s.active_comp().unwrap().layer(clip).unwrap().effects().unwrap().groups().next().unwrap().clone(); + assert!(matches!(g.prop(ct::METHOD_USED).map(|p| &p.value), Some(effectcraft_keyframe::Value::Str(m)) if !m.is_empty())); + + // Pick floor points on frame 12 (their 3D positions lie on y = 1.5 in the true scene: the + // ones seen below the horizon at the bottom of the frame). + s.execute("time.set", json!({"time": frame_time(12).seconds()})).unwrap(); + let pts = s.execute("camera.points", json!({})).unwrap(); + let list = pts["points"].as_array().unwrap(); + assert!(list.len() > 30, "{}", list.len()); + let floor: Vec = list.iter().filter(|p| p["comp"][1].as_f64().unwrap() > H as f64 * 0.8).map(|p| p["id"].as_u64().unwrap()).take(8).collect(); + assert!(floor.len() >= 3, "{floor:?}"); + s.execute("camera.selectPoints", json!({"points": floor})).unwrap(); + assert_eq!(s.state.camera_points.len(), floor.len()); + + // Create Solid and Camera: one undo step, camera + solid. + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + let cam = LayerId(r["camera"].as_u64().unwrap()); + let solid = LayerId(r["layers"][0].as_u64().unwrap()); + let comp = s.active_comp().unwrap().clone(); + assert!(comp.layer(cam).unwrap().is_camera()); + assert_eq!(comp.layer(cam).unwrap().name, "3D Tracker Camera"); + assert_eq!(comp.layers[0].id, cam); + let sl = comp.layer(solid).unwrap(); + assert!(sl.switches.three_d); + assert!(sl.name.starts_with("Track Solid")); + // The camera is keyed on every frame. + assert_eq!(comp.layer(cam).unwrap().props.prop("transform/position").unwrap().keys.len(), 25); + + // Every solved point projects through the created camera exactly where the solve puts it, + // and onto its tracked position (the footage fills the comp, so layer pixels are comp + // pixels): within 1 px for 95 % of the observations (the rest are the solve's outliers: + // features on occlusion edges), 0.5 px on average. + let cid = s.active_comp_id().unwrap(); + let uid = g.uid; + let pl = crate::camera_track::placed(&s.project, cid, clip, uid).unwrap(); + let tracks = ct::tracks(&crate::camera_track::static_params(&g)).unwrap(); + let mut errs = vec![]; + for k in [0u32, 6, 12, 18, 24] { + let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(k)); + let camst = active_camera(&ctx); + for sp in pl.solve.visible(k as usize) { + let Some(obs) = tracks.track(sp.id).and_then(|t| t.at(k)) else { continue }; + let w = pl.to_world(sp.pos); + let q = camst.project(W as f64, H as f64, vec3(w[0], w[1], w[2])).unwrap(); + let own = pl.solve.frames[k as usize].project(pl.solve.size, sp.pos).unwrap(); + assert!((q.x - own[0]).hypot(q.y - own[1]) < 0.01, "camera layer disagrees with the solve"); + errs.push((q.x - obs[0]).hypot(q.y - obs[1])); + } + } + assert!(errs.len() > 100, "{}", errs.len()); + errs.sort_by(f64::total_cmp); + let p95 = errs[errs.len() * 95 / 100]; + let mean = errs.iter().sum::() / errs.len() as f64; + assert!(p95 < 1.0 && mean < 0.5, "p95 {p95} px, mean {mean} px"); + + // The solid lies on the selected points' plane. + let tr = sl.props.sub("transform").unwrap(); + let pos = tr.get("position").unwrap().value.as_vec3(); + let ori = tr.get("orientation").unwrap().value.as_vec3(); + let normal = linalg::mv(&linalg::from_euler_xyz(ori), [0.0, 0.0, -1.0]); + let size = r["target"]["size"].as_f64().unwrap(); + for id in &s.state.camera_points { + let w = pl.to_world(pl.solve.point(*id).unwrap().pos); + let d = linalg::dot(linalg::sub(w, pos), normal).abs(); + assert!(d < 0.05 * size.max(1.0) + 1.0, "point {id} is {d} px off the plane"); + } + // The solid faces the camera. + let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(12)); + let eye = active_camera(&ctx).eye; + assert!(linalg::dot(normal, linalg::sub([eye.x, eye.y, eye.z], pos)) > 0.0); + + // Undo removes both layers; redo brings them back. + let before = comp.layers.len(); + assert!(s.undo()); + assert_eq!(s.active_comp().unwrap().layers.len(), before - 2); + assert!(s.redo()); + assert_eq!(s.active_comp().unwrap().layers.len(), before); + + // A second create reuses the camera: Create Multiple Nulls and Camera. + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "null", "multiple": true})).unwrap(); + assert_eq!(r["camera"].as_u64().unwrap(), cam.0); + assert_eq!(r["layers"].as_array().unwrap().len(), floor.len()); + // Text and Shadow Catcher (+ light). + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "text"})).unwrap(); + assert_eq!(r["layers"].as_array().unwrap().len(), 1); + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "shadowCatcher"})).unwrap(); + let ids: Vec = r["layers"].as_array().unwrap().iter().map(|v| LayerId(v.as_u64().unwrap())).collect(); + let comp = s.active_comp().unwrap(); + assert!(comp.layer(ids[0]).unwrap().name.starts_with("Shadow Catcher")); + assert_eq!(comp.layer(ids[0]).unwrap().props.prop("materialOptions/acceptsShadows").unwrap().value.as_enum(), 2); + assert!(comp.layer(ids[1]).unwrap().is_light()); + assert!(s.execute_checked("camera.createFromSolve", json!({"kind": "teapot"})).is_err()); + + // Set Ground Plane and Origin: the target becomes the origin; a null created there sits at + // (0, 0, 0) lying on the X-Z plane (its front facing up, −y). + s.execute_checked("camera.setGroundPlane", json!({})).unwrap(); + assert_eq!(status(&mut s)["groundPlane"], true); + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "null"})).unwrap(); + let null = LayerId(r["layers"][0].as_u64().unwrap()); + let tr = s.active_comp().unwrap().layer(null).unwrap().props.sub("transform").unwrap().clone(); + let pos = tr.get("position").unwrap().value.as_vec3(); + assert!(linalg::norm(pos) < 1e-6, "{pos:?}"); + let up = linalg::mv(&linalg::from_euler_xyz(tr.get("orientation").unwrap().value.as_vec3()), [0.0, 0.0, -1.0]); + assert!(linalg::norm(linalg::sub(up, [0.0, -1.0, 0.0])) < 1e-6, "{up:?}"); + + // An explicit target (comp world) places the layer there, its front facing the normal. + let r = s + .execute_checked("camera.createFromSolve", json!({"kind": "solid", "target": {"center": [10.0, 0.0, 20.0], "normal": [0.0, 0.0, -1.0], "size": 80.0}})) + .unwrap(); + let l = s.active_comp().unwrap().layer(LayerId(r["layers"][0].as_u64().unwrap())).unwrap().clone(); + let tr = l.props.sub("transform").unwrap(); + let pos = tr.get("position").unwrap().value.as_vec3(); + assert!(linalg::norm(linalg::sub(pos, [10.0, 0.0, 20.0])) < 1e-6, "{pos:?}"); + let front = linalg::mv(&linalg::from_euler_xyz(tr.get("orientation").unwrap().value.as_vec3()), [0.0, 0.0, -1.0]); + assert!(linalg::norm(linalg::sub(front, [0.0, 0.0, -1.0])) < 1e-6, "{front:?}"); + + // Serde keeps the analysis. + let back = effectcraft_project::Project::from_json(&s.project.to_json()).unwrap(); + let g2 = back.comp(cid).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap(); + let sv = ct::solve(&crate::camera_track::static_params(g2)).unwrap(); + assert!(sv.ground.is_some()); + assert_eq!(sv.points.len(), pl.solve.points.len()); +} + +#[test] +fn invalidation_resolve_cancel_and_undo() { + let (mut s, clip) = tracked(json!({"shotType": "specify", "aov": 2.0 * (W as f64 / 2.0 / F).atan().to_degrees()})); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + assert_eq!(st["shotType"], "Specify Angle of View"); + assert!((st["focalLength"].as_f64().unwrap() - F).abs() < 0.01 * F); + let cid = s.active_comp_id().unwrap(); + let uid = st["effect"].as_u64().unwrap(); + // Undo removes the analysis (one step); redo restores it. + assert!(s.undo()); + assert_eq!(status(&mut s)["analyzed"], false); + assert!(s.redo()); + assert_eq!(status(&mut s)["solved"], true); + + // Changing the Solve Method keeps the tracks and only re-solves. + s.execute_checked("prop.set", json!({"layer": clip.0, "path": "effects/#1/advanced/solveMethod", "value": 1})).unwrap(); + let st = status(&mut s); + assert_eq!(st["analyzed"], true); + assert_eq!(st["solved"], false); + assert_eq!(st["pending"], true); + s.execute_checked("camera.analyze", json!({"wait": true})).unwrap(); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + assert_eq!(st["methodUsed"], "Typical"); + + // Deleting points re-solves without them. + let pts = s.execute("camera.points", json!({})).unwrap(); + let del: Vec = pts["points"].as_array().unwrap().iter().take(3).map(|p| p["id"].as_u64().unwrap()).collect(); + s.execute_checked("camera.deletePoints", json!({"points": del, "wait": true})).unwrap(); + s.poll_camera(false); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + assert!(st["deleted"].as_u64().unwrap() >= 3); + let g = s.project.comp(cid).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().clone(); + let sv = ct::solve(&crate::camera_track::static_params(&g)).unwrap(); + assert!(sv.points.iter().all(|p| !del.contains(&(p.id as u64)))); + + // Trimming the layer changes its frames: tracks and solve are cleared and queued. + s.edit("trim", None, |p, _| { + p.comp_mut(cid).unwrap().layer_mut(clip).unwrap().in_point = frame_time(2); + Ok(()) + }) + .unwrap(); + let st = status(&mut s); + assert_eq!(st["analyzed"], false); + assert_eq!(st["pending"], true); + + // A background analysis can be cancelled: nothing is written. + s.execute_checked("camera.analyze", json!({})).unwrap(); + assert!(s.is_camera_analyzing() || s.camera_job.is_some()); + let banner = status(&mut s)["banner"].as_str().map(str::to_string); + if s.is_camera_analyzing() { + assert!(banner.unwrap().starts_with("Analyzing in background (step 1 of 2)")); + } + s.execute("camera.cancel", json!({})).ok(); + let t0 = std::time::Instant::now(); + while s.camera_job.is_some() { + assert!(t0.elapsed().as_secs() < 120); + std::thread::sleep(std::time::Duration::from_millis(5)); + s.poll_camera(false); + } + // (When the job finished before the cancel landed it wrote a valid analysis.) + let st = status(&mut s); + assert_eq!(st["running"], false); + assert!(s.execute_checked("track.camera", json!({"layer": clip.0, "shotType": "bogus"})).is_err()); + assert!(s.execute_checked("track.camera", json!({"layer": clip.0, "aov": 500})).is_err()); +} + +#[test] +fn effect_parameters_match_after_effects() { + let spec = effectcraft_effects::registry().iter().find(|e| e.id == ct::ID).unwrap(); + assert_eq!(spec.name, "3D Camera Tracker"); + assert_eq!(spec.category, "Perspective"); + let names: Vec<&str> = spec.params.iter().filter(|p| !matches!(p.ui, effectcraft_project::ParamUi::Hidden)).map(|p| p.name).collect(); + for n in [ + "Shot Type", + "Horizontal Angle of View", + "Show Track Points", + "Render Track Points", + "Track Point Size", + "Target Size", + "Solve Method", + "Detailed Analysis", + "Auto-delete Points Across Time", + "Hide Warning Banner", + ] { + assert!(names.contains(&n), "{n}"); + } + // The Animation menu entry is a real command now. + let c = crate::find_command("track.camera").unwrap(); + assert_eq!(c.label, "Track Camera"); +} diff --git a/crates/engine/src/tests_stubs.rs b/crates/engine/src/tests_stubs.rs index 17ffda7f..725cf600 100644 --- a/crates/engine/src/tests_stubs.rs +++ b/crates/engine/src/tests_stubs.rs @@ -332,7 +332,7 @@ fn implemented_stubs_are_enabled_and_workspaces_are_frontend() { assert!(s.drain_events().iter().any(|e| matches!(e, Event::Frontend { command, .. } if command == id))); } // What remains a stub is still disabled. - for id in ["view.displayColorManagement", "file.createProxy", "track.camera"] { + for id in ["view.displayColorManagement", "file.createProxy"] { assert!(!s.is_enabled(id), "{id}"); } } diff --git a/crates/track/Cargo.toml b/crates/track/Cargo.toml index 70e08ae0..33ffacae 100644 --- a/crates/track/Cargo.toml +++ b/crates/track/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-track" -description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, corner-pin / transform solves, stabilization and Roto Brush segmentation (graph cut, matting)" +description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, corner-pin / transform solves, stabilization, Roto Brush segmentation (graph cut, matting) and 3D camera solving (structure from motion)" version.workspace = true edition.workspace = true license.workspace = true @@ -14,3 +14,4 @@ effectcraft-raster = { workspace = true } rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +web-time = { workspace = true } diff --git a/crates/track/src/camtrack/bundle.rs b/crates/track/src/camtrack/bundle.rs new file mode 100644 index 00000000..72cb2f17 --- /dev/null +++ b/crates/track/src/camtrack/bundle.rs @@ -0,0 +1,470 @@ +//! Sparse bundle adjustment: Levenberg–Marquardt over cameras and points with the **Schur +//! complement** eliminating the points (Triggs, McLauchlan, Hartley & Fitzgibbon, "Bundle +//! Adjustment — A Modern Synthesis", 2000, §6; H&Z Appendix 6). Residuals are reprojection +//! errors in pixels with a **Huber** robust loss (applied by iteratively reweighted least +//! squares, Triggs et al. §3.3). +//! +//! Cameras are `(R, C, f)`; rotations are updated multiplicatively (`R ← exp([δ]×) R`), focal +//! lengths in log space. The focal length is held fixed, shared by all cameras, or per camera +//! (with a smoothness prior between consecutive cameras). Tripod solves hold the centres fixed. + +use rayon::prelude::*; + +use super::geometry::Pose; +use super::linalg::*; + +/// A camera in the adjustment. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct BaCam { + pub pose: Pose, + /// Focal length (pixels). + pub f: f64, +} + +/// One observation: camera `cam` sees point `pt` at `uv` (pixels relative to the principal +/// point). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct BaObs { + pub cam: u32, + pub pt: u32, + pub uv: [f64; 2], +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum FocalMode { + Fixed, + Shared, + /// Per camera, with prior weight `k` on the log-focal difference of consecutive cameras. + PerCamera(f64), +} + +#[derive(Clone, Debug)] +pub struct BaOpts { + pub focal: FocalMode, + /// Motion only: the points stay fixed. + pub fix_points: bool, + /// The camera centres stay fixed (tripod). + pub fix_centers: bool, + /// Cameras that stay fixed (gauge), by index; missing entries are free. + pub fixed: Vec, + /// Huber threshold (pixels). + pub huber: f64, + pub max_iter: usize, +} + +impl Default for BaOpts { + fn default() -> Self { + BaOpts { focal: FocalMode::Fixed, fix_points: false, fix_centers: false, fixed: vec![], huber: 2.0, max_iter: 30 } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct BaReport { + pub initial_cost: f64, + pub cost: f64, + pub iterations: usize, +} + +const BEHIND: f64 = 1e6; + +fn huber(e2: f64, k: f64) -> f64 { + if e2 <= k * k { e2 } else { 2.0 * k * e2.sqrt() - k * k } +} + +/// Residual (pixels) of one observation, `None` behind the camera. +pub fn residual(c: &BaCam, x: V3, uv: [f64; 2]) -> Option<[f64; 2]> { + let p = c.pose.to_cam(x); + if p[2] <= 1e-9 { + return None; + } + Some([c.f * p[0] / p[2] - uv[0], c.f * p[1] / p[2] - uv[1]]) +} + +/// Total robust cost. +pub fn cost(cams: &[BaCam], pts: &[V3], obs: &[BaObs], o: &BaOpts) -> f64 { + let c: f64 = obs + .par_iter() + .map(|ob| match residual(&cams[ob.cam as usize], pts[ob.pt as usize], ob.uv) { + Some(r) => huber(r[0] * r[0] + r[1] * r[1], o.huber), + None => BEHIND, + }) + .sum(); + c + prior_cost(cams, o) +} + +fn prior_cost(cams: &[BaCam], o: &BaOpts) -> f64 { + match o.focal { + FocalMode::PerCamera(k) => cams.windows(2).map(|w| (k * (w[0].f.ln() - w[1].f.ln())).powi(2)).sum(), + _ => 0.0, + } +} + +/// Parameter layout. +struct Layout { + /// First pose parameter of each camera (`usize::MAX` when its pose is fixed). + start: Vec, + /// Per-camera focal parameter (`usize::MAX` when none). Gauge-fixed cameras keep a free + /// focal length. + fidx: Vec, + /// Index of the shared focal parameter. + shared: Option, + n: usize, + rot_only: bool, +} + +fn layout(ncam: usize, o: &BaOpts) -> Layout { + let per_cam_f = matches!(o.focal, FocalMode::PerCamera(_)); + let d = if o.fix_centers { 3 } else { 6 }; + let mut start = vec![usize::MAX; ncam]; + let mut fidx = vec![usize::MAX; ncam]; + let mut n = 0; + for i in 0..ncam { + if !o.fixed.get(i).copied().unwrap_or(false) { + start[i] = n; + n += d; + } + if per_cam_f { + fidx[i] = n; + n += 1; + } + } + let shared = matches!(o.focal, FocalMode::Shared).then(|| { + n += 1; + n - 1 + }); + Layout { start, fidx, shared, n, rot_only: o.fix_centers } +} + +/// Camera-side Jacobian entries of one observation: (parameter index, d r / d param). +fn cam_jac(l: &Layout, ci: usize, c: &BaCam, xc: V3, a: &[[f64; 3]; 2], p: [f64; 2], out: &mut Vec<(usize, [f64; 2])>) { + out.clear(); + let s = l.start[ci]; + if s != usize::MAX { + // dXc/dδ = −[Xc]× + let sk = skew(xc); + for k in 0..3 { + let g = [-sk[0][k], -sk[1][k], -sk[2][k]]; + out.push((s + k, [dot(a[0], g), dot(a[1], g)])); + } + if !l.rot_only { + // dXc/dC = −R + for k in 0..3 { + let g = [-c.pose.r[0][k], -c.pose.r[1][k], -c.pose.r[2][k]]; + out.push((s + 3 + k, [dot(a[0], g), dot(a[1], g)])); + } + } + } + if l.fidx[ci] != usize::MAX { + out.push((l.fidx[ci], p)); + } + if let Some(g) = l.shared { + out.push((g, p)); + } +} + +/// Run Levenberg–Marquardt. `cams` and `pts` are updated in place. +pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> BaReport { + let l = layout(cams.len(), o); + let npts = pts.len(); + // Observations by point. + let mut by_pt: Vec> = vec![vec![]; npts]; + for (i, ob) in obs.iter().enumerate() { + by_pt[ob.pt as usize].push(i as u32); + } + let mut cur = cost(cams, pts, obs, o); + let initial = cur; + let t0 = web_time::Instant::now(); + let mut tries = 0usize; + let dbg = std::env::var_os("EC_CAMTRACK_DEBUG").is_some() && l.n > 200; + let mut lambda = 1e-4; + let mut iters = 0; + if l.n == 0 && o.fix_points { + return BaReport { initial_cost: initial, cost: cur, iterations: 0 }; + } + let n = l.n; + let parallel = n <= 700; + for _ in 0..o.max_iter { + iters += 1; + // ---- Normal equations (Gauss–Newton with IRLS weights). + // Per point: V (3×3), bp (3), W entries (cam-side index → 3). + struct PtSys { + v: [[f64; 3]; 3], + bp: V3, + w: Vec<(usize, V3)>, + } + let build_point = |j: usize, s_acc: &mut Vec, bc: &mut Vec| -> PtSys { + let mut ps = PtSys { v: [[0.0; 3]; 3], bp: [0.0; 3], w: vec![] }; + let mut jc: Vec<(usize, [f64; 2])> = Vec::with_capacity(8); + for &oi in &by_pt[j] { + let ob = &obs[oi as usize]; + let ci = ob.cam as usize; + let c = &cams[ci]; + let xc = c.pose.to_cam(pts[j]); + if xc[2] <= 1e-9 { + continue; + } + let iz = 1.0 / xc[2]; + let p = [c.f * xc[0] * iz, c.f * xc[1] * iz]; + let r = [p[0] - ob.uv[0], p[1] - ob.uv[1]]; + let e = (r[0] * r[0] + r[1] * r[1]).sqrt(); + let w = if e <= o.huber { 1.0 } else { o.huber / e }; + let a = [[c.f * iz, 0.0, -c.f * xc[0] * iz * iz], [0.0, c.f * iz, -c.f * xc[1] * iz * iz]]; + cam_jac(&l, ci, c, xc, &a, p, &mut jc); + // Point Jacobian: A R. + let jx = if o.fix_points { + [[0.0; 3]; 2] + } else { + let r0 = mtv(&c.pose.r, a[0]); + let r1 = mtv(&c.pose.r, a[1]); + [r0, r1] + }; + for (ia, ja) in &jc { + bc[*ia] -= w * (ja[0] * r[0] + ja[1] * r[1]); + for (ib, jb) in &jc { + s_acc[ia * n + ib] += w * (ja[0] * jb[0] + ja[1] * jb[1]); + } + } + if !o.fix_points { + for a1 in 0..3 { + ps.bp[a1] -= w * (jx[0][a1] * r[0] + jx[1][a1] * r[1]); + for b1 in 0..3 { + ps.v[a1][b1] += w * (jx[0][a1] * jx[0][b1] + jx[1][a1] * jx[1][b1]); + } + } + for (ia, ja) in &jc { + let wv = + [w * (ja[0] * jx[0][0] + ja[1] * jx[1][0]), w * (ja[0] * jx[0][1] + ja[1] * jx[1][1]), w * (ja[0] * jx[0][2] + ja[1] * jx[1][2])]; + // Each observation of a point is on another camera: only the shared focal + // length repeats. + let found = if Some(*ia) == l.shared { ps.w.iter_mut().find(|e| e.0 == *ia) } else { None }; + match found { + Some(e) => e.1 = add(e.1, wv), + None => ps.w.push((*ia, wv)), + } + } + } + } + ps + }; + let (mut s_mat, mut bc, systems): (Vec, Vec, Vec) = if parallel && npts > 64 { + let chunks = rayon::current_num_threads().clamp(1, 16); + let per = npts.div_ceil(chunks); + let parts: Vec<(Vec, Vec, Vec)> = (0..chunks) + .into_par_iter() + .map(|t| { + let (mut s, mut b) = (vec![0.0; n * n], vec![0.0; n]); + let v = (t * per..((t + 1) * per).min(npts)).map(|j| build_point(j, &mut s, &mut b)).collect(); + (s, b, v) + }) + .collect(); + let mut s = vec![0.0; n * n]; + let mut b = vec![0.0; n]; + let mut sys = Vec::with_capacity(npts); + for (ps, pb, pv) in parts { + for (x, y) in s.iter_mut().zip(ps) { + *x += y; + } + for (x, y) in b.iter_mut().zip(pb) { + *x += y; + } + sys.extend(pv); + } + (s, b, sys) + } else { + let mut s = vec![0.0; n * n]; + let mut b = vec![0.0; n]; + let sys = (0..npts).map(|j| build_point(j, &mut s, &mut b)).collect(); + (s, b, sys) + }; + // Focal prior. + if let FocalMode::PerCamera(k) = o.focal { + for i in 0..cams.len().saturating_sub(1) { + let r = k * (cams[i].f.ln() - cams[i + 1].f.ln()); + let (a, b) = (l.fidx[i], l.fidx[i + 1]); + let ia = (a != usize::MAX).then_some(a); + let ib = (b != usize::MAX).then_some(b); + for (idx, g) in [(ia, k), (ib, -k)] { + let Some(x) = idx else { continue }; + bc[x] -= g * r; + for (idy, h) in [(ia, k), (ib, -k)] { + if let Some(y) = idy { + s_mat[x * n + y] += g * h; + } + } + } + } + } + let base_s = s_mat.clone(); + let base_bc = bc.clone(); + let mean_diag = ((0..n).map(|i| base_s[i * n + i]).sum::() / n.max(1) as f64).max(1e-12); + // ---- Try steps with increasing damping. + let mut accepted = false; + for _try in 0..12 { + tries += 1; + let mut s = base_s.clone(); + let mut rhs = base_bc.clone(); + for i in 0..n { + let d = s[i * n + i]; + s[i * n + i] = d + lambda * d.max(1e-6 * mean_diag); + } + // Schur complement over the points: S −= W V⁻¹ Wᵀ, rhs −= W V⁻¹ bp (in parallel + // chunks of points when the reduced system is small enough to copy per thread). + let vinv: Vec> = if o.fix_points { + vec![None; npts] + } else { + systems + .par_iter() + .map(|ps| { + let mut v = ps.v; + let vd = (v[0][0] + v[1][1] + v[2][2]) / 3.0; + for k in 0..3 { + v[k][k] += lambda * v[k][k].max(1e-6 * vd.max(1e-12)) + 1e-12; + } + inv3(&v) + }) + .collect() + }; + if !o.fix_points { + let reduce = |range: std::ops::Range, s: &mut [f64], rhs: &mut [f64]| { + for j in range { + let (Some(vi), ps) = (&vinv[j], &systems[j]) else { continue }; + let vb = mv(vi, ps.bp); + let wv: Vec = ps.w.iter().map(|(_, w)| mv(vi, *w)).collect(); + for (a, (ia, wa)) in ps.w.iter().enumerate() { + rhs[*ia] -= dot(*wa, vb); + for (ib, wb) in &ps.w[a..] { + let d = dot(wv[a], *wb); + s[ia * n + ib] -= d; + if ia != ib { + s[ib * n + ia] -= d; + } + } + } + } + }; + let chunks = rayon::current_num_threads().clamp(1, 8); + if n <= 1200 && npts >= 64 && chunks > 1 { + let per = npts.div_ceil(chunks); + let parts: Vec<(Vec, Vec)> = (0..chunks) + .into_par_iter() + .map(|t| { + let (mut ds, mut dr) = (vec![0.0; n * n], vec![0.0; n]); + reduce(t * per..((t + 1) * per).min(npts), &mut ds, &mut dr); + (ds, dr) + }) + .collect(); + for (ds, dr) in parts { + s.par_iter_mut().zip(ds.par_iter()).for_each(|(x, y)| *x += y); + for (x, y) in rhs.iter_mut().zip(dr) { + *x += y; + } + } + } else { + reduce(0..npts, &mut s, &mut rhs); + } + } + let mut dc = rhs; + if n > 0 && !cholesky_solve(&mut s, n, &mut dc) { + lambda *= 10.0; + continue; + } + // Apply. + let mut nc: Vec = cams.to_vec(); + for (i, c) in nc.iter_mut().enumerate() { + let st = l.start[i]; + if st != usize::MAX { + let dr = [dc[st], dc[st + 1], dc[st + 2]]; + c.pose.r = mm(&rodrigues(dr), &c.pose.r); + if !l.rot_only { + c.pose.c = add(c.pose.c, [dc[st + 3], dc[st + 4], dc[st + 5]]); + } + } + if l.fidx[i] != usize::MAX { + c.f *= dc[l.fidx[i]].clamp(-0.5, 0.5).exp(); + } + if let Some(g) = l.shared { + c.f *= dc[g].clamp(-0.5, 0.5).exp(); + } + } + let mut np: Vec = pts.to_vec(); + if !o.fix_points { + np.par_iter_mut().enumerate().for_each(|(j, x)| { + let Some(vi) = &vinv[j] else { return }; + let ps = &systems[j]; + let mut b = ps.bp; + for (ia, w) in &ps.w { + b = sub(b, scale(*w, dc[*ia])); + } + *x = add(*x, mv(vi, b)); + }); + } + let c2 = cost(&nc, &np, obs, o); + if c2.is_finite() && c2 < cur { + cams.copy_from_slice(&nc); + pts.copy_from_slice(&np); + let gain = cur - c2; + cur = c2; + lambda = (lambda / 3.0).max(1e-9); + accepted = gain > 1e-6 * cur.max(1e-12); + if !accepted { + // Converged (tiny improvement). + if dbg { + eprintln!( + "ba n={} pts={} obs={} iters={iters} tries={tries} {:.2}s cost {initial:.1} -> {cur:.1}", + l.n, + npts, + obs.len(), + t0.elapsed().as_secs_f64() + ); + } + return BaReport { initial_cost: initial, cost: cur, iterations: iters }; + } + break; + } + lambda *= 4.0; + } + if !accepted { + break; + } + } + if dbg { + eprintln!("ba n={} pts={} obs={} iters={iters} tries={tries} {:.2}s cost {initial:.1} -> {cur:.1}", l.n, npts, obs.len(), t0.elapsed().as_secs_f64()); + } + BaReport { initial_cost: initial, cost: cur, iterations: iters } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::camtrack::geometry::Rng; + + #[test] + fn bundle_adjustment_converges_from_noise() { + let mut rng = Rng(99); + let mut u = || (rng.next_u64() % 100_000) as f64 / 100_000.0 - 0.5; + let truth: Vec = (0..80).map(|_| [u() * 6.0, u() * 4.0, 6.0 + u() * 4.0]).collect(); + let f = 900.0; + let cams_true: Vec = (0..6) + .map(|i| BaCam { pose: Pose { r: rodrigues([0.0, -0.05 * i as f64, 0.01 * i as f64]), c: [0.3 * i as f64, 0.05 * i as f64, 0.0] }, f }) + .collect(); + let mut obs = vec![]; + for (ci, c) in cams_true.iter().enumerate() { + for (pi, x) in truth.iter().enumerate() { + let r = residual(c, *x, [0.0, 0.0]).unwrap(); + obs.push(BaObs { cam: ci as u32, pt: pi as u32, uv: r }); + } + } + let mut cams = cams_true.clone(); + for (i, c) in cams.iter_mut().enumerate().skip(2) { + c.pose.r = mm(&rodrigues([0.01, -0.01 * i as f64 * 0.3, 0.005]), &c.pose.r); + c.pose.c = add(c.pose.c, [0.02, -0.03, 0.01]); + } + let mut pts: Vec = truth.iter().map(|x| add(*x, [u() * 0.1, u() * 0.1, u() * 0.2])).collect(); + let mut fixed = vec![false; 6]; + fixed[0] = true; + fixed[1] = true; + let rep = adjust(&mut cams, &mut pts, &obs, &BaOpts { fixed, max_iter: 50, ..Default::default() }); + let rms = (rep.cost / obs.len() as f64).sqrt(); + assert!(rms < 1e-4, "{rep:?}"); + } +} diff --git a/crates/track/src/camtrack/features.rs b/crates/track/src/camtrack/features.rs new file mode 100644 index 00000000..30b019e5 --- /dev/null +++ b/crates/track/src/camtrack/features.rs @@ -0,0 +1,115 @@ +//! Step 1 of the camera analysis: long feature tracks through the clip. +//! +//! Every frame, the active features are followed from the previous frame with pyramidal +//! Lucas–Kanade and a forward–backward check ([`crate::klt::track`]); a feature that fails ends +//! its track. When fewer than ~85 % of the wanted features remain, new Shi–Tomasi corners are +//! detected away from the active ones and start new tracks. Tracks shorter than three frames are +//! dropped at the end. + +use crate::Frame; +use crate::klt::{CornerOpts, GrayPyramid, LkOpts, analysis_factor, good_features, track}; + +use super::{CameraTracks, Track2D}; + +/// Analysis options. +#[derive(Clone, Copy, Debug, Default)] +pub struct AnalyzeOpts { + /// Detailed Analysis: twice the resolution and more features. + pub detailed: bool, +} + +/// Incremental feature tracking: push the clip's frames in order. +pub struct TrackAnalyzer { + opts: AnalyzeOpts, + prev: Option, + /// (track index, level-0 plane position) of the live features. + active: Vec<(usize, [f64; 2])>, + tracks: Vec, + frame: u32, + size: [f64; 2], + factor: f64, +} + +impl TrackAnalyzer { + pub fn new(size: [f64; 2], opts: AnalyzeOpts) -> TrackAnalyzer { + TrackAnalyzer { opts, prev: None, active: vec![], tracks: vec![], frame: 0, size, factor: 1.0 } + } + + pub fn frames(&self) -> u32 { + self.frame + } + + /// Features currently followed. + pub fn active(&self) -> usize { + self.active.len() + } + + fn params(&self) -> (u32, usize) { + if self.opts.detailed { (1280, 900) } else { (720, 500) } + } + + /// Analyse the next frame. + pub fn push(&mut self, frame: &Frame) { + let (max_side, want) = self.params(); + let factor = analysis_factor(frame.img.width, frame.img.height, max_side); + self.factor = factor as f64; + let pyr = GrayPyramid::from_image(frame.img, frame.offset, factor, 4); + let k = self.frame; + match &self.prev { + Some(prev) if prev.width() == pyr.width() && prev.height() == pyr.height() && prev.offset == pyr.offset => { + let pts: Vec<[f64; 2]> = self.active.iter().map(|a| a.1).collect(); + let q = track(prev, &pyr, &pts, None, &LkOpts { radius: 6, iterations: 30, epsilon: 0.005, fb_max: 0.5, ..Default::default() }); + let mut next = Vec::with_capacity(self.active.len()); + for ((ti, _), q) in self.active.iter().zip(q) { + if let Some(q) = q { + let l = pyr.to_layer(q); + self.tracks[*ti].pts.push([l[0] as f32, l[1] as f32]); + next.push((*ti, q)); + } + } + self.active = next; + } + _ => self.active.clear(), + } + // Re-detect where there are no features. + if (self.active.len() as f64) < want as f64 * 0.85 { + let md = (pyr.width().max(pyr.height()) as f64 / (want as f64).sqrt() * 0.6).clamp(5.0, 40.0); + let cell = md; + let (gw, gh) = ((pyr.width() as f64 / cell).ceil() as usize + 1, (pyr.height() as f64 / cell).ceil() as usize + 1); + let mut occ = vec![false; gw * gh]; + for (_, p) in &self.active { + let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize); + for yy in cy.saturating_sub(1)..=(cy + 1).min(gh - 1) { + for xx in cx.saturating_sub(1)..=(cx + 1).min(gw - 1) { + occ[yy * gw + xx] = true; + } + } + } + let free = |p: [f64; 2]| { + let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize); + !occ.get(cy * gw + cx).copied().unwrap_or(true) + }; + let opts = CornerOpts { max_features: want - self.active.len().min(want), min_distance: md, quality: 0.001, window: 2, border: 8 }; + for p in good_features(&pyr, &opts, Some(&free)) { + let l = pyr.to_layer(p); + let id = self.tracks.len() as u32; + self.tracks.push(Track2D { id, start: k, pts: vec![[l[0] as f32, l[1] as f32]] }); + self.active.push((id as usize, p)); + } + } + self.prev = Some(pyr); + self.frame += 1; + } + + pub fn finish(self, start: f64, frame_duration: f64) -> CameraTracks { + let mut tracks: Vec = self.tracks.into_iter().filter(|t| t.pts.len() >= 3).collect(); + // Round to 1/100 pixel (smaller JSON) and renumber densely. + for (i, t) in tracks.iter_mut().enumerate() { + t.id = i as u32; + for p in t.pts.iter_mut() { + *p = [(p[0] * 100.0).round() / 100.0, (p[1] * 100.0).round() / 100.0]; + } + } + CameraTracks { version: 1, size: self.size, start, frame_duration, frames: self.frame, factor: self.factor, detailed: self.opts.detailed, tracks } + } +} diff --git a/crates/track/src/camtrack/geometry.rs b/crates/track/src/camtrack/geometry.rs new file mode 100644 index 00000000..6956ac2b --- /dev/null +++ b/crates/track/src/camtrack/geometry.rs @@ -0,0 +1,432 @@ +//! Two-view and multi-view geometry for the camera solver, from Hartley & Zisserman (H&Z), +//! *Multiple View Geometry in Computer Vision* (2nd ed.) and Ma et al., *An Invitation to 3-D +//! Vision* (2004). +//! +//! All image points here are **normalised** (calibrated) coordinates `x = (u − c) / f`. +//! A camera is `(R, C)` with `X_cam = R (X − C)`. + +use super::linalg::*; + +/// A deterministic xorshift generator for RANSAC samples. +pub struct Rng(pub u64); +impl Rng { + pub fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + pub fn below(&mut self, n: usize) -> usize { + (self.next_u64() % n.max(1) as u64) as usize + } + /// `k` distinct indices below `n`. + pub fn sample(&mut self, n: usize, k: usize, out: &mut Vec) { + out.clear(); + let mut guard = 0; + while out.len() < k && guard < 1000 { + guard += 1; + let c = self.below(n); + if !out.contains(&c) { + out.push(c); + } + } + } +} + +/// Hartley normalisation (H&Z §4.4.4): centroid to the origin, mean distance √2. +fn normalizer(p: &[[f64; 2]]) -> M3 { + let n = p.len().max(1) as f64; + let c = [p.iter().map(|q| q[0]).sum::() / n, p.iter().map(|q| q[1]).sum::() / n]; + let d = p.iter().map(|q| (q[0] - c[0]).hypot(q[1] - c[1])).sum::() / n; + let s = if d > 1e-12 { std::f64::consts::SQRT_2 / d } else { 1.0 }; + [[s, 0.0, -s * c[0]], [0.0, s, -s * c[1]], [0.0, 0.0, 1.0]] +} + +fn h2(m: &M3, p: [f64; 2]) -> V3 { + mv(m, [p[0], p[1], 1.0]) +} + +/// The essential matrix from ≥ 8 correspondences (`x2ᵀ E x1 = 0`) by the normalised 8-point +/// algorithm (H&Z Alg. 11.1), with the essential constraint (two equal singular values, one +/// zero) enforced (H&Z §9.6.1 / Result 9.18). +pub fn essential_8pt(x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Option { + let n = x1.len().min(x2.len()); + if n < 8 { + return None; + } + let (t1, t2) = (normalizer(&x1[..n]), normalizer(&x2[..n])); + let mut a = Vec::with_capacity(n * 9); + for i in 0..n { + let p = h2(&t1, x1[i]); + let q = h2(&t2, x2[i]); + a.extend_from_slice(&[q[0] * p[0], q[0] * p[1], q[0], q[1] * p[0], q[1] * p[1], q[1], p[0], p[1], 1.0]); + } + let e = null_vector(&a, n, 9); + let en = [[e[0], e[1], e[2]], [e[3], e[4], e[5]], [e[6], e[7], e[8]]]; + // Denormalise: E = T2ᵀ Ê T1. + let e = mm(&mt(&t2), &mm(&en, &t1)); + let (u, s, v) = svd3(&e); + let m = (s[0] + s[1]) * 0.5; + if m <= 0.0 || !m.is_finite() { + return None; + } + let d = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]]; + Some(mm(&u, &mm(&d, &mt(&v)))) +} + +/// Squared Sampson distance of a correspondence to `x2ᵀ E x1 = 0` (H&Z §11.4.3), in the units of +/// the coordinates. +pub fn sampson2(e: &M3, p: [f64; 2], q: [f64; 2]) -> f64 { + let x1 = [p[0], p[1], 1.0]; + let x2 = [q[0], q[1], 1.0]; + let ex1 = mv(e, x1); + let etx2 = mtv(e, x2); + let num = dot(x2, ex1); + let den = ex1[0] * ex1[0] + ex1[1] * ex1[1] + etx2[0] * etx2[0] + etx2[1] * etx2[1]; + if den < 1e-300 { f64::INFINITY } else { num * num / den } +} + +/// RANSAC essential matrix (8-point samples, Sampson distance below `thr`), refitted on the +/// consensus set. Returns the matrix and the inlier mask. +pub fn ransac_essential(x1: &[[f64; 2]], x2: &[[f64; 2]], thr: f64, iterations: usize, seed: u64) -> Option<(M3, Vec)> { + let n = x1.len().min(x2.len()); + if n < 8 { + return None; + } + let t2 = thr * thr; + let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1); + let mut best: Option<(usize, f64, M3)> = None; + let mut idx = vec![]; + let mut max_iter = iterations; + let mut it = 0; + let (mut s1, mut s2) = (vec![[0.0; 2]; 8], vec![[0.0; 2]; 8]); + while it < max_iter { + it += 1; + rng.sample(n, 8, &mut idx); + if idx.len() < 8 { + break; + } + for (k, &i) in idx.iter().enumerate() { + s1[k] = x1[i]; + s2[k] = x2[i]; + } + let Some(e) = essential_8pt(&s1, &s2) else { continue }; + let (mut c, mut err) = (0, 0.0); + for i in 0..n { + let d = sampson2(&e, x1[i], x2[i]); + if d < t2 { + c += 1; + err += d; + } + } + if best.as_ref().is_none_or(|b| c > b.0 || (c == b.0 && err < b.1)) { + best = Some((c, err, e)); + // Adaptive iteration count for 99 % confidence. + let w = c as f64 / n as f64; + let p_good = w.powi(8); + if p_good > 1e-9 { + let need = ((1.0 - 0.99f64).ln() / (1.0 - p_good).max(1e-12).ln()).ceil(); + if need.is_finite() { + max_iter = max_iter.min((need as usize).max(30)); + } + } + } + } + let (_, _, mut e) = best?; + // Refit on the inliers (twice: the consensus may grow). + let mut inl = vec![false; n]; + for _ in 0..2 { + for i in 0..n { + inl[i] = sampson2(&e, x1[i], x2[i]) < t2; + } + let (a, b): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..n).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip(); + match essential_8pt(&a, &b) { + Some(e2) => e = e2, + None => break, + } + } + for i in 0..n { + inl[i] = sampson2(&e, x1[i], x2[i]) < t2; + } + Some((e, inl)) +} + +/// The four `(R, t)` with `x2 ~ R x1 + t` (|t| = 1) of an essential matrix (H&Z Result 9.19). +pub fn decompose_essential(e: &M3) -> [(M3, V3); 4] { + let (mut u, _, mut v) = svd3(e); + if det3(&u) < 0.0 { + for row in u.iter_mut() { + row[2] = -row[2]; + } + } + if det3(&v) < 0.0 { + for row in v.iter_mut() { + row[2] = -row[2]; + } + } + let w = [[0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]]; + let r1 = mm(&u, &mm(&w, &mt(&v))); + let r2 = mm(&u, &mm(&mt(&w), &mt(&v))); + let t = col(&u, 2); + [(r1, t), (r1, scale(t, -1.0)), (r2, t), (r2, scale(t, -1.0))] +} + +/// A camera `(R, C)`. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Pose { + pub r: M3, + pub c: V3, +} + +impl Pose { + pub const IDENTITY: Pose = Pose { r: I3, c: [0.0; 3] }; + /// From `x_cam = R X + t`. + pub fn from_rt(r: M3, t: V3) -> Pose { + Pose { r, c: scale(mtv(&r, t), -1.0) } + } + pub fn t(&self) -> V3 { + scale(mv(&self.r, self.c), -1.0) + } + pub fn to_cam(&self, x: V3) -> V3 { + mv(&self.r, sub(x, self.c)) + } +} + +/// Linear (DLT) triangulation from normalised observations (H&Z §12.2), minimising the +/// algebraic error over all views. +pub fn triangulate(views: &[(Pose, [f64; 2])]) -> Option { + if views.len() < 2 { + return None; + } + let mut ata = [0.0f64; 16]; + for (p, x) in views { + let t = p.t(); + let rows = [[p.r[0][0], p.r[0][1], p.r[0][2], t[0]], [p.r[1][0], p.r[1][1], p.r[1][2], t[1]], [p.r[2][0], p.r[2][1], p.r[2][2], t[2]]]; + let a0: Vec = (0..4).map(|j| x[0] * rows[2][j] - rows[0][j]).collect(); + let a1: Vec = (0..4).map(|j| x[1] * rows[2][j] - rows[1][j]).collect(); + for a in [a0, a1] { + let nrm = a.iter().map(|v| v * v).sum::().sqrt().max(1e-300); + for i in 0..4 { + for j in 0..4 { + ata[i * 4 + j] += a[i] * a[j] / (nrm * nrm); + } + } + } + } + let (_, vecs) = sym_eigen(&ata, 4); + let h = [vecs[0], vecs[4], vecs[8], vecs[12]]; + if h[3].abs() < 1e-12 { + return None; + } + let x = [h[0] / h[3], h[1] / h[3], h[2] / h[3]]; + x.iter().all(|v| v.is_finite()).then_some(x) +} + +/// Angle (radians) between the rays from two camera centres to `x`. +pub fn ray_angle(a: V3, b: V3, x: V3) -> f64 { + let (u, v) = (normalize(sub(x, a)), normalize(sub(x, b))); + dot(u, v).clamp(-1.0, 1.0).acos() +} + +/// Squared reprojection error (normalised units) of `x` in `pose` against observation `o` +/// (`None` behind the camera). +pub fn reproj2(pose: &Pose, x: V3, o: [f64; 2]) -> Option { + let c = pose.to_cam(x); + if c[2] <= 1e-9 { + return None; + } + let (u, v) = (c[0] / c[2] - o[0], c[1] / c[2] - o[1]); + Some(u * u + v * v) +} + +/// Pick the essential decomposition with the most points in front of both cameras; returns +/// the second camera (the first is the identity) and that count. +pub fn select_pose(cands: &[(M3, V3)], x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Option<(Pose, usize)> { + let mut best: Option<(Pose, usize)> = None; + for (r, t) in cands { + let p2 = Pose::from_rt(*r, *t); + let mut c = 0; + for i in 0..x1.len() { + if let Some(x) = triangulate(&[(Pose::IDENTITY, x1[i]), (p2, x2[i])]) + && x[2] > 0.0 + && p2.to_cam(x)[2] > 0.0 + { + c += 1; + } + } + if best.as_ref().is_none_or(|b| c > b.1) { + best = Some((p2, c)); + } + } + best +} + +/// Decompose a calibrated planar homography `H ~ R + t nᵀ / d` (normalised coordinates, +/// `x2 ~ H x1`) into its motions `(R, t/d, n)` (Ma et al. §5.3.3). Up to four solutions; those +/// that put the plane behind the first camera are dropped. +pub fn decompose_homography(h: &M3, x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Vec<(M3, V3, V3)> { + let (_, s, _) = svd3(h); + if s[1] <= 1e-12 { + return vec![]; + } + let mut hn = mscale(h, 1.0 / s[1]); + // Sign: x2ᵀ H x1 > 0 for the correspondences (positive depth). + let pos = x1.iter().zip(x2).filter(|(p, q)| dot([q[0], q[1], 1.0], mv(&hn, [p[0], p[1], 1.0])) > 0.0).count(); + if pos * 2 < x1.len() { + hn = mscale(&hn, -1.0); + } + let hth = mm(&mt(&hn), &hn); + let a: Vec = hth.iter().flatten().copied().collect(); + let (vals, vecs) = sym_eigen(&a, 3); + // Descending: σ1² ≥ σ2² = 1 ≥ σ3². + let v1 = [vecs[2], vecs[5], vecs[8]]; + let v2 = [vecs[1], vecs[4], vecs[7]]; + let v3 = [vecs[0], vecs[3], vecs[6]]; + let (s1, s3) = (vals[2].max(1.0), vals[0].clamp(0.0, 1.0)); + let mut out = vec![]; + if s1 - s3 < 1e-10 { + // H is a rotation: no translation (pure rotation or plane at infinity). + return vec![(nearest_rotation(&hn), [0.0; 3], [0.0, 0.0, 1.0])]; + } + let k = (s1 - s3).sqrt(); + let (a1, a3) = ((1.0 - s3).sqrt(), (s1 - 1.0).sqrt()); + let u1 = scale(add(scale(v1, a1), scale(v3, a3)), 1.0 / k); + let u2 = scale(sub(scale(v1, a1), scale(v3, a3)), 1.0 / k); + for u in [u1, u2] { + let uu = from_cols(v2, u, cross(v2, u)); + let hv2 = mv(&hn, v2); + let hu = mv(&hn, u); + let ww = from_cols(hv2, hu, cross(hv2, hu)); + let r = mm(&ww, &mt(&uu)); + let n = cross(v2, u); + let t = mv(&madd(&hn, &r, -1.0), n); + for sgn in [1.0, -1.0] { + let (n2, t2) = (scale(n, sgn), scale(t, sgn)); + // The plane must be in front of the first camera: nᵀ x1 > 0 (with n pointing away). + let front = x1.iter().filter(|p| dot(n2, [p[0], p[1], 1.0]) > 0.0).count(); + if front * 2 >= x1.len() { + out.push((r, t2, n2)); + } + } + } + out +} + +/// Least-squares plane through points: (centroid, unit normal, RMS in-plane spread). +pub fn fit_plane(p: &[V3]) -> Option<(V3, V3, f64)> { + if p.len() < 3 { + return None; + } + let k = 1.0 / p.len() as f64; + let c = scale(p.iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [0.0; 9]; + for q in p { + let d = sub(*q, c); + for i in 0..3 { + for j in 0..3 { + cov[i * 3 + j] += d[i] * d[j] * k; + } + } + } + let (vals, vecs) = sym_eigen(&cov, 3); + if vals[1] <= 1e-18 { + return None; + } + let n = normalize([vecs[0], vecs[3], vecs[6]]); + Some((c, n, (vals[1] + vals[2]).max(0.0).sqrt())) +} + +/// The rotation `R` minimising `Σ |b − R a|²` over unit rays (orthogonal Procrustes / Kabsch). +pub fn procrustes(a: &[V3], b: &[V3]) -> Option { + if a.len() < 2 { + return None; + } + let mut m = [[0.0; 3]; 3]; + for (x, y) in a.iter().zip(b) { + m = madd(&m, &outer(*y, *x), 1.0); + } + let (u, _, v) = svd3(&m); + let mut r = mm(&u, &mt(&v)); + if det3(&r) < 0.0 { + let mut u2 = u; + for row in u2.iter_mut() { + row[2] = -row[2]; + } + r = mm(&u2, &mt(&v)); + } + Some(r) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scene() -> (Vec, Pose) { + let mut rng = Rng(12345); + let pts: Vec = (0..60) + .map(|_| { + let f = |r: &mut Rng| (r.next_u64() % 10_000) as f64 / 10_000.0 - 0.5; + [f(&mut rng) * 4.0, f(&mut rng) * 3.0, 5.0 + f(&mut rng) * 3.0] + }) + .collect(); + let p2 = Pose { r: rodrigues([0.02, -0.15, 0.03]), c: [0.6, 0.1, -0.2] }; + (pts, p2) + } + + fn proj(p: &Pose, x: V3) -> [f64; 2] { + let c = p.to_cam(x); + [c[0] / c[2], c[1] / c[2]] + } + + #[test] + fn essential_recovers_relative_pose() { + let (pts, p2) = scene(); + let x1: Vec<[f64; 2]> = pts.iter().map(|x| proj(&Pose::IDENTITY, *x)).collect(); + let mut x2: Vec<[f64; 2]> = pts.iter().map(|x| proj(&p2, *x)).collect(); + // Outliers. + x2[3] = [0.3, -0.2]; + x2[17] = [-0.1, 0.25]; + let (e, inl) = ransac_essential(&x1, &x2, 1e-3, 500, 7).unwrap(); + assert!(!inl[3] && !inl[17]); + assert!(inl.iter().filter(|b| **b).count() >= 57); + let (pose, n) = select_pose(&decompose_essential(&e), &x1, &x2).unwrap(); + assert!(n >= 57); + let r_err = norm(log_rot(&mm(&pose.r, &mt(&p2.r)))); + assert!(r_err < 1e-6, "{r_err}"); + let dir = normalize(p2.c); + assert!(norm(sub(normalize(pose.c), dir)) < 1e-5, "{:?} {:?}", pose.c, dir); + } + + #[test] + fn homography_decomposition_contains_the_motion() { + // Plane z = 4 (normal (0,0,1) in camera 1), seen after a motion. + let p2 = Pose { r: rodrigues([0.05, 0.2, -0.04]), c: [0.8, -0.3, 0.4] }; + let pts: Vec = (0..30).map(|i| [((i * 37) % 17) as f64 / 8.0 - 1.0, ((i * 11) % 13) as f64 / 6.0 - 1.0, 4.0]).collect(); + let x1: Vec<[f64; 2]> = pts.iter().map(|x| proj(&Pose::IDENTITY, *x)).collect(); + let x2: Vec<[f64; 2]> = pts.iter().map(|x| proj(&p2, *x)).collect(); + // H = R + t nᵀ / d with t = −R C, n = (0,0,1), d = 4. + let t = p2.t(); + let h = madd(&p2.r, &outer(t, [0.0, 0.0, 0.25]), 1.0); + let sols = decompose_homography(&mscale(&h, 3.7), &x1, &x2); + assert!(!sols.is_empty()); + let ok = sols + .iter() + .any(|(r, td, n)| norm(log_rot(&mm(r, &mt(&p2.r)))) < 1e-6 && norm(sub(*td, scale(t, 0.25))) < 1e-6 && norm(sub(*n, [0.0, 0.0, 1.0])) < 1e-6); + assert!(ok, "{sols:?}"); + } + + #[test] + fn triangulation_and_plane() { + let (pts, p2) = scene(); + for x in &pts[..5] { + let y = triangulate(&[(Pose::IDENTITY, proj(&Pose::IDENTITY, *x)), (p2, proj(&p2, *x))]).unwrap(); + assert!(norm(sub(*x, y)) < 1e-8); + } + let plane: Vec = (0..10).map(|i| [i as f64, (i * i % 7) as f64, 2.0 * i as f64 + 1.0]).collect(); + let (_, n, _) = fit_plane(&plane).unwrap(); + // All points satisfy z = 2x + 1: normal ∝ (2, 0, −1). + assert!(norm(cross(n, normalize([2.0, 0.0, -1.0]))) < 1e-9); + } +} diff --git a/crates/track/src/camtrack/linalg.rs b/crates/track/src/camtrack/linalg.rs new file mode 100644 index 00000000..b719cc8e --- /dev/null +++ b/crates/track/src/camtrack/linalg.rs @@ -0,0 +1,486 @@ +//! Small dense linear algebra for the camera solver, written from the textbook methods: +//! +//! - 3-vectors / 3 × 3 matrices and rotations (Rodrigues' formula and its inverse); +//! - the **one-sided Jacobi SVD** (Hestenes 1958; Golub & Van Loan, *Matrix Computations*, +//! §8.6.4), accurate for the small, badly scaled systems of the 8-point algorithm; +//! - the **cyclic Jacobi eigenvalue method** for symmetric matrices (Golub & Van Loan §8.5); +//! - **Cholesky** factorisation for the bundle adjustment's normal equations; +//! - the closed-form **similarity alignment** of two point sets (Umeyama, "Least-squares +//! estimation of transformation parameters between two point patterns", PAMI 1991). + +pub type V3 = [f64; 3]; +pub type M3 = [[f64; 3]; 3]; + +pub const I3: M3 = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]; + +pub fn add(a: V3, b: V3) -> V3 { + [a[0] + b[0], a[1] + b[1], a[2] + b[2]] +} +pub fn sub(a: V3, b: V3) -> V3 { + [a[0] - b[0], a[1] - b[1], a[2] - b[2]] +} +pub fn scale(a: V3, k: f64) -> V3 { + [a[0] * k, a[1] * k, a[2] * k] +} +pub fn dot(a: V3, b: V3) -> f64 { + a[0] * b[0] + a[1] * b[1] + a[2] * b[2] +} +pub fn cross(a: V3, b: V3) -> V3 { + [a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]] +} +pub fn norm(a: V3) -> f64 { + dot(a, a).sqrt() +} +pub fn normalize(a: V3) -> V3 { + let n = norm(a); + if n > 0.0 { scale(a, 1.0 / n) } else { a } +} +pub fn mv(m: &M3, v: V3) -> V3 { + [dot(m[0], v), dot(m[1], v), dot(m[2], v)] +} +/// `mᵀ v`. +pub fn mtv(m: &M3, v: V3) -> V3 { + [m[0][0] * v[0] + m[1][0] * v[1] + m[2][0] * v[2], m[0][1] * v[0] + m[1][1] * v[1] + m[2][1] * v[2], m[0][2] * v[0] + m[1][2] * v[1] + m[2][2] * v[2]] +} +pub fn mm(a: &M3, b: &M3) -> M3 { + let mut o = [[0.0; 3]; 3]; + for (i, row) in o.iter_mut().enumerate() { + for (j, v) in row.iter_mut().enumerate() { + *v = a[i][0] * b[0][j] + a[i][1] * b[1][j] + a[i][2] * b[2][j]; + } + } + o +} +pub fn mt(a: &M3) -> M3 { + [[a[0][0], a[1][0], a[2][0]], [a[0][1], a[1][1], a[2][1]], [a[0][2], a[1][2], a[2][2]]] +} +pub fn madd(a: &M3, b: &M3, k: f64) -> M3 { + let mut o = *a; + for i in 0..3 { + for j in 0..3 { + o[i][j] += k * b[i][j]; + } + } + o +} +pub fn mscale(a: &M3, k: f64) -> M3 { + madd(&[[0.0; 3]; 3], a, k) +} +pub fn det3(m: &M3) -> f64 { + m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0]) + m[0][2] * (m[1][0] * m[2][1] - m[1][1] * m[2][0]) +} +pub fn skew(v: V3) -> M3 { + [[0.0, -v[2], v[1]], [v[2], 0.0, -v[0]], [-v[1], v[0], 0.0]] +} +pub fn outer(a: V3, b: V3) -> M3 { + [[a[0] * b[0], a[0] * b[1], a[0] * b[2]], [a[1] * b[0], a[1] * b[1], a[1] * b[2]], [a[2] * b[0], a[2] * b[1], a[2] * b[2]]] +} +pub fn col(m: &M3, j: usize) -> V3 { + [m[0][j], m[1][j], m[2][j]] +} +pub fn from_cols(a: V3, b: V3, c: V3) -> M3 { + [[a[0], b[0], c[0]], [a[1], b[1], c[1]], [a[2], b[2], c[2]]] +} +pub fn inv3(m: &M3) -> Option { + let d = det3(m); + if d.abs() < 1e-300 { + return None; + } + let c = |r0: usize, c0: usize, r1: usize, c1: usize| m[r0][c0] * m[r1][c1] - m[r0][c1] * m[r1][c0]; + let adj = [[c(1, 1, 2, 2), -c(0, 1, 2, 2), c(0, 1, 1, 2)], [-c(1, 0, 2, 2), c(0, 0, 2, 2), -c(0, 0, 1, 2)], [c(1, 0, 2, 1), -c(0, 0, 2, 1), c(0, 0, 1, 1)]]; + Some(mscale(&adj, 1.0 / d)) +} + +/// Rotation matrix of an axis-angle vector (Rodrigues' formula). +pub fn rodrigues(w: V3) -> M3 { + let th = norm(w); + let k = skew(w); + if th < 1e-12 { + return madd(&I3, &k, 1.0); + } + let a = th.sin() / th; + let b = (1.0 - th.cos()) / (th * th); + madd(&madd(&I3, &k, a), &mm(&k, &k), b) +} + +/// Axis-angle vector of a rotation matrix (inverse of [`rodrigues`]). +pub fn log_rot(r: &M3) -> V3 { + let c = ((r[0][0] + r[1][1] + r[2][2] - 1.0) * 0.5).clamp(-1.0, 1.0); + let th = c.acos(); + let v = [r[2][1] - r[1][2], r[0][2] - r[2][0], r[1][0] - r[0][1]]; + if th < 1e-9 { + return scale(v, 0.5); + } + if std::f64::consts::PI - th < 1e-6 { + // Near π: the axis is the dominant column of R + I. + let b = madd(r, &I3, 1.0); + let j = (0..3).max_by(|&a, &bb| b[a][a].total_cmp(&b[bb][bb])).unwrap_or(0); + let axis = normalize(col(&b, j)); + return scale(axis, th); + } + scale(v, th / (2.0 * th.sin())) +} + +/// The rotation closest to `m` (Frobenius norm), via the SVD. +pub fn nearest_rotation(m: &M3) -> M3 { + let (u, _, v) = svd3(m); + let mut r = mm(&u, &mt(&v)); + if det3(&r) < 0.0 { + let mut u2 = u; + for row in u2.iter_mut() { + row[2] = -row[2]; + } + r = mm(&u2, &mt(&v)); + } + r +} + +/// Spherical interpolation between two rotations. +pub fn slerp(a: &M3, b: &M3, t: f64) -> M3 { + let d = log_rot(&mm(b, &mt(a))); + mm(&rodrigues(scale(d, t)), a) +} + +/// Thin SVD `A = U diag(s) Vᵀ` of an `m × n` row-major matrix (one-sided Jacobi). `U` is +/// `max(m, n) × n` (rows beyond `m` belong to zero padding), `s` is sorted descending and `V` +/// is `n × n` (columns are the right singular vectors). +pub fn svd(a: &[f64], m: usize, n: usize) -> (Vec, Vec, Vec) { + let rows = m.max(n); + let mut u = vec![0.0; rows * n]; + u[..m * n].copy_from_slice(&a[..m * n]); + let mut v = vec![0.0; n * n]; + for i in 0..n { + v[i * n + i] = 1.0; + } + for _sweep in 0..60 { + let mut off = 0.0f64; + for p in 0..n { + for q in p + 1..n { + let (mut alpha, mut beta, mut gamma) = (0.0, 0.0, 0.0); + for r in 0..rows { + let (x, y) = (u[r * n + p], u[r * n + q]); + alpha += x * x; + beta += y * y; + gamma += x * y; + } + if gamma == 0.0 { + continue; + } + let c0 = gamma.abs() / (alpha * beta).sqrt().max(1e-300); + off = off.max(c0); + if c0 < 1e-15 { + continue; + } + let zeta = (beta - alpha) / (2.0 * gamma); + let t = zeta.signum() / (zeta.abs() + (1.0 + zeta * zeta).sqrt()); + let t = if zeta == 0.0 { 1.0 } else { t }; + let c = 1.0 / (1.0 + t * t).sqrt(); + let s = c * t; + for r in 0..rows { + let (x, y) = (u[r * n + p], u[r * n + q]); + u[r * n + p] = c * x - s * y; + u[r * n + q] = s * x + c * y; + } + for r in 0..n { + let (x, y) = (v[r * n + p], v[r * n + q]); + v[r * n + p] = c * x - s * y; + v[r * n + q] = s * x + c * y; + } + } + } + if off < 1e-15 { + break; + } + } + let mut s: Vec = (0..n).map(|j| (0..rows).map(|r| u[r * n + j] * u[r * n + j]).sum::().sqrt()).collect(); + for (j, sj) in s.iter().enumerate() { + if *sj > 1e-300 { + for r in 0..rows { + u[r * n + j] /= sj; + } + } + } + // Sort descending. + let mut order: Vec = (0..n).collect(); + order.sort_by(|&a, &b| s[b].total_cmp(&s[a])); + let (mut u2, mut v2) = (vec![0.0; rows * n], vec![0.0; n * n]); + for (k, &j) in order.iter().enumerate() { + for r in 0..rows { + u2[r * n + k] = u[r * n + j]; + } + for r in 0..n { + v2[r * n + k] = v[r * n + j]; + } + } + s = order.iter().map(|&j| s[j]).collect(); + (u2, s, v2) +} + +/// SVD of a 3 × 3 matrix: (U, singular values descending, V) with `m = U diag(s) Vᵀ`. +pub fn svd3(m: &M3) -> (M3, V3, M3) { + let a: Vec = m.iter().flatten().copied().collect(); + let (u, s, v) = svd(&a, 3, 3); + let g = |x: &[f64]| [[x[0], x[1], x[2]], [x[3], x[4], x[5]], [x[6], x[7], x[8]]]; + let mut um = g(&u); + // Columns of U for zero singular values are undefined: complete them to an orthonormal basis. + if s[2] < 1e-300 { + let c = cross(col(&um, 0), col(&um, 1)); + for (r, row) in um.iter_mut().enumerate() { + row[2] = c[r]; + } + } + (um, [s[0], s[1], s[2]], g(&v)) +} + +/// The unit vector `x` minimising `|A x|` (`m × n` row-major): the right singular vector of +/// the smallest singular value. +pub fn null_vector(a: &[f64], m: usize, n: usize) -> Vec { + let (_, _, v) = svd(a, m, n); + (0..n).map(|r| v[r * n + n - 1]).collect() +} + +/// Eigen-decomposition of a symmetric `n × n` matrix (cyclic Jacobi): eigenvalues ascending +/// and the matching eigenvectors as the columns of the returned row-major matrix. +pub fn sym_eigen(a: &[f64], n: usize) -> (Vec, Vec) { + let mut a = a.to_vec(); + let mut v = vec![0.0; n * n]; + for i in 0..n { + v[i * n + i] = 1.0; + } + for _ in 0..100 { + let mut off = 0.0; + for p in 0..n { + for q in p + 1..n { + off += a[p * n + q] * a[p * n + q]; + } + } + if off < 1e-30 { + break; + } + for p in 0..n { + for q in p + 1..n { + let apq = a[p * n + q]; + if apq.abs() < 1e-300 { + continue; + } + let theta = (a[q * n + q] - a[p * n + p]) / (2.0 * apq); + let t = theta.signum() / (theta.abs() + (theta * theta + 1.0).sqrt()); + let t = if theta == 0.0 { 1.0 } else { t }; + let c = 1.0 / (t * t + 1.0).sqrt(); + let s = t * c; + for k in 0..n { + let (akp, akq) = (a[k * n + p], a[k * n + q]); + a[k * n + p] = c * akp - s * akq; + a[k * n + q] = s * akp + c * akq; + } + for k in 0..n { + let (apk, aqk) = (a[p * n + k], a[q * n + k]); + a[p * n + k] = c * apk - s * aqk; + a[q * n + k] = s * apk + c * aqk; + } + for k in 0..n { + let (vkp, vkq) = (v[k * n + p], v[k * n + q]); + v[k * n + p] = c * vkp - s * vkq; + v[k * n + q] = s * vkp + c * vkq; + } + } + } + } + let mut order: Vec = (0..n).collect(); + order.sort_by(|&x, &y| a[x * n + x].total_cmp(&a[y * n + y])); + let vals = order.iter().map(|&j| a[j * n + j]).collect(); + let mut vecs = vec![0.0; n * n]; + for (k, &j) in order.iter().enumerate() { + for r in 0..n { + vecs[r * n + k] = v[r * n + j]; + } + } + (vals, vecs) +} + +/// Dot product, four lanes (lets the compiler vectorise). +fn dot_n(a: &[f64], b: &[f64]) -> f64 { + let mut acc = [0.0f64; 4]; + let (ca, cb) = (a.chunks_exact(4), b.chunks_exact(4)); + let (ra, rb) = (ca.remainder(), cb.remainder()); + for (x, y) in ca.zip(cb) { + for k in 0..4 { + acc[k] += x[k] * y[k]; + } + } + let mut s = acc[0] + acc[1] + acc[2] + acc[3]; + for (x, y) in ra.iter().zip(rb) { + s += x * y; + } + s +} + +/// Solve `A x = b` for a symmetric positive definite `A` (`n × n`, row-major) in place by +/// Cholesky factorisation. Returns false when `A` is not positive definite. +pub fn cholesky_solve(a: &mut [f64], n: usize, b: &mut [f64]) -> bool { + for j in 0..n { + let mut d = a[j * n + j]; + for k in 0..j { + d -= a[j * n + k] * a[j * n + k]; + } + if d <= 0.0 || !d.is_finite() { + return false; + } + let d = d.sqrt(); + a[j * n + j] = d; + let (head, tail) = a.split_at_mut((j + 1) * n); + let rowj = &head[j * n..j * n + j]; + // Rows below j (sequential: per-column parallel dispatch costs more than it saves). + for row in tail.chunks_mut(n) { + row[j] = (row[j] - dot_n(&row[..j], rowj)) / d; + } + } + // Forward: L y = b. + for i in 0..n { + let mut s = b[i]; + for k in 0..i { + s -= a[i * n + k] * b[k]; + } + b[i] = s / a[i * n + i]; + } + // Back: Lᵀ x = y. + for i in (0..n).rev() { + let mut s = b[i]; + for k in i + 1..n { + s -= a[k * n + i] * b[k]; + } + b[i] = s / a[i * n + i]; + } + b.iter().all(|v| v.is_finite()) +} + +/// A similarity transform `x ↦ s R x + t`. +#[derive(Clone, Copy, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct Similarity { + pub s: f64, + pub r: M3, + pub t: V3, +} + +impl Default for Similarity { + fn default() -> Self { + Similarity { s: 1.0, r: I3, t: [0.0; 3] } + } +} + +impl Similarity { + pub fn apply(&self, x: V3) -> V3 { + add(scale(mv(&self.r, x), self.s), self.t) + } +} + +/// The similarity minimising `Σ |dst − (s R src + t)|²` (Umeyama 1991). +pub fn umeyama(src: &[V3], dst: &[V3]) -> Option { + let n = src.len().min(dst.len()); + if n < 3 { + return None; + } + let k = 1.0 / n as f64; + let ms = scale(src[..n].iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let md = scale(dst[..n].iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [[0.0; 3]; 3]; + let mut var = 0.0; + for i in 0..n { + let (a, b) = (sub(src[i], ms), sub(dst[i], md)); + cov = madd(&cov, &outer(b, a), k); + var += dot(a, a) * k; + } + if var < 1e-300 { + return None; + } + let (u, d, v) = svd3(&cov); + let mut sgn = [1.0, 1.0, 1.0]; + if det3(&u) * det3(&v) < 0.0 { + sgn[2] = -1.0; + } + let ud = from_cols(scale(col(&u, 0), sgn[0]), scale(col(&u, 1), sgn[1]), scale(col(&u, 2), sgn[2])); + let r = mm(&ud, &mt(&v)); + let s = (d[0] * sgn[0] + d[1] * sgn[1] + d[2] * sgn[2]) / var; + let t = sub(md, scale(mv(&r, ms), s)); + Some(Similarity { s, r, t }) +} + +/// Euler angles (degrees) `[x, y, z]` with `m = Rz(z) Ry(y) Rx(x)` (After Effects' Orientation +/// order: X, then Y, then Z applied to the vector). +pub fn euler_xyz(m: &M3) -> V3 { + let sy = (-m[2][0]).clamp(-1.0, 1.0); + let y = sy.asin(); + let (x, z) = if sy.abs() < 0.999_999 { (m[2][1].atan2(m[2][2]), m[1][0].atan2(m[0][0])) } else { ((-m[1][2]).atan2(m[1][1]), 0.0) }; + [x.to_degrees(), y.to_degrees(), z.to_degrees()] +} + +/// `Rz(z) Ry(y) Rx(x)` from degrees. +pub fn from_euler_xyz(e: V3) -> M3 { + let (sx, cx) = e[0].to_radians().sin_cos(); + let (sy, cy) = e[1].to_radians().sin_cos(); + let (sz, cz) = e[2].to_radians().sin_cos(); + let rx = [[1.0, 0.0, 0.0], [0.0, cx, -sx], [0.0, sx, cx]]; + let ry = [[cy, 0.0, sy], [0.0, 1.0, 0.0], [-sy, 0.0, cy]]; + let rz = [[cz, -sz, 0.0], [sz, cz, 0.0], [0.0, 0.0, 1.0]]; + mm(&rz, &mm(&ry, &rx)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn svd_reconstructs() { + let a = [3.0, 1.0, 2.0, -1.0, 4.0, 0.5, 2.0, 2.0, 7.0, 0.1, -3.0, 1.0]; + let (u, s, v) = svd(&a, 4, 3); + for r in 0..4 { + for c in 0..3 { + let x: f64 = (0..3).map(|k| u[r * 3 + k] * s[k] * v[c * 3 + k]).sum(); + assert!((x - a[r * 3 + c]).abs() < 1e-10); + } + } + assert!(s[0] >= s[1] && s[1] >= s[2]); + } + + #[test] + fn rotations_round_trip() { + let w = [0.3, -1.2, 0.7]; + let r = rodrigues(w); + let w2 = log_rot(&r); + assert!(norm(sub(w, w2)) < 1e-12); + let e = euler_xyz(&r); + let r2 = from_euler_xyz(e); + for i in 0..3 { + for j in 0..3 { + assert!((r[i][j] - r2[i][j]).abs() < 1e-12); + } + } + assert!((det3(&r) - 1.0).abs() < 1e-12); + } + + #[test] + fn eigen_and_cholesky() { + let a = [4.0, 1.0, 0.5, 1.0, 3.0, 0.2, 0.5, 0.2, 2.0]; + let (vals, vecs) = sym_eigen(&a, 3); + for k in 0..3 { + let v = [vecs[k], vecs[3 + k], vecs[6 + k]]; + let av = [dot([a[0], a[1], a[2]], v), dot([a[3], a[4], a[5]], v), dot([a[6], a[7], a[8]], v)]; + assert!(norm(sub(av, scale(v, vals[k]))) < 1e-10); + } + let mut m = a; + let mut b = [1.0, 2.0, 3.0]; + assert!(cholesky_solve(&mut m, 3, &mut b)); + let r = [dot([a[0], a[1], a[2]], b), dot([a[3], a[4], a[5]], b), dot([a[6], a[7], a[8]], b)]; + assert!(norm(sub(r, [1.0, 2.0, 3.0])) < 1e-12); + } + + #[test] + fn umeyama_recovers_a_similarity() { + let t = Similarity { s: 2.5, r: rodrigues([0.1, 0.4, -0.3]), t: [1.0, -2.0, 5.0] }; + let src = [[0.0, 0.0, 0.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [1.0, 2.0, 3.0]]; + let dst: Vec = src.iter().map(|p| t.apply(*p)).collect(); + let e = umeyama(&src, &dst).unwrap(); + assert!((e.s - 2.5).abs() < 1e-10); + assert!(norm(sub(e.t, t.t)) < 1e-9); + } +} diff --git a/crates/track/src/camtrack/mod.rs b/crates/track/src/camtrack/mod.rs new file mode 100644 index 00000000..76c0637f --- /dev/null +++ b/crates/track/src/camtrack/mod.rs @@ -0,0 +1,380 @@ +//! The 3D Camera Tracker's analysis: structure from motion on a single clip, clean-room from the +//! textbook pipeline (Hartley & Zisserman, *Multiple View Geometry in Computer Vision*, 2nd ed., +//! 2004; Triggs et al., "Bundle Adjustment — A Modern Synthesis", 2000): +//! +//! 1. **Feature tracks** ([`features::TrackAnalyzer`]): Shi–Tomasi corners followed through the +//! clip with pyramidal Lucas–Kanade and a forward–backward check ([`crate::klt`]); lost +//! features end their track and new ones are detected where the frame has none, so tracks +//! are as long as possible. +//! 2. **Solve** ([`solve`]): keyframes are selected by parallax; a two-view reconstruction +//! initialises the scene from either the **essential matrix** (normalised 8-point algorithm +//! inside RANSAC, H&Z §11.2 and §9.6) or, for mostly flat scenes, a **homography +//! decomposition** (Ma, Soatto, Košecká & Sastry, *An Invitation to 3-D Vision*, 2004, §5.3); +//! the other keyframes are added incrementally (**resection** by robust pose refinement, +//! **triangulation** of new tracks by the linear method of H&Z §12.2); a sparse +//! **Levenberg–Marquardt bundle adjustment** with the Schur complement over the points +//! ([`bundle`]) refines everything; the frames between keyframes are resected last. Inlier +//! thresholds start from the analysis resolution and are re-estimated from the residuals +//! (median absolute residual) once the keyframes are solved, so features on moving objects, +//! occlusion edges and screen-locked overlays drop out. +//! 3. **Focal length**: *Specify Angle of View* fixes it; *Fixed Angle of View* searches it +//! (the reprojection error of the keyframe reconstruction over a log-spaced range of focal +//! lengths, refined by golden-section search) and then adjusts it in the bundle adjustment +//! as one shared parameter; *Variable Zoom* starts from the same estimate and adjusts a focal +//! length per frame, with a weak smoothness prior between neighbouring frames. +//! 4. **Tripod pan** shots (no camera translation) are solved as a pure rotation: frame-to-frame +//! rotations from the tracked rays (orthogonal Procrustes) refined by the same bundle +//! adjustment with the camera centres fixed. *Auto Detect* solves both ways and keeps the +//! tripod solution when it explains the tracks as well as the general one (a quick rotation fit +//! at the general solution's focal length skips the tripod search when it clearly fails). +//! +//! The camera model has square pixels, the principal point at the layer centre and no lens +//! distortion. Camera space is x right, y down, z forward (After Effects' convention); a camera +//! is `(R, C, f)` with `X_cam = R (X − C)` and pixel `u = f x / z + w / 2`. +//! +//! The solve is stored in a canonical frame: the first solved frame's camera is at the origin +//! looking along +z, scaled so the median depth of the points it sees is 1. [`CameraSolve::world`] +//! maps it into composition space for a camera zoom, optionally re-based on a ground plane. + +pub mod bundle; +pub mod features; +pub mod geometry; +pub mod linalg; +mod solver; +#[cfg(test)] +mod tests; + +use serde::{Deserialize, Serialize}; + +pub use features::{AnalyzeOpts, TrackAnalyzer}; +use linalg::{M3, Similarity, V3}; +pub use solver::{SolveError, solve}; + +/// One feature track: positions (layer pixels) on consecutive frames from `start`. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Track2D { + pub id: u32, + pub start: u32, + pub pts: Vec<[f32; 2]>, +} + +impl Track2D { + pub fn end(&self) -> u32 { + self.start + self.pts.len() as u32 + } + /// Position on frame `k` when the track covers it. + pub fn at(&self, k: u32) -> Option<[f64; 2]> { + let i = k.checked_sub(self.start)? as usize; + self.pts.get(i).map(|p| [p[0] as f64, p[1] as f64]) + } +} + +/// The tracked features of a clip (step 1 of the analysis). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct CameraTracks { + pub version: u32, + /// Layer size (pixels). + pub size: [f64; 2], + /// Layer time (seconds) of frame 0, and the frame duration. + pub start: f64, + pub frame_duration: f64, + pub frames: u32, + /// Analysis downsampling factor (image pixels per analysis pixel): the tracking noise scale. + pub factor: f64, + pub detailed: bool, + pub tracks: Vec, +} + +impl CameraTracks { + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } + pub fn from_json(s: &str) -> Option { + if s.trim().is_empty() { + return None; + } + serde_json::from_str(s).ok() + } + pub fn is_empty(&self) -> bool { + self.frames == 0 + } + pub fn track(&self, id: u32) -> Option<&Track2D> { + self.tracks.binary_search_by_key(&id, |t| t.id).ok().map(|i| &self.tracks[i]) + } +} + +/// 3D Camera Tracker ▸ Shot Type. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ShotType { + #[default] + FixedAngle, + VariableZoom, + SpecifyAngle, +} + +impl ShotType { + pub const ALL: [ShotType; 3] = [ShotType::FixedAngle, ShotType::VariableZoom, ShotType::SpecifyAngle]; + pub fn label(self) -> &'static str { + match self { + ShotType::FixedAngle => "Fixed Angle of View", + ShotType::VariableZoom => "Variable Zoom", + ShotType::SpecifyAngle => "Specify Angle of View", + } + } + pub fn from_name(s: &str) -> Option { + let k = s.to_ascii_lowercase().replace([' ', '-', '_'], ""); + match k.as_str() { + "fixed" | "fixedangle" | "fixedangleofview" => Some(ShotType::FixedAngle), + "variable" | "variablezoom" | "zoom" => Some(ShotType::VariableZoom), + "specify" | "specifyangle" | "specifyangleofview" => Some(ShotType::SpecifyAngle), + _ => None, + } + } +} + +/// 3D Camera Tracker ▸ Advanced ▸ Solve Method (and Method Used). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum SolveMethod { + #[default] + Auto, + Typical, + MostlyFlat, + TripodPan, +} + +impl SolveMethod { + pub const ALL: [SolveMethod; 4] = [SolveMethod::Auto, SolveMethod::Typical, SolveMethod::MostlyFlat, SolveMethod::TripodPan]; + pub fn label(self) -> &'static str { + match self { + SolveMethod::Auto => "Auto Detect", + SolveMethod::Typical => "Typical", + SolveMethod::MostlyFlat => "Mostly Flat Scene", + SolveMethod::TripodPan => "Tripod Pan", + } + } + pub fn from_name(s: &str) -> Option { + let k = s.to_ascii_lowercase().replace([' ', '-', '_'], ""); + match k.as_str() { + "auto" | "autodetect" => Some(SolveMethod::Auto), + "typical" => Some(SolveMethod::Typical), + "flat" | "mostlyflat" | "mostlyflatscene" => Some(SolveMethod::MostlyFlat), + "tripod" | "tripodpan" => Some(SolveMethod::TripodPan), + _ => None, + } + } +} + +/// What a solve is made with. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolveSettings { + pub shot: ShotType, + /// Horizontal angle of view (degrees) for [`ShotType::SpecifyAngle`]. + pub hfov: f64, + pub method: SolveMethod, + /// Track ids the user deleted. + pub deleted: Vec, +} + +/// The solved camera of one frame. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolvedFrame { + /// World → camera rotation (axis-angle). + pub rot: V3, + /// Camera centre. + pub center: V3, + /// Focal length (layer pixels). + pub focal: f64, + pub solved: bool, +} + +impl SolvedFrame { + pub fn r(&self) -> M3 { + linalg::rodrigues(self.rot) + } + /// Camera-space position of a world point. + pub fn to_cam(&self, x: V3) -> V3 { + linalg::mv(&self.r(), linalg::sub(x, self.center)) + } + /// Project a world point to layer pixels (`None` behind the camera). + pub fn project(&self, size: [f64; 2], x: V3) -> Option<[f64; 2]> { + let c = self.to_cam(x); + (c[2] > 1e-9).then(|| [self.focal * c[0] / c[2] + size[0] * 0.5, self.focal * c[1] / c[2] + size[1] * 0.5]) + } +} + +/// A solved 3D track point. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolvedPoint { + /// The feature track it was made from. + pub id: u32, + pub pos: V3, + /// Average reprojection error (pixels). + pub error: f32, + /// Frames `[first, last]` the point was tracked on. + pub first: u32, + pub last: u32, +} + +/// A ground plane and origin (canonical solve coordinates). +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct Ground { + pub origin: V3, + /// Unit normal on the cameras' side ("up"). + pub normal: V3, + /// Unit in-plane direction that becomes world +x. + pub x_axis: V3, +} + +/// A plane through solved points: After Effects' target. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct Target { + pub center: V3, + /// Unit normal facing the camera. + pub normal: V3, + /// Diameter in canonical units. + pub size: f64, +} + +/// The camera solve (step 2 of the analysis). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct CameraSolve { + pub version: u32, + pub size: [f64; 2], + pub start: f64, + pub frame_duration: f64, + pub shot: ShotType, + /// The model that was used (Auto Detect resolves to one of the others). + pub method_used: SolveMethod, + /// Average reprojection error (pixels) over all solved observations. + pub average_error: f64, + pub frames: Vec, + pub points: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub ground: Option, +} + +impl CameraSolve { + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } + pub fn from_json(s: &str) -> Option { + if s.trim().is_empty() { + return None; + } + serde_json::from_str(s).ok() + } + pub fn is_empty(&self) -> bool { + self.frames.is_empty() + } + /// Frame index at a layer time (held at the ends). + pub fn frame_at(&self, layer_time: f64) -> Option { + if self.frames.is_empty() || self.frame_duration <= 0.0 { + return None; + } + let i = ((layer_time - self.start) / self.frame_duration).round(); + Some(i.clamp(0.0, (self.frames.len() - 1) as f64) as usize) + } + pub fn point(&self, id: u32) -> Option<&SolvedPoint> { + self.points.iter().find(|p| p.id == id) + } + /// Points tracked on frame `k`. + pub fn visible(&self, k: usize) -> impl Iterator { + let k = k as u32; + self.points.iter().filter(move |p| p.first <= k && k <= p.last) + } + /// Median focal length of the solved frames (layer pixels). + pub fn focal(&self) -> f64 { + let mut f: Vec = self.frames.iter().filter(|c| c.solved).map(|c| c.focal).collect(); + if f.is_empty() { + return 0.0; + } + f.sort_by(f64::total_cmp); + f[f.len() / 2] + } + /// Horizontal angle of view (degrees) of focal length `f`. + pub fn hfov(&self, f: f64) -> f64 { + 2.0 * (self.size[0] * 0.5 / f.max(1e-9)).atan().to_degrees() + } + + /// The plane through points `ids` seen from frame `k` (needs three points; with more the + /// plane is a least-squares fit). + pub fn target(&self, ids: &[u32], k: usize) -> Option { + let pts: Vec = ids.iter().filter_map(|i| self.point(*i)).map(|p| p.pos).collect(); + let cam = self.frames.get(k).filter(|c| c.solved).or_else(|| self.frames.iter().find(|c| c.solved))?; + geometry::fit_plane(&pts).map(|(c, mut n, spread)| { + if linalg::dot(n, linalg::sub(cam.center, c)) < 0.0 { + n = linalg::scale(n, -1.0); + } + // After Effects sizes the target from the camera distance when the points are close. + let depth = linalg::norm(linalg::sub(cam.center, c)); + Target { center: c, normal: n, size: (spread * 2.0).max(depth * 0.12) } + }) + } + + /// A ground plane through `ids` (origin at their centroid, normal towards frame `k`'s + /// camera, +x along that camera's right vector projected on the plane). + pub fn ground_from(&self, ids: &[u32], k: usize) -> Option { + let t = self.target(ids, k)?; + let cam = self.frames.get(k).filter(|c| c.solved).or_else(|| self.frames.iter().find(|c| c.solved))?; + let right = linalg::mtv(&cam.r(), [1.0, 0.0, 0.0]); + let mut x = linalg::sub(right, linalg::scale(t.normal, linalg::dot(right, t.normal))); + if linalg::norm(x) < 1e-6 { + let fwd = linalg::mtv(&cam.r(), [0.0, 0.0, 1.0]); + x = linalg::cross(fwd, t.normal); + } + Some(Ground { origin: t.center, normal: t.normal, x_axis: linalg::normalize(x) }) + } + + /// Canonical solve coordinates → composition world, for a camera whose zoom on the first + /// solved frame is `zoom0` (comp pixels) in a `comp` sized composition. Without a ground + /// plane, the first solved frame's camera becomes After Effects' default camera (at + /// `(w/2, h/2, −zoom)` looking down +z, so points at the median depth land near z = 0). With + /// one, the ground plane becomes the X-Z plane through the origin, normal along −y (up). + pub fn world(&self, comp: [f64; 2], zoom0: f64) -> Similarity { + match &self.ground { + Some(g) => { + // Rows: canonical directions that map to world x, y, z. + let y = linalg::scale(g.normal, -1.0); + let x = g.x_axis; + let z = linalg::cross(x, y); + let r = [x, y, z]; + let t = linalg::scale(linalg::mv(&r, g.origin), -zoom0); + Similarity { s: zoom0, r, t } + } + None => Similarity { s: zoom0, r: linalg::I3, t: [comp[0] * 0.5, comp[1] * 0.5, -zoom0] }, + } + } + + /// The first solved frame. + pub fn first_solved(&self) -> Option { + self.frames.iter().position(|c| c.solved) + } +} + +/// Deterministic colour of a track point (After Effects draws each target in its own colour). +pub fn point_color(id: u32) -> [f32; 3] { + let mut h = id.wrapping_mul(0x9E37_79B9) ^ 0x5bd1_e995; + h ^= h >> 15; + let hue = (h % 360) as f32 / 60.0; + let x = 1.0 - (hue % 2.0 - 1.0).abs(); + let (r, g, b) = match hue as u32 { + 0 => (1.0, x, 0.0), + 1 => (x, 1.0, 0.0), + 2 => (0.0, 1.0, x), + 3 => (0.0, x, 1.0), + 4 => (x, 0.0, 1.0), + _ => (1.0, 0.0, x), + }; + // Bright, slightly desaturated. + [0.25 + 0.75 * r, 0.25 + 0.75 * g, 0.25 + 0.75 * b] +} diff --git a/crates/track/src/camtrack/solver.rs b/crates/track/src/camtrack/solver.rs new file mode 100644 index 00000000..85edd2bd --- /dev/null +++ b/crates/track/src/camtrack/solver.rs @@ -0,0 +1,953 @@ +//! The camera solve pipeline (see the module docs of [`super`]). + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::atomic::{AtomicBool, Ordering}; + +use rayon::prelude::*; + +use super::bundle::{self, BaCam, BaObs, BaOpts, FocalMode}; +use super::geometry::{self, Pose}; +use super::linalg::*; +use super::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, SolvedFrame, SolvedPoint}; +use crate::fit::{Model, ransac}; + +fn debug() -> bool { + std::env::var_os("EC_CAMTRACK_DEBUG").is_some() +} + +/// Why a solve failed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SolveError(pub String); + +impl std::fmt::Display for SolveError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +fn err(m: &str) -> Result { + Err(SolveError(m.to_string())) +} + +/// Tracks in solver form: pixels relative to the principal point. +struct Data { + n: usize, + w: f64, + /// (track id, first frame, positions). + tracks: Vec<(u32, usize, Vec<[f64; 2]>)>, + /// Per frame: (track index, position). + by_frame: Vec>, + /// Tracking noise scale (pixels): starts from the analysis resolution and is re-estimated + /// from the residuals once the keyframes are solved. + noise_bits: std::sync::atomic::AtomicU64, +} + +impl Data { + fn new(t: &CameraTracks, deleted: &[u32]) -> Data { + let n = t.frames as usize; + let c = [t.size[0] * 0.5, t.size[1] * 0.5]; + let del: HashSet = deleted.iter().copied().collect(); + let mut tracks = vec![]; + let mut by_frame = vec![vec![]; n]; + for tr in t.tracks.iter().filter(|tr| !del.contains(&tr.id) && tr.pts.len() >= 3) { + let ti = tracks.len() as u32; + let pts: Vec<[f64; 2]> = tr.pts.iter().map(|p| [p[0] as f64 - c[0], p[1] as f64 - c[1]]).collect(); + for (k, p) in pts.iter().enumerate() { + if let Some(v) = by_frame.get_mut(tr.start as usize + k) { + v.push((ti, *p)); + } + } + tracks.push((tr.id, tr.start as usize, pts)); + } + let noise = (0.5 * t.factor).max(0.5); + Data { n, w: t.size[0], tracks, by_frame, noise_bits: std::sync::atomic::AtomicU64::new(noise.to_bits()) } + } + fn at(&self, ti: u32, k: usize) -> Option<[f64; 2]> { + let (_, s, p) = &self.tracks[ti as usize]; + k.checked_sub(*s).and_then(|i| p.get(i).copied()) + } + fn noise(&self) -> f64 { + f64::from_bits(self.noise_bits.load(Ordering::Relaxed)) + } + fn set_noise(&self, v: f64) { + self.noise_bits.store(v.to_bits(), Ordering::Relaxed); + } + fn inlier_thr(&self) -> f64 { + 4.0 * self.noise() + } + fn huber(&self) -> f64 { + self.noise() + } +} + +/// Parallax-based keyframes: a new keyframe when the median feature motion since the last +/// exceeds 1.5 % of the width, when fewer than 70 % of its features survive, after 12 frames, or +/// before the shared features drop below 24. The first and last frames with features are +/// keyframes. +fn keyframes(d: &Data) -> Vec { + let Some(first) = (0..d.n).find(|k| d.by_frame[*k].len() >= 8) else { return vec![] }; + let mut kf = vec![first]; + let mut last = first; + let mut j = first + 1; + while j < d.n { + let base = &d.by_frame[last]; + let mut disp = vec![]; + for (ti, p) in base { + if let Some(q) = d.at(*ti, j) { + disp.push((q[0] - p[0]).hypot(q[1] - p[1])); + } + } + if disp.len() < 24 && j > last + 1 { + // Too few shared features: the previous frame becomes a keyframe. + last = j - 1; + kf.push(last); + continue; + } + disp.sort_by(f64::total_cmp); + let med = disp.get(disp.len() / 2).copied().unwrap_or(0.0); + let ratio = disp.len() as f64 / base.len().max(1) as f64; + if med > 0.015 * d.w || ratio < 0.7 || disp.len() < 24 || j - last >= 12 { + last = j; + kf.push(j); + } + j += 1; + } + if let Some(lastf) = (0..d.n).rev().find(|k| d.by_frame[*k].len() >= 8) + && *kf.last().expect("one") < lastf + { + kf.push(lastf); + } + kf +} + +/// A (partial) reconstruction. +#[derive(Clone, Default)] +struct Recon { + cams: BTreeMap, + pts: HashMap, + /// Rejected observations (track index, frame). + outliers: HashSet<(u32, usize)>, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Kind { + General, + Tripod, +} + +#[derive(Clone, Copy, PartialEq, Debug)] +struct Cfg { + kind: Kind, + /// Focal mode of the global adjustments. + focal: FocalMode, + /// Registration estimates a focal length per frame. + per_frame_f: bool, +} + +const F_PRIOR: f64 = 30.0; + +impl Recon { + /// Observations among `frames` (inliers, with a 3D point). + fn observations(&self, d: &Data, frames: &[usize]) -> (Vec, Vec, Vec, Vec) { + let cams: Vec = frames.iter().map(|k| self.cams[k]).collect(); + let mut pidx: HashMap = HashMap::new(); + let mut pts = vec![]; + let mut ids = vec![]; + let mut obs = vec![]; + for (ci, k) in frames.iter().enumerate() { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + let pi = *pidx.entry(*ti).or_insert_with(|| { + pts.push(*x); + ids.push(*ti); + (pts.len() - 1) as u32 + }); + obs.push(BaObs { cam: ci as u32, pt: pi, uv: *uv }); + } + } + (cams, pts, obs, ids) + } + + /// Bundle-adjust the cameras of `frames` (the first stays fixed) and their points. + fn adjust(&mut self, d: &Data, frames: &[usize], cfg: &Cfg, focal: FocalMode, iters: usize) -> f64 { + let (mut cams, mut pts, obs, ids) = self.observations(d, frames); + if obs.is_empty() { + return f64::INFINITY; + } + let mut fixed = vec![false; cams.len()]; + fixed[0] = true; + let rep = bundle::adjust( + &mut cams, + &mut pts, + &obs, + &BaOpts { focal, fix_points: false, fix_centers: cfg.kind == Kind::Tripod, fixed, huber: d.huber(), max_iter: iters }, + ); + for (k, c) in frames.iter().zip(&cams) { + self.cams.insert(*k, *c); + } + for (ti, x) in ids.iter().zip(&pts) { + self.pts.insert(*ti, *x); + } + rep.cost + } + + /// Mark observations above the inlier threshold as outliers; drop points left with fewer + /// than two observations among the registered frames. + fn reject(&mut self, d: &Data) { + let thr2 = d.inlier_thr().powi(2); + let mut count: HashMap = HashMap::new(); + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + match bundle::residual(c, *x, *uv) { + Some(r) if r[0] * r[0] + r[1] * r[1] <= thr2 => *count.entry(*ti).or_default() += 1, + _ => { + self.outliers.insert((*ti, *k)); + } + } + } + } + self.pts.retain(|ti, _| count.get(ti).copied().unwrap_or(0) >= 2); + } + + /// Triangulate tracks seen on ≥ 2 registered frames that have no point yet. + fn triangulate(&mut self, d: &Data, kind: Kind, min_angle_deg: f64, check: f64) { + let mut seen: HashMap> = HashMap::new(); + for k in self.cams.keys() { + for (ti, _) in &d.by_frame[*k] { + if !self.pts.contains_key(ti) { + seen.entry(*ti).or_default().push(*k); + } + } + } + let thr2 = (d.inlier_thr() * check).powi(2); + let new: Vec<(u32, V3)> = seen + .par_iter() + .filter(|(_, ks)| ks.len() >= 2) + .filter_map(|(ti, ks)| { + let views: Vec<(Pose, [f64; 2], &BaCam)> = ks + .iter() + .map(|k| { + let c = &self.cams[k]; + let uv = d.at(*ti, *k).expect("seen"); + (c.pose, [uv[0] / c.f, uv[1] / c.f], c) + }) + .collect(); + let x = match kind { + Kind::Tripod => normalize(views.iter().fold([0.0; 3], |a, (p, o, _)| add(a, normalize(mtv(&p.r, [o[0], o[1], 1.0]))))), + Kind::General => { + let v: Vec<(Pose, [f64; 2])> = views.iter().map(|(p, o, _)| (*p, *o)).collect(); + let x = geometry::triangulate(&v)?; + // Enough parallax between the most separated views. + let (a, b) = (views[0].0.c, views[views.len() - 1].0.c); + if geometry::ray_angle(a, b, x).to_degrees() < min_angle_deg { + return None; + } + x + } + }; + for (k, (_, _, c)) in ks.iter().zip(&views) { + let uv = d.at(*ti, *k).expect("seen"); + let r = bundle::residual(c, x, uv)?; + if r[0] * r[0] + r[1] * r[1] > thr2 { + return None; + } + } + Some((*ti, x)) + }) + .collect(); + self.pts.extend(new); + } + + /// Resect frame `k` from the existing points starting at `init` (pose, and the focal + /// length when `per_frame_f`). Returns false when too few points agree. + fn register(&mut self, d: &Data, k: usize, init: BaCam, cfg: &Cfg) -> bool { + let mut pts = vec![]; + let mut obs = vec![]; + let mut ids = vec![]; + for (ti, uv) in &d.by_frame[k] { + if let Some(x) = self.pts.get(ti) { + obs.push(BaObs { cam: 0, pt: pts.len() as u32, uv: *uv }); + pts.push(*x); + ids.push(*ti); + } + } + if obs.len() < 6 { + return false; + } + let thr2 = d.inlier_thr().powi(2); + let opts = |huber: f64| BaOpts { + focal: if cfg.per_frame_f { FocalMode::PerCamera(0.0) } else { FocalMode::Fixed }, + fix_points: true, + fix_centers: cfg.kind == Kind::Tripod, + fixed: vec![false], + huber, + max_iter: 40, + }; + let mut best: Option<(usize, BaCam)> = None; + // Robust start (large Huber threshold), then tighten; from the prediction and, failing + // that, from the nearest registered camera. + let nearest = self.cams.iter().min_by_key(|(c, _)| c.abs_diff(k)).map(|(_, c)| *c); + for start in std::iter::once(init).chain(nearest) { + if best.as_ref().is_some_and(|b| b.0 as f64 >= 0.8 * obs.len() as f64) { + break; + } + let mut cams = vec![start]; + bundle::adjust(&mut cams, &mut pts.clone(), &obs, &opts(d.inlier_thr() * 4.0)); + bundle::adjust(&mut cams, &mut pts.clone(), &obs, &opts(d.huber())); + let inl = obs.iter().filter(|o| bundle::residual(&cams[0], pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2)).count(); + if best.as_ref().is_none_or(|b| inl > b.0) { + best = Some((inl, cams[0])); + } + } + let Some((inl, cam)) = best else { return false }; + if inl < 6 || (inl as f64) < 0.5 * obs.len() as f64 { + return false; + } + // Refine on the inliers only. + let keep: Vec = + obs.iter().filter(|o| bundle::residual(&cam, pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2)).copied().collect(); + let mut cams = vec![cam]; + bundle::adjust(&mut cams, &mut pts.clone(), &keep, &opts(d.huber())); + for o in &obs { + let ok = bundle::residual(&cams[0], pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2); + if !ok { + self.outliers.insert((ids[o.pt as usize], k)); + } + } + self.cams.insert(k, cams[0]); + true + } + + /// How well the reconstruction explains every observation on its frames: the mean of the + /// reprojection error truncated at the inlier threshold, an observation without a point + /// counting as the threshold. + fn score(&self, d: &Data) -> f64 { + let thr = d.inlier_thr(); + let mut s = 0.0; + let mut n = 0usize; + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + n += 1; + s += match self.pts.get(ti).and_then(|x| bundle::residual(c, *x, *uv)) { + Some(r) => r[0].hypot(r[1]).min(thr), + None => thr, + }; + } + } + if n > 0 { s / n as f64 } else { f64::INFINITY } + } + + /// Average reprojection error (pixels) and observation count over the inliers. + fn error(&self, d: &Data) -> (f64, usize) { + let mut s = 0.0; + let mut n = 0; + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + if let Some(r) = bundle::residual(c, *x, *uv) { + s += r[0].hypot(r[1]); + n += 1; + } + } + } + (if n > 0 { s / n as f64 } else { f64::INFINITY }, n) + } +} + +/// Interpolated / extrapolated starting camera for frame `k` from the registered ones. +fn predict(r: &Recon, k: usize) -> Option { + let before = r.cams.range(..k).next_back().map(|(a, b)| (*a, *b)); + let after = r.cams.range(k + 1..).next().map(|(a, b)| (*a, *b)); + match (before, after) { + (Some((ka, a)), Some((kb, b))) => { + let t = (k - ka) as f64 / (kb - ka) as f64; + Some(BaCam { pose: Pose { r: slerp(&a.pose.r, &b.pose.r, t), c: add(scale(a.pose.c, 1.0 - t), scale(b.pose.c, t)) }, f: a.f * (b.f / a.f).powf(t) }) + } + (Some((ka, a)), None) => { + // Constant velocity from the two previous cameras. + match r.cams.range(..ka).next_back() { + Some((kp, p)) => { + let t = (k - ka) as f64 / (ka - kp) as f64; + let dr = log_rot(&mm(&a.pose.r, &mt(&p.pose.r))); + let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &a.pose.r); + let c = add(a.pose.c, scale(sub(a.pose.c, p.pose.c), t.min(2.0))); + Some(BaCam { pose: Pose { r: rr, c }, f: a.f }) + } + None => Some(a), + } + } + (None, Some((kb, b))) => match r.cams.range(kb + 1..).next() { + Some((kn, nx)) => { + let t = (kb - k) as f64 / (kn - kb) as f64; + let dr = log_rot(&mm(&b.pose.r, &mt(&nx.pose.r))); + let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &b.pose.r); + let c = add(b.pose.c, scale(sub(b.pose.c, nx.pose.c), t.min(2.0))); + Some(BaCam { pose: Pose { r: rr, c }, f: b.f }) + } + None => Some(b), + }, + (None, None) => None, + } +} + +/// Shared tracks of frames `a` and `b`: (track index, pixels in a, pixels in b). +fn shared(d: &Data, a: usize, b: usize) -> Vec<(u32, [f64; 2], [f64; 2])> { + d.by_frame[a].iter().filter_map(|(ti, p)| d.at(*ti, b).map(|q| (*ti, *p, q))).collect() +} + +/// Two-view initialisations of frames `a`, `b` at focal length `f`: the second camera and the +/// triangulated points (the first camera is the identity). +fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> Vec<(Pose, HashMap)> { + let sh = shared(d, a, b); + if sh.len() < 12 { + return vec![]; + } + let x1: Vec<[f64; 2]> = sh.iter().map(|s| [s.1[0] / f, s.1[1] / f]).collect(); + let x2: Vec<[f64; 2]> = sh.iter().map(|s| [s.2[0] / f, s.2[1] / f]).collect(); + let thr = 2.0 * d.noise() / f; + let mut cands: Vec = vec![]; + if use_e && let Some((e, inl)) = geometry::ransac_essential(&x1, &x2, thr, 400, (a * 7919 + b) as u64) { + let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip(); + if let Some((pose, _)) = geometry::select_pose(&geometry::decompose_essential(&e), &p1, &p2) { + cands.push(pose); + } + } + if use_h { + let (s1, s2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = sh.iter().map(|s| (s.1, s.2)).unzip(); + if let Some(fit) = ransac(Model::Homography, &s1, &s2, 2.0 * d.noise(), 500, (a * 31 + b) as u64) { + let h = fit.h.0; + // Normalised: K⁻¹ H K with K = diag(f, f, 1). + let hn = [[h[0][0], h[0][1], h[0][2] / f], [h[1][0], h[1][1], h[1][2] / f], [h[2][0] * f, h[2][1] * f, h[2][2]]]; + let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| fit.inliers[*i]).map(|i| (x1[i], x2[i])).unzip(); + for (r, td, _) in geometry::decompose_homography(&hn, &p1, &p2) { + if norm(td) > 1e-6 { + cands.push(Pose::from_rt(r, normalize(td))); + } + } + } + } + let thr2 = (d.inlier_thr() / f).powi(2); + cands + .into_iter() + .filter_map(|pose| { + let mut pts = HashMap::new(); + for (i, s) in sh.iter().enumerate() { + let Some(x) = geometry::triangulate(&[(Pose::IDENTITY, x1[i]), (pose, x2[i])]) else { continue }; + let ok = geometry::reproj2(&Pose::IDENTITY, x, x1[i]).is_some_and(|e| e < thr2) + && geometry::reproj2(&pose, x, x2[i]).is_some_and(|e| e < thr2) + && geometry::ray_angle([0.0; 3], pose.c, x).to_degrees() > 0.3; + if ok { + pts.insert(s.0, x); + } + } + (pts.len() >= 10 && pts.len() * 3 >= sh.len()).then_some((pose, pts)) + }) + .collect() +} + +/// Grow a reconstruction over keyframes `kfs` from an initial pair (`kfs[0]`, `kfs[j]`). +fn grow(d: &Data, kfs: &[usize], j: usize, init: &(Pose, HashMap), f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let (a, b) = (kfs[0], kfs[j]); + let mut r = Recon::default(); + r.cams.insert(a, BaCam { pose: Pose::IDENTITY, f }); + r.cams.insert(b, BaCam { pose: init.0, f }); + r.pts = init.1.clone(); + let inner = |r: &mut Recon, frames: &[usize]| { + let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed }; + r.adjust(d, frames, cfg, focal, 10); + r.reject(d); + }; + inner(&mut r, &[a, b]); + let order: Vec = kfs[1..j].iter().chain(&kfs[j + 1..]).copied().collect(); + let mut done = 0; + for k in order { + if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return None; + } + let start = predict(&r, k)?; + if !r.register(d, k, start, cfg) { + // A failed keyframe ends the reconstruction (a later frame is too far from the known + // points); the caller resects the rest from what is known. + break; + } + r.triangulate(d, cfg.kind, 1.0, 1.0); + done += 1; + let frames: Vec = r.cams.keys().copied().collect(); + if done % 3 == 0 || frames.len() <= 6 { + inner(&mut r, &frames); + } + } + let frames: Vec = r.cams.keys().copied().collect(); + inner(&mut r, &frames); + (r.cams.len() >= 2 && r.pts.len() >= 8).then_some(r) +} + +/// Tripod reconstruction over keyframes `kfs` at focal length `f`. +fn grow_tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let mut r = Recon::default(); + r.cams.insert(kfs[0], BaCam { pose: Pose::IDENTITY, f }); + let ray = |p: [f64; 2], f: f64| normalize([p[0] / f, p[1] / f, 1.0]); + for w in kfs.windows(2) { + if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return None; + } + let (a, b) = (w[0], w[1]); + let sh = shared(d, a, b); + if sh.len() < 6 { + break; + } + let ra: Vec = sh.iter().map(|s| ray(s.1, f)).collect(); + let rb: Vec = sh.iter().map(|s| ray(s.2, f)).collect(); + let mut rel = geometry::procrustes(&ra, &rb)?; + // One round of outlier trimming. + let errs: Vec = ra.iter().zip(&rb).map(|(x, y)| norm(sub(*y, mv(&rel, *x)))).collect(); + let mut sorted = errs.clone(); + sorted.sort_by(f64::total_cmp); + let lim = (sorted[sorted.len() / 2] * 3.0).max(d.inlier_thr() / f); + let (ia, ib): (Vec, Vec) = ra.iter().zip(&rb).zip(&errs).filter(|(_, e)| **e <= lim).map(|((x, y), _)| (*x, *y)).unzip(); + if ia.len() >= 4 { + rel = geometry::procrustes(&ia, &ib)?; + } + let pa = r.cams[&a].pose.r; + r.cams.insert(b, BaCam { pose: Pose { r: mm(&rel, &pa), c: [0.0; 3] }, f }); + } + r.triangulate(d, Kind::Tripod, 0.0, 20.0); + let frames: Vec = r.cams.keys().copied().collect(); + let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed }; + r.adjust(d, &frames, cfg, focal, 15); + r.reject(d); + r.triangulate(d, Kind::Tripod, 0.0, 1.0); + r.adjust(d, &frames, cfg, focal, 15); + (r.cams.len() >= 2 && r.pts.len() >= 8).then_some(r) +} + +/// Focal length (pixels) of a horizontal angle of view. +fn focal_of(w: f64, hfov: f64) -> f64 { + w * 0.5 / (hfov.clamp(1.0, 170.0).to_radians() * 0.5).tan() +} + +struct Attempt { + recon: Recon, + err: f64, + method: SolveMethod, +} + +/// Choose the initial pair: the first keyframe and the later keyframe (among the next few) +/// that keeps the most points while having enough parallax. +fn best_init(d: &Data, kfs: &[usize], f: f64, method: SolveMethod) -> Option<(usize, (Pose, HashMap), bool)> { + let (use_e, use_h) = match method { + SolveMethod::Typical => (true, false), + SolveMethod::MostlyFlat => (false, true), + _ => (true, true), + }; + let mut best: Option<(f64, usize, (Pose, HashMap), bool)> = None; + for j in 1..kfs.len().min(6) { + let cands = init_pair(d, kfs[0], kfs[j], f, use_e, false); + let hc = if use_h { init_pair(d, kfs[0], kfs[j], f, false, true) } else { vec![] }; + for (c, is_h) in cands.into_iter().map(|c| (c, false)).chain(hc.into_iter().map(|c| (c, true))) { + // Score: points × median triangulation angle (capped). + let mut ang: Vec = c.1.values().map(|x| geometry::ray_angle([0.0; 3], c.0.c, *x).to_degrees()).collect(); + ang.sort_by(f64::total_cmp); + let med = ang.get(ang.len() / 2).copied().unwrap_or(0.0); + let score = c.1.len() as f64 * med.min(4.0); + if best.as_ref().is_none_or(|b| score > b.0) { + best = Some((score, j, c, is_h)); + } + } + } + best.map(|b| (b.1, b.2, b.3)) +} + +/// All initial candidates for the pair chosen by [`best_init`] (E and H decompositions). +fn init_candidates(d: &Data, kfs: &[usize], f: f64, method: SolveMethod) -> Vec<(usize, (Pose, HashMap), bool)> { + let Some((j, first, is_h)) = best_init(d, kfs, f, method) else { return vec![] }; + let mut out = vec![(j, first, is_h)]; + if matches!(method, SolveMethod::Auto | SolveMethod::MostlyFlat) { + for c in init_pair(d, kfs[0], kfs[j], f, false, true) { + if out.iter().all(|o| norm(log_rot(&mm(&o.1.0.r, &mt(&c.0.r)))) > 1e-3 || norm(sub(o.1.0.c, c.0.c)) > 1e-3) { + out.push((j, c, true)); + } + } + } + out +} + +/// A general (translating camera) reconstruction of keyframes `kfs` at focal length `f`. +fn general(d: &Data, kfs: &[usize], f: f64, method: SolveMethod, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let cands = init_candidates(d, kfs, f, method); + let res: Vec = cands + .par_iter() + .filter_map(|(j, init, is_h)| { + let r = grow(d, kfs, *j, init, f, cfg, cancel)?; + let (_, n) = r.error(d); + // Prefer reconstructions covering more keyframes. + let cover = r.cams.len() as f64 / kfs.len() as f64; + (n > 0).then(|| Attempt { err: r.score(d) / cover.max(0.05), recon: r, method: if *is_h { SolveMethod::MostlyFlat } else { SolveMethod::Typical } }) + }) + .collect(); + res.into_iter().min_by(|a, b| a.err.total_cmp(&b.err)) +} + +fn tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let r = grow_tripod(d, kfs, f, cfg, cancel)?; + let cover = r.cams.len() as f64 / kfs.len() as f64; + Some(Attempt { err: r.score(d) / cover.max(0.05), recon: r, method: SolveMethod::TripodPan }) +} + +/// Search the focal length minimising `eval(f)` (a reprojection error): a log-spaced scan of +/// horizontal angles of view, then golden-section refinement around the best. +fn search_focal(w: f64, eval: &(dyn Fn(f64) -> f64 + Sync)) -> f64 { + let fovs = [8.0, 12.0, 16.0, 20.0, 25.0, 30.0, 36.0, 43.0, 51.0, 60.0, 70.0, 82.0, 96.0, 112.0]; + let fs: Vec = fovs.iter().map(|a| focal_of(w, *a)).collect(); + let errs: Vec = fs.par_iter().map(|f| eval(*f)).collect(); + if debug() { + eprintln!("focal scan: {:?}", fs.iter().zip(&errs).map(|(f, e)| format!("{f:.0}:{e:.3}")).collect::>()); + } + let i = (0..fs.len()).min_by(|a, b| errs[*a].total_cmp(&errs[*b])).unwrap_or(5); + let (mut lo, mut hi) = (fs[(i + 1).min(fs.len() - 1)].ln(), fs[i.saturating_sub(1)].ln()); + if lo > hi { + std::mem::swap(&mut lo, &mut hi); + } + let g = 0.618_033_988_75; + let (mut x1, mut x2) = (hi - g * (hi - lo), lo + g * (hi - lo)); + let (mut e1, mut e2) = rayon::join(|| eval(x1.exp()), || eval(x2.exp())); + for _ in 0..7 { + if e1 < e2 { + hi = x2; + x2 = x1; + e2 = e1; + x1 = hi - g * (hi - lo); + e1 = eval(x1.exp()); + } else { + lo = x1; + x1 = x2; + e1 = e2; + x2 = lo + g * (hi - lo); + e2 = eval(x2.exp()); + } + } + let best = if e1 < e2 { (x1, e1) } else { (x2, e2) }; + if best.1 <= errs[i] { best.0.exp() } else { fs[i] } +} + +/// Solve the camera of a tracked clip. +pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBool>) -> Result { + let d = Data::new(tracks, &s.deleted); + if d.n < 2 { + return err("the clip must be at least two frames long"); + } + let kfs = keyframes(&d); + if kfs.len() < 2 { + return err("not enough features were tracked to solve the camera"); + } + let cancelled = || cancel.is_some_and(|c| c.load(Ordering::Relaxed)); + let fixed_f = (s.shot == ShotType::SpecifyAngle).then(|| focal_of(d.w, if s.hfov > 0.0 { s.hfov } else { 40.0 })); + let variable = s.shot == ShotType::VariableZoom; + let gcfg = Cfg { kind: Kind::General, focal: FocalMode::Fixed, per_frame_f: false }; + let tcfg = Cfg { kind: Kind::Tripod, ..gcfg }; + // The focal-length search runs on the first keyframes only. + let mini: Vec = kfs.iter().copied().take(10).collect(); + let try_general = matches!(s.method, SolveMethod::Auto | SolveMethod::Typical | SolveMethod::MostlyFlat); + let try_tripod = matches!(s.method, SolveMethod::Auto | SolveMethod::TripodPan); + + let t0 = web_time::Instant::now(); + let lap = |what: &str| { + if debug() { + eprintln!("{what}: {:.2} s", t0.elapsed().as_secs_f64()); + } + }; + let mut results: Vec<(Attempt, f64)> = vec![]; + if try_general { + let f = match fixed_f { + Some(f) => f, + None => { + search_focal(d.w, &|f| general(&d, &mini, f, s.method, &Cfg { per_frame_f: variable, ..gcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)) + } + }; + if cancelled() { + return err("cancelled"); + } + if let Some(a) = general(&d, &kfs, f, s.method, &Cfg { per_frame_f: variable, ..gcfg }, cancel) { + results.push((a, f)); + } + } + lap("general"); + // Auto Detect: a quick tripod check at the general solve's focal length; when a rotation + // clearly cannot explain the tracks, the tripod search is skipped. + let tripod_plausible = match (s.method, results.first()) { + (SolveMethod::Auto, Some((g, f))) => { + let quick = tripod(&d, &mini, *f, &tcfg, cancel).map(|a| a.err).unwrap_or(f64::INFINITY); + if debug() { + eprintln!("quick tripod check: {quick:.3} vs general {:.3}", g.err); + } + quick <= 3.0 * g.err + d.noise() + } + _ => true, + }; + if try_tripod && tripod_plausible && !cancelled() { + let f = match fixed_f { + Some(f) => f, + None => search_focal(d.w, &|f| tripod(&d, &mini, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)), + }; + if let Some(a) = tripod(&d, &kfs, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel) { + results.push((a, f)); + } + } + if cancelled() { + return err("cancelled"); + } + // Auto Detect keeps the tripod model when it explains the tracks about as well. + let pick = { + let g = results.iter().position(|r| r.0.method != SolveMethod::TripodPan); + let t = results.iter().position(|r| r.0.method == SolveMethod::TripodPan); + match (g, t) { + (Some(g), Some(t)) => { + let (eg, et) = (results[g].0.err, results[t].0.err); + // A general solve whose camera hardly moves relative to the scene depth is a pan. + let rg = &results[g].0.recon; + let cs: Vec = rg.cams.values().map(|c| c.pose.c).collect(); + let base = cs.iter().flat_map(|a| cs.iter().map(move |b| norm(sub(*a, *b)))).fold(0.0, f64::max); + let c0 = rg.cams.values().next().expect("camera"); + let mut depth: Vec = rg.pts.values().map(|x| norm(sub(*x, c0.pose.c))).collect(); + depth.sort_by(f64::total_cmp); + let med = depth.get(depth.len() / 2).copied().unwrap_or(1.0); + if debug() { + eprintln!("auto: general {eg:.4} tripod {et:.4} baseline/depth {:.4}", base / med); + } + if et <= (eg * 1.25).max(eg + 0.05 * d.noise()) || base / med < 0.01 { t } else { g } + } + (Some(g), None) => g, + (None, Some(t)) => t, + (None, None) => return err("the camera could not be solved (not enough parallax or features)"), + } + }; + if debug() { + for (a, f) in &results { + eprintln!("candidate {:?}: err {:.4} f {f:.1} cams {} pts {}", a.method, a.err, a.recon.cams.len(), a.recon.pts.len()); + } + } + let (att, _) = results.swap_remove(pick); + let kind = if att.method == SolveMethod::TripodPan { Kind::Tripod } else { Kind::General }; + let cfg = Cfg { kind, focal: FocalMode::Fixed, per_frame_f: variable }; + let mut r = att.recon; + let focal = match (fixed_f, variable) { + (Some(_), _) => FocalMode::Fixed, + (None, true) => FocalMode::PerCamera(F_PRIOR), + (None, false) => FocalMode::Shared, + }; + lap("tripod"); + // Keyframes: adjust with the focal length free, reject, re-triangulate, adjust again. + let kf_frames: Vec = r.cams.keys().copied().collect(); + r.adjust(&d, &kf_frames, &cfg, focal, 40); + r.reject(&d); + // Re-estimate the tracking noise from the inlier residuals (robustly, from their median: + // for 2D Gaussian noise of deviation σ the median distance is 1.1774 σ), then tighten the + // thresholds: observations of moving objects and occlusion edges become outliers. + let mut res: Vec = vec![]; + for (k, c) in &r.cams { + for (ti, uv) in &d.by_frame[*k] { + if r.outliers.contains(&(*ti, *k)) { + continue; + } + if let Some(e) = r.pts.get(ti).and_then(|x| bundle::residual(c, *x, *uv)) { + res.push(e[0].hypot(e[1])); + } + } + } + if res.len() > 20 { + res.sort_by(f64::total_cmp); + let sigma = res[res.len() / 2] / 1.1774; + let floor = 0.05 * tracks.factor.max(1.0); + d.set_noise((1.5 * sigma).clamp(floor, d.noise())); + if debug() { + eprintln!("noise: σ {sigma:.3} → {:.3}", d.noise()); + } + r.outliers.clear(); + r.reject(&d); + } + r.triangulate(&d, kind, 1.0, 1.0); + r.adjust(&d, &kf_frames, &cfg, focal, 40); + r.reject(&d); + r.adjust(&d, &kf_frames, &cfg, focal, 20); + if cancelled() { + return err("cancelled"); + } + lap("keyframe adjustment"); + // Every other frame: resect from the points (nearest registered camera first). + let mut todo: Vec = (0..d.n).filter(|k| !r.cams.contains_key(k)).collect(); + todo.sort_by_key(|k| r.cams.keys().map(|c| c.abs_diff(*k)).min().unwrap_or(usize::MAX)); + let base = r.clone(); + let resected: Vec<(usize, Option)> = todo + .par_iter() + .map(|k| { + let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() }; + let ok = predict(&local, *k).is_some_and(|start| local.register(&d, *k, start, &cfg)); + (*k, if ok { local.cams.get(k).copied() } else { None }) + }) + .collect(); + for (k, c) in resected { + if let Some(c) = c { + r.cams.insert(k, c); + } + } + if cancelled() { + return err("cancelled"); + } + lap("resection"); + // Points for the tracks that only lived between keyframes, then the final adjustment over + // every frame (when that stays affordable). + r.triangulate(&d, kind, 1.0, 1.0); + let all: Vec = r.cams.keys().copied().collect(); + r.reject(&d); + if all.len() <= 160 { + r.adjust(&d, &all, &cfg, focal, 25); + r.reject(&d); + } else { + // Points only (cameras fixed) would be cheap but the keyframe solution is already + // adjusted; refine each non-keyframe once more against the final points. + let base = r.clone(); + let refined: Vec<(usize, Option)> = all + .par_iter() + .filter(|k| !kf_frames.contains(k)) + .map(|k| { + let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() }; + let start = local.cams[k]; + let ok = local.register(&d, *k, start, &cfg); + (*k, if ok { local.cams.get(k).copied() } else { None }) + }) + .collect(); + for (k, c) in refined.into_iter() { + if let Some(c) = c { + r.cams.insert(k, c); + } + } + r.reject(&d); + } + // Method Used: the chosen model; Auto Detect calls a general solve "Mostly Flat Scene" when + // its points are close to one plane. + let method = match (s.method, att.method) { + (_, SolveMethod::TripodPan) => SolveMethod::TripodPan, + (SolveMethod::Auto, _) => { + let p: Vec = r.pts.values().copied().collect(); + let flat = cov_ratio(&p) < 0.08; + if flat { SolveMethod::MostlyFlat } else { SolveMethod::Typical } + } + (m, _) => m, + }; + lap("final adjustment"); + Ok(finish(&d, tracks, s, r, method, kind)) +} + +/// Thickness of a point cloud: √(smallest / middle eigenvalue of its covariance). +fn cov_ratio(p: &[V3]) -> f64 { + if p.len() < 3 { + return 1.0; + } + let k = 1.0 / p.len() as f64; + let c = scale(p.iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [0.0; 9]; + for q in p { + let d = sub(*q, c); + for i in 0..3 { + for j in 0..3 { + cov[i * 3 + j] += d[i] * d[j] * k; + } + } + } + let (vals, _) = sym_eigen(&cov, 3); + (vals[0].max(0.0) / vals[1].max(1e-300)).sqrt() +} + +/// Canonical frame, per-frame output (unsolved frames interpolated), points and errors. +fn finish(d: &Data, tracks: &CameraTracks, s: &SolveSettings, r: Recon, method: SolveMethod, kind: Kind) -> CameraSolve { + let (k0, c0) = r.cams.iter().next().map(|(k, c)| (*k, *c)).expect("solved"); + // Scale: median depth of the points seen by the first solved camera is 1. + let mut depths: Vec = d.by_frame[k0].iter().filter_map(|(ti, _)| r.pts.get(ti)).map(|x| c0.pose.to_cam(*x)[2]).filter(|z| *z > 0.0).collect(); + depths.sort_by(f64::total_cmp); + let sc = match kind { + Kind::Tripod => 1.0, + Kind::General => depths.get(depths.len() / 2).copied().filter(|v| *v > 1e-12).unwrap_or(1.0), + }; + let r0 = c0.pose.r; + let cc = c0.pose.c; + let to_world = |x: V3| scale(mv(&r0, sub(x, cc)), 1.0 / sc); + let mut frames = vec![SolvedFrame::default(); d.n]; + for (k, c) in &r.cams { + let rr = mm(&c.pose.r, &mt(&r0)); + frames[*k] = SolvedFrame { rot: log_rot(&rr), center: to_world(c.pose.c), focal: c.f, solved: true }; + } + // Hold / interpolate unsolved frames so the camera path stays continuous. + let solved: Vec = (0..d.n).filter(|k| frames[*k].solved).collect(); + for k in 0..d.n { + if frames[k].solved { + continue; + } + let a = solved.iter().rev().find(|x| **x < k).copied(); + let b = solved.iter().find(|x| **x > k).copied(); + frames[k] = match (a, b) { + (Some(a), Some(b)) => { + let t = (k - a) as f64 / (b - a) as f64; + let (fa, fb) = (frames[a], frames[b]); + let rr = slerp(&fa.r(), &fb.r(), t); + SolvedFrame { + rot: log_rot(&rr), + center: add(scale(fa.center, 1.0 - t), scale(fb.center, t)), + focal: fa.focal + (fb.focal - fa.focal) * t, + solved: false, + } + } + (Some(a), None) => SolvedFrame { solved: false, ..frames[a] }, + (None, Some(b)) => SolvedFrame { solved: false, ..frames[b] }, + (None, None) => frames[k], + }; + } + let mut points = vec![]; + let (mut esum, mut en) = (0.0, 0usize); + let mut ids: Vec<&u32> = r.pts.keys().collect(); + ids.sort(); + for ti in ids { + let x = r.pts[ti]; + let (id, start, pts) = &d.tracks[*ti as usize]; + let (mut s1, mut n1) = (0.0, 0); + for (i, uv) in pts.iter().enumerate() { + let k = start + i; + if r.outliers.contains(&(*ti, k)) { + continue; + } + if let Some(c) = r.cams.get(&k) + && let Some(res) = bundle::residual(c, x, *uv) + { + s1 += res[0].hypot(res[1]); + n1 += 1; + } + } + if n1 < 2 { + continue; + } + esum += s1; + en += n1; + points.push(SolvedPoint { id: *id, pos: to_world(x), error: (s1 / n1 as f64) as f32, first: *start as u32, last: (start + pts.len() - 1) as u32 }); + } + CameraSolve { + version: 1, + size: tracks.size, + start: tracks.start, + frame_duration: tracks.frame_duration, + shot: s.shot, + method_used: method, + average_error: if en > 0 { esum / en as f64 } else { 0.0 }, + frames, + points, + ground: None, + } +} diff --git a/crates/track/src/camtrack/tests.rs b/crates/track/src/camtrack/tests.rs new file mode 100644 index 00000000..4b5a8346 --- /dev/null +++ b/crates/track/src/camtrack/tests.rs @@ -0,0 +1,438 @@ +//! Synthetic scenes: random 3D points (or a plane) seen by a known moving camera with a known +//! focal length, projected directly (with sub-pixel noise) into feature tracks. + +use super::geometry::Rng; +use super::linalg::*; +use super::*; + +const W: f64 = 1280.0; +const H: f64 = 720.0; + +struct Truth { + /// Per frame: (R world→cam, centre, focal). + cams: Vec<(M3, V3, f64)>, +} + +fn noise(rng: &mut Rng) -> f64 { + // Sum of uniforms ≈ Gaussian, σ ≈ 0.1 px. + let u: f64 = (0..4).map(|_| (rng.next_u64() % 10_000) as f64 / 10_000.0 - 0.5).sum(); + u * 0.17 +} + +/// Project `pts` through the cameras into tracks (a track ends when its point leaves the frame; +/// re-entering starts a new track). +fn tracks_of(pts: &[V3], t: &Truth, seed: u64, noise_on: bool) -> CameraTracks { + let mut rng = Rng(seed); + let mut tracks: Vec = vec![]; + for x in pts { + let mut cur: Option = None; + for (k, (r, c, f)) in t.cams.iter().enumerate() { + let p = mv(r, sub(*x, *c)); + let uv = (p[2] > 0.1).then(|| [f * p[0] / p[2] + W / 2.0, f * p[1] / p[2] + H / 2.0]); + match uv.filter(|u| u[0] >= 2.0 && u[1] >= 2.0 && u[0] < W - 2.0 && u[1] < H - 2.0) { + Some(u) => { + let (nx, ny) = if noise_on { (noise(&mut rng), noise(&mut rng)) } else { (0.0, 0.0) }; + let q = [(u[0] + nx) as f32, (u[1] + ny) as f32]; + match &mut cur { + Some(tr) => tr.pts.push(q), + None => cur = Some(Track2D { id: 0, start: k as u32, pts: vec![q] }), + } + } + None => { + if let Some(tr) = cur.take() { + tracks.push(tr); + } + } + } + } + if let Some(tr) = cur.take() { + tracks.push(tr); + } + } + tracks.retain(|t| t.pts.len() >= 3); + for (i, tr) in tracks.iter_mut().enumerate() { + tr.id = i as u32; + } + CameraTracks { version: 1, size: [W, H], start: 0.0, frame_duration: 1.0 / 30.0, frames: t.cams.len() as u32, factor: 1.0, detailed: false, tracks } +} + +fn random_points(n: usize, seed: u64, center: V3, ext: V3) -> Vec { + let mut rng = Rng(seed); + (0..n) + .map(|_| { + let mut u = || (rng.next_u64() % 100_000) as f64 / 100_000.0 - 0.5; + [center[0] + u() * ext[0], center[1] + u() * ext[1], center[2] + u() * ext[2]] + }) + .collect() +} + +/// A camera dollying sideways and forward while panning to keep the scene in view. +fn moving_camera(frames: usize, f: impl Fn(usize) -> f64) -> Truth { + let cams = (0..frames) + .map(|k| { + let t = k as f64 / (frames - 1) as f64; + let c = [-3.0 + 6.0 * t, -0.6 + 0.8 * t + 0.5 * (std::f64::consts::PI * t).sin(), -1.0 + 2.0 * t - 1.5 * (t - 0.5) * (t - 0.5)]; + // Look at the scene centre (0, 0, 10). + let fwd = normalize(sub([0.3 * t, 0.2, 10.0], c)); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + let down = cross(fwd, right); + let r = [right, down, fwd]; + // A little roll. + let roll = rodrigues([0.0, 0.0, 0.05 * (t * 6.0).sin()]); + (mm(&roll, &r), c, f(k)) + }) + .collect(); + Truth { cams } +} + +/// Align the solved centres to the true ones (similarity) and return the RMS path error +/// relative to the path length, plus the worst rotation error (degrees). +fn path_error(s: &CameraSolve, t: &Truth) -> (f64, f64) { + let solved: Vec = (0..s.frames.len()).filter(|k| s.frames[*k].solved).collect(); + let src: Vec = solved.iter().map(|k| s.frames[*k].center).collect(); + let dst: Vec = solved.iter().map(|k| t.cams[*k].1).collect(); + let sim = umeyama(&src, &dst).expect("alignment"); + let len = norm(sub(dst[0], dst[dst.len() - 1])).max(1e-9); + let rms = (src.iter().zip(&dst).map(|(a, b)| dot(sub(sim.apply(*a), *b), sub(sim.apply(*a), *b))).sum::() / src.len() as f64).sqrt(); + // Rotation: R_true ≈ R_solved · simᵀ. + let mut worst: f64 = 0.0; + for k in &solved { + let rs = mm(&s.frames[*k].r(), &mt(&sim.r)); + let e = norm(log_rot(&mm(&rs, &mt(&t.cams[*k].0)))).to_degrees(); + worst = worst.max(e); + } + (rms / len, worst) +} + +fn settings(shot: ShotType, method: SolveMethod) -> SolveSettings { + SolveSettings { shot, hfov: 0.0, method, deleted: vec![] } +} + +#[test] +fn fixed_angle_solve_recovers_path_and_focal() { + let f = 1100.0; + let t = moving_camera(40, |_| f); + let pts = random_points(400, 1, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 2, true); + let t0 = std::time::Instant::now(); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), None).expect("solve"); + eprintln!("fixed: {:?} err {:.3} px focal {:.1} in {:?}", s.method_used, s.average_error, s.focal(), t0.elapsed()); + assert!(s.frames.iter().all(|c| c.solved)); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.02, "focal {}", s.focal()); + let (pe, re) = path_error(&s, &t); + assert!(pe < 0.01, "path {pe}"); + assert!(re < 0.5, "rotation {re}"); + // Canonical frame: first camera at the origin. + assert!(norm(s.frames[0].center) < 1e-9 && norm(s.frames[0].rot) < 1e-9); + // Serde round trip. + let j = s.to_json(); + assert_eq!(CameraSolve::from_json(&j).unwrap().points.len(), s.points.len()); +} + +#[test] +fn auto_detect_picks_a_general_model_for_a_translating_camera() { + let f = 900.0; + let t = moving_camera(30, |_| f); + let pts = random_points(300, 3, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 4, true); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + assert_eq!(s.method_used, SolveMethod::Typical); + assert!(s.average_error < 0.5); + assert!((s.focal() - f).abs() / f < 0.02, "focal {}", s.focal()); +} + +#[test] +fn specified_angle_of_view_is_kept() { + let f = 1000.0; + let t = moving_camera(25, |_| f); + let pts = random_points(300, 5, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 6, true); + let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); + let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: vec![] }, None).unwrap(); + assert!((s.focal() - f).abs() < 1e-6); + assert!(s.average_error < 0.5); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.01, "{pe}"); +} + +#[test] +fn variable_zoom_recovers_per_frame_focal() { + let t = moving_camera(40, |k| 950.0 + 6.0 * k as f64); + let pts = random_points(450, 7, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]); + let tr = tracks_of(&pts, &t, 8, true); + let s = solve(&tr, &settings(ShotType::VariableZoom, SolveMethod::Typical), None).expect("solve"); + eprintln!("variable: err {:.3}", s.average_error); + assert!(s.average_error < 0.5); + for (k, c) in s.frames.iter().enumerate() { + let f = t.cams[k].2; + assert!((c.focal - f).abs() / f < 0.02, "frame {k}: {} vs {f}", c.focal); + } +} + +#[test] +fn tripod_pan_solve() { + let f = 1200.0; + let cams = (0..40) + .map(|k| { + let a = -0.4 + 0.8 * k as f64 / 39.0; + (mm(&rodrigues([0.0, 0.0, 0.02]), &mm(&rodrigues([0.05 * a, 0.0, 0.0]), &rodrigues([0.0, a, 0.0]))), [0.0; 3], f) + }) + .collect(); + let t = Truth { cams }; + // Points all around at varying distances (rotation only: depth is unobservable). + let pts = random_points(600, 9, [0.0, 0.0, 0.0], [60.0, 20.0, 60.0]).into_iter().filter(|p| p[2] > 3.0).collect::>(); + let tr = tracks_of(&pts, &t, 10, true); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("tripod: {:?} err {:.3} focal {:.1}", s.method_used, s.average_error, s.focal()); + assert_eq!(s.method_used, SolveMethod::TripodPan); + assert!(s.average_error < 0.5); + assert!((s.focal() - f).abs() / f < 0.02, "{}", s.focal()); + for (k, c) in s.frames.iter().enumerate() { + assert!(norm(c.center) < 1e-9); + // Relative rotation to frame 0 matches. + let rel_s = mm(&c.r(), &mt(&s.frames[0].r())); + let rel_t = mm(&t.cams[k].0, &mt(&t.cams[0].0)); + assert!(norm(log_rot(&mm(&rel_s, &mt(&rel_t)))).to_degrees() < 0.2); + } +} + +#[test] +fn mostly_flat_scene_and_ground_plane() { + let f = 1000.0; + // A textured floor y = 2 (below the camera), seen by a camera moving forward and sideways. + let cams = (0..40) + .map(|k| { + let t = k as f64 / 39.0; + let c = [-2.0 + 4.0 * t, -0.3 * (std::f64::consts::PI * t).sin(), 2.0 * t]; + let fwd = normalize([0.1 - 0.2 * t, 0.45, 1.0]); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + let down = cross(fwd, right); + ([right, down, fwd], c, f) + }) + .collect(); + let t = Truth { cams }; + let mut rng = Rng(11); + let pts: Vec = (0..500).map(|_| [((rng.next_u64() % 2000) as f64 / 100.0) - 10.0, 2.0, 2.0 + (rng.next_u64() % 2000) as f64 / 100.0]).collect(); + let tr = tracks_of(&pts, &t, 12, true); + for method in [SolveMethod::MostlyFlat, SolveMethod::Auto] { + let s = solve(&tr, &settings(ShotType::FixedAngle, method), None).expect("solve"); + eprintln!("flat {method:?}: {:?} err {:.3} focal {:.1}", s.method_used, s.average_error, s.focal()); + assert_eq!(s.method_used, SolveMethod::MostlyFlat); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.02, "{}", s.focal()); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.01, "{pe}"); + } + // Ground plane from the solved (coplanar) points: every point lies on it. + let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); + let mut s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::MostlyFlat, deleted: vec![] }, None).expect("solve"); + let ids: Vec = s.visible(0).map(|p| p.id).take(12).collect(); + let g = s.ground_from(&ids, 0).expect("ground"); + let spread = s.points.iter().map(|p| dot(sub(p.pos, g.origin), g.normal).abs()).fold(0.0, f64::max); + let depth = norm(sub(s.frames[0].center, g.origin)); + assert!(spread < 0.01 * depth, "{spread}"); + // Up points towards the camera. + assert!(dot(sub(s.frames[0].center, g.origin), g.normal) > 0.0); + s.ground = Some(g); + let w = s.world([W, H], 1000.0); + // In the world, ground points have y ≈ 0 and the cameras are above (y < 0). + for p in s.points.iter().take(20) { + assert!(w.apply(p.pos)[1].abs() < 0.01 * 1000.0 * depth, "{:?}", w.apply(p.pos)); + } + assert!(w.apply(s.frames[0].center)[1] < 0.0); + assert!(norm(w.apply(g.origin)) < 1e-6); +} + +#[test] +fn deleted_tracks_are_ignored_and_cancel_stops() { + let t = moving_camera(20, |_| 1000.0); + let pts = random_points(250, 13, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 14, true); + let deleted: Vec = (0..40).collect(); + let hfov = 2.0 * (W * 0.5 / 1000.0f64).atan().to_degrees(); + let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: deleted.clone() }, None).unwrap(); + assert!(s.points.iter().all(|p| !deleted.contains(&p.id))); + let stop = std::sync::atomic::AtomicBool::new(true); + let r = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), Some(&stop)); + assert!(r.is_err()); +} + +/// End to end: rendered frames of a textured scene through the KLT feature tracker. +#[test] +fn rendered_scene_end_to_end() { + use effectcraft_raster::Image; + let (w, h) = (480u32, 270u32); + let f = 420.0; + // Three textured planes at different depths (a floor, a back wall and a box face). + fn tex(u: f64, v: f64) -> f32 { + fn hsh(i: i64, j: i64) -> f64 { + let mut x = (i.wrapping_mul(73856093) ^ j.wrapping_mul(19349663)) as u64; + x ^= x >> 13; + x = x.wrapping_mul(0x5bd1e995); + x ^= x >> 15; + (x % 1000) as f64 / 1000.0 + } + fn noise(x: f64, y: f64) -> f64 { + let (i, j) = (x.floor(), y.floor()); + let (tx, ty) = (x - i, y - j); + let (sx, sy) = (tx * tx * (3.0 - 2.0 * tx), ty * ty * (3.0 - 2.0 * ty)); + let (i, j) = (i as i64, j as i64); + let a = hsh(i, j) + (hsh(i + 1, j) - hsh(i, j)) * sx; + let b = hsh(i, j + 1) + (hsh(i + 1, j + 1) - hsh(i, j + 1)) * sx; + a + (b - a) * sy + } + (0.15 + 0.5 * noise(u * 4.0, v * 4.0) + 0.3 * noise(u * 11.0 + 7.0, v * 11.0 + 3.0)) as f32 + } + let frames = 24; + let t = Truth { + cams: (0..frames) + .map(|k| { + let s = k as f64 / (frames - 1) as f64; + let c = [-1.0 + 2.0 * s, -0.2, 0.5 * s]; + let fwd = normalize(sub([0.0, 0.3, 8.0], c)); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + ([right, cross(fwd, right), fwd], c, f) + }) + .collect(), + }; + // Planes: (point, normal, u axis, v axis). + let planes: [(V3, V3, V3, V3); 3] = [ + ([0.0, 1.5, 0.0], [0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]), + ([0.0, 0.0, 10.0], [0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0]), + ([-1.0, 0.0, 6.0], [0.3, 0.0, -1.0], [1.0, 0.0, 0.3], [0.0, 1.0, 0.0]), + ]; + let render = |k: usize| { + let (r, c, f) = t.cams[k]; + let mut img = Image::new(w, h); + for y in 0..h { + for x in 0..w { + let d = mtv(&r, [(x as f64 + 0.5 - w as f64 / 2.0) / f, (y as f64 + 0.5 - h as f64 / 2.0) / f, 1.0]); + let mut best = (f64::INFINITY, 0.0f32); + for (i, (p0, n, ua, va)) in planes.iter().enumerate() { + let den = dot(*n, d); + if den.abs() < 1e-9 { + continue; + } + let tt = dot(*n, sub(*p0, c)) / den; + let hit = add(c, scale(d, tt)); + // The box face is limited in extent. + if i == 2 && (dot(sub(hit, *p0), *ua).abs() > 1.2 || dot(sub(hit, *p0), *va).abs() > 1.0) { + continue; + } + if tt > 0.1 && tt < best.0 { + best = (tt, tex(dot(sub(hit, *p0), *ua) + 10.0 * i as f64, dot(sub(hit, *p0), *va))); + } + } + let v = best.1; + img.set(x, y, [v, v, v, 1.0]); + } + } + img + }; + let mut an = TrackAnalyzer::new([w as f64, h as f64], AnalyzeOpts::default()); + for k in 0..frames { + let img = render(k); + an.push(&crate::Frame::new(&img)); + } + let tr = an.finish(0.0, 1.0 / 24.0); + assert!(tr.tracks.len() > 100, "{}", tr.tracks.len()); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("rendered: {:?} err {:.3} focal {:.1} points {}", s.method_used, s.average_error, s.focal(), s.points.len()); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.05, "{}", s.focal()); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.03, "{pe}"); +} + +/// A long clip (300 frames, ~900 points): `cargo test -p effectcraft-track --release -- --ignored`. +#[test] +#[ignore] +fn perf_long_clip() { + let f = 1100.0; + let t = moving_camera(300, |_| f); + let pts = random_points(900, 21, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]); + let tr = tracks_of(&pts, &t, 22, true); + let t0 = std::time::Instant::now(); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("300 frames, {} tracks: {:?} err {:.3} focal {:.1} in {:?}", tr.tracks.len(), s.method_used, s.average_error, s.focal(), t0.elapsed()); + assert!(s.average_error < 0.5); +} + +/// Solve tracks saved from the app (`EC_CAMTRACK_FILE=tracks.json cargo test -p effectcraft-track +/// --release solve_saved_tracks -- --ignored --nocapture`): a debugging aid. +#[test] +#[ignore] +fn solve_saved_tracks() { + let Some(path) = std::env::var_os("EC_CAMTRACK_FILE") else { return }; + let mut tr = CameraTracks::from_json(&std::fs::read_to_string(path).unwrap()).unwrap(); + if let Some(maxy) = std::env::var("EC_CAMTRACK_MAXY").ok().and_then(|v| v.parse::().ok()) { + tr.tracks.retain(|t| t.pts.iter().all(|p| p[1] < maxy)); + } + let shot = if std::env::var_os("EC_CAMTRACK_VARIABLE").is_some() { ShotType::VariableZoom } else { ShotType::FixedAngle }; + let t0 = std::time::Instant::now(); + // With the true cameras (`[{pos, poi, zoom}]` per frame, two-node), how consistent are the + // tracks themselves? + if let Some(tp) = std::env::var_os("EC_CAMTRACK_TRUTH") { + let v: Vec = serde_json::from_str(&std::fs::read_to_string(tp).unwrap()).unwrap(); + let g = |x: &serde_json::Value| -> V3 { [x[0].as_f64().unwrap(), x[1].as_f64().unwrap(), x[2].as_f64().unwrap()] }; + let cams: Vec<(super::geometry::Pose, f64)> = v + .iter() + .map(|c| { + let (pos, poi) = (g(&c["pos"]), g(&c["poi"])); + let fwd = normalize(sub(poi, pos)); + let d0 = [0.0, 1.0, 0.0]; + let down = normalize(sub(d0, scale(fwd, dot(d0, fwd)))); + let right = cross(down, fwd); + (super::geometry::Pose { r: [right, down, fwd], c: pos }, c["zoom"].as_f64().unwrap()) + }) + .collect(); + let (w, h) = (tr.size[0], tr.size[1]); + let mut means = vec![]; + let mut drop_ids = vec![]; + for t in &tr.tracks { + let views: Vec<(super::geometry::Pose, [f64; 2])> = t + .pts + .iter() + .enumerate() + .map(|(i, p)| { + let (pose, z) = cams[t.start as usize + i]; + (pose, [(p[0] as f64 - w / 2.0) / z, (p[1] as f64 - h / 2.0) / z]) + }) + .collect(); + let Some(x) = super::geometry::triangulate(&views) else { continue }; + let e: Vec = views.iter().filter_map(|(p, o)| super::geometry::reproj2(p, x, *o).map(|e| e.sqrt() * cams[0].1)).collect(); + means.push((e.iter().sum::() / e.len().max(1) as f64, t.pts[0])); + if std::env::var_os("EC_CAMTRACK_TRUTH_FILTER").is_some() && means.last().unwrap().0 > 1.0 { + drop_ids.push(t.id); + } + } + tr.tracks.retain(|t| !drop_ids.contains(&t.id)); + means.sort_by(|a, b| a.0.total_cmp(&b.0)); + eprintln!( + "true-camera track errors (px): median {:.2}; quartiles {:?}", + means[means.len() / 2].0, + [means[means.len() / 4].0, means[means.len() * 3 / 4].0] + ); + for m in means.iter().step_by(6) { + eprintln!(" {:.2} at {:?}", m.0, m.1); + } + } + let mut st = settings(shot, SolveMethod::Auto); + if let Some(a) = std::env::var("EC_CAMTRACK_AOV").ok().and_then(|v| v.parse::().ok()) { + st.shot = ShotType::SpecifyAngle; + st.hfov = a; + } + let s = solve(&tr, &st, None).expect("solve"); + eprintln!( + "{} frames, {} tracks: {:?} err {:.3} focal {:.1} (hfov {:.1}) points {} in {:?}", + tr.frames, + tr.tracks.len(), + s.method_used, + s.average_error, + s.focal(), + s.hfov(s.focal()), + s.points.len(), + t0.elapsed() + ); +} diff --git a/crates/track/src/lib.rs b/crates/track/src/lib.rs index 629097cf..1c6e6a15 100644 --- a/crates/track/src/lib.rs +++ b/crates/track/src/lib.rs @@ -18,6 +18,7 @@ //! Coordinates are *layer* pixels: the frame image's pixel `(i, j)` covers `[i, i+1) × [j, j+1)` //! after subtracting the frame's `offset` (see [`Frame`]). +pub mod camtrack; pub mod fit; pub mod klt; pub mod mask; diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index c9c5e8fe..0976c47a 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -750,6 +750,11 @@ impl EffectcraftApp { self.session.poll_roto(true); ctx.request_repaint_after(std::time::Duration::from_millis(100)); } + // 3D Camera Tracker: likewise (tracking, then solving). + if self.session.camera_job.is_some() || !self.session.camera_pending.is_empty() { + self.session.poll_camera(true); + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } self.apply_prefs(&ctx); self.handle_events(&ctx); self.tick_autosave(&ctx); diff --git a/crates/ui-egui/src/panels/camera_tracker_ui.rs b/crates/ui-egui/src/panels/camera_tracker_ui.rs new file mode 100644 index 00000000..41fb7204 --- /dev/null +++ b/crates/ui-egui/src/panels/camera_tracker_ui.rs @@ -0,0 +1,438 @@ +//! 3D Camera Tracker in the Composition viewer and Effect Controls: +//! +//! - while the effect is selected, the solved 3D track points are drawn as coloured targets +//! sized by depth (or the 2D source tracks with Show Track Points ▸ 2D Source); +//! - hovering between three points shows the target plane they span (a disc in perspective); +//! click a point (Shift adds / toggles) or drag a marquee to select points, click inside a +//! triangle to select its three points; the selection's target is drawn; +//! - right-click: Set Ground Plane and Origin, Create Text / Solid / Null / Shadow Catcher … +//! and Camera, Create Multiple …, Delete Selected Points; Delete removes the selection; +//! - the "Analyzing in background (step 1 of 2)" / "Solving camera" banner; +//! - the Effect Controls block with Analyze / Cancel / Create Camera, Method Used and Average +//! Error. +//! +//! Everything is dispatched to the `camera.*` commands. + +use std::sync::Arc; + +use effectcraft_engine::Session; +use effectcraft_engine::effects::camera_tracker as ct; +use effectcraft_engine::geom::{Mat3, vec2 as gv2}; +use effectcraft_engine::keyframe::Value; +use effectcraft_engine::project::{Comp, Layer, LayerId, PropGroup, Uid}; +use effectcraft_engine::render::EvalCtx; +use effectcraft_engine::track::camtrack::{CameraSolve, CameraTracks, Target, point_color}; +use egui::{Align2, Color32, Pos2, Rect, Sense, Stroke, pos2, vec2}; +use serde_json::json; + +use super::viewer::ViewerMap; +use crate::theme::Tokens; +use crate::{EffectcraftApp, widgets}; + +/// The tracker whose points the viewer shows. +struct Active { + layer: LayerId, + uid: Uid, + solve: Option>, + tracks: Option>, + show_3d: bool, + point_size: f64, + target_size: f64, + /// Layer pixels → comp pixels. + l2c: Mat3, + frame: usize, +} + +/// A drawn point: (id, screen position, screen radius). +type Hit = (u32, Pos2, f32); + +fn tracker_group(l: &Layer, uid: Uid) -> Option<&PropGroup> { + l.props.find_group(uid).filter(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == ct::ID)) +} + +/// The selected 3D Camera Tracker effect on an active layer (After Effects shows the points +/// while the effect is selected). +fn active(s: &Session, comp: &Comp, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3) -> Option { + let time = ectx.time; + for (lid, uid) in &s.state.selected_props { + let Some(l) = comp.layer(*lid).filter(|l| l.is_active_at(time)) else { continue }; + let Some(g) = tracker_group(l, *uid) else { continue }; + let params = effectcraft_engine::camera_track::static_params(g); + let val = |id: &str| effectcraft_engine::effects::warp_stab::param(¶ms, id).cloned(); + let solve = ct::solve(¶ms); + let tracks = ct::tracks(¶ms); + let lt = l.layer_time(time).seconds(); + let frame = solve.as_ref().and_then(|sv| sv.frame_at(lt)).or_else(|| { + let t = tracks.as_ref()?; + (t.frame_duration > 0.0).then(|| ((lt - t.start) / t.frame_duration).round().clamp(0.0, t.frames.saturating_sub(1) as f64) as usize) + }); + return Some(Active { + layer: *lid, + uid: *uid, + solve, + tracks, + show_3d: val("showTrackPoints").map(|v| v.as_enum()).unwrap_or(1) == 1, + point_size: val("trackPointSize").map(|v| v.as_f64()).unwrap_or(100.0), + target_size: val("targetSize").map(|v| v.as_f64()).unwrap_or(100.0), + l2c: l2c(ectx, l), + frame: frame.unwrap_or(0), + }); + } + None +} + +fn to_screen(a: &Active, map: &ViewerMap, q: [f64; 2]) -> Pos2 { + let c = a.l2c.apply(gv2(q[0], q[1])); + map.to_screen([c.x, c.y]) +} + +fn col(id: u32) -> Color32 { + let c = point_color(id); + Color32::from_rgb((c[0] * 255.0) as u8, (c[1] * 255.0) as u8, (c[2] * 255.0) as u8) +} + +/// Comp-pixel scale of the layer → comp mapping. +fn l2c_scale(m: &Mat3) -> f32 { + m.0[0][0].hypot(m.0[1][0]) as f32 +} + +/// Draw the target disc (concentric rings in perspective) of `t` on frame `k`. +fn draw_target(p: &egui::Painter, a: &Active, map: &ViewerMap, sv: &CameraSolve, t: &Target) { + let Some(cam) = sv.frames.get(a.frame) else { return }; + let r = t.size * 0.5 * a.target_size / 100.0; + for (i, frac) in [1.0, 0.66, 0.33].into_iter().enumerate() { + let ring = ct::target_circle(t.center, t.normal, r * frac, 48); + let pts: Vec = ring.iter().filter_map(|x| cam.project(sv.size, *x)).map(|q| to_screen(a, map, q)).collect(); + if pts.len() < 3 { + return; + } + let fill = if i % 2 == 0 { Color32::from_rgba_unmultiplied(0xd0, 0x30, 0x30, 70) } else { Color32::from_rgba_unmultiplied(0xff, 0xff, 0xff, 50) }; + p.add(egui::Shape::convex_polygon(pts.clone(), fill, Stroke::new(1.0, Color32::from_rgb(0xe0, 0x40, 0x40)))); + } + if let Some(c) = cam.project(sv.size, t.center) { + let c = to_screen(a, map, c); + p.circle_filled(c, 3.0, Color32::from_rgb(0xff, 0xd0, 0x40)); + } +} + +/// The three drawn points forming a triangle around `hp` (nearest first). +fn triangle_at(hits: &[Hit], hp: Pos2) -> Option<[u32; 3]> { + let mut near: Vec<&Hit> = hits.iter().collect(); + near.sort_by(|x, y| x.1.distance_sq(hp).total_cmp(&y.1.distance_sq(hp))); + let near: Vec<&Hit> = near.into_iter().take(8).collect(); + let inside = |a: Pos2, b: Pos2, c: Pos2| { + let s = |p: Pos2, q: Pos2| (q.x - p.x) * (hp.y - p.y) - (q.y - p.y) * (hp.x - p.x); + let (d1, d2, d3) = (s(a, b), s(b, c), s(c, a)); + !((d1 < 0.0 || d2 < 0.0 || d3 < 0.0) && (d1 > 0.0 || d2 > 0.0 || d3 > 0.0)) + }; + for i in 0..near.len() { + for j in i + 1..near.len() { + for k in j + 1..near.len() { + let (a, b, c) = (near[i], near[j], near[k]); + // Skip slivers. + let area = ((b.1 - a.1).x * (c.1 - a.1).y - (b.1 - a.1).y * (c.1 - a.1).x).abs(); + if area > 30.0 && inside(a.1, b.1, c.1) { + return Some([a.0, b.0, c.0]); + } + } + } + } + None +} + +fn run(app: &mut EffectcraftApp, ctx: &egui::Context, cmd: &str, p: serde_json::Value) { + if let Err(e) = crate::menus::invoke(app, ctx, cmd, p) { + app.ui.status = e; + } +} + +/// The viewer overlay and its interaction (call after the viewer's own gestures so it takes +/// the clicks while a 3D Camera Tracker is selected). +pub fn viewer_hook( + app: &mut EffectcraftApp, + ui: &mut egui::Ui, + painter: &egui::Painter, + map: &ViewerMap, + ectx: &EvalCtx, + l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3, +) { + let ctx = ui.ctx().clone(); + let comp = ectx.comp; + banner(app, painter, map, ectx, l2c); + if !app.ui.viewer.show_layer_controls { + return; + } + let Some(a) = active(&app.session, comp, ectx, l2c) else { return }; + let selected = app.session.state.camera_points.clone(); + let lscale = l2c_scale(&a.l2c) * map.zoom; + let mut hits: Vec = vec![]; + match (&a.solve, a.show_3d) { + (Some(sv), true) => { + let pts = ct::projected(sv, a.frame); + let med = ct::median_depth(&pts); + for (id, q, z) in pts { + let sp = to_screen(&a, map, q); + if !map.area.expand(20.0).contains(sp) { + continue; + } + let r = (ct::target_radius(a.point_size, z, med) as f32 * lscale).clamp(2.0, 30.0); + let c = col(id); + let sel = selected.contains(&id); + if sel { + painter.circle_filled(sp, r, c); + painter.circle_stroke(sp, r + 1.5, Stroke::new(1.5, Color32::WHITE)); + } else { + painter.circle_stroke(sp, r, Stroke::new((r * 0.3).clamp(1.0, 3.0), c)); + painter.circle_filled(sp, (r * 0.25).max(1.0), c); + } + hits.push((id, sp, r)); + } + } + _ => { + // 2D source tracks (also shown before the solve finishes). + if let Some(tr) = &a.tracks { + for t in &tr.tracks { + let Some(q) = t.at(a.frame as u32) else { continue }; + let sp = to_screen(&a, map, q); + let c = col(t.id); + let r = (3.0 * a.point_size as f32 / 100.0).clamp(1.5, 12.0); + painter.line_segment([sp - vec2(r, r), sp + vec2(r, r)], Stroke::new(1.2, c)); + painter.line_segment([sp - vec2(r, -r), sp + vec2(r, -r)], Stroke::new(1.2, c)); + hits.push((t.id, sp, r)); + } + } + } + } + let Some(sv) = a.solve.clone() else { return }; + if app.ui.tool != crate::state::Tool::Selection { + return; + } + // Interaction over the image. + let resp = ui.interact(map.area, egui::Id::new("viewer-camera-tracker"), Sense::click_and_drag()); + app.auto.add("viewer.cameraTracker", map.area, "3D Camera Tracker points"); + for (id, sp, r) in hits.iter().take(400) { + app.auto.add(&format!("viewer.cameraTracker.point.{id}"), Rect::from_center_size(*sp, vec2(r * 2.0 + 4.0, r * 2.0 + 4.0)), "track point"); + } + let hover = resp.hover_pos(); + let point_at = |p: Pos2| hits.iter().filter(|h| h.1.distance(p) <= h.2 + 3.0).min_by(|x, y| x.1.distance(p).total_cmp(&y.1.distance(p))).map(|h| h.0); + let tri = hover.filter(|p| point_at(*p).is_none()).and_then(|p| triangle_at(&hits, p)); + let mid = egui::Id::new("viewer-camera-tracker-marquee"); + let marquee: Option = ui.data(|d| d.get_temp(mid)); + // Hover target / the selection's target. + if marquee.is_none() { + if let Some(t3) = tri + && let Some(t) = effectcraft_engine::camera_track::target_for(&sv, &t3, a.frame) + { + let ps: Vec = t3.iter().filter_map(|i| hits.iter().find(|h| h.0 == *i)).map(|h| h.1).collect(); + if ps.len() == 3 { + painter.add(egui::Shape::closed_line(ps, Stroke::new(1.0, Color32::from_white_alpha(200)))); + } + draw_target(painter, &a, map, &sv, &t); + } else if !selected.is_empty() + && let Some(t) = effectcraft_engine::camera_track::target_for(&sv, &selected, a.frame) + { + draw_target(painter, &a, map, &sv, &t); + } + } + let mods = ui.input(|i| i.modifiers); + if resp.drag_started() + && let Some(p) = ui.input(|i| i.pointer.press_origin()) + { + ui.data_mut(|d| d.insert_temp(mid, p)); + } + if let (Some(start), Some(p)) = (marquee, resp.interact_pointer_pos()) { + let r = Rect::from_two_pos(start, p); + let t = app.tokens; + painter.rect_filled(r, 0.0, t.accent.gamma_multiply(0.12)); + painter.rect_stroke(r, 0.0, Stroke::new(1.0, t.accent), egui::StrokeKind::Middle); + if resp.drag_stopped() { + let ids: Vec = hits.iter().filter(|h| r.contains(h.1)).map(|h| h.0).collect(); + ui.data_mut(|d| d.remove::(mid)); + run(app, &ctx, "camera.selectPoints", json!({"points": ids, "add": mods.shift})); + } + } else if resp.drag_stopped() { + ui.data_mut(|d| d.remove::(mid)); + } + if resp.clicked() + && let Some(p) = resp.interact_pointer_pos() + { + match (point_at(p), triangle_at(&hits, p)) { + (Some(id), _) => run(app, &ctx, "camera.selectPoints", json!({"points": [id], "toggle": mods.shift})), + (None, Some(t3)) => run(app, &ctx, "camera.selectPoints", json!({"points": t3, "add": mods.shift})), + (None, None) => run(app, &ctx, "camera.selectPoints", json!({"points": []})), + } + } + // Right-click: the points under the cursor (selection, else the hovered triangle). + let cid = egui::Id::new("viewer-camera-tracker-menu-points"); + if resp.secondary_clicked() { + let pts: Vec = if !selected.is_empty() { + selected.clone() + } else if let Some(p) = resp.interact_pointer_pos() { + point_at(p).map(|i| vec![i]).or_else(|| triangle_at(&hits, p).map(|t| t.to_vec())).unwrap_or_default() + } else { + vec![] + }; + ui.data_mut(|d| d.insert_temp(cid, pts)); + } + let base = json!({"layer": a.layer.0, "effect": a.uid}); + resp.context_menu(|ui| { + let pts: Vec = ui.data(|d| d.get_temp(cid)).unwrap_or_default(); + let with = |kind: Option<&str>, multiple: bool| { + let mut q = base.clone(); + q["points"] = json!(pts); + if let Some(k) = kind { + q["kind"] = json!(k); + } + if multiple { + q["multiple"] = json!(true); + } + q + }; + let some = !pts.is_empty(); + let many = pts.len() > 1; + let mut pick: Option<(&str, serde_json::Value)> = None; + if ui.add_enabled(some, egui::Button::new("Set Ground Plane and Origin")).clicked() { + pick = Some(("camera.setGroundPlane", with(None, false))); + } + ui.separator(); + for (label, kind) in [ + ("Create Text and Camera", "text"), + ("Create Solid and Camera", "solid"), + ("Create Null and Camera", "null"), + ("Create Shadow Catcher, Camera and Light", "shadowCatcher"), + ] { + if ui.add_enabled(some, egui::Button::new(label)).clicked() { + pick = Some(("camera.createFromSolve", with(Some(kind), false))); + } + } + ui.separator(); + for (label, kind) in + [("Create Multiple Text Layers and Camera", "text"), ("Create Multiple Solids and Camera", "solid"), ("Create Multiple Nulls and Camera", "null")] + { + if ui.add_enabled(many, egui::Button::new(label)).clicked() { + pick = Some(("camera.createFromSolve", with(Some(kind), true))); + } + } + ui.separator(); + if ui.add_enabled(some, egui::Button::new("Delete Selected Points")).clicked() { + pick = Some(("camera.deletePoints", with(None, false))); + } + if let Some((cmd, q)) = pick { + ui.close(); + run(app, &ctx, cmd, q); + } + }); + if resp.hovered() + && !selected.is_empty() + && ui.input(|i| i.key_pressed(egui::Key::Delete) || i.key_pressed(egui::Key::Backspace)) + && !ctx.egui_wants_keyboard_input() + { + run(app, &ctx, "camera.deletePoints", json!({"layer": a.layer.0, "effect": a.uid, "points": selected})); + } +} + +/// "Analyzing in background (step 1 of 2)" / "Solving camera" over the analysed layer, and a +/// warning when the solve failed (unless Hide Warning Banner). +fn banner(app: &mut EffectcraftApp, painter: &egui::Painter, map: &ViewerMap, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3) { + let comp = ectx.comp; + let time = ectx.time; + let cid = ectx.comp_id; + let draw = |app: &mut EffectcraftApp, layer: &Layer, text: String, color: Color32, id: &str| { + let m = l2c(ectx, layer); + let (w, h) = effectcraft_engine::render::source_size(&app.session.project, layer); + let c = m.apply(gv2(w as f64 / 2.0, h as f64 / 2.0)); + let at = map.to_screen([c.x, c.y]); + let galley = painter.layout_no_wrap(text.clone(), Tokens::medium(13.0), Color32::WHITE); + let r = Rect::from_center_size(at, galley.size() + vec2(28.0, 14.0)).intersect(map.area); + painter.rect_filled(r, 3.0, color); + painter.galley(pos2(r.center().x - galley.size().x / 2.0, r.center().y - galley.size().y / 2.0), galley, Color32::WHITE); + app.auto.add(id, r, &text); + }; + if let Some((cc, cl, _)) = app.session.camera_target() + && cc == cid + && let Some(pr) = app.session.camera_progress() + && let Some(layer) = comp.layer(cl) + && layer.is_active_at(time) + { + draw(app, layer, pr.banner(), Color32::from_rgba_unmultiplied(0x1d, 0x4f, 0x9c, 0xe0), "viewer.cameraBanner"); + return; + } + // A failed solve: tracks without a solve while nothing is running or queued. + for l in comp.layers.iter().filter(|l| l.is_active_at(time)) { + let Some(fx) = l.effects() else { continue }; + for g in fx.groups().filter(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == ct::ID)) { + let params = effectcraft_engine::camera_track::static_params(g); + let hide = effectcraft_engine::effects::warp_stab::param(¶ms, "advanced/hideWarningBanner").is_some_and(Value::as_bool); + let solve_key = matches!(g.prop(ct::SOLVE_KEY).map(|p| &p.value), Some(Value::Str(s)) if !s.is_empty()); + let failed = solve_key && ct::tracks(¶ms).is_some() && ct::solve(¶ms).is_none(); + if failed && !hide && !app.session.camera_pending.iter().any(|(_, ll, u)| *ll == l.id && *u == g.uid) { + draw(app, l, "Unable to solve camera".into(), Color32::from_rgba_unmultiplied(0x9c, 0x2a, 0x1d, 0xe0), "viewer.cameraWarning"); + return; + } + } + } +} + +/// Effect Controls: Analyze / Cancel, Create Camera and the solve report. +#[allow(clippy::too_many_arguments)] +pub fn editor( + app: &mut EffectcraftApp, + ui: &mut egui::Ui, + p: &egui::Painter, + layer: &Layer, + g: &PropGroup, + r: Rect, + actions: &mut Vec<(String, serde_json::Value)>, +) { + let t = app.tokens; + let running = app.session.camera_target().is_some_and(|(_, l, u)| l == layer.id && u == g.uid) && app.session.is_camera_analyzing(); + let busy = app.session.is_camera_analyzing(); + let me = json!({"layer": layer.id.0, "effect": g.uid}); + let b1 = Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 4.0), vec2(84.0, 22.0)); + let b2 = Rect::from_min_size(pos2(b1.max.x + 8.0, b1.min.y), vec2(84.0, 22.0)); + let b3 = Rect::from_min_size(pos2(b2.max.x + 8.0, b1.min.y), vec2(110.0, 22.0)); + if widgets::text_button(ui, b1, "Analyze", false, &t, egui::Id::new(("ct-analyze", g.uid))).clicked() && !busy { + actions.push(("camera.analyze".into(), me.clone())); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.analyze", g.uid), b1, "Analyze"); + if running { + if widgets::text_button(ui, b2, "Cancel", false, &t, egui::Id::new(("ct-cancel", g.uid))).clicked() { + actions.push(("camera.cancel".into(), json!({}))); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.cancel", g.uid), b2, "Cancel"); + } + let params = effectcraft_engine::camera_track::static_params(g); + let solve = ct::solve(¶ms); + if solve.is_some() { + if widgets::text_button(ui, b3, "Create Camera", false, &t, egui::Id::new(("ct-camera", g.uid))).clicked() { + actions.push(("camera.create".into(), me.clone())); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.createCamera", g.uid), b3, "Create Camera"); + } + let status = match app.session.camera_progress() { + Some(pr) if running => pr.banner(), + _ => match &solve { + Some(sv) => format!("Method Used: {} Average Error: {:.2} pixels", sv.method_used.label(), sv.average_error), + None if ct::tracks(¶ms).is_some() && app.session.camera_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => { + "Waiting to solve".into() + } + None if ct::tracks(¶ms).is_some() => "Unable to solve camera: try another Solve Method or Shot Type".into(), + None if app.session.camera_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => "Waiting to analyze".into(), + None => "Not analyzed: click Analyze".into(), + }, + }; + p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim); + if let Some(sv) = &solve { + let f = sv.focal(); + p.text( + pos2(r.min.x + 24.0, r.min.y + 56.0), + Align2::LEFT_CENTER, + format!("{} solved points · Horizontal Angle of View {:.1}°", sv.points.len(), sv.hfov(f)), + Tokens::ui(11.5), + t.text_faint, + ); + } +} + +/// Height of [`editor`]. +pub const EDITOR_HEIGHT: f32 = 68.0; diff --git a/crates/ui-egui/src/panels/effect_controls.rs b/crates/ui-egui/src/panels/effect_controls.rs index 03afd94c..90d62892 100644 --- a/crates/ui-egui/src/panels/effect_controls.rs +++ b/crates/ui-egui/src/panels/effect_controls.rs @@ -574,6 +574,7 @@ fn editor_height(effect: &str, width: f32) -> f32 { match effect { "ec.color.curves" => 34.0 + curves_size(width) + 26.0, effectcraft_engine::effects::warp_stab::ID => 50.0, + effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::EDITOR_HEIGHT, effectcraft_engine::effects::roto::ID => 50.0, "ec.color.levels" | "ec.color.levelsic" => 34.0 + 80.0 + 58.0, "ec.color.autolevels" | "ec.color.autocontrast" | "ec.color.autocolor" => 24.0 + 80.0 + 12.0, @@ -950,6 +951,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { match effect.as_str() { "ec.color.curves" => curves_editor(app, ui, &bp, &layer, g, &ectx, er, &mut actions), effectcraft_engine::effects::warp_stab::ID => warp_editor(app, ui, &bp, &layer, g, er, &mut actions), + effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::editor(app, ui, &bp, &layer, g, er, &mut actions), effectcraft_engine::effects::roto::ID => roto_editor(app, ui, &bp, &layer, g, er, &mut actions), "ec.color.levels" | "ec.color.levelsic" => levels_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions), _ => histogram_only(app, ui, &bp, g, er), diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index e8c13dce..3cf2dae1 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -1,6 +1,7 @@ //! Panel bodies. pub mod anim_tools; +pub mod camera_tracker_ui; pub mod comp_settings; pub mod dialogs; pub mod dialogs_3d; diff --git a/crates/ui-egui/src/panels/tracker.rs b/crates/ui-egui/src/panels/tracker.rs index d1f456b2..364e114e 100644 --- a/crates/ui-egui/src/panels/tracker.rs +++ b/crates/ui-egui/src/panels/tracker.rs @@ -130,11 +130,14 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let cur = current(app); let can_track = has_comp && source.is_some(); - // Track Camera / Warp Stabilizer (not available) and Track Motion / Stabilize Motion. + // Track Camera / Warp Stabilizer and Track Motion / Stabilize Motion. let r1 = Rect::from_min_size(pos2(x0, y), vec2(half, 22.0)); let r2 = Rect::from_min_size(pos2(x0 + half + 6.0, y), vec2(half, 22.0)); let src_p = source.map(|l| json!({"layer": l.0})).unwrap_or_else(|| json!({})); - button(app, ui, r1, "Track Camera", false, "tracker.trackCamera"); + let can_camera = app.session.is_enabled("track.camera") && !app.session.is_camera_analyzing(); + if button(app, ui, r1, "Track Camera", can_camera, "tracker.trackCamera") { + run(app, &ctx, "track.camera", json!({})); + } let can_warp = app.session.is_enabled("track.warpStabilizer") && !app.session.is_warp_analyzing(); if button(app, ui, r2, "Warp Stabilizer", can_warp, "tracker.warpStabilizer") { run(app, &ctx, "track.warpStabilizer", json!({})); diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index d847edb8..c9dd70db 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -1144,6 +1144,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { app.session.open_comp(*item); } } + // 3D Camera Tracker points, targets and their menu (on top of the viewer's gestures while + // the effect is selected), and its analysis banner. + super::camera_tracker_ui::viewer_hook(app, ui, &painter, &map, &ectx, &|c, l| l2c(c, l).0); // Effect point controls and crosshair/eyedropper picks (on top of the viewer's gestures). crate::panels::effect_controls::viewer_hook(app, ui, &painter, &map, &ectx, &|c, l| l2c(c, l).0); diff --git a/crates/ui-egui/tests/ui_camera_tracker.rs b/crates/ui-egui/tests/ui_camera_tracker.rs new file mode 100644 index 00000000..45374fba --- /dev/null +++ b/crates/ui-egui/tests/ui_camera_tracker.rs @@ -0,0 +1,175 @@ +//! Headless checks for the 3D Camera Tracker UI (egui_kittest, UI logic only): the viewer's track +//! points and selection, the Effect Controls buttons and the Tracker panel's Track Camera button. +//! The solve is a synthetic one written into the effect (the analysis itself is tested in the +//! engine and track crates). + +use effectcraft_engine::Session; +use effectcraft_engine::effects::camera_tracker as ct; +use effectcraft_engine::track::camtrack::{CameraSolve, ShotType, SolveMethod, SolvedFrame, SolvedPoint}; +use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::dock::PanelKind; +use egui_kittest::Harness; +use serde_json::json; + +const W: f64 = 640.0; +const H: f64 = 360.0; + +fn app() -> (EffectcraftApp, u64, u64) { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Shot", "width": W, "height": H, "duration": 2})).unwrap(); + let plate = s.execute("layer.newSolid", json!({"name": "Plate", "color": "#406080"})).unwrap()["layer"].as_u64().unwrap(); + let r = s.execute("effect.apply", json!({"effect": ct::ID, "layers": [plate]})).unwrap(); + let uid = r["effects"][0].as_u64().unwrap(); + // A synthetic solve: a static camera (focal 600) looking at a 6 × 4 grid of points on z = 1. + let frames = 60; + let solve = CameraSolve { + version: 1, + size: [W, H], + start: 0.0, + frame_duration: 1.0 / 30.0, + shot: ShotType::FixedAngle, + method_used: SolveMethod::Typical, + average_error: 0.21, + frames: vec![SolvedFrame { rot: [0.0; 3], center: [0.0; 3], focal: 600.0, solved: true }; frames], + points: (0..24) + .map(|i| SolvedPoint { + id: i, + pos: [((i % 6) as f64 - 2.5) * 0.15, ((i / 6) as f64 - 1.5) * 0.12, 1.0 + 0.01 * (i % 3) as f64], + error: 0.2, + first: 0, + last: frames as u32 - 1, + }) + .collect(), + ground: None, + }; + s.execute("prop.set", json!({"layer": plate, "path": "effects/#1/solve", "value": solve.to_json()})).unwrap(); + s.camera_pending.clear(); + s.state.selected_props = vec![(effectcraft_engine::project::LayerId(plate), uid)]; + s.execute("layer.select", json!({"layers": [plate]})).unwrap(); + s.state.selected_props = vec![(effectcraft_engine::project::LayerId(plate), uid)]; + (EffectcraftApp::new(s), plate, uid) +} + +fn ids(h: &Harness<'_, EffectcraftApp>) -> Vec { + h.state().auto.previous.iter().chain(h.state().auto.elements.iter()).map(|e| e.id.clone()).collect() +} + +fn rect(h: &Harness<'_, EffectcraftApp>, id: &str) -> egui::Rect { + let e = h.state().auto.find(id).unwrap_or_else(|| panic!("no {id}")); + egui::Rect::from_min_size(egui::pos2(e.rect[0], e.rect[1]), egui::vec2(e.rect[2], e.rect[3])) +} + +fn click(h: &mut Harness<'_, EffectcraftApp>, at: egui::Pos2, modifiers: egui::Modifiers) { + h.input_mut().events.push(egui::Event::ModifiersChanged(modifiers)); + h.input_mut().events.push(egui::Event::PointerMoved(at)); + h.input_mut().events.push(egui::Event::PointerButton { pos: at, button: egui::PointerButton::Primary, pressed: true, modifiers }); + h.input_mut().events.push(egui::Event::PointerButton { pos: at, button: egui::PointerButton::Primary, pressed: false, modifiers }); + h.run_steps(3); + h.input_mut().events.push(egui::Event::ModifiersChanged(Default::default())); + h.step(); +} + +#[test] +fn viewer_points_selection_and_effect_controls() { + let (a, plate, uid) = app(); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).build_eframe(|_| a); + h.state_mut().show_panel(PanelKind::Composition); + h.run_steps(4); + let all = ids(&h); + assert!(all.iter().any(|i| i == "viewer.cameraTracker"), "track point overlay registered"); + let mut pts: Vec<&String> = all.iter().filter(|i| i.starts_with("viewer.cameraTracker.point.")).collect(); + pts.sort(); + pts.dedup(); + let n = pts.len(); + assert_eq!(n, 24, "every visible point is drawn"); + // Click a point: it is selected; Shift-click another adds it. + let r = rect(&h, "viewer.cameraTracker.point.7"); + click(&mut h, r.center(), Default::default()); + assert_eq!(h.state().session.state.camera_points, vec![7]); + let r = rect(&h, "viewer.cameraTracker.point.9"); + click(&mut h, r.center(), egui::Modifiers::SHIFT); + assert_eq!(h.state().session.state.camera_points, vec![7, 9]); + // Marquee over the first row selects its six points. + let (a0, a5) = (rect(&h, "viewer.cameraTracker.point.0"), rect(&h, "viewer.cameraTracker.point.5")); + let from = a0.left_top() - egui::vec2(6.0, 6.0); + let to = a5.right_bottom() + egui::vec2(6.0, 6.0); + h.input_mut().events.push(egui::Event::PointerMoved(from)); + h.input_mut().events.push(egui::Event::PointerButton { pos: from, button: egui::PointerButton::Primary, pressed: true, modifiers: Default::default() }); + h.step(); + for i in 1..=8 { + h.input_mut().events.push(egui::Event::PointerMoved(from + (to - from) * (i as f32 / 8.0))); + h.step(); + } + h.input_mut().events.push(egui::Event::PointerButton { pos: to, button: egui::PointerButton::Primary, pressed: false, modifiers: Default::default() }); + h.run_steps(3); + let mut sel = h.state().session.state.camera_points.clone(); + sel.sort(); + assert_eq!(sel, vec![0, 1, 2, 3, 4, 5]); + // The right-click menu's Create Solid and Camera (via its command). + let ctx = h.ctx.clone(); + effectcraft_ui_egui::menus::invoke(h.state_mut(), &ctx, "camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + let comp = h.state().session.active_comp().unwrap(); + assert!(comp.layers.iter().any(|l| l.name == "3D Tracker Camera" && l.is_camera())); + assert!(comp.layers.iter().any(|l| l.name.starts_with("Track Solid"))); + + // Effect Controls: Analyze and Create Camera. + h.state_mut().session.execute("layer.select", json!({"layers": [plate]})).unwrap(); + h.state_mut().show_panel(PanelKind::EffectControls); + h.run_steps(4); + let all = ids(&h); + for want in [format!("effectControls.cameraTracker.{uid}.analyze"), format!("effectControls.cameraTracker.{uid}.createCamera")] { + assert!(all.contains(&want), "missing {want}"); + } + // Tracker panel: Track Camera is available for the selected layer. + h.state_mut().show_panel(PanelKind::Tracker); + h.run_steps(3); + assert!(ids(&h).iter().any(|i| i == "tracker.trackCamera")); + assert!(h.state().session.is_enabled("track.camera")); +} + +/// Step the UI until background frame renders have landed. +fn settle(h: &mut Harness<'_, EffectcraftApp>) { + for _ in 0..600 { + h.step(); + if h.state().frames.inflight() == 0 && h.state().frames.last_ms.lock().map(|v| *v > 0.0).unwrap_or(false) { + break; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + for _ in 0..4 { + h.step(); + } +} + +#[test] +#[ignore = "analyses the demo's 3D Showcase and renders with wgpu; run with --ignored (set EC_SNAPSHOT_DIR to keep PNGs)"] +fn snapshot_showcase_track() { + let dir = std::env::var("EC_SNAPSHOT_DIR").ok(); + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + s.execute("comp.new", json!({"name": "Camera Track", "width": 1920, "height": 1080, "duration": 4, "frameRate": 29.97})).unwrap(); + s.execute("layer.addItem", json!({"item": "3D Showcase"})).unwrap(); + s.execute("layer.select", json!({"layers": ["#1"]})).unwrap(); + s.execute("track.camera", json!({"layer": "#1", "wait": true})).unwrap(); + s.execute("time.set", json!({"time": 2.0})).unwrap(); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).with_pixels_per_point(1.0).wgpu().build_eframe(|_| EffectcraftApp::new(s)); + h.state_mut().show_panel(PanelKind::Composition); + settle(&mut h); + let pts: Vec = ids(&h).into_iter().filter(|i| i.starts_with("viewer.cameraTracker.point.")).collect(); + assert!(pts.len() > 10, "{}", pts.len()); + // Hover between points to show a target, then select three and show the selection's target. + let r = rect(&h, &pts[0]); + h.input_mut().events.push(egui::Event::PointerMoved(r.center() + egui::vec2(12.0, 12.0))); + h.run_steps(2); + if let Some(d) = &dir { + h.render().expect("render").save(format!("{d}/camera_tracker_hover.png")).unwrap(); + } + let sel: Vec = pts.iter().take(5).filter_map(|i| i.rsplit('.').next()?.parse().ok()).collect(); + h.state_mut().session.execute("camera.selectPoints", json!({"points": sel})).unwrap(); + h.state_mut().session.execute("camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + h.state_mut().session.execute("layer.select", json!({"layers": ["#3"]})).unwrap(); + settle(&mut h); + if let Some(d) = &dir { + h.render().expect("render").save(format!("{d}/camera_tracker_solid.png")).unwrap(); + } +} diff --git a/docs/agents.md b/docs/agents.md index f49ddc22..8689b9da 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -147,6 +147,34 @@ effectcraft-cli exec warp.analyze '{"layer":"#1","wait":true}' shaky.ecproj --sa effectcraft-cli exec warp.status '{"layer":"#1"}' shaky.ecproj --json ``` +### 3D Camera Tracker + +Animation ▸ Track Camera and the Tracker panel's button (`track.camera {layer?, shotType?: +fixed|variable|specify, aov?, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}`) apply +Effect ▸ Perspective ▸ 3D Camera Tracker (or reuse the layer's) and analyse it in the background: +"Analyzing in background (step 1 of 2)" tracks features, "Solving camera" solves. +`camera.analyze {layer?, effect?, wait?}` re-runs it, `camera.cancel` stops it without writing +anything. `camera.solveStatus` reports `progress`/`banner`, `analyzed`, `solved`, `methodUsed`, +`averageError` (pixels), `focalLength`, `horizontalAngleOfView`, the current frame's `camera` +(position, orientation, zoom) and `groundPlane`. `camera.points {time?}` lists the solved points +visible now with their `id`, `comp` position, `depth`, `world` position and `error`. + +Select points with `camera.selectPoints {points, add?, toggle?}` (the viewer's click / Shift-click / +marquee), then the right-click menu's commands: `camera.setGroundPlane {points?}` (Set Ground Plane +and Origin), `camera.createFromSolve {kind: text|solid|null|shadowCatcher, points? | target: +{center, normal, size?}, multiple?}` (Create Text / Solid / Null / Shadow Catcher, Camera and Light, +Create Multiple …), `camera.deletePoints {points?, wait?}` (re-solves; with Auto-delete Points +Across Time the same feature's other tracks go too) and `camera.create` (the Create Camera button). +Each is one undo step; the first create adds the one-node "3D Tracker Camera" keyed on every frame. +Changing the layer's frames clears the analysis; changing `effects/#1/shotType`, +`effects/#1/horizontalAngleOfView`, `effects/#1/advanced/solveMethod` re-solves the stored tracks. + +```sh +effectcraft-cli run shot.ecproj track.camera '{"layer":"#1","wait":true}' \ + camera.points '{}' --json +effectcraft-cli run shot.ecproj camera.createFromSolve '{"kind":"solid","points":[12,40,77]}' --save +``` + ### Roto Brush & Refine Edge The Roto Brush tool (Alt+W cycles Roto Brush / Refine Edge) paints in the Layer panel; agents use diff --git a/docs/architecture.md b/docs/architecture.md index 8e9ed63c..67d94fb9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -219,6 +219,22 @@ auto-scale; cached per analysis and settings) and warps each frame; Synthesize E borders from neighbouring frames read with `EffectHost::self_at`. The effect lives in the effects crate, hence the `effects → track` edge. +**3D Camera Tracker** (`effects::camera_tracker`, `engine::camera_track`, `effectcraft-track`'s +`camtrack`): `camera.analyze` / `track.camera` render the layer's input to the effect on a +background thread and run structure from motion in two steps. Step 1 follows Shi–Tomasi features +through the clip with pyramidal Lucas–Kanade (forward–backward checked, re-detected where the frame +has none) into long 2D tracks. Step 2 solves the camera: parallax keyframes; a two-view start from +the essential matrix (normalised 8-point in RANSAC) or a planar homography decomposition; incremental +resection and triangulation; a sparse Levenberg–Marquardt bundle adjustment (Schur complement over +the points, Huber loss) with the focal length fixed, shared, or per frame; a log-spaced and +golden-section focal search for Fixed Angle of View / Variable Zoom; a rotation-only model for tripod +pans, chosen by Auto Detect when it explains the tracks as well. Tracks and solve are stored as JSON +in hidden effect parameters with keys: changing the layer's frames clears both, changing Shot Type, +Angle of View, Solve Method or deleting points only re-solves. The solve's canonical frame maps to +comp space so the first frame's camera is the default comp camera (or so a chosen ground plane is +the X-Z plane at the origin); `camera.createFromSolve` keys a one-node "3D Tracker Camera" on every +frame and places text, solids, nulls or a shadow catcher and light on the target plane. + **Roto Brush & Refine Edge** (`effectcraft_track::roto`, `effects::roto`, `engine::roto`): strokes (foreground, background, Refine Edge) are stored as JSON in the effect's hidden Strokes parameter with the base frame and segmentation span. Each frame is segmented by graph cut diff --git a/docs/parity.md b/docs/parity.md index 497b3858..0f3103ac 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -15,7 +15,7 @@ rendered, does not count as done. | P1 (professional daily use) | ≈ 38% | | P2 (long tail) | ≈ 17% | | Features done / partial / missing | 18 / 60 / 14 of 92 | -| **Effects** | **297 of 298** After Effects 2026 effects by name (99.7%, after M9.11 and Warp Stabilizer; was 257); only the 3D Camera Tracker is left | +| **Effects** | **298 of 298** After Effects 2026 effects by name (100%, after M9.11, Warp Stabilizer and the 3D Camera Tracker; was 257) | | Remaining work | ≈ **167 agent-hours** (139 h of features + ≈ 20% for 1:1 polish against After Effects) | | Wall-clock estimate | ≈ **42–50 hours** with five agents working in parallel and one integrating; ≈ 31–35 h for P0 + P1 only | @@ -73,13 +73,12 @@ library groundwork for mask tracking and Warp Stabilizer. | Remaining (audit-scale agent-hours) | ≈ 167 | ≈ 100 | | Wall clock, five parallel agents | 42–50 h | **≈ 8–10 h at today's measured pace; ≈ 25 h by the audit's conservative figures** | -What is left is concentrated in large systems: the 3D camera tracker, -the Warp Stabilizer and mask-tracking UI on top of +What is left is concentrated in large systems: the Warp Stabilizer and mask-tracking UI on top of the new tracking library, and wasm threads. (The missing effect categories, 3D Channel, Immersive Video and OCIO, landed in M9.11; the scripting object model in M14.4; Roto Brush and Refine Edge in M6.7; native macOS menus, the Composition Flowchart, Timeline column/search depth, the Home screen and every After Effects workspace in the UI-polish wave; PSD/SVG import, WebM/WAV/AIFF output and -the disk cache in the formats wave.) +the disk cache in the formats wave; the 3D Camera Tracker in M12.6.) ## By area @@ -95,18 +94,18 @@ the disk cache in the formats wave.) | Animation | 65% | 10.4 | puppet, Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8) | | Text | ≈ 86% | 1.3 | vertical type is basic (upright characters, no tate-chu-yoko), no OpenType feature panel, no extruded strokes (extruded, bevelled text landed in M7.6); per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API landed (M9.9–M9.10) | | Web | 60% | 2.5 | threads, browser storage, audio | -| 3D | 72% | 12.0 | 3D camera tracker, multi-view layouts, Advanced 3D motion blur and blend modes, cameras/lights from models; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) landed in M7.4–M7.6 | -| Effects | ≈ 75% | 5.0 | GPU versions of more effects and the 3D Camera Tracker (every other After Effects effect exists since M9.11 and M12.5) | +| 3D | 76% | 10.0 | multi-view layouts, Advanced 3D motion blur and blend modes, cameras/lights from models; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) landed in M7.4–M7.6 | +| Effects | ≈ 75% | 5.0 | GPU versions of more effects (every After Effects effect exists since M9.11, M12.5 and M12.6) | | Interface | 70% | 6.0 | Timeline horizontal scrolling of many columns, a richer Learn area (native macOS menu bar, Timeline columns/search/reveal-add, Home screen with recent projects and all AE workspaces landed; viewer rulers/snapping/channels/snapshots landed in M0.13) | | Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves | | Masks & roto | 68% | 6.0 | variable-width mask feather points, Roto Brush's learned (3.0) segmentation model (mask tracking and Mask Interpolation landed in M6.6; Roto Brush & Refine Edge with graph-cut segmentation, flow propagation, edge matting, decontamination and Freeze in M6.7) | | Preview | 62% | 5.0 | GPU 3D and adjustment layers (persistent disk cache with the blue cache bar landed; region of interest, snapshots, exposure and Fast Previews landed in M0.13) | -| Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking and Subspace Warp's mesh warp (Rolling Shutter Repair landed in M9.11; point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) | +| Tracking | ≈ 85% | 2.5 | face tracking, Subspace Warp's mesh warp, lens distortion in the camera solve (Rolling Shutter Repair landed in M9.11; point tracker, mask tracking, Warp Stabilizer and the 3D Camera Tracker landed in M6.x / M12.5 / M12.6) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | ## Effects still missing -The 3D Camera Tracker (built on the tracking library by other work). Boris FX +None: the 3D Camera Tracker landed in M12.6. Boris FX Mocha and Cineware are third-party and not counted. ### Added in M9.11 (39 effects)