308 lines
15 KiB
Rust
308 lines
15 KiB
Rust
//! The 3D Camera Tracker end to end through the commands, on synthetic footage rendered in-test:
|
||
//! textured planes (a floor, a back wall and a box face) seen through a known moving camera.
|
||
|
||
use effectcraft_effects::camera_tracker as ct;
|
||
use effectcraft_geom::vec3;
|
||
use effectcraft_project::LayerId;
|
||
use effectcraft_raster::Image;
|
||
use effectcraft_render::EvalCtx;
|
||
use effectcraft_render::three_d::camera::active_camera;
|
||
use effectcraft_track::camtrack::linalg::{self, M3, V3};
|
||
use serde_json::{Value, json};
|
||
|
||
use crate::Session;
|
||
use crate::tests_track::{H, W, frame_time, setup, texture};
|
||
|
||
const F: f64 = 260.0;
|
||
|
||
/// The true camera on frame `f`: (world → camera rotation, centre).
|
||
fn true_camera(f: u32) -> (M3, V3) {
|
||
let t = f as f64 / 24.0;
|
||
let c = [-1.0 + 2.0 * t, -0.3 + 0.2 * (t * 3.0).sin(), 0.5 * t];
|
||
let fwd = linalg::normalize(linalg::sub([0.2 * t, 0.4, 8.0], c));
|
||
let right = linalg::normalize(linalg::cross([0.0, 1.0, 0.0], fwd));
|
||
([right, linalg::cross(fwd, right), fwd], c)
|
||
}
|
||
|
||
/// Planes: (point, normal, u axis, v axis, half extent or 0 = infinite).
|
||
const PLANES: [(V3, V3, V3, V3, f64); 3] = [
|
||
([0.0, 1.5, 0.0], [0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0], 0.0),
|
||
([0.0, 0.0, 10.0], [0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0], 0.0),
|
||
([-1.2, 0.2, 6.0], [0.3, 0.0, -1.0], [0.957_826, 0.0, 0.287_348], [0.0, 1.0, 0.0], 1.1),
|
||
];
|
||
|
||
fn scene_frame(f: u32) -> Image {
|
||
let (r, c) = true_camera(f);
|
||
let mut img = Image::new(W, H);
|
||
for y in 0..H {
|
||
for x in 0..W {
|
||
let d = linalg::mtv(&r, [(x as f64 + 0.5 - W as f64 / 2.0) / F, (y as f64 + 0.5 - H as f64 / 2.0) / F, 1.0]);
|
||
let mut best = (f64::INFINITY, 0.2f32);
|
||
for (i, (p0, n, ua, va, ext)) in PLANES.iter().enumerate() {
|
||
let den = linalg::dot(*n, d);
|
||
if den.abs() < 1e-9 {
|
||
continue;
|
||
}
|
||
let tt = linalg::dot(*n, linalg::sub(*p0, c)) / den;
|
||
if tt <= 0.1 || tt >= best.0 {
|
||
continue;
|
||
}
|
||
let hit = linalg::add(c, linalg::scale(d, tt));
|
||
let (u, v) = (linalg::dot(linalg::sub(hit, *p0), *ua), linalg::dot(linalg::sub(hit, *p0), *va));
|
||
if *ext > 0.0 && (u.abs() > *ext || v.abs() > *ext) {
|
||
continue;
|
||
}
|
||
best = (tt, texture(u * 40.0, v * 40.0, 11 + i as u32));
|
||
}
|
||
img.set(x, y, [best.1, best.1, best.1, 1.0]);
|
||
}
|
||
}
|
||
img
|
||
}
|
||
|
||
/// The comp with the clip tracked (blocking) with `params`.
|
||
fn tracked(params: Value) -> (Session, LayerId) {
|
||
let (mut s, clip, _) = setup(scene_frame);
|
||
s.execute("layer.select", json!({"layers": [clip.0]})).unwrap();
|
||
assert!(s.is_enabled("track.camera"));
|
||
let mut p = params;
|
||
p["wait"] = json!(true);
|
||
let r = s.execute_checked("track.camera", p).unwrap();
|
||
assert_eq!(r["frames"], 25, "{r}");
|
||
assert!(s.camera_job.is_none());
|
||
(s, clip)
|
||
}
|
||
|
||
fn status(s: &mut Session) -> Value {
|
||
s.execute("camera.solveStatus", json!({})).unwrap()
|
||
}
|
||
|
||
#[test]
|
||
fn track_camera_solves_and_creates_layers_on_the_plane() {
|
||
let (mut s, clip) = tracked(json!({}));
|
||
let st = status(&mut s);
|
||
assert_eq!(st["solved"], true, "{st}");
|
||
let err = st["averageError"].as_f64().unwrap();
|
||
assert!(err < 0.5, "average error {err}");
|
||
let f = st["focalLength"].as_f64().unwrap();
|
||
assert!((f - F).abs() / F < 0.05, "focal {f}");
|
||
assert_ne!(st["methodUsed"], "Tripod Pan");
|
||
// Method Used / Average Error are reported in the effect.
|
||
let g = s.active_comp().unwrap().layer(clip).unwrap().effects().unwrap().groups().next().unwrap().clone();
|
||
assert!(matches!(g.prop(ct::METHOD_USED).map(|p| &p.value), Some(effectcraft_keyframe::Value::Str(m)) if !m.is_empty()));
|
||
|
||
// Pick floor points on frame 12 (their 3D positions lie on y = 1.5 in the true scene: the
|
||
// ones seen below the horizon at the bottom of the frame).
|
||
s.execute("time.set", json!({"time": frame_time(12).seconds()})).unwrap();
|
||
let pts = s.execute("camera.points", json!({})).unwrap();
|
||
let list = pts["points"].as_array().unwrap();
|
||
assert!(list.len() > 30, "{}", list.len());
|
||
let floor: Vec<u64> = list.iter().filter(|p| p["comp"][1].as_f64().unwrap() > H as f64 * 0.8).map(|p| p["id"].as_u64().unwrap()).take(8).collect();
|
||
assert!(floor.len() >= 3, "{floor:?}");
|
||
s.execute("camera.selectPoints", json!({"points": floor})).unwrap();
|
||
assert_eq!(s.state.camera_points.len(), floor.len());
|
||
|
||
// Create Solid and Camera: one undo step, camera + solid.
|
||
let r = s.execute_checked("camera.createFromSolve", json!({"kind": "solid"})).unwrap();
|
||
let cam = LayerId(r["camera"].as_u64().unwrap());
|
||
let solid = LayerId(r["layers"][0].as_u64().unwrap());
|
||
let comp = s.active_comp().unwrap().clone();
|
||
assert!(comp.layer(cam).unwrap().is_camera());
|
||
assert_eq!(comp.layer(cam).unwrap().name, "3D Tracker Camera");
|
||
assert_eq!(comp.layers[0].id, cam);
|
||
let sl = comp.layer(solid).unwrap();
|
||
assert!(sl.switches.three_d);
|
||
assert!(sl.name.starts_with("Track Solid"));
|
||
// The camera is keyed on every frame.
|
||
assert_eq!(comp.layer(cam).unwrap().props.prop("transform/position").unwrap().keys.len(), 25);
|
||
|
||
// Every solved point projects through the created camera exactly where the solve puts it,
|
||
// and onto its tracked position (the footage fills the comp, so layer pixels are comp
|
||
// pixels): within 1 px for 95 % of the observations (the rest are the solve's outliers:
|
||
// features on occlusion edges), 0.5 px on average.
|
||
let cid = s.active_comp_id().unwrap();
|
||
let uid = g.uid;
|
||
let pl = crate::camera_track::placed(&s.project, cid, clip, uid).unwrap();
|
||
let tracks = ct::tracks(&crate::camera_track::static_params(&g)).unwrap();
|
||
let mut errs = vec![];
|
||
for k in [0u32, 6, 12, 18, 24] {
|
||
let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(k));
|
||
let camst = active_camera(&ctx);
|
||
for sp in pl.solve.visible(k as usize) {
|
||
let Some(obs) = tracks.track(sp.id).and_then(|t| t.at(k)) else { continue };
|
||
let w = pl.to_world(sp.pos);
|
||
let q = camst.project(W as f64, H as f64, vec3(w[0], w[1], w[2])).unwrap();
|
||
let own = pl.solve.frames[k as usize].project(pl.solve.size, sp.pos).unwrap();
|
||
assert!((q.x - own[0]).hypot(q.y - own[1]) < 0.01, "camera layer disagrees with the solve");
|
||
errs.push((q.x - obs[0]).hypot(q.y - obs[1]));
|
||
}
|
||
}
|
||
assert!(errs.len() > 100, "{}", errs.len());
|
||
errs.sort_by(f64::total_cmp);
|
||
let p95 = errs[errs.len() * 95 / 100];
|
||
let mean = errs.iter().sum::<f64>() / errs.len() as f64;
|
||
assert!(p95 < 1.0 && mean < 0.5, "p95 {p95} px, mean {mean} px");
|
||
|
||
// The solid lies on the selected points' plane.
|
||
let tr = sl.props.sub("transform").unwrap();
|
||
let pos = tr.get("position").unwrap().value.as_vec3();
|
||
let ori = tr.get("orientation").unwrap().value.as_vec3();
|
||
let normal = linalg::mv(&linalg::from_euler_xyz(ori), [0.0, 0.0, -1.0]);
|
||
let size = r["target"]["size"].as_f64().unwrap();
|
||
for id in &s.state.camera_points {
|
||
let w = pl.to_world(pl.solve.point(*id).unwrap().pos);
|
||
let d = linalg::dot(linalg::sub(w, pos), normal).abs();
|
||
assert!(d < 0.05 * size.max(1.0) + 1.0, "point {id} is {d} px off the plane");
|
||
}
|
||
// The solid faces the camera.
|
||
let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(12));
|
||
let eye = active_camera(&ctx).eye;
|
||
assert!(linalg::dot(normal, linalg::sub([eye.x, eye.y, eye.z], pos)) > 0.0);
|
||
|
||
// Undo removes both layers; redo brings them back.
|
||
let before = comp.layers.len();
|
||
assert!(s.undo());
|
||
assert_eq!(s.active_comp().unwrap().layers.len(), before - 2);
|
||
assert!(s.redo());
|
||
assert_eq!(s.active_comp().unwrap().layers.len(), before);
|
||
|
||
// A second create reuses the camera: Create Multiple Nulls and Camera.
|
||
let r = s.execute_checked("camera.createFromSolve", json!({"kind": "null", "multiple": true})).unwrap();
|
||
assert_eq!(r["camera"].as_u64().unwrap(), cam.0);
|
||
assert_eq!(r["layers"].as_array().unwrap().len(), floor.len());
|
||
// Text and Shadow Catcher (+ light).
|
||
let r = s.execute_checked("camera.createFromSolve", json!({"kind": "text"})).unwrap();
|
||
assert_eq!(r["layers"].as_array().unwrap().len(), 1);
|
||
let r = s.execute_checked("camera.createFromSolve", json!({"kind": "shadowCatcher"})).unwrap();
|
||
let ids: Vec<LayerId> = r["layers"].as_array().unwrap().iter().map(|v| LayerId(v.as_u64().unwrap())).collect();
|
||
let comp = s.active_comp().unwrap();
|
||
assert!(comp.layer(ids[0]).unwrap().name.starts_with("Shadow Catcher"));
|
||
assert_eq!(comp.layer(ids[0]).unwrap().props.prop("materialOptions/acceptsShadows").unwrap().value.as_enum(), 2);
|
||
assert!(comp.layer(ids[1]).unwrap().is_light());
|
||
assert!(s.execute_checked("camera.createFromSolve", json!({"kind": "teapot"})).is_err());
|
||
|
||
// Set Ground Plane and Origin: the target becomes the origin; a null created there sits at
|
||
// (0, 0, 0) lying on the X-Z plane (its front facing up, −y).
|
||
s.execute_checked("camera.setGroundPlane", json!({})).unwrap();
|
||
assert_eq!(status(&mut s)["groundPlane"], true);
|
||
let r = s.execute_checked("camera.createFromSolve", json!({"kind": "null"})).unwrap();
|
||
let null = LayerId(r["layers"][0].as_u64().unwrap());
|
||
let tr = s.active_comp().unwrap().layer(null).unwrap().props.sub("transform").unwrap().clone();
|
||
let pos = tr.get("position").unwrap().value.as_vec3();
|
||
assert!(linalg::norm(pos) < 1e-6, "{pos:?}");
|
||
let up = linalg::mv(&linalg::from_euler_xyz(tr.get("orientation").unwrap().value.as_vec3()), [0.0, 0.0, -1.0]);
|
||
assert!(linalg::norm(linalg::sub(up, [0.0, -1.0, 0.0])) < 1e-6, "{up:?}");
|
||
|
||
// An explicit target (comp world) places the layer there, its front facing the normal.
|
||
let r = s
|
||
.execute_checked("camera.createFromSolve", json!({"kind": "solid", "target": {"center": [10.0, 0.0, 20.0], "normal": [0.0, 0.0, -1.0], "size": 80.0}}))
|
||
.unwrap();
|
||
let l = s.active_comp().unwrap().layer(LayerId(r["layers"][0].as_u64().unwrap())).unwrap().clone();
|
||
let tr = l.props.sub("transform").unwrap();
|
||
let pos = tr.get("position").unwrap().value.as_vec3();
|
||
assert!(linalg::norm(linalg::sub(pos, [10.0, 0.0, 20.0])) < 1e-6, "{pos:?}");
|
||
let front = linalg::mv(&linalg::from_euler_xyz(tr.get("orientation").unwrap().value.as_vec3()), [0.0, 0.0, -1.0]);
|
||
assert!(linalg::norm(linalg::sub(front, [0.0, 0.0, -1.0])) < 1e-6, "{front:?}");
|
||
|
||
// Serde keeps the analysis.
|
||
let back = effectcraft_project::Project::from_json(&s.project.to_json()).unwrap();
|
||
let g2 = back.comp(cid).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap();
|
||
let sv = ct::solve(&crate::camera_track::static_params(g2)).unwrap();
|
||
assert!(sv.ground.is_some());
|
||
assert_eq!(sv.points.len(), pl.solve.points.len());
|
||
}
|
||
|
||
#[test]
|
||
fn invalidation_resolve_cancel_and_undo() {
|
||
let (mut s, clip) = tracked(json!({"shotType": "specify", "aov": 2.0 * (W as f64 / 2.0 / F).atan().to_degrees()}));
|
||
let st = status(&mut s);
|
||
assert_eq!(st["solved"], true, "{st}");
|
||
assert_eq!(st["shotType"], "Specify Angle of View");
|
||
assert!((st["focalLength"].as_f64().unwrap() - F).abs() < 0.01 * F);
|
||
let cid = s.active_comp_id().unwrap();
|
||
let uid = st["effect"].as_u64().unwrap();
|
||
// Undo removes the analysis (one step); redo restores it.
|
||
assert!(s.undo());
|
||
assert_eq!(status(&mut s)["analyzed"], false);
|
||
assert!(s.redo());
|
||
assert_eq!(status(&mut s)["solved"], true);
|
||
|
||
// Changing the Solve Method keeps the tracks and only re-solves.
|
||
s.execute_checked("prop.set", json!({"layer": clip.0, "path": "effects/#1/advanced/solveMethod", "value": 1})).unwrap();
|
||
let st = status(&mut s);
|
||
assert_eq!(st["analyzed"], true);
|
||
assert_eq!(st["solved"], false);
|
||
assert_eq!(st["pending"], true);
|
||
s.execute_checked("camera.analyze", json!({"wait": true})).unwrap();
|
||
let st = status(&mut s);
|
||
assert_eq!(st["solved"], true, "{st}");
|
||
assert_eq!(st["methodUsed"], "Typical");
|
||
|
||
// Deleting points re-solves without them.
|
||
let pts = s.execute("camera.points", json!({})).unwrap();
|
||
let del: Vec<u64> = pts["points"].as_array().unwrap().iter().take(3).map(|p| p["id"].as_u64().unwrap()).collect();
|
||
s.execute_checked("camera.deletePoints", json!({"points": del, "wait": true})).unwrap();
|
||
s.poll_camera(false);
|
||
let st = status(&mut s);
|
||
assert_eq!(st["solved"], true, "{st}");
|
||
assert!(st["deleted"].as_u64().unwrap() >= 3);
|
||
let g = s.project.comp(cid).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().clone();
|
||
let sv = ct::solve(&crate::camera_track::static_params(&g)).unwrap();
|
||
assert!(sv.points.iter().all(|p| !del.contains(&(p.id as u64))));
|
||
|
||
// Trimming the layer changes its frames: tracks and solve are cleared and queued.
|
||
s.edit("trim", None, |p, _| {
|
||
p.comp_mut(cid).unwrap().layer_mut(clip).unwrap().in_point = frame_time(2);
|
||
Ok(())
|
||
})
|
||
.unwrap();
|
||
let st = status(&mut s);
|
||
assert_eq!(st["analyzed"], false);
|
||
assert_eq!(st["pending"], true);
|
||
|
||
// A background analysis can be cancelled: nothing is written.
|
||
s.execute_checked("camera.analyze", json!({})).unwrap();
|
||
assert!(s.is_camera_analyzing() || s.camera_job.is_some());
|
||
let banner = status(&mut s)["banner"].as_str().map(str::to_string);
|
||
if s.is_camera_analyzing() {
|
||
assert!(banner.unwrap().starts_with("Analyzing in background (step 1 of 2)"));
|
||
}
|
||
s.execute("camera.cancel", json!({})).ok();
|
||
let t0 = std::time::Instant::now();
|
||
while s.camera_job.is_some() {
|
||
assert!(t0.elapsed().as_secs() < 120);
|
||
std::thread::sleep(std::time::Duration::from_millis(5));
|
||
s.poll_camera(false);
|
||
}
|
||
// (When the job finished before the cancel landed it wrote a valid analysis.)
|
||
let st = status(&mut s);
|
||
assert_eq!(st["running"], false);
|
||
assert!(s.execute_checked("track.camera", json!({"layer": clip.0, "shotType": "bogus"})).is_err());
|
||
assert!(s.execute_checked("track.camera", json!({"layer": clip.0, "aov": 500})).is_err());
|
||
}
|
||
|
||
#[test]
|
||
fn effect_parameters_match_after_effects() {
|
||
let spec = effectcraft_effects::registry().iter().find(|e| e.id == ct::ID).unwrap();
|
||
assert_eq!(spec.name, "3D Camera Tracker");
|
||
assert_eq!(spec.category, "Perspective");
|
||
let names: Vec<&str> = spec.params.iter().filter(|p| !matches!(p.ui, effectcraft_project::ParamUi::Hidden)).map(|p| p.name).collect();
|
||
for n in [
|
||
"Shot Type",
|
||
"Horizontal Angle of View",
|
||
"Show Track Points",
|
||
"Render Track Points",
|
||
"Track Point Size",
|
||
"Target Size",
|
||
"Solve Method",
|
||
"Detailed Analysis",
|
||
"Auto-delete Points Across Time",
|
||
"Hide Warning Banner",
|
||
] {
|
||
assert!(names.contains(&n), "{n}");
|
||
}
|
||
// The Animation menu entry is a real command now.
|
||
let c = crate::find_command("track.camera").unwrap();
|
||
assert_eq!(c.label, "Track Camera");
|
||
}
|