141 lines
8.0 KiB
Rust
141 lines
8.0 KiB
Rust
//! Face tracking end to end through the commands, on synthetic face footage generated in-test
|
|
//! (a moving, rotating face that opens its mouth and blinks).
|
|
|
|
use effectcraft_keyframe::{ShapePath, Value as KV};
|
|
use effectcraft_track::face::{FaceParams, LANDMARKS, MEASUREMENTS, render_synthetic, synth_landmarks};
|
|
use serde_json::json;
|
|
|
|
use crate::mask_track::MaskMethod;
|
|
use crate::tests_track::{H, W, frame_time, setup};
|
|
|
|
fn face(f: u32) -> FaceParams {
|
|
let t = f as f64;
|
|
FaceParams {
|
|
center: [160.0 + 25.0 * (t * 0.2).sin(), 120.0 + 8.0 * (t * 0.15).cos()],
|
|
height: 70.0 * (1.0 + 0.05 * (t * 0.1).sin()),
|
|
roll: 12.0 * (t * 0.18).sin(),
|
|
yaw: 10.0 * (t * 0.12).sin(),
|
|
mouth_open: 0.5 + 0.45 * (t * 0.3).sin(),
|
|
eye_open: [0.85, 0.85],
|
|
..Default::default()
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn face_tracking_keys_outline_points_and_measurements() {
|
|
let (mut s, clip, _) = setup(|f| render_synthetic(W, H, &face(f), 5));
|
|
assert_eq!(MaskMethod::from_name("faceDetailed"), Some(MaskMethod::FaceDetailed));
|
|
assert_eq!(MaskMethod::from_name("Face Tracking (Outline Only)"), Some(MaskMethod::FaceOutline));
|
|
// A rough mask around the face.
|
|
let c = face(0).center;
|
|
let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect(c, 120.0, 160.0).vertices, "closed": true})).unwrap();
|
|
let mask = r["mask"].as_u64().unwrap();
|
|
s.execute("time.set", json!({"time": 0})).unwrap();
|
|
let r = s.execute_checked("track.mask", json!({"layer": clip.0, "mask": mask, "method": "faceDetailed", "direction": "forward", "wait": true})).unwrap();
|
|
assert_eq!(r["frames"], 24, "{r}");
|
|
assert_eq!(s.state.mask_track_method, MaskMethod::FaceDetailed);
|
|
let comp = s.active_comp().unwrap();
|
|
let layer = comp.layer(clip).unwrap();
|
|
// The mask follows the face outline.
|
|
let path = layer.props.find_group(mask).unwrap().get("path").unwrap();
|
|
assert_eq!(path.keys.len(), 25);
|
|
for f in [0, 12, 24] {
|
|
let KV::Path(p) = path.value_at(frame_time(f)) else { panic!() };
|
|
let fp = face(f);
|
|
let (sn, cs) = fp.roll.to_radians().sin_cos();
|
|
for v in &p.vertices {
|
|
let d = [(v[0] - fp.center[0]) / fp.height, (v[1] - fp.center[1]) / fp.height];
|
|
let (u, w) = (cs * d[0] + sn * d[1], -sn * d[0] + cs * d[1]);
|
|
let r = ((u / 0.75).powi(2) + w * w).sqrt();
|
|
assert!((r - 1.0).abs() < 0.06, "frame {f}: vertex {v:?} at {r}");
|
|
}
|
|
}
|
|
// Face Track Points: one keyed point per landmark, on the face.
|
|
let fx = layer.effects().unwrap();
|
|
let pts = fx.groups().find(|g| g.match_id == effectcraft_effects::face_track::POINTS_ID).expect("Face Track Points");
|
|
let mut worst: f64 = 0.0;
|
|
for f in [0, 6, 12, 18, 24] {
|
|
let truth = synth_landmarks(&face(f));
|
|
for (i, (id, _, _)) in LANDMARKS.iter().enumerate() {
|
|
let pr = pts.get(id).unwrap();
|
|
assert_eq!(pr.keys.len(), 25, "{id}");
|
|
let KV::Vec2(v) = pr.value_at(frame_time(f)) else { panic!() };
|
|
worst = worst.max((v[0] - truth[i][0]).hypot(v[1] - truth[i][1]) / face(f).height);
|
|
}
|
|
}
|
|
assert!(worst < 0.1, "worst landmark error {worst} face heights");
|
|
// Extract & Copy Face Measurements.
|
|
assert!(s.is_enabled("track.extractFaceMeasurements"));
|
|
let r = s.execute_checked("track.extractFaceMeasurements", json!({"layer": clip.0})).unwrap();
|
|
assert_eq!(r["frames"], 25);
|
|
assert!(r["clipboard"].as_str().unwrap().contains("Mouth Openness"));
|
|
assert_eq!(s.state.key_clipboard.len(), MEASUREMENTS.len());
|
|
assert!(s.state.clip_is_keys);
|
|
let layer = s.active_comp().unwrap().layer(clip).unwrap();
|
|
let m = layer.effects().unwrap().groups().find(|g| g.match_id == effectcraft_effects::face_track::MEASUREMENTS_ID).expect("Face Measurements");
|
|
let mo = m.get("mouthOpenness").unwrap();
|
|
assert_eq!(mo.keys.len(), 25);
|
|
// Mouth openness follows the footage: widest open where the generator opens it most.
|
|
let open = |f: u32| mo.value_at(frame_time(f)).as_f64();
|
|
let (fmax, fmin) = (
|
|
(0..25).max_by(|a, b| face(*a).mouth_open.total_cmp(&face(*b).mouth_open)).unwrap(),
|
|
(0..25).min_by(|a, b| face(*a).mouth_open.total_cmp(&face(*b).mouth_open)).unwrap(),
|
|
);
|
|
assert!(open(fmax) > open(fmin) + 20.0, "{} vs {}", open(fmax), open(fmin));
|
|
let roll = m.get("headOrientationZ").unwrap();
|
|
for f in [0, 10, 20] {
|
|
assert!((roll.value_at(frame_time(f)).as_f64() - face(f).roll).abs() < 3.0);
|
|
}
|
|
// One undo step for the extraction.
|
|
assert!(s.undo());
|
|
assert!(!s.active_comp().unwrap().layer(clip).unwrap().effects().unwrap().groups().any(|g| g.match_id == effectcraft_effects::face_track::MEASUREMENTS_ID));
|
|
// Outline Only keys just the mask.
|
|
let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect(c, 120.0, 160.0).vertices, "closed": true})).unwrap();
|
|
let m2 = r["mask"].as_u64().unwrap();
|
|
s.execute("time.set", json!({"time": 0})).unwrap();
|
|
s.execute_checked("track.mask", json!({"layer": clip.0, "mask": m2, "method": "faceOutline", "direction": "frameForward", "wait": true})).unwrap();
|
|
let layer = s.active_comp().unwrap().layer(clip).unwrap();
|
|
assert_eq!(layer.props.find_group(m2).unwrap().get("path").unwrap().keys.len(), 2);
|
|
// A mask without a face reports it.
|
|
let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect([20.0, 20.0], 30.0, 30.0).vertices, "closed": true})).unwrap();
|
|
let m3 = r["mask"].as_u64().unwrap();
|
|
s.execute("time.set", json!({"time": 0})).unwrap();
|
|
let _ = s.execute("track.mask", json!({"layer": clip.0, "mask": m3, "method": "faceOutline", "direction": "forward", "wait": true}));
|
|
let layer = s.active_comp().unwrap().layer(clip).unwrap();
|
|
assert!(layer.props.find_group(m3).unwrap().get("path").unwrap().keys.is_empty());
|
|
}
|
|
|
|
/// With MediaPipe Face Landmarker installed (`EFFECTCRAFT_FACE_LANDMARKER` = path to
|
|
/// `face_landmarker.task`, else skipped): Track Mask ▸ Face Tracking uses it, and the pupils and
|
|
/// eye corners it keys sit on the synthetic face's.
|
|
#[test]
|
|
fn face_tracking_uses_the_chosen_model() {
|
|
let Ok(model) = std::env::var("EFFECTCRAFT_FACE_LANDMARKER") else { return };
|
|
let (mut s, clip, _) = setup(|f| render_synthetic(W, H, &face(f), 5));
|
|
let dir = std::env::temp_dir().join(format!("ec-face-model-{}", std::process::id()));
|
|
s.models_dir = Some(dir.clone());
|
|
s.execute_checked("face.model.install", json!({"path": model})).unwrap();
|
|
// `wait`: loaded before the track starts (as scripts need).
|
|
let r = s.execute_checked("face.model.select", json!({"id": "mediapipe-face", "wait": true})).unwrap();
|
|
assert_eq!(r["active"], "mediapipe-face");
|
|
let c = face(0).center;
|
|
let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect(c, 120.0, 160.0).vertices, "closed": true})).unwrap();
|
|
let mask = r["mask"].as_u64().unwrap();
|
|
s.execute("time.set", json!({"time": 0})).unwrap();
|
|
let r = s.execute_checked("track.mask", json!({"layer": clip.0, "mask": mask, "method": "faceDetailed", "direction": "forward", "wait": true})).unwrap();
|
|
assert_eq!((r["frames"].as_u64(), r["faceModel"].as_str()), (Some(24), Some("mediapipe-face")), "{r}");
|
|
let layer = s.active_comp().unwrap().layer(clip).unwrap().clone();
|
|
let pts = layer.effects().unwrap().groups().find(|g| g.match_id == effectcraft_effects::face_track::POINTS_ID).expect("Face Track Points");
|
|
let mut worst: f64 = 0.0;
|
|
for f in [0, 8, 16, 24] {
|
|
let truth = synth_landmarks(&face(f));
|
|
for id in ["leftPupil", "rightPupil", "leftEyeOuter", "rightEyeOuter"] {
|
|
let i = effectcraft_track::face::landmark(id).unwrap();
|
|
let KV::Vec2(v) = pts.get(id).unwrap().value_at(frame_time(f)) else { panic!() };
|
|
worst = worst.max((v[0] - truth[i][0]).hypot(v[1] - truth[i][1]) / face(f).height);
|
|
}
|
|
}
|
|
assert!(worst < 0.05, "worst eye error {worst} face heights");
|
|
let _ = std::fs::remove_dir_all(dir);
|
|
}
|