diff --git a/crates/engine/src/cmd/masks.rs b/crates/engine/src/cmd/masks.rs index 24a94cbc..b78c7647 100644 --- a/crates/engine/src/cmd/masks.rs +++ b/crates/engine/src/cmd/masks.rs @@ -690,6 +690,63 @@ pub fn specs() -> Vec { Ok(json!({"spots": out, "added": found.len()})) } ), + cmd!( + "spot.findBlemishes", + "Find Blemishes", + [], + None, + "{sensitivity?: 0..100 (50), add?: bool (true), maxPerFace?: 1..20 (6)} — find small blemishes (darker or redder round spots on skin) on the faces of the active photo and add a heal spot on each (one undo step); needs the YuNet face detector (`faces.models.download {id: \"yunet-2023mar\"}`). Faces narrower than 160 px at 1600 px, and faces with more than 12 candidates (stubble, freckles, pores), get none → {faces, spots: [{x, y, size, face}], added}", + has_active, + |s, p| { + let c = "spot.findBlemishes"; + let id = s.active().ok_or_else(|| bad(c, "no active photo"))?; + let detector = super::face_detect::detector(s).map_err(|e| bad(c, e))?; + // the uncropped photo, so positions are the spots' own coordinates + let job = s.render_job(id, 1600, 1600, false, false).ok_or_else(|| bad(c, "no photo"))?; + let img = job.run().rendered.map_err(|e| bad(c, e))?.image; + let rgb: Vec = img.data.iter().flat_map(|px| [px[0], px[1], px[2]]).collect(); + let faces = + detector.detect(&rgb, img.width, img.height, &lightcraft_faces::yunet::Options::default()).map_err(|e| bad(c, e.to_string()))?; + let boxes: Vec = faces + .iter() + .map(|f| lightcraft_pipeline::blemish::FaceBox { + x0: f64::from(f.x0), + y0: f64::from(f.y0), + x1: f64::from(f.x1), + y1: f64::from(f.y1), + landmarks: f.landmarks.map(|(x, y)| (f64::from(x), f64::from(y))), + }) + .collect(); + let max_per_face = (f64_or(p, "maxPerFace", 6.0) as usize).clamp(1, 20); + let mut found = lightcraft_pipeline::blemish::detect(&img, &boxes, f64_or(p, "sensitivity", 50.0) as f32); + let mut per_face = std::collections::BTreeMap::new(); + found.retain(|b| { + let n = per_face.entry(b.face).or_insert(0usize); + *n += 1; + *n <= max_per_face + }); + let out: Vec = found.iter().map(|b| json!({"x": b.x, "y": b.y, "size": b.radius, "face": b.face})).collect(); + if !p.get("add").and_then(Value::as_bool).unwrap_or(true) || found.is_empty() { + return Ok(json!({"faces": boxes.len(), "spots": out, "added": 0})); + } + let mut dd = (*s.develop_of(id).unwrap_or_default()).clone(); + let base = Spot::default(); + for b in &found { + let mut spot = Spot { + mode: SpotMode::Heal, + points: vec![Point::new(b.x, b.y)], + size: b.radius.clamp(SPOT_SIZE.0, SPOT_SIZE.1), + feather: base.feather, + opacity: 100.0, + source_offset: None, + }; + spot.source_offset = pick_source(s, id, &dd, &spot, None); + dd.spots.push(spot); + } + s.set_develop(id, dd, "Find Blemishes")?; + Ok(json!({"faces": boxes.len(), "spots": out, "added": found.len()})) + } + ), cmd!( "spot.add", "Add Remove Spot", diff --git a/crates/pipeline/src/blemish.rs b/crates/pipeline/src/blemish.rs new file mode 100644 index 00000000..61b3f263 --- /dev/null +++ b/crates/pipeline/src/blemish.rs @@ -0,0 +1,288 @@ +//! Blemish detection on faces: small, round spots that are darker or redder than the skin around +//! them. Classical, in the spirit of [`crate::dust`]: a blurred copy gives the surrounding skin, +//! a spot has to stand out from both the surroundings and the local texture, and each candidate +//! blob must be small and roughly round. The face boxes and landmarks come from a face detector +//! (YuNet) so eyes, brows, nostrils and lips are never taken for blemishes. +//! +//! Deliberately conservative: a face with many candidates has stubble, freckles or pores, not +//! blemishes, and gets none; the rest are capped per face. Removing freckles is a different +//! feature and never a default. + +use lightcraft_raster::{Plane, Rgba8, blur::gaussian}; + +/// A face in fractions of the image (0..1, y down): its box and the five landmarks a YuNet +/// detector gives, in its order (right eye, left eye, nose tip, right and left mouth corner, +/// from the subject's point of view). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct FaceBox { + pub x0: f64, + pub y0: f64, + pub x1: f64, + pub y1: f64, + pub landmarks: [(f64, f64); 5], +} + +/// A found blemish: which face, centre (0..1 of the image's width / height), radius (fraction +/// of the long edge) and how much it stands out (0..1). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Blemish { + pub face: usize, + pub x: f64, + pub y: f64, + pub radius: f64, + pub strength: f32, +} + +/// Faces narrower than this (in pixels of the analysed image) are left alone: a blemish is not +/// resolved below it, and what is found there is lash shadow and nostril edges. +pub const MIN_FACE_PX: f64 = 160.0; +/// More candidates than this on one face means texture (stubble, freckles), not blemishes. +pub const MAX_CANDIDATES_PER_FACE: usize = 12; +/// Most blemishes kept per face, strongest first. +pub const MAX_PER_FACE: usize = 6; + +/// Pixels of `img` inside the box that are skin: an ellipse over the face, gated by skin colour, +/// with the eyes, brows, nostrils and mouth cut out around the landmarks. +fn skin_mask(img: &Rgba8, f: &FaceBox, x0: usize, y0: usize, cw: usize, ch: usize) -> Vec { + let (w, h) = (img.width as f64, img.height as f64); + let (fx, fy, fw, fh) = (f.x0 * w, f.y0 * h, (f.x1 - f.x0) * w, (f.y1 - f.y0) * h); + let (cx, cy) = (fx + fw / 2.0, fy + fh * 0.55); + let (rx, ry) = (fw * 0.55, fh * 0.70); + let lm = |i: usize| (f.landmarks[i].0 * w, f.landmarks[i].1 * h); + let (re, le, nose, mr, ml) = (lm(0), lm(1), lm(2), lm(3), lm(4)); + let mouth = ((mr.0 + ml.0) / 2.0, (mr.1 + ml.1) / 2.0); + let eye_y = (re.1 + le.1) / 2.0; + let mut out = vec![false; cw * ch]; + for yy in 0..ch { + for xx in 0..cw { + let (px, py) = ((x0 + xx) as f64 + 0.5, (y0 + yy) as f64 + 0.5); + let in_ellipse = ((px - cx) / rx).powi(2) + ((py - cy) / ry).powi(2) <= 1.0; + if !in_ellipse { + continue; + } + // brows and eyes: a band from above the brows to just below the eyes + if py > eye_y - fh * 0.32 && py < eye_y + fh * 0.05 { + continue; + } + let near = |(lx, ly): (f64, f64), r: f64| (px - lx).powi(2) + (py - ly).powi(2) <= r * r; + if near(re, fw * 0.12) || near(le, fw * 0.12) || near(mouth, fw * 0.16) { + continue; + } + // nostrils: an ellipse at and below the nose tip + if ((px - nose.0) / (fw * 0.20)).powi(2) + ((py - nose.1 - fw * 0.02) / (fw * 0.16)).powi(2) <= 1.0 { + continue; + } + let Some(p) = img.data.get((y0 + yy) * img.width + x0 + xx) else { continue }; + let (r, g, b) = (f64::from(p[0]), f64::from(p[1]), f64::from(p[2])); + let cr = 128.0 + 0.5 * r - 0.4187 * g - 0.0813 * b; + let cb = 128.0 - 0.1687 * r - 0.3313 * g + 0.5 * b; + if let Some(o) = out.get_mut(yy * cw + xx) { + *o = (133.0..180.0).contains(&cr) && (77.0..130.0).contains(&cb); + } + } + } + out +} + +/// Blemishes on the given faces of a rendered image; `sensitivity` 0..100 (higher finds fainter +/// spots). Faces narrower than [`MIN_FACE_PX`] get none; a face with more than +/// [`MAX_CANDIDATES_PER_FACE`] candidates gets none; at most [`MAX_PER_FACE`] per face. +pub fn detect(img: &Rgba8, faces: &[FaceBox], sensitivity: f32) -> Vec { + let (w, h) = (img.width, img.height); + if w < 32 || h < 32 { + return Vec::new(); + } + let long = w.max(h) as f64; + let thr = 0.09 - 0.05 * (sensitivity / 100.0).clamp(0.0, 1.0); + let mut out = Vec::new(); + for (fi, f) in faces.iter().enumerate() { + let fw = (f.x1 - f.x0) * w as f64; + let fh = (f.y1 - f.y0) * h as f64; + if !(fw.is_finite() && fh.is_finite()) || fw < MIN_FACE_PX || fh <= 0.0 { + continue; + } + // the face with a margin, clipped to the image + let clamp = |v: f64, hi: usize| (v.max(0.0) as usize).min(hi); + let x0 = clamp(f.x0 * w as f64 - fw * 0.15, w); + let y0 = clamp(f.y0 * h as f64 - fh * 0.15, h); + let x1 = clamp(f.x1 * w as f64 + fw * 0.15, w); + let y1 = clamp(f.y1 * h as f64 + fh * 0.15, h); + let (cw, ch) = (x1.saturating_sub(x0), y1.saturating_sub(y0)); + if cw < 16 || ch < 16 { + continue; + } + let skin = skin_mask(img, f, x0, y0, cw, ch); + let px = |xx: usize, yy: usize| img.data.get((y0 + yy) * w + x0 + xx).copied().unwrap_or([0; 4]); + let lum = Plane::from_fn(cw, ch, |xx, yy| { + let p = px(xx, yy); + (0.2126 * f32::from(p[0]) + 0.7152 * f32::from(p[1]) + 0.0722 * f32::from(p[2])) / 255.0 + }); + // redness: how much red exceeds green, which a pimple does and even skin does not + let red = Plane::from_fn(cw, ch, |xx, yy| { + let p = px(xx, yy); + (f32::from(p[0]) - f32::from(p[1])) / 255.0 + }); + let sigma = (0.035 * fw as f32).max(2.0); + let around_l = gaussian(&lum, sigma); + let around_r = gaussian(&red, sigma); + let detail = Plane::from_fn(cw, ch, |xx, yy| { + let i = yy * cw + xx; + (lum.data.get(i).copied().unwrap_or(0.0) - around_l.data.get(i).copied().unwrap_or(0.0)).abs() + }); + let texture = gaussian(&detail, (0.06 * fw as f32).max(3.0)); + let resp: Vec = (0..cw * ch) + .map(|i| { + let dark = around_l.data.get(i).copied().unwrap_or(0.0) - lum.data.get(i).copied().unwrap_or(0.0); + let redder = red.data.get(i).copied().unwrap_or(0.0) - around_r.data.get(i).copied().unwrap_or(0.0); + dark.max(redder).max(0.0) + }) + .collect(); + let candidate = |i: usize| { + skin.get(i).copied().unwrap_or(false) + && resp.get(i).copied().unwrap_or(0.0) > thr + && resp.get(i).copied().unwrap_or(0.0) > 2.5 * texture.data.get(i).copied().unwrap_or(0.0) + }; + // blobs of candidate pixels (4-connected), small and roundish + let mut seen = vec![false; cw * ch]; + let mut found: Vec = Vec::new(); + let (rmin, rmax) = ((0.004 * fw).max(1.5), 0.05 * fw); + for start in 0..cw * ch { + if seen.get(start).copied().unwrap_or(true) || !candidate(start) { + continue; + } + let mut stack = vec![start]; + if let Some(s) = seen.get_mut(start) { + *s = true; + } + let (mut n, mut sx, mut sy, mut peak) = (0usize, 0.0f64, 0.0f64, 0.0f32); + let (mut bx0, mut bx1, mut by0, mut by1) = (cw, 0, ch, 0); + while let Some(i) = stack.pop() { + let (xx, yy) = (i % cw, i / cw); + n += 1; + sx += xx as f64; + sy += yy as f64; + peak = peak.max(resp.get(i).copied().unwrap_or(0.0)); + (bx0, bx1, by0, by1) = (bx0.min(xx), bx1.max(xx), by0.min(yy), by1.max(yy)); + if n as f64 > rmax * rmax * 4.0 { + break; // too big to be a blemish: a shadow or a feature + } + for j in [i.wrapping_sub(1), i + 1, i.wrapping_sub(cw), i + cw] { + let ok = j < cw * ch && !seen.get(j).copied().unwrap_or(true) && (j % cw).abs_diff(xx) <= 1; + if ok && candidate(j) { + if let Some(s) = seen.get_mut(j) { + *s = true; + } + stack.push(j); + } + } + } + let (bw, bh) = ((bx1 - bx0 + 1) as f64, (by1 - by0 + 1) as f64); + let r = bw.max(bh) / 2.0; + let round = bw.max(bh) / bw.min(bh) < 3.0 && n as f64 >= 0.35 * bw * bh && n >= 3; + if r >= rmin && r <= rmax && round { + found.push(Blemish { + face: fi, + x: (x0 as f64 + sx / n as f64 + 0.5) / w as f64, + y: (y0 as f64 + sy / n as f64 + 0.5) / h as f64, + radius: (r / long * 1.6).max(fw * 0.012 / long), + strength: peak, + }); + } + } + if found.len() > MAX_CANDIDATES_PER_FACE { + continue; // texture, not blemishes + } + found.sort_by(|a, b| b.strength.total_cmp(&a.strength)); + found.truncate(MAX_PER_FACE); + out.extend(found); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A grey frame with one skin-coloured face ellipse `fw` wide at the centre, landmarks where + /// a face has them, and `dots` dark spots on the cheeks. + fn scene(w: usize, h: usize, fw: f64, dots: &[(f64, f64)]) -> (Rgba8, FaceBox) { + let (cx, cy, fh) = (w as f64 / 2.0, h as f64 / 2.0, fw * 1.3); + let mut img = Rgba8::filled(w, h, [120, 120, 120, 255]); + for y in 0..h { + for x in 0..w { + let (px, py) = (x as f64 + 0.5, y as f64 + 0.5); + if ((px - cx) / (fw * 0.5)).powi(2) + ((py - cy) / (fh * 0.5)).powi(2) <= 1.0 { + img.data[y * w + x] = [222, 180, 160, 255]; // skin + } + } + } + for &(dx, dy) in dots { + let (sx, sy) = (cx + dx * fw, cy + dy * fh); + for y in 0..h { + for x in 0..w { + if (x as f64 + 0.5 - sx).powi(2) + (y as f64 + 0.5 - sy).powi(2) <= (0.012 * fw).max(1.5).powi(2) { + img.data[y * w + x] = [150, 90, 85, 255]; // a dark red spot + } + } + } + } + let n = |x: f64, y: f64| (x / w as f64, y / h as f64); + let face = FaceBox { + x0: (cx - fw / 2.0) / w as f64, + y0: (cy - fh / 2.0) / h as f64, + x1: (cx + fw / 2.0) / w as f64, + y1: (cy + fh / 2.0) / h as f64, + landmarks: [ + n(cx - fw * 0.2, cy - fh * 0.12), + n(cx + fw * 0.2, cy - fh * 0.12), + n(cx, cy + fh * 0.08), + n(cx - fw * 0.15, cy + fh * 0.28), + n(cx + fw * 0.15, cy + fh * 0.28), + ], + }; + (img, face) + } + + #[test] + fn finds_spots_on_the_cheeks_and_nothing_else() { + let dots = [(-0.28, 0.08), (0.3, 0.12), (-0.2, 0.3)]; + let (img, face) = scene(800, 600, 300.0, &dots); + let found = detect(&img, &[face], 50.0); + assert_eq!(found.len(), 3, "{found:?}"); + for (dx, dy) in dots { + let (ex, ey) = (0.5 + dx * 300.0 / 800.0, 0.5 + dy * 390.0 / 600.0); + assert!(found.iter().any(|b| (b.x - ex).abs() < 0.01 && (b.y - ey).abs() < 0.01), "{found:?} lacks {ex},{ey}"); + } + assert!(found.iter().all(|b| b.face == 0 && b.radius > 0.0 && b.radius < 0.02 && b.strength > 0.0)); + let (clean, face) = scene(800, 600, 300.0, &[]); + assert!(detect(&clean, &[face], 50.0).is_empty(), "a clean face has no blemishes"); + } + + #[test] + fn features_and_small_faces_are_left_alone() { + // spots on the eyes, the nose tip and the mouth: all inside the cut-outs + let (img, face) = scene(800, 600, 300.0, &[(-0.2, -0.12), (0.2, -0.12), (0.0, 0.1), (0.0, 0.28)]); + assert!(detect(&img, &[face], 80.0).is_empty(), "{:?}", detect(&img, &[face], 80.0)); + // the same cheek spots on a face below the size floor + let (img, face) = scene(800, 600, 120.0, &[(-0.28, 0.08), (0.3, 0.12)]); + assert!(detect(&img, &[face], 50.0).is_empty()); + } + + #[test] + fn a_speckled_face_is_texture_not_blemishes() { + // 24 spots on the cheeks, all outside the eye, nose and mouth cut-outs + let dots: Vec<(f64, f64)> = + [-0.42, -0.32, -0.22, 0.22, 0.32, 0.42].iter().flat_map(|&dx| (0..4).map(move |j| (dx, 0.0 + 0.08 * j as f64))).collect(); + let (img, face) = scene(800, 600, 300.0, &dots); + assert!(detect(&img, &[face], 50.0).is_empty(), "more than {MAX_CANDIDATES_PER_FACE} candidates means stubble or freckles"); + } + + #[test] + fn hostile_faces_do_not_panic() { + let (img, _) = scene(64, 48, 20.0, &[]); + let weird = FaceBox { x0: -2.0, y0: 0.5, x1: 9.0, y1: f64::NAN, landmarks: [(f64::INFINITY, 0.0); 5] }; + let inverted = FaceBox { x0: 0.9, y0: 0.9, x1: 0.1, y1: 0.1, landmarks: [(0.5, 0.5); 5] }; + assert!(detect(&img, &[weird, inverted], 100.0).is_empty()); + assert!(detect(&Rgba8::new(0, 0), &[inverted], 50.0).is_empty()); + } +} diff --git a/crates/pipeline/src/lib.rs b/crates/pipeline/src/lib.rs index 05f0ff8a..e8fe57c9 100644 --- a/crates/pipeline/src/lib.rs +++ b/crates/pipeline/src/lib.rs @@ -24,6 +24,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] pub mod auto; +pub mod blemish; pub mod colorops; pub mod cull; pub mod dust; diff --git a/docs/faces.md b/docs/faces.md index 7adc1fca..5c8e5431 100644 --- a/docs/faces.md +++ b/docs/faces.md @@ -297,3 +297,12 @@ suggestions it would make and how many were right. All of this is reachable from the control channel, the CLI and MCP: `faces.models.list`, `faces.models.inspect {path}`, `faces.models.install {path, acknowledged: true, activate?}`, `faces.models.download {id, acknowledged: true}` (then `faces.models.downloads`, which also installs what has arrived, and `faces.models.downloadCancel {id}`), `faces.models.remove {id}`, `faces.models.select {id}`, `faces.enable {enabled?}`, `faces.detect {ids?, apply?}`, `faces.index {budgetMs?, ids?}`, `faces.pump` (what the app calls every frame), `faces.suggest {ids?, threshold?, margin?}`, `faces.person {name, more?}` (a person's faces and the unnamed faces that look like them), `faces.unnamed {limit?}` (every unnamed face, look-alikes together, with suggested names), `faces.setName {id?, index, name}`, `faces.nameFaces {faces: [{photo, index}], name}` (name many at once, one undo step) and `faces.evaluate`. `acknowledged` must be `true`: the caller has shown the user the terms and the user agreed. Installing makes the model the one in use and switches recognition on unless `activate` is `false`. + +## Find Blemishes + +Photo ▸ Find Blemishes (`spot.findBlemishes`) uses the installed detector's boxes and landmarks to look for small, +round, darker-or-redder spots on each face's skin and adds a heal spot on each, the way Find Dust Spots does for +sensor dust. Eyes, brows, nostrils and the mouth are cut out around the landmarks; faces narrower than 160 px in +the 1600 px analysis image get nothing, as do faces with more than 12 candidates (stubble, freckles and pores are +texture, not blemishes); at most six per face. It needs no model beyond the detector and never changes anything +without a spot you can see and delete (`crates/pipeline/src/blemish.rs`). diff --git a/docs/parity-checklist.md b/docs/parity-checklist.md index 1e3f4ae1..88dfc2eb 100644 --- a/docs/parity-checklist.md +++ b/docs/parity-checklist.md @@ -34,7 +34,7 @@ cited here still exists. Formerly docs/parity.md. | F. Edit panel — global adjustments (EDIT) | 43 | 2 | 3 | 1 | 28/28 (100%) | 13/14 (93%) | | G. Profiles (PROF) | 7 | 1 | 3 | 0 | 3/4 (75%) | 3/3 (100%) | | H. Crop & rotate (CROP) | 9 | 0 | 0 | 1 | 6/6 (100%) | 3/3 (100%) | -| I. Remove / healing (REM) | 7 | 1 | 2 | 2 | 4/4 (100%) | 2/3 (67%) | +| I. Remove / healing (REM) | 8 | 1 | 2 | 2 | 4/4 (100%) | 2/3 (67%) | | J. Red eye (EYE) | 2 | 0 | 0 | 0 | — | 1/1 (100%) | | K. Masking (MASK) | 13 | 5 | 5 | 0 | 7/8 (88%) | 5/5 (100%) | | L. Presets (PRE) | 6 | 0 | 1 | 1 | 2/2 (100%) | 2/2 (100%) | @@ -270,6 +270,7 @@ differences and missing bindings are in [ui-parity.md](ui-parity.md#shortcuts). | LR-REM-PEOPLE | Remove people (generative) | OOS | 🚫 | | | | LR-REM-REFLECT | Remove reflections | P2 | ⬜ | | | | LR-REM-DUST | Dust detection | P2 | ✅ | `cmd:spot.findDust`, `crates/pipeline/src/dust.rs` | Remove panel ▸ Find Dust Spots: small, soft, round dark spots on smooth areas become heal spots (sensitivity; one undo step); Visualize Spots for checking by eye | +| LR-REM-BLEMISH | Find blemishes on faces (beyond Lightroom) | P2 | ✅ | `cmd:spot.findBlemishes`, `crates/pipeline/src/blemish.rs`, `crates/engine/src/cmd/face_detect.rs` | Remove panel ▸ Find Blemishes: with the YuNet detector installed, small darker-or-redder round spots on each face's skin (eyes, brows, nostrils and mouth cut out around the landmarks) become heal spots (one undo step); faces under 160 px and faces with more than 12 candidates (stubble, freckles) get none, at most 6 per face; sensitivity 0–100. Classical, no learned model beyond the detector; not a Lightroom feature | | LR-REM-SYNC | Sync spots | P1 | ✅ | `cmd:develop.copy` (`groups`), `crates/develop/src/presets.rs` | | ## J. Red eye (EYE)