diff --git a/NOTICE b/NOTICE index 1fcf11e7..f3f6499f 100644 --- a/NOTICE +++ b/NOTICE @@ -15,11 +15,17 @@ list (author, source, license) is in assets/ATTRIBUTION.md. Inter (assets/fonts/Inter-*.ttf) — SIL Open Font License 1.1. See assets/fonts/OFL-Inter.txt. Showcase photographs by Ansel Adams, Dorothea Lange and NASA (docs/images/) — public domain. See assets/ATTRIBUTION.md. -crates/segment ports the SAM 3 model code of Hugging Face Transformers +crates/segment (lightcraft-segment) ports the SAM 3 model code of Hugging Face Transformers (https://github.com/huggingface/transformers, src/transformers/models/sam3 and sam3_tracker), Copyright The HuggingFace Team and Meta Platforms, Inc., licensed under the Apache License 2.0. -The SAM 3 weights are not part of LightCraft: users download the facebook/sam3 checkpoint -themselves, under Meta's SAM License. +The port was modified by the LightCraft contributors (2026): translated from Python/PyTorch to +Rust on candle and restructured; each ported file says so. Because of this, crates/segment is +licensed under the Apache License 2.0 only (LICENSE-APACHE), not under the MIT option. + +The SAM 3 weights are not part of LightCraft and are never distributed with it. A user downloads +the facebook/sam3 checkpoint only when they choose to (in the app, or by hand), under Meta's SAM +License (https://github.com/facebookresearch/sam3/blob/main/LICENSE), a separate, non-OSI +licence. See docs/ai-masks.md. Builds made with craft-fonts (the optional CRAFT_FONTS_DIR build input; all official releases) also embed the fonts listed in https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md diff --git a/ROADMAP.md b/ROADMAP.md index fdfbec0c..f2dcb9bc 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -39,7 +39,7 @@ duplicate Local entries #22, black GPU exports on an Intel iGPU #78). | Nikon / Sony / older-Canon raw shooter | ~65% | Camera colour fidelity and coverage (ARW and NEF preview estimates are only a starting point) | | Canon CR3 / Fujifilm / Olympus shooter | ~35% | Their raws open as embedded previews only (CR3: the full-size JPEG with full metadata) | | Lightroom Classic power user | ~45% | Print, Book, Map, publish, tethering | -| Relies on AI (masks, denoise) | ~25% | No segmentation or AI denoise models | +| Relies on AI (masks, denoise) | ~25% | Object / Describe masks via optional SAM 3 (download not yet hosted); no AI denoise | ## Where we're going @@ -57,7 +57,9 @@ Priorities, in order. Each points at tracker rows in [`docs/parity.md`](docs/par (references stay in the local `plan/`), then tune against the numbers. 5. **Lens profiles of our own** (LR-EDIT-OPTICS-PROFILE, P1). 6. **AI model strategy** (maintainer decision): which permissively licensed models (or our own training) for - segmentation masks and denoise; then pure-Rust inference. Unblocks M12 and Enhance. + segmentation masks and denoise; then pure-Rust inference. Unblocks M12 and Enhance. Object / Describe masks now + run Meta's SAM 3 in pure Rust (optional, user-consented download under the non-OSI SAM License; CDN mirrors still + to be set up, docs/ai-masks.md); Subject / Sky / People and Enhance are still open. 7. **Then:** HDR (Q), the Classic output modules (Print first, then Map view, Book, Slideshow), video (R), localisation and accessibility. diff --git a/apps/lightcraft/src/main.rs b/apps/lightcraft/src/main.rs index 7617cba1..e46fd2ee 100644 --- a/apps/lightcraft/src/main.rs +++ b/apps/lightcraft/src/main.rs @@ -406,7 +406,9 @@ OPTIONS: ENVIRONMENT: LIGHTCRAFT_GPU_BACKEND=dx12|vulkan|metal|auto|off graphics backend (default: DX12 on Windows, Metal on macOS, Vulkan on Linux; off = render on the CPU); else WGPU_BACKEND. LIGHTCRAFT_GPU=0: CPU rendering. - LIGHTCRAFT_SAM3_DIR=DIR the SAM 3 model for Object / Describe masks (default: /models/sam3) + LIGHTCRAFT_SAM3_DIR=DIR the SAM 3 model for Object / Describe masks (default: /models/sam3; + optional: LightCraft offers to download it when first needed) + LIGHTCRAFT_SAM3_MIRRORS=URL,… where to download the SAM 3 model from (base URLs, tried in order) "; /// Warnings and errors (failed commands, AI mask analysis) on stderr; `LIGHTCRAFT_LOG=info` @@ -504,8 +506,11 @@ fn main() -> eframe::Result { Box::new(move |cc| { let (mut session, problem) = open_session(in_memory, library_dir, seed_demo && files.is_empty()); // AI masks: the SAM 3 checkpoint (facebook/sam3) in /models/sam3, or LIGHTCRAFT_SAM3_DIR + // (never required: without it, AI masks offer to download it; see docs/ai-masks.md) session.segmenter.dir = std::env::var_os("LIGHTCRAFT_SAM3_DIR").map(std::path::PathBuf::from).or_else(|| config_dir().map(|d| d.join("models").join("sam3"))); + // the user's own download locations, one base URL per line (LIGHTCRAFT_SAM3_MIRRORS too) + session.segmenter.mirrors_file = config_dir().map(|d| d.join("models").join("sam3-mirrors.txt")); let mut app = LightcraftApp::new(session, services()); if let Some(ui) = prefs { app.ui = ui; diff --git a/crates/develop/src/segmask.rs b/crates/develop/src/segmask.rs index 724ebdf5..172d53fc 100644 --- a/crates/develop/src/segmask.rs +++ b/crates/develop/src/segmask.rs @@ -55,6 +55,34 @@ impl SegMask { } } +/// Most zoomed-in detail patches one AI mask component keeps (the app makes one; each costs a +/// decode per render, so a hostile document can't make rendering explode). +pub const MAX_DETAIL: usize = 4; + +/// Deserialize a component's detail patches, keeping at most [`MAX_DETAIL`] (the rest are +/// skipped without being decoded). +pub(crate) fn de_detail<'de, D: serde::Deserializer<'de>>(d: D) -> Result, D::Error> { + struct Capped; + impl<'de> serde::de::Visitor<'de> for Capped { + type Value = Vec; + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("a list of segmentations") + } + fn visit_seq>(self, mut seq: A) -> Result, A::Error> { + let mut out = Vec::new(); + while out.len() < MAX_DETAIL { + match seq.next_element::()? { + Some(m) => out.push(m), + None => return Ok(out), + } + } + while seq.next_element::()?.is_some() {} + Ok(out) + } + } + d.deserialize_seq(Capped) +} + const B64: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; fn base64_encode(bytes: &[u8]) -> String { @@ -118,6 +146,30 @@ mod tests { } } + #[test] + fn a_hostile_document_keeps_at_most_max_detail_patches() { + let patch = serde_json::to_value(SegMask::from_logits(4, &[1.0; 16])).unwrap(); + let shape = serde_json::json!({"kind": "object", "hint": [{"x": 0.5, "y": 0.5}], "detail": vec![patch; 10_000]}); + let s: crate::MaskShape = serde_json::from_value(shape).unwrap(); + let crate::MaskShape::Object { detail, .. } = s else { panic!("an Object") }; + assert_eq!(detail.len(), MAX_DETAIL); + // and a whole develop document with them still loads + let mut d = crate::DevelopSettings::default(); + d.masks.push(crate::Mask { + id: 1, + components: vec![crate::MaskComponent { + name: None, + op: crate::MaskOp::Add, + invert: false, + shape: crate::MaskShape::Prompt { text: "x".into(), seg: None, detail: vec![SegMask::from_logits(4, &[1.0; 16]); 50], edge: 0.0 }, + }], + ..Default::default() + }); + let back: crate::DevelopSettings = serde_json::from_str(&serde_json::to_string(&d).unwrap()).unwrap(); + let crate::MaskShape::Prompt { detail, .. } = &back.masks[0].components[0].shape else { panic!("a Prompt") }; + assert_eq!(detail.len(), MAX_DETAIL); + } + #[test] fn damaged_data_is_none() { let mut m = SegMask::from_logits(8, &[1.0; 64]); diff --git a/crates/develop/src/settings.rs b/crates/develop/src/settings.rs index 736586fe..800f229b 100644 --- a/crates/develop/src/settings.rs +++ b/crates/develop/src/settings.rs @@ -782,7 +782,7 @@ pub enum MaskShape { seg: Option, /// Zoomed-in passes over parts of the image (higher resolution than `seg`), used /// inside their rectangles. - #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[serde(default, skip_serializing_if = "Vec::is_empty", deserialize_with = "crate::segmask::de_detail")] detail: Vec, /// Edge −100..100: below 0 harder (a steeper transition), above 0 softer (feathered). #[serde(default, skip_serializing_if = "is_zero")] @@ -794,7 +794,7 @@ pub enum MaskShape { text: String, #[serde(default, skip_serializing_if = "Option::is_none")] seg: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[serde(default, skip_serializing_if = "Vec::is_empty", deserialize_with = "crate::segmask::de_detail")] detail: Vec, /// Edge −100..100, as for `Object`. #[serde(default, skip_serializing_if = "is_zero")] diff --git a/crates/engine/src/cmd/masks.rs b/crates/engine/src/cmd/masks.rs index da96f967..1615fb06 100644 --- a/crates/engine/src/cmd/masks.rs +++ b/crates/engine/src/cmd/masks.rs @@ -5,7 +5,7 @@ use lightcraft_geom::Point; use serde_json::{Value, json}; use super::{CommandSpec, bad, bool_or, cmd, f64_or, has_active, ok, point, str_param}; -use crate::segment::Click; +use crate::segment::{Click, MAX_CLICKS}; use crate::{Result, Session}; fn shape_from(kind: &str, p: &Value, c: &str) -> Result { @@ -39,8 +39,10 @@ fn shape_from(kind: &str, p: &Value, c: &str) -> Result { .unwrap_or_default(); MaskShape::ColorRange { samples, refine: f64_or(p, "refine", 50.0) } } - "object" => MaskShape::Object { hint: points(p, "points"), exclude: points(p, "exclude"), seg: None, detail: vec![], edge: 0.0 }, - "prompt" => MaskShape::Prompt { text: str_param(p, "text").unwrap_or_default().trim().to_string(), seg: None, detail: vec![], edge: 0.0 }, + "object" => MaskShape::Object { hint: points(p, "points"), exclude: points(p, "exclude"), seg: seg_param(p), detail: vec![], edge: 0.0 }, + "prompt" => { + MaskShape::Prompt { text: str_param(p, "text").unwrap_or_default().trim().to_string(), seg: seg_param(p), detail: vec![], edge: 0.0 } + } other => { return Err(bad( c, @@ -58,19 +60,50 @@ fn points(p: &Value, key: &str) -> Vec { .unwrap_or_default() } -/// Compute the segmentation of a new AI shape (Object with clicks, Prompt): an Object -/// without clicks stays empty until the photo is clicked; a Prompt that matches nothing is -/// an error (nothing to select). -fn resolve_ai(s: &mut Session, shape: &mut MaskShape, c: &str) -> Result<()> { +/// A segmentation passed in (`seg`: a stored or replayed AI mask; the model isn't needed). +fn seg_param(p: &Value) -> Option { + p.get("seg").and_then(|v| serde_json::from_value(v.clone()).ok()) +} + +/// What a new AI shape still needs once it is added. +enum Later { + Nothing, + /// Background mode: these Object clicks are segmented on the worker. + Clicks(Vec, Vec), + /// Background mode: the Describe selection is computed first; the mask is added when found. + Text(String), +} + +/// Compute the segmentation of a new AI shape (Object with clicks, Prompt) — or, in background +/// mode, say what to queue. An Object without clicks stays empty until the photo is clicked +/// (the photo is analyzed meanwhile); a Prompt that matches nothing is an error. Shapes given +/// with their `seg` need no model. +fn resolve_ai(s: &mut Session, shape: &mut MaskShape, c: &str) -> Result { let id = s.active().ok_or_else(|| bad(c, "no active photo"))?; + let background = s.segmenter.background; match shape { + MaskShape::Object { seg: Some(_), .. } | MaskShape::Prompt { seg: Some(_), .. } => {} + MaskShape::Object { hint, exclude, .. } if !hint.is_empty() && background => { + s.segmenter.model_dir().map_err(ai_err)?; + return Ok(Later::Clicks(std::mem::take(hint), std::mem::take(exclude))); + } MaskShape::Object { hint, exclude, seg, .. } if !hint.is_empty() => { *seg = Some(s.segment_clicks(id, &clicks_of(hint, exclude)).map_err(ai_err)?); } MaskShape::Object { .. } => { - // the clicks come later: start analyzing the photo now, and refuse an Object mask that - // could never be computed (no model, missing file) rather than leave an empty one - s.segment_prepare(id).map_err(ai_err)?; + // the clicks come later: refuse an Object mask that could never be computed (no + // model) rather than leave an empty one; in the app, analyze the photo meanwhile + s.segmenter.model_dir().map_err(ai_err)?; + if background { + s.segment_prepare(id).map_err(ai_err)?; + } + } + MaskShape::Prompt { text, .. } if background => { + s.segmenter.model_dir().map_err(ai_err)?; + if text.trim().is_empty() { + return Err(ai_err("describe what to select")); + } + return Ok(Later::Text(text.clone())); } MaskShape::Prompt { text, seg, .. } => { let found = s.segment_text(id, text).map_err(ai_err)?; @@ -78,7 +111,17 @@ fn resolve_ai(s: &mut Session, shape: &mut MaskShape, c: &str) -> Result<()> { } _ => {} } - Ok(()) + Ok(Later::Nothing) +} + +/// Queue what a new AI component (component `comp` of mask `mask`) still needs. +fn queue_later(s: &mut Session, later: Later, mask: u32, comp: usize, mut out: Value) -> Result { + if let Later::Clicks(hint, exclude) = later { + let id = s.active().ok_or_else(|| ai_err("no active photo"))?; + s.segment_clicks_later(id, mask, comp, hint, exclude).map_err(ai_err)?; + out["pending"] = json!(true); + } + Ok(out) } /// AI mask failures are shown as they are (not as "invalid parameters"). @@ -86,7 +129,7 @@ fn ai_err(msg: impl Into) -> crate::EngineError { crate::EngineError::Other(msg.into()) } -fn clicks_of(include: &[Point], exclude: &[Point]) -> Vec { +pub(crate) fn clicks_of(include: &[Point], exclude: &[Point]) -> Vec { include.iter().map(|p| Click { at: *p, include: true }).chain(exclude.iter().map(|p| Click { at: *p, include: false })).collect() } @@ -115,15 +158,20 @@ pub fn specs() -> Vec { "Create New Mask", [], None, - "{kind: brush|linear|radial|sky|subject|background|luminanceRange|colorRange|object|prompt, ...shape params (start/end, center/rx/ry/angle/feather, lo/hi…; object: points/exclude [[x,y],…]; prompt: text), name?}", + "{kind: brush|linear|radial|sky|subject|background|luminanceRange|colorRange|object|prompt, ...shape params (start/end, center/rx/ry/angle/feather, lo/hi…; object: points/exclude [[x,y],…]; prompt: text; object/prompt: seg? a stored segmentation, no model needed), name?} — AI kinds need the SAM 3 model; in the app a prompt returns {pending} and the mask appears when found", has_active, |s, p| { let kind = str_param(p, "kind").unwrap_or("radial").to_string(); let mut shape = shape_from(&kind, p, "mask.add")?; - resolve_ai(s, &mut shape, "mask.add")?; + let later = resolve_ai(s, &mut shape, "mask.add")?; let name = str_param(p, "name").map(str::to_string); + if let Later::Text(text) = &later { + let id = s.active().ok_or_else(|| bad("mask.add", "no active photo"))?; + s.segment_text_later(id, None, "add", name, text).map_err(ai_err)?; + return Ok(json!({"pending": true})); + } let next = s.active().and_then(|id| s.develop_of(id)).map(|d| d.next_mask_id()).unwrap_or(1); - masks_edit(s, "mask.add", "Add Mask", |masks, active| { + let out = masks_edit(s, "mask.add", "Add Mask", |masks, active| { masks.push(Mask { id: next, name: name.unwrap_or_else(|| format!("Mask {next}")), @@ -132,7 +180,8 @@ pub fn specs() -> Vec { }); *active = Some(next); Ok(()) - }) + })?; + queue_later(s, later, next, 0, out) } ), cmd!( @@ -144,17 +193,29 @@ pub fn specs() -> Vec { has_active, |s, p| { let kind = str_param(p, "kind").unwrap_or("brush").to_string(); - let mut shape = shape_from(&kind, p, "mask.addComponent")?; - resolve_ai(s, &mut shape, "mask.addComponent")?; let op: MaskOp = serde_json::from_value(p.get("op").cloned().unwrap_or(json!("add"))).map_err(|e| bad("mask.addComponent", e.to_string()))?; let active = s.active_mask; let mid = mask_id(p, active, "mask.addComponent")?; - masks_edit(s, "mask.addComponent", "Edit Mask", |masks, _| { + let id = s.active().ok_or_else(|| bad("mask.addComponent", "no active photo"))?; + if !s.develop_of(id).is_some_and(|d| d.masks.iter().any(|m| m.id == mid)) { + return Err(bad("mask.addComponent", format!("no mask {mid}"))); + } + let mut shape = shape_from(&kind, p, "mask.addComponent")?; + let later = resolve_ai(s, &mut shape, "mask.addComponent")?; + if let Later::Text(text) = &later { + let op = p.get("op").and_then(Value::as_str).unwrap_or("add"); + s.segment_text_later(id, Some(mid), op, None, text).map_err(ai_err)?; + return Ok(json!({"pending": true})); + } + let mut comp = 0; + let out = masks_edit(s, "mask.addComponent", "Edit Mask", |masks, _| { let i = find(masks, mid, "mask.addComponent")?; masks[i].components.push(MaskComponent { name: None, op, invert: bool_or(p, "invert", false), shape }); + comp = masks[i].components.len() - 1; Ok(()) - }) + })?; + queue_later(s, later, mid, comp, out) } ), cmd!( @@ -290,11 +351,23 @@ pub fn specs() -> Vec { return Err(bad(c, "not an Object component")); }; let (mut hint, mut ex, edge) = (hint.clone(), ex.clone(), *edge); + // clicks still on their way to the model count too + if let Some(pending) = s.segmenter.pending_clicks().filter(|q| (q.photo, q.mask, q.comp) == (id, mid, k)) { + (hint, ex) = (pending.hint.clone(), pending.exclude.clone()); + } + if hint.len() + ex.len() >= MAX_CLICKS { + return Err(ai_err(format!("an Object selection takes at most {MAX_CLICKS} clicks; start a new one"))); + } if exclude { ex.push(at) } else { hint.push(at) } + if s.segmenter.background && !hint.is_empty() { + let out = json!({"include": hint.len(), "exclude": ex.len(), "pending": true}); + s.segment_clicks_later(id, mid, k, hint, ex).map_err(ai_err)?; + return Ok(out); + } let seg = if hint.is_empty() { None } else { Some(s.segment_clicks(id, &clicks_of(&hint, &ex)).map_err(ai_err)?) }; let out = json!({"include": hint.len(), "exclude": ex.len()}); masks_edit(s, c, "Object Mask", |masks, _| { @@ -346,6 +419,63 @@ pub fn specs() -> Vec { Ok(json!({"busy": s.segmenter.busy()})) } ), + cmd!( + query "segment.model.status", + "AI Mask Model Status", + [], + None, + "{} — whether this build has AI masks, whether the SAM 3 model is installed (and where), loaded, busy, and the download's progress → {available, installed, dir, loaded, busy, analyzing, sizeBytes, license, licenseUrl, mirrors, download: {running, done, total, file, error, finished}}", + super::always, + |s, _| { + let g = &s.segmenter; + Ok(json!({ + "available": crate::segment::Segmenter::AVAILABLE, + "installed": g.installed(), + "dir": g.dir.as_ref().map(|d| d.display().to_string()), + "loaded": g.loaded(), + "busy": g.busy(), + "analyzing": g.analyzing(), + "sizeBytes": crate::segment::MODEL_BYTES, + "license": crate::segment::LICENSE_NAME, + "licenseUrl": crate::segment::LICENSE_URL, + "mirrors": g.mirrors().len(), + "download": g.download_status(), + })) + } + ), + cmd!( + query "segment.model.download", + "Download AI Mask Model", + [], + None, + "{acknowledged: true} — download the SAM 3 model (about 3.4 GB, Meta's SAM License, not LightCraft's) in the background, from the configured mirrors; only after the user agreed to it. Watch segment.model.status; segment.model.cancel stops it (it resumes later) → {started, installed, downloading}", + super::always, + |s, p| { + let c = "segment.model.download"; + if p.get("acknowledged").and_then(Value::as_bool) != Some(true) { + return Err(bad( + c, + format!( + "the SAM 3 model is a {:.1} GB download under Meta's {} ({}): pass `acknowledged: true` once the user has agreed to download it", + crate::segment::MODEL_BYTES as f64 / 1e9, + crate::segment::LICENSE_NAME, + crate::segment::LICENSE_URL + ), + )); + } + let started = s.segmenter.start_download().map_err(ai_err)?; + Ok(json!({"started": started, "installed": s.segmenter.installed(), "downloading": s.segmenter.download_status().running})) + } + ), + cmd!( + query "segment.model.cancel", + "Cancel AI Mask Model Download", + [], + None, + "{} — stop the SAM 3 download (what has arrived is kept, and a new download resumes from it) → {cancelled}", + super::always, + |s, _| Ok(json!({"cancelled": s.segmenter.cancel_download()})) + ), cmd!( "mask.refine", "Refine Mask Edges", diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 5edcad67..ec39222a 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -727,6 +727,8 @@ mod tests_persist; #[cfg(test)] mod tests_prefs; #[cfg(test)] +mod tests_segment; +#[cfg(test)] mod tests_settings_files; #[cfg(test)] mod tests_spots; diff --git a/crates/engine/src/segment.rs b/crates/engine/src/segment.rs deleted file mode 100644 index 41aed09e..00000000 --- a/crates/engine/src/segment.rs +++ /dev/null @@ -1,588 +0,0 @@ -//! AI masks: Object (clicks) and Describe (text) selections computed with SAM 3 -//! (`lightcraft-segment`, cargo feature `sam`; the desktop app enables it). -//! -//! The model sees the photo as developed, uncropped and without masks, at 1008 px — the frame -//! mask coordinates live in — so a stored segmentation lines up with the photo at any size. -//! Encoding the image is the slow part (a few seconds on a laptop GPU); it runs once per photo -//! and look, on a worker thread when [`Session::segment_prepare`] is called (the Masking panel -//! does this when an AI mask is started), and each click then takes milliseconds. -//! -//! Without the feature (web, CLI) every request reports that this build has no model; stored -//! segmentations still render (they are plain data in the develop settings). - -use std::path::PathBuf; - -use lightcraft_catalog::PhotoId; -use lightcraft_develop::{MaskShape, SegMask}; -use lightcraft_geom::Point; - -use crate::Session; - -/// Long edge of the image the model sees. -pub const INPUT_EDGE: usize = 1008; - -/// One click of an Object selection (normalized image coordinates). -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct Click { - pub at: Point, - pub include: bool, -} - -/// What a request runs on: the photo, its look (settings without masks), the model input. -#[cfg_attr(not(feature = "sam"), allow(dead_code))] -struct Input { - key: u64, - rgb: Vec, - w: usize, - h: usize, -} - -#[cfg(feature = "sam")] -mod model { - use std::sync::atomic::{AtomicBool, Ordering}; - use std::sync::{Arc, Mutex, PoisonError}; - - use lightcraft_segment::{Encoded, Sam3}; - - #[derive(Default)] - pub struct Inner { - pub model: Option, - /// The last encoded image and its key. - pub cache: Option<(u64, Encoded)>, - } - - #[derive(Clone, Default)] - pub struct Shared { - pub inner: Arc>, - pub busy: Arc, - /// A zoomed-in detail pass is running. - pub detail_busy: Arc, - /// Finished detail passes, waiting for [`crate::Session::segment_poll`]. - pub results: Arc>>, - } - - impl Shared { - pub fn lock(&self) -> std::sync::MutexGuard<'_, Inner> { - self.inner.lock().unwrap_or_else(PoisonError::into_inner) - } - - pub fn busy(&self) -> bool { - self.busy.load(Ordering::Relaxed) - } - - pub fn set_busy(&self, v: bool) { - self.busy.store(v, Ordering::Relaxed); - } - } -} - -/// The model (loaded on first use) and the last encoded photo. -#[derive(Default)] -pub struct Segmenter { - /// Where the `facebook/sam3` checkpoint lives (set by the app). - pub dir: Option, - #[cfg(feature = "sam")] - shared: model::Shared, -} - -impl Segmenter { - /// Whether this build can compute AI masks at all. - pub const AVAILABLE: bool = cfg!(feature = "sam"); - - /// Whether a worker is preparing a photo right now. - pub fn busy(&self) -> bool { - #[cfg(feature = "sam")] - return self.shared.busy(); - #[cfg(not(feature = "sam"))] - false - } - - /// Whether a zoomed-in detail pass is running. - pub fn detail_busy(&self) -> bool { - #[cfg(feature = "sam")] - return self.shared.detail_busy.load(std::sync::atomic::Ordering::Relaxed); - #[cfg(not(feature = "sam"))] - false - } - - /// The model directory, if its files are there. - pub fn model_dir(&self) -> Result { - if !Self::AVAILABLE { - return Err("AI masks are not available in this build".into()); - } - let dir = self.dir.clone().ok_or("no folder is set for the SAM 3 model")?; - #[cfg(feature = "sam")] - if !lightcraft_segment::is_model_dir(&dir) { - return Err(format!( - "the SAM 3 model is not installed: put the facebook/sam3 files (model.safetensors, vocab.json, merges.txt) in {}", - dir.display() - )); - } - Ok(dir) - } -} - -/// Mix `v` into a 64-bit hash. -fn mix(h: u64, v: u64) -> u64 { - (h ^ v).wrapping_mul(0x0100_0000_01b3).rotate_left(29) -} - -/// A finished zoomed-in pass: for which component (and the shape it was computed from, so a -/// component changed meanwhile is left alone), and the patch (`None`: nothing found there). -#[cfg_attr(not(feature = "sam"), allow(dead_code))] -pub struct DetailResult { - photo: PhotoId, - mask: u32, - comp: usize, - shape: MaskShape, - patch: Option, -} - -/// What a detail pass re-runs. -#[cfg_attr(not(feature = "sam"), allow(dead_code))] -enum Prompt { - Clicks(Vec), - Text(String), -} - -/// Objects covering more than this much of the photo gain little from a zoomed pass. -const DETAIL_MAX_AREA: f64 = 0.4; - -/// The part of the photo to zoom into for `seg`'s selection: its bounding box with a margin of -/// a quarter of its size, at least 12 % of the long edge, at most 2:1, inside the photo -/// (`aspect` = width / height). `None` when nothing is selected or it is most of the photo. -pub fn detail_region(seg: &SegMask, aspect: f64) -> Option<[f64; 4]> { - let logits = seg.logits()?; - let r = seg.bounds()?; - let side = seg.side as usize; - let (mut x0, mut y0, mut x1, mut y1) = (usize::MAX, usize::MAX, 0, 0); - for (i, l) in logits.iter().enumerate() { - if *l > 0.0 { - let (x, y) = (i % side, i / side); - (x0, y0, x1, y1) = (x0.min(x), y0.min(y), x1.max(x + 1), y1.max(y + 1)); - } - } - if x0 == usize::MAX { - return None; - } - let (rw, rh) = (r[2] - r[0], r[3] - r[1]); - let nx = |c: usize| r[0] + c as f64 / side as f64 * rw; - let ny = |c: usize| r[1] + c as f64 / side as f64 * rh; - let (bx0, by0, bx1, by1) = (nx(x0), ny(y0), nx(x1), ny(y1)); - if (bx1 - bx0) * (by1 - by0) > DETAIL_MAX_AREA { - return None; - } - // in units of the photo's height: x × aspect - let a = aspect.clamp(0.1, 10.0); - let (cx, cy) = ((bx0 + bx1) / 2.0 * a, (by0 + by1) / 2.0); - let size = ((bx1 - bx0) * a).max(by1 - by0); - let m = 0.25 * size; - let mut w = ((bx1 - bx0) * a + 2.0 * m).max(0.12 * a.max(1.0)); - let mut h = ((by1 - by0) + 2.0 * m).max(0.12 * a.max(1.0)); - w = w.max(h / 2.0).min(a); - h = h.max(w / 2.0).min(1.0); - let x = (cx - w / 2.0).clamp(0.0, (a - w).max(0.0)); - let y = (cy - h / 2.0).clamp(0.0, (1.0 - h).max(0.0)); - let out = [x / a, y, (x + w) / a, y + h]; - // a region nearly the whole photo isn't worth a second pass - ((out[2] - out[0]) * (out[3] - out[1]) < 0.8).then_some(out) -} - -impl Session { - /// The key and settings of photo `id` as the model sees it: its look without masks, uncropped. - fn segment_key(&self, id: PhotoId) -> Option<(u64, lightcraft_develop::DevelopSettings)> { - let p = self.catalog.photo(id)?; - let mut d = (*p.develop).clone(); - d.masks.clear(); - let content = crate::media::content_key(p); - let mut h = mix(0xcbf2_9ce4_8422_2325, d.hash64()); - h = mix(h, id.0); - for b in content.bytes() { - h = mix(h, u64::from(b)); - } - Some((h, d)) - } - - /// Render the model input for `id`. - #[cfg_attr(not(feature = "sam"), allow(dead_code))] - fn segment_input(&mut self, id: PhotoId) -> Result { - let (key, d) = self.segment_key(id).ok_or("no such photo")?; - let job = self.preview_job(id, INPUT_EDGE, INPUT_EDGE, false, &d).ok_or("no such photo")?; - let r = job.run(); - self.accept(&r); - let img = match r.rendered { - Ok(r) => r.image, - Err(e) => { - // the usual reason: the file was moved or deleted (and there's no smart preview) - if let Some(lightcraft_catalog::Source::File { path }) = self.catalog.photo(id).map(|p| &p.source) - && !std::path::Path::new(path).exists() - { - return Err(format!( - "This photo's file is missing (moved or deleted), so it can't be analyzed for AI masks: {path}. Library ▸ Find Missing Photos can relink it." - )); - } - return Err(format!("couldn't render the photo for AI masks: {e}")); - } - }; - let rgb = img.data.iter().flat_map(|p| [p[0], p[1], p[2]]).collect(); - Ok(Input { key, rgb, w: img.width, h: img.height }) - } - - /// Start preparing photo `id` for AI masks on a worker thread (load the model, encode the - /// image). Returns at once; does nothing when it is ready or being prepared. - pub fn segment_prepare(&mut self, id: PhotoId) -> Result<(), String> { - let dir = self.segmenter.model_dir()?; - #[cfg(feature = "sam")] - { - let (key, _) = self.segment_key(id).ok_or("no such photo")?; - let shared = self.segmenter.shared.clone(); - if shared.busy() { - return Ok(()); - } - if let Ok(inner) = shared.inner.try_lock() - && inner.cache.as_ref().is_some_and(|c| c.0 == key) - { - return Ok(()); - } - let input = self.segment_input(id)?; - shared.set_busy(true); - let spawned = std::thread::Builder::new().name("sam3-prepare".into()).spawn(move || { - let mut inner = shared.lock(); - if let Err(e) = prepare(&mut inner, &dir, &input) { - log::warn!("SAM 3: {e}"); - } - shared.set_busy(false); - }); - if let Err(e) = spawned { - self.segmenter.shared.set_busy(false); - return Err(format!("could not start the model: {e}")); - } - } - #[cfg(not(feature = "sam"))] - let _ = (dir, id); - Ok(()) - } - - /// Segment the object `clicks` pick on photo `id` (waits for a running prepare). - pub fn segment_clicks(&mut self, id: PhotoId, clicks: &[Click]) -> Result { - let dir = self.segmenter.model_dir()?; - if !clicks.iter().any(|c| c.include) { - return Err("click the object to include it first".into()); - } - #[cfg(feature = "sam")] - { - let input = self.segment_input_if_needed(id)?; - let shared = self.segmenter.shared.clone(); - let mut inner = shared.lock(); - let enc = encoded(&mut inner, &dir, input)?; - let clicks: Vec = - clicks.iter().map(|c| lightcraft_segment::Click { x: c.at.x as f32, y: c.at.y as f32, positive: c.include }).collect(); - let (model, enc) = enc; - let p = model.segment_clicks(enc, &clicks).map_err(|e| e.to_string())?; - Ok(SegMask::from_logits(lightcraft_segment::MASK_SIDE, &p.logits)) - } - #[cfg(not(feature = "sam"))] - { - let _ = (dir, id); - Err("AI masks are not available in this build".into()) - } - } - - /// Segment everything `text` describes on photo `id`; `Ok(None)` when nothing matches. - pub fn segment_text(&mut self, id: PhotoId, text: &str) -> Result, String> { - let dir = self.segmenter.model_dir()?; - let text = text.trim(); - if text.is_empty() { - return Err("describe what to select".into()); - } - #[cfg(feature = "sam")] - { - let input = self.segment_input_if_needed(id)?; - let shared = self.segmenter.shared.clone(); - let mut inner = shared.lock(); - let (model, enc) = encoded(&mut inner, &dir, input)?; - // "car, road": each phrase on its own, merged (the per-pixel maximum) - let mut merged: Option> = None; - for phrase in text.split([',', ';']).map(str::trim).filter(|p| !p.is_empty()) { - if let Some(p) = model.segment_text(enc, phrase, 0.5).map_err(|e| e.to_string())? { - merged = Some(match merged { - None => p.logits, - Some(m) => m.iter().zip(&p.logits).map(|(a, b)| a.max(*b)).collect(), - }); - } - } - Ok(merged.map(|l| SegMask::from_logits(lightcraft_segment::MASK_SIDE, &l))) - } - #[cfg(not(feature = "sam"))] - { - let _ = (dir, id); - Err("AI masks are not available in this build".into()) - } - } - - /// Render the part `region` (normalized) of photo `id` for the model, from a source sharp - /// enough for it. - #[cfg_attr(not(feature = "sam"), allow(dead_code))] - fn segment_region_input(&mut self, id: PhotoId, region: [f64; 4]) -> Result { - let (_, mut d) = self.segment_key(id).ok_or("no such photo")?; - d.crop.geometry = lightcraft_geom::CropGeometry { rect: lightcraft_geom::Rect::new(region[0], region[1], region[2], region[3]), angle: 0.0 }; - d.crop.flip_h = false; - d.crop.flip_v = false; - d.geometry.constrain_crop = false; - let mut job = self.preview_job(id, INPUT_EDGE, INPUT_EDGE, true, &d).ok_or("no such photo")?; - let frac = (region[2] - region[0]).max(region[3] - region[1]).max(0.01); - let p = self.catalog.photo(id).ok_or("no such photo")?.clone(); - job.level = crate::media::SourceLevel::for_size((INPUT_EDGE as f64 / frac) as usize); - job.source = self.media.source_ref(&p, job.level); - let r = job.run(); - self.accept(&r); - let img = r.rendered.map_err(|e| format!("couldn't render the photo for AI masks: {e}"))?.image; - let rgb = img.data.iter().flat_map(|p| [p[0], p[1], p[2]]).collect(); - Ok(Input { key: 0, rgb, w: img.width, h: img.height }) - } - - /// Start a zoomed-in pass for component `comp` of mask `mask` on photo `id` (an Object or - /// Describe selection): the photo around the selection is rendered at a higher resolution - /// and segmented again, on a worker thread; [`Session::segment_poll`] applies the result. - /// `Ok(false)` when there is nothing to refine (no selection yet, or it is most of the photo). - pub fn segment_detail(&mut self, id: PhotoId, mask: u32, comp: usize) -> Result { - let dir = self.segmenter.model_dir()?; - let d = self.develop_of(id).ok_or("no such photo")?; - let shape = d.masks.iter().find(|m| m.id == mask).and_then(|m| m.components.get(comp)).map(|c| c.shape.clone()).ok_or("no such mask")?; - let (seg, prompt) = match &shape { - MaskShape::Object { hint, exclude, seg: Some(seg), .. } if !hint.is_empty() => ( - seg.clone(), - Prompt::Clicks( - hint.iter().map(|p| Click { at: *p, include: true }).chain(exclude.iter().map(|p| Click { at: *p, include: false })).collect(), - ), - ), - MaskShape::Prompt { text, seg: Some(seg), .. } => (seg.clone(), Prompt::Text(text.clone())), - _ => return Ok(false), - }; - let p = self.catalog.photo(id).ok_or("no such photo")?; - let (pw, ph) = (f64::from(p.width.max(1)), f64::from(p.height.max(1))); - let aspect = if d.orientation.swaps_axes() { ph / pw } else { pw / ph }; - let Some(region) = detail_region(&seg, aspect) else { return Ok(false) }; - #[cfg(feature = "sam")] - { - let shared = self.segmenter.shared.clone(); - if shared.detail_busy.swap(true, std::sync::atomic::Ordering::Relaxed) { - return Ok(false); - } - let input = match self.segment_region_input(id, region) { - Ok(i) => i, - Err(e) => { - shared.detail_busy.store(false, std::sync::atomic::Ordering::Relaxed); - return Err(e); - } - }; - let shape = strip_detail(shape); - let spawned = std::thread::Builder::new().name("sam3-detail".into()).spawn({ - let shared = shared.clone(); - move || { - let patch = { - let mut inner = shared.lock(); - detail_pass(&mut inner, &dir, &input, &prompt, region) - }; - match patch { - Ok(patch) => { - let r = DetailResult { photo: id, mask, comp, shape, patch }; - shared.results.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push(r); - } - Err(e) => log::warn!("SAM 3 detail pass: {e}"), - } - shared.detail_busy.store(false, std::sync::atomic::Ordering::Relaxed); - } - }); - if let Err(e) = spawned { - shared.detail_busy.store(false, std::sync::atomic::Ordering::Relaxed); - return Err(format!("could not start the detail pass: {e}")); - } - Ok(true) - } - #[cfg(not(feature = "sam"))] - { - let _ = (dir, prompt, region, mask); - Ok(false) - } - } - - /// Apply finished detail passes (to components that haven't changed since). True when a - /// mask changed. - pub fn segment_poll(&mut self) -> bool { - #[cfg(feature = "sam")] - { - let done: Vec = - std::mem::take(&mut *self.segmenter.shared.results.lock().unwrap_or_else(std::sync::PoisonError::into_inner)); - let mut changed = false; - for r in done { - let Some(patch) = r.patch else { continue }; - let Some(d) = self.develop_of(r.photo) else { continue }; - let mut d = (*d).clone(); - let Some(c) = d.masks.iter_mut().find(|m| m.id == r.mask).and_then(|m| m.components.get_mut(r.comp)) else { continue }; - if strip_detail(c.shape.clone()) != r.shape { - continue; // clicked again meanwhile: a newer pass will follow - } - match &mut c.shape { - MaskShape::Object { detail, .. } | MaskShape::Prompt { detail, .. } => *detail = vec![patch], - _ => continue, - } - match self.set_develop(r.photo, d, "AI Mask Detail") { - Ok(()) => changed = true, - Err(e) => log::warn!("AI mask detail: {e}"), - } - } - changed - } - #[cfg(not(feature = "sam"))] - false - } - - /// The model input for `id`, unless the cache already holds it (or a prepare is running, - /// which will have it). - #[cfg(feature = "sam")] - fn segment_input_if_needed(&mut self, id: PhotoId) -> Result { - let (key, _) = self.segment_key(id).ok_or("no such photo")?; - if self.segmenter.shared.busy() { - return Ok(Either::Key(key)); - } - let cached = self.segmenter.shared.lock().cache.as_ref().is_some_and(|c| c.0 == key); - if cached { Ok(Either::Key(key)) } else { Ok(Either::Input(self.segment_input(id)?)) } - } -} - -#[cfg(feature = "sam")] -enum Either { - /// Already encoded (or being encoded) under this key. - Key(u64), - Input(Input), -} - -#[cfg(feature = "sam")] -fn load(inner: &mut model::Inner, dir: &std::path::Path) -> Result<(), String> { - if inner.model.is_none() { - let t = web_time::Instant::now(); - inner.model = Some(lightcraft_segment::Sam3::load(dir).map_err(|e| e.to_string())?); - log::info!("SAM 3 loaded in {:?}", t.elapsed()); - } - Ok(()) -} - -/// Load the model and encode `input` into the cache (with the click features ready). -#[cfg(feature = "sam")] -fn prepare(inner: &mut model::Inner, dir: &std::path::Path, input: &Input) -> Result<(), String> { - load(inner, dir)?; - let model = inner.model.as_mut().ok_or("the model did not load")?; - let t = web_time::Instant::now(); - let mut enc = model.encode(&input.rgb, input.w, input.h).map_err(|e| e.to_string())?; - model.prepare_clicks(&mut enc).map_err(|e| e.to_string())?; - log::info!("SAM 3 encoded {}×{} in {:?}", input.w, input.h, t.elapsed()); - inner.cache = Some((input.key, enc)); - Ok(()) -} - -/// The model and the encoded image for `want` (encoding it now when needed). -#[cfg(feature = "sam")] -fn encoded<'a>( - inner: &'a mut model::Inner, - dir: &std::path::Path, - want: Either, -) -> Result<(&'a mut lightcraft_segment::Sam3, &'a mut lightcraft_segment::Encoded), String> { - match want { - Either::Input(input) => prepare(inner, dir, &input)?, - Either::Key(key) => { - if !inner.cache.as_ref().is_some_and(|c| c.0 == key) { - return Err("the photo changed while it was being prepared; try again".into()); - } - } - } - let model::Inner { model, cache } = inner; - match (model.as_mut(), cache.as_mut()) { - (Some(m), Some((_, e))) => Ok((m, e)), - _ => Err("the model is not ready".into()), - } -} - -/// `shape` without its detail patches (what a detail pass was computed from) -#[cfg_attr(not(feature = "sam"), allow(dead_code))] -fn strip_detail(mut shape: MaskShape) -> MaskShape { - // (nor its edge setting: moving the slider meanwhile keeps the pass) - if let MaskShape::Object { detail, edge, .. } | MaskShape::Prompt { detail, edge, .. } = &mut shape { - detail.clear(); - *edge = 0.0; - } - shape -} - -/// Segment `input` (the photo's part `region`) with `prompt`, as a patch over `region`. -#[cfg(feature = "sam")] -fn detail_pass(inner: &mut model::Inner, dir: &std::path::Path, input: &Input, prompt: &Prompt, region: [f64; 4]) -> Result, String> { - load(inner, dir)?; - let model = inner.model.as_mut().ok_or("the model did not load")?; - let t = web_time::Instant::now(); - let mut enc = model.encode(&input.rgb, input.w, input.h).map_err(|e| e.to_string())?; - let (rw, rh) = (region[2] - region[0], region[3] - region[1]); - let logits = match prompt { - Prompt::Clicks(clicks) => { - let inside: Vec = clicks - .iter() - .map(|c| ((c.at.x - region[0]) / rw, (c.at.y - region[1]) / rh, c.include)) - .filter(|(x, y, _)| (0.0..=1.0).contains(x) && (0.0..=1.0).contains(y)) - .map(|(x, y, positive)| lightcraft_segment::Click { x: x as f32, y: y as f32, positive }) - .collect(); - if !inside.iter().any(|c| c.positive) { - return Ok(None); - } - Some(model.segment_clicks(&mut enc, &inside).map_err(|e| e.to_string())?.logits) - } - Prompt::Text(text) => { - let mut merged: Option> = None; - for phrase in text.split([',', ';']).map(str::trim).filter(|p| !p.is_empty()) { - if let Some(p) = model.segment_text(&mut enc, phrase, 0.5).map_err(|e| e.to_string())? { - merged = Some(match merged { - None => p.logits, - Some(m) => m.iter().zip(&p.logits).map(|(a, b)| a.max(*b)).collect(), - }); - } - } - merged - } - }; - log::info!("SAM 3 detail pass ({}×{}) in {:?}", input.w, input.h, t.elapsed()); - Ok(logits.map(|l| SegMask::from_logits_in(lightcraft_segment::MASK_SIDE, &l, region))) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn seg_box(side: usize, x0: usize, y0: usize, x1: usize, y1: usize) -> SegMask { - let l: Vec = - (0..side * side).map(|i| if (x0..x1).contains(&(i % side)) && (y0..y1).contains(&(i / side)) { 8.0 } else { -8.0 }).collect(); - SegMask::from_logits(side, &l) - } - - #[test] - fn detail_region_frames_the_selection_with_a_margin() { - // a small object in the middle of a square photo - let r = detail_region(&seg_box(100, 40, 45, 50, 55), 1.0).unwrap(); - assert!(r[0] < 0.4 && r[2] > 0.5 && r[1] < 0.45 && r[3] > 0.55, "contains it: {r:?}"); - assert!(r[2] - r[0] < 0.25 && r[3] - r[1] < 0.25, "zoomed in: {r:?}"); - assert!(r.iter().all(|v| (0.0..=1.0).contains(v))); - // at the corner: shifted inside the photo, not cut - let r = detail_region(&seg_box(100, 0, 0, 5, 5), 1.5).unwrap(); - assert!(r[0] >= 0.0 && r[1] >= 0.0 && r[0] < 1e-9 && r[1] < 1e-9); - // a thin object stays at most 2:1 (in photo pixels, 1.5:1 photo) - let r = detail_region(&seg_box(100, 10, 50, 90, 52), 1.5).unwrap(); - let (w, h) = ((r[2] - r[0]) * 1.5, r[3] - r[1]); - assert!(w / h <= 2.0 + 1e-9, "{w} × {h}"); - } - - #[test] - fn no_region_for_nothing_or_most_of_the_photo() { - assert!(detail_region(&seg_box(50, 0, 0, 0, 0), 1.0).is_none()); - assert!(detail_region(&seg_box(50, 2, 2, 48, 48), 1.0).is_none()); - let damaged = SegMask { side: 50, data: "?".into(), rect: None }; - assert!(detail_region(&damaged, 1.0).is_none()); - } -} diff --git a/crates/engine/src/segment/download.rs b/crates/engine/src/segment/download.rs new file mode 100644 index 00000000..875d652f --- /dev/null +++ b/crates/engine/src/segment/download.rs @@ -0,0 +1,146 @@ +//! The model download as a background task the session starts, watches and cancels without +//! ever blocking: progress lives in atomics, messages behind a mutex only ever `try_lock`ed +//! by the session (the download thread holds it for a few instructions at a time). + +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +use lightcraft_segment::fetch::{self, Options, Progress}; + +/// What the session sees of a download. +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)] +pub struct Status { + pub running: bool, + pub done: u64, + pub total: u64, + /// The file being fetched. + pub file: String, + /// The last download's error (cleared when a new one starts). + pub error: Option, + /// The last download finished and the files are in place. + pub finished: bool, +} + +#[derive(Default)] +struct Text { + file: String, + error: Option, +} + +#[derive(Default)] +struct Shared { + running: AtomicBool, + cancel: AtomicBool, + finished: AtomicBool, + done: AtomicU64, + total: AtomicU64, + text: Mutex, +} + +/// Clears `running` when the download thread ends, however it ends. +struct Running(Arc); + +impl Drop for Running { + fn drop(&mut self) { + self.0.running.store(false, Ordering::SeqCst); + } +} + +#[derive(Default)] +pub struct Downloader { + shared: Arc, + /// The last text seen (when the download thread held the lock at that moment). + last: Mutex, +} + +fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, Text> { + m.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + +impl Downloader { + pub fn status(&self) -> Status { + let s = &self.shared; + if let Ok(t) = s.text.try_lock() + && let Ok(mut last) = self.last.try_lock() + { + last.file.clone_from(&t.file); + last.error.clone_from(&t.error); + } + // (only the session's thread locks `last`) + let (file, error) = self.last.try_lock().map(|l| (l.file.clone(), l.error.clone())).unwrap_or_default(); + Status { + running: s.running.load(Ordering::SeqCst), + done: s.done.load(Ordering::SeqCst), + total: s.total.load(Ordering::SeqCst), + file, + error, + finished: s.finished.load(Ordering::SeqCst), + } + } + + pub fn running(&self) -> bool { + self.shared.running.load(Ordering::SeqCst) + } + + /// Ask a running download to stop (it keeps its partial files to resume later). + pub fn cancel(&self) -> bool { + let running = self.running(); + if running { + self.shared.cancel.store(true, Ordering::SeqCst); + } + running + } + + /// Start downloading `files` from `mirrors` into `dir` on a background thread. False when + /// one is already running. + pub fn start(&self, files: &'static [fetch::FileSpec], mirrors: Vec, dir: PathBuf, opts: Options) -> Result { + let s = self.shared.clone(); + if s.running.swap(true, Ordering::SeqCst) { + return Ok(false); + } + s.cancel.store(false, Ordering::SeqCst); + s.finished.store(false, Ordering::SeqCst); + s.done.store(0, Ordering::SeqCst); + s.total.store(files.iter().filter_map(|f| f.size).sum(), Ordering::SeqCst); + // nothing else runs while `running` was false + *lock(&s.text) = Text::default(); + *lock(&self.last) = Text::default(); + let guard = Running(s.clone()); + let spawned = std::thread::Builder::new().name("sam3-download".into()).spawn(move || { + let s = guard.0.clone(); + let mut last_file = String::new(); + let mut on_progress = |p: &Progress| { + s.done.store(p.done, Ordering::SeqCst); + s.total.store(p.total, Ordering::SeqCst); + if p.file != last_file { + last_file.clone_from(&p.file); + lock(&s.text).file.clone_from(&p.file); + } + }; + let r = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| fetch::download(files, &mirrors, &dir, &opts, &s.cancel, &mut on_progress))); + let error = match r { + Ok(Ok(())) => { + s.finished.store(true, Ordering::SeqCst); + log::info!("SAM 3 model downloaded to {}", dir.display()); + None + } + Ok(Err(e)) => Some(e.to_string()), + Err(_) => Some("the download failed unexpectedly".to_string()), + }; + if let Some(e) = &error { + log::warn!("SAM 3 download: {e}"); + } + lock(&s.text).error = error; + drop(guard); + }); + match spawned { + Ok(_) => Ok(true), + Err(e) => { + // the closure (and its guard) was dropped: `running` is false again + Err(format!("could not start the download: {e}")) + } + } + } +} diff --git a/crates/engine/src/segment/mod.rs b/crates/engine/src/segment/mod.rs new file mode 100644 index 00000000..84397042 --- /dev/null +++ b/crates/engine/src/segment/mod.rs @@ -0,0 +1,715 @@ +//! AI masks: Object (clicks) and Describe (text) selections computed with SAM 3 +//! (`lightcraft-segment`, cargo feature `sam`; the desktop app enables it). +//! +//! **Nothing requires the model.** It is not part of LightCraft (SAM License); the user +//! downloads it when they first use an AI mask and agree to (`segment.model.download`), or +//! puts the files in the model folder themselves. Without it, AI mask requests fail with a +//! clear "not installed" error; renders, exports and every other feature never touch it +//! (stored segmentations are plain data in the develop settings). +//! +//! The model sees the photo as developed, uncropped and without masks, at 1008 px — the frame +//! mask coordinates live in — so a stored segmentation lines up with the photo at any size. +//! Everything that touches the model (loading it, rendering and encoding the photo, clicks, +//! descriptions, detail passes) runs on one worker thread ([`worker`]); the session talks to it +//! over channels and never waits on a lock. With [`Segmenter::background`] (the desktop app) +//! commands return at once and [`Session::segment_poll`] applies the results; without it (CLI, +//! MCP, tests) they wait for the worker. + +#[cfg(feature = "sam")] +mod download; +#[cfg(feature = "sam")] +mod worker; + +use std::path::PathBuf; + +use lightcraft_catalog::PhotoId; +use lightcraft_develop::{MaskShape, SegMask}; +use lightcraft_geom::Point; + +use crate::Session; + +/// Long edge of the image the model sees. +pub const INPUT_EDGE: usize = 1008; +/// Download size of the model (`model.safetensors`; the tokenizer files add ~2 MB). +pub const MODEL_BYTES: u64 = 3_439_938_512; +/// The model's licence (not LightCraft's): shown before downloading. +pub const LICENSE_NAME: &str = "SAM License (Meta)"; +pub const LICENSE_URL: &str = "https://github.com/facebookresearch/sam3/blob/main/LICENSE"; +/// How errors about a missing model start (the UI offers the download on it). +pub const NOT_INSTALLED: &str = "The SAM 3 model is not installed"; +/// Longest a waiting command (CLI, MCP) waits for the model (loading on a CPU can take minutes). +#[cfg_attr(not(feature = "sam"), allow(dead_code))] +const WAIT: std::time::Duration = std::time::Duration::from_secs(15 * 60); +/// Most clicks one Object selection takes (the model's prompt grows with each). +pub const MAX_CLICKS: usize = 64; + +/// One click of an Object selection (normalized image coordinates). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Click { + pub at: Point, + pub include: bool, +} + +/// What a request's result is for (how [`Session::segment_poll`] applies it). +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(not(feature = "sam"), allow(dead_code))] +pub(crate) enum Tag { + /// A command waiting for the result itself. + Wait, + /// Load the model and encode the photo ahead of the first click. + Prepare, + /// Object clicks on component `comp` of mask `mask`. + Clicks { photo: PhotoId, mask: u32, comp: usize, hint: Vec, exclude: Vec }, + /// A Describe selection: a new mask (`mask: None`) or a component combined by `op`. + NewPrompt { photo: PhotoId, mask: Option, op: String, name: Option, text: String }, + /// A zoomed-in pass for a component, computed from `shape` (without its detail). + Detail { photo: PhotoId, mask: u32, comp: usize, shape: MaskShape }, +} + +/// The model input: 8-bit RGB. +#[cfg_attr(not(feature = "sam"), allow(dead_code))] +pub(crate) struct Input { + rgb: Vec, + w: usize, + h: usize, +} + +/// What a detail pass re-runs. +#[cfg_attr(not(feature = "sam"), allow(dead_code))] +pub(crate) enum Prompt { + Clicks(Vec), + Text(String), +} + +/// Object clicks sent to the worker and not applied yet (background mode): further clicks add +/// to these, and the loupe shows them. +#[derive(Clone, Debug, PartialEq)] +pub struct PendingClicks { + pub photo: PhotoId, + pub mask: u32, + pub comp: usize, + pub hint: Vec, + pub exclude: Vec, +} + +/// What [`Session::segment_poll`] did. +#[derive(Debug, Default)] +pub struct Polled { + /// A mask changed. + pub changed: bool, + /// Errors and notices from background work, for the user. + pub messages: Vec, + /// A mask that got a new selection (worth a detail pass). + pub refine: Option, +} + +/// The model download (`segment.model.*`). +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)] +pub struct DownloadStatus { + pub running: bool, + pub done: u64, + pub total: u64, + pub file: String, + pub error: Option, + pub finished: bool, +} + +/// The model (on its worker thread), the download, and AI mask requests in flight. +pub struct Segmenter { + /// Where the `facebook/sam3` files live (set by the app; `None`: no AI masks). + pub dir: Option, + /// The user's list of download mirrors (one base URL per line), if any. + pub mirrors_file: Option, + /// Run requests in the background and apply them in [`Session::segment_poll`] (the + /// desktop app); otherwise commands wait for their result (CLI, MCP, tests). + pub background: bool, + pending: Option, + messages: Vec, + #[cfg(feature = "sam")] + worker: worker::Worker, + #[cfg(feature = "sam")] + download: download::Downloader, + #[cfg(feature = "sam")] + results: (std::sync::mpsc::Sender, std::sync::mpsc::Receiver), +} + +// (derivable only without the `sam` feature: the results channel has no Default) +#[allow(clippy::derivable_impls)] +impl Default for Segmenter { + fn default() -> Self { + Segmenter { + dir: None, + mirrors_file: None, + background: false, + pending: None, + messages: Vec::new(), + #[cfg(feature = "sam")] + worker: worker::Worker::default(), + #[cfg(feature = "sam")] + download: download::Downloader::default(), + #[cfg(feature = "sam")] + results: std::sync::mpsc::channel(), + } + } +} + +impl Segmenter { + /// Whether this build can compute AI masks at all. + pub const AVAILABLE: bool = cfg!(feature = "sam"); + + /// Whether the model's files are in place. + pub fn installed(&self) -> bool { + #[cfg(feature = "sam")] + if let Some(dir) = &self.dir { + return lightcraft_segment::is_model_dir(dir); + } + false + } + + /// Whether requests are queued or running (loading, analyzing the photo, a click…). + pub fn busy(&self) -> bool { + #[cfg(feature = "sam")] + return self.worker.pending() > 0; + #[cfg(not(feature = "sam"))] + false + } + + /// Whether the slow part runs now: loading the model or analyzing a photo. + pub fn analyzing(&self) -> bool { + #[cfg(feature = "sam")] + return self.worker.encoding(); + #[cfg(not(feature = "sam"))] + false + } + + /// Whether a zoomed-in detail pass is queued or running. + pub fn detail_busy(&self) -> bool { + #[cfg(feature = "sam")] + return self.worker.detail() > 0; + #[cfg(not(feature = "sam"))] + false + } + + /// Whether the model is in memory (it is unloaded after a while without use). + pub fn loaded(&self) -> bool { + #[cfg(feature = "sam")] + return self.worker.loaded(); + #[cfg(not(feature = "sam"))] + false + } + + /// Clicks waiting for their segmentation. + pub fn pending_clicks(&self) -> Option<&PendingClicks> { + self.pending.as_ref() + } + + /// The model download's state. + pub fn download_status(&self) -> DownloadStatus { + #[cfg(feature = "sam")] + { + let s = self.download.status(); + DownloadStatus { running: s.running, done: s.done, total: s.total, file: s.file, error: s.error, finished: s.finished } + } + #[cfg(not(feature = "sam"))] + DownloadStatus::default() + } + + /// Where the model is downloaded from, in order (the user's mirrors first). + pub fn mirrors(&self) -> Vec { + #[cfg(feature = "sam")] + { + let env = std::env::var(lightcraft_segment::fetch::MIRRORS_ENV).ok(); + lightcraft_segment::fetch::mirrors(env.as_deref(), self.mirrors_file.as_deref()) + } + #[cfg(not(feature = "sam"))] + Vec::new() + } + + /// The model folder, if the model is there; otherwise an error that says what to do. + pub fn model_dir(&self) -> Result { + if !Self::AVAILABLE { + return Err("AI masks are not available in this build".into()); + } + let dir = self.dir.clone().ok_or("no folder is set for the SAM 3 model")?; + if self.installed() { + return Ok(dir); + } + let d = self.download_status(); + if d.running { + let pct = d.done.saturating_mul(100).checked_div(d.total).unwrap_or(0); + return Err(format!("{NOT_INSTALLED} yet: it is downloading ({pct} %).")); + } + Err(format!( + "{NOT_INSTALLED}. Download it (about {:.1} GB, {LICENSE_NAME}) when LightCraft offers it, with `segment.model.download {{\"acknowledged\": true}}`, or put model.safetensors, vocab.json and merges.txt in {}.", + MODEL_BYTES as f64 / 1e9, + dir.display() + )) + } + + /// Start downloading the model (on a background thread; see `segment.model.download`). + /// `Ok(false)` when it is installed or downloading already. + pub fn start_download(&self) -> Result { + if !Self::AVAILABLE { + return Err("AI masks are not available in this build".into()); + } + let dir = self.dir.clone().ok_or("no folder is set for the SAM 3 model")?; + if self.installed() { + return Ok(false); + } + #[cfg(feature = "sam")] + { + let mirrors = self.mirrors(); + if mirrors.is_empty() { + return Err(lightcraft_segment::fetch::DownloadError::NoMirrors.to_string()); + } + self.download.start(lightcraft_segment::fetch::SAM3_FILES, mirrors, dir, lightcraft_segment::fetch::Options::default()) + } + #[cfg(not(feature = "sam"))] + { + let _ = dir; + Ok(false) + } + } + + /// Stop a running download (its partial files stay, to resume). False when none runs. + pub fn cancel_download(&self) -> bool { + #[cfg(feature = "sam")] + return self.download.cancel(); + #[cfg(not(feature = "sam"))] + false + } +} + +/// Mix `v` into a 64-bit hash. +#[cfg_attr(not(feature = "sam"), allow(dead_code))] +fn mix(h: u64, v: u64) -> u64 { + (h ^ v).wrapping_mul(0x0100_0000_01b3).rotate_left(29) +} + +/// Objects covering more than this much of the photo gain little from a zoomed pass. +const DETAIL_MAX_AREA: f64 = 0.4; + +/// The part of the photo to zoom into for `seg`'s selection: its bounding box with a margin of +/// a quarter of its size, at least 12 % of the long edge, at most 2:1, inside the photo +/// (`aspect` = width / height). `None` when nothing is selected or it is most of the photo. +pub fn detail_region(seg: &SegMask, aspect: f64) -> Option<[f64; 4]> { + let logits = seg.logits()?; + let r = seg.bounds()?; + let side = seg.side as usize; + let (mut x0, mut y0, mut x1, mut y1) = (usize::MAX, usize::MAX, 0, 0); + for (i, l) in logits.iter().enumerate() { + if *l > 0.0 { + let (x, y) = (i % side, i / side); + (x0, y0, x1, y1) = (x0.min(x), y0.min(y), x1.max(x + 1), y1.max(y + 1)); + } + } + if x0 == usize::MAX { + return None; + } + let (rw, rh) = (r[2] - r[0], r[3] - r[1]); + let nx = |c: usize| r[0] + c as f64 / side as f64 * rw; + let ny = |c: usize| r[1] + c as f64 / side as f64 * rh; + let (bx0, by0, bx1, by1) = (nx(x0), ny(y0), nx(x1), ny(y1)); + if (bx1 - bx0) * (by1 - by0) > DETAIL_MAX_AREA { + return None; + } + // in units of the photo's height: x × aspect + let a = if aspect.is_finite() { aspect.clamp(0.1, 10.0) } else { 1.0 }; + let (cx, cy) = ((bx0 + bx1) / 2.0 * a, (by0 + by1) / 2.0); + let size = ((bx1 - bx0) * a).max(by1 - by0); + let m = 0.25 * size; + let mut w = ((bx1 - bx0) * a + 2.0 * m).max(0.12 * a.max(1.0)); + let mut h = ((by1 - by0) + 2.0 * m).max(0.12 * a.max(1.0)); + w = w.max(h / 2.0).min(a); + h = h.max(w / 2.0).min(1.0); + let x = (cx - w / 2.0).clamp(0.0, (a - w).max(0.0)); + let y = (cy - h / 2.0).clamp(0.0, (1.0 - h).max(0.0)); + let out = [x / a, y, (x + w) / a, y + h]; + // a region nearly the whole photo isn't worth a second pass + ((out[2] - out[0]) * (out[3] - out[1]) < 0.8).then_some(out) +} + +/// `shape` without its detail patches (what a detail pass was computed from) +#[cfg_attr(not(feature = "sam"), allow(dead_code))] +fn strip_detail(mut shape: MaskShape) -> MaskShape { + // (nor its edge setting: moving the slider meanwhile keeps the pass) + if let MaskShape::Object { detail, edge, .. } | MaskShape::Prompt { detail, edge, .. } = &mut shape { + detail.clear(); + *edge = 0.0; + } + shape +} + +fn clicks_of(include: &[Point], exclude: &[Point]) -> Vec { + include.iter().map(|p| Click { at: *p, include: true }).chain(exclude.iter().map(|p| Click { at: *p, include: false })).collect() +} + +impl Session { + /// The key and settings of photo `id` as the model sees it: its look without masks, uncropped. + #[cfg_attr(not(feature = "sam"), allow(dead_code))] + fn segment_key(&self, id: PhotoId) -> Option<(u64, lightcraft_develop::DevelopSettings)> { + let p = self.catalog.photo(id)?; + let mut d = (*p.develop).clone(); + d.masks.clear(); + let content = crate::media::content_key(p); + let mut h = mix(0xcbf2_9ce4_8422_2325, d.hash64()); + h = mix(h, id.0); + for b in content.bytes() { + h = mix(h, u64::from(b)); + } + Some((h, d)) + } + + /// Clicks (in normalized photo coordinates) for the model. + #[cfg(feature = "sam")] + fn model_clicks(clicks: &[Click]) -> Vec { + clicks.iter().map(|c| lightcraft_segment::Click { x: c.at.x as f32, y: c.at.y as f32, positive: c.include }).collect() + } + + /// A worker request about photo `id` (the render job for its model input goes along; the + /// worker runs it only when its cache doesn't hold the photo). `region` renders that part + /// of the photo, from a source sharp enough for it (detail passes). + #[cfg(feature = "sam")] + fn segment_job( + &mut self, + id: PhotoId, + kind: worker::Kind, + tag: Tag, + reply: std::sync::mpsc::Sender, + region: Option<[f64; 4]>, + ) -> Result { + let dir = self.segmenter.model_dir()?; + let (key, mut d) = self.segment_key(id).ok_or("no such photo")?; + let p = self.catalog.photo(id).ok_or("no such photo")?.clone(); + let missing = match &p.source { + lightcraft_catalog::Source::File { path } if !std::path::Path::new(path).exists() => Some(path.clone()), + _ => None, + }; + let render = match region { + None => self.preview_job(id, INPUT_EDGE, INPUT_EDGE, false, &d).ok_or("no such photo")?, + Some(region) => { + d.crop.geometry = + lightcraft_geom::CropGeometry { rect: lightcraft_geom::Rect::new(region[0], region[1], region[2], region[3]), angle: 0.0 }; + d.crop.flip_h = false; + d.crop.flip_v = false; + d.geometry.constrain_crop = false; + let mut job = self.preview_job(id, INPUT_EDGE, INPUT_EDGE, true, &d).ok_or("no such photo")?; + let frac = (region[2] - region[0]).max(region[3] - region[1]).max(0.01); + job.level = crate::media::SourceLevel::for_size((INPUT_EDGE as f64 / frac) as usize); + job.source = self.media.source_ref(&p, job.level); + job + } + }; + Ok(worker::Job { dir, key: if region.is_some() { 0 } else { key }, render: Some(render), missing, kind, tag, reply }) + } + + /// Send a request and wait for its result (CLI, MCP, tests; or when not in background mode). + #[cfg(feature = "sam")] + fn segment_wait(&mut self, id: PhotoId, kind: worker::Kind, region: Option<[f64; 4]>) -> Result, String> { + let (tx, rx) = std::sync::mpsc::channel(); + let job = self.segment_job(id, kind, Tag::Wait, tx, region)?; + self.segmenter.worker.submit(job)?; + match rx.recv_timeout(WAIT) { + Ok(o) => o.result, + Err(_) => Err("the AI model took too long; try again".into()), + } + } + + /// Send a request whose result [`Session::segment_poll`] applies. + #[cfg(feature = "sam")] + fn segment_queue(&mut self, id: PhotoId, kind: worker::Kind, tag: Tag, region: Option<[f64; 4]>) -> Result<(), String> { + let reply = self.segmenter.results.0.clone(); + let job = self.segment_job(id, kind, tag, reply, region)?; + self.segmenter.worker.submit(job) + } + + /// Start preparing photo `id` for AI masks (load the model, encode the image), so the first + /// click is fast. In background mode it returns at once. + pub fn segment_prepare(&mut self, id: PhotoId) -> Result<(), String> { + self.segmenter.model_dir()?; + #[cfg(feature = "sam")] + { + if self.segmenter.background { + return self.segment_queue(id, worker::Kind::Prepare, Tag::Prepare, None); + } + self.segment_wait(id, worker::Kind::Prepare, None)?; + } + #[cfg(not(feature = "sam"))] + let _ = id; + Ok(()) + } + + /// Segment the object `clicks` pick on photo `id`, waiting for the model. + pub fn segment_clicks(&mut self, id: PhotoId, clicks: &[Click]) -> Result { + self.segmenter.model_dir()?; + if !clicks.iter().any(|c| c.include) { + return Err("click the object to include it first".into()); + } + #[cfg(feature = "sam")] + { + self.segment_wait(id, worker::Kind::Clicks(Self::model_clicks(clicks)), None)?.ok_or_else(|| "nothing was selected".to_string()) + } + #[cfg(not(feature = "sam"))] + { + let _ = id; + Err("AI masks are not available in this build".into()) + } + } + + /// Segment everything `text` describes on photo `id`, waiting for the model; `Ok(None)` + /// when nothing matches. + pub fn segment_text(&mut self, id: PhotoId, text: &str) -> Result, String> { + self.segmenter.model_dir()?; + let text = text.trim(); + if text.is_empty() { + return Err("describe what to select".into()); + } + #[cfg(feature = "sam")] + { + self.segment_wait(id, worker::Kind::Text(text.to_string()), None) + } + #[cfg(not(feature = "sam"))] + { + let _ = id; + Err("AI masks are not available in this build".into()) + } + } + + /// Background mode: queue Object clicks for component `comp` of mask `mask`; the result + /// (clicks and segmentation, one undo step) is applied by [`Session::segment_poll`]. + pub(crate) fn segment_clicks_later(&mut self, id: PhotoId, mask: u32, comp: usize, hint: Vec, exclude: Vec) -> Result<(), String> { + self.segmenter.model_dir()?; + if hint.is_empty() { + return Err("click the object to include it first".into()); + } + #[cfg(feature = "sam")] + { + let clicks = Self::model_clicks(&clicks_of(&hint, &exclude)); + let tag = Tag::Clicks { photo: id, mask, comp, hint: hint.clone(), exclude: exclude.clone() }; + self.segment_queue(id, worker::Kind::Clicks(clicks), tag, None)?; + self.segmenter.pending = Some(PendingClicks { photo: id, mask, comp, hint, exclude }); + Ok(()) + } + #[cfg(not(feature = "sam"))] + { + let _ = (id, mask, comp, exclude); + Err("AI masks are not available in this build".into()) + } + } + + /// Background mode: queue a Describe selection; the mask (or component, combined by `op` + /// into mask `mask`) is created by [`Session::segment_poll`] when something matches. + pub(crate) fn segment_text_later(&mut self, id: PhotoId, mask: Option, op: &str, name: Option, text: &str) -> Result<(), String> { + self.segmenter.model_dir()?; + let text = text.trim(); + if text.is_empty() { + return Err("describe what to select".into()); + } + #[cfg(feature = "sam")] + { + let tag = Tag::NewPrompt { photo: id, mask, op: op.to_string(), name, text: text.to_string() }; + self.segment_queue(id, worker::Kind::Text(text.to_string()), tag, None) + } + #[cfg(not(feature = "sam"))] + { + let _ = (id, mask, op, name); + Err("AI masks are not available in this build".into()) + } + } + + /// Start a zoomed-in pass for component `comp` of mask `mask` on photo `id` (an Object or + /// Describe selection): the photo around the selection is rendered at a higher resolution + /// and segmented again. In background mode [`Session::segment_poll`] applies the result; + /// otherwise it is applied before this returns. `Ok(false)` when there is nothing to + /// refine (no selection yet, or it is most of the photo) or a pass is running already. + pub fn segment_detail(&mut self, id: PhotoId, mask: u32, comp: usize) -> Result { + self.segmenter.model_dir()?; + let d = self.develop_of(id).ok_or("no such photo")?; + let shape = d.masks.iter().find(|m| m.id == mask).and_then(|m| m.components.get(comp)).map(|c| c.shape.clone()).ok_or("no such mask")?; + let (seg, prompt) = match &shape { + // (a document with absurdly many clicks isn't run through the model) + MaskShape::Object { hint, exclude, seg: Some(seg), .. } if !hint.is_empty() && hint.len() + exclude.len() <= MAX_CLICKS => { + (seg.clone(), Prompt::Clicks(clicks_of(hint, exclude))) + } + MaskShape::Prompt { text, seg: Some(seg), .. } => (seg.clone(), Prompt::Text(text.clone())), + _ => return Ok(false), + }; + let p = self.catalog.photo(id).ok_or("no such photo")?; + let (pw, ph) = (f64::from(p.width.max(1)), f64::from(p.height.max(1))); + let aspect = if d.orientation.swaps_axes() { ph / pw } else { pw / ph }; + let Some(region) = detail_region(&seg, aspect) else { return Ok(false) }; + #[cfg(feature = "sam")] + { + let kind = worker::Kind::Detail { prompt, region }; + let tag = Tag::Detail { photo: id, mask, comp, shape: strip_detail(shape) }; + if self.segmenter.background { + if self.segmenter.detail_busy() { + return Ok(false); + } + self.segment_queue(id, kind, tag, Some(region))?; + } else { + let patch = self.segment_wait(id, kind, Some(region))?; + let outcome = worker::Outcome { tag, result: Ok(patch), superseded: false }; + let mut polled = Polled::default(); + self.apply_outcome(outcome, &mut polled); + } + Ok(true) + } + #[cfg(not(feature = "sam"))] + { + let _ = (prompt, region, mask); + Ok(false) + } + } + + /// Apply finished background requests (to masks that haven't changed since) and collect + /// their messages. Cheap when nothing is pending: call it every frame. + pub fn segment_poll(&mut self) -> Polled { + #[cfg_attr(not(feature = "sam"), allow(unused_mut))] + let mut polled = Polled { messages: std::mem::take(&mut self.segmenter.messages), ..Default::default() }; + #[cfg(feature = "sam")] + { + // (before draining: a worker that is idle now has sent everything) + let idle = !self.segmenter.busy(); + let done: Vec = self.segmenter.results.1.try_iter().collect(); + for o in done { + self.apply_outcome(o, &mut polled); + } + // clicks whose request was lost (the worker restarted) aren't pending forever + if idle { + self.segmenter.pending = None; + } + } + polled + } + + #[cfg(feature = "sam")] + fn apply_outcome(&mut self, o: worker::Outcome, polled: &mut Polled) { + if o.superseded { + return; + } + match o.tag { + Tag::Wait => {} + Tag::Prepare => { + if let Err(e) = o.result { + polled.messages.push(e); + } + } + Tag::Clicks { photo, mask, comp, hint, exclude } => { + let latest = PendingClicks { photo, mask, comp, hint, exclude }; + if self.segmenter.pending.as_ref() != Some(&latest) { + return; // newer clicks are on their way + } + self.segmenter.pending = None; + let seg = match o.result { + Ok(Some(seg)) => seg, + Ok(None) => return, + Err(e) => return polled.messages.push(e), + }; + let Some(d) = self.develop_of(photo) else { return }; + let mut d = (*d).clone(); + let Some(c) = d.masks.iter_mut().find(|m| m.id == mask).and_then(|m| m.components.get_mut(comp)) else { return }; + let MaskShape::Object { edge, .. } = c.shape else { return }; + c.shape = MaskShape::Object { hint: latest.hint, exclude: latest.exclude, seg: Some(seg), detail: vec![], edge }; + match self.set_develop(photo, d, "Object Mask") { + Ok(()) => { + polled.changed = true; + polled.refine = Some(mask); + } + Err(e) => polled.messages.push(e.to_string()), + } + } + Tag::NewPrompt { photo, mask, op, name, text } => { + let seg = match o.result { + Ok(Some(seg)) => seg, + Ok(None) => return polled.messages.push(format!("Nothing matching “{text}” was found in this photo.")), + Err(e) => return polled.messages.push(e), + }; + if self.active() != Some(photo) { + return polled.messages.push(format!("The photo changed before “{text}” was found; try again.")); + } + let Ok(seg) = serde_json::to_value(seg) else { return }; + let mut p = serde_json::json!({"kind": "prompt", "text": text, "seg": seg}); + if let Some(n) = name { + p["name"] = serde_json::json!(n); + } + let r = match mask { + None => self.execute("mask.add", &p), + Some(m) => { + if !self.develop_of(photo).is_some_and(|d| d.masks.iter().any(|x| x.id == m)) { + return polled.messages.push("The mask was removed before the selection was found.".into()); + } + self.active_mask = Some(m); + p["op"] = serde_json::json!(op); + self.execute("mask.addComponent", &p) + } + }; + match r { + Ok(_) => { + polled.changed = true; + polled.refine = self.active_mask; + } + Err(e) => polled.messages.push(e.to_string()), + } + } + Tag::Detail { photo, mask, comp, shape } => { + let patch = match o.result { + Ok(Some(p)) => p, + Ok(None) => return, + Err(e) => return log::warn!("SAM 3 detail pass: {e}"), + }; + let Some(d) = self.develop_of(photo) else { return }; + let mut d = (*d).clone(); + let Some(c) = d.masks.iter_mut().find(|m| m.id == mask).and_then(|m| m.components.get_mut(comp)) else { return }; + if strip_detail(c.shape.clone()) != shape { + return; // clicked again meanwhile: a newer pass will follow + } + match &mut c.shape { + MaskShape::Object { detail, .. } | MaskShape::Prompt { detail, .. } => *detail = vec![patch], + _ => return, + } + match self.set_develop(photo, d, "AI Mask Detail") { + Ok(()) => polled.changed = true, + Err(e) => log::warn!("AI mask detail: {e}"), + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn seg_box(side: usize, x0: usize, y0: usize, x1: usize, y1: usize) -> SegMask { + let l: Vec = + (0..side * side).map(|i| if (x0..x1).contains(&(i % side)) && (y0..y1).contains(&(i / side)) { 8.0 } else { -8.0 }).collect(); + SegMask::from_logits(side, &l) + } + + #[test] + fn detail_region_frames_the_selection_with_a_margin() { + // a small object in the middle of a square photo + let r = detail_region(&seg_box(100, 40, 45, 50, 55), 1.0).unwrap(); + assert!(r[0] < 0.4 && r[2] > 0.5 && r[1] < 0.45 && r[3] > 0.55, "contains it: {r:?}"); + assert!(r[2] - r[0] < 0.25 && r[3] - r[1] < 0.25, "zoomed in: {r:?}"); + assert!(r.iter().all(|v| (0.0..=1.0).contains(v))); + // at the corner: shifted inside the photo, not cut + let r = detail_region(&seg_box(100, 0, 0, 5, 5), 1.5).unwrap(); + assert!(r[0] >= 0.0 && r[1] >= 0.0 && r[0] < 1e-9 && r[1] < 1e-9); + // a thin object stays at most 2:1 (in photo pixels, 1.5:1 photo) + let r = detail_region(&seg_box(100, 10, 50, 90, 52), 1.5).unwrap(); + let (w, h) = ((r[2] - r[0]) * 1.5, r[3] - r[1]); + assert!(w / h <= 2.0 + 1e-9, "{w} × {h}"); + // a hostile aspect ratio stays sane + assert!(detail_region(&seg_box(100, 40, 45, 50, 55), f64::NAN).is_some_and(|r| r.iter().all(|v| v.is_finite()))); + } + + #[test] + fn no_region_for_nothing_or_most_of_the_photo() { + assert!(detail_region(&seg_box(50, 0, 0, 0, 0), 1.0).is_none()); + assert!(detail_region(&seg_box(50, 2, 2, 48, 48), 1.0).is_none()); + let damaged = SegMask { side: 50, data: "?".into(), rect: None }; + assert!(detail_region(&damaged, 1.0).is_none()); + } +} diff --git a/crates/engine/src/segment/worker.rs b/crates/engine/src/segment/worker.rs new file mode 100644 index 00000000..9ded0fbe --- /dev/null +++ b/crates/engine/src/segment/worker.rs @@ -0,0 +1,426 @@ +//! The SAM 3 worker thread: it owns the model and the last encoded photo, and runs every +//! request (load, render the model input, encode, clicks, descriptions, detail passes) in +//! turn. The session only sends jobs and receives outcomes over channels, so no thread ever +//! waits on a lock the model holds, and the UI thread never runs the model. +//! +//! Every job runs under `catch_unwind`: a panic inside candle becomes an error outcome, drops +//! the (possibly inconsistent) model and leaves the worker ready for the next job. The busy +//! counters are decremented by drop guards, so they can't stay stuck. The model is unloaded +//! after [`IDLE_UNLOAD`] without requests (it holds several GB). + +use std::collections::VecDeque; +use std::panic::AssertUnwindSafe; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::mpsc::{self, RecvTimeoutError}; +use std::time::Duration; + +use lightcraft_develop::SegMask; +use lightcraft_segment::{Encoded, Sam3}; + +use super::{Input, Prompt, Tag}; +use crate::media::RenderJob; + +/// Unload the model after this long without a request. +pub const IDLE_UNLOAD: Duration = Duration::from_secs(10 * 60); + +/// What to compute. +pub enum Kind { + /// Load the model and encode the photo (so the first click is fast). + Prepare, + Clicks(Vec), + Text(String), + /// A zoomed-in pass over `region` (normalized) of the photo, rendered by the job's `render`. + Detail { + prompt: Prompt, + region: [f64; 4], + }, + /// Panics (tests of the worker's recovery). + #[cfg(test)] + Panic, +} + +/// One request. +pub struct Job { + /// The model folder. + pub dir: PathBuf, + /// Which photo and look the request is about (the encode cache's key). + pub key: u64, + /// Renders the model input when the cache doesn't hold `key` (always for a detail pass). + pub render: Option, + /// The photo's file, when it is missing on disk (for a clearer error if rendering fails). + pub missing: Option, + pub kind: Kind, + pub tag: Tag, + pub reply: mpsc::Sender, +} + +/// A finished request: the segmentation (`None`: nothing selected / nothing to report). +pub struct Outcome { + pub tag: Tag, + pub result: Result, String>, + /// A newer request for the same selection came in; this one was skipped. + pub superseded: bool, +} + +/// Counters the session reads without locking. +#[derive(Default)] +pub struct Shared { + /// Requests other than detail passes, queued or running. + pub pending: AtomicUsize, + /// Detail passes queued or running. + pub detail: AtomicUsize, + /// The model is in memory. + pub loaded: AtomicBool, + /// Loading the model or encoding a photo right now (the slow parts). + pub encoding: AtomicBool, +} + +/// Decrements a counter when dropped (also when unwinding). +struct Done<'a>(&'a AtomicUsize); + +impl Drop for Done<'_> { + fn drop(&mut self) { + let _ = self.0.try_update(Ordering::SeqCst, Ordering::SeqCst, |n| Some(n.saturating_sub(1))); + } +} + +/// Sets a flag while alive. +struct Flag<'a>(&'a AtomicBool); + +impl<'a> Flag<'a> { + fn raise(f: &'a AtomicBool) -> Self { + f.store(true, Ordering::SeqCst); + Flag(f) + } +} + +impl Drop for Flag<'_> { + fn drop(&mut self) { + self.0.store(false, Ordering::SeqCst); + } +} + +/// The session's handle on the worker thread (started on the first request, restarted if it +/// ever ends). +#[derive(Default)] +pub struct Worker { + tx: Option>, + thread: Option>, + pub shared: Arc, +} + +impl Worker { + fn counter(&self, job: &Job) -> &AtomicUsize { + if matches!(job.kind, Kind::Detail { .. }) { &self.shared.detail } else { &self.shared.pending } + } + + /// Whether the thread has ended (it never should); its counters are then meaningless. + fn dead(&self) -> bool { + self.thread.as_ref().is_some_and(|t| t.is_finished()) + } + + pub fn pending(&self) -> usize { + if self.dead() { 0 } else { self.shared.pending.load(Ordering::SeqCst) } + } + + pub fn detail(&self) -> usize { + if self.dead() { 0 } else { self.shared.detail.load(Ordering::SeqCst) } + } + + pub fn encoding(&self) -> bool { + !self.dead() && self.shared.encoding.load(Ordering::SeqCst) + } + + pub fn loaded(&self) -> bool { + !self.dead() && self.shared.loaded.load(Ordering::SeqCst) + } + + /// Queue `job`; returns at once. + pub fn submit(&mut self, job: Job) -> Result<(), String> { + if self.dead() { + // a thread that ended took its counters' meaning with it + self.tx = None; + self.thread = None; + self.shared = Arc::new(Shared::default()); + } + self.counter(&job).fetch_add(1, Ordering::SeqCst); + let job = match &self.tx { + Some(tx) => match tx.send(job) { + Ok(()) => return Ok(()), + Err(mpsc::SendError(job)) => job, + }, + None => job, + }; + // (re)start the thread + let (tx, rx) = mpsc::channel(); + let shared = self.shared.clone(); + match std::thread::Builder::new().name("sam3".into()).spawn(move || run(&rx, &shared, IDLE_UNLOAD)) { + Ok(t) => { + self.thread = Some(t); + self.tx = Some(tx.clone()); + tx.send(job).map_err(|_| "the AI mask worker stopped".to_string()) + } + Err(e) => { + let _ = self.counter(&job).try_update(Ordering::SeqCst, Ordering::SeqCst, |n| Some(n.saturating_sub(1))); + Err(format!("could not start the AI mask worker: {e}")) + } + } + } +} + +/// The model and the last encoded photo. +#[derive(Default)] +struct State { + model: Option<(PathBuf, Sam3)>, + cache: Option<(u64, Encoded)>, +} + +/// Whether `later` makes `job` pointless (it asks for the same selection again). +fn supersedes(later: &Job, job: &Job) -> bool { + match (&later.tag, &job.tag) { + (Tag::Prepare, Tag::Prepare) => later.key == job.key, + (Tag::Clicks { photo: a, mask: am, comp: ac, .. }, Tag::Clicks { photo: b, mask: bm, comp: bc, .. }) => (a, am, ac) == (b, bm, bc), + (Tag::Detail { photo: a, mask: am, comp: ac, .. }, Tag::Detail { photo: b, mask: bm, comp: bc, .. }) => (a, am, ac) == (b, bm, bc), + _ => false, + } +} + +fn run(rx: &mpsc::Receiver, shared: &Shared, idle: Duration) { + let mut state = State::default(); + let mut queue: VecDeque = VecDeque::new(); + loop { + if queue.is_empty() { + match rx.recv_timeout(idle) { + Ok(j) => queue.push_back(j), + Err(RecvTimeoutError::Timeout) => { + if state.model.is_some() { + state = State::default(); + shared.loaded.store(false, Ordering::SeqCst); + log::info!("SAM 3 unloaded after {} minutes without use", idle.as_secs() / 60); + } + continue; + } + Err(RecvTimeoutError::Disconnected) => return, + } + } + while let Ok(j) = rx.try_recv() { + queue.push_back(j); + } + let Some(mut job) = queue.pop_front() else { continue }; + let is_detail = matches!(job.kind, Kind::Detail { .. }); + let _done = Done(if is_detail { &shared.detail } else { &shared.pending }); + if queue.iter().any(|later| supersedes(later, &job)) { + let _ = job.reply.send(Outcome { tag: job.tag, result: Ok(None), superseded: true }); + continue; + } + let result = match std::panic::catch_unwind(AssertUnwindSafe(|| run_job(&mut state, shared, &mut job))) { + Ok(r) => r, + Err(p) => { + let why = p.downcast_ref::<&str>().map(|s| s.to_string()).or_else(|| p.downcast_ref::().cloned()).unwrap_or_default(); + log::error!("SAM 3 panicked: {why}"); + // don't trust what the model kept: load it again next time + state = State::default(); + Err("the AI model failed unexpectedly (it will be reloaded on the next try)".to_string()) + } + }; + if result.is_err() && !is_detail { + state.cache = None; + } + shared.loaded.store(state.model.is_some(), Ordering::SeqCst); + let _ = job.reply.send(Outcome { tag: job.tag, result, superseded: false }); + } +} + +/// The model for `dir`, loading it when needed. +fn model<'a>(state: &'a mut State, shared: &Shared, dir: &std::path::Path) -> Result<&'a mut Sam3, String> { + if state.model.as_ref().is_none_or(|(d, _)| d != dir) { + state.model = None; + state.cache = None; + let _busy = Flag::raise(&shared.encoding); + let t = web_time::Instant::now(); + let m = Sam3::load(dir).map_err(|e| e.to_string())?; + log::info!("SAM 3 loaded in {:?}", t.elapsed()); + state.model = Some((dir.to_path_buf(), m)); + shared.loaded.store(true, Ordering::SeqCst); + } + state.model.as_mut().map(|(_, m)| m).ok_or_else(|| "the model did not load".to_string()) +} + +/// Render the model input. +fn render(job: &mut Job) -> Result { + let r = job.render.take().ok_or("the photo is not available")?.run(); + let img = match r.rendered { + Ok(r) => r.image, + Err(e) => { + return Err(match &job.missing { + Some(path) => format!( + "This photo's file is missing (moved or deleted), so it can't be analyzed for AI masks: {path}. Library ▸ Find Missing Photos can relink it." + ), + None => format!("couldn't render the photo for AI masks: {e}"), + }); + } + }; + let rgb = img.data.iter().flat_map(|p| [p[0], p[1], p[2]]).collect(); + Ok(Input { rgb, w: img.width, h: img.height }) +} + +/// The model and the encoded photo `job.key` (rendering and encoding it when needed). +fn encoded<'a>(state: &'a mut State, shared: &Shared, job: &mut Job) -> Result<(&'a mut Sam3, &'a mut Encoded), String> { + model(state, shared, &job.dir)?; + if state.cache.as_ref().is_none_or(|(k, _)| *k != job.key) { + state.cache = None; + let input = render(job)?; + let _busy = Flag::raise(&shared.encoding); + let m = state.model.as_mut().map(|(_, m)| m).ok_or("the model did not load")?; + let t = web_time::Instant::now(); + let mut enc = m.encode(&input.rgb, input.w, input.h).map_err(|e| e.to_string())?; + m.prepare_clicks(&mut enc).map_err(|e| e.to_string())?; + log::info!("SAM 3 encoded {}×{} in {:?}", input.w, input.h, t.elapsed()); + state.cache = Some((job.key, enc)); + } + let State { model, cache } = state; + match (model.as_mut(), cache.as_mut()) { + (Some((_, m)), Some((_, e))) => Ok((m, e)), + _ => Err("the model is not ready".into()), + } +} + +/// Every phrase of `text` ("car, road"), merged (the per-pixel maximum); `None` when nothing +/// matches. +fn text_logits(model: &mut Sam3, enc: &mut Encoded, text: &str) -> Result>, String> { + let mut merged: Option> = None; + for phrase in text.split([',', ';']).map(str::trim).filter(|p| !p.is_empty()) { + if let Some(p) = model.segment_text(enc, phrase, 0.5).map_err(|e| e.to_string())? { + merged = Some(match merged { + None => p.logits, + Some(m) => m.iter().zip(&p.logits).map(|(a, b)| a.max(*b)).collect(), + }); + } + } + Ok(merged) +} + +fn run_job(state: &mut State, shared: &Shared, job: &mut Job) -> Result, String> { + let side = lightcraft_segment::MASK_SIDE; + match std::mem::replace(&mut job.kind, Kind::Prepare) { + Kind::Prepare => { + encoded(state, shared, job)?; + Ok(None) + } + Kind::Clicks(clicks) => { + let (m, enc) = encoded(state, shared, job)?; + let p = m.segment_clicks(enc, &clicks).map_err(|e| e.to_string())?; + Ok(Some(SegMask::from_logits(side, &p.logits))) + } + Kind::Text(text) => { + let (m, enc) = encoded(state, shared, job)?; + Ok(text_logits(m, enc, &text)?.map(|l| SegMask::from_logits(side, &l))) + } + #[cfg(test)] + #[allow(clippy::panic)] + Kind::Panic => panic!("a test panic"), + Kind::Detail { prompt, region } => { + let input = render(job)?; + let m = model(state, shared, &job.dir)?; + let t = web_time::Instant::now(); + let mut enc = m.encode(&input.rgb, input.w, input.h).map_err(|e| e.to_string())?; + let (rw, rh) = (region[2] - region[0], region[3] - region[1]); + if !(rw > 0.0 && rh > 0.0) { + return Ok(None); + } + let logits = match prompt { + Prompt::Clicks(clicks) => { + let inside: Vec = clicks + .iter() + .map(|c| ((c.at.x - region[0]) / rw, (c.at.y - region[1]) / rh, c.include)) + .filter(|(x, y, _)| (0.0..=1.0).contains(x) && (0.0..=1.0).contains(y)) + .map(|(x, y, positive)| lightcraft_segment::Click { x: x as f32, y: y as f32, positive }) + .collect(); + if !inside.iter().any(|c| c.positive) { + return Ok(None); + } + Some(m.segment_clicks(&mut enc, &inside).map_err(|e| e.to_string())?.logits) + } + Prompt::Text(text) => text_logits(m, &mut enc, &text)?, + }; + log::info!("SAM 3 detail pass ({}×{}) in {:?}", input.w, input.h, t.elapsed()); + Ok(logits.map(|l| SegMask::from_logits_in(side, &l, region))) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn job(tag: Tag, reply: &mpsc::Sender) -> Job { + // an empty folder: loading fails with an error (no model files) + let dir = std::env::temp_dir().join(format!("lc-sam3-worker-none-{}", std::process::id())); + Job { dir, key: 1, render: None, missing: None, kind: Kind::Prepare, tag, reply: reply.clone() } + } + + #[test] + fn errors_come_back_and_the_counters_return_to_zero() { + let mut w = Worker::default(); + let (tx, rx) = mpsc::channel(); + for _ in 0..3 { + w.submit(job(Tag::Wait, &tx)).unwrap(); + } + for _ in 0..3 { + let o = rx.recv_timeout(Duration::from_secs(30)).unwrap(); + assert!(o.result.unwrap_err().contains("not found")); + } + let t = web_time::Instant::now(); + while w.pending() > 0 && t.elapsed() < Duration::from_secs(5) { + std::thread::yield_now(); + } + assert_eq!(w.pending(), 0); + assert!(!w.encoding() && !w.loaded()); + } + + #[test] + fn a_panic_in_a_job_is_an_error_and_the_worker_carries_on() { + let shared = Shared::default(); + let (tx, rx) = mpsc::channel(); + let (jtx, jrx) = mpsc::channel(); + let mut j = job(Tag::Wait, &tx); + j.kind = Kind::Panic; + jtx.send(j).unwrap(); + jtx.send(job(Tag::Wait, &tx)).unwrap(); + shared.pending.store(2, Ordering::SeqCst); + drop(jtx); + run(&jrx, &shared, Duration::from_secs(60)); + let first = rx.recv().unwrap(); + assert!(first.result.unwrap_err().contains("failed unexpectedly")); + let second = rx.recv().unwrap(); + assert!(second.result.is_err(), "the next job still runs"); + assert_eq!(shared.pending.load(Ordering::SeqCst), 0, "counters reset"); + assert!(!shared.encoding.load(Ordering::SeqCst)); + } + + #[test] + fn a_newer_request_for_the_same_selection_supersedes_a_queued_one() { + let (tx, rx) = mpsc::channel(); + let (jtx, jrx) = mpsc::channel(); + let clicks = |n: usize| Tag::Clicks { + photo: lightcraft_catalog::PhotoId(1), + mask: 1, + comp: 0, + hint: vec![lightcraft_geom::Point::new(0.5, 0.5); n], + exclude: vec![], + }; + for n in 1..=3 { + jtx.send(job(clicks(n), &tx)).unwrap(); + } + drop(jtx); + let shared = Shared::default(); + shared.pending.store(3, Ordering::SeqCst); + run(&jrx, &shared, Duration::from_secs(60)); + let out: Vec = rx.try_iter().collect(); + assert_eq!(out.len(), 3); + assert!(out[0].superseded && out[1].superseded && !out[2].superseded); + assert_eq!(shared.pending.load(Ordering::SeqCst), 0); + } +} diff --git a/crates/engine/src/tests_segment.rs b/crates/engine/src/tests_segment.rs new file mode 100644 index 00000000..59aee18c --- /dev/null +++ b/crates/engine/src/tests_segment.rs @@ -0,0 +1,232 @@ +//! AI masks without the SAM 3 model (it is never required): every request is a clear error that +//! comes back at once, nothing waits on the network or the model, a damaged model is an error +//! (not a crash or a stuck "busy"), and the download command needs consent and fails cleanly. +//! No test touches the internet: downloads go to a local server. + +use std::time::{Duration, Instant}; + +use serde_json::json; + +use crate::Session; +use crate::segment::{NOT_INSTALLED, Segmenter}; + +fn tmp(name: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!("lc-sam3-engine-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + d +} + +/// A folder that looks installed but holds no usable model. +fn damaged_model(name: &str) -> std::path::PathBuf { + let d = tmp(name); + std::fs::create_dir_all(&d).unwrap(); + std::fs::write(d.join("model.safetensors"), b"\x10\x00\x00\x00\x00\x00\x00\x00{not json at all}").unwrap(); + std::fs::write(d.join("vocab.json"), b"{}").unwrap(); + std::fs::write(d.join("merges.txt"), b"#version: 0.2\n").unwrap(); + d +} + +/// Poll until the worker is idle (or `limit` passes), collecting messages. +fn settle(s: &mut Session, limit: Duration) -> Vec { + let t = Instant::now(); + let mut messages = Vec::new(); + loop { + let p = s.segment_poll(); + messages.extend(p.messages); + if !s.segmenter.busy() && !s.segmenter.detail_busy() { + messages.extend(s.segment_poll().messages); + return messages; + } + assert!(t.elapsed() < limit, "the worker never finished"); + std::thread::sleep(Duration::from_millis(10)); + } +} + +#[test] +fn status_and_model_commands_without_the_model() { + let mut s = Session::with_demo(); + let st = s.execute("segment.model.status", &json!({})).unwrap(); + assert_eq!(st["available"], Segmenter::AVAILABLE); + assert_eq!(st["installed"], false); + assert_eq!(st["download"]["running"], false); + assert!(st["license"].as_str().unwrap().contains("SAM License")); + // downloading needs the user's consent, and names the size and the licence + let e = s.execute("segment.model.download", &json!({})).unwrap_err().to_string(); + assert!(e.contains("acknowledged") && e.contains("SAM License") && e.contains("GB"), "{e}"); + let e = s.execute("segment.model.download", &json!({"acknowledged": "yes"})).unwrap_err().to_string(); + assert!(e.contains("acknowledged"), "{e}"); + assert_eq!(s.execute("segment.model.cancel", &json!({})).unwrap()["cancelled"], false); + if !Segmenter::AVAILABLE { + assert!(s.execute("segment.model.download", &json!({"acknowledged": true})).is_err()); + return; + } + // no folder, then no download location: clear errors, nothing started + let e = s.execute("segment.model.download", &json!({"acknowledged": true})).unwrap_err().to_string(); + assert!(e.contains("no folder"), "{e}"); + let dir = tmp("nomirrors"); + s.segmenter.dir = Some(dir.clone()); + s.segmenter.mirrors_file = Some(dir.join("none.txt")); + if s.segmenter.mirrors().is_empty() { + let e = s.execute("segment.model.download", &json!({"acknowledged": true})).unwrap_err().to_string(); + assert!(e.contains("LIGHTCRAFT_SAM3_MIRRORS"), "{e}"); + } + assert!(!s.segmenter.download_status().running); +} + +#[cfg(feature = "sam")] +#[test] +fn a_failing_download_ends_with_an_error_and_never_blocks() { + use std::io::{BufRead, BufReader, Write}; + // a local mirror that has nothing + let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}/sam3", l.local_addr().unwrap()); + std::thread::spawn(move || { + for mut c in l.incoming().flatten() { + let mut r = BufReader::new(c.try_clone().unwrap()); + let mut line = String::new(); + while r.read_line(&mut line).unwrap_or(0) > 2 { + line.clear(); + } + let _ = c.write_all(b"HTTP/1.1 404 Not Found\r\nContent-Length: 13\r\nConnection: close\r\n\r\n

404

\n"); + } + }); + let dir = tmp("download404"); + let mirrors = dir.with_extension("mirrors.txt"); + std::fs::write(&mirrors, format!("# test mirror\n{base}\n")).unwrap(); + let mut s = Session::with_demo(); + s.segmenter.dir = Some(dir.clone()); + s.segmenter.mirrors_file = Some(mirrors.clone()); + let t = Instant::now(); + let r = s.execute("segment.model.download", &json!({"acknowledged": true})).unwrap(); + assert_eq!(r["started"], true); + assert!(t.elapsed() < Duration::from_secs(1), "starting a download returns at once"); + // a second request while it runs starts nothing + let again = s.execute("segment.model.download", &json!({"acknowledged": true})).unwrap(); + assert!(again["started"] == false || again["downloading"] == false); + let t = Instant::now(); + while s.segmenter.download_status().running { + assert!(t.elapsed() < Duration::from_secs(30), "the download never ended"); + // AI mask requests meanwhile: an error at once, never a wait + let q = Instant::now(); + let e = s.execute("mask.add", &json!({"kind": "prompt", "text": "sky"})).unwrap_err().to_string(); + assert!(e.contains(NOT_INSTALLED), "{e}"); + assert!(q.elapsed() < Duration::from_millis(500)); + std::thread::sleep(Duration::from_millis(20)); + } + let st = s.segmenter.download_status(); + assert!(!st.finished); + let e = st.error.unwrap(); + assert!(e.contains("not found"), "{e}"); + assert!(!s.segmenter.installed()); + // nothing was left behind under a real name, and no error page was saved + for f in ["model.safetensors", "vocab.json", "merges.txt", "vocab.json.part"] { + assert!(!dir.join(f).exists(), "{f}"); + } + let _ = std::fs::remove_dir_all(&dir); + let _ = std::fs::remove_file(&mirrors); +} + +/// The app (background mode) never waits: with the model missing every request fails at once; +/// with a damaged model the requests are queued, fail on the worker, and come back as messages. +#[test] +fn background_requests_never_wait_and_failures_come_back_as_messages() { + let mut s = Session::with_demo(); + s.segmenter.background = true; + let id = s.active().unwrap(); + s.segmenter.dir = Some(tmp("missing")); + for (cmd, p) in [("mask.add", json!({"kind": "object"})), ("mask.add", json!({"kind": "prompt", "text": "sky"})), ("segment.prepare", json!({}))] + { + let t = Instant::now(); + let e = s.execute(cmd, &p).unwrap_err().to_string(); + assert!(t.elapsed() < Duration::from_millis(500), "{cmd}"); + if Segmenter::AVAILABLE { + assert!(e.starts_with(NOT_INSTALLED), "{cmd}: {e}"); + } else { + assert!(e.contains("not available"), "{cmd}: {e}"); + } + } + assert!(s.develop_of(id).unwrap().masks.is_empty()); + if !Segmenter::AVAILABLE { + return; + } + // a model folder whose weights are damaged + let dir = damaged_model("damaged"); + s.segmenter.dir = Some(dir.clone()); + assert!(s.segmenter.installed()); + let t = Instant::now(); + s.execute("mask.add", &json!({"kind": "object"})).unwrap(); + let r = s.execute("mask.objectPoint", &json!({"x": 0.5, "y": 0.5})).unwrap(); + assert_eq!(r["pending"], true); + assert!(s.segmenter.pending_clicks().is_some_and(|p| p.hint.len() == 1)); + // a second click while the first is on its way builds on it + let r = s.execute("mask.objectPoint", &json!({"x": 0.6, "y": 0.5, "exclude": true})).unwrap(); + assert_eq!((r["include"].as_u64(), r["exclude"].as_u64()), (Some(1), Some(1))); + let r = s.execute("mask.add", &json!({"kind": "prompt", "text": "the sky"})).unwrap(); + assert_eq!(r["pending"], true); + assert!(t.elapsed() < Duration::from_secs(1), "nothing waited for the model: {:?}", t.elapsed()); + let messages = settle(&mut s, Duration::from_secs(60)); + assert!(!messages.is_empty() && messages.iter().all(|m| !m.is_empty()), "{messages:?}"); + assert!(messages.iter().any(|m| m.contains("model.safetensors") || m.contains("SAM 3")), "{messages:?}"); + assert!(s.segmenter.pending_clicks().is_none()); + // the Object mask is there, without a selection; no Describe mask was made from nothing + let d = s.develop_of(id).unwrap(); + assert_eq!(d.masks.len(), 1); + assert!(matches!(&d.masks[0].components[0].shape, lightcraft_develop::MaskShape::Object { seg: None, .. })); + assert!(!s.segmenter.busy() && !s.segmenter.loaded()); + let _ = std::fs::remove_dir_all(&dir); +} + +/// CLI / MCP (waiting mode): a damaged model is an error from the command, quickly. +#[test] +fn waiting_requests_with_a_damaged_model_are_errors() { + if !Segmenter::AVAILABLE { + return; + } + let mut s = Session::with_demo(); + let id = s.active().unwrap(); + let dir = damaged_model("damaged-wait"); + s.segmenter.dir = Some(dir.clone()); + let e = s.execute("mask.add", &json!({"kind": "prompt", "text": "sky"})).unwrap_err().to_string(); + assert!(e.contains("model.safetensors") || e.contains("SAM 3"), "{e}"); + let e = s.execute("mask.add", &json!({"kind": "object", "points": [[0.5, 0.5]]})).unwrap_err().to_string(); + assert!(!e.is_empty()); + assert!(s.develop_of(id).unwrap().masks.is_empty()); + assert!(!s.segmenter.busy()); + // a stored segmentation needs no model at all (replayed or pasted masks) + let seg = serde_json::to_value(lightcraft_develop::SegMask::from_logits(4, &[5.0; 16])).unwrap(); + s.segmenter.dir = None; + s.execute("mask.add", &json!({"kind": "prompt", "text": "sky", "seg": seg})).unwrap(); + assert_eq!(s.develop_of(id).unwrap().masks.len(), 1); + let _ = std::fs::remove_dir_all(&dir); +} + +/// Too many clicks on one Object selection are refused, not sent to the model. +#[test] +fn object_clicks_are_capped() { + if !Segmenter::AVAILABLE { + return; + } + let mut s = Session::with_demo(); + s.segmenter.background = true; + let dir = damaged_model("clicks"); + s.segmenter.dir = Some(dir.clone()); + let id = s.active().unwrap(); + let mut d = (*s.develop_of(id).unwrap()).clone(); + let many = vec![lightcraft_geom::Point::new(0.5, 0.5); crate::segment::MAX_CLICKS]; + d.masks.push(lightcraft_develop::Mask { + id: 1, + components: vec![lightcraft_develop::MaskComponent { + name: None, + op: lightcraft_develop::MaskOp::Add, + invert: false, + shape: lightcraft_develop::MaskShape::Object { hint: many, exclude: vec![], seg: None, detail: vec![], edge: 0.0 }, + }], + ..Default::default() + }); + s.set_develop(id, d, "test").unwrap(); + s.active_mask = Some(1); + let e = s.execute("mask.objectPoint", &json!({"x": 0.5, "y": 0.5})).unwrap_err().to_string(); + assert!(e.contains("at most"), "{e}"); + assert!(!s.segmenter.busy()); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/pipeline/src/masks.rs b/crates/pipeline/src/masks.rs index 01985b5b..a71667bc 100644 --- a/crates/pipeline/src/masks.rs +++ b/crates/pipeline/src/masks.rs @@ -228,7 +228,8 @@ impl SegGrid { /// data selects nothing. fn sample_seg(seg: Option<&SegMask>, detail: &[SegMask], gain: f32, frame: &Frame, w: usize, h: usize, out: &mut Plane) { let coarse = seg.and_then(SegGrid::new); - let patches: Vec = detail.iter().filter_map(SegGrid::new).collect(); + // (capped: a hostile document can hold many) + let patches: Vec = detail.iter().take(lightcraft_develop::segmask::MAX_DETAIL).filter_map(SegGrid::new).collect(); if coarse.is_none() && patches.is_empty() { return; } @@ -466,6 +467,21 @@ mod tests { assert!(a.get(190, 50) < 0.01, "inside the patch, its unselected half"); } + #[test] + fn a_hostile_number_of_detail_patches_is_capped() { + // built in memory (a deserialized document is capped already): only the first + // MAX_DETAIL are decoded and sampled, so this needs 4 × 4 MB, not 20 000 × 4 MB + let side = 1024; + let l: Vec = vec![12.0; side * side]; + let patch = SegMask::from_logits_in(side, &l, [0.0, 0.0, 0.5, 1.0]); + let shape = MaskShape::Object { hint: vec![Point::new(0.2, 0.5)], exclude: vec![], seg: None, detail: vec![patch; 20_000], edge: 0.0 }; + let (w, h) = (64, 32); + let t = std::time::Instant::now(); + let a = shape_alpha(&shape, &frame(w, h), w, h, &Rgb32f::new(w, h), &Plane::new(w, h), 0.0); + assert!(a.get(5, 16) > 0.99 && a.get(60, 16) < 0.01); + assert!(t.elapsed() < std::time::Duration::from_secs(20), "{:?}", t.elapsed()); + } + #[test] fn the_edge_setting_hardens_or_feathers_the_transition() { let side = 16; diff --git a/crates/ui-egui/src/control.rs b/crates/ui-egui/src/control.rs index b592557a..1b9d22bb 100644 --- a/crates/ui-egui/src/control.rs +++ b/crates/ui-egui/src/control.rs @@ -278,7 +278,9 @@ pub fn handle(app: &mut LightcraftApp, ctx: &egui::Context, req: &ControlRequest Some(d) => { let r = crate::panels::dialogs::confirm_dialog(app, &d); // the import review stays open on an error, as with its button - if r.is_err() && matches!(d, crate::state::Dialog::Import { .. }) { + if (r.is_err() && matches!(d, crate::state::Dialog::Import { .. } | crate::state::Dialog::SamModel { .. })) + || (r.is_ok() && crate::panels::dialogs::keeps_open(app, &d)) + { app.ui.dialog = Some(d); } wrap(r) diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index bdc2780f..49ea140f 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -203,7 +203,9 @@ pub struct LightcraftApp { } impl LightcraftApp { - pub fn new(session: Session, services: Services) -> Self { + pub fn new(mut session: Session, services: Services) -> Self { + // AI mask requests run on the model's worker; frames apply their results (never wait) + session.segmenter.background = true; Self { session, ui: UiState::default(), @@ -358,13 +360,42 @@ impl LightcraftApp { self.ui.toast = Some((text.into(), t + secs)); } - /// AI masks: start a zoomed-in detail pass once the clicking stops (`ui.detail_due`), and - /// apply finished passes. + /// AI masks: apply finished background requests (clicks, descriptions, detail passes) and + /// show their errors; start a zoomed-in detail pass once the clicking stops + /// (`ui.detail_due`); watch the model download. Never waits for the model. fn ai_mask_detail(&mut self, ctx: &egui::Context) { - if self.session.segment_poll() { + let polled = self.session.segment_poll(); + let now = ctx.input(|i| i.time); + if polled.changed { ctx.request_repaint(); } - let now = ctx.input(|i| i.time); + if let Some(mask) = polled.refine { + self.ui.detail_due = Some((now + 0.3, mask)); + } + if let Some(e) = polled.messages.into_iter().last() { + self.ai_error(ctx, e, None); + } + let seg = &self.session.segmenter; + if seg.busy() || seg.pending_clicks().is_some() { + ctx.request_repaint_after(std::time::Duration::from_millis(50)); + } + let download = seg.download_status(); + if download.running { + ctx.request_repaint_after(std::time::Duration::from_millis(250)); + } + // a download finishing while its dialog is closed: say so once + if self.ui.sam_downloading && !download.running { + self.ui.sam_downloading = false; + let open = matches!(self.ui.dialog, Some(state::Dialog::SamModel { .. })); + match (&download.error, open) { + (_, true) => {} + (Some(e), false) => self.toast_error(ctx, format!("The SAM 3 download failed: {e}")), + (None, false) if download.finished => { + self.toast_error(ctx, crate::i18n::tr("The SAM 3 model is installed: Object and Describe masks are ready.")) + } + (None, false) => {} + } + } if let Some((due, mask)) = self.ui.detail_due { if now >= due && !self.session.segmenter.busy() && !self.session.segmenter.detail_busy() { self.ui.detail_due = None; @@ -380,6 +411,24 @@ impl LightcraftApp { } } + /// An AI mask error: when the model isn't installed, the dialog that offers to download it + /// (`then`: the AI mask to start afterwards); otherwise a toast. + pub fn ai_error(&mut self, ctx: &egui::Context, e: impl Into, then: Option<(&str, &str)>) { + let e = e.into(); + if lightcraft_engine::segment::Segmenter::AVAILABLE && e.starts_with(lightcraft_engine::segment::NOT_INSTALLED) { + self.offer_sam_download(then); + } else { + self.toast_error(ctx, e); + } + } + + /// Open the dialog offering the SAM 3 download (never downloads by itself). + pub fn offer_sam_download(&mut self, then: Option<(&str, &str)>) { + if self.ui.dialog.is_none() || matches!(self.ui.dialog, Some(state::Dialog::SamModel { .. })) { + self.ui.dialog = Some(state::Dialog::SamModel { then: then.map(|(k, o)| (k.to_string(), o.to_string())), error: None }); + } + } + /// A toast for an error the user has to read and act on (stays 6 s). pub fn toast_error(&mut self, ctx: &egui::Context, text: impl Into) { let t = ctx.input(|i| i.time); diff --git a/crates/ui-egui/src/panels/detail.rs b/crates/ui-egui/src/panels/detail.rs index 67c01f47..3b3622be 100644 --- a/crates/ui-egui/src/panels/detail.rs +++ b/crates/ui-egui/src/panels/detail.rs @@ -698,9 +698,6 @@ fn targeted_drag(app: &mut LightcraftApp, ui: &mut egui::Ui, resp: &egui::Respon } } -/// The colour of a mask hovered in the Masks list. -const HOVER_RED: [u8; 3] = [230, 30, 40]; - /// The diagnostic overlay the loupe shows (the selected mask, Point Color's visualized range, /// Visualize Spots). pub(crate) fn view_overlay(app: &LightcraftApp, d: &DevelopSettings) -> lightcraft_pipeline::Overlay { @@ -708,13 +705,6 @@ pub(crate) fn view_overlay(app: &LightcraftApp, d: &DevelopSettings) -> lightcra if app.ui.fullscreen { return Overlay::None; } - // a mask hovered in the Masks list shows in red; the selected one only with the overlay on (O) - if app.ui.right == RightPanel::Masking - && let Some(m) = app.ui.hover_mask.and_then(|id| d.masks.iter().find(|m| m.id == id)) - && !m.components.is_empty() - { - return Overlay::Mask { id: m.id.min(u16::MAX as u32) as u16, view: MaskView::default(), color: HOVER_RED, opacity: 55 }; - } if app.ui.right == RightPanel::Masking && app.ui.mask_overlay && let Some(m) = app.session.active_mask.and_then(|id| d.masks.iter().find(|m| m.id == id)) @@ -1183,9 +1173,15 @@ fn mask_overlay(app: &mut LightcraftApp, ui: &mut egui::Ui, resp: &egui::Respons } } // Object selections (SAM 3) of the selected mask: each click, green to include, red to exclude + // (clicks still on their way to the model included) + let pending = app.session.segmenter.pending_clicks().cloned(); for m in d.masks.iter().filter(|m| Some(m.id) == active) { - for c in &m.components { + for (k, c) in m.components.iter().enumerate() { if let MaskShape::Object { hint, exclude, .. } = &c.shape { + let (hint, exclude) = match pending.as_ref().filter(|q| (q.mask, q.comp) == (m.id, k)) { + Some(q) => (&q.hint, &q.exclude), + None => (hint, exclude), + }; for (pts, col) in [(hint, Color32::from_rgb(40, 200, 90)), (exclude, Color32::from_rgb(230, 60, 60))] { for q in pts { let q = map.screen(*q); @@ -1212,7 +1208,7 @@ fn mask_overlay(app: &mut LightcraftApp, ui: &mut egui::Ui, resp: &egui::Respons app.ui.detail_due = Some((ui.input(|i| i.time) + 1.0, mid)); } } - Err(e) => app.toast_error(ui.ctx(), e), + Err(e) => app.ai_error(ui.ctx(), e, Some(("object", "new"))), } } return; diff --git a/crates/ui-egui/src/panels/dialogs.rs b/crates/ui-egui/src/panels/dialogs.rs index a9ecdcfe..67d495de 100644 --- a/crates/ui-egui/src/panels/dialogs.rs +++ b/crates/ui-egui/src/panels/dialogs.rs @@ -94,6 +94,7 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) { Dialog::Merge { opts } => opts.title(), Dialog::Settings { .. } => "Settings", Dialog::ConfirmDelete { .. } => "Delete Photos", + Dialog::SamModel { .. } => "Download the SAM 3 Model?", Dialog::About => "About LightCraft", Dialog::Shortcuts => "Keyboard Shortcuts", } @@ -700,6 +701,7 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) { Dialog::Merge { opts } => crate::merge::body(app, ui, opts), Dialog::Import { opts } => crate::import::body(app, ui, opts), Dialog::Settings { tab } => crate::panels::settings::body(app, ui, tab), + Dialog::SamModel { error, .. } => sam_model_body(app, ui, error.as_deref()), Dialog::ConfirmDelete { count } => { let what = if *count == 1 { crate::i18n::tr("this photo").to_string() } else { crate::i18n::tr_format!("these {count} photos", count = count) }; ui.label(crate::i18n::tr_format!("Move {what} to Recently Deleted?", what = what)); @@ -767,8 +769,19 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) { ui.add_space(4.0); ui.horizontal(|ui| { let informational = matches!(dlg, Dialog::About | Dialog::Shortcuts | Dialog::Settings { .. }); - if !informational && ui.button(crate::i18n::tr("Cancel")).clicked() { - close = true; + let sam = &app.session.segmenter; + let (sam_installed, sam_running, sam_failed) = (sam.installed(), sam.download_status().running, sam.download_status().error.is_some()); + let cancel = match &dlg { + Dialog::SamModel { .. } if sam_running || sam_installed => "Close", + Dialog::SamModel { .. } => "Not Now", + _ => "Cancel", + }; + if !informational { + let r = ui.button(crate::i18n::tr(cancel)); + crate::widgets::register(ui.ctx(), "button:dialogCancel", r.rect); + if r.clicked() { + close = true; + } } let add_label; let ok = match &dlg { @@ -780,10 +793,18 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) { } Dialog::Merge { .. } => "Merge", Dialog::ConfirmDelete { .. } => "Delete", + Dialog::SamModel { then: Some(_), .. } if sam_installed => "Continue", + Dialog::SamModel { .. } if sam_installed || sam_running => "", + Dialog::SamModel { error, .. } if error.is_some() || sam_failed => "Try Again", + Dialog::SamModel { .. } => "Download", _ if informational => "Close", _ => "OK", }; - if ui.button(crate::i18n::tr(ok)).clicked() { + let r = (!ok.is_empty()).then(|| ui.button(crate::i18n::tr(ok))); + if let Some(r) = &r { + crate::widgets::register(ui.ctx(), "button:dialogOk", r.rect); + } + if r.is_some_and(|r| r.clicked()) { if informational { close = true; } else { @@ -803,6 +824,17 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) { match confirm_dialog(app, &dlg) { // the import review stays open on an error (e.g. an unusable folder template) Err(e) if matches!(dlg, Dialog::Import { .. }) => app.toast(ctx, e), + // the SAM 3 dialog stays open to show the download (or why it can't start) + Err(e) if matches!(dlg, Dialog::SamModel { .. }) => { + if let Dialog::SamModel { error, .. } = &mut dlg { + *error = Some(e); + } + } + Ok(_) if keeps_open(app, &dlg) => { + if let Dialog::SamModel { error, .. } = &mut dlg { + *error = None; + } + } _ => close = true, } } @@ -823,8 +855,86 @@ pub fn fmt_gap(v: f64) -> String { } } +/// Whether a dialog stays open after its action succeeded (the SAM 3 dialog while the model +/// downloads). +pub fn keeps_open(app: &LightcraftApp, dlg: &Dialog) -> bool { + matches!(dlg, Dialog::SamModel { .. }) && !app.session.segmenter.installed() +} + +/// The SAM 3 dialog: what the model is, its size and licence, and the download's progress. +fn sam_model_body(app: &mut LightcraftApp, ui: &mut egui::Ui, error: Option<&str>) { + use lightcraft_engine::segment::{LICENSE_NAME, LICENSE_URL, MODEL_BYTES}; + let t = Tokens::get(ui.ctx()); + let seg = &app.session.segmenter; + let d = seg.download_status(); + if seg.installed() { + ui.label(crate::i18n::tr("The SAM 3 model is installed: Object and Describe masks are ready.")); + return; + } + let gb = |b: u64| b as f64 / 1e9; + ui.label(crate::i18n::tr( + "Object and Describe masks use SAM 3, Meta's segmentation model. It isn't part of LightCraft, and everything else works without it.", + )); + let dir = seg.dir.as_ref().map(|d| d.display().to_string()).unwrap_or_default(); + ui.label(format!("{} {:.1} GB · {dir}", crate::i18n::tr("Download it once (about):"), gb(MODEL_BYTES))); + ui.label( + egui::RichText::new(format!( + "{} {LICENSE_NAME} — {}", + crate::i18n::tr("Licence:"), + crate::i18n::tr("Meta's terms, not LightCraft's. Downloading it means accepting them.") + )) + .color(t.text_label), + ); + let r = ui.link(crate::i18n::tr("Read the SAM License")).on_hover_text(LICENSE_URL); + crate::widgets::register(ui.ctx(), "link:samLicense", r.rect); + if r.clicked() { + let _ = crate::links::open(app, LICENSE_URL); + } + let seg = &app.session.segmenter; + if d.running { + ui.add_space(4.0); + let frac = if d.total > 0 { d.done as f64 / d.total as f64 } else { 0.0 }; + let text = format!("{:.2} / {:.2} GB · {}", gb(d.done), gb(d.total), d.file); + let r = ui.add(egui::ProgressBar::new(frac as f32).text(text)); + crate::widgets::register(ui.ctx(), "progress:samDownload", r.rect); + let r = ui.button(crate::i18n::tr("Cancel Download")); + crate::widgets::register(ui.ctx(), "button:samCancel", r.rect); + if r.clicked() { + app.session.segmenter.cancel_download(); + } + ui.label(egui::RichText::new(crate::i18n::tr("You can close this: the download continues, and resumes if interrupted.")).color(t.text_dim)); + ui.ctx().request_repaint_after(std::time::Duration::from_millis(250)); + return; + } + if seg.mirrors().is_empty() { + ui.label( + egui::RichText::new(crate::i18n::tr( + "This build has no download location for the model yet: put the files in the folder above yourself (see docs/ai-masks.md).", + )) + .color(t.text_dim), + ); + } + if let Some(e) = error.map(str::to_string).or(d.error) { + ui.label(egui::RichText::new(format!("{} {e}", crate::i18n::tr("The download didn't work:"))).color(egui::Color32::from_rgb(230, 90, 80))); + } +} + pub fn confirm_dialog(app: &mut LightcraftApp, dlg: &Dialog) -> Result { match dlg { + Dialog::SamModel { then, .. } => { + if app.session.segmenter.installed() { + // installed: start what the user was doing + if let Some((kind, op)) = then { + crate::panels::masking::begin_ai(app, kind, op)?; + } + return Ok(serde_json::Value::Null); + } + let r = app.run("segment.model.download", json!({"acknowledged": true})); + if r.is_ok() { + app.ui.sam_downloading = true; + } + r + } Dialog::NewAlbum { name, folder } => app.run("album.create", json!({"name": name, "folder": folder, "addSelected": !folder})), Dialog::RenameAlbum { id, name } => app.run("album.rename", json!({"id": id, "name": name})), Dialog::TextPrompt { value, command, params, key, .. } => { diff --git a/crates/ui-egui/src/panels/masking.rs b/crates/ui-egui/src/panels/masking.rs index 86fa7aad..2e150a60 100644 --- a/crates/ui-egui/src/panels/masking.rs +++ b/crates/ui-egui/src/panels/masking.rs @@ -106,7 +106,7 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) { app.toast(ui.ctx(), "Click the photo to pick a colour · ⇧-click adds more"); } "object" => start_object(app, ui.ctx(), "new"), - "prompt" => app.ui.describe = Some(("new".into(), String::new())), + "prompt" => start_describe(app, "new"), "brush" | "linear" | "radial" => { app.ui.tool = kind.to_string(); if *kind != "brush" { @@ -124,9 +124,12 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) { } }); describe_field(app, ui, true); - let status = if app.session.segmenter.busy() { + let seg = &app.session.segmenter; + let status = if seg.analyzing() { Some("Analyzing the photo for AI masks…") - } else if app.session.segmenter.detail_busy() || app.ui.detail_due.is_some() { + } else if seg.busy() { + Some("Selecting…") + } else if seg.detail_busy() || app.ui.detail_due.is_some() { Some("Refining the mask's detail…") } else { None @@ -136,6 +139,21 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) { ui.label(egui::RichText::new(crate::i18n::tr(status)).color(t.text_dim)); ui.ctx().request_repaint_after(std::time::Duration::from_millis(250)); } + // the model download, while its dialog is closed + let download = seg.download_status(); + if download.running && app.ui.dialog.is_none() { + ui.add_space(4.0); + ui.horizontal(|ui| { + let pct = if download.total > 0 { download.done as f64 / download.total as f64 } else { 0.0 }; + let r = ui.add( + egui::ProgressBar::new(pct as f32).desired_width(ui.available_width() - 70.0).text(format!("SAM 3: {} %", (pct * 100.0).floor())), + ); + register(ui.ctx(), "maskSamProgress", r.rect); + if text_button(ui, "maskSamDetails", crate::i18n::tr("Details"), false).clicked() { + app.offer_sam_download(None); + } + }); + } }); divider(ui); // mask list @@ -144,17 +162,7 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) { ui.label(egui::RichText::new(crate::i18n::tr("Masks")).font(t.semibold(13.0)).color(t.text_label)); ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { let on = app.ui.mask_overlay; - if icon_button( - ui, - "maskOverlay", - Icon::Eye, - vec2(24.0, 24.0), - on, - true, - "Always show the selected mask (O); hovering a mask shows it", - ) - .clicked() - { + if icon_button(ui, "maskOverlay", Icon::Eye, vec2(24.0, 24.0), on, true, "Show overlay (O)").clicked() { app.ui.mask_overlay = !on; } }); @@ -163,7 +171,6 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) { ui.label(egui::RichText::new(crate::i18n::tr("No masks yet. Choose a mask type above.")).color(t.text_dim)); } let count = d.masks.len(); - app.ui.hover_mask = None; for (index, m) in d.masks.iter().enumerate() { let sel = app.session.active_mask == Some(m.id); if let Some((rid, name)) = app.ui.renaming_mask.as_mut().filter(|(rid, _)| *rid == m.id) { @@ -187,10 +194,6 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) { } let (r, resp) = ui.allocate_exact_size(vec2(ui.available_width(), 30.0), Sense::click()); register(ui.ctx(), format!("mask:{}", m.id), r); - // the mask under the pointer shows on the photo (red overlay) - if ui.rect_contains_pointer(r) { - app.ui.hover_mask = Some(m.id); - } ui.painter().rect_filled( r, 4.0, @@ -297,9 +300,6 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) { }); let resp = resp.inner; register(ui.ctx(), format!("component:{i}"), resp.rect); - if ui.rect_contains_pointer(resp.rect) { - app.ui.hover_mask = Some(m.id); - } if resp.double_clicked() { app.ui.renaming_component = Some((m.id, i, label.clone())); } @@ -635,21 +635,62 @@ fn component_row_menu(app: &mut LightcraftApp, ui: &mut egui::Ui, mask: u32, k: } } +/// Whether the SAM 3 model is missing in a build that could use it: then the download is +/// offered (with `then` to start afterwards) instead of starting an AI mask. +fn needs_model(app: &mut LightcraftApp, kind: &str, op: &str) -> bool { + let seg = &app.session.segmenter; + let missing = lightcraft_engine::segment::Segmenter::AVAILABLE && seg.dir.is_some() && !seg.installed(); + if missing { + app.offer_sam_download(Some((kind, op))); + } + missing +} + /// Start an Object selection (SAM 3 clicks): a new mask, or a component of the selected one -/// combined by `op`; the photo is analyzed meanwhile. -fn start_object(app: &mut LightcraftApp, ctx: &egui::Context, op: &str) { +/// combined by `op`; the photo is analyzed meanwhile (in the background). +pub(crate) fn start_object(app: &mut LightcraftApp, ctx: &egui::Context, op: &str) { + if needs_model(app, "object", op) { + return; + } let r = if op == "new" { app.run("mask.add", json!({"kind": "object"})) } else { app.run("mask.addComponent", json!({"op": op, "kind": "object"})) }; match r { Ok(_) => { app.ui.tool = "object".into(); - // a missing model is reported now rather than at the first click app.toast(ctx, "Click the object to select it · ⌥-click leaves a part out"); } - Err(e) => app.toast_error(ctx, e), + Err(e) => app.ai_error(ctx, e, Some(("object", op))), } } +/// Start an AI mask of `kind` (object|prompt) combined by `op` (new|add|subtract|intersect), +/// after the model was installed. +pub(crate) fn begin_ai(app: &mut LightcraftApp, kind: &str, op: &str) -> Result { + match kind { + "object" => { + let r = if op == "new" { + app.run("mask.add", json!({"kind": "object"})) + } else { + app.run("mask.addComponent", json!({"op": op, "kind": "object"})) + }?; + app.ui.tool = "object".into(); + Ok(r) + } + _ => { + app.ui.describe = Some((op.to_string(), String::new())); + Ok(serde_json::Value::Null) + } + } +} + +/// Open the Describe field (a new mask, or a component combined by `op`). +pub(crate) fn start_describe(app: &mut LightcraftApp, op: &str) { + if needs_model(app, "prompt", op) { + return; + } + app.ui.describe = Some((op.to_string(), String::new())); +} + /// The Describe field: type what to select ("sky", "the red car") and press Return. fn describe_field(app: &mut LightcraftApp, ui: &mut egui::Ui, new_mask: bool) { let Some((op, mut text)) = app.ui.describe.clone() else { return }; @@ -691,13 +732,9 @@ fn describe_field(app: &mut LightcraftApp, ui: &mut egui::Ui, new_mask: bool) { app.run("mask.addComponent", json!({"op": op, "kind": "prompt", "text": text})) }; match r { - Ok(_) => { - app.ui.describe = None; - if let Some(mid) = app.session.active_mask { - app.ui.detail_due = Some((ui.input(|i| i.time) + 0.3, mid)); - } - } - Err(e) => app.toast_error(ui.ctx(), e), + // the mask appears when the model has found it (a detail pass follows) + Ok(_) => app.ui.describe = None, + Err(e) => app.ai_error(ui.ctx(), e, Some(("prompt", op.as_str()))), } } } @@ -711,7 +748,7 @@ fn component_menu(app: &mut LightcraftApp, ui: &mut egui::Ui, op: &str) { let b = ui.button(crate::i18n::tr("Describe…")); register(ui.ctx(), format!("maskComp:{op}:prompt"), b.rect); if b.clicked() { - app.ui.describe = Some((op.to_string(), String::new())); + start_describe(app, op); } for (kind, label) in [ ("brush", "Brush"), diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index 5b8b21d2..bdbbcf29 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -291,9 +291,9 @@ pub struct UiState { /// (`new` mask, or `add`/`subtract`/`intersect` on the selected one) and the text typed. #[serde(skip)] pub describe: Option<(String, String)>, - /// The mask under the pointer in the Masks list (shown on the photo in red). + /// A SAM 3 download was started from the app (to report its end once). #[serde(skip)] - pub hover_mask: Option, + pub sam_downloading: bool, /// When to start the zoomed-in detail pass of an AI mask (app time) and which mask: set by /// each click or description, so the pass runs once the clicking stops. #[serde(skip)] @@ -517,6 +517,15 @@ pub enum Dialog { Settings { tab: String, }, + /// Object and Describe masks need the SAM 3 model, which isn't installed: offer to download + /// it (size, licence, progress). `then`: the AI mask to start once it is there (`kind` + /// object|prompt, `op` new|add|subtract|intersect). + SamModel { + then: Option<(String, String)>, + /// Why the download couldn't start (shown in the dialog). + #[serde(default)] + error: Option, + }, /// Confirm moving photos to Recently Deleted. ConfirmDelete { count: usize, @@ -553,7 +562,7 @@ impl Default for UiState { histogram: true, soft_proof: false, proof: lightcraft_engine::pipeline::Proof { dest_warning: false, ..Default::default() }, - mask_overlay: false, + mask_overlay: true, mask_overlay_mode: "color".into(), mask_overlay_color: [230, 30, 40], mask_overlay_opacity: 50.0, @@ -573,8 +582,8 @@ impl Default for UiState { focus_search: false, renaming_mask: None, describe: None, - hover_mask: None, detail_due: None, + sam_downloading: false, renaming_component: None, quit: false, dragging_photos: None, diff --git a/crates/ui-egui/src/tests_masking.rs b/crates/ui-egui/src/tests_masking.rs index 5be82af4..c3aac95c 100644 --- a/crates/ui-egui/src/tests_masking.rs +++ b/crates/ui-egui/src/tests_masking.rs @@ -47,20 +47,14 @@ fn mask_overlay_keys_and_pins() { exec(&mut h, "mask.add", json!({"kind": "radial", "center": [0.3, 0.4], "rx": 0.1, "ry": 0.1})); exec(&mut h, "mask.add", json!({"kind": "linear", "start": [0.7, 0.2], "end": [0.7, 0.6]})); assert_eq!(h.app.session.active_mask, Some(2)); - // no overlay until a mask is hovered in the Masks list; then that mask, in red + // the loupe asks the renderer for the selected mask's overlay let d = develop(&h); - assert_eq!(crate::panels::detail::view_overlay(&h.app, &d), Overlay::None); - h.app.ui.hover_mask = Some(1); - assert_eq!(crate::panels::detail::view_overlay(&h.app, &d), Overlay::Mask { id: 1, view: MaskView::Color, color: [230, 30, 40], opacity: 55 }); - h.app.ui.hover_mask = None; - // O shows the selected mask all the time (and toggles it off again) - h.request("ui.key", json!({"key": "o"}), T); - assert!(h.app.ui.mask_overlay); let o = crate::panels::detail::view_overlay(&h.app, &d); assert_eq!(o, Overlay::Mask { id: 2, view: MaskView::Color, color: [230, 30, 40], opacity: 50 }); + // O toggles it, Shift+O cycles the colour (and leaves the crop overlay alone) h.request("ui.key", json!({"key": "o"}), T); + assert!(!h.app.ui.mask_overlay); assert_eq!(crate::panels::detail::view_overlay(&h.app, &d), Overlay::None); - // Shift+O cycles the colour (and leaves the crop overlay alone) h.request("ui.key", json!({"key": "o"}), T); let crop = h.app.ui.crop_overlay; let colour = h.app.ui.mask_overlay_color; @@ -626,3 +620,53 @@ fn soft_proofing_flags_out_of_gamut_colours_and_makes_proof_copies() { assert!(!h.app.ui.soft_proof); assert_ne!(h.app.session.catalog.stack_of(copy.id).unwrap().collapsed, stack.collapsed, "S toggled the stack"); } + +/// Object / Describe without the SAM 3 model: the download is offered (never started without a +/// yes), the dialog says why it can't start when no mirror is configured, nothing freezes and no +/// empty mask is left behind. +#[test] +fn ai_masks_without_the_model_offer_the_download() { + use crate::state::Dialog; + use lightcraft_engine::segment::Segmenter; + let mut h = detail("panel.masking"); + let dir = std::env::temp_dir().join(format!("lc-ui-no-sam3-{}", std::process::id())); + h.app.session.segmenter.dir = Some(dir.clone()); + h.app.session.segmenter.mirrors_file = Some(dir.join("none.txt")); + let t = std::time::Instant::now(); + let r = h.request("ui.clickWidget", json!({"id": "maskNew:object"}), T); + assert_eq!(r["ok"], true, "{r}"); + assert!(develop(&h).masks.is_empty()); + if !Segmenter::AVAILABLE { + // a build without AI masks says so (a toast), no dialog + assert_eq!(h.app.ui.dialog, None); + return; + } + assert_eq!(h.app.ui.dialog, Some(Dialog::SamModel { then: Some(("object".into(), "new".into())), error: None })); + assert!(!h.app.session.segmenter.download_status().running, "nothing downloads without a yes"); + if h.app.session.segmenter.mirrors().is_empty() { + // Download: no location configured in this build → the reason, in the dialog + let r = h.request("ui.dialog.confirm", json!({}), T); + assert_eq!(r["ok"], false, "{r}"); + assert!(matches!(h.app.ui.dialog, Some(Dialog::SamModel { .. })), "stays open"); + let r = h.request("ui.clickWidget", json!({"id": "button:dialogOk"}), T); + assert_eq!(r["ok"], true, "{r}"); + let Some(Dialog::SamModel { error: Some(e), .. }) = &h.app.ui.dialog else { panic!("{:?}", h.app.ui.dialog) }; + assert!(e.contains("LIGHTCRAFT_SAM3_MIRRORS"), "{e}"); + } + // (a frame with the message laid out, so the buttons are where they are drawn) + h.step(); + h.step(); + let r = h.request("ui.clickWidget", json!({"id": "button:dialogCancel"}), T); + assert_eq!(r["ok"], true, "{r}"); + assert_eq!(h.app.ui.dialog, None); + // Describe asks too + let r = h.request("ui.clickWidget", json!({"id": "maskNew:prompt"}), T); + assert_eq!(r["ok"], true, "{r}"); + assert_eq!(h.app.ui.dialog, Some(Dialog::SamModel { then: Some(("prompt".into(), "new".into())), error: None })); + assert_eq!(h.app.ui.describe, None); + // a click command from an agent gets the not-installed error at once, and the app offers the model + let e = h.app.run("mask.add", json!({"kind": "prompt", "text": "sky"})).unwrap_err(); + assert!(e.starts_with(lightcraft_engine::segment::NOT_INSTALLED), "{e}"); + assert!(develop(&h).masks.is_empty()); + assert!(t.elapsed() < SETTLE, "{:?}", t.elapsed()); +} diff --git a/docs/ai-masks.md b/docs/ai-masks.md index dc3e6726..26d4f717 100644 --- a/docs/ai-masks.md +++ b/docs/ai-masks.md @@ -4,6 +4,10 @@ LightCraft selects objects with **SAM 3** (Segment Anything with Concepts, Meta pure Rust by `crates/segment` on [candle](https://github.com/huggingface/candle): on the GPU through Metal on macOS, on the CPU elsewhere (for now). +**The model is optional.** It is not part of LightCraft, and nothing requires it: without it, +Object and Describe offer to download it, every other feature (and every existing mask, see +below) works, and no render, export or command ever waits for it. + ## Using them In the Masking panel (M): @@ -19,31 +23,87 @@ In the Masking panel (M): the people out of a sky mask; − ▸ Object then clicks remove one object. - **Edge** (per selection, −100…100): below 0 a crisper border, above 0 a feathered one (up to 2 % of the long edge, so previews and exports match). -- Hover a mask in the Masks list to see it in red on the photo; O shows the selected one all the - time. - **Detail:** about a second after the last click (or right after a description), the photo around the selection is analyzed again, zoomed in, from a sharper source (~5 s on an M4 Pro, - in the background); the result is kept as a high-resolution patch, so small objects get 5–10× - finer edges than one pass over the whole photo gives. + in the background); the result is kept as a high-resolution patch (at most 4 per selection), + so small objects get 5–10× finer edges than one pass over the whole photo gives. The first Object or Describe on a photo analyzes it (the image encoder runs once per photo and look: ~3 s on an Apple M4 Pro, plus a one-time ~6 s for compiling GPU kernels and loading the model after launch). That starts in the background as soon as you pick Object; after it, each -click takes ~35 ms and each description ~0.6 s. +click takes ~35 ms and each description ~0.6 s. Everything the model does runs on its own worker +thread: the window stays responsive, a click shows its dot at once and the selection follows +when it's computed ("Selecting…" / "Analyzing the photo…" under the mask tiles). The model is +unloaded after 10 minutes without use (it takes several GB of memory) and loaded again when +needed. The result is stored with the mask: the model's 288 × 288 mask logits over the uncropped photo (quantized, compressed, ~10–30 KB). Renders and exports sample them at any resolution and never need the model, so masks survive moving the library to a machine without it, and the web build renders them. Editing the photo's look later does not recompute a mask (click again to update). -## Installing the model +## Getting the model -The weights are not part of LightCraft: they are Meta's `facebook/sam3` checkpoint on Hugging -Face, under the SAM License, and access is gated. Once: +### In the app (recommended) -1. Open , accept the license and wait for approval. -2. Create a read token (Hugging Face ▸ Settings ▸ Access Tokens), or log in with `hf auth login`. -3. Run the installer from the repository: +The first time you pick Object or Describe without the model, LightCraft asks: + +> **Download the SAM 3 model?** Object and Describe masks use SAM 3, Meta's segmentation model. +> It isn't part of LightCraft, and everything else works without it. Download it once (about +> 3.4 GB) · *folder*. Licence: SAM License (Meta) — Meta's terms, not LightCraft's. Downloading +> it means accepting them. [Read the SAM License] — **Not Now** / **Download** + +Nothing is downloaded until you choose **Download**. The download runs in the background (close +the dialog and keep working; the Masking panel shows its progress), can be cancelled, resumes +where it stopped if interrupted (also after quitting), and tries the next download location when +one fails or stalls. Each file is written as `.part` and moved into the model folder only +when complete and verified — `model.safetensors` must have the exact size and SHA-256 of Meta's +official checkpoint; a file that doesn't match is deleted, never used. If no location works, the +dialog says why and LightCraft carries on without the model. + +Agents and scripts use the same commands (control channel, `lightcraft-cli mcp --connect`): +`segment.model.status`, `segment.model.download {"acknowledged": true}` (refused without the +explicit `acknowledged`: pass it only after the user agreed to the download and the licence), +`segment.model.cancel`. + +### Download locations + +The model is fetched from an ordered list of mirrors: base URLs where `/model.safetensors`, +`/vocab.json` and `/merges.txt` live. The list is, in order: + +1. `LIGHTCRAFT_SAM3_MIRRORS` (environment variable; URLs separated by commas or spaces); +2. the file `models/sam3-mirrors.txt` in LightCraft's settings folder (one URL per line, `#` + comments) — `~/Library/Application Support/LightCraft/` (macOS), `%APPDATA%\LightCraft\` + (Windows), `~/.config/lightcraft/` (Linux); +3. the built-in list, `DEFAULT_MIRRORS` in `crates/segment/src/fetch/mod.rs`. + +> **Maintainers:** the built-in list is **empty** until LightCraft's own CDN locations exist +> (see the `TODO(maintainer)` there): add them in order of preference, host the three files +> unchanged, and pin `vocab.json` / `merges.txt` (size + SHA-256) in `SAM3_FILES` at the same +> time. Until then the in-app download needs a user-configured mirror, and the dialog says so. + +Hugging Face's `facebook/sam3` can't be a default: it is **gated** (each person must accept the +licence there, wait for approval and download with their own token). Someone with access can +use it as their mirror: + +```sh +LIGHTCRAFT_SAM3_MIRRORS=https://huggingface.co/facebook/sam3/resolve/main \ +LIGHTCRAFT_SAM3_TOKEN=hf_... lightcraft +``` + +(`LIGHTCRAFT_SAM3_TOKEN` is sent as a bearer token only over https, and only to the mirror's own +host — not to the storage host it redirects to.) + +The download is plain HTTP/1.1 over TLS in pure Rust (rustls with the RustCrypto provider and +the Mozilla root certificates; no OpenSSL, `ring` or `aws-lc`, nothing compiled from C), with a +15 s connect timeout and a 30 s stall timeout; it follows redirects but doesn't use HTTP proxies. + +### By hand + +Put `model.safetensors`, `vocab.json` and `merges.txt` from `facebook/sam3` in the model folder: +`models/sam3/` in the settings folder above, or the folder `LIGHTCRAFT_SAM3_DIR` names. Developers +can use the installer scripts, which download from Hugging Face with your token and verify the +weights: ```sh HF_TOKEN=hf_... tools/install-sam3.sh # macOS, Linux @@ -54,15 +114,79 @@ tools/install-sam3.sh --check # verify an installation $env:HF_TOKEN = "hf_..."; .\tools\install-sam3.ps1 # Windows ``` -It downloads `model.safetensors` (3.4 GB, fp32; resumes when interrupted, checked against the -official SHA-256), `vocab.json`, `merges.txt` and the small config files into the folder -LightCraft looks in: `~/Library/Application Support/LightCraft/models/sam3/` (macOS), -`%APPDATA%\LightCraft\models\sam3` (Windows), `~/.config/lightcraft/models/sam3` (Linux). -`--dir` (or `LIGHTCRAFT_SAM3_DIR`, which LightCraft also reads) picks another folder; `--repo` -another Hugging Face repository with the same files. Restart LightCraft afterwards. +## Licence of the model -Nothing else is needed at run time: the model runs in LightCraft itself (pure Rust, no Python). -Without the files, Object and Describe say where they are expected. +LightCraft's code is MIT OR Apache-2.0; `crates/segment` (a port of Apache-2.0 code) is +Apache-2.0. **The SAM 3 weights are neither**: they are Meta's, released under the **SAM +License** (), a custom licence that +is not an OSI open-source licence. It grants a non-exclusive, worldwide, non-transferable, +royalty-free licence to use, reproduce, distribute, copy, modify and create derivative works of +the "SAM Materials", subject to its terms, including: anyone redistributing them must provide a +copy of the licence with them, and derivative works are distributed under its terms; uses +subject to ITAR, military or warfare purposes, nuclear industries, espionage, guns or illegal +weapons, and reverse engineering are prohibited, and trade controls (sanctions, export rules) +apply; the materials come "as is", without warranty; Meta may terminate the licence on a breach +(then the materials must be deleted); California law governs. Read the licence itself — this +summary is not legal advice. + +Therefore: + +- **The weights are never part of this repository, its releases or installers**, and must not be + committed. +- They reach a user's computer only when that user asks for them, after the dialog names the + licence (or when they install them by hand). +- Whoever hosts a mirror redistributes the SAM Materials and must follow the licence (ship it + next to the files). + +## Commands (control channel, MCP) + +| Command | Params | | +|---|---|---| +| `mask.add` | `{kind: "object", points?: [[x,y],…], exclude?: [[x,y],…], seg?}` | an Object mask (empty until clicked) | +| `mask.add` | `{kind: "prompt", text, seg?}` | a Describe mask | +| `mask.addComponent` | `{op: add\|subtract\|intersect, kind: "object"\|"prompt", …}` | the same as a component | +| `mask.objectPoint` | `{x, y, exclude?: bool, id?}` | one click on the selected mask's Object selection (≤ 64 clicks) | +| `mask.refineDetail` | `{id?, component?}` | the zoomed-in detail pass → `{started}` | +| `segment.prepare` | `{}` | load the model and analyze the active photo → `{busy}` | +| `segment.model.status` | `{}` | `{available, installed, dir, loaded, busy, analyzing, sizeBytes, license, licenseUrl, mirrors, download}` | +| `segment.model.download` | `{acknowledged: true}` | start the download in the background → `{started}` | +| `segment.model.cancel` | `{}` | stop it (it resumes next time) → `{cancelled}` | + +Coordinates are normalized to the uncropped, oriented photo, like every mask shape. `seg` passes +a stored segmentation (as `mask.list` / the develop settings hold it): the mask is made without +the model. Without the model, the AI kinds fail with an error starting "The SAM 3 model is not +installed" that says how to get it. + +In the desktop app the commands don't wait for the model: `mask.objectPoint` returns +`{pending: true}` and the selection is applied when it's computed (one undo step per settled +selection); `mask.add {kind: "prompt"}` returns `{pending: true}` and the mask appears when +something is found (or a message says nothing was). `lightcraft-cli` and the headless MCP server +wait and return the result (when built with the `sam` feature; the default CLI has no AI masks). + +## Implementation notes + +- `crates/segment` ports the Hugging Face `transformers` implementation (Apache-2.0; see + `NOTICE` — the ported files say so and that they were modified): the 32-layer ViT backbone + with 2-D RoPE and windowed attention, the feature pyramids, the SAM 2-style prompt encoder and + two-way mask decoder for clicks, and the CLIP text encoder, DETR encoder/decoder (box + relative-position bias, presence token) and pixel decoder for text. The CLIP tokenizer is a + small BPE in `tokenizer.rs`. `fetch/` is the downloader (original code). +- candle is pinned at 0.9.2: later releases make `candle-core` depend on `tokenizers` with the + Oniguruma C library, and the product is pure Rust. Weights are read with positional reads (no + memory map: `unsafe` stays in `lightcraft-sysmem`), only the tensors a path needs, and every + tensor's byte range is checked against the file's length when it is opened (a truncated or + damaged file is an error, never a crash). +- The engine (`crates/engine/src/segment/`) owns one worker thread with the model and the last + encoded photo; the session sends it jobs and receives results over channels, never holding a + lock the model holds. Each job runs under `catch_unwind`: a panic inside candle becomes an + error and the model is reloaded on the next request; busy counters are reset by drop guards. +- Hostile documents: a selection keeps at most 4 detail patches (when read and when rendered), + each at most 1024² logits; more than 64 clicks are never sent to the model. +- Accuracy: `crates/segment/tests/reference.rs` compares every stage against the Python + reference (`tools/sam3_reference.py`); on Metal the backbone matches to 7e-5 relative error + and masks to ~5e-5. f16 was tried and rejected (overflow, ~7 % faster). +- The engine feature `sam` (on in the desktop app, off for web and CLI) gates the model; without + it the commands report that the build has no AI masks. **Developers:** the accuracy test against the Python reference needs `torch` and `transformers` (`tools/requirements-sam3-reference.txt`): @@ -72,32 +196,3 @@ python3 -m venv .venv-sam3 && .venv-sam3/bin/pip install -r tools/requirements-s .venv-sam3/bin/python tools/sam3_reference.py photo.jpg ref.safetensors LIGHTCRAFT_SAM3_DIR= LIGHTCRAFT_SAM3_REF=ref.safetensors cargo test -p lightcraft-segment --release -- --nocapture ``` - -## Commands (control channel, MCP) - -| Command | Params | | -|---|---|---| -| `mask.add` | `{kind: "object", points?: [[x,y],…], exclude?: [[x,y],…]}` | an Object mask (empty until clicked) | -| `mask.add` | `{kind: "prompt", text}` | a Describe mask | -| `mask.addComponent` | `{op: add\|subtract\|intersect, kind: "object"\|"prompt", …}` | the same as a component | -| `mask.objectPoint` | `{x, y, exclude?: bool, id?}` | one click on the selected mask's Object selection | -| `mask.refineDetail` | `{id?, component?}` | the zoomed-in detail pass, in the background → `{started}` | -| `segment.prepare` | `{}` | analyze the active photo in the background → `{busy}` | - -Coordinates are normalized to the uncropped, oriented photo, like every mask shape. - -## Implementation notes - -- `crates/segment` ports the Hugging Face `transformers` implementation (Apache-2.0; see - `NOTICE`): the 32-layer ViT backbone with 2-D RoPE and windowed attention, the feature - pyramids, the SAM 2-style prompt encoder and two-way mask decoder for clicks, and the CLIP text - encoder, DETR encoder/decoder (box relative-position bias, presence token) and pixel decoder - for text. The CLIP tokenizer is a small BPE in `tokenizer.rs`. -- candle is pinned at 0.9.2: later releases make `candle-core` depend on `tokenizers` with the - Oniguruma C library, and the product is pure Rust. Weights are read with positional reads (no - memory map: `unsafe` stays in `lightcraft-sysmem`), and only the tensors a path needs. -- Accuracy: `crates/segment/tests/reference.rs` compares every stage against the Python - reference (`tools/sam3_reference.py`); on Metal the backbone matches to 7e-5 relative error - and masks to ~5e-5. f16 was tried and rejected (overflow, ~7 % faster). -- The engine feature `sam` (on in the desktop app, off for web and CLI) gates the model; without - it the commands report that the build has no AI masks. diff --git a/docs/parity.md b/docs/parity.md index c15d1c09..82729971 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -69,9 +69,10 @@ Take the first one nobody is working on. then tune tone, highlights, texture/clarity/dehaze, NR and sharpening against it. 5. **LR-EDIT-OPTICS-PROFILE** (P1): a lens-profile database of our own (embedded DNG/maker corrections work today). 6. **AI masks and Enhance** (LR-MASK-SUBJECT / SKY / PEOPLE / OBJECTS, LR-EDIT-DETAIL-DENOISE, SUPERRES, LENSBLUR): - Object and Describe masks run SAM 3 in pure Rust (`crates/segment`) with user-installed weights; Subject / Sky / - People could use the same model with fixed prompts. Bundling or recommending weights (SAM License) and denoise / - super-resolution models remain a maintainer decision. + Object and Describe masks run SAM 3 in pure Rust (`crates/segment`); the weights (SAM License) are never bundled: + the app offers a consented, verified download, but **LightCraft's CDN mirrors are not configured yet** (the + built-in list is empty, so today users need their own mirror or a manual install). Subject / Sky / People could + use the same model with fixed prompts; denoise / super-resolution models remain a maintainer decision. 7. **HDR** (Q. HDR, LR-EXP-HDR), **video** (R. Video), **Classic output modules** (Map view, Book, Slideshow module, Print, publish): large, well understood, lower priority than 1–5. @@ -329,7 +330,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins. | LR-MASK-SUBJECT | Select subject | P2 | 🟡 | `cmd:mask.add` (`subject`), `crates/pipeline/src/masks.rs` | saliency heuristic, no segmentation model | | LR-MASK-SKY | Select sky | P2 | 🟡 | `cmd:mask.add` (`sky`) | heuristic | | LR-MASK-BACKGROUND | Select background | P2 | 🟡 | `cmd:mask.add` (`background`) | inverse of the subject heuristic | -| LR-MASK-OBJECTS | Object selection | P2 | 🟡 | `cmd:mask.add` (`object`, `prompt`), `cmd:mask.objectPoint`, `cmd:segment.prepare`, `crates/segment`, `crates/engine/src/segment.rs` | SAM 3 in pure Rust (candle; Metal on macOS, CPU elsewhere): Object tile → click to include, ⌥-click to leave out; Describe tile → a text prompt selects every instance ("sky", "the red car"); both also as Add/Subtract/Intersect components. The segmentation is stored with the mask (288² logits), so renders and exports never need the model. Weights are not bundled: the user installs the `facebook/sam3` checkpoint (SAM License) — see docs/ai-masks.md. + / − under the selected mask combine selections; comma lists (`car, road`); a zoomed-in detail pass (`cmd:mask.refineDetail`, background) for 5–10× finer edges on small objects; per-selection Edge (hard ↔ soft); hovering a mask shows it in red. No brush/box object mode; the first click on a photo waits for its analysis (~4 s on an M4 Pro) | +| LR-MASK-OBJECTS | Object selection | P2 | 🟡 | `cmd:mask.add` (`object`, `prompt`), `cmd:mask.objectPoint`, `cmd:mask.refineDetail`, `cmd:segment.prepare`, `cmd:segment.model.status`, `cmd:segment.model.download`, `cmd:segment.model.cancel`, `crates/segment`, `crates/engine/src/segment/mod.rs` | SAM 3 in pure Rust (candle; Metal on macOS, CPU elsewhere): Object tile → click to include, ⌥-click to leave out; Describe tile → a text prompt selects every instance ("sky", "the red car"); both also as Add/Subtract/Intersect components; + / − under the selected mask; comma lists (`car, road`); a zoomed-in detail pass for 5–10× finer edges on small objects; per-selection Edge (hard ↔ soft). The model runs on its own worker thread (the UI never waits; panics become errors), is unloaded after 10 min idle, and is optional: the segmentation is stored with the mask (288² logits), so renders and exports never need it. Weights are never bundled (SAM License): without them the app offers a consented background download (mirrors, resume, timeouts, SHA-256) — **but no default download location is configured yet** (users set `LIGHTCRAFT_SAM3_MIRRORS` or install by hand; see docs/ai-masks.md). Not verified against Lightroom's Select Object; no brush/box object mode; the first click on a photo waits for its analysis (~4 s on an M4 Pro, much longer on CPU) | | LR-MASK-PEOPLE | People parts | P2 | ⬜ | | | | LR-MASK-LANDSCAPE | Landscape classes | P2 | ⬜ | | shape exists, evaluates empty | | LR-MASK-BRUSH | Brush mask | P0 | ✅ | `cmd:tool.brush`, `cmd:mask.brushStroke` (`autoMask`), `crates/pipeline/src/masks.rs` | size/feather/flow/density/erase; Auto Mask: dabs weighted by similarity to the colour under the dab centre, refined by a guided filter on luminance (CPU + GPU); no A/B brushes, no pressure | diff --git a/docs/whats-new.md b/docs/whats-new.md index 12984b9a..00480132 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -131,6 +131,12 @@ lost everything at quit; a temporary session shows a banner the whole time and never writes to your library. ### Editing +- AI masks with SAM 3 (Object and Describe in the Masking panel): click an object to select it (⌥-click leaves a + part out), or type what to select ("sky", "the red car", "car, road"); both combine with other masks, have an + Edge setting, and get a sharper zoomed-in pass in the background. The model runs inside LightCraft in pure Rust + and never freezes the window. It is optional and not part of LightCraft (Meta's SAM License, about 3.4 GB): the + first time you use an AI mask, LightCraft asks before downloading it, shows the progress, can cancel and resume, + and checks the file before using it. Masks keep their selection, so they render and export without the model. - Auto Sync: edits apply to every selected photo. Auto B&W mix. Automatic versions. - Colour-range masks: click the photo to sample. Luminance ranges: range bar, smoothness, luminance map. - ⌘-drag to straighten, ⇧G Guided Upright, a grid while transforming. diff --git a/tools/install-sam3.ps1 b/tools/install-sam3.ps1 index bd43fd98..0036c561 100644 --- a/tools/install-sam3.ps1 +++ b/tools/install-sam3.ps1 @@ -1,5 +1,6 @@ # Install the SAM 3 model for LightCraft's Object and Describe masks on Windows -# (see docs/ai-masks.md; tools/install-sam3.sh is the macOS/Linux version). +# (see docs/ai-masks.md; tools/install-sam3.sh is the macOS/Linux version). The desktop app +# offers to download the model itself when an AI mask first needs it; this is for developers. # # facebook/sam3 is gated: accept the SAM License at https://huggingface.co/facebook/sam3, wait # for approval, then: $env:HF_TOKEN = "hf_..." ; .\tools\install-sam3.ps1 @@ -34,7 +35,26 @@ if ($Token) { $Headers["Authorization"] = "Bearer $Token" } $Base = "https://huggingface.co/$Repo/resolve/main" foreach ($f in $Files + "model.safetensors") { Write-Host "fetching $f" - Invoke-WebRequest -Uri "$Base/$f" -Headers $Headers -OutFile (Join-Path $Dir $f) + # into a .part file, moved into place only when complete (and, for the weights, verified): + # a failed or damaged download never ends up under the real name + $final = Join-Path $Dir $f + $part = "$final.part" + Remove-Item -Force -ErrorAction SilentlyContinue $part + try { + Invoke-WebRequest -Uri "$Base/$f" -Headers $Headers -OutFile $part + } catch { + Remove-Item -Force -ErrorAction SilentlyContinue $part + throw "download of $Repo/$f failed: $($_.Exception.Message)" + } + if ($f -eq "model.safetensors") { + Write-Host "checking model.safetensors (SHA-256 of 3.4 GB)..." + $got = (Get-FileHash $part -Algorithm SHA256).Hash.ToLower() + if ($got -ne $Sha256) { + Remove-Item -Force $part + throw "the download is damaged or not the official checkpoint (got $got); deleted it" + } + } + Move-Item -Force $part $final } Test-Install Write-Host "Restart LightCraft; Object and Describe in the Masking panel now use it." diff --git a/tools/install-sam3.sh b/tools/install-sam3.sh index 86554c39..4ec9d0a9 100755 --- a/tools/install-sam3.sh +++ b/tools/install-sam3.sh @@ -1,6 +1,9 @@ #!/bin/sh # Install the SAM 3 model for LightCraft's Object and Describe masks (see docs/ai-masks.md). # +# The desktop app offers to download the model itself when an AI mask first needs it; this script +# is for developers (reference tests, benchmarks) and for installing from Hugging Face directly. +# # The weights are not part of LightCraft: they are Meta's facebook/sam3 checkpoint on Hugging # Face, under the SAM License. Access is gated: open https://huggingface.co/facebook/sam3, accept # the license, wait for approval, then run this with a token that can read it: @@ -75,20 +78,29 @@ fi mkdir -p "$DIR" BASE="https://huggingface.co/$REPO/resolve/main" -fetch() { # fetch FILE DEST [resume] +# fetch FILE DEST [resume]: HTTP errors never reach DEST (--fail writes no error page), and a +# resume the server refuses (416) starts over instead of counting as done. +fetch() { set -- "$1" "$2" "${3:-}" if [ -n "$TOKEN" ]; then auth="Authorization: Bearer $TOKEN"; else auth="X-No-Auth: 1"; fi - code=$(curl -sSL ${3:+-C -} -H "$auth" -w '%{http_code}' -o "$2" "$BASE/$1") || code=$? + code=$(curl -fsSL ${3:+-C -} -H "$auth" -w '%{http_code}' -o "$2" "$BASE/$1") || true case "$code" in - 200|206|416) return 0 ;; + 200|206) return 0 ;; + 416) + echo "the partial download of $1 doesn't match the server's; starting over" >&2 + rm -f "$2" + code=$(curl -fsSL -H "$auth" -w '%{http_code}' -o "$2" "$BASE/$1") || true + [ "$code" = 200 ] && return 0 + echo "download of $REPO/$1 failed ($code)" >&2; return 1 ;; 401|403) echo "access denied to $REPO/$1 (has your request for facebook/sam3 been approved?)" >&2; return 1 ;; - *) echo "download of $REPO/$1 failed ($code)" >&2; return 1 ;; + *) echo "download of $REPO/$1 failed (${code:-no response})" >&2; return 1 ;; esac } for f in $FILES; do echo "fetching $f" - fetch "$f" "$DIR/$f" + # small files: whole, into place only when complete + fetch "$f" "$DIR/$f.part" && mv -f "$DIR/$f.part" "$DIR/$f" || { rm -f "$DIR/$f.part"; exit 1; } done if [ -s "$DIR/model.safetensors" ] && [ "$(sha256 "$DIR/model.safetensors")" = "$SHA256" ]; then @@ -96,13 +108,21 @@ if [ -s "$DIR/model.safetensors" ] && [ "$(sha256 "$DIR/model.safetensors")" = " else echo "fetching model.safetensors (3.4 GB; resumes if interrupted)" part="$DIR/model.safetensors.part" - [ -s "$DIR/model.safetensors" ] && mv -f "$DIR/model.safetensors" "$part" - fetch model.safetensors "$part" resume + # a wrong model.safetensors is never resumed from: it goes + rm -f "$DIR/model.safetensors" + if [ -f "$part" ] && [ "$(wc -c < "$part" | tr -d ' ')" -ge "$SIZE" ]; then rm -f "$part"; fi + fetch model.safetensors "$part" resume || exit 1 have=$(wc -c < "$part" | tr -d ' ') - if [ "$have" != "$SIZE" ]; then + if [ "$have" -lt "$SIZE" ]; then echo "incomplete download ($have of $SIZE bytes); run the script again to resume" >&2 exit 1 fi + echo "checking model.safetensors (SHA-256 of 3.4 GB, takes a few seconds)…" + if [ "$have" != "$SIZE" ] || [ "$(sha256 "$part")" != "$SHA256" ]; then + rm -f "$part" + echo "the download is damaged or not the official checkpoint; deleted it, run the script again" >&2 + exit 1 + fi mv -f "$part" "$DIR/model.safetensors" fi