From 853ada7754a213df27bfa0da6278088c8a64d4e0 Mon Sep 17 00:00:00 2001 From: Walter Boring Date: Thu, 8 Oct 2026 12:45:33 -0400 Subject: [PATCH 01/48] LR-IMP-FORMATS: HEIC/HEIF decode behind the optional `heif` feature New crate `lightcraft-heif` (a port of PhotoCraft's photocraft-heif): a thin, panic-guarded wrapper around heic-rs (pure Rust, pinned =0.1.1) - 8/10-bit, alpha, grid tiles, container irot/imir/clap, ICC/EXIF/XMP. `lightcraft-codecs` decodes HEIF behind its non-default `heif` feature (HEVC patents are the distributor's call, the same policy as PhotoCraft); the apps forward it. HEIC files stay recognised without the feature: opening one is a clear "isn't included in this build" error. Fuzz-found heic-rs panics arrive as regression fixtures (inline bytes, no media committed). Verified end-to-end on real iPhone HEICs (lightcraft-cli render, rotation and metadata correct). Note for reviewers: measured here, the engine currently ignores the decoded image's source-space matrix on export for ANY format (a true Display-P3 PNG round-trips unchanged); the ICC and space are carried for all codecs, so this inherits - and does not worsen - existing behaviour. Co-Authored-By: opencode --- Cargo.lock | 17 +++ Cargo.toml | 1 + README.md | 3 +- apps/lightcraft-cli/Cargo.toml | 4 + apps/lightcraft-web/Cargo.toml | 5 + apps/lightcraft/Cargo.toml | 5 + crates/codecs/Cargo.toml | 5 + crates/codecs/src/heif.rs | 125 ++++++++++++++++++ crates/codecs/src/lib.rs | 3 +- crates/codecs/src/sniff.rs | 3 +- crates/heif/Cargo.toml | 20 +++ crates/heif/README.md | 33 +++++ crates/heif/src/lib.rs | 229 +++++++++++++++++++++++++++++++++ docs/parity.md | 5 +- docs/whats-new.md | 5 + xtask/src/layers.rs | 1 + 16 files changed, 459 insertions(+), 5 deletions(-) create mode 100644 crates/codecs/src/heif.rs create mode 100644 crates/heif/Cargo.toml create mode 100644 crates/heif/README.md create mode 100644 crates/heif/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index fd03d210..675f546e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2296,6 +2296,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "heic-rs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adf9dde634bb6d5b718bacd01bef80192f2334306b2f785519a0136cca7b9618" +dependencies = [ + "rayon", +] + [[package]] name = "hermit-abi" version = "0.5.3" @@ -2837,6 +2846,7 @@ dependencies = [ "jxl-oxide", "lightcraft-color", "lightcraft-geom", + "lightcraft-heif", "lightcraft-raster", "moxcms 0.9.1", "png", @@ -2935,6 +2945,13 @@ dependencies = [ "wgpu", ] +[[package]] +name = "lightcraft-heif" +version = "0.4.0" +dependencies = [ + "heic-rs", +] + [[package]] name = "lightcraft-mcp" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 77b20666..31408eb3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ lightcraft-engine = { path = "crates/engine" } lightcraft-ui-egui = { path = "crates/ui-egui" } lightcraft-mcp = { path = "crates/mcp" } lightcraft-fetch = { path = "crates/fetch" } +lightcraft-heif = { path = "crates/heif" } lightcraft-segment = { path = "crates/segment" } # foundation diff --git a/README.md b/README.md index 7444912e..06041b48 100644 --- a/README.md +++ b/README.md @@ -260,7 +260,8 @@ lightcraft --control 7980 ~/Pictures/trip - **Pure Rust, no C.** Our own RAW decoders (DNG, Canon CR2, Sony ARW, Nikon NEF, Fujifilm RAF incl. X-Trans, Panasonic RW2 / Leica RWL, Pentax PEF, Olympus ORF), our own colour science, our own pipeline. JPEG, PNG, TIFF, WebP, - PSD composites and JPEG XL open today. + PSD composites and JPEG XL open today; HEIC/HEIF photos too with `--features heif` (HEVC is a build-time choice, + as in PhotoCraft). - **Scene-referred & wide-gamut.** Linear Rec.2020 float internally, Bradford-adapted white balance, gamut mapping instead of clipping, a filmic shoulder for raw and pixel-exact pass-through for JPEGs you haven't touched. - **Resolution-independent edits.** Radii and brush sizes are relative to the image, so a 400 px preview, your diff --git a/apps/lightcraft-cli/Cargo.toml b/apps/lightcraft-cli/Cargo.toml index 3f60fe32..ab8aea72 100644 --- a/apps/lightcraft-cli/Cargo.toml +++ b/apps/lightcraft-cli/Cargo.toml @@ -29,3 +29,7 @@ dhat = { workspace = true, optional = true } # every allocation (live/peak bytes in `library.memory` / `ui.inspect`) and writes dhat-heap.json # (open it in DHAT's viewer: allocation sites by bytes at the peak). Slows allocation-heavy work. dhat-heap = ["dep:dhat"] + +# Open HEIC/HEIF photos (the optional lightcraft-heif crate). Not default so distributors choose +# it; official builds (CI, packaging/, release) build with `--features heif`. +heif = ["lightcraft-codecs/heif"] diff --git a/apps/lightcraft-web/Cargo.toml b/apps/lightcraft-web/Cargo.toml index a3331a60..bd0bd2dd 100644 --- a/apps/lightcraft-web/Cargo.toml +++ b/apps/lightcraft-web/Cargo.toml @@ -71,3 +71,8 @@ web-sys = { workspace = true, features = [ "WorkerType", "WritableStream", ] } + +[features] +# Open HEIC/HEIF photos (the optional lightcraft-heif crate). Not default so distributors choose +# it; official builds (CI, packaging/, release) build with `--features heif`. +heif = ["lightcraft-codecs/heif"] diff --git a/apps/lightcraft/Cargo.toml b/apps/lightcraft/Cargo.toml index a29299da..d87f4161 100644 --- a/apps/lightcraft/Cargo.toml +++ b/apps/lightcraft/Cargo.toml @@ -29,3 +29,8 @@ rfd = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] muda = { workspace = true } + +[features] +# Open HEIC/HEIF photos (the optional lightcraft-heif crate). Not default so distributors choose +# it; official builds (CI, packaging/, release) build with `--features heif`. +heif = ["lightcraft-codecs/heif"] diff --git a/crates/codecs/Cargo.toml b/crates/codecs/Cargo.toml index 68d5e8cd..73f4d9b7 100644 --- a/crates/codecs/Cargo.toml +++ b/crates/codecs/Cargo.toml @@ -12,6 +12,7 @@ workspace = true lightcraft-geom = { workspace = true } lightcraft-color = { workspace = true } lightcraft-raster = { workspace = true } +lightcraft-heif = { workspace = true, optional = true } serde = { workspace = true } bytemuck = { workspace = true } thiserror = { workspace = true } @@ -34,6 +35,10 @@ ravif = { workspace = true, optional = true } default = ["parallel", "jxl", "avif"] parallel = ["dep:rayon", "lightcraft-raster/parallel", "jpeg-decoder/rayon"] jxl = ["dep:jxl-oxide"] +# HEIC/HEIF decode via the optional lightcraft-heif crate (heic-rs, pure Rust). Off by default so +# distributors choose it at build time (HEVC patents are a distributor's call, same policy as +# PhotoCraft); official builds and CI enable it (`--features heif` on the apps). +heif = ["dep:lightcraft-heif"] ## AVIF encoding via ravif/rav1e (pure Rust, BSD). Native targets only; a no-op on wasm32. avif = ["dep:ravif"] diff --git a/crates/codecs/src/heif.rs b/crates/codecs/src/heif.rs new file mode 100644 index 00000000..cb451f0c --- /dev/null +++ b/crates/codecs/src/heif.rs @@ -0,0 +1,125 @@ +//! HEIF / HEIC (the iPhone and Mac photo format), read-only. +//! +//! Decoding lives in the optional `lightcraft-heif` crate (heic-rs, a pure-Rust HEVC +//! still-picture decoder), enabled by this crate's `heif` feature — off by default because HEVC +//! is patent-encumbered and whether a build carries an HEVC decoder is the distributor's call +//! (the same policy as the sibling PhotoCraft). Without the feature, HEIF files are still +//! detected and opening one is an [`Error::Unsupported`] error, never a panic. +//! +//! HEIF records orientation in the container (`irot`/`imir`, plus a `clap` crop), not in EXIF, +//! so the decoder applies them and `orientation` is reported as 1: the EXIF Orientation tag only +//! mirrors what the container says, and applying it as well would turn the photo twice. +//! The samples come back in the container's own primaries with its ICC profile (if any); like +//! every decoder here, the colour interpretation happens in [`crate::convert::finish`]. + +#[cfg(feature = "heif")] +use crate::convert::{Buf, Meta, Model, Raw, finish}; +use crate::{DecodeOptions, Decoded, Error, Format, Result}; + +const F: Format = Format::Heif; + +/// The reason given when this build has no HEIF decoder. +#[cfg(not(feature = "heif"))] +pub(crate) const NOT_IN_BUILD: &str = "HEIC/HEIF support isn't included in this build of LightCraft"; + +/// The reason a HEIF file that no build can open is refused (this crate decodes HEVC stills +/// only; the wrapper names the exact construct in its own errors). +#[cfg(feature = "heif")] +const NOT_DECODED: &str = "the image is not a decodable HEVC still (image sequence, overlay, alpha-less grid or AVIF-coded item)"; + +#[cfg(not(feature = "heif"))] +pub(crate) fn decode(_bytes: &[u8], _opts: &DecodeOptions) -> Result { + Err(Error::Unsupported(F, NOT_IN_BUILD)) +} + +#[cfg(feature = "heif")] +fn err(e: lightcraft_heif::Error) -> Error { + match e { + // This crate's Unsupported takes a fixed reason; the dynamic wording lives in the + // lightcraft-heif error itself. + lightcraft_heif::Error::Unsupported(_) => Error::Unsupported(F, NOT_DECODED), + lightcraft_heif::Error::Limit(m) | lightcraft_heif::Error::Malformed(m) => Error::Malformed(F, m), + } +} + +#[cfg(feature = "heif")] +pub(crate) fn decode(bytes: &[u8], opts: &DecodeOptions) -> Result { + // The container alone: the declared size is checked before any pixel is decoded. + let info = lightcraft_heif::probe(bytes).map_err(err)?; + if (info.width as u64).saturating_mul(info.height as u64) > opts.max_pixels { + return Err(Error::TooLarge(info.width as u64, info.height as u64)); + } + let options = lightcraft_heif::Options { max_pixels: opts.max_pixels, apply_transforms: true }; + let decoded = lightcraft_heif::decode(bytes, &options).map_err(err)?; + // The decoder re-reads the container; trust its output's shape, not the probe's. + let buf = if decoded.sixteen_bit { + let mut v = Vec::with_capacity(decoded.data.len() / 2); + for pair in decoded.data.chunks_exact(2).filter_map(|p| <[u8; 2]>::try_from(p).ok()) { + v.push(u16::from_ne_bytes(pair)); + } + Buf::U16(v) + } else { + Buf::U8(decoded.data) + }; + let raw = Raw { + width: decoded.width as usize, + height: decoded.height as usize, + model: Model::Rgb, + alpha: decoded.has_alpha, + premultiplied: false, + buf, + bit_depth: if decoded.sixteen_bit { 16 } else { 8 }, + }; + let meta = Meta { icc: decoded.icc, exif: decoded.exif, xmp: decoded.xmp, orientation: Some(1), ..Default::default() }; + finish(F, raw, meta, (decoded.width, decoded.height), opts) +} + +#[cfg(test)] +mod tests { + use crate::{Format, sniff}; + + /// The container header both builds recognise, whatever the feature says. + const HEIC_HEADER: &[u8] = b"\0\0\0\x18ftypheic\0\0\0\0mif1heic"; + + #[test] + fn heif_is_always_recognised() { + assert_eq!(sniff(HEIC_HEADER), Some(Format::Heif)); + } + + #[test] + fn can_decode_follows_the_feature() { + assert_eq!(Format::Heif.can_decode(), cfg!(feature = "heif")); + } + + #[cfg(not(feature = "heif"))] + #[test] + fn without_the_feature_heif_is_a_clear_unsupported_error() { + let r = crate::decode(HEIC_HEADER, crate::DecodeOptions::default()); + assert!(matches!(&r, Err(crate::Error::Unsupported(Format::Heif, why)) if why.contains("isn't included in this build")), "{r:?}"); + } + + /// Found by PhotoCraft's `decode_heif` fuzz target: a malformed box makes heic-rs 0.1.1 slice + /// out of range (`boxes.rs:130`). It must be an error, not a crash. + #[cfg(feature = "heif")] + const HEIC_RS_BOX_PANIC: [u8; 72] = [ + 0x00, 0x00, 0x00, 0x24, 0x66, 0x74, 0x79, 0x70, 0x68, 0x65, 0x69, 0x63, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x69, 0x66, 0x31, 0x4d, 0x69, 0x50, + 0x72, 0x6d, 0x69, 0x61, 0x66, 0x4d, 0x69, 0x48, 0x42, 0x72, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x65, 0x74, 0x61, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x75, 0x75, 0x69, 0x64, 0x00, 0x00, 0x00, 0x00, 0x03, 0x08, 0x08, 0x08, 0x00, 0x03, 0x01, 0x03, 0x70, + 0x00, 0xa8, 0x00, + ]; + + #[cfg(feature = "heif")] + #[test] + fn a_heic_rs_panic_is_a_malformed_error() { + let r = crate::decode(&HEIC_RS_BOX_PANIC, crate::DecodeOptions::default()); + assert!(matches!(&r, Err(crate::Error::Malformed(Format::Heif, _))), "{r:?}"); + } + + #[cfg(feature = "heif")] + #[test] + fn an_image_sequence_is_unsupported() { + // An ftyp with no meta box: a video track, not a still. + let r = crate::decode(b"\0\0\0\x18ftypmsf1\0\0\0\0msf1hevc", crate::DecodeOptions::default()); + assert!(matches!(&r, Err(crate::Error::Unsupported(Format::Heif, _))), "{r:?}"); + } +} diff --git a/crates/codecs/src/lib.rs b/crates/codecs/src/lib.rs index 708aa9b7..74537e56 100644 --- a/crates/codecs/src/lib.rs +++ b/crates/codecs/src/lib.rs @@ -17,6 +17,7 @@ mod convert; pub mod encode; pub mod exif; +mod heif; pub mod icc; mod jpeg; pub mod jpeg_par; @@ -171,8 +172,8 @@ pub fn decode_unguarded(bytes: &[u8], format: Format, opts: &DecodeOptions) -> R Format::Jxl => jxl::decode(bytes, &opts), #[cfg(not(feature = "jxl"))] Format::Jxl => Err(Error::Unsupported(format, "built without the `jxl` feature")), + Format::Heif => heif::decode(bytes, &opts), Format::Avif => Err(Error::Unsupported(format, "no pure-Rust, permissively licensed AV1 decoder yet")), - Format::Heif => Err(Error::Unsupported(format, "no pure-Rust, permissively licensed HEVC decoder yet")), Format::RawTiffLike | Format::RawOther => Err(Error::Unsupported(format, "camera raw: decode with lightcraft-raw")), } } diff --git a/crates/codecs/src/sniff.rs b/crates/codecs/src/sniff.rs index a02d87d2..50fa3ccf 100644 --- a/crates/codecs/src/sniff.rs +++ b/crates/codecs/src/sniff.rs @@ -66,7 +66,8 @@ impl Format { pub fn can_decode(self) -> bool { match self { Format::Jxl => cfg!(feature = "jxl"), - Format::Avif | Format::Heif | Format::RawTiffLike | Format::RawOther => false, + Format::Heif => cfg!(feature = "heif"), + Format::Avif | Format::RawTiffLike | Format::RawOther => false, _ => true, } } diff --git a/crates/heif/Cargo.toml b/crates/heif/Cargo.toml new file mode 100644 index 00000000..9a12bb2e --- /dev/null +++ b/crates/heif/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "lightcraft-heif" +description = "Optional HEIF/HEIC still-image decoder for LightCraft (a thin, panic-guarded wrapper around heic-rs)" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +# Pure Rust, no unsafe. Pinned exactly: a young, single-owner crate that parses untrusted files; +# moving the pin is a reviewed change (re-run the heif tests and the codecs corpus tests). +# Mirrors PhotoCraft's photocraft-heif (same team, same decision): heic-rs was chosen over the +# other permissive candidates (heif-oxide lacks 4:2:2; heifer was published the same week). +heic-rs = { version = "=0.1.1", default-features = false, features = ["std"] } + +# The tiles of a HEIC photo decode in parallel on native; wasm stays single-threaded. +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +heic-rs = { version = "=0.1.1", default-features = false, features = ["std", "parallel"] } + +[lints] +workspace = true diff --git a/crates/heif/README.md b/crates/heif/README.md new file mode 100644 index 00000000..e008a01c --- /dev/null +++ b/crates/heif/README.md @@ -0,0 +1,33 @@ +# lightcraft-heif + +The optional HEIF / HEIC decoder of LightCraft: iPhone and Mac photos. A thin, panic-guarded wrapper +around [`heic-rs`](https://github.com/tbraun96/heic-rs), a pure-Rust HEVC still-picture decoder +(no `unsafe`, MIT OR Apache-2.0): single pictures and grid-tiled photos, 8- and 10-bit (10-bit +decodes to 16-bit), alpha auxiliary images, the container's rotation/mirror/crop, ICC, EXIF and XMP. +Read-only: writing would need an HEVC encoder. + +```rust +let info = lightcraft_heif::probe(&bytes)?; // size, depth, alpha: check limits first +let img = lightcraft_heif::decode(&bytes, &lightcraft_heif::Options::default())?; +// img.width, img.height, img.has_alpha, img.sixteen_bit, img.data (RGB/RGBA), img.icc, img.exif, img.xmp +``` + +Errors are `Error::Unsupported` (image sequences, overlays, …), `Error::Limit` or +`Error::Malformed`. It never panics: every call into heic-rs runs under `catch_unwind`, and a panic +inside it (PhotoCraft's fuzzing found two in 0.1.1) becomes `Error::Malformed`. The crate is a port +of PhotoCraft's `photocraft-heif` — the sibling apps deliberately share one decoder, one pin and +one set of regression fixtures. + +## Why it is a separate, optional crate + +- **A young decoder.** heic-rs is new and has a single maintainer. It is pinned exactly + (`=0.1.1`); moving the pin is a reviewed change (re-run the crate tests and + `cargo test -p lightcraft-codecs --features heif`). +- **HEVC patents are a distributor's call.** HEVC is patent-encumbered in some jurisdictions, so + whether a build includes an HEVC decoder is a build-time choice. `lightcraft-codecs` uses this + crate only behind its non-default `heif` feature; the apps forward it + (`cargo build -p lightcraft --features heif`). Without it, HEIC files are still recognised and + opening one is a clear "HEIC/HEIF support isn't included in this build" error — the same policy + as PhotoCraft. + +The crate depends on no other LightCraft crate; `codecs` → `heif` is the only edge between them. diff --git a/crates/heif/src/lib.rs b/crates/heif/src/lib.rs new file mode 100644 index 00000000..7b19b9b9 --- /dev/null +++ b/crates/heif/src/lib.rs @@ -0,0 +1,229 @@ +//! `lightcraft-heif`: the optional HEIF / HEIC (iPhone and Mac photo) decoder. +//! +//! A thin wrapper around `heic-rs`, a pure-Rust HEVC still-picture decoder: single pictures and +//! grid-tiled photos, 8- and 10-bit (10-bit decodes to 16-bit), alpha auxiliary images, ICC, +//! EXIF and XMP. Its API is plain data ([`Info`], [`Decoded`], [`Error`]) so the crate knows +//! nothing about the rest of LightCraft; `lightcraft-codecs` adapts it behind its `heif` feature. +//! A port of PhotoCraft's `photocraft-heif` (same team, same licence) so the sibling apps share +//! one decision, one dependency pin and one set of regression fixtures. +//! +//! HEIF records orientation in the container (`irot`/`imir`, plus a `clap` crop), not in EXIF. +//! [`Options::apply_transforms`] applies them; with it off, the pixels come back as coded. +//! +//! Never panics: heic-rs is young, so every call into it runs under `catch_unwind` and a panic +//! inside it becomes [`Error::Malformed`]. + +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +use std::borrow::Cow; + +use heic_rs::PixelLayout; + +/// What a HEIF file declares, read from the container alone (no pixel is decoded). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Info { + /// Size after the container transforms (rotation, mirror, crop): what a viewer shows. + pub width: u32, + pub height: u32, + /// Size as coded, before the transforms. + pub coded_width: u32, + pub coded_height: u32, + /// The primary image has an alpha auxiliary image. + pub has_alpha: bool, + /// More than 8 bits per sample: it decodes to 16-bit. + pub sixteen_bit: bool, +} + +/// Decode settings. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Options { + /// Refuse images with more pixels than this (checked before decoding). + pub max_pixels: u64, + /// Apply the container's rotation, mirror and crop (`irot`/`imir`/`clap`). + pub apply_transforms: bool, +} + +impl Default for Options { + fn default() -> Self { + Options { max_pixels: 1 << 28, apply_transforms: true } + } +} + +/// A decoded image: interleaved RGB or RGBA samples, row-major, no padding. 16-bit samples are +/// native-endian `u16`s. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Decoded { + pub width: u32, + pub height: u32, + /// Four channels (RGBA) instead of three (RGB). + pub has_alpha: bool, + /// Two bytes per sample instead of one. + pub sixteen_bit: bool, + pub data: Vec, + /// The primary image's ICC profile. + pub icc: Option>, + /// EXIF as a TIFF structure (without HEIF's offset header). Its Orientation tag only mirrors + /// what the container says; it is returned verbatim. + pub exif: Option>, + /// The XMP packet. + pub xmp: Option, +} + +/// Why a file could not be decoded. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Error { + /// A valid file using something this decoder does not handle (image sequences, overlays…). + Unsupported(String), + /// The image is larger than [`Options::max_pixels`] or holds an oversized box. + Limit(String), + /// Broken or truncated data (or a decoder bug, reported the same way). + Malformed(String), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::Unsupported(m) => write!(f, "unsupported HEIF: {m}"), + Error::Limit(m) => write!(f, "limit exceeded: {m}"), + Error::Malformed(m) => write!(f, "malformed HEIF: {m}"), + } + } +} + +impl std::error::Error for Error {} + +const SEQUENCE: &str = "this file holds an image sequence; only HEIF still images can be opened"; + +fn err(e: heic_rs::Error) -> Error { + match e { + heic_rs::Error::Unsupported(what) => Error::Unsupported(what.to_string()), + // The primary item is neither an HEVC picture nor a grid of them. + heic_rs::Error::MissingBox("hvcC") => Error::Unsupported("the image is an overlay, an identity derivation or not HEVC-coded".into()), + heic_rs::Error::PixelLimit { .. } | heic_rs::Error::BoxTooLarge { .. } => Error::Limit(e.to_string()), + e => Error::Malformed(e.to_string()), + } +} + +/// Runs `f`, turning a heic-rs panic into an error. Fuzzing in the sibling app found two +/// out-of-range slices in heic-rs 0.1.1 on malformed files: in its box parser (`boxes.rs:130`) +/// and during reconstruction (`hevc/decode/recon.rs:70`). +fn guarded(f: impl FnOnce() -> Result) -> Result { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).unwrap_or_else(|_| Err(Error::Malformed("the HEIF decoder failed on this file".into()))) +} + +/// Reads the container's declared size, depth and alpha without decoding pixels, so callers can +/// check their limits first. +pub fn probe(bytes: &[u8]) -> Result { + guarded(|| probe_unguarded(bytes)) +} + +fn probe_unguarded(bytes: &[u8]) -> Result { + let info = heic_rs::probe(bytes).map_err(|e| match e { + // No `meta` box: no still image, only a `moov` image sequence (a video track). + heic_rs::Error::MissingBox("meta") => Error::Unsupported(SEQUENCE.into()), + e => err(e), + })?; + Ok(Info { + width: info.width, + height: info.height, + coded_width: info.coded_width, + coded_height: info.coded_height, + has_alpha: info.has_alpha, + sixteen_bit: info.bit_depth > 8, + }) +} + +/// Decodes the primary image, with its metadata. +pub fn decode(bytes: &[u8], options: &Options) -> Result { + guarded(|| decode_unguarded(bytes, options)) +} + +fn decode_unguarded(bytes: &[u8], options: &Options) -> Result { + let info = probe_unguarded(bytes)?; + let pixel_layout = match (info.has_alpha, info.sixteen_bit) { + (true, true) => PixelLayout::Rgba16, + (true, false) => PixelLayout::Rgba8, + (false, true) => PixelLayout::Rgb16, + (false, false) => PixelLayout::Rgb8, + }; + let raw = heic_rs::DecodeOptions::default() + .with_layout(pixel_layout) + .with_max_pixels(Some(options.max_pixels)) + .with_transforms(options.apply_transforms) + .with_alpha(info.has_alpha); + let decoded = heic_rs::decode(bytes, &raw).map_err(err)?; + let (icc, exif, xmp) = metadata(bytes); + Ok(Decoded { + width: decoded.width, + height: decoded.height, + has_alpha: info.has_alpha, + sixteen_bit: info.sixteen_bit, + data: decoded.data, + icc, + exif, + xmp, + }) +} + +/// The primary image's ICC profile, EXIF (TIFF structure) and XMP packet. Metadata is a +/// courtesy: a broken metadata item never fails a decode whose pixels came out fine. +fn metadata(bytes: &[u8]) -> (Option>, Option>, Option) { + let Ok(ctx) = heic_rs::context::Context::open(bytes) else { + return (None, None, None); + }; + let id = ctx.meta.primary; + let icc = ctx.props(id).ok().and_then(|p| ctx.icc(&p).map(<[u8]>::to_vec)); + let exif = ctx.exif(id).ok().flatten().map(<[u8]>::to_vec); + let xmp = ctx.xmp_item(id).and_then(|x| ctx.item_data(x).ok()).and_then(|d| match d { + Cow::Borrowed(b) => String::from_utf8(b.to_vec()).ok(), + Cow::Owned(b) => String::from_utf8(b).ok(), + }); + (icc, exif, xmp.map(|x| x.trim_end_matches('\0').to_string())) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Found by PhotoCraft's `decode_heif` fuzz target: a malformed box makes heic-rs 0.1.1 slice + /// out of range (`boxes.rs:130`). It must be an error, not a crash. + const HEIC_RS_BOX_PANIC: [u8; 72] = [ + 0x00, 0x00, 0x00, 0x24, 0x66, 0x74, 0x79, 0x70, 0x68, 0x65, 0x69, 0x63, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x69, 0x66, 0x31, 0x4d, 0x69, 0x50, + 0x72, 0x6d, 0x69, 0x61, 0x66, 0x4d, 0x69, 0x48, 0x42, 0x72, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x65, 0x74, 0x61, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x75, 0x75, 0x69, 0x64, 0x00, 0x00, 0x00, 0x00, 0x03, 0x08, 0x08, 0x08, 0x00, 0x03, 0x01, 0x03, 0x70, + 0x00, 0xa8, 0x00, + ]; + + #[test] + fn a_heic_rs_panic_is_a_malformed_error() { + for apply_transforms in [true, false] { + let r = decode(&HEIC_RS_BOX_PANIC, &Options { apply_transforms, ..Default::default() }); + assert!(matches!(r, Err(Error::Malformed(_))), "{r:?}"); + } + } + + #[test] + fn image_sequence_is_unsupported_with_a_clear_message() { + // An ftyp and nothing else: a HEIF image sequence would have a moov here instead of a meta. + let bytes = b"\0\0\0\x18ftypmsf1\0\0\0\0msf1hevc"; + for r in [probe(bytes).map(|_| ()), decode(bytes, &Options::default()).map(|_| ())] { + assert!(matches!(&r, Err(Error::Unsupported(m)) if m.contains("sequence")), "{r:?}"); + } + } + + #[test] + fn garbage_is_an_error() { + for bytes in [&b""[..], b"\0\0\0\x18ftypheic\0\0\0\0mif1heic", &[0xFF; 64]] { + assert!(probe(bytes).is_err()); + assert!(decode(bytes, &Options::default()).is_err()); + } + } + + #[test] + fn errors_display_their_kind() { + assert!(Error::Unsupported("x".into()).to_string().contains("unsupported")); + assert!(Error::Limit("x".into()).to_string().contains("limit")); + assert!(Error::Malformed("x".into()).to_string().contains("malformed")); + } +} diff --git a/docs/parity.md b/docs/parity.md index d5a31ea9..301749de 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -61,7 +61,8 @@ Take the first one nobody is working on. 1. **LR-PROF-CAMERACOLOR** (P0): our own camera colour calibration. Sony ARW, Nikon NEF and Panasonic RW2 now get a guarded file-local fit to their own embedded JPEG (colour matrix + hue/saturation/value table + tone and chroma curves, relative WB; per-model profiles pooled from many photos via `lightcraft-cli calibrate` for ARW and NEF; ARW: 7 of 10 public samples accepted, mean ΔE vs the camera JPEG 17–26 → 3–10; NEF: 13 of 13 decodable samples from 6 bodies accepted, ΔE 13–46 → 3–8, one mixed-light scene 13 → 12; RW2: 140 of 174 public samples accepted, from 93 of the 114 bodies with a preview, median ΔE 5.0); measured calibration and fidelity remain missing. Other non-DNG raws and rejected fits still use a neutral matrix. Expand validated preview fitting and use matrices the files carry themselves; never Adobe data. 2. **LR-IMP-FORMATS** (P0): **CR3** first (every Canon body since ~2018), then compressed RAF / ORF, Nikon - "lossy after split" NEF, Canon sRAW; HEIC/AVIF decode. Clean-room, from prose descriptions only (see + "lossy after split" NEF, Canon sRAW; AVIF decode (HEIC decodes behind the codecs' non-default `heif` feature: + the optional `lightcraft-heif` crate on heic-rs, the same decoder and policy as PhotoCraft). Clean-room, from prose descriptions only (see `crates/raw/src/vendor/nefc.rs` for how compressed NEF was done). Until decoded, such photos are `preview_only`. 3. **LR-IMP-CAMERA-COVERAGE** (P0): per-model verification; grow the CC0 corpus and fix per-model bugs (like the CR2 colour-filter layout, fixed in #85 by reading the file's own tag). @@ -124,7 +125,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins. | LR-IMP-PROFILES | Import profiles & presets | P1 | ✅ | `cmd:file.importPresets`, `cmd:preset.import`, `cmd:profile.import`, `crates/engine/src/preset_import.rs`, `crates/engine/src/preset_luminar.rs` | presets: .lcpreset, XMP, classic .lrtemplate, photos carrying edits ("DNG presets"), Luminar looks (.lmp, .mplumpack collections; sliders with a counterpart), .zip bundles, folders (folder → group), drag & drop; masks carried over; unmapped settings reported; profiles: .cube 3D LUTs; Adobe profile formats deliberately unsupported | | LR-IMP-LOCAL | Work on files in place | P0 | ✅ | `cmd:library.browse`, `cmd:photo.addToLibrary`, `cmd:library.import` (mode add), `cmd:local.addRoot`, `cmd:local.hide`, `cmd:local.restoreHidden`, `crates/engine/src/cmd/browse.rs`, `crates/ui-egui/src/panels/left.rs` (`local_section`), `crates/ui-egui/src/panels/grid.rs` (`folder_header`) | Local: browse Pictures / Desktop / Downloads / Home or any folder without adding it (breadcrumb, Include subfolders, Add N to My Photos); browsed photos stay out of All Photos, albums and counts; folders picked with Browse Folder… (or Keep in Local / Add to Local on a folder row) stay listed across restarts, and browsing a folder inside a listed one highlights it inside that tree (opened down to it, siblings reachable) instead of adding a root; edits go to XMP sidecars; importing promotes them; right-click a top-level location → Remove from Local hides the shortcut only (nothing on disk or in the catalog changes), saved with the UI state, and “Show N hidden locations” (or Browse Folder… on it) brings it back; a folder is one location however its path is spelled (`/` or `\`, trailing separator, `.`/`..`, drive-letter case; `folder_key` in `crates/catalog/src/query.rs`); browsed photos nobody changed are forgotten once their folder has not been browsed for 30 days (Settings → Performance → Local folders, 0 = never; checked when the library opens, or `cmd:library.forgetLocal` with `dryRun`; files and sidecars stay, browsing again brings them back; see `crates/catalog/src/local.rs` for what counts as a change) | | LR-IMP-SIDECAR-SPLIT | Separate XMP sidecar variants | P2 | ⬜ | `cmd:library.xmpPreferences` | sidecar naming option exists (stem/full), no split sidecars | -| LR-IMP-FORMATS | Supported formats | P0 | 🟡 | `crates/codecs/src/lib.rs`, `crates/raw/src/lib.rs` | JPEG, PNG, TIFF, WebP, JXL, PSD, GIF, BMP; DNG (incl. lossy / Smart Preview DNG), CR2 (colour-filter layout from the file's `CR2CFAPattern` tag, issue #85), ARW (pre-2017 bodies: white balance and black level from enciphered maker-note / `SR2SubIFD` data, issue #148; downsized Sony lossless YCbCr 4:2:0 / 4:2:2 tiles decoded to linear RGB, verified on private ILCE-7M4 3:2 & 4:3 examples including 16-bit full-resolution export), NEF (uncompressed, lossless and lossy compressed 12/14-bit, `crates/raw/src/vendor/nefc.rs`), RAF, RW2 / Leica RWL / Panasonic RAW in every raw format (compressed 4 and 6, the prefix-coded strips of format 8 from the GH6 on, packed 2/5/7, the 16-bit words of the 2005–2007 bodies; black levels, in-camera aspect crops and mapped-out defect pixels; 178 CC0 files from 118 bodies checked, `crates/raw/src/vendor/rw2.rs`), PEF, ORF. 12-bit NEF black level: maker note `0x003d` is in 14-bit units (D750, D780, D850, D7500, Z 50 checked). CR3 opens from its full-size embedded JPEG with full metadata (Exif, GPS, XMP from the `CMT1`/`CMT2`/`CMT4` boxes, `crates/meta/src/cr3.rs`) but its raw data is not decoded yet. Missing: CR3 raw decoding (CRX), compressed RAF/ORF, NEFs labelled compressed (34713) but stored uncompressed without a `0x0096` table (Z 6 packed 14-bit, D850 12-bit uncompressed), NEF "lossy after split" (preview only: imported with `preview_only` = the decoder's reason, rendered as a rendered JPEG, flagged by a grid badge and loupe/Edit/Info notices, `previewOnly` in `catalog.query`; Reload clears it once the file decodes), HEIC/AVIF decode | +| LR-IMP-FORMATS | Supported formats | P0 | 🟡 | `crates/codecs/src/lib.rs`, `crates/raw/src/lib.rs` | JPEG, PNG, TIFF, WebP, JXL, PSD, GIF, BMP; DNG (incl. lossy / Smart Preview DNG), CR2 (colour-filter layout from the file's `CR2CFAPattern` tag, issue #85), ARW (pre-2017 bodies: white balance and black level from enciphered maker-note / `SR2SubIFD` data, issue #148; downsized Sony lossless YCbCr 4:2:0 / 4:2:2 tiles decoded to linear RGB, verified on private ILCE-7M4 3:2 & 4:3 examples including 16-bit full-resolution export), NEF (uncompressed, lossless and lossy compressed 12/14-bit, `crates/raw/src/vendor/nefc.rs`), RAF, RW2 / Leica RWL / Panasonic RAW in every raw format (compressed 4 and 6, the prefix-coded strips of format 8 from the GH6 on, packed 2/5/7, the 16-bit words of the 2005–2007 bodies; black levels, in-camera aspect crops and mapped-out defect pixels; 178 CC0 files from 118 bodies checked, `crates/raw/src/vendor/rw2.rs`), PEF, ORF. 12-bit NEF black level: maker note `0x003d` is in 14-bit units (D750, D780, D850, D7500, Z 50 checked). CR3 opens from its full-size embedded JPEG with full metadata (Exif, GPS, XMP from the `CMT1`/`CMT2`/`CMT4` boxes, `crates/meta/src/cr3.rs`) but its raw data is not decoded yet. Missing: CR3 raw decoding (CRX), compressed RAF/ORF, NEFs labelled compressed (34713) but stored uncompressed without a `0x0096` table (Z 6 packed 14-bit, D850 12-bit uncompressed), NEF "lossy after split" (preview only: imported with `preview_only` = the decoder's reason, rendered as a rendered JPEG, flagged by a grid badge and loupe/Edit/Info notices, `previewOnly` in `catalog.query`; Reload clears it once the file decodes), HEIC decodes behind the non-default `heif` feature (`crates/heif`, heic-rs, same policy as PhotoCraft: 8/10-bit, alpha, container irot/imir/clap, ICC/EXIF/XMP); AVIF decode | | LR-IMP-CAMERA-COVERAGE | Camera coverage, verified per model | P0 | 🟡 | `crates/raw/tests/corpus.rs`, `xtask/src/main.rs` | a supported container is not the same as every camera that writes it decoding correctly: ~55 CC0 corpus files (plus ~60 NEFs checked by hand and 178 Panasonic / Leica RW2, RWL and RAW files from 118 bodies, checked against each file's own JPEG) against the >1,000 models Lightroom lists; per-model differences still surface (e.g. the CR2 colour-filter layout differed by model until it was read from the file's own tag, issue #85, checked on 35 CR2 bodies; pre-2017 Sony ARWs opened green until their scrambled white balance / black level were read, issue #148, checked on 16 ARWs from 15 Sony bodies). Grow the CC0 corpus per model and test each sample decodes with plausible colour | | LR-IMP-CULL-AT-IMPORT | Culling analysis at import | P2 | 🟡 | `cmd:photo.analyze` | run Assisted Culling on the imported photos (they're selected after an import); not automatic | | LR-IMP-MOVE | Move on import [Classic] | P1 | ✅ | `cmd:library.import` (`mode` move), `crates/engine/src/import_move.rs`, `crates/engine/src/tests_import_move.rs`, `crates/ui-egui/src/import.rs` | Transfer ▸ Move with Copy's destination, folders (day / month / one folder / custom template), rename template and example destination; XMP sidecars (both namings) move along; each source is removed only after its destination is written (hard link on the same volume, else copied, synced and compared byte for byte) and its catalog record is saved; failed, duplicate, unchecked files keep their sources, taken names get -1, -2…, never overwritten; files already in the destination/library are added in place; sources that can't be removed (read-only card) are kept and reported (`kept`). Undo removes the photos from the library but leaves the moved files at the destination. No Copy as DNG while moving | diff --git a/docs/whats-new.md b/docs/whats-new.md index ea0243f4..4c7bbc07 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -6,6 +6,11 @@ - Sony ILCE-7M4 downsized lossless ARWs now decode subsampled YCbCr tiles into linear RGB, preserving RAW editing & full-resolution export instead of using embedded JPEG previews. +### Formats +- HEIC / HEIF photos (iPhone and Mac) open now: the optional `lightcraft-heif` crate (heic-rs, pure Rust) behind + codecs' `heif` feature — 8- and 10-bit, alpha, grid tiles, the container's rotation/mirror/crop, ICC, EXIF and XMP. + Off by default (HEVC patents are the distributor's call, same as PhotoCraft); official builds pass `--features heif`. + ### Presets and profiles - Import presets from other editors: XMP presets, classic `.lrtemplate` files, "DNG presets" from mobile apps and `.zip` bundles of any of these — whole folders at once, grouped by pack. Masks inside presets come along. diff --git a/xtask/src/layers.rs b/xtask/src/layers.rs index 28d52677..90400ffb 100644 --- a/xtask/src/layers.rs +++ b/xtask/src/layers.rs @@ -39,6 +39,7 @@ pub const TABLE: &[(&str, Class)] = &[ ("tiff", Class::Layer(0)), ("sysmem", Class::Layer(0)), ("fetch", Class::Layer(0)), + ("heif", Class::Layer(0)), ("raw", Class::Layer(1)), ("codecs", Class::Layer(1)), ("meta", Class::Layer(1)), From 0613f925ded1dd57f8089eff2cb18087b74fb1ea Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 8 Oct 2026 11:47:59 -0500 Subject: [PATCH 02/48] Support remote SAM3 inference through a private Metal worker --- crates/engine/src/cmd/masks.rs | 1 + crates/engine/src/segment/mod.rs | 13 +- crates/engine/src/segment/worker.rs | 75 +++++-- .../segment/src/bin/lightcraft-sam3-worker.rs | 8 + crates/segment/src/lib.rs | 2 + crates/segment/src/remote.rs | 207 ++++++++++++++++++ crates/segment/tests/remote.rs | 21 ++ crates/ui-egui/src/panels/masking.rs | 2 +- docs/ai-masks.md | 37 +++- docs/whats-new.md | 3 + 10 files changed, 350 insertions(+), 19 deletions(-) create mode 100644 crates/segment/src/bin/lightcraft-sam3-worker.rs create mode 100644 crates/segment/src/remote.rs create mode 100644 crates/segment/tests/remote.rs diff --git a/crates/engine/src/cmd/masks.rs b/crates/engine/src/cmd/masks.rs index 1615fb06..dd1fa77c 100644 --- a/crates/engine/src/cmd/masks.rs +++ b/crates/engine/src/cmd/masks.rs @@ -432,6 +432,7 @@ pub fn specs() -> Vec { "available": crate::segment::Segmenter::AVAILABLE, "installed": g.installed(), "dir": g.dir.as_ref().map(|d| d.display().to_string()), + "remote": g.remote_endpoint(), "loaded": g.loaded(), "busy": g.busy(), "analyzing": g.analyzing(), diff --git a/crates/engine/src/segment/mod.rs b/crates/engine/src/segment/mod.rs index 3cf14712..ce927878 100644 --- a/crates/engine/src/segment/mod.rs +++ b/crates/engine/src/segment/mod.rs @@ -157,7 +157,14 @@ impl Segmenter { /// Whether this build can compute AI masks at all. pub const AVAILABLE: bool = cfg!(feature = "sam"); - /// Whether the model's files are in place. + pub fn remote_endpoint(&self) -> Option { + #[cfg(feature = "sam")] + return lightcraft_segment::remote::configured(); + #[cfg(not(feature = "sam"))] + None + } + + /// Whether the local model files are in place. pub fn installed(&self) -> bool { #[cfg(feature = "sam")] if let Some(dir) = &self.dir { @@ -230,6 +237,10 @@ impl Segmenter { if !Self::AVAILABLE { return Err("AI masks are not available in this build".into()); } + #[cfg(feature = "sam")] + if lightcraft_segment::remote::configured().is_some() { + return Ok(PathBuf::new()); + } let dir = self.dir.clone().ok_or("no folder is set for the SAM 3 model")?; if self.installed() { return Ok(dir); diff --git a/crates/engine/src/segment/worker.rs b/crates/engine/src/segment/worker.rs index 9ded0fbe..cb16ab6b 100644 --- a/crates/engine/src/segment/worker.rs +++ b/crates/engine/src/segment/worker.rs @@ -17,7 +17,10 @@ use std::sync::mpsc::{self, RecvTimeoutError}; use std::time::Duration; use lightcraft_develop::SegMask; -use lightcraft_segment::{Encoded, Sam3}; +use lightcraft_segment::{ + Encoded, Sam3, + remote::{self, Operation, Request, text_logits}, +}; use super::{Input, Prompt, Tag}; use crate::media::RenderJob; @@ -175,6 +178,7 @@ impl Worker { struct State { model: Option<(PathBuf, Sam3)>, cache: Option<(u64, Encoded)>, + remote: Option, } /// Whether `later` makes `job` pointless (it asks for the same selection again). @@ -195,7 +199,7 @@ fn run(rx: &mpsc::Receiver, shared: &Shared, idle: Duration) { match rx.recv_timeout(idle) { Ok(j) => queue.push_back(j), Err(RecvTimeoutError::Timeout) => { - if state.model.is_some() { + if state.model.is_some() || state.remote.is_some() { state = State::default(); shared.loaded.store(false, Ordering::SeqCst); log::info!("SAM 3 unloaded after {} minutes without use", idle.as_secs() / 60); @@ -228,7 +232,7 @@ fn run(rx: &mpsc::Receiver, shared: &Shared, idle: Duration) { if result.is_err() && !is_detail { state.cache = None; } - shared.loaded.store(state.model.is_some(), Ordering::SeqCst); + shared.loaded.store(state.model.is_some() || state.remote.is_some(), Ordering::SeqCst); let _ = job.reply.send(Outcome { tag: job.tag, result, superseded: false }); } } @@ -280,29 +284,68 @@ fn encoded<'a>(state: &'a mut State, shared: &Shared, job: &mut Job) -> Result<( log::info!("SAM 3 encoded {}×{} in {:?}", input.w, input.h, t.elapsed()); state.cache = Some((job.key, enc)); } - let State { model, cache } = state; + let State { model, cache, .. } = state; match (model.as_mut(), cache.as_mut()) { (Some((_, m)), Some((_, e))) => Ok((m, e)), _ => Err("the model is not ready".into()), } } -/// Every phrase of `text` ("car, road"), merged (the per-pixel maximum); `None` when nothing -/// matches. -fn text_logits(model: &mut Sam3, enc: &mut Encoded, text: &str) -> Result>, String> { - let mut merged: Option> = None; - for phrase in text.split([',', ';']).map(str::trim).filter(|p| !p.is_empty()) { - if let Some(p) = model.segment_text(enc, phrase, 0.5).map_err(|e| e.to_string())? { - merged = Some(match merged { - None => p.logits, - Some(m) => m.iter().zip(&p.logits).map(|(a, b)| a.max(*b)).collect(), - }); - } +fn remote_job(state: &mut State, shared: &Shared, job: &mut Job, endpoint: &str) -> Result, String> { + let _busy = Flag::raise(&shared.encoding); + if state.remote.is_none() { + state.remote = Some(remote::Client::connect(endpoint)?); } - Ok(merged) + let client = state.remote.as_mut().ok_or("remote SAM unavailable")?; + let mut region = None; + let operation = match std::mem::replace(&mut job.kind, Kind::Prepare) { + Kind::Prepare => Operation::Prepare, + Kind::Text(text) => Operation::Text(text), + Kind::Clicks(clicks) => Operation::Clicks(clicks.iter().map(|c| [c.x, c.y, if c.positive { 1.0 } else { 0.0 }]).collect()), + Kind::Detail { prompt, region: r } => { + region = Some(r); + let (rw, rh) = (r[2] - r[0], r[3] - r[1]); + if !(rw > 0.0 && rh > 0.0) { + return Ok(None); + } + match prompt { + Prompt::Text(text) => Operation::Text(text), + Prompt::Clicks(clicks) => { + let inside: Vec<_> = clicks + .iter() + .map(|c| [((c.at.x - r[0]) / rw) as f32, ((c.at.y - r[1]) / rh) as f32, if c.include { 1.0 } else { 0.0 }]) + .filter(|p| (0.0..=1.0).contains(&p[0]) && (0.0..=1.0).contains(&p[1])) + .collect(); + if !inside.iter().any(|p| p[2] == 1.0) { + return Ok(None); + } + Operation::Clicks(inside) + } + } + } + #[cfg(test)] + Kind::Panic => return Err("test panic request".into()), + }; + let input = if region.is_some() || client.key != Some(job.key) { render(job)? } else { Input { rgb: Vec::new(), w: 0, h: 0 } }; + let request = Request { width: input.w, height: input.h, temporary: region.is_some(), operation }; + let logits = client.request(&request, &input.rgb)?; + if region.is_none() { + client.key = Some(job.key); + } + Ok(logits.map(|l| match region { + Some(r) => SegMask::from_logits_in(lightcraft_segment::MASK_SIDE, &l, r), + None => SegMask::from_logits(lightcraft_segment::MASK_SIDE, &l), + })) } fn run_job(state: &mut State, shared: &Shared, job: &mut Job) -> Result, String> { + if let Some(endpoint) = remote::configured() { + let result = remote_job(state, shared, job, &endpoint); + if result.is_err() { + state.remote = None; + } + return result.map_err(|e| format!("Remote SAM 3 failed: {e}. Check the worker and SSH tunnel, then retry.")); + } let side = lightcraft_segment::MASK_SIDE; match std::mem::replace(&mut job.kind, Kind::Prepare) { Kind::Prepare => { diff --git a/crates/segment/src/bin/lightcraft-sam3-worker.rs b/crates/segment/src/bin/lightcraft-sam3-worker.rs new file mode 100644 index 00000000..9929d2eb --- /dev/null +++ b/crates/segment/src/bin/lightcraft-sam3-worker.rs @@ -0,0 +1,8 @@ +fn main() -> Result<(), Box> { + let mut args = std::env::args().skip(1); + let dir = args.next().ok_or("usage: lightcraft-sam3-worker MODEL_DIR [127.0.0.1:8793]")?; + let address = args.next().unwrap_or_else(|| "127.0.0.1:8793".into()).parse()?; + let device = candle_core::Device::new_metal(0)?; + lightcraft_segment::remote::serve(address, std::path::Path::new(&dir), device)?; + Ok(()) +} diff --git a/crates/segment/src/lib.rs b/crates/segment/src/lib.rs index 42aed538..a4c3acad 100644 --- a/crates/segment/src/lib.rs +++ b/crates/segment/src/lib.rs @@ -32,6 +32,8 @@ mod clip; mod detector; #[cfg(not(target_arch = "wasm32"))] pub mod fetch; +#[cfg(not(target_arch = "wasm32"))] +pub mod remote; pub mod mask; #[cfg(not(target_arch = "wasm32"))] mod neck; diff --git a/crates/segment/src/remote.rs b/crates/segment/src/remote.rs new file mode 100644 index 00000000..cdee096f --- /dev/null +++ b/crates/segment/src/remote.rs @@ -0,0 +1,207 @@ +//! Private SAM 3 transport. Bind the worker to loopback and forward it over SSH. +use std::io::{self, Read, Write}; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::path::Path; +use std::time::Duration; + +use crate::{Click, Encoded, INPUT, MASK_SIDE, Sam3}; +use serde::{Deserialize, Serialize}; + +const CONTROL_LIMIT: usize = 16 * 1024; +const RESPONSE_LIMIT: usize = 4 * 1024 * 1024; +const TIMEOUT: Duration = Duration::from_secs(9 * 60); +pub const ENV: &str = "LIGHTCRAFT_SAM3_REMOTE"; + +#[derive(Serialize, Deserialize)] +pub enum Operation { + Prepare, + Clicks(Vec<[f32; 3]>), + Text(String), +} + +#[derive(Serialize, Deserialize)] +pub struct Request { + pub width: usize, + pub height: usize, + pub temporary: bool, + pub operation: Operation, +} + +impl Request { + pub fn validate(&self) -> Result<(), String> { + if self.width > INPUT || self.height > INPUT || (self.width == 0) != (self.height == 0) { + return Err("invalid SAM image dimensions".into()); + } + if self.temporary && self.width == 0 { + return Err("detail pass requires an image".into()); + } + match &self.operation { + Operation::Prepare => {} + Operation::Text(text) if text.len() <= 4096 && text.split([',', ';']).any(|p| !p.trim().is_empty()) => {} + Operation::Clicks(clicks) + if !clicks.is_empty() + && clicks.len() <= 64 + && clicks.iter().all(|c| { + c.iter().all(|v| v.is_finite()) && (0.0..=1.0).contains(&c[0]) && (0.0..=1.0).contains(&c[1]) && (c[2] == 0.0 || c[2] == 1.0) + }) + && clicks.iter().any(|c| c[2] == 1.0) => {} + _ => return Err("invalid SAM prompt".into()), + } + Ok(()) + } +} + +pub fn configured() -> Option { + std::env::var(ENV).ok().filter(|v| !v.trim().is_empty()) +} + +pub fn read_frame(reader: &mut impl Read, limit: usize) -> io::Result> { + let mut size = [0; 4]; + reader.read_exact(&mut size)?; + let size = u32::from_be_bytes(size) as usize; + if size > limit { + return Err(io::Error::new(io::ErrorKind::InvalidData, "SAM frame exceeds limit")); + } + let mut data = vec![0; size]; + reader.read_exact(&mut data)?; + Ok(data) +} + +fn write_frame(writer: &mut impl Write, data: &[u8]) -> io::Result<()> { + let size = u32::try_from(data.len()).map_err(io::Error::other)?; + writer.write_all(&size.to_be_bytes())?; + writer.write_all(data) +} + +pub struct Client { + stream: TcpStream, + pub key: Option, +} + +impl Client { + pub fn connect(endpoint: &str) -> Result { + let address: SocketAddr = endpoint.parse().map_err(|e| format!("SAM endpoint must be an IP:port: {e}"))?; + if !address.ip().is_loopback() { + return Err("SAM endpoint must be loopback; use an SSH tunnel for remote access".into()); + } + let stream = TcpStream::connect_timeout(&address, Duration::from_secs(5)).map_err(|e| e.to_string())?; + stream.set_read_timeout(Some(TIMEOUT)).map_err(|e| e.to_string())?; + stream.set_write_timeout(Some(Duration::from_secs(30))).map_err(|e| e.to_string())?; + Ok(Self { stream, key: None }) + } + + pub fn request(&mut self, request: &Request, rgb: &[u8]) -> Result>, String> { + request.validate()?; + if rgb.len() != request.width * request.height * 3 { + return Err("SAM RGB buffer size does not match dimensions".into()); + } + let bytes = serde_json::to_vec(request).map_err(|e| e.to_string())?; + write_frame(&mut self.stream, &bytes).map_err(|e| e.to_string())?; + write_frame(&mut self.stream, rgb).map_err(|e| e.to_string())?; + let response = read_frame(&mut self.stream, RESPONSE_LIMIT).map_err(|e| e.to_string())?; + let result: Result>, String> = serde_json::from_slice(&response).map_err(|e| e.to_string())?; + let logits = result?; + if logits.as_ref().is_some_and(|v| v.len() != MASK_SIDE * MASK_SIDE || v.iter().any(|n| !n.is_finite())) { + return Err("invalid SAM mask response".into()); + } + Ok(logits) + } +} + +pub fn text_logits(model: &mut Sam3, enc: &mut Encoded, text: &str) -> Result>, String> { + let mut merged: Option> = None; + for phrase in text.split([',', ';']).map(str::trim).filter(|p| !p.is_empty()) { + if let Some(p) = model.segment_text(enc, phrase, 0.5).map_err(|e| e.to_string())? { + if let Some(m) = &mut merged { + for (a, b) in m.iter_mut().zip(p.logits) { + *a = a.max(b); + } + } else { + merged = Some(p.logits); + } + } + } + Ok(merged) +} + +fn predict(model: &mut Sam3, enc: &mut Encoded, operation: Operation) -> Result>, String> { + match operation { + Operation::Prepare => { + model.prepare_clicks(enc).map_err(|e| e.to_string())?; + Ok(None) + } + Operation::Text(text) => text_logits(model, enc, &text), + Operation::Clicks(points) => { + let clicks: Vec<_> = points.into_iter().map(|p| Click { x: p[0], y: p[1], positive: p[2] == 1.0 }).collect(); + Ok(Some(model.segment_clicks(enc, &clicks).map_err(|e| e.to_string())?.logits)) + } + } +} + +fn serve_client(stream: &mut TcpStream, dir: &Path, device: candle_core::Device) -> Result<(), String> { + stream.set_read_timeout(Some(Duration::from_secs(10 * 60))).map_err(|e| e.to_string())?; + stream.set_write_timeout(Some(Duration::from_secs(30))).map_err(|e| e.to_string())?; + let mut model = None; + let mut cache = None; + loop { + let frame = read_frame(stream, CONTROL_LIMIT).map_err(|e| e.to_string())?; + let request: Request = serde_json::from_slice(&frame).map_err(|e| e.to_string())?; + request.validate()?; + let rgb = read_frame(stream, request.width * request.height * 3).map_err(|e| e.to_string())?; + if rgb.len() != request.width * request.height * 3 { + return Err("invalid RGB payload".into()); + } + let started = std::time::Instant::now(); + let result = (|| { + if model.is_none() { + model = Some(Sam3::load_on(dir, device.clone()).map_err(|e| e.to_string())?); + } + let m = model.as_mut().ok_or("SAM model unavailable")?; + let mut temporary = None; + if request.width > 0 { + let encoded = m.encode(&rgb, request.width, request.height).map_err(|e| e.to_string())?; + if request.temporary { + temporary = Some(encoded); + } else { + cache = Some(encoded); + } + } + let enc = if request.temporary { temporary.as_mut() } else { cache.as_mut() }.ok_or("SAM image cache empty")?; + predict(m, enc, request.operation) + })(); + eprintln!( + "SAM3 Metal request elapsed={:.3}s image={}x{} detail={} success={}", + started.elapsed().as_secs_f64(), + request.width, + request.height, + request.temporary, + result.is_ok() + ); + let response = serde_json::to_vec(&result).map_err(|e| e.to_string())?; + write_frame(stream, &response).map_err(|e| e.to_string())?; + if result.is_err() { + return Err("SAM inference failed; session released".into()); + } + } +} + +pub fn serve(address: SocketAddr, dir: &Path, device: candle_core::Device) -> Result<(), String> { + if !address.ip().is_loopback() || !device.is_metal() { + return Err("SAM worker requires a loopback address and a Metal device".into()); + } + if !crate::is_model_dir(dir) { + return Err("SAM model files are missing".into()); + } + let listener = TcpListener::bind(address).map_err(|e| e.to_string())?; + eprintln!("SAM3 worker listening on {address}; device=Metal; idle unload=600s"); + for stream in listener.incoming() { + let mut stream = stream.map_err(|e| e.to_string())?; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| serve_client(&mut stream, dir, device.clone()))); + match result { + Ok(Err(error)) => eprintln!("SAM3 session closed: {error}"), + Err(_) => eprintln!("SAM3 session panicked; model released"), + Ok(Ok(())) => {} + } + } + Ok(()) +} diff --git a/crates/segment/tests/remote.rs b/crates/segment/tests/remote.rs new file mode 100644 index 00000000..ee20ea20 --- /dev/null +++ b/crates/segment/tests/remote.rs @@ -0,0 +1,21 @@ +use lightcraft_segment::remote::{Operation, Request, read_frame}; +use std::io::Cursor; + +#[test] +fn rejects_unbounded_and_truncated_frames() { + assert!(read_frame(&mut Cursor::new(u32::MAX.to_be_bytes()), 1024).is_err()); + assert!(read_frame(&mut Cursor::new([0, 0, 0, 3, 1]), 1024).is_err()); +} + +#[test] +fn validates_images_and_prompt_boundaries() { + let request = |width, height, operation| Request { width, height, temporary: false, operation }; + assert!(request(1009, 1, Operation::Prepare).validate().is_err()); + assert!(request(0, 1, Operation::Prepare).validate().is_err()); + assert!(request(0, 0, Operation::Text("dog".into())).validate().is_ok()); + assert!(request(4, 3, Operation::Clicks(vec![[0.5, 0.5, 1.0]])).validate().is_ok()); + assert!(request(4, 3, Operation::Clicks(vec![[1.5, 0.5, 1.0]])).validate().is_err()); + assert!(request(4, 3, Operation::Clicks(vec![[0.5, f32::NAN, 1.0]])).validate().is_err()); + assert!(request(4, 3, Operation::Clicks(vec![[0.5, 0.5, 0.0]])).validate().is_err()); + assert!(request(4, 3, Operation::Text(" ".into())).validate().is_err()); +} diff --git a/crates/ui-egui/src/panels/masking.rs b/crates/ui-egui/src/panels/masking.rs index 2e150a60..12ddb031 100644 --- a/crates/ui-egui/src/panels/masking.rs +++ b/crates/ui-egui/src/panels/masking.rs @@ -639,7 +639,7 @@ fn component_row_menu(app: &mut LightcraftApp, ui: &mut egui::Ui, mask: u32, k: /// offered (with `then` to start afterwards) instead of starting an AI mask. fn needs_model(app: &mut LightcraftApp, kind: &str, op: &str) -> bool { let seg = &app.session.segmenter; - let missing = lightcraft_engine::segment::Segmenter::AVAILABLE && seg.dir.is_some() && !seg.installed(); + let missing = lightcraft_engine::segment::Segmenter::AVAILABLE && seg.remote_endpoint().is_none() && seg.dir.is_some() && !seg.installed(); if missing { app.offer_sam_download(Some((kind, op))); } diff --git a/docs/ai-masks.md b/docs/ai-masks.md index 5a32d690..00de5016 100644 --- a/docs/ai-masks.md +++ b/docs/ai-masks.md @@ -42,6 +42,39 @@ The result is stored with the mask: the model's 288 × 288 mask logits over the need the model, so masks survive moving the library to a machine without it, and the web build renders them. Editing the photo's look later does not recompute a mask (click again to update). +## Remote Metal inference + +Native builds with `sam` can keep the editor, RAW files, library, and exports local while +a Mac runs SAM 3. Build both ends from the same revision. On the Mac, install the model +as described below, then run: + +```sh +cargo build --release --locked -p lightcraft-segment --bin lightcraft-sam3-worker +target/release/lightcraft-sam3-worker "$HOME/Library/Application Support/LightCraft/models/sam3" 127.0.0.1:8793 +``` + +On the editing computer, start an SSH tunnel in one terminal and LightCraft in another: + +```sh +ssh -N -T -o ExitOnForwardFailure=yes -L 127.0.0.1:8793:127.0.0.1:8793 user@your-mac +LIGHTCRAFT_SAM3_REMOTE=127.0.0.1:8793 lightcraft +``` + +Object clicks, Describe prompts, and detail refinement use the remote worker. Only the +rendered model input (long edge at most 1008 pixels) and prompts cross the tunnel. The +worker caches the current photo; detail crops do not replace that cache. Returned masks +stay in the local library, so later adjustments and exports work without the connection. + +The worker requires Metal and accepts one editing connection at a time. It loads weights +on demand and releases them when the client disconnects or remains idle for ten minutes. +Both endpoints require loopback addresses: SSH provides authentication and encryption. +The listener has no independent authentication; do not expose it through a public proxy. + +No local weights are required in remote mode. An unavailable worker produces an explicit +error, not a silent CPU fallback. Restore the tunnel and retry. Unset +`LIGHTCRAFT_SAM3_REMOTE` to use the local model again. This offloads segmentation only, +not RAW development, denoising, or upscaling. The model licence below still applies. + ## Getting the model ### In the app (recommended) @@ -148,10 +181,12 @@ Therefore: | `mask.objectPoint` | `{x, y, exclude?: bool, id?}` | one click on the selected mask's Object selection (≤ 64 clicks) | | `mask.refineDetail` | `{id?, component?}` | the zoomed-in detail pass → `{started}` | | `segment.prepare` | `{}` | load the model and analyze the active photo → `{busy}` | -| `segment.model.status` | `{}` | `{available, installed, dir, loaded, busy, analyzing, sizeBytes, license, licenseUrl, mirrors, download}` | +| `segment.model.status` | `{}` | `{available, installed, remote, dir, loaded, busy, analyzing, sizeBytes, license, licenseUrl, mirrors, download}` | | `segment.model.download` | `{acknowledged: true}` | start the download in the background → `{started}` | | `segment.model.cancel` | `{}` | stop it (it resumes next time) → `{cancelled}` | +`remote` reports the configured endpoint, not its health. `installed` describes local weights only. + Coordinates are normalized to the uncropped, oriented photo, like every mask shape. `seg` passes a stored segmentation (as `mask.list` / the develop settings hold it): the mask is made without the model. Without the model, the AI kinds fail with an error starting "The SAM 3 model is not diff --git a/docs/whats-new.md b/docs/whats-new.md index ea0243f4..658ff4e0 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -135,6 +135,9 @@ lost everything at quit; a temporary session shows a banner the whole time and never writes to your library. ### Editing +- Optional remote SAM 3: keep the native editor local and run Object, Describe, and detail + inference on a Mac through SSH. Saved masks still render and export offline. + See [remote Metal inference](ai-masks.md#remote-metal-inference). - AI masks with SAM 3 (Object and Describe in the Masking panel): click an object to select it (⌥-click leaves a part out), or type what to select ("sky", "the red car", "car, road"); both combine with other masks, have an Edge setting, and get a sharper zoomed-in pass in the background. The model runs inside LightCraft in pure Rust From caa102aabde2aa1629773cfac66ea2f9483f1901 Mon Sep 17 00:00:00 2001 From: radialmonster Date: Thu, 8 Oct 2026 20:40:23 -0400 Subject: [PATCH 03/48] MENU-BAR: pointing at another menu title switches the open menu in the in-window bar --- crates/ui-egui/src/menubar.rs | 68 +++++++++++++++++++++++++++-- crates/ui-egui/src/tests_menubar.rs | 61 ++++++++++++++++++++++++++ docs/parity.md | 2 +- 3 files changed, 126 insertions(+), 5 deletions(-) diff --git a/crates/ui-egui/src/menubar.rs b/crates/ui-egui/src/menubar.rs index 97c2d634..2925d0c2 100644 --- a/crates/ui-egui/src/menubar.rs +++ b/crates/ui-egui/src/menubar.rs @@ -752,10 +752,29 @@ pub fn show_in_window(app: &mut LightcraftApp, ui: &mut egui::Ui, max_width: f32 if total <= max_width { let saved = ui.spacing().item_spacing.x; ui.spacing_mut().item_spacing.x = TITLE_GAP; - for (title, items) in &bar { - let r = ui.add(egui::Button::new(egui::RichText::new(crate::i18n::tr(title)).font(font.clone()).color(t.text_label)).frame(false)); - crate::widgets::register(ui.ctx(), format!("menu:{title}"), r.rect); - egui::Popup::menu(&r).show(|ui| crate::menu_level::level(ui, 1, bar_bottom, |ui| nodes_ui(ui, items, mac, &mut clicked, 1, bar_bottom))); + // All titles first, then their popups, so a title the pointer moves onto can take over + // from the open menu before either is drawn (no frame with two menus or none). + let titles: Vec = bar + .iter() + .map(|(title, _)| { + let r = ui.add(egui::Button::new(egui::RichText::new(crate::i18n::tr(title)).font(font.clone()).color(t.text_label)).frame(false)); + crate::widgets::register(ui.ctx(), format!("menu:{title}"), r.rect); + r + }) + .collect(); + let confirmed = switch_open_title(ui.ctx(), &titles); + for ((_, items), r) in bar.iter().zip(&titles) { + // (opening and closing by click is done in `switch_open_title`) + // (the click confirming a menu a hover opened must not close it again) + let close = if confirmed == Some(egui::Popup::default_response_id(r)) { + egui::PopupCloseBehavior::IgnoreClicks + } else { + egui::PopupCloseBehavior::CloseOnClick + }; + egui::Popup::menu(r) + .close_behavior(close) + .open_memory(None) + .show(|ui| crate::menu_level::level(ui, 1, bar_bottom, |ui| nodes_ui(ui, items, mac, &mut clicked, 1, bar_bottom))); } ui.spacing_mut().item_spacing.x = saved; } else { @@ -781,6 +800,47 @@ pub fn show_in_window(app: &mut LightcraftApp, ui: &mut egui::Ui, max_width: f32 ui.cursor().left() - start } +/// Native menu-bar behaviour for the top-level titles: while one menu is open, resting the pointer +/// on another title closes it and opens that one, and a click on another title opens it in that +/// same click (instead of only closing the open menu as a click outside). Without an open menu +/// hovering does nothing; clicking a title toggles its menu. egui keeps one popup open at a time. +/// Returns the menu whose title was just clicked right after a hover opened it. +fn switch_open_title(ctx: &egui::Context, titles: &[egui::Response]) -> Option { + let popup = egui::Popup::default_response_id; + let key = egui::Id::new("lc-menubar-hover-opened"); + let open = titles.iter().position(|r| egui::Popup::is_id_open(ctx, popup(r))); + // the menu a hover opened, until it closes or is clicked + let mut confirmed = None; + let mut hover_opened: Option = ctx.data(|d| d.get_temp(key)).filter(|id| open.is_some_and(|o| popup(&titles[o]) == *id)); + for (i, r) in titles.iter().enumerate() { + if r.clicked() { + // the click that follows a hover switch confirms the menu the hover opened + if hover_opened == Some(popup(r)) { + confirmed = Some(popup(r)); + } else { + egui::Popup::toggle_id(ctx, popup(r)); + } + hover_opened = None; + } else if r.hovered() && open.is_some_and(|o| o != i) { + egui::Popup::open_id(ctx, popup(r)); + hover_opened = Some(popup(r)); + // a pointer move alone doesn't schedule the frame that draws the new menu + ctx.request_repaint(); + } + } + // (leaving the titles ends it: a later click on the open title closes the menu) + let hover_opened = hover_opened.filter(|_| titles.iter().any(|r| r.hovered())); + ctx.data_mut(|d| match hover_opened { + Some(id) => { + d.insert_temp(key, id); + } + None => { + d.remove::(key); + } + }); + confirmed +} + /// A submenu row showing `text` at `depth`, whose rows `children` draws one level deeper, /// bounded like every level. The row is registered as `menusub: