From 0613f925ded1dd57f8089eff2cb18087b74fb1ea Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Thu, 8 Oct 2026 11:47:59 -0500 Subject: [PATCH 01/46] Support remote SAM3 inference through a private Metal worker --- crates/engine/src/cmd/masks.rs | 1 + crates/engine/src/segment/mod.rs | 13 +- crates/engine/src/segment/worker.rs | 75 +++++-- .../segment/src/bin/lightcraft-sam3-worker.rs | 8 + crates/segment/src/lib.rs | 2 + crates/segment/src/remote.rs | 207 ++++++++++++++++++ crates/segment/tests/remote.rs | 21 ++ crates/ui-egui/src/panels/masking.rs | 2 +- docs/ai-masks.md | 37 +++- docs/whats-new.md | 3 + 10 files changed, 350 insertions(+), 19 deletions(-) create mode 100644 crates/segment/src/bin/lightcraft-sam3-worker.rs create mode 100644 crates/segment/src/remote.rs create mode 100644 crates/segment/tests/remote.rs diff --git a/crates/engine/src/cmd/masks.rs b/crates/engine/src/cmd/masks.rs index 1615fb06..dd1fa77c 100644 --- a/crates/engine/src/cmd/masks.rs +++ b/crates/engine/src/cmd/masks.rs @@ -432,6 +432,7 @@ pub fn specs() -> Vec { "available": crate::segment::Segmenter::AVAILABLE, "installed": g.installed(), "dir": g.dir.as_ref().map(|d| d.display().to_string()), + "remote": g.remote_endpoint(), "loaded": g.loaded(), "busy": g.busy(), "analyzing": g.analyzing(), diff --git a/crates/engine/src/segment/mod.rs b/crates/engine/src/segment/mod.rs index 3cf14712..ce927878 100644 --- a/crates/engine/src/segment/mod.rs +++ b/crates/engine/src/segment/mod.rs @@ -157,7 +157,14 @@ impl Segmenter { /// Whether this build can compute AI masks at all. pub const AVAILABLE: bool = cfg!(feature = "sam"); - /// Whether the model's files are in place. + pub fn remote_endpoint(&self) -> Option { + #[cfg(feature = "sam")] + return lightcraft_segment::remote::configured(); + #[cfg(not(feature = "sam"))] + None + } + + /// Whether the local model files are in place. pub fn installed(&self) -> bool { #[cfg(feature = "sam")] if let Some(dir) = &self.dir { @@ -230,6 +237,10 @@ impl Segmenter { if !Self::AVAILABLE { return Err("AI masks are not available in this build".into()); } + #[cfg(feature = "sam")] + if lightcraft_segment::remote::configured().is_some() { + return Ok(PathBuf::new()); + } let dir = self.dir.clone().ok_or("no folder is set for the SAM 3 model")?; if self.installed() { return Ok(dir); diff --git a/crates/engine/src/segment/worker.rs b/crates/engine/src/segment/worker.rs index 9ded0fbe..cb16ab6b 100644 --- a/crates/engine/src/segment/worker.rs +++ b/crates/engine/src/segment/worker.rs @@ -17,7 +17,10 @@ use std::sync::mpsc::{self, RecvTimeoutError}; use std::time::Duration; use lightcraft_develop::SegMask; -use lightcraft_segment::{Encoded, Sam3}; +use lightcraft_segment::{ + Encoded, Sam3, + remote::{self, Operation, Request, text_logits}, +}; use super::{Input, Prompt, Tag}; use crate::media::RenderJob; @@ -175,6 +178,7 @@ impl Worker { struct State { model: Option<(PathBuf, Sam3)>, cache: Option<(u64, Encoded)>, + remote: Option, } /// Whether `later` makes `job` pointless (it asks for the same selection again). @@ -195,7 +199,7 @@ fn run(rx: &mpsc::Receiver, shared: &Shared, idle: Duration) { match rx.recv_timeout(idle) { Ok(j) => queue.push_back(j), Err(RecvTimeoutError::Timeout) => { - if state.model.is_some() { + if state.model.is_some() || state.remote.is_some() { state = State::default(); shared.loaded.store(false, Ordering::SeqCst); log::info!("SAM 3 unloaded after {} minutes without use", idle.as_secs() / 60); @@ -228,7 +232,7 @@ fn run(rx: &mpsc::Receiver, shared: &Shared, idle: Duration) { if result.is_err() && !is_detail { state.cache = None; } - shared.loaded.store(state.model.is_some(), Ordering::SeqCst); + shared.loaded.store(state.model.is_some() || state.remote.is_some(), Ordering::SeqCst); let _ = job.reply.send(Outcome { tag: job.tag, result, superseded: false }); } } @@ -280,29 +284,68 @@ fn encoded<'a>(state: &'a mut State, shared: &Shared, job: &mut Job) -> Result<( log::info!("SAM 3 encoded {}×{} in {:?}", input.w, input.h, t.elapsed()); state.cache = Some((job.key, enc)); } - let State { model, cache } = state; + let State { model, cache, .. } = state; match (model.as_mut(), cache.as_mut()) { (Some((_, m)), Some((_, e))) => Ok((m, e)), _ => Err("the model is not ready".into()), } } -/// Every phrase of `text` ("car, road"), merged (the per-pixel maximum); `None` when nothing -/// matches. -fn text_logits(model: &mut Sam3, enc: &mut Encoded, text: &str) -> Result>, String> { - let mut merged: Option> = None; - for phrase in text.split([',', ';']).map(str::trim).filter(|p| !p.is_empty()) { - if let Some(p) = model.segment_text(enc, phrase, 0.5).map_err(|e| e.to_string())? { - merged = Some(match merged { - None => p.logits, - Some(m) => m.iter().zip(&p.logits).map(|(a, b)| a.max(*b)).collect(), - }); - } +fn remote_job(state: &mut State, shared: &Shared, job: &mut Job, endpoint: &str) -> Result, String> { + let _busy = Flag::raise(&shared.encoding); + if state.remote.is_none() { + state.remote = Some(remote::Client::connect(endpoint)?); } - Ok(merged) + let client = state.remote.as_mut().ok_or("remote SAM unavailable")?; + let mut region = None; + let operation = match std::mem::replace(&mut job.kind, Kind::Prepare) { + Kind::Prepare => Operation::Prepare, + Kind::Text(text) => Operation::Text(text), + Kind::Clicks(clicks) => Operation::Clicks(clicks.iter().map(|c| [c.x, c.y, if c.positive { 1.0 } else { 0.0 }]).collect()), + Kind::Detail { prompt, region: r } => { + region = Some(r); + let (rw, rh) = (r[2] - r[0], r[3] - r[1]); + if !(rw > 0.0 && rh > 0.0) { + return Ok(None); + } + match prompt { + Prompt::Text(text) => Operation::Text(text), + Prompt::Clicks(clicks) => { + let inside: Vec<_> = clicks + .iter() + .map(|c| [((c.at.x - r[0]) / rw) as f32, ((c.at.y - r[1]) / rh) as f32, if c.include { 1.0 } else { 0.0 }]) + .filter(|p| (0.0..=1.0).contains(&p[0]) && (0.0..=1.0).contains(&p[1])) + .collect(); + if !inside.iter().any(|p| p[2] == 1.0) { + return Ok(None); + } + Operation::Clicks(inside) + } + } + } + #[cfg(test)] + Kind::Panic => return Err("test panic request".into()), + }; + let input = if region.is_some() || client.key != Some(job.key) { render(job)? } else { Input { rgb: Vec::new(), w: 0, h: 0 } }; + let request = Request { width: input.w, height: input.h, temporary: region.is_some(), operation }; + let logits = client.request(&request, &input.rgb)?; + if region.is_none() { + client.key = Some(job.key); + } + Ok(logits.map(|l| match region { + Some(r) => SegMask::from_logits_in(lightcraft_segment::MASK_SIDE, &l, r), + None => SegMask::from_logits(lightcraft_segment::MASK_SIDE, &l), + })) } fn run_job(state: &mut State, shared: &Shared, job: &mut Job) -> Result, String> { + if let Some(endpoint) = remote::configured() { + let result = remote_job(state, shared, job, &endpoint); + if result.is_err() { + state.remote = None; + } + return result.map_err(|e| format!("Remote SAM 3 failed: {e}. Check the worker and SSH tunnel, then retry.")); + } let side = lightcraft_segment::MASK_SIDE; match std::mem::replace(&mut job.kind, Kind::Prepare) { Kind::Prepare => { diff --git a/crates/segment/src/bin/lightcraft-sam3-worker.rs b/crates/segment/src/bin/lightcraft-sam3-worker.rs new file mode 100644 index 00000000..9929d2eb --- /dev/null +++ b/crates/segment/src/bin/lightcraft-sam3-worker.rs @@ -0,0 +1,8 @@ +fn main() -> Result<(), Box> { + let mut args = std::env::args().skip(1); + let dir = args.next().ok_or("usage: lightcraft-sam3-worker MODEL_DIR [127.0.0.1:8793]")?; + let address = args.next().unwrap_or_else(|| "127.0.0.1:8793".into()).parse()?; + let device = candle_core::Device::new_metal(0)?; + lightcraft_segment::remote::serve(address, std::path::Path::new(&dir), device)?; + Ok(()) +} diff --git a/crates/segment/src/lib.rs b/crates/segment/src/lib.rs index 42aed538..a4c3acad 100644 --- a/crates/segment/src/lib.rs +++ b/crates/segment/src/lib.rs @@ -32,6 +32,8 @@ mod clip; mod detector; #[cfg(not(target_arch = "wasm32"))] pub mod fetch; +#[cfg(not(target_arch = "wasm32"))] +pub mod remote; pub mod mask; #[cfg(not(target_arch = "wasm32"))] mod neck; diff --git a/crates/segment/src/remote.rs b/crates/segment/src/remote.rs new file mode 100644 index 00000000..cdee096f --- /dev/null +++ b/crates/segment/src/remote.rs @@ -0,0 +1,207 @@ +//! Private SAM 3 transport. Bind the worker to loopback and forward it over SSH. +use std::io::{self, Read, Write}; +use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::path::Path; +use std::time::Duration; + +use crate::{Click, Encoded, INPUT, MASK_SIDE, Sam3}; +use serde::{Deserialize, Serialize}; + +const CONTROL_LIMIT: usize = 16 * 1024; +const RESPONSE_LIMIT: usize = 4 * 1024 * 1024; +const TIMEOUT: Duration = Duration::from_secs(9 * 60); +pub const ENV: &str = "LIGHTCRAFT_SAM3_REMOTE"; + +#[derive(Serialize, Deserialize)] +pub enum Operation { + Prepare, + Clicks(Vec<[f32; 3]>), + Text(String), +} + +#[derive(Serialize, Deserialize)] +pub struct Request { + pub width: usize, + pub height: usize, + pub temporary: bool, + pub operation: Operation, +} + +impl Request { + pub fn validate(&self) -> Result<(), String> { + if self.width > INPUT || self.height > INPUT || (self.width == 0) != (self.height == 0) { + return Err("invalid SAM image dimensions".into()); + } + if self.temporary && self.width == 0 { + return Err("detail pass requires an image".into()); + } + match &self.operation { + Operation::Prepare => {} + Operation::Text(text) if text.len() <= 4096 && text.split([',', ';']).any(|p| !p.trim().is_empty()) => {} + Operation::Clicks(clicks) + if !clicks.is_empty() + && clicks.len() <= 64 + && clicks.iter().all(|c| { + c.iter().all(|v| v.is_finite()) && (0.0..=1.0).contains(&c[0]) && (0.0..=1.0).contains(&c[1]) && (c[2] == 0.0 || c[2] == 1.0) + }) + && clicks.iter().any(|c| c[2] == 1.0) => {} + _ => return Err("invalid SAM prompt".into()), + } + Ok(()) + } +} + +pub fn configured() -> Option { + std::env::var(ENV).ok().filter(|v| !v.trim().is_empty()) +} + +pub fn read_frame(reader: &mut impl Read, limit: usize) -> io::Result> { + let mut size = [0; 4]; + reader.read_exact(&mut size)?; + let size = u32::from_be_bytes(size) as usize; + if size > limit { + return Err(io::Error::new(io::ErrorKind::InvalidData, "SAM frame exceeds limit")); + } + let mut data = vec![0; size]; + reader.read_exact(&mut data)?; + Ok(data) +} + +fn write_frame(writer: &mut impl Write, data: &[u8]) -> io::Result<()> { + let size = u32::try_from(data.len()).map_err(io::Error::other)?; + writer.write_all(&size.to_be_bytes())?; + writer.write_all(data) +} + +pub struct Client { + stream: TcpStream, + pub key: Option, +} + +impl Client { + pub fn connect(endpoint: &str) -> Result { + let address: SocketAddr = endpoint.parse().map_err(|e| format!("SAM endpoint must be an IP:port: {e}"))?; + if !address.ip().is_loopback() { + return Err("SAM endpoint must be loopback; use an SSH tunnel for remote access".into()); + } + let stream = TcpStream::connect_timeout(&address, Duration::from_secs(5)).map_err(|e| e.to_string())?; + stream.set_read_timeout(Some(TIMEOUT)).map_err(|e| e.to_string())?; + stream.set_write_timeout(Some(Duration::from_secs(30))).map_err(|e| e.to_string())?; + Ok(Self { stream, key: None }) + } + + pub fn request(&mut self, request: &Request, rgb: &[u8]) -> Result>, String> { + request.validate()?; + if rgb.len() != request.width * request.height * 3 { + return Err("SAM RGB buffer size does not match dimensions".into()); + } + let bytes = serde_json::to_vec(request).map_err(|e| e.to_string())?; + write_frame(&mut self.stream, &bytes).map_err(|e| e.to_string())?; + write_frame(&mut self.stream, rgb).map_err(|e| e.to_string())?; + let response = read_frame(&mut self.stream, RESPONSE_LIMIT).map_err(|e| e.to_string())?; + let result: Result>, String> = serde_json::from_slice(&response).map_err(|e| e.to_string())?; + let logits = result?; + if logits.as_ref().is_some_and(|v| v.len() != MASK_SIDE * MASK_SIDE || v.iter().any(|n| !n.is_finite())) { + return Err("invalid SAM mask response".into()); + } + Ok(logits) + } +} + +pub fn text_logits(model: &mut Sam3, enc: &mut Encoded, text: &str) -> Result>, String> { + let mut merged: Option> = None; + for phrase in text.split([',', ';']).map(str::trim).filter(|p| !p.is_empty()) { + if let Some(p) = model.segment_text(enc, phrase, 0.5).map_err(|e| e.to_string())? { + if let Some(m) = &mut merged { + for (a, b) in m.iter_mut().zip(p.logits) { + *a = a.max(b); + } + } else { + merged = Some(p.logits); + } + } + } + Ok(merged) +} + +fn predict(model: &mut Sam3, enc: &mut Encoded, operation: Operation) -> Result>, String> { + match operation { + Operation::Prepare => { + model.prepare_clicks(enc).map_err(|e| e.to_string())?; + Ok(None) + } + Operation::Text(text) => text_logits(model, enc, &text), + Operation::Clicks(points) => { + let clicks: Vec<_> = points.into_iter().map(|p| Click { x: p[0], y: p[1], positive: p[2] == 1.0 }).collect(); + Ok(Some(model.segment_clicks(enc, &clicks).map_err(|e| e.to_string())?.logits)) + } + } +} + +fn serve_client(stream: &mut TcpStream, dir: &Path, device: candle_core::Device) -> Result<(), String> { + stream.set_read_timeout(Some(Duration::from_secs(10 * 60))).map_err(|e| e.to_string())?; + stream.set_write_timeout(Some(Duration::from_secs(30))).map_err(|e| e.to_string())?; + let mut model = None; + let mut cache = None; + loop { + let frame = read_frame(stream, CONTROL_LIMIT).map_err(|e| e.to_string())?; + let request: Request = serde_json::from_slice(&frame).map_err(|e| e.to_string())?; + request.validate()?; + let rgb = read_frame(stream, request.width * request.height * 3).map_err(|e| e.to_string())?; + if rgb.len() != request.width * request.height * 3 { + return Err("invalid RGB payload".into()); + } + let started = std::time::Instant::now(); + let result = (|| { + if model.is_none() { + model = Some(Sam3::load_on(dir, device.clone()).map_err(|e| e.to_string())?); + } + let m = model.as_mut().ok_or("SAM model unavailable")?; + let mut temporary = None; + if request.width > 0 { + let encoded = m.encode(&rgb, request.width, request.height).map_err(|e| e.to_string())?; + if request.temporary { + temporary = Some(encoded); + } else { + cache = Some(encoded); + } + } + let enc = if request.temporary { temporary.as_mut() } else { cache.as_mut() }.ok_or("SAM image cache empty")?; + predict(m, enc, request.operation) + })(); + eprintln!( + "SAM3 Metal request elapsed={:.3}s image={}x{} detail={} success={}", + started.elapsed().as_secs_f64(), + request.width, + request.height, + request.temporary, + result.is_ok() + ); + let response = serde_json::to_vec(&result).map_err(|e| e.to_string())?; + write_frame(stream, &response).map_err(|e| e.to_string())?; + if result.is_err() { + return Err("SAM inference failed; session released".into()); + } + } +} + +pub fn serve(address: SocketAddr, dir: &Path, device: candle_core::Device) -> Result<(), String> { + if !address.ip().is_loopback() || !device.is_metal() { + return Err("SAM worker requires a loopback address and a Metal device".into()); + } + if !crate::is_model_dir(dir) { + return Err("SAM model files are missing".into()); + } + let listener = TcpListener::bind(address).map_err(|e| e.to_string())?; + eprintln!("SAM3 worker listening on {address}; device=Metal; idle unload=600s"); + for stream in listener.incoming() { + let mut stream = stream.map_err(|e| e.to_string())?; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| serve_client(&mut stream, dir, device.clone()))); + match result { + Ok(Err(error)) => eprintln!("SAM3 session closed: {error}"), + Err(_) => eprintln!("SAM3 session panicked; model released"), + Ok(Ok(())) => {} + } + } + Ok(()) +} diff --git a/crates/segment/tests/remote.rs b/crates/segment/tests/remote.rs new file mode 100644 index 00000000..ee20ea20 --- /dev/null +++ b/crates/segment/tests/remote.rs @@ -0,0 +1,21 @@ +use lightcraft_segment::remote::{Operation, Request, read_frame}; +use std::io::Cursor; + +#[test] +fn rejects_unbounded_and_truncated_frames() { + assert!(read_frame(&mut Cursor::new(u32::MAX.to_be_bytes()), 1024).is_err()); + assert!(read_frame(&mut Cursor::new([0, 0, 0, 3, 1]), 1024).is_err()); +} + +#[test] +fn validates_images_and_prompt_boundaries() { + let request = |width, height, operation| Request { width, height, temporary: false, operation }; + assert!(request(1009, 1, Operation::Prepare).validate().is_err()); + assert!(request(0, 1, Operation::Prepare).validate().is_err()); + assert!(request(0, 0, Operation::Text("dog".into())).validate().is_ok()); + assert!(request(4, 3, Operation::Clicks(vec![[0.5, 0.5, 1.0]])).validate().is_ok()); + assert!(request(4, 3, Operation::Clicks(vec![[1.5, 0.5, 1.0]])).validate().is_err()); + assert!(request(4, 3, Operation::Clicks(vec![[0.5, f32::NAN, 1.0]])).validate().is_err()); + assert!(request(4, 3, Operation::Clicks(vec![[0.5, 0.5, 0.0]])).validate().is_err()); + assert!(request(4, 3, Operation::Text(" ".into())).validate().is_err()); +} diff --git a/crates/ui-egui/src/panels/masking.rs b/crates/ui-egui/src/panels/masking.rs index 2e150a60..12ddb031 100644 --- a/crates/ui-egui/src/panels/masking.rs +++ b/crates/ui-egui/src/panels/masking.rs @@ -639,7 +639,7 @@ fn component_row_menu(app: &mut LightcraftApp, ui: &mut egui::Ui, mask: u32, k: /// offered (with `then` to start afterwards) instead of starting an AI mask. fn needs_model(app: &mut LightcraftApp, kind: &str, op: &str) -> bool { let seg = &app.session.segmenter; - let missing = lightcraft_engine::segment::Segmenter::AVAILABLE && seg.dir.is_some() && !seg.installed(); + let missing = lightcraft_engine::segment::Segmenter::AVAILABLE && seg.remote_endpoint().is_none() && seg.dir.is_some() && !seg.installed(); if missing { app.offer_sam_download(Some((kind, op))); } diff --git a/docs/ai-masks.md b/docs/ai-masks.md index 5a32d690..00de5016 100644 --- a/docs/ai-masks.md +++ b/docs/ai-masks.md @@ -42,6 +42,39 @@ The result is stored with the mask: the model's 288 × 288 mask logits over the need the model, so masks survive moving the library to a machine without it, and the web build renders them. Editing the photo's look later does not recompute a mask (click again to update). +## Remote Metal inference + +Native builds with `sam` can keep the editor, RAW files, library, and exports local while +a Mac runs SAM 3. Build both ends from the same revision. On the Mac, install the model +as described below, then run: + +```sh +cargo build --release --locked -p lightcraft-segment --bin lightcraft-sam3-worker +target/release/lightcraft-sam3-worker "$HOME/Library/Application Support/LightCraft/models/sam3" 127.0.0.1:8793 +``` + +On the editing computer, start an SSH tunnel in one terminal and LightCraft in another: + +```sh +ssh -N -T -o ExitOnForwardFailure=yes -L 127.0.0.1:8793:127.0.0.1:8793 user@your-mac +LIGHTCRAFT_SAM3_REMOTE=127.0.0.1:8793 lightcraft +``` + +Object clicks, Describe prompts, and detail refinement use the remote worker. Only the +rendered model input (long edge at most 1008 pixels) and prompts cross the tunnel. The +worker caches the current photo; detail crops do not replace that cache. Returned masks +stay in the local library, so later adjustments and exports work without the connection. + +The worker requires Metal and accepts one editing connection at a time. It loads weights +on demand and releases them when the client disconnects or remains idle for ten minutes. +Both endpoints require loopback addresses: SSH provides authentication and encryption. +The listener has no independent authentication; do not expose it through a public proxy. + +No local weights are required in remote mode. An unavailable worker produces an explicit +error, not a silent CPU fallback. Restore the tunnel and retry. Unset +`LIGHTCRAFT_SAM3_REMOTE` to use the local model again. This offloads segmentation only, +not RAW development, denoising, or upscaling. The model licence below still applies. + ## Getting the model ### In the app (recommended) @@ -148,10 +181,12 @@ Therefore: | `mask.objectPoint` | `{x, y, exclude?: bool, id?}` | one click on the selected mask's Object selection (≤ 64 clicks) | | `mask.refineDetail` | `{id?, component?}` | the zoomed-in detail pass → `{started}` | | `segment.prepare` | `{}` | load the model and analyze the active photo → `{busy}` | -| `segment.model.status` | `{}` | `{available, installed, dir, loaded, busy, analyzing, sizeBytes, license, licenseUrl, mirrors, download}` | +| `segment.model.status` | `{}` | `{available, installed, remote, dir, loaded, busy, analyzing, sizeBytes, license, licenseUrl, mirrors, download}` | | `segment.model.download` | `{acknowledged: true}` | start the download in the background → `{started}` | | `segment.model.cancel` | `{}` | stop it (it resumes next time) → `{cancelled}` | +`remote` reports the configured endpoint, not its health. `installed` describes local weights only. + Coordinates are normalized to the uncropped, oriented photo, like every mask shape. `seg` passes a stored segmentation (as `mask.list` / the develop settings hold it): the mask is made without the model. Without the model, the AI kinds fail with an error starting "The SAM 3 model is not diff --git a/docs/whats-new.md b/docs/whats-new.md index ea0243f4..658ff4e0 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -135,6 +135,9 @@ lost everything at quit; a temporary session shows a banner the whole time and never writes to your library. ### Editing +- Optional remote SAM 3: keep the native editor local and run Object, Describe, and detail + inference on a Mac through SSH. Saved masks still render and export offline. + See [remote Metal inference](ai-masks.md#remote-metal-inference). - AI masks with SAM 3 (Object and Describe in the Masking panel): click an object to select it (⌥-click leaves a part out), or type what to select ("sky", "the red car", "car, road"); both combine with other masks, have an Edge setting, and get a sharper zoomed-in pass in the background. The model runs inside LightCraft in pure Rust From a46c0071a0320faebc253da402700f26fa51166d Mon Sep 17 00:00:00 2001 From: Nathaniel Ledford Date: Thu, 8 Oct 2026 19:57:27 -0400 Subject: [PATCH 02/46] Albums: pin that albums are created only inside folders Co-Authored-By: Claude Sonnet 5.5 --- crates/engine/src/tests.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 978e8067..9806a81e 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -118,6 +118,29 @@ fn albums_crud() { s.execute("album.delete", &json!({"id": f})).unwrap(); } +/// Given an album and a folder, when something is created inside each, then only the folder +/// accepts it: the sidebar only lists children of folders, so a child of a plain album would +/// be invisible. +#[test] +fn albums_and_smart_albums_are_created_only_inside_folders() { + let mut s = demo(); + let folder = s.execute("album.create", &json!({"name": "Trips", "folder": true})).unwrap()["id"].as_u64().unwrap(); + let album = s.execute("album.create", &json!({"name": "Best", "parent": folder})).unwrap()["id"].as_u64().unwrap(); + for (cmd, params) in [ + ("album.create", json!({"name": "Nested", "parent": album})), + ("album.create", json!({"name": "Nested", "folder": true, "parent": album})), + ("album.createSmart", json!({"name": "Nested", "parent": album})), + ("album.create", json!({"name": "Nested", "parent": 987_654})), + ] { + assert!(s.execute(cmd, ¶ms).is_err(), "{cmd} {params} must be refused"); + } + let n = s.catalog.albums().count(); + let sub = s.execute("album.create", &json!({"name": "Sub", "folder": true, "parent": folder})).unwrap()["id"].as_u64().unwrap(); + let smart = s.execute("album.createSmart", &json!({"name": "Rated", "parent": sub})).unwrap()["id"].as_u64().unwrap(); + assert_eq!(s.catalog.albums().count(), n + 2); + assert_eq!(s.catalog.album(lightcraft_catalog::AlbumId(smart)).unwrap().parent, Some(lightcraft_catalog::AlbumId(sub))); +} + #[test] fn filters_and_delete_restore() { let mut s = demo(); From e8b18ae636cbecf3d4d1d68a79a8fc56d1222b95 Mon Sep 17 00:00:00 2001 From: Nathaniel Ledford Date: Thu, 8 Oct 2026 20:01:56 -0400 Subject: [PATCH 03/46] Sidebar: one disclosure-triangle helper for the Local, By Date, Folders and Keywords trees Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/panels/left.rs | 74 ++++++++++--------------------- 1 file changed, 24 insertions(+), 50 deletions(-) diff --git a/crates/ui-egui/src/panels/left.rs b/crates/ui-egui/src/panels/left.rs index 868a71cd..304a4586 100644 --- a/crates/ui-egui/src/panels/left.rs +++ b/crates/ui-egui/src/panels/left.rs @@ -107,6 +107,25 @@ fn row_named( resp } +/// The disclosure triangle at the left of a tree row (`indent` is the row's): right-pointing +/// when closed, down when open, registered as widget `widget` for the control channel. Returns +/// its response: the caller flips the open state on `clicked()`. +fn disclosure_triangle(ui: &mut egui::Ui, row: &egui::Response, indent: f32, open: bool, id: egui::Id, widget: String) -> egui::Response { + let t = Tokens::get(ui.ctx()); + let c = pos2(row.rect.left() + 10.0 + indent, row.rect.center().y); + let tri = Rect::from_center_size(c, vec2(14.0, 14.0)); + let tr = ui.interact(tri, id, Sense::click()); + register(ui.ctx(), widget, tri); + let col = if tr.hovered() { t.text } else { t.text_dim }; + let pts = if open { + vec![c + vec2(-4.0, -2.0), c + vec2(4.0, -2.0), c + vec2(0.0, 3.0)] + } else { + vec![c + vec2(-2.0, -4.0), c + vec2(3.0, 0.0), c + vec2(-2.0, 4.0)] + }; + ui.painter().add(egui::Shape::convex_polygon(pts, col, egui::Stroke::NONE)); + tr +} + /// A collapsible section header (Albums, Local, By Date, Keywords): the bold title with a /// disclosure chevron after it; a click folds or unfolds the section (kept in the UI state, so it /// survives restarts). Returns the header's rect and whether the section is now open. @@ -516,7 +535,6 @@ fn folder_tree( reveal: bool, transient: bool, ) { - let t = Tokens::get(ui.ctx()); let open_id = egui::Id::new(("folder-open", path.to_string())); let sel = current.is_some_and(|c| same_folder(c, path)); let on_the_way = !sel && current.is_some_and(|c| lightcraft_catalog::query::folder_within(c, path)); @@ -534,17 +552,7 @@ fn folder_tree( } } let resp = row(app, ui, &format!("local:{path}"), Icon::Folder, name, None, sel, indent + 12.0).on_hover_text(path); - let c = pos2(resp.rect.left() + 10.0 + indent, resp.rect.center().y); - let tri = Rect::from_center_size(c, vec2(14.0, 14.0)); - let tr = ui.interact(tri, egui::Id::new(("folder-tri", path.to_string())), Sense::click()); - register(ui.ctx(), format!("folderToggle:{path}"), tri); - let col = if tr.hovered() { t.text } else { t.text_dim }; - let pts = if open { - vec![c + vec2(-4.0, -2.0), c + vec2(4.0, -2.0), c + vec2(0.0, 3.0)] - } else { - vec![c + vec2(-2.0, -4.0), c + vec2(3.0, 0.0), c + vec2(-2.0, 4.0)] - }; - ui.painter().add(egui::Shape::convex_polygon(pts, col, egui::Stroke::NONE)); + let tr = disclosure_triangle(ui, &resp, indent, open, egui::Id::new(("folder-tri", path.to_string())), format!("folderToggle:{path}")); if tr.clicked() { open = !open; ui.data_mut(|d| d.insert_temp(open_id, open)); @@ -607,23 +615,12 @@ fn folder_tree( /// One By Date row (`key`: `YYYY`, `YYYY-MM` or `YYYY-MM-DD`); returns whether it is open. fn date_row(app: &mut LightcraftApp, ui: &mut egui::Ui, key: &str, label: &str, count: usize, indent: f32) -> bool { - let t = Tokens::get(ui.ctx()); let open_id = egui::Id::new(("date-open", key.to_string())); let mut open: bool = ui.data(|d| d.get_temp(open_id)).unwrap_or(false); let sel = app.session.filter.date.as_deref() == Some(key); let resp = row(app, ui, &format!("date:{key}"), Icon::Clock, label, Some(count), sel, indent); if key.len() < 10 { - let c = pos2(resp.rect.left() + 10.0 + indent, resp.rect.center().y); - let tri = Rect::from_center_size(c, vec2(14.0, 14.0)); - let tr = ui.interact(tri, egui::Id::new(("date-tri", key.to_string())), Sense::click()); - register(ui.ctx(), format!("dateToggle:{key}"), tri); - let col = if tr.hovered() { t.text } else { t.text_dim }; - let pts = if open { - vec![c + vec2(-4.0, -2.0), c + vec2(4.0, -2.0), c + vec2(0.0, 3.0)] - } else { - vec![c + vec2(-2.0, -4.0), c + vec2(3.0, 0.0), c + vec2(-2.0, 4.0)] - }; - ui.painter().add(egui::Shape::convex_polygon(pts, col, egui::Stroke::NONE)); + let tr = disclosure_triangle(ui, &resp, indent, open, egui::Id::new(("date-tri", key.to_string())), format!("dateToggle:{key}")); if tr.clicked() { open = !open; ui.data_mut(|d| d.insert_temp(open_id, open)); @@ -838,7 +835,6 @@ fn reveal_chosen(app: &LightcraftApp, ui: &egui::Ui, tree: &[FolderNode]) { } fn folder_rows(app: &mut LightcraftApp, ui: &mut egui::Ui, nodes: &[FolderNode], indent: f32) { - let t = Tokens::get(ui.ctx()); for n in nodes { let key = lightcraft_catalog::query::folder_key(&n.path); let open_id = egui::Id::new(("libfolder-open", key.clone())); @@ -855,18 +851,7 @@ fn folder_rows(app: &mut LightcraftApp, ui: &mut egui::Ui, nodes: &[FolderNode], let resp = row_named(app, ui, &format!("libfolder:{}", n.path), Icon::Folder, &name, Some(&name), Some(n.count), sel, indent); let mut toggled = false; if !n.children.is_empty() { - // disclosure triangle left of the icon - let c = pos2(resp.rect.left() + 10.0 + indent, resp.rect.center().y); - let tri = Rect::from_center_size(c, vec2(14.0, 14.0)); - let tr = ui.interact(tri, egui::Id::new(("libfolder-tri", key)), Sense::click()); - register(ui.ctx(), format!("libraryFolderToggle:{}", n.path), tri); - let col = if tr.hovered() { t.text } else { t.text_dim }; - let pts = if open { - vec![c + vec2(-4.0, -2.0), c + vec2(4.0, -2.0), c + vec2(0.0, 3.0)] - } else { - vec![c + vec2(-2.0, -4.0), c + vec2(3.0, 0.0), c + vec2(-2.0, 4.0)] - }; - ui.painter().add(egui::Shape::convex_polygon(pts, col, egui::Stroke::NONE)); + let tr = disclosure_triangle(ui, &resp, indent, open, egui::Id::new(("libfolder-tri", key)), format!("libraryFolderToggle:{}", n.path)); toggled = tr.clicked(); tr.widget_info(|| { egui::WidgetInfo::labeled(egui::WidgetType::Button, true, if open { crate::i18n::tr("Collapse") } else { crate::i18n::tr("Expand") }) @@ -977,25 +962,14 @@ fn keywords_section(app: &mut LightcraftApp, ui: &mut egui::Ui) { } fn keyword_rows(app: &mut LightcraftApp, ui: &mut egui::Ui, nodes: &[KeywordNode], indent: f32) { - let t = Tokens::get(ui.ctx()); for n in nodes { let open_id = egui::Id::new(("kw-open", n.path.to_lowercase())); let mut open: bool = ui.data(|d| d.get_temp(open_id)).unwrap_or(false); let sel = app.session.filter.keyword.as_deref().is_some_and(|k| k.eq_ignore_ascii_case(&n.path)); let resp = row(app, ui, &format!("keyword:{}", n.path), Icon::Tag, &n.name, Some(n.count), sel, indent); if !n.children.is_empty() { - // disclosure triangle left of the icon - let c = pos2(resp.rect.left() + 10.0 + indent, resp.rect.center().y); - let tri = Rect::from_center_size(c, vec2(14.0, 14.0)); - let tr = ui.interact(tri, egui::Id::new(("kw-tri", n.path.to_lowercase())), Sense::click()); - register(ui.ctx(), format!("keywordToggle:{}", n.path), tri); - let col = if tr.hovered() { t.text } else { t.text_dim }; - let pts = if open { - vec![c + vec2(-4.0, -2.0), c + vec2(4.0, -2.0), c + vec2(0.0, 3.0)] - } else { - vec![c + vec2(-2.0, -4.0), c + vec2(3.0, 0.0), c + vec2(-2.0, 4.0)] - }; - ui.painter().add(egui::Shape::convex_polygon(pts, col, egui::Stroke::NONE)); + let tr = + disclosure_triangle(ui, &resp, indent, open, egui::Id::new(("kw-tri", n.path.to_lowercase())), format!("keywordToggle:{}", n.path)); if tr.clicked() { open = !open; ui.data_mut(|d| d.insert_temp(open_id, open)); From 96a9c6e04daea57ef16a2510ad20a1d034d6c6c5 Mon Sep 17 00:00:00 2001 From: Nathaniel Ledford Date: Thu, 8 Oct 2026 20:04:29 -0400 Subject: [PATCH 04/46] Albums: folders get the same disclosure triangle as the other sidebar trees Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/panels/left.rs | 4 +++- crates/ui-egui/src/tests_panels.rs | 36 ++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/crates/ui-egui/src/panels/left.rs b/crates/ui-egui/src/panels/left.rs index 304a4586..5771c9d4 100644 --- a/crates/ui-egui/src/panels/left.rs +++ b/crates/ui-egui/src/panels/left.rs @@ -651,7 +651,9 @@ fn albums_tree(app: &mut LightcraftApp, ui: &mut egui::Ui, all: &[Album], parent let open_id = egui::Id::new(("folder-open", a.id.0)); let open: bool = ui.data(|d| d.get_temp(open_id)).unwrap_or(true); let resp = row(app, ui, &format!("folder:{}", a.id.0), Icon::Folder, &a.name, None, false, indent); - if resp.clicked() { + // a folder is no source, so its row folds it too; the triangle is the same click, aimed + let tri = disclosure_triangle(ui, &resp, indent, open, egui::Id::new(("album-tri", a.id.0)), format!("albumToggle:{}", a.id.0)); + if resp.clicked() || tri.clicked() { ui.data_mut(|d| d.insert_temp(open_id, !open)); } folder_menu(app, &resp, a); diff --git a/crates/ui-egui/src/tests_panels.rs b/crates/ui-egui/src/tests_panels.rs index aad86fd5..d0cf302f 100644 --- a/crates/ui-egui/src/tests_panels.rs +++ b/crates/ui-egui/src/tests_panels.rs @@ -220,6 +220,42 @@ fn sidebar_sections_collapse_and_remember_it() { assert!(!h.app.ui.sidebar_section_collapsed("albums"), "the plus does not fold Albums"); } +/// Albums nest in folders like the other sidebar trees: a folder row has a disclosure triangle +/// (`albumToggle:`), plain albums have none, and folding a folder hides what is inside it. +#[test] +fn album_folders_have_a_disclosure_triangle() { + let mut h = demo([1400.0, 900.0], json!({"view": "photoGrid", "leftPanel": true})); + let make = |h: &mut Headless, params: serde_json::Value| h.app.session.execute("album.create", ¶ms).unwrap()["id"].as_u64().unwrap(); + let trips = make(&mut h, json!({"name": "Trips", "folder": true})); + let europe = make(&mut h, json!({"name": "Europe", "folder": true, "parent": trips})); + let best = make(&mut h, json!({"name": "Best", "parent": europe})); + let loose = make(&mut h, json!({"name": "Loose"})); + h.step(); + h.step(); + let has = |h: &Headless, id: &str| h.app.widgets.iter().any(|(w, _)| w == id); + let toggle = |h: &mut Headless, id: u64| { + let r = h.request("ui.clickWidget", json!({"id": format!("albumToggle:{id}")}), T); + assert_eq!(r["ok"], true, "{r}"); + h.step(); + h.step(); + }; + // folders start open and show their albums; only folders get a triangle + assert!(has(&h, &format!("source:album:{best}")), "open by default"); + assert!(has(&h, &format!("albumToggle:{trips}")) && has(&h, &format!("albumToggle:{europe}"))); + assert!(!has(&h, &format!("albumToggle:{best}")) && !has(&h, &format!("albumToggle:{loose}")), "albums have no triangle"); + // folding the outer folder hides everything inside it, the inner triangle included + toggle(&mut h, trips); + assert!(!has(&h, &format!("source:folder:{europe}")) && !has(&h, &format!("source:album:{best}"))); + assert!(has(&h, &format!("source:album:{loose}")), "siblings stay"); + // opening it again leaves the inner folder as it was + toggle(&mut h, trips); + assert!(has(&h, &format!("source:album:{best}"))); + toggle(&mut h, europe); + assert!(!has(&h, &format!("source:album:{best}")) && has(&h, &format!("source:folder:{europe}"))); + // the triangle is not the row: clicking it does not make the folder the source + assert_ne!(h.app.session.source, lightcraft_engine::LibrarySource::Album(lightcraft_catalog::AlbumId(europe))); +} + /// A headless app over a library of file-backed photos that exist only in the catalog. fn folders_app(paths: &[&str]) -> Headless { folders_app_sized(paths, [1400.0, 900.0]) From 79403a5680aeda5fd322811f54dcf409f0b8d37c Mon Sep 17 00:00:00 2001 From: Nathaniel Ledford Date: Thu, 8 Oct 2026 20:08:53 -0400 Subject: [PATCH 05/46] Albums: create albums, smart albums and folders inside a folder from its context menu The dialog.new* commands take the folder to create in, the dialogs remember it, and the Albums tree opens the folders down to the new row. Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/menus.rs | 15 +++++-- crates/ui-egui/src/panels/dialogs.rs | 23 +++++++--- crates/ui-egui/src/panels/left.rs | 47 +++++++++++++++++---- crates/ui-egui/src/state.rs | 12 ++++++ crates/ui-egui/src/tests_panels.rs | 63 ++++++++++++++++++++++++++++ 5 files changed, 143 insertions(+), 17 deletions(-) diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index 62461749..9829e057 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -230,6 +230,11 @@ fn adjust_brush(app: &mut LightcraftApp, k: f32, df: f32) -> Value { } /// An sRGB colour from `"#rrggbb"` or `[r, g, b]` (0..255). +/// The `parent` folder id of a `dialog.new*` command (none: the top level). +fn parent_param(p: &Value) -> Option { + p.get("parent").and_then(Value::as_u64) +} + pub fn parse_rgb(v: &Value) -> Option<[u8; 3]> { if let Some(s) = v.as_str() { let h = s.strip_prefix('#').unwrap_or(s); @@ -807,11 +812,13 @@ pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option { - app.ui.dialog = Some(Dialog::NewAlbum { name: p.get("name").and_then(Value::as_str).unwrap_or("").into(), folder: true }); + app.ui.dialog = + Some(Dialog::NewAlbum { name: p.get("name").and_then(Value::as_str).unwrap_or("").into(), folder: true, parent: parent_param(p) }); Ok(Value::Null) } "dialog.newAlbum" => { - app.ui.dialog = Some(Dialog::NewAlbum { name: p.get("name").and_then(Value::as_str).unwrap_or("").into(), folder: false }); + app.ui.dialog = + Some(Dialog::NewAlbum { name: p.get("name").and_then(Value::as_str).unwrap_or("").into(), folder: false, parent: parent_param(p) }); Ok(Value::Null) } "dialog.autoStack" => { @@ -838,17 +845,19 @@ pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option Dialog::SmartRules { id: None, name: p.get("name").and_then(Value::as_str).unwrap_or("").into(), rules: lightcraft_catalog::RuleSet { rules: vec![crate::panels::rules_editor::new_rule()], ..Default::default() }, + parent: parent_param(p), }, }); Ok(Value::Null) } "dialog.newSmartAlbum" => { - app.ui.dialog = Some(Dialog::NewSmartAlbum { name: p.get("name").and_then(Value::as_str).unwrap_or("").into() }); + app.ui.dialog = Some(Dialog::NewSmartAlbum { name: p.get("name").and_then(Value::as_str).unwrap_or("").into(), parent: parent_param(p) }); Ok(Value::Null) } "dialog.captureTime" => { diff --git a/crates/ui-egui/src/panels/dialogs.rs b/crates/ui-egui/src/panels/dialogs.rs index cd087776..4eff2f39 100644 --- a/crates/ui-egui/src/panels/dialogs.rs +++ b/crates/ui-egui/src/panels/dialogs.rs @@ -231,7 +231,7 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) { } }); } - Dialog::SmartRules { id, name, rules } => { + Dialog::SmartRules { id, name, rules, .. } => { let r = ui.add(egui::TextEdit::singleline(name).hint_text(crate::i18n::tr("Name")).desired_width(f32::INFINITY)); crate::widgets::register(ui.ctx(), "field:smartName", r.rect); ui.add_space(6.0); @@ -252,7 +252,7 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) { .color(t.text_dim), ); } - Dialog::NewSmartAlbum { name } => { + Dialog::NewSmartAlbum { name, .. } => { let r = ui.add(egui::TextEdit::singleline(name).hint_text(crate::i18n::tr("Name")).desired_width(f32::INFINITY)); r.request_focus(); if r.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)) { @@ -979,6 +979,13 @@ fn sam_model_body(app: &mut LightcraftApp, ui: &mut egui::Ui, error: Option<&str } } +/// Runs a command that creates an album and asks the Albums tree to open the folders down to it. +fn created_in(app: &mut LightcraftApp, command: &str, params: serde_json::Value) -> Result { + let r = app.run(command, params)?; + app.ui.reveal_album = r.get("id").and_then(serde_json::Value::as_u64); + Ok(r) +} + pub fn confirm_dialog(app: &mut LightcraftApp, dlg: &Dialog) -> Result { match dlg { Dialog::SamModel { then, .. } => { @@ -995,7 +1002,9 @@ pub fn confirm_dialog(app: &mut LightcraftApp, dlg: &Dialog) -> Result app.run("album.create", json!({"name": name, "folder": folder, "addSelected": !folder})), + Dialog::NewAlbum { name, folder, parent } => { + created_in(app, "album.create", json!({"name": name, "folder": folder, "addSelected": !folder, "parent": parent})) + } Dialog::RenameAlbum { id, name } => app.run("album.rename", json!({"id": id, "name": name})), Dialog::TextPrompt { value, command, params, key, .. } => { let mut p = params.clone(); @@ -1033,7 +1042,7 @@ pub fn confirm_dialog(app: &mut LightcraftApp, dlg: &Dialog) -> Result { + Dialog::SmartRules { id, name, rules, parent } => { let name = if name.trim().is_empty() { "Smart Album".to_string() } else { name.trim().to_string() }; match id { Some(id) => { @@ -1042,10 +1051,12 @@ pub fn confirm_dialog(app: &mut LightcraftApp, dlg: &Dialog) -> Result app.run("album.createSmart", json!({"name": name, "rules": {"ruleSet": rules}})), + None => created_in(app, "album.createSmart", json!({"name": name, "rules": {"ruleSet": rules}, "parent": parent})), } } - Dialog::NewSmartAlbum { name } => app.run("album.createSmart", json!({"name": if name.trim().is_empty() { "Smart Album" } else { name }})), + Dialog::NewSmartAlbum { name, parent } => { + created_in(app, "album.createSmart", json!({"name": if name.trim().is_empty() { "Smart Album" } else { name }, "parent": parent})) + } Dialog::CreatePreset { name, group, groups } => app.run( "preset.create", json!({ diff --git a/crates/ui-egui/src/panels/left.rs b/crates/ui-egui/src/panels/left.rs index 5771c9d4..100493e2 100644 --- a/crates/ui-egui/src/panels/left.rs +++ b/crates/ui-egui/src/panels/left.rs @@ -201,25 +201,34 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui) { let plus = icon_button(&mut hdr, "albumNew", Icon::Plus, vec2(26.0, 26.0), false, true, "Create Album"); egui::Popup::menu(&plus).show(|ui| { if ui.button(crate::i18n::tr("Create Album…")).clicked() { - app.ui.dialog = Some(crate::state::Dialog::NewAlbum { name: String::new(), folder: false }); + app.ui.dialog = Some(crate::state::Dialog::NewAlbum { name: String::new(), folder: false, parent: None }); } if ui.button(crate::i18n::tr("Create Smart Album…")).clicked() { app.ui.dialog = Some(crate::state::Dialog::SmartRules { id: None, name: String::new(), rules: lightcraft_catalog::RuleSet { rules: vec![crate::panels::rules_editor::new_rule()], ..Default::default() }, + parent: None, }); } if ui.button(crate::i18n::tr("Create Smart Album from Filter…")).clicked() { - app.ui.dialog = Some(crate::state::Dialog::NewSmartAlbum { name: String::new() }); + app.ui.dialog = Some(crate::state::Dialog::NewSmartAlbum { name: String::new(), parent: None }); } if ui.button(crate::i18n::tr("Create Folder…")).clicked() { - app.ui.dialog = Some(crate::state::Dialog::NewAlbum { name: String::new(), folder: true }); + app.ui.dialog = Some(crate::state::Dialog::NewAlbum { name: String::new(), folder: true, parent: None }); } }); if albums_open { let albums: Vec = app.session.catalog.albums().cloned().collect(); - albums_tree(app, ui, &albums, None, 0.0); + // the album just made: the folders down to it open, once + let reveal = app.ui.reveal_album.take(); + let mut open_to = Vec::new(); + let mut cur = reveal.map(AlbumId).and_then(|id| albums.iter().find(|a| a.id == id)).and_then(|a| a.parent); + while let Some(p) = cur.filter(|p| !open_to.contains(p) && open_to.len() < 64) { + open_to.push(p); + cur = albums.iter().find(|a| a.id == p).and_then(|a| a.parent); + } + albums_tree(app, ui, &albums, None, 0.0, &open_to); } ui.add_space(10.0); local_section(app, ui); @@ -643,13 +652,18 @@ fn browse_all_photos(app: &mut LightcraftApp, on: bool) { } } -fn albums_tree(app: &mut LightcraftApp, ui: &mut egui::Ui, all: &[Album], parent: Option, indent: f32) { +/// `open_to`: folders forced open this frame (the way down to an album that was just made). +fn albums_tree(app: &mut LightcraftApp, ui: &mut egui::Ui, all: &[Album], parent: Option, indent: f32, open_to: &[AlbumId]) { let mut kids: Vec<&Album> = all.iter().filter(|a| a.parent == parent).collect(); kids.sort_by_key(|a| (!a.folder, a.name.to_lowercase())); for a in kids { if a.folder { let open_id = egui::Id::new(("folder-open", a.id.0)); - let open: bool = ui.data(|d| d.get_temp(open_id)).unwrap_or(true); + let mut open: bool = ui.data(|d| d.get_temp(open_id)).unwrap_or(true); + if open_to.contains(&a.id) && !open { + open = true; + ui.data_mut(|d| d.insert_temp(open_id, true)); + } let resp = row(app, ui, &format!("folder:{}", a.id.0), Icon::Folder, &a.name, None, false, indent); // a folder is no source, so its row folds it too; the triangle is the same click, aimed let tri = disclosure_triangle(ui, &resp, indent, open, egui::Id::new(("album-tri", a.id.0)), format!("albumToggle:{}", a.id.0)); @@ -658,7 +672,7 @@ fn albums_tree(app: &mut LightcraftApp, ui: &mut egui::Ui, all: &[Album], parent } folder_menu(app, &resp, a); if open { - albums_tree(app, ui, all, Some(a.id), indent + 16.0); + albums_tree(app, ui, all, Some(a.id), indent + 16.0, open_to); } } else { let sel = app.session.source == LibrarySource::Album(a.id); @@ -707,6 +721,23 @@ fn drop_target(app: &mut LightcraftApp, ui: &mut egui::Ui, resp: &egui::Response fn folder_menu(app: &mut LightcraftApp, resp: &egui::Response, a: &Album) { resp.context_menu(|ui| { + if a.folder { + // the same commands as File ▸ New…, aimed at this folder + ui.menu_button(crate::i18n::tr("New"), |ui| { + for (label, command) in [ + ("Album…", "dialog.newAlbum"), + ("Smart Album…", "dialog.smartAlbum"), + ("Smart Album from Filter…", "dialog.newSmartAlbum"), + ("Folder…", "dialog.newFolder"), + ] { + if ui.button(crate::i18n::tr(label)).clicked() { + let _ = app.run(command, json!({"parent": a.id.0})); + ui.close(); + } + } + }); + ui.separator(); + } if !a.folder && !a.is_smart() && ui.button(crate::i18n::tr("Add Selected Photos")).clicked() { let _ = app.run("album.addPhotos", json!({"id": a.id.0})); } @@ -725,7 +756,7 @@ fn folder_menu(app: &mut LightcraftApp, resp: &egui::Response, a: &Album) { if a.is_smart() && ui.button(crate::i18n::tr("Edit Smart Album…")).clicked() { // older smart albums keep their filter fields; the editor works on the rule set let rules = a.smart.as_ref().and_then(|f| f.rule_set.clone()).unwrap_or_default(); - app.ui.dialog = Some(crate::state::Dialog::SmartRules { id: Some(a.id.0), name: a.name.clone(), rules }); + app.ui.dialog = Some(crate::state::Dialog::SmartRules { id: Some(a.id.0), name: a.name.clone(), rules, parent: None }); } if a.is_smart() && ui.button(crate::i18n::tr("Update Rules from Current Filter")).clicked() { let _ = app.run("album.setRules", json!({"id": a.id.0, "fromView": true})); diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index 17f5c58b..5f00f5a5 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -372,6 +372,9 @@ pub struct UiState { /// Photos being dragged from the grid (dropped on an album to add them). #[serde(skip)] pub dragging_photos: Option>, + /// An album just made: the Albums tree opens the folders down to it, once. + #[serde(skip)] + pub reveal_album: Option, /// Selected curve channel in the Curve flyout. pub curve_channel: String, /// Selected mixer mode: "hue" | "saturation" | "luminance" | "all". @@ -465,9 +468,12 @@ impl Dialog { #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] #[serde(tag = "kind", rename_all = "camelCase")] pub enum Dialog { + /// `parent`: the folder to create it in (none: the top level). NewAlbum { name: String, folder: bool, + #[serde(default)] + parent: Option, }, RenameAlbum { id: u64, @@ -525,6 +531,8 @@ pub enum Dialog { /// Save the current view (source + filter) as a smart album. NewSmartAlbum { name: String, + #[serde(default)] + parent: Option, }, /// Help ▸ What's New. WhatsNew, @@ -549,6 +557,9 @@ pub enum Dialog { id: Option, name: String, rules: lightcraft_catalog::RuleSet, + /// The folder a new smart album is created in (ignored when editing). + #[serde(default)] + parent: Option, }, /// `groups`: the settings groups the preset includes (`SettingsGroup` ids). CreatePreset { @@ -663,6 +674,7 @@ impl Default for UiState { renaming_component: None, quit: false, dragging_photos: None, + reveal_album: None, curve_channel: "parametric".into(), mixer_mode: "hue".into(), grading_mode: "3way".into(), diff --git a/crates/ui-egui/src/tests_panels.rs b/crates/ui-egui/src/tests_panels.rs index d0cf302f..edb71fde 100644 --- a/crates/ui-egui/src/tests_panels.rs +++ b/crates/ui-egui/src/tests_panels.rs @@ -256,6 +256,69 @@ fn album_folders_have_a_disclosure_triangle() { assert_ne!(h.app.session.source, lightcraft_engine::LibrarySource::Album(lightcraft_catalog::AlbumId(europe))); } +/// Creating from a folder's menu: every `dialog.new*` command takes the folder to create in, the +/// dialog remembers it, and the new row shows up even when that folder (and the one holding it) +/// was folded shut. +#[test] +fn albums_are_created_inside_the_chosen_folder() { + let mut h = demo([1400.0, 900.0], json!({"view": "photoGrid", "leftPanel": true})); + let make = |h: &mut Headless, params: serde_json::Value| h.app.session.execute("album.create", ¶ms).unwrap()["id"].as_u64().unwrap(); + let trips = make(&mut h, json!({"name": "Trips", "folder": true})); + let europe = make(&mut h, json!({"name": "Europe", "folder": true, "parent": trips})); + h.step(); + h.step(); + let has = |h: &Headless, id: &str| h.app.widgets.iter().any(|(w, _)| w == id); + let create = |h: &mut Headless, command: &str, name: &str, parent: Option| -> u64 { + let before: Vec = h.app.session.catalog.albums().map(|a| a.id.0).collect(); + let mut params = json!({"name": name}); + if let Some(p) = parent { + params["parent"] = json!(p); + } + let r = h.request("engine.execute", json!({"command": command, "params": params}), T); + assert_eq!(r["ok"], true, "{r}"); + let r = h.request("ui.dialog.confirm", json!({}), T); + assert_eq!(r["ok"], true, "{r}"); + h.step(); + h.step(); + let new: Vec = h.app.session.catalog.albums().map(|a| a.id.0).filter(|i| !before.contains(i)).collect(); + assert_eq!(new.len(), 1, "{command} made exactly one album: {new:?}"); + new[0] + }; + let parent_of = |h: &Headless, id: u64| h.app.session.catalog.album(lightcraft_catalog::AlbumId(id)).unwrap().parent.map(|p| p.0); + // fold both folders: the new rows must still be visible afterwards + for f in [europe, trips] { + let r = h.request("ui.clickWidget", json!({"id": format!("albumToggle:{f}")}), T); + assert_eq!(r["ok"], true, "{r}"); + h.step(); + h.step(); + } + assert!(!has(&h, &format!("source:folder:{europe}")), "Trips is folded"); + let album = create(&mut h, "dialog.newAlbum", "Day 1", Some(europe)); + assert_eq!(parent_of(&h, album), Some(europe)); + assert!(has(&h, &format!("source:album:{album}")), "the new album is shown, its folders opened to it"); + let folder = create(&mut h, "dialog.newFolder", "Italy", Some(europe)); + assert_eq!(parent_of(&h, folder), Some(europe)); + assert!(h.app.session.catalog.album(lightcraft_catalog::AlbumId(folder)).unwrap().folder); + let smart = create(&mut h, "dialog.smartAlbum", "Rated", Some(folder)); + assert_eq!(parent_of(&h, smart), Some(folder)); + assert!(h.app.session.catalog.album(lightcraft_catalog::AlbumId(smart)).unwrap().is_smart()); + let from_view = create(&mut h, "dialog.newSmartAlbum", "From view", Some(trips)); + assert_eq!(parent_of(&h, from_view), Some(trips)); + // without a folder they are still made at the top level + let top = create(&mut h, "dialog.newAlbum", "Loose", None); + assert_eq!(parent_of(&h, top), None); +} + +/// Saved or scripted dialogs from before the folder parameter still load. +#[test] +fn album_dialogs_without_a_parent_still_deserialize() { + use crate::state::Dialog; + let d: Dialog = serde_json::from_value(json!({"kind": "newAlbum", "name": "x", "folder": false})).unwrap(); + assert!(matches!(d, Dialog::NewAlbum { parent: None, .. })); + let d: Dialog = serde_json::from_value(json!({"kind": "newSmartAlbum", "name": "x"})).unwrap(); + assert!(matches!(d, Dialog::NewSmartAlbum { parent: None, .. })); +} + /// A headless app over a library of file-backed photos that exist only in the catalog. fn folders_app(paths: &[&str]) -> Headless { folders_app_sized(paths, [1400.0, 900.0]) From 4abf70e254d20b3cce8350710baab4248fadc54a Mon Sep 17 00:00:00 2001 From: Nathaniel Ledford Date: Thu, 8 Oct 2026 20:08:53 -0400 Subject: [PATCH 06/46] Parity: album folders fold like the other trees and create inside themselves Co-Authored-By: Claude Sonnet 5.5 --- docs/parity.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/parity.md b/docs/parity.md index 2eef5106..e6d908d8 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -151,7 +151,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins. | LR-LIB-RECENT-ADDED | Recently added | P1 | ✅ | `cmd:library.source` (`recentlyAdded`), `crates/engine/src/view.rs` (`RECENT_DAYS`) | photos imported in the 30 days up to the latest import, newest import first, grouped by import day | | LR-LIB-BYDATE | Browse by date | P1 | ✅ | `cmd:library.filter` (`date`), `crates/ui-egui/src/panels/left.rs` (`date_row`), `crates/catalog/src/query.rs` (`date_groups`) | year → month → day tree with library-wide counts; a row filters the current source by that date. Active filters show as removable chips under the grid header (`crates/ui-egui/src/panels/chips.rs`, with “+N more” and Clear all), the header counts “N of M photos” (matching of the source's total), a badge on the top-bar filter icon counts them, and an empty result says no photos match the active filters | | LR-LIB-ALBUM | Albums | P0 | ✅ | `cmd:album.create`, `cmd:album.rename`, `cmd:album.delete`, `cmd:album.addPhotos`, `cmd:album.removePhotos`, `cmd:dialog.newAlbum` | no drag photos onto album (LR-BEHAV-DRAGDROP), no album sort | -| LR-LIB-FOLDER | Folders of albums | P0 | ✅ | `cmd:album.create` (`folder`), `cmd:album.move` | moving is command-only (no drag, no "Move to" menu) | +| LR-LIB-FOLDER | Folders of albums | P0 | ✅ | `cmd:album.create` (`folder`, `parent`), `cmd:album.move`, `cmd:dialog.newAlbum` / `cmd:dialog.newFolder` / `cmd:dialog.smartAlbum` / `cmd:dialog.newSmartAlbum` (`parent`), `crates/ui-egui/src/panels/left.rs` (`albums_tree`, `folder_menu`, `disclosure_triangle`), `crates/ui-egui/src/tests_panels.rs` | folders nest and fold with the same disclosure triangle as Local / By Date / Folders / Keywords (`albumToggle:`; the row folds them too; open by default); right-click a folder ▸ New ▸ Album… / Smart Album… / Smart Album from Filter… / Folder… creates inside it (the `dialog.new*` commands take `parent`), opening the folders down to the new row; creating is only allowed inside folders; moving is by the Move to menu or `album.move` (no drag yet), no custom order (alphabetical, folders first) | | LR-LIB-SMARTALBUM | Smart albums | P1 | ✅ | `cmd:album.createSmart`, `cmd:album.setRules`, `crates/catalog/src/query.rs`, `crates/ui-egui/src/panels/filterbar.rs`, `crates/ui-egui/src/lib.rs` (`album_counts`) | saved filters (rating/flag/label/kind/edited/keyword/camera/lens/date range/text/album), live; match-all only (no any/none rule groups, no rule editor dialog — rules come from the filter bar or `album.setRules`); sidebar counts are cached per catalog revision (and per minute while an “in the last…” rule is used), counted without building id lists | | LR-LIB-SHARED-ALBUM | Shared albums | P2 | ⬜ | | needs a sharing service | | LR-LIB-OFFLINE | Keep album offline | P2 | 🚫 | | not applicable: local-first library | @@ -620,7 +620,7 @@ visible menu bar yet: the menu model is only exposed through the control channel | MENU-HELP-SYSINFO | System info | P2 | ✅ | `cmd:app.systemInfo`, `cmd:library.info` | Help ▸ System Info…: version, OS, CPU threads, GPU, memory budget, preview size, library, timings; Copy to Clipboard; JSON for agents (`open: false`) | | MENU-CTX-GRID | Photo context menu | P0 | ✅ | `crates/ui-egui/src/panels/grid.rs` (`context_menu`) | rate, flag, label, add to / remove from album, rename, virtual copy, version, stack, copy / paste / paste selected, reset, merge, rotate, show in Finder, export / export with preset, set as album cover (in an album), delete (restore / delete permanently in Recently Deleted) | | MENU-CTX-DETAIL | Loupe context menu | P1 | ✅ | `crates/ui-egui/src/panels/detail.rs` | Zoom submenu (fit, 100%, in, out), then the photo menu | -| MENU-CTX-ALBUM | Album / folder row menu | P0 | ✅ | `crates/ui-egui/src/panels/left.rs` (`folder_menu`), `cmd:album.move`, `cmd:dialog.export` | add selected, export album (dialog / preset), move to a folder or the top level, rename, delete; smart albums: update rules | +| MENU-CTX-ALBUM | Album / folder row menu | P0 | ✅ | `crates/ui-egui/src/panels/left.rs` (`folder_menu`), `cmd:album.move`, `cmd:dialog.export` | add selected, export album (dialog / preset), New ▸ Album / Smart Album / Smart Album from Filter / Folder (on a folder: made inside it), move to a folder or the top level, rename, delete; smart albums: update rules | | MENU-CTX-MASK | Mask / component menu | P0 | ✅ | `crates/ui-egui/src/panels/masking.rs` (`mask_menu`, `component_row_menu`), `cmd:mask.component` | mask rows: duplicate (and invert), invert, show/hide, move, rename, delete; component rows (right-click or "…"): invert, duplicate, mode (add / subtract / intersect), intersect with / subtract a new component, rename (also double-click), delete (the last one deletes the mask) | | MENU-CTX-PRESET | Preset menu | P1 | ✅ | `crates/ui-egui/src/panels/presets.rs` | favourite, update with current settings, rename, move to group, delete; group: export | | MENU-CTX-PROFILE | Profile favourites | P2 | ✅ | `cmd:profile.favorite`, `crates/ui-egui/src/panels/profiles.rs` | star in the profile browser; Favorites group first | From 5443ae28f9adf515048f18e10483160129ca5390 Mon Sep 17 00:00:00 2001 From: Nathaniel Ledford Date: Thu, 8 Oct 2026 20:14:20 -0400 Subject: [PATCH 07/46] Albums: address review of folder creation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Show a dialog's error and keep it open instead of closing it, open the folder menu from the triangle too, draw a triangle only on folders that hold something, reveal even when the section is folded, reuse the translated Create … labels, and fix a misplaced doc comment. Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/menus.rs | 2 +- crates/ui-egui/src/panels/dialogs.rs | 13 +++++++++- crates/ui-egui/src/panels/left.rs | 23 ++++++++++------ crates/ui-egui/src/tests_panels.rs | 39 ++++++++++++++++++++++++++-- 4 files changed, 65 insertions(+), 12 deletions(-) diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index 9829e057..17ed2ef6 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -229,12 +229,12 @@ fn adjust_brush(app: &mut LightcraftApp, k: f32, df: f32) -> Value { } } -/// An sRGB colour from `"#rrggbb"` or `[r, g, b]` (0..255). /// The `parent` folder id of a `dialog.new*` command (none: the top level). fn parent_param(p: &Value) -> Option { p.get("parent").and_then(Value::as_u64) } +/// An sRGB colour from `"#rrggbb"` or `[r, g, b]` (0..255). pub fn parse_rgb(v: &Value) -> Option<[u8; 3]> { if let Some(s) = v.as_str() { let h = s.strip_prefix('#').unwrap_or(s); diff --git a/crates/ui-egui/src/panels/dialogs.rs b/crates/ui-egui/src/panels/dialogs.rs index 4eff2f39..b1c168e6 100644 --- a/crates/ui-egui/src/panels/dialogs.rs +++ b/crates/ui-egui/src/panels/dialogs.rs @@ -847,7 +847,18 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) { match confirm_dialog(app, &dlg) { // the import review stays open on an error (e.g. an unusable folder template), and so // does Export (e.g. no folder chosen) so the choices aren't lost - Err(e) if matches!(dlg, Dialog::Import { .. } | Dialog::Export { .. }) => app.toast(ctx, e), + Err(e) + if matches!( + dlg, + Dialog::Import { .. } + | Dialog::Export { .. } + | Dialog::NewAlbum { .. } + | Dialog::NewSmartAlbum { .. } + | Dialog::SmartRules { .. } + ) => + { + app.toast(ctx, e) + } // the SAM 3 dialog stays open to show the download (or why it can't start) Err(e) if matches!(dlg, Dialog::SamModel { .. }) => { if let Dialog::SamModel { error, .. } = &mut dlg { diff --git a/crates/ui-egui/src/panels/left.rs b/crates/ui-egui/src/panels/left.rs index 100493e2..12db28b2 100644 --- a/crates/ui-egui/src/panels/left.rs +++ b/crates/ui-egui/src/panels/left.rs @@ -218,10 +218,10 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui) { app.ui.dialog = Some(crate::state::Dialog::NewAlbum { name: String::new(), folder: true, parent: None }); } }); + // the album just made: the folders down to it open, once (also when the section is shut) + let reveal = app.ui.reveal_album.take(); if albums_open { let albums: Vec = app.session.catalog.albums().cloned().collect(); - // the album just made: the folders down to it open, once - let reveal = app.ui.reveal_album.take(); let mut open_to = Vec::new(); let mut cur = reveal.map(AlbumId).and_then(|id| albums.iter().find(|a| a.id == id)).and_then(|a| a.parent); while let Some(p) = cur.filter(|p| !open_to.contains(p) && open_to.len() < 64) { @@ -666,8 +666,15 @@ fn albums_tree(app: &mut LightcraftApp, ui: &mut egui::Ui, all: &[Album], parent } let resp = row(app, ui, &format!("folder:{}", a.id.0), Icon::Folder, &a.name, None, false, indent); // a folder is no source, so its row folds it too; the triangle is the same click, aimed - let tri = disclosure_triangle(ui, &resp, indent, open, egui::Id::new(("album-tri", a.id.0)), format!("albumToggle:{}", a.id.0)); - if resp.clicked() || tri.clicked() { + let has_children = all.iter().any(|c| c.parent == Some(a.id)); + let mut toggled = resp.clicked(); + if has_children { + let tri = disclosure_triangle(ui, &resp, indent, open, egui::Id::new(("album-tri", a.id.0)), format!("albumToggle:{}", a.id.0)); + toggled |= tri.clicked(); + // the triangle sits on the row and takes its clicks: the menu opens from it too + folder_menu(app, &tri, a); + } + if toggled { ui.data_mut(|d| d.insert_temp(open_id, !open)); } folder_menu(app, &resp, a); @@ -725,10 +732,10 @@ fn folder_menu(app: &mut LightcraftApp, resp: &egui::Response, a: &Album) { // the same commands as File ▸ New…, aimed at this folder ui.menu_button(crate::i18n::tr("New"), |ui| { for (label, command) in [ - ("Album…", "dialog.newAlbum"), - ("Smart Album…", "dialog.smartAlbum"), - ("Smart Album from Filter…", "dialog.newSmartAlbum"), - ("Folder…", "dialog.newFolder"), + ("Create Album…", "dialog.newAlbum"), + ("Create Smart Album…", "dialog.smartAlbum"), + ("Create Smart Album from Filter…", "dialog.newSmartAlbum"), + ("Create Folder…", "dialog.newFolder"), ] { if ui.button(crate::i18n::tr(label)).clicked() { let _ = app.run(command, json!({"parent": a.id.0})); diff --git a/crates/ui-egui/src/tests_panels.rs b/crates/ui-egui/src/tests_panels.rs index edb71fde..5f81ee36 100644 --- a/crates/ui-egui/src/tests_panels.rs +++ b/crates/ui-egui/src/tests_panels.rs @@ -252,8 +252,37 @@ fn album_folders_have_a_disclosure_triangle() { assert!(has(&h, &format!("source:album:{best}"))); toggle(&mut h, europe); assert!(!has(&h, &format!("source:album:{best}")) && has(&h, &format!("source:folder:{europe}"))); - // the triangle is not the row: clicking it does not make the folder the source - assert_ne!(h.app.session.source, lightcraft_engine::LibrarySource::Album(lightcraft_catalog::AlbumId(europe))); + // an empty folder has nothing to open: no triangle until it holds something + let empty = make(&mut h, json!({"name": "Empty", "folder": true})); + h.step(); + h.step(); + assert!(has(&h, &format!("source:folder:{empty}")) && !has(&h, &format!("albumToggle:{empty}"))); + make(&mut h, json!({"name": "First", "parent": empty})); + h.step(); + h.step(); + assert!(has(&h, &format!("albumToggle:{empty}")), "it gets one with its first child"); +} + +/// A dialog that cannot do its job (here: the chosen parent is a plain album, as when the folder +/// was deleted or turned out not to be one) stays open with the error shown, the typed name kept. +#[test] +fn creating_in_a_non_folder_keeps_the_dialog_open() { + let mut h = demo([1400.0, 900.0], json!({"view": "photoGrid", "leftPanel": true})); + let plain = h.app.session.execute("album.create", &json!({"name": "Plain"})).unwrap()["id"].as_u64().unwrap(); + let before = h.app.session.catalog.albums().count(); + let r = h.request("engine.execute", json!({"command": "dialog.newAlbum", "params": {"name": "Kept", "parent": plain}}), T); + assert_eq!(r["ok"], true, "{r}"); + h.step(); + let r = h.request("ui.clickWidget", json!({"id": "button:dialogOk"}), T); + assert_eq!(r["ok"], true, "{r}"); + h.step(); + h.step(); + assert_eq!(h.app.session.catalog.albums().count(), before, "nothing was made"); + assert!( + matches!(&h.app.ui.dialog, Some(crate::state::Dialog::NewAlbum { name, .. }) if name == "Kept"), + "the dialog stays, with its text: {:?}", + h.app.ui.dialog + ); } /// Creating from a folder's menu: every `dialog.new*` command takes the folder to create in, the @@ -265,6 +294,7 @@ fn albums_are_created_inside_the_chosen_folder() { let make = |h: &mut Headless, params: serde_json::Value| h.app.session.execute("album.create", ¶ms).unwrap()["id"].as_u64().unwrap(); let trips = make(&mut h, json!({"name": "Trips", "folder": true})); let europe = make(&mut h, json!({"name": "Europe", "folder": true, "parent": trips})); + make(&mut h, json!({"name": "Seed", "parent": europe})); h.step(); h.step(); let has = |h: &Headless, id: &str| h.app.widgets.iter().any(|(w, _)| w == id); @@ -317,6 +347,11 @@ fn album_dialogs_without_a_parent_still_deserialize() { assert!(matches!(d, Dialog::NewAlbum { parent: None, .. })); let d: Dialog = serde_json::from_value(json!({"kind": "newSmartAlbum", "name": "x"})).unwrap(); assert!(matches!(d, Dialog::NewSmartAlbum { parent: None, .. })); + let mut v = serde_json::to_value(Dialog::SmartRules { id: None, name: "x".into(), rules: Default::default(), parent: Some(3) }).unwrap(); + assert_eq!(v["parent"], 3); + v.as_object_mut().unwrap().remove("parent"); + let d: Dialog = serde_json::from_value(v).unwrap(); + assert!(matches!(d, Dialog::SmartRules { parent: None, .. })); } /// A headless app over a library of file-backed photos that exist only in the catalog. From aaae39fae0b6c2dbf8416278778592d8a672a8b5 Mon Sep 17 00:00:00 2001 From: Nathaniel Ledford Date: Thu, 8 Oct 2026 20:20:38 -0400 Subject: [PATCH 08/46] Albums: drag an album or folder onto a folder to move it A folder under the pointer is outlined when the drop is allowed and opens after resting there; a drop zone under the tree takes an album back to the top level. The move is album.move, so it is one undo step. Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/lib.rs | 1 + crates/ui-egui/src/panels/left.rs | 133 +++++++++++++++++++++++++++- crates/ui-egui/src/state.rs | 4 + crates/ui-egui/src/tests_panels.rs | 137 +++++++++++++++++++++++++++++ 4 files changed, 272 insertions(+), 3 deletions(-) diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index fd2c0611..14250ab3 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -909,6 +909,7 @@ impl LightcraftApp { export_task::poll(self, &ctx); pick::poll(self, &ctx); panels::grid::drag_feedback(self, &ctx); + panels::left::album_drag_feedback(self, &ctx); panels::toast(self, &ctx); self.widgets = widgets::take_registry(&ctx); self.end_frame(t0); diff --git a/crates/ui-egui/src/panels/left.rs b/crates/ui-egui/src/panels/left.rs index 12db28b2..e851473f 100644 --- a/crates/ui-egui/src/panels/left.rs +++ b/crates/ui-egui/src/panels/left.rs @@ -52,10 +52,27 @@ fn row_named( count: Option, selected: bool, indent: f32, +) -> egui::Response { + row_sensed(app, ui, id, icon, label, spoken, count, selected, indent, Sense::click()) +} + +/// [`row_named`] with the given `sense` (rows that can be dragged add `drag`). +#[allow(clippy::too_many_arguments)] +fn row_sensed( + app: &mut LightcraftApp, + ui: &mut egui::Ui, + id: &str, + icon: Icon, + label: &str, + spoken: Option<&str>, + count: Option, + selected: bool, + indent: f32, + sense: Sense, ) -> egui::Response { let label = if matches!(id, "all" | "recentlyAdded" | "picks" | "missing" | "recentlyDeleted") { crate::i18n::tr(label) } else { label }; let t = Tokens::get(ui.ctx()); - let (r, resp) = ui.allocate_exact_size(vec2(ui.available_width(), 29.0), Sense::click()); + let (r, resp) = ui.allocate_exact_size(vec2(ui.available_width(), 29.0), sense); register(ui.ctx(), format!("source:{id}"), r); let said = spoken.unwrap_or(label); let name = match count { @@ -229,6 +246,7 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui) { cur = albums.iter().find(|a| a.id == p).and_then(|a| a.parent); } albums_tree(app, ui, &albums, None, 0.0, &open_to); + top_level_drop_zone(app, ui); } ui.add_space(10.0); local_section(app, ui); @@ -664,9 +682,15 @@ fn albums_tree(app: &mut LightcraftApp, ui: &mut egui::Ui, all: &[Album], parent open = true; ui.data_mut(|d| d.insert_temp(open_id, true)); } - let resp = row(app, ui, &format!("folder:{}", a.id.0), Icon::Folder, &a.name, None, false, indent); + let resp = row_sensed(app, ui, &format!("folder:{}", a.id.0), Icon::Folder, &a.name, None, None, false, indent, Sense::click_and_drag()); // a folder is no source, so its row folds it too; the triangle is the same click, aimed let has_children = all.iter().any(|c| c.parent == Some(a.id)); + if resp.drag_started() { + app.ui.dragging_album = Some(a.id.0); + } + if album_drag_over(app, ui, &resp, a, has_children.then_some(&mut open)) { + ui.data_mut(|d| d.insert_temp(open_id, true)); + } let mut toggled = resp.clicked(); if has_children { let tri = disclosure_triangle(ui, &resp, indent, open, egui::Id::new(("album-tri", a.id.0)), format!("albumToggle:{}", a.id.0)); @@ -691,7 +715,10 @@ fn albums_tree(app: &mut LightcraftApp, ui: &mut egui::Ui, all: &[Album], parent let target = app.session.target_album.filter(|t| app.session.catalog.album(*t).is_some()).or_else(|| app.session.catalog.quick_collection()); let label = if target == Some(a.id) { format!("{} +", a.name) } else { a.name.clone() }; - let mut resp = row(app, ui, &format!("album:{}", a.id.0), icon, &label, Some(n), sel, indent); + let mut resp = row_sensed(app, ui, &format!("album:{}", a.id.0), icon, &label, None, Some(n), sel, indent, Sense::click_and_drag()); + if resp.drag_started() { + app.ui.dragging_album = Some(a.id.0); + } if !a.is_smart() { drop_target(app, ui, &resp, a); } @@ -706,6 +733,106 @@ fn albums_tree(app: &mut LightcraftApp, ui: &mut egui::Ui, all: &[Album], parent } } +/// How long a dragged album must rest on a closed folder before it opens. +const HOVER_OPEN_SECS: f64 = 0.6; + +/// Whether the album `dragged` may be dropped into the folder `target` (`None`: the top level): +/// a folder other than where it already is, and not itself or something inside it. +fn can_drop_album(app: &LightcraftApp, dragged: AlbumId, target: Option) -> bool { + let Some(d) = app.session.catalog.album(dragged) else { return false }; + match target { + None => d.parent.is_some(), + Some(t) => app.session.catalog.album(t).is_some_and(|f| f.folder) && d.parent != Some(t) && !is_within(app, t, dragged), + } +} + +/// Moves the dragged album (`album.move`, one undo step) and ends the drag; the folders down to it +/// open so it shows in its new place. +fn drop_album(app: &mut LightcraftApp, ui: &egui::Ui, dragged: AlbumId, target: Option) { + match app.run("album.move", json!({"id": dragged.0, "parent": target.map(|t| t.0)})) { + Ok(_) => { + app.ui.reveal_album = Some(dragged.0); + ui.ctx().request_repaint(); + } + Err(e) => app.toast(ui.ctx(), e), + } + app.ui.dragging_album = None; +} + +/// A folder row while an album is dragged: outlined under the pointer when the drop is allowed, +/// opened (`open`, when it has something inside) once the pointer rests on it, and the album moved +/// in on release. Returns whether it opened the folder. +fn album_drag_over(app: &mut LightcraftApp, ui: &mut egui::Ui, resp: &egui::Response, folder: &Album, open: Option<&mut bool>) -> bool { + let hover_id = egui::Id::new(("album-hover", folder.id.0)); + let Some(dragged) = app.ui.dragging_album.map(AlbumId) else { + ui.data_mut(|d| d.remove_temp::(hover_id)); + return false; + }; + if !ui.input(|i| i.pointer.latest_pos()).is_some_and(|p| resp.rect.contains(p)) { + ui.data_mut(|d| d.remove_temp::(hover_id)); + return false; + } + let now = ui.input(|i| i.time); + let since = ui.data_mut(|d| *d.get_temp_mut_or_insert_with(hover_id, || now)); + let mut opened = false; + if let Some(open) = open + && !*open + { + if now - since >= HOVER_OPEN_SECS { + *open = true; + opened = true; + } else { + ui.ctx().request_repaint_after(std::time::Duration::from_millis(100)); + } + } + if can_drop_album(app, dragged, Some(folder.id)) { + let t = Tokens::get(ui.ctx()); + ui.painter().rect_stroke(resp.rect.shrink2(vec2(8.0, 1.0)), 4.0, egui::Stroke::new(1.5, t.accent), egui::StrokeKind::Inside); + if ui.input(|i| i.pointer.any_released()) { + drop_album(app, ui, dragged, Some(folder.id)); + } + } + opened +} + +/// While an album that sits in a folder is dragged: a row under the tree that takes it back to the +/// top level. +fn top_level_drop_zone(app: &mut LightcraftApp, ui: &mut egui::Ui) { + let Some(dragged) = app.ui.dragging_album.map(AlbumId) else { return }; + if !can_drop_album(app, dragged, None) { + return; + } + let t = Tokens::get(ui.ctx()); + let (r, _) = ui.allocate_exact_size(vec2(ui.available_width(), 29.0), Sense::hover()); + register(ui.ctx(), "albumDrop:top", r); + let over = ui.input(|i| i.pointer.latest_pos()).is_some_and(|p| r.contains(p)); + let stroke = if over { egui::Stroke::new(1.5, t.accent) } else { egui::Stroke::new(1.0, t.text_dim.gamma_multiply(0.5)) }; + ui.painter().rect_stroke(r.shrink2(vec2(8.0, 1.0)), 4.0, stroke, egui::StrokeKind::Inside); + ui.painter().text(pos2(r.left() + 18.0, r.center().y), Align2::LEFT_CENTER, crate::i18n::tr("Top Level"), t.font(13.5), t.text_dim); + if over && ui.input(|i| i.pointer.any_released()) { + drop_album(app, ui, dragged, None); + } +} + +/// While an album is dragged in the sidebar: its name at the pointer; the drag ends when the +/// button is up (the drop targets act on the release frame, before this runs). +pub fn album_drag_feedback(app: &mut LightcraftApp, ctx: &egui::Context) { + let Some(id) = app.ui.dragging_album else { return }; + let (released, down, pos) = ctx.input(|i| (i.pointer.any_released(), i.pointer.any_down(), i.pointer.latest_pos())); + if released || !down { + app.ui.dragging_album = None; + return; + } + let (Some(pos), Some(name)) = (pos, app.session.catalog.album(AlbumId(id)).map(|a| a.name.clone())) else { return }; + let t = Tokens::get(ctx); + ctx.set_cursor_icon(egui::CursorIcon::Grabbing); + egui::Area::new(egui::Id::new("drag-album")).order(egui::Order::Tooltip).interactable(false).fixed_pos(pos + vec2(14.0, 10.0)).show(ctx, |ui| { + egui::Frame::NONE.fill(t.accent).corner_radius(10.0).inner_margin(egui::Margin::symmetric(9, 3)).show(ui, |ui| { + ui.label(egui::RichText::new(name).color(egui::Color32::WHITE).font(t.semibold(12.0))); + }); + }); +} + /// An album row while photos are dragged from the grid: highlighted under the pointer; a /// release there adds them. fn drop_target(app: &mut LightcraftApp, ui: &mut egui::Ui, resp: &egui::Response, a: &Album) { diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index 5f00f5a5..bbf936dd 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -375,6 +375,9 @@ pub struct UiState { /// An album just made: the Albums tree opens the folders down to it, once. #[serde(skip)] pub reveal_album: Option, + /// An album or folder row being dragged in the sidebar (dropped on a folder to move it there). + #[serde(skip)] + pub dragging_album: Option, /// Selected curve channel in the Curve flyout. pub curve_channel: String, /// Selected mixer mode: "hue" | "saturation" | "luminance" | "all". @@ -675,6 +678,7 @@ impl Default for UiState { quit: false, dragging_photos: None, reveal_album: None, + dragging_album: None, curve_channel: "parametric".into(), mixer_mode: "hue".into(), grading_mode: "3way".into(), diff --git a/crates/ui-egui/src/tests_panels.rs b/crates/ui-egui/src/tests_panels.rs index 5f81ee36..ba67867c 100644 --- a/crates/ui-egui/src/tests_panels.rs +++ b/crates/ui-egui/src/tests_panels.rs @@ -354,6 +354,143 @@ fn album_dialogs_without_a_parent_still_deserialize() { assert!(matches!(d, Dialog::SmartRules { parent: None, .. })); } +/// An album tree to drag in: `Archive` (empty folder), `Trips` ⊃ `Sub` (folder) + `Best`, `Loose`. +struct AlbumTree { + h: Headless, + archive: u64, + trips: u64, + sub: u64, + best: u64, + loose: u64, +} + +fn album_tree() -> AlbumTree { + let mut h = demo([1400.0, 900.0], json!({"view": "photoGrid", "leftPanel": true})); + let make = |h: &mut Headless, params: serde_json::Value| h.app.session.execute("album.create", ¶ms).unwrap()["id"].as_u64().unwrap(); + let archive = make(&mut h, json!({"name": "Archive", "folder": true})); + let trips = make(&mut h, json!({"name": "Trips", "folder": true})); + let sub = make(&mut h, json!({"name": "Sub", "folder": true, "parent": trips})); + let best = make(&mut h, json!({"name": "Best", "parent": trips})); + let loose = make(&mut h, json!({"name": "Loose"})); + h.step(); + h.step(); + AlbumTree { h, archive, trips, sub, best, loose } +} + +impl AlbumTree { + fn row(&self, id: u64) -> egui::Rect { + let h = &self.h; + let name = ["source:folder:", "source:album:"].iter().map(|p| format!("{p}{id}")).find(|n| h.app.widgets.iter().any(|(w, _)| w == n)); + widget(h, &name.unwrap_or_else(|| panic!("no row for album {id}"))) + } + /// The middle of the row just under the last row of the Albums tree (where the top-level drop zone is). + fn below_tree(&self) -> egui::Pos2 { + let bottom = (self.h.app.widgets.iter()) + .filter(|(w, r)| (w.starts_with("source:album:") || w.starts_with("source:folder:")) && r.left() < 300.0) + .map(|(_, r)| r.bottom()) + .fold(0.0, f32::max); + egui::pos2(100.0, bottom + 14.5) + } + fn parent(&self, id: u64) -> Option { + self.h.app.session.catalog.album(lightcraft_catalog::AlbumId(id)).unwrap().parent.map(|p| p.0) + } + /// Press at `from`, move through each `(point, frames held there)`, release at the last one. + fn drag(&mut self, from: egui::Pos2, path: &[(egui::Pos2, usize)]) { + let mut ev = |e| self.h.app.synthetic.push(e); + ev(egui::Event::PointerMoved(from)); + ev(egui::Event::PointerButton { pos: from, button: egui::PointerButton::Primary, pressed: true, modifiers: Default::default() }); + let mut at = from; + for (to, hold) in path { + for i in 1..=8 { + at = from + (*to - from) * (i as f32 / 8.0); + ev(egui::Event::PointerMoved(at)); + } + for _ in 0..*hold { + ev(egui::Event::PointerMoved(*to)); + } + at = *to; + } + ev(egui::Event::PointerButton { pos: at, button: egui::PointerButton::Primary, pressed: false, modifiers: Default::default() }); + for _ in 0..400 { + if self.h.app.synthetic.is_empty() { + break; + } + self.h.step(); + } + self.h.step(); + self.h.step(); + } + fn drag_row(&mut self, id: u64, onto: u64) { + let (from, to) = (self.row(id).center(), self.row(onto).center()); + self.drag(from, &[(to, 3)]); + } +} + +/// Given albums and folders in the sidebar, when an album or folder is dragged onto a folder, +/// then it moves into it (as Move to does); onto a plain album, itself or something inside it, it +/// stays where it was. +#[test] +fn dragging_an_album_onto_a_folder_moves_it() { + let mut t = album_tree(); + t.drag_row(t.loose, t.archive); + assert_eq!(t.parent(t.loose), Some(t.archive), "an album into a folder"); + assert!(t.h.app.ui.dragging_album.is_none(), "the drag ended"); + // it is shown inside its new folder, which opened for it + assert!(t.h.app.widgets.iter().any(|(w, _)| *w == format!("source:album:{}", t.loose))); + t.drag_row(t.sub, t.archive); + assert_eq!(t.parent(t.sub), Some(t.archive), "a folder into a folder"); + // refused: onto a plain album, onto itself, into its own subfolder + t.drag_row(t.best, t.loose); + assert_eq!(t.parent(t.best), Some(t.trips), "a plain album is no destination"); + let (from, aside) = (t.row(t.archive).center(), t.row(t.archive).center() + egui::vec2(40.0, 0.0)); + t.drag(from, &[(aside, 3)]); + assert_eq!(t.parent(t.archive), None, "not onto itself"); + t.drag_row(t.archive, t.sub); + assert_eq!(t.parent(t.archive), None, "not into something inside it"); + assert!(t.h.app.ui.dragging_album.is_none()); + // a plain click still only opens the album + let c = t.row(t.best).center(); + t.drag(c, &[(c, 0)]); + assert_eq!(t.h.app.session.source, lightcraft_engine::LibrarySource::Album(lightcraft_catalog::AlbumId(t.best))); +} + +/// Dragging out of a folder: the drop zone under the tree puts the album back at the top level; +/// it only shows when there is somewhere to go, and the move is one undo step. +#[test] +fn dragging_an_album_to_the_top_level() { + let mut t = album_tree(); + let has = |t: &AlbumTree, id: &str| t.h.app.widgets.iter().any(|(w, _)| w == id); + assert!(!has(&t, "albumDrop:top"), "no zone unless an album is being dragged"); + // below the last row of the tree, where the zone appears once an album with a folder is dragged + let below = t.below_tree(); + let from = t.row(t.best).center(); + t.drag(from, &[(below, 3)]); + assert_eq!(t.parent(t.best), None, "moved out of Trips"); + // a top-level album has nowhere to go: dropping it there does nothing + let (from, below) = (t.row(t.best).center(), t.below_tree()); + t.drag(from, &[(below, 3)]); + assert_eq!(t.parent(t.best), None); + t.h.app.session.execute("edit.undo", &json!({})).unwrap(); + assert_eq!(t.parent(t.best), Some(t.trips), "one undo puts it back"); +} + +/// Holding a dragged album over a closed folder opens it, so the drop can go into a subfolder. +#[test] +fn a_dragged_album_opens_the_folder_it_hovers_over() { + let mut t = album_tree(); + let r = t.h.request("ui.clickWidget", json!({"id": format!("albumToggle:{}", t.trips)}), T); + assert_eq!(r["ok"], true, "{r}"); + t.h.step(); + t.h.step(); + assert!(!t.h.app.widgets.iter().any(|(w, _)| *w == format!("source:folder:{}", t.sub)), "Trips is folded"); + // where Sub's row will be once Trips is open: one row below it + let trips = t.row(t.trips); + let inside = trips.center() + egui::vec2(0.0, trips.height()); + let from = t.row(t.loose).center(); + t.drag(from, &[(trips.center(), 90), (inside, 3)]); + assert_eq!(t.parent(t.loose), Some(t.sub), "dropped on Sub, which was only there after the hover"); +} + /// A headless app over a library of file-backed photos that exist only in the catalog. fn folders_app(paths: &[&str]) -> Headless { folders_app_sized(paths, [1400.0, 900.0]) From 63335e2b0acc5aaaccd8bb489bcee7e7dfe186d5 Mon Sep 17 00:00:00 2001 From: Nathaniel Ledford Date: Thu, 8 Oct 2026 20:20:43 -0400 Subject: [PATCH 09/46] Parity: albums and folders move by dragging onto a folder Co-Authored-By: Claude Sonnet 5.5 --- docs/parity.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/parity.md b/docs/parity.md index e6d908d8..ca6d974e 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -151,7 +151,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins. | LR-LIB-RECENT-ADDED | Recently added | P1 | ✅ | `cmd:library.source` (`recentlyAdded`), `crates/engine/src/view.rs` (`RECENT_DAYS`) | photos imported in the 30 days up to the latest import, newest import first, grouped by import day | | LR-LIB-BYDATE | Browse by date | P1 | ✅ | `cmd:library.filter` (`date`), `crates/ui-egui/src/panels/left.rs` (`date_row`), `crates/catalog/src/query.rs` (`date_groups`) | year → month → day tree with library-wide counts; a row filters the current source by that date. Active filters show as removable chips under the grid header (`crates/ui-egui/src/panels/chips.rs`, with “+N more” and Clear all), the header counts “N of M photos” (matching of the source's total), a badge on the top-bar filter icon counts them, and an empty result says no photos match the active filters | | LR-LIB-ALBUM | Albums | P0 | ✅ | `cmd:album.create`, `cmd:album.rename`, `cmd:album.delete`, `cmd:album.addPhotos`, `cmd:album.removePhotos`, `cmd:dialog.newAlbum` | no drag photos onto album (LR-BEHAV-DRAGDROP), no album sort | -| LR-LIB-FOLDER | Folders of albums | P0 | ✅ | `cmd:album.create` (`folder`, `parent`), `cmd:album.move`, `cmd:dialog.newAlbum` / `cmd:dialog.newFolder` / `cmd:dialog.smartAlbum` / `cmd:dialog.newSmartAlbum` (`parent`), `crates/ui-egui/src/panels/left.rs` (`albums_tree`, `folder_menu`, `disclosure_triangle`), `crates/ui-egui/src/tests_panels.rs` | folders nest and fold with the same disclosure triangle as Local / By Date / Folders / Keywords (`albumToggle:`; the row folds them too; open by default); right-click a folder ▸ New ▸ Album… / Smart Album… / Smart Album from Filter… / Folder… creates inside it (the `dialog.new*` commands take `parent`), opening the folders down to the new row; creating is only allowed inside folders; moving is by the Move to menu or `album.move` (no drag yet), no custom order (alphabetical, folders first) | +| LR-LIB-FOLDER | Folders of albums | P0 | ✅ | `cmd:album.create` (`folder`, `parent`), `cmd:album.move`, `cmd:dialog.newAlbum` / `cmd:dialog.newFolder` / `cmd:dialog.smartAlbum` / `cmd:dialog.newSmartAlbum` (`parent`), `crates/ui-egui/src/panels/left.rs` (`albums_tree`, `folder_menu`, `disclosure_triangle`), `crates/ui-egui/src/tests_panels.rs` | folders nest and fold with the same disclosure triangle as Local / By Date / Folders / Keywords (`albumToggle:`; the row folds them too; open by default); right-click a folder ▸ New ▸ Album… / Smart Album… / Smart Album from Filter… / Folder… creates inside it (the `dialog.new*` commands take `parent`), opening the folders down to the new row; creating is only allowed inside folders; moving: drag an album or folder row onto a folder (outlined when allowed; a closed folder opens after 0.6 s under the pointer; a "Top Level" zone under the tree appears while dragging something that sits in a folder), the Move to menu or `album.move` — one undo step each; not onto a plain album, itself or something inside it; no custom order (alphabetical, folders first) | | LR-LIB-SMARTALBUM | Smart albums | P1 | ✅ | `cmd:album.createSmart`, `cmd:album.setRules`, `crates/catalog/src/query.rs`, `crates/ui-egui/src/panels/filterbar.rs`, `crates/ui-egui/src/lib.rs` (`album_counts`) | saved filters (rating/flag/label/kind/edited/keyword/camera/lens/date range/text/album), live; match-all only (no any/none rule groups, no rule editor dialog — rules come from the filter bar or `album.setRules`); sidebar counts are cached per catalog revision (and per minute while an “in the last…” rule is used), counted without building id lists | | LR-LIB-SHARED-ALBUM | Shared albums | P2 | ⬜ | | needs a sharing service | | LR-LIB-OFFLINE | Keep album offline | P2 | 🚫 | | not applicable: local-first library | @@ -512,7 +512,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins. | LR-BEHAV-WEB-SAFETY | Library safety in the browser build | P1 | 🟡 | `cmd:file.backupLibrary`, `cmd:file.restoreLibrary`, `apps/lightcraft-web/src/safety.rs`, `apps/lightcraft-web/src/backup.rs`, `apps/lightcraft-web/src/files.rs` | web only (experimental): Back Up Library (zip of the catalog and every stored original) and a non-destructive Restore; failed saves show as unsaved and are retried; a photo that can't be stored isn't added; one tab per library (Web Locks); notices for storage that isn't persistent and for a library that can't open; `?reset` asks first; a panic shows a message. Missing: `?safe` start, deleting originals of removed photos, zip64 (backups > 4 GB) | | LR-BEHAV-GPU | GPU acceleration | P0 | ✅ | `crates/gpu/src/render.rs`, `cmd:app.gpu`, `docs/gpu-pipeline.md` | CPU fallback per render on device limits, out of memory, device errors / loss, incomplete or black results (`perf.gpuReason`, `perf.gpuFallback`); short submissions for slow iGPUs; backend choice (DX12 only on Windows, `LIGHTCRAFT_GPU_BACKEND` / `WGPU_BACKEND`), device created after the window opens, crash sentinel (issue #136); no tiling beyond the buffer limit (CPU) | | LR-BEHAV-RENDER-FIDELITY | Rendering matches Lightroom | P1 | 🟡 | `crates/pipeline/src/lib.rs`, `crates/pipeline/src/tone.rs` | every slider exists and works, but the character of the result (tone curve shape, highlight recovery, texture / clarity / dehaze, noise reduction, sharpening, default look) is tuned by eye; there is no systematic side-by-side comparison against Lightroom on the same CC0 raws. Needs a fidelity suite: Lightroom reference renders kept only in the local `plan/` (never committed), compared per slider and preset with a perceptual metric | -| LR-BEHAV-DRAGDROP | Drag and drop | P1 | ✅ | `crates/ui-egui/src/lib.rs`, `crates/ui-egui/src/panels/grid.rs` (`drag_feedback`), `crates/ui-egui/src/panels/left.rs` (`drop_target`) | files → app (import); grid photos → an album row (adds the selection, with a count badge while dragging) | +| LR-BEHAV-DRAGDROP | Drag and drop | P1 | ✅ | `crates/ui-egui/src/lib.rs`, `crates/ui-egui/src/panels/grid.rs` (`drag_feedback`), `crates/ui-egui/src/panels/left.rs` (`drop_target`, `album_drag_over`, `top_level_drop_zone`) | files → app (import); grid photos → an album row (adds the selection, with a count badge while dragging); sidebar album / folder rows → a folder (moves it; its name follows the pointer) | | LR-BEHAV-TOAST | Toast notifications | P1 | ✅ | `crates/ui-egui/src/panels/mod.rs` | | | LR-BEHAV-SIDEBAR-COLLAPSE | Collapsible left-sidebar sections | P2 | ✅ | `crates/ui-egui/src/panels/left.rs` (`sidebar_section_header`), `crates/ui-egui/src/state.rs` (`collapsed_sidebar`), `crates/ui-egui/src/tests_panels.rs` | click the Albums, Local, By Date or Keywords header (or drive `ui.clickWidget` `sidebarSection:`) to fold or unfold the section; a chevron after the title shows the state; the choice is kept across restarts (`collapsedSidebar` in the UI state). UI-only (no command), like the By Date / Keywords disclosure triangles; photos dropped on a folded Albums section have no target | | LR-BEHAV-PANEL-RESIZE | Resizable side panels | P1 | ✅ | `crates/ui-egui/src/panels/mod.rs` (`resizable_side`), `crates/ui-egui/src/state.rs` (`LEFT_WIDTH`, `RIGHT_WIDTH`), `crates/ui-egui/src/tests_panels.rs` | drag the left sidebar's right edge (200–480 pt) or the right panel's left edge (250–520 pt); the photo area keeps ≥ 360 pt; widths are kept across panels, views and restarts (`leftWidth` / `rightWidth` in the UI state); the Presets column stays fixed | From 67aed0b4f4fdceb2079e63f9aea935a84a37a814 Mon Sep 17 00:00:00 2001 From: Nathaniel Ledford Date: Thu, 8 Oct 2026 20:25:03 -0400 Subject: [PATCH 10/46] Albums: a drag ends with the main button, Esc cancels it, hidden rows are no targets Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/panels/left.rs | 35 ++++++++++++++++++++---------- crates/ui-egui/src/tests_panels.rs | 29 +++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 11 deletions(-) diff --git a/crates/ui-egui/src/panels/left.rs b/crates/ui-egui/src/panels/left.rs index e851473f..065d1cfc 100644 --- a/crates/ui-egui/src/panels/left.rs +++ b/crates/ui-egui/src/panels/left.rs @@ -733,6 +733,12 @@ fn albums_tree(app: &mut LightcraftApp, ui: &mut egui::Ui, all: &[Album], parent } } +/// Whether the pointer is over `rect` and over what the panel shows of it (a row scrolled out of +/// sight is not a drop target). +fn pointer_over(ui: &egui::Ui, rect: Rect) -> bool { + ui.input(|i| i.pointer.latest_pos()).is_some_and(|p| rect.contains(p) && ui.clip_rect().contains(p)) +} + /// How long a dragged album must rest on a closed folder before it opens. const HOVER_OPEN_SECS: f64 = 0.6; @@ -764,12 +770,11 @@ fn drop_album(app: &mut LightcraftApp, ui: &egui::Ui, dragged: AlbumId, target: /// in on release. Returns whether it opened the folder. fn album_drag_over(app: &mut LightcraftApp, ui: &mut egui::Ui, resp: &egui::Response, folder: &Album, open: Option<&mut bool>) -> bool { let hover_id = egui::Id::new(("album-hover", folder.id.0)); - let Some(dragged) = app.ui.dragging_album.map(AlbumId) else { - ui.data_mut(|d| d.remove_temp::(hover_id)); - return false; - }; - if !ui.input(|i| i.pointer.latest_pos()).is_some_and(|p| resp.rect.contains(p)) { - ui.data_mut(|d| d.remove_temp::(hover_id)); + let Some(dragged) = app.ui.dragging_album.map(AlbumId) else { return false }; + if !pointer_over(ui, resp.rect) { + if ui.data(|d| d.get_temp::(hover_id)).is_some() { + ui.data_mut(|d| d.remove_temp::(hover_id)); + } return false; } let now = ui.input(|i| i.time); @@ -788,7 +793,7 @@ fn album_drag_over(app: &mut LightcraftApp, ui: &mut egui::Ui, resp: &egui::Resp if can_drop_album(app, dragged, Some(folder.id)) { let t = Tokens::get(ui.ctx()); ui.painter().rect_stroke(resp.rect.shrink2(vec2(8.0, 1.0)), 4.0, egui::Stroke::new(1.5, t.accent), egui::StrokeKind::Inside); - if ui.input(|i| i.pointer.any_released()) { + if ui.input(|i| i.pointer.primary_released()) { drop_album(app, ui, dragged, Some(folder.id)); } } @@ -805,11 +810,11 @@ fn top_level_drop_zone(app: &mut LightcraftApp, ui: &mut egui::Ui) { let t = Tokens::get(ui.ctx()); let (r, _) = ui.allocate_exact_size(vec2(ui.available_width(), 29.0), Sense::hover()); register(ui.ctx(), "albumDrop:top", r); - let over = ui.input(|i| i.pointer.latest_pos()).is_some_and(|p| r.contains(p)); + let over = pointer_over(ui, r); let stroke = if over { egui::Stroke::new(1.5, t.accent) } else { egui::Stroke::new(1.0, t.text_dim.gamma_multiply(0.5)) }; ui.painter().rect_stroke(r.shrink2(vec2(8.0, 1.0)), 4.0, stroke, egui::StrokeKind::Inside); ui.painter().text(pos2(r.left() + 18.0, r.center().y), Align2::LEFT_CENTER, crate::i18n::tr("Top Level"), t.font(13.5), t.text_dim); - if over && ui.input(|i| i.pointer.any_released()) { + if over && ui.input(|i| i.pointer.primary_released()) { drop_album(app, ui, dragged, None); } } @@ -818,9 +823,17 @@ fn top_level_drop_zone(app: &mut LightcraftApp, ui: &mut egui::Ui) { /// button is up (the drop targets act on the release frame, before this runs). pub fn album_drag_feedback(app: &mut LightcraftApp, ctx: &egui::Context) { let Some(id) = app.ui.dragging_album else { return }; - let (released, down, pos) = ctx.input(|i| (i.pointer.any_released(), i.pointer.any_down(), i.pointer.latest_pos())); - if released || !down { + let (released, down, pos, esc) = + ctx.input(|i| (i.pointer.primary_released(), i.pointer.primary_down(), i.pointer.latest_pos(), i.key_pressed(egui::Key::Escape))); + if released || !down || esc { app.ui.dragging_album = None; + // forget how long the pointer rested on each folder + let ids: Vec = app.session.catalog.albums().filter(|a| a.folder).map(|a| a.id.0).collect(); + ctx.data_mut(|d| { + for id in ids { + d.remove_temp::(egui::Id::new(("album-hover", id))); + } + }); return; } let (Some(pos), Some(name)) = (pos, app.session.catalog.album(AlbumId(id)).map(|a| a.name.clone())) else { return }; diff --git a/crates/ui-egui/src/tests_panels.rs b/crates/ui-egui/src/tests_panels.rs index ba67867c..c9ee05ca 100644 --- a/crates/ui-egui/src/tests_panels.rs +++ b/crates/ui-egui/src/tests_panels.rs @@ -396,6 +396,10 @@ impl AlbumTree { } /// Press at `from`, move through each `(point, frames held there)`, release at the last one. fn drag(&mut self, from: egui::Pos2, path: &[(egui::Pos2, usize)]) { + self.drag_with(from, path, vec![]); + } + /// [`Self::drag`] with `before_release` events sent just before the button comes up. + fn drag_with(&mut self, from: egui::Pos2, path: &[(egui::Pos2, usize)], before_release: Vec) { let mut ev = |e| self.h.app.synthetic.push(e); ev(egui::Event::PointerMoved(from)); ev(egui::Event::PointerButton { pos: from, button: egui::PointerButton::Primary, pressed: true, modifiers: Default::default() }); @@ -410,6 +414,9 @@ impl AlbumTree { } at = *to; } + for e in before_release { + ev(e); + } ev(egui::Event::PointerButton { pos: at, button: egui::PointerButton::Primary, pressed: false, modifiers: Default::default() }); for _ in 0..400 { if self.h.app.synthetic.is_empty() { @@ -454,6 +461,28 @@ fn dragging_an_album_onto_a_folder_moves_it() { assert_eq!(t.h.app.session.source, lightcraft_engine::LibrarySource::Album(lightcraft_catalog::AlbumId(t.best))); } +/// A drag ends only with the main button, and Esc abandons it: a release of another button over a +/// folder does not drop there, and Esc then release drops nothing. +#[test] +fn an_album_drag_ignores_other_buttons_and_esc_cancels_it() { + let mut t = album_tree(); + let (from, to) = (t.row(t.loose).center(), t.row(t.archive).center()); + let m = egui::Modifiers::default(); + let secondary = |pressed| egui::Event::PointerButton { pos: to, button: egui::PointerButton::Secondary, pressed, modifiers: m }; + // a right click over the folder mid-drag; the drag then ends elsewhere (on the Loose row itself) + t.drag_with(from, &[(to, 3), (from, 2)], vec![secondary(true), secondary(false)]); + assert_eq!(t.parent(t.loose), None, "the secondary release is not a drop"); + assert!(t.h.app.ui.dragging_album.is_none()); + let esc = |pressed| egui::Event::Key { key: egui::Key::Escape, physical_key: None, pressed, repeat: false, modifiers: m }; + let (from, to) = (t.row(t.loose).center(), t.row(t.archive).center()); + t.drag_with(from, &[(to, 3)], vec![esc(true), esc(false)]); + assert_eq!(t.parent(t.loose), None, "Esc abandons the drag, the release drops nothing"); + assert!(t.h.app.ui.dragging_album.is_none()); + // and a drag that is left alone still drops + t.drag_row(t.loose, t.archive); + assert_eq!(t.parent(t.loose), Some(t.archive)); +} + /// Dragging out of a folder: the drop zone under the tree puts the album back at the top level; /// it only shows when there is somewhere to go, and the move is one undo step. #[test] From 39f5f114de066cdc41ea61fc6fe584d3d9bbe80f Mon Sep 17 00:00:00 2001 From: Nathaniel Ledford Date: Thu, 8 Oct 2026 20:30:52 -0400 Subject: [PATCH 11/46] Catalog: albums can be ordered by hand (format version 3) Album.order and Op::SetAlbumOrder; album_children lists folders first, then by place, then by name. Moving an album drops its place and undo restores it. Co-Authored-By: Claude Sonnet 5.5 --- crates/catalog/src/journal.rs | 3 +- crates/catalog/src/lib.rs | 34 ++++++- crates/catalog/src/local.rs | 1 + crates/catalog/src/model.rs | 6 +- crates/catalog/src/tests.rs | 14 ++- crates/catalog/src/tests_album_order.rs | 102 +++++++++++++++++++++ crates/catalog/src/tests_format_version.rs | 16 +++- crates/catalog/src/tests_journal.rs | 2 + crates/engine/src/cmd/library.rs | 2 +- crates/engine/src/demo.rs | 16 +++- 10 files changed, 187 insertions(+), 9 deletions(-) create mode 100644 crates/catalog/src/tests_album_order.rs diff --git a/crates/catalog/src/journal.rs b/crates/catalog/src/journal.rs index 96187cfa..5973975d 100644 --- a/crates/catalog/src/journal.rs +++ b/crates/catalog/src/journal.rs @@ -45,6 +45,7 @@ //! | (none) | early builds: a log without a snapshot | | //! | 1 | up to v0.2.0 | | //! | 2 | after v0.2.0 | `Op::SetBrowsed`, `Catalog.browsed`, `Photo.local_baseline` | +//! | 3 | albums by hand | `Op::SetAlbumOrder`, `Album.order` | //! //! Rules: //! - **Bump [`VERSION`]** (and add a row above) in the change that adds an [`Op`] variant or a @@ -69,7 +70,7 @@ pub const LOG: &str = "catalog.log"; const FORMAT: &str = "lightcraft-catalog"; /// The catalog format this build writes (and the newest it reads). See the module docs → /// *Format versions*; bump it whenever an [`Op`] variant or a serialized field is added. -pub const VERSION: u32 = 2; +pub const VERSION: u32 = 3; /// When [`Journal::wants_snapshot`] says it's time to compact the log. #[derive(Clone, Copy, Debug)] diff --git a/crates/catalog/src/lib.rs b/crates/catalog/src/lib.rs index 2934c7a6..8a19b6e2 100644 --- a/crates/catalog/src/lib.rs +++ b/crates/catalog/src/lib.rs @@ -136,6 +136,11 @@ pub enum Op { id: AlbumId, parent: Option, }, + /// Where the album stands among its siblings (`None`: by name). Format version 3. + SetAlbumOrder { + id: AlbumId, + order: Option, + }, SetAlbumPhotos { id: AlbumId, photos: Vec, @@ -282,6 +287,18 @@ impl Catalog { pub fn albums(&self) -> impl Iterator { self.albums.values() } + /// The albums and folders directly inside `parent` (`None`: the top level), in the order the + /// sidebar lists them: folders first, then by their place if the user ordered them (those + /// without one after, by name), else by name. + pub fn album_children(&self, parent: Option) -> Vec<&Album> { + let mut kids: Vec<&Album> = self.albums.values().filter(|a| a.parent == parent).collect(); + kids.sort_by_cached_key(|a| (!a.folder, a.order.is_none(), a.order.unwrap_or(0), a.name.to_lowercase(), a.id)); + kids + } + /// Whether anything inside `parent` has a place of its own (the folder is ordered by hand). + pub fn album_children_are_ordered(&self, parent: Option) -> bool { + self.albums.values().any(|a| a.parent == parent && a.order.is_some()) + } /// The Quick Collection, once something was added to it. pub fn quick_collection(&self) -> Option { self.albums.values().find(|a| a.quick).map(|a| a.id) @@ -479,7 +496,20 @@ impl Catalog { } } let a = self.album_mut(id)?; - Op::MoveAlbum { id, parent: std::mem::replace(&mut a.parent, parent) } + if a.parent == parent { + return Ok(Op::MoveAlbum { id, parent }); + } + // its place belonged to the old neighbours: the undo gives it back with the folder + let old_order = a.order.take(); + let old_parent = std::mem::replace(&mut a.parent, parent); + match old_order { + Some(_) => Op::Batch { ops: vec![Op::MoveAlbum { id, parent: old_parent }, Op::SetAlbumOrder { id, order: old_order }] }, + None => Op::MoveAlbum { id, parent: old_parent }, + } + } + Op::SetAlbumOrder { id, order } => { + let a = self.album_mut(id)?; + Op::SetAlbumOrder { id, order: std::mem::replace(&mut a.order, order) } } Op::SetAlbumPhotos { id, photos } => { let a = self.album_mut(id)?; @@ -671,6 +701,8 @@ impl Catalog { #[cfg(test)] mod tests; #[cfg(test)] +mod tests_album_order; +#[cfg(test)] mod tests_background; #[cfg(test)] mod tests_folders; diff --git a/crates/catalog/src/local.rs b/crates/catalog/src/local.rs index 79a4a42d..4291f45f 100644 --- a/crates/catalog/src/local.rs +++ b/crates/catalog/src/local.rs @@ -295,6 +295,7 @@ impl Op { Op::RemoveAlbum { .. } | Op::RenameAlbum { .. } | Op::MoveAlbum { .. } + | Op::SetAlbumOrder { .. } | Op::SetAlbumRules { .. } | Op::RemoveStack { .. } | Op::SetLabelName { .. } diff --git a/crates/catalog/src/model.rs b/crates/catalog/src/model.rs index ae53b486..957f5b68 100644 --- a/crates/catalog/src/model.rs +++ b/crates/catalog/src/model.rs @@ -405,12 +405,16 @@ pub struct Album { /// The Quick Collection: a regular album that B adds to (one per library). #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub quick: bool, + /// Its place among the albums of the same kind in its folder, once the user ordered them by + /// hand (see [`crate::Catalog::album_children`]); `None`: listed by name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub order: Option, } impl Album { /// A regular (manual) album. pub fn new(id: AlbumId, name: impl Into) -> Album { - Album { id, name: name.into(), parent: None, folder: false, photos: Vec::new(), cover: None, smart: None, quick: false } + Album { id, name: name.into(), parent: None, folder: false, photos: Vec::new(), cover: None, smart: None, quick: false, order: None } } pub fn is_smart(&self) -> bool { self.smart.is_some() diff --git a/crates/catalog/src/tests.rs b/crates/catalog/src/tests.rs index 49f0ac4d..8d34d5ef 100644 --- a/crates/catalog/src/tests.rs +++ b/crates/catalog/src/tests.rs @@ -58,12 +58,22 @@ fn albums_and_folders() { let a = photo(&mut c, "a.jpg", "2026-04-01"); let f = c.alloc_album_id(); c.apply(Op::AddAlbum { - album: Album { id: f, name: "Trips".into(), parent: None, folder: true, photos: vec![], cover: None, smart: None, quick: false }, + album: Album { id: f, name: "Trips".into(), parent: None, folder: true, photos: vec![], cover: None, smart: None, quick: false, order: None }, }) .unwrap(); let al = c.alloc_album_id(); c.apply(Op::AddAlbum { - album: Album { id: al, name: "Alps".into(), parent: Some(f), folder: false, photos: vec![a], cover: None, smart: None, quick: false }, + album: Album { + id: al, + name: "Alps".into(), + parent: Some(f), + folder: false, + photos: vec![a], + cover: None, + smart: None, + quick: false, + order: None, + }, }) .unwrap(); assert_eq!(c.albums_of(a), vec![al]); diff --git a/crates/catalog/src/tests_album_order.rs b/crates/catalog/src/tests_album_order.rs new file mode 100644 index 00000000..dfd463f9 --- /dev/null +++ b/crates/catalog/src/tests_album_order.rs @@ -0,0 +1,102 @@ +//! Album order: the children of a folder (or of the top level) are listed folders first, then +//! alphabetically, until some of them are given a place of their own (`Album.order`); moving an +//! album to another folder drops its place, and undoing the move brings it back. + +use crate::*; + +fn add(c: &mut Catalog, name: &str, parent: Option, folder: bool) -> AlbumId { + let id = c.alloc_album_id(); + c.apply(Op::AddAlbum { album: Album { parent, folder, ..Album::new(id, name) } }).unwrap(); + id +} + +fn names(c: &Catalog, parent: Option) -> Vec { + c.album_children(parent).iter().map(|a| a.name.clone()).collect() +} + +fn place(c: &mut Catalog, id: AlbumId, order: Option) { + c.apply(Op::SetAlbumOrder { id, order }).unwrap(); +} + +/// Given albums that were never ordered, when they are listed, then folders come first and each +/// group is alphabetical regardless of case. +#[test] +fn children_are_folders_first_then_alphabetical() { + let mut c = Catalog::new(); + add(&mut c, "beta", None, false); + add(&mut c, "Alpha", None, false); + add(&mut c, "zoo", None, true); + add(&mut c, "Archive", None, true); + assert_eq!(names(&c, None), ["Archive", "zoo", "Alpha", "beta"]); + assert!(!c.album_children_are_ordered(None)); +} + +/// Given some children with a place of their own, then they follow it; the others come after, by +/// name; folders stay ahead of albums. +#[test] +fn ordered_children_follow_their_order() { + let mut c = Catalog::new(); + let a = add(&mut c, "A", None, false); + let b = add(&mut c, "B", None, false); + let cc = add(&mut c, "C", None, false); + add(&mut c, "D", None, false); + let f = add(&mut c, "F", None, true); + place(&mut c, cc, Some(0)); + place(&mut c, a, Some(1)); + assert_eq!(names(&c, None), ["F", "C", "A", "B", "D"]); + assert!(c.album_children_are_ordered(None)); + // equal places fall back to the name, so a damaged catalog still lists deterministically + place(&mut c, b, Some(1)); + assert_eq!(names(&c, None), ["F", "C", "A", "B", "D"]); + // another container is unaffected + let inner = add(&mut c, "Z", Some(f), false); + add(&mut c, "Y", Some(f), false); + assert_eq!(names(&c, Some(f)), ["Y", "Z"]); + assert!(!c.album_children_are_ordered(Some(f))); + place(&mut c, inner, Some(0)); + assert_eq!(names(&c, Some(f)), ["Z", "Y"]); +} + +/// A place is one undo step, and `None` puts the album back among the alphabetical ones. +#[test] +fn a_place_can_be_set_cleared_and_undone() { + let mut c = Catalog::new(); + let a = add(&mut c, "A", None, false); + add(&mut c, "B", None, false); + let inv = c.apply(Op::SetAlbumOrder { id: a, order: Some(5) }).unwrap(); + assert_eq!(c.album(a).unwrap().order, Some(5)); + c.apply(inv).unwrap(); + assert_eq!(c.album(a).unwrap().order, None); + assert!(c.apply(Op::SetAlbumOrder { id: AlbumId(999), order: Some(1) }).is_err(), "no such album"); +} + +/// Moving to another folder drops the album's place there (it belonged to the old neighbours), +/// and undoing the move gives it back. +#[test] +fn moving_drops_the_place_and_undo_restores_it() { + let mut c = Catalog::new(); + let from = add(&mut c, "From", None, true); + let to = add(&mut c, "To", None, true); + let a = add(&mut c, "A", Some(from), false); + place(&mut c, a, Some(3)); + let inv = c.apply(Op::MoveAlbum { id: a, parent: Some(to) }).unwrap(); + assert_eq!((c.album(a).unwrap().parent, c.album(a).unwrap().order), (Some(to), None)); + c.apply(inv).unwrap(); + assert_eq!((c.album(a).unwrap().parent, c.album(a).unwrap().order), (Some(from), Some(3))); + // "moving" to where it already is changes nothing + c.apply(Op::MoveAlbum { id: a, parent: Some(from) }).unwrap(); + assert_eq!(c.album(a).unwrap().order, Some(3)); +} + +/// Catalogs written before albums had an order load as they were; an unset order is not written. +#[test] +fn albums_without_an_order_load_and_save_unchanged() { + let old = r#"{"id":4,"name":"Old","parent":null,"folder":false,"photos":[],"cover":null}"#; + let a: Album = serde_json::from_str(old).unwrap(); + assert_eq!(a.order, None); + assert!(!serde_json::to_string(&a).unwrap().contains("order")); + let mut a = a; + a.order = Some(2); + let back: Album = serde_json::from_str(&serde_json::to_string(&a).unwrap()).unwrap(); + assert_eq!(back.order, Some(2)); +} diff --git a/crates/catalog/src/tests_format_version.rs b/crates/catalog/src/tests_format_version.rs index 3f2e8816..cb80ce67 100644 --- a/crates/catalog/src/tests_format_version.rs +++ b/crates/catalog/src/tests_format_version.rs @@ -47,6 +47,19 @@ fn v1_library_loads_and_is_upgraded() { assert_eq!((r.replayed, r.upgraded_from), (1, None)); } +/// A library saved before albums could be ordered by hand opens as it was and is rewritten as v3. +#[test] +fn v2_library_loads_and_is_upgraded() { + let (base, log, full) = legacy_parts(); + let m = MemStore::new(); + m.set(SNAPSHOT, format!("{{\"format\":\"lightcraft-catalog\",\"version\":2,\"seq\":2,\"catalog\":{}}}\n", base.to_snapshot()).into_bytes()); + m.set(LOG, log.into_bytes()); + let (_, c, r) = Journal::open(Box::new(m.clone())).unwrap(); + assert_eq!(c.to_snapshot(), full.to_snapshot()); + assert_eq!((r.replayed, r.upgraded_from), (1, Some(2))); + assert_eq!(snapshot_version(&m), 3); +} + #[test] fn versionless_log_only_library_is_upgraded() { let mut c = Catalog::new(); @@ -179,8 +192,9 @@ fn op_variants_are_versioned() { | Op::SetLabelName { .. } | Op::Batch { .. } => 1, Op::SetBrowsed { .. } => 2, + Op::SetAlbumOrder { .. } => 3, } } - let newest = since(&Op::SetBrowsed { folder: String::new(), at: None }); + let newest = since(&Op::SetAlbumOrder { id: AlbumId(0), order: None }); assert_eq!(newest, VERSION, "the newest op's version must be the current format version"); } diff --git a/crates/catalog/src/tests_journal.rs b/crates/catalog/src/tests_journal.rs index 0c9513e3..0594b6d8 100644 --- a/crates/catalog/src/tests_journal.rs +++ b/crates/catalog/src/tests_journal.rs @@ -59,6 +59,7 @@ fn op_for(c: &mut Catalog, kind: u8, a: u8, b: u8) -> Op { cover: None, smart: None, quick: false, + order: None, }, } } @@ -77,6 +78,7 @@ fn op_for(c: &mut Catalog, kind: u8, a: u8, b: u8) -> Op { 13 => Op::Batch { ops: c.remove_from_stacks_ops(&[pid]) }, 14 => c.set_top_ops(pid).unwrap_or(Op::Batch { ops: vec![] }), 9 if b.is_multiple_of(5) => Op::SetLabelName { label: ColorLabel::ALL[a as usize % 5], name: (a % 3 != 1).then(|| format!("Label {b}")) }, + 9 if b % 7 == 3 => Op::SetAlbumOrder { id: aid, order: (!b.is_multiple_of(3)).then_some(u32::from(b % 5)) }, 9 if b.is_multiple_of(2) => { Op::SetFile { id: pid, file_name: format!("Renamed_{b}.jpg"), source: Source::File { path: format!("/p/Renamed_{b}.jpg") } } } diff --git a/crates/engine/src/cmd/library.rs b/crates/engine/src/cmd/library.rs index 436ad123..4ff64eb3 100644 --- a/crates/engine/src/cmd/library.rs +++ b/crates/engine/src/cmd/library.rs @@ -784,7 +784,7 @@ pub fn specs() -> Vec { let cover = photos.first().copied(); s.commit( if folder { "New Folder" } else { "New Album" }, - Op::AddAlbum { album: Album { id, name, parent, folder, photos, cover, smart: None, quick: false } }, + Op::AddAlbum { album: Album { id, name, parent, folder, photos, cover, smart: None, quick: false, order: None } }, )?; Ok(json!({"id": id.0})) }), diff --git a/crates/engine/src/demo.rs b/crates/engine/src/demo.rs index 2f392cb7..f9d8b7c9 100644 --- a/crates/engine/src/demo.rs +++ b/crates/engine/src/demo.rs @@ -62,7 +62,17 @@ pub fn load(s: &mut Session) { } let folder = s.catalog.alloc_album_id(); ops.push(Op::AddAlbum { - album: Album { id: folder, name: "Travel 2026".into(), parent: None, folder: true, photos: vec![], cover: None, smart: None, quick: false }, + album: Album { + id: folder, + name: "Travel 2026".into(), + parent: None, + folder: true, + photos: vec![], + cover: None, + smart: None, + quick: false, + order: None, + }, }); let by_kw = |kw: &str| -> Vec<_> { scenes.iter().zip(&ids).filter(|(sc, _)| sc.meta.keywords.contains(&kw)).map(|(_, id)| *id).collect() }; for (name, parent, photos) in [ @@ -75,7 +85,9 @@ pub fn load(s: &mut Session) { ] { let id = s.catalog.alloc_album_id(); let cover = photos.first().copied(); - ops.push(Op::AddAlbum { album: Album { id, name: name.into(), parent, folder: false, photos, cover, smart: None, quick: false } }); + ops.push(Op::AddAlbum { + album: Album { id, name: name.into(), parent, folder: false, photos, cover, smart: None, quick: false, order: None }, + }); } for op in ops { let _ = s.catalog.apply(op); From 2c5cb708bb50c26793ea9a83b5818a55254a307a Mon Sep 17 00:00:00 2001 From: Nathaniel Ledford Date: Thu, 8 Oct 2026 20:31:23 -0400 Subject: [PATCH 12/46] Engine: album.reorder and album.sort Place an album or folder before a sibling of its kind (also in another folder), or last of its kind, in one undo step; album.sort goes back to by name for one folder. Co-Authored-By: Claude Sonnet 5.5 --- crates/engine/src/cmd/library.rs | 78 +++++++++++++++ crates/engine/src/lib.rs | 2 + crates/engine/src/tests_album_order.rs | 132 +++++++++++++++++++++++++ 3 files changed, 212 insertions(+) create mode 100644 crates/engine/src/tests_album_order.rs diff --git a/crates/engine/src/cmd/library.rs b/crates/engine/src/cmd/library.rs index 4ff64eb3..11f66589 100644 --- a/crates/engine/src/cmd/library.rs +++ b/crates/engine/src/cmd/library.rs @@ -119,6 +119,14 @@ fn album_param(p: &Value, key: &str, c: &str) -> Result { p.get(key).and_then(Value::as_u64).map(AlbumId).ok_or_else(|| bad(c, format!("missing album `{key}`"))) } +/// An optional album id: absent or `null` is `None`, anything but a number is an error. +fn opt_album_param(p: &Value, key: &str, c: &str) -> Result> { + match p.get(key) { + None | Some(Value::Null) => Ok(None), + Some(v) => v.as_u64().map(|n| Some(AlbumId(n))).ok_or_else(|| bad(c, format!("`{key}` must be an album id or null"))), + } +} + fn strs(p: &Value, key: &str) -> Vec { p.get(key).and_then(Value::as_array).map(|a| a.iter().filter_map(Value::as_str).map(str::to_string).collect()).unwrap_or_default() } @@ -880,6 +888,76 @@ pub fn specs() -> Vec { s.commit("Move Album", Op::MoveAlbum { id, parent })?; ok() }), + cmd!( + "album.reorder", + "Reorder Album", + [], + None, + "{id, parent?: folderId|null, before?: albumId|null} — places the album (or folder) in `parent` (default: where it is; null: the top level) before the sibling `before` of the same kind, or last of its kind; the folder is then ordered by hand. One undo step. → {changed}", + always, + |s, p| { + let id = album_param(p, "id", "album.reorder")?; + let me = s.catalog.album(id).ok_or_else(|| bad("album.reorder", "no such album"))?; + let parent = if p.get("parent").is_some() { opt_album_param(p, "parent", "album.reorder")? } else { me.parent }; + let before = opt_album_param(p, "before", "album.reorder")?; + let mut order: Vec = s.catalog.album_children(parent).into_iter().filter(|a| a.id != id).map(|a| a.id).collect(); + let at = match before { + Some(b) => { + let at = order.iter().position(|x| *x == b).ok_or_else(|| bad("album.reorder", "`before` is not inside that folder"))?; + if s.catalog.album(b).is_none_or(|x| x.folder != me.folder) { + return Err(bad("album.reorder", "folders are placed among folders and albums among albums")); + } + at + } + None if me.folder => order.iter().take_while(|x| s.catalog.album(**x).is_some_and(|a| a.folder)).count(), + None => order.len(), + }; + order.insert(at.min(order.len()), id); + let moved = me.parent != parent; + let mut ops = Vec::new(); + if moved { + ops.push(Op::MoveAlbum { id, parent }); + } + for (i, aid) in order.iter().enumerate() { + let want = u32::try_from(i).ok(); + // a moved album starts without a place in its new folder + let have = if moved && *aid == id { None } else { s.catalog.album(*aid).and_then(|a| a.order) }; + if have != want { + ops.push(Op::SetAlbumOrder { id: *aid, order: want }); + } + } + if ops.is_empty() { + return Ok(json!({"changed": 0})); + } + let changed = ops.len(); + s.commit("Reorder Album", Op::Batch { ops })?; + Ok(json!({"changed": changed})) + } + ), + cmd!( + "album.sort", + "Sort Albums by Name", + [], + None, + "{parent?: folderId|null} — drops the hand order of the albums inside the folder (default: the top level), so they are listed by name → {changed}", + always, + |s, p| { + let parent = opt_album_param(p, "parent", "album.sort")?; + let ops: Vec = s + .catalog + .album_children(parent) + .into_iter() + .filter(|a| a.order.is_some()) + .map(|a| Op::SetAlbumOrder { id: a.id, order: None }) + .collect(); + if ops.is_empty() { + return Ok(json!({"changed": 0})); + } + let changed = ops.len(); + s.commit("Sort Albums by Name", Op::Batch { ops })?; + Ok(json!({"changed": changed})) + } + ), cmd!("album.addPhotos", "Add to Album", ["Photo"], None, "{id: albumId, ids?: [photoIds]} (default: selection)", has_selection, |s, p| { let id = album_param(p, "id", "album.addPhotos")?; let targets = ids_param(p).unwrap_or_else(|| s.targets(&Value::Null)); diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index f26b4ca7..0ecc431e 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -781,6 +781,8 @@ pub fn json_delta(old: &Value, new: &Value) -> Option { #[cfg(test)] mod tests; #[cfg(test)] +mod tests_album_order; +#[cfg(test)] mod tests_color; #[cfg(test)] mod tests_export; diff --git a/crates/engine/src/tests_album_order.rs b/crates/engine/src/tests_album_order.rs new file mode 100644 index 00000000..324c0db5 --- /dev/null +++ b/crates/engine/src/tests_album_order.rs @@ -0,0 +1,132 @@ +//! Ordering albums by hand: `album.reorder` puts an album (or folder) before a sibling, or last of +//! its kind, in one undo step, also when it changes folder; `album.sort` goes back to by name. + +use lightcraft_catalog::AlbumId; +use serde_json::{Value, json}; + +use crate::Session; + +fn make(s: &mut Session, name: &str, parent: Option, folder: bool) -> u64 { + let mut p = json!({"name": name, "folder": folder}); + if let Some(parent) = parent { + p["parent"] = json!(parent); + } + s.execute("album.create", &p).unwrap()["id"].as_u64().unwrap() +} + +fn names(s: &Session, parent: Option) -> Vec { + s.catalog.album_children(parent.map(AlbumId)).iter().map(|a| a.name.clone()).collect() +} + +fn reorder(s: &mut Session, id: u64, params: Value) -> Result { + let mut p = params; + p["id"] = json!(id); + s.execute("album.reorder", &p).map_err(|e| e.to_string()) +} + +/// Given three albums listed by name, when one is placed before another, then the list follows; +/// "no sibling" puts it last (among albums). +#[test] +fn an_album_is_placed_before_a_sibling_or_last() { + let mut s = Session::new(); + let (a, b, c) = (make(&mut s, "A", None, false), make(&mut s, "B", None, false), make(&mut s, "C", None, false)); + reorder(&mut s, c, json!({"before": a})).unwrap(); + assert_eq!(names(&s, None), ["C", "A", "B"]); + assert!(s.catalog.album_children_are_ordered(None)); + reorder(&mut s, c, json!({"before": null})).unwrap(); + assert_eq!(names(&s, None), ["A", "B", "C"]); + reorder(&mut s, a, json!({})).unwrap(); + assert_eq!(names(&s, None), ["B", "C", "A"], "no `before`: last"); + reorder(&mut s, b, json!({"before": b})).expect_err("not before itself"); + let _ = (a, b); +} + +/// Folders stay ahead of albums: a folder is placed among folders, an album among albums. +#[test] +fn folders_and_albums_keep_to_their_own_group() { + let mut s = Session::new(); + let (f1, f2) = (make(&mut s, "F1", None, true), make(&mut s, "F2", None, true)); + let (a1, a2) = (make(&mut s, "A1", None, false), make(&mut s, "A2", None, false)); + reorder(&mut s, f2, json!({"before": f1})).unwrap(); + reorder(&mut s, a2, json!({"before": a1})).unwrap(); + assert_eq!(names(&s, None), ["F2", "F1", "A2", "A1"]); + reorder(&mut s, f1, json!({"before": a1})).expect_err("a folder is not placed among albums"); + reorder(&mut s, a1, json!({"before": f1})).expect_err("an album is not placed among folders"); + reorder(&mut s, f2, json!({"before": null})).unwrap(); + assert_eq!(names(&s, None), ["F1", "F2", "A2", "A1"], "last of the folders"); +} + +/// Placing an album in another folder moves it there, and one undo undoes both. +#[test] +fn placing_across_folders_moves_it_and_undoes_in_one_step() { + let mut s = Session::new(); + let folder = make(&mut s, "Trips", None, true); + let (x, y) = (make(&mut s, "X", Some(folder), false), make(&mut s, "Y", Some(folder), false)); + let loose = make(&mut s, "Loose", None, false); + reorder(&mut s, y, json!({"before": x})).unwrap(); + assert_eq!(names(&s, Some(folder)), ["Y", "X"]); + reorder(&mut s, loose, json!({"parent": folder, "before": x})).unwrap(); + assert_eq!(names(&s, Some(folder)), ["Y", "Loose", "X"]); + assert_eq!(s.catalog.album(AlbumId(loose)).unwrap().parent, Some(AlbumId(folder))); + s.execute("edit.undo", &json!({})).unwrap(); + assert_eq!(names(&s, Some(folder)), ["Y", "X"]); + assert_eq!(s.catalog.album(AlbumId(loose)).unwrap().parent, None); + assert_eq!(s.catalog.album(AlbumId(loose)).unwrap().order, None); +} + +/// Bad requests change nothing: a `before` that is not in the target folder, a target that is not +/// a folder or lies inside the album, an unknown album. +#[test] +fn bad_placements_are_refused_and_change_nothing() { + let mut s = Session::new(); + let f = make(&mut s, "F", None, true); + let inner = make(&mut s, "Inner", Some(f), true); + let (a, b) = (make(&mut s, "A", None, false), make(&mut s, "B", Some(f), false)); + let before = s.catalog.to_snapshot(); + reorder(&mut s, a, json!({"before": b})).expect_err("B is in F, not at the top level"); + reorder(&mut s, a, json!({"parent": b})).expect_err("a plain album is no folder"); + reorder(&mut s, f, json!({"parent": inner})).expect_err("a folder into itself"); + reorder(&mut s, 987_654, json!({})).expect_err("no such album"); + reorder(&mut s, a, json!({"before": 987_654})).expect_err("no such sibling"); + assert_eq!(s.catalog.to_snapshot(), before, "nothing changed"); +} + +/// `album.sort` drops the hand order of one folder (and only that one): by name again. +#[test] +fn sorting_goes_back_to_by_name_for_one_folder() { + let mut s = Session::new(); + let f = make(&mut s, "F", None, true); + let (x, y) = (make(&mut s, "X", Some(f), false), make(&mut s, "Y", Some(f), false)); + let (a, b) = (make(&mut s, "A", None, false), make(&mut s, "B", None, false)); + reorder(&mut s, y, json!({"before": x})).unwrap(); + reorder(&mut s, b, json!({"before": a})).unwrap(); + assert_eq!(names(&s, Some(f)), ["Y", "X"]); + assert_eq!(names(&s, None), ["F", "B", "A"]); + let r = s.execute("album.sort", &json!({"parent": f})).unwrap(); + assert_eq!(r["changed"], 2); + assert_eq!(names(&s, Some(f)), ["X", "Y"]); + assert_eq!(names(&s, None), ["F", "B", "A"], "the top level keeps its order"); + assert!(!s.catalog.album_children_are_ordered(Some(AlbumId(f)))); + // nothing to do: no undo step is made + let r = s.execute("album.sort", &json!({"parent": f})).unwrap(); + assert_eq!(r["changed"], 0); + s.execute("edit.undo", &json!({})).unwrap(); + assert_eq!(names(&s, Some(f)), ["Y", "X"], "the one undo step was the sort"); + let r = s.execute("album.sort", &json!({})).unwrap(); + assert_eq!(r["changed"], 3, "every child of the top level had a place: F, B and A"); + assert_eq!(names(&s, None), ["F", "A", "B"]); +} + +/// Moving an album with `album.move` out of an ordered folder leaves the old neighbours' order. +#[test] +fn moving_out_of_an_ordered_folder_keeps_the_others_in_place() { + let mut s = Session::new(); + let f = make(&mut s, "F", None, true); + let (x, y, z) = (make(&mut s, "X", Some(f), false), make(&mut s, "Y", Some(f), false), make(&mut s, "Z", Some(f), false)); + reorder(&mut s, z, json!({"before": x})).unwrap(); + assert_eq!(names(&s, Some(f)), ["Z", "X", "Y"]); + s.execute("album.move", &json!({"id": x, "parent": null})).unwrap(); + assert_eq!(names(&s, Some(f)), ["Z", "Y"]); + assert_eq!(names(&s, None), ["F", "X"]); + let _ = y; +} From 34a4b61f7ed1b66a85cb656d4d4cbf705f7297e4 Mon Sep 17 00:00:00 2001 From: Nathaniel Ledford Date: Thu, 8 Oct 2026 20:33:36 -0400 Subject: [PATCH 13/46] Albums: one-pass children map, before-itself is a no-op, more order tests Co-Authored-By: Claude Sonnet 5.5 --- crates/catalog/src/lib.rs | 18 +++++++++++++- crates/catalog/src/tests_album_order.rs | 20 ++++++++++++++++ crates/engine/src/cmd/library.rs | 8 +++++-- crates/engine/src/tests_album_order.rs | 31 +++++++++++++++++++++++-- 4 files changed, 72 insertions(+), 5 deletions(-) diff --git a/crates/catalog/src/lib.rs b/crates/catalog/src/lib.rs index 8a19b6e2..884ec658 100644 --- a/crates/catalog/src/lib.rs +++ b/crates/catalog/src/lib.rs @@ -292,9 +292,19 @@ impl Catalog { /// without one after, by name), else by name. pub fn album_children(&self, parent: Option) -> Vec<&Album> { let mut kids: Vec<&Album> = self.albums.values().filter(|a| a.parent == parent).collect(); - kids.sort_by_cached_key(|a| (!a.folder, a.order.is_none(), a.order.unwrap_or(0), a.name.to_lowercase(), a.id)); + kids.sort_by_cached_key(|a| album_order_key(a)); kids } + /// [`Self::album_children`] of every folder (and of the top level, under `None`) in one pass: + /// what a tree view that draws all of them wants each frame. + pub fn album_children_by_parent(&self) -> std::collections::HashMap, Vec<&Album>> { + let mut map: std::collections::HashMap, Vec<&Album>> = std::collections::HashMap::new(); + for a in self.albums.values() { + map.entry(a.parent).or_default().push(a); + } + map.values_mut().for_each(|kids| kids.sort_by_cached_key(|a| album_order_key(a))); + map + } /// Whether anything inside `parent` has a place of its own (the folder is ordered by hand). pub fn album_children_are_ordered(&self, parent: Option) -> bool { self.albums.values().any(|a| a.parent == parent && a.order.is_some()) @@ -698,6 +708,12 @@ impl Catalog { } } +/// How siblings are listed: folders first, then the ones with a place of their own by it, then by +/// name; the id settles any tie. +fn album_order_key(a: &Album) -> (bool, bool, u32, String, AlbumId) { + (!a.folder, a.order.is_none(), a.order.unwrap_or(0), a.name.to_lowercase(), a.id) +} + #[cfg(test)] mod tests; #[cfg(test)] diff --git a/crates/catalog/src/tests_album_order.rs b/crates/catalog/src/tests_album_order.rs index dfd463f9..a6fb597b 100644 --- a/crates/catalog/src/tests_album_order.rs +++ b/crates/catalog/src/tests_album_order.rs @@ -57,6 +57,26 @@ fn ordered_children_follow_their_order() { assert_eq!(names(&c, Some(f)), ["Z", "Y"]); } +/// The one-pass map lists every folder's children exactly as `album_children` does. +#[test] +fn the_children_map_matches_the_children_of_each_folder() { + let mut c = Catalog::new(); + let f = add(&mut c, "F", None, true); + let g = add(&mut c, "G", Some(f), true); + for (name, parent) in [("b", None), ("a", None), ("y", Some(f)), ("x", Some(f)), ("q", Some(g))] { + add(&mut c, name, parent, false); + } + let x = c.albums().find(|a| a.name == "x").unwrap().id; + place(&mut c, x, Some(7)); + let map = c.album_children_by_parent(); + for parent in [None, Some(f), Some(g)] { + let from_map: Vec = map.get(&parent).map(|v| v.iter().map(|a| a.id).collect()).unwrap_or_default(); + let direct: Vec = c.album_children(parent).iter().map(|a| a.id).collect(); + assert_eq!(from_map, direct, "{parent:?}"); + } + assert!(map.get(&Some(AlbumId(999))).is_none()); +} + /// A place is one undo step, and `None` puts the album back among the alphabetical ones. #[test] fn a_place_can_be_set_cleared_and_undone() { diff --git a/crates/engine/src/cmd/library.rs b/crates/engine/src/cmd/library.rs index 11f66589..d900eacf 100644 --- a/crates/engine/src/cmd/library.rs +++ b/crates/engine/src/cmd/library.rs @@ -893,13 +893,17 @@ pub fn specs() -> Vec { "Reorder Album", [], None, - "{id, parent?: folderId|null, before?: albumId|null} — places the album (or folder) in `parent` (default: where it is; null: the top level) before the sibling `before` of the same kind, or last of its kind; the folder is then ordered by hand. One undo step. → {changed}", + "{id, parent?: folderId|null, before?: albumId|null} — places the album (or folder) in `parent` (default: where it is; null: the top level) before the sibling `before` of the same kind, or last of its kind; the folder is then ordered by hand. One undo step. → {changed}: how many edits it took (0: already there)", always, |s, p| { let id = album_param(p, "id", "album.reorder")?; let me = s.catalog.album(id).ok_or_else(|| bad("album.reorder", "no such album"))?; let parent = if p.get("parent").is_some() { opt_album_param(p, "parent", "album.reorder")? } else { me.parent }; let before = opt_album_param(p, "before", "album.reorder")?; + if before == Some(id) { + // before itself: where it is + return Ok(json!({"changed": 0})); + } let mut order: Vec = s.catalog.album_children(parent).into_iter().filter(|a| a.id != id).map(|a| a.id).collect(); let at = match before { Some(b) => { @@ -939,7 +943,7 @@ pub fn specs() -> Vec { "Sort Albums by Name", [], None, - "{parent?: folderId|null} — drops the hand order of the albums inside the folder (default: the top level), so they are listed by name → {changed}", + "{parent?: folderId|null} — drops the hand order of the albums inside the folder (default: the top level), so they are listed by name → {changed}: how many albums lost their place", always, |s, p| { let parent = opt_album_param(p, "parent", "album.sort")?; diff --git a/crates/engine/src/tests_album_order.rs b/crates/engine/src/tests_album_order.rs index 324c0db5..cff524cb 100644 --- a/crates/engine/src/tests_album_order.rs +++ b/crates/engine/src/tests_album_order.rs @@ -37,8 +37,8 @@ fn an_album_is_placed_before_a_sibling_or_last() { assert_eq!(names(&s, None), ["A", "B", "C"]); reorder(&mut s, a, json!({})).unwrap(); assert_eq!(names(&s, None), ["B", "C", "A"], "no `before`: last"); - reorder(&mut s, b, json!({"before": b})).expect_err("not before itself"); - let _ = (a, b); + assert_eq!(reorder(&mut s, b, json!({"before": b})).unwrap()["changed"], 0, "before itself is where it is"); + assert_eq!(names(&s, None), ["B", "C", "A"]); } /// Folders stay ahead of albums: a folder is placed among folders, an album among albums. @@ -74,6 +74,33 @@ fn placing_across_folders_moves_it_and_undoes_in_one_step() { assert_eq!(s.catalog.album(AlbumId(loose)).unwrap().order, None); } +/// Undo and redo of placing across folders, of sorting, and of moving a folder that has children. +#[test] +fn placing_and_sorting_undo_and_redo() { + let mut s = Session::new(); + let top = make(&mut s, "Top", None, true); + let child = make(&mut s, "Child", Some(top), false); + let (a, b) = (make(&mut s, "A", None, false), make(&mut s, "B", None, false)); + reorder(&mut s, b, json!({"before": a})).unwrap(); + // a folder with children, placed inside another folder + let other = make(&mut s, "Other", None, true); + reorder(&mut s, top, json!({"parent": other})).unwrap(); + assert_eq!(names(&s, Some(other)), ["Top"]); + assert_eq!(names(&s, Some(top)), ["Child"], "its children came along"); + s.execute("edit.undo", &json!({})).unwrap(); + assert_eq!(s.catalog.album(AlbumId(top)).unwrap().parent, None); + s.execute("edit.redo", &json!({})).unwrap(); + assert_eq!(s.catalog.album(AlbumId(top)).unwrap().parent, Some(AlbumId(other))); + // sort, undo, redo + s.execute("album.sort", &json!({})).unwrap(); + assert_eq!(names(&s, None), ["Other", "A", "B"]); + s.execute("edit.undo", &json!({})).unwrap(); + assert_eq!(names(&s, None), ["Other", "B", "A"], "the hand order is back"); + s.execute("edit.redo", &json!({})).unwrap(); + assert_eq!(names(&s, None), ["Other", "A", "B"]); + assert_eq!(s.catalog.album(AlbumId(child)).unwrap().parent, Some(AlbumId(top))); +} + /// Bad requests change nothing: a `before` that is not in the target folder, a target that is not /// a folder or lies inside the album, an unknown album. #[test] From 8036f98acac1447664509a1ef3af2f0b1d966d3a Mon Sep 17 00:00:00 2001 From: Nathaniel Ledford Date: Thu, 8 Oct 2026 20:43:30 -0400 Subject: [PATCH 14/46] Catalog: tidy a test assertion Co-Authored-By: Claude Sonnet 5.5 --- crates/catalog/src/tests_album_order.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/catalog/src/tests_album_order.rs b/crates/catalog/src/tests_album_order.rs index a6fb597b..112633f4 100644 --- a/crates/catalog/src/tests_album_order.rs +++ b/crates/catalog/src/tests_album_order.rs @@ -74,7 +74,7 @@ fn the_children_map_matches_the_children_of_each_folder() { let direct: Vec = c.album_children(parent).iter().map(|a| a.id).collect(); assert_eq!(from_map, direct, "{parent:?}"); } - assert!(map.get(&Some(AlbumId(999))).is_none()); + assert!(!map.contains_key(&Some(AlbumId(999)))); } /// A place is one undo step, and `None` puts the album back among the alphabetical ones. From 085c25b10b56341f278bdc44ca289e49a8d1b15d Mon Sep 17 00:00:00 2001 From: Nathaniel Ledford Date: Thu, 8 Oct 2026 20:43:31 -0400 Subject: [PATCH 15/46] =?UTF-8?q?Albums:=20drop=20beside=20a=20row=20to=20?= =?UTF-8?q?place=20it=20by=20hand;=20Sort=20A=E2=80=93Z=20goes=20back=20to?= =?UTF-8?q?=20by=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dropping on the edge of a row places the album there (album.reorder) with a line showing where; a folder's middle still means inside. A closed folder opens only when the drop would go inside it. Sort Contents A–Z and Sort Albums A–Z (album.sort) appear once a folder is ordered by hand. Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/panels/left.rs | 174 +++++++++++++++++++++++------ crates/ui-egui/src/tests_panels.rs | 121 +++++++++++++++++++- 2 files changed, 258 insertions(+), 37 deletions(-) diff --git a/crates/ui-egui/src/panels/left.rs b/crates/ui-egui/src/panels/left.rs index 065d1cfc..64b3a98b 100644 --- a/crates/ui-egui/src/panels/left.rs +++ b/crates/ui-egui/src/panels/left.rs @@ -234,18 +234,27 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui) { if ui.button(crate::i18n::tr("Create Folder…")).clicked() { app.ui.dialog = Some(crate::state::Dialog::NewAlbum { name: String::new(), folder: true, parent: None }); } + // only once the albums were put in an order by hand + if app.session.catalog.album_children_are_ordered(None) { + ui.separator(); + if ui.button(crate::i18n::tr("Sort Albums A–Z")).on_hover_text(crate::i18n::tr("Go back to listing them by name")).clicked() { + let _ = app.run("album.sort", json!({})); + } + } }); // the album just made: the folders down to it open, once (also when the section is shut) let reveal = app.ui.reveal_album.take(); if albums_open { - let albums: Vec = app.session.catalog.albums().cloned().collect(); + let kids: AlbumKids = + app.session.catalog.album_children_by_parent().into_iter().map(|(k, v)| (k, v.into_iter().cloned().collect())).collect(); + let cat = &app.session.catalog; let mut open_to = Vec::new(); - let mut cur = reveal.map(AlbumId).and_then(|id| albums.iter().find(|a| a.id == id)).and_then(|a| a.parent); + let mut cur = reveal.map(AlbumId).and_then(|id| cat.album(id)).and_then(|a| a.parent); while let Some(p) = cur.filter(|p| !open_to.contains(p) && open_to.len() < 64) { open_to.push(p); - cur = albums.iter().find(|a| a.id == p).and_then(|a| a.parent); + cur = cat.album(p).and_then(|a| a.parent); } - albums_tree(app, ui, &albums, None, 0.0, &open_to); + albums_tree(app, ui, &kids, None, 0.0, &open_to); top_level_drop_zone(app, ui); } ui.add_space(10.0); @@ -670,10 +679,12 @@ fn browse_all_photos(app: &mut LightcraftApp, on: bool) { } } +/// The albums inside each folder (`None`: the top level), in the order they are listed. +type AlbumKids = std::collections::HashMap, Vec>; + /// `open_to`: folders forced open this frame (the way down to an album that was just made). -fn albums_tree(app: &mut LightcraftApp, ui: &mut egui::Ui, all: &[Album], parent: Option, indent: f32, open_to: &[AlbumId]) { - let mut kids: Vec<&Album> = all.iter().filter(|a| a.parent == parent).collect(); - kids.sort_by_key(|a| (!a.folder, a.name.to_lowercase())); +fn albums_tree(app: &mut LightcraftApp, ui: &mut egui::Ui, all: &AlbumKids, parent: Option, indent: f32, open_to: &[AlbumId]) { + let Some(kids) = all.get(&parent) else { return }; for a in kids { if a.folder { let open_id = egui::Id::new(("folder-open", a.id.0)); @@ -684,11 +695,11 @@ fn albums_tree(app: &mut LightcraftApp, ui: &mut egui::Ui, all: &[Album], parent } let resp = row_sensed(app, ui, &format!("folder:{}", a.id.0), Icon::Folder, &a.name, None, None, false, indent, Sense::click_and_drag()); // a folder is no source, so its row folds it too; the triangle is the same click, aimed - let has_children = all.iter().any(|c| c.parent == Some(a.id)); + let has_children = all.get(&Some(a.id)).is_some_and(|v| !v.is_empty()); if resp.drag_started() { app.ui.dragging_album = Some(a.id.0); } - if album_drag_over(app, ui, &resp, a, has_children.then_some(&mut open)) { + if album_drag_over(app, ui, &resp, a, has_children.then_some(&mut open), indent) { ui.data_mut(|d| d.insert_temp(open_id, true)); } let mut toggled = resp.clicked(); @@ -719,6 +730,7 @@ fn albums_tree(app: &mut LightcraftApp, ui: &mut egui::Ui, all: &[Album], parent if resp.drag_started() { app.ui.dragging_album = Some(a.id.0); } + album_drag_over(app, ui, &resp, a, None, indent); if !a.is_smart() { drop_target(app, ui, &resp, a); } @@ -752,10 +764,10 @@ fn can_drop_album(app: &LightcraftApp, dragged: AlbumId, target: Option } } -/// Moves the dragged album (`album.move`, one undo step) and ends the drag; the folders down to it -/// open so it shows in its new place. -fn drop_album(app: &mut LightcraftApp, ui: &egui::Ui, dragged: AlbumId, target: Option) { - match app.run("album.move", json!({"id": dragged.0, "parent": target.map(|t| t.0)})) { +/// Ends a drag that ran `result`: the folders down to the album open so it shows in its new place. +fn finish_drop(app: &mut LightcraftApp, ui: &egui::Ui, dragged: AlbumId, result: Result) { + forget_hover(app, ui.ctx()); + match result { Ok(_) => { app.ui.reveal_album = Some(dragged.0); ui.ctx().request_repaint(); @@ -765,24 +777,103 @@ fn drop_album(app: &mut LightcraftApp, ui: &egui::Ui, dragged: AlbumId, target: app.ui.dragging_album = None; } -/// A folder row while an album is dragged: outlined under the pointer when the drop is allowed, -/// opened (`open`, when it has something inside) once the pointer rests on it, and the album moved -/// in on release. Returns whether it opened the folder. -fn album_drag_over(app: &mut LightcraftApp, ui: &mut egui::Ui, resp: &egui::Response, folder: &Album, open: Option<&mut bool>) -> bool { - let hover_id = egui::Id::new(("album-hover", folder.id.0)); +/// Forgets how long the pointer rested on each folder (a new drag starts counting again). +fn forget_hover(app: &LightcraftApp, ctx: &egui::Context) { + let ids: Vec = app.session.catalog.albums().filter(|a| a.folder).map(|a| a.id.0).collect(); + ctx.data_mut(|d| { + for id in ids { + d.remove_temp::(egui::Id::new(("album-hover", id))); + } + }); +} + +/// Moves the dragged album into a folder (`album.move`, one undo step; `None`: the top level). +fn drop_album(app: &mut LightcraftApp, ui: &egui::Ui, dragged: AlbumId, target: Option) { + let r = app.run("album.move", json!({"id": dragged.0, "parent": target.map(|t| t.0)})); + finish_drop(app, ui, dragged, r); +} + +/// Places the dragged album in `parent` before the sibling `before` (`None`: last of its kind), by +/// hand (`album.reorder`, one undo step). +fn place_album(app: &mut LightcraftApp, ui: &egui::Ui, dragged: AlbumId, parent: Option, before: Option) { + let r = app.run("album.reorder", json!({"id": dragged.0, "parent": parent.map(|p| p.0), "before": before.map(|b| b.0)})); + finish_drop(app, ui, dragged, r); +} + +/// What dropping the dragged album on a row would do. +#[derive(Clone, Copy, Debug, PartialEq)] +enum AlbumDrop { + /// Into the folder. + Into(AlbumId), + /// Beside the row: in `parent`, before `before` (`None`: last of its kind); the line is drawn + /// above the row (`above`) or below it. + Place { parent: Option, before: Option, above: bool }, +} + +/// What dropping `dragged` at `frac` (0 = top edge, 1 = bottom edge) of the row of `target` does, +/// if anything. An album over a folder goes inside it; a folder over a folder goes inside by its +/// middle and beside it by its edges; an album over an album goes before it by its top half and +/// after it by its bottom half; a folder over an album does nothing. Placing where it already +/// is, or inside itself, is nothing too. `expanded`: `target` is a folder shown open. +fn album_drop_at(app: &LightcraftApp, dragged: AlbumId, target: &Album, frac: f32, expanded: bool) -> Option { + let cat = &app.session.catalog; + let d = cat.album(dragged)?; + let into = || can_drop_album(app, dragged, Some(target.id)).then_some(AlbumDrop::Into(target.id)); + if d.folder != target.folder { + return if target.folder { into() } else { None }; + } + if target.id == dragged { + return None; + } + let after = frac >= if target.folder { 0.7 } else { 0.5 }; + // the middle of a folder, and the bottom of an open one (its first child is next), mean inside + if target.folder && frac >= 0.3 && (!after || expanded) { + return into(); + } + let parent = target.parent; + if parent.is_some_and(|p| is_within(app, p, dragged)) { + return None; + } + let sibs = cat.album_children(parent); + let at = sibs.iter().position(|a| a.id == target.id)?; + let was = sibs.iter().position(|a| a.id == dragged); + if !after { + (was.map(|w| w + 1) != Some(at)).then_some(AlbumDrop::Place { parent, before: Some(target.id), above: true }) + } else { + let next = sibs.iter().skip(at + 1).find(|a| a.id != dragged).filter(|a| a.folder == target.folder).map(|a| a.id); + (was != Some(at + 1)).then_some(AlbumDrop::Place { parent, before: next, above: false }) + } +} + +/// A row while an album is dragged: a folder outlined under the pointer when the drop would go +/// inside it, a line above or below the row when it would go beside it, and the move or placement +/// done on release. A closed folder with something inside (`open`) opens once the pointer rests on +/// it; returns whether it did. +fn album_drag_over(app: &mut LightcraftApp, ui: &mut egui::Ui, resp: &egui::Response, row: &Album, open: Option<&mut bool>, indent: f32) -> bool { + let hover_id = egui::Id::new(("album-hover", row.id.0)); let Some(dragged) = app.ui.dragging_album.map(AlbumId) else { return false }; - if !pointer_over(ui, resp.rect) { + let pos = ui.input(|i| i.pointer.latest_pos()).filter(|_| pointer_over(ui, resp.rect)); + let Some(pos) = pos else { if ui.data(|d| d.get_temp::(hover_id)).is_some() { ui.data_mut(|d| d.remove_temp::(hover_id)); } return false; - } - let now = ui.input(|i| i.time); - let since = ui.data_mut(|d| *d.get_temp_mut_or_insert_with(hover_id, || now)); + }; + let expanded = open.as_deref().copied().unwrap_or(false); + let frac = ((pos.y - resp.rect.top()) / resp.rect.height().max(1.0)).clamp(0.0, 1.0); + let drop = album_drop_at(app, dragged, row, frac, expanded); let mut opened = false; - if let Some(open) = open + // only a drop that goes inside opens the folder: on its edges the rows would shift under the pointer + let inside = matches!(drop, Some(AlbumDrop::Into(_))); + if !inside && ui.data(|d| d.get_temp::(hover_id)).is_some() { + ui.data_mut(|d| d.remove_temp::(hover_id)); + } + if inside + && let Some(open) = open && !*open { + let now = ui.input(|i| i.time); + let since = ui.data_mut(|d| *d.get_temp_mut_or_insert_with(hover_id, || now)); if now - since >= HOVER_OPEN_SECS { *open = true; opened = true; @@ -790,12 +881,25 @@ fn album_drag_over(app: &mut LightcraftApp, ui: &mut egui::Ui, resp: &egui::Resp ui.ctx().request_repaint_after(std::time::Duration::from_millis(100)); } } - if can_drop_album(app, dragged, Some(folder.id)) { - let t = Tokens::get(ui.ctx()); - ui.painter().rect_stroke(resp.rect.shrink2(vec2(8.0, 1.0)), 4.0, egui::Stroke::new(1.5, t.accent), egui::StrokeKind::Inside); - if ui.input(|i| i.pointer.primary_released()) { - drop_album(app, ui, dragged, Some(folder.id)); + let t = Tokens::get(ui.ctx()); + let released = ui.input(|i| i.pointer.primary_released()); + match drop { + Some(AlbumDrop::Into(folder)) => { + ui.painter().rect_stroke(resp.rect.shrink2(vec2(8.0, 1.0)), 4.0, egui::Stroke::new(1.5, t.accent), egui::StrokeKind::Inside); + if released { + drop_album(app, ui, dragged, Some(folder)); + } } + Some(AlbumDrop::Place { parent, before, above }) => { + let y = if above { resp.rect.top() + 0.5 } else { resp.rect.bottom() - 0.5 }; + let (x0, x1) = (resp.rect.left() + 18.0 + indent, ui.clip_rect().right() - 8.0); + ui.painter().circle_stroke(pos2(x0, y), 3.0, egui::Stroke::new(1.5, t.accent)); + ui.painter().line_segment([pos2(x0 + 3.0, y), pos2(x1.max(x0 + 3.0), y)], egui::Stroke::new(2.0, t.accent)); + if released { + place_album(app, ui, dragged, parent, before); + } + } + None => {} } opened } @@ -827,13 +931,7 @@ pub fn album_drag_feedback(app: &mut LightcraftApp, ctx: &egui::Context) { ctx.input(|i| (i.pointer.primary_released(), i.pointer.primary_down(), i.pointer.latest_pos(), i.key_pressed(egui::Key::Escape))); if released || !down || esc { app.ui.dragging_album = None; - // forget how long the pointer rested on each folder - let ids: Vec = app.session.catalog.albums().filter(|a| a.folder).map(|a| a.id.0).collect(); - ctx.data_mut(|d| { - for id in ids { - d.remove_temp::(egui::Id::new(("album-hover", id))); - } - }); + forget_hover(app, ctx); return; } let (Some(pos), Some(name)) = (pos, app.session.catalog.album(AlbumId(id)).map(|a| a.name.clone())) else { return }; @@ -883,6 +981,12 @@ fn folder_menu(app: &mut LightcraftApp, resp: &egui::Response, a: &Album) { } } }); + if app.session.catalog.album_children_are_ordered(Some(a.id)) + && ui.button(crate::i18n::tr("Sort Contents A–Z")).on_hover_text(crate::i18n::tr("Go back to listing them by name")).clicked() + { + let _ = app.run("album.sort", json!({"parent": a.id.0})); + ui.close(); + } ui.separator(); } if !a.folder && !a.is_smart() && ui.button(crate::i18n::tr("Add Selected Photos")).clicked() { diff --git a/crates/ui-egui/src/tests_panels.rs b/crates/ui-egui/src/tests_panels.rs index c9ee05ca..364fdc2b 100644 --- a/crates/ui-egui/src/tests_panels.rs +++ b/crates/ui-egui/src/tests_panels.rs @@ -427,6 +427,19 @@ impl AlbumTree { self.h.step(); self.h.step(); } + /// Drag `id` to `frac` of the way down the row of `onto` (0 = its top edge, 1 = its bottom edge). + fn drag_to(&mut self, id: u64, onto: u64, frac: f32) { + let (from, r) = (self.row(id).center(), self.row(onto)); + self.drag(from, &[(egui::pos2(r.left() + 100.0, r.top() + frac * r.height()), 3)]); + } + fn id_of(&self, name: &str) -> u64 { + self.h.app.session.catalog.albums().find(|a| a.name == name).unwrap_or_else(|| panic!("no album {name}")).id.0 + } + /// The names inside `parent`, folders or albums only. + fn kids(&self, parent: Option, folders: bool) -> Vec { + let cat = &self.h.app.session.catalog; + cat.album_children(parent.map(lightcraft_catalog::AlbumId)).iter().filter(|a| a.folder == folders).map(|a| a.name.clone()).collect() + } fn drag_row(&mut self, id: u64, onto: u64) { let (from, to) = (self.row(id).center(), self.row(onto).center()); self.drag(from, &[(to, 3)]); @@ -446,9 +459,10 @@ fn dragging_an_album_onto_a_folder_moves_it() { assert!(t.h.app.widgets.iter().any(|(w, _)| *w == format!("source:album:{}", t.loose))); t.drag_row(t.sub, t.archive); assert_eq!(t.parent(t.sub), Some(t.archive), "a folder into a folder"); - // refused: onto a plain album, onto itself, into its own subfolder + // refused: onto itself, into its own subfolder; onto a plain album it only goes beside it t.drag_row(t.best, t.loose); - assert_eq!(t.parent(t.best), Some(t.trips), "a plain album is no destination"); + assert_eq!(t.parent(t.best), Some(t.archive), "a plain album is no destination: Best lands beside Loose, not inside"); + assert_eq!(t.parent(t.loose), Some(t.archive)); let (from, aside) = (t.row(t.archive).center(), t.row(t.archive).center() + egui::vec2(40.0, 0.0)); t.drag(from, &[(aside, 3)]); assert_eq!(t.parent(t.archive), None, "not onto itself"); @@ -483,6 +497,109 @@ fn an_album_drag_ignores_other_buttons_and_esc_cancels_it() { assert_eq!(t.parent(t.loose), Some(t.archive)); } +/// Given the top-level albums Garden, Loose, Night Sky, Portfolio listed by name, when an album is +/// dropped on the top or bottom half of another, then it is placed before or after it and the +/// list is ordered by hand from then on. +#[test] +fn dropping_on_the_edge_of_an_album_places_it_there() { + let mut t = album_tree(); + let id = |t: &AlbumTree, name: &str| t.h.app.session.catalog.albums().find(|a| a.name == name).unwrap().id.0; + let (garden, night, portfolio) = (id(&t, "Garden"), id(&t, "Night Sky"), id(&t, "Portfolio")); + assert_eq!(t.kids(None, false), ["Garden", "Loose", "Night Sky", "Portfolio"]); + t.drag_to(portfolio, garden, 0.2); + assert_eq!(t.kids(None, false), ["Portfolio", "Garden", "Loose", "Night Sky"], "before Garden"); + t.drag_to(garden, night, 0.8); + assert_eq!(t.kids(None, false), ["Portfolio", "Loose", "Night Sky", "Garden"], "after Night Sky"); + assert!(t.h.app.session.catalog.album_children_are_ordered(None)); + // one undo step per drop + t.h.app.session.execute("edit.undo", &json!({})).unwrap(); + assert_eq!(t.kids(None, false), ["Portfolio", "Garden", "Loose", "Night Sky"]); + assert!(t.h.app.ui.dragging_album.is_none()); +} + +/// Dropping beside an album in another folder moves the album there, at that place. +#[test] +fn dropping_beside_an_album_in_another_folder_moves_it_there() { + let mut t = album_tree(); + t.drag_to(t.loose, t.best, 0.2); + assert_eq!(t.parent(t.loose), Some(t.trips)); + assert_eq!(t.kids(Some(t.trips), false), ["Loose", "Best"]); +} + +/// Where nothing would change, nothing is done: no edit, so no hand order appears. +#[test] +fn dropping_where_it_already_is_changes_nothing() { + let mut t = album_tree(); + let garden = t.id_of("Garden"); + assert_eq!(t.kids(None, false)[..2], ["Garden", "Loose"]); + t.drag_to(t.loose, garden, 0.8); + t.drag_to(garden, t.loose, 0.2); + assert!(!t.h.app.session.catalog.album_children_are_ordered(None), "Loose already follows Garden"); + // a real placement orders the albums; the same drop again then changes nothing + t.drag_to(t.loose, garden, 0.2); + assert!(t.h.app.session.catalog.album_children_are_ordered(None)); + let snapshot = t.h.app.session.catalog.to_snapshot(); + t.drag_to(t.loose, garden, 0.2); + assert_eq!(t.h.app.session.catalog.to_snapshot(), snapshot); +} + +/// The bottom of an open folder is inside it; from the bottom edge of a closed one a folder goes +/// after it (last of the folders); a folder over an album does nothing. +#[test] +fn folders_go_after_a_closed_folder_and_inside_an_open_one() { + let mut t = album_tree(); + t.drag_to(t.archive, t.trips, 0.9); + assert_eq!(t.parent(t.archive), Some(t.trips), "the bottom of an open folder is inside it"); + let travel = t.id_of("Travel 2026"); + assert_eq!(t.kids(None, true), ["Travel 2026", "Trips"]); + let r = t.h.request("ui.clickWidget", json!({"id": format!("albumToggle:{}", t.trips)}), T); + assert_eq!(r["ok"], true, "{r}"); + t.h.step(); + t.h.step(); + t.drag_to(travel, t.trips, 0.9); + assert_eq!(t.kids(None, true), ["Trips", "Travel 2026"], "after Trips, the last folder"); + assert_eq!(t.parent(travel), None); + // folders are not placed among albums + t.drag_to(t.trips, t.loose, 0.5); + assert_eq!(t.kids(None, true), ["Trips", "Travel 2026"]); +} + +/// Resting on a closed folder's edge (where the drop goes beside it) does not open it: the rows +/// would shift under the pointer. Only a drop that goes inside opens it. +#[test] +fn hovering_a_folders_edge_does_not_open_it() { + let mut t = album_tree(); + let r = t.h.request("ui.clickWidget", json!({"id": format!("albumToggle:{}", t.trips)}), T); + assert_eq!(r["ok"], true, "{r}"); + t.h.step(); + t.h.step(); + let (from, r) = (t.row(t.archive).center(), t.row(t.trips)); + t.drag(from, &[(egui::pos2(r.left() + 100.0, r.top() + 0.9 * r.height()), 90)]); + assert_eq!(t.parent(t.archive), None, "dropped after Trips, not inside it"); + assert_eq!(t.kids(None, true), ["Travel 2026", "Trips", "Archive"]); +} + +/// Folders are placed among folders by their edges; their middle still means "inside"; an album +/// over a folder is always "inside"; where nothing would change nothing is done or ordered. +#[test] +fn folders_are_placed_by_their_edges_and_entered_by_their_middle() { + let mut t = album_tree(); + assert_eq!(t.kids(None, true), ["Archive", "Travel 2026", "Trips"]); + t.drag_to(t.trips, t.archive, 0.1); + assert_eq!(t.kids(None, true), ["Trips", "Archive", "Travel 2026"], "Trips before Archive"); + assert_eq!(t.parent(t.trips), None); + t.drag_to(t.sub, t.archive, 0.5); + assert_eq!(t.parent(t.sub), Some(t.archive), "the middle of a folder is inside it"); + // an album anywhere over a folder row goes inside, even at its edge + t.drag_to(t.loose, t.archive, 0.05); + assert_eq!(t.parent(t.loose), Some(t.archive)); + // already in place: no change, so no hand order is made + let mut t = album_tree(); + let travel = t.h.app.session.catalog.albums().find(|a| a.name == "Travel 2026").unwrap().id.0; + t.drag_to(travel, t.trips, 0.1); + assert!(!t.h.app.session.catalog.album_children_are_ordered(None), "Travel 2026 already sits right before Trips"); +} + /// Dragging out of a folder: the drop zone under the tree puts the album back at the top level; /// it only shows when there is somewhere to go, and the move is one undo step. #[test] From 922ea25d881c972101f96b784719da53fbfb8fa0 Mon Sep 17 00:00:00 2001 From: Nathaniel Ledford Date: Thu, 8 Oct 2026 20:43:31 -0400 Subject: [PATCH 16/46] Docs: album tree, drag and hand order (parity, what's new) Co-Authored-By: Claude Sonnet 5.5 --- docs/parity.md | 6 +++--- docs/whats-new.md | 13 +++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/docs/parity.md b/docs/parity.md index ca6d974e..0d1c7d10 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -150,8 +150,8 @@ under the UI command that wraps it (a no-op error); the UI command now wins. | LR-LIB-ALLPHOTOS | All photos | P0 | ✅ | `cmd:library.source`, `crates/ui-egui/src/panels/left.rs` | | | LR-LIB-RECENT-ADDED | Recently added | P1 | ✅ | `cmd:library.source` (`recentlyAdded`), `crates/engine/src/view.rs` (`RECENT_DAYS`) | photos imported in the 30 days up to the latest import, newest import first, grouped by import day | | LR-LIB-BYDATE | Browse by date | P1 | ✅ | `cmd:library.filter` (`date`), `crates/ui-egui/src/panels/left.rs` (`date_row`), `crates/catalog/src/query.rs` (`date_groups`) | year → month → day tree with library-wide counts; a row filters the current source by that date. Active filters show as removable chips under the grid header (`crates/ui-egui/src/panels/chips.rs`, with “+N more” and Clear all), the header counts “N of M photos” (matching of the source's total), a badge on the top-bar filter icon counts them, and an empty result says no photos match the active filters | -| LR-LIB-ALBUM | Albums | P0 | ✅ | `cmd:album.create`, `cmd:album.rename`, `cmd:album.delete`, `cmd:album.addPhotos`, `cmd:album.removePhotos`, `cmd:dialog.newAlbum` | no drag photos onto album (LR-BEHAV-DRAGDROP), no album sort | -| LR-LIB-FOLDER | Folders of albums | P0 | ✅ | `cmd:album.create` (`folder`, `parent`), `cmd:album.move`, `cmd:dialog.newAlbum` / `cmd:dialog.newFolder` / `cmd:dialog.smartAlbum` / `cmd:dialog.newSmartAlbum` (`parent`), `crates/ui-egui/src/panels/left.rs` (`albums_tree`, `folder_menu`, `disclosure_triangle`), `crates/ui-egui/src/tests_panels.rs` | folders nest and fold with the same disclosure triangle as Local / By Date / Folders / Keywords (`albumToggle:`; the row folds them too; open by default); right-click a folder ▸ New ▸ Album… / Smart Album… / Smart Album from Filter… / Folder… creates inside it (the `dialog.new*` commands take `parent`), opening the folders down to the new row; creating is only allowed inside folders; moving: drag an album or folder row onto a folder (outlined when allowed; a closed folder opens after 0.6 s under the pointer; a "Top Level" zone under the tree appears while dragging something that sits in a folder), the Move to menu or `album.move` — one undo step each; not onto a plain album, itself or something inside it; no custom order (alphabetical, folders first) | +| LR-LIB-ALBUM | Albums | P0 | ✅ | `cmd:album.create`, `cmd:album.rename`, `cmd:album.delete`, `cmd:album.addPhotos`, `cmd:album.removePhotos`, `cmd:dialog.newAlbum` | photos dragged from the grid onto an album row are added to it (LR-BEHAV-DRAGDROP) | +| LR-LIB-FOLDER | Folders of albums | P0 | ✅ | `cmd:album.create` (`folder`, `parent`), `cmd:album.move`, `cmd:album.reorder`, `cmd:album.sort`, `cmd:dialog.newAlbum` / `cmd:dialog.newFolder` / `cmd:dialog.smartAlbum` / `cmd:dialog.newSmartAlbum` (`parent`), `crates/ui-egui/src/panels/left.rs` (`albums_tree`, `folder_menu`, `disclosure_triangle`), `crates/ui-egui/src/tests_panels.rs` | folders nest and fold with the same disclosure triangle as Local / By Date / Folders / Keywords (`albumToggle:`; the row folds them too; open by default); right-click a folder ▸ New ▸ Create Album… / Create Smart Album… / Create Smart Album from Filter… / Create Folder… creates inside it (the `dialog.new*` commands take `parent`), opening the folders down to the new row; creating is only allowed inside folders; moving: drag an album or folder row onto a folder (outlined when allowed; a closed folder opens after 0.6 s under the pointer; a "Top Level" zone under the tree appears while dragging something that sits in a folder), the Move to menu or `album.move` — one undo step each; not onto a plain album, itself or something inside it; drop on the top / bottom half of an album or the edge of a folder to place it there by hand (`cmd:album.reorder`, a line shows the place; ordered per folder, folders stay first, new albums go after the ordered ones; right-click ▸ Sort Contents A–Z or ＋ ▸ Sort Albums A–Z = `cmd:album.sort`) | | LR-LIB-SMARTALBUM | Smart albums | P1 | ✅ | `cmd:album.createSmart`, `cmd:album.setRules`, `crates/catalog/src/query.rs`, `crates/ui-egui/src/panels/filterbar.rs`, `crates/ui-egui/src/lib.rs` (`album_counts`) | saved filters (rating/flag/label/kind/edited/keyword/camera/lens/date range/text/album), live; match-all only (no any/none rule groups, no rule editor dialog — rules come from the filter bar or `album.setRules`); sidebar counts are cached per catalog revision (and per minute while an “in the last…” rule is used), counted without building id lists | | LR-LIB-SHARED-ALBUM | Shared albums | P2 | ⬜ | | needs a sharing service | | LR-LIB-OFFLINE | Keep album offline | P2 | 🚫 | | not applicable: local-first library | @@ -620,7 +620,7 @@ visible menu bar yet: the menu model is only exposed through the control channel | MENU-HELP-SYSINFO | System info | P2 | ✅ | `cmd:app.systemInfo`, `cmd:library.info` | Help ▸ System Info…: version, OS, CPU threads, GPU, memory budget, preview size, library, timings; Copy to Clipboard; JSON for agents (`open: false`) | | MENU-CTX-GRID | Photo context menu | P0 | ✅ | `crates/ui-egui/src/panels/grid.rs` (`context_menu`) | rate, flag, label, add to / remove from album, rename, virtual copy, version, stack, copy / paste / paste selected, reset, merge, rotate, show in Finder, export / export with preset, set as album cover (in an album), delete (restore / delete permanently in Recently Deleted) | | MENU-CTX-DETAIL | Loupe context menu | P1 | ✅ | `crates/ui-egui/src/panels/detail.rs` | Zoom submenu (fit, 100%, in, out), then the photo menu | -| MENU-CTX-ALBUM | Album / folder row menu | P0 | ✅ | `crates/ui-egui/src/panels/left.rs` (`folder_menu`), `cmd:album.move`, `cmd:dialog.export` | add selected, export album (dialog / preset), New ▸ Album / Smart Album / Smart Album from Filter / Folder (on a folder: made inside it), move to a folder or the top level, rename, delete; smart albums: update rules | +| MENU-CTX-ALBUM | Album / folder row menu | P0 | ✅ | `crates/ui-egui/src/panels/left.rs` (`folder_menu`), `cmd:album.move`, `cmd:dialog.export` | add selected, export album (dialog / preset), New ▸ Create Album / Smart Album / Smart Album from Filter / Folder (on a folder: made inside it), move to a folder or the top level, rename, delete; smart albums: update rules | | MENU-CTX-MASK | Mask / component menu | P0 | ✅ | `crates/ui-egui/src/panels/masking.rs` (`mask_menu`, `component_row_menu`), `cmd:mask.component` | mask rows: duplicate (and invert), invert, show/hide, move, rename, delete; component rows (right-click or "…"): invert, duplicate, mode (add / subtract / intersect), intersect with / subtract a new component, rename (also double-click), delete (the last one deletes the mask) | | MENU-CTX-PRESET | Preset menu | P1 | ✅ | `crates/ui-egui/src/panels/presets.rs` | favourite, update with current settings, rename, move to group, delete; group: export | | MENU-CTX-PROFILE | Profile favourites | P2 | ✅ | `cmd:profile.favorite`, `crates/ui-egui/src/panels/profiles.rs` | star in the profile browser; Favorites group first | diff --git a/docs/whats-new.md b/docs/whats-new.md index f503d56f..5d9a173b 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -2,6 +2,19 @@ ## October 2026 +### Albums tree +- Album folders fold with the same disclosure triangle as Local, By Date, Folders and Keywords. +- Right-click a folder ▸ New ▸ Create Album… / Create Smart Album… / Create Smart Album from Filter… / Create Folder… + makes it inside that folder (`dialog.newAlbum`, `dialog.newFolder`, `dialog.smartAlbum`, `dialog.newSmartAlbum` take `parent`). +- Drag an album or folder onto a folder to move it there (a closed folder opens after 0.6 s under the pointer; a + "Top Level" strip under the tree takes it back out; Esc cancels). Drop on the top or bottom half of another album, + or the edge of another folder, to place it before or after it: that folder is then ordered by hand. Right-click ▸ + Sort Contents A–Z (or ＋ ▸ Sort Albums A–Z at the top level) goes back to by name. To put a folder after an open + folder, fold that folder first (the bottom of an open folder means "inside"). Agents use `album.reorder` + (`id`, `parent?`, `before?`) and `album.sort`. +- The catalog format is now version 3 (`Album.order`, `Op::SetAlbumOrder`). Once this version has opened a library, + older versions refuse it, as with every format change. + ### Library keyboard culling - Colour labels tint thumbnail surrounds in Square Grid and the Detail filmstrip, and a translucent footer along the bottom of the photo in Photo Grid. Label confirmations use a matching pale colour. - Setting or clearing a colour label shows a brief bottom confirmation, like rating a photo; custom label names appear in the message too. From 24b80b4ee3666a98e19bb14d0a99e9e70ad0fdb1 Mon Sep 17 00:00:00 2001 From: Nathaniel Ledford Date: Thu, 8 Oct 2026 20:52:08 -0400 Subject: [PATCH 17/46] Albums: the Albums header is the way back to the top level It replaces the strip under the tree, which could be scrolled out of view and shifted the rows when it appeared; the header is always visible and nothing moves. Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/panels/left.rs | 20 ++++++++++-------- crates/ui-egui/src/tests_panels.rs | 33 ++++++++++++++---------------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/crates/ui-egui/src/panels/left.rs b/crates/ui-egui/src/panels/left.rs index 64b3a98b..98182936 100644 --- a/crates/ui-egui/src/panels/left.rs +++ b/crates/ui-egui/src/panels/left.rs @@ -208,6 +208,7 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui) { ui.add_space(10.0); // Albums header let (ar, albums_open) = sidebar_section_header(app, ui, "albums", "Albums"); + top_level_drop_target(app, ui, ar); // the + stays at the visible edge when the sidebar is scrolled sideways let plus_right = (ar.left() + viewport.max.x).min(ar.right()); let mut hdr = ui.new_child( @@ -255,7 +256,6 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui) { cur = cat.album(p).and_then(|a| a.parent); } albums_tree(app, ui, &kids, None, 0.0, &open_to); - top_level_drop_zone(app, ui); } ui.add_space(10.0); local_section(app, ui); @@ -904,20 +904,22 @@ fn album_drag_over(app: &mut LightcraftApp, ui: &mut egui::Ui, resp: &egui::Resp opened } -/// While an album that sits in a folder is dragged: a row under the tree that takes it back to the -/// top level. -fn top_level_drop_zone(app: &mut LightcraftApp, ui: &mut egui::Ui) { +/// While an album that sits in a folder is dragged: the "Albums" header takes it back to the top +/// level (outlined, with a hint). The header stays in view however long the tree is, and nothing +/// moves when it appears. +fn top_level_drop_target(app: &mut LightcraftApp, ui: &mut egui::Ui, header: Rect) { let Some(dragged) = app.ui.dragging_album.map(AlbumId) else { return }; if !can_drop_album(app, dragged, None) { return; } let t = Tokens::get(ui.ctx()); - let (r, _) = ui.allocate_exact_size(vec2(ui.available_width(), 29.0), Sense::hover()); - register(ui.ctx(), "albumDrop:top", r); - let over = pointer_over(ui, r); + register(ui.ctx(), "albumDrop:top", header); + let over = pointer_over(ui, header); let stroke = if over { egui::Stroke::new(1.5, t.accent) } else { egui::Stroke::new(1.0, t.text_dim.gamma_multiply(0.5)) }; - ui.painter().rect_stroke(r.shrink2(vec2(8.0, 1.0)), 4.0, stroke, egui::StrokeKind::Inside); - ui.painter().text(pos2(r.left() + 18.0, r.center().y), Align2::LEFT_CENTER, crate::i18n::tr("Top Level"), t.font(13.5), t.text_dim); + ui.painter().rect_stroke(header.shrink2(vec2(8.0, 3.0)), 4.0, stroke, egui::StrokeKind::Inside); + // right-aligned, left of the + button + let edge = header.right().min(ui.clip_rect().right()); + ui.painter().text(pos2(edge - 56.0, header.center().y), Align2::RIGHT_CENTER, crate::i18n::tr("Top Level"), t.font(12.5), t.text_dim); if over && ui.input(|i| i.pointer.primary_released()) { drop_album(app, ui, dragged, None); } diff --git a/crates/ui-egui/src/tests_panels.rs b/crates/ui-egui/src/tests_panels.rs index 364fdc2b..98b6da64 100644 --- a/crates/ui-egui/src/tests_panels.rs +++ b/crates/ui-egui/src/tests_panels.rs @@ -383,13 +383,9 @@ impl AlbumTree { let name = ["source:folder:", "source:album:"].iter().map(|p| format!("{p}{id}")).find(|n| h.app.widgets.iter().any(|(w, _)| w == n)); widget(h, &name.unwrap_or_else(|| panic!("no row for album {id}"))) } - /// The middle of the row just under the last row of the Albums tree (where the top-level drop zone is). - fn below_tree(&self) -> egui::Pos2 { - let bottom = (self.h.app.widgets.iter()) - .filter(|(w, r)| (w.starts_with("source:album:") || w.starts_with("source:folder:")) && r.left() < 300.0) - .map(|(_, r)| r.bottom()) - .fold(0.0, f32::max); - egui::pos2(100.0, bottom + 14.5) + /// The middle of the "Albums" header, where an album is dropped to take it to the top level. + fn albums_header(&self) -> egui::Pos2 { + widget(&self.h, "sidebarSection:albums").center() } fn parent(&self, id: u64) -> Option { self.h.app.session.catalog.album(lightcraft_catalog::AlbumId(id)).unwrap().parent.map(|p| p.0) @@ -600,22 +596,23 @@ fn folders_are_placed_by_their_edges_and_entered_by_their_middle() { assert!(!t.h.app.session.catalog.album_children_are_ordered(None), "Travel 2026 already sits right before Trips"); } -/// Dragging out of a folder: the drop zone under the tree puts the album back at the top level; -/// it only shows when there is somewhere to go, and the move is one undo step. +/// Dragging out of a folder: the "Albums" header is the way back to the top level (it stays in +/// view however long the tree is), only offered for an album that sits in a folder, and the move +/// is one undo step. The drop does not fold the section. #[test] -fn dragging_an_album_to_the_top_level() { +fn dragging_an_album_to_the_albums_header_takes_it_to_the_top_level() { let mut t = album_tree(); let has = |t: &AlbumTree, id: &str| t.h.app.widgets.iter().any(|(w, _)| w == id); - assert!(!has(&t, "albumDrop:top"), "no zone unless an album is being dragged"); - // below the last row of the tree, where the zone appears once an album with a folder is dragged - let below = t.below_tree(); - let from = t.row(t.best).center(); - t.drag(from, &[(below, 3)]); + assert!(!has(&t, "albumDrop:top"), "no target unless an album is being dragged"); + let (from, header) = (t.row(t.best).center(), t.albums_header()); + t.drag(from, &[(header, 3)]); assert_eq!(t.parent(t.best), None, "moved out of Trips"); - // a top-level album has nowhere to go: dropping it there does nothing - let (from, below) = (t.row(t.best).center(), t.below_tree()); - t.drag(from, &[(below, 3)]); + assert!(!t.h.app.ui.sidebar_section_collapsed("albums"), "dropping on the header does not fold it"); + // a top-level album has nowhere to go: dropping it on the header does nothing + let (from, header) = (t.row(t.best).center(), t.albums_header()); + t.drag(from, &[(header, 3)]); assert_eq!(t.parent(t.best), None); + assert!(!t.h.app.ui.sidebar_section_collapsed("albums")); t.h.app.session.execute("edit.undo", &json!({})).unwrap(); assert_eq!(t.parent(t.best), Some(t.trips), "one undo puts it back"); } From 5fefe91640ec13ca7dfe65e254fe119ab559dc40 Mon Sep 17 00:00:00 2001 From: Nathaniel Ledford Date: Thu, 8 Oct 2026 20:52:08 -0400 Subject: [PATCH 18/46] Docs: top-level drop is on the Albums header Co-Authored-By: Claude Sonnet 5.5 --- docs/parity.md | 4 ++-- docs/whats-new.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/parity.md b/docs/parity.md index 0d1c7d10..b1e36b87 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -151,7 +151,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins. | LR-LIB-RECENT-ADDED | Recently added | P1 | ✅ | `cmd:library.source` (`recentlyAdded`), `crates/engine/src/view.rs` (`RECENT_DAYS`) | photos imported in the 30 days up to the latest import, newest import first, grouped by import day | | LR-LIB-BYDATE | Browse by date | P1 | ✅ | `cmd:library.filter` (`date`), `crates/ui-egui/src/panels/left.rs` (`date_row`), `crates/catalog/src/query.rs` (`date_groups`) | year → month → day tree with library-wide counts; a row filters the current source by that date. Active filters show as removable chips under the grid header (`crates/ui-egui/src/panels/chips.rs`, with “+N more” and Clear all), the header counts “N of M photos” (matching of the source's total), a badge on the top-bar filter icon counts them, and an empty result says no photos match the active filters | | LR-LIB-ALBUM | Albums | P0 | ✅ | `cmd:album.create`, `cmd:album.rename`, `cmd:album.delete`, `cmd:album.addPhotos`, `cmd:album.removePhotos`, `cmd:dialog.newAlbum` | photos dragged from the grid onto an album row are added to it (LR-BEHAV-DRAGDROP) | -| LR-LIB-FOLDER | Folders of albums | P0 | ✅ | `cmd:album.create` (`folder`, `parent`), `cmd:album.move`, `cmd:album.reorder`, `cmd:album.sort`, `cmd:dialog.newAlbum` / `cmd:dialog.newFolder` / `cmd:dialog.smartAlbum` / `cmd:dialog.newSmartAlbum` (`parent`), `crates/ui-egui/src/panels/left.rs` (`albums_tree`, `folder_menu`, `disclosure_triangle`), `crates/ui-egui/src/tests_panels.rs` | folders nest and fold with the same disclosure triangle as Local / By Date / Folders / Keywords (`albumToggle:`; the row folds them too; open by default); right-click a folder ▸ New ▸ Create Album… / Create Smart Album… / Create Smart Album from Filter… / Create Folder… creates inside it (the `dialog.new*` commands take `parent`), opening the folders down to the new row; creating is only allowed inside folders; moving: drag an album or folder row onto a folder (outlined when allowed; a closed folder opens after 0.6 s under the pointer; a "Top Level" zone under the tree appears while dragging something that sits in a folder), the Move to menu or `album.move` — one undo step each; not onto a plain album, itself or something inside it; drop on the top / bottom half of an album or the edge of a folder to place it there by hand (`cmd:album.reorder`, a line shows the place; ordered per folder, folders stay first, new albums go after the ordered ones; right-click ▸ Sort Contents A–Z or ＋ ▸ Sort Albums A–Z = `cmd:album.sort`) | +| LR-LIB-FOLDER | Folders of albums | P0 | ✅ | `cmd:album.create` (`folder`, `parent`), `cmd:album.move`, `cmd:album.reorder`, `cmd:album.sort`, `cmd:dialog.newAlbum` / `cmd:dialog.newFolder` / `cmd:dialog.smartAlbum` / `cmd:dialog.newSmartAlbum` (`parent`), `crates/ui-egui/src/panels/left.rs` (`albums_tree`, `folder_menu`, `disclosure_triangle`), `crates/ui-egui/src/tests_panels.rs` | folders nest and fold with the same disclosure triangle as Local / By Date / Folders / Keywords (`albumToggle:`; the row folds them too; open by default); right-click a folder ▸ New ▸ Create Album… / Create Smart Album… / Create Smart Album from Filter… / Create Folder… creates inside it (the `dialog.new*` commands take `parent`), opening the folders down to the new row; creating is only allowed inside folders; moving: drag an album or folder row onto a folder (outlined when allowed; a closed folder opens after 0.6 s under the pointer; the "Albums" header takes an item that sits in a folder back to the top level while it is dragged), the Move to menu or `album.move` — one undo step each; not onto a plain album, itself or something inside it; drop on the top / bottom half of an album or the edge of a folder to place it there by hand (`cmd:album.reorder`, a line shows the place; ordered per folder, folders stay first, new albums go after the ordered ones; right-click ▸ Sort Contents A–Z or ＋ ▸ Sort Albums A–Z = `cmd:album.sort`) | | LR-LIB-SMARTALBUM | Smart albums | P1 | ✅ | `cmd:album.createSmart`, `cmd:album.setRules`, `crates/catalog/src/query.rs`, `crates/ui-egui/src/panels/filterbar.rs`, `crates/ui-egui/src/lib.rs` (`album_counts`) | saved filters (rating/flag/label/kind/edited/keyword/camera/lens/date range/text/album), live; match-all only (no any/none rule groups, no rule editor dialog — rules come from the filter bar or `album.setRules`); sidebar counts are cached per catalog revision (and per minute while an “in the last…” rule is used), counted without building id lists | | LR-LIB-SHARED-ALBUM | Shared albums | P2 | ⬜ | | needs a sharing service | | LR-LIB-OFFLINE | Keep album offline | P2 | 🚫 | | not applicable: local-first library | @@ -512,7 +512,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins. | LR-BEHAV-WEB-SAFETY | Library safety in the browser build | P1 | 🟡 | `cmd:file.backupLibrary`, `cmd:file.restoreLibrary`, `apps/lightcraft-web/src/safety.rs`, `apps/lightcraft-web/src/backup.rs`, `apps/lightcraft-web/src/files.rs` | web only (experimental): Back Up Library (zip of the catalog and every stored original) and a non-destructive Restore; failed saves show as unsaved and are retried; a photo that can't be stored isn't added; one tab per library (Web Locks); notices for storage that isn't persistent and for a library that can't open; `?reset` asks first; a panic shows a message. Missing: `?safe` start, deleting originals of removed photos, zip64 (backups > 4 GB) | | LR-BEHAV-GPU | GPU acceleration | P0 | ✅ | `crates/gpu/src/render.rs`, `cmd:app.gpu`, `docs/gpu-pipeline.md` | CPU fallback per render on device limits, out of memory, device errors / loss, incomplete or black results (`perf.gpuReason`, `perf.gpuFallback`); short submissions for slow iGPUs; backend choice (DX12 only on Windows, `LIGHTCRAFT_GPU_BACKEND` / `WGPU_BACKEND`), device created after the window opens, crash sentinel (issue #136); no tiling beyond the buffer limit (CPU) | | LR-BEHAV-RENDER-FIDELITY | Rendering matches Lightroom | P1 | 🟡 | `crates/pipeline/src/lib.rs`, `crates/pipeline/src/tone.rs` | every slider exists and works, but the character of the result (tone curve shape, highlight recovery, texture / clarity / dehaze, noise reduction, sharpening, default look) is tuned by eye; there is no systematic side-by-side comparison against Lightroom on the same CC0 raws. Needs a fidelity suite: Lightroom reference renders kept only in the local `plan/` (never committed), compared per slider and preset with a perceptual metric | -| LR-BEHAV-DRAGDROP | Drag and drop | P1 | ✅ | `crates/ui-egui/src/lib.rs`, `crates/ui-egui/src/panels/grid.rs` (`drag_feedback`), `crates/ui-egui/src/panels/left.rs` (`drop_target`, `album_drag_over`, `top_level_drop_zone`) | files → app (import); grid photos → an album row (adds the selection, with a count badge while dragging); sidebar album / folder rows → a folder (moves it; its name follows the pointer) | +| LR-BEHAV-DRAGDROP | Drag and drop | P1 | ✅ | `crates/ui-egui/src/lib.rs`, `crates/ui-egui/src/panels/grid.rs` (`drag_feedback`), `crates/ui-egui/src/panels/left.rs` (`drop_target`, `album_drag_over`, `top_level_drop_target`) | files → app (import); grid photos → an album row (adds the selection, with a count badge while dragging); sidebar album / folder rows → a folder (moves it; its name follows the pointer) | | LR-BEHAV-TOAST | Toast notifications | P1 | ✅ | `crates/ui-egui/src/panels/mod.rs` | | | LR-BEHAV-SIDEBAR-COLLAPSE | Collapsible left-sidebar sections | P2 | ✅ | `crates/ui-egui/src/panels/left.rs` (`sidebar_section_header`), `crates/ui-egui/src/state.rs` (`collapsed_sidebar`), `crates/ui-egui/src/tests_panels.rs` | click the Albums, Local, By Date or Keywords header (or drive `ui.clickWidget` `sidebarSection:`) to fold or unfold the section; a chevron after the title shows the state; the choice is kept across restarts (`collapsedSidebar` in the UI state). UI-only (no command), like the By Date / Keywords disclosure triangles; photos dropped on a folded Albums section have no target | | LR-BEHAV-PANEL-RESIZE | Resizable side panels | P1 | ✅ | `crates/ui-egui/src/panels/mod.rs` (`resizable_side`), `crates/ui-egui/src/state.rs` (`LEFT_WIDTH`, `RIGHT_WIDTH`), `crates/ui-egui/src/tests_panels.rs` | drag the left sidebar's right edge (200–480 pt) or the right panel's left edge (250–520 pt); the photo area keeps ≥ 360 pt; widths are kept across panels, views and restarts (`leftWidth` / `rightWidth` in the UI state); the Presets column stays fixed | diff --git a/docs/whats-new.md b/docs/whats-new.md index 5d9a173b..3004617a 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -7,7 +7,7 @@ - Right-click a folder ▸ New ▸ Create Album… / Create Smart Album… / Create Smart Album from Filter… / Create Folder… makes it inside that folder (`dialog.newAlbum`, `dialog.newFolder`, `dialog.smartAlbum`, `dialog.newSmartAlbum` take `parent`). - Drag an album or folder onto a folder to move it there (a closed folder opens after 0.6 s under the pointer; a - "Top Level" strip under the tree takes it back out; Esc cancels). Drop on the top or bottom half of another album, + "Albums" header takes it back to the top level; Esc cancels). Drop on the top or bottom half of another album, or the edge of another folder, to place it before or after it: that folder is then ordered by hand. Right-click ▸ Sort Contents A–Z (or ＋ ▸ Sort Albums A–Z at the top level) goes back to by name. To put a folder after an open folder, fold that folder first (the bottom of an open folder means "inside"). Agents use `album.reorder` From 8ff45f5237c6bc83305dc84a23fba0d403504775 Mon Sep 17 00:00:00 2001 From: Nathaniel Ledford Date: Thu, 8 Oct 2026 20:59:12 -0400 Subject: [PATCH 19/46] Sidebar: dragging albums or photos near its top or bottom edge scrolls it Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/panels/left.rs | 31 +++++++++++++ crates/ui-egui/src/tests_panels.rs | 72 ++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/crates/ui-egui/src/panels/left.rs b/crates/ui-egui/src/panels/left.rs index 98182936..8863e5d1 100644 --- a/crates/ui-egui/src/panels/left.rs +++ b/crates/ui-egui/src/panels/left.rs @@ -179,6 +179,7 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui) { let counts = app.caches.counts(&app.session.catalog); let (total, picks, deleted) = (counts.total, counts.picks, counts.deleted); egui::ScrollArea::both().id_salt("left-scroll").auto_shrink([false, false]).show_viewport(ui, |ui, viewport| { + drag_auto_scroll(app, ui); // rows are as wide as the widest one needs (last frame), at least the panel let wide = viewport.width().max(content_width(ui.ctx())); ui.set_min_width(wide); @@ -751,6 +752,36 @@ fn pointer_over(ui: &egui::Ui, rect: Rect) -> bool { ui.input(|i| i.pointer.latest_pos()).is_some_and(|p| rect.contains(p) && ui.clip_rect().contains(p)) } +/// How close to the top or bottom edge of the sidebar (points) a dragged album starts scrolling it. +const SCROLL_EDGE: f32 = 36.0; +/// The fastest the sidebar scrolls for a drag (points per second). +const SCROLL_MAX_SPEED: f32 = 700.0; + +/// How fast the sidebar scrolls (points per second, positive = down) for a dragged item with the +/// pointer at `y`, the visible part being `top..bottom`: none in the middle, faster the nearer to +/// the edge, the top speed at and past it. +pub fn auto_scroll_speed(y: f32, top: f32, bottom: f32) -> f32 { + let depth = |dist: f32| ((SCROLL_EDGE - dist) / SCROLL_EDGE).clamp(0.0, 1.0); + SCROLL_MAX_SPEED * (depth(bottom - y) - depth(y - top)) +} + +/// While an album, or photos from the grid, are dragged over the sidebar: scrolls it when the +/// pointer is near its top or bottom edge, so rows beyond the visible part can be reached. Call +/// inside the scroll area. +fn drag_auto_scroll(app: &LightcraftApp, ui: &egui::Ui) { + if app.ui.dragging_album.is_none() && app.ui.dragging_photos.is_none() { + return; + } + let view = ui.clip_rect(); + let Some(p) = ui.input(|i| i.pointer.latest_pos()).filter(|p| p.x >= view.left() && p.x <= view.right()) else { return }; + let speed = auto_scroll_speed(p.y, view.top(), view.bottom()); + if speed != 0.0 { + // a positive delta moves the content down: the view goes up + ui.scroll_with_delta_animation(vec2(0.0, -speed * ui.input(|i| i.stable_dt).min(0.1)), egui::style::ScrollAnimation::none()); + ui.ctx().request_repaint(); + } +} + /// How long a dragged album must rest on a closed folder before it opens. const HOVER_OPEN_SECS: f64 = 0.6; diff --git a/crates/ui-egui/src/tests_panels.rs b/crates/ui-egui/src/tests_panels.rs index 98b6da64..66eb360e 100644 --- a/crates/ui-egui/src/tests_panels.rs +++ b/crates/ui-egui/src/tests_panels.rs @@ -596,6 +596,78 @@ fn folders_are_placed_by_their_edges_and_entered_by_their_middle() { assert!(!t.h.app.session.catalog.album_children_are_ordered(None), "Travel 2026 already sits right before Trips"); } +/// Holding a dragged album near the top or bottom edge of the sidebar scrolls it that way; in the +/// middle, or beside the sidebar, nothing scrolls. (Each drag ends with Esc: nothing is dropped.) +#[test] +fn a_dragged_album_scrolls_the_sidebar_at_its_edges() { + let mut t = album_tree(); + for i in 0..60 { + t.h.app.session.execute("album.create", &json!({"name": format!("Z{i:02}")})).unwrap(); + } + t.h.step(); + t.h.step(); + let last = t.id_of("Z59"); + let panel = widget(&t.h, "panel:left_panel"); + let (cx, top, bottom) = (panel.center().x, panel.top(), panel.bottom()); + let start = t.row(last).top(); + assert!(start > bottom, "the last album starts below the visible sidebar: {start} vs {bottom}"); + // a row in the middle of the visible sidebar: where a drag starts + let pick = |t: &AlbumTree| { + let rows = t.h.app.widgets.iter().filter(|(w, r)| w.starts_with("source:album:") && r.top() > top + 120.0 && r.bottom() < bottom - 120.0); + rows.map(|(_, r)| r.center()).next().expect("a visible album row") + }; + let esc = |pressed| egui::Event::Key { key: egui::Key::Escape, physical_key: None, pressed, repeat: false, modifiers: Default::default() }; + let hold = |t: &mut AlbumTree, to: egui::Pos2| { + let from = pick(t); + t.drag_with(from, &[(to, 90)], vec![esc(true), esc(false)]); + }; + hold(&mut t, egui::pos2(cx, panel.center().y)); + assert_eq!(t.row(last).top(), start, "no scrolling in the middle"); + hold(&mut t, egui::pos2(panel.right() + 200.0, bottom - 6.0)); + assert_eq!(t.row(last).top(), start, "no scrolling beside the sidebar"); + hold(&mut t, egui::pos2(cx, bottom - 6.0)); + let scrolled = t.row(last).top(); + assert!(scrolled < start - 100.0, "the bottom edge scrolls down: {start} -> {scrolled}"); + hold(&mut t, egui::pos2(cx, top + 70.0)); + let back = t.row(last).top(); + assert!(back > scrolled + 100.0, "the top edge scrolls back up: {scrolled} -> {back}"); + assert!(t.h.app.ui.dragging_album.is_none()); +} + +/// Photos dragged from the grid scroll the sidebar the same way, to reach an album further down. +#[test] +fn dragged_photos_scroll_the_sidebar_at_its_edges() { + let mut t = album_tree(); + for i in 0..60 { + t.h.app.session.execute("album.create", &json!({"name": format!("Z{i:02}")})).unwrap(); + } + t.h.step(); + t.h.step(); + let (last, panel) = (t.id_of("Z59"), widget(&t.h, "panel:left_panel")); + let start = t.row(last).top(); + let first = t.h.app.session.visible_cloned()[0].0; + let thumb = widget(&t.h, &format!("thumb:{first}")).center(); + t.drag(thumb, &[(egui::pos2(panel.center().x, panel.bottom() - 6.0), 90)]); + assert!(t.row(last).top() < start - 100.0, "scrolled down: {start} -> {}", t.row(last).top()); + assert!(t.h.app.ui.dragging_photos.is_none(), "the drag ended"); +} + +/// How fast the sidebar scrolls for a pointer at `y` (+ down) is zero in the middle, ramps up +/// within the edge band and stays at the top speed past the edge. +#[test] +fn auto_scroll_speed_ramps_up_at_the_edges() { + use crate::panels::left::auto_scroll_speed; + let (top, bottom) = (100.0, 700.0); + assert_eq!(auto_scroll_speed(400.0, top, bottom), 0.0); + assert_eq!(auto_scroll_speed(top + 60.0, top, bottom), 0.0, "outside the band"); + let (slow, fast, past) = + (auto_scroll_speed(bottom - 30.0, top, bottom), auto_scroll_speed(bottom - 5.0, top, bottom), auto_scroll_speed(bottom + 80.0, top, bottom)); + assert!(0.0 < slow && slow < fast && fast <= past, "{slow} {fast} {past}"); + assert_eq!(past, auto_scroll_speed(bottom + 400.0, top, bottom), "capped"); + let up = auto_scroll_speed(top + 5.0, top, bottom); + assert!(up < 0.0 && (up + fast).abs() < 1e-3, "the same ramp upwards: {up} vs {fast}"); +} + /// Dragging out of a folder: the "Albums" header is the way back to the top level (it stays in /// view however long the tree is), only offered for an album that sits in a folder, and the move /// is one undo step. The drop does not fold the section. From 733846b326c714f385aedd9c2305e12dfd8e7c6a Mon Sep 17 00:00:00 2001 From: Nathaniel Ledford Date: Thu, 8 Oct 2026 20:59:12 -0400 Subject: [PATCH 20/46] Docs: sidebar auto-scroll while dragging Co-Authored-By: Claude Sonnet 5.5 --- docs/parity.md | 2 +- docs/whats-new.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/parity.md b/docs/parity.md index b1e36b87..f8e1e317 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -512,7 +512,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins. | LR-BEHAV-WEB-SAFETY | Library safety in the browser build | P1 | 🟡 | `cmd:file.backupLibrary`, `cmd:file.restoreLibrary`, `apps/lightcraft-web/src/safety.rs`, `apps/lightcraft-web/src/backup.rs`, `apps/lightcraft-web/src/files.rs` | web only (experimental): Back Up Library (zip of the catalog and every stored original) and a non-destructive Restore; failed saves show as unsaved and are retried; a photo that can't be stored isn't added; one tab per library (Web Locks); notices for storage that isn't persistent and for a library that can't open; `?reset` asks first; a panic shows a message. Missing: `?safe` start, deleting originals of removed photos, zip64 (backups > 4 GB) | | LR-BEHAV-GPU | GPU acceleration | P0 | ✅ | `crates/gpu/src/render.rs`, `cmd:app.gpu`, `docs/gpu-pipeline.md` | CPU fallback per render on device limits, out of memory, device errors / loss, incomplete or black results (`perf.gpuReason`, `perf.gpuFallback`); short submissions for slow iGPUs; backend choice (DX12 only on Windows, `LIGHTCRAFT_GPU_BACKEND` / `WGPU_BACKEND`), device created after the window opens, crash sentinel (issue #136); no tiling beyond the buffer limit (CPU) | | LR-BEHAV-RENDER-FIDELITY | Rendering matches Lightroom | P1 | 🟡 | `crates/pipeline/src/lib.rs`, `crates/pipeline/src/tone.rs` | every slider exists and works, but the character of the result (tone curve shape, highlight recovery, texture / clarity / dehaze, noise reduction, sharpening, default look) is tuned by eye; there is no systematic side-by-side comparison against Lightroom on the same CC0 raws. Needs a fidelity suite: Lightroom reference renders kept only in the local `plan/` (never committed), compared per slider and preset with a perceptual metric | -| LR-BEHAV-DRAGDROP | Drag and drop | P1 | ✅ | `crates/ui-egui/src/lib.rs`, `crates/ui-egui/src/panels/grid.rs` (`drag_feedback`), `crates/ui-egui/src/panels/left.rs` (`drop_target`, `album_drag_over`, `top_level_drop_target`) | files → app (import); grid photos → an album row (adds the selection, with a count badge while dragging); sidebar album / folder rows → a folder (moves it; its name follows the pointer) | +| LR-BEHAV-DRAGDROP | Drag and drop | P1 | ✅ | `crates/ui-egui/src/lib.rs`, `crates/ui-egui/src/panels/grid.rs` (`drag_feedback`), `crates/ui-egui/src/panels/left.rs` (`drop_target`, `album_drag_over`, `top_level_drop_target`, `drag_auto_scroll`) | files → app (import); grid photos → an album row (adds the selection, with a count badge while dragging); sidebar album / folder rows → a folder (moves it; its name follows the pointer); a drag of albums or grid photos near the top or bottom edge of the sidebar scrolls it (faster nearer the edge, only while the pointer is over the sidebar) | | LR-BEHAV-TOAST | Toast notifications | P1 | ✅ | `crates/ui-egui/src/panels/mod.rs` | | | LR-BEHAV-SIDEBAR-COLLAPSE | Collapsible left-sidebar sections | P2 | ✅ | `crates/ui-egui/src/panels/left.rs` (`sidebar_section_header`), `crates/ui-egui/src/state.rs` (`collapsed_sidebar`), `crates/ui-egui/src/tests_panels.rs` | click the Albums, Local, By Date or Keywords header (or drive `ui.clickWidget` `sidebarSection:`) to fold or unfold the section; a chevron after the title shows the state; the choice is kept across restarts (`collapsedSidebar` in the UI state). UI-only (no command), like the By Date / Keywords disclosure triangles; photos dropped on a folded Albums section have no target | | LR-BEHAV-PANEL-RESIZE | Resizable side panels | P1 | ✅ | `crates/ui-egui/src/panels/mod.rs` (`resizable_side`), `crates/ui-egui/src/state.rs` (`LEFT_WIDTH`, `RIGHT_WIDTH`), `crates/ui-egui/src/tests_panels.rs` | drag the left sidebar's right edge (200–480 pt) or the right panel's left edge (250–520 pt); the photo area keeps ≥ 360 pt; widths are kept across panels, views and restarts (`leftWidth` / `rightWidth` in the UI state); the Presets column stays fixed | diff --git a/docs/whats-new.md b/docs/whats-new.md index 3004617a..c82eb1af 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -7,7 +7,7 @@ - Right-click a folder ▸ New ▸ Create Album… / Create Smart Album… / Create Smart Album from Filter… / Create Folder… makes it inside that folder (`dialog.newAlbum`, `dialog.newFolder`, `dialog.smartAlbum`, `dialog.newSmartAlbum` take `parent`). - Drag an album or folder onto a folder to move it there (a closed folder opens after 0.6 s under the pointer; a - "Albums" header takes it back to the top level; Esc cancels). Drop on the top or bottom half of another album, + "Albums" header takes it back to the top level; Esc cancels; holding a dragged album or photos near the top or bottom edge of the sidebar scrolls it). Drop on the top or bottom half of another album, or the edge of another folder, to place it before or after it: that folder is then ordered by hand. Right-click ▸ Sort Contents A–Z (or ＋ ▸ Sort Albums A–Z at the top level) goes back to by name. To put a folder after an open folder, fold that folder first (the bottom of an open folder means "inside"). Agents use `album.reorder` From 7935999d944905a472547412b4ce65398df1fe2b Mon Sep 17 00:00:00 2001 From: Daniel Voshart <205798369+voshart@users.noreply.github.com> Date: Thu, 8 Oct 2026 16:45:34 -0400 Subject: [PATCH 21/46] LR-EDIT-DETAIL-DENOISE: non-destructive AI RAW denoise with pure-Rust CPU/GPU Recover the unpublished denoise work against current upstream. Preserve later opt-in model setup, shared Rust HTTPS downloads, pending-photo resume and an Amount-independent switch. Keep original RAWs unchanged and share the current color finish between ordinary and denoised sources. Bound public tiling inputs, refuse corrupt-cache exports, and retain synthetic/real-model coverage. Include two Windows path spelling fixes in existing tests. No model weights, face code, media or Tract runtime are included. Validation: cargo xtask ci (all seven gates); installed RawNIND CPU/GPU agreement 1.38e-6; native CPU preview/export on a public 24 MP Sony RAW, unchanged original SHA-256. Co-authored-by: Codex Co-authored-by: Claude Sonnet 5.5 Generated with Codex. --- Cargo.lock | 315 ++++- Cargo.toml | 2 + ROADMAP.md | 10 +- apps/lightcraft-cli/Cargo.toml | 2 + apps/lightcraft-cli/src/main.rs | 10 +- apps/lightcraft/Cargo.toml | 4 + apps/lightcraft/src/main.rs | 6 +- crates/denoise/Cargo.toml | 28 + crates/denoise/README.md | 14 + crates/denoise/src/archive.rs | 271 ++++ crates/denoise/src/bayer.rs | 191 +++ crates/denoise/src/cpu.rs | 386 ++++++ crates/denoise/src/cpu/tests.rs | 151 +++ crates/denoise/src/hash.rs | 46 + crates/denoise/src/known.rs | 112 ++ crates/denoise/src/lib.rs | 42 + crates/denoise/src/licence.rs | 34 + crates/denoise/src/manifest.rs | 217 +++ crates/denoise/src/net.rs | 345 +++++ crates/denoise/src/onnx.rs | 369 +++++ crates/denoise/src/onnx_proto.rs | 295 ++++ crates/denoise/src/product.rs | 595 +++++++++ crates/denoise/src/reference.rs | 133 ++ crates/denoise/src/run.rs | 582 ++++++++ crates/denoise/src/runtime.rs | 423 ++++++ crates/denoise/src/synthetic.rs | 234 ++++ crates/denoise/src/synthetic_proto.rs | 44 + crates/denoise/src/tiles.rs | 100 ++ crates/denoise/tests/real_model.rs | 248 ++++ crates/develop/src/controls.rs | 1 + crates/develop/src/lib.rs | 44 + crates/develop/src/settings.rs | 8 + crates/engine/Cargo.toml | 5 + crates/engine/src/cmd/denoise.rs | 543 ++++++++ crates/engine/src/cmd/mod.rs | 2 + crates/engine/src/config.rs | 23 + crates/engine/src/denoise.rs | 1629 +++++++++++++++++++++++ crates/engine/src/files.rs | 168 ++- crates/engine/src/lib.rs | 15 + crates/engine/src/library.rs | 4 + crates/engine/src/media.rs | 244 +++- crates/engine/src/model_download.rs | 363 +++++ crates/engine/src/smart.rs | 2 +- crates/engine/src/tests_denoise.rs | 699 ++++++++++ crates/engine/src/tests_xmp.rs | 9 +- crates/engine/tests/denoise_cpu.rs | 101 ++ crates/engine/tests/denoise_real.rs | 117 ++ crates/gpu/Cargo.toml | 1 + crates/gpu/src/lib.rs | 12 + crates/gpu/src/nn.rs | 1048 +++++++++++++++ crates/gpu/src/wgsl/nn_conv.wgsl | 162 +++ crates/gpu/src/wgsl/nn_pool.wgsl | 28 + crates/mcp/src/headless.rs | 4 +- crates/raw/src/lib.rs | 60 +- crates/ui-egui/Cargo.toml | 1 + crates/ui-egui/src/headless.rs | 56 + crates/ui-egui/src/lib.rs | 16 + crates/ui-egui/src/menus.rs | 1 + crates/ui-egui/src/model_setup.rs | 133 ++ crates/ui-egui/src/panels/denoise.rs | 647 +++++++++ crates/ui-egui/src/panels/dialogs.rs | 11 +- crates/ui-egui/src/panels/edit.rs | 10 +- crates/ui-egui/src/panels/mod.rs | 1 + crates/ui-egui/src/panels/settings.rs | 14 +- crates/ui-egui/src/state.rs | 4 + crates/ui-egui/src/tests_model_setup.rs | 171 +++ docs/denoise.md | 118 ++ docs/parity.md | 4 +- docs/whats-new.md | 4 + xtask/src/layers.rs | 1 + 70 files changed, 11557 insertions(+), 136 deletions(-) create mode 100644 crates/denoise/Cargo.toml create mode 100644 crates/denoise/README.md create mode 100644 crates/denoise/src/archive.rs create mode 100644 crates/denoise/src/bayer.rs create mode 100644 crates/denoise/src/cpu.rs create mode 100644 crates/denoise/src/cpu/tests.rs create mode 100644 crates/denoise/src/hash.rs create mode 100644 crates/denoise/src/known.rs create mode 100644 crates/denoise/src/lib.rs create mode 100644 crates/denoise/src/licence.rs create mode 100644 crates/denoise/src/manifest.rs create mode 100644 crates/denoise/src/net.rs create mode 100644 crates/denoise/src/onnx.rs create mode 100644 crates/denoise/src/onnx_proto.rs create mode 100644 crates/denoise/src/product.rs create mode 100644 crates/denoise/src/reference.rs create mode 100644 crates/denoise/src/run.rs create mode 100644 crates/denoise/src/runtime.rs create mode 100644 crates/denoise/src/synthetic.rs create mode 100644 crates/denoise/src/synthetic_proto.rs create mode 100644 crates/denoise/src/tiles.rs create mode 100644 crates/denoise/tests/real_model.rs create mode 100644 crates/engine/src/cmd/denoise.rs create mode 100644 crates/engine/src/config.rs create mode 100644 crates/engine/src/denoise.rs create mode 100644 crates/engine/src/model_download.rs create mode 100644 crates/engine/src/tests_denoise.rs create mode 100644 crates/engine/tests/denoise_cpu.rs create mode 100644 crates/engine/tests/denoise_real.rs create mode 100644 crates/gpu/src/nn.rs create mode 100644 crates/gpu/src/wgsl/nn_conv.wgsl create mode 100644 crates/gpu/src/wgsl/nn_pool.wgsl create mode 100644 crates/ui-egui/src/model_setup.rs create mode 100644 crates/ui-egui/src/panels/denoise.rs create mode 100644 crates/ui-egui/src/tests_model_setup.rs create mode 100644 docs/denoise.md diff --git a/Cargo.lock b/Cargo.lock index 86d4bca6..6de52a46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -209,7 +209,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" dependencies = [ - "equator", + "equator 0.4.2", ] [[package]] @@ -967,6 +967,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "coe-rs" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8f1e641542c07631228b1e0dc04b69ae3c1d58ef65d5691a439711d805c698" + [[package]] name = "color" version = "0.3.3" @@ -1256,6 +1262,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76" +[[package]] +name = "dyn-stack" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e53799688f5632f364f8fb387488dd05db9fe45db7011be066fc20e7027f8b" +dependencies = [ + "bytemuck", + "reborrow", +] + [[package]] name = "dyn-stack" version = "0.13.2" @@ -1537,13 +1553,33 @@ version = "0.36.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773fa9c96dd0dbef887e39d0ed6177f141cce4d68e6041df77570aa3702dfa13" +[[package]] +name = "equator" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3b0a88aa91d0ad2b9684e4479aed31a17d3f9051bdbbc634bd2c01bc5a5eee8" +dependencies = [ + "equator-macro 0.1.9", +] + [[package]] name = "equator" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" dependencies = [ - "equator-macro", + "equator-macro 0.4.2", +] + +[[package]] +name = "equator-macro" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60d08acb9849f7fb4401564f251be5a526829183a3645a90197dea8e786cf3ae" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -1608,6 +1644,42 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "faer-core" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da0f6262445261106407bdfb548df4f8f1c89bcc3cff1bb6d08e7f525b575a" +dependencies = [ + "bytemuck", + "coe-rs", + "dyn-stack 0.10.0", + "equator 0.1.10", + "faer-entity", + "gemm 0.17.1", + "matrixcompare-core", + "num-complex", + "num-traits", + "paste", + "pulp 0.18.22", + "reborrow", + "seq-macro", +] + +[[package]] +name = "faer-entity" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fb12f36da3162b3fefc0c7343d905d5bd2c2bf444a7f63ceeaa2bc003311d3" +dependencies = [ + "bytemuck", + "coe-rs", + "libm", + "num-complex", + "num-traits", + "pulp 0.18.22", + "reborrow", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -1783,13 +1855,33 @@ dependencies = [ "slab", ] +[[package]] +name = "gemm" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ab24cc62135b40090e31a76a9b2766a501979f3070fa27f689c27ec04377d32" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-c32 0.17.1", + "gemm-c64 0.17.1", + "gemm-common 0.17.1", + "gemm-f16 0.17.1", + "gemm-f32 0.17.1", + "gemm-f64 0.17.1", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", + "seq-macro", +] + [[package]] name = "gemm" version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab96b703d31950f1aeddded248bc95543c9efc7ac9c4a21fda8703a83ee35451" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-c32 0.18.2", "gemm-c64 0.18.2", "gemm-common 0.18.2", @@ -1799,7 +1891,7 @@ dependencies = [ "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", "seq-macro", ] @@ -1809,7 +1901,7 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa0673db364b12263d103b68337a68fbecc541d6f6b61ba72fe438654709eacb" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-c32 0.19.0", "gemm-c64 0.19.0", "gemm-common 0.19.0", @@ -1819,7 +1911,22 @@ dependencies = [ "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", + "seq-macro", +] + +[[package]] +name = "gemm-c32" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c030d0b983d1e34a546b86e08f600c11696fde16199f971cd46c12e67512c0" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common 0.17.1", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", "seq-macro", ] @@ -1829,12 +1936,12 @@ version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6db9fd9f40421d00eea9dd0770045a5603b8d684654816637732463f4073847" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-common 0.18.2", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", "seq-macro", ] @@ -1844,12 +1951,27 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "086936dbdcb99e37aad81d320f98f670e53c1e55a98bee70573e83f95beb128c" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-common 0.19.0", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", + "seq-macro", +] + +[[package]] +name = "gemm-c64" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbb5f2e79fefb9693d18e1066a557b4546cd334b226beadc68b11a8f9431852a" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common 0.17.1", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", "seq-macro", ] @@ -1859,12 +1981,12 @@ version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfcad8a3d35a43758330b635d02edad980c1e143dc2f21e6fd25f9e4eada8edf" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-common 0.18.2", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", "seq-macro", ] @@ -1874,15 +1996,34 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20c8aeeeec425959bda4d9827664029ba1501a90a0d1e6228e48bef741db3a3f" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-common 0.19.0", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", "seq-macro", ] +[[package]] +name = "gemm-common" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2e7ea062c987abcd8db95db917b4ffb4ecdfd0668471d8dc54734fdff2354e8" +dependencies = [ + "bytemuck", + "dyn-stack 0.10.0", + "half", + "num-complex", + "num-traits", + "once_cell", + "paste", + "pulp 0.18.22", + "raw-cpuid 10.7.0", + "seq-macro", + "sysctl 0.5.5", +] + [[package]] name = "gemm-common" version = "0.18.2" @@ -1890,7 +2031,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a352d4a69cbe938b9e2a9cb7a3a63b7e72f9349174a2752a558a8a563510d0f3" dependencies = [ "bytemuck", - "dyn-stack", + "dyn-stack 0.13.2", "half", "libm", "num-complex", @@ -1898,10 +2039,10 @@ dependencies = [ "once_cell", "paste", "pulp 0.21.5", - "raw-cpuid", + "raw-cpuid 11.6.0", "rayon", "seq-macro", - "sysctl", + "sysctl 0.6.0", ] [[package]] @@ -1911,7 +2052,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88027625910cc9b1085aaaa1c4bc46bb3a36aad323452b33c25b5e4e7c8e2a3e" dependencies = [ "bytemuck", - "dyn-stack", + "dyn-stack 0.13.2", "half", "libm", "num-complex", @@ -1919,10 +2060,27 @@ dependencies = [ "once_cell", "paste", "pulp 0.22.3", - "raw-cpuid", + "raw-cpuid 11.6.0", "rayon", "seq-macro", - "sysctl", + "sysctl 0.6.0", +] + +[[package]] +name = "gemm-f16" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ca4c06b9b11952071d317604acb332e924e817bd891bec8dfb494168c7cedd4" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common 0.17.1", + "gemm-f32 0.17.1", + "half", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", + "seq-macro", ] [[package]] @@ -1931,14 +2089,14 @@ version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cff95ae3259432f3c3410eaa919033cd03791d81cebd18018393dc147952e109" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-common 0.18.2", "gemm-f32 0.18.2", "half", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", "rayon", "seq-macro", ] @@ -1949,30 +2107,45 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3df7a55202e6cd6739d82ae3399c8e0c7e1402859b30e4cb780e61525d9486e" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-common 0.19.0", "gemm-f32 0.19.0", "half", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", "rayon", "seq-macro", ] +[[package]] +name = "gemm-f32" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9a69f51aaefbd9cf12d18faf273d3e982d9d711f60775645ed5c8047b4ae113" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common 0.17.1", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", + "seq-macro", +] + [[package]] name = "gemm-f32" version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc8d3d4385393304f407392f754cd2dc4b315d05063f62cf09f47b58de276864" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-common 0.18.2", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", "seq-macro", ] @@ -1982,12 +2155,27 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02e0b8c9da1fbec6e3e3ab2ce6bc259ef18eb5f6f0d3e4edf54b75f9fd41a81c" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-common 0.19.0", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", + "seq-macro", +] + +[[package]] +name = "gemm-f64" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa397a48544fadf0b81ec8741e5c0fba0043008113f71f2034def1935645d2b0" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common 0.17.1", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", "seq-macro", ] @@ -1997,12 +2185,12 @@ version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35b2a4f76ce4b8b16eadc11ccf2e083252d8237c1b589558a49b0183545015bd" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-common 0.18.2", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", "seq-macro", ] @@ -2012,12 +2200,12 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "056131e8f2a521bfab322f804ccd652520c79700d81209e9d9275bbdecaadc6a" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-common 0.19.0", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", "seq-macro", ] @@ -2859,6 +3047,22 @@ dependencies = [ "serde", ] +[[package]] +name = "lightcraft-denoise" +version = "0.4.0" +dependencies = [ + "faer-core", + "lightcraft-raster", + "miniz_oxide 0.8.9", + "proptest", + "rayon", + "serde", + "serde_json", + "sha2", + "thiserror 2.0.21", + "web-time", +] + [[package]] name = "lightcraft-develop" version = "0.4.0" @@ -2881,7 +3085,9 @@ dependencies = [ "lightcraft-catalog", "lightcraft-codecs", "lightcraft-color", + "lightcraft-denoise", "lightcraft-develop", + "lightcraft-fetch", "lightcraft-geom", "lightcraft-gpu", "lightcraft-merge", @@ -2928,6 +3134,7 @@ version = "0.4.0" dependencies = [ "bytemuck", "lightcraft-color", + "lightcraft-denoise", "lightcraft-develop", "lightcraft-geom", "lightcraft-pipeline", @@ -3082,6 +3289,7 @@ dependencies = [ "lightcraft-catalog", "lightcraft-codecs", "lightcraft-color", + "lightcraft-denoise", "lightcraft-develop", "lightcraft-engine", "lightcraft-geom", @@ -3174,6 +3382,12 @@ dependencies = [ "libc", ] +[[package]] +name = "matrixcompare-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0bdabb30db18805d5290b3da7ceaccbddba795620b86c02145d688e04900a73" + [[package]] name = "maybe-rayon" version = "0.1.1" @@ -4291,6 +4505,18 @@ dependencies = [ "unarray", ] +[[package]] +name = "pulp" +version = "0.18.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0a01a0dc67cf4558d279f0c25b0962bd08fc6dec0137699eae304103e882fe6" +dependencies = [ + "bytemuck", + "libm", + "num-complex", + "reborrow", +] + [[package]] name = "pulp" version = "0.21.5" @@ -4317,7 +4543,7 @@ dependencies = [ "num-complex", "paste", "pulp-wasm-simd-flag", - "raw-cpuid", + "raw-cpuid 11.6.0", "reborrow", "version_check", ] @@ -4518,6 +4744,15 @@ dependencies = [ "rgb", ] +[[package]] +name = "raw-cpuid" +version = "10.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "raw-cpuid" version = "11.6.0" @@ -5245,6 +5480,20 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "sysctl" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea" +dependencies = [ + "bitflags 2.13.2", + "byteorder", + "enum-as-inner", + "libc", + "thiserror 1.0.69", + "walkdir", +] + [[package]] name = "sysctl" version = "0.6.0" diff --git a/Cargo.toml b/Cargo.toml index 77b20666..b23fae21 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ repository = "https://github.com/storytold/lightcraft" [workspace.dependencies] # internal +lightcraft-denoise = { path = "crates/denoise" } lightcraft-geom = { path = "crates/geom" } lightcraft-sysmem = { path = "crates/sysmem" } lightcraft-color = { path = "crates/color" } @@ -34,6 +35,7 @@ lightcraft-segment = { path = "crates/segment" } # foundation thiserror = "2" +sha2 = "0.10" serde = { version = "1", features = ["derive", "rc"] } serde_json = { version = "1", features = ["float_roundtrip"] } log = "0.4" diff --git a/ROADMAP.md b/ROADMAP.md index 5e793e8b..18d4dae1 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -25,7 +25,7 @@ duplicate Local entries #22, black GPU exports on an Intel iGPU #78). | **Feature checklist** | 79.7% | P0 core and P1 nearly complete: import (Add / Copy / Move, templates, devices), library, grid/loupe/compare/survey, every Edit slider, curves, colour grading, masking tools, crop/Upright, heal/clone, presets/profiles, versions/history, sync, export, menus, shortcuts | P0: full-resolution zoom detail; P1: lens-profile database, content-aware fill (patch synthesis), video playback/trim | | **RAW coverage** (formats people shoot) | ~55% | DNG (all kinds), CR2, CR3 lossless Bayer and version 0x100/0x200 C-RAW (M50/R100/R8 pixel-exact corpus), ARW, NEF (uncompressed + Huffman lossless/lossy), RAF (uncompressed, lossless/lossy compressed Bayer and X-Trans), uncompressed ORF, RW2 / RWL / Panasonic RAW (every raw format, checked on 178 files from 118 Panasonic and Leica bodies), PEF; every container's embedded preview (incl. CR3) | **CR3** (every Canon since ~2018), compressed ORF, Nikon lossy-after-split, Canon sRAW, HEIC/AVIF. NEFs labelled compressed but stored uncompressed (Z 6 packed 14-bit, D850 12-bit uncompressed). Per-model verification is thin (~55 corpus files vs >1,000 models) | | **Colour & image quality** | ~55–65% | Pipeline is complete and fast; GPU path CPU-exact within 1/255 | **No measured camera calibration database**: ARW, NEF, RW2, RAF, CR3, CR2 and PEF have guarded camera-JPEG colour estimates, with bundled ILCE-7M4, X-H2S and X-T4 profiles (docs/camera-preview-colour.md); other non-DNG raws and rejected estimates use a neutral matrix. Colour fidelity remains incomplete. No lens-profile database. No measured fidelity against Lightroom (tone, highlights, texture/clarity, NR, sharpening are tuned by eye) | -| **AI & computational** | ~15–20% | Assisted culling (focus, bursts), auto tone, HDR/panorama merge; subject/sky/background masks as classical heuristics | Real segmentation masks (subject, sky, people, objects, landscape, depth), AI denoise, super resolution, lens blur, generative remove, faces/people, natural-language search. **Blocked on a model strategy** (licensable weights or our own training; pure-Rust inference is feasible) | +| **AI & computational** | ~15–20% | Assisted culling (focus, bursts), auto tone, HDR/panorama merge; subject/sky/background masks as classical heuristics; non-destructive Bayer RAW denoise with opt-in weights on pure-Rust CPU/GPU | Real segmentation masks (subject, sky, people, objects, landscape, depth), denoise model policy / linear RGB / quality verification, super resolution, lens blur, generative remove, faces/people, natural-language search. **Blocked on a model strategy** (licensable weights or our own training; pure-Rust inference is feasible) | | **Workflow & library** | ~85% (single machine) | Robust catalog (journal + snapshots, background compaction, crash-tested), 85k-photo libraries stay responsive, Local browsing with automatic cleanup, XMP interop, keywords, smart albums, Move import | Opening an 85k library takes 1.7–4.7 s; no cloud sync (out of scope), no tablet companion (#74, roadmap), shared albums, publish services, tethering | | **Classic modules** | ~30% | Geotagging from GPX track logs, soft proofing (partial), slideshow (basic) | **Map view, Book, Print, Slideshow module, Web, publish services**: ~40 tracker rows ⬜ | | **HDR & video** | 0% | | HDR edit/display/export; video play/trim/edit/export | @@ -40,7 +40,7 @@ duplicate Local entries #22, black GPU exports on an Intel iGPU #78). | Fujifilm raw shooter | ~65% | Uncompressed and lossless/lossy compressed RAF decoded; camera colour calibration remains missing. Verified on 13 bodies, including 14/16-bit GFX; see `docs/raf-compression.md` | | Canon CR3 / Olympus compressed-raw shooter | ~45% | CR3 lossless Bayer and version 0x100/0x200 C-RAW develop from sensor data (M50, R100, R8 verified pixel for pixel); other CRX variants and compressed ORF open as embedded JPEG previews. Camera colour and broader model verification remain | | Lightroom Classic power user | ~45% | Print, Book, Map, publish, tethering | -| Relies on AI (masks, denoise) | ~25% | Object / Describe masks via optional SAM 3 (download not yet hosted); no AI denoise | +| Relies on AI (masks, denoise) | ~25% | Object / Describe masks via optional SAM 3 (download not yet hosted); Bayer AI denoise needs separately installed weights; X-Trans/linear RGB, model policy and quality parity remain open | ## Where we're going @@ -59,9 +59,9 @@ Priorities, in order. Each points at tracker rows in [`docs/parity.md`](docs/par (references stay in the local `plan/`), then tune against the numbers. 5. **Lens profiles of our own** (LR-EDIT-OPTICS-PROFILE, P1). 6. **AI model strategy** (maintainer decision): which permissively licensed models (or our own training) for - segmentation masks and denoise; then pure-Rust inference. Unblocks M12 and Enhance. Object / Describe masks now + segmentation masks and denoise. Bayer denoise inference is implemented in pure Rust with an opt-in GPL model; release policy and independently licensed weights remain open. Unblocks M12 and Enhance. Object / Describe masks now run Meta's SAM 3 in pure Rust (optional, user-consented download under the non-OSI SAM License; CDN mirrors still - to be set up, docs/ai-masks.md); Subject / Sky / People and Enhance are still open. + to be set up, docs/ai-masks.md); Subject / Sky / People, Raw Details and Super Resolution are still open. 7. **Then:** HDR (Q), the Classic output modules (Print first, then Map view, Book, Slideshow), video (R), localisation and accessibility. @@ -86,7 +86,7 @@ yet guarantee original-pixel detail; visible-region rendering remains a gap (LR- | M4 | Crop, geometry, optics | crop tool + overlays, straighten, Upright (auto/level/vertical/full/guided), manual transforms, CA, defringe, manual lens corrections | 6–10 | ✅ | | M5 | Performance | source pyramids, wgpu compute pipeline (CPU oracle), draft/full renders, prefetch, budgets (16 ms slider updates on 24 MP) | 10–15 | 🚧 (stage cache, source pyramid, wgpu pipeline, prefetch, memory budget ✅; colour NR at half resolution, GPU histogram ⬜) | | M6 | Masking | brush, linear/radial gradients, colour/luminance/depth range, add/subtract/intersect/invert, all local adjustments, masks panel | 8–12 | 🚧 (brush/linear/radial/colour/luminance range, add/subtract/intersect, masks panel ✅; depth range, AI masks ⬜) | -| M7 | Detail | sharpening + masking preview, luminance/colour NR, Denoise, Raw Details, Super Resolution | 6–10 | 🚧 (sharpening, luminance/colour NR ✅; AI Denoise, Raw Details, Super Resolution ⬜) | +| M7 | Detail | sharpening + masking preview, luminance/colour NR, Denoise, Raw Details, Super Resolution | 6–10 | 🚧 (sharpening, luminance/colour NR ✅; AI Denoise 🟡 (Bayer, optional weights; policy/fidelity open); Raw Details, Super Resolution ⬜) | | M8 | Heal / Remove | content-aware remove (PatchMatch), heal, clone, brush spots, visualize spots, red/pet eye | 6–10 | 🚧 (heal, clone, auto source, visualize spots, red/pet eye ✅; PatchMatch remove ⬜) | | M9 | Presets, profiles, versions, sync | preset browser + amount, create/import presets, profile browser, versions, history, copy/paste/sync settings | 5–8 | ✅ | | M10 | Export & share | export dialog (JPEG/PNG/TIFF/DNG/AVIF/JXL/original), sizing, sharpening, metadata, watermark, naming, batch jobs, XMP sidecars, HDR export | 6–10 | 🚧 (all formats incl. DNG/original, sizing, presets, background jobs ✅; JXL encode, HDR export ⬜) | diff --git a/apps/lightcraft-cli/Cargo.toml b/apps/lightcraft-cli/Cargo.toml index 3f60fe32..72d46858 100644 --- a/apps/lightcraft-cli/Cargo.toml +++ b/apps/lightcraft-cli/Cargo.toml @@ -25,6 +25,8 @@ serde_json = { workspace = true } dhat = { workspace = true, optional = true } [features] +default = ["denoise"] +denoise = ["lightcraft-engine/denoise"] # Heap profiling: `cargo run --release -p lightcraft-cli --features dhat-heap -- snapshot …` counts # every allocation (live/peak bytes in `library.memory` / `ui.inspect`) and writes dhat-heap.json # (open it in DHAT's viewer: allocation sites by bytes at the peak). Slows allocation-heavy work. diff --git a/apps/lightcraft-cli/src/main.rs b/apps/lightcraft-cli/src/main.rs index 6c5e6e44..c6dfdc2a 100644 --- a/apps/lightcraft-cli/src/main.rs +++ b/apps/lightcraft-cli/src/main.rs @@ -391,7 +391,7 @@ fn merge(args: &[String]) -> Result<(), String> { } i += 1; } - let mut s = Session::new().with_fs(); + let mut s = Session::new().with_fs().with_default_denoise_models(); let paths = expand_paths(&files); let r = s.execute("library.import", &json!({"paths": paths})).map_err(|e| e.to_string())?; let mut ids: Vec = r["imported"].as_array().map(|a| a.iter().filter_map(Value::as_u64).collect()).unwrap_or_default(); @@ -652,7 +652,7 @@ fn render(args: &[String]) -> Result<(), String> { } let input = input.ok_or("render: missing input file")?; let output = output.ok_or("render: missing -o OUTPUT")?; - let mut s = Session::new().with_fs(); + let mut s = Session::new().with_fs().with_default_denoise_models(); let abs = expand_paths(std::slice::from_ref(&input)); let r = s.execute("library.import", &json!({"paths": abs})).map_err(|e| e.to_string())?; let id = r["imported"][0].as_u64().ok_or_else(|| format!("{input}: not a readable photo"))?; @@ -734,12 +734,12 @@ fn snapshot(args: &[String]) -> Result<(), String> { let t0 = Instant::now(); let mut session = match &library { Some(dir) => { - let mut s = Session::new().with_fs(); + let mut s = Session::new().with_fs().with_default_denoise_models(); s.open_library(dir, false).map_err(|e| library_error(dir, e))?; s } - None if files.is_empty() => Session::with_demo().with_fs(), - None => Session::new().with_fs(), + None if files.is_empty() => Session::with_demo().with_fs().with_default_denoise_models(), + None => Session::new().with_fs().with_default_denoise_models(), }; if !files.is_empty() { let ti = Instant::now(); diff --git a/apps/lightcraft/Cargo.toml b/apps/lightcraft/Cargo.toml index a29299da..3f3f40ba 100644 --- a/apps/lightcraft/Cargo.toml +++ b/apps/lightcraft/Cargo.toml @@ -9,6 +9,10 @@ default-run = "lightcraft" [lints] workspace = true +[features] +default = ["denoise"] +denoise = ["lightcraft-engine/denoise"] + [dependencies] lightcraft-engine = { workspace = true, features = ["sam"] } lightcraft-sysmem = { workspace = true } diff --git a/apps/lightcraft/src/main.rs b/apps/lightcraft/src/main.rs index 4c7bb424..44f0548d 100644 --- a/apps/lightcraft/src/main.rs +++ b/apps/lightcraft/src/main.rs @@ -453,15 +453,15 @@ fn services(ctx: egui::Context, log_file: Option<&std::path::Path>) -> Services /// and offers Try Again / Choose Another Library… / Continue Without Saving / Quit (issue #100). fn open_session(in_memory: bool, dir: Option, seed_demo: bool) -> (Session, Option) { if in_memory { - return (if seed_demo { Session::with_demo() } else { Session::new() }.with_fs().with_system_clock(), None); + return (if seed_demo { Session::with_demo() } else { Session::new() }.with_fs().with_default_denoise_models().with_system_clock(), None); } - let unopened = || Session::new().with_fs().with_system_clock(); + let unopened = || Session::new().with_fs().with_default_denoise_models().with_system_clock(); let Some(dir) = dir else { log::warn!("no library location (set --library or LIGHTCRAFT_LIBRARY)"); return (unopened(), Some(LibraryProblem::new("", "There is no home folder to keep the library in. Choose a folder for it."))); }; let t0 = std::time::Instant::now(); - let mut s = Session::new().with_fs().with_system_clock(); + let mut s = Session::new().with_fs().with_default_denoise_models().with_system_clock(); match s.open_library(&dir, seed_demo) { Ok(r) => { let (replayed, torn) = (r.replayed, r.torn_bytes); diff --git a/crates/denoise/Cargo.toml b/crates/denoise/Cargo.toml new file mode 100644 index 00000000..55c44a14 --- /dev/null +++ b/crates/denoise/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "lightcraft-denoise" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "LightCraft AI denoise (L1): denoise-model manifests, Bayer packing, tiling and blending, the cache file, and pure-Rust CPU inference" + +[lints] +workspace = true + +[features] +default = [] + +[dependencies] +# Safe views over Rust-intrinsic gemm kernels. Pin the core-only release: newer backends +# generate assembly. No BLAS, cc/build scripts, nightly/AMX, or inner thread pool. +faer = { package = "faer-core", version = "=0.17.1", default-features = false, features = ["std"] } +lightcraft-raster = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +thiserror = { workspace = true } +miniz_oxide = { workspace = true } +rayon = { workspace = true } +web-time = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/crates/denoise/README.md b/crates/denoise/README.md new file mode 100644 index 00000000..cb69e382 --- /dev/null +++ b/crates/denoise/README.md @@ -0,0 +1,14 @@ +# lightcraft-denoise + +L1 pure-Rust AI RAW denoise primitives and CPU inference. No weights are bundled. +See [user workflow and limitations](../../docs/denoise.md). + +`manifest` and `known` validate opt-in models; `onnx` reads a bounded operator subset +into `net`; `cpu` executes it and `reference` supplies scalar checks. `bayer`, `tiles` +and `run` pack sensor data and blend overlapping tiles while preserving clipped +highlights. `product` reads/writes the disposable `.lcdn` cache. `runtime` supplies +the installation self-test. Synthetic networks are generated in Rust for tests. + +Run `cargo test -p lightcraft-denoise`. Real-model tests are ignored by default and +require an explicitly supplied local model. The GPU twin is `lightcraft-gpu::nn`; +model management, background jobs and export integration live in `lightcraft-engine`. diff --git a/crates/denoise/src/archive.rs b/crates/denoise/src/archive.rs new file mode 100644 index 00000000..ddaf7084 --- /dev/null +++ b/crates/denoise/src/archive.rs @@ -0,0 +1,271 @@ +//! Taking one file out of a `.zip` (a `.dtmodel` is one), the little of the format a model download needs: stored and +//! deflated entries, found by name, checked against their CRC-32 and a size limit, written next to where they are +//! going and moved into place. It is a stranger's file, so every length and offset in it is checked, a name is only +//! ever compared (never used as a path), and zip64, encrypted and multi-disk archives are refused. + +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::Path; + +/// How far from the end the end-of-central-directory record can start (a 22-byte record and a 64 KiB comment). +const EOCD_SEARCH: u64 = 22 + 65535; +/// Largest central directory we read. +const MAX_DIRECTORY: u64 = 1 << 20; + +#[derive(Debug, PartialEq, thiserror::Error)] +pub enum ArchiveError { + #[error("the archive could not be read: {0}")] + Io(String), + #[error("not a usable archive: {0}")] + Format(String), + #[error("the archive has no file called `{0}`")] + Missing(String), +} + +impl From for ArchiveError { + fn from(e: std::io::Error) -> Self { + ArchiveError::Io(e.to_string()) + } +} + +fn format(why: &str) -> Result { + Err(ArchiveError::Format(why.into())) +} + +fn u16_at(b: &[u8], at: usize) -> Option { + Some(u16::from_le_bytes(b.get(at..at.checked_add(2)?)?.try_into().ok()?)) +} + +fn u32_at(b: &[u8], at: usize) -> Option { + Some(u32::from_le_bytes(b.get(at..at.checked_add(4)?)?.try_into().ok()?)) +} + +/// CRC-32 (IEEE) of `bytes`. +fn crc32(bytes: &[u8]) -> u32 { + let mut table = [0u32; 256]; + for (i, t) in table.iter_mut().enumerate() { + let mut c = i as u32; + for _ in 0..8 { + c = if c & 1 != 0 { 0xedb8_8320 ^ (c >> 1) } else { c >> 1 }; + } + *t = c; + } + !bytes.iter().fold(!0u32, |c, &b| table[((c ^ u32::from(b)) & 0xff) as usize] ^ (c >> 8)) +} + +struct Entry { + method: u16, + crc: u32, + compressed: u64, + size: u64, + local_header: u64, +} + +/// Find the entry called `name` (exactly, or as the last part of a longer name: `folder/name`) in the central directory. +fn find(f: &mut (impl Read + Seek), name: &str) -> Result { + let len = f.seek(SeekFrom::End(0))?; + let tail_len = len.min(EOCD_SEARCH); + f.seek(SeekFrom::Start(len - tail_len))?; + let mut tail = vec![0u8; tail_len as usize]; + f.read_exact(&mut tail)?; + let eocd = (0..tail.len().saturating_sub(21)).rev().find(|&i| u32_at(&tail, i) == Some(0x0605_4b50)); + let Some(e) = eocd else { return format("this is not a zip file") }; + let (entries, size, offset) = (u16_at(&tail, e + 10), u32_at(&tail, e + 12), u32_at(&tail, e + 16)); + let (Some(entries), Some(size), Some(offset)) = (entries, size, offset) else { return format("the end of the archive is cut short") }; + if entries == 0xffff || size == u32::MAX || offset == u32::MAX { + return format("zip64 archives are not supported"); + } + if u64::from(size) > MAX_DIRECTORY || u64::from(offset) + u64::from(size) > len { + return format("the list of files is not where it says"); + } + f.seek(SeekFrom::Start(u64::from(offset)))?; + let mut dir = vec![0u8; size as usize]; + f.read_exact(&mut dir)?; + let mut at = 0usize; + for _ in 0..entries { + if u32_at(&dir, at) != Some(0x0201_4b50) { + return format("the list of files is damaged"); + } + let get = |o: usize| at.checked_add(o); + let (flags, method, crc) = (u16_at(&dir, at + 8), u16_at(&dir, at + 10), u32_at(&dir, at + 16)); + let (compressed, uncompressed) = (u32_at(&dir, at + 20), u32_at(&dir, at + 24)); + let (n, x, c) = (u16_at(&dir, at + 28), u16_at(&dir, at + 30), u16_at(&dir, at + 32)); + let local = u32_at(&dir, at + 42); + let (Some(flags), Some(method), Some(crc), Some(compressed), Some(uncompressed), Some(n), Some(x), Some(c), Some(local), Some(start)) = + (flags, method, crc, compressed, uncompressed, n, x, c, local, get(46)) + else { + return format("the list of files is cut short"); + }; + let Some(entry_name) = dir.get(start..start + usize::from(n)) else { return format("a file name is cut short") }; + let matches = entry_name == name.as_bytes() || entry_name.strip_suffix(name.as_bytes()).is_some_and(|p| p.ends_with(b"/")); + if matches { + if flags & 1 != 0 { + return format("encrypted files are not supported"); + } + if compressed == u32::MAX || uncompressed == u32::MAX { + return format("zip64 files are not supported"); + } + return Ok(Entry { method, crc, compressed: u64::from(compressed), size: u64::from(uncompressed), local_header: u64::from(local) }); + } + at = start + usize::from(n) + usize::from(x) + usize::from(c); + } + Err(ArchiveError::Missing(name.into())) +} + +/// Take the file called `name` out of the archive at `archive` and write it to `dest` (through a temporary file next +/// to it); `max_bytes` bounds its size. Returns the number of bytes written. +pub fn extract(archive: &Path, name: &str, dest: &Path, max_bytes: u64) -> Result { + let mut f = std::io::BufReader::new(std::fs::File::open(archive)?); + let e = find(&mut f, name)?; + if e.size > max_bytes || e.compressed > max_bytes { + return format("the file in the archive is larger than allowed"); + } + f.seek(SeekFrom::Start(e.local_header))?; + let mut head = [0u8; 30]; + f.read_exact(&mut head)?; + if u32_at(&head, 0) != Some(0x0403_4b50) { + return format("the file's header is not where it says"); + } + let skip = u64::from(u16_at(&head, 26).unwrap_or(0)) + u64::from(u16_at(&head, 28).unwrap_or(0)); + f.seek_relative(i64::try_from(skip).map_err(|_| ArchiveError::Format("a header is too long".into()))?)?; + let mut packed = vec![0u8; e.compressed as usize]; + f.read_exact(&mut packed)?; + let data = match e.method { + 0 => packed, + 8 => { + miniz_oxide::inflate::decompress_to_vec_with_limit(&packed, max_bytes as usize).map_err(|err| ArchiveError::Format(format!("{err:?}")))? + } + m => return format(&format!("compression method {m} is not supported")), + }; + if data.len() as u64 != e.size || crc32(&data) != e.crc { + return format("the file in the archive is damaged"); + } + let tmp = dest.with_extension("part"); + let written = (|| -> std::io::Result<()> { + let mut out = std::fs::File::create(&tmp)?; + out.write_all(&data)?; + out.flush()?; + drop(out); + std::fs::rename(&tmp, dest) + })(); + if let Err(err) = written { + let _ = std::fs::remove_file(&tmp); + return Err(err.into()); + } + Ok(data.len() as u64) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A zip with the given (name, bytes, deflate) entries, written by hand. + pub(crate) fn make_zip(files: &[(&str, &[u8], bool)]) -> Vec { + let mut out = Vec::new(); + let mut dir = Vec::new(); + for (name, data, deflate) in files { + let packed = if *deflate { miniz_oxide::deflate::compress_to_vec(data, 6) } else { data.to_vec() }; + let offset = out.len() as u32; + let method: u16 = if *deflate { 8 } else { 0 }; + out.extend_from_slice(&0x0403_4b50u32.to_le_bytes()); + out.extend_from_slice(&[20, 0, 0, 0]); + out.extend_from_slice(&method.to_le_bytes()); + out.extend_from_slice(&[0, 0, 0, 0]); + out.extend_from_slice(&crc32(data).to_le_bytes()); + out.extend_from_slice(&(packed.len() as u32).to_le_bytes()); + out.extend_from_slice(&(data.len() as u32).to_le_bytes()); + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&[0, 0]); + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(&packed); + dir.extend_from_slice(&0x0201_4b50u32.to_le_bytes()); + dir.extend_from_slice(&[20, 0, 20, 0, 0, 0]); + dir.extend_from_slice(&method.to_le_bytes()); + dir.extend_from_slice(&[0, 0, 0, 0]); + dir.extend_from_slice(&crc32(data).to_le_bytes()); + dir.extend_from_slice(&(packed.len() as u32).to_le_bytes()); + dir.extend_from_slice(&(data.len() as u32).to_le_bytes()); + dir.extend_from_slice(&(name.len() as u16).to_le_bytes()); + dir.extend_from_slice(&[0; 12]); + dir.extend_from_slice(&offset.to_le_bytes()); + dir.extend_from_slice(name.as_bytes()); + } + let cd_offset = out.len() as u32; + out.extend_from_slice(&dir); + out.extend_from_slice(&0x0605_4b50u32.to_le_bytes()); + out.extend_from_slice(&[0, 0, 0, 0]); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(dir.len() as u32).to_le_bytes()); + out.extend_from_slice(&cd_offset.to_le_bytes()); + out.extend_from_slice(&[0, 0]); + out + } + + fn temp(name: &str) -> std::path::PathBuf { + std::env::temp_dir().join(format!("lc-denoise-zip-{name}-{}", std::process::id())) + } + + #[test] + fn a_stored_and_a_deflated_file_come_out_whole() { + let big: Vec = (0..200_000u32).map(|i| (i * 7 % 251) as u8).collect(); + let zip = make_zip(&[("m/config.json", b"{}", false), ("m/model.onnx", &big, true), ("m/other.onnx", b"zz", true)]); + let (a, out) = (temp("a.zip"), temp("a.onnx")); + std::fs::write(&a, &zip).unwrap(); + assert_eq!(extract(&a, "model.onnx", &out, 1 << 20).unwrap(), big.len() as u64); + assert_eq!(std::fs::read(&out).unwrap(), big); + assert_eq!(extract(&a, "m/config.json", &out, 1 << 20).unwrap(), 2); + assert_eq!(std::fs::read(&out).unwrap(), b"{}"); + // a name must match whole path parts: `del.onnx` is not `model.onnx` + assert!(matches!(extract(&a, "del.onnx", &out, 1 << 20), Err(ArchiveError::Missing(_)))); + assert!(matches!(extract(&a, "nothing", &out, 1 << 20), Err(ArchiveError::Missing(_)))); + // the size limit applies + assert!(matches!(extract(&a, "model.onnx", &out, 1000), Err(ArchiveError::Format(_)))); + for p in [a, out] { + let _ = std::fs::remove_file(p); + } + } + + /// Opt-in: `LC_DENOISE_DTMODEL= cargo test -p lightcraft-denoise -- --ignored archive` + #[test] + #[ignore = "needs the real download: set LC_DENOISE_DTMODEL"] + fn the_real_download_gives_the_model_it_should() { + let Some(path) = std::env::var_os("LC_DENOISE_DTMODEL") else { return }; + let known = crate::known::find("rawnind-bayer").unwrap(); + let (d, entry) = (known.download.unwrap(), known.entry.unwrap()); + assert_eq!(crate::hash::sha256_file(Path::new(&path)).unwrap(), d.sha256); + let out = temp("real.onnx"); + let n = extract(Path::new(&path), entry, &out, crate::manifest::MAX_MODEL_BYTES).unwrap(); + assert_eq!(Some(n), known.manifest.size_bytes); + assert_eq!(Some(crate::hash::sha256_file(&out).unwrap()), known.manifest.sha256); + let _ = std::fs::remove_file(out); + } + + #[test] + fn damaged_and_hostile_archives_are_errors_not_panics() { + let zip = make_zip(&[("m/model.onnx", &vec![9u8; 5000], true)]); + let (a, out) = (temp("b.zip"), temp("b.onnx")); + // cut anywhere + for n in (0..zip.len()).step_by(3) { + std::fs::write(&a, &zip[..n]).unwrap(); + assert!(extract(&a, "model.onnx", &out, 1 << 20).is_err(), "cut at {n}"); + } + // one byte changed anywhere: an error, or (when it falls in an unused field) the right file, never a panic + for at in 0..zip.len() { + let mut bad = zip.clone(); + bad[at] ^= 0x5a; + std::fs::write(&a, &bad).unwrap(); + if extract(&a, "model.onnx", &out, 1 << 20).is_ok() { + assert_eq!(std::fs::read(&out).unwrap(), vec![9u8; 5000], "a changed byte at {at} gave another file"); + } + } + // not a zip at all, and an empty file + for junk in [Vec::new(), b"PK".to_vec(), vec![0u8; 100_000]] { + std::fs::write(&a, junk).unwrap(); + assert!(extract(&a, "model.onnx", &out, 1 << 20).is_err()); + } + assert!(extract(&temp("missing.zip"), "model.onnx", &out, 1 << 20).is_err()); + for p in [a, out] { + let _ = std::fs::remove_file(p); + } + } +} diff --git a/crates/denoise/src/bayer.rs b/crates/denoise/src/bayer.rs new file mode 100644 index 00000000..a4a00d94 --- /dev/null +++ b/crates/denoise/src/bayer.rs @@ -0,0 +1,191 @@ +//! Bayer geometry: which way a mosaic is laid out, and how a patch of it is packed into the four planes +//! (red, green on the red row, green on the blue row, blue) a denoise model takes. +//! +//! Every Bayer layout is the same 2 × 2 cell shifted by one pixel at most, so a layout is just where its +//! first red sample is. The cell grid starts one pixel before the picture when that red sample is in +//! column or row 1, and the picture is read with its edge mirrored (without repeating the edge sample, so +//! the colour of a mirrored sample is the colour that belongs at that place). + +/// Where the first red sample of a Bayer mosaic sits, `(dx, dy)` in `{0, 1}²`: RGGB is `(0, 0)`, GRBG `(1, 0)`, +/// GBRG `(0, 1)`, BGGR `(1, 1)`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Layout { + pub dx: usize, + pub dy: usize, +} + +impl Layout { + /// The layout of a 2 × 2 colour pattern, row-major, colours 0 = red, 1 = green, 2 = blue; `None` when + /// it is not a Bayer cell (one red, one blue, the greens on the other diagonal). + pub fn from_cell(cell: [u8; 4]) -> Option { + let at = |c: u8| cell.iter().position(|&v| v == c); + let red = at(0)?; + let blue = at(2)?; + let greens = cell.iter().filter(|&&v| v == 1).count(); + // red and blue sit on one diagonal: indices 0+3 or 1+2 + if greens != 2 || red + blue != 3 { + return None; + } + Some(Layout { dx: red % 2, dy: red / 2 }) + } + + /// The cell grid's origin: `-1` along an axis whose first red sample is at 1, so that cell 0 starts on a red column or row. + fn origin(self) -> (isize, isize) { + (isize::try_from(self.dx).unwrap_or(isize::MAX).saturating_neg(), isize::try_from(self.dy).unwrap_or(isize::MAX).saturating_neg()) + } + + /// Cells across and down covering a `width × height` mosaic. + pub fn cells(self, width: usize, height: usize) -> (usize, usize) { + let cells = |n: usize, phase: usize| n / 2 + (n % 2 + phase.min(1)).div_ceil(2); + (cells(width, self.dx), cells(height, self.dy)) + } + + /// The mosaic position of cell `(cx, cy)`'s top-left sample (may lie outside the picture). + pub fn position(self, cx: isize, cy: isize) -> (isize, isize) { + let (ox, oy) = self.origin(); + (ox.saturating_add(cx.saturating_mul(2)), oy.saturating_add(cy.saturating_mul(2))) + } +} + +/// `i` mirrored into `0..n` (ping-pong, edge not repeated), so `-1` is `1` and `n` is `n - 2`. `n == 0` gives 0. +pub fn reflect(i: isize, n: usize) -> usize { + if n <= 1 { + return 0; + } + if let Some(period) = isize::try_from(n).ok().and_then(|n| (n - 1).checked_mul(2)) { + let m = i.rem_euclid(period); + return (if m < n as isize { m } else { period - m }) as usize; + } + let period = 2 * (n as i128 - 1); + let m = (i as i128).rem_euclid(period); + // The reflected value is in 0..n, so this conversion always fits usize. + (if m < n as i128 { m } else { period - m }) as usize +} + +/// Pack the `tile × tile` cells whose top-left cell is `(cx0, cy0)` into `out` as four planes `[R, G1, G2, B]` +/// (`4 · tile²` values, NCHW). G1 is the green beside the red sample, G2 the green under it. Samples outside +/// the picture are mirrored. Returns `false` (and writes nothing) when the sizes do not fit. +pub fn pack_tile(mosaic: &[f32], width: usize, height: usize, layout: Layout, cx0: isize, cy0: isize, tile: usize, out: &mut [f32]) -> bool { + let Some(plane) = tile.checked_mul(tile) else { return false }; + let Some(values) = plane.checked_mul(4) else { return false }; + if width == 0 + || height == 0 + || tile == 0 + || layout.dx > 1 + || layout.dy > 1 + || width.checked_mul(height) != Some(mosaic.len()) + || out.len() != values + { + return false; + } + let Ok(last) = isize::try_from(tile - 1) else { return false }; + let (ox, oy) = layout.origin(); + for (start, origin) in [(cx0, ox), (cy0, oy)] { + if start.checked_mul(2).and_then(|v| v.checked_add(origin)).is_none() + || start.checked_add(last).and_then(|v| v.checked_mul(2)).and_then(|v| v.checked_add(origin)).and_then(|v| v.checked_add(1)).is_none() + { + return false; + } + } + // the four samples of a cell, as (dx, dy) from its top-left sample + const SUB: [(isize, isize); 4] = [(0, 0), (1, 0), (0, 1), (1, 1)]; + for (p, &(sx, sy)) in SUB.iter().enumerate() { + let Some(dst) = out.get_mut(p * plane..(p + 1) * plane) else { return false }; + for (j, row) in dst.chunks_exact_mut(tile).enumerate() { + let (_, y) = layout.position(cx0, cy0 + j as isize); + let ry = reflect(y + sy, height); + let Some(src) = mosaic.get(ry * width..(ry + 1) * width) else { return false }; + for (i, v) in row.iter_mut().enumerate() { + let (x, _) = layout.position(cx0 + i as isize, cy0); + *v = src.get(reflect(x + sx, width)).copied().unwrap_or(0.0); + } + } + } + true +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hostile_sizes_and_positions_never_panic_or_write_output() { + let layout = Layout::from_cell([0, 1, 1, 2]).unwrap(); + let mut out = [7.0; 4]; + for (w, h, tile, x) in [(usize::MAX, 2, 1, 0), (1, 1, usize::MAX, 0), (1, 1, 0, 0), (1, 1, 1, isize::MAX), (1, 1, 1, isize::MIN)] { + assert!(!pack_tile(&[0.5], w, h, layout, x, 0, tile, &mut out)); + assert_eq!(out, [7.0; 4]); + } + for n in [1, 2, isize::MAX as usize, usize::MAX] { + for i in [isize::MIN, -1, 0, isize::MAX] { + assert!(reflect(i, n) < n); + } + } + assert_eq!(layout.cells(usize::MAX, usize::MAX), (usize::MAX / 2 + 1, usize::MAX / 2 + 1)); + } + + #[test] + fn layouts_from_cells() { + assert_eq!(Layout::from_cell([0, 1, 1, 2]), Some(Layout { dx: 0, dy: 0 })); + assert_eq!(Layout::from_cell([1, 0, 2, 1]), Some(Layout { dx: 1, dy: 0 })); + assert_eq!(Layout::from_cell([1, 2, 0, 1]), Some(Layout { dx: 0, dy: 1 })); + assert_eq!(Layout::from_cell([2, 1, 1, 0]), Some(Layout { dx: 1, dy: 1 })); + // not Bayer: two reds, a missing blue, greens on the wrong diagonal + assert_eq!(Layout::from_cell([0, 0, 1, 2]), None); + assert_eq!(Layout::from_cell([0, 1, 2, 1]), None); + assert_eq!(Layout::from_cell([1, 1, 1, 1]), None); + } + + #[test] + fn reflection_keeps_the_colour_of_a_site() { + assert_eq!(reflect(-1, 10), 1); + assert_eq!(reflect(-2, 10), 2); + assert_eq!(reflect(10, 10), 8); + assert_eq!(reflect(11, 10), 7); + assert_eq!(reflect(25, 10), 7); + assert_eq!(reflect(5, 0), 0); + assert_eq!(reflect(5, 1), 0); + for i in -40..60 { + assert_eq!(reflect(i, 11) % 2, (i.rem_euclid(2)) as usize, "parity at {i}"); + } + } + + #[test] + fn packing_puts_each_colour_in_its_plane_for_every_layout() { + // a mosaic whose sample value says which colour it is: R = 10, G = 20, B = 30 + for cell in [[0u8, 1, 1, 2], [1, 0, 2, 1], [1, 2, 0, 1], [2, 1, 1, 0]] { + let layout = Layout::from_cell(cell).unwrap(); + let (w, h) = (10usize, 8usize); + let mosaic: Vec = (0..w * h).map(|i| [10.0, 20.0, 30.0][cell[(i / w % 2) * 2 + i % w % 2] as usize]).collect(); + let (nx, ny) = layout.cells(w, h); + let tile = nx.max(ny); + let mut out = vec![0.0; 4 * tile * tile]; + assert!(pack_tile(&mosaic, w, h, layout, 0, 0, tile, &mut out)); + let plane = tile * tile; + for (p, want) in [10.0, 20.0, 20.0, 30.0].into_iter().enumerate() { + assert!(out[p * plane..(p + 1) * plane].iter().all(|&v| v == want), "layout {cell:?} plane {p}"); + } + } + } + + #[test] + fn g1_is_beside_the_red_sample_and_g2_under_it() { + // RGGB with a distinct value at every site + let (w, h) = (4, 4); + let mosaic: Vec = (0..16).map(|i| i as f32).collect(); + let mut out = vec![0.0; 4 * 4]; + assert!(pack_tile(&mosaic, w, h, Layout { dx: 0, dy: 0 }, 0, 0, 2, &mut out)); + // cell (0,0): R = site (0,0)=0, G1 = site (1,0)=1, G2 = site (0,1)=4, B = site (1,1)=5 + assert_eq!([out[0], out[4], out[8], out[12]], [0.0, 1.0, 4.0, 5.0]); + // cell (1,0): sites (2,0)=2, (3,0)=3, (2,1)=6, (3,1)=7 + assert_eq!([out[1], out[5], out[9], out[13]], [2.0, 3.0, 6.0, 7.0]); + } + + #[test] + fn packing_rejects_wrong_sizes() { + let mut out = vec![0.0; 16]; + assert!(!pack_tile(&[0.0; 15], 4, 4, Layout { dx: 0, dy: 0 }, 0, 0, 2, &mut out)); + assert!(!pack_tile(&[0.0; 16], 4, 4, Layout { dx: 0, dy: 0 }, 0, 0, 3, &mut out)); + assert!(!pack_tile(&[], 0, 0, Layout { dx: 0, dy: 0 }, 0, 0, 2, &mut out)); + } +} diff --git a/crates/denoise/src/cpu.rs b/crates/denoise/src/cpu.rs new file mode 100644 index 00000000..a856fc25 --- /dev/null +++ b/crates/denoise/src/cpu.rs @@ -0,0 +1,386 @@ +//! Single-thread CPU execution of the checked denoise network. +//! +//! Convolutions lower at most 256 spatial positions at a time, then use faer's safe views over pure-Rust GEMM. +//! No inner thread pool: `run::denoise_bayer` already runs tiles on the application's Rayon pool. Tensor lifetimes +//! are planned once; buffers and im2col storage are recycled between calls, independently for concurrent callers. + +use crate::net::{Net, NetError, Op}; +use crate::run::{Error, TileRunner}; +use faer::{Parallelism, mat}; +use std::sync::{Arc, Mutex}; + +const BLOCK: usize = 256; +/// Cap all tensor buffers and convolution scratch for one tile together (512 MiB). +const MAX_VALUES: usize = 128 * 1024 * 1024; +const MAX_TILE: usize = 2048; +const MAX_IDLE: usize = 16; + +type Result = std::result::Result; +fn bad(why: &str) -> NetError { + NetError::Invalid(why.into()) +} +fn count(c: usize, side: usize) -> Result { + c.checked_mul(side) + .and_then(|n| n.checked_mul(side)) + .filter(|&n| n > 0 && n <= MAX_VALUES) + .ok_or_else(|| bad("a tensor has no size or is too large")) +} +fn zeros(n: usize) -> Result> { + let mut v = Vec::new(); + v.try_reserve_exact(n).map_err(|_| bad("not enough memory for a denoise tile"))?; + v.resize(n, 0.0); + Ok(v) +} +fn sources(op: &Op) -> (usize, Option) { + match op { + Op::Conv(c) => (c.src, c.src2), + Op::ConvT2(c) => (c.src, None), + Op::MaxPool2 { src } | Op::DepthToSpace2 { src } => (*src, None), + } +} +#[derive(Clone, Copy)] +struct Tensor { + slot: usize, + side: usize, + channels: usize, +} +struct Plan { + net: Net, + tensors: Vec, + sizes: Vec, + lower: usize, + product: usize, + up_weights: Vec>>, +} +struct Workspace { + buffers: Vec>, + lower: Vec, + product: Vec, +} + +/// A checked Net on square CHW tiles. Clones share immutable weights and the idle workspace cache. +#[derive(Clone)] +pub struct NetRunner { + plan: Arc, + idle: Arc>>, +} + +impl NetRunner { + pub fn new(net: &Net, tile: usize) -> Result { + if tile == 0 + || tile > MAX_TILE + || !net.fits_tile(tile) + || net.in_channels != 4 + || net.out_channels() != 3 + || net.shape(0).is_none_or(|s| s.channels != net.in_channels) + { + return Err(bad("the tile or Bayer-to-RGB contract does not fit the network")); + } + let mut last = vec![0; net.ops().len() + 1]; + for (i, op) in net.ops().iter().enumerate() { + let (a, b) = sources(op); + for id in std::iter::once(a).chain(b) { + *last.get_mut(id).ok_or_else(|| bad("missing tensor"))? = i + 1; + } + } + let mut tensors = vec![Tensor { slot: 0, side: tile, channels: net.in_channels }]; + let mut sizes = vec![count(net.in_channels, tile)?]; + let mut busy = vec![true]; + let (mut lower, mut product) = (0, 0); + let mut up_weights = Vec::new(); + for (i, op) in net.ops().iter().enumerate() { + let (src, src2) = sources(op); + let a = *tensors.get(src).ok_or_else(|| bad("missing tensor"))?; + let (channels, side) = match op { + Op::Conv(c) => { + if let Some(s) = src2 + && tensors.get(s).is_none_or(|b| b.side != a.side) + { + return Err(bad("a join has different sizes")); + } + (c.cout, a.side) + } + Op::ConvT2(c) => (c.cout, a.side.checked_mul(2).ok_or_else(|| bad("size overflow"))?), + Op::MaxPool2 { .. } => { + if a.side < 2 || !a.side.is_multiple_of(2) { + return Err(bad("pooling does not divide the tile")); + } + (a.channels, a.side / 2) + } + Op::DepthToSpace2 { .. } => (a.channels / 4, a.side.checked_mul(2).ok_or_else(|| bad("size overflow"))?), + }; + let n = count(channels, side)?; + let slot = busy + .iter() + .zip(&sizes) + .enumerate() + .filter(|(_, (used, _))| !**used) + .min_by_key(|(_, (_, size))| (n.saturating_sub(**size), **size)) + .map_or(busy.len(), |(i, _)| i); + if slot == sizes.len() { + sizes.push(n); + busy.push(true); + } else { + let sz = sizes.get_mut(slot).ok_or_else(|| bad("missing buffer"))?; + *sz = (*sz).max(n); + *busy.get_mut(slot).ok_or_else(|| bad("missing buffer"))? = true; + } + tensors.push(Tensor { slot, side, channels }); + // Release sources only after the output has its own buffer (also handles reading the same tensor twice). + for id in std::iter::once(src).chain(src2) { + if last.get(id) == Some(&(i + 1)) { + let t = tensors.get(id).ok_or_else(|| bad("missing tensor"))?; + *busy.get_mut(t.slot).ok_or_else(|| bad("missing buffer"))? = false; + } + } + let (k, m, positions) = match op { + Op::Conv(c) => (c.cin * c.k * c.k, c.cout, a.side * a.side), + Op::ConvT2(c) => (c.cin, c.cout * 4, a.side * a.side), + _ => (0, 0, 0), + }; + lower = lower.max(k.saturating_mul(BLOCK.min(positions))); + product = product.max(m.saturating_mul(BLOCK.min(positions))); + let packed = if let Op::ConvT2(c) = op { + let mut w = zeros(c.weight.len())?; + // [cin][cout][dy][dx] -> [cout*4][cin], so an up-convolution is one GEMM and a shuffle. + for (ci, row) in c.weight.chunks_exact(c.cout * 4).enumerate() { + for (co, &v) in row.iter().enumerate() { + *w.get_mut(co * c.cin + ci).ok_or_else(|| bad("invalid transposed weights"))? = v; + } + } + Some(w) + } else { + None + }; + up_weights.push(packed); + } + let total = sizes + .iter() + .try_fold(lower.saturating_add(product), |n, &s| n.checked_add(s)) + .filter(|&n| n <= MAX_VALUES) + .ok_or_else(|| bad("the tile needs too much working memory"))?; + let _ = total; + Ok(Self { plan: Arc::new(Plan { net: net.clone(), tensors, sizes, lower, product, up_weights }), idle: Arc::new(Mutex::new(Vec::new())) }) + } + /// Immutable data for the GPU runner, without re-reading the model. + pub fn net(&self) -> &Net { + &self.plan.net + } + pub fn bytes_per_tile(&self) -> usize { + (self.plan.sizes.iter().sum::() + self.plan.lower + self.plan.product) * 4 + } + fn workspace(&self) -> Result { + if let Some(w) = self.idle.lock().unwrap_or_else(std::sync::PoisonError::into_inner).pop() { + return Ok(w); + } + Ok(Workspace { + buffers: self.plan.sizes.iter().map(|&n| zeros(n)).collect::>()?, + lower: zeros(self.plan.lower)?, + product: zeros(self.plan.product)?, + }) + } + fn execute(&self, input: &[f32], w: &mut Workspace) -> Result> { + let first = self.plan.tensors.first().ok_or_else(|| bad("no input"))?; + let want = count(first.channels, first.side)?; + w.buffers.first_mut().and_then(|b| b.get_mut(..want)).ok_or_else(|| bad("missing input buffer"))?.copy_from_slice(input); + let profile = std::env::var_os("LIGHTCRAFT_PROFILE").is_some(); + for (i, op) in self.plan.net.ops().iter().enumerate() { + let started = web_time::Instant::now(); + let out = *self.plan.tensors.get(i + 1).ok_or_else(|| bad("missing output tensor"))?; + let (before, rest) = w.buffers.split_at_mut_checked(out.slot).ok_or_else(|| bad("missing output buffer"))?; + let (dest, after) = rest.split_first_mut().ok_or_else(|| bad("missing output buffer"))?; + let get = |id: usize| -> Result<(&[f32], Tensor)> { + let t = *self.plan.tensors.get(id).ok_or_else(|| bad("missing source tensor"))?; + let data = if t.slot < out.slot { + before.get(t.slot) + } else { + after.get(t.slot.checked_sub(out.slot + 1).ok_or_else(|| bad("aliased buffers"))?) + } + .and_then(|b| b.get(..count(t.channels, t.side).ok()?)) + .ok_or_else(|| bad("missing source buffer"))?; + Ok((data, t)) + }; + let (src, src2) = sources(op); + let (a, shape) = get(src)?; + let b = src2.map(get).transpose()?.map(|v| v.0); + let dest = dest.get_mut(..count(out.channels, out.side)?).ok_or_else(|| bad("short output buffer"))?; + match op { + Op::Conv(c) => conv(c, a, b, shape.side, dest, &mut w.lower, &mut w.product)?, + Op::ConvT2(c) => up( + c, + a, + shape.side, + dest, + self.plan.up_weights.get(i).and_then(Option::as_deref).ok_or_else(|| bad("missing up weights"))?, + &mut w.lower, + &mut w.product, + )?, + Op::MaxPool2 { .. } => pool(a, shape.side, dest)?, + Op::DepthToSpace2 { .. } => shuffle(a, shape.side, dest)?, + } + if dest.iter().any(|v| !v.is_finite()) { + return Err(bad("the model produced non-finite values")); + } + if profile { + eprintln!("[denoise-cpu] layer {i}: {}x{}x{} {:.2} ms", out.channels, out.side, out.side, started.elapsed().as_secs_f64() * 1e3); + } + } + let t = self.plan.tensors.last().ok_or_else(|| bad("no output"))?; + let data = w.buffers.get(t.slot).and_then(|b| b.get(..count(t.channels, t.side).ok()?)).ok_or_else(|| bad("missing result"))?; + let mut out = zeros(data.len())?; + out.copy_from_slice(data); + Ok(out) + } +} +impl TileRunner for NetRunner { + fn run(&self, input: &[f32]) -> std::result::Result, Error> { + let first = self.plan.tensors.first().ok_or_else(|| Error::Runtime("missing input shape".into()))?; + let want = count(first.channels, first.side).map_err(|e| Error::Runtime(e.to_string()))?; + if input.len() != want || input.iter().any(|v| !v.is_finite()) { + return Err(Error::Input("the tile has the wrong size or non-finite values".into())); + } + let mut w = self.workspace().map_err(|e| Error::Runtime(e.to_string()))?; + let result = self.execute(input, &mut w).map_err(|e| Error::Runtime(e.to_string())); + let mut idle = self.idle.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if idle.len() < MAX_IDLE { + idle.push(w); + } + result + } +} +fn activate(x: f32, bias: f32, alpha: Option) -> f32 { + let x = x + bias; + if x < 0.0 { x * alpha.unwrap_or(1.0) } else { x } +} + +/// All views are over exact, checked slices. The matrix library's assertions therefore never see file-derived sizes. +fn multiply(w: &[f32], x: &[f32], y: &mut [f32], m: usize, k: usize, n: usize) -> Result<()> { + if m.checked_mul(k) != Some(w.len()) || k.checked_mul(n) != Some(x.len()) || m.checked_mul(n) != Some(y.len()) || m == 0 || k == 0 || n == 0 { + return Err(bad("invalid matrix dimensions")); + } + faer::mul::matmul( + mat::from_row_major_slice_mut::(y, m, n), + mat::from_row_major_slice::(w, m, k), + mat::from_row_major_slice::(x, k, n), + None, + 1.0, + Parallelism::None, + ); + Ok(()) +} +fn lower(a: &[f32], b: Option<&[f32]>, side: usize, k: usize, start: usize, n: usize, dest: &mut [f32]) -> Result<()> { + let pad = k / 2; + let pixels = side * side; + // Whole row segments are copied, so bounds checks and edge branches stay out of the multiply loop. + for (ci, plane) in a.chunks_exact(pixels).chain(b.into_iter().flat_map(|b| b.chunks_exact(pixels))).enumerate() { + for ky in 0..k { + for kx in 0..k { + let row = (ci * k * k + ky * k + kx) * n; + let out = dest.get_mut(row..row + n).ok_or_else(|| bad("short im2col buffer"))?; + let mut at = 0; + while at < n { + let p = start + at; + let (y, x) = (p / side, p % side); + let len = (side - x).min(n - at); + if let Some(yy) = y.checked_add(ky).and_then(|v| v.checked_sub(pad)).filter(|&v| v < side) { + let lo = x.max(pad.saturating_sub(kx)); + let hi = (x + len).min((side + pad).saturating_sub(kx)); + if lo < hi { + let from = yy * side + lo + kx - pad; + let to = at + lo - x; + out.get_mut(at..to).ok_or_else(|| bad("short im2col padding"))?.fill(0.0); + out.get_mut(to..to + hi - lo) + .ok_or_else(|| bad("short im2col row"))? + .copy_from_slice(plane.get(from..from + hi - lo).ok_or_else(|| bad("short source row"))?); + out.get_mut(to + hi - lo..at + len).ok_or_else(|| bad("short im2col padding"))?.fill(0.0); + } else { + out.get_mut(at..at + len).ok_or_else(|| bad("short im2col row"))?.fill(0.0); + } + } else { + out.get_mut(at..at + len).ok_or_else(|| bad("short im2col row"))?.fill(0.0); + } + at += len; + } + } + } + } + Ok(()) +} +fn conv(c: &crate::net::Conv, a: &[f32], b: Option<&[f32]>, side: usize, dest: &mut [f32], scratch: &mut [f32], product: &mut [f32]) -> Result<()> { + let pixels = side * side; + let k = c.cin * c.k * c.k; + for start in (0..pixels).step_by(BLOCK) { + let n = BLOCK.min(pixels - start); + let x = scratch.get_mut(..k * n).ok_or_else(|| bad("short im2col storage"))?; + lower(a, b, side, c.k, start, n, x)?; + let y = product.get_mut(..c.cout * n).ok_or_else(|| bad("short product storage"))?; + multiply(&c.weight, x, y, c.cout, k, n)?; + for ((row, &bias), out) in y.chunks_exact(n).zip(&c.bias).zip(dest.chunks_exact_mut(pixels)) { + let to = out.get_mut(start..start + n).ok_or_else(|| bad("short convolution output"))?; + for (d, &v) in to.iter_mut().zip(row) { + *d = activate(v, bias, c.leaky); + } + } + } + Ok(()) +} +fn up(c: &crate::net::ConvT2, a: &[f32], side: usize, dest: &mut [f32], weights: &[f32], scratch: &mut [f32], product: &mut [f32]) -> Result<()> { + let pixels = side * side; + let out_side = 2 * side; + for start in (0..pixels).step_by(BLOCK) { + let n = BLOCK.min(pixels - start); + let x = scratch.get_mut(..c.cin * n).ok_or_else(|| bad("short up input storage"))?; + for (to, from) in x.chunks_exact_mut(n).zip(a.chunks_exact(pixels)) { + to.copy_from_slice(from.get(start..start + n).ok_or_else(|| bad("short up input"))?); + } + let y = product.get_mut(..c.cout * 4 * n).ok_or_else(|| bad("short up product storage"))?; + multiply(weights, x, y, c.cout * 4, c.cin, n)?; + for (co, (&bias, out)) in c.bias.iter().zip(dest.chunks_exact_mut(4 * pixels)).enumerate() { + for d in 0..4 { + let row = y.get((4 * co + d) * n..(4 * co + d + 1) * n).ok_or_else(|| bad("short up product"))?; + for (j, &v) in row.iter().enumerate() { + let p = start + j; + let at = (2 * (p / side) + d / 2) * out_side + 2 * (p % side) + d % 2; + *out.get_mut(at).ok_or_else(|| bad("short up output"))? = activate(v, bias, c.leaky); + } + } + } + } + Ok(()) +} +fn pool(a: &[f32], side: usize, dest: &mut [f32]) -> Result<()> { + let s = side / 2; + for (input, out) in a.chunks_exact(side * side).zip(dest.chunks_exact_mut(s * s)) { + for (y, row) in out.chunks_exact_mut(s).enumerate() { + let lo = input.get(2 * y * side..(2 * y + 1) * side).ok_or_else(|| bad("short pool input"))?; + let hi = input.get((2 * y + 1) * side..(2 * y + 2) * side).ok_or_else(|| bad("short pool input"))?; + for ((d, lo), hi) in row.iter_mut().zip(lo.as_chunks::<2>().0).zip(hi.as_chunks::<2>().0) { + let [a, b] = *lo; + let [c, e] = *hi; + *d = a.max(b).max(c).max(e); + } + } + } + Ok(()) +} +fn shuffle(a: &[f32], side: usize, dest: &mut [f32]) -> Result<()> { + let pixels = side * side; + for (input, out) in a.chunks_exact(4 * pixels).zip(dest.chunks_exact_mut(4 * pixels)) { + for d in 0..4 { + let plane = input.get(d * pixels..(d + 1) * pixels).ok_or_else(|| bad("short shuffle input"))?; + for (y, row) in plane.chunks_exact(side).enumerate() { + let at = (2 * y + d / 2) * 2 * side; + let to = out.get_mut(at..at + 2 * side).ok_or_else(|| bad("short shuffle output"))?; + for (to, &v) in to.as_chunks_mut::<2>().0.iter_mut().zip(row) { + if let Some(t) = to.get_mut(d % 2) { + *t = v; + } + } + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/denoise/src/cpu/tests.rs b/crates/denoise/src/cpu/tests.rs new file mode 100644 index 00000000..16a496b5 --- /dev/null +++ b/crates/denoise/src/cpu/tests.rs @@ -0,0 +1,151 @@ +use super::*; +use crate::{ + net::{Conv, ConvT2}, + reference, synthetic, +}; + +fn random(n: usize, seed: u32) -> Vec { + let mut s = seed | 1; + (0..n) + .map(|_| { + s ^= s << 13; + s ^= s >> 17; + s ^= s << 5; + (s >> 8) as f32 / (1u32 << 23) as f32 - 1.0 + }) + .collect() +} +fn net(tile: usize, ch: u64, depth: usize, seed: u64) -> Net { + let p = std::env::temp_dir().join(format!("lc-purecpu-{}-{tile}-{ch}-{depth}-{seed}.onnx", std::process::id())); + std::fs::write(&p, synthetic::unet_onnx(tile as u64, ch, depth, seed)).unwrap(); + let n = crate::onnx::read(&p).unwrap(); + std::fs::remove_file(p).unwrap(); + n +} +fn agree(a: &[f32], b: &[f32]) { + assert_eq!(a.len(), b.len()); + assert!(a.iter().chain(b).all(|v| v.is_finite())); + let scale = b.iter().fold(1e-6f32, |m, v| m.max(v.abs())); + let error = a.iter().zip(b).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale; + assert!(error <= 1e-3, "relative maximum error {error:e}"); +} +#[test] +fn all_operators_match_scalar_on_random_nets_and_odd_and_tiny_shapes() { + for (t, ch, d) in [(1, 3, 0), (3, 5, 0), (5, 7, 0), (6, 3, 1), (20, 5, 2), (36, 9, 2), (64, 8, 3)] { + for seed in [1, 73] { + let n = net(t, ch, d, seed); + let r = NetRunner::new(&n, t).unwrap(); + let x = random(4 * t * t, seed as u32); + let want = reference::run(&n, t, &x).unwrap(); + agree(&r.run(&x).unwrap(), &want); + assert_eq!(r.run(&x).unwrap(), r.run(&x).unwrap()); + } + } +} + +#[test] +fn malformed_values_fail_before_execution_and_workspace_sizes_are_bounded() { + let mut n = net(4, 3, 1, 11); + for t in [0, 1, 3, usize::MAX, 4096] { + assert!(NetRunner::new(&n, t).is_err()); + } + for ch in [0, 3, usize::MAX] { + n.in_channels = ch; + assert!(NetRunner::new(&n, 4).is_err()); + } + let n = Net::new( + 4, + vec![ + Op::Conv(Conv { src: 0, src2: None, cin: 4, cout: 4096, k: 1, weight: vec![0.1; 4 * 4096], bias: vec![0.0; 4096], leaky: None }), + Op::Conv(Conv { src: 1, src2: None, cin: 4096, cout: 12, k: 1, weight: vec![0.1; 4096 * 12], bias: vec![0.0; 12], leaky: None }), + Op::DepthToSpace2 { src: 2 }, + ], + ) + .unwrap(); + assert!(NetRunner::new(&n, 2048).is_err()); + let ops = (0..100).map(|i| Op::MaxPool2 { src: i }).collect(); + let n = Net::new(4, ops).unwrap(); + assert!(NetRunner::new(&n, 16).is_err()); + let c = Conv { src: 0, src2: None, cin: 4, cout: 12, k: 1, weight: vec![0.25; 48], bias: vec![0.0; 12], leaky: None }; + for mode in 0..6 { + let mut bad = c.clone(); + match mode { + 0 => bad.src = usize::MAX, + 1 => bad.weight.pop().map(|_| ()).unwrap(), + 2 => bad.bias[0] = f32::NAN, + 3 => bad.leaky = Some(f32::INFINITY), + 4 => bad.cout = usize::MAX, + _ => bad.k = usize::MAX, + } + assert!(Net::new(4, vec![Op::Conv(bad), Op::DepthToSpace2 { src: 1 }]).is_err()); + } +} +#[test] +fn bad_tiles_and_arithmetic_overflow_are_errors_and_do_not_poison_reuse() { + let c = Conv { src: 0, src2: None, cin: 4, cout: 12, k: 1, weight: vec![f32::MAX; 48], bias: vec![0.0; 12], leaky: None }; + let n = Net::new(4, vec![Op::Conv(c), Op::DepthToSpace2 { src: 1 }]).unwrap(); + let r = NetRunner::new(&n, 1).unwrap(); + assert!(r.run(&[]).is_err()); + assert!(r.run(&[f32::NAN; 4]).is_err()); + assert!(r.run(&[f32::INFINITY; 4]).is_err()); + assert_eq!(r.idle.lock().unwrap().len(), 0, "invalid input never allocates a workspace"); + assert!(r.run(&[2.0; 4]).is_err()); + assert_eq!(r.run(&[0.0; 4]).unwrap(), vec![0.0; 12]); + // Matrix shape checks are the boundary in front of the dependency's asserting constructors. + assert!(multiply(&[], &[], &mut [], usize::MAX, 2, 2).is_err()); +} +#[test] +fn shared_runner_serves_concurrent_tiles_without_cross_contamination() { + let n = net(12, 5, 1, 97); + let r = NetRunner::new(&n, 12).unwrap(); + let inputs: Vec<_> = (0..8).map(|s| random(4 * 12 * 12, s)).collect(); + let answers: Vec<_> = inputs.iter().map(|x| reference::run(&n, 12, x).unwrap()).collect(); + std::thread::scope(|scope| { + let mut handles = Vec::new(); + for (x, want) in inputs.iter().zip(&answers) { + let r = &r; + handles.push(scope.spawn(move || { + for _ in 0..3 { + agree(&r.run(x).unwrap(), want); + } + })); + } + for h in handles { + h.join().unwrap(); + } + }); +} +#[test] +fn join_may_read_the_same_tensor_twice_and_leaky_up_convolution_is_supported() { + let c = ConvT2 { src: 0, cin: 4, cout: 3, weight: random(48, 2), bias: vec![0.1, -0.1, 0.0], leaky: Some(0.13) }; + let n = Net::new( + 4, + vec![ + Op::ConvT2(c), + Op::Conv(Conv { src: 1, src2: Some(1), cin: 6, cout: 3, k: 3, weight: random(162, 3), bias: vec![0.0; 3], leaky: Some(0.0) }), + ], + ) + .unwrap(); + let x = random(4 * 3 * 3, 7); + agree(&NetRunner::new(&n, 3).unwrap().run(&x).unwrap(), &reference::run(&n, 3, &x).unwrap()); +} + +#[test] +fn reused_im2col_clears_all_edge_padding_between_images() { + for t in [1, 3, 5, 36] { + let n = net(t, 5, 0, 121); + let r = NetRunner::new(&n, t).unwrap(); + for x in [vec![1.0; 4 * t * t], vec![0.0; 4 * t * t], random(4 * t * t, 9)] { + agree(&r.run(&x).unwrap(), &reference::run(&n, t, &x).unwrap()); + } + } +} +#[test] +#[ignore = "needs LC_DENOISE_MODEL; one tile with LIGHTCRAFT_PROFILE=1"] +fn profile_one_real_tile() { + let model = std::env::var_os("LC_DENOISE_MODEL").expect("set LC_DENOISE_MODEL"); + let m = crate::known::find("rawnind-bayer").unwrap().manifest; + let r = crate::runtime::CpuRunner::load(std::path::Path::new(&model), &m).unwrap(); + println!("workspace {:.1} MB", r.bytes_per_tile() as f64 / 1e6); + r.run(&crate::runtime::check_tile(512)).unwrap(); +} diff --git a/crates/denoise/src/hash.rs b/crates/denoise/src/hash.rs new file mode 100644 index 00000000..50f83be3 --- /dev/null +++ b/crates/denoise/src/hash.rs @@ -0,0 +1,46 @@ +//! SHA-256 of a model file, streamed (a model can be hundreds of megabytes). + +use std::io::{self, Read}; +use std::path::Path; + +use sha2::{Digest, Sha256}; + +/// Lowercase hex SHA-256 of everything `r` yields. +pub fn sha256_reader(mut r: R) -> io::Result { + let mut h = Sha256::new(); + let mut buf = vec![0u8; 1 << 20]; + loop { + let n = r.read(&mut buf)?; + if n == 0 { + break; + } + h.update(buf.get(..n).unwrap_or(&[])); + } + Ok(hex(&h.finalize())) +} + +/// Lowercase hex SHA-256 of a file. +pub fn sha256_file(path: &Path) -> io::Result { + sha256_reader(io::BufReader::new(std::fs::File::open(path)?)) +} + +/// Lowercase hex SHA-256 of `bytes`. +pub fn sha256_hex(bytes: &[u8]) -> String { + hex(&Sha256::digest(bytes)) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn known_digests() { + assert_eq!(sha256_hex(b""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + assert_eq!(sha256_hex(b"abc"), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + assert_eq!(sha256_reader(&b"abc"[..]).unwrap(), sha256_hex(b"abc")); + } +} diff --git a/crates/denoise/src/known.rs b/crates/denoise/src/known.rs new file mode 100644 index 00000000..9e0e7ea1 --- /dev/null +++ b/crates/denoise/src/known.rs @@ -0,0 +1,112 @@ +//! The denoise models LightCraft knows about: what each one is, who may use it, and where to fetch it. +//! +//! None is part of LightCraft. A model with an address here can be downloaded when the user asks and has accepted +//! its terms; anything else is brought by the user as an `.onnx` file with a `denoise-model.json` beside it. + +use crate::licence::{Commercial, Licence}; + +use crate::manifest::{DenoiserManifest, Domain, Gain}; + +/// A known model, and how to get it. +pub struct Known { + pub manifest: DenoiserManifest, + /// What to fetch (an archive or the `.onnx` itself); `None` when the user must bring the file. + pub download: Option, + /// The `.onnx` inside the downloaded archive (`None`: the download is the `.onnx`). + pub entry: Option<&'static str>, +} + +/// RawNIND's UtNet2 for Bayer sensors, as published for darktable (`rawdenoise-nind.dtmodel`, a zip with a Bayer and a +/// linear model). Its weights are GPL-3.0; the training pictures are CC BY 4.0 and CC0. +fn rawnind_bayer() -> Known { + let manifest = DenoiserManifest { + id: "rawnind-bayer".into(), + name: "RawNIND UtNet2 (Bayer)".into(), + version: "1.0".into(), + licence: Licence { + name: "GPL-3.0".into(), + commercial: Commercial::Yes, + url: Some("https://www.gnu.org/licenses/gpl-3.0.html".into()), + notice: "The weights are published under the GNU General Public License v3. LightCraft does not include them: they \ + are downloaded to your computer at your request and used by it only. If you pass the model file on, the \ + GPL applies to that copy." + .into(), + }, + source: Some("https://github.com/darktable-org/darktable-ai/tree/master/models/rawdenoise-nind".into()), + sha256: Some("da27509dab6a2915da67e988acd86cf71f9d5bbc8d1aa0ed32933578a887b901".into()), + size_bytes: Some(31_056_425), + provenance: "Trained on RawNIND, real photographs of the same scenes taken noisy and clean (images CC BY 4.0 and CC0, \ + published on Wikimedia Commons and by the University of Louvain). Paper: arXiv 2501.08924." + .into(), + domain: Domain::BayerToRgb, + tile: 512, + overlap: 64, + gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 }, + }; + let download = Download { + id: manifest.id.clone(), + url: "https://github.com/darktable-org/darktable-ai/releases/download/release-5.6.0/rawdenoise-nind.dtmodel".into(), + file_name: "rawdenoise-nind.dtmodel".into(), + size_bytes: 57_700_134, + sha256: "d71b5f1e727c85a359e6f74dca9e2016c9d8fc3e2f7ac3e9b347d80ceca969af".into(), + }; + Known { manifest, download: Some(download), entry: Some("model_bayer.onnx") } +} + +/// Every model LightCraft knows. +pub fn all() -> Vec { + vec![rawnind_bayer()] +} + +/// The known model with this id. +pub fn find(id: &str) -> Option { + all().into_iter().find(|k| k.manifest.id == id) +} + +/// The known model whose `.onnx` has this SHA-256. +pub fn by_sha256(sha256: &str) -> Option { + all().into_iter().map(|k| k.manifest).find(|m| m.sha256.as_deref() == Some(sha256)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::manifest::validate; + + #[test] + fn known_models_are_valid_unique_and_downloaded_over_https() { + let all = all(); + assert!(!all.is_empty()); + for (i, k) in all.iter().enumerate() { + validate(&k.manifest).unwrap(); + assert!(all.iter().skip(i + 1).all(|o| o.manifest.id != k.manifest.id), "unique ids"); + assert!(!k.manifest.licence.name.is_empty() && !k.manifest.licence.notice.is_empty(), "terms are shown before use"); + if let Some(d) = &k.download { + assert_eq!(d.id, k.manifest.id); + assert!(d.url.starts_with("https://") && d.sha256.len() == 64 && d.size_bytes > 0); + // a name that cannot leave the staging folder + assert!(!d.file_name.contains(['/', '\\']) && !d.file_name.starts_with('.')); + } + // a model that may not be used commercially is never offered for download + if k.manifest.licence.commercial == Commercial::No { + assert!(k.download.is_none(), "{}", k.manifest.id); + } + } + assert!(find("rawnind-bayer").is_some() && find("nothing").is_none()); + assert_eq!(by_sha256("da27509dab6a2915da67e988acd86cf71f9d5bbc8d1aa0ed32933578a887b901").map(|m| m.id), Some("rawnind-bayer".into())); + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Download { + pub id: String, + pub url: String, + /// The name the file is kept under while it waits for the user to accept its terms. + pub file_name: String, + pub size_bytes: u64, + pub sha256: String, +} + +pub fn host(url: &str) -> &str { + url.strip_prefix("https://").and_then(|r| r.split('/').next()).unwrap_or("") +} diff --git a/crates/denoise/src/lib.rs b/crates/denoise/src/lib.rs new file mode 100644 index 00000000..c7c76b58 --- /dev/null +++ b/crates/denoise/src/lib.rs @@ -0,0 +1,42 @@ +//! AI denoise for LightCraft, as a step between the sensor and the demosaic that the rest of the develop +//! pipeline never sees: the raw file stays untouched, the denoised picture is cached data that can always be +//! made again, and the amount is an ordinary develop setting. +//! +//! - [`manifest`]: what a denoise model takes and gives, and who may use it (untrusted JSON, validated). +//! - [`known`] and [`archive`]: the models LightCraft can fetch, and taking the model file out of the `.zip` it comes in. +//! - [`bayer`]: Bayer layouts and the packing of a mosaic into the four planes a model takes. +//! - [`tiles`] and [`run`]: cutting a picture into overlapping tiles for a fixed-size model, bringing each tile's +//! output scale back to the input's, keeping clipped highlights as they were, and blending the tiles together. +//! The model itself is a [`run::TileRunner`], so all of this is tested without one. +//! - [`product`]: the cache file the denoised picture is kept in (half floats, compressed in strips) and the +//! blend with the plain demosaic that the Amount slider controls. +//! - [`cpu`] and [`runtime`]: a single-thread pure-Rust CPU runner and its installation self-test. +//! - [`net`] and [`onnx`]: checked plain network data and bounded ONNX decoding, shared by CPU and GPU. +//! +//! Everything here treats model files, manifests and cache files as hostile input: sizes are capped, numbers must +//! be finite, and a malformed file is an error, never a panic. +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +pub mod archive; +pub mod bayer; +pub mod cpu; +pub mod known; +pub mod manifest; +pub mod net; +pub mod onnx; +mod onnx_proto; +pub mod product; +pub mod reference; +pub mod run; +pub mod runtime; +#[doc(hidden)] +pub mod synthetic; +pub mod tiles; + +pub use manifest::{DenoiserManifest, Domain, Gain, ManifestError}; +pub use run::{Control, Error, Params, TileRunner, denoise_bayer}; + +pub mod hash; +pub mod licence; +pub mod synthetic_proto; diff --git a/crates/denoise/src/licence.rs b/crates/denoise/src/licence.rs new file mode 100644 index 00000000..1d89146b --- /dev/null +++ b/crates/denoise/src/licence.rs @@ -0,0 +1,34 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Commercial { + Yes, + /// Research or personal use only. Never bundled or redistributed by LightCraft. + No, + /// The licence or the training data is unclear: the user is told so before installing. + Unknown, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct Licence { + /// SPDX id or a short name ("MIT", "Apache-2.0", "InsightFace non-commercial research"). + pub name: String, + pub commercial: Commercial, + pub url: Option, + /// Plain-language terms shown before the model is enabled. + pub notice: String, +} + +impl Default for Licence { + fn default() -> Self { + Self { name: String::new(), commercial: Commercial::Unknown, url: None, notice: String::new() } + } +} + +pub fn valid_id(id: &str) -> bool { + let mut bytes = id.bytes(); + let first_ok = bytes.next().is_some_and(|b| b.is_ascii_lowercase() || b.is_ascii_digit()); + first_ok && id.len() <= 64 && bytes.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || matches!(b, b'.' | b'_' | b'-')) +} diff --git a/crates/denoise/src/manifest.rs b/crates/denoise/src/manifest.rs new file mode 100644 index 00000000..9fbc9f49 --- /dev/null +++ b/crates/denoise/src/manifest.rs @@ -0,0 +1,217 @@ +//! A denoise model's manifest: what the model takes and gives, how big its tiles are and who may use it, as +//! plain data. +//! +//! It comes from a `denoise-model.json` next to the `.onnx` file or from the user's `catalog.json`. Both are +//! untrusted: [`validate`] must pass before a manifest is used, and [`parse`] applies it after reading JSON of +//! bounded size. The licence, id and address rules are the face models' (`crate::licence`). + +use crate::licence::{Licence, valid_id}; +use serde::{Deserialize, Serialize}; + +/// Largest `denoise-model.json` we read. +pub const MAX_MANIFEST_BYTES: usize = 64 * 1024; +/// Largest model file we accept (bytes). +pub const MAX_MODEL_BYTES: u64 = 2 * 1024 * 1024 * 1024; + +/// What the model takes and what it gives. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Domain { + /// The sensor's Bayer mosaic (black subtracted, white at 1, not white balanced) packed as four planes + /// `[R, G1, G2, B]` of its 2 × 2 cells, forced to RGGB; gives camera RGB at the mosaic's resolution, so + /// the picture is denoised and demosaiced in one step and must not be demosaiced again. + BayerToRgb, +} + +/// How the model's output scale is brought back to the input's. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "camelCase")] +pub enum Gain { + /// The output already has the input's scale. + None, + /// The output has a scale of its own, `nominal` times the input's (a million, for one model) and a little + /// different from tile to tile: each tile is scaled so that its channel means equal the input's. A tile too + /// dark to measure, or whose measured scale is further than `maxDeviation` (a fraction, 0.05 = 5 %) from + /// `nominal`, is scaled as if it were within that limit. + MatchMean { nominal: f32, max_deviation: f32 }, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DenoiserManifest { + /// Lowercase letters, digits, `.` `_` `-`. + pub id: String, + pub name: String, + pub version: String, + #[serde(default)] + pub licence: Licence, + /// Where the weights come from (an `https://` page or file), for the user's information. + #[serde(default)] + pub source: Option, + /// Lowercase hex SHA-256 of the `.onnx` file, when known. + #[serde(default)] + pub sha256: Option, + #[serde(default)] + pub size_bytes: Option, + /// What the model was trained on, in plain words; says "undisclosed" when it is. + #[serde(default)] + pub provenance: String, + pub domain: Domain, + /// Side of the square the model runs on, in packed cells (the file's fixed input size). + pub tile: u32, + /// Cells neighbouring tiles share; the picture is blended across them. + pub overlap: u32, + pub gain: Gain, +} + +#[derive(Debug, PartialEq, thiserror::Error)] +pub enum ManifestError { + #[error("`{0}` is not valid: {1}")] + Field(&'static str, String), + #[error("not a denoise-model manifest: {0}")] + Parse(String), + #[error("a manifest may be at most {MAX_MANIFEST_BYTES} bytes")] + TooLarge, +} + +fn bad(field: &'static str, why: impl Into) -> Result { + Err(ManifestError::Field(field, why.into())) +} + +fn text(field: &'static str, s: &str, min: usize, max: usize) -> Result<(), ManifestError> { + let n = s.chars().count(); + if n < min || n > max { + return bad(field, format!("must be {min} to {max} characters")); + } + if s.chars().any(|c| c.is_control() && c != '\n') { + return bad(field, "must not contain control characters"); + } + Ok(()) +} + +fn url(field: &'static str, u: &Option) -> Result<(), ManifestError> { + match u { + None => Ok(()), + Some(u) if u.len() <= 500 && u.starts_with("https://") && !u.chars().any(|c| c.is_control() || c.is_whitespace()) => Ok(()), + Some(_) => bad(field, "must be an https:// address of at most 500 characters"), + } +} + +/// Check a manifest from outside before trusting it. +pub fn validate(m: &DenoiserManifest) -> Result<(), ManifestError> { + if !valid_id(&m.id) { + return bad("id", "1 to 64 lowercase letters, digits, `.`, `_` or `-`, starting with a letter or digit"); + } + text("name", &m.name, 1, 80)?; + text("version", &m.version, 1, 40)?; + text("provenance", &m.provenance, 0, 1000)?; + text("licence.name", &m.licence.name, 0, 80)?; + text("licence.notice", &m.licence.notice, 0, 2000)?; + url("licence.url", &m.licence.url)?; + url("source", &m.source)?; + if let Some(h) = &m.sha256 + && !(h.len() == 64 && h.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))) + { + return bad("sha256", "must be 64 lowercase hex digits"); + } + if let Some(n) = m.size_bytes + && (n == 0 || n > MAX_MODEL_BYTES) + { + return bad("sizeBytes", format!("must be between 1 and {MAX_MODEL_BYTES}")); + } + // a U-Net with four poolings wants a multiple of 16; nothing smaller than 64 or larger than 2048 is a tile + if !(64..=2048).contains(&m.tile) || !m.tile.is_multiple_of(16) { + return bad("tile", "must be a multiple of 16 from 64 to 2048"); + } + // tiles must overlap by less than half, or one tile's blend ramps would run into its other neighbour's + if !m.overlap.is_multiple_of(2) || m.overlap < 16 || m.overlap.saturating_mul(2) >= m.tile { + return bad("overlap", "must be even, at least 16 and less than half the tile"); + } + if let Gain::MatchMean { nominal, max_deviation } = m.gain { + if !(nominal.is_finite() && (1e-6..=1e12).contains(&nominal)) { + return bad("gain.nominal", "must be between 0.000001 and 1000000000000"); + } + if !(max_deviation.is_finite() && (0.0..=0.5).contains(&max_deviation)) { + return bad("gain.maxDeviation", "must be between 0 and 0.5"); + } + } + Ok(()) +} + +/// Read and validate a `denoise-model.json`. +pub fn parse(bytes: &[u8]) -> Result { + if bytes.len() > MAX_MANIFEST_BYTES { + return Err(ManifestError::TooLarge); + } + let bytes = bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]).unwrap_or(bytes); + let m: DenoiserManifest = serde_json::from_slice(bytes).map_err(|e| ManifestError::Parse(e.to_string()))?; + validate(&m)?; + Ok(m) +} + +#[cfg(test)] +mod tests { + use super::*; + + pub(crate) fn sample() -> DenoiserManifest { + DenoiserManifest { + id: "nind-bayer".into(), + name: "Bayer denoiser".into(), + version: "1".into(), + licence: Licence::default(), + source: Some("https://example.org/model.onnx".into()), + sha256: None, + size_bytes: Some(31_000_000), + provenance: "trained on real raw noise pairs".into(), + domain: Domain::BayerToRgb, + tile: 512, + overlap: 64, + gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 }, + } + } + + #[test] + fn a_good_manifest_round_trips() { + let m = sample(); + let json = serde_json::to_vec(&m).unwrap(); + assert_eq!(parse(&json).unwrap(), m); + } + + #[test] + fn bad_fields_are_named() { + let cases: Vec<(&str, Box)> = vec![ + ("id", Box::new(|m| m.id = "../evil".into())), + ("id", Box::new(|m| m.id = String::new())), + ("name", Box::new(|m| m.name = "x\u{7}".into())), + ("source", Box::new(|m| m.source = Some("http://insecure".into()))), + ("sha256", Box::new(|m| m.sha256 = Some("XYZ".into()))), + ("sizeBytes", Box::new(|m| m.size_bytes = Some(0))), + ("tile", Box::new(|m| m.tile = 500)), + ("tile", Box::new(|m| m.tile = 4096)), + ("overlap", Box::new(|m| m.overlap = 300)), + ("overlap", Box::new(|m| m.overlap = 63)), + ("gain.maxDeviation", Box::new(|m| m.gain = Gain::MatchMean { nominal: 1.0, max_deviation: f32::NAN })), + ("gain.maxDeviation", Box::new(|m| m.gain = Gain::MatchMean { nominal: 1.0, max_deviation: 2.0 })), + ("gain.nominal", Box::new(|m| m.gain = Gain::MatchMean { nominal: 0.0, max_deviation: 0.1 })), + ("gain.nominal", Box::new(|m| m.gain = Gain::MatchMean { nominal: f32::INFINITY, max_deviation: 0.1 })), + ]; + for (field, edit) in cases { + let mut m = sample(); + edit(&mut m); + match validate(&m) { + Err(ManifestError::Field(f, _)) => assert_eq!(f, field), + other => panic!("{field}: {other:?}"), + } + } + } + + #[test] + fn parse_is_bounded_and_tolerant() { + assert_eq!(parse(&vec![b' '; MAX_MANIFEST_BYTES + 1]), Err(ManifestError::TooLarge)); + assert!(matches!(parse(b"{ nope"), Err(ManifestError::Parse(_)))); + assert!(matches!(parse(b""), Err(ManifestError::Parse(_)))); + let mut with_bom = vec![0xEF, 0xBB, 0xBF]; + with_bom.extend(serde_json::to_vec(&sample()).unwrap()); + assert!(parse(&with_bom).is_ok()); + } +} diff --git a/crates/denoise/src/net.rs b/crates/denoise/src/net.rs new file mode 100644 index 00000000..42241833 --- /dev/null +++ b/crates/denoise/src/net.rs @@ -0,0 +1,345 @@ +//! A denoise network as plain data: the handful of layer kinds a U-Net denoiser is made of, with their weights. +//! +//! [`onnx`](crate::onnx) reads the file into this description, shared by the pure-Rust CPU and GPU runners. +//! Unsupported operators, groups, dilation or activations are reported as errors when loading the model. +//! +//! The description is checked ([`Net::new`]) because it comes from a stranger's file: channel counts must chain, +//! weights must have the sizes the shapes need and be finite, and the depth must stay small. + +/// Most layers a network may have. +pub const MAX_OPS: usize = 512; +/// Most channels a layer may have. +pub const MAX_CHANNELS: usize = 4096; +/// Most weights (numbers) a network may have in all. +pub const MAX_WEIGHTS: usize = 128 * 1024 * 1024; + +#[derive(Debug, PartialEq, thiserror::Error)] +pub enum NetError { + /// The model uses something this denoise network description cannot express. + #[error("not supported by the denoise runners: {0}")] + Unsupported(String), + /// The model is not a well-formed network. + #[error("not a valid network: {0}")] + Invalid(String), +} + +fn invalid(why: impl Into) -> Result { + Err(NetError::Invalid(why.into())) +} + +/// A tensor is named by its number: 0 is the network's input, and layer `i` makes tensor `i + 1`. +pub type TensorId = usize; + +/// A convolution with stride 1 and "same" padding (`k` is 1 or 3), over one tensor or two joined along the channels +/// (`src` first), then a bias and optionally a leaky ReLU. +#[derive(Clone, Debug, PartialEq)] +pub struct Conv { + pub src: TensorId, + /// A second source, whose channels follow `src`'s (a skip connection's concatenation). + pub src2: Option, + pub cin: usize, + pub cout: usize, + pub k: usize, + /// `[cout][cin][k][k]`, as ONNX has it. + pub weight: Vec, + pub bias: Vec, + /// `Some(alpha)`: a leaky ReLU with this slope follows. + pub leaky: Option, +} + +/// A transposed convolution with a 2 × 2 kernel and stride 2: every input pixel becomes a 2 × 2 block. +#[derive(Clone, Debug, PartialEq)] +pub struct ConvT2 { + pub src: TensorId, + pub cin: usize, + pub cout: usize, + /// `[cin][cout][2][2]`, as ONNX has it. + pub weight: Vec, + pub bias: Vec, + pub leaky: Option, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Op { + Conv(Conv), + ConvT2(ConvT2), + /// 2 × 2 maximum, stride 2. + MaxPool2 { + src: TensorId, + }, + /// Depth-to-space with block size 2 in "CRD" order: output channel `c` at `(2y + dy, 2x + dx)` is input channel + /// `4c + 2dy + dx` at `(y, x)`. + DepthToSpace2 { + src: TensorId, + }, +} + +/// What a tensor looks like: its channels, and its resolution as a power of two below the input's (a negative +/// level is above it). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Shape { + pub channels: usize, + pub level: i32, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Net { + pub in_channels: usize, + ops: Vec, + shapes: Vec, +} + +impl Net { + /// Check `ops` (layer `i` makes tensor `i + 1`) over an input of `in_channels` and describe every tensor. + pub fn new(in_channels: usize, ops: Vec) -> Result { + if in_channels == 0 || in_channels > MAX_CHANNELS { + return invalid("the input has no channels or too many"); + } + if ops.is_empty() || ops.len() > MAX_OPS { + return invalid("the network has no layers or too many"); + } + let mut shapes = vec![Shape { channels: in_channels, level: 0 }]; + let mut weights = 0usize; + for (i, op) in ops.iter().enumerate() { + let shape_of = |t: TensorId| { + shapes + .get(t) + .copied() + .filter(|_| t <= i) + .ok_or_else(|| NetError::Invalid(format!("layer {i} reads tensor {t}, which does not exist yet"))) + }; + let out = match op { + Op::Conv(c) => { + let a = shape_of(c.src)?; + let (cin, level) = match c.src2 { + Some(s2) => { + let b = shape_of(s2)?; + if b.level != a.level { + return invalid(format!("layer {i} joins tensors of different resolutions")); + } + (a.channels + b.channels, a.level) + } + None => (a.channels, a.level), + }; + if c.cin != cin || c.cout == 0 || c.cout > MAX_CHANNELS || !(c.k == 1 || c.k == 3) { + return invalid(format!( + "layer {i}: a convolution of {} to {} channels with a {}-wide kernel does not fit", + c.cin, c.cout, c.k + )); + } + check_weights( + i, + &c.weight, + c.cout.checked_mul(cin).and_then(|n| n.checked_mul(c.k * c.k)), + &c.bias, + c.cout, + c.leaky, + &mut weights, + )?; + Shape { channels: c.cout, level } + } + Op::ConvT2(c) => { + let a = shape_of(c.src)?; + if c.cin != a.channels || c.cout == 0 || c.cout > MAX_CHANNELS { + return invalid(format!("layer {i}: a transposed convolution of {} to {} channels does not fit", c.cin, c.cout)); + } + check_weights(i, &c.weight, c.cin.checked_mul(c.cout).and_then(|n| n.checked_mul(4)), &c.bias, c.cout, c.leaky, &mut weights)?; + Shape { channels: c.cout, level: a.level - 1 } + } + Op::MaxPool2 { src } => { + let a = shape_of(*src)?; + Shape { channels: a.channels, level: a.level + 1 } + } + Op::DepthToSpace2 { src } => { + let a = shape_of(*src)?; + if !a.channels.is_multiple_of(4) { + return invalid(format!("layer {i}: depth-to-space needs a multiple of 4 channels")); + } + Shape { channels: a.channels / 4, level: a.level - 1 } + } + }; + shapes.push(out); + } + Ok(Net { in_channels, ops, shapes }) + } + + /// The layers: layer `i` makes tensor `i + 1`. + pub fn ops(&self) -> &[Op] { + &self.ops + } + + /// The shape of tensor `t` (0 is the input). + pub fn shape(&self, t: TensorId) -> Option { + self.shapes.get(t).copied() + } + + /// The tensor the network gives (the last layer's). + pub fn output(&self) -> TensorId { + self.ops.len() + } + + pub fn out_channels(&self) -> usize { + self.shapes.last().map_or(0, |s| s.channels) + } + + /// The deepest resolution level: a tile must be a multiple of `2^this` cells wide. + pub fn depth(&self) -> u32 { + self.shapes.iter().map(|s| s.level.max(0) as u32).max().unwrap_or(0) + } + + /// Can the network run on a `tile × tile` input and give `[out_channels, 2·tile, 2·tile]`? (Pooling must divide + /// the tile, and the output must be one level above the input.) + pub fn fits_tile(&self, tile: usize) -> bool { + let Some(multiple) = 1usize.checked_shl(self.depth()) else { return false }; + tile > 0 && tile.is_multiple_of(multiple) && self.shapes.last().is_some_and(|s| s.level == -1) + } + + /// Multiply-adds for one tile, for estimating time. + pub fn macs(&self, tile: usize) -> u64 { + let pixels = |level: i32| -> u64 { + let side = if level >= 0 { + tile.checked_shr(level as u32).unwrap_or(0) + } else { + tile.saturating_mul(1usize.checked_shl(level.unsigned_abs()).unwrap_or(usize::MAX)) + }; + (side as u64).saturating_mul(side as u64) + }; + self.ops + .iter() + .enumerate() + .map(|(i, op)| { + let out = self.shapes.get(i + 1).map_or(0, |s| pixels(s.level)); + match op { + Op::Conv(c) => out.saturating_mul((c.cin * c.cout * c.k * c.k) as u64), + // every input pixel makes a 2 × 2 block: as many multiply-adds as the 2 × 2-wide kernel has weights + Op::ConvT2(c) => pixels(self.shapes.get(c.src).map_or(0, |s| s.level)).saturating_mul((c.cin * c.cout * 4) as u64), + _ => 0, + } + }) + .fold(0u64, u64::saturating_add) + } +} + +fn check_weights( + layer: usize, + weight: &[f32], + want: Option, + bias: &[f32], + cout: usize, + leaky: Option, + total: &mut usize, +) -> Result<(), NetError> { + if want != Some(weight.len()) || bias.len() != cout { + return invalid(format!("layer {layer}: the weights are not the size its shape needs")); + } + *total = total.saturating_add(weight.len()).saturating_add(bias.len()); + if *total > MAX_WEIGHTS { + return invalid("the network has too many weights"); + } + if weight.iter().chain(bias).any(|v| !v.is_finite()) || leaky.is_some_and(|a| !a.is_finite()) { + return invalid(format!("layer {layer}: a weight is not a finite number")); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn conv(src: TensorId, src2: Option, cin: usize, cout: usize, k: usize) -> Op { + Op::Conv(Conv { src, src2, cin, cout, k, weight: vec![0.1; cout * cin * k * k], bias: vec![0.0; cout], leaky: Some(0.2) }) + } + + fn convt(src: TensorId, cin: usize, cout: usize) -> Op { + Op::ConvT2(ConvT2 { src, cin, cout, weight: vec![0.1; cin * cout * 4], bias: vec![0.0; cout], leaky: None }) + } + + /// A two-level network of the shape the denoisers have: 4 → 8 channels, pool, 16 channels, up, join, 12, and the + /// 1 × 1 convolution to the colours. + fn unet() -> Vec { + vec![ + conv(0, None, 4, 8, 3), // 1: 8 ch, level 0 + Op::MaxPool2 { src: 1 }, // 2: level 1 + conv(2, None, 8, 16, 3), // 3: 16 ch, level 1 + convt(3, 16, 8), // 4: 8 ch, level 0 + conv(4, Some(1), 16, 8, 3), // 5: join [up, skip] + conv(5, None, 8, 12, 1), // 6: 12 ch + Op::DepthToSpace2 { src: 6 }, // 7: 3 ch, level -1 + ] + } + + #[test] + fn deep_networks_and_huge_cost_queries_never_overflow_or_truncate_depth() { + let n = Net::new(4, unet()).unwrap(); + assert_eq!(n.macs(usize::MAX), u64::MAX); + assert_eq!(n.macs(0), 0); + let mut ops = Vec::new(); + for i in 0..21 { + ops.push(Op::MaxPool2 { src: i }); + } + for i in 21..42 { + ops.push(convt(i, 4, 4)); + } + ops.push(conv(42, None, 4, 12, 1)); + ops.push(Op::DepthToSpace2 { src: 43 }); + let n = Net::new(4, ops).unwrap(); + assert!(!n.fits_tile(1 << 20)); + assert!(n.fits_tile(1 << 21)); + let _ = n.macs(16); + } + + #[test] + fn a_u_net_is_described_with_its_shapes_and_costs() { + let net = Net::new(4, unet()).unwrap(); + assert_eq!(net.out_channels(), 3); + assert_eq!(net.shape(3), Some(Shape { channels: 16, level: 1 })); + assert_eq!(net.shape(7), Some(Shape { channels: 3, level: -1 })); + assert_eq!(net.depth(), 1); + assert!(net.fits_tile(64) && !net.fits_tile(63) && !net.fits_tile(0)); + // 64 × 64 cells: layer 1 = 4096·4·8·9, and so on; just check it is the sum of what the layers cost + let t = 64u64; + let want = t * t * 4 * 8 * 9 + (t / 2) * (t / 2) * 8 * 16 * 9 + (t / 2) * (t / 2) * 16 * 8 * 4 + t * t * 16 * 8 * 9 + t * t * 8 * 12; + assert_eq!(net.macs(64), want); + } + + #[test] + fn a_network_that_does_not_chain_is_refused() { + // wrong input channel count for the first convolution + assert!(Net::new(3, unet()).is_err()); + // a layer reading a tensor that does not exist yet + let mut ops = unet(); + ops[2] = conv(9, None, 8, 16, 3); + assert!(Net::new(4, ops).is_err()); + // joining tensors of different resolutions + let mut ops = unet(); + ops[4] = conv(3, Some(1), 24, 8, 3); + assert!(Net::new(4, ops).is_err()); + // weights of the wrong size, and not finite + let mut ops = unet(); + if let Op::Conv(c) = &mut ops[0] { + c.weight.pop(); + } + assert!(Net::new(4, ops).is_err()); + let mut ops = unet(); + if let Op::Conv(c) = &mut ops[0] { + c.bias[0] = f32::NAN; + } + assert!(Net::new(4, ops).is_err()); + // a kernel that is neither 1 nor 3 wide, no layers, depth-to-space on a channel count it cannot split + let mut ops = unet(); + if let Op::Conv(c) = &mut ops[0] { + c.k = 5; + } + assert!(Net::new(4, ops).is_err()); + assert!(Net::new(4, vec![]).is_err()); + assert!(Net::new(4, vec![Op::DepthToSpace2 { src: 0 }]).is_ok()); + assert!(Net::new(4, vec![Op::DepthToSpace2 { src: 0 }, Op::DepthToSpace2 { src: 1 }]).is_err()); + assert!(Net::new(2, vec![Op::DepthToSpace2 { src: 0 }]).is_err()); + } + + #[test] + fn a_network_that_does_not_end_one_level_up_does_not_fit_a_tile() { + let net = Net::new(4, unet()[..6].to_vec()).unwrap(); + assert!(!net.fits_tile(64), "no depth-to-space: the output is not at twice the resolution"); + } +} diff --git a/crates/denoise/src/onnx.rs b/crates/denoise/src/onnx.rs new file mode 100644 index 00000000..fc84a36c --- /dev/null +++ b/crates/denoise/src/onnx.rs @@ -0,0 +1,369 @@ +//! Reading an `.onnx` file into a [`Net`] with bounded, pure-Rust protobuf decoding. +//! +//! Only what the denoise U-Nets are made of is understood: `Conv` (3 × 3 "same" or 1 × 1, stride 1, no groups, no +//! dilation) optionally followed by `LeakyRelu`, `ConvTranspose` (2 × 2, stride 2) likewise, `MaxPool` (2 × 2, stride 2), +//! a two-input `Concat` on the channels that feeds a convolution, and a final `DepthToSpace` (2, `CRD`). Any other +//! operator or attribute is [`NetError::Unsupported`], which is reported to the user by both runners. + +use std::collections::HashMap; +use std::path::Path; + +use crate::onnx_proto::{self, AttributeProto, NodeProto, TensorProto, ValueInfoProto}; +use std::io::Read; + +use crate::net::{Conv, ConvT2, MAX_WEIGHTS, Net, NetError, Op, TensorId}; + +fn unsupported(why: impl Into) -> Result { + Err(NetError::Unsupported(why.into())) +} + +fn invalid(why: impl Into) -> Result { + Err(NetError::Invalid(why.into())) +} + +/// Read the model at `path`. +pub fn read(path: &Path) -> Result { + let file = std::fs::File::open(path).map_err(|e| NetError::Invalid(e.to_string()))?; + let size = usize::try_from(file.metadata().map_err(|e| NetError::Invalid(e.to_string()))?.len()) + .ok() + .filter(|&n| n <= onnx_proto::MAX_BYTES) + .ok_or_else(|| NetError::Invalid("model file too large".into()))?; + let mut bytes = Vec::new(); + bytes.try_reserve_exact(size).map_err(|_| NetError::Invalid("not enough memory to load the model".into()))?; + file.take(onnx_proto::MAX_BYTES as u64 + 1).read_to_end(&mut bytes).map_err(|e| NetError::Invalid(e.to_string()))?; + read_bytes(&bytes) +} + +fn read_bytes(bytes: &[u8]) -> Result { + let graph = onnx_proto::decode(bytes)?; + + let initialisers: HashMap<&str, &TensorProto> = graph.initializer.iter().map(|t| (t.name.as_str(), t)).collect(); + if initialisers.len() != graph.initializer.len() { + return invalid("duplicate initializer names"); + } + let inputs: Vec<_> = graph.input.iter().filter(|i| !initialisers.contains_key(i.name.as_str())).collect(); + let ([input], [output]) = (inputs.as_slice(), graph.output.as_slice()) else { return unsupported("not one input and one output") }; + let in_channels = match channel_dim(input) { + Some(c) => c, + None => return unsupported("the input's channel count is not written in the file"), + }; + + // how many nodes read each name (a graph output counts as a reader) + let mut readers: HashMap<&str, usize> = HashMap::new(); + for n in &graph.node { + for i in &n.input { + *readers.entry(i.as_str()).or_default() += 1; + } + } + *readers.entry(output.name.as_str()).or_default() += 1; + + let mut ids: HashMap = HashMap::new(); + ids.insert(input.name.clone(), 0); + // a Concat is not a layer of its own: the convolution that reads it takes both sources + let mut joins: HashMap = HashMap::new(); + let mut ops: Vec = Vec::new(); + let mut weights_total = 0usize; + + let nodes = &graph.node; + let mut i = 0; + while i < nodes.len() { + let Some(node) = nodes.get(i) else { break }; + i += 1; + if node.output.len() != 1 || node.output.first().is_none_or(|n| n.is_empty() || ids.contains_key(n) || joins.contains_key(n)) { + return invalid("a layer has missing, repeated or multiple outputs"); + } + let allowed: &[&str] = match node.op_type.as_str() { + "Conv" => &["kernel_shape", "pads", "strides", "dilations", "group", "auto_pad"], + "ConvTranspose" => &["kernel_shape", "pads", "strides", "dilations", "group", "output_padding", "auto_pad"], + "MaxPool" => &["kernel_shape", "pads", "strides", "dilations", "ceil_mode", "storage_order", "auto_pad"], + "Concat" => &["axis"], + "DepthToSpace" => &["blocksize", "mode"], + _ => &[], + }; + if node.attribute.iter().any(|a| !allowed.contains(&a.name.as_str())) { + return unsupported("an unknown operator attribute"); + } + let this = ops.len() + 1; + let lookup = |name: &str| -> Result { + ids.get(name).copied().ok_or_else(|| NetError::Invalid(format!("`{}` reads `{name}`, which nothing makes", node.op_type))) + }; + // a LeakyRelu straight after this layer, which nothing else reads, is part of it + let mut fuse_leaky = |out: &str| -> Result<(Option, String), NetError> { + if let Some(next) = nodes.get(i) + && next.op_type == "LeakyRelu" + && next.input.first().map(String::as_str) == Some(out) + && readers.get(out).copied() == Some(1) + { + if next.input.len() != 1 || next.output.len() != 1 || next.attribute.iter().any(|a| a.name != "alpha") { + return invalid("malformed LeakyRelu"); + } + let alpha = attr(next, "alpha").map_or(0.01, |a| a.f); + i += 1; + let name = next.output.first().cloned().ok_or_else(|| NetError::Invalid("LeakyRelu without an output".into()))?; + return Ok((Some(alpha), name)); + } + Ok((None, out.to_string())) + }; + match node.op_type.as_str() { + "Concat" => { + if attr(node, "axis").map(|a| a.i) != Some(1) { + return unsupported("Concat on another axis than the channels"); + } + let [a, b] = node.input.as_slice() else { return unsupported("Concat of other than two tensors") }; + let out = node.output.first().ok_or_else(|| NetError::Invalid("Concat without an output".into()))?; + if readers.get(out.as_str()).copied() != Some(1) { + return unsupported("a Concat that is read by more than one layer"); + } + joins.insert(out.clone(), (lookup(a)?, lookup(b)?)); + } + "Conv" => { + let (w, b) = weights_of(node, &initialisers, &mut weights_total)?; + let (cout, cin, kh, kw) = four(&w.0)?; + let b = if b.is_empty() { vec![0.0; cout] } else { b }; + if kh != kw || !(kh == 1 || kh == 3) { + return unsupported(format!("a {kh} × {kw} convolution")); + } + if ints_or(node, "strides", &[1, 1]) != [1, 1] + || ints_or(node, "dilations", &[1, 1]) != [1, 1] + || attr(node, "group").is_some_and(|a| a.i != 1) + { + return unsupported("a convolution with strides, dilation or groups"); + } + if attr(node, "kernel_shape").is_some_and(|a| a.ints != [kh as i64, kw as i64]) { + return invalid("kernel shape does not match the weights"); + } + let pad = (kh as i64 - 1) / 2; + if ints_or(node, "pads", &[0; 4]) != [pad; 4] || attr(node, "auto_pad").is_some_and(|a| !a.s.is_empty() && a.s != b"NOTSET") { + return unsupported("a convolution that is not padded to keep its size"); + } + let src_name = node.input.first().ok_or_else(|| NetError::Invalid("Conv without an input".into()))?; + let (src, src2) = match joins.remove(src_name) { + Some((a, b)) => (a, Some(b)), + None => (lookup(src_name)?, None), + }; + let out = node.output.first().ok_or_else(|| NetError::Invalid("Conv without an output".into()))?; + let (leaky, out_name) = fuse_leaky(out)?; + ids.insert(out_name, this); + ops.push(Op::Conv(Conv { src, src2, cin, cout, k: kh, weight: w.1, bias: b, leaky })); + } + "ConvTranspose" => { + let (w, b) = weights_of(node, &initialisers, &mut weights_total)?; + let (cin, cout, kh, kw) = four(&w.0)?; + let b = if b.is_empty() { vec![0.0; cout] } else { b }; + if (kh, kw) != (2, 2) + || ints_or(node, "strides", &[1, 1]) != [2, 2] + || ints_or(node, "pads", &[0; 4]) != [0; 4] + || ints_or(node, "dilations", &[1, 1]) != [1, 1] + || attr(node, "group").is_some_and(|a| a.i != 1) + || attr(node, "output_padding").is_some_and(|a| a.ints.iter().any(|&v| v != 0)) + { + return unsupported("a transposed convolution other than 2 × 2 with stride 2"); + } + if attr(node, "auto_pad").is_some_and(|a| !a.s.is_empty() && a.s != b"NOTSET") { + return unsupported("transposed convolution auto padding"); + } + if attr(node, "kernel_shape").is_some_and(|a| a.ints != [2, 2]) { + return invalid("transposed kernel shape does not match the weights"); + } + let src = lookup(node.input.first().map_or("", String::as_str))?; + let out = node.output.first().ok_or_else(|| NetError::Invalid("ConvTranspose without an output".into()))?; + let (leaky, out_name) = fuse_leaky(out)?; + ids.insert(out_name, this); + ops.push(Op::ConvT2(ConvT2 { src, cin, cout, weight: w.1, bias: b, leaky })); + } + "MaxPool" => { + if ints_or(node, "kernel_shape", &[]) != [2, 2] + || ints_or(node, "strides", &[1, 1]) != [2, 2] + || ints_or(node, "pads", &[0; 4]) != [0; 4] + || ints_or(node, "dilations", &[1, 1]) != [1, 1] + || attr(node, "ceil_mode").is_some_and(|a| a.i != 0) + || attr(node, "storage_order").is_some_and(|a| a.i != 0) + || attr(node, "auto_pad").is_some_and(|a| !a.s.is_empty() && a.s != b"NOTSET") + { + return unsupported("a max-pool other than 2 × 2 with stride 2"); + } + let src = lookup(node.input.first().map_or("", String::as_str))?; + let out = node.output.first().ok_or_else(|| NetError::Invalid("MaxPool without an output".into()))?; + ids.insert(out.clone(), this); + ops.push(Op::MaxPool2 { src }); + } + "DepthToSpace" => { + if attr(node, "blocksize").map(|a| a.i) != Some(2) || attr(node, "mode").map(|a| a.s.as_slice()) != Some(&b"CRD"[..]) { + return unsupported("depth-to-space other than block size 2 in CRD order"); + } + let src = lookup(node.input.first().map_or("", String::as_str))?; + let out = node.output.first().ok_or_else(|| NetError::Invalid("DepthToSpace without an output".into()))?; + ids.insert(out.clone(), this); + ops.push(Op::DepthToSpace2 { src }); + } + other => return unsupported(format!("the `{other}` operator")), + } + } + if !joins.is_empty() { + return unsupported("a Concat whose result no convolution reads"); + } + if ids.get(output.name.as_str()) != Some(&ops.len()) { + return unsupported("the output is not the last layer"); + } + Net::new(in_channels, ops) +} + +fn attr<'a>(n: &'a NodeProto, name: &str) -> Option<&'a AttributeProto> { + n.attribute.iter().find(|a| a.name == name) +} + +/// The integer list attribute `name`, or `default` when the node does not have it. +fn ints_or(n: &NodeProto, name: &str, default: &[i64]) -> Vec { + attr(n, name).map_or_else(|| default.to_vec(), |a| a.ints.clone()) +} + +fn four(dims: &[i64]) -> Result<(usize, usize, usize, usize), NetError> { + match dims { + [a, b, c, d] if dims.iter().all(|&v| v > 0 && v <= 1 << 20) => Ok((*a as usize, *b as usize, *c as usize, *d as usize)), + _ => invalid("a weight tensor that is not four-dimensional"), + } +} + +type Weights = ((Vec, Vec), Vec); + +/// The weight and bias initialisers of a convolution node. +fn weights_of(node: &NodeProto, initialisers: &HashMap<&str, &TensorProto>, total: &mut usize) -> Result { + let mut get = |idx: usize| -> Result<(Vec, Vec), NetError> { + let name = node.input.get(idx).ok_or_else(|| NetError::Invalid(format!("`{}` without its weights", node.op_type)))?; + let t = initialisers.get(name.as_str()).ok_or_else(|| NetError::Unsupported(format!("weights that are computed (`{name}`)")))?; + let values = floats(t)?; + *total = total.saturating_add(values.len()); + if *total > MAX_WEIGHTS { + return invalid("the network has too many weights"); + } + Ok((t.dims.clone(), values)) + }; + let w = get(1)?; + let b = if node.input.len() > 2 { + let (dims, values) = get(2)?; + if dims.len() != 1 || values.is_empty() { + return invalid("a bias tensor must be a nonempty vector"); + } + values + } else { + Vec::new() + }; + Ok((w, b)) +} + +fn floats(t: &TensorProto) -> Result, NetError> { + if t.data_type != 1 { + return unsupported("weights that are not 32-bit floats"); + } + let want: usize = t + .dims + .iter() + .try_fold(1usize, |n, &d| usize::try_from(d).ok().and_then(|d| n.checked_mul(d))) + .ok_or_else(|| NetError::Invalid("a tensor with a negative or huge size".into()))?; + if want > MAX_WEIGHTS { + return invalid("a tensor with too many numbers"); + } + if !t.raw_data.is_empty() && (!t.float_data.is_empty() || !t.raw_data.len().is_multiple_of(4)) { + return invalid("ambiguous or truncated tensor data"); + } + let actual = if t.raw_data.is_empty() { t.float_data.len() } else { t.raw_data.len() / 4 }; + if actual != want { + return invalid("tensor data does not match its dimensions"); + } + let mut values = Vec::new(); + values.try_reserve_exact(want).map_err(|_| NetError::Invalid("not enough memory for weights".into()))?; + if !t.raw_data.is_empty() { + values.extend(t.raw_data.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b))); + } else { + values.extend_from_slice(&t.float_data); + } + if values.len() != want { + return invalid(format!("tensor `{}` has {} numbers, not the {want} its shape says", t.name, values.len())); + } + Ok(values) +} + +/// The channel count in an input's declared shape `[batch, channels, …]`. +fn channel_dim(v: &ValueInfoProto) -> Option { + v.channels +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::synthetic::unet_onnx; + + fn temp_model(name: &str, bytes: &[u8]) -> std::path::PathBuf { + let p = std::env::temp_dir().join(format!("lc-denoise-onnx-{name}-{}.onnx", std::process::id())); + std::fs::write(&p, bytes).unwrap(); + p + } + + #[test] + fn a_u_net_is_read_with_its_layers_fused() { + let (tile, depth) = (32usize, 2usize); + let p = temp_model("unet", &unet_onnx(tile as u64, 8, depth, 7)); + let net = read(&p).unwrap(); + // per level two convolutions; a pool per level but the last; going up a transposed convolution and two convolutions + // per level (the join is part of the first); then the 1 × 1 convolution and depth-to-space + assert_eq!(net.ops().len(), 2 * (depth + 1) + depth + depth * 3 + 2); + assert!(matches!(net.ops().last(), Some(crate::net::Op::DepthToSpace2 { .. }))); + assert!(net.ops().iter().any(|o| matches!(o, crate::net::Op::Conv(c) if c.src2.is_some())), "the joins became two-source convolutions"); + assert!( + net.ops().iter().all(|o| !matches!(o, crate::net::Op::Conv(c) if c.leaky.is_none() && c.k == 3)), + "every 3 × 3 convolution has its activation" + ); + assert_eq!((net.in_channels, net.out_channels(), net.depth()), (4, 3, 2)); + assert!(net.fits_tile(tile) && !net.fits_tile(30)); + + let _ = std::fs::remove_file(p); + } + + #[test] + fn an_explicit_empty_or_nonvector_bias_is_not_treated_as_missing() { + let w = TensorProto { name: "w".into(), dims: vec![12, 4, 1, 1], data_type: 1, float_data: vec![0.1; 48], ..Default::default() }; + let n = NodeProto { input: vec!["data".into(), "w".into(), "b".into()], ..Default::default() }; + for dims in [vec![0], vec![3, 4]] { + let b = TensorProto { + name: "b".into(), + float_data: vec![0.0; dims.iter().product::() as usize], + dims, + data_type: 1, + ..Default::default() + }; + let inits = HashMap::from([("w", &w), ("b", &b)]); + assert!(weights_of(&n, &inits, &mut 0).is_err()); + } + } + + #[test] + fn truncated_and_mutated_protobuf_is_bounded_and_never_panics() { + let good = unet_onnx(4, 3, 1, 7); + for n in (0..good.len()).step_by(31) { + let _ = read_bytes(&good[..n]); + } + for at in (0..good.len()).step_by(37) { + let mut bad = good.clone(); + bad[at] ^= 0xff; + let _ = read_bytes(&bad); + } + assert!(read_bytes(&[0x3a, 0xff, 0xff, 0xff, 0xff, 0x7f]).is_err()); + assert!(read_bytes(&[0x3a, 0, 0x3a, 0]).is_err()); + } + proptest::proptest! { + #[test] + fn arbitrary_model_bytes_never_panic(bytes in proptest::collection::vec(proptest::prelude::any::(), 0..4096)) { + let _ = read_bytes(&bytes); + } + } + + #[test] + fn what_the_description_cannot_say_is_unsupported_and_garbage_is_an_error() { + // not a model + let p = temp_model("junk", b"this is not an onnx file"); + assert!(matches!(read(&p), Err(NetError::Invalid(_)) | Err(NetError::Unsupported(_)))); + let _ = std::fs::remove_file(p); + // a missing file + assert!(read(Path::new("/does/not/exist.onnx")).is_err()); + } +} diff --git a/crates/denoise/src/onnx_proto.rs b/crates/denoise/src/onnx_proto.rs new file mode 100644 index 00000000..d8bbba41 --- /dev/null +++ b/crates/denoise/src/onnx_proto.rs @@ -0,0 +1,295 @@ +//! Bounded, borrowed decoding of the ONNX fields used by denoise, from the public Apache-2.0 schema: +//! https://github.com/onnx/onnx/blob/main/onnx/onnx.proto3 +//! No generated code, protoc, recursive graph decoding, or external tensor files. + +use crate::net::{MAX_OPS, MAX_WEIGHTS, NetError}; + +const MAX_ITEMS: usize = MAX_OPS * 4; +const MAX_STRING: usize = 4096; +pub(super) const MAX_BYTES: usize = MAX_WEIGHTS * 4 + 16 * 1024 * 1024; + +type Result = std::result::Result; +fn bad() -> NetError { + NetError::Invalid("malformed or oversized ONNX protobuf".into()) +} + +struct Fields<'a> { + rest: &'a [u8], + count: usize, +} +enum Wire<'a> { + Int(u64), + Bytes(&'a [u8]), + F32(f32), + Other, +} + +fn varint(rest: &mut &[u8]) -> Result { + let mut value = 0u64; + for shift in (0..70).step_by(7) { + let (b, tail) = rest.split_first().ok_or_else(bad)?; + *rest = tail; + if shift == 63 && *b > 1 { + return Err(bad()); + } + value |= u64::from(*b & 127) << shift; + if b & 128 == 0 { + return Ok(value); + } + } + Err(bad()) +} + +impl<'a> Fields<'a> { + fn new(rest: &'a [u8]) -> Self { + Self { rest, count: 0 } + } + fn take(&mut self, n: usize) -> Result<&'a [u8]> { + let out = self.rest.get(..n).ok_or_else(bad)?; + self.rest = self.rest.get(n..).ok_or_else(bad)?; + Ok(out) + } + fn next(&mut self) -> Result)>> { + if self.rest.is_empty() { + return Ok(None); + } + self.count += 1; + if self.count > MAX_WEIGHTS { + return Err(bad()); + } + let tag = varint(&mut self.rest)?; + let number = u32::try_from(tag >> 3).map_err(|_| bad())?; + if number == 0 || number >= 1 << 29 { + return Err(bad()); + } + let wire = match tag & 7 { + 0 => Wire::Int(varint(&mut self.rest)?), + 1 => { + self.take(8)?; + Wire::Other + } + 2 => { + let n = usize::try_from(varint(&mut self.rest)?).map_err(|_| bad())?; + Wire::Bytes(self.take(n)?) + } + 5 => { + let b: [u8; 4] = self.take(4)?.try_into().map_err(|_| bad())?; + Wire::F32(f32::from_le_bytes(b)) + } + _ => return Err(bad()), + }; + Ok(Some((number, wire))) + } +} + +fn string(b: &[u8]) -> Result { + if b.len() > MAX_STRING { + return Err(bad()); + } + std::str::from_utf8(b).map(str::to_owned).map_err(|_| bad()) +} +fn push(v: &mut Vec, value: T, cap: usize) -> Result<()> { + if v.len() >= cap { + return Err(bad()); + } + v.try_reserve(1).map_err(|_| bad())?; + v.push(value); + Ok(()) +} +fn ints(w: Wire<'_>, out: &mut Vec, cap: usize) -> Result<()> { + match w { + Wire::Int(v) => push(out, v as i64, cap), + Wire::Bytes(mut b) => { + while !b.is_empty() { + let v = varint(&mut b)?; + push(out, v as i64, cap)?; + } + Ok(()) + } + _ => Err(bad()), + } +} + +#[derive(Default)] +pub(super) struct AttributeProto { + pub name: String, + pub f: f32, + pub i: i64, + pub s: Vec, + pub ints: Vec, +} +#[derive(Default)] +pub(super) struct NodeProto { + pub input: Vec, + pub output: Vec, + pub op_type: String, + pub attribute: Vec, +} +#[derive(Default)] +pub(super) struct TensorProto<'a> { + pub name: String, + pub dims: Vec, + pub data_type: i32, + pub raw_data: &'a [u8], + pub float_data: Vec, +} +pub(super) struct ValueInfoProto { + pub name: String, + pub channels: Option, +} +#[derive(Default)] +pub(super) struct Graph<'a> { + pub node: Vec, + pub initializer: Vec>, + pub input: Vec, + pub output: Vec, +} + +fn attribute(b: &[u8]) -> Result { + let mut a = AttributeProto::default(); + let mut fields = Fields::new(b); + while let Some((n, w)) = fields.next()? { + match (n, w) { + (1, Wire::Bytes(b)) => a.name = string(b)?, + (2, Wire::F32(v)) => a.f = v, + (3, Wire::Int(v)) => a.i = v as i64, + (4, Wire::Bytes(b)) => { + if b.len() > MAX_STRING { + return Err(bad()); + } + a.s = b.to_vec(); + } + (8, w) => ints(w, &mut a.ints, 16)?, + (20, Wire::Int(1 | 2 | 3 | 7)) => {} + _ => return Err(NetError::Unsupported("unknown attribute value".into())), + } + } + Ok(a) +} +fn node(b: &[u8]) -> Result { + let mut a = NodeProto::default(); + let mut fields = Fields::new(b); + while let Some((n, w)) = fields.next()? { + match (n, w) { + (1, Wire::Bytes(b)) => push(&mut a.input, string(b)?, 4)?, + (2, Wire::Bytes(b)) => push(&mut a.output, string(b)?, 2)?, + (4, Wire::Bytes(b)) => a.op_type = string(b)?, + (5, Wire::Bytes(b)) => push(&mut a.attribute, attribute(b)?, 16)?, + (7, Wire::Bytes(b)) if !b.is_empty() => return Err(NetError::Unsupported("a custom operator domain".into())), + _ => {} + } + } + Ok(a) +} +fn tensor(b: &[u8]) -> Result> { + let mut a = TensorProto::default(); + let mut fields = Fields::new(b); + while let Some((n, w)) = fields.next()? { + match (n, w) { + (1, w) => ints(w, &mut a.dims, 8)?, + (2, Wire::Int(v)) => a.data_type = i32::try_from(v).map_err(|_| bad())?, + (4, Wire::F32(v)) => push(&mut a.float_data, v, MAX_WEIGHTS)?, + (4, Wire::Bytes(b)) => { + if !b.len().is_multiple_of(4) { + return Err(bad()); + } + for chunk in b.as_chunks::<4>().0 { + push(&mut a.float_data, f32::from_le_bytes(*chunk), MAX_WEIGHTS)?; + } + } + (8, Wire::Bytes(b)) => a.name = string(b)?, + (9, Wire::Bytes(b)) => { + if b.len() > MAX_WEIGHTS * 4 { + return Err(bad()); + } + a.raw_data = b; + } + (13, _) | (14, Wire::Int(1..)) => return Err(NetError::Unsupported("external tensor data".into())), + _ => {} + } + } + Ok(a) +} +fn channel_shape(b: &[u8]) -> Result> { + let mut fields = Fields::new(b); + let mut dim = 0; + let mut channels = None; + while let Some((n, w)) = fields.next()? { + if let (1, Wire::Bytes(b)) = (n, w) { + if dim >= 8 { + return Err(bad()); + } + let mut df = Fields::new(b); + while let Some((n, w)) = df.next()? { + if let (1, Wire::Int(v)) = (n, w) + && dim == 1 + { + channels = usize::try_from(v).ok().filter(|&v| v > 0); + } + } + dim += 1; + } + } + Ok(channels) +} +fn channel_type(b: &[u8]) -> Result> { + let mut fields = Fields::new(b); + while let Some((n, w)) = fields.next()? { + if let (1, Wire::Bytes(b)) = (n, w) { + let mut tf = Fields::new(b); + let (mut elem, mut channels) = (0, None); + while let Some((n, w)) = tf.next()? { + match (n, w) { + (1, Wire::Int(v)) => elem = v, + (2, Wire::Bytes(b)) => channels = channel_shape(b)?, + _ => {} + } + } + return Ok(if elem == 1 { channels } else { None }); + } + } + Ok(None) +} +fn value_info(b: &[u8]) -> Result { + let mut out = ValueInfoProto { name: String::new(), channels: None }; + let mut fields = Fields::new(b); + while let Some((n, w)) = fields.next()? { + match (n, w) { + (1, Wire::Bytes(b)) => out.name = string(b)?, + (2, Wire::Bytes(b)) => out.channels = channel_type(b)?, + _ => {} + } + } + Ok(out) +} +fn graph(b: &[u8]) -> Result> { + let mut out = Graph::default(); + let mut fields = Fields::new(b); + while let Some((n, w)) = fields.next()? { + match (n, w) { + (1, Wire::Bytes(b)) => push(&mut out.node, node(b)?, MAX_ITEMS)?, + (5, Wire::Bytes(b)) => push(&mut out.initializer, tensor(b)?, MAX_ITEMS)?, + (11, Wire::Bytes(b)) => push(&mut out.input, value_info(b)?, MAX_ITEMS)?, + (12, Wire::Bytes(b)) => push(&mut out.output, value_info(b)?, 1)?, + (15, _) => return Err(NetError::Unsupported("sparse initializers".into())), + _ => {} + } + } + Ok(out) +} +pub(super) fn decode(b: &[u8]) -> Result> { + if b.len() > MAX_BYTES { + return Err(bad()); + } + let mut fields = Fields::new(b); + let mut out = None; + while let Some((n, w)) = fields.next()? { + if let (7, Wire::Bytes(b)) = (n, w) { + if out.is_some() { + return Err(bad()); + } + out = Some(graph(b)?); + } + } + out.ok_or_else(bad) +} diff --git a/crates/denoise/src/product.rs b/crates/denoise/src/product.rs new file mode 100644 index 00000000..1e393885 --- /dev/null +++ b/crates/denoise/src/product.rs @@ -0,0 +1,595 @@ +//! The denoised picture as cached data, and the blend the Amount slider controls. +//! +//! A product is camera RGB at the sensor's resolution (what the model gave, before the raw's own opcodes and +//! crop), kept as half floats in strips of rows, each strip byte-shuffled and deflated with its own checksum. +//! It is derived data: any problem reading it (missing, truncated, corrupt, made for another photo or model) is +//! just a cache miss, never a panic, and the picture can be made again. +//! +//! File: `LCDN`, version, header length, header JSON (`width`, `height`, `strip`, `key`), then for each strip its +//! compressed length and CRC-32, then the strips' bytes. `key` says what the product was made from (the photo's +//! content, the model, this code's version, the settings it depends on); a file with another key is stale. + +use std::io::{Read, Write}; +use std::path::Path; + +use lightcraft_raster::Rgb32f; +use rayon::prelude::*; +use serde::{Deserialize, Serialize}; + +const MAGIC: &[u8; 4] = b"LCDN"; +const VERSION: u8 = 1; +const MAX_HEADER: usize = 4096; +const MAX_SIDE: usize = 1 << 17; +/// Most pixels a product may have (the denoiser's own limit, [`crate::run::MAX_PIXELS`]). +const MAX_PIXELS: usize = crate::run::MAX_PIXELS; +/// Rows per strip: about a megapixel of three planes at 24 MP widths. +const STRIP_ROWS: usize = 128; +/// Bumped when the meaning of a product changes (included in keys by the caller). +pub const ALGORITHM: u32 = 1; + +#[derive(Debug, thiserror::Error)] +pub enum ProductError { + #[error("the denoised picture could not be read or written: {0}")] + Io(String), + #[error("not a usable denoised picture: {0}")] + Format(String), + /// A product made from something else (another photo state, model or version). + #[error("the denoised picture is out of date")] + Stale, +} + +impl From for ProductError { + fn from(e: std::io::Error) -> Self { + ProductError::Io(e.to_string()) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Header { + pub width: usize, + pub height: usize, + pub strip: usize, + pub key: String, +} + +/// `f32` as IEEE half-float bits, rounded to nearest even; beyond the range it saturates, NaN becomes 0. +pub fn to_f16(v: f32) -> u16 { + if !v.is_finite() { + return if v.is_nan() { + 0 + } else if v > 0.0 { + 0x7bff + } else { + 0xfbff + }; + } + let b = v.to_bits(); + let sign = ((b >> 16) & 0x8000) as u16; + let exp = ((b >> 23) & 0xff) as i32 - 127 + 15; + let man = b & 0x7f_ffff; + if exp >= 31 { + return sign | 0x7bff; + } + if exp <= 0 { + if exp < -10 { + return sign; + } + let man = man | 0x80_0000; + let shift = (14 - exp) as u32; + let (half, rem, halfway) = (man >> shift, man & ((1u32 << shift) - 1), 1u32 << (shift - 1)); + let mut h = half as u16; + if rem > halfway || (rem == halfway && h & 1 == 1) { + h += 1; + } + return sign | h; + } + let mut h = (((exp as u32) << 10) | (man >> 13)) as u16; + let rem = man & 0x1fff; + if rem > 0x1000 || (rem == 0x1000 && h & 1 == 1) { + h += 1; + } + if h >= 0x7c00 { + h = 0x7bff; + } + sign | h +} + +/// Half-float bits as `f32`. +pub fn from_f16(h: u16) -> f32 { + let sign = if h & 0x8000 != 0 { -1.0f32 } else { 1.0 }; + let exp = i32::from((h >> 10) & 0x1f); + let man = f32::from(h & 0x3ff); + match exp { + 0 => sign * man * 2f32.powi(-24), + 31 => sign * 65504.0, + _ => sign * (1.0 + man / 1024.0) * 2f32.powi(exp - 15), + } +} + +fn crc(bytes: &[u8]) -> u32 { + // CRC-32 (IEEE), bitwise: strips are small and this runs once per strip + let mut table = [0u32; 256]; + for (i, t) in table.iter_mut().enumerate() { + let mut c = i as u32; + for _ in 0..8 { + c = if c & 1 != 0 { 0xedb8_8320 ^ (c >> 1) } else { c >> 1 }; + } + *t = c; + } + !bytes.iter().fold(!0u32, |c, &b| table[((c ^ u32::from(b)) & 0xff) as usize] ^ (c >> 8)) +} + +/// One strip as bytes: for each plane (R, G, B) the low bytes of its half floats, then the high bytes. +fn encode_strip(img: &Rgb32f, y0: usize, y1: usize) -> Vec { + let n = (y1 - y0) * img.width; + let mut raw = vec![0u8; 6 * n]; + let rows = img.data.get(y0 * img.width..y1 * img.width).unwrap_or(&[]); + for (i, px) in rows.iter().enumerate() { + for (c, &v) in px.iter().enumerate() { + let h = to_f16(v); + if let Some(lo) = raw.get_mut(2 * c * n + i) { + *lo = h as u8; + } + if let Some(hi) = raw.get_mut((2 * c + 1) * n + i) { + *hi = (h >> 8) as u8; + } + } + } + miniz_oxide::deflate::compress_to_vec(&raw, 3) +} + +/// Write `img` to `path` (through a temporary file next to it, so a reader never sees half a product). +pub fn write(path: &Path, img: &Rgb32f, key: &str) -> Result<(), ProductError> { + if img.width == 0 || img.height == 0 || img.width > MAX_SIDE || img.height > MAX_SIDE || img.data.len() != img.width * img.height { + return Err(ProductError::Format("the picture has no usable size".into())); + } + let strips: Vec<(usize, usize)> = (0..img.height).step_by(STRIP_ROWS).map(|y| (y, (y + STRIP_ROWS).min(img.height))).collect(); + let packed: Vec> = strips.par_iter().map(|&(a, b)| encode_strip(img, a, b)).collect(); + let header = serde_json::to_vec(&Header { width: img.width, height: img.height, strip: STRIP_ROWS, key: key.to_string() }) + .map_err(|e| ProductError::Format(e.to_string()))?; + let tmp = path.with_extension("tmp"); + let result = (|| -> Result<(), ProductError> { + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir)?; + } + let mut f = std::io::BufWriter::new(std::fs::File::create(&tmp)?); + f.write_all(MAGIC)?; + f.write_all(&[VERSION, 0, 0, 0])?; + f.write_all(&(header.len() as u32).to_le_bytes())?; + f.write_all(&header)?; + for p in &packed { + f.write_all(&(p.len() as u32).to_le_bytes())?; + f.write_all(&crc(p).to_le_bytes())?; + } + for p in &packed { + f.write_all(p)?; + } + f.flush()?; + drop(f); + std::fs::rename(&tmp, path)?; + Ok(()) + })(); + if result.is_err() { + let _ = std::fs::remove_file(&tmp); + } + result +} + +/// The parsed start of a product file: header, each strip's (compressed length, CRC-32), and the file length the +/// start says the whole file has. +struct Start { + header: Header, + table: Vec<(usize, u32)>, + expected_len: u64, +} + +fn read_start(r: &mut impl Read) -> Result { + let mut fixed = [0u8; 12]; + r.read_exact(&mut fixed)?; + if fixed.get(0..4) != Some(&MAGIC[..]) || fixed.get(4) != Some(&VERSION) { + return Err(ProductError::Format("not a denoised picture of this version".into())); + } + let len = fixed.get(8..12).and_then(|b| <[u8; 4]>::try_from(b).ok()).map(u32::from_le_bytes).unwrap_or(u32::MAX) as usize; + if len > MAX_HEADER { + return Err(ProductError::Format("the header is too large".into())); + } + let mut hb = vec![0u8; len]; + r.read_exact(&mut hb)?; + let h: Header = serde_json::from_slice(&hb).map_err(|e| ProductError::Format(e.to_string()))?; + let pixels = h.width.saturating_mul(h.height); + if h.width == 0 || h.height == 0 || h.width > MAX_SIDE || h.height > MAX_SIDE || pixels > MAX_PIXELS || h.strip != STRIP_ROWS { + return Err(ProductError::Format("the size is not usable".into())); + } + let n = h.height.div_ceil(h.strip); + let mut table = Vec::with_capacity(n); + let mut total = 12 + len as u64 + 8 * n as u64; + for _ in 0..n { + let mut e = [0u8; 8]; + r.read_exact(&mut e)?; + let l = u32::from_le_bytes([e[0], e[1], e[2], e[3]]) as usize; + let c = u32::from_le_bytes([e[4], e[5], e[6], e[7]]); + // a strip never compresses to much more than its raw size + if l > 12 * h.strip * h.width + 4096 { + return Err(ProductError::Format("a strip is larger than it can be".into())); + } + total += l as u64; + table.push((l, c)); + } + Ok(Start { header: h, table, expected_len: total }) +} + +/// The header of the product at `path`, when it is one. +pub fn read_header_at(path: &Path) -> Result { + let mut f = std::io::BufReader::new(std::fs::File::open(path)?); + Ok(read_start(&mut f)?.header) +} + +/// Whether `path` holds a complete-looking product made from `key` (the header and the file's length are checked; +/// strips are checked when read). +pub fn is_current(path: &Path, key: &str) -> bool { + let check = || -> Result { + let mut f = std::io::BufReader::new(std::fs::File::open(path)?); + let s = read_start(&mut f)?; + Ok(s.header.key == key && std::fs::metadata(path)?.len() == s.expected_len) + }; + check().unwrap_or(false) +} + +/// A rectangle of a product, in the product's pixels. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Window { + pub x: usize, + pub y: usize, + pub width: usize, + pub height: usize, +} + +/// Half floats as `f32`, for all 65536 bit patterns (a table is much faster than computing each). +fn f16_table() -> &'static [f32] { + static TABLE: std::sync::OnceLock> = std::sync::OnceLock::new(); + TABLE.get_or_init(|| (0..=u16::MAX).map(from_f16).collect()) +} + +/// Strips decoded together (they are inflated in parallel, then folded in order). +const GROUP: usize = 8; + +/// One strip, checked and inflated: for each plane the low bytes of its half floats, then the high bytes. +fn decode_strip(index: usize, packed: &[u8], expect_crc: u32, n: usize) -> Result, ProductError> { + if crc(packed) != expect_crc { + return Err(ProductError::Format(format!("strip {index} is damaged"))); + } + let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(packed, 6 * n).map_err(|e| ProductError::Format(format!("strip {index}: {e:?}")))?; + if raw.len() != 6 * n { + return Err(ProductError::Format(format!("strip {index} has the wrong size"))); + } + Ok(raw) +} + +/// Read the product at `path`, made from `key`, with every `factor × factor` block of pixels averaged into one +/// (`factor` 1 = full size). The result is camera RGB. +/// +/// With a `window`, only that rectangle is read and the blocks start at its corner; only whole blocks are made, so +/// the result is `window.width / factor × window.height / factor` (the same blocks a binned development of the same +/// crop makes). Without one the whole product is read and blocks at the right and bottom edges may be smaller. +/// +/// With `clip` (white is 1), a block in which any sample of a colour is at or above it takes that colour's +/// maximum instead of the mean, so a clipped highlight stays clipped for highlight reconstruction. +pub fn read_window(path: &Path, key: &str, window: Option, factor: usize, clip: Option) -> Result { + let factor = factor.clamp(1, 1 << 12); + let mut f = std::io::BufReader::new(std::fs::File::open(path)?); + let Start { header: h, table, expected_len } = read_start(&mut f)?; + if h.key != key { + return Err(ProductError::Stale); + } + if std::fs::metadata(path)?.len() != expected_len { + return Err(ProductError::Format("the file is not the length its header says".into())); + } + let w = window.unwrap_or(Window { x: 0, y: 0, width: h.width, height: h.height }); + let fits = |a: usize, b: usize, limit: usize| a.checked_add(b).is_some_and(|e| e <= limit); + if w.width == 0 || w.height == 0 || !fits(w.x, w.width, h.width) || !fits(w.y, w.height, h.height) { + return Err(ProductError::Format("the window does not lie inside the picture".into())); + } + // whole blocks for a window; partial ones at the edges when the whole picture is read + let whole = window.is_none(); + let (ow, oh) = if whole { (w.width.div_ceil(factor), w.height.div_ceil(factor)) } else { (w.width / factor, w.height / factor) }; + if ow == 0 || oh == 0 { + return Err(ProductError::Format("the window is smaller than one block".into())); + } + // the picture rows and columns the blocks cover + let (x_end, y_end) = if whole { (w.x + w.width, w.y + w.height) } else { (w.x + ow * factor, w.y + oh * factor) }; + let mut out = Rgb32f { width: ow, height: oh, data: vec![[0.0; 3]; ow * oh] }; + let lut = f16_table(); + let value = |raw: &[u8], c: usize, n: usize, i: usize| -> f32 { + let (lo, hi) = (raw.get(2 * c * n + i).copied().unwrap_or(0), raw.get((2 * c + 1) * n + i).copied().unwrap_or(0)); + lut.get(usize::from(lo) | usize::from(hi) << 8).copied().unwrap_or(0.0) + }; + // which strips are needed, and where each starts in the file (after the header and the strip table) + let wanted = |s: usize| s * h.strip < y_end && (s + 1) * h.strip > w.y; + let (mut sum, mut peak) = (vec![[0f32; 3]; ow], vec![[f32::MIN; 3]; ow]); + let (mut rows_in, mut oy) = (0usize, 0usize); + let strips: Vec = (0..table.len()).filter(|&s| wanted(s)).collect(); + // skip the strips before the first one wanted + let skipped: usize = table.iter().take(strips.first().copied().unwrap_or(0)).map(|e| e.0).sum(); + f.seek_relative(i64::try_from(skipped).map_err(|_| ProductError::Format("the file is too large".into()))?)?; + for group in strips.chunks(GROUP) { + let mut packed = Vec::with_capacity(group.len()); + for &s in group { + let Some(&(len, _)) = table.get(s) else { return Err(ProductError::Format("a strip is missing".into())) }; + let mut p = vec![0u8; len]; + f.read_exact(&mut p)?; + packed.push(p); + } + let decoded: Vec, ProductError>> = group + .par_iter() + .zip(packed.par_iter()) + .map(|(&s, p)| { + let rows = h.height.saturating_sub(s * h.strip).min(h.strip); + decode_strip(s, p, table.get(s).map_or(0, |e| e.1), rows * h.width) + }) + .collect(); + for (&s, raw) in group.iter().zip(decoded) { + let raw = raw?; + let y0 = s * h.strip; + let rows = h.height.saturating_sub(y0).min(h.strip); + let n = rows * h.width; + for r in 0..rows { + let y = y0 + r; + if y < w.y || y >= y_end { + continue; + } + let base = r * h.width; + for (ox, (sm, pk)) in sum.iter_mut().zip(peak.iter_mut()).enumerate() { + let x0 = w.x + ox * factor; + let x1 = (x0 + factor).min(x_end); + for x in x0..x1 { + for c in 0..3 { + let v = value(&raw, c, n, base + x); + if let (Some(s_), Some(p_)) = (sm.get_mut(c), pk.get_mut(c)) { + *s_ += v; + *p_ = p_.max(v); + } + } + } + } + rows_in += 1; + if rows_in == factor || y + 1 == y_end { + for (ox, (sm, pk)) in sum.iter_mut().zip(peak.iter_mut()).enumerate() { + // the block's size at the right and bottom edges may be smaller + let bw = (x_end - (w.x + ox * factor)).min(factor); + let k = 1.0 / (rows_in * bw).max(1) as f32; + if let Some(o) = out.data.get_mut(oy * ow + ox) { + for c in 0..3 { + let (s_, p_) = (sm.get(c).copied().unwrap_or(0.0), pk.get(c).copied().unwrap_or(0.0)); + if let Some(slot) = o.get_mut(c) { + *slot = if factor > 1 && clip.is_some_and(|t| p_ >= t) { p_ } else { s_ * k }; + } + } + } + *sm = [0.0; 3]; + *pk = [f32::MIN; 3]; + } + rows_in = 0; + oy += 1; + } + } + } + } + Ok(out) +} + +/// [`read_window`] of the whole product: every `factor × factor` block averaged into one. +pub fn read(path: &Path, key: &str, factor: usize) -> Result { + read_window(path, key, None, factor, None) +} + +/// `base + (denoised − base) · amount` for every pixel (`amount` 0 to 1), in place in `base`. The two pictures must +/// be the same size; if they are not, `base` is left as it was and `false` is returned. +pub fn blend(base: &mut Rgb32f, denoised: &Rgb32f, amount: f32) -> bool { + if base.width != denoised.width || base.height != denoised.height { + return false; + } + let a = if amount.is_finite() { amount.clamp(0.0, 1.0) } else { 0.0 }; + base.data.par_iter_mut().zip(denoised.data.par_iter()).for_each(|(b, d)| { + for c in 0..3 { + b[c] += (d[c] - b[c]) * a; + } + }); + true +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp(name: &str) -> std::path::PathBuf { + std::env::temp_dir().join(format!("lc-denoise-{name}-{}.lcdn", std::process::id())) + } + + fn picture(w: usize, h: usize) -> Rgb32f { + Rgb32f { + width: w, + height: h, + data: (0..w * h).map(|i| [(i % w) as f32 / w as f32, (i / w) as f32 / h as f32, 0.001 + 0.5 * ((i % 7) as f32 / 7.0)]).collect(), + } + } + + #[test] + fn half_floats_round_trip_closely() { + for v in [0.0f32, 1.0, -1.0, 0.5, 0.333_333, 1e-4, 6.1e-5, 3e-6, 123.456, 65000.0, -0.002] { + let back = from_f16(to_f16(v)); + assert!((back - v).abs() <= v.abs() * 0.001 + 1e-7, "{v} -> {back}"); + } + assert_eq!(from_f16(to_f16(1e10)), 65504.0); + assert_eq!(to_f16(f32::NAN), 0); + assert_eq!(from_f16(to_f16(f32::NEG_INFINITY)), -65504.0); + } + + #[test] + fn a_product_survives_a_round_trip_at_full_and_reduced_size() { + let (w, h) = (301, 517); + let img = picture(w, h); + let p = temp("round"); + write(&p, &img, "k1").unwrap(); + assert!(is_current(&p, "k1") && !is_current(&p, "k2")); + assert_eq!(read_header_at(&p).unwrap().width, w); + let full = read(&p, "k1", 1).unwrap(); + assert_eq!((full.width, full.height), (w, h)); + for (a, b) in full.data.iter().zip(&img.data) { + for c in 0..3 { + assert!((a[c] - b[c]).abs() < 0.002, "{a:?} vs {b:?}"); + } + } + // averaged into blocks of 4: the sizes round up, and the mean of the picture is kept + let small = read(&p, "k1", 4).unwrap(); + assert_eq!((small.width, small.height), (w.div_ceil(4), h.div_ceil(4))); + let mean = |i: &Rgb32f| i.data.iter().map(|p| f64::from(p[1])).sum::() / i.data.len() as f64; + assert!((mean(&small) - mean(&img)).abs() < 0.01); + // a factor that does not divide the strip height is the same average + let odd = read(&p, "k1", 5).unwrap(); + assert_eq!((odd.width, odd.height), (w.div_ceil(5), h.div_ceil(5))); + assert!((mean(&odd) - mean(&img)).abs() < 0.01); + assert!(matches!(read(&p, "other", 1), Err(ProductError::Stale))); + std::fs::remove_file(&p).unwrap(); + } + + /// What binning a window of `img` by `k` means, written the slow obvious way. + fn blocks(img: &Rgb32f, win: Window, k: usize, clip: Option) -> Rgb32f { + let (ow, oh) = (win.width / k, win.height / k); + let mut out = Rgb32f { width: ow, height: oh, data: vec![[0.0; 3]; ow * oh] }; + for by in 0..oh { + for bx in 0..ow { + for c in 0..3 { + let vals: Vec = (0..k * k).map(|i| img.data[(win.y + by * k + i / k) * img.width + win.x + bx * k + i % k][c]).collect(); + let max = vals.iter().copied().fold(f32::MIN, f32::max); + out.data[by * ow + bx][c] = + if k > 1 && clip.is_some_and(|t| max >= t) { max } else { vals.iter().sum::() / vals.len() as f32 }; + } + } + } + out + } + + #[test] + fn a_window_is_read_as_the_blocks_of_that_crop() { + let (w, h) = (301, 517); + let mut img = picture(w, h); + // a clipped sample in the middle of a block + img.data[(130 + 2) * w + 61 + 1] = [1.0, 0.4, 0.4]; + let p = temp("window"); + write(&p, &img, "k1").unwrap(); + let near = |a: &Rgb32f, b: &Rgb32f| { + assert_eq!((a.width, a.height), (b.width, b.height)); + for (x, y) in a.data.iter().zip(&b.data) { + for c in 0..3 { + assert!((x[c] - y[c]).abs() < 0.003, "{x:?} vs {y:?}"); + } + } + }; + // windows at odd places, across strip boundaries (128 rows), with factors that do and do not divide them + for (win, k) in [ + (Window { x: 3, y: 5, width: 200, height: 300 }, 1), + (Window { x: 3, y: 5, width: 200, height: 300 }, 4), + (Window { x: 61, y: 130, width: 121, height: 250 }, 5), + (Window { x: 0, y: 0, width: w, height: h }, 2), + (Window { x: 1, y: 380, width: 90, height: 137 }, 3), + ] { + let got = read_window(&p, "k1", Some(win), k, None).unwrap(); + near(&got, &blocks(&img, win, k, None)); + } + // with a clip level the block with the clipped sample keeps its highest value, the others stay means + let win = Window { x: 61, y: 130, width: 40, height: 20 }; + let clipped = read_window(&p, "k1", Some(win), 4, Some(0.99)).unwrap(); + near(&clipped, &blocks(&img, win, 4, Some(0.99))); + assert!(clipped.data[0][0] > 0.99, "the clipped block stays clipped: {:?}", clipped.data[0]); + let plain = read_window(&p, "k1", Some(win), 4, None).unwrap(); + assert!(plain.data[0][0] < 0.99, "an average falls below the clip level"); + // windows that do not fit, or are smaller than a block, are errors + for bad in [ + Window { x: 0, y: 0, width: 0, height: 5 }, + Window { x: 290, y: 0, width: 20, height: 5 }, + Window { x: 0, y: 510, width: 5, height: 10 }, + Window { x: usize::MAX, y: 0, width: 2, height: 2 }, + Window { x: 0, y: 0, width: 3, height: 3 }, + ] { + assert!(read_window(&p, "k1", Some(bad), 4, None).is_err(), "{bad:?}"); + } + std::fs::remove_file(&p).unwrap(); + } + + #[test] + fn a_smooth_picture_compresses_well() { + let img = Rgb32f { + width: 512, + height: 512, + data: (0..512 * 512) + .map(|i| { + let v = 0.2 + 0.0001 * (i % 512) as f32; + [v, v * 0.8, v * 0.6] + }) + .collect(), + }; + let p = temp("size"); + write(&p, &img, "k").unwrap(); + let bytes = std::fs::metadata(&p).unwrap().len() as usize; + assert!(bytes < 512 * 512 * 6 / 2, "{bytes} bytes for a smooth picture"); + std::fs::remove_file(&p).unwrap(); + } + + #[test] + fn damaged_files_are_errors_not_panics() { + let img = picture(100, 300); + let p = temp("damage"); + write(&p, &img, "k").unwrap(); + let good = std::fs::read(&p).unwrap(); + // truncated at every length + for cut in (0..good.len()).step_by(37) { + std::fs::write(&p, &good[..cut]).unwrap(); + assert!(read(&p, "k", 1).is_err(), "cut at {cut}"); + assert!(!is_current(&p, "k")); + } + // a flipped byte anywhere is caught by a checksum, the header parse, or the key + for at in (0..good.len()).step_by(53) { + let mut bad = good.clone(); + bad[at] ^= 0xff; + std::fs::write(&p, &bad).unwrap(); + let _ = read(&p, "k", 1); // must not panic; most are errors + } + // a header that claims an absurd size + let mut huge = b"LCDN\x01\0\0\0".to_vec(); + let hdr = br#"{"width":4000000000,"height":4000000000,"strip":128,"key":"k"}"#; + huge.extend((hdr.len() as u32).to_le_bytes()); + huge.extend(hdr); + std::fs::write(&p, &huge).unwrap(); + assert!(matches!(read(&p, "k", 1), Err(ProductError::Format(_)))); + std::fs::write(&p, b"").unwrap(); + assert!(read(&p, "k", 1).is_err()); + assert!(read(&temp("missing"), "k", 1).is_err()); + std::fs::remove_file(&p).unwrap(); + } + + #[test] + fn writing_rejects_unusable_pictures() { + let p = temp("bad"); + assert!(write(&p, &Rgb32f { width: 0, height: 0, data: vec![] }, "k").is_err()); + assert!(write(&p, &Rgb32f { width: 3, height: 3, data: vec![[0.0; 3]; 8] }, "k").is_err()); + assert!(!p.exists()); + } + + #[test] + fn blend_mixes_by_amount_and_rejects_other_sizes() { + let mut base = Rgb32f { width: 2, height: 1, data: vec![[0.0; 3], [1.0; 3]] }; + let dn = Rgb32f { width: 2, height: 1, data: vec![[1.0; 3], [0.0; 3]] }; + assert!(blend(&mut base, &dn, 0.25)); + assert_eq!(base.data, vec![[0.25; 3], [0.75; 3]]); + assert!(blend(&mut base, &dn, f32::NAN)); + assert_eq!(base.data, vec![[0.25; 3], [0.75; 3]]); + assert!(!blend(&mut base, &Rgb32f { width: 1, height: 1, data: vec![[0.0; 3]] }, 1.0)); + assert!(blend(&mut base, &dn, 1.0)); + assert_eq!(base.data, dn.data); + } + + #[test] + fn crc_matches_the_standard_check_value() { + assert_eq!(crc(b"123456789"), 0xcbf4_3926); + } +} diff --git a/crates/denoise/src/reference.rs b/crates/denoise/src/reference.rs new file mode 100644 index 00000000..ef58b5b9 --- /dev/null +++ b/crates/denoise/src/reference.rs @@ -0,0 +1,133 @@ +//! A plain-loop interpreter of a [`Net`], slow and obviously right: the reference the parser and the GPU runner are +//! tested against (tract's answer on the real model is the other). It is not used on photos. + +use crate::net::{Net, NetError, Op, TensorId}; +use crate::run::{Error, TileRunner}; + +/// A tensor in `[channels][side][side]` order. +struct T { + channels: usize, + side: usize, + data: Vec, +} + +/// Run `net` on `input` (`in_channels` planes of `tile × tile`). +pub fn run(net: &Net, tile: usize, input: &[f32]) -> Result, NetError> { + if !net.fits_tile(tile) || input.len() != net.in_channels * tile * tile { + return Err(NetError::Invalid("the tile does not fit the network".into())); + } + let mut tensors: Vec = vec![T { channels: net.in_channels, side: tile, data: input.to_vec() }]; + let get = |tensors: &[T], id: TensorId| -> Result { + if id < tensors.len() { Ok(id) } else { Err(NetError::Invalid("a layer reads a tensor that does not exist yet".into())) } + }; + for op in net.ops() { + let out = match op { + Op::Conv(c) => { + let a = get(&tensors, c.src)?; + let b = c.src2.map(|s| get(&tensors, s)).transpose()?; + let side = tensors[a].side; + let pad = (c.k - 1) / 2; + let mut data = vec![0f32; c.cout * side * side]; + // the input channels in order: the first source's, then the second's + let planes: Vec<&[f32]> = + std::iter::once(&tensors[a]).chain(b.map(|b| &tensors[b])).flat_map(|t| t.data.chunks(side * side)).collect(); + for co in 0..c.cout { + for y in 0..side { + for x in 0..side { + let mut sum = c.bias[co]; + for (ci, plane) in planes.iter().enumerate() { + for ky in 0..c.k { + for kx in 0..c.k { + let (yy, xx) = ((y + ky) as isize - pad as isize, (x + kx) as isize - pad as isize); + if yy < 0 || xx < 0 || yy >= side as isize || xx >= side as isize { + continue; + } + sum += plane[yy as usize * side + xx as usize] * c.weight[((co * c.cin + ci) * c.k + ky) * c.k + kx]; + } + } + } + data[(co * side + y) * side + x] = leaky(sum, c.leaky); + } + } + } + T { channels: c.cout, side, data } + } + Op::ConvT2(c) => { + let a = get(&tensors, c.src)?; + let side = tensors[a].side; + let out_side = 2 * side; + let mut data = vec![0f32; c.cout * out_side * out_side]; + for co in 0..c.cout { + for y in 0..side { + for x in 0..side { + for dy in 0..2 { + for dx in 0..2 { + let mut sum = c.bias[co]; + for ci in 0..c.cin { + sum += tensors[a].data[(ci * side + y) * side + x] * c.weight[((ci * c.cout + co) * 2 + dy) * 2 + dx]; + } + data[(co * out_side + 2 * y + dy) * out_side + 2 * x + dx] = leaky(sum, c.leaky); + } + } + } + } + } + T { channels: c.cout, side: out_side, data } + } + Op::MaxPool2 { src } => { + let a = get(&tensors, *src)?; + let (channels, side) = (tensors[a].channels, tensors[a].side / 2); + let mut data = vec![0f32; channels * side * side]; + for ch in 0..channels { + for y in 0..side { + for x in 0..side { + let at = |dy: usize, dx: usize| tensors[a].data[(ch * 2 * side + 2 * y + dy) * 2 * side + 2 * x + dx]; + data[(ch * side + y) * side + x] = at(0, 0).max(at(0, 1)).max(at(1, 0)).max(at(1, 1)); + } + } + } + T { channels, side, data } + } + Op::DepthToSpace2 { src } => { + let a = get(&tensors, *src)?; + let (channels, side) = (tensors[a].channels / 4, tensors[a].side); + let out_side = 2 * side; + let mut data = vec![0f32; channels * out_side * out_side]; + for ch in 0..channels { + for y in 0..side { + for x in 0..side { + for dy in 0..2 { + for dx in 0..2 { + data[(ch * out_side + 2 * y + dy) * out_side + 2 * x + dx] = + tensors[a].data[((4 * ch + 2 * dy + dx) * side + y) * side + x]; + } + } + } + } + } + T { channels, side: out_side, data } + } + }; + tensors.push(out); + } + tensors.pop().map(|t| t.data).ok_or_else(|| NetError::Invalid("no layers".into())) +} + +fn leaky(v: f32, alpha: Option) -> f32 { + match alpha { + Some(a) if v < 0.0 => v * a, + _ => v, + } +} + +/// The reference as a [`TileRunner`] (for tests of the code around the model). +pub struct ReferenceRunner { + pub net: Net, + pub tile: usize, +} + +impl TileRunner for ReferenceRunner { + fn run(&self, input: &[f32]) -> Result, Error> { + run(&self.net, self.tile, input).map_err(|e| Error::Runtime(e.to_string())) + } +} diff --git a/crates/denoise/src/run.rs b/crates/denoise/src/run.rs new file mode 100644 index 00000000..5bc763ce --- /dev/null +++ b/crates/denoise/src/run.rs @@ -0,0 +1,582 @@ +//! Denoising a whole Bayer mosaic with a tile model: cut it into overlapping tiles, run the model on each, bring +//! each tile's scale back to the input's, keep clipped highlights as they were, and blend the tiles together. +//! +//! The model itself is a [`TileRunner`], so everything around it (packing, tiling, scale matching, highlight +//! protection, blending, cancelling, progress) is tested without one. + +use std::sync::atomic::{AtomicBool, Ordering}; + +use lightcraft_raster::Rgb32f; +use rayon::prelude::*; + +use crate::bayer::{Layout, pack_tile}; +use crate::manifest::Gain; +use crate::tiles::{starts, weight}; + +/// Most pixels one picture may have (guards the allocations that follow from its size). +pub const MAX_PIXELS: usize = 400_000_000; +/// A tile whose mean signal is below this (white = 1) is too dark to measure its scale from. +const MIN_SIGNAL: f32 = 0.003; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("{0}")] + Input(String), + #[error("the model gave {0}")] + Model(String), + #[error("the model could not run: {0}")] + Runtime(String), + #[error("the model could not be loaded: {0}")] + Load(String), + #[error("cancelled")] + Cancelled, +} + +/// Something that runs the model on one tile. +pub trait TileRunner: Sync { + /// `input` is the tile as four planes `[R, G1, G2, B]` of `tile × tile` cells (NCHW, batch 1); the answer is + /// camera RGB, three planes of `2·tile × 2·tile` pixels (CHW), in the model's own scale. + fn run(&self, input: &[f32]) -> Result, Error>; +} + +pub struct Params { + /// Side of the model's square, in cells. + pub tile: usize, + /// Cells neighbouring tiles share. + pub overlap: usize, + pub gain: Gain, + /// White is 1; samples at or above this (minus a little) are clipped and keep their value. + pub clip: Option, + /// Tiles run at once (each model call may itself be single-threaded). + pub parallel: usize, +} + +#[derive(Default)] +pub struct Control<'a> { + pub cancel: Option<&'a AtomicBool>, + /// `(tiles done, tiles in all)`. + pub progress: Option<&'a (dyn Fn(usize, usize) + Sync)>, +} + +/// The tile grid of a picture: `(starts across, starts down)`. +pub fn grid(width: usize, height: usize, layout: Layout, tile: usize, overlap: usize) -> (Vec, Vec) { + let (nx, ny) = layout.cells(width, height); + (starts(nx, tile, overlap), starts(ny, tile, overlap)) +} + +/// How many model calls [`denoise_bayer`] makes for a picture. +pub fn tile_count(width: usize, height: usize, layout: Layout, tile: usize, overlap: usize) -> usize { + let (sx, sy) = grid(width, height, layout, tile, overlap); + sx.len().saturating_mul(sy.len()) +} + +/// Denoise (and demosaic) the normalised mosaic `mosaic` of `width × height` samples, laid out as `layout`. +/// Gives camera RGB at the mosaic's resolution. +pub fn denoise_bayer( + mosaic: &[f32], + width: usize, + height: usize, + layout: Layout, + runner: &dyn TileRunner, + p: &Params, + ctl: &Control<'_>, +) -> Result { + let pixels = width + .checked_mul(height) + .filter(|&n| n > 0 && n <= MAX_PIXELS) + .ok_or_else(|| Error::Input("the picture has no size, or is too large".into()))?; + if mosaic.len() != pixels { + return Err(Error::Input("the mosaic does not have width × height samples".into())); + } + if !(16..=2048).contains(&p.tile) || p.overlap >= p.tile.div_ceil(2) || layout.dx > 1 || layout.dy > 1 { + return Err(Error::Input("the tile size and overlap do not fit".into())); + } + if let Gain::MatchMean { nominal, max_deviation } = p.gain + && !(nominal.is_finite() && (1e-6..=1e12).contains(&nominal) && max_deviation.is_finite() && (0.0..=0.5).contains(&max_deviation)) + { + return Err(Error::Input("the model gain is not finite or within its supported range".into())); + } + if p.clip.is_some_and(|v| !v.is_finite() || v <= 0.0) { + return Err(Error::Input("the highlight clip must be finite and positive".into())); + } + let (nx, ny) = layout.cells(width, height); + let (sx, sy) = (starts(nx, p.tile, p.overlap), starts(ny, p.tile, p.overlap)); + let (cols, rows) = (sx.len(), sy.len()); + if cols == 0 || rows == 0 { + return Err(Error::Input("the picture needs too many tiles".into())); + } + let tiles: Vec<(usize, usize, bool, bool, bool, bool)> = sy + .iter() + .enumerate() + .flat_map(|(j, &y0)| sx.iter().enumerate().map(move |(i, &x0)| (x0, y0, i > 0, i + 1 < cols, j > 0, j + 1 < rows))) + .collect(); + let total = tiles.len(); + + let mut acc = vec![[0f32; 3]; pixels]; + let mut wsum = vec![0f32; nx * ny]; + let group = p.parallel.clamp(1, 64); + let mut done = 0; + // where the time goes, printed under `LIGHTCRAFT_PROFILE` + let (mut t_wait, mut t_total) = (std::time::Duration::ZERO, std::time::Duration::ZERO); + let began = std::time::Instant::now(); + + // One group of tiles at a time: pack, run the model, check and bring each to the input's scale, in parallel. + // Blending (sequential: the tiles add into one picture, in tile order so the result does not depend on timing) of + // the previous group runs while the next group is on the model, so neither the card nor the cores wait for it. + type Tile = (usize, usize, bool, bool, bool, bool); + let compute = |chunk: &[Tile]| -> Vec, Error>> { + chunk + .par_iter() + .map(|&(x0, y0, bx, ax, by, ay)| { + let mut input = vec![0f32; 4 * p.tile * p.tile]; + if !pack_tile(mosaic, width, height, layout, x0 as isize, y0 as isize, p.tile, &mut input) { + return Err(Error::Input("a tile could not be packed".into())); + } + let mut out = runner.run(&input)?; + let side = 2 * p.tile; + if out.len() != 3 * side * side { + return Err(Error::Model(format!("{} numbers for a tile, not the {} expected", out.len(), 3 * side * side))); + } + if out.iter().any(|v| !v.is_finite()) { + return Err(Error::Model("numbers that are not finite".into())); + } + finish_tile(&input, &mut out, p, (bx, ax, by, ay)); + Ok(out) + }) + .collect() + }; + let mut blend = |chunk: &[Tile], outs: Vec, Error>>| -> Result<(), Error> { + for (&(x0, y0, bx, ax, by, ay), r) in chunk.iter().zip(outs) { + blend_tile(&r?, &mut acc, &mut wsum, (width, height), (nx, ny), layout, (x0, y0), p, (bx, ax, by, ay)); + done += 1; + if let Some(f) = ctl.progress { + f(done, total); + } + } + Ok(()) + }; + let mut previous: Option<(&[Tile], Vec, Error>>)> = None; + for chunk in tiles.chunks(group) { + if ctl.cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return Err(Error::Cancelled); + } + let (outs, blended) = rayon::join( + || compute(chunk), + || match previous.take() { + Some((c, o)) => { + let started = std::time::Instant::now(); + let r = blend(c, o); + (r, started.elapsed()) + } + None => (Ok(()), std::time::Duration::ZERO), + }, + ); + blended.0?; + t_wait += blended.1; + previous = Some((chunk, outs)); + } + if let Some((c, o)) = previous.take() { + let started = std::time::Instant::now(); + blend(c, o)?; + t_wait += started.elapsed(); + } + t_total += began.elapsed(); + let started = std::time::Instant::now(); + // normalise by the weights + let (ox, oy) = (layout.position(0, 0).0, layout.position(0, 0).1); + acc.par_chunks_mut(width).enumerate().for_each(|(y, row)| { + let cy = ((y as isize - oy) / 2) as usize; + for (x, px) in row.iter_mut().enumerate() { + let cx = ((x as isize - ox) / 2) as usize; + let w = wsum.get(cy * nx + cx).copied().unwrap_or(0.0); + if w > 0.0 { + *px = px.map(|v| v / w); + } + } + }); + if std::env::var_os("LIGHTCRAFT_PROFILE").is_some() { + eprintln!( + "[denoise] {total} tiles, {} at once: tiles {:.0} ms (blending, {:.0} ms of it, overlaps the model), normalise {:.0} ms", + group, + t_total.as_secs_f64() * 1e3, + t_wait.as_secs_f64() * 1e3, + started.elapsed().as_secs_f64() * 1e3 + ); + } + Ok(Rgb32f { width, height, data: acc }) +} + +/// Bring a tile's output to the input's scale and keep clipped highlights as they were (in place). +fn finish_tile(input: &[f32], out: &mut [f32], p: &Params, edges: (bool, bool, bool, bool)) { + let t = p.tile; + let side = 2 * t; + if let Gain::MatchMean { nominal, max_deviation } = p.gain { + // the part of the tile whose scale is measured: away from the edges that blend into a neighbour + let m = p.overlap / 2; + let (i0, i1) = (if edges.0 { m } else { 0 }, t - if edges.1 { m } else { 0 }); + let (j0, j1) = (if edges.2 { m } else { 0 }, t - if edges.3 { m } else { 0 }); + let cells = ((i1 - i0) * (j1 - j0)) as f64; + let plane_sum = |pl: usize| -> f64 { + (j0..j1) + .map(|j| input.get(pl * t * t + j * t + i0..pl * t * t + j * t + i1).map_or(0.0, |r| r.iter().map(|&v| f64::from(v)).sum::())) + .sum() + }; + let in_mean = [plane_sum(0) / cells, (plane_sum(1) + plane_sum(2)) / (2.0 * cells), plane_sum(3) / cells]; + for (c, &mean_in) in in_mean.iter().enumerate() { + let sum_out: f64 = (2 * j0..2 * j1) + .map(|y| { + out.get(c * side * side + y * side + 2 * i0..c * side * side + y * side + 2 * i1) + .map_or(0.0, |r| r.iter().map(|&v| f64::from(v)).sum::()) + }) + .sum(); + let mean_out = sum_out / (4.0 * cells); + let measured = if mean_in as f32 >= MIN_SIGNAL && mean_out > 0.0 { (mean_out / mean_in) as f32 } else { nominal }; + let g = measured.clamp(nominal * (1.0 - max_deviation), nominal * (1.0 + max_deviation)); + if g.is_finite() && g > 0.0 { + let inv = 1.0 / g; + if let Some(plane) = out.get_mut(c * side * side..(c + 1) * side * side) { + plane.iter_mut().for_each(|v| *v *= inv); + } + } + } + } + if let Some(clip) = p.clip { + protect_clipped(input, out, t, clip); + } +} + +/// Where the mosaic is at (or within a cell of) white, the output becomes the plain cell values again, so a clipped +/// area stays exactly as clipped as the camera recorded it (highlight recovery later depends on that). +fn protect_clipped(input: &[f32], out: &mut [f32], t: usize, clip: f32) { + let side = 2 * t; + let plane = t * t; + let (Some(r), Some(g1), Some(g2), Some(b)) = + (input.get(0..plane), input.get(plane..2 * plane), input.get(2 * plane..3 * plane), input.get(3 * plane..4 * plane)) + else { + return; + }; + let mut m: Vec = (0..plane).map(|i| r[i].max(g1[i]).max(g2[i]).max(b[i])).collect(); + if !m.iter().any(|&v| v >= clip) { + return; + } + // 3×3 maximum over cells (separable) + let dilate = |src: &[f32], horizontal: bool| -> Vec { + (0..plane) + .map(|i| { + let (x, y) = (i % t, i / t); + let mut best = src[i]; + for d in [-1isize, 1] { + let (nx, ny) = if horizontal { (x as isize + d, y as isize) } else { (x as isize, y as isize + d) }; + if nx >= 0 && ny >= 0 && (nx as usize) < t && (ny as usize) < t { + best = best.max(src[ny as usize * t + nx as usize]); + } + } + best + }) + .collect() + }; + m = dilate(&dilate(&m, true), false); + let lo = clip - 0.03; + for j in 0..t { + for i in 0..t { + let k = j * t + i; + let s = ((m[k] - lo) / (clip - lo)).clamp(0.0, 1.0); + if s <= 0.0 { + continue; + } + let s = s * s * (3.0 - 2.0 * s); + let plain = [r[k], 0.5 * (g1[k] + g2[k]), b[k]]; + for (c, &pv) in plain.iter().enumerate() { + for (sy, sx) in [(0, 0), (0, 1), (1, 0), (1, 1)] { + if let Some(v) = out.get_mut(c * side * side + (2 * j + sy) * side + 2 * i + sx) { + *v += (pv - *v) * s; + } + } + } + } + } +} + +/// Add a finished tile to the picture with its blend weights. +#[allow(clippy::too_many_arguments)] +fn blend_tile( + out: &[f32], + acc: &mut [[f32; 3]], + wsum: &mut [f32], + size: (usize, usize), + cells: (usize, usize), + layout: Layout, + at: (usize, usize), + p: &Params, + edges: (bool, bool, bool, bool), +) { + let (width, height) = size; + let (nx, ny) = cells; + let (t, side) = (p.tile, 2 * p.tile); + let fade = p.overlap; + let wx: Vec = (0..t).map(|i| weight(i, t, fade, edges.0, edges.1)).collect(); + let wy: Vec = (0..t).map(|j| weight(j, t, fade, edges.2, edges.3)).collect(); + for j in 0..t { + let cy = at.1 + j; + for i in 0..t { + let cx = at.0 + i; + // cells past the picture's edge (a picture smaller than one tile is padded) belong to no pixel + if cx >= nx || cy >= ny { + continue; + } + let (Some(&a), Some(&b)) = (wx.get(i), wy.get(j)) else { continue }; + let w = a * b; + if let Some(s) = wsum.get_mut(cy * nx + cx) { + *s += w; + } + let (px0, py0) = layout.position(cx as isize, cy as isize); + for (sy, sx) in [(0isize, 0isize), (0, 1), (1, 0), (1, 1)] { + let (x, y) = (px0 + sx, py0 + sy); + if x < 0 || y < 0 || x as usize >= width || y as usize >= height { + continue; + } + let o = (2 * j + sy as usize) * side + 2 * i + sx as usize; + let (Some(&r), Some(&g), Some(&bl)) = (out.get(o), out.get(side * side + o), out.get(2 * side * side + o)) else { continue }; + if let Some(px) = acc.get_mut(y as usize * width + x as usize) { + px[0] += w * r; + px[1] += w * g; + px[2] += w * bl; + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A stand-in model: the cell's samples as an RGB picture at twice the resolution (a plain 2 × 2 "demosaic"), + /// times `scale`, so the output has a scale of its own like the real model's. + struct Mock { + tile: usize, + scale: f32, + } + impl TileRunner for Mock { + fn run(&self, input: &[f32]) -> Result, Error> { + let (t, side) = (self.tile, 2 * self.tile); + let plane = t * t; + let mut out = vec![0.0; 3 * side * side]; + for j in 0..t { + for i in 0..t { + let k = j * t + i; + let rgb = [input[k], 0.5 * (input[plane + k] + input[2 * plane + k]), input[3 * plane + k]]; + for (c, v) in rgb.iter().enumerate() { + for (sy, sx) in [(0, 0), (0, 1), (1, 0), (1, 1)] { + out[c * side * side + (2 * j + sy) * side + 2 * i + sx] = v * self.scale; + } + } + } + } + Ok(out) + } + } + + fn params(tile: usize, overlap: usize, gain: Gain) -> Params { + Params { tile, overlap, gain, clip: None, parallel: 2 } + } + + #[test] + fn overflowing_tiles_and_nan_gain_are_errors_before_the_model_runs() { + let layout = Layout::from_cell([0, 1, 1, 2]).unwrap(); + let model = Mock { tile: 16, scale: 1.0 }; + for (tile, overlap, gain) in [ + (usize::MAX, 0, Gain::None), + (16, usize::MAX, Gain::None), + (16, 0, Gain::MatchMean { nominal: 1.0, max_deviation: f32::NAN }), + (16, 0, Gain::MatchMean { nominal: f32::INFINITY, max_deviation: 0.1 }), + ] { + assert!(matches!(denoise_bayer(&[0.5], 1, 1, layout, &model, ¶ms(tile, overlap, gain), &Control::default()), Err(Error::Input(_)))); + } + } + + /// A smooth picture whose colour at a sample is `base[colour] · (1 + shading)`. + fn mosaic(w: usize, h: usize, cell: [u8; 4]) -> Vec { + (0..w * h) + .map(|i| { + let (x, y) = (i % w, i / w); + let c = cell[(y % 2) * 2 + x % 2] as usize; + [0.5, 0.3, 0.2][c] * (0.6 + 0.4 * ((x as f32 / 23.0).sin() * (y as f32 / 31.0).cos())) + }) + .collect() + } + + fn reference(m: &[f32], w: usize, h: usize, layout: Layout) -> Rgb32f { + // what the mock gives for the whole picture in one tile + let (nx, ny) = layout.cells(w, h); + let t = nx.max(ny).div_ceil(16) * 16; + denoise_bayer(m, w, h, layout, &Mock { tile: t, scale: 1.0 }, ¶ms(t, 16, Gain::None), &Control::default()).unwrap() + } + + #[test] + fn tiles_blend_to_the_same_picture_as_one_tile_for_every_layout() { + for cell in [[0u8, 1, 1, 2], [1, 0, 2, 1], [1, 2, 0, 1], [2, 1, 1, 0]] { + let layout = Layout::from_cell(cell).unwrap(); + let (w, h) = (150usize, 110usize); + let m = mosaic(w, h, cell); + let want = reference(&m, w, h, layout); + let got = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.0 }, ¶ms(32, 8, Gain::None), &Control::default()).unwrap(); + assert_eq!((got.width, got.height), (w, h)); + for (a, b) in got.data.iter().zip(&want.data) { + for c in 0..3 { + assert!((a[c] - b[c]).abs() < 1e-5, "layout {cell:?}: {a:?} vs {b:?}"); + } + } + } + } + + #[test] + fn the_picture_does_not_depend_on_how_many_tiles_run_at_once_or_how_long_each_takes() { + // tiles that finish in a different order every time: blending still adds them in tile order + struct Jittery(Mock); + impl TileRunner for Jittery { + fn run(&self, input: &[f32]) -> Result, Error> { + let wait = (input.iter().take(64).sum::() * 1000.0) as u64 % 7; + std::thread::sleep(std::time::Duration::from_millis(wait)); + self.0.run(input) + } + } + let layout = Layout { dx: 0, dy: 1 }; + let (w, h) = (150usize, 110usize); + let m = mosaic(w, h, [1, 0, 2, 1]); + let run = |parallel: usize| { + let mut p = params(32, 8, Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 }); + p.parallel = parallel; + denoise_bayer(&m, w, h, layout, &Jittery(Mock { tile: 32, scale: 1.01e6 }), &p, &Control::default()).unwrap() + }; + let one = run(1); + for parallel in [2, 3, 8] { + assert_eq!(run(parallel).data, one.data, "{parallel} at once"); + } + } + + #[test] + fn a_failing_tile_or_a_cancel_ends_the_picture_without_a_panic() { + struct Fails(Mock, std::sync::atomic::AtomicUsize); + impl TileRunner for Fails { + fn run(&self, input: &[f32]) -> Result, Error> { + if self.1.fetch_add(1, Ordering::Relaxed) == 4 { + return Err(Error::Runtime("boom".into())); + } + self.0.run(input) + } + } + let layout = Layout { dx: 0, dy: 0 }; + let (w, h) = (150usize, 110usize); + let m = mosaic(w, h, [0, 1, 1, 2]); + let r = denoise_bayer( + &m, + w, + h, + layout, + &Fails(Mock { tile: 32, scale: 1.0 }, Default::default()), + ¶ms(32, 8, Gain::None), + &Control::default(), + ); + assert!(matches!(r, Err(Error::Runtime(_))), "{r:?}"); + let stop = AtomicBool::new(true); + let ctl = Control { cancel: Some(&stop), progress: None }; + let r = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.0 }, ¶ms(32, 8, Gain::None), &ctl); + assert!(matches!(r, Err(Error::Cancelled))); + } + + #[test] + fn a_scale_of_its_own_is_taken_out_tile_by_tile() { + let layout = Layout { dx: 0, dy: 0 }; + let (w, h) = (150usize, 110usize); + let m = mosaic(w, h, [0, 1, 1, 2]); + let want = reference(&m, w, h, layout); + let gain = Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 }; + // the scale differs a little from nominal; each tile's own scale is measured and removed + let got = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.02e6 }, ¶ms(32, 8, gain), &Control::default()).unwrap(); + let mean = |img: &Rgb32f, c: usize| img.data.iter().map(|p| f64::from(p[c])).sum::() / img.data.len() as f64; + for c in 0..3 { + let (a, b) = (mean(&got, c), mean(&want, c)); + assert!((a - b).abs() / b < 0.01, "channel {c}: {a} vs {b}"); + } + // a scale far from nominal is clamped rather than trusted + let wild = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 3.0e6 }, ¶ms(32, 8, gain), &Control::default()).unwrap(); + assert!(mean(&wild, 1) > 2.0 * mean(&want, 1)); + } + + #[test] + fn a_picture_smaller_than_a_tile_is_padded_by_mirroring() { + let layout = Layout { dx: 1, dy: 1 }; + let (w, h) = (20usize, 14usize); + let m = mosaic(w, h, [2, 1, 1, 0]); + let got = denoise_bayer(&m, w, h, layout, &Mock { tile: 64, scale: 1.0 }, ¶ms(64, 16, Gain::None), &Control::default()).unwrap(); + assert_eq!((got.width, got.height), (w, h)); + assert!(got.data.iter().all(|p| p.iter().all(|v| v.is_finite() && *v > 0.0))); + } + + #[test] + fn clipped_highlights_stay_clipped() { + // a model that darkens everything by 10 % would pull a clipped plateau below white + struct Dim(Mock); + impl TileRunner for Dim { + fn run(&self, input: &[f32]) -> Result, Error> { + Ok(self.0.run(input)?.into_iter().map(|v| v * 0.9).collect()) + } + } + let layout = Layout { dx: 0, dy: 0 }; + let (w, h) = (96usize, 96usize); + let m: Vec = (0..w * h).map(|i| if i % w > 40 && i % w < 70 { 1.0 } else { 0.4 }).collect(); + let mut p = params(32, 8, Gain::None); + p.clip = Some(0.99); + let got = denoise_bayer(&m, w, h, layout, &Dim(Mock { tile: 32, scale: 1.0 }), &p, &Control::default()).unwrap(); + let mid = got.data[48 * w + 55]; + assert!(mid.iter().all(|&v| v > 0.995), "plateau {mid:?}"); + let dark = got.data[48 * w + 10]; + assert!(dark.iter().all(|&v| (v - 0.36).abs() < 0.01), "unclipped area still goes through the model: {dark:?}"); + } + + #[test] + fn cancelling_and_progress_work() { + let layout = Layout { dx: 0, dy: 0 }; + let (w, h) = (150usize, 110usize); + let m = mosaic(w, h, [0, 1, 1, 2]); + let cancel = AtomicBool::new(true); + let r = denoise_bayer( + &m, + w, + h, + layout, + &Mock { tile: 32, scale: 1.0 }, + ¶ms(32, 8, Gain::None), + &Control { cancel: Some(&cancel), progress: None }, + ); + assert!(matches!(r, Err(Error::Cancelled))); + let seen = std::sync::Mutex::new(Vec::new()); + let f = |d: usize, n: usize| seen.lock().unwrap().push((d, n)); + denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.0 }, ¶ms(32, 8, Gain::None), &Control { cancel: None, progress: Some(&f) }) + .unwrap(); + let seen = seen.into_inner().unwrap(); + let total = tile_count(w, h, layout, 32, 8); + assert!(total > 1); + assert_eq!(seen.last(), Some(&(total, total))); + assert!(seen.windows(2).all(|p| p[1].0 == p[0].0 + 1)); + } + + #[test] + fn hostile_inputs_are_errors() { + let layout = Layout { dx: 0, dy: 0 }; + let mock = Mock { tile: 32, scale: 1.0 }; + let p = params(32, 8, Gain::None); + assert!(matches!(denoise_bayer(&[], 0, 0, layout, &mock, &p, &Control::default()), Err(Error::Input(_)))); + assert!(matches!(denoise_bayer(&[0.0; 10], 4, 4, layout, &mock, &p, &Control::default()), Err(Error::Input(_)))); + assert!(matches!(denoise_bayer(&[0.0; 16], 4, 4, layout, &mock, ¶ms(32, 16, Gain::None), &Control::default()), Err(Error::Input(_)))); + struct Bad(Vec); + impl TileRunner for Bad { + fn run(&self, _: &[f32]) -> Result, Error> { + Ok(self.0.clone()) + } + } + assert!(matches!(denoise_bayer(&[0.5; 16], 4, 4, layout, &Bad(vec![0.0; 5]), &p, &Control::default()), Err(Error::Model(_)))); + assert!(matches!(denoise_bayer(&[0.5; 16], 4, 4, layout, &Bad(vec![f32::NAN; 3 * 64 * 64]), &p, &Control::default()), Err(Error::Model(_)))); + } +} diff --git a/crates/denoise/src/runtime.rs b/crates/denoise/src/runtime.rs new file mode 100644 index 00000000..4314537a --- /dev/null +++ b/crates/denoise/src/runtime.rs @@ -0,0 +1,423 @@ +//! Model loading, a shared installation self-test, and deterministic test tiles. +//! `CpuRunner` is the pure-Rust executor; scalar operator tests live in `reference`. + +use std::path::Path; + +use web_time::Instant; + +use serde::Serialize; + +use crate::manifest::{DenoiserManifest, Gain}; +use crate::run::{Error, TileRunner}; + +/// What `self_test` found. +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SelfTest { + pub ok: bool, + /// Milliseconds to load and optimise the model. + pub load_ms: f64, + /// Median milliseconds to run one tile. + pub tile_ms: f64, + /// Megapixels of picture one tile covers, for estimating a whole photo's time. + pub tile_megapixels: f64, + /// The model's output scale relative to its input on the test picture (what `Gain::MatchMean` takes out). + pub scale: f64, + /// Each check: what was checked and whether it held. + pub checks: Vec<(String, bool)>, +} + +/// The default pure-Rust CPU model, with the same manifest and installation self-test as the reference. +#[derive(Clone)] +pub struct CpuRunner { + runner: crate::cpu::NetRunner, + manifest: DenoiserManifest, + load_ms: f64, +} +impl CpuRunner { + pub fn load(path: &Path, manifest: &DenoiserManifest) -> Result { + crate::manifest::validate(manifest).map_err(|e| Error::Load(e.to_string()))?; + let started = Instant::now(); + let net = crate::onnx::read(path).map_err(|e| Error::Load(e.to_string()))?; + let runner = crate::cpu::NetRunner::new(&net, manifest.tile as usize).map_err(|e| Error::Load(e.to_string()))?; + Ok(Self { runner, manifest: manifest.clone(), load_ms: started.elapsed().as_secs_f64() * 1e3 }) + } + pub fn manifest(&self) -> &DenoiserManifest { + &self.manifest + } + pub fn net(&self) -> &crate::net::Net { + self.runner.net() + } + pub fn bytes_per_tile(&self) -> usize { + self.runner.bytes_per_tile() + } + pub fn self_test(&self) -> SelfTest { + self_test(self, &self.manifest, self.load_ms) + } +} +impl TileRunner for CpuRunner { + fn run(&self, input: &[f32]) -> Result, Error> { + self.runner.run(input) + } +} +fn self_test(runner: &dyn TileRunner, manifest: &DenoiserManifest, load_ms: f64) -> SelfTest { + let t = manifest.tile as usize; + let mut checks: Vec<(String, bool)> = Vec::new(); + let (input, base) = test_tile(t); + let (mut times, mut scale) = (Vec::new(), 0.0f64); + let mut run = || { + let started = Instant::now(); + let r = runner.run(&input); + times.push(started.elapsed().as_secs_f64() * 1000.0); + r + }; + let (a, b) = (run(), run()); + match (&a, &b) { + (Ok(a), Ok(b)) => { + checks.push((format!("gives {} finite numbers per tile", a.len()), true)); + checks.push(( + "gives the same answer for the same tile".into(), + a.iter().zip(b).all(|(x, y)| (x - y).abs() <= 1e-4 * x.abs().max(y.abs()).max(1e-9)), + )); + let side = 2 * t; + let mean_in = base.iter().map(|&v| f64::from(v)).sum::() / base.len().max(1) as f64; + let mean_out = a.iter().map(|&v| f64::from(v)).sum::() / a.len().max(1) as f64; + scale = if mean_in > 0.0 { mean_out / mean_in } else { 0.0 }; + if let Gain::MatchMean { nominal, max_deviation } = manifest.gain { + let ratio = scale / f64::from(nominal); + let tolerance = f64::from(max_deviation).max(0.05) * 4.0; + checks.push((format!("its scale is the {nominal:.3e} the description says (found {scale:.3e})"), (ratio - 1.0).abs() <= tolerance)); + } + // noise left after bringing the answer back to the input's scale, against the noise that went in + let g = if scale > 0.0 { 1.0 / scale } else { 1.0 }; + let (mut noise_in, mut noise_out, mut n_in, mut n_out) = (0.0f64, 0.0f64, 0usize, 0usize); + for (i, (&v, &s)) in input.iter().zip(base.iter().cycle()).enumerate() { + // centre of each plane only: edges are the model's guess + let (x, y) = ((i % (t * t)) % t, (i % (t * t)) / t); + if x >= t / 4 && x < 3 * t / 4 && y >= t / 4 && y < 3 * t / 4 { + noise_in += f64::from(v - s).powi(2); + n_in += 1; + } + } + for c in 0..3 { + for y in side / 4..3 * side / 4 { + for x in side / 4..3 * side / 4 { + if let (Some(&o), Some(&s)) = (a.get(c * side * side + y * side + x), base.get((y / 2) * t + x / 2)) { + noise_out += (f64::from(o) * g - f64::from(s)).powi(2); + n_out += 1; + } + } + } + } + let (rms_in, rms_out) = ((noise_in / n_in.max(1) as f64).sqrt(), (noise_out / n_out.max(1) as f64).sqrt()); + checks.push((format!("takes noise down (from {rms_in:.4} to {rms_out:.4})"), rms_out < 0.8 * rms_in)); + } + (Err(e), _) | (_, Err(e)) => checks.push((format!("runs: {e}"), false)), + } + times.sort_by(|x, y| x.total_cmp(y)); + SelfTest { + ok: !checks.is_empty() && checks.iter().all(|(_, ok)| *ok), + load_ms, + tile_ms: times.get(times.len() / 2).copied().unwrap_or(0.0), + tile_megapixels: (2 * t * 2 * t) as f64 / 1e6, + scale, + checks, + } +} + +/// A deterministic test tile: four planes of a smooth gradient with noise on top, and the gradient alone for one +/// plane (`tile × tile`). +pub fn test_tile(t: usize) -> (Vec, Vec) { + if t == 0 || t > 2048 { + return (Vec::new(), Vec::new()); + } + let mut seed = 0x2545_f491u32; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + (seed >> 8) as f32 / (1u32 << 24) as f32 + }; + let base: Vec = (0..t * t).map(|i| 0.12 + 0.3 * (((i % t) + (i / t)) as f32 / (2 * t) as f32)).collect(); + let input = (0..4 * t * t) + .map(|i| { + // sum of four uniforms: close to Gaussian, deviation 0.0577 × 2 before scaling + let n = next() + next() + next() + next() - 2.0; + base.get(i % (t * t)).copied().unwrap_or(0.0) + 0.05 * n + }) + .collect(); + (input, base) +} + +/// A tile for checking that two runners of one model agree: the whole range a raw photo has rather than the test tile's +/// mid-greys — a noisy ramp from black to white across the tile, a flat black corner and a clipped white one (four +/// planes of `tile × tile`). +pub fn check_tile(t: usize) -> Vec { + if t == 0 || t > 2048 { + return Vec::new(); + } + let mut seed = 0x9e37_79b9u32; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + (seed >> 8) as f32 / (1u32 << 24) as f32 + }; + let (edge, span) = ((t / 8).max(1), (2 * t).saturating_sub(2).max(1) as f32); + (0..4 * t * t) + .map(|i| { + let (x, y) = ((i % (t * t)) % t, (i % (t * t)) / t); + let n = next() + next() + next() + next() - 2.0; + if x < edge && y < edge { + 0.0 + } else if x + edge >= t && y + edge >= t { + 1.0 + } else { + ((x + y) as f32 / span + 0.05 * n).clamp(0.0, 1.0) + } + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::licence::Licence; + use crate::manifest::Domain; + use crate::synthetic_proto::{len_field, value_info_bytes}; + + fn varint(mut v: u64, out: &mut Vec) { + loop { + let b = (v & 0x7f) as u8; + v >>= 7; + if v == 0 { + out.push(b); + return; + } + out.push(b | 0x80); + } + } + + fn varint_field(n: u64, v: u64, out: &mut Vec) { + varint(n << 3, out); + varint(v, out); + } + + fn node(op: &str, inputs: &[&str], output: &str, ints: &[(&str, i64)]) -> Vec { + let mut n = Vec::new(); + for i in inputs { + len_field(1, i.as_bytes(), &mut n); + } + len_field(2, output.as_bytes(), &mut n); + len_field(4, op.as_bytes(), &mut n); + for (name, value) in ints { + let mut a = Vec::new(); + len_field(1, name.as_bytes(), &mut a); + varint_field(3, *value as u64, &mut a); + varint_field(20, 2, &mut a); + len_field(5, &a, &mut n); + } + n + } + + fn tensor(name: &str, dims: &[u64], values: &[f32]) -> Vec { + let mut t = Vec::new(); + for d in dims { + varint_field(1, *d, &mut t); + } + varint_field(2, 1, &mut t); + len_field(8, name.as_bytes(), &mut t); + let raw: Vec = values.iter().flat_map(|v| v.to_le_bytes()).collect(); + len_field(9, &raw, &mut t); + t + } + + /// A tiny working denoiser-shaped network: `[1, 4, t, t]` → a 1 × 1 convolution to 12 channels (each output + /// channel is `scale` times one input plane, so the scale is known) → depth-to-space → `[1, 3, 2t, 2t]`. + fn tiny_model(t: u64, scale: f32) -> Vec { + tiny_model_with(t, scale, 12) + } + + /// The same with another number of convolution channels; only 12 can be arranged into the three colours. + fn tiny_model_with(t: u64, scale: f32, channels: u64) -> Vec { + let mut graph = Vec::new(); + len_field(1, &node("Conv", &["data", "w"], if channels == 12 { "c" } else { "rgb" }, &[("group", 1)]), &mut graph); + if channels == 12 { + let mut depth_to_space = node("DepthToSpace", &["c"], "rgb", &[("blocksize", 2)]); + let mut mode = Vec::new(); + len_field(1, b"mode", &mut mode); + len_field(4, b"CRD", &mut mode); + varint_field(20, 3, &mut mode); + len_field(5, &mode, &mut depth_to_space); + len_field(1, &depth_to_space, &mut graph); + } + // CRD output channel colour·4 + dy·2 + dx reads input plane [R, G1, B] of the tile. + let mut w = vec![0f32; channels as usize * 4]; + for oc in 0..channels as usize { + let colour = (oc / 4).min(2); + let plane = [0, 1, 3][colour]; + w[oc * 4 + plane] = scale; + } + len_field(5, &tensor("w", &[channels, 4, 1, 1], &w), &mut graph); + len_field(11, &value_info_bytes("data", 1, &[Ok(1), Ok(4), Ok(t), Ok(t)]), &mut graph); + len_field(12, &value_info_bytes("rgb", 1, &[Ok(1), Ok(3), Ok(2 * t), Ok(2 * t)]), &mut graph); + let mut ops = Vec::new(); + len_field(1, b"", &mut ops); + varint_field(2, 13, &mut ops); + let mut model = Vec::new(); + varint_field(1, 8, &mut model); + len_field(7, &graph, &mut model); + len_field(8, &ops, &mut model); + model + } + + fn manifest(tile: u32, nominal: f32) -> DenoiserManifest { + DenoiserManifest { + id: "tiny".into(), + name: "Tiny".into(), + version: "1".into(), + licence: Licence::default(), + source: None, + sha256: None, + size_bytes: None, + provenance: String::new(), + domain: Domain::BayerToRgb, + tile, + overlap: 16, + gain: Gain::MatchMean { nominal, max_deviation: 0.05 }, + } + } + + fn temp_model(name: &str, bytes: &[u8]) -> std::path::PathBuf { + let p = std::env::temp_dir().join(format!("lc-denoise-runtime-{name}-{}.onnx", std::process::id())); + std::fs::write(&p, bytes).unwrap(); + p + } + + #[test] + fn a_working_model_loads_runs_and_has_the_scale_it_says() { + let p = temp_model("ok", &tiny_model(64, 1000.0)); + let r = CpuRunner::load(&p, &manifest(64, 1000.0)).unwrap(); + let input: Vec = (0..4 * 64 * 64).map(|i| (i % 97) as f32 / 97.0).collect(); + let out = r.run(&input).unwrap(); + assert_eq!(out.len(), 3 * 128 * 128); + // output (colour c, pixel (2y+dy, 2x+dx)) = scale × input plane [R, G1, B][c] at (y, x) + for (c, plane) in [0usize, 1, 3].into_iter().enumerate() { + let (y, x) = (5usize, 9usize); + assert!((out[c * 128 * 128 + (2 * y + 1) * 128 + 2 * x] - 1000.0 * input[plane * 64 * 64 + y * 64 + x]).abs() < 1e-2); + } + let _ = std::fs::remove_file(p); + } + + #[test] + fn the_self_test_checks_scale_and_noise() { + // a model that only multiplies cannot take noise down: it must fail that check but pass the others + let p = temp_model("selftest", &tiny_model(64, 1000.0)); + let t = CpuRunner::load(&p, &manifest(64, 1000.0)).unwrap().self_test(); + assert!(!t.ok, "{t:?}"); + assert!(t.checks.iter().any(|(c, ok)| c.contains("same answer") && *ok), "{t:?}"); + assert!(t.checks.iter().any(|(c, ok)| c.contains("scale") && *ok), "{t:?}"); + assert!(t.checks.iter().any(|(c, ok)| c.contains("noise") && !*ok), "{t:?}"); + // and one that claims the wrong scale fails that check + let q = CpuRunner::load(&p, &manifest(64, 5.0)).unwrap().self_test(); + assert!(q.checks.iter().any(|(c, ok)| c.contains("scale") && !*ok), "{q:?}"); + let _ = std::fs::remove_file(p); + } + + #[test] + fn wrong_sizes_and_broken_files_are_errors() { + let p = temp_model("sizes", &tiny_model(64, 1.0)); + let r = CpuRunner::load(&p, &manifest(64, 1.0)).unwrap(); + assert!(matches!(r.run(&[0.0; 10]), Err(Error::Input(_)))); + // a fully convolutional model works at whatever tile size the manifest asks for + let wide = CpuRunner::load(&p, &manifest(128, 1.0)).unwrap(); + assert_eq!(wide.run(&vec![0.5; 4 * 128 * 128]).unwrap().len(), 3 * 256 * 256); + // a model that gives another number of values than the manifest promises is an error + let q = temp_model("channels", &tiny_model_with(64, 1.0, 8)); + assert!(CpuRunner::load(&q, &manifest(64, 1.0)).is_err(), "shape is refused before execution"); + let _ = std::fs::remove_file(q); + for (name, bytes) in [("junk", b"nonsense".to_vec()), ("empty", Vec::new())] { + let q = temp_model(name, &bytes); + assert!(CpuRunner::load(&q, &manifest(64, 1.0)).is_err(), "{name}"); + let _ = std::fs::remove_file(q); + } + assert!(CpuRunner::load(&std::env::temp_dir().join("does-not-exist.onnx"), &manifest(64, 1.0)).is_err()); + let _ = std::fs::remove_file(p); + } + + #[test] + fn the_check_tile_covers_black_to_white() { + for t in [1, 2, 7, 64] { + let c = check_tile(t); + assert_eq!(c.len(), 4 * t * t); + assert!(c.iter().all(|v| (0.0..=1.0).contains(v)), "{t}"); + } + let c = check_tile(64); + let near = |lo: f32, hi: f32| c.iter().filter(|v| (lo..=hi).contains(*v)).count(); + assert!(near(0.0, 0.0) >= 4 * 8 * 8 && near(1.0, 1.0) >= 4 * 8 * 8, "a flat black and a clipped white corner"); + for band in 0..10 { + let lo = band as f32 / 10.0; + assert!(near(lo, lo + 0.1) > 100, "values around {lo}"); + } + assert_eq!(check_tile(64), c, "the same tile every time"); + } + + #[test] + fn truncated_and_corrupted_models_never_panic() { + let good = tiny_model(64, 1.0); + for n in (0..good.len()).step_by(5) { + let q = temp_model("trunc", &good[..n]); + let _ = CpuRunner::load(&q, &manifest(64, 1.0)); + let _ = std::fs::remove_file(q); + } + for at in (0..good.len()).step_by(7) { + let mut bad = good.clone(); + if let Some(b) = bad.get_mut(at) { + *b = b.wrapping_add(0x55); + } + let q = temp_model("flip", &bad); + if let Ok(r) = CpuRunner::load(&q, &manifest(64, 1.0)) { + let _ = r.run(&vec![0.5; 4 * 64 * 64]); + } + let _ = std::fs::remove_file(q); + } + } + + /// Opt-in: `LC_DENOISE_MODEL= cargo test -p lightcraft-denoise --release + /// -- --ignored --nocapture` self-tests a real model (RawNIND's Bayer model: tile 512, scale about 1e6). + #[test] + #[ignore = "needs a real model file: set LC_DENOISE_MODEL"] + fn a_real_model_passes_the_self_test() { + let Some(path) = std::env::var_os("LC_DENOISE_MODEL") else { return }; + let m = manifest(512, 1.0e6); + let r = CpuRunner::load(Path::new(&path), &m).unwrap(); + let t = r.self_test(); + println!("load {:.0} ms | {:.0} ms per tile | scale {:.4e} | {:?}", t.load_ms, t.tile_ms, t.scale, t.checks); + assert!(t.ok); + } +} + +#[cfg(test)] +mod cpu_tests { + use super::*; + #[test] + fn manifest_and_test_tile_sizes_are_checked_without_tract() { + for t in [0, 2049, usize::MAX] { + assert!(test_tile(t).0.is_empty()); + assert!(check_tile(t).is_empty()); + } + let path = std::env::temp_dir().join(format!("lc-cpu-load-{}.onnx", std::process::id())); + std::fs::write(&path, crate::synthetic::unet_onnx(64, 5, 1, 7)).unwrap(); + let mut m = crate::known::find("rawnind-bayer").unwrap().manifest; + m.tile = 64; + m.overlap = 16; + m.gain = Gain::None; + let r = CpuRunner::load(&path, &m).unwrap(); + let x = check_tile(64); + assert_eq!(r.run(&x).unwrap(), r.run(&x).unwrap()); + assert!(r.self_test().checks.iter().any(|(text, ok)| text.contains("same answer") && *ok)); + m.tile = u32::MAX; + assert!(CpuRunner::load(&path, &m).is_err()); + std::fs::remove_file(path).unwrap(); + } +} diff --git a/crates/denoise/src/synthetic.rs b/crates/denoise/src/synthetic.rs new file mode 100644 index 00000000..8ceafca7 --- /dev/null +++ b/crates/denoise/src/synthetic.rs @@ -0,0 +1,234 @@ +//! A small ONNX U-Net built in memory, for tests of the layers that read and run denoise networks (here, in the GPU +//! runner and in the engine): the same layer kinds and wiring as the real denoisers, with deterministic random weights. +//! Nothing here is a model anyone would use. + +use crate::synthetic_proto::{len_field, value_info_bytes}; + +fn varint(mut v: u64, out: &mut Vec) { + loop { + let b = (v & 0x7f) as u8; + v >>= 7; + if v == 0 { + out.push(b); + return; + } + out.push(b | 0x80); + } +} + +fn varint_field(n: u64, v: u64, out: &mut Vec) { + varint(n << 3, out); + varint(v, out); +} + +enum A<'a> { + Int(i64), + Ints(&'a [i64]), + Float(f32), + Str(&'a str), +} + +fn node(op: &str, inputs: &[&str], output: &str, attrs: &[(&str, A)]) -> Vec { + let mut n = Vec::new(); + for i in inputs { + len_field(1, i.as_bytes(), &mut n); + } + len_field(2, output.as_bytes(), &mut n); + len_field(4, op.as_bytes(), &mut n); + for (name, value) in attrs { + let mut a = Vec::new(); + len_field(1, name.as_bytes(), &mut a); + match value { + A::Int(v) => { + varint_field(3, *v as u64, &mut a); + varint_field(20, 2, &mut a); + } + A::Ints(vs) => { + for v in *vs { + varint_field(8, *v as u64, &mut a); + } + varint_field(20, 7, &mut a); + } + A::Float(v) => { + varint((2 << 3) | 5, &mut a); + a.extend_from_slice(&v.to_le_bytes()); + varint_field(20, 1, &mut a); + } + A::Str(s) => { + len_field(4, s.as_bytes(), &mut a); + varint_field(20, 3, &mut a); + } + } + len_field(5, &a, &mut n); + } + n +} + +fn tensor(name: &str, dims: &[u64], values: &[f32]) -> Vec { + let mut t = Vec::new(); + for d in dims { + varint_field(1, *d, &mut t); + } + varint_field(2, 1, &mut t); + len_field(8, name.as_bytes(), &mut t); + let raw: Vec = values.iter().flat_map(|v| v.to_le_bytes()).collect(); + len_field(9, &raw, &mut t); + t +} + +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> f32 { + self.0 ^= self.0 >> 12; + self.0 ^= self.0 << 25; + self.0 ^= self.0 >> 27; + ((self.0.wrapping_mul(0x2545_f491_4f6c_dd1d) >> 40) as f32 / (1u64 << 24) as f32) * 2.0 - 1.0 + } +} + +struct Builder { + nodes: Vec>, + inits: Vec>, + rng: Rng, + n: usize, +} + +impl Builder { + fn name(&mut self, kind: &str) -> String { + self.n += 1; + format!("{kind}{}", self.n) + } + + /// Random weights scaled so activations keep a sane size through the layers. + fn weights(&mut self, name: &str, dims: &[u64], fan_in: usize) -> String { + let count: u64 = dims.iter().product(); + let scale = (3.0 / fan_in as f32).sqrt(); + let values: Vec = (0..count).map(|_| self.rng.next() * scale).collect(); + self.inits.push(tensor(name, dims, &values)); + name.to_string() + } + + fn bias(&mut self, name: &str, n: u64) -> String { + let values: Vec = (0..n).map(|_| self.rng.next() * 0.1).collect(); + self.inits.push(tensor(name, &[n], &values)); + name.to_string() + } + + /// `Conv` (+ `LeakyRelu` when `leaky`); gives the name of the tensor that results. + fn conv(&mut self, src: &str, cin: u64, cout: u64, k: u64, leaky: bool) -> String { + let (w, b, out) = (self.name("w"), self.name("b"), self.name("conv")); + let w = self.weights(&w, &[cout, cin, k, k], (cin * k * k) as usize); + let b = self.bias(&b, cout); + let pad = (k as i64 - 1) / 2; + self.nodes.push(node( + "Conv", + &[src, &w, &b], + &out, + &[("kernel_shape", A::Ints(&[k as i64, k as i64])), ("pads", A::Ints(&[pad; 4])), ("strides", A::Ints(&[1, 1])), ("group", A::Int(1))], + )); + if !leaky { + return out; + } + let act = self.name("act"); + self.nodes.push(node("LeakyRelu", &[&out], &act, &[("alpha", A::Float(0.2))])); + act + } + + fn pool(&mut self, src: &str) -> String { + let out = self.name("pool"); + self.nodes.push(node( + "MaxPool", + &[src], + &out, + &[("kernel_shape", A::Ints(&[2, 2])), ("strides", A::Ints(&[2, 2])), ("pads", A::Ints(&[0; 4]))], + )); + out + } + + fn up(&mut self, src: &str, cin: u64, cout: u64) -> String { + let (w, b, out) = (self.name("uw"), self.name("ub"), self.name("up")); + let w = self.weights(&w, &[cin, cout, 2, 2], (cin * 4) as usize); + let b = self.bias(&b, cout); + self.nodes.push(node( + "ConvTranspose", + &[src, &w, &b], + &out, + &[("kernel_shape", A::Ints(&[2, 2])), ("strides", A::Ints(&[2, 2])), ("pads", A::Ints(&[0; 4]))], + )); + out + } + + fn concat(&mut self, a: &str, b: &str) -> String { + let out = self.name("cat"); + self.nodes.push(node("Concat", &[a, b], &out, &[("axis", A::Int(1))])); + out + } +} + +/// A denoiser-shaped U-Net as ONNX bytes: `[1, 4, tile, tile]` → `[1, 3, 2·tile, 2·tile]`, with `depth` poolings +/// (so `tile` must be a multiple of `2^depth`) and `ch` channels on the first level, doubling on each (every count a +/// multiple of 4). Each level is two 3 × 3 convolutions with a leaky ReLU, as in the real networks; the output is a +/// 1 × 1 convolution to 12 channels and depth-to-space. +pub fn unet_onnx(tile: u64, ch: u64, depth: usize, seed: u64) -> Vec { + let mut b = Builder { nodes: Vec::new(), inits: Vec::new(), rng: Rng(seed | 1), n: 0 }; + let mut skips: Vec<(String, u64)> = Vec::new(); + let (mut cur, mut c_in, mut c) = ("data".to_string(), 4u64, ch); + for level in 0..=depth { + let x = b.conv(&cur, c_in, c, 3, true); + let x = b.conv(&x, c, c, 3, true); + if level < depth { + skips.push((x.clone(), c)); + cur = b.pool(&x); + c_in = c; + c *= 2; + } else { + cur = x; + } + } + for (skip, sc) in skips.into_iter().rev() { + let up = b.up(&cur, c, sc); + let joined = b.concat(&up, &skip); + let x = b.conv(&joined, 2 * sc, sc, 3, true); + cur = b.conv(&x, sc, sc, 3, true); + c = sc; + } + let rgb = b.conv(&cur, c, 12, 1, false); + b.nodes.push(node("DepthToSpace", &[&rgb], "rgb", &[("blocksize", A::Int(2)), ("mode", A::Str("CRD"))])); + + let mut graph = Vec::new(); + for n in &b.nodes { + len_field(1, n, &mut graph); + } + for t in &b.inits { + len_field(5, t, &mut graph); + } + len_field(11, &value_info_bytes("data", 1, &[Ok(1), Ok(4), Ok(tile), Ok(tile)]), &mut graph); + len_field(12, &value_info_bytes("rgb", 1, &[Ok(1), Ok(3), Ok(2 * tile), Ok(2 * tile)]), &mut graph); + let mut ops = Vec::new(); + len_field(1, b"", &mut ops); + varint_field(2, 13, &mut ops); + let mut model = Vec::new(); + varint_field(1, 8, &mut model); + len_field(7, &graph, &mut model); + len_field(8, &ops, &mut model); + model +} + +/// A tiny smoothing network for installing a real CPU model in setup-flow tests. +/// It averages a 3 × 3 neighborhood of the four Bayer planes and returns three identical channels. +pub fn smoothing_onnx() -> Vec { + let mut graph = Vec::new(); + len_field(1, &node("Conv", &["data", "weights"], "samples", &[("kernel_shape", A::Ints(&[3, 3])), ("pads", A::Ints(&[1, 1, 1, 1]))]), &mut graph); + len_field(1, &node("DepthToSpace", &["samples"], "rgb", &[("blocksize", A::Int(2)), ("mode", A::Str("CRD"))]), &mut graph); + len_field(5, &tensor("weights", &[12, 4, 3, 3], &[1.0 / 36.0; 12 * 4 * 3 * 3]), &mut graph); + len_field(11, &value_info_bytes("data", 1, &[Ok(1), Ok(4), Ok(64), Ok(64)]), &mut graph); + len_field(12, &value_info_bytes("rgb", 1, &[Ok(1), Ok(3), Ok(128), Ok(128)]), &mut graph); + let mut model = Vec::new(); + varint_field(1, 8, &mut model); + len_field(7, &graph, &mut model); + let mut ops = Vec::new(); + varint_field(2, 13, &mut ops); + len_field(8, &ops, &mut model); + model +} diff --git a/crates/denoise/src/synthetic_proto.rs b/crates/denoise/src/synthetic_proto.rs new file mode 100644 index 00000000..a78d586b --- /dev/null +++ b/crates/denoise/src/synthetic_proto.rs @@ -0,0 +1,44 @@ +pub(crate) fn varint(mut v: u64, out: &mut Vec) { + loop { + let b = (v & 0x7f) as u8; + v >>= 7; + if v == 0 { + out.push(b); + return; + } + out.push(b | 0x80); + } +} + +pub fn len_field(n: u64, body: &[u8], out: &mut Vec) { + varint(n << 3 | 2, out); + varint(body.len() as u64, out); + out.extend_from_slice(body); +} + +pub(crate) fn varint_field(n: u64, v: u64, out: &mut Vec) { + varint(n << 3, out); + varint(v, out); +} + +/// A graph input or output: `Ok(n)` is a fixed dimension, `Err(name)` a named (dynamic) one. +pub fn value_info_bytes(name: &str, elem: u64, dims: &[std::result::Result]) -> Vec { + let mut shape = Vec::new(); + for d in dims { + let mut dim = Vec::new(); + match d { + Ok(v) => varint_field(1, *v, &mut dim), + Err(p) => len_field(2, p.as_bytes(), &mut dim), + } + len_field(1, &dim, &mut shape); + } + let mut tensor = Vec::new(); + varint_field(1, elem, &mut tensor); + len_field(2, &shape, &mut tensor); + let mut ty = Vec::new(); + len_field(1, &tensor, &mut ty); + let mut vi = Vec::new(); + len_field(1, name.as_bytes(), &mut vi); + len_field(2, &ty, &mut vi); + vi +} diff --git a/crates/denoise/src/tiles.rs b/crates/denoise/src/tiles.rs new file mode 100644 index 00000000..59898c5a --- /dev/null +++ b/crates/denoise/src/tiles.rs @@ -0,0 +1,100 @@ +//! Cutting a picture into overlapping tiles and putting the results back together. +//! +//! A model runs on a fixed square, so a picture is covered by tiles spaced evenly along each axis, each sharing +//! at least `overlap` cells with its neighbours. Each tile's influence fades towards its own edges (except at the +//! picture's edge), and the result is the sum of tile × weight divided by the sum of weights, so any set of +//! overlapping tiles blends consistently and a seam between two tiles of slightly different scale is a smooth ramp. + +/// Where the tiles start along one axis of `n` cells: `[0]` when the axis fits one tile (the tile is then padded by +/// mirroring), else evenly spaced starts, the first at 0 and the last at `n − tile`, each tile sharing at least +/// `overlap` cells with the next. +pub fn starts(n: usize, tile: usize, overlap: usize) -> Vec { + if n <= tile || tile == 0 { + return vec![0]; + } + let step = tile.saturating_sub(overlap).max(1); + // tiles needed so that start k·spacing with spacing ≤ step reaches n − tile + let Some(k) = (n - tile).div_ceil(step).checked_add(1).filter(|&k| k <= 1_000_000) else { return Vec::new() }; + let last = n - tile; + (0..k).map(|i| if k == 1 { 0 } else { ((i as u128 * last as u128 + (k - 1) as u128 / 2) / (k - 1) as u128) as usize }).collect() +} + +/// Weight of the cell at offset `p` (0-based, inside a tile of `tile` cells) along one axis: 1 in the middle, easing +/// down to a sliver over `fade` cells at an edge that has a neighbour, 1 right up to an edge that is the picture's. +pub fn weight(p: usize, tile: usize, fade: usize, has_before: bool, has_after: bool) -> f32 { + let fade = fade.max(1) as f32; + let ramp = |d: usize| { + let t = ((d as f32 + 0.5) / fade).min(1.0); + // smoothstep, never exactly 0 so that a cell is always covered + (t * t * (3.0 - 2.0 * t)).max(1e-3) + }; + let before = if has_before { ramp(p) } else { 1.0 }; + let after = if has_after { ramp(tile.saturating_sub(1).saturating_sub(p)) } else { 1.0 }; + before.min(after) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hostile_axes_have_bounded_tile_counts_and_checked_spacing() { + assert!(starts(usize::MAX, 1, 0).is_empty()); + assert!(starts(usize::MAX, 2, usize::MAX).is_empty()); + let s = starts(usize::MAX, usize::MAX / 2, 0); + assert_eq!(s.first(), Some(&0)); + assert_eq!(s.last(), Some(&(usize::MAX - usize::MAX / 2))); + assert!(s.windows(2).all(|s| s[0] < s[1])); + } + + #[test] + fn one_tile_when_it_fits() { + assert_eq!(starts(100, 512, 64), vec![0]); + assert_eq!(starts(512, 512, 64), vec![0]); + assert_eq!(starts(0, 512, 64), vec![0]); + assert_eq!(starts(10, 0, 0), vec![0]); + } + + #[test] + fn tiles_cover_the_axis_and_overlap_enough() { + for n in [513usize, 600, 900, 1000, 1024, 1500, 3024, 3600, 5000, 12345] { + let s = starts(n, 512, 64); + assert_eq!(s[0], 0, "n={n}"); + assert_eq!(*s.last().unwrap(), n - 512, "n={n}"); + for w in s.windows(2) { + assert!(w[1] > w[0], "starts increase, n={n}"); + assert!(w[0] + 512 >= w[1] + 64, "at least 64 shared cells between {} and {}, n={n}", w[0], w[1]); + } + } + } + + #[test] + fn weights_fade_only_towards_neighbours() { + let (t, f) = (512, 64); + // a lone tile: full weight everywhere + assert!((0..t).all(|p| weight(p, t, f, false, false) == 1.0)); + // the first tile fades on its far side only + assert_eq!(weight(0, t, f, false, true), 1.0); + assert!(weight(t - 1, t, f, false, true) < 0.01); + // the last tile fades on its near side only + assert!(weight(0, t, f, true, false) < 0.01); + assert_eq!(weight(t - 1, t, f, true, false), 1.0); + // a middle tile is 1 in the middle and everywhere positive + assert_eq!(weight(t / 2, t, f, true, true), 1.0); + assert!((0..t).all(|p| weight(p, t, f, true, true) > 0.0)); + } + + #[test] + fn two_overlapping_tiles_blend_to_a_partition_of_unity() { + // at every cell of the shared region the normalised weights are in [0, 1] and sum to 1 + let (t, f) = (512usize, 64usize); + let s = starts(900, t, 64); + assert_eq!(s.len(), 2); + for cell in s[1]..s[0] + t { + let a = weight(cell - s[0], t, f, false, true); + let b = weight(cell - s[1], t, f, true, false); + let (na, nb) = (a / (a + b), b / (a + b)); + assert!((na + nb - 1.0).abs() < 1e-6 && na >= 0.0 && nb >= 0.0); + } + } +} diff --git a/crates/denoise/tests/real_model.rs b/crates/denoise/tests/real_model.rs new file mode 100644 index 00000000..bea7e712 --- /dev/null +++ b/crates/denoise/tests/real_model.rs @@ -0,0 +1,248 @@ +//! Opt-in end-to-end check of the real RawNIND Bayer model on real mosaics with noise added: +//! +//! ```text +//! LC_DENOISE_MODEL= LC_DENOISE_MOSAICS= LC_DENOISE_OUT= \ +//! cargo test --release -p lightcraft-denoise --test real_model -- --ignored --nocapture +//! ``` +//! +//! `LC_DENOISE_MOSAICS` holds `.mosaic.f32` (little-endian f32, normalised: black 0, white 1, not white +//! balanced), `.rgb.f32` (a clean demosaic of it, camera RGB, interleaved) and `.txt` (`width`, +//! `height`, `cfa`, `wb` lines). Noise is Poisson-Gaussian, made here. Optional: `LC_DENOISE_PARALLEL` (tiles at +//! once, default 8), `LC_DENOISE_ONLY` (one name), `LC_DENOISE_ISO` (noise strength, default 1.0; 0 adds none), +//! `LC_DENOISE_FULL` (the whole frame instead of a 2048 × 1536 crop). + +use std::path::{Path, PathBuf}; +use std::time::Instant; + +use lightcraft_denoise::bayer::Layout; +use lightcraft_denoise::manifest::{DenoiserManifest, Domain, Gain}; +use lightcraft_denoise::run::{Control, Params, denoise_bayer}; +use lightcraft_denoise::runtime::CpuRunner; + +struct Rng(u64); +impl Rng { + fn next(&mut self) -> f64 { + self.0 ^= self.0 >> 12; + self.0 ^= self.0 << 25; + self.0 ^= self.0 >> 27; + ((self.0.wrapping_mul(0x2545_f491_4f6c_dd1d) >> 11) as f64 + 0.5) / (1u64 << 53) as f64 + } + fn gauss(&mut self) -> f64 { + (-2.0 * self.next().ln()).sqrt() * (std::f64::consts::TAU * self.next()).cos() + } +} + +fn read_f32(path: &Path) -> Vec { + std::fs::read(path).unwrap().as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect() +} + +struct Dump { + width: usize, + height: usize, + layout: Layout, + wb: [f32; 3], + mosaic: Vec, + rgb: Vec, +} + +fn load(dir: &Path, name: &str) -> Dump { + let text = std::fs::read_to_string(dir.join(format!("{name}.txt"))).unwrap(); + let field = |k: &str| text.lines().find_map(|l| l.strip_prefix(k).map(|v| v.trim().to_string())).unwrap(); + let cell = match field("cfa").as_str() { + "RGGB" => [0, 1, 1, 2], + "BGGR" => [2, 1, 1, 0], + "GRBG" => [1, 0, 2, 1], + "GBRG" => [1, 2, 0, 1], + other => panic!("cfa {other}"), + }; + let wb: Vec = field("wb").split_whitespace().map(|v| v.parse().unwrap()).collect(); + Dump { + width: field("width").parse().unwrap(), + height: field("height").parse().unwrap(), + layout: Layout::from_cell(cell).unwrap(), + wb: [wb[0], wb[1], wb[2]], + mosaic: read_f32(&dir.join(format!("{name}.mosaic.f32"))), + rgb: read_f32(&dir.join(format!("{name}.rgb.f32"))), + } +} + +/// Crop with an even origin so the Bayer phase is kept. +fn crop(d: &Dump, x0: usize, y0: usize, w: usize, h: usize) -> (Vec, Vec) { + let mut m = Vec::with_capacity(w * h); + let mut c = Vec::with_capacity(w * h * 3); + for y in y0..y0 + h { + m.extend_from_slice(&d.mosaic[y * d.width + x0..y * d.width + x0 + w]); + c.extend_from_slice(&d.rgb[(y * d.width + x0) * 3..(y * d.width + x0 + w) * 3]); + } + (m, c) +} + +/// 0 = red, 1 = green, 2 = blue at a mosaic position. +fn site(layout: Layout, x: usize, y: usize) -> usize { + match ((x + layout.dx) % 2, (y + layout.dy) % 2) { + (0, 0) => 0, + (1, 1) => 2, + _ => 1, + } +} + +/// The simplest demosaic (each missing colour is the mean of the same-coloured samples in the 3 × 3 around), +/// as the baseline a user would get from the noisy picture without AI. +fn bilinear(m: &[f32], w: usize, h: usize, layout: Layout) -> Vec { + let mut out = vec![0f32; w * h * 3]; + for y in 0..h { + for x in 0..w { + let mut sum = [0f32; 3]; + let mut n = [0f32; 3]; + for yy in y.saturating_sub(1)..(y + 2).min(h) { + for xx in x.saturating_sub(1)..(x + 2).min(w) { + let c = site(layout, xx, yy); + sum[c] += m[yy * w + xx]; + n[c] += 1.0; + } + } + for c in 0..3 { + out[(y * w + x) * 3 + c] = if n[c] > 0.0 { sum[c] / n[c] } else { 0.0 }; + } + } + } + out +} + +/// Signal-to-error ratio in dB of `got` against `want` (both interleaved RGB), ignoring a border. +fn snr(got: &[f32], want: &[f32], w: usize, h: usize, border: usize) -> f64 { + let (mut sig, mut err) = (0f64, 0f64); + for y in border..h - border { + for x in border..w - border { + for c in 0..3 { + let (g, t) = (f64::from(got[(y * w + x) * 3 + c]), f64::from(want[(y * w + x) * 3 + c])); + sig += t * t; + err += (g - t).powi(2); + } + } + } + 10.0 * (sig / err.max(1e-30)).log10() +} + +fn crc32(data: &[u8]) -> u32 { + let mut c = !0u32; + for &b in data { + c ^= u32::from(b); + for _ in 0..8 { + c = if c & 1 != 0 { 0xedb8_8320 ^ (c >> 1) } else { c >> 1 }; + } + } + !c +} + +/// A viewable PNG of interleaved camera-RGB panels side by side: white balanced, one exposure for all, gamma. +fn write_png(path: &Path, panels: &[&[f32]], w: usize, h: usize, wb: [f32; 3], exposure: f32) { + let total_w = w * panels.len(); + let mut raw = Vec::with_capacity((total_w * 3 + 1) * h); + for y in 0..h { + raw.push(0u8); + for p in panels { + for x in 0..w { + for c in 0..3 { + let v = (p[(y * w + x) * 3 + c] * wb[c] * exposure).clamp(0.0, 1.0); + raw.push((v.powf(1.0 / 2.2) * 255.0 + 0.5) as u8); + } + } + } + } + let z = miniz_oxide::deflate::compress_to_vec_zlib(&raw, 6); + let mut png = b"\x89PNG\r\n\x1a\n".to_vec(); + let mut chunk = |kind: &[u8; 4], body: &[u8]| { + png.extend_from_slice(&(body.len() as u32).to_be_bytes()); + let mut k = kind.to_vec(); + k.extend_from_slice(body); + png.extend_from_slice(&k); + png.extend_from_slice(&crc32(&k).to_be_bytes()); + }; + let mut ihdr = Vec::new(); + ihdr.extend_from_slice(&(total_w as u32).to_be_bytes()); + ihdr.extend_from_slice(&(h as u32).to_be_bytes()); + ihdr.extend_from_slice(&[8, 2, 0, 0, 0]); + chunk(b"IHDR", &ihdr); + chunk(b"IDAT", &z); + chunk(b"IEND", &[]); + std::fs::write(path, png).unwrap(); +} + +#[test] +#[ignore = "needs a real model and mosaics: see the module docs"] +fn the_real_model_denoises_real_mosaics() { + let (Some(model), Some(dir)) = (std::env::var_os("LC_DENOISE_MODEL"), std::env::var_os("LC_DENOISE_MOSAICS")) else { return }; + let dir = PathBuf::from(dir); + let out_dir = std::env::var_os("LC_DENOISE_OUT").map(PathBuf::from); + let parallel: usize = std::env::var("LC_DENOISE_PARALLEL").ok().and_then(|v| v.parse().ok()).unwrap_or(8); + let iso: f64 = std::env::var("LC_DENOISE_ISO").ok().and_then(|v| v.parse().ok()).unwrap_or(1.0); + let names: Vec = match std::env::var("LC_DENOISE_ONLY") { + Ok(n) => vec![n], + Err(_) => ["arw-sony-a7m3-compressed", "cr2-canon-6d", "nef-nikon-d5100-lossless", "dng-google-pixel2xl", "pef-pentax-k3"] + .map(String::from) + .to_vec(), + }; + let manifest = DenoiserManifest { + id: "rawnind-bayer".into(), + name: "RawNIND Bayer".into(), + version: "1".into(), + licence: Default::default(), + source: None, + sha256: None, + size_bytes: None, + provenance: String::new(), + domain: Domain::BayerToRgb, + tile: 512, + overlap: 64, + gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 }, + }; + let runner = CpuRunner::load(Path::new(&model), &manifest).unwrap(); + let params = Params { tile: 512, overlap: 64, gain: manifest.gain, clip: Some(1.0), parallel }; + for name in names { + let d = load(&dir, &name); + // a crop of about 2 × 1.5 tiles, away from the edges, even origin + let full = std::env::var_os("LC_DENOISE_FULL").is_some(); + let (w, h) = if full { (d.width & !1, d.height & !1) } else { (2048usize, 1536usize) }; + let (x0, y0) = (((d.width - w) / 3) & !1, ((d.height - h) / 2) & !1); + let (clean, reference) = crop(&d, x0, y0, w, h); + // Poisson-Gaussian noise: variance a·signal + b, as a sensor at high ISO + let (a, b) = (1.5e-3 * iso, (4.0e-3 * iso.sqrt()).powi(2)); + let mut rng = Rng(0x9e37_79b9_7f4a_7c15 ^ name.len() as u64); + let noisy: Vec = clean + .iter() + .map(|&v| { + let v = f64::from(v); + let sigma = (a * v.max(0.0) + b).sqrt(); + ((v + sigma * rng.gauss()) as f32).min(1.0) + }) + .collect(); + let started = Instant::now(); + let done = std::sync::atomic::AtomicUsize::new(0); + let progress = |n: usize, total: usize| done.store(n * 1000 / total, std::sync::atomic::Ordering::Relaxed); + let denoised = denoise_bayer(&noisy, w, h, d.layout, &runner, ¶ms, &Control { cancel: None, progress: Some(&progress) }).unwrap(); + let secs = started.elapsed().as_secs_f64(); + let baseline = bilinear(&noisy, w, h, d.layout); + let clean_bilinear = bilinear(&clean, w, h, d.layout); + let den: Vec = denoised.data.iter().flat_map(|p| p.iter().copied()).collect(); + let (s_noisy, s_den) = (snr(&baseline, &reference, w, h, 64), snr(&den, &reference, w, h, 64)); + let s_floor = snr(&clean_bilinear, &reference, w, h, 64); + println!( + "{name}: {w}×{h} crop, {} tiles, parallel {parallel}, {secs:.1} s | SNR vs clean demosaic: noisy bilinear {s_noisy:.2} dB, denoised {s_den:.2} dB (clean bilinear {s_floor:.2} dB)", + lightcraft_denoise::run::tile_count(w, h, d.layout, 512, 64) + ); + if let Some(out) = &out_dir { + std::fs::create_dir_all(out).unwrap(); + let mean = reference.iter().map(|&v| f64::from(v)).sum::() / reference.len() as f64; + let exposure = (0.18 / mean.max(1e-6)) as f32; + // three panels, a 512 × 384 window each, at 1:1 + let (pw, ph, px, py) = (512usize, 384usize, 700usize, 500usize); + let window = + |buf: &[f32]| -> Vec { (0..ph).flat_map(|y| buf[((py + y) * w + px) * 3..((py + y) * w + px + pw) * 3].to_vec()).collect() }; + write_png(&out.join(format!("{name}.png")), &[&window(&baseline), &window(&den), &window(&reference)], pw, ph, d.wb, exposure); + } + if iso >= 4.0 { + assert!(s_den > s_noisy + 2.0, "{name}: denoised {s_den:.2} dB is not clearly better than noisy {s_noisy:.2} dB"); + } + } +} diff --git a/crates/develop/src/controls.rs b/crates/develop/src/controls.rs index 5696d670..c8a001b7 100644 --- a/crates/develop/src/controls.rs +++ b/crates/develop/src/controls.rs @@ -207,6 +207,7 @@ controls! { "grain.amount" => grain.amount, "Grain", Grain, 0, 100, 0, 1, 0, Plain; "grain.size" => grain.size, "Size", Grain, 0, 100, 25, 1, 0, Plain; "grain.roughness" => grain.roughness, "Roughness", Grain, 0, 100, 50, 1, 0, Plain; + "enhance.denoise" => enhance.denoise, "Amount", Detail, 0, 100, 0, 1, 0, Plain; "detail.sharpenAmount" => detail.sharpen_amount, "Sharpening", Detail, 0, 150, 0, 1, 0, Plain; "detail.sharpenRadius" => detail.sharpen_radius, "Radius", Detail, 0.5, 3, 1, 0.1, 1, Plain; "detail.sharpenDetail" => detail.sharpen_detail, "Detail", Detail, 0, 100, 25, 1, 0, Plain; diff --git a/crates/develop/src/lib.rs b/crates/develop/src/lib.rs index bfce1893..b7c95251 100644 --- a/crates/develop/src/lib.rs +++ b/crates/develop/src/lib.rs @@ -94,6 +94,13 @@ impl DevelopSettings { std::borrow::Cow::Owned(d) } + /// How much of the AI-denoised picture to mix in, 0 (none) to 1 (all of it): the Denoise amount, unless the Detail + /// section is switched off or the value is not a number. + pub fn denoise_amount(&self) -> f32 { + let a = self.enhance.denoise; + if a.is_finite() && self.enhance.denoise_enabled() && self.section_enabled("detail") { (a / 100.0).clamp(0.0, 1.0) as f32 } else { 0.0 } + } + pub fn set_section_enabled(&mut self, section: &str, on: bool) { self.disabled_sections.retain(|s| s != section); if !on { @@ -112,6 +119,7 @@ impl DevelopSettings { self.curve = d; } Section::Color => self.wb.mode = WbMode::AsShot, + Section::Detail => self.enhance.denoise_on = None, _ => {} } } @@ -193,4 +201,40 @@ mod tests { s.set_section_enabled("effects", true); assert!(s.section_enabled("effects")); } + + #[test] + fn the_denoise_amount_is_a_slider_and_follows_the_detail_section() { + let mut s = DevelopSettings::default(); + assert_eq!(s.denoise_amount(), 0.0); + assert!(controls::set(&mut s, "enhance.denoise", 60.0)); + assert_eq!(controls::get(&s, "enhance.denoise"), Some(60.0)); + assert!((s.denoise_amount() - 0.6).abs() < 1e-6); + // the slider's range is 0 to 100; values outside it are brought in + controls::set(&mut s, "enhance.denoise", 500.0); + assert_eq!(s.denoise_amount(), 1.0); + controls::set(&mut s, "enhance.denoise", -5.0); + assert_eq!(s.denoise_amount(), 0.0); + // switching the Detail section off switches the denoise off with it; a settings file with a bad number does nothing + controls::set(&mut s, "enhance.denoise", 80.0); + s.set_section_enabled("detail", false); + assert_eq!(s.denoise_amount(), 0.0); + s.set_section_enabled("detail", true); + s.enhance.denoise = f64::NAN; + assert_eq!(s.denoise_amount(), 0.0); + // it is part of Detail: a reset of the section clears it + controls::set(&mut s, "enhance.denoise", 40.0); + s.reset_section(Section::Detail); + assert_eq!(s.enhance.denoise, 0.0); + // The later independent switch preserves Amount when off, and can be on at zero. + controls::set(&mut s, "enhance.denoise", 75.0); + s.enhance.denoise_on = Some(false); + assert_eq!(s.denoise_amount(), 0.0); + assert_eq!(s.enhance.denoise, 75.0); + s.enhance.denoise_on = Some(true); + assert_eq!(s.denoise_amount(), 0.75); + controls::set(&mut s, "enhance.denoise", 0.0); + assert!(s.enhance.denoise_enabled()); + s.reset_section(Section::Detail); + assert_eq!(s.enhance.denoise_on, None); + } } diff --git a/crates/develop/src/settings.rs b/crates/develop/src/settings.rs index 910cd07d..5f0db8ab 100644 --- a/crates/develop/src/settings.rs +++ b/crates/develop/src/settings.rs @@ -915,6 +915,14 @@ pub struct LensBlur { #[serde(default)] pub struct Enhance { pub denoise: f64, + #[serde(skip_serializing_if = "Option::is_none")] + pub denoise_on: Option, pub raw_details: bool, pub super_resolution: bool, } + +impl Enhance { + pub fn denoise_enabled(&self) -> bool { + self.denoise_on.unwrap_or(self.denoise > 0.0) + } +} diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index a916d180..536df716 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -21,6 +21,7 @@ web-time = { workspace = true } lightcraft-preview = { workspace = true } lightcraft-codecs = { workspace = true } lightcraft-raw = { workspace = true } +lightcraft-denoise = { workspace = true } lightcraft-meta = { workspace = true } miniz_oxide = { workspace = true } lightcraft-merge = { workspace = true } @@ -38,6 +39,10 @@ lightcraft-segment = { workspace = true, optional = true } [features] # AI masks with SAM 3 (Object / Describe). Off for the web and CLI builds. sam = ["dep:lightcraft-segment"] +denoise = [] + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +lightcraft-fetch = { workspace = true } [dev-dependencies] libc = { workspace = true } diff --git a/crates/engine/src/cmd/denoise.rs b/crates/engine/src/cmd/denoise.rs new file mode 100644 index 00000000..c5103791 --- /dev/null +++ b/crates/engine/src/cmd/denoise.rs @@ -0,0 +1,543 @@ +//! Denoise commands (`denoise.*`): the models (list, install, download, test, remove, choose), the settings, and the +//! work (queue, cancel, status, pump, clear). The machinery they drive is [`crate::denoise`]. +//! +//! Models live in the folder the host provides (`Session::denoise.models_dir`): one subfolder per model with +//! `model.onnx`, `denoise-model.json` (its manifest) and `installed.json` (what the user accepted and the file's hash), +//! plus `settings.json` beside them. Model files, archives and manifests are hostile input: sizes are capped, ids are +//! validated before they become folder names, and nothing outside a model's own folder is ever touched. + +use std::path::{Path, PathBuf}; +use std::sync::PoisonError; + +use lightcraft_catalog::PhotoId; +use lightcraft_denoise::archive; +use lightcraft_denoise::hash::sha256_file; +use lightcraft_denoise::known::{self, Known}; +use lightcraft_denoise::manifest::{self, DenoiserManifest, MAX_MANIFEST_BYTES, MAX_MODEL_BYTES}; +use serde_json::{Value, json}; + +use super::{CommandSpec, always, bad, bool_or, cmd, str_param}; +use crate::denoise::{Pace, RunOn, Settings, forget_gpu_set_ups, installed_models, read_capped, read_settings, write_atomic, write_settings}; +use crate::model_download::{self, State}; +use crate::{EngineError, Result, Session}; + +fn fail(what: &str, e: impl std::fmt::Display) -> EngineError { + EngineError::Other(format!("{what}: {e}")) +} + +fn models_dir(s: &Session, cmd: &str) -> Result { + s.denoise.models_dir.clone().ok_or_else(|| bad(cmd, "this build has nowhere to keep denoise models (the desktop app does)")) +} + +/// What a command says about a model. +fn row(m: &DenoiserManifest, download_host: Option, installed: bool, selected: bool, accepted: &Value) -> Value { + json!({ + "id": m.id, + "name": m.name, + "version": m.version, + "licence": m.licence, + "provenance": m.provenance, + "source": m.source, + "sizeBytes": m.size_bytes, + "sha256": m.sha256, + "tile": m.tile, + "known": known::find(&m.id).is_some(), + // a pinned address LightCraft can fetch it from (the user presses Download), and which site that is + "downloadHost": download_host, + "installed": installed, + "selected": selected, + "accepted": accepted, + }) +} + +fn host_of(k: &Known) -> Option { + k.download.as_ref().map(|d| lightcraft_denoise::known::host(&d.url).to_string()) +} + +fn list(s: &mut Session, _: &Value) -> Result { + s.denoise_refresh_active(true); + let dir = s.denoise.models_dir.clone(); + let st = dir.as_deref().map(read_settings).unwrap_or_default(); + let installed = dir.as_deref().map(installed_models).unwrap_or_default(); + let on_disk = |id: &str| installed.iter().find(|i| i.manifest.id == id); + let chosen = |id: &str| st.model.as_deref() == Some(id); + let mut models: Vec = Vec::new(); + for k in known::all() { + let found = on_disk(&k.manifest.id); + models.push(row(&k.manifest, host_of(&k), found.is_some(), chosen(&k.manifest.id), found.map_or(&Value::Null, |f| &f.accepted))); + } + for i in installed.iter().filter(|i| known::find(&i.manifest.id).is_none()) { + models.push(row(&i.manifest, None, true, chosen(&i.manifest.id), &i.accepted)); + } + Ok(json!({ + "dir": dir.as_ref().map(|d| d.display().to_string()), + "productsDir": s.denoise_products_dir().map(|d| d.display().to_string()), + "model": st.model, + // whether this build can run denoise models (the `denoise` feature: our pure-Rust CPU/GPU runners) + "runtime": cfg!(feature = "denoise"), + "auto": st.auto(), + "cacheGb": st.cache_bytes() >> 30, + "threads": st.threads, + "runOn": st.run_on().name(), + "models": models, + })) +} + +/// What a file the user pointed at is, as far as installing it goes. +struct Source { + manifest: DenoiserManifest, + /// The `.onnx` inside the archive; `None` when the file is the `.onnx`. + entry: Option<&'static str>, + file_name: String, +} + +fn identify(path: &Path, c: &str, sha_known: Option<&str>) -> Result { + let meta = std::fs::metadata(path).map_err(|e| bad(c, format!("cannot read `{}`: {e}", path.display())))?; + if !meta.is_file() { + return Err(bad(c, format!("`{}` is not a file", path.display()))); + } + if meta.len() == 0 || meta.len() > MAX_MODEL_BYTES { + return Err(bad(c, "a model file must be between 1 byte and 2 GiB")); + } + let sha256 = match sha_known { + Some(h) => h.to_string(), + None => sha256_file(path).map_err(|e| fail(&format!("could not read {}", path.display()), e))?, + }; + let file_name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + // an archive LightCraft knows (the file darktable publishes the model in) + if let Some(k) = known::all().into_iter().find(|k| k.download.as_ref().is_some_and(|d| d.sha256 == sha256)) { + return Ok(Source { manifest: k.manifest, entry: k.entry, file_name }); + } + // the `.onnx` of a known model + if let Some(m) = known::by_sha256(&sha256) { + return Ok(Source { manifest: m, entry: None, file_name }); + } + // an `.onnx` with its manifest beside it + let sibling = path.with_file_name("denoise-model.json"); + if path.extension().is_some_and(|e| e.eq_ignore_ascii_case("onnx")) + && let Some(bytes) = read_capped(&sibling, MAX_MANIFEST_BYTES) + { + let m = manifest::parse(&bytes).map_err(|e| bad(c, format!("{}: {e}", sibling.display())))?; + return Ok(Source { manifest: m, entry: None, file_name }); + } + Err(bad( + c, + "LightCraft does not know this file. A denoise model is an .onnx file with a denoise-model.json beside it that says how to use it \ + (see docs/denoise.md), or an archive LightCraft can fetch itself (`denoise.models.download`)", + )) +} + +/// Install the model in `src`. A model that passes its self-test becomes the one in use (`activate`, the default): installing +/// a model is how someone says they want it. `verified` is for a download whose hash was already checked. +fn install_file(s: &mut Session, c: &str, src: &Path, acknowledged: bool, activate: bool, verified: bool) -> Result { + let dir = models_dir(s, c)?; + let ident = identify(src, c, None)?; + let m = ident.manifest.clone(); + manifest::validate(&m).map_err(|e| bad(c, e.to_string()))?; + if !acknowledged { + return Err(bad( + c, + "the licence has not been accepted: show the user its terms (see `denoise.models.list`) and pass `acknowledged: true` once they agree", + )); + } + let home = dir.join(&m.id); + std::fs::create_dir_all(&home).map_err(|e| fail("could not create the model's folder", e))?; + let (part, final_path) = (home.join("model.onnx.part"), home.join("model.onnx")); + let _ = std::fs::remove_file(&part); + let placed: Result<()> = match ident.entry { + Some(entry) => { + archive::extract(src, entry, &part, MAX_MODEL_BYTES).map(|_| ()).map_err(|e| fail("could not take the model out of the archive", e)) + } + None if verified && std::fs::rename(src, &part).is_ok() => Ok(()), + None => std::fs::copy(src, &part).map(|_| ()).map_err(|e| fail("could not copy the model (is the disk full?)", e)), + }; + let sha256 = placed.and_then(|_| sha256_file(&part).map_err(|e| fail("could not check the model", e))).and_then(|h| match &m.sha256 { + Some(want) if *want != h => Err(EngineError::Other("the model file is not the one its description names (its SHA-256 differs)".into())), + _ => Ok(h), + }); + let sha256 = match sha256 { + Ok(h) => h, + Err(e) => { + let _ = std::fs::remove_file(&part); + return Err(e); + } + }; + let _ = std::fs::remove_file(&final_path); + if let Err(e) = std::fs::rename(&part, &final_path) { + let _ = std::fs::remove_file(&part); + return Err(fail("could not finish the copy", e)); + } + // a model that does not work is never left installed + let run_on = s.denoise.settings.run_on(); + let test = (s.denoise.loader)(&final_path, &m).and_then(|model| model.self_test(run_on)); + let test = match test { + Ok(t) => t, + Err(e) => { + let _ = std::fs::remove_file(&final_path); + let _ = std::fs::remove_file(home.join("denoise-model.json")); + let _ = std::fs::remove_dir(&home); + return Err(bad(c, e)); + } + }; + write_atomic(&home.join("denoise-model.json"), &serde_json::to_vec_pretty(&m).map_err(|e| fail("manifest", e))?).map_err(EngineError::Other)?; + let accepted = json!({ + "acceptedAt": (s.clock)(), + "licence": m.licence.name, + "commercial": m.licence.commercial, + "fileName": ident.file_name, + "sha256": sha256, + "selfTest": test, + }); + write_atomic(&home.join("installed.json"), &serde_json::to_vec_pretty(&accepted).map_err(|e| fail("record", e))?).map_err(EngineError::Other)?; + // a file that was downloaded and waited for the user's acceptance has served: the installed copy is the model now + if src.parent() == Some(dir.join(model_download::STAGING).as_path()) { + let _ = std::fs::remove_file(src); + s.denoise.downloads.staged_file_used(src); + } + let mut st = read_settings(&dir); + if activate { + st.model = Some(m.id.clone()); + write_settings(&dir, &st).map_err(EngineError::Other)?; + } + s.denoise.touch(); + s.denoise_refresh_active(true); + let host = known::find(&m.id).and_then(|k| host_of(&k)); + Ok(json!({"installed": row(&m, host, true, st.model.as_deref() == Some(m.id.as_str()), &accepted), "model": st.model})) +} + +fn install(s: &mut Session, p: &Value) -> Result { + const C: &str = "denoise.models.install"; + let path = str_param(p, "path").ok_or_else(|| bad(C, "missing `path`"))?; + install_file(s, C, Path::new(path), bool_or(p, "acknowledged", false), bool_or(p, "activate", true), false) +} + +/// What an error says, without the command it came from (for a line shown to the user). +fn plain(e: &EngineError) -> String { + match e { + EngineError::BadParams { msg, .. } => msg.clone(), + other => other.to_string(), + } +} + +/// Install every download that has arrived: each was accepted when it was started, so it is installed and chosen without +/// another question. Called by `denoise.models.downloads` and by every frame's `denoise.pump`. +pub(crate) fn finish_downloads(s: &mut Session) { + const C: &str = "denoise.models.download"; + for (id, path, _sha) in s.denoise.downloads.finished() { + match install_file(s, C, &path, true, true, true) { + Ok(_) => s.denoise.downloads.set_outcome(&id, State::Installed), + Err(e) => { + // a file that cannot be installed is of no use to anyone: throw it away, say why + let _ = std::fs::remove_file(&path); + s.denoise.downloads.set_outcome(&id, State::Failed(plain(&e))); + } + } + } +} + +fn download(s: &mut Session, p: &Value) -> Result { + const C: &str = "denoise.models.download"; + let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?; + let dir = models_dir(s, C)?; + let k = known::find(id).ok_or_else(|| bad(C, "LightCraft has no download for that model: get the file from its page, then add it"))?; + let spec = k.download.clone().ok_or_else(|| bad(C, "LightCraft has no download for that model: get the file from its page, then add it"))?; + if !cfg!(feature = "denoise") { + return Err(bad(C, "this build cannot run denoise models, so there is nothing to download for it")); + } + if !bool_or(p, "acknowledged", false) { + return Err(bad( + C, + "the licence has not been accepted: show the user the model's terms (see `denoise.models.list`) and pass `acknowledged: true` once they agree", + )); + } + if installed_models(&dir).iter().any(|i| i.manifest.id == spec.id) { + return Err(bad(C, "that model is already installed")); + } + let host = lightcraft_denoise::known::host(&spec.url).to_string(); + s.denoise.downloads.start(spec, &dir).map_err(|e| bad(C, e))?; + Ok(json!({"started": id, "from": host})) +} + +fn download_row(id: &str, state: &State) -> Value { + let from = known::find(id).and_then(|k| host_of(&k)); + match state { + State::Running { bytes, total } => json!({"id": id, "state": "running", "bytes": bytes, "total": total, "from": from}), + State::Done { path, .. } => json!({"id": id, "state": "done", "path": path.display().to_string(), "from": from}), + State::Installed => json!({"id": id, "state": "installed", "from": from}), + State::Failed(why) => json!({"id": id, "state": "failed", "error": why, "from": from}), + State::Cancelled => json!({"id": id, "state": "cancelled", "from": from}), + } +} + +fn downloads(s: &mut Session, _: &Value) -> Result { + finish_downloads(s); + let all: Vec = s.denoise.downloads.snapshot().iter().map(|(id, st)| download_row(id, st)).collect(); + Ok(json!({"running": s.denoise.downloads.running(), "downloads": all})) +} + +fn download_cancel(s: &mut Session, p: &Value) -> Result { + const C: &str = "denoise.models.downloadCancel"; + let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?; + Ok(json!({"discarded": s.denoise.downloads.discard(id)})) +} + +fn test(s: &mut Session, p: &Value) -> Result { + const C: &str = "denoise.models.test"; + let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?; + let dir = models_dir(s, C)?; + let found = installed_models(&dir).into_iter().find(|i| i.manifest.id == id).ok_or_else(|| bad(C, "that model is not installed"))?; + let run_on = s.denoise.settings.run_on(); + let result = (s.denoise.loader)(&found.onnx, &found.manifest).and_then(|m| m.self_test(run_on)); + let (ok, detail) = match &result { + Ok(v) => (true, v.clone()), + Err(e) => (false, json!(e)), + }; + // keep the latest outcome with the model's record + let mut record = found.accepted; + if let Some(o) = record.as_object_mut() { + o.insert("selfTest".into(), if ok { detail.clone() } else { json!({"ok": false, "error": detail}) }); + write_atomic(&found.onnx.with_file_name("installed.json"), &serde_json::to_vec_pretty(&record).map_err(|e| fail("record", e))?) + .map_err(EngineError::Other)?; + } + Ok(json!({"id": id, "ok": ok, "result": detail})) +} + +fn remove(s: &mut Session, p: &Value) -> Result { + const C: &str = "denoise.models.remove"; + let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?; + let dir = models_dir(s, C)?; + if !lightcraft_denoise::licence::valid_id(id) { + return Err(bad(C, "not a model id")); + } + let home = dir.join(id); + // only a folder that is a model (it has a manifest) is ever deleted + if !home.join("denoise-model.json").is_file() { + return Err(bad(C, "that model is not installed")); + } + std::fs::remove_dir_all(&home).map_err(|e| fail("could not remove the model", e))?; + let mut st = read_settings(&dir); + if st.model.as_deref() == Some(id) { + st.model = installed_models(&dir).into_iter().map(|i| i.manifest.id).find(|m| m != id); + write_settings(&dir, &st).map_err(EngineError::Other)?; + } + s.denoise.touch(); + s.denoise_refresh_active(true); + Ok(json!({"removed": id, "model": st.model})) +} + +fn select(s: &mut Session, p: &Value) -> Result { + const C: &str = "denoise.models.select"; + let dir = models_dir(s, C)?; + let id = match p.get("id") { + None | Some(Value::Null) => None, + Some(v) => Some(v.as_str().ok_or_else(|| bad(C, "`id` must be a model id or null"))?.to_string()), + }; + if let Some(id) = &id + && !installed_models(&dir).iter().any(|i| &i.manifest.id == id) + { + return Err(bad(C, "that denoise model is not installed")); + } + let mut st = read_settings(&dir); + st.model = id; + write_settings(&dir, &st).map_err(EngineError::Other)?; + s.denoise.touch(); + s.denoise_refresh_active(true); + Ok(json!({"model": st.model})) +} + +fn settings(s: &mut Session, p: &Value) -> Result { + const C: &str = "denoise.settings"; + let dir = models_dir(s, C)?; + let mut st: Settings = read_settings(&dir); + if let Some(v) = p.get("auto") { + st.auto = Some(v.as_bool().ok_or_else(|| bad(C, "`auto` must be true or false"))?); + } + if let Some(v) = p.get("cacheGb") { + let n = + v.as_u64().filter(|n| (1..=10_000).contains(n)).ok_or_else(|| bad(C, "`cacheGb` must be a whole number of gigabytes from 1 to 10000"))?; + st.cache_gb = Some(n as u32); + } + if let Some(v) = p.get("threads") { + st.threads = match v { + Value::Null => None, + v => Some(v.as_u64().filter(|n| (1..=64).contains(n)).ok_or_else(|| bad(C, "`threads` must be from 1 to 64, or null"))? as u32), + }; + } + let run_on = match p.get("runOn") { + None => None, + Some(v) => Some(v.as_str().and_then(RunOn::parse).ok_or_else(|| bad(C, "`runOn` must be auto, gpu or cpu"))?), + }; + if let Some(r) = run_on { + st.run_on = Some(r.name().to_string()); + st.gpu = None; + } + write_settings(&dir, &st).map_err(EngineError::Other)?; + if run_on.is_some() { + // a choice of where denoise runs is also a go-ahead to try the card again: a set-up that crashed or gave up + // before is forgotten, and the loaded model sets the card up afresh for the next photo + forget_gpu_set_ups(&dir); + if let Some(a) = &s.denoise.active { + *a.model.lock().unwrap_or_else(PoisonError::into_inner) = None; + } + } + s.denoise.touch(); + s.denoise_refresh_active(true); + // a smaller limit takes effect now + s.denoise_evict(None); + Ok(json!({"auto": st.auto(), "cacheGb": st.cache_bytes() >> 30, "threads": st.threads, "runOn": st.run_on().name()})) +} + +fn ids_param(p: &Value) -> Option> { + p.get("ids").and_then(Value::as_array).map(|a| a.iter().filter_map(Value::as_u64).map(PhotoId).collect()) +} + +fn status_json(s: &Session, photo: Option) -> Value { + let (files, bytes) = s.denoise_cache_usage(); + let st = &s.denoise.settings; + let model = s.denoise.active.as_ref().map(|a| json!({"id": a.id, "name": a.manifest.name})); + let mut v = json!({ + "enabled": s.denoise.active.is_some(), + "model": model, + "runtime": cfg!(feature = "denoise"), + "productsDir": s.denoise_products_dir().map(|d| d.display().to_string()), + "queued": s.denoise.queued(), + "running": s.denoise_running_json(), + "ready": s.media.denoise.len(), + "made": s.denoise_made(), + "auto": st.auto(), + "runOn": st.run_on().name(), + "device": s.denoise_device(), + "cache": {"files": files, "bytes": bytes, "limitBytes": st.cache_bytes()}, + "generation": s.denoise.generation, + }); + if let (Some(id), Some(o)) = (photo, v.as_object_mut()) { + o.insert("photo".into(), serde_json::to_value(s.denoise_photo_state(id)).unwrap_or(Value::Null)); + } + v +} + +/// A per-photo switch, independent of the selection and Auto Sync. Turning on starts at 50%. +fn toggle(s: &mut Session, p: &Value) -> Result { + const C: &str = "denoise.toggle"; + let id = match p.get("id") { + Some(v) => PhotoId(v.as_u64().ok_or_else(|| bad(C, "`id` must be a photo id"))?), + None => s.active().ok_or_else(|| bad(C, "select a raw photo"))?, + }; + let photo = s.catalog.photo(id).ok_or_else(|| bad(C, "no such photo"))?; + if !crate::denoise::eligible(photo) { + return Err(bad(C, "AI Denoise applies to raw photos developed from their sensor data")); + } + let mut d = (*s.develop_of(id).ok_or_else(|| bad(C, "no such photo"))?).clone(); + let on = match p.get("enabled") { + Some(v) => v.as_bool().ok_or_else(|| bad(C, "`enabled` must be true or false"))?, + None => !d.enhance.denoise_enabled(), + }; + if on { + if !d.enhance.denoise.is_finite() { + return Err(bad(C, "the photo has an invalid Denoise amount")); + } + s.denoise_refresh_active(true); + if s.denoise.active.is_none() { + return Err(bad(C, "AI Denoise needs a model: install one in Settings > AI Denoise")); + } + } + // The switch is its own setting: the Amount stays where it is (0 included) when it is flipped, so flipping compares + // with and without. A photo that never had an Amount starts at 50 the first time. + let amount = if on && d.enhance.denoise_on.is_none() && d.enhance.denoise <= 0.0 { 50.0 } else { d.enhance.denoise }; + if d.enhance.denoise != amount || d.enhance.denoise_on != Some(on) { + s.end_interaction()?; + d.enhance.denoise = amount; + d.enhance.denoise_on = Some(on); + let op = s.develop_op(id, d, "AI Denoise").ok_or_else(|| bad(C, "no such photo"))?; + s.commit("AI Denoise", op)?; + } + if on { + s.denoise_enqueue(&[id], true, false); + } else { + s.denoise_cancel(Some(id), false); + } + Ok(json!({"id": id.0, "enabled": on, "amount": amount})) +} + +fn queue(s: &mut Session, p: &Value) -> Result { + const C: &str = "denoise.queue"; + s.denoise_refresh_active(true); + if s.denoise.active.is_none() { + return Err(bad(C, "no denoise model is chosen: install one first (`denoise.models.download` or `denoise.models.install`)")); + } + if s.denoise_products_dir().is_none() { + return Err(bad(C, "denoise keeps its pictures in the library's folder: open a library on disk first")); + } + let ids = match ids_param(p) { + Some(ids) => ids, + None => match str_param(p, "scope").unwrap_or("selected") { + "selected" => s.selection.active.iter().chain(s.selection.ids.iter()).copied().collect(), + "visible" => s.visible_cloned().to_vec(), + "withAmount" => s.catalog.photos().filter(|ph| ph.develop.denoise_amount() > 0.0).map(|ph| ph.id).collect(), + other => return Err(bad(C, format!("unknown scope `{other}`: use selected, visible or withAmount"))), + }, + }; + let queued = s.denoise_enqueue(&ids, false, bool_or(p, "retry", false)); + let mut v = status_json(s, None); + if let Some(o) = v.as_object_mut() { + o.insert("added".into(), json!(queued)); + } + Ok(v) +} + +fn cancel(s: &mut Session, p: &Value) -> Result { + let photo = p.get("id").and_then(Value::as_u64).map(PhotoId); + let n = s.denoise_cancel(photo, bool_or(p, "all", photo.is_none())); + Ok(json!({"cancelled": n})) +} + +fn status(s: &mut Session, p: &Value) -> Result { + s.denoise_refresh_active(false); + Ok(status_json(s, p.get("id").and_then(Value::as_u64).map(PhotoId))) +} + +/// `denoise.pump {pace?}`: cheap to call every frame. Takes in what the background job has finished and keeps one going. +fn pump(s: &mut Session, p: &Value) -> Result { + s.denoise_pump(Pace::parse(p.get("pace").and_then(Value::as_str))); + Ok(json!({ + "active": s.denoise.active.is_some(), + "running": s.denoise_running_json(), + "queued": s.denoise.queued(), + "ready": s.media.denoise.len(), + "generation": s.denoise.generation, + "downloading": s.denoise.downloads.running(), + })) +} + +fn clear(s: &mut Session, _: &Value) -> Result { + let (files, bytes) = s.denoise_clear(); + Ok(json!({"deleted": files, "bytes": bytes})) +} + +pub fn specs() -> Vec { + vec![ + cmd!(query "denoise.models.list", "Denoise Models", [], None, "{} → {dir, productsDir, model, runtime, auto, cacheGb, threads, runOn, models: [{id, name, version, licence{name, commercial, url, notice}, provenance, source, sizeBytes, sha256, tile, known, downloadHost, installed, selected, accepted}]}", always, list), + cmd!(query "denoise.models.install", "Install Denoise Model", [], None, "{path, acknowledged: true, activate?: true} → {installed, model} — install a denoise model from a file: an .onnx with a denoise-model.json beside it, or an archive LightCraft knows (the darktable `.dtmodel`). `acknowledged` must be true: the user has been shown the model's licence and accepted it. Unless `activate` is false it becomes the model in use", always, install), + cmd!(query "denoise.models.download", "Download Denoise Model", [], None, "{id, acknowledged: true} → {started, from} — fetch a model LightCraft has a pinned address for (see `downloadHost` in the list) in the background over HTTPS with lightcraft-fetch. `acknowledged` must be true: the user has been shown the model's terms and accepted them. It is checked against its size and SHA-256, installed and chosen by itself; `denoise.models.downloads` shows how far it is", always, download), + cmd!(query "denoise.models.downloads", "Denoise Model Downloads", [], None, "{} → {running, downloads: [{id, state: running | done | installed | failed | cancelled, bytes, total, error, from}]} — also installs any download that has arrived", always, downloads), + cmd!(query "denoise.models.downloadCancel", "Cancel Denoise Model Download", [], None, "{id} → {discarded} — stop a download, or delete a finished one that was not installed", always, download_cancel), + cmd!(query "denoise.models.test", "Test Denoise Model", [], None, "{id} → {ok, result} — load an installed model and check it gives sensible pictures; needs the denoise runtime", always, test), + cmd!(query "denoise.models.remove", "Remove Denoise Model", [], None, "{id} → {removed, model} — delete an installed model (its cached pictures stay until they are evicted)", always, remove), + cmd!(query "denoise.models.select", "Choose Denoise Model", [], None, "{id: installed model | null} → {model} — null switches denoise off", always, select), + cmd!(query "denoise.settings", "Denoise Settings", [], None, "{auto?: bool, cacheGb?: 1..10000, threads?: 1..64 | null, runOn?: auto | gpu | cpu} → the settings — `auto`: make the picture of the photos being looked at that have a Denoise amount; `cacheGb`: how much the cached pictures may take; `threads`: most tiles run at once; `runOn`: where the model runs — `auto` (the default) the graphics card where it is faster than the processor (both timed on this computer when the card is set up), `gpu` the card whenever it can run the model, `cpu` the processor. Setting `runOn` also lets the card be tried again after a set-up that failed or closed LightCraft", always, settings), + cmd!( + "denoise.toggle", + "AI Denoise", + [], + None, + "{id?: photoId, enabled?: bool} → {id, enabled, amount} — turn AI Denoise on (50% initially) or off for one raw photo, independent of selection and Auto Sync; needs an installed model to turn on", + always, + toggle + ), + cmd!(query "denoise.queue", "Denoise Photos", [], None, "{ids?: [photoId], scope?: selected | visible | withAmount, retry?: false} → {added, …status} — make the denoised pictures of raw photos that have none, in the background. `retry` tries photos that failed before", always, queue), + cmd!(query "denoise.cancel", "Cancel Denoise", [], None, "{id?, all?} → {cancelled} — stop the photo being made and/or forget the ones waiting (all of them when `id` is omitted)", always, cancel), + cmd!(query "denoise.status", "Denoise Status", [], None, "{id?} → {enabled, model, runtime, productsDir, queued, running: {photo, done, total, seconds} | null, ready, made, auto, runOn, device: {kind: gpu | cpu | pending | none, adapter?, reason?, cardMs?, cpuMs?, fellBack?}, cache: {files, bytes, limitBytes}, generation, photo?: {state: notApplicable | noModel | ready | queued | running | failed | idle, …}}", always, status), + cmd!(query "denoise.pump", "Denoise Pump", [], None, "{pace?: pause | light | normal | full} → {active, running, queued, ready, generation, downloading} — called every frame by the app: takes in finished work and keeps one photo going", always, pump), + cmd!(query "denoise.clear", "Clear Denoise Cache", [], None, "{} → {deleted, bytes} — delete every cached denoised picture of this library (they are made again when wanted)", always, clear), + ] +} diff --git a/crates/engine/src/cmd/mod.rs b/crates/engine/src/cmd/mod.rs index 9fc83116..9f1d255c 100644 --- a/crates/engine/src/cmd/mod.rs +++ b/crates/engine/src/cmd/mod.rs @@ -10,6 +10,7 @@ mod color; pub(crate) mod convert; mod cull; pub mod curves; +pub(crate) mod denoise; mod develop; mod edit; mod export; @@ -140,6 +141,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(missing::specs()); v.extend(metadata::specs()); v.extend(filters::specs()); + v.extend(denoise::specs()); v }) } diff --git a/crates/engine/src/config.rs b/crates/engine/src/config.rs new file mode 100644 index 00000000..96632f7a --- /dev/null +++ b/crates/engine/src/config.rs @@ -0,0 +1,23 @@ +//! Shared, opt-in per-user denoise model storage for native hosts. + +use crate::Session; +use std::path::PathBuf; + +pub fn default_denoise_models_dir() -> Option { + std::env::var_os("LIGHTCRAFT_DENOISE_MODELS") + .filter(|v| !v.is_empty()) + .map(PathBuf::from) + .or_else(|| crate::camera_profiles::config_dir().map(|d| d.join("denoise-models"))) +} + +impl Session { + pub fn with_default_denoise_models(mut self) -> Self { + self.set_denoise_models_dir(default_denoise_models_dir()); + self + } + + pub fn set_denoise_models_dir(&mut self, dir: Option) { + self.denoise.models_dir = dir; + self.denoise.touch(); + } +} diff --git a/crates/engine/src/denoise.rs b/crates/engine/src/denoise.rs new file mode 100644 index 00000000..37d79dcd --- /dev/null +++ b/crates/engine/src/denoise.rs @@ -0,0 +1,1629 @@ +//! AI denoise, the session's side (`docs/denoise.md`): which photos have a denoised picture, making the missing ones +//! in the background, and keeping the cache within its size. +//! +//! The denoised picture of a raw photo is *cached data*, never a file in the library: it is made from the untouched +//! original by the model the user chose, kept in `/denoise/.lcdn`, and thrown away whenever space is +//! needed (it is made again when it is next wanted). Its name is a hash of the model, the algorithm and the photo's +//! content, so a changed file or another model simply does not find it. The Denoise *amount* is an ordinary develop +//! setting (`enhance.denoise`): a render mixes the plain and the denoised picture by it ([`crate::media`]). +//! +//! What runs where: the session thread only decides (which photos, which model) and keeps an index of what is ready +//! ([`crate::media::DenoiseIndex`], memory only). One background thread at a time decodes a photo, runs the model on +//! its mosaic tile by tile and writes the product ([`make_product`]); an export that finds its photo's product missing +//! makes it itself, on its own thread, so it never comes out without the denoise it was asked for. +//! +//! Failure is a message, never a panic: a file that is not a Bayer raw, a model that does not load, a full disk, a +//! panic inside the runtime and a cancelled job all end as an [`Outcome`] the pump records. + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex, OnceLock, PoisonError}; +use std::time::{Duration, SystemTime}; + +use lightcraft_catalog::{Photo, PhotoId, Source}; +use lightcraft_denoise::bayer::Layout; +use lightcraft_denoise::manifest::DenoiserManifest; +use lightcraft_denoise::run::{Control, Params, TileRunner, denoise_bayer}; +use lightcraft_denoise::{Error as RunError, product}; +use lightcraft_develop::DevelopSettings; +use lightcraft_preview::Hasher128; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::media::{DenoiseSpec, MakeProduct, PairLoader}; +use crate::merge::ByteReader; +use crate::model_download::Downloads; +use crate::{Selection, Session}; + +/// Extension of a product file. +pub const EXT: &str = "lcdn"; +/// Where the products of a library are kept, inside the library's folder (safe to delete, like `thumbs/`). +pub const DIR: &str = "denoise"; +/// What the cache may take up unless the user says otherwise (GB). A product is about as large as the raw file's +/// decoded samples, compressed: ~100 MB for 24 megapixels. +pub const DEFAULT_CACHE_GB: u32 = 20; +/// Samples at or above this (white = 1) are clipped: the model leaves them as the camera recorded them. +const CLIP: f32 = 0.99; +/// Most photos waiting in the queue. +const MAX_QUEUE: usize = 5000; +/// Most selected photos the automatic queue looks at in one pump. +const AUTO_LOOK: usize = 64; +/// How long a look at the model settings is trusted. +const SETTINGS_TTL: Duration = Duration::from_secs(2); +/// Stack of the threads that run the model (the runtime recurses through its graph). +const STACK: usize = 16 << 20; + +// --------------------------------------------------------------------------------------------------------------------- +// the model + +/// Where the model runs (Settings ▸ AI Denoise ▸ Speed; `denoise.settings {runOn}`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum RunOn { + /// The graphics card where it is faster than the processor at the pace the work goes at (both are timed on this + /// computer when the card is set up), else the processor. + #[default] + Auto, + /// The graphics card whenever it can run the model, even where the processor would be faster. + Gpu, + /// Always the processor. + Cpu, +} + +impl RunOn { + /// `auto`, `gpu` or `cpu`. + pub fn parse(name: &str) -> Option { + match name { + "auto" => Some(RunOn::Auto), + "gpu" => Some(RunOn::Gpu), + "cpu" => Some(RunOn::Cpu), + _ => None, + } + } + + pub fn name(self) -> &'static str { + match self { + RunOn::Auto => "auto", + RunOn::Gpu => "gpu", + RunOn::Cpu => "cpu", + } + } +} + +/// A loaded denoise model: what runs tiles, and a self-check. +pub(crate) trait Model: Send + Sync { + /// What runs a photo's tiles, and how many to run at once, when `threads` may be used and the setting is `run_on` + /// (a graphics card is kept busy by a few). The first time the card may be used it is set up and checked here. + fn runner(&self, run_on: RunOn, threads: usize) -> (&dyn TileRunner, usize); + /// Check it does something sensible: the test's result, or why not. Unless `run_on` is the processor the graphics + /// card is set up and checked too, and the result says where the work will run. + fn self_test(&self, run_on: RunOn) -> Result; + /// Where the tiles run: `{kind: "gpu", adapter}`, `{kind: "cpu", reason?}` or `{kind: "pending"}` (the card is not + /// set up yet), with the card's and the processor's time for the check tile (`cardMs`, `cpuMs`) once both were + /// timed. Never starts anything. + fn device(&self, _run_on: RunOn) -> Value { + json!({"kind": "cpu"}) + } +} + +/// Loads the model at a path as its manifest describes it. +pub(crate) type Loader = Arc Result, String> + Send + Sync>; + +/// Tiles run at once on a graphics card: enough to keep it fed while others are packed and blended. +#[cfg(feature = "denoise")] +const GPU_PARALLEL: usize = 4; +/// How far the GPU's answer on the check tile may be from the CPU's (of the answer's largest value) before it is not used. +#[cfg(feature = "denoise")] +const GPU_AGREE: f32 = 1e-3; +/// Longest the card may take to set up (build its kernels, run the check tile, time it) before the processor does the +/// work instead. +#[cfg(feature = "denoise")] +const GPU_SETUP_LIMIT: Duration = Duration::from_secs(30); +/// Tiles the card may get wrong (an error, numbers that are not finite) before it stops being used for the model. +#[cfg(feature = "denoise")] +const GPU_MAX_FALLBACKS: usize = 3; +/// The file beside a model while the graphics card is set up for it (see [`set_up_card`]). +pub(crate) const GPU_SETUP_MARKER: &str = "gpu-setup.marker"; +/// Why the card is not tried when a set-up marker is found. +#[cfg(feature = "denoise")] +const GPU_CRASHED: &str = "LightCraft closed while it was setting up the graphics card for AI Denoise last time"; +/// Why the card is not used when its set-up ran out of time (the start of it). +#[cfg(feature = "denoise")] +const GPU_SLOW: &str = "setting up the graphics card took longer than"; +/// Why the card is not used while a set-up that ran out of time is still going. +#[cfg(feature = "denoise")] +const GPU_STUCK: &str = "the graphics card is still setting up from before (it took too long)"; + +/// A set-up that may work another time (a crash, the time limit), unlike one the card or the model cannot do. +#[cfg(feature = "denoise")] +fn worth_retrying(why: &str) -> bool { + why == GPU_CRASHED || why == GPU_STUCK || why.starts_with(GPU_SLOW) +} + +/// The model on our pure-Rust CPU runner, and on the graphics card when there is one that can run it: the card is set up for the +/// first photo that may use it, checked against the CPU's answer on a check tile and timed against it, and used while +/// it keeps working; a tile it gets wrong is run on the CPU, so a photo is always finished. +#[cfg(feature = "denoise")] +struct Network { + cpu: lightcraft_denoise::runtime::CpuRunner, + path: PathBuf, + tile: usize, + gpu: OnceLock>, + /// Tiles the card got wrong (an error, numbers that are not finite) and the CPU ran. + fallbacks: AtomicUsize, + /// Tiles at once of the photo started last (0: none yet), to say where the work runs. + last_threads: AtomicUsize, +} + +#[cfg(feature = "denoise")] +struct GpuSide { + runner: Box, + adapter: String, + /// The check tile on the card, the fastest of a few runs (ms). + card_ms: f64, + /// The check tile on the CPU, on one thread (ms). + cpu_ms: f64, +} + +/// Whether the card (`card_ms` a tile) is faster than the CPU running `threads` tiles at once, from the CPU's time for +/// one tile alone (`cpu_ms`). Tiles at once share the memory and the cores' second threads, so each one more counts as +/// half a tile's speed: on a 16-core laptop one tile alone took 1.4 s and sixteen at once finished one every 0.2 s. +#[cfg(feature = "denoise")] +fn card_is_faster(card_ms: f64, cpu_ms: f64, threads: usize) -> bool { + let at_once = 1.0 + 0.5 * threads.saturating_sub(1) as f64; + card_ms.is_finite() && card_ms <= cpu_ms / at_once +} + +/// Card set-ups of this process that have not finished, by marker (their markers are not a crash's). +#[cfg(feature = "denoise")] +static SETTING_UP: Mutex> = Mutex::new(Vec::new()); + +/// Removes a set-up's marker when the set-up ends, however it ends — but not when the process dies inside the driver. +#[cfg(feature = "denoise")] +struct SetUpEnded(PathBuf); + +#[cfg(feature = "denoise")] +impl Drop for SetUpEnded { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + SETTING_UP.lock().unwrap_or_else(PoisonError::into_inner).retain(|m| m != &self.0); + } +} + +/// Set the graphics card up with `card` on a thread of its own while `meanwhile` runs here, and give up on the card when +/// it takes longer than `limit` (a driver building kernels for minutes, or stuck). The `marker` file exists while `card` +/// runs: one found before it starts was left by a LightCraft that closed during a set-up (a driver crash takes the app +/// with it), so the card is not tried again until the user chooses where denoise runs (`denoise.settings {runOn}`). +#[cfg(feature = "denoise")] +fn set_up_card( + marker: &Path, + limit: Duration, + card: impl FnOnce() -> Result + Send + 'static, + meanwhile: impl FnOnce() -> R, +) -> Result<(T, R), String> { + let ended = { + let mut running = SETTING_UP.lock().unwrap_or_else(PoisonError::into_inner); + if running.iter().any(|m| m == marker) { + return Err(GPU_STUCK.into()); + } + if marker.exists() { + return Err(GPU_CRASHED.into()); + } + let secs = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).map_or(0, |d| d.as_secs()); + // a marker that cannot be written only loses the crash guard + let _ = std::fs::write(marker, format!("setting up the graphics card for AI Denoise (unix time {secs})\n")); + running.push(marker.to_path_buf()); + SetUpEnded(marker.to_path_buf()) + }; + let started = std::time::Instant::now(); + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::Builder::new() + .name("denoise-gpu-setup".into()) + .stack_size(STACK) + .spawn(move || { + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(card)).unwrap_or_else(|_| Err("the GPU runner gave up".into())); + drop(ended); + let _ = tx.send(r); + }) + .map_err(|e| format!("could not start setting up the graphics card: {e}"))?; + let here = meanwhile(); + match rx.recv_timeout(limit.saturating_sub(started.elapsed())) { + Ok(r) => r.map(|t| (t, here)), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => Err(format!("{GPU_SLOW} {} s, so it is not used", limit.as_secs_f64())), + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => Err("the GPU runner gave up".into()), + } +} + +/// The card's time for one tile once it is warm: the fastest of up to three runs, stopping after about two seconds (a +/// slow card is slow enough to tell at once). +#[cfg(feature = "denoise")] +fn card_tile_ms(runner: &dyn TileRunner, input: &[f32]) -> f64 { + if runner.run(input).is_err() { + return f64::INFINITY; + } + let began = std::time::Instant::now(); + let mut best = f64::INFINITY; + for _ in 0..3 { + let one = std::time::Instant::now(); + if runner.run(input).is_err() { + break; + } + best = best.min(one.elapsed().as_secs_f64() * 1000.0); + if began.elapsed() > Duration::from_secs(2) { + break; + } + } + best +} + +/// Set the card up for the model at `path`: kernels built, the check tile run on both and compared, both timed. +#[cfg(feature = "denoise")] +fn make_gpu(cpu: &lightcraft_denoise::runtime::CpuRunner, path: &Path, tile: usize) -> Result { + let input = Arc::new(lightcraft_denoise::runtime::check_tile(tile)); + let (on_card, net) = (input.clone(), cpu.net().clone()); + let card = move || -> Result<_, String> { + let runner = lightcraft_gpu::nn::runner(&net, tile)?; + let got = runner.run(&on_card).map_err(|e| format!("the GPU could not run the check tile: {e}"))?; + let card_ms = card_tile_ms(&runner, &on_card); + Ok((runner, got, card_ms)) + }; + let on_cpu = || { + let started = std::time::Instant::now(); + (cpu.run(&input), started.elapsed().as_secs_f64() * 1000.0) + }; + let ((runner, got, card_ms), (want, cpu_ms)) = set_up_card(&path.with_file_name(GPU_SETUP_MARKER), GPU_SETUP_LIMIT, card, on_cpu)?; + let want = want.map_err(|e| format!("the CPU could not check the GPU: {e}"))?; + let scale = want.iter().fold(0f32, |m, v| m.max(v.abs())).max(1e-12); + let worst = want.iter().zip(&got).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale; + if got.len() != want.len() || !got.iter().all(|v| v.is_finite()) || worst > GPU_AGREE { + return Err(format!("its answer on the check tile is not the CPU's (off by {worst:.1e} of the largest value): not used")); + } + Ok(GpuSide { adapter: runner.adapter(), runner: Box::new(runner), card_ms, cpu_ms }) +} + +#[cfg(feature = "denoise")] +impl Network { + /// The card when it can be used now, else why not. `start` sets it up when that has not been tried yet; without it + /// nothing is started. + fn card(&self, start: bool) -> Result<&GpuSide, String> { + let side = if start { + Some(self.gpu.get_or_init(|| { + let made = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| make_gpu(&self.cpu, &self.path, self.tile))) + .unwrap_or_else(|_| Err("the GPU runner gave up".into())); + if let Err(why) = &made { + log::info!("denoise: running on the CPU ({why})"); + } + made + })) + } else { + self.gpu.get() + }; + let g = match side { + None => return Err("the graphics card is not set up yet".into()), + Some(Err(why)) => return Err(why.clone()), + Some(Ok(g)) => g, + }; + if let Some(why) = lightcraft_gpu::nn::broken_reason() { + return Err(format!("the graphics card stopped working ({why})")); + } + if self.fallbacks.load(Ordering::Relaxed) >= GPU_MAX_FALLBACKS { + return Err(format!("{} got {GPU_MAX_FALLBACKS} tiles wrong", g.adapter)); + } + Ok(g) + } + + /// The card, when a photo made `threads` tiles at a time under `run_on` runs on it; else why the CPU does. + fn choose(&self, run_on: RunOn, threads: usize, start: bool) -> Result<&GpuSide, String> { + if run_on == RunOn::Cpu { + return Err("the processor is chosen in Settings".into()); + } + let g = self.card(start)?; + if run_on == RunOn::Auto && !card_is_faster(g.card_ms, g.cpu_ms, threads) { + return Err(format!("it is faster than {} on this computer", g.adapter)); + } + Ok(g) + } + + fn fell_back(&self, why: &str) { + let n = self.fallbacks.fetch_add(1, Ordering::Relaxed) + 1; + if n == 1 { + log::warn!("denoise: a tile came back wrong from the GPU ({why}); the CPU runs it"); + } else if n == GPU_MAX_FALLBACKS { + log::warn!("denoise: the GPU got {n} tiles wrong; the CPU does the rest"); + } + } +} + +/// The card, with the CPU running any tile it gets wrong. Handed out only for photos that run on the card. +#[cfg(feature = "denoise")] +impl TileRunner for Network { + fn run(&self, input: &[f32]) -> Result, RunError> { + if let Ok(g) = self.card(false) { + match g.runner.run(input) { + Ok(out) if out.iter().all(|v| v.is_finite()) => return Ok(out), + Ok(_) => self.fell_back("numbers that are not finite"), + Err(e) => self.fell_back(&e.to_string()), + } + } + self.cpu.run(input) + } +} + +#[cfg(feature = "denoise")] +impl Model for Network { + fn runner(&self, run_on: RunOn, threads: usize) -> (&dyn TileRunner, usize) { + let threads = threads.max(1); + self.last_threads.store(threads, Ordering::Relaxed); + match self.choose(run_on, threads, true) { + Ok(_) => (self, threads.min(GPU_PARALLEL)), + Err(_) => (&self.cpu, threads), + } + } + + fn self_test(&self, run_on: RunOn) -> Result { + let t = self.cpu.self_test(); + if !t.ok { + let failed: Vec<&str> = t.checks.iter().filter(|(_, ok)| !*ok).map(|(c, _)| c.as_str()).collect(); + return Err(format!("the model failed its self-test: it does not {}", failed.join(", and does not "))); + } + let mut v = serde_json::to_value(&t).map_err(|e| e.to_string())?; + // the card is set up and checked here, so the test says where the work will run + if run_on != RunOn::Cpu { + let _ = self.card(true); + } + if let Some(o) = v.as_object_mut() { + o.insert("device".into(), self.device(run_on)); + } + Ok(v) + } + + fn device(&self, run_on: RunOn) -> Value { + if run_on != RunOn::Cpu && self.gpu.get().is_none() { + return json!({"kind": "pending"}); + } + let threads = match self.last_threads.load(Ordering::Relaxed) { + 0 => Pace::Full.parallel(std::thread::available_parallelism().map_or(1, |n| n.get()), None), + n => n, + }; + let mut v = match self.choose(run_on, threads, false) { + Ok(g) => json!({"kind": "gpu", "adapter": g.adapter}), + // a set-up that crashed or ran out of time can be tried again (`denoise.settings {runOn}`) + Err(why) => json!({"kind": "cpu", "retry": worth_retrying(&why), "reason": why}), + }; + if let (Some(Ok(g)), Some(o)) = (self.gpu.get(), v.as_object_mut()) { + // measured on this computer, on the check tile: the card's best run, and one tile alone on the CPU + o.insert("cardMs".into(), json!(g.card_ms)); + o.insert("cpuMs".into(), json!(g.cpu_ms)); + o.insert("fellBack".into(), json!(self.fallbacks.load(Ordering::Relaxed))); + } + v + } +} + +/// The loader of this build: our pure-Rust executor with the `denoise` feature, else one that says why nothing can run. +pub(crate) fn default_loader() -> Loader { + #[cfg(feature = "denoise")] + { + Arc::new(|path, manifest| { + let cpu = lightcraft_denoise::runtime::CpuRunner::load(path, manifest).map_err(|e| e.to_string())?; + Ok(Arc::new(Network { + cpu, + path: path.to_path_buf(), + tile: manifest.tile as usize, + gpu: OnceLock::new(), + fallbacks: AtomicUsize::new(0), + last_threads: AtomicUsize::new(0), + }) as Arc) + }) + } + #[cfg(not(feature = "denoise"))] + { + Arc::new(|_, _| Err("this build cannot run denoise models".to_string())) + } +} + +/// A model that is loaded on first use and kept, shared by the background job and the exports. +pub(crate) type ModelSlot = Arc>>>; + +// --------------------------------------------------------------------------------------------------------------------- +// settings and installed models + +/// `settings.json` in the models folder. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub(crate) struct Settings { + /// The installed model in use; `None`: denoise is off. + pub model: Option, + /// Photos with a Denoise amount above 0 that are being looked at (active, selected) get their picture made without + /// being asked (`None`: yes). + pub auto: Option, + /// Cache size limit (GB; `None`: [`DEFAULT_CACHE_GB`]). + pub cache_gb: Option, + /// Most tiles run at once (`None`: by how hard the pump says to work). + pub threads: Option, + /// Where the model runs: `auto`, `gpu` or `cpu` (`None` or anything else: `auto`; see [`RunOn`]). + pub run_on: Option, + /// What came before `runOn`: `false` was the processor only. + pub gpu: Option, +} + +impl Settings { + pub fn auto(&self) -> bool { + self.auto.unwrap_or(true) + } + + pub fn run_on(&self) -> RunOn { + match self.run_on.as_deref().and_then(RunOn::parse) { + Some(r) => r, + None if self.gpu == Some(false) => RunOn::Cpu, + None => RunOn::Auto, + } + } + + pub fn cache_bytes(&self) -> u64 { + u64::from(self.cache_gb.unwrap_or(DEFAULT_CACHE_GB).clamp(1, 10_000)) << 30 + } +} + +pub(crate) fn read_capped(path: &Path, max: usize) -> Option> { + use std::io::Read; + let mut buf = Vec::new(); + std::fs::File::open(path).ok()?.take(max as u64 + 1).read_to_end(&mut buf).ok()?; + (buf.len() <= max).then_some(buf) +} + +pub(crate) fn read_settings(dir: &Path) -> Settings { + read_capped(&dir.join("settings.json"), 4096).and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default() +} + +/// Write a file next to its final name and rename it into place. +pub(crate) fn write_atomic(path: &Path, bytes: &[u8]) -> Result<(), String> { + let part = path.with_extension("part"); + std::fs::write(&part, bytes).map_err(|e| format!("could not write {}: {e}", path.display()))?; + std::fs::rename(&part, path).map_err(|e| { + let _ = std::fs::remove_file(&part); + format!("could not write {}: {e}", path.display()) + }) +} + +pub(crate) fn write_settings(dir: &Path, st: &Settings) -> Result<(), String> { + std::fs::create_dir_all(dir).map_err(|e| format!("could not create the denoise models folder: {e}"))?; + write_atomic(&dir.join("settings.json"), &serde_json::to_vec_pretty(st).map_err(|e| e.to_string())?) +} + +/// An installed model folder: `/model.onnx`, `denoise-model.json`, `installed.json`. +pub(crate) struct Installed { + pub manifest: DenoiserManifest, + pub onnx: PathBuf, + /// What was recorded when it was installed (terms accepted, hash of the file, self-test). + pub accepted: Value, +} + +impl Installed { + /// Names the model file the cache is made with: a hash of the file as installed, so another file with the same id + /// never finds this one's products. + pub fn fingerprint(&self) -> String { + let sha = self.accepted.get("sha256").and_then(Value::as_str).or(self.manifest.sha256.as_deref()).unwrap_or(""); + let len = std::fs::metadata(&self.onnx).map(|m| m.len()).unwrap_or(0); + format!("{}@{}:{len}", self.manifest.id, sha) + } +} + +/// The models in the folder. A folder that is not a valid model (wrong name, no model file, a manifest that does not +/// validate or does not match its folder) is ignored, never trusted. +pub(crate) fn installed_models(dir: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() }; + let mut out = Vec::new(); + for e in entries.flatten() { + let p = e.path(); + let Some(name) = p.file_name().and_then(|n| n.to_str()).map(str::to_string) else { continue }; + let onnx = p.join("model.onnx"); + if !lightcraft_denoise::licence::valid_id(&name) || !p.is_dir() || !onnx.is_file() { + continue; + } + let Some(m) = read_capped(&p.join("denoise-model.json"), lightcraft_denoise::manifest::MAX_MANIFEST_BYTES) + .and_then(|b| lightcraft_denoise::manifest::parse(&b).ok()) + else { + continue; + }; + if m.id != name { + continue; + } + let accepted = read_capped(&p.join("installed.json"), 8192).and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or(Value::Null); + out.push(Installed { manifest: m, onnx, accepted }); + } + out.sort_by(|a, b| a.manifest.id.cmp(&b.manifest.id)); + out +} + +/// Forget that setting the graphics card up for a model closed LightCraft before (the markers [`set_up_card`] leaves): +/// the user chose where denoise runs, so the card may be tried again. +pub(crate) fn forget_gpu_set_ups(dir: &Path) { + for i in installed_models(dir) { + let _ = std::fs::remove_file(i.onnx.with_file_name(GPU_SETUP_MARKER)); + } +} + +/// The model in use. +pub(crate) struct Active { + pub id: String, + pub manifest: DenoiserManifest, + pub onnx: PathBuf, + pub fingerprint: String, + pub model: ModelSlot, +} + +// --------------------------------------------------------------------------------------------------------------------- +// products + +/// The photos whose files can be denoised at all: raw files developed from their sensor data and read from disk. +pub(crate) fn eligible(p: &Photo) -> bool { + p.develops_raw() && matches!(p.source, Source::File { .. }) && !p.deleted +} + +/// A product's name: what it was made from and with. +pub(crate) fn product_key(fingerprint: &str, p: &Photo) -> String { + let mut h = Hasher128::new(); + h.str("lcdn").u64(u64::from(product::ALGORITHM)).str(fingerprint).str(&crate::media::content_key(p)); + h.finish().to_string() +} + +pub(crate) fn product_path(dir: &Path, key: &str) -> PathBuf { + dir.join(format!("{key}.{EXT}")) +} + +/// The `.lcdn` files in `dir`: path, size, last change. +fn products_in(dir: &Path) -> Vec<(PathBuf, u64, SystemTime)> { + let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() }; + entries + .flatten() + .filter(|e| e.path().extension().is_some_and(|x| x == EXT)) + .filter_map(|e| { + let m = e.metadata().ok()?; + m.is_file().then(|| (e.path(), m.len(), m.modified().unwrap_or(SystemTime::UNIX_EPOCH))) + }) + .collect() +} + +/// Make sure only one thread writes a product at a time: a second one waits, then finds it made. +fn making() -> &'static (Mutex>, Condvar) { + static MAKING: OnceLock<(Mutex>, Condvar)> = OnceLock::new(); + MAKING.get_or_init(|| (Mutex::new(HashSet::new()), Condvar::new())) +} + +struct Slot(PathBuf); + +impl Slot { + fn enter(path: &Path) -> Slot { + let (set, cv) = making(); + let mut g = set.lock().unwrap_or_else(PoisonError::into_inner); + while g.contains(path) { + g = cv.wait(g).unwrap_or_else(PoisonError::into_inner); + } + g.insert(path.to_path_buf()); + Slot(path.to_path_buf()) + } +} + +impl Drop for Slot { + fn drop(&mut self) { + let (set, cv) = making(); + set.lock().unwrap_or_else(PoisonError::into_inner).remove(&self.0); + cv.notify_all(); + } +} + +// --------------------------------------------------------------------------------------------------------------------- +// making a product + +/// Everything a thread needs to make one product, detached from the session. +#[derive(Clone)] +pub(crate) struct JobSpec { + /// The original file. + pub path: String, + pub read: Option, + pub product: PathBuf, + pub key: String, + pub manifest: DenoiserManifest, + pub onnx: PathBuf, + pub model: ModelSlot, + pub loader: Loader, + /// Tiles run at once. + pub parallel: usize, + /// Where the model runs (the setting). + pub run_on: RunOn, +} + +/// Why a product was not made. +#[derive(Debug, PartialEq)] +pub(crate) enum MakeError { + /// Denoise does not apply to this file (not a Bayer raw): nothing is wrong. + Unsupported(String), + Failed(String), + Cancelled, +} + +impl From for MakeError { + fn from(e: RunError) -> Self { + match e { + RunError::Cancelled => MakeError::Cancelled, + other => MakeError::Failed(other.to_string()), + } + } +} + +/// Progress and cancel of a running job. +#[derive(Default)] +pub(crate) struct Progress { + done: AtomicUsize, + total: AtomicUsize, + cancel: AtomicBool, +} + +impl Progress { + /// Ask the job to stop at the next tile. + pub(crate) fn cancel_now(&self) { + self.cancel.store(true, Ordering::Relaxed); + } +} + +/// The loaded model, loading it first when it is not (kept for the next photo). +fn loaded(spec: &JobSpec) -> Result, MakeError> { + if let Some(m) = spec.model.lock().unwrap_or_else(PoisonError::into_inner).clone() { + return Ok(m); + } + let m = (spec.loader)(&spec.onnx, &spec.manifest).map_err(|e| MakeError::Failed(format!("the denoise model could not be loaded: {e}")))?; + *spec.model.lock().unwrap_or_else(PoisonError::into_inner) = Some(m.clone()); + Ok(m) +} + +/// The cell `[(0,0), (1,0), (0,1), (1,1)]` colours of a Bayer mosaic and so its layout. +fn layout_of(cfa: &lightcraft_raw::Cfa) -> Option { + Layout::from_cell([cfa.color_at(0, 0), cfa.color_at(1, 0), cfa.color_at(0, 1), cfa.color_at(1, 1)]) +} + +/// Make the product of `spec` unless it is there: decode the photo, run the model over its mosaic, write the result. +/// `Ok(true)` when it was made now, `Ok(false)` when it already existed. +pub(crate) fn make_product(spec: &JobSpec, progress: Option<&Progress>) -> Result { + let _slot = Slot::enter(&spec.product); + if product::is_current(&spec.product, &spec.key) { + return Ok(false); + } + let began = std::time::Instant::now(); + let cancelled = || progress.is_some_and(|p| p.cancel.load(Ordering::Relaxed)); + if cancelled() { + return Err(MakeError::Cancelled); + } + let bytes = match &spec.read { + Some(r) => r(&spec.path).map_err(MakeError::Failed)?, + None => std::fs::read(&spec.path).map_err(|e| MakeError::Failed(format!("{}: {e}", spec.path)))?, + }; + // the decode and the normalised mosaic wait their turn at the process-wide memory gate, as every decode does + let (mosaic, layout) = { + let _held = crate::memory::work_gate().acquire(bytes.len().saturating_mul(4)); + if lightcraft_raw::probe(&bytes).is_none() { + return Err(MakeError::Unsupported("not a raw file".into())); + } + let raw = match lightcraft_raw::decode(&bytes) { + Ok(r) => r, + Err(lightcraft_raw::RawError::Unsupported(why)) => return Err(MakeError::Unsupported(why)), + Err(e) => return Err(MakeError::Failed(e.to_string())), + }; + drop(bytes); + if raw.cpp != 1 || !raw.cfa.as_ref().is_some_and(lightcraft_raw::Cfa::is_bayer) { + return Err(MakeError::Unsupported( + "only Bayer raw files can be denoised (this one has another sensor layout or is already demosaiced)".into(), + )); + } + let n = raw.normalized().map_err(|e| MakeError::Failed(e.to_string()))?; + drop(raw); + let layout = + n.cfa.as_ref().and_then(layout_of).ok_or_else(|| MakeError::Unsupported("the colour filter layout is not a Bayer pattern".into()))?; + (n, layout) + }; + if cancelled() { + return Err(MakeError::Cancelled); + } + let decoded = began.elapsed(); + let model = loaded(spec)?; + let (runner, parallel) = model.runner(spec.run_on, spec.parallel.max(1)); + let params = Params { + tile: spec.manifest.tile as usize, + overlap: spec.manifest.overlap as usize, + gain: spec.manifest.gain, + clip: Some(CLIP), + parallel: parallel.max(1), + }; + let report = |done: usize, total: usize| { + if let Some(p) = progress { + p.total.store(total, Ordering::Relaxed); + p.done.store(done, Ordering::Relaxed); + } + }; + let ctl = Control { cancel: progress.map(|p| &p.cancel), progress: Some(&report) }; + // the tiles run on threads of their own, so a busy window's renders keep theirs + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(params.parallel) + .stack_size(STACK) + .thread_name(|i| format!("denoise-{i}")) + .build() + .map_err(|e| MakeError::Failed(format!("could not start the worker threads: {e}")))?; + let rgb = pool.install(|| denoise_bayer(&mosaic.data, mosaic.width, mosaic.height, layout, runner, ¶ms, &ctl))?; + drop(mosaic); + if cancelled() { + return Err(MakeError::Cancelled); + } + let ran = began.elapsed(); + product::write(&spec.product, &rgb, &spec.key).map_err(|e| MakeError::Failed(format!("could not save the denoised picture: {e}")))?; + if std::env::var_os("LIGHTCRAFT_PROFILE").is_some() { + eprintln!( + "[denoise] read + decode {:.0} ms, model (set-up, tiles, blend) {:.0} ms, write {:.0} ms", + decoded.as_secs_f64() * 1e3, + (ran - decoded).as_secs_f64() * 1e3, + (began.elapsed() - ran).as_secs_f64() * 1e3 + ); + } + Ok(true) +} + +// --------------------------------------------------------------------------------------------------------------------- +// the session's state + +/// How hard the background work may go (the app picks it from what the user is doing, like the face scan). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Pace { + /// Start nothing new. + Pause, + /// Two tiles at once. + Light, + /// Half the machine. + Normal, + /// Most of the machine: the user is watching the progress. + Full, +} + +impl Pace { + /// `pause`, `light`, `normal` or `full`; anything else is `normal`. + pub fn parse(name: Option<&str>) -> Pace { + match name { + Some("pause") => Pace::Pause, + Some("light") => Pace::Light, + Some("full") => Pace::Full, + _ => Pace::Normal, + } + } + + /// Tiles to run at once on a machine with `cores` threads (`cap`: the user's own limit). + pub fn parallel(self, cores: usize, cap: Option) -> usize { + let n = match self { + Pace::Pause | Pace::Light => 2, + Pace::Normal => cores / 2, + Pace::Full => cores * 4 / 5, + } + .clamp(1, 16); + cap.map_or(n, |c| n.min(c.max(1) as usize)) + } +} + +/// How a photo stands, for the interface. +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(tag = "state", rename_all = "camelCase")] +pub enum PhotoState { + /// Not a photo denoise applies to (not a raw, offline, a demo). + NotApplicable, + /// No model is chosen: nothing can be made. + NoModel, + /// Its picture is made: the Amount slider works. + Ready, + Queued { + /// Photos ahead of it (0 = next). + ahead: usize, + }, + Running { + done: usize, + total: usize, + }, + /// Making it failed (`why`), or denoise does not apply to this file (`unsupported`). + Failed { + why: String, + unsupported: bool, + }, + /// Nothing has asked for it yet. + Idle, +} + +#[derive(Clone)] +struct Failure { + key: String, + why: String, + unsupported: bool, +} + +enum Outcome { + Done, + Failed(String, bool), + Cancelled, +} + +struct Running { + photo: PhotoId, + key: String, + product: PathBuf, + progress: Arc, + outcome: Arc>>, + started: web_time::Instant, +} + +pub(crate) struct State { + /// Where the models are kept (`None` where there is no file system). + pub models_dir: Option, + /// Models being downloaded at the user's request. + pub downloads: Downloads, + /// How a model is loaded (tests put another in). + pub loader: Loader, + pub(crate) active: Option, + active_seen: Option, + /// `settings.json` as last read (with the model choice). + pub(crate) settings: Settings, + /// The index must be made again (the library, the model or the cache changed). + dirty: bool, + /// The catalog revision and time of the last index, to look again when photos changed. + reindexed: Option<(u64, web_time::Instant)>, + queue: VecDeque, + running: Option, + failed: HashMap, + /// Counts every change of what is ready, so the interface knows to draw again. + pub generation: u64, + /// Photos made since the session started. + made: u64, +} + +impl Default for State { + fn default() -> Self { + State { + models_dir: None, + downloads: Downloads::default(), + loader: default_loader(), + active: None, + active_seen: None, + settings: Settings::default(), + dirty: true, + reindexed: None, + queue: VecDeque::new(), + running: None, + failed: HashMap::new(), + generation: 0, + made: 0, + } + } +} + +impl Drop for State { + /// Quitting stops the job instead of leaving it to finish a photo nobody will see. + fn drop(&mut self) { + if let Some(r) = &self.running { + r.progress.cancel_now(); + } + } +} + +impl State { + /// The library was replaced: nothing carries over (photo ids are per library). + pub fn library_changed(&mut self) { + if let Some(r) = self.running.take() { + r.progress.cancel_now(); + } + self.queue.clear(); + self.failed.clear(); + self.dirty = true; + self.generation += 1; + } + + /// The model choice or the cache changed on disk: look again at the next pump. + pub fn touch(&mut self) { + self.active_seen = None; + self.dirty = true; + } + + pub fn queued(&self) -> usize { + self.queue.len() + } +} + +impl Session { + /// Where the active model's tiles run (see [`Model::device`]); `{kind: "none"}` while no model is loaded. + pub(crate) fn denoise_device(&self) -> Value { + let loaded = self.denoise.active.as_ref().and_then(|a| a.model.lock().unwrap_or_else(PoisonError::into_inner).clone()); + match loaded { + Some(m) => m.device(self.denoise.settings.run_on()), + None => json!({"kind": "none"}), + } + } + + /// The folder the products of the open library are kept in; `None` without a library on disk. + pub(crate) fn denoise_products_dir(&self) -> Option { + self.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.join(DIR)) + } + + /// Read which model is in use (at most every couple of seconds): the manifest and file of the model the settings + /// name, if it is installed. + pub(crate) fn denoise_refresh_active(&mut self, force: bool) { + let fresh = self.denoise.active_seen.is_some_and(|t| t.elapsed() < SETTINGS_TTL); + if fresh && !force { + return; + } + self.denoise.active_seen = Some(web_time::Instant::now()); + self.denoise.settings = self.denoise.models_dir.as_deref().map(read_settings).unwrap_or_default(); + let chosen = self.denoise.models_dir.as_deref().and_then(|dir| { + let id = self.denoise.settings.model.clone()?; + installed_models(dir).into_iter().find(|i| i.manifest.id == id) + }); + let found = chosen.map(|i| (i.fingerprint(), i)); + let same = match (&self.denoise.active, &found) { + (None, None) => true, + (Some(a), Some((fp, _))) => a.fingerprint == *fp, + _ => false, + }; + if same { + return; + } + // another model: what was made with the old one is not this one's, and what failed may work now + self.denoise.active = found.map(|(fingerprint, i)| Active { + id: i.manifest.id.clone(), + manifest: i.manifest, + onnx: i.onnx, + fingerprint, + model: Arc::new(Mutex::new(None)), + }); + if let Some(r) = self.denoise.running.take() { + r.progress.cancel_now(); + } + self.denoise.queue.clear(); + self.denoise.failed.clear(); + self.denoise.dirty = true; + } + + fn denoise_spec(&self, p: &Photo) -> Option<(DenoiseSpec, String)> { + let (dir, active) = (self.denoise_products_dir()?, self.denoise.active.as_ref()?); + let key = product_key(&active.fingerprint, p); + Some((DenoiseSpec { product: product_path(&dir, &key), key: key.clone(), make: None }, key)) + } + + /// Make the index of ready photos from the products on disk (a directory listing and a hash per photo: no file is + /// opened). Pictures that appeared or vanished drop the decoded sources they affect. + pub(crate) fn denoise_reindex(&mut self) { + self.denoise.dirty = false; + self.denoise.reindexed = Some((self.catalog.revision, web_time::Instant::now())); + let mut now: HashMap = HashMap::new(); + if let (Some(dir), Some(active)) = (self.denoise_products_dir(), self.denoise.active.as_ref()) { + let names: HashSet = + products_in(&dir).into_iter().filter_map(|(p, _, _)| p.file_stem().and_then(|s| s.to_str()).map(str::to_string)).collect(); + if !names.is_empty() { + for p in self.catalog.photos().filter(|p| eligible(p)) { + let key = product_key(&active.fingerprint, p); + if names.contains(&key) { + now.insert(p.id, DenoiseSpec { product: product_path(&dir, &key), key, make: None }); + } + } + } + } + let mut changed = false; + for id in self.media.denoise.ids() { + if !now.contains_key(&id) { + self.media.denoise.remove(id); + self.media.forget(id); + changed = true; + } + } + for (id, spec) in now { + if self.media.denoise.spec(id).is_none_or(|old| old.key != spec.key) { + self.media.denoise.set(id, spec); + self.media.forget(id); + changed = true; + } + } + if changed { + self.denoise.generation += 1; + } + } + + /// A photo whose file changed (its content key is not the one its index entry was made for) has no picture. + fn denoise_check_photo(&mut self, id: PhotoId) { + let Some(p) = self.catalog.photo(id).cloned() else { return }; + let Some((spec, _)) = self.denoise_spec(&p) else { return }; + match self.media.denoise.spec(id) { + Some(have) if have.key != spec.key => { + self.media.denoise.remove(id); + self.media.forget(id); + self.denoise.generation += 1; + } + None if spec.product.is_file() && product::is_current(&spec.product, &spec.key) => { + // made by an export, or by another session + self.media.denoise.set(id, spec); + self.media.forget(id); + self.denoise.generation += 1; + } + _ => {} + } + } + + /// How photo `id` stands. + pub fn denoise_photo_state(&self, id: PhotoId) -> PhotoState { + let Some(p) = self.catalog.photo(id) else { return PhotoState::NotApplicable }; + if !eligible(p) { + return PhotoState::NotApplicable; + } + if self.denoise.active.is_none() { + // the model folder has not been looked at yet (just started): not "no model" until it has been + return if self.denoise.active_seen.is_none() { PhotoState::Idle } else { PhotoState::NoModel }; + } + if self.denoise_products_dir().is_none() { + return PhotoState::Failed { + why: "AI Denoise keeps its pictures in the library folder, and this session has none.".into(), + unsupported: true, + }; + } + if self.media.denoise.spec(id).is_some() { + return PhotoState::Ready; + } + if let Some(r) = self.denoise.running.as_ref().filter(|r| r.photo == id) { + return PhotoState::Running { done: r.progress.done.load(Ordering::Relaxed), total: r.progress.total.load(Ordering::Relaxed) }; + } + if let Some(ahead) = self.denoise.queue.iter().position(|q| *q == id) { + return PhotoState::Queued { ahead: ahead + usize::from(self.denoise.running.is_some()) }; + } + let key = self.denoise_spec(p).map(|s| s.1); + match self.denoise.failed.get(&id) { + Some(f) if Some(&f.key) == key.as_ref() => PhotoState::Failed { why: f.why.clone(), unsupported: f.unsupported }, + _ => PhotoState::Idle, + } + } + + /// Ask for photos' pictures to be made (the ones that are not made, queued or known to be impossible). With `retry` + /// a photo that failed before is tried again. A photo that is not raw is skipped. Returns how many were queued. + pub(crate) fn denoise_enqueue(&mut self, ids: &[PhotoId], front: bool, retry: bool) -> usize { + self.denoise_refresh_active(false); + if self.denoise.active.is_none() || self.denoise_products_dir().is_none() { + return 0; + } + let mut added = 0; + for id in ids { + let Some(p) = self.catalog.photo(*id).cloned() else { continue }; + if !eligible(&p) || self.media.denoise.spec(*id).is_some() || self.denoise.queue.contains(id) { + continue; + } + if self.denoise.running.as_ref().is_some_and(|r| r.photo == *id) || self.denoise.queue.len() >= MAX_QUEUE { + continue; + } + let Some((_, key)) = self.denoise_spec(&p) else { continue }; + if retry { + self.denoise.failed.remove(id); + } else if self.denoise.failed.get(id).is_some_and(|f| f.key == key) { + continue; + } + if front { + self.denoise.queue.push_front(*id); + } else { + self.denoise.queue.push_back(*id); + } + added += 1; + } + added + } + + /// Stop the job that is running, and with `all` forget everything that waits. + pub(crate) fn denoise_cancel(&mut self, photo: Option, all: bool) -> usize { + let mut n = 0; + if let Some(r) = &self.denoise.running + && photo.is_none_or(|p| p == r.photo) + { + r.progress.cancel_now(); + n += 1; + } + let before = self.denoise.queue.len(); + match photo { + Some(p) => self.denoise.queue.retain(|q| *q != p), + None if all => self.denoise.queue.clear(), + None => {} + } + n + before - self.denoise.queue.len() + } + + /// The photos the automatic queue looks at: the one open and the ones selected, those with a Denoise amount. + fn denoise_wanted(&self) -> Vec { + let Selection { active, ids, .. } = &self.selection; + active + .iter() + .chain(ids.iter().take(AUTO_LOOK)) + .copied() + .filter(|id| self.catalog.photo(*id).is_some_and(|p| p.develop.denoise_amount() > 0.0)) + .collect() + } + + fn denoise_job_spec(&self, p: &Photo, parallel: usize) -> Option { + let Source::File { path } = &p.source else { return None }; + let (spec, _) = self.denoise_spec(p)?; + let active = self.denoise.active.as_ref()?; + Some(JobSpec { + path: path.clone(), + read: self.media.file_bytes.clone(), + product: spec.product, + key: spec.key, + manifest: active.manifest.clone(), + onnx: active.onnx.clone(), + model: active.model.clone(), + loader: self.denoise.loader.clone(), + parallel, + run_on: self.denoise.settings.run_on(), + }) + } + + /// Start the next queued photo on its own thread. + fn denoise_start_next(&mut self, parallel: usize) { + while let Some(id) = self.denoise.queue.pop_front() { + let Some(p) = self.catalog.photo(id).cloned() else { continue }; + if !eligible(&p) || self.media.denoise.spec(id).is_some() { + continue; + } + let Some(spec) = self.denoise_job_spec(&p, parallel) else { continue }; + // another session or an export may have made it since it was queued + if product::is_current(&spec.product, &spec.key) { + self.denoise_check_photo(id); + continue; + } + let progress = Arc::new(Progress::default()); + let outcome: Arc>> = Arc::new(Mutex::new(None)); + let (key, product) = (spec.key.clone(), spec.product.clone()); + let (worker_progress, worker_outcome) = (progress.clone(), outcome.clone()); + let started = std::thread::Builder::new().name("denoise".into()).stack_size(STACK).spawn(move || { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| make_product(&spec, Some(&worker_progress)))); + let o = match result { + Ok(Ok(_)) => Outcome::Done, + Ok(Err(MakeError::Cancelled)) => Outcome::Cancelled, + Ok(Err(MakeError::Unsupported(why))) => Outcome::Failed(why, true), + Ok(Err(MakeError::Failed(why))) => Outcome::Failed(why, false), + Err(_) => Outcome::Failed("the denoiser stopped unexpectedly".into(), false), + }; + *worker_outcome.lock().unwrap_or_else(PoisonError::into_inner) = Some(o); + }); + match started { + Ok(_) => { + self.denoise.running = Some(Running { photo: id, key, product, progress, outcome, started: web_time::Instant::now() }); + } + Err(e) => { + self.denoise.failed.insert(id, Failure { key, why: format!("could not start the denoiser: {e}"), unsupported: false }); + continue; + } + } + return; + } + } + + /// Take in the running job's result. + fn denoise_poll(&mut self) { + let Some(r) = &self.denoise.running else { return }; + let Some(outcome) = r.outcome.lock().unwrap_or_else(PoisonError::into_inner).take() else { return }; + let Some(r) = self.denoise.running.take() else { return }; + match outcome { + Outcome::Done => { + self.denoise.made += 1; + self.denoise_check_photo(r.photo); + self.denoise_evict(Some(&r.product)); + } + Outcome::Failed(why, unsupported) => { + self.denoise.failed.insert(r.photo, Failure { key: r.key, why, unsupported }); + } + Outcome::Cancelled => {} + } + // the picture's state changed: the interface draws it again + self.denoise.generation += 1; + } + + /// Keep the cache within its limit: the pictures nobody asks for go first, the oldest first; `keep` (the one just + /// made) never goes. The index forgets what was deleted. + pub(crate) fn denoise_evict(&mut self, keep: Option<&Path>) { + let Some(models) = self.denoise.models_dir.clone() else { return }; + self.denoise_evict_to(read_settings(&models).cache_bytes(), keep); + } + + /// [`Self::denoise_evict`] for a limit of `limit` bytes. + pub(crate) fn denoise_evict_to(&mut self, limit: u64, keep: Option<&Path>) { + let Some(dir) = self.denoise_products_dir() else { return }; + let mut files = products_in(&dir); + let mut total: u64 = files.iter().map(|f| f.1).sum(); + if total <= limit { + return; + } + // the pictures photos use now (a Denoise amount above 0) are the last to go + let wanted: HashSet = self + .media + .denoise + .ids() + .into_iter() + .filter(|id| self.catalog.photo(*id).is_some_and(|p| p.develop.denoise_amount() > 0.0)) + .filter_map(|id| self.media.denoise.spec(id).map(|s| s.product.clone())) + .collect(); + files.sort_by_key(|(path, _, modified)| (wanted.contains(path), *modified)); + let mut gone: HashSet = HashSet::new(); + for (path, size, _) in files { + if total <= limit { + break; + } + if keep == Some(path.as_path()) { + continue; + } + if std::fs::remove_file(&path).is_ok() { + total = total.saturating_sub(size); + gone.insert(path); + } + } + if gone.is_empty() { + return; + } + for id in self.media.denoise.ids() { + if self.media.denoise.spec(id).is_some_and(|s| gone.contains(&s.product)) { + self.media.denoise.remove(id); + self.media.forget(id); + } + } + self.denoise.generation += 1; + } + + /// Delete every product of this library (they are made again when wanted). + pub(crate) fn denoise_clear(&mut self) -> (usize, u64) { + self.denoise_cancel(None, true); + let (mut files, mut bytes) = (0, 0); + if let Some(dir) = self.denoise_products_dir() { + for (path, size, _) in products_in(&dir) { + if std::fs::remove_file(&path).is_ok() { + files += 1; + bytes += size; + } + } + } + self.denoise.failed.clear(); + self.denoise.touch(); + (files, bytes) + } + + /// Per frame: take in what finished, look at what the user is looking at, and keep one job going. Cheap when + /// nothing is happening (no file is opened unless a photo with a Denoise amount lacks its picture). + pub(crate) fn denoise_pump(&mut self, pace: Pace) { + crate::cmd::denoise::finish_downloads(self); + self.denoise_refresh_active(false); + // photos that changed (a replaced file has another key) must not keep the picture of what they were + let stale = !self.media.denoise.is_empty() + && self.denoise.reindexed.is_none_or(|(rev, at)| rev != self.catalog.revision && at.elapsed() > SETTINGS_TTL); + if self.denoise.dirty || stale { + self.denoise_reindex(); + } + self.denoise_poll(); + if self.denoise.active.is_none() || self.denoise_products_dir().is_none() { + return; + } + let (cap, auto) = (self.denoise.settings.threads, self.denoise.settings.auto()); + let wanted = self.denoise_wanted(); + for id in &wanted { + self.denoise_check_photo(*id); + } + if auto { + // the photo being edited goes first + let (open, others): (Vec, Vec) = wanted.into_iter().partition(|id| self.selection.active == Some(*id)); + self.denoise_enqueue(&others, false, false); + self.denoise_enqueue(&open, true, false); + } + if self.denoise.running.is_none() && pace != Pace::Pause { + let cores = std::thread::available_parallelism().map_or(1, |n| n.get()); + self.denoise_start_next(pace.parallel(cores, cap)); + } + } + + /// The statistics `denoise.status` reports besides the queue (reads the cache folder). + pub(crate) fn denoise_cache_usage(&self) -> (usize, u64) { + match self.denoise_products_dir() { + Some(dir) => { + let files = products_in(&dir); + (files.len(), files.iter().map(|f| f.1).sum()) + } + None => (0, 0), + } + } + + /// Are pictures made without being asked for photos being looked at (Settings ▸ AI Denoise)? + pub fn denoise_auto(&self) -> bool { + self.denoise.settings.auto() + } + + /// Is a picture being made, or waiting for its turn with a model to make it? (Headless runs wait for this.) + pub fn denoise_busy(&self) -> bool { + self.denoise.running.is_some() || (!self.denoise.queue.is_empty() && self.denoise.active.is_some()) + } + + /// Seconds the running job has been going, and its progress. + pub(crate) fn denoise_running_json(&self) -> Value { + match &self.denoise.running { + Some(r) => json!({ + "photo": r.photo.0, + "done": r.progress.done.load(Ordering::Relaxed), + "total": r.progress.total.load(Ordering::Relaxed), + "seconds": r.started.elapsed().as_secs_f64(), + }), + None => Value::Null, + } + } + + pub(crate) fn denoise_made(&self) -> u64 { + self.denoise.made + } + + /// What an export of photo `id` with `settings` needs for its Denoise amount: its denoised picture, made now when the + /// queue has not got to it. `Ok(None)` when denoise does not apply (amount 0, not a Bayer raw), `Err` when it should + /// have and cannot (no model chosen, nowhere to keep the picture, the model failing). + pub(crate) fn denoise_for_export(&mut self, id: PhotoId, settings: &DevelopSettings) -> Result, String> { + if settings.denoise_amount() <= 0.0 { + return Ok(None); + } + let Some(p) = self.catalog.photo(id).cloned() else { return Ok(None) }; + let Some(loader) = self.media.denoise.loader.clone() else { return Ok(None) }; + if !eligible(&p) { + return Ok(None); + } + self.denoise_refresh_active(false); + let name = &p.file_name; + if self.denoise.active.is_none() { + return Err(format!("{name}: Denoise is on for this photo but no denoise model is chosen; choose one in Settings, or set Denoise to 0")); + } + if self.denoise_products_dir().is_none() { + return Err(format!("{name}: Denoise needs a library on disk to keep its pictures in")); + } + self.denoise_check_photo(id); + let Some(job) = self.denoise_job_spec(&p, Pace::Full.parallel(std::thread::available_parallelism().map_or(1, |n| n.get()), None)) else { + return Ok(None); + }; + let label = name.clone(); + let spec = job.clone(); + let make: MakeProduct = Arc::new(move || match make_product(&spec, None) { + Ok(_) => Ok(true), + Err(MakeError::Unsupported(_)) => Ok(false), + Err(MakeError::Failed(why)) => Err(format!("{label}: could not denoise: {why}")), + Err(MakeError::Cancelled) => Err(format!("{label}: denoise was cancelled")), + }); + Ok(Some((DenoiseSpec { product: job.product, key: job.key, make: Some(make) }, loader))) + } +} + +#[cfg(all(test, feature = "denoise"))] +mod gpu_tests { + use super::*; + use lightcraft_denoise::manifest::{Domain, Gain}; + + /// A folder of the test's own: the card's set-up leaves its marker beside the model. + fn folder(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("lc-engine-gpu-{}-{name}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + /// The model file is read again when the card is set up (for the first photo), so it stays until the test ends. + fn model(name: &str, tile: u64) -> (Arc, lightcraft_denoise::runtime::CpuRunner, PathBuf) { + let path = folder(name).join("model.onnx"); + std::fs::write(&path, lightcraft_denoise::synthetic::unet_onnx(tile, 8, 2, 7)).unwrap(); + let manifest = DenoiserManifest { + id: "synthetic".into(), + name: "Synthetic".into(), + version: "1".into(), + licence: Default::default(), + source: None, + sha256: None, + size_bytes: None, + provenance: String::new(), + domain: Domain::BayerToRgb, + tile: tile as u32, + overlap: 16, + gain: Gain::MatchMean { nominal: 1.0, max_deviation: 0.05 }, + }; + let loaded = default_loader()(&path, &manifest).unwrap(); + let cpu = lightcraft_denoise::runtime::CpuRunner::load(&path, &manifest).unwrap(); + (loaded, cpu, path) + } + + fn worst(want: &[f32], got: &[f32]) -> f32 { + assert_eq!(want.len(), got.len()); + let scale = want.iter().fold(1e-9f32, |m, v| m.max(v.abs())); + want.iter().zip(got).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale + } + + #[test] + fn a_tile_on_the_card_is_the_cpus_tile() { + let (model, cpu, path) = model("same", 64); + let (input, _) = lightcraft_denoise::runtime::test_tile(64); + let want = cpu.run(&input).unwrap(); + let (runner, at_once) = model.runner(RunOn::Gpu, 16); + assert!(worst(&want, &runner.run(&input).unwrap()) < 1e-3); + let device = model.device(RunOn::Gpu); + eprintln!("device: {device}"); + match device["kind"].as_str() { + Some("gpu") => { + assert!(device["adapter"].as_str().is_some_and(|a| !a.is_empty())); + assert!(at_once <= GPU_PARALLEL, "a card is fed by a few tiles at once"); + assert!(device["cardMs"].as_f64().is_some_and(|ms| ms > 0.0) && device["cpuMs"].as_f64().is_some_and(|ms| ms > 0.0), "{device}"); + } + Some("cpu") => { + assert!(device["reason"].as_str().is_some_and(|r| !r.is_empty()), "the CPU is only used for a reason: {device}"); + assert_eq!(at_once, 16); + } + other => panic!("the first photo decides where the work runs, not {other:?}"), + } + assert!(!path.with_file_name(GPU_SETUP_MARKER).exists(), "a set-up that ended leaves no marker"); + let _ = std::fs::remove_dir_all(path.parent().unwrap()); + } + + #[test] + fn with_the_processor_chosen_everything_runs_on_the_cpu() { + let (model, cpu, path) = model("cpu", 64); + let (input, _) = lightcraft_denoise::runtime::test_tile(64); + let (runner, at_once) = model.runner(RunOn::Cpu, 16); + assert_eq!(runner.run(&input).unwrap(), cpu.run(&input).unwrap(), "the CPU path is the CPU runner, bit for bit"); + assert_eq!(at_once, 16, "the threads asked for are the threads used"); + assert_eq!(model.device(RunOn::Cpu)["kind"], "cpu"); + assert_eq!(model.device(RunOn::Auto)["kind"], "pending", "the card was not even set up"); + let _ = std::fs::remove_dir_all(path.parent().unwrap()); + } + + #[test] + fn the_card_is_used_where_it_is_faster_than_the_processor() { + // measured: a tile alone on one CPU thread 1.4 s; an RTX 4090 Laptop 13 ms; a Radeon 610M 490 ms + for threads in [1, 2, 8, 16, 64] { + assert!(card_is_faster(13.0, 1400.0, threads), "a big card wins at {threads}"); + } + assert!(card_is_faster(490.0, 1400.0, 2), "a small card beats the two threads background work gets"); + assert!(!card_is_faster(490.0, 1400.0, 16), "and loses to sixteen"); + assert!(!card_is_faster(f64::INFINITY, 1400.0, 1) && !card_is_faster(f64::NAN, 1400.0, 1), "a card that never ran is never faster"); + assert!(!card_is_faster(10.0, 0.0, 1)); + } + + /// A card that gives numbers that are not finite. + struct Garbage(AtomicUsize); + + impl TileRunner for Garbage { + fn run(&self, input: &[f32]) -> Result, RunError> { + self.0.fetch_add(1, Ordering::Relaxed); + let n = input.len() / 4 * 12; + Ok(vec![f32::NAN; n]) + } + } + + #[test] + fn a_tile_the_card_gets_wrong_is_run_on_the_processor_and_three_stop_the_card() { + let (_, cpu, path) = model("garbage", 64); + let card = GpuSide { runner: Box::new(Garbage(AtomicUsize::new(0))), adapter: "Test card".into(), card_ms: 1.0, cpu_ms: 1000.0 }; + let model = Network { + cpu: cpu.clone(), + path: path.clone(), + tile: 32, + gpu: OnceLock::from(Ok(card)), + fallbacks: AtomicUsize::new(0), + last_threads: AtomicUsize::new(0), + }; + let (input, _) = lightcraft_denoise::runtime::test_tile(64); + let want = cpu.run(&input).unwrap(); + let (runner, at_once) = model.runner(RunOn::Auto, 8); + assert_eq!(at_once, GPU_PARALLEL, "the (fast) card is chosen"); + for _ in 0..GPU_MAX_FALLBACKS { + assert_eq!(runner.run(&input).unwrap(), want, "a wrong tile is the CPU's tile"); + } + let device = model.device(RunOn::Auto); + assert_eq!(device["kind"], "cpu", "{device}"); + assert!(device["reason"].as_str().unwrap().contains("tiles wrong"), "{device}"); + assert_eq!(device["fellBack"], GPU_MAX_FALLBACKS); + let (_, at_once) = model.runner(RunOn::Gpu, 8); + assert_eq!(at_once, 8, "the next photo runs on the CPU, even with the card chosen"); + let _ = std::fs::remove_dir_all(path.parent().unwrap()); + } + + #[test] + fn a_slow_card_is_left_out_unless_it_is_chosen() { + let (_, cpu, path) = model("slow", 64); + let slow = GpuSide { runner: Box::new(cpu.clone()), adapter: "Slow card".into(), card_ms: 490.0, cpu_ms: 1400.0 }; + let model = Network { + cpu, + path: path.clone(), + tile: 32, + gpu: OnceLock::from(Ok(slow)), + fallbacks: AtomicUsize::new(0), + last_threads: AtomicUsize::new(0), + }; + assert_eq!(model.runner(RunOn::Auto, 16).1, 16, "sixteen threads beat it"); + let device = model.device(RunOn::Auto); + assert_eq!((device["kind"].as_str(), device["reason"].as_str()), (Some("cpu"), Some("it is faster than Slow card on this computer"))); + assert_eq!(device["cardMs"], 490.0, "the times measured are reported"); + assert_eq!(device["retry"], false, "trying again would not make it faster"); + assert_eq!(model.runner(RunOn::Auto, 2).1, GPU_PARALLEL.min(2), "two threads do not"); + assert_eq!(model.device(RunOn::Auto)["kind"], "gpu"); + assert_eq!(model.runner(RunOn::Gpu, 16).1, GPU_PARALLEL, "chosen, it is used anyway"); + let _ = std::fs::remove_dir_all(path.parent().unwrap()); + } + + #[test] + fn a_set_up_marker_left_behind_keeps_the_card_off() { + let (model, _, path) = model("crashed", 64); + let marker = path.with_file_name(GPU_SETUP_MARKER); + std::fs::write(&marker, "setting up the graphics card for AI Denoise (unix time 0)").unwrap(); + let (_, at_once) = model.runner(RunOn::Gpu, 4); + assert_eq!(at_once, 4, "the CPU does the work"); + let device = model.device(RunOn::Gpu); + assert_eq!((device["kind"].as_str(), device["reason"].as_str()), (Some("cpu"), Some(GPU_CRASHED)), "{device}"); + assert_eq!(device["retry"], true, "the interface offers to try again"); + assert!(marker.exists(), "it stays until the user chooses where denoise runs"); + let _ = std::fs::remove_dir_all(path.parent().unwrap()); + } + + #[test] + fn a_set_up_runs_beside_the_cpu_and_removes_its_marker() { + let dir = folder("setup"); + let marker = dir.join(GPU_SETUP_MARKER); + let seen = Arc::new(AtomicBool::new(false)); + let (m, s) = (marker.clone(), seen.clone()); + let r = set_up_card( + &marker, + Duration::from_secs(10), + move || { + s.store(m.exists(), Ordering::Relaxed); + Ok(7) + }, + || "cpu", + ); + assert_eq!(r, Ok((7, "cpu"))); + assert!(seen.load(Ordering::Relaxed), "the marker is there while the card is set up"); + assert!(!marker.exists(), "and gone after"); + // a set-up that fails or panics removes it too + assert_eq!(set_up_card(&marker, Duration::from_secs(10), || Err::<(), _>("no card".into()), || ()), Err("no card".into())); + assert!(!marker.exists()); + let r = set_up_card(&marker, Duration::from_secs(10), || -> Result<(), String> { panic!("driver") }, || ()); + assert!(r.is_err() && !marker.exists(), "{r:?}"); + // a marker found before the set-up starts means the last one took LightCraft down: the card is not touched + std::fs::write(&marker, "left behind").unwrap(); + let touched = Arc::new(AtomicBool::new(false)); + let t = touched.clone(); + let r = set_up_card( + &marker, + Duration::from_secs(10), + move || { + t.store(true, Ordering::Relaxed); + Ok(()) + }, + || (), + ); + assert_eq!(r, Err(GPU_CRASHED.to_string())); + assert!(!touched.load(Ordering::Relaxed) && marker.exists()); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn a_set_up_that_takes_too_long_is_given_up_on() { + let dir = folder("slowsetup"); + let marker = dir.join(GPU_SETUP_MARKER); + let started = std::time::Instant::now(); + let slow = || -> Result<(), String> { + std::thread::sleep(Duration::from_millis(600)); + Ok(()) + }; + let r = set_up_card(&marker, Duration::from_millis(50), slow, || ()); + assert!(r.as_ref().is_err_and(|e| e.contains("took longer") && worth_retrying(e)), "{r:?}"); + assert!(started.elapsed() < Duration::from_millis(500), "the caller does not wait for the card"); + // while the first is still stuck, another set-up of the same model does not start a second one + let again = set_up_card(&marker, Duration::from_secs(5), || Ok(()), || ()); + assert!(again.is_err_and(|e| e.contains("still setting up") && worth_retrying(&e))); + // once it ends its marker goes, and the card can be set up again + while marker.exists() { + assert!(started.elapsed() < Duration::from_secs(10), "the stuck set-up ended"); + std::thread::sleep(Duration::from_millis(20)); + } + while SETTING_UP.lock().unwrap().iter().any(|m| m == &marker) { + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(set_up_card(&marker, Duration::from_secs(5), || Ok(1), || 2), Ok((1, 2))); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/engine/src/files.rs b/crates/engine/src/files.rs index c620be92..5ef823db 100644 --- a/crates/engine/src/files.rs +++ b/crates/engine/src/files.rs @@ -11,7 +11,7 @@ use lightcraft_pipeline::SourceInfo; use lightcraft_raster::Rgb32f; use lightcraft_raster::resample::{Filter, fit}; -use crate::media::{FileLoader, FileProbe, PreviewLoader, ProbeInfo}; +use crate::media::{DenoiseSpec, FileLoader, FileProbe, PairLoader, PreviewLoader, ProbeInfo}; fn meta_of(m: &lightcraft_meta::Metadata) -> (Meta, Option) { let shutter = m.exposure_time.map(|t| if t >= 1.0 { format!("{t:.0}") } else { format!("1/{:.0}", 1.0 / t) }).unwrap_or_default(); @@ -220,22 +220,56 @@ pub fn bin_factor(raw: &lightcraft_raw::RawImage, max_edge: usize) -> Option Result<(Rgb32f, SourceInfo), String> { - rayon::scope(|_| load_bytes_now(std::borrow::Cow::Borrowed(bytes), max_edge)) + rayon::scope(|_| load_bytes_now(std::borrow::Cow::Borrowed(bytes), max_edge, None).map(|(img, _, info)| (img, info))) } /// [`load_bytes`] taking the file's bytes: a raw file's bytes are freed as soon as it is decoded /// (less memory held while it is developed). pub fn load_vec(bytes: Vec, max_edge: usize) -> Result<(Rgb32f, SourceInfo), String> { - rayon::scope(move |_| load_bytes_now(std::borrow::Cow::Owned(bytes), max_edge)) + rayon::scope(move |_| load_bytes_now(std::borrow::Cow::Owned(bytes), max_edge, None).map(|(img, _, info)| (img, info))) } -fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result<(Rgb32f, SourceInfo), String> { +/// [`load_vec`] that also develops the photo's cached denoised picture (`spec`) from the same decode: the plain +/// picture, the denoised one (`None` when it cannot be read or does not fit this raw: the plain one is then all +/// there is) and the source facts. Both pictures are the same size. +pub fn load_vec_pair(bytes: Vec, max_edge: usize, spec: &DenoiseSpec) -> Result<(Rgb32f, Option, SourceInfo), String> { + rayon::scope(move |_| load_bytes_now(std::borrow::Cow::Owned(bytes), max_edge, Some(spec))) +} + +/// The denoised camera RGB of `raw` from its cached product, as the plain development would have it: the default +/// crop, and `factor` × `factor` blocks averaged into one pixel when the plain one was binned. `None` when the product +/// is missing, damaged, made from something else or not the size of this raw's active area. +fn denoised_camera_rgb(raw: &lightcraft_raw::RawImage, spec: &DenoiseSpec, factor: usize) -> Option { + use lightcraft_denoise::product::{self, Window}; + let a = raw.active_area; + let header = product::read_header_at(&spec.product).ok()?; + if header.key != spec.key || (header.width, header.height) != (a.width, a.height) { + return None; + } + if raw.opcodes.list3.is_empty() { + let c = raw.develop_crop(a.width, a.height); + let window = Window { x: c.x, y: c.y, width: c.width, height: c.height }; + product::read_window(&spec.product, &spec.key, Some(window), factor, Some(HIGHLIGHT_CLIP)).ok() + } else { + // (the plain development is not binned either when the file has opcodes of this kind: `factor` is 1) + let whole = product::read(&spec.product, &spec.key, 1).ok()?; + Some(raw.finish_demosaiced(whole)) + } +} + +fn load_bytes_now( + bytes: std::borrow::Cow<'_, [u8]>, + max_edge: usize, + denoise: Option<&DenoiseSpec>, +) -> Result<(Rgb32f, Option, SourceInfo), String> { if lightcraft_raw::probe(&bytes).is_some() { let mut raw = match lightcraft_raw::decode(&bytes).map_err(|e| preview_reason(&bytes, e)) { Ok(r) => r, Err(Ok(why)) => { // show the camera's embedded JPEG (rendered, not raw) until the variant is supported - return load_embedded_preview(&bytes, max_edge).ok_or(format!("unsupported raw ({why}) without an embedded preview")); + return load_embedded_preview(&bytes, max_edge) + .map(|(img, info)| (img, None, info)) + .ok_or(format!("unsupported raw ({why}) without an embedded preview")); } Err(Err(e)) => return Err(e), }; @@ -250,71 +284,72 @@ fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result< // Previews and thumbnails bin the mosaic straight to (about) the size they need; only // larger levels (exports, 1:1) demosaic the whole sensor. let t0 = web_time::Instant::now(); - let binned = match bin_factor(&raw, max_edge) { + let bin = bin_factor(&raw, max_edge); + let binned = match bin { Some(k) => raw.develop_binned(k, HIGHLIGHT_CLIP).map_err(|e| e.to_string())?, None => None, }; - let mut img = match binned { + let factor = if binned.is_some() { bin.unwrap_or(1) } else { 1 }; + let img = match binned { Some(img) => img, None => { let method = if max_edge <= 600 { lightcraft_raw::Method::Bilinear } else { lightcraft_raw::Method::Ahd }; raw.develop(method).map_err(|e| e.to_string())? } }; + // the same development from the denoised mosaic, when the photo has been denoised + let twin = denoise.and_then(|spec| denoised_camera_rgb(&raw, spec, factor)).filter(|d| d.width == img.width && d.height == img.height); // the samples aren't needed any more (the colour model below reads only the tags) raw.data = lightcraft_raw::RawData::U16(Vec::new()); - let mut stages = vec![("develop", t0.elapsed())]; - stages.push(("transform", t0.elapsed())); - lightcraft_raw::highlight::reconstruct(&mut img, t.wb, HIGHLIGHT_CLIP); - stages.push(("highlights", t0.elapsed())); + let wb = t.wb; let m = camera_look.as_ref().map(|p| p.matrix.mul(&t.matrix)).unwrap_or(t.matrix).to_f32(); let hue_sat = camera_look.as_ref().and_then(|p| p.hue_sat.as_ref()).and_then(crate::camera_preview::HueSat::new); let gain = 2f32.powf(t.baseline_exposure as f32); - let wb = t.wb; // A DNG's own profile look (hue/saturation map, look table), DNG spec chapter 6. let tables = lightcraft_raw::profile::ProfileTables::new(&raw.color.profile, lightcraft_raw::color::illuminant_weight(&raw.color, xy)); - img.map_in_place(|p| { - let c = [p[0] * wb[0], p[1] * wb[1], p[2] * wb[2]]; - let rgb = [ - m[0][0] * c[0] + m[0][1] * c[1] + m[0][2] * c[2], - m[1][0] * c[0] + m[1][1] * c[1] + m[1][2] * c[2], - m[2][0] * c[0] + m[2][1] * c[1] + m[2][2] * c[2], - ]; - let rgb = match &tables { - Some(tables) => tables.apply(rgb, gain), - None => rgb.map(|v| v * gain), - }; - // after the baseline exposure, as when it was fitted - hue_sat.as_ref().map_or(rgb, |h| h.apply(rgb)).map(|v| v.max(0.0)) - }); - stages.push(("colour", t0.elapsed())); - let img = fit(&img, max_edge, max_edge, Filter::Box); - stages.push(("fit", t0.elapsed())); - let img = img.into_oriented(raw.orientation); - stages.push(("orient", t0.elapsed())); + // camera RGB → what the pipeline takes: clipped highlights rebuilt, white balance and the colour model applied, + // fitted to `max_edge` and upright (the same for the plain and the denoised picture) + let finish = |mut img: Rgb32f| -> Rgb32f { + lightcraft_raw::highlight::reconstruct(&mut img, wb, HIGHLIGHT_CLIP); + img.map_in_place(|p| { + let c = [p[0] * wb[0], p[1] * wb[1], p[2] * wb[2]]; + let rgb = [ + m[0][0] * c[0] + m[0][1] * c[1] + m[0][2] * c[2], + m[1][0] * c[0] + m[1][1] * c[1] + m[1][2] * c[2], + m[2][0] * c[0] + m[2][1] * c[1] + m[2][2] * c[2], + ]; + let rgb = match &tables { + Some(tables) => tables.apply(rgb, gain), + None => rgb.map(|v| v * gain), + }; + hue_sat.as_ref().map_or(rgb, |h| h.apply(rgb)).map(|v| v.max(0.0)) + }); + let img = fit(&img, max_edge, max_edge, Filter::Box); + img.into_oriented(raw.orientation) + }; + let develop_ms = t0.elapsed(); + let img = finish(img); + let twin = twin.map(finish); if lightcraft_pipeline::profiling() { - let mut prev = std::time::Duration::ZERO; - let parts: Vec = stages - .iter() - .map(|(n, t)| { - let d = *t - prev; - prev = *t; - format!("{n} {:.1}", d.as_secs_f64() * 1e3) - }) - .collect(); - eprintln!("[profile] raw source {}×{} (max {max_edge}, ms after decode): {}", img.width, img.height, parts.join(", ")); + let finish_ms = (t0.elapsed() - develop_ms).as_secs_f64() * 1e3; + eprintln!( + "[profile] raw source {}×{} (max {max_edge}): develop {:.1} ms, finish {finish_ms:.1} ms (after decode)", + img.width, + img.height, + develop_ms.as_secs_f64() * 1e3 + ); } let (temp, tint) = xy_to_temp_tint(xy); let relative = crate::camera_preview::file_local_look(raw.format) && t.matrix_is_fallback; let camera_tone = camera_look.as_ref().map(|p| p.tone).or_else(|| raw.color.profile.tone_curve.as_ref().and_then(dng_tone_curve)); let (temp, tint) = if relative { (6500.0, 0.0) } else { (temp.round(), tint.round()) }; - return Ok((img, SourceInfo { raw: true, as_shot_temp: temp, as_shot_tint: tint, lens, relative_wb: relative, camera_tone })); + return Ok((img, twin, SourceInfo { raw: true, as_shot_temp: temp, as_shot_tint: tint, lens, relative_wb: relative, camera_tone })); } let d = lightcraft_codecs::decode(&bytes, fit_box(max_edge)).map_err(|e| e.to_string())?; drop(bytes); let img = d.to_working(); let img = if img.width.max(img.height) > max_edge { fit(&img, max_edge, max_edge, Filter::Mitchell) } else { img }; - Ok((img.into_oriented(Orientation::from_exif(d.orientation)), SourceInfo::default())) + Ok((img.into_oriented(Orientation::from_exif(d.orientation)), None, SourceInfo::default())) } /// A DNG `ProfileToneCurve` (linear in, linear out, 1.0 = white after exposure compensation) as the @@ -408,6 +443,21 @@ pub fn fs_preview_loader() -> PreviewLoader { Arc::new(|path: &str, max_edge: usize| embedded_preview_srgb(&std::fs::read(path).ok()?, max_edge)) } +/// The filesystem-backed [`PairLoader`]: the file is read and decoded once, and developed as usual and from the photo's +/// denoised picture. It holds two pictures at once, so it counts for more at the memory gate than a plain load. +pub fn fs_pair_loader() -> PairLoader { + Arc::new(|path: &str, max_edge: usize, spec: &DenoiseSpec| { + let len = std::fs::metadata(path).map(|m| m.len() as usize).unwrap_or(0); + let weight = len * if max_edge <= crate::media::SourceLevel::Thumb.max_edge() { 4 } else { 10 }; + let gate = crate::memory::work_gate(); + let _permit = if crate::memory::is_background() { gate.acquire(weight) } else { gate.acquire_urgent(weight) }; + let bytes = std::fs::read(path).map_err(|e| format!("{path}: {e}"))?; + let r = load_vec_pair(bytes, max_edge, spec); + crate::memory::release(); + r + }) +} + /// Filesystem-backed hooks (native). On the web the host installs bytes-based hooks instead. /// /// Their working memory is bounded by [`crate::memory::work_gate`]: background loads (grid @@ -443,6 +493,7 @@ impl crate::Session { self.media.file_loader = Some(l); self.media.file_probe = Some(p); self.media.preview_loader = Some(fs_preview_loader()); + self.media.denoise.loader = Some(fs_pair_loader()); self } } @@ -483,6 +534,37 @@ mod tests { assert!(probe_bytes("x.jpg", b"not an image").is_err()); } + #[test] + fn cached_denoise_uses_the_same_white_balance_profile_and_orientation_as_plain_raw() { + use lightcraft_denoise::product; + use lightcraft_raw::{DngWriteOptions, Method, Orientation, profile::HsvTable, write_dng}; + let dir = crate::tests_xmp::temp_dir("denoise-color-finish"); + let fixture = crate::tests_xmp::synthetic_dng_with(None, Default::default()); + let mut raw = lightcraft_raw::decode(&fixture).unwrap(); + raw.orientation = Orientation::Rotate90; + raw.color.baseline_exposure = 0.5; + raw.color.profile.look_table = + Some(HsvTable { hue_divisions: 1, sat_divisions: 2, val_divisions: 1, data: vec![[23.0, 0.65, 0.8]; 2], srgb_value: false }); + let bytes = write_dng(&raw, &DngWriteOptions::default()).unwrap(); + let decoded = lightcraft_raw::decode(&bytes).unwrap(); + assert!(decoded.color.profile.look_table.is_some()); + // A cache containing the ordinary demosaic must undergo the identical downstream finish. + let camera_rgb = decoded.develop(Method::Ahd).unwrap(); + let key = "synthetic-color-finish"; + let path = dir.join("same.lcdn"); + product::write(&path, &camera_rgb, key).unwrap(); + let spec = DenoiseSpec { product: path, key: key.into(), make: None }; + let (ordinary, _) = load_bytes(&bytes, 2048).unwrap(); + let (plain, cached, _) = load_vec_pair(bytes, 2048, &spec).unwrap(); + assert_eq!(plain.data, ordinary.data, "requesting a twin must not change the ordinary path"); + let cached = cached.unwrap(); + assert_eq!((plain.width, plain.height), (24, 32)); + assert_eq!((cached.width, cached.height), (plain.width, plain.height)); + let worst = plain.data.iter().zip(&cached.data).flat_map(|(a, b)| a.iter().zip(b).map(|(a, b)| (a - b).abs())).fold(0.0f32, f32::max); + assert!(worst < 0.002, "color finish differs beyond cache half-float rounding: {worst}"); + let _ = std::fs::remove_dir_all(dir); + } + /// Luminance (from linear RGB) minus its 9 × 9 local mean: tone differences between a render and a camera JPEG /// mostly cancel, edges remain. fn detail(w: usize, h: usize, lum: impl Fn(usize, usize) -> f32) -> Vec { diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index f26b4ca7..5f298de4 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -14,9 +14,11 @@ pub mod availability; mod camera_preview; pub mod camera_profiles; pub mod cmd; +pub mod config; pub mod crs; pub mod crs_masks; pub mod demo; +pub mod denoise; pub mod devices; pub mod export; pub mod files; @@ -33,6 +35,7 @@ pub mod logging; pub mod media; pub mod memory; pub mod merge; +mod model_download; pub mod originals; pub mod preset_import; pub mod preset_luminar; @@ -132,8 +135,10 @@ pub struct Interaction { /// Source of [`Session::visible_shared`] generations (process-wide, so two sessions never share one). static VISIBLE_GEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); +static LIBRARY_GEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); pub struct Session { + library_generation: u64, /// Auto Sync: edits to the active photo also change the other selected photos (the settings /// that changed, nothing else). pub auto_sync: bool, @@ -157,6 +162,8 @@ pub struct Session { /// Set by a command whose change must not rewrite the photo's XMP sidecar even with auto-write on /// (a catalog-only edit of data the sidecar writer does not emit); consumed when the command ends. pub(crate) skip_auto_write: bool, + /// AI denoise: the model in use, the photos that have their picture and the work in progress. + pub(crate) denoise: denoise::State, /// Copied develop settings (partial JSON) for Paste. pub clipboard: Option, /// The folder on disk the [`LibrarySource::Folder`] view browses. @@ -247,8 +254,13 @@ impl Default for Session { } impl Session { + pub fn library_generation(&self) -> u64 { + self.library_generation + } + pub fn new() -> Session { Session { + library_generation: LIBRARY_GEN.fetch_add(1, std::sync::atomic::Ordering::Relaxed), auto_sync: false, catalog: Catalog::new(), source: LibrarySource::All, @@ -263,6 +275,7 @@ impl Session { redo: Vec::new(), interaction: None, skip_auto_write: false, + denoise: Default::default(), clipboard: None, meta_clipboard: None, browse: None, @@ -783,6 +796,8 @@ mod tests; #[cfg(test)] mod tests_color; #[cfg(test)] +mod tests_denoise; +#[cfg(test)] mod tests_export; #[cfg(test)] mod tests_folders; diff --git a/crates/engine/src/library.rs b/crates/engine/src/library.rs index c1b91c4c..384d90ea 100644 --- a/crates/engine/src/library.rs +++ b/crates/engine/src/library.rs @@ -297,6 +297,9 @@ impl Session { { log::error!("library: {e}"); } + // Photo ids belong to one library; denoise work must not carry into the next. + self.denoise.library_changed(); + self.media.denoise.clear(); let (mut journal, mut catalog, report) = Journal::open(catalog)?; // A loaded catalog counts revisions from 0, like every other one. Caches (sidebar counts, // keyword tree, the grid's list…) are keyed on the revision, so give each library loaded @@ -305,6 +308,7 @@ impl Session { static LOADS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); catalog.revision = LOADS.fetch_add(1, std::sync::atomic::Ordering::Relaxed).wrapping_add(1) << 32; self.catalog = catalog; + self.library_generation = crate::LIBRARY_GEN.fetch_add(1, std::sync::atomic::Ordering::Relaxed); self.undo.clear(); self.redo.clear(); self.interaction = None; diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rs index 1ed89e47..479831c9 100644 --- a/crates/engine/src/media.rs +++ b/crates/engine/src/media.rs @@ -125,6 +125,44 @@ pub type FileLoader = Arc Result<(Rgb32f, SourceInfo), St /// `max_edge` (set by the app). `None`: no usable preview. pub type PreviewLoader = Arc Option + Send + Sync>; +/// Makes a photo's denoised picture when it is not cached yet (an export does not wait for the background queue): +/// `Ok(true)` once it is there, `Ok(false)` when denoise does not apply to this photo (render it plain), `Err` when it +/// should have worked and did not. +pub type MakeProduct = Arc Result + Send + Sync>; + +/// A raw photo's cached AI-denoised picture (see [`crate::denoise`]): where the file is and the key it must have been +/// made with. A file that is missing, damaged or made for something else is a cache miss, never an error. +#[derive(Clone)] +pub struct DenoiseSpec { + pub product: std::path::PathBuf, + pub key: String, + /// Run before the picture is read, by a job that must not render without it (an export). + pub make: Option, +} + +/// Decodes a raw file once and develops it twice: as usual, and from the denoised picture in `spec` when that can be +/// read (`None` when it cannot). Both are at most `max_edge` long and the same size. +pub type PairLoader = Arc Result<(Rgb32f, Option, SourceInfo), String> + Send + Sync>; + +/// A photo's denoised picture next to its plain one, and the last mix of the two that was asked for. +pub struct Twin { + pub image: Arc, + /// The Amount (in thousandths) and the picture mixed for it, so dragging other sliders keeps one source picture. + mix: std::sync::Mutex)>>, +} + +impl Twin { + pub fn new(image: Arc) -> Twin { + Twin { image, mix: std::sync::Mutex::new(None) } + } + + /// Bytes held, the last mix included. + fn bytes(&self) -> usize { + let mixed = self.mix.lock().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref().map_or(0, |m| m.1.data.len() * 12); + self.image.data.len() * 12 + mixed + } +} + /// Pixels and the source interpretation learned while decoding; they must be cached together. #[derive(Clone)] pub struct DecodedSource { @@ -133,11 +171,45 @@ pub struct DecodedSource { /// A smart preview's stored camera tone curve: the one decoder fact its pixels need that the /// catalog's header facts lack (used when `info` is `None`). pub camera_tone: Option, + /// The same picture developed from the AI-denoised mosaic, when the photo has one cached. + pub denoised: Option>, } impl DecodedSource { pub fn new(image: Arc, info: Option) -> Self { - DecodedSource { image, info, camera_tone: None } + DecodedSource { image, info, camera_tone: None, denoised: None } + } + + /// The picture to develop for a Denoise `amount` (0 to 1, see [`DevelopSettings::denoise_amount`]): the plain one, + /// the denoised one, or a mix of the two. A mix is made once per amount and kept, so the stages after it are + /// reused until the amount changes. + pub fn image_for(&self, amount: f32) -> Arc { + let Some(twin) = &self.denoised else { return self.image.clone() }; + if amount.is_nan() || amount <= 0.0 || twin.image.width != self.image.width || twin.image.height != self.image.height { + return self.image.clone(); + } + if amount >= 1.0 { + return twin.image.clone(); + } + let thousandths = (amount * 1000.0).round().clamp(1.0, 999.0) as u32; + let mut memo = twin.mix.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some((t, img)) = memo.as_ref() + && *t == thousandths + { + return img.clone(); + } + let mut mixed = (*self.image).clone(); + if !lightcraft_denoise::product::blend(&mut mixed, &twin.image, thousandths as f32 / 1000.0) { + return self.image.clone(); + } + let mixed = Arc::new(mixed); + *memo = Some((thousandths, mixed.clone())); + mixed + } + + /// Bytes held: the picture, its denoised twin and the last mix. + pub fn bytes(&self) -> usize { + self.image.data.len() * 12 + 64 + std::mem::size_of::() + self.denoised.as_ref().map_or(0, |t| t.bytes()) } /// What to render these pixels against: the decoder's facts, else `header` (the catalog's) @@ -161,6 +233,8 @@ pub enum SourceRef { /// The photo's smart preview, used when the original can't be read (offline drive). The /// UI thread never checks whether the original is there; the render worker finds out. fallback: Option, + /// The photo has a cached denoised picture: load it with the plain one (see [`PairLoader`]). + denoise: Option<(DenoiseSpec, PairLoader)>, }, /// A smart preview standing in for a missing original. Smart { @@ -177,15 +251,45 @@ impl SourceRef { let image = match self { SourceRef::Loaded(a) => return Ok((**a).clone()), SourceRef::Demo { scene, max_edge } => Arc::new(scene.render_fit(*max_edge)), - SourceRef::File { path, max_edge, loader, fallback } => { - let r = match loader { + SourceRef::File { path, max_edge, loader, fallback, denoise } => { + let plain = || match loader { Some(l) => l(path, *max_edge).map(|(image, info)| DecodedSource::new(Arc::new(image), Some(info))), None => Err(format!("no decoder available for {path}")), }; + // the denoised picture comes with the plain one from a single decode; when it cannot be read the + // pair loader gives the plain picture alone (the cached file is never a reason to fail a render) + let r = match denoise { + Some((spec, pair)) => { + // a job that needs the denoised picture has it made first; if that fails the job fails (no smart + // preview stands in: an export must not quietly come out without the denoise it was asked for) + let made = match &spec.make { + Some(make) => make()?, + None => true, + }; + if made { + pair(path, *max_edge, spec).and_then(|(image, twin, info)| { + if spec.make.is_some() && twin.is_none() { + return Err("the cached AI Denoise picture could not be read; clear the denoise cache and retry".into()); + } + Ok(DecodedSource { + image: Arc::new(image), + info: Some(info), + camera_tone: None, + denoised: twin.map(|t| Arc::new(Twin::new(Arc::new(t)))), + }) + }) + } else { + plain() + } + } + None => plain(), + }; // An offline original renders from its smart preview (no decoder facts: header ones). return match (r, fallback) { #[cfg(not(target_arch = "wasm32"))] - (Err(e), Some(sp)) if crate::smart::is_valid(sp) => crate::smart::load(sp).map_err(|e2| format!("{e}; smart preview: {e2}")), + (Err(e), Some(sp)) if denoise.as_ref().is_none_or(|(spec, _)| spec.make.is_none()) && crate::smart::is_valid(sp) => { + crate::smart::load(sp).map_err(|e2| format!("{e}; smart preview: {e2}")) + } (r, _) => r, }; } @@ -224,6 +328,54 @@ pub struct MediaCache { scenes: Vec, /// Rendered thumbnails (memory, plus disk once a library is attached). pub rendered: Arc, + /// Which photos have a cached denoised picture, and how to read it. + pub denoise: DenoiseIndex, +} + +/// The photos that have a cached denoised picture for the denoise model in use. Only memory: nothing here touches the +/// file system, so it can be asked on the UI thread. +#[derive(Default)] +pub struct DenoiseIndex { + ready: std::collections::HashMap, + /// Reads the pictures (`None`: this build cannot, and nothing is ever ready). + pub loader: Option, +} + +impl DenoiseIndex { + /// What to load for `id` besides its plain picture, and how. + pub fn get(&self, id: PhotoId) -> Option<(&DenoiseSpec, &PairLoader)> { + self.ready.get(&id).zip(self.loader.as_ref()) + } + + pub fn spec(&self, id: PhotoId) -> Option<&DenoiseSpec> { + self.ready.get(&id) + } + + pub fn set(&mut self, id: PhotoId, spec: DenoiseSpec) { + self.ready.insert(id, spec); + } + + pub fn remove(&mut self, id: PhotoId) -> bool { + self.ready.remove(&id).is_some() + } + + pub fn clear(&mut self) { + self.ready.clear(); + } + + pub fn len(&self) -> usize { + self.ready.len() + } + + pub fn is_empty(&self) -> bool { + self.ready.is_empty() + } + + pub fn ids(&self) -> Vec { + let mut v: Vec = self.ready.keys().copied().collect(); + v.sort(); + v + } } impl Default for MediaCache { @@ -244,6 +396,7 @@ impl Default for MediaCache { availability: Default::default(), scenes: Vec::new(), rendered: Arc::new(PreviewCache::memory(rendered_budget(budget))), + denoise: DenoiseIndex::default(), } } } @@ -253,10 +406,6 @@ fn rendered_budget(share: usize) -> usize { RENDERED_MEM_BYTES.min(share / 4) } -fn source_bytes(img: &Rgb32f) -> usize { - img.data.len() * 12 + 64 + std::mem::size_of::() -} - impl MediaCache { /// Keep rendered thumbnails on disk in `dir` as well. The cache this replaces is retired /// (its files stay): render jobs still holding it can't write through it any more, even @@ -319,8 +468,8 @@ impl MediaCache { let Some((tick, which)) = candidates.into_iter().flatten().min() else { break }; let freed = match which { 0 => self.thumbs.pop_oldest(), - 1 => self.previews.iter().position(|e| e.2 == tick).map(|i| source_bytes(&self.previews.remove(i).1.image)), - 2 => self.full.take().map(|e| source_bytes(&e.1.image)), + 1 => self.previews.iter().position(|e| e.2 == tick).map(|i| self.previews.remove(i).1.bytes()), + 2 => self.full.take().map(|e| e.1.bytes()), _ => self.rendered.evict_oldest(), }; match freed { @@ -372,7 +521,7 @@ impl MediaCache { let tick = lightcraft_preview::next_tick(); match level { SourceLevel::Thumb => { - let cost = source_bytes(&img.image); + let cost = img.bytes(); self.thumbs.insert(id, img, cost); } SourceLevel::Preview => { @@ -415,13 +564,42 @@ impl MediaCache { /// Decoded sources held: (thumbnail level, preview level, full size). pub fn usage(&self) -> (crate::memory::Usage, crate::memory::Usage, crate::memory::Usage) { use crate::memory::Usage; - let previews = Usage::new(self.previews.len(), self.previews.iter().map(|e| source_bytes(&e.1.image)).sum()); - let full = self.full.as_ref().map(|e| Usage::new(1, source_bytes(&e.1.image))).unwrap_or_default(); + let previews = Usage::new(self.previews.len(), self.previews.iter().map(|e| e.1.bytes()).sum()); + let full = self.full.as_ref().map(|e| Usage::new(1, e.1.bytes())).unwrap_or_default(); (Usage::new(self.thumbs.len(), self.thumbs.cost()), previews, full) } + /// What a render of photo `id` with settings `s` adds to its identity for the denoised picture it will use: 0 unless + /// it uses one (the Denoise amount is above 0 and the photo has a cached denoised picture). + pub fn denoise_salt(&self, id: PhotoId, s: &DevelopSettings) -> u64 { + match self.denoise.spec(id) { + Some(spec) if s.denoise_amount() > 0.0 => Hasher128::new().str(&spec.key).finish().0 as u64 | 1, + _ => 0, + } + } + + /// How to load photo `p` at `level`: from the cache when its source is there, else from its file (with its denoised + /// picture beside it when it has one), its smart preview or the demo scene. pub fn source_ref(&mut self, p: &Photo, level: SourceLevel) -> SourceRef { - if let Some(a) = self.get_source(p.id, level) { + self.source_ref_with(p, level, true) + } + + /// [`Self::source_ref`] that never takes a decoded source from memory: for a job that needs something the cached + /// one lacks (an export, whose denoised picture is made when it runs). + pub fn source_ref_uncached(&mut self, p: &Photo, level: SourceLevel) -> SourceRef { + self.source_ref_with(p, level, false) + } + + fn source_ref_with(&mut self, p: &Photo, level: SourceLevel, cached: bool) -> SourceRef { + let mut r = self.source_ref_plain(p, level, cached); + if let (SourceRef::File { denoise, .. }, Some((spec, loader))) = (&mut r, self.denoise.get(p.id)) { + *denoise = Some((spec.clone(), loader.clone())); + } + r + } + + fn source_ref_plain(&mut self, p: &Photo, level: SourceLevel, cached: bool) -> SourceRef { + if cached && let Some(a) = self.get_source(p.id, level) { return SourceRef::Loaded(Box::new(a)); } // Procedural scenes have a nominal size: "full" is that size, not unbounded. @@ -459,10 +637,12 @@ impl MediaCache { } match self.scenes.iter().find(|s| s.id == *scene) { Some(s) => SourceRef::Demo { scene: Box::new(s.clone()), max_edge }, - None => SourceRef::File { path: format!("demo:{scene}"), max_edge, loader: None, fallback: None }, + None => SourceRef::File { path: format!("demo:{scene}"), max_edge, loader: None, fallback: None, denoise: None }, } } - Source::File { path } => SourceRef::File { path: path.clone(), max_edge, loader: self.file_loader.clone(), fallback: None }, + Source::File { path } => { + SourceRef::File { path: path.clone(), max_edge, loader: self.file_loader.clone(), fallback: None, denoise: None } + } } } } @@ -573,7 +753,8 @@ impl RenderJob { let was_loaded = matches!(self.source, SourceRef::Loaded(_)); match self.source.load_source() { Ok(source) => { - let src = &source.image; + // the plain picture, or (Denoise amount above 0 and a denoised picture cached) the mix the amount asks for + let src = &source.image_for(self.settings.denoise_amount()); let info = source.info_or(self.info); // Thumbnails (many small jobs side by side) stay on the CPU; views and exports use // the GPU when there is one. @@ -757,22 +938,23 @@ impl crate::Session { let content = source_key.0 as u64; // Temporary "before" settings must not replace the photo's cached develop hash. let settings_hash = if before { settings.hash64() } else { self.media.settings_hashes.get(id, &settings) }; + let salt = self.media.denoise_salt(id, &settings); let key = settings_hash ^ ((max_w as u64) << 40) ^ ((max_h as u64) << 20) ^ (apply_crop as u64) ^ (level as u64) << 60 ^ id.0.wrapping_mul(0x9e37_79b9_7f4a_7c15) - ^ content.rotate_left(17); + ^ content.rotate_left(17) + ^ salt; let cache = thumb_bucket.map(|b| { - let k = Hasher128::new() - .str(&content_key) - .u64(settings_hash) - .u64(b as u64) - .u64(RENDER_CACHE_VERSION) - .u64(crate::camera_profiles::cache_key()) - .finish(); - (self.media.rendered.clone(), k) + let mut k = Hasher128::new(); + k.str(&content_key).u64(settings_hash).u64(b as u64).u64(RENDER_CACHE_VERSION).u64(crate::camera_profiles::cache_key()); + // (only a render that uses a denoised picture has it in its key, so every other thumbnail stays valid) + if salt != 0 { + k.u64(salt); + } + (self.media.rendered.clone(), k.finish()) }); Some(RenderJob { request_id: 0, @@ -1052,6 +1234,15 @@ impl crate::Session { depth: lightcraft_pipeline::OutputDepth, ) -> Result { let mut job = self.render_job(id, max_w, max_h, false, true).ok_or("no such photo")?; + // An export is made from the denoised picture its Denoise amount asks for, made when the job runs if the + // background queue has not got to it. (A source kept in memory may lack the picture, so the job reads the file.) + if let Some(wanted) = self.denoise_for_export(id, &job.settings)? { + let p = self.catalog.photo(id).ok_or("no such photo")?.clone(); + job.source = self.media.source_ref_uncached(&p, job.level); + if let SourceRef::File { denoise, .. } = &mut job.source { + *denoise = Some(wanted); + } + } job.request.space = space; job.request.depth = depth; job.key ^= (space as u64 + 1).wrapping_mul(0xa076_1d64_78bd_642f) ^ (depth as u64 + 1).wrapping_mul(0xe703_7ed1_a0b4_28db); @@ -1219,6 +1410,7 @@ mod tests { Ok((image, info)) })), fallback: None, + denoise: None, }; job.info = SourceInfo::default(); // Header facts cannot override decoder facts. job.settings = Arc::new(DevelopSettings::default()); diff --git a/crates/engine/src/model_download.rs b/crates/engine/src/model_download.rs new file mode 100644 index 00000000..f07c373d --- /dev/null +++ b/crates/engine/src/model_download.rs @@ -0,0 +1,363 @@ +//! Downloading a denoise model explicitly requested after showing its terms. +//! +//! The transfer is [`lightcraft_fetch`] (pure-Rust HTTPS with resume, size and SHA-256 checks), on a background thread, +//! so the UI never waits for the network. Only a model listed in [`lightcraft_denoise::known::Download`] can be fetched: +//! the address and hash come from our pinned catalog or the user's model catalog, and no token is sent with it. What arrives +//! matches the model's recorded size and SHA-256 or it is thrown away; it then waits in a staging folder until the model +//! is installed (`denoise.models.install`, or by itself once the user has accepted the model's terms in the download). +//! +//! Failure is a message, never a panic: no network, a stalled or truncated transfer, a full disk, a file that is not the +//! one expected, and a panic inside the thread itself all end as [`State::Failed`]. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use lightcraft_denoise::known::Download; + +/// The folder inside the models folder where a downloaded file waits to be installed. Its name starts with a dot, so it +/// can never be a model's own folder (ids start with a letter or digit). +pub const STAGING: &str = ".downloads"; + +/// Where a download stands. +#[derive(Clone, Debug, PartialEq)] +pub enum State { + /// Fetching (`bytes` of `total` so far); the file is checked as the last step. + Running { + bytes: u64, + total: u64, + }, + /// Fetched and verified (its SHA-256 is `sha256`), waiting in the staging folder at `path` to be installed. + Done { + path: PathBuf, + sha256: String, + }, + /// Installed and in use: nothing more to do but tell the user. + Installed, + Failed(String), + Cancelled, +} + +struct Shared { + state: Mutex, + cancel: AtomicBool, +} + +impl Shared { + fn new(total: u64) -> Self { + Shared { state: Mutex::new(State::Running { bytes: 0, total }), cancel: AtomicBool::new(false) } + } + + fn set(&self, s: State) { + *self.state.lock().unwrap_or_else(PoisonError::into_inner) = s; + } + + fn get(&self) -> State { + self.state.lock().unwrap_or_else(PoisonError::into_inner).clone() + } +} + +/// The downloads of one session, by model id. +#[derive(Default)] +pub struct Downloads { + jobs: HashMap>, +} + +impl Downloads { + /// Start fetching `spec` into `/.downloads/`. Different models can download side by side; the same + /// one is started once. + pub fn start(&mut self, spec: Download, models_dir: &Path) -> Result<(), String> { + if self.jobs.get(&spec.id).is_some_and(|j| matches!(j.get(), State::Running { .. })) { + return Err("that model is already downloading".into()); + } + let staging = models_dir.join(STAGING); + std::fs::create_dir_all(&staging).map_err(|e| format!("could not create the download folder: {e}"))?; + let shared = Arc::new(Shared::new(spec.size_bytes)); + let worker = shared.clone(); + let id = spec.id.clone(); + std::thread::Builder::new() + .name("denoise-model-download".into()) + .spawn(move || { + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| run(&spec, &spec.url, &staging, &worker))); + if outcome.is_err() { + worker.set(State::Failed("the download stopped unexpectedly".into())); + } + }) + .map_err(|e| format!("could not start the download: {e}"))?; + self.jobs.insert(id, shared); + Ok(()) + } + + /// Every download this session has started and not discarded, by model id. + pub fn snapshot(&self) -> Vec<(String, State)> { + let mut all: Vec<_> = self.jobs.iter().map(|(id, j)| (id.clone(), j.get())).filter(|(_, s)| *s != State::Cancelled).collect(); + all.sort_by(|a, b| a.0.cmp(&b.0)); + all + } + + pub fn running(&self) -> bool { + self.jobs.values().any(|j| matches!(j.get(), State::Running { .. })) + } + + /// Stop a running download, or discard a finished one (its staged file is deleted). `true` if there was one. + pub fn discard(&mut self, id: &str) -> bool { + let Some(job) = self.jobs.get(id) else { return false }; + job.cancel.store(true, Ordering::Relaxed); + match job.get() { + // the thread notices, deletes the partial file and marks itself cancelled + State::Running { .. } => {} + State::Done { path, .. } => { + let _ = std::fs::remove_file(path); + self.jobs.remove(id); + } + State::Installed | State::Failed(_) | State::Cancelled => { + self.jobs.remove(id); + } + } + true + } + + /// The staged file at `path` has been installed (its copy is the model now) or is no longer wanted. + pub fn staged_file_used(&mut self, path: &Path) { + self.jobs.retain(|_, j| !matches!(j.get(), State::Done { path: p, .. } if p == path)); + } + + /// Downloads that have arrived and been checked, ready to install: (model id, staged file, its SHA-256). + pub fn finished(&self) -> Vec<(String, PathBuf, String)> { + let mut out: Vec<_> = self + .jobs + .iter() + .filter_map(|(id, j)| match j.get() { + State::Done { path, sha256 } => Some((id.clone(), path, sha256)), + _ => None, + }) + .collect(); + out.sort_by(|a, b| a.0.cmp(&b.0)); + out + } + + /// Record how installing a finished download went (`Installed` or `Failed`). + pub fn set_outcome(&mut self, id: &str, state: State) { + match self.jobs.get(id) { + Some(j) => j.set(state), + None => { + let j = Shared::new(0); + j.set(state); + self.jobs.insert(id.to_string(), Arc::new(j)); + } + } + } +} + +impl Drop for Downloads { + /// Quitting while a download runs stops it instead of leaving it behind. + fn drop(&mut self) { + for job in self.jobs.values() { + job.cancel.store(true, Ordering::Relaxed); + } + let started = std::time::Instant::now(); + while self.running() && started.elapsed() < Duration::from_secs(1) { + std::thread::sleep(Duration::from_millis(10)); + } + } +} + +/// What went wrong, in words the user can act on. +#[cfg(not(target_arch = "wasm32"))] +fn explain(e: &lightcraft_fetch::DownloadError) -> String { + use lightcraft_fetch::DownloadError; + match e { + DownloadError::AllMirrorsFailed { errors, .. } => { + let detail: String = errors.join("; ").chars().take(300).collect(); + format!("could not download the model ({detail}): check the internet connection, or get the file from its page and add it yourself") + } + other => other.to_string(), + } +} + +/// Fetch `spec` from its exact URL into `staging`, check it and leave it there. +#[cfg(not(target_arch = "wasm32"))] +fn fetch(spec: &Download, mirror: &str, staging: &Path, shared: &Shared) -> Result<(), lightcraft_fetch::DownloadError> { + let file = lightcraft_fetch::FileSpec { name: &spec.file_name, size: Some(spec.size_bytes), sha256: Some(&spec.sha256), max: spec.size_bytes }; + lightcraft_fetch::download_url(&file, mirror, staging, &lightcraft_fetch::Options::default(), &shared.cancel, &mut |p| { + shared.set(State::Running { bytes: p.done.min(spec.size_bytes), total: spec.size_bytes }); + }) +} + +#[cfg(target_arch = "wasm32")] +fn fetch(_: &Download, _: &str, _: &Path, _: &Shared) -> Result<(), String> { + Err("the web build downloads no models".into()) +} + +/// The whole job: reuse a verified earlier copy, else fetch, check and stage. Always ends in a final [`State`]. +fn run(spec: &Download, mirror: &str, staging: &Path, shared: &Shared) { + let target = staging.join(&spec.file_name); + let part = staging.join(format!("{}.part", spec.file_name)); + match fetch(spec, mirror, staging, shared) { + Ok(()) => shared.set(State::Done { path: target, sha256: spec.sha256.to_string() }), + #[cfg(not(target_arch = "wasm32"))] + Err(lightcraft_fetch::DownloadError::Cancelled) => { + let _ = std::fs::remove_file(&part); + shared.set(State::Cancelled); + } + #[cfg(not(target_arch = "wasm32"))] + Err(e) => { + shared.set(State::Failed(explain(&e))); + } + #[cfg(target_arch = "wasm32")] + Err(e) => { + let _ = std::fs::remove_file(&part); + shared.set(State::Failed(e)); + } + } +} + +#[cfg(all(test, not(target_arch = "wasm32")))] +mod tests { + use super::*; + use lightcraft_denoise::hash::sha256_hex; + use std::io::{BufRead, BufReader, Write}; + use std::net::TcpListener; + + fn scratch(name: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("lc-face-dl-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d + } + + fn spec_for(bytes: &[u8]) -> Download { + Download { + id: "test-model".into(), + url: "https://example.invalid/m/test-model.onnx".into(), + file_name: "test-model.onnx".into(), + size_bytes: bytes.len() as u64, + sha256: sha256_hex(bytes), + } + } + + /// A local server that answers every request with `status` and `body` (the whole file, no ranges). + fn serve(status: &'static str, body: Vec) -> String { + let l = TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}/m", l.local_addr().unwrap()); + std::thread::spawn(move || { + for mut s in l.incoming().flatten() { + let mut r = BufReader::new(s.try_clone().unwrap()); + loop { + let mut line = String::new(); + if r.read_line(&mut line).unwrap_or(0) == 0 || line.trim().is_empty() { + break; + } + } + let _ = s.write_all(format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()).as_bytes()); + let _ = s.write_all(&body); + } + }); + base + } + + #[test] + fn fetches_checks_and_stages_a_file() { + let dir = scratch("ok"); + let bytes: Vec = (0..200_000u32).map(|i| (i % 251) as u8).collect(); + let (staging, spec) = (dir.join(STAGING), spec_for(&bytes)); + std::fs::create_dir_all(&staging).unwrap(); + let mirror = serve("200 OK", bytes.clone()); + let shared = Shared::new(spec.size_bytes); + run(&spec, &mirror, &staging, &shared); + let State::Done { path, sha256 } = shared.get() else { panic!("{:?}", shared.get()) }; + assert_eq!((std::fs::read(&path).unwrap(), sha256), (bytes, spec.sha256.to_string())); + assert!(!staging.join("test-model.onnx.part").exists()); + // asked again with the copy still staged: no fetch needed (nothing listens there) + let again = Shared::new(spec.size_bytes); + run(&spec, "http://127.0.0.1:1/m", &staging, &again); + assert!(matches!(again.get(), State::Done { .. }), "{:?}", again.get()); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn a_file_that_does_not_match_is_thrown_away() { + let dir = scratch("bad"); + let staging = dir.join(STAGING); + std::fs::create_dir_all(&staging).unwrap(); + // the right size and the wrong content + let spec = spec_for(&[7u8; 5000]); + let shared = Shared::new(spec.size_bytes); + run(&spec, &serve("200 OK", vec![8u8; 5000]), &staging, &shared); + assert!(matches!(shared.get(), State::Failed(_)), "{:?}", shared.get()); + // nothing is left behind: not the file, not a partial one + assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn an_unreachable_or_refusing_server_is_a_message() { + let dir = scratch("down"); + let staging = dir.join(STAGING); + std::fs::create_dir_all(&staging).unwrap(); + let spec = spec_for(b"abc"); + for mirror in ["http://127.0.0.1:1/m".to_string(), serve("404 Not Found", b"not found".to_vec())] { + let shared = Shared::new(3); + run(&spec, &mirror, &staging, &shared); + let State::Failed(why) = shared.get() else { panic!("{:?}", shared.get()) }; + assert!(why.contains("internet connection"), "{why}"); + } + assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn a_cancelled_download_leaves_nothing() { + let dir = scratch("cancel"); + let staging = dir.join(STAGING); + std::fs::create_dir_all(&staging).unwrap(); + let spec = spec_for(b"abc"); + let shared = Shared::new(3); + shared.cancel.store(true, Ordering::Relaxed); + run(&spec, &serve("200 OK", b"abc".to_vec()), &staging, &shared); + assert_eq!(shared.get(), State::Cancelled); + assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn dropping_the_session_stops_a_running_download() { + let shared = Arc::new(Shared::new(10)); + // a stand-in for the download thread: notices the cancel flag the way the downloader does + let worker = shared.clone(); + let handle = std::thread::spawn(move || { + while !worker.cancel.load(Ordering::Relaxed) { + std::thread::sleep(Duration::from_millis(5)); + } + worker.set(State::Cancelled); + }); + let mut d = Downloads::default(); + d.jobs.insert("m".into(), shared.clone()); + assert!(d.running()); + let started = std::time::Instant::now(); + drop(d); + assert!(started.elapsed() < Duration::from_millis(900), "quitting must not wait for a transfer to finish"); + assert_eq!(shared.get(), State::Cancelled); + handle.join().unwrap(); + } + + #[test] + fn discarding_a_finished_download_deletes_its_file() { + let dir = scratch("discard"); + let file = dir.join("staged.onnx"); + std::fs::write(&file, b"x").unwrap(); + let mut d = Downloads::default(); + let shared = Arc::new(Shared::new(1)); + shared.set(State::Done { path: file.clone(), sha256: String::new() }); + d.jobs.insert("m".into(), shared); + assert_eq!(d.snapshot().len(), 1); + assert!(!d.running()); + assert!(d.discard("m")); + assert!(!file.exists()); + assert!(d.snapshot().is_empty()); + assert!(!d.discard("m"), "nothing left to discard"); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/engine/src/smart.rs b/crates/engine/src/smart.rs index 8deff7bd..a48fcd04 100644 --- a/crates/engine/src/smart.rs +++ b/crates/engine/src/smart.rs @@ -190,7 +190,7 @@ pub fn is_valid(path: &Path) -> bool { /// Load the proxy at `path`. pub fn load(path: &Path) -> Result { let b = std::fs::read(path).map_err(|e| format!("{}: {e}", path.display()))?; - decode(&b).map(|(image, camera_tone)| crate::media::DecodedSource { image: Arc::new(image), info: None, camera_tone }) + decode(&b).map(|(image, camera_tone)| crate::media::DecodedSource { image: Arc::new(image), info: None, camera_tone, denoised: None }) } #[cfg(test)] diff --git a/crates/engine/src/tests_denoise.rs b/crates/engine/src/tests_denoise.rs new file mode 100644 index 00000000..172f9082 --- /dev/null +++ b/crates/engine/src/tests_denoise.rs @@ -0,0 +1,699 @@ +//! AI denoise through the engine: installing a model, making a photo's denoised picture in the background and for an +//! export, the Amount slider mixing it in, and the cache. A tiny stand-in model (it halves every sample) takes the +//! place of a real one, which is enough to see what the machinery does with its answer; how well real models denoise is +//! measured separately (`crates/denoise/tests/real_model.rs`). + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use lightcraft_denoise::manifest::{DenoiserManifest, Domain, Gain}; +use lightcraft_denoise::run::TileRunner; +use serde_json::{Value, json}; + +use crate::Session; +use crate::denoise::{Loader, Model, PhotoState, RunOn}; +use crate::export::{ExportOptions, export_photo}; +use crate::tests_xmp::{synthetic_dng_of, temp_dir}; + +/// A model that makes every output pixel half of its cell's colour: a picture that is plainly not the plain one. +struct Dim { + runs: Arc, +} + +impl TileRunner for Dim { + fn run(&self, input: &[f32]) -> Result, lightcraft_denoise::Error> { + self.runs.fetch_add(1, Ordering::Relaxed); + let t = ((input.len() / 4) as f64).sqrt() as usize; + let side = 2 * t; + let mut out = vec![0f32; 3 * side * side]; + for y in 0..side { + for x in 0..side { + let i = (y / 2) * t + x / 2; + let (r, g, b) = (input[i], (input[t * t + i] + input[2 * t * t + i]) / 2.0, input[3 * t * t + i]); + out[y * side + x] = r * 0.5; + out[side * side + y * side + x] = g * 0.5; + out[2 * side * side + y * side + x] = b * 0.5; + } + } + Ok(out) + } +} + +impl Model for Dim { + fn runner(&self, _: RunOn, threads: usize) -> (&dyn TileRunner, usize) { + (self, threads) + } + + fn self_test(&self, _: RunOn) -> Result { + Ok(json!({"ok": true})) + } +} + +fn dim_loader(runs: &Arc) -> Loader { + let runs = runs.clone(); + Arc::new(move |_, _| Ok(Arc::new(Dim { runs: runs.clone() }) as Arc)) +} + +fn manifest(id: &str) -> DenoiserManifest { + DenoiserManifest { + id: id.into(), + name: "Stand-in".into(), + version: "1".into(), + licence: Default::default(), + source: None, + sha256: None, + size_bytes: None, + provenance: "made up for a test".into(), + domain: Domain::BayerToRgb, + tile: 64, + overlap: 16, + gain: Gain::None, + } +} + +/// The model's `.onnx` (not a real one: the stand-in loader never reads it) with its manifest beside it. +fn model_files(dir: &Path, id: &str) -> PathBuf { + let src = dir.join(format!("src-{id}")); + std::fs::create_dir_all(&src).unwrap(); + std::fs::write(src.join("denoise-model.json"), serde_json::to_vec(&manifest(id)).unwrap()).unwrap(); + std::fs::write(src.join("model.onnx"), format!("not an onnx file, only a stand-in for {id}")).unwrap(); + src.join("model.onnx") +} + +/// A Bayer DNG that is a smooth ramp, big enough for a 64-cell tile and a second one. +fn ramp_dng(w: usize, h: usize, cfa: &str) -> Vec { + let data: Vec = (0..w * h).map(|i| 600 + ((i % w) * 30 + (i / w) * 35) as u16).collect(); + synthetic_dng_of(w, h, cfa, data, None, Default::default()) +} + +struct Setup { + dir: PathBuf, + s: Session, + runs: Arc, +} + +/// A session with a library on disk holding one raw photo, and the stand-in model installed and chosen. +fn setup(tag: &str, with_model: bool) -> Setup { + let dir = temp_dir(tag); + let originals = dir.join("originals"); + std::fs::create_dir_all(&originals).unwrap(); + std::fs::write(originals.join("ramp.dng"), ramp_dng(192, 160, "RGGB")).unwrap(); + let runs = Arc::new(AtomicUsize::new(0)); + let mut s = Session::new().with_fs(); + s.open_library(dir.join("library"), false).unwrap(); + s.execute("library.import", &json!({"paths": [originals.join("ramp.dng").to_string_lossy()]})).unwrap(); + let id = s.catalog.photos().next().unwrap().id; + s.execute("library.select", &json!({"ids": [id.0]})).unwrap(); + s.set_denoise_models_dir(Some(dir.join("denoise-models"))); + s.denoise.loader = dim_loader(&runs); + if with_model { + let onnx = model_files(&dir, "stand-in"); + s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap(); + } + Setup { dir, s, runs } +} + +fn photo(s: &Session) -> lightcraft_catalog::PhotoId { + s.catalog.photos().next().unwrap().id +} + +/// Pump until `n` photos are ready (the job runs on its own thread). +fn pump_until_ready(s: &mut Session, n: u64) -> Value { + for _ in 0..600 { + let r = s.execute("denoise.pump", &json!({"pace": "full"})).unwrap(); + if r["ready"].as_u64() >= Some(n) { + return r; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + panic!("not ready: {}", s.execute("denoise.status", &json!({})).unwrap()); +} + +fn mean(img: &lightcraft_raster::Rgba8) -> f64 { + let sum: u64 = img.data.iter().map(|p| u64::from(p[0]) + u64::from(p[1]) + u64::from(p[2])).sum(); + sum as f64 / (img.data.len() * 3) as f64 +} + +fn rendered_mean(s: &mut Session, id: lightcraft_catalog::PhotoId) -> f64 { + mean(&s.render_now(id, 160, 160).unwrap().image) +} + +fn set_amount(s: &mut Session, amount: f64) { + s.execute("develop.set", &json!({"control": "enhance.denoise", "value": amount})).unwrap(); +} + +#[test] +fn a_model_is_installed_chosen_and_removed_with_the_terms_accepted() { + let mut x = setup("model", false); + let onnx = model_files(&x.dir, "stand-in"); + // the terms come first + let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy()})).unwrap_err().to_string(); + assert!(e.contains("licence has not been accepted"), "{e}"); + let r = x.s.execute("denoise.models.list", &json!({})).unwrap(); + assert_eq!(r["model"], Value::Null); + assert_eq!(r["runtime"], cfg!(feature = "denoise")); + // a file with no manifest is not guessed at + let stray = x.dir.join("stray.onnx"); + std::fs::write(&stray, b"who knows").unwrap(); + let e = x.s.execute("denoise.models.install", &json!({"path": stray.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string(); + assert!(e.contains("does not know this file"), "{e}"); + // installed, it is the model in use + let r = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap(); + assert_eq!((r["installed"]["id"].clone(), r["model"].clone()), (json!("stand-in"), json!("stand-in")), "{r}"); + assert!(r["installed"]["accepted"]["sha256"].as_str().is_some_and(|h| h.len() == 64), "the file's hash is recorded: {r}"); + assert!(x.s.denoise.active.is_some()); + assert_eq!(x.s.execute("denoise.status", &json!({})).unwrap()["model"]["id"], "stand-in"); + // it can be checked again, switched off and on, and removed + assert_eq!(x.s.execute("denoise.models.test", &json!({"id": "stand-in"})).unwrap()["ok"], true); + assert_eq!(x.s.execute("denoise.models.select", &json!({"id": null})).unwrap()["model"], Value::Null); + assert!(x.s.denoise.active.is_none()); + assert!(x.s.execute("denoise.models.select", &json!({"id": "nothing"})).is_err()); + x.s.execute("denoise.models.select", &json!({"id": "stand-in"})).unwrap(); + assert!(x.s.denoise.active.is_some()); + let r = x.s.execute("denoise.models.remove", &json!({"id": "stand-in"})).unwrap(); + assert_eq!(r["model"], Value::Null); + assert!(x.s.denoise.active.is_none()); + assert!(!x.dir.join("denoise-models").join("stand-in").exists()); + // ids that are not folder names, and folders that are not models, are never touched + for bad in ["../library", "", "a/b", "settings.json"] { + assert!(x.s.execute("denoise.models.remove", &json!({"id": bad})).is_err(), "{bad}"); + } + assert!(x.dir.join("library").is_dir()); + // a model that fails its self-test is not left installed + x.s.denoise.loader = Arc::new(|_, _| Err("it does not load".into())); + let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string(); + assert!(e.contains("it does not load"), "{e}"); + assert!(!x.dir.join("denoise-models").join("stand-in").exists()); + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn a_manifest_that_does_not_validate_or_name_the_file_is_refused() { + let mut x = setup("manifest", false); + let onnx = model_files(&x.dir, "stand-in"); + let json_path = onnx.with_file_name("denoise-model.json"); + // a tile the model cannot run on + let mut m = manifest("stand-in"); + m.tile = 50; + std::fs::write(&json_path, serde_json::to_vec(&m).unwrap()).unwrap(); + let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string(); + assert!(e.contains("tile"), "{e}"); + // a hash that is not the file's + let mut m = manifest("stand-in"); + m.sha256 = Some("0".repeat(64)); + std::fs::write(&json_path, serde_json::to_vec(&m).unwrap()).unwrap(); + let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string(); + assert!(e.contains("SHA-256"), "{e}"); + assert!(!x.dir.join("denoise-models").join("stand-in").join("model.onnx").exists(), "nothing half-installed is left"); + // an id that would leave the folder + let mut m = manifest("../escape"); + m.id = "../escape".into(); + std::fs::write(&json_path, serde_json::to_vec(&m).unwrap()).unwrap(); + assert!(x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).is_err()); + assert!(!x.dir.join("escape").exists()); + // garbage instead of a manifest + std::fs::write(&json_path, b"{ not json").unwrap(); + assert!(x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).is_err()); + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn a_photo_is_denoised_in_the_background_and_the_amount_mixes_it_in() { + let mut x = setup("background", true); + let id = photo(&x.s); + // nothing is made until a photo has an amount + let r = x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap(); + assert_eq!((r["queued"].clone(), r["running"].clone(), r["ready"].clone()), (json!(0), Value::Null, json!(0)), "{r}"); + assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle); + let plain = rendered_mean(&mut x.s, id); + let key_before = x.s.render_job(id, 160, 160, false, true).unwrap().key; + + set_amount(&mut x.s, 100.0); + // the photo being edited is picked up by itself, and made on another thread + let r = pump_until_ready(&mut x.s, 1); + assert_eq!(r["queued"], 0, "{r}"); + assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready); + assert!(x.runs.load(Ordering::Relaxed) >= 4, "a 192 × 160 mosaic is four 64-cell tiles: {}", x.runs.load(Ordering::Relaxed)); + let products: Vec<_> = std::fs::read_dir(x.dir.join("library").join("denoise")).unwrap().flatten().collect(); + assert_eq!(products.len(), 1, "one cached picture, no DNG or anything else in the library"); + assert!(products[0].path().extension().is_some_and(|e| e == "lcdn")); + let originals: Vec<_> = std::fs::read_dir(x.dir.join("originals")).unwrap().flatten().collect(); + assert_eq!(originals.len(), 1, "the original is untouched and alone"); + + // the render now comes from the denoised picture: darker at 100, in between at 50, the plain one at 0 + let full = rendered_mean(&mut x.s, id); + assert!(full < plain - 3.0, "denoised {full} vs plain {plain}"); + set_amount(&mut x.s, 50.0); + let half = rendered_mean(&mut x.s, id); + assert!(half < plain - 1.0 && half > full + 1.0, "{full} < {half} < {plain}"); + set_amount(&mut x.s, 0.0); + assert!((rendered_mean(&mut x.s, id) - plain).abs() < 0.01, "amount 0 is the plain picture again"); + // …and a render that uses it has another identity than one that does not + assert_ne!(key_before, { + set_amount(&mut x.s, 100.0); + x.s.render_job(id, 160, 160, false, true).unwrap().key + }); + // switching the Detail section off switches the denoise off with it + set_amount(&mut x.s, 100.0); + x.s.execute("develop.sectionEnabled", &json!({"section": "detail", "enabled": false})).unwrap(); + assert!((rendered_mean(&mut x.s, id) - plain).abs() < 0.01); + assert_eq!(x.s.media.denoise_salt(id, &x.s.catalog.photo(id).unwrap().develop), 0); + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn what_was_made_is_found_again_by_a_new_session_and_a_new_model_starts_over() { + let mut x = setup("again", true); + let id = photo(&x.s); + set_amount(&mut x.s, 100.0); + pump_until_ready(&mut x.s, 1); + let made = x.runs.load(Ordering::Relaxed); + x.s.close_library().unwrap(); + drop(x.s); + + // reopened: ready without running the model again + let mut s2 = Session::new().with_fs(); + s2.set_denoise_models_dir(Some(x.dir.join("denoise-models"))); + s2.denoise.loader = dim_loader(&x.runs); + s2.open_library(x.dir.join("library"), false).unwrap(); + s2.denoise_pump(crate::denoise::Pace::Full); + assert_eq!(s2.denoise_photo_state(id), PhotoState::Ready); + assert_eq!(x.runs.load(Ordering::Relaxed), made, "nothing was made again"); + + // another model's pictures are different pictures + let onnx = model_files(&x.dir, "other"); + s2.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap(); + s2.denoise_pump(crate::denoise::Pace::Pause); + assert!( + matches!(s2.denoise_photo_state(id), PhotoState::Queued { .. }), + "the first model's picture is not this model's: it is to be made: {:?}", + s2.denoise_photo_state(id) + ); + // and choosing the first again finds its picture where it was + s2.execute("denoise.models.select", &json!({"id": "stand-in"})).unwrap(); + s2.denoise_pump(crate::denoise::Pace::Pause); + assert_eq!(s2.denoise_photo_state(id), PhotoState::Ready); + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn an_export_makes_its_picture_when_the_queue_has_not_and_never_comes_out_without_it() { + let mut x = setup("export", true); + let id = photo(&x.s); + let o = ExportOptions::from_json(&json!({"format": "png", "width": 160, "height": 160})); + let decode = |bytes: &[u8]| { + let d = lightcraft_codecs::decode(bytes, lightcraft_codecs::DecodeOptions::fit(160, 160)).unwrap(); + let w = d.to_working(); + w.data.iter().map(|p| f64::from(p[0] + p[1] + p[2])).sum::() / (w.data.len() * 3) as f64 + }; + let plain = decode(&export_photo(&mut x.s, id, &o, 1).unwrap().bytes); + assert_eq!(x.runs.load(Ordering::Relaxed), 0, "no amount, no denoise"); + set_amount(&mut x.s, 100.0); + // no pump: the export makes the picture itself + let denoised = decode(&export_photo(&mut x.s, id, &o, 1).unwrap().bytes); + assert!(x.runs.load(Ordering::Relaxed) >= 4); + assert!(denoised < plain * 0.9, "denoised {denoised} vs plain {plain}"); + let products = std::fs::read_dir(x.dir.join("library").join("denoise")).unwrap().flatten().count(); + assert_eq!(products, 1); + // a second export finds it + let runs = x.runs.load(Ordering::Relaxed); + let again = decode(&export_photo(&mut x.s, id, &o, 1).unwrap().bytes); + assert_eq!(x.runs.load(Ordering::Relaxed), runs); + assert!((again - denoised).abs() < 1e-6); + // the pump takes in what the export made + x.s.denoise_pump(crate::denoise::Pace::Pause); + assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready); + + // no model: the export says so instead of leaving the denoise out + x.s.execute("denoise.models.select", &json!({"id": null})).unwrap(); + let e = export_photo(&mut x.s, id, &o, 1).err().unwrap(); + assert!(e.contains("no denoise model is chosen") && e.contains("ramp.dng"), "{e}"); + set_amount(&mut x.s, 0.0); + assert!(export_photo(&mut x.s, id, &o, 1).is_ok(), "without an amount nothing is asked of the model"); + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn a_model_that_fails_fails_the_export_and_is_not_retried_by_the_queue() { + let mut x = setup("fails", true); + let id = photo(&x.s); + x.s.denoise.loader = Arc::new(|_, _| Err("the model went away".into())); + // (the model chosen is read afresh: it was loaded by the install's self-test only) + set_amount(&mut x.s, 80.0); + let o = ExportOptions::from_json(&json!({"format": "png", "width": 100, "height": 100})); + let e = export_photo(&mut x.s, id, &o, 1).err().unwrap(); + assert!(e.contains("ramp.dng") && e.contains("the model went away"), "{e}"); + // the background job fails too, says why once and does not try again every frame + for _ in 0..200 { + x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap(); + if matches!(x.s.denoise_photo_state(id), PhotoState::Failed { .. }) { + break; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + match x.s.denoise_photo_state(id) { + PhotoState::Failed { why, unsupported } => assert!(why.contains("the model went away") && !unsupported, "{why}"), + other => panic!("{other:?}"), + } + let r = x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap(); + assert_eq!((r["queued"].clone(), r["running"].clone()), (json!(0), Value::Null), "{r}"); + // asking again by hand does retry + x.s.denoise.loader = dim_loader(&x.runs); + let r = x.s.execute("denoise.queue", &json!({"retry": true})).unwrap(); + assert_eq!(r["added"], 1, "{r}"); + pump_until_ready(&mut x.s, 1); + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn a_corrupt_cache_with_a_current_header_never_silently_exports_the_plain_photo() { + let mut x = setup("corrupt-export", true); + let id = photo(&x.s); + set_amount(&mut x.s, 100.0); + let options = ExportOptions::from_json(&json!({"format": "png", "width": 160, "height": 160})); + export_photo(&mut x.s, id, &options, 1).unwrap(); + x.s.denoise_pump(crate::denoise::Pace::Pause); + let spec = x.s.media.denoise.spec(id).unwrap().clone(); + let mut bytes = std::fs::read(&spec.product).unwrap(); + for byte in bytes.iter_mut().rev().take(64) { + *byte ^= 0xff; + } + std::fs::write(&spec.product, bytes).unwrap(); + assert!(lightcraft_denoise::product::is_current(&spec.product, &spec.key), "header/length still look current"); + x.s.media.forget(id); + let error = export_photo(&mut x.s, id, &options, 1).err().unwrap(); + assert!(error.contains("AI Denoise picture could not be read"), "{error}"); + let _ = std::fs::remove_dir_all(x.dir); +} + +#[test] +fn only_raw_photos_on_disk_are_denoised_and_nothing_is_queued_without_a_model() { + let dir = temp_dir("eligible"); + let mut demo = Session::with_demo(); + demo.set_denoise_models_dir(Some(dir.join("models"))); + let id = demo.active().unwrap(); + assert_eq!(demo.denoise_photo_state(id), PhotoState::NotApplicable, "a generated photo has no sensor data"); + let e = demo.execute("denoise.queue", &json!({})).unwrap_err().to_string(); + assert!(e.contains("no denoise model is chosen"), "{e}"); + // no models folder at all (the web build): the commands say so, nothing panics + let mut web = Session::with_demo(); + for cmd in ["denoise.models.install", "denoise.models.select", "denoise.models.remove", "denoise.settings"] { + assert!(web.execute(cmd, &json!({"path": "x", "acknowledged": true, "id": "x"})).is_err(), "{cmd}"); + } + assert_eq!(web.execute("denoise.pump", &json!({})).unwrap()["active"], false); + assert_eq!(web.execute("denoise.status", &json!({})).unwrap()["enabled"], false); + // no library open: nowhere to keep pictures, and the queue says so + let x = setup("eligible2", true); + let mut bare = Session::new().with_fs(); + bare.set_denoise_models_dir(Some(x.dir.join("denoise-models"))); + assert!(bare.denoise_products_dir().is_none()); + let e = bare.execute("denoise.queue", &json!({})).unwrap_err().to_string(); + assert!(e.contains("open a library on disk first"), "{e}"); + assert_eq!(bare.execute("denoise.pump", &json!({})).unwrap()["queued"], 0); + let _ = std::fs::remove_dir_all(&dir); + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn files_that_cannot_be_denoised_fail_with_a_reason_and_never_a_panic() { + let dir = temp_dir("makefails"); + let spec = |path: &Path| crate::denoise::JobSpec { + path: path.to_string_lossy().into_owned(), + read: None, + product: dir.join("p.lcdn"), + key: "k".into(), + manifest: manifest("stand-in"), + onnx: dir.join("model.onnx"), + model: Default::default(), + loader: dim_loader(&Arc::new(AtomicUsize::new(0))), + parallel: 2, + run_on: RunOn::Auto, + }; + use crate::denoise::{MakeError, make_product}; + // not a file + assert!(matches!(make_product(&spec(&dir.join("missing.dng")), None), Err(MakeError::Failed(_)))); + // not a raw + let junk = dir.join("junk.dng"); + std::fs::write(&junk, b"this is not a raw file at all").unwrap(); + assert!(matches!(make_product(&spec(&junk), None), Err(MakeError::Unsupported(_)))); + // a raw cut short + let whole = ramp_dng(192, 160, "RGGB"); + let cut = dir.join("cut.dng"); + std::fs::write(&cut, &whole[..whole.len() / 3]).unwrap(); + assert!(make_product(&spec(&cut), None).is_err()); + // a model that gives the wrong number of values is an error from the model, not a crash + struct Wrong; + impl TileRunner for Wrong { + fn run(&self, _: &[f32]) -> Result, lightcraft_denoise::Error> { + Ok(vec![0.0; 7]) + } + } + impl Model for Wrong { + fn runner(&self, _: RunOn, threads: usize) -> (&dyn TileRunner, usize) { + (self, threads) + } + fn self_test(&self, _: RunOn) -> Result { + Err("no".into()) + } + } + let good = dir.join("good.dng"); + std::fs::write(&good, &whole).unwrap(); + let mut s = spec(&good); + s.loader = Arc::new(|_, _| Ok(Arc::new(Wrong) as Arc)); + match make_product(&s, None) { + Err(MakeError::Failed(why)) => assert!(why.contains("7 numbers"), "{why}"), + other => panic!("{other:?}"), + } + assert!(!dir.join("p.lcdn").exists(), "a failed job leaves no product"); + // a job told to stop stops before doing anything + let progress = crate::denoise::Progress::default(); + progress.cancel_now(); + assert_eq!(make_product(&spec(&good), Some(&progress)), Err(MakeError::Cancelled)); + // a Bayer layout of the other kinds is denoised too, and a product that exists is not made again + for cfa in ["BGGR", "GRBG", "GBRG"] { + let f = dir.join(format!("{cfa}.dng")); + std::fs::write(&f, ramp_dng(192, 160, cfa)).unwrap(); + let mut s = spec(&f); + s.product = dir.join(format!("{cfa}.lcdn")); + assert_eq!(make_product(&s, None), Ok(true), "{cfa}"); + assert_eq!(make_product(&s, None), Ok(false), "{cfa} is already there"); + } + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn two_threads_making_the_same_picture_make_it_once() { + let dir = temp_dir("twice"); + let f = dir.join("a.dng"); + std::fs::write(&f, ramp_dng(192, 160, "RGGB")).unwrap(); + let runs = Arc::new(AtomicUsize::new(0)); + let spec = crate::denoise::JobSpec { + path: f.to_string_lossy().into_owned(), + read: None, + product: dir.join("p.lcdn"), + key: "k".into(), + manifest: manifest("stand-in"), + onnx: dir.join("model.onnx"), + model: Default::default(), + loader: dim_loader(&runs), + parallel: 2, + run_on: RunOn::Auto, + }; + let made: Vec = std::thread::scope(|sc| { + let hs: Vec<_> = (0..4).map(|_| sc.spawn(|| crate::denoise::make_product(&spec, None).unwrap())).collect(); + hs.into_iter().map(|h| h.join().unwrap()).collect() + }); + assert_eq!(made.iter().filter(|m| **m).count(), 1, "{made:?}"); + assert_eq!(runs.load(Ordering::Relaxed), 4, "one photo's four tiles, once"); + assert!(lightcraft_denoise::product::is_current(&spec.product, "k")); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn the_cache_is_kept_within_its_limit_and_what_photos_use_goes_last() { + let mut x = setup("evict", true); + let id = photo(&x.s); + set_amount(&mut x.s, 100.0); + pump_until_ready(&mut x.s, 1); + let used = x.dir.join("library").join("denoise"); + // two older pictures nobody uses + for (i, name) in ["aaaa", "bbbb"].iter().enumerate() { + let f = used.join(format!("{name}.lcdn")); + std::fs::write(&f, vec![7u8; 1000]).unwrap(); + let t = std::time::SystemTime::now() - std::time::Duration::from_secs(1000 - i as u64); + std::fs::File::options().write(true).open(&f).unwrap().set_modified(t).unwrap(); + } + let size_of = |p: &Path| std::fs::metadata(p).unwrap().len(); + let mine: PathBuf = std::fs::read_dir(&used).unwrap().flatten().map(|e| e.path()).find(|p| p.file_stem().is_some_and(|s| s.len() == 32)).unwrap(); + // room for the photo's own picture and one stray: the oldest stray goes, then the other + x.s.denoise_evict_to(size_of(&mine) + 1500, None); + assert!(!used.join("aaaa.lcdn").exists() && used.join("bbbb.lcdn").exists() && mine.exists()); + x.s.denoise_evict_to(size_of(&mine) + 10, None); + assert!(!used.join("bbbb.lcdn").exists() && mine.exists()); + assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready); + // too small for even that: the photo's own picture is the last thing to go, and the photo knows + x.s.denoise_evict_to(10, None); + assert!(!mine.exists()); + assert_ne!(x.s.denoise_photo_state(id), PhotoState::Ready); + // …and it is made again, because the photo still has its amount + pump_until_ready(&mut x.s, 1); + assert!(mine.exists()); + // clearing deletes it all + let r = x.s.execute("denoise.clear", &json!({})).unwrap(); + assert_eq!(r["deleted"], 1, "{r}"); + assert_eq!(std::fs::read_dir(&used).unwrap().flatten().count(), 0); + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn settings_are_checked_saved_and_a_paused_pump_starts_nothing() { + let mut x = setup("settings", true); + let r = x.s.execute("denoise.settings", &json!({"auto": false, "cacheGb": 5, "threads": 3})).unwrap(); + assert_eq!((r["auto"].clone(), r["cacheGb"].clone(), r["threads"].clone()), (json!(false), json!(5), json!(3)), "{r}"); + for bad in [ + json!({"cacheGb": 0}), + json!({"cacheGb": "big"}), + json!({"threads": 0}), + json!({"threads": 1000}), + json!({"auto": "yes"}), + json!({"runOn": "fastest"}), + json!({"runOn": true}), + ] { + assert!(x.s.execute("denoise.settings", &bad).is_err(), "{bad}"); + } + assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["cacheGb"], 5, "a refused change changes nothing"); + // the faster of the card and the processor unless the user says otherwise, and the interface can read the choice back + assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["runOn"], "auto"); + assert_eq!(x.s.execute("denoise.settings", &json!({"runOn": "cpu"})).unwrap()["runOn"], "cpu"); + assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["runOn"], "cpu"); + let status = x.s.execute("denoise.status", &json!({})).unwrap(); + assert_eq!((status["runOn"].clone(), status["device"]["kind"].clone()), (json!("cpu"), json!("none")), "no model is loaded yet: {status}"); + // the setting before `runOn` still reads as what it meant + let models = x.s.denoise.models_dir.clone().unwrap(); + let mut st = crate::denoise::read_settings(&models); + (st.run_on, st.gpu) = (None, Some(false)); + crate::denoise::write_settings(&models, &st).unwrap(); + assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["runOn"], "cpu"); + // choosing where it runs lets the card be tried again after a set-up that closed LightCraft + let onnx = crate::denoise::installed_models(&models).first().unwrap().onnx.clone(); + let marker = onnx.with_file_name(crate::denoise::GPU_SETUP_MARKER); + std::fs::write(&marker, "setting up").unwrap(); + assert_eq!(x.s.execute("denoise.settings", &json!({"runOn": "auto"})).unwrap()["runOn"], "auto"); + assert!(!marker.exists(), "the crash is forgotten"); + assert_eq!(crate::denoise::read_settings(&models).gpu, None, "the old setting is not kept beside the new one"); + // with auto off nothing is started by itself, and the photo is made when asked + let id = photo(&x.s); + set_amount(&mut x.s, 100.0); + for _ in 0..3 { + x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap(); + } + assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle); + x.s.execute("denoise.settings", &json!({"auto": true})).unwrap(); + // paused: it is queued but nothing starts + for _ in 0..3 { + let r = x.s.execute("denoise.pump", &json!({"pace": "pause"})).unwrap(); + assert_eq!(r["running"], Value::Null); + } + assert!(matches!(x.s.denoise_photo_state(id), PhotoState::Queued { .. }), "{:?}", x.s.denoise_photo_state(id)); + // …and cancelling forgets it + assert_eq!(x.s.execute("denoise.cancel", &json!({})).unwrap()["cancelled"], 1); + assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle); + // when the pace allows, it is made + x.s.execute("denoise.queue", &json!({"scope": "withAmount"})).unwrap(); + pump_until_ready(&mut x.s, 1); + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn a_photo_whose_file_changed_has_no_picture_until_it_is_made_again() { + let mut x = setup("changed", true); + let id = photo(&x.s); + set_amount(&mut x.s, 100.0); + pump_until_ready(&mut x.s, 1); + // the photo's content is something else now (a replaced file): its old picture is not its picture + let p = x.s.catalog.photo(id).unwrap().clone(); + x.s.commit( + "test", + lightcraft_catalog::Op::SetContent { + id, + width: p.width, + height: p.height, + file_size: p.file_size + 1, + content_hash: Some("ffffffffffffffffffffffffffffffff".into()), + preview_only: None, + }, + ) + .unwrap(); + x.s.denoise_pump(crate::denoise::Pace::Pause); + assert_ne!(x.s.denoise_photo_state(id), PhotoState::Ready); + // the old picture is not used by a render either + assert_eq!(x.s.media.denoise_salt(id, &x.s.catalog.photo(id).unwrap().develop), 0); + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn a_photo_waits_for_its_turn_unless_pictures_are_not_made_on_their_own_or_have_nowhere_to_go() { + let mut x = setup("waits", true); + let id = photo(&x.s); + set_amount(&mut x.s, 80.0); + // automatic off: the open photo is left alone until it is asked for + x.s.execute("denoise.settings", &json!({"auto": false})).unwrap(); + assert!(!x.s.denoise_auto()); + x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap(); + assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle); + assert!(!x.s.denoise_busy()); + x.s.execute("denoise.queue", &json!({"ids": [id.0]})).unwrap(); + assert!(x.s.denoise_busy(), "queued with a model: the headless runs wait for it"); + pump_until_ready(&mut x.s, 1); + assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready); + assert!(!x.s.denoise_busy()); + // a session whose library is not on disk has nowhere to keep it: the photo says so instead of waiting for ever + x.s.denoise.settings.auto = Some(true); + if let Some(l) = x.s.library.as_mut() { + l.on_disk = false; + } + match x.s.denoise_photo_state(id) { + PhotoState::Failed { why, unsupported } => assert!(unsupported && why.contains("library folder"), "{why}"), + other => panic!("{other:?}"), + } + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn the_denoise_switch_validates_arguments_and_accepts_explicit_targets_without_selection() { + let mut x = setup("toggle", false); + let id = photo(&x.s); + for params in [json!({"id": "bad"}), json!({"id": u64::MAX}), json!({"id": id.0, "enabled": "yes"})] { + assert!(x.s.execute("denoise.toggle", ¶ms).is_err()); + } + assert!(x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).is_err()); + assert_eq!(x.s.develop_of(id).unwrap().enhance.denoise, 0.0); + let model = model_files(&x.dir, "toggle-model"); + x.s.execute("denoise.models.install", &json!({"path": model.to_string_lossy(), "acknowledged": true})).unwrap(); + x.s.selection = Default::default(); + let undo = x.s.undo.len(); + let r = x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap(); + assert_eq!(r["amount"], 50.0); + assert_eq!(x.s.undo.len(), undo + 1); + x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": false})).unwrap(); + assert_eq!(x.s.develop_of(id).unwrap().denoise_amount(), 0.0); + // the switch is its own setting: flipping it keeps the Amount (so it compares with and without), and an Amount of 0 + // does not switch it off + x.s.execute("library.select", &json!({"ids": [id.0]})).unwrap(); + x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap(); + x.s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 80.0})).unwrap(); + let off = x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": false})).unwrap(); + assert_eq!(off["amount"], 80.0); + let d = x.s.develop_of(id).unwrap(); + assert!(!d.enhance.denoise_enabled() && d.denoise_amount() == 0.0); + assert_eq!(x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap()["amount"], 80.0); + assert!((x.s.develop_of(id).unwrap().denoise_amount() - 0.8).abs() < 1e-6); + x.s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 0.0})).unwrap(); + assert!(x.s.develop_of(id).unwrap().enhance.denoise_enabled(), "sliding to 0 leaves it on"); + assert_eq!(x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap()["amount"], 0.0); + // with no `enabled`, the toggle flips what the switch says + assert_eq!(x.s.execute("denoise.toggle", &json!({"id": id.0})).unwrap()["enabled"], false); + drop(x.s); + let _ = std::fs::remove_dir_all(x.dir); +} diff --git a/crates/engine/src/tests_xmp.rs b/crates/engine/src/tests_xmp.rs index fede1fec..ab801e23 100644 --- a/crates/engine/src/tests_xmp.rs +++ b/crates/engine/src/tests_xmp.rs @@ -335,10 +335,15 @@ fn synthetic_dng(xmp: &str) -> Vec { /// A tiny Bayer DNG written by our own DNG writer (optional embedded XMP, camera metadata). pub(crate) fn synthetic_dng_with(xmp: Option<&str>, metadata: lightcraft_meta::Metadata) -> Vec { - use lightcraft_raw::*; let (w, h) = (32usize, 24usize); - let cfa = Cfa::bayer("RGGB").unwrap(); let data: Vec = (0..w * h).map(|i| 256 + ((i % w) * 300 + (i / w) * 200) as u16).collect(); + synthetic_dng_of(w, h, "RGGB", data, xmp, metadata) +} + +/// A synthetic Bayer DNG for denoise and cache tests. +pub(crate) fn synthetic_dng_of(w: usize, h: usize, cfa: &str, data: Vec, xmp: Option<&str>, metadata: lightcraft_meta::Metadata) -> Vec { + use lightcraft_raw::*; + let cfa = Cfa::bayer(cfa).unwrap(); let raw = RawImage { format: RawFormat::Dng, width: w, diff --git a/crates/engine/tests/denoise_cpu.rs b/crates/engine/tests/denoise_cpu.rs new file mode 100644 index 00000000..d01b7c5a --- /dev/null +++ b/crates/engine/tests/denoise_cpu.rs @@ -0,0 +1,101 @@ +//! Opt-in same-machine denoise model-stage benchmark on one public Bayer raw. +//! LC_DENOISE_MODEL= LC_DENOISE_RAW= cargo test --release +//! -p lightcraft-engine --features denoise --test denoise_cpu -- --ignored --nocapture +//! LC_DENOISE_ONLY selects a file (default arw-sony-a7m3-compressed.arw). +//! The whole model stage includes packing, scale/clip protection, blending and normalization; +//! file decode, model loading and GPU setup are outside the timer. It uses the application's tile pool convention. +#![cfg(feature = "denoise")] + +use lightcraft_denoise::{ + TileRunner, + bayer::Layout, + run::{Control, Params, denoise_bayer}, + runtime::{CpuRunner, check_tile}, +}; +use lightcraft_raster::Rgb32f; +use std::{path::Path, time::Instant}; + +fn agree(a: &[f32], b: &[f32]) -> f32 { + assert_eq!(a.len(), b.len()); + assert!(a.iter().chain(b).all(|v| v.is_finite())); + let scale = b.iter().fold(1e-6f32, |m, v| m.max(v.abs())); + let error = a.iter().zip(b).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale; + assert!(error <= 1e-3, "maximum relative error {error:e}"); + error +} +fn tile_time(name: &str, r: &dyn TileRunner, x: &[f32]) { + r.run(x).unwrap(); + let mut times: Vec<_> = (0..5) + .map(|_| { + let t = Instant::now(); + r.run(x).unwrap(); + t.elapsed().as_secs_f64() * 1000.0 + }) + .collect(); + times.sort_by(f64::total_cmp); + println!("{name}, one 512-cell tile, one thread: {:.1} ms median ({:.1}..{:.1})", times[2], times[0], times[4]); +} +fn photo_time(name: &str, r: &dyn TileRunner, raw: &lightcraft_raw::Normalized, layout: Layout, p: &Params) -> Rgb32f { + let pool = rayon::ThreadPoolBuilder::new().num_threads(p.parallel).build().unwrap(); + let run = || pool.install(|| denoise_bayer(&raw.data, raw.width, raw.height, layout, r, p, &Control::default())).unwrap(); + drop(run()); // warm the workers and their workspace cache + let mut times = Vec::new(); + let mut answer = Rgb32f { width: 0, height: 0, data: Vec::new() }; + for i in 0..3 { + let t = Instant::now(); + answer = run(); + let secs = t.elapsed().as_secs_f64(); + times.push(secs); + println!("{name}, whole model stage, {} workers, run {i}: {secs:.3} s", p.parallel); + } + times.sort_by(f64::total_cmp); + println!("{name}, whole model stage median {:.3} s", times[1]); + answer +} +#[test] +#[ignore = "needs LC_DENOISE_MODEL and LC_DENOISE_RAW; release timing and real-model accuracy"] +fn denoise_stage_compares_cpu_and_gpu() { + let model = std::env::var_os("LC_DENOISE_MODEL").expect("set LC_DENOISE_MODEL"); + let folder = std::env::var_os("LC_DENOISE_RAW").expect("set LC_DENOISE_RAW to public CC0 corpus/raw"); + let name = std::env::var("LC_DENOISE_ONLY").unwrap_or_else(|_| "arw-sony-a7m3-compressed.arw".into()); + let bytes = std::fs::read(Path::new(&folder).join(&name)).unwrap(); + let raw = lightcraft_raw::decode(&bytes).unwrap().normalized().unwrap(); + let cfa = raw.cfa.as_ref().unwrap(); + assert!(cfa.is_bayer()); + let layout = Layout::from_cell([cfa.color_at(0, 0), cfa.color_at(1, 0), cfa.color_at(0, 1), cfa.color_at(1, 1)]).unwrap(); + let m = lightcraft_denoise::known::find("rawnind-bayer").unwrap().manifest; + let cpu = CpuRunner::load(Path::new(&model), &m).unwrap(); + let threads = std::env::var("LC_DENOISE_PARALLEL") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or_else(|| std::thread::available_parallelism().map_or(1, |n| n.get()).clamp(1, 16)); + let p = Params { tile: 512, overlap: 64, gain: m.gain, clip: Some(1.0), parallel: threads }; + println!( + "{name}: {}x{}, {:.2} MP, {} tiles, {} CPU workers, {:.1} MB/workspace, {:.1} GMAC/tile", + raw.width, + raw.height, + raw.width as f64 * raw.height as f64 / 1e6, + lightcraft_denoise::run::tile_count(raw.width, raw.height, layout, 512, 64), + threads, + cpu.bytes_per_tile() as f64 / 1e6, + cpu.net().macs(512) as f64 / 1e9 + ); + let x = check_tile(512); + let want = cpu.run(&x).unwrap(); + tile_time("pure Rust", &cpu, &x); + let done = photo_time("pure Rust", &cpu, &raw, layout, &p); + + match lightcraft_gpu::nn::runner(cpu.net(), 512) { + Ok(gpu) => { + println!("GPU: {}", gpu.adapter()); + println!("CPU/check tile vs GPU: {:e}", agree(&want, &gpu.run(&x).unwrap())); + tile_time("GPU", &gpu, &x); + let gp = Params { parallel: 4, ..p }; + let answer = photo_time("GPU", &gpu, &raw, layout, &gp); + let a: Vec<_> = done.data.iter().flatten().copied().collect(); + let b: Vec<_> = answer.data.iter().flatten().copied().collect(); + println!("CPU/whole photo vs GPU: {:e}", agree(&a, &b)); + } + Err(e) => println!("GPU unavailable: {e}"), + } +} diff --git a/crates/engine/tests/denoise_real.rs b/crates/engine/tests/denoise_real.rs new file mode 100644 index 00000000..8d858175 --- /dev/null +++ b/crates/engine/tests/denoise_real.rs @@ -0,0 +1,117 @@ +//! Opt-in end-to-end check of AI denoise with the real RawNIND Bayer model, through the engine: install it, make a real +//! raw photo's picture in the background, render and export it with the Amount slider, and see what a file that is not +//! Bayer says. +//! +//! ```text +//! LC_DENOISE_MODEL= LC_DENOISE_RAW= \ +//! cargo test --release -p lightcraft-engine --features denoise --test denoise_real -- --ignored --nocapture +//! ``` +//! +//! `LC_DENOISE_RAW` is a folder of CC0 raw files (`cargo xtask corpus --download` → `corpus/raw`); every file in it is +//! tried. Optional: `LC_DENOISE_ONLY` (a file name to try alone), `LC_DENOISE_RUN_ON=cpu|gpu|auto` (where the model runs; `cpu` to compare with the card). +#![cfg(feature = "denoise")] + +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use lightcraft_engine::Session; +use serde_json::{Value, json}; + +fn temp(tag: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("lc-denoise-real-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +/// Mean absolute second difference of the luma: the fine detail and the noise of a picture. +fn roughness(img: &lightcraft_raster::Rgba8) -> f64 { + let (w, h) = (img.width, img.height); + let luma = |x: usize, y: usize| { + let p = img.data[y * w + x]; + 0.299 * f64::from(p[0]) + 0.587 * f64::from(p[1]) + 0.114 * f64::from(p[2]) + }; + let (mut sum, mut n) = (0.0, 0usize); + for y in 1..h - 1 { + for x in 1..w - 1 { + sum += (4.0 * luma(x, y) - luma(x - 1, y) - luma(x + 1, y) - luma(x, y - 1) - luma(x, y + 1)).abs(); + n += 1; + } + } + sum / n.max(1) as f64 +} + +fn files(dir: &Path) -> Vec { + let only = std::env::var("LC_DENOISE_ONLY").ok(); + let mut v: Vec = std::fs::read_dir(dir) + .unwrap() + .flatten() + .map(|e| e.path()) + .filter(|p| p.is_file() && only.as_deref().is_none_or(|o| p.file_name().is_some_and(|n| n.to_string_lossy() == o))) + .collect(); + v.sort(); + v +} + +#[test] +#[ignore = "needs the real model and raw files (see the file's header)"] +fn the_real_model_denoises_real_raws_through_the_engine() { + let (Some(model), Some(raws)) = (std::env::var_os("LC_DENOISE_MODEL"), std::env::var_os("LC_DENOISE_RAW")) else { + panic!("set LC_DENOISE_MODEL and LC_DENOISE_RAW"); + }; + let dir = temp("e2e"); + let mut s = Session::new().with_fs(); + s.open_library(dir.join("library"), false).unwrap(); + s.set_denoise_models_dir(Some(dir.join("models"))); + let t = Instant::now(); + let r = s.execute("denoise.models.install", &json!({"path": model.to_string_lossy(), "acknowledged": true})).unwrap(); + println!("installed {} in {:.1} s: self-test {}", r["installed"]["id"], t.elapsed().as_secs_f64(), r["installed"]["accepted"]["selfTest"]); + assert_eq!(r["installed"]["id"], "rawnind-bayer"); + if let Ok(run_on) = std::env::var("LC_DENOISE_RUN_ON") { + s.execute("denoise.settings", &json!({"runOn": run_on})).unwrap(); + } + + for file in files(Path::new(&raws)) { + let name = file.file_name().unwrap().to_string_lossy().into_owned(); + let imported = s.execute("library.import", &json!({"paths": [file.to_string_lossy()]})).unwrap(); + let Some(id) = imported["imported"].as_array().and_then(|a| a.first()).and_then(Value::as_u64) else { + println!("{name}: not imported ({imported})"); + continue; + }; + s.execute("library.select", &json!({"ids": [id]})).unwrap(); + s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 100})).unwrap(); + let photo = lightcraft_engine::catalog::PhotoId(id); + let started = Instant::now(); + let mut state = Value::Null; + while started.elapsed() < Duration::from_secs(600) { + s.execute("denoise.pump", &json!({"pace": "full"})).unwrap(); + state = s.execute("denoise.status", &json!({"id": id})).unwrap()["photo"].clone(); + if matches!(state["state"].as_str(), Some("ready" | "failed")) { + break; + } + std::thread::sleep(Duration::from_millis(250)); + } + println!("{name}: {} in {:.1} s", state, started.elapsed().as_secs_f64()); + if state["state"] == "failed" { + // a file that is not a Bayer raw says why, and its export is the plain one + assert_eq!(state["unsupported"], true, "{name}: {state}"); + continue; + } + assert_eq!(state["state"], "ready", "{name}: {state}"); + let denoised = s.render_now(photo, 1600, 1600).unwrap().image; + s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 0})).unwrap(); + let plain = s.render_now(photo, 1600, 1600).unwrap().image; + let (a, b) = (roughness(&plain), roughness(&denoised)); + println!("{name}: roughness at 1600 px: plain {a:.3}, denoised {b:.3} ({:+.1} %)", 100.0 * (b - a) / a); + assert!(b < a, "{name}: the denoised picture is not smoother"); + // the export is made from the same picture, without asking the queue + s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 100})).unwrap(); + let o = lightcraft_engine::export::ExportOptions::from_json(&json!({"format": "jpeg", "width": 1200, "height": 1200})); + let e = lightcraft_engine::export::export_photo(&mut s, photo, &o, 1).unwrap(); + assert!(e.bytes.len() > 1000); + } + let status = s.execute("denoise.status", &json!({})).unwrap(); + println!("device: {}", status["device"]); + println!("cache: {}", status["cache"]); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/gpu/Cargo.toml b/crates/gpu/Cargo.toml index 765e4c6d..314f69c8 100644 --- a/crates/gpu/Cargo.toml +++ b/crates/gpu/Cargo.toml @@ -9,6 +9,7 @@ description = "LightCraft GPU develop pipeline: wgpu compute (WGSL) twin of the workspace = true [dependencies] +lightcraft-denoise = { workspace = true } lightcraft-geom = { workspace = true } lightcraft-color = { workspace = true } lightcraft-raster = { workspace = true } diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index 312d4263..b4e32ba9 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -30,6 +30,8 @@ pub mod backend; #[cfg(not(target_arch = "wasm32"))] mod ctx; #[cfg(not(target_arch = "wasm32"))] +pub mod nn; +#[cfg(not(target_arch = "wasm32"))] mod params; #[cfg(not(target_arch = "wasm32"))] mod render; @@ -145,6 +147,16 @@ pub fn enabled() -> bool { ENABLED.load(Ordering::Relaxed) && !BROKEN.load(Ordering::Relaxed) && !env_disabled() } +/// Why GPU compute that has a device of its own (denoise) is switched off by the user's settings, or `None`: the +/// environment switches and the rendering preference, but not a render-device failure (the devices are separate). +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn switched_off() -> Option { + if env_disabled() { + return Some(if backend::env_off() { "disabled by LIGHTCRAFT_GPU_BACKEND=off" } else { "disabled by LIGHTCRAFT_GPU=0" }.into()); + } + (!ENABLED.load(Ordering::Relaxed)).then(|| "disabled by the GPU rendering preference (app.gpu)".into()) +} + /// `LIGHTCRAFT_GPU=0` (or `LIGHTCRAFT_GPU_BACKEND=off`): no GPU for the whole process. fn env_disabled() -> bool { static OFF: std::sync::OnceLock = std::sync::OnceLock::new(); diff --git a/crates/gpu/src/nn.rs b/crates/gpu/src/nn.rs new file mode 100644 index 00000000..4231dae8 --- /dev/null +++ b/crates/gpu/src/nn.rs @@ -0,0 +1,1048 @@ +//! Denoise networks on the GPU: a [`NetRunner`] executes a [`Net`] (the plain description of a U-Net's layers) with +//! WGSL compute kernels, one tile at a time, behind the same [`TileRunner`] interface the CPU runner has. +//! +//! The runner has a device of its own, so a denoise job and the interactive renders are separate contexts that the +//! driver time-slices, and a failure of one never takes the other down. It respects the same switches as GPU rendering +//! (`LIGHTCRAFT_GPU`, `LIGHTCRAFT_GPU_BACKEND`, the GPU preference) and the same crash sentinel around device creation. +//! `LIGHTCRAFT_GPU_ADAPTER` picks an adapter by (part of) its name, e.g. to try another card. +//! +//! Every failure is an `Err` (never a panic), and a device that errs or is lost stops being used: callers then run the +//! CPU runner. The kernels (`wgsl/nn_conv.wgsl`, `wgsl/nn_pool.wgsl`) are tested against the plain-loop reference +//! interpreter in `lightcraft_denoise::reference` and, with the real model, against the pure-Rust CPU runner. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Condvar, Mutex, OnceLock}; +use std::time::Duration; + +use bytemuck::{Pod, Zeroable}; +use lightcraft_denoise::net::{Net, Op}; +use lightcraft_denoise::run::{Error, TileRunner}; +use wgpu::util::DeviceExt; + +/// Longest wait for one tile's GPU work before the device is given up on. +const TIMEOUT: Duration = Duration::from_secs(60); +/// The largest tile side (in input cells) the dispatch sizes and 32-bit indexing are checked for. +const MAX_TILE: usize = 1024; +/// The largest single buffer used: indices in the kernels are 32-bit element offsets. +const MAX_BUFFER: u64 = 2 << 30; +/// Tiles that can be on the GPU at once (each has buffers of its own). +const WORKSPACES: usize = 2; + +// --------------------------------------------------------------------------------------------------------------------- +// the device + +struct Dev { + device: wgpu::Device, + queue: wgpu::Queue, + info: wgpu::AdapterInfo, + limits: wgpu::Limits, +} + +static DEV: OnceLock> = OnceLock::new(); +static BROKEN: AtomicBool = AtomicBool::new(false); +static BROKEN_WHY: Mutex> = Mutex::new(None); + +fn mark_broken(why: &str) { + let mut w = BROKEN_WHY.lock().unwrap_or_else(|e| e.into_inner()); + if w.is_none() { + *w = Some(why.to_string()); + } + BROKEN.store(true, Ordering::Relaxed); +} + +/// Why the GPU stopped being used for denoise (a device error), if it did. +pub fn broken_reason() -> Option { + BROKEN.load(Ordering::Relaxed).then(|| BROKEN_WHY.lock().unwrap_or_else(|e| e.into_inner()).clone().unwrap_or_else(|| "device error".into())) +} + +fn dev() -> Result<&'static Dev, String> { + if let Some(why) = broken_reason() { + return Err(format!("stopped after a GPU failure: {why}")); + } + if let Some(why) = crate::switched_off() { + return Err(why); + } + DEV.get_or_init(create_device).as_ref().map_err(|e| e.clone()) +} + +fn create_device() -> Result { + let Some(backends) = crate::backend::compute_backends() else { return Err("disabled by LIGHTCRAFT_GPU_BACKEND=off".into()) }; + crate::backend::with_init_marker(backends, || { + std::panic::catch_unwind(|| make_device(backends)).unwrap_or_else(|_| Err("device creation panicked".into())) + }) +} + +fn make_device(backends: wgpu::Backends) -> Result { + let mut desc = wgpu::InstanceDescriptor::new_without_display_handle(); + desc.backends = backends; + let instance = wgpu::Instance::new(desc); + let wanted = std::env::var("LIGHTCRAFT_GPU_ADAPTER").ok().map(|s| s.trim().to_lowercase()).filter(|s| !s.is_empty()); + let adapter = match &wanted { + Some(w) => { + let all = pollster::block_on(instance.enumerate_adapters(backends)); + let names: Vec = all.iter().map(|a| a.get_info().name).collect(); + all.into_iter() + .find(|a| { + let i = a.get_info(); + i.device_type != wgpu::DeviceType::Cpu && i.name.to_lowercase().contains(w.as_str()) + }) + .ok_or_else(|| format!("no GPU adapter named like `{w}` (found: {})", names.join(", ")))? + } + None => pollster::block_on( + instance.request_adapter(&wgpu::RequestAdapterOptions { power_preference: wgpu::PowerPreference::HighPerformance, ..Default::default() }), + ) + .map_err(|e| format!("no GPU adapter among {backends:?} ({e})"))?, + }; + let info = adapter.get_info(); + if info.device_type == wgpu::DeviceType::Cpu { + return Err(format!("software adapter ({}) skipped: the CPU is faster", info.name)); + } + let limits = adapter.limits(); + if limits.max_storage_buffers_per_shader_stage < 6 + || limits.max_compute_invocations_per_workgroup < 256 + || limits.max_compute_workgroup_size_x < 256 + { + return Err(format!("{} is too limited for the denoise kernels", info.name)); + } + if limits.max_compute_workgroup_storage_size < 16 * 1024 { + return Err(format!("{} has too little workgroup memory for the denoise kernels", info.name)); + } + let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor { + label: Some("lightcraft-denoise"), + required_limits: limits.clone(), + ..Default::default() + })) + .map_err(|e| format!("{}: device creation failed ({e})", info.name))?; + device.on_uncaptured_error(std::sync::Arc::new(|e| { + log::error!("gpu (denoise): {e}"); + mark_broken(&format!("device error: {e}")); + })); + device.set_device_lost_callback(|reason, msg| { + log::error!("gpu (denoise): device lost ({reason:?}): {msg}"); + mark_broken(&format!("device lost ({reason:?}): {msg}")); + }); + log::info!("gpu (denoise): {} ({:?})", info.name, info.backend); + Ok(Dev { device, queue, info, limits }) +} + +/// The adapter denoise would use ("NVIDIA GeForce RTX 4090 Laptop GPU (Dx12)"), or why it cannot use one. Creates the +/// device on first use. +pub fn adapter() -> Result { + dev().map(|d| format!("{} ({:?})", d.info.name, d.info.backend)) +} + +// --------------------------------------------------------------------------------------------------------------------- +// the plan: what to run, in what buffers + +#[repr(C)] +#[derive(Clone, Copy, Pod, Zeroable)] +struct ConvParams { + h: u32, + w: u32, + c0: u32, + c1: u32, + n: u32, + npad: u32, + k: u32, + mode: u32, + cout: u32, + leaky: u32, + alpha: f32, + pad: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Pod, Zeroable)] +struct PoolParams { + h: u32, + w: u32, + c4: u32, + total: u32, +} + +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +enum Kernel { + Conv { bm: u32, bn: u32, bk: u32 }, + Pool, +} + +/// Where a step's tensor lives. +#[derive(Clone, Copy, Debug)] +enum Place { + Input, + Slot(usize), + Output, +} + +struct Step { + kernel: Kernel, + groups: [u32; 3], + src: Place, + src2: Option, + dst: Place, + /// Index into the per-layer GPU data (weights, bias), for the convolutions. + layer: Option, + params: Vec, +} + +struct Layout { + steps: Vec, + /// Bytes of each activation buffer. + slots: Vec, + in_floats: usize, + out_floats: usize, +} + +/// Weights arranged for the kernel: `[k][npad]` rows with `k = tap · cin + ci` (columns are output channels, zero past +/// `n`), and the bias padded the same. +struct Layer { + weight: Vec, + bias: Vec, +} + +fn unsupported(why: impl Into) -> Result { + Err(why.into()) +} + +/// The kernel shape for `n` output columns: 64 × 64 blocks when `n` is a multiple of 64, else 128 × 32. +fn blocks(n: usize, cin: usize) -> (u32, u32, u32) { + let bn = if n.is_multiple_of(64) { 64 } else { 32 }; + let bk = if cin.is_multiple_of(16) { + 16 + } else if cin.is_multiple_of(8) { + 8 + } else { + 4 + }; + (4096 / bn, bn, bk) +} + +fn round_up(v: usize, to: usize) -> usize { + v.div_ceil(to) * to +} + +/// Work out the steps, the buffers and the arranged weights for `net` on `tile × tile` input cells. +fn plan(net: &Net, tile: usize, max_binding: u64) -> Result<(Layout, Vec), String> { + if !net.fits_tile(tile) { + return unsupported("the tile does not fit the network"); + } + if tile > MAX_TILE { + return unsupported("the tile is larger than the kernels' dispatches reach"); + } + let side = |t: usize| -> usize { + let level = net.shape(t).map_or(0, |s| s.level); + if level >= 0 { tile >> level } else { tile << (-level) } + }; + let chan = |t: usize| net.shape(t).map_or(0, |s| s.channels); + if !net.in_channels.is_multiple_of(4) { + return unsupported("the input channel count is not a multiple of 4"); + } + let ops = net.ops(); + // the network must end in a 1 × 1 convolution and a depth-to-space (they run as one kernel) + let n_ops = ops.len(); + let ends_right = matches!( + (n_ops.checked_sub(2).and_then(|i| ops.get(i)), ops.last()), + (Some(Op::Conv(c)), Some(Op::DepthToSpace2 { src })) if c.k == 1 && c.src2.is_none() && *src == n_ops - 1 && c.cout == 12 && c.leaky.is_none() + ); + if !ends_right { + return unsupported("the network does not end in a 1 × 1 convolution to 12 channels and depth-to-space"); + } + // the last layer that reads each tensor + let mut last_use = vec![0usize; n_ops + 1]; + for (i, op) in ops.iter().enumerate() { + let reads: Vec = match op { + Op::Conv(c) => std::iter::once(c.src).chain(c.src2).collect(), + Op::ConvT2(c) => vec![c.src], + Op::MaxPool2 { src } | Op::DepthToSpace2 { src } => vec![*src], + }; + for t in reads { + if let Some(l) = last_use.get_mut(t) { + *l = i; + } + } + } + + use Place::{Input, Output, Slot}; + let mut slots: Vec = Vec::new(); + let mut free: Vec = Vec::new(); + let mut place: Vec> = vec![None; n_ops + 1]; + if let Some(p) = place.first_mut() { + *p = Some(Input); + } + let mut steps: Vec = Vec::new(); + let mut layers: Vec = Vec::new(); + /// A buffer of at least `bytes`: the smallest free one that is big enough, else the biggest free one grown, else new. + fn alloc(bytes: u64, slots: &mut Vec, free: &mut Vec) -> usize { + let size = |s: usize| slots.get(s).copied().unwrap_or(0); + let fits = free.iter().enumerate().filter(|&(_, &s)| size(s) >= bytes).min_by_key(|&(_, &s)| size(s)).map(|(i, _)| i); + let pick = fits.or_else(|| free.iter().enumerate().max_by_key(|&(_, &s)| size(s)).map(|(i, _)| i)); + match pick { + Some(i) => { + let s = free.swap_remove(i); + if let Some(b) = slots.get_mut(s) { + *b = (*b).max(bytes); + } + s + } + None => { + slots.push(bytes); + slots.len() - 1 + } + } + } + let get = |place: &[Option], t: usize| place.get(t).copied().flatten().ok_or_else(|| format!("tensor {t} has no buffer")); + + for (i, op) in ops.iter().enumerate() { + let out_t = i + 1; + match op { + Op::Conv(c) => { + if !c.cin.is_multiple_of(4) || !c.cout.is_multiple_of(4) { + return unsupported("a convolution whose channel counts are not multiples of 4"); + } + let s = side(c.src); + let last = i + 2 == n_ops; + let (n, mode) = (c.cout, if last { 2u32 } else { 0u32 }); + let (bm, bn, bk) = blocks(n, c.cin); + let npad = round_up(n, bn as usize); + // arranged weights: row (tap · cin + ci), column co + let taps = c.k * c.k; + let mut weight = vec![0f32; taps * c.cin * npad]; + for co in 0..c.cout { + for ci in 0..c.cin { + for ky in 0..c.k { + for kx in 0..c.k { + let v = c.weight.get(((co * c.cin + ci) * c.k + ky) * c.k + kx).copied().unwrap_or(0.0); + if let Some(w) = weight.get_mut(((ky * c.k + kx) * c.cin + ci) * npad + co) { + *w = v; + } + } + } + } + } + let mut bias = vec![0f32; npad]; + if let Some(b) = bias.get_mut(..c.cout) { + b.copy_from_slice(&c.bias); + } + let m = s * s; + let params = ConvParams { + h: s as u32, + w: s as u32, + c0: chan(c.src) as u32, + c1: c.src2.map_or(0, chan) as u32, + n: n as u32, + npad: npad as u32, + k: c.k as u32, + mode, + cout: c.cout as u32, + leaky: u32::from(c.leaky.is_some()), + alpha: c.leaky.unwrap_or(0.0), + pad: 0, + }; + let src = get(&place, c.src)?; + let src2 = c.src2.map(|t| get(&place, t)).transpose()?; + let dst = if last { + Output + } else { + let slot = alloc((m * c.cout * 4) as u64, &mut slots, &mut free); + Slot(slot) + }; + if let Some(p) = place.get_mut(out_t) { + *p = Some(dst); + } + layers.push(Layer { weight, bias }); + steps.push(Step { + kernel: Kernel::Conv { bm, bn, bk }, + groups: [m.div_ceil(bm as usize) as u32, (npad as u32) / bn, 1], + src, + src2, + dst, + layer: Some(layers.len() - 1), + params: bytemuck::bytes_of(¶ms).to_vec(), + }); + } + Op::ConvT2(c) => { + if !c.cin.is_multiple_of(4) || !c.cout.is_multiple_of(4) { + return unsupported("a transposed convolution whose channel counts are not multiples of 4"); + } + let s = side(c.src); + let n = 4 * c.cout; + let (bm, bn, bk) = blocks(n, c.cin); + let npad = round_up(n, bn as usize); + // arranged weights: row ci, column (ky · 2 + kx) · cout + co + let mut weight = vec![0f32; c.cin * npad]; + for ci in 0..c.cin { + for co in 0..c.cout { + for ky in 0..2 { + for kx in 0..2 { + let v = c.weight.get(((ci * c.cout + co) * 2 + ky) * 2 + kx).copied().unwrap_or(0.0); + if let Some(w) = weight.get_mut(ci * npad + (ky * 2 + kx) * c.cout + co) { + *w = v; + } + } + } + } + } + let mut bias = vec![0f32; npad]; + for q in 0..4 { + for co in 0..c.cout { + if let (Some(b), Some(v)) = (bias.get_mut(q * c.cout + co), c.bias.get(co)) { + *b = *v; + } + } + } + let m = s * s; + let params = ConvParams { + h: s as u32, + w: s as u32, + c0: c.cin as u32, + c1: 0, + n: n as u32, + npad: npad as u32, + k: 1, + mode: 1, + cout: c.cout as u32, + leaky: u32::from(c.leaky.is_some()), + alpha: c.leaky.unwrap_or(0.0), + pad: 0, + }; + let src = get(&place, c.src)?; + let slot = alloc((4 * m * c.cout * 4) as u64, &mut slots, &mut free); + if let Some(p) = place.get_mut(out_t) { + *p = Some(Slot(slot)); + } + layers.push(Layer { weight, bias }); + steps.push(Step { + kernel: Kernel::Conv { bm, bn, bk }, + groups: [m.div_ceil(bm as usize) as u32, (npad as u32) / bn, 1], + src, + src2: None, + dst: Slot(slot), + layer: Some(layers.len() - 1), + params: bytemuck::bytes_of(¶ms).to_vec(), + }); + } + Op::MaxPool2 { src } => { + let (s, c) = (side(*src), chan(*src)); + if !c.is_multiple_of(4) { + return unsupported("a max-pool over a channel count that is not a multiple of 4"); + } + let total = (s / 2) * (s / 2) * (c / 4); + let params = PoolParams { h: s as u32, w: s as u32, c4: (c / 4) as u32, total: total as u32 }; + let from = get(&place, *src)?; + let slot = alloc((total * 16) as u64, &mut slots, &mut free); + if let Some(p) = place.get_mut(out_t) { + *p = Some(Slot(slot)); + } + let groups = total.div_ceil(64); + steps.push(Step { + kernel: Kernel::Pool, + groups: [groups.min(8192) as u32, groups.div_ceil(8192) as u32, 1], + src: from, + src2: None, + dst: Slot(slot), + layer: None, + params: bytemuck::bytes_of(¶ms).to_vec(), + }); + } + // run as part of the 1 × 1 convolution before it + Op::DepthToSpace2 { .. } => {} + } + // buffers whose last reader was this layer are free for the next layer's result + for t in 0..=i { + if last_use.get(t).copied() == Some(i) + && let Some(Some(Slot(s))) = place.get(t) + && !free.contains(s) + { + free.push(*s); + } + } + } + if slots.iter().any(|&b| b > max_binding) { + return unsupported("an activation buffer is larger than this GPU allows"); + } + if layers.iter().any(|l| (l.weight.len() as u64).saturating_mul(4) > max_binding) { + return unsupported("a weight buffer is larger than this GPU allows"); + } + Ok((Layout { steps, slots, in_floats: net.in_channels * tile * tile, out_floats: 3 * (2 * tile) * (2 * tile) }, layers)) +} + +// --------------------------------------------------------------------------------------------------------------------- +// the GPU objects + +struct Pipelines { + conv: HashMap<(u32, u32, u32), wgpu::ComputePipeline>, + conv_layout: wgpu::BindGroupLayout, + pool: wgpu::ComputePipeline, + pool_layout: wgpu::BindGroupLayout, + /// How long each kernel took to build (the driver compiles them), for the log and the tests. + built: Vec<(String, f64)>, +} + +fn entry(binding: u32, ty: wgpu::BufferBindingType) -> wgpu::BindGroupLayoutEntry { + wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { ty, has_dynamic_offset: false, min_binding_size: None }, + count: None, + } +} + +fn pipeline(d: &Dev, label: &str, src: &str, layout: &wgpu::BindGroupLayout) -> wgpu::ComputePipeline { + let module = d.device.create_shader_module(wgpu::ShaderModuleDescriptor { label: Some(label), source: wgpu::ShaderSource::Wgsl(src.into()) }); + let pl = d.device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: None, bind_group_layouts: &[Some(layout)], immediate_size: 0 }); + d.device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor { + label: Some(label), + layout: Some(&pl), + module: &module, + entry_point: Some("main"), + // The kernels write all of their workgroup memory before reading any of it, so the zeroing wgpu adds by default + // is not needed; on DX12 it is written out as one assignment per element, which makes the shader compiler take + // a minute over a 2k-element array. + compilation_options: wgpu::PipelineCompilationOptions { zero_initialize_workgroup_memory: false, ..Default::default() }, + cache: None, + }) +} + +fn conv_source(bm: u32, bn: u32, bk: u32) -> String { + include_str!("wgsl/nn_conv.wgsl") + .replace("{{BM}}", &bm.to_string()) + .replace("{{BN}}", &bn.to_string()) + .replace("{{BK}}", &bk.to_string()) + .replace("{{A_VECS}}", &(bk * bm / 4).to_string()) + .replace("{{B_VECS}}", &(bk * bn / 4).to_string()) +} + +fn conv_bgl(d: &Dev) -> wgpu::BindGroupLayout { + use wgpu::BufferBindingType::{Storage, Uniform}; + d.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: None, + entries: &[ + entry(0, Uniform), + entry(1, Storage { read_only: true }), + entry(2, Storage { read_only: true }), + entry(3, Storage { read_only: true }), + entry(4, Storage { read_only: true }), + entry(5, Storage { read_only: false }), + ], + }) +} + +fn pool_bgl(d: &Dev) -> wgpu::BindGroupLayout { + use wgpu::BufferBindingType::{Storage, Uniform}; + d.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: None, + entries: &[entry(0, Uniform), entry(1, Storage { read_only: true }), entry(2, Storage { read_only: false })], + }) +} + +fn build_pipelines(d: &Dev, layout: &Layout) -> Pipelines { + let (conv_layout, pool_layout) = (conv_bgl(d), pool_bgl(d)); + let mut conv = HashMap::new(); + let mut built = Vec::new(); + for s in &layout.steps { + if let Kernel::Conv { bm, bn, bk } = s.kernel { + conv.entry((bm, bn, bk)).or_insert_with(|| { + let started = std::time::Instant::now(); + let p = pipeline(d, "nn_conv", &conv_source(bm, bn, bk), &conv_layout); + built.push((format!("conv {bm}x{bn}x{bk}"), started.elapsed().as_secs_f64() * 1000.0)); + p + }); + } + } + let started = std::time::Instant::now(); + let pool = pipeline(d, "nn_pool", include_str!("wgsl/nn_pool.wgsl"), &pool_layout); + built.push(("pool".into(), started.elapsed().as_secs_f64() * 1000.0)); + Pipelines { conv, conv_layout, pool, pool_layout, built } +} + +/// One tile's buffers and the bound steps. +struct Workspace { + input: wgpu::Buffer, + output: wgpu::Buffer, + staging: wgpu::Buffer, + bound: Vec<(Kernel, [u32; 3], wgpu::BindGroup)>, + _slots: Vec, +} + +struct Shared { + dev: &'static Dev, + layout: Layout, + pipes: Pipelines, + weights: Vec<(wgpu::Buffer, wgpu::Buffer)>, + uniforms: Vec, + dummy: wgpu::Buffer, +} + +fn storage(d: &Dev, label: &str, bytes: u64, extra: wgpu::BufferUsages) -> wgpu::Buffer { + d.device.create_buffer(&wgpu::BufferDescriptor { + label: Some(label), + size: bytes.max(16), + usage: wgpu::BufferUsages::STORAGE | extra, + mapped_at_creation: false, + }) +} + +impl Shared { + fn workspace(&self) -> Workspace { + let d = self.dev; + let slots: Vec = self.layout.slots.iter().map(|&b| storage(d, "activations", b, wgpu::BufferUsages::empty())).collect(); + let input = storage(d, "input", (self.layout.in_floats * 4) as u64, wgpu::BufferUsages::COPY_DST); + let output = storage(d, "output", (self.layout.out_floats * 4) as u64, wgpu::BufferUsages::COPY_SRC); + let staging = d.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("readback"), + size: (self.layout.out_floats * 4) as u64, + usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + let buf = |p: Place| -> &wgpu::Buffer { + match p { + Place::Input => &input, + Place::Output => &output, + Place::Slot(i) => slots.get(i).unwrap_or(&self.dummy), + } + }; + let mut bound = Vec::new(); + for (i, s) in self.layout.steps.iter().enumerate() { + let uniform = self.uniforms.get(i).unwrap_or(&self.dummy); + let bg = match s.kernel { + Kernel::Conv { .. } => { + let (w, b) = s.layer.and_then(|l| self.weights.get(l)).map_or((&self.dummy, &self.dummy), |(w, b)| (w, b)); + d.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: None, + layout: &self.pipes.conv_layout, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: uniform.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 1, resource: buf(s.src).as_entire_binding() }, + wgpu::BindGroupEntry { binding: 2, resource: s.src2.map_or(&self.dummy, buf).as_entire_binding() }, + wgpu::BindGroupEntry { binding: 3, resource: w.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 4, resource: b.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 5, resource: buf(s.dst).as_entire_binding() }, + ], + }) + } + Kernel::Pool => d.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: None, + layout: &self.pipes.pool_layout, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: uniform.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 1, resource: buf(s.src).as_entire_binding() }, + wgpu::BindGroupEntry { binding: 2, resource: buf(s.dst).as_entire_binding() }, + ], + }), + }; + bound.push((s.kernel, s.groups, bg)); + } + Workspace { input, output, staging, bound, _slots: slots } + } +} + +/// A network on the GPU for one tile size. Cheap to share between threads: each call takes a free workspace. +pub struct NetRunner { + shared: Shared, + tile: usize, + in_channels: usize, + free: Mutex>, + /// Workspaces made so far (they are made when a call finds none free, up to [`WORKSPACES`]). + made: Mutex, + wake: Condvar, + bytes: u64, +} + +/// Put `net` on the GPU for `tile × tile` input cells, or say why it cannot be (no GPU, a layer or a size the kernels +/// do not do, the device refusing the buffers). Creates the device on first use and compiles the kernels. +pub fn runner(net: &Net, tile: usize) -> Result { + let d = dev()?; + let max_binding = d.limits.max_storage_buffer_binding_size.min(d.limits.max_buffer_size).min(MAX_BUFFER); + let (layout, layers) = plan(net, tile, max_binding)?; + let scope = [ + d.device.push_error_scope(wgpu::ErrorFilter::OutOfMemory), + d.device.push_error_scope(wgpu::ErrorFilter::Validation), + d.device.push_error_scope(wgpu::ErrorFilter::Internal), + ]; + let pipes = build_pipelines(d, &layout); + let weights: Vec<(wgpu::Buffer, wgpu::Buffer)> = layers + .iter() + .map(|l| { + let make = |v: &[f32]| { + d.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("weights"), + contents: bytemuck::cast_slice(v), + usage: wgpu::BufferUsages::STORAGE, + }) + }; + (make(&l.weight), make(&l.bias)) + }) + .collect(); + let uniforms: Vec = layout + .steps + .iter() + .map(|s| { + d.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("params"), + contents: &s.params, + usage: wgpu::BufferUsages::UNIFORM, + }) + }) + .collect(); + let dummy = storage(d, "dummy", 16, wgpu::BufferUsages::empty()); + let bytes: u64 = layout.slots.iter().sum::() + ((layout.in_floats + layout.out_floats) * 8) as u64; + let shared = Shared { dev: d, layout, pipes, weights, uniforms, dummy }; + let first = shared.workspace(); + let [oom, validation, internal] = scope; + let errors = [pollster::block_on(internal.pop()), pollster::block_on(validation.pop()), pollster::block_on(oom.pop())]; + if let Some(e) = errors.into_iter().flatten().next() { + return Err(format!("{}: the denoise kernels could not be set up ({e})", d.info.name)); + } + Ok(NetRunner { shared, tile, in_channels: net.in_channels, free: Mutex::new(vec![first]), made: Mutex::new(1), wake: Condvar::new(), bytes }) +} + +impl NetRunner { + /// The adapter this runner uses. + pub fn adapter(&self) -> String { + format!("{} ({:?})", self.shared.dev.info.name, self.shared.dev.info.backend) + } + + /// How long each kernel took to build, e.g. `conv 64x64x16: 812 ms`. + pub fn build_report(&self) -> String { + self.shared.pipes.built.iter().map(|(k, ms)| format!("{k}: {ms:.0} ms")).collect::>().join(", ") + } + + /// Device memory one tile in flight uses (bytes), without the weights. + pub fn bytes_per_tile(&self) -> u64 { + self.bytes + } + + /// A free workspace, a new one when fewer than [`WORKSPACES`] exist, else wait for one to come back. + fn take(&self) -> Result { + let mut free = self.free.lock().unwrap_or_else(|e| e.into_inner()); + loop { + if let Some(w) = free.pop() { + return Ok(w); + } + { + let mut made = self.made.lock().unwrap_or_else(|e| e.into_inner()); + if *made < WORKSPACES { + *made += 1; + drop(made); + drop(free); + let d = self.shared.dev; + let scopes = + [d.device.push_error_scope(wgpu::ErrorFilter::OutOfMemory), d.device.push_error_scope(wgpu::ErrorFilter::Validation)]; + let w = self.shared.workspace(); + let [oom, validation] = scopes; + let (validation, oom) = (pollster::block_on(validation.pop()), pollster::block_on(oom.pop())); + if let Some(e) = oom.or(validation) { + self.forget(); + return Err(Error::Runtime(format!("the GPU could not make room for another tile: {e}"))); + } + return Ok(w); + } + } + free = self.wake.wait(free).unwrap_or_else(|e| e.into_inner()); + } + } + + fn give(&self, w: Workspace) { + self.free.lock().unwrap_or_else(|e| e.into_inner()).push(w); + self.wake.notify_one(); + } + + /// A workspace is gone (dropped after a failure): room for a new one, and a waiting caller may make it. + fn forget(&self) { + let mut made = self.made.lock().unwrap_or_else(|e| e.into_inner()); + *made = made.saturating_sub(1); + drop(made); + self.wake.notify_one(); + } + + fn execute(&self, w: &Workspace, input: &[f32]) -> Result, Error> { + let d = self.shared.dev; + let (t, c) = (self.tile, self.in_channels); + // planar → channels innermost + let mut nhwc = vec![0f32; input.len()]; + for (ch, plane) in input.chunks_exact(t * t).enumerate() { + for (i, v) in plane.iter().enumerate() { + if let Some(o) = nhwc.get_mut(i * c + ch) { + *o = *v; + } + } + } + let mut calls = Vec::with_capacity(w.bound.len()); + for (kernel, groups, bg) in &w.bound { + let pipe = match kernel { + Kernel::Conv { bm, bn, bk } => self.shared.pipes.conv.get(&(*bm, *bn, *bk)), + Kernel::Pool => Some(&self.shared.pipes.pool), + }; + let Some(pipe) = pipe else { return Err(Error::Runtime("a kernel is missing".into())) }; + calls.push((pipe, groups, bg)); + } + let scopes = [ + d.device.push_error_scope(wgpu::ErrorFilter::OutOfMemory), + d.device.push_error_scope(wgpu::ErrorFilter::Validation), + d.device.push_error_scope(wgpu::ErrorFilter::Internal), + ]; + d.queue.write_buffer(&w.input, 0, bytemuck::cast_slice(&nhwc)); + let mut enc = d.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: None }); + { + let mut pass = enc.begin_compute_pass(&wgpu::ComputePassDescriptor { label: Some("denoise tile"), timestamp_writes: None }); + for (pipe, groups, bg) in calls { + pass.set_pipeline(pipe); + pass.set_bind_group(0, bg, &[]); + pass.dispatch_workgroups(groups[0], groups[1], groups[2]); + } + } + let bytes = (self.shared.layout.out_floats * 4) as u64; + enc.copy_buffer_to_buffer(&w.output, 0, &w.staging, 0, bytes); + d.queue.submit([enc.finish()]); + let slice = w.staging.slice(..bytes); + let (tx, rx) = std::sync::mpsc::channel(); + slice.map_async(wgpu::MapMode::Read, move |r| { + let _ = tx.send(r); + }); + let waited = d.device.poll(wgpu::PollType::Wait { submission_index: None, timeout: Some(TIMEOUT) }); + let [oom, validation, internal] = scopes; + let errors = [pollster::block_on(internal.pop()), pollster::block_on(validation.pop()), pollster::block_on(oom.pop())]; + let [internal, validation, oom] = errors; + if let Some(e) = oom { + // not a broken device: this tile runs on the CPU, and the next may fit + return Err(Error::Runtime(format!("the GPU ran out of memory: {e}"))); + } + if let Some(e) = internal.or(validation) { + mark_broken(&format!("device error: {e}")); + return Err(Error::Runtime(format!("the GPU reported an error: {e}"))); + } + if let Err(e) = waited { + mark_broken(&format!("the GPU did not finish a tile: {e}")); + return Err(Error::Runtime(format!("the GPU did not finish: {e}"))); + } + match rx.recv_timeout(Duration::from_secs(10)) { + Ok(Ok(())) => {} + Ok(Err(e)) => return Err(Error::Runtime(format!("reading the result back failed: {e}"))), + Err(e) => return Err(Error::Runtime(format!("reading the result back failed: {e}"))), + } + let out = { + let data = slice.get_mapped_range().map_err(|e| Error::Runtime(format!("reading the result back failed: {e:?}")))?; + data.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect::>() + }; + w.staging.unmap(); + Ok(out) + } +} + +impl TileRunner for NetRunner { + fn run(&self, input: &[f32]) -> Result, Error> { + if let Some(why) = broken_reason() { + return Err(Error::Runtime(format!("the GPU is not used any more: {why}"))); + } + if input.len() != self.shared.layout.in_floats { + return Err(Error::Input("the tile is not the size the model wants".into())); + } + let w = self.take()?; + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.execute(&w, input))); + match r { + Ok(Ok(out)) => { + self.give(w); + Ok(out) + } + Ok(Err(e)) => { + // its readback buffer may be left half-mapped: a fresh workspace is made when one is next needed + drop(w); + self.forget(); + Err(e) + } + Err(_) => { + // a panic inside wgpu: this workspace is not given back, and the GPU is not used again + mark_broken("a denoise tile panicked"); + drop(w); + self.forget(); + Err(Error::Runtime("the GPU runner gave up".into())) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use lightcraft_denoise::{onnx, reference, synthetic}; + + fn net(tile: u64, ch: u64, depth: usize, seed: u64) -> Net { + let path = std::env::temp_dir().join(format!("lc-nn-{}-{tile}-{ch}-{depth}-{seed}.onnx", std::process::id())); + std::fs::write(&path, synthetic::unet_onnx(tile, ch, depth, seed)).unwrap(); + let n = onnx::read(&path).unwrap(); + let _ = std::fs::remove_file(&path); + n + } + + fn picture(len: usize, seed: u64) -> Vec { + let mut s = seed | 1; + (0..len) + .map(|_| { + s ^= s >> 12; + s ^= s << 25; + s ^= s >> 27; + (s.wrapping_mul(0x2545_f491_4f6c_dd1d) >> 40) as f32 / (1u64 << 24) as f32 + }) + .collect() + } + + /// The runner, or `None` (with a note) on a machine with no usable GPU. + fn gpu(net: &Net, tile: usize) -> Option { + match runner(net, tile) { + Ok(r) => Some(r), + Err(e) => { + eprintln!("skipped: no GPU runner here ({e})"); + None + } + } + } + + fn worst(a: &[f32], b: &[f32]) -> f32 { + assert_eq!(a.len(), b.len()); + let scale = b.iter().fold(1e-3f32, |m, v| m.max(v.abs())); + a.iter().zip(b).fold(0f32, |m, (x, y)| m.max((x - y).abs())) / scale + } + + #[test] + fn the_gpu_gives_what_the_reference_gives() { + // (tile, channels, depth): a tile that is not a multiple of the block size, odd channel counts, a deeper net + for (tile, ch, depth, seed) in + [(1u64, 4u64, 0usize, 1u64), (3, 4, 0, 8), (6, 4, 1, 9), (20, 12, 2, 2), (32, 8, 2, 3), (48, 12, 2, 4), (64, 16, 3, 5)] + { + let n = net(tile, ch, depth, seed); + let Some(g) = gpu(&n, tile as usize) else { return }; + let x = picture(4 * (tile * tile) as usize, seed); + let want = reference::run(&n, tile as usize, &x).unwrap(); + let got = g.run(&x).unwrap(); + let cpu = lightcraft_denoise::cpu::NetRunner::new(&n, tile as usize).unwrap().run(&x).unwrap(); + assert!(worst(&got, &cpu) < 1e-3, "GPU and pure-Rust CPU differ"); + let e = worst(&got, &want); + eprintln!("{}: tile {tile}, {ch} channels, depth {depth}: off by {e:e} of the output's size", g.adapter()); + assert!(e < 2e-4, "tile {tile}, {ch} channels, depth {depth}: the GPU is off by {e:e} of the output's size"); + } + } + + #[test] + fn a_runner_serves_many_tiles_from_many_threads() { + let n = net(32, 8, 2, 7); + let Some(g) = gpu(&n, 32) else { return }; + let inputs: Vec> = (0..12).map(|i| picture(4 * 32 * 32, 100 + i)).collect(); + let wants: Vec> = inputs.iter().map(|x| reference::run(&n, 32, x).unwrap()).collect(); + std::thread::scope(|s| { + for chunk in inputs.chunks(3).zip(wants.chunks(3)) { + let g = &g; + s.spawn(move || { + for (x, want) in chunk.0.iter().zip(chunk.1) { + let e = worst(&g.run(x).unwrap(), want); + assert!(e < 2e-4, "off by {e:e}"); + } + }); + } + }); + } + + /// The real model on the GPU against the pure-Rust CPU runner, with timings (the numbers in docs/denoise.md come from here): + /// `LC_DENOISE_MODEL= cargo test --release -p lightcraft-gpu --lib real_model -- --ignored --nocapture` + #[test] + #[ignore = "needs the real model: see the doc comment"] + fn real_model_on_the_gpu_matches_cpu() { + use lightcraft_denoise::manifest::{DenoiserManifest, Domain, Gain}; + use lightcraft_denoise::runtime::CpuRunner; + use std::time::Instant; + let Some(model) = std::env::var_os("LC_DENOISE_MODEL") else { return }; + let net = onnx::read(std::path::Path::new(&model)).unwrap(); + let tile = 512usize; + let started = Instant::now(); + let g = runner(&net, tile).unwrap(); + println!("{}: set up in {:.2} s, {:.0} MB per tile in flight", g.adapter(), started.elapsed().as_secs_f64(), g.bytes_per_tile() as f64 / 1e6); + println!("kernels: {}", g.build_report()); + let manifest = DenoiserManifest { + id: "rawnind-bayer".into(), + name: "RawNIND Bayer".into(), + version: "1".into(), + licence: Default::default(), + source: None, + sha256: None, + size_bytes: None, + provenance: String::new(), + domain: Domain::BayerToRgb, + tile: 512, + overlap: 64, + gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 }, + }; + let cpu = CpuRunner::load(std::path::Path::new(&model), &manifest).unwrap(); + // a gradient with noise, in the model's input range + let noise = picture(4 * tile * tile, 42); + let input: Vec = + (0..4 * tile * tile).map(|i| (0.2 + 0.3 * ((i % tile) as f32 / tile as f32) + 0.1 * (noise[i] - 0.5)).clamp(0.0, 1.0)).collect(); + let t = Instant::now(); + let want = cpu.run(&input).unwrap(); + println!("CPU: {:.0} ms for one tile (one call runs on one core)", t.elapsed().as_secs_f64() * 1000.0); + let t = Instant::now(); + let got = g.run(&input).unwrap(); + println!("gpu, first tile: {:.0} ms", t.elapsed().as_secs_f64() * 1000.0); + let e = worst(&got, &want); + println!("worst difference {e:e} of the output's size"); + assert!(e < 1e-3, "the GPU does not give the CPU answer ({e:e})"); + let mut times: Vec = (0..8) + .map(|_| { + let t = Instant::now(); + g.run(&input).unwrap(); + t.elapsed().as_secs_f64() * 1000.0 + }) + .collect(); + times.sort_by(|a, b| a.total_cmp(b)); + println!("gpu, one tile at a time: median {:.0} ms (best {:.0}, worst {:.0})", times[times.len() / 2], times[0], times[times.len() - 1]); + // two threads, as the denoiser feeds it + let t = Instant::now(); + std::thread::scope(|s| { + for _ in 0..2 { + s.spawn(|| { + for _ in 0..8 { + g.run(&input).unwrap(); + } + }); + } + }); + println!("gpu, 16 tiles from 2 threads: {:.0} ms per tile", t.elapsed().as_secs_f64() * 1000.0 / 16.0); + } + + /// How long the driver takes to build one conv kernel variant: `LC_SHADER= LC_BLOCKS=128,32,16`. + #[test] + #[ignore = "a probe for shader compile times"] + fn time_a_shader_build() { + let Some(path) = std::env::var_os("LC_SHADER") else { return }; + let blocks: Vec = std::env::var("LC_BLOCKS").unwrap_or_else(|_| "128,32,16".into()).split(',').filter_map(|v| v.parse().ok()).collect(); + let (bm, bn, bk) = (blocks[0], blocks[1], blocks[2]); + let src = std::fs::read_to_string(path) + .unwrap() + .replace("{{BM}}", &bm.to_string()) + .replace("{{BN}}", &bn.to_string()) + .replace("{{BK}}", &bk.to_string()) + .replace("{{A_VECS}}", &(bk * bm / 4).to_string()) + .replace("{{B_VECS}}", &(bk * bn / 4).to_string()); + let d = dev().unwrap(); + let layout = conv_bgl(d); + let started = std::time::Instant::now(); + let _ = pipeline(d, "probe", &src, &layout); + println!("{} {bm}x{bn}x{bk}: built in {:.0} ms", d.info.backend.to_str(), started.elapsed().as_secs_f64() * 1000.0); + } + + #[test] + fn a_tile_of_the_wrong_size_is_an_error_not_a_crash() { + let n = net(16, 4, 1, 9); + let Some(g) = gpu(&n, 16) else { return }; + assert!(g.run(&[0.0; 10]).is_err()); + assert!(g.run(&[]).is_err()); + } + + #[test] + fn networks_the_kernels_cannot_run_are_refused() { + let n = net(16, 4, 1, 11); + // a tile the network does not fit + assert!(plan(&n, 15, u64::MAX).is_err()); + // a buffer limit smaller than the activations + assert!(plan(&n, 16, 64).is_err()); + } +} diff --git a/crates/gpu/src/wgsl/nn_conv.wgsl b/crates/gpu/src/wgsl/nn_conv.wgsl new file mode 100644 index 00000000..d991e9e7 --- /dev/null +++ b/crates/gpu/src/wgsl/nn_conv.wgsl @@ -0,0 +1,162 @@ +// Convolution for the denoise networks, as an implicit matrix product: every output pixel is a row, every input +// channel of every kernel tap is a column of K, and the output channels are the columns of N. +// +// out[m][n] = bias[n] + sum_k A[m][k] * W[k][n] A[m][k] = input pixel (m + tap offset), channel ci +// +// Tensors are NHWC, channels innermost, read four at a time (every channel count is a multiple of 4). A workgroup of +// 256 threads makes a BM x BN block of the output, 4 x 4 values per thread, and walks K in steps of BK through +// workgroup memory. The source can be two tensors whose channels follow each other (a skip connection's join, so it +// is never made), and a leaky ReLU is applied on the way out. Specialised by the host: BM, BN, BK (BM * BN = 4096). +// +// Output modes: +// 0 NHWC at the input's size. +// 1 a 2x2 stride-2 transposed convolution: the columns are (dy, dx, co) and each lands in the output pixel +// (2y + dy, 2x + dx) of an output twice the size. +// 2 the last 1x1 convolution followed by depth-to-space (CRD): columns 4c + 2dy + dx land in plane c at +// (2y + dy, 2x + dx) of a planar (CHW) output twice the size. + +struct P { + h: u32, + w: u32, + c0: u32, + c1: u32, + n: u32, + npad: u32, + k: u32, + mode: u32, + cout: u32, + leaky: u32, + alpha: f32, + pad: u32, +} + +@group(0) @binding(0) var p: P; +@group(0) @binding(1) var src0: array>; +@group(0) @binding(2) var src1: array>; +@group(0) @binding(3) var wgt: array>; +@group(0) @binding(4) var bias: array>; +@group(0) @binding(5) var dst: array; + +const BM: u32 = {{BM}}u; +const BN: u32 = {{BN}}u; +const BK: u32 = {{BK}}u; +const TX: u32 = BN / 4u; +const A_ITEMS: u32 = BM * (BK / 4u); +const B_ITEMS: u32 = BK * (BN / 4u); + +// A[k][m] four pixels to a vec4, B[k][n] four channels to a vec4 +var As: array, {{A_VECS}}>; +var Bs: array, {{B_VECS}}>; + +@compute @workgroup_size(256) +fn main(@builtin(workgroup_id) wg: vec3, @builtin(local_invocation_index) tid: u32) { + let m0 = wg.x * BM; + let n0 = wg.y * BN; + let tx = tid % TX; + let ty = tid / TX; + let hw = p.h * p.w; + let cin = p.c0 + p.c1; + let taps = p.k * p.k; + let pad = (p.k - 1u) / 2u; + let chunks = cin / BK; + + var acc0 = vec4(0.0); + var acc1 = vec4(0.0); + var acc2 = vec4(0.0); + var acc3 = vec4(0.0); + + for (var tap = 0u; tap < taps; tap++) { + let ty_off = i32(tap / p.k) - i32(pad); + let tx_off = i32(tap % p.k) - i32(pad); + for (var chunk = 0u; chunk < chunks; chunk++) { + let ci0 = chunk * BK; + for (var item = tid; item < A_ITEMS; item += 256u) { + let ml = item / (BK / 4u); + let q = item % (BK / 4u); + let m = m0 + ml; + var v = vec4(0.0); + if (m < hw) { + let y = i32(m / p.w) + ty_off; + let x = i32(m % p.w) + tx_off; + if (y >= 0 && y < i32(p.h) && x >= 0 && x < i32(p.w)) { + let pix = u32(y) * p.w + u32(x); + let ci = ci0 + q * 4u; + if (ci < p.c0) { + v = src0[pix * (p.c0 / 4u) + ci / 4u]; + } else { + v = src1[pix * (p.c1 / 4u) + (ci - p.c0) / 4u]; + } + } + } + let base = (q * 4u) * (BM / 4u) + ml / 4u; + let lane = ml % 4u; + As[base][lane] = v.x; + As[base + BM / 4u][lane] = v.y; + As[base + 2u * (BM / 4u)][lane] = v.z; + As[base + 3u * (BM / 4u)][lane] = v.w; + } + for (var item = tid; item < B_ITEMS; item += 256u) { + let r = item / (BN / 4u); + let c4 = item % (BN / 4u); + let krow = tap * cin + ci0 + r; + Bs[item] = wgt[krow * (p.npad / 4u) + n0 / 4u + c4]; + } + workgroupBarrier(); + for (var kk = 0u; kk < BK; kk++) { + let a = As[kk * (BM / 4u) + ty]; + let b = Bs[kk * TX + tx]; + acc0 += a.x * b; + acc1 += a.y * b; + acc2 += a.z * b; + acc3 += a.w * b; + } + workgroupBarrier(); + } + } + + let n = n0 + tx * 4u; + if (n >= p.n) { + return; + } + let bv = bias[n / 4u]; + for (var i = 0u; i < 4u; i++) { + let m = m0 + ty * 4u + i; + if (m >= hw) { + continue; + } + var v = bv; + if (i == 0u) { v += acc0; } else if (i == 1u) { v += acc1; } else if (i == 2u) { v += acc2; } else { v += acc3; } + if (p.leaky != 0u) { + v = select(v * p.alpha, v, v >= vec4(0.0)); + } + if (p.mode == 0u) { + let o = m * p.n + n; + dst[o] = v.x; + dst[o + 1u] = v.y; + dst[o + 2u] = v.z; + dst[o + 3u] = v.w; + } else if (p.mode == 1u) { + let q = n / p.cout; + let co = n % p.cout; + let y = m / p.w; + let x = m % p.w; + let opix = (2u * y + q / 2u) * (2u * p.w) + 2u * x + q % 2u; + let o = opix * p.cout + co; + dst[o] = v.x; + dst[o + 1u] = v.y; + dst[o + 2u] = v.z; + dst[o + 3u] = v.w; + } else { + let c = n / 4u; + let y = m / p.w; + let x = m % p.w; + let ow = 2u * p.w; + let plane = c * (2u * p.h) * ow; + let o = plane + (2u * y) * ow + 2u * x; + dst[o] = v.x; + dst[o + 1u] = v.y; + dst[o + ow] = v.z; + dst[o + ow + 1u] = v.w; + } + } +} diff --git a/crates/gpu/src/wgsl/nn_pool.wgsl b/crates/gpu/src/wgsl/nn_pool.wgsl new file mode 100644 index 00000000..6117eb06 --- /dev/null +++ b/crates/gpu/src/wgsl/nn_pool.wgsl @@ -0,0 +1,28 @@ +// 2x2 maximum with stride 2 over an NHWC tensor read and written four channels at a time. + +struct P { + h: u32, + w: u32, + c4: u32, + total: u32, +} + +@group(0) @binding(0) var p: P; +@group(0) @binding(1) var src: array>; +@group(0) @binding(2) var dst: array>; + +@compute @workgroup_size(64) +fn main(@builtin(global_invocation_id) gid: vec3, @builtin(num_workgroups) nwg: vec3) { + let o = gid.x + gid.y * nwg.x * 64u; + if (o >= p.total) { + return; + } + let c = o % p.c4; + let pix = o / p.c4; + let ow = p.w / 2u; + let oy = pix / ow; + let ox = pix % ow; + let i00 = ((2u * oy) * p.w + 2u * ox) * p.c4 + c; + let i10 = i00 + p.w * p.c4; + dst[o] = max(max(src[i00], src[i00 + p.c4]), max(src[i10], src[i10 + p.c4])); +} diff --git a/crates/mcp/src/headless.rs b/crates/mcp/src/headless.rs index bdebec60..7d2d929c 100644 --- a/crates/mcp/src/headless.rs +++ b/crates/mcp/src/headless.rs @@ -31,7 +31,7 @@ impl Drop for Headless { impl Default for Headless { fn default() -> Self { - Self::new(Session::new().with_fs()) + Self::new(Session::new().with_fs().with_default_denoise_models()) } } @@ -42,7 +42,7 @@ impl Headless { /// A headless session with the procedurally generated demo library. pub fn demo() -> Self { - Self::new(Session::with_demo().with_fs()) + Self::new(Session::with_demo().with_fs().with_default_denoise_models()) } fn photo_or_active(&self, p: &Value) -> Result { diff --git a/crates/raw/src/lib.rs b/crates/raw/src/lib.rs index d63e5254..ec52b3dd 100644 --- a/crates/raw/src/lib.rs +++ b/crates/raw/src/lib.rs @@ -677,14 +677,27 @@ impl RawImage { /// The result is camera RGB (not white balanced), white level = 1.0, not oriented. pub fn develop(&self, method: Method) -> Result { let n = self.normalized()?; - let mut rgb = demosaic(&n, method); + let rgb = demosaic(&n, method); drop(n); + Ok(self.finish_demosaiced(rgb)) + } + + /// The last steps of [`Self::develop`] for camera RGB made some other way (a denoiser that demosaics too) from + /// [`Self::normalized`]'s mosaic: apply `OpcodeList3`, then the default crop. + pub fn finish_demosaiced(&self, mut rgb: Rgb32f) -> Rgb32f { opcodes::apply_list3(&self.opcodes.list3, &mut rgb); - let c = self.crop.clipped(rgb.width, rgb.height); - if c.width == 0 || c.height == 0 || (c.x == 0 && c.y == 0 && c.width == rgb.width && c.height == rgb.height) { - return Ok(rgb); + let c = self.develop_crop(rgb.width, rgb.height); + if c.x == 0 && c.y == 0 && c.width == rgb.width && c.height == rgb.height { + return rgb; } - Ok(rgb.into_crop(c.x, c.y, c.width, c.height)) + rgb.into_crop(c.x, c.y, c.width, c.height) + } + + /// The part of a `width × height` demosaiced picture (the active area) that [`Self::develop`] keeps: the default + /// crop, or all of it when there is none. + pub fn develop_crop(&self, width: usize, height: usize) -> Rect { + let c = self.crop.clipped(width, height); + if c.width == 0 || c.height == 0 { Rect::new(0, 0, width, height) } else { c } } } @@ -717,6 +730,43 @@ mod tests { } } + #[test] + fn a_picture_made_elsewhere_is_finished_like_develop_does() { + let (w, h) = (12usize, 10usize); + let raw = RawImage { + format: RawFormat::Dng, + width: w, + height: h, + cpp: 1, + data: RawData::U16((0..w * h).map(|i| (100 + (i * 37) % 900) as u16).collect()), + cfa: Cfa::bayer("RGGB"), + bits: 12, + black: BlackLevel::uniform(64.0), + white: vec![1023.0], + active_area: Rect::new(2, 2, 8, 6), + crop: Rect::new(1, 1, 5, 4), + orientation: Orientation::Normal, + color: ColorData::default(), + wb_multipliers: None, + linearized: false, + opcodes: OpcodeLists::default(), + metadata: Metadata::default(), + }; + let direct = raw.develop(Method::Bilinear).unwrap(); + let by_hand = raw.finish_demosaiced(demosaic(&raw.normalized().unwrap(), Method::Bilinear)); + assert_eq!((direct.width, direct.height), (5, 4)); + assert_eq!(direct, by_hand); + // the crop is what develop keeps; no usable crop keeps everything + assert_eq!(raw.develop_crop(8, 6), Rect::new(1, 1, 5, 4)); + let mut whole = raw.clone(); + whole.crop = Rect::default(); + assert_eq!(whole.develop_crop(8, 6), Rect::new(0, 0, 8, 6)); + assert_eq!(whole.develop(Method::Bilinear).unwrap().width, 8); + // a crop that pokes out is clipped to the picture + whole.crop = Rect::new(6, 4, 10, 10); + assert_eq!(whole.develop_crop(8, 6), Rect::new(6, 4, 2, 2)); + } + #[test] fn black_levels() { let b = BlackLevel { repeat_rows: 2, repeat_cols: 2, values: vec![1.0, 2.0, 3.0, 4.0], delta_h: vec![0.5, 0.0], delta_v: vec![0.0, 10.0] }; diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml index 49bdfc90..7836d96c 100644 --- a/crates/ui-egui/Cargo.toml +++ b/crates/ui-egui/Cargo.toml @@ -26,6 +26,7 @@ log = { workspace = true } [dev-dependencies] lightcraft-codecs = { workspace = true } +lightcraft-denoise = { workspace = true } [build-dependencies] serde_json = { workspace = true } diff --git a/crates/ui-egui/src/headless.rs b/crates/ui-egui/src/headless.rs index 206d75b1..48688b6a 100644 --- a/crates/ui-egui/src/headless.rs +++ b/crates/ui-egui/src/headless.rs @@ -194,6 +194,7 @@ impl Headless { || self.app.scan.is_some() || self.app.import.is_some() || self.app.export.is_some() + || self.app.session.denoise_busy() || !self.app.tasks.is_empty() || !self.app.synthetic.is_empty() || !self.events.is_empty() @@ -291,6 +292,61 @@ impl Headless { #[cfg(test)] mod tests { + #[test] + fn detail_offers_ai_denoise_for_raw_photos_and_leads_to_its_settings() { + use crate::state::Dialog; + use lightcraft_catalog::{MediaKind, Op, Photo, PhotoId, Source}; + let mut h = demo([1400.0, 900.0]); + let t = Duration::from_secs(10); + let dir = std::env::temp_dir().join(format!("lc-ui-denoise-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + h.app.session.set_denoise_models_dir(Some(dir.join("denoise-models"))); + // a raw photo (its file need not exist: nothing here reads it) + let id = PhotoId(h.app.session.catalog.photos().map(|p| p.id.0).max().unwrap_or(0) + 1); + let mut raw = Photo::new( + id, + Source::File { path: dir.join("IMG_1.dng").to_string_lossy().into_owned() }, + "IMG_1.dng", + "DNG", + 6000, + 4000, + "2026-10-01T00:00:00", + ); + raw.kind = MediaKind::Raw; + h.app.session.commit("setup", Op::AddPhoto { photo: Box::new(raw) }).unwrap(); + let widgets = |h: &mut Headless| h.request("ui.widgets", json!({}), t).to_string(); + h.request("engine.execute", json!({"command": "library.select", "params": {"ids": [id.0]}}), t); + h.request("ui.set", json!({"view": "detail"}), t); + h.app.ui.open_sections = vec!["detail".to_string()]; + h.settle(SETTLE); + h.step(); + h.step(); + let w = widgets(&mut h); + assert!(w.contains("\"denoise:setup\""), "a raw photo with no model is offered the setup: {w}"); + // one click lands in Settings ▸ AI Denoise, where the model LightCraft knows is listed + let r = h.request("ui.clickWidget", json!({"id": "denoise:setup"}), t); + assert_eq!(r["ok"], true, "{r}"); + h.step(); + h.step(); + assert_eq!(h.app.ui.dialog, Some(Dialog::Settings { tab: "denoise".into() })); + h.settle(SETTLE); + let listed = h.app.session.execute("denoise.models.list", &json!({})).unwrap(); + assert!(listed["models"].as_array().unwrap().iter().any(|m| m["id"] == "rawnind-bayer" && m["installed"] == false), "{listed}"); + h.app.ui.dialog = None; + // a photo that is not raw: no offer + h.request( + "engine.execute", + json!({"command": "library.select", "params": {"ids": [h.app.session.catalog.photos().find(|p| p.id != id).unwrap().id.0]}}), + t, + ); + h.settle(SETTLE); + h.step(); + h.step(); + assert!(!widgets(&mut h).contains("\"denoise:setup\""), "nothing is offered for a photo denoise does not apply to"); + let _ = std::fs::remove_dir_all(&dir); + } + use super::*; /// Generous: renders are slow when the machine is loaded (parallel builds), and a timed-out diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index aeab2794..be2095fc 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -19,6 +19,7 @@ pub mod menu_level; pub mod menubar; pub mod menus; pub mod merge; +mod model_setup; pub mod panels; pub mod pick; pub mod region; @@ -157,6 +158,7 @@ pub struct Perf { } pub struct LightcraftApp { + pub(crate) model_setup: model_setup::Pending, /// Per-catalog-revision caches of library-wide results the panels show every frame /// (expensive on big libraries). pub caches: Caches, @@ -312,6 +314,7 @@ impl LightcraftApp { gpu_applied: None, memory_applied: None, library_problem: None, + model_setup: Default::default(), } } @@ -324,6 +327,9 @@ impl LightcraftApp { /// Run a UI or engine command by id. The single entry point for every frontend path. pub fn run(&mut self, id: &str, params: Value) -> Result { + if let Some(result) = model_setup::intercept(self, id, ¶ms) { + return result; + } if let Some(r) = menus::run_ui_command(self, id, ¶ms) { return r; } @@ -751,6 +757,8 @@ impl LightcraftApp { ctx.request_repaint_after(std::time::Duration::from_secs(3)); } self.session.persist_if_dirty(); + panels::denoise::pump(self, ctx); + model_setup::pump(self, ctx); self.collect_screenshots(ctx); self.issue_screenshots(ctx); if self.fonts_ready { @@ -988,6 +996,11 @@ pub struct Caches { album_counts: Option<(u64, std::sync::Arc>)>, /// How often the album counts were recomputed (tests check that unchanged frames don't). pub album_count_scans: usize, + /// AI denoise: what the pump last saw, the model list and the downloads being watched. + pub denoise: panels::denoise::Ui, + /// When the user last dragged, typed or scrolled, and when they last moved the pointer (egui time). + pub last_input: f64, + pub last_move: f64, /// The grid's date runs, layout and indexes (by the visible list's generation). pub grid: panels::grid::GridCache, /// What the grid did on its frames (benchmarks and tests check unchanged frames stay cheap). @@ -1213,3 +1226,6 @@ mod cache_tests { assert_eq!(c.album_count_scans, 1); } } + +#[cfg(test)] +mod tests_model_setup; diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index 5ca01cb4..96fc1dc9 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -47,6 +47,7 @@ pub fn language_from_command(id: &str) -> Option { } pub const UI_COMMANDS: &[UiCommand] = &[ + ("modelSetup.cancel", "Cancel Pending Photo Action", None, ""), ("view.photoGrid", "Photo Grid", None, "View"), ("view.squareGrid", "Square Grid", None, "View"), // G: Photo Grid ↔ Square Grid (from other views: the photo grid) diff --git a/crates/ui-egui/src/model_setup.rs b/crates/ui-egui/src/model_setup.rs new file mode 100644 index 00000000..238a4afe --- /dev/null +++ b/crates/ui-egui/src/model_setup.rs @@ -0,0 +1,133 @@ +//! Resume a denoise request after model setup, bound to its original photo and library. + +use crate::{LightcraftApp, state::Dialog}; +use lightcraft_catalog::{PhotoId, Source}; +use lightcraft_engine::denoise::PhotoState; +use serde_json::{Value, json}; + +struct Request { + library: u64, + photo: PhotoId, + source: Source, +} + +impl Request { + fn capture(app: &LightcraftApp, id: PhotoId) -> Result { + let photo = app.session.catalog.photo(id).filter(|p| !p.deleted).ok_or("the requested photo is no longer available")?; + Ok(Self { library: app.session.library_generation(), photo: id, source: photo.source.clone() }) + } + + fn valid(&self, app: &LightcraftApp) -> bool { + self.library == app.session.library_generation() + && app.session.catalog.photo(self.photo).is_some_and(|p| !p.deleted && p.source == self.source) + } +} + +#[derive(Default)] +pub(crate) struct Pending { + denoise: Option, + next_check: f64, +} + +pub(crate) fn denoise_requested(app: &LightcraftApp, id: PhotoId) -> bool { + app.model_setup.denoise.as_ref().is_some_and(|r| r.photo == id && r.valid(app)) +} + +pub(crate) fn intercept(app: &mut LightcraftApp, command: &str, params: &Value) -> Option> { + if matches!(command, "app.openLibrary" | "file.restoreLibrary") { + app.model_setup = Pending::default(); + return None; + } + if command == "modelSetup.cancel" { + if params.get("kind").and_then(Value::as_str) != Some("denoise") { + return Some(Err("modelSetup.cancel: use kind denoise".into())); + } + app.model_setup.denoise = None; + app.ui.status = "Pending photo action cancelled".into(); + return Some(Ok(json!({"cancelled": true}))); + } + if command == "denoise.models.downloadCancel" { + app.model_setup.denoise = None; + } + if command != "denoise.toggle" { + return None; + } + let id = match params.get("id") { + Some(v) => match v.as_u64() { + Some(id) => PhotoId(id), + None => return Some(Err("denoise.toggle: id must be a photo id".into())), + }, + None => match app.session.active() { + Some(id) => id, + None => return Some(Err("select a raw photo".into())), + }, + }; + let enabled = app.session.develop_of(id).is_some_and(|d| d.enhance.denoise_enabled()); + let on = match params.get("enabled") { + Some(v) => match v.as_bool() { + Some(on) => on, + None => return Some(Err("denoise.toggle: enabled must be true or false".into())), + }, + None => !enabled && !denoise_requested(app, id), + }; + if !on { + app.model_setup.denoise = None; + return Some(app.session.execute("denoise.toggle", &json!({"id": id.0, "enabled": false})).map_err(|e| e.to_string())); + } + let _ = app.session.execute("denoise.status", &json!({"id": id.0})); + if !matches!(app.session.denoise_photo_state(id), PhotoState::NoModel) { + return None; + } + if app.session.execute("denoise.models.list", &json!({})).ok().is_none_or(|v| v["dir"].is_null()) { + return None; + } + Some(Request::capture(app, id).map(|request| { + app.model_setup.denoise = Some(request); + app.model_setup.next_check = 0.0; + app.ui.dialog = Some(Dialog::Settings { tab: "denoise".into() }); + app.ui.status = "Install a denoise model to enable AI Denoise on this photo".into(); + json!({"setupRequired": true, "photo": id.0}) + })) +} + +pub(crate) fn notice(app: &mut LightcraftApp, ui: &mut egui::Ui, kind: &str) { + if kind != "denoise" || !app.model_setup.denoise.as_ref().is_some_and(|r| r.valid(app)) { + return; + } + ui.add(egui::Label::new("After installation, AI Denoise will turn on for the requesting photo.").wrap()); + let r = ui.small_button("Cancel pending action"); + crate::widgets::register(ui.ctx(), "modelSetup:cancel:denoise", r.rect); + if r.clicked() { + let _ = app.run("modelSetup.cancel", json!({"kind": "denoise"})); + } + ui.add_space(6.0); +} + +pub(crate) fn pump(app: &mut LightcraftApp, ctx: &egui::Context) { + if app.model_setup.denoise.is_none() { + return; + } + if app.session.interaction.is_some() { + ctx.request_repaint_after(std::time::Duration::from_millis(250)); + return; + } + let now = ctx.input(|i| i.time); + if now < app.model_setup.next_check { + return; + } + app.model_setup.next_check = now + 0.25; + let Some(request) = app.model_setup.denoise.take() else { return }; + if !request.valid(app) { + app.ui.status = "Pending photo action cancelled because its library or photo changed".into(); + return; + } + if matches!(app.session.denoise_photo_state(request.photo), PhotoState::NoModel | PhotoState::NotApplicable) { + app.model_setup.denoise = Some(request); + ctx.request_repaint_after(std::time::Duration::from_millis(250)); + return; + } + match app.run("denoise.toggle", json!({"id": request.photo.0, "enabled": true})) { + Ok(_) => app.ui.status = "AI Denoise is enabled on the requesting photo".into(), + Err(error) => app.ui.status = error, + } +} diff --git a/crates/ui-egui/src/panels/denoise.rs b/crates/ui-egui/src/panels/denoise.rs new file mode 100644 index 00000000..bd69b0cf --- /dev/null +++ b/crates/ui-egui/src/panels/denoise.rs @@ -0,0 +1,647 @@ +//! AI denoise in the interface: the line under the Detail panel's Denoise slider (what the picture is doing), the +//! Settings ▸ Denoise tab (the model and the cache) and the per-frame pump that keeps the engine's background work going. +//! +//! The Amount slider is an ordinary develop control (`enhance.denoise`); everything here is about the cleaned picture the +//! slider mixes in. It is cached data made in the background for the photos being looked at, and never a file in the +//! library. A model is fetched only when the user presses Download and accepts its terms. + +use egui::RichText; +use lightcraft_catalog::PhotoId; +use lightcraft_engine::denoise::PhotoState; +use serde_json::{Value, json}; + +use super::settings::{check, choices, heading, hint}; +use crate::LightcraftApp; +use crate::theme::Tokens; +use crate::widgets::register; + +/// How long a read of the model list is reused (it is a few small files, but not for every frame). +const REFRESH_SECS: f64 = 1.5; +/// Cache limits offered (GB). +const CACHE_LIMITS: [u32; 6] = [5, 10, 20, 50, 100, 250]; + +/// What the interface keeps between frames. +#[derive(Default)] +pub struct Ui { + next_pump: f64, + generation: u64, + active: bool, + /// (tiles done, tiles in all) of the photo being made. + running: Option<(u64, u64)>, + queued: u64, + /// Bumped when a model is installed, removed or chosen, so Settings re-reads the list at once. + pub epoch: u64, + /// Models the user pressed Download for that have not been installed yet. + dl_watch: Vec, + list: Option<(u64, f64, Value)>, +} + +/// Called every frame: keeps the engine's denoise work going and asks for a redraw when a photo's picture appears (the +/// loupe and the thumbnails then render from it) or while one is being made (the progress). +pub fn pump(app: &mut LightcraftApp, ctx: &egui::Context) { + watch_downloads(app, ctx); + let now = ctx.input(|i| i.time); + let (working, moving) = ctx.input(|i| { + let keys = i.events.iter().any(|e| matches!(e, egui::Event::Key { .. } | egui::Event::Text(_) | egui::Event::MouseWheel { .. })); + (i.pointer.any_down() || keys, i.pointer.is_moving()) + }); + if working { + app.caches.last_input = now; + } + if moving || working { + app.caches.last_move = now; + } + if now < app.caches.denoise.next_pump { + return; + } + let watching = matches!(&app.ui.dialog, Some(crate::state::Dialog::Settings { tab }) if tab == "denoise"); + let pace = window_pace(app, ctx, now, watching); + let Ok(v) = app.session.execute("denoise.pump", &json!({"pace": pace})) else { return }; + let c = &mut app.caches.denoise; + let generation = v["generation"].as_u64().unwrap_or(0); + if generation != c.generation { + c.generation = generation; + ctx.request_repaint(); + } + c.active = v["active"] == true; + c.queued = v["queued"].as_u64().unwrap_or(0); + c.running = v["running"].is_object().then(|| (v["running"]["done"].as_u64().unwrap_or(0), v["running"]["total"].as_u64().unwrap_or(0))); + let busy = c.running.is_some() || c.queued > 0; + c.next_pump = now + + if busy { + 0.1 + } else if c.active { + 0.5 + } else { + 1.0 + }; + // frames are drawn only when something asks for one: while a photo is being made, ask again for its progress + if busy { + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } +} + +/// The engine's model list, re-read at most every [`REFRESH_SECS`], or at once after an action changed it. +fn models(app: &mut LightcraftApp, ctx: &egui::Context) -> Value { + let now = ctx.input(|i| i.time); + let epoch = app.caches.denoise.epoch; + if let Some((e, at, v)) = &app.caches.denoise.list + && *e == epoch + && now - at < REFRESH_SECS + { + return v.clone(); + } + let v = app.run("denoise.models.list", json!({})).unwrap_or(Value::Null); + app.caches.denoise.list = Some((epoch, now, v.clone())); + v +} + +/// Follows the downloads the user started: keeps redrawing while one runs (the progress bar), and says so when a +/// model has been installed and is in use (the engine does all of that by itself). +fn watch_downloads(app: &mut LightcraftApp, ctx: &egui::Context) { + if app.caches.denoise.dl_watch.is_empty() { + return; + } + let Ok(v) = app.session.execute("denoise.models.downloads", &json!({})) else { + app.caches.denoise.dl_watch.clear(); + return; + }; + let rows: Vec = v["downloads"].as_array().cloned().unwrap_or_default(); + let (mut keep, mut installed) = (Vec::new(), Vec::new()); + for id in std::mem::take(&mut app.caches.denoise.dl_watch) { + match rows.iter().find(|r| r["id"] == id.as_str()).and_then(|r| r["state"].as_str()) { + Some("running" | "done") => keep.push(id), + Some("installed") => installed.push(id), + _ => {} + } + } + app.caches.denoise.dl_watch = keep; + for id in installed { + let _ = app.session.execute("denoise.models.downloadCancel", &json!({"id": id})); + app.caches.denoise.epoch += 1; + let text = "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works"; + app.ui.status = text.into(); + app.toast(ctx, text); + } + ctx.request_repaint_after(std::time::Duration::from_millis(120)); +} + +/// Per-photo AI Denoise switch, including a request waiting for model installation. +pub fn detail_toggle(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId, on: bool, applicable: bool) { + let mut enabled = on || crate::model_setup::denoise_requested(app, id); + egui::Frame::NONE.inner_margin(egui::Margin { left: 24, right: 22, top: 6, bottom: 2 }).show(ui, |ui| { + let r = ui.add_enabled(applicable, egui::Checkbox::new(&mut enabled, crate::i18n::tr("AI Denoise"))); + register(ui.ctx(), "denoise:enabled", r.rect); + if r.changed() { + let _ = app.run("denoise.toggle", json!({"id": id.0, "enabled": enabled})); + } + }); +} + +/// The line under the Denoise slider for the photo `id`, whose Amount is `amount`. +pub fn detail_status(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId, amount: f64) { + let t = Tokens::get(ui.ctx()); + let state = app.session.denoise_photo_state(id); + let pad = egui::Margin { left: 24, right: 22, top: 0, bottom: 6 }; + let note = |ui: &mut egui::Ui, text: &str, color: egui::Color32| { + ui.add(egui::Label::new(RichText::new(crate::i18n::tr(text)).size(11.0).color(color)).wrap()); + }; + match state { + PhotoState::NotApplicable | PhotoState::Ready => {} + PhotoState::NoModel => { + egui::Frame::NONE.inner_margin(pad).show(ui, |ui| { + note(ui, "AI Denoise needs a model: a one-time download, in Settings.", t.text_dim); + ui.add_space(2.0); + let r = ui.small_button(crate::i18n::tr("Set up AI Denoise…")); + register(ui.ctx(), "denoise:setup", r.rect); + if r.clicked() { + let _ = app.run("denoise.toggle", json!({"id": id.0, "enabled": true})); + } + }); + } + PhotoState::Idle => { + if amount > 0.0 && app.session.denoise_auto() { + // the pump has not met this photo yet: let it come round soon instead of waiting out its pause + let now = ui.ctx().input(|i| i.time); + app.caches.denoise.next_pump = app.caches.denoise.next_pump.min(now + 0.05); + ui.ctx().request_repaint_after(std::time::Duration::from_millis(60)); + egui::Frame::NONE.inner_margin(pad).show(ui, |ui| note(ui, "Waiting to start…", t.text_dim)); + } else if amount > 0.0 { + egui::Frame::NONE.inner_margin(pad).show(ui, |ui| { + note(ui, "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).", t.text_dim); + ui.add_space(2.0); + let r = ui.small_button(crate::i18n::tr("Make it now")); + register(ui.ctx(), "denoise:make", r.rect); + if r.clicked() { + let _ = app.run("denoise.queue", json!({"ids": [id.0]})); + } + }); + } + } + PhotoState::Queued { ahead } => { + egui::Frame::NONE.inner_margin(pad).show(ui, |ui| { + let text = if ahead == 0 { "Next up for AI Denoise…".to_string() } else { format!("In line for AI Denoise ({ahead} ahead)…") }; + note(ui, &text, t.text_dim); + }); + } + PhotoState::Running { done, total } => { + egui::Frame::NONE.inner_margin(pad).show(ui, |ui| { + let frac = if total > 0 { (done as f32 / total as f32).clamp(0.0, 1.0) } else { 0.0 }; + let text = + if total > 0 { format!("Making the denoised picture… {}%", (frac * 100.0).round() as u32) } else { "Starting…".to_string() }; + let bar = ui.add(egui::ProgressBar::new(frac).desired_width(ui.available_width().min(240.0)).text(RichText::new(text).size(11.0))); + register(ui.ctx(), "denoise:progress", bar.rect); + }); + } + PhotoState::Failed { why, unsupported } => { + egui::Frame::NONE.inner_margin(pad).show(ui, |ui| { + let color = if unsupported { t.text_dim } else { t.caution }; + note(ui, &sentence(&why), color); + if !unsupported { + let r = ui.small_button(crate::i18n::tr("Try again")); + register(ui.ctx(), "denoise:retry", r.rect); + if r.clicked() { + let _ = app.run("denoise.queue", json!({"ids": [id.0], "retry": true})); + } + } + }); + } + } +} + +/// The Settings ▸ Denoise tab. +pub fn settings_tab(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens) { + let list = models(app, ui.ctx()); + heading(ui, t, "AI Denoise"); + crate::model_setup::notice(app, ui, "denoise"); + if list["dir"].is_null() { + hint(ui, t, "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)"); + return; + } + hint( + ui, + t, + "Cleans noise out of raw photos with an AI model, before the picture is built: the AI Denoise Amount slider under Detail sets how much of it you see. \ + Your files are never changed and no extra files are made in your library: the cleaned picture is cached data that is made again when needed.", + ); + let runtime = list["runtime"].as_bool() == Some(true); + if !runtime { + hint(ui, t, "This build cannot run denoise models."); + } + let all: Vec = list["models"].as_array().cloned().unwrap_or_default(); + let downloads = app + .session + .execute("denoise.models.downloads", &json!({})) + .map(|v| v["downloads"].as_array().cloned().unwrap_or_default()) + .unwrap_or_default(); + heading(ui, t, "Models"); + for m in &all { + let dl = downloads.iter().find(|d| d["id"] == m["id"]); + model_row(app, ui, t, m, dl, runtime); + } + if all.iter().all(|m| m["selected"] != true) && all.iter().any(|m| m["installed"] == true) { + hint(ui, t, "AI Denoise is off: press Use on a model."); + } + hint(ui, t, "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction."); + work_section(app, ui, t, &list); +} + +/// What the cache holds, its limit, and what is being made. +fn work_section(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, list: &Value) { + let status = app.session.execute("denoise.status", &json!({})).unwrap_or(Value::Null); + if status["enabled"] != true { + return; + } + heading(ui, t, "Speed"); + let current = list["runOn"].as_str().unwrap_or("auto").to_string(); + let mut run_on = current.as_str(); + let options = [("auto", "Automatic"), ("gpu", "Graphics card"), ("cpu", "Processor")]; + if ui.horizontal(|ui| choices(ui, "denoiseRunOn", &options, &mut run_on)).inner { + let _ = app.run("denoise.settings", json!({"runOn": run_on})); + app.caches.denoise.epoch += 1; + } + hint( + ui, + t, + "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.", + ); + hint(ui, t, &device_line(&status["device"], run_on)); + if run_on != "cpu" && status["device"]["retry"] == true { + let r = ui.small_button(crate::i18n::tr("Try the graphics card again")); + register(ui.ctx(), "denoise:retryGpu", r.rect); + if r.clicked() { + // choosing where it runs, even the same again, forgets the failed set-up + let _ = app.run("denoise.settings", json!({"runOn": run_on})); + app.caches.denoise.epoch += 1; + } + } + heading(ui, t, "Photos"); + let mut auto = list["auto"].as_bool().unwrap_or(true); + if check(ui, "settings.denoiseAuto", &mut auto, "Make the denoised picture of the photos I am looking at") { + let _ = app.run("denoise.settings", json!({"auto": auto})); + app.caches.denoise.epoch += 1; + } + hint(ui, t, "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing."); + let (running, queued) = (app.caches.denoise.running, app.caches.denoise.queued); + match running { + Some((done, total)) => { + let frac = if total > 0 { (done as f32 / total as f32).clamp(0.0, 1.0) } else { 0.0 }; + let name = status["running"]["photo"] + .as_u64() + .and_then(|p| app.session.catalog.photo(PhotoId(p)).map(|p| p.file_name.clone())) + .unwrap_or_default(); + let text = + format!("Denoising {name}: {done} of {total} tiles{}", if queued > 0 { format!(" · {queued} waiting") } else { String::new() }); + let bar = ui.add(egui::ProgressBar::new(frac).desired_width(380.0).text(RichText::new(text).size(11.5))); + register(ui.ctx(), "denoise:settingsProgress", bar.rect); + } + None if queued > 0 => hint(ui, t, &count(queued, "photo waiting", "photos waiting")), + None => { + let ready = status["ready"].as_u64().unwrap_or(0); + hint(ui, t, &count(ready, "photo has its denoised picture.", "photos have their denoised picture.")) + } + } + ui.horizontal(|ui| { + let r = ui.button("Denoise all photos with an Amount"); + register(ui.ctx(), "denoise:queueAll", r.rect); + if r.clicked() { + let _ = app.run("denoise.queue", json!({"scope": "withAmount"})); + } + if running.is_some() || queued > 0 { + let r = ui.button("Stop"); + register(ui.ctx(), "denoise:stop", r.rect); + if r.clicked() { + let _ = app.run("denoise.cancel", json!({})); + } + } + }); + heading(ui, t, "Cache"); + let (files, bytes) = (status["cache"]["files"].as_u64().unwrap_or(0), status["cache"]["bytes"].as_u64().unwrap_or(0)); + hint( + ui, + t, + &format!( + "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.", + count(files, "cached picture", "cached pictures"), + mb(Some(bytes)) + ), + ); + ui.horizontal(|ui| { + ui.label(RichText::new(crate::i18n::tr("Limit")).color(t.text_label)); + let current = list["cacheGb"].as_u64().unwrap_or(20) as u32; + let mut choice = current; + egui::ComboBox::from_id_salt("denoise-cache-limit").selected_text(format!("{current} GB")).show_ui(ui, |ui| { + for gb in CACHE_LIMITS { + ui.selectable_value(&mut choice, gb, format!("{gb} GB")); + } + }); + if choice != current { + let _ = app.run("denoise.settings", json!({"cacheGb": choice})); + app.caches.denoise.epoch += 1; + } + let r = ui.button("Clear cache"); + register(ui.ctx(), "denoise:clear", r.rect); + if r.clicked() { + let _ = app.run("denoise.clear", json!({})); + } + }); +} + +/// Where the work runs, in a sentence (`denoise.status` → `device`), for the setting `run_on`. No timings: they are this +/// computer's, and only the comparison matters. +fn device_line(d: &Value, run_on: &str) -> String { + match d["kind"].as_str() { + _ if run_on == "cpu" => "The processor does the work.".into(), + Some("gpu") => format!("Running on {}.", d["adapter"].as_str().unwrap_or("the graphics card")), + Some("cpu") => { + format!("Running on the processor: {}.", d["reason"].as_str().unwrap_or("the graphics card is not used").trim_end_matches('.')) + } + _ => "The graphics card is set up when the next photo is made.".into(), + } +} + +fn model_row(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, m: &Value, dl: Option<&Value>, can_run: bool) { + let id = m["id"].as_str().unwrap_or("").to_string(); + let (installed, selected) = (m["installed"].as_bool() == Some(true), m["selected"].as_bool() == Some(true)); + let dl_state = if installed { None } else { dl.and_then(|d| d["state"].as_str()) }; + let host = if can_run { m["downloadHost"].as_str().unwrap_or("").to_string() } else { String::new() }; + egui::Frame::NONE.inner_margin(egui::Margin::symmetric(0, 3)).show(ui, |ui| { + ui.horizontal(|ui| { + ui.vertical(|ui| { + ui.horizontal(|ui| { + ui.label(RichText::new(m["name"].as_str().unwrap_or("")).color(t.text)); + let size = mb(m["sizeBytes"].as_u64()); + if !size.is_empty() { + ui.label(RichText::new(size).color(t.text_dim)); + } + if selected { + ui.label(RichText::new("in use").color(t.accent)); + } else if installed { + ui.label(RichText::new("installed").color(t.text_dim)); + } + }); + ui.label(RichText::new(licence_line(m)).font(t.font(11.5)).color(if m["licence"]["commercial"] == "yes" { + t.text_dim + } else { + t.caution + })); + match m["accepted"]["selfTest"].as_object() { + Some(test) if test.get("ok").and_then(Value::as_bool) == Some(true) => { + ui.label(RichText::new("Tested and works on this computer").font(t.font(11.5)).color(t.text_dim)); + } + Some(_) => { + ui.label(RichText::new("Failed its last test").font(t.font(11.5)).color(t.caution)); + } + None => {} + } + match (dl_state, dl) { + (Some("running"), Some(d)) => { + let (got, total) = (d["bytes"].as_u64().unwrap_or(0), d["total"].as_u64().unwrap_or(0)); + let frac = if total > 0 { (got as f32 / total as f32).clamp(0.0, 1.0) } else { 0.0 }; + let text = if total > 0 && got >= total { + "Checking…".to_string() + } else { + format!("{} of {} from {host}", mb(Some(got)), mb(Some(total))) + }; + let bar = ui.add(egui::ProgressBar::new(frac).desired_width(260.0).text(RichText::new(text).font(t.font(11.5)))); + register(ui.ctx(), format!("denoise:progress:{id}"), bar.rect); + } + (Some("done"), _) => { + ui.label(RichText::new("Downloaded and checked. Installing…").font(t.font(11.5)).color(t.text_dim)); + } + (Some("failed"), Some(d)) => { + let why = sentence(d["error"].as_str().unwrap_or("The download failed")); + ui.add(egui::Label::new(RichText::new(why).font(t.font(11.5)).color(t.caution)).wrap()); + } + _ => {} + } + }); + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + if installed { + let r = ui.button("Remove"); + register(ui.ctx(), format!("denoise:remove:{id}"), r.rect); + if r.clicked() { + let _ = app.run("denoise.models.remove", json!({"id": id})); + app.caches.denoise.epoch += 1; + } + if selected { + let r = ui.button("Turn off"); + register(ui.ctx(), format!("denoise:off:{id}"), r.rect); + if r.clicked() { + let _ = app.run("denoise.models.select", json!({"id": null})); + app.caches.denoise.epoch += 1; + } + } else { + let r = ui.button("Use"); + register(ui.ctx(), format!("denoise:use:{id}"), r.rect); + if r.clicked() { + let _ = app.run("denoise.models.select", json!({"id": id})); + app.caches.denoise.epoch += 1; + } + } + } else { + match dl_state { + Some("running") => { + let r = ui.button("Cancel"); + register(ui.ctx(), format!("denoise:cancelDownload:{id}"), r.rect); + if r.clicked() { + let _ = app.run("denoise.models.downloadCancel", json!({"id": id})); + app.caches.denoise.dl_watch.retain(|w| w != &id); + } + } + // the engine installs it within a moment; nothing to press + Some("done") => {} + _ => { + if let Some(url) = m["source"].as_str() { + let r = ui + .button("Open page") + .on_hover_text("Opens the model's own page in your browser, to read about it or get the file yourself."); + register(ui.ctx(), format!("denoise:get:{id}"), r.rect); + if r.clicked() { + open_page(app, url); + } + } + if !host.is_empty() { + let label = if dl_state == Some("failed") { "Try again" } else { "Download" }; + let tip = format!( + "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.", + mb(m["sizeBytes"].as_u64()) + ); + let r = ui.button(label).on_hover_text(tip); + register(ui.ctx(), format!("denoise:download:{id}"), r.rect); + if r.clicked() { + open_download_dialog(app, m); + } + } + } + } + } + }); + }); + }); +} + +/// Pressing Download: the model's terms first, in the licence dialog. Accepting them starts the download; +/// nothing is fetched before. +fn open_download_dialog(app: &mut LightcraftApp, m: &Value) { + let info = json!({ + "kind": "known", + "domain": "denoise", + "fileName": m["id"], + "sizeBytes": m["sizeBytes"], + "model": m, + "alreadyInstalled": false, + "download": m["id"], + "host": m["downloadHost"], + "assumptions": [], + "reason": null, + }); + app.ui.dialog = Some(crate::state::Dialog::DenoiseModel { info, accepted: false }); +} + +/// The dialog's OK for a denoise model: start the download and watch it. +pub fn install(app: &mut LightcraftApp, info: &Value, accepted: bool) -> Result { + if !accepted { + return Err("Tick the box to accept the model's terms first".into()); + } + app.caches.denoise.epoch += 1; + let id = info["download"].as_str().ok_or("There is no download for this model")?; + let r = app.run("denoise.models.download", json!({"id": id, "acknowledged": true})); + if r.is_ok() { + app.caches.denoise.dl_watch.push(id.to_string()); + // the progress is in Settings: stay there + app.ui.dialog = Some(crate::state::Dialog::Settings { tab: "denoise".into() }); + } + r +} + +/// "1 photo" / "3 photos": the count with the right noun. +fn count(n: u64, one: &str, many: &str) -> String { + format!("{n} {}", if n == 1 { one } else { many }) +} + +pub(super) fn window_pace(app: &LightcraftApp, ctx: &egui::Context, now: f64, watching: bool) -> &'static str { + let (focused, minimized) = ctx.input(|i| (i.focused, i.raw.viewports.get(&i.raw.viewport_id).and_then(|v| v.minimized).unwrap_or(false))); + pace_for(focused, minimized, now - app.caches.last_input, now - app.caches.last_move, watching) +} + +fn pace_for(focused: bool, minimized: bool, worked: f64, moved: f64, watching: bool) -> &'static str { + if minimized { + "light" + } else if !focused { + // still on screen, but the user is working in another app: no input here to get in the way of + "normal" + } else if worked < 0.4 { + "pause" + } else if moved < 3.0 { + "light" + } else if watching { + "full" + } else { + "normal" + } +} + +pub(super) fn mb(bytes: Option) -> String { + match bytes { + Some(b) if b >= 10_000_000 => format!("{} MB", (b as f64 / 1e6).round() as u64), + Some(b) if b >= 1_000_000 => format!("{:.1} MB", b as f64 / 1e6), + Some(b) => format!("{} KB", (b as f64 / 1e3).round().max(1.0) as u64), + None => String::new(), + } +} + +pub(super) fn sentence(s: &str) -> String { + let s = s.trim(); + let mut c = s.chars(); + let mut out: String = c.next().map(|f| f.to_uppercase().collect()).unwrap_or_default(); + out.push_str(c.as_str()); + if !out.is_empty() && !out.ends_with(['.', '!', '?']) { + out.push('.'); + } + out +} + +pub(super) fn licence_line(m: &Value) -> String { + let name = m["licence"]["name"].as_str().filter(|s| !s.is_empty()).unwrap_or("Unknown licence"); + let terms = match m["licence"]["commercial"].as_str() { + Some("yes") => "commercial use allowed", + Some("no") => "non-commercial use only", + _ => "terms unclear", + }; + format!("{name} · {terms}") +} + +pub(super) fn open_page(app: &mut LightcraftApp, url: &str) { + if let Some(f) = app.services.open_url.as_mut() { + let _ = f(url); + } +} + +pub fn model_dialog(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, info: &Value, accepted: &mut bool) { + let kind = info["kind"].as_str().unwrap_or("unsupported"); + let file = info["fileName"].as_str().unwrap_or(""); + if kind == "unsupported" { + ui.label(RichText::new("This file cannot be used yet").font(t.semibold(13.5)).color(t.caution)); + ui.add( + egui::Label::new( + RichText::new(sentence(info["reason"].as_str().unwrap_or("It is not a denoise model LightCraft understands"))).color(t.text_label), + ) + .wrap(), + ); + ui.label(RichText::new(format!("{file} · {}", mb(info["sizeBytes"].as_u64()))).color(t.text_dim)); + return; + } + let m = &info["model"]; + ui.horizontal(|ui| { + ui.label(RichText::new(m["name"].as_str().unwrap_or(file)).font(t.semibold(14.0)).color(t.text)); + ui.label(RichText::new(mb(info["sizeBytes"].as_u64())).color(t.text_dim)); + }); + if info["alreadyInstalled"] == true { + ui.label(RichText::new("Already installed. Installing again is harmless.").color(t.text_dim)); + } + let download = info["download"].is_string(); + if download { + let host = info["host"].as_str().unwrap_or("its own repository"); + let after = if info["domain"] == "denoise" { "starts using it for AI Denoise" } else { "makes AI Denoise available" }; + ui.add( + egui::Label::new( + RichText::new(format!("Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.")) + .color(t.text_label), + ) + .wrap(), + ); + } + let commercial = m["licence"]["commercial"].as_str().unwrap_or("unknown"); + ui.label(RichText::new(licence_line(m)).font(t.semibold(12.5)).color(if commercial == "yes" { t.text } else { t.caution })); + let notice = m["licence"]["notice"].as_str().unwrap_or(""); + if !notice.is_empty() { + ui.add(egui::Label::new(RichText::new(notice).color(t.text_label)).wrap()); + } + let provenance = m["provenance"].as_str().unwrap_or(""); + if !provenance.is_empty() { + ui.add(egui::Label::new(RichText::new(format!("Trained on: {provenance}")).color(t.text_dim)).wrap()); + } + if kind == "draft" { + ui.add_space(2.0); + ui.label(RichText::new("LightCraft does not know this model, so it assumed:").color(t.text_label)); + for a in info["assumptions"].as_array().into_iter().flatten().filter_map(Value::as_str) { + ui.add(egui::Label::new(RichText::new(format!("• {a}")).font(t.font(12.0)).color(t.text_dim)).wrap()); + } + } + if let Some(url) = m["source"].as_str().or(m["licence"]["url"].as_str()) { + let r = ui.link("Open the model's page"); + register(ui.ctx(), "denoiseModel:page", r.rect); + if r.clicked() { + open_page(app, url); + } + } + ui.add_space(4.0); + check(ui, "denoiseModel.accept", accepted, "I have read these terms and accept them for my own use"); + let closing = if download { + "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it." + } else { + "The model is kept on this computer only. LightCraft never uploads or shares it." + }; + ui.label(RichText::new(closing).font(t.font(11.5)).color(t.text_dim)); +} diff --git a/crates/ui-egui/src/panels/dialogs.rs b/crates/ui-egui/src/panels/dialogs.rs index cd087776..da400bc2 100644 --- a/crates/ui-egui/src/panels/dialogs.rs +++ b/crates/ui-egui/src/panels/dialogs.rs @@ -85,6 +85,7 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) { Dialog::NewSmartAlbum { .. } => "Create Smart Album", Dialog::AllMetadata { .. } => "All Metadata", Dialog::SystemInfo { .. } => "System Info", + Dialog::DenoiseModel { .. } => "AI Denoise Model", Dialog::WhatsNew => "What's New", Dialog::Cull { .. } => "Assisted Culling", Dialog::SmartRules { id: None, .. } => "New Smart Album", @@ -174,6 +175,7 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) { } }); } + Dialog::DenoiseModel { info, accepted } => crate::panels::denoise::model_dialog(app, ui, &t, info, accepted), Dialog::SystemInfo { rows } => { egui::Grid::new("sysinfo").num_columns(2).spacing([16.0, 4.0]).striped(true).show(ui, |ui| { for (k, v) in rows.iter() { @@ -784,7 +786,8 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) { } ui.add_space(4.0); ui.horizontal(|ui| { - let informational = matches!(dlg, Dialog::About | Dialog::Shortcuts | Dialog::Settings { .. }); + let unusable_model = matches!(&dlg, Dialog::DenoiseModel { info, .. } if info["kind"] == "unsupported"); + let informational = unusable_model || matches!(dlg, Dialog::About | Dialog::Shortcuts | Dialog::Settings { .. }); let sam = &app.session.segmenter; let (sam_installed, sam_running, sam_failed) = (sam.installed(), sam.download_status().running, sam.download_status().error.is_some()); // no download location in this build: nothing to offer but the manual install @@ -810,6 +813,8 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) { add_label.as_str() } Dialog::Merge { .. } => "Merge", + Dialog::DenoiseModel { info, .. } if info["download"].is_string() => "Accept & Download", + Dialog::DenoiseModel { .. } => "Install", Dialog::ConfirmDelete { .. } => "Delete", Dialog::RemoveFolder { .. } => "Remove", Dialog::SamModel { then: Some(_), .. } if sam_installed => "Continue", @@ -819,7 +824,8 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) { _ if informational => "Close", _ => "OK", }; - let r = (!ok.is_empty()).then(|| ui.button(crate::i18n::tr(ok))); + let can_confirm = informational || !matches!(&dlg, Dialog::DenoiseModel { accepted: false, .. }); + let r = (!ok.is_empty()).then(|| ui.add_enabled(can_confirm, egui::Button::new(crate::i18n::tr(ok)))); if let Some(r) = &r { crate::widgets::register(ui.ctx(), "button:dialogOk", r.rect); } @@ -1025,6 +1031,7 @@ pub fn confirm_dialog(app: &mut LightcraftApp, dlg: &Dialog) -> Result app.run("keyword.rename", json!({"from": from, "to": to})), Dialog::MergeKeywords { from, into } => app.run("keyword.merge", json!({"from": from, "into": into})), Dialog::AutoStack { gap } => app.run("stack.auto", json!({"gap": gap})), + Dialog::DenoiseModel { info, accepted } => crate::panels::denoise::install(app, info, *accepted), Dialog::AllMetadata { .. } | Dialog::SystemInfo { .. } | Dialog::WhatsNew => Ok(serde_json::Value::Null), Dialog::Cull { reject_below, pick_best } => { let mut p = json!({"pickBest": pick_best}); diff --git a/crates/ui-egui/src/panels/edit.rs b/crates/ui-egui/src/panels/edit.rs index 2188afa6..51a54bc9 100644 --- a/crates/ui-egui/src/panels/edit.rs +++ b/crates/ui-egui/src/panels/edit.rs @@ -272,7 +272,15 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) { for c in ["detail.sharpenAmount", "detail.sharpenRadius", "detail.sharpenDetail", "detail.sharpenMasking"] { control(app, ui, d, c, c == "detail.sharpenAmount" || d.detail.sharpen_amount > 0.0); } - sub_title(ui, crate::i18n::tr("Noise Reduction")); + // AI Denoise is its own tool, apart from the manual noise reduction sliders below it + sub_title(ui, crate::i18n::tr("AI Denoise")); + // AI Denoise: how much of the photo's cleaned picture to mix in (raw files only), and what that picture is doing + let applicable = !matches!(app.session.denoise_photo_state(id), lightcraft_engine::denoise::PhotoState::NotApplicable); + let on = d.enhance.denoise_enabled(); + super::denoise::detail_toggle(app, ui, id, on, applicable); + control(app, ui, d, "enhance.denoise", applicable && on); + super::denoise::detail_status(app, ui, id, if on { d.enhance.denoise } else { 0.0 }); + sub_title(ui, crate::i18n::tr("Manual Noise Reduction")); for c in ["detail.nrLuminance", "detail.nrDetail", "detail.nrContrast", "detail.nrColor", "detail.nrColorDetail", "detail.nrColorSmoothness"] { control(app, ui, d, c, true); diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 30075648..1d228538 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -4,6 +4,7 @@ pub mod bottombar; pub mod chips; pub mod compare; pub mod crop_overlay; +pub mod denoise; pub mod detail; pub mod dialogs; pub mod edit; diff --git a/crates/ui-egui/src/panels/settings.rs b/crates/ui-egui/src/panels/settings.rs index db5ca5a1..11285776 100644 --- a/crates/ui-egui/src/panels/settings.rs +++ b/crates/ui-egui/src/panels/settings.rs @@ -1,4 +1,4 @@ -//! The Settings dialog (⌘,): General, Import, Performance, Interface. +//! The Settings dialog (⌘,): General, Import, Performance, Interface, AI Denoise. //! //! Changes apply immediately (no OK/Cancel). Where they are stored: //! - **app settings** ([`crate::state::AppSettings`]: startup view, delete confirmation, GPU, @@ -17,7 +17,8 @@ use crate::theme::Tokens; use crate::widgets::register; /// (id, label) of the tabs, in order. -pub const TABS: &[(&str, &str)] = &[("general", "General"), ("import", "Import"), ("performance", "Performance"), ("interface", "Interface")]; +pub const TABS: &[(&str, &str)] = + &[("general", "General"), ("import", "Import"), ("performance", "Performance"), ("interface", "Interface"), ("denoise", "AI Denoise")]; /// Thumbnail cache sizes offered (MB). const CACHE_SIZES: [u32; 5] = [512, 1024, 2048, 4096, 8192]; @@ -43,11 +44,12 @@ pub fn body(app: &mut LightcraftApp, ui: &mut egui::Ui, tab: &mut String) { "import" => import_tab(app, ui, &t), "performance" => performance_tab(app, ui, &t), "interface" => interface_tab(app, ui, &t), + "denoise" => super::denoise::settings_tab(app, ui, &t), _ => general_tab(app, ui, &t), } } -fn heading(ui: &mut egui::Ui, t: &Tokens, text: &str) { +pub(super) fn heading(ui: &mut egui::Ui, t: &Tokens, text: &str) { ui.add_space(4.0); ui.label(RichText::new(crate::i18n::tr(text)).font(t.semibold(12.5)).color(t.text)); } @@ -63,19 +65,19 @@ fn row(ui: &mut egui::Ui, t: &Tokens, label: &str, add: impl FnOnce(&mut egui .inner } -fn hint(ui: &mut egui::Ui, t: &Tokens, text: &str) { +pub(super) fn hint(ui: &mut egui::Ui, t: &Tokens, text: &str) { ui.label(RichText::new(crate::i18n::tr(text)).size(11.0).color(t.text_dim)); } /// A checkbox addressable as `check:{id}`; true when toggled. -fn check(ui: &mut egui::Ui, id: &str, value: &mut bool, label: &str) -> bool { +pub(super) fn check(ui: &mut egui::Ui, id: &str, value: &mut bool, label: &str) -> bool { let r = ui.checkbox(value, crate::i18n::tr(label)); register(ui.ctx(), format!("check:{id}"), r.rect); r.changed() } /// Mutually exclusive buttons (`button:{id}-{index}`). -fn choices(ui: &mut egui::Ui, id: &str, options: &[(V, &str)], value: &mut V) -> bool { +pub(super) fn choices(ui: &mut egui::Ui, id: &str, options: &[(V, &str)], value: &mut V) -> bool { let mut changed = false; ui.spacing_mut().item_spacing.x = 4.0; for (i, (v, l)) in options.iter().enumerate() { diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index 17f5c58b..45949bb8 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -535,6 +535,10 @@ pub enum Dialog { pick_best: bool, }, /// Help ▸ System Info: (label, value) rows. + DenoiseModel { + info: serde_json::Value, + accepted: bool, + }, SystemInfo { rows: Vec<(String, String)>, }, diff --git a/crates/ui-egui/src/tests_model_setup.rs b/crates/ui-egui/src/tests_model_setup.rs new file mode 100644 index 00000000..892df22d --- /dev/null +++ b/crates/ui-egui/src/tests_model_setup.rs @@ -0,0 +1,171 @@ +//! Synthetic setup flows: model installation must resume the requested action on its original targets. +use std::path::{Path, PathBuf}; + +use lightcraft_catalog::{MediaKind, Op, Photo, PhotoId, Source}; +use lightcraft_denoise::manifest::{DenoiserManifest, Domain, Gain}; +use lightcraft_engine::Session; +use serde_json::json; + +use crate::{LightcraftApp, Services, model_setup, state::Dialog}; + +fn setup(name: &str) -> (LightcraftApp, PathBuf, PhotoId) { + let dir = std::env::temp_dir().join(format!("lc-model-flow-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let mut s = Session::new().with_fs(); + s.open_library(dir.join("library"), true).unwrap(); + s.set_denoise_models_dir(Some(dir.join("denoise"))); + s.execute("denoise.settings", &json!({"runOn": "cpu"})).unwrap(); + let id = PhotoId(s.catalog.photos().map(|p| p.id.0).max().unwrap() + 1); + let mut raw = Photo::new( + id, + Source::File { path: dir.join("synthetic.dng").to_string_lossy().into_owned() }, + "synthetic.dng", + "DNG", + 128, + 128, + "2026-10-07", + ); + raw.kind = MediaKind::Raw; + s.commit("fixture", Op::AddPhoto { photo: Box::new(raw) }).unwrap(); + (LightcraftApp::new(s, Services::default()), dir, id) +} + +fn install_denoise(app: &mut LightcraftApp, dir: &Path) { + let source = dir.join("source"); + std::fs::create_dir_all(&source).unwrap(); + let bytes = lightcraft_denoise::synthetic::smoothing_onnx(); + std::fs::write(source.join("model.onnx"), &bytes).unwrap(); + let m = DenoiserManifest { + id: "test-smoothing".into(), + name: "Synthetic smoothing".into(), + version: "1".into(), + licence: Default::default(), + source: None, + sha256: None, + size_bytes: Some(bytes.len() as u64), + provenance: "Synthetic tests".into(), + domain: Domain::BayerToRgb, + tile: 64, + overlap: 16, + gain: Gain::None, + }; + std::fs::write(source.join("denoise-model.json"), serde_json::to_vec(&m).unwrap()).unwrap(); + app.run("denoise.models.install", json!({"path": source.join("model.onnx").to_string_lossy(), "acknowledged": true})).unwrap(); +} + +fn amount(app: &LightcraftApp, id: PhotoId) -> f64 { + app.session.develop_of(id).unwrap().enhance.denoise +} +fn resume(app: &mut LightcraftApp) { + model_setup::pump(app, &egui::Context::default()); +} + +#[test] +fn denoise_installation_enables_only_the_requesting_photo_once_and_is_undoable() { + let (mut app, dir, raw) = setup("denoise"); + let other = app.session.active().unwrap(); + app.session.auto_sync = true; + let r = app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap(); + assert_eq!(r["setupRequired"], true); + assert_eq!(app.ui.dialog, Some(Dialog::Settings { tab: "denoise".into() })); + assert_eq!(amount(&app, raw), 0.0, "setup alone never changes the photo"); + app.run("library.select", json!({"ids": [other.0]})).unwrap(); + let undo = app.session.undo.len(); + install_denoise(&mut app, &dir); + resume(&mut app); + assert_eq!(amount(&app, raw), 50.0); + assert_eq!(amount(&app, other), 0.0); + assert_eq!(app.session.active(), Some(other)); + assert_eq!(app.session.undo.len(), undo + 1); + resume(&mut app); + assert_eq!(app.session.undo.len(), undo + 1, "a later frame cannot apply it again"); + app.run("edit.undo", json!({})).unwrap(); + assert_eq!(amount(&app, raw), 0.0); + app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap(); + app.run("develop.set", json!({"ids": [raw.0], "control": "enhance.denoise", "value": 75})).unwrap(); + app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap(); + assert_eq!(amount(&app, raw), 75.0, "enabling an already chosen amount keeps it"); + app.run("denoise.toggle", json!({"id": raw.0, "enabled": false})).unwrap(); + assert_eq!(amount(&app, raw), 75.0, "switching off keeps the Amount"); + assert_eq!(app.session.develop_of(raw).unwrap().denoise_amount(), 0.0, "but mixes nothing in"); + drop(app); + let _ = std::fs::remove_dir_all(dir); +} + +#[test] +fn cancelled_denoise_requests_do_not_apply_after_installation() { + for (name, command, params) in [ + ("cancel-action", "modelSetup.cancel", json!({"kind": "denoise"})), + ("cancel-toggle", "denoise.toggle", json!({"enabled": false})), + ("cancel-download", "denoise.models.downloadCancel", json!({"id": "rawnind-bayer"})), + ] { + let (mut app, dir, raw) = setup(name); + app.run("library.select", json!({"ids": [raw.0]})).unwrap(); + app.run("denoise.toggle", json!({"enabled": true})).unwrap(); + app.run(command, params).unwrap(); + install_denoise(&mut app, &dir); + resume(&mut app); + assert_eq!(amount(&app, raw), 0.0, "{name}"); + drop(app); + let _ = std::fs::remove_dir_all(dir); + } +} + +#[test] +fn pending_actions_do_not_cross_libraries_or_relinked_sources() { + for changed in ["library", "source"] { + let (mut app, dir, raw) = setup(changed); + app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap(); + if changed == "library" { + app.session.open_library(dir.join("other-library"), true).unwrap(); + } else { + app.session + .commit( + "relink", + Op::Relink { + id: raw, + file_name: "another.dng".into(), + source: Source::File { path: dir.join("another.dng").to_string_lossy().into_owned() }, + format: Some("DNG".into()), + }, + ) + .unwrap(); + } + install_denoise(&mut app, &dir); + resume(&mut app); + assert!(!model_setup::denoise_requested(&app, raw)); + assert!(app.session.catalog.photos().all(|p| p.develop.enhance.denoise == 0.0)); + assert!(app.ui.status.contains("cancelled")); + drop(app); + let _ = std::fs::remove_dir_all(dir); + } +} + +#[test] +fn hostile_or_inapplicable_toggle_arguments_are_errors_without_changes() { + let (mut app, dir, raw) = setup("hostile"); + let jpeg = app.session.active().unwrap(); + for params in [json!({"id": "wrong"}), json!({"id": u64::MAX}), json!({"id": raw.0, "enabled": 1}), json!({"id": jpeg.0, "enabled": true})] { + assert!(app.run("denoise.toggle", params).is_err()); + assert_eq!(amount(&app, raw), 0.0); + } + assert!(app.run("modelSetup.cancel", json!({"kind": "nonsense"})).is_err()); + drop(app); + let _ = std::fs::remove_dir_all(dir); +} + +#[test] +fn model_installation_waits_for_an_active_slider_gesture_to_finish() { + let (mut app, dir, raw) = setup("interaction"); + app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap(); + app.session.begin_interaction("Exposure").unwrap(); + install_denoise(&mut app, &dir); + resume(&mut app); + assert_eq!(amount(&app, raw), 0.0); + assert!(app.session.interaction.is_some()); + app.session.end_interaction().unwrap(); + resume(&mut app); + assert_eq!(amount(&app, raw), 50.0); + drop(app); + let _ = std::fs::remove_dir_all(dir); +} diff --git a/docs/denoise.md b/docs/denoise.md new file mode 100644 index 00000000..b8fd2901 --- /dev/null +++ b/docs/denoise.md @@ -0,0 +1,118 @@ +# AI Denoise + +Edit ▸ Detail ▸ **AI Denoise** is a per-photo switch above **Amount** (0–100). +Turning it on starts at 50, or preserves an already positive Amount. Turning it off +preserves Amount and mixes none of the cleaned picture. It affects only the requesting +photo, independently of Auto Sync, and is undoable. Manual luminance and colour noise +reduction remain separate controls. + +If no model is selected, turning it on opens **Settings ▸ AI Denoise** without changing +the photo. After installation, the pending request enables the original photo even if +the selection changed. Cancel the pending action in Settings. Cancelled downloads, +deleted or relinked photos, switching libraries and quitting cancel the request. +An active slider gesture finishes before the pending action resumes. + +## Models and terms + +**No model weights are bundled.** Models live in the per-user `denoise-models` folder +(override with `LIGHTCRAFT_DENOISE_MODELS`). Settings offers a pinned, verified download +of RawNIND UtNet2 for Bayer sensors, or installation of a user-supplied ONNX file with +a `denoise-model.json` beside it. Terms must be acknowledged before installation or +download. Downloads use the shared pure-Rust `lightcraft-fetch` client, with resume, +progress, cancellation, size limits and SHA-256 verification. + +RawNIND's offered ONNX weights are **GPL-3.0**, as recorded in the pinned model catalog. +They are downloaded separately after consent and read by our own Rust executor. +Whether releases should offer this download remains a maintainer policy decision; +permissively licensed or independently trained weights remain a gap. This is not a +claim that optional downloading settles licensing questions. No GPL decoder or neural +runtime code is included. Model metadata is in `crates/denoise/src/known.rs`. + +Installation checks the manifest, model hash and size, then runs a synthetic self-test: +finite output of the promised shape, repeatability, the declared scale and reduced +noise. Unsupported operators, malformed files and a failed self-test return errors. + +The current contract is `bayerToRgb`: normalized, black-subtracted, non-white-balanced +RGGB planes `[1, 4, tile, tile]` produce camera RGB `[1, 3, 2*tile, 2*tile]`. +RawNIND uses a 512-cell tile, 64-cell overlap and a mean-matched nominal gain of 1e6. +See `DenoiserManifest` for the validated JSON schema. + +## Photos, previews and exports + +Original RAW files stay untouched. A cleaned picture is cached under +`/denoise/.lcdn`, keyed by source content, model and algorithm version. +The compressed half-float cache has checked headers, bounded windows and checksums. +It can be cleared or recreated; it is not a new catalog photo or a duplicate DNG. + +Plain and cleaned camera RGB go through the same highlight reconstruction, white +balance, colour matrix, DNG profile tables, camera hue/saturation look, crop and +orientation. Amount blends the two before the remaining develop pipeline, so edits +and exports use the same data. At zero Amount or with Detail disabled, the ordinary +render path remains in use. + +The queue prioritizes the open and selected photos, pauses after input, and adjusts +its worker count while the user works. Progress, cancellation and retry are available +in Settings and through commands. The default cache limit is 20 GB; unused products +are evicted oldest first. A failed file is not continuously retried. + +An export with a nonzero effective Amount creates a missing product synchronously. +It returns a clear error if it cannot produce the denoised result; it never silently +exports the plain image instead. Unsupported sensor layouts retain manual noise +reduction and report their limitation. + +## Pure-Rust CPU and GPU execution + +The CPU executor reads a bounded subset of ONNX into checked network data. Convolutions +use bounded im2col blocks and `faer-core 0.17.1` with its Rust-intrinsic GEMM kernels, +without BLAS, generated assembly or an inner thread pool. Tiles share the outer worker +pool. A scalar implementation checks the optimized operators on synthetic networks. +Tract is not a workspace dependency or a product feature. + +The GPU executor uses our WGSL kernels through wgpu. It checks a real model's output +against the CPU and falls back on errors, unsupported networks or a slower device. +Setup has a time limit and a crash marker. Automatic selection estimates throughput +from a check tile; this estimate can choose imperfectly on unmeasured hardware. +Settings offers Automatic, Graphics card and Processor. GPU tests cover supported +synthetic networks against the scalar executor; a real-model test compares CPU/GPU. + +### Historical performance, measured 2026-10-07 + +These are the contributor's measurements **before the current upstream integration**, +on a Ryzen 9 7945HX and RTX 4090 Laptop GPU (DX12), release build, model SHA-256 +`da27509dab6a2915da67e988acd86cf71f9d5bbc8d1aa0ed32933578a887b901`. +The 24.34 MP Sony a7 III photo is 35 tiles. Numbers exclude model loading, source +decode, GPU setup, cache encoding and export, but include packing and blending. + +| Work | Pure-Rust CPU | Prior Tract reference | GPU | +|---|---:|---:|---:| +| One warm 512-cell tile, one caller | 1148 ms | 1463 ms | 15.1 ms | +| Whole photo, 16 CPU / 4 GPU feeding workers | 5.185 s | 9.534 s | 0.582 s | + +The largest relative differences were 1.68e-6 CPU/reference and 1.57e-6 CPU/GPU for +the check tile, below the required 1e-3 tolerance. Reference comparisons were made +on the older contributor branch; that reference runtime is intentionally excluded +from the current workspace. These numbers are not a fresh benchmark of this rebase. + +Fresh checks can be run using an already installed model and CC0 raw samples: + +```text +LC_DENOISE_MODEL= cargo test --release -p lightcraft-gpu --lib real_model_on_the_gpu_matches_cpu -- --ignored --nocapture +LC_DENOISE_MODEL= LC_DENOISE_RAW= cargo test --release -p lightcraft-engine --features denoise --test denoise_cpu -- --ignored --nocapture +LC_DENOISE_MODEL= LC_DENOISE_RAW= cargo test --release -p lightcraft-engine --features denoise --test denoise_real -- --ignored --nocapture +``` + +## Commands and limits + +UI, CLI, control channel and MCP dispatch `denoise.toggle`, `denoise.settings`, +`denoise.queue`, `.cancel`, `.status`, `.pump`, `.clear`, and `denoise.models.*` +(`list`, `install`, `download`, `downloads`, `downloadCancel`, `test`, `remove`, +`select`). The develop control is `enhance.denoise`. + +Native apps enable inference through their `denoise` feature. A build without it and +the web build report inference unavailable. Manifest/cache primitives still compile +for WASM. Settings and models remain opt-in; nothing is downloaded on launch. + +This implementation supports **Bayer RAW only**. X-Trans, Foveon, demosaiced DNG and +non-RAW images remain unsupported by this model contract. No measured quality parity +with Lightroom is claimed. Apple/Metal and wider hardware verification, model policy, +a linear-RGB contract and a side-by-side fidelity suite remain future work. diff --git a/docs/parity.md b/docs/parity.md index 90dc8a74..a4718949 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -79,7 +79,7 @@ Take the first one nobody is working on. Object and Describe masks run SAM 3 in pure Rust (`crates/segment`); the weights (SAM License) are never bundled: the app offers a consented, verified download, but **LightCraft's CDN mirrors are not configured yet** (the built-in list is empty, so today users need their own mirror or a manual install). Subject / Sky / People could - use the same model with fixed prompts; denoise / super-resolution models remain a maintainer decision. + use the same model with fixed prompts. Bayer RAW denoise now runs on pure-Rust CPU/GPU with separately installed weights; its opt-in GPL model policy, linear RGB and quality comparison remain open. Super-resolution models remain a maintainer decision. 7. **HDR** (Q. HDR, LR-EXP-HDR), **video** (R. Video), **Classic output modules** (Map view, Book, Slideshow module, Print, publish): large, well understood, lower priority than 1–5. 8. **LR-VIEW-ZOOM** (P0): pinch zoom and two-finger pan now work, with stable image cursors between gesture events. @@ -262,7 +262,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins. | LR-EDIT-DETAIL-SHARPEN | Sharpening | P0 | ✅ | `ctl:detail.sharpenAmount`, `ctl:detail.sharpenRadius`, `ctl:detail.sharpenDetail`, `ctl:detail.sharpenMasking` | no Alt-drag mask preview | | LR-EDIT-DETAIL-NR | Luminance noise reduction | P0 | ✅ | `ctl:detail.nrLuminance`, `ctl:detail.nrDetail`, `ctl:detail.nrContrast` | | | LR-EDIT-DETAIL-CNR | Colour noise reduction | P0 | ✅ | `ctl:detail.nrColor`, `ctl:detail.nrColorDetail`, `ctl:detail.nrColorSmoothness` | | -| LR-EDIT-DETAIL-DENOISE | AI denoise | P2 | ⬜ | | settings field reserved, not rendered | +| LR-EDIT-DETAIL-DENOISE | AI denoise | P2 | 🟡 | cmd:denoise.toggle, cmd:denoise.models.install, cmd:denoise.queue, ctl:enhance.denoise, `crates/denoise/`, `docs/denoise.md` | non-destructive Bayer RAW, opt-in weights, pure-Rust CPU/GPU; X-Trans/linear RGB, model policy and Lightroom fidelity remain | | LR-EDIT-DETAIL-RAWDETAILS | Improved demosaic toggle | P2 | ⬜ | | | | LR-EDIT-DETAIL-SUPERRES | Super resolution | P2 | ⬜ | | | | LR-EDIT-DETAIL-AISHARPEN | AI sharpen | OOS | 🚫 | | | diff --git a/docs/whats-new.md b/docs/whats-new.md index f9b295bd..b172faaa 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -2,6 +2,10 @@ ## October 2026 +### AI RAW denoise +- Detail has a per-photo AI Denoise switch and Amount. Bayer RAW photos use a disposable cache, with matching previews and exports and pure-Rust CPU/GPU inference. +- Models are installed separately after accepting their terms; no weights are bundled. See [setup and limitations](denoise.md). + ### Library keyboard culling - Colour labels tint thumbnail surrounds in Square Grid and the Detail filmstrip, and a translucent footer along the bottom of the photo in Photo Grid. Label confirmations use a matching pale colour. - Setting or clearing a colour label shows a brief bottom confirmation, like rating a photo; custom label names appear in the message too. diff --git a/xtask/src/layers.rs b/xtask/src/layers.rs index 28d52677..e330a8e1 100644 --- a/xtask/src/layers.rs +++ b/xtask/src/layers.rs @@ -39,6 +39,7 @@ pub const TABLE: &[(&str, Class)] = &[ ("tiff", Class::Layer(0)), ("sysmem", Class::Layer(0)), ("fetch", Class::Layer(0)), + ("denoise", Class::Layer(1)), ("raw", Class::Layer(1)), ("codecs", Class::Layer(1)), ("meta", Class::Layer(1)), From fd4019300aec2ce4b91e44b2ed5b4852c4fd8265 Mon Sep 17 00:00:00 2001 From: Daniel Voshart <205798369+voshart@users.noreply.github.com> Date: Thu, 8 Oct 2026 18:47:57 -0400 Subject: [PATCH 22/46] LR-EDIT-DETAIL-DENOISE: address installation and model-policy review Generated with Codex. --- Cargo.lock | 17 ++ Cargo.toml | 3 +- NOTICE | 9 + apps/lightcraft-cli/Cargo.toml | 1 + apps/lightcraft-web/src/web.rs | 1 + apps/lightcraft/Cargo.toml | 1 + apps/lightcraft/src/main.rs | 7 + crates/denoise-core/Cargo.toml | 26 +++ crates/denoise-core/README.md | 7 + .../{denoise => denoise-core}/src/archive.rs | 16 +- crates/{denoise => denoise-core}/src/bayer.rs | 0 crates/{denoise => denoise-core}/src/hash.rs | 0 crates/{denoise => denoise-core}/src/known.rs | 28 ++- crates/denoise-core/src/lib.rs | 16 ++ .../{denoise => denoise-core}/src/licence.rs | 0 .../{denoise => denoise-core}/src/manifest.rs | 0 .../{denoise => denoise-core}/src/product.rs | 39 +++- crates/{denoise => denoise-core}/src/run.rs | 25 ++- crates/{denoise => denoise-core}/src/tiles.rs | 0 crates/denoise/Cargo.toml | 10 +- crates/denoise/README.md | 8 +- crates/denoise/src/cpu.rs | 2 +- crates/denoise/src/lib.rs | 17 +- crates/denoise/src/runtime.rs | 18 +- crates/denoise/tests/real_model.rs | 4 +- crates/engine/Cargo.toml | 6 +- crates/engine/src/cmd/denoise.rs | 186 ++++++++++++++---- crates/engine/src/denoise.rs | 29 ++- crates/engine/src/files.rs | 33 ++-- crates/engine/src/lib.rs | 4 + crates/engine/src/media.rs | 21 +- crates/engine/src/model_download.rs | 12 +- crates/engine/src/tests_denoise.rs | 62 +++++- crates/gpu/Cargo.toml | 6 +- crates/gpu/src/lib.rs | 4 +- crates/ui-egui/Cargo.toml | 2 +- crates/ui-egui/locales/de-formats.json | 21 +- crates/ui-egui/locales/de.json | 91 ++++++++- crates/ui-egui/locales/es-formats.json | 21 +- crates/ui-egui/locales/es.json | 91 ++++++++- crates/ui-egui/locales/ja-formats.json | 21 +- crates/ui-egui/locales/ja.json | 91 ++++++++- crates/ui-egui/locales/pt-br-formats.json | 21 +- crates/ui-egui/locales/pt-br.json | 91 ++++++++- crates/ui-egui/locales/ru-formats.json | 21 +- crates/ui-egui/locales/ru.json | 92 ++++++++- crates/ui-egui/locales/zh-hans-formats.json | 21 +- crates/ui-egui/locales/zh-hans.json | 91 ++++++++- crates/ui-egui/locales/zh-hant-formats.json | 21 +- crates/ui-egui/locales/zh-hant.json | 91 ++++++++- crates/ui-egui/src/headless.rs | 5 +- crates/ui-egui/src/lib.rs | 2 + crates/ui-egui/src/model_setup.rs | 13 +- crates/ui-egui/src/panels/denoise.rs | 155 +++++++++------ crates/ui-egui/src/panels/edit.rs | 2 +- crates/ui-egui/src/tests_model_setup.rs | 35 +++- docs/denoise.md | 31 ++- docs/parity.md | 4 +- xtask/src/layers.rs | 2 + 59 files changed, 1437 insertions(+), 217 deletions(-) create mode 100644 crates/denoise-core/Cargo.toml create mode 100644 crates/denoise-core/README.md rename crates/{denoise => denoise-core}/src/archive.rs (94%) rename crates/{denoise => denoise-core}/src/bayer.rs (100%) rename crates/{denoise => denoise-core}/src/hash.rs (100%) rename crates/{denoise => denoise-core}/src/known.rs (81%) create mode 100644 crates/denoise-core/src/lib.rs rename crates/{denoise => denoise-core}/src/licence.rs (100%) rename crates/{denoise => denoise-core}/src/manifest.rs (100%) rename crates/{denoise => denoise-core}/src/product.rs (94%) rename crates/{denoise => denoise-core}/src/run.rs (97%) rename crates/{denoise => denoise-core}/src/tiles.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 6de52a46..d082de8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3052,6 +3052,22 @@ name = "lightcraft-denoise" version = "0.4.0" dependencies = [ "faer-core", + "lightcraft-denoise-core", + "lightcraft-raster", + "miniz_oxide 0.8.9", + "proptest", + "rayon", + "serde", + "serde_json", + "sha2", + "thiserror 2.0.21", + "web-time", +] + +[[package]] +name = "lightcraft-denoise-core" +version = "0.4.0" +dependencies = [ "lightcraft-raster", "miniz_oxide 0.8.9", "proptest", @@ -3086,6 +3102,7 @@ dependencies = [ "lightcraft-codecs", "lightcraft-color", "lightcraft-denoise", + "lightcraft-denoise-core", "lightcraft-develop", "lightcraft-fetch", "lightcraft-geom", diff --git a/Cargo.toml b/Cargo.toml index b23fae21..daff5840 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,8 @@ repository = "https://github.com/storytold/lightcraft" [workspace.dependencies] # internal -lightcraft-denoise = { path = "crates/denoise" } +lightcraft-denoise-core = { path = "crates/denoise-core" } +lightcraft-denoise = { path = "crates/denoise", default-features = false } lightcraft-geom = { path = "crates/geom" } lightcraft-sysmem = { path = "crates/sysmem" } lightcraft-color = { path = "crates/color" } diff --git a/NOTICE b/NOTICE index f3f6499f..254e2628 100644 --- a/NOTICE +++ b/NOTICE @@ -27,6 +27,15 @@ the facebook/sam3 checkpoint only when they choose to (in the app, or by hand), License (https://github.com/facebookresearch/sam3/blob/main/LICENSE), a separate, non-OSI licence. See docs/ai-masks.md. +AI denoise weights are not part of LightCraft and are never distributed with it. +The default build offers user-supplied models only. The optional rawnind-model +feature offers a separately downloaded RawNIND UtNet2 Bayer checkpoint under +GPL-3.0 (https://www.gnu.org/licenses/gpl-3.0.html), after the user accepts its terms. +Commercial use is subject to those terms, including applicable source, licence and +notice obligations on redistribution. Enabling that offer in releases remains a +maintainer decision. The ONNX reader and CPU/GPU executor are our Rust implementation; +no darktable, RawSpeed, LibRaw or other GPL decoder/runtime code is included. + Builds made with craft-fonts (the optional CRAFT_FONTS_DIR build input; all official releases) also embed the fonts listed in https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md (the Japanese fonts BIZ UDPGothic, BIZ UDMincho and Shippori Mincho; the web build embeds only BIZ diff --git a/apps/lightcraft-cli/Cargo.toml b/apps/lightcraft-cli/Cargo.toml index 72d46858..bafe3287 100644 --- a/apps/lightcraft-cli/Cargo.toml +++ b/apps/lightcraft-cli/Cargo.toml @@ -27,6 +27,7 @@ dhat = { workspace = true, optional = true } [features] default = ["denoise"] denoise = ["lightcraft-engine/denoise"] +rawnind-model = ["denoise", "lightcraft-engine/rawnind-model"] # Heap profiling: `cargo run --release -p lightcraft-cli --features dhat-heap -- snapshot …` counts # every allocation (live/peak bytes in `library.memory` / `ui.inspect`) and writes dhat-heap.json # (open it in DHAT's viewer: allocation sites by bytes at the peak). Slows allocation-heavy work. diff --git a/apps/lightcraft-web/src/web.rs b/apps/lightcraft-web/src/web.rs index 9ffbf170..0b5d4c26 100644 --- a/apps/lightcraft-web/src/web.rs +++ b/apps/lightcraft-web/src/web.rs @@ -205,6 +205,7 @@ fn services(originals: Originals, backend: Option, files: Files, frozen open_picker(originals.clone(), backend.clone(), ctx.clone()); Vec::new() // files arrive asynchronously and are imported on a later frame })), + pick_denoise_model: None, // Preset files: browser pickers are asynchronous; not wired on the web yet. pick_preset_files: None, pick_tracklog: None, diff --git a/apps/lightcraft/Cargo.toml b/apps/lightcraft/Cargo.toml index 3f3f40ba..f5fe2763 100644 --- a/apps/lightcraft/Cargo.toml +++ b/apps/lightcraft/Cargo.toml @@ -12,6 +12,7 @@ workspace = true [features] default = ["denoise"] denoise = ["lightcraft-engine/denoise"] +rawnind-model = ["denoise", "lightcraft-engine/rawnind-model"] [dependencies] lightcraft-engine = { workspace = true, features = ["sam"] } diff --git a/apps/lightcraft/src/main.rs b/apps/lightcraft/src/main.rs index 44f0548d..e6629c00 100644 --- a/apps/lightcraft/src/main.rs +++ b/apps/lightcraft/src/main.rs @@ -387,6 +387,13 @@ fn services(ctx: egui::Context, log_file: Option<&std::path::Path>) -> Services .map(|p| p.to_string_lossy().to_string()) .collect() })), + pick_denoise_model: Some(Box::new(|| { + rfd::FileDialog::new() + .add_filter_nocase(lightcraft_ui_egui::i18n::tr("ONNX model"), &["onnx"]) + .pick_file() + .map(|p| vec![p.to_string_lossy().into_owned()]) + .unwrap_or_default() + })), pick_preset_files: Some(Box::new(|| { rfd::FileDialog::new() .set_title(lightcraft_ui_egui::i18n::tr("Import Presets")) diff --git a/crates/denoise-core/Cargo.toml b/crates/denoise-core/Cargo.toml new file mode 100644 index 00000000..b65f90ae --- /dev/null +++ b/crates/denoise-core/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "lightcraft-denoise-core" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "LightCraft denoise model metadata, Bayer tiling and cache formats (L1; no inference backend)" + +[lints] +workspace = true + +[features] +default = [] +rawnind-model = [] + +[dependencies] +lightcraft-raster = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +thiserror = { workspace = true } +miniz_oxide = { workspace = true } +rayon = { workspace = true } +web-time = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/crates/denoise-core/README.md b/crates/denoise-core/README.md new file mode 100644 index 00000000..74d1e324 --- /dev/null +++ b/crates/denoise-core/README.md @@ -0,0 +1,7 @@ +# lightcraft-denoise-core + +L1 model metadata, bounded archive/cache formats and Bayer packing/tiling helpers. +No weights or inference backend. See [the user workflow](../../docs/denoise.md). +`lightcraft-denoise` re-exports these types for CPU/GPU users; the engine uses this +crate directly when inference is disabled. The default download catalog is empty. +Run `cargo test -p lightcraft-denoise-core`; optional catalog checks use `--features rawnind-model`. diff --git a/crates/denoise/src/archive.rs b/crates/denoise-core/src/archive.rs similarity index 94% rename from crates/denoise/src/archive.rs rename to crates/denoise-core/src/archive.rs index ddaf7084..ccfea471 100644 --- a/crates/denoise/src/archive.rs +++ b/crates/denoise-core/src/archive.rs @@ -127,13 +127,21 @@ pub fn extract(archive: &Path, name: &str, dest: &Path, max_bytes: u64) -> Resul } let skip = u64::from(u16_at(&head, 26).unwrap_or(0)) + u64::from(u16_at(&head, 28).unwrap_or(0)); f.seek_relative(i64::try_from(skip).map_err(|_| ArchiveError::Format("a header is too long".into()))?)?; - let mut packed = vec![0u8; e.compressed as usize]; + let len = usize::try_from(e.compressed) + .ok() + .filter(|&n| n <= 256 * 1024 * 1024) + .ok_or_else(|| ArchiveError::Format("the compressed model is too large".into()))?; + if e.size > 256 * 1024 * 1024 { + return format("the extracted model is too large"); + } + let mut packed = Vec::new(); + packed.try_reserve_exact(len).map_err(|_| ArchiveError::Format("not enough memory for the model archive".into()))?; + packed.resize(len, 0u8); f.read_exact(&mut packed)?; let data = match e.method { 0 => packed, - 8 => { - miniz_oxide::inflate::decompress_to_vec_with_limit(&packed, max_bytes as usize).map_err(|err| ArchiveError::Format(format!("{err:?}")))? - } + 8 => miniz_oxide::inflate::decompress_to_vec_with_limit(&packed, max_bytes.min(256 * 1024 * 1024) as usize) + .map_err(|err| ArchiveError::Format(format!("{err:?}")))?, m => return format(&format!("compression method {m} is not supported")), }; if data.len() as u64 != e.size || crc32(&data) != e.crc { diff --git a/crates/denoise/src/bayer.rs b/crates/denoise-core/src/bayer.rs similarity index 100% rename from crates/denoise/src/bayer.rs rename to crates/denoise-core/src/bayer.rs diff --git a/crates/denoise/src/hash.rs b/crates/denoise-core/src/hash.rs similarity index 100% rename from crates/denoise/src/hash.rs rename to crates/denoise-core/src/hash.rs diff --git a/crates/denoise/src/known.rs b/crates/denoise-core/src/known.rs similarity index 81% rename from crates/denoise/src/known.rs rename to crates/denoise-core/src/known.rs index 9e0e7ea1..f9a51109 100644 --- a/crates/denoise/src/known.rs +++ b/crates/denoise-core/src/known.rs @@ -3,9 +3,12 @@ //! None is part of LightCraft. A model with an address here can be downloaded when the user asks and has accepted //! its terms; anything else is brought by the user as an `.onnx` file with a `denoise-model.json` beside it. +#[cfg(any(feature = "rawnind-model", test))] use crate::licence::{Commercial, Licence}; -use crate::manifest::{DenoiserManifest, Domain, Gain}; +use crate::manifest::DenoiserManifest; +#[cfg(any(feature = "rawnind-model", test))] +use crate::manifest::{Domain, Gain}; /// A known model, and how to get it. pub struct Known { @@ -18,6 +21,7 @@ pub struct Known { /// RawNIND's UtNet2 for Bayer sensors, as published for darktable (`rawdenoise-nind.dtmodel`, a zip with a Bayer and a /// linear model). Its weights are GPL-3.0; the training pictures are CC BY 4.0 and CC0. +#[cfg(any(feature = "rawnind-model", test))] fn rawnind_bayer() -> Known { let manifest = DenoiserManifest { id: "rawnind-bayer".into(), @@ -29,7 +33,7 @@ fn rawnind_bayer() -> Known { url: Some("https://www.gnu.org/licenses/gpl-3.0.html".into()), notice: "The weights are published under the GNU General Public License v3. LightCraft does not include them: they \ are downloaded to your computer at your request and used by it only. If you pass the model file on, the \ - GPL applies to that copy." + GPL applies to that copy. Commercial use is subject to the GPL terms, including applicable source, licence and notice obligations on redistribution." .into(), }, source: Some("https://github.com/darktable-org/darktable-ai/tree/master/models/rawdenoise-nind".into()), @@ -55,7 +59,14 @@ fn rawnind_bayer() -> Known { /// Every model LightCraft knows. pub fn all() -> Vec { - vec![rawnind_bayer()] + #[cfg(feature = "rawnind-model")] + { + vec![rawnind_bayer()] + } + #[cfg(not(feature = "rawnind-model"))] + { + Vec::new() + } } /// The known model with this id. @@ -76,7 +87,8 @@ mod tests { #[test] fn known_models_are_valid_unique_and_downloaded_over_https() { let all = all(); - assert!(!all.is_empty()); + assert_eq!(all.is_empty(), !cfg!(feature = "rawnind-model")); + validate(&rawnind_bayer().manifest).unwrap(); for (i, k) in all.iter().enumerate() { validate(&k.manifest).unwrap(); assert!(all.iter().skip(i + 1).all(|o| o.manifest.id != k.manifest.id), "unique ids"); @@ -92,8 +104,12 @@ mod tests { assert!(k.download.is_none(), "{}", k.manifest.id); } } - assert!(find("rawnind-bayer").is_some() && find("nothing").is_none()); - assert_eq!(by_sha256("da27509dab6a2915da67e988acd86cf71f9d5bbc8d1aa0ed32933578a887b901").map(|m| m.id), Some("rawnind-bayer".into())); + assert_eq!(find("rawnind-bayer").is_some(), cfg!(feature = "rawnind-model")); + assert!(find("nothing").is_none()); + assert_eq!( + by_sha256("da27509dab6a2915da67e988acd86cf71f9d5bbc8d1aa0ed32933578a887b901").map(|m| m.id), + if cfg!(feature = "rawnind-model") { Some("rawnind-bayer".into()) } else { None } + ); } } diff --git a/crates/denoise-core/src/lib.rs b/crates/denoise-core/src/lib.rs new file mode 100644 index 00000000..568ed9f8 --- /dev/null +++ b/crates/denoise-core/src/lib.rs @@ -0,0 +1,16 @@ +//! Model metadata, bounded archive/cache formats and Bayer tiling, independent of inference. +//! Used by featureless engine and web builds; no neural-network or matrix backend is linked. +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +pub mod archive; +pub mod bayer; +pub mod hash; +pub mod known; +pub mod licence; +pub mod manifest; +pub mod product; +pub mod run; +pub mod tiles; + +pub use manifest::{DenoiserManifest, Domain, Gain, ManifestError}; +pub use run::{Control, Error, Params, TileRunner, denoise_bayer}; diff --git a/crates/denoise/src/licence.rs b/crates/denoise-core/src/licence.rs similarity index 100% rename from crates/denoise/src/licence.rs rename to crates/denoise-core/src/licence.rs diff --git a/crates/denoise/src/manifest.rs b/crates/denoise-core/src/manifest.rs similarity index 100% rename from crates/denoise/src/manifest.rs rename to crates/denoise-core/src/manifest.rs diff --git a/crates/denoise/src/product.rs b/crates/denoise-core/src/product.rs similarity index 94% rename from crates/denoise/src/product.rs rename to crates/denoise-core/src/product.rs index 1e393885..5a607c8f 100644 --- a/crates/denoise/src/product.rs +++ b/crates/denoise-core/src/product.rs @@ -52,6 +52,13 @@ pub struct Header { pub key: String, } +fn allocate(n: usize, value: T) -> Result, ProductError> { + let mut v = Vec::new(); + v.try_reserve_exact(n).map_err(|_| ProductError::Format("not enough memory for the denoised picture".into()))?; + v.resize(n, value); + Ok(v) +} + /// `f32` as IEEE half-float bits, rounded to nearest even; beyond the range it saturates, NaN becomes 0. pub fn to_f16(v: f32) -> u16 { if !v.is_finite() { @@ -120,9 +127,9 @@ fn crc(bytes: &[u8]) -> u32 { } /// One strip as bytes: for each plane (R, G, B) the low bytes of its half floats, then the high bytes. -fn encode_strip(img: &Rgb32f, y0: usize, y1: usize) -> Vec { +fn encode_strip(img: &Rgb32f, y0: usize, y1: usize) -> Result, ProductError> { let n = (y1 - y0) * img.width; - let mut raw = vec![0u8; 6 * n]; + let mut raw = allocate(6 * n, 0u8)?; let rows = img.data.get(y0 * img.width..y1 * img.width).unwrap_or(&[]); for (i, px) in rows.iter().enumerate() { for (c, &v) in px.iter().enumerate() { @@ -135,16 +142,17 @@ fn encode_strip(img: &Rgb32f, y0: usize, y1: usize) -> Vec { } } } - miniz_oxide::deflate::compress_to_vec(&raw, 3) + Ok(miniz_oxide::deflate::compress_to_vec(&raw, 3)) } /// Write `img` to `path` (through a temporary file next to it, so a reader never sees half a product). pub fn write(path: &Path, img: &Rgb32f, key: &str) -> Result<(), ProductError> { - if img.width == 0 || img.height == 0 || img.width > MAX_SIDE || img.height > MAX_SIDE || img.data.len() != img.width * img.height { + let pixels = img.width.checked_mul(img.height).filter(|&n| n <= MAX_PIXELS); + if img.width == 0 || img.height == 0 || img.width > MAX_SIDE || img.height > MAX_SIDE || pixels != Some(img.data.len()) { return Err(ProductError::Format("the picture has no usable size".into())); } let strips: Vec<(usize, usize)> = (0..img.height).step_by(STRIP_ROWS).map(|y| (y, (y + STRIP_ROWS).min(img.height))).collect(); - let packed: Vec> = strips.par_iter().map(|&(a, b)| encode_strip(img, a, b)).collect(); + let packed: Vec> = strips.par_iter().map(|&(a, b)| encode_strip(img, a, b)).collect::>()?; let header = serde_json::to_vec(&Header { width: img.width, height: img.height, strip: STRIP_ROWS, key: key.to_string() }) .map_err(|e| ProductError::Format(e.to_string()))?; let tmp = path.with_extension("tmp"); @@ -193,7 +201,7 @@ fn read_start(r: &mut impl Read) -> Result { if len > MAX_HEADER { return Err(ProductError::Format("the header is too large".into())); } - let mut hb = vec![0u8; len]; + let mut hb = allocate(len, 0u8)?; r.read_exact(&mut hb)?; let h: Header = serde_json::from_slice(&hb).map_err(|e| ProductError::Format(e.to_string()))?; let pixels = h.width.saturating_mul(h.height); @@ -297,7 +305,7 @@ pub fn read_window(path: &Path, key: &str, window: Option, factor: usize } // the picture rows and columns the blocks cover let (x_end, y_end) = if whole { (w.x + w.width, w.y + w.height) } else { (w.x + ow * factor, w.y + oh * factor) }; - let mut out = Rgb32f { width: ow, height: oh, data: vec![[0.0; 3]; ow * oh] }; + let mut out = Rgb32f { width: ow, height: oh, data: allocate(ow * oh, [0.0; 3])? }; let lut = f16_table(); let value = |raw: &[u8], c: usize, n: usize, i: usize| -> f32 { let (lo, hi) = (raw.get(2 * c * n + i).copied().unwrap_or(0), raw.get((2 * c + 1) * n + i).copied().unwrap_or(0)); @@ -305,7 +313,7 @@ pub fn read_window(path: &Path, key: &str, window: Option, factor: usize }; // which strips are needed, and where each starts in the file (after the header and the strip table) let wanted = |s: usize| s * h.strip < y_end && (s + 1) * h.strip > w.y; - let (mut sum, mut peak) = (vec![[0f32; 3]; ow], vec![[f32::MIN; 3]; ow]); + let (mut sum, mut peak) = (allocate(ow, [0f32; 3])?, allocate(ow, [f32::MIN; 3])?); let (mut rows_in, mut oy) = (0usize, 0usize); let strips: Vec = (0..table.len()).filter(|&s| wanted(s)).collect(); // skip the strips before the first one wanted @@ -315,7 +323,7 @@ pub fn read_window(path: &Path, key: &str, window: Option, factor: usize let mut packed = Vec::with_capacity(group.len()); for &s in group { let Some(&(len, _)) = table.get(s) else { return Err(ProductError::Format("a strip is missing".into())) }; - let mut p = vec![0u8; len]; + let mut p = allocate(len, 0u8)?; f.read_exact(&mut p)?; packed.push(p); } @@ -401,6 +409,19 @@ pub fn blend(base: &mut Rgb32f, denoised: &Rgb32f, amount: f32) -> bool { mod tests { use super::*; + #[test] + fn oversized_product_header_is_rejected_before_allocating_pixels() { + let p = temp("oversized-header"); + let header = serde_json::to_vec(&Header { width: 10001, height: 10000, strip: STRIP_ROWS, key: "k".into() }).unwrap(); + let mut bytes = Vec::from(MAGIC.as_slice()); + bytes.extend_from_slice(&[VERSION, 0, 0, 0]); + bytes.extend_from_slice(&(header.len() as u32).to_le_bytes()); + bytes.extend_from_slice(&header); + std::fs::write(&p, bytes).unwrap(); + assert!(read_header_at(&p).is_err()); + let _ = std::fs::remove_file(p); + } + fn temp(name: &str) -> std::path::PathBuf { std::env::temp_dir().join(format!("lc-denoise-{name}-{}.lcdn", std::process::id())) } diff --git a/crates/denoise/src/run.rs b/crates/denoise-core/src/run.rs similarity index 97% rename from crates/denoise/src/run.rs rename to crates/denoise-core/src/run.rs index 5bc763ce..1df96a9e 100644 --- a/crates/denoise/src/run.rs +++ b/crates/denoise-core/src/run.rs @@ -14,7 +14,7 @@ use crate::manifest::Gain; use crate::tiles::{starts, weight}; /// Most pixels one picture may have (guards the allocations that follow from its size). -pub const MAX_PIXELS: usize = 400_000_000; +pub const MAX_PIXELS: usize = 100_000_000; /// A tile whose mean signal is below this (white = 1) is too dark to measure its scale from. const MIN_SIGNAL: f32 = 0.003; @@ -32,6 +32,13 @@ pub enum Error { Cancelled, } +fn allocate(n: usize, value: T) -> Result, Error> { + let mut v = Vec::new(); + v.try_reserve_exact(n).map_err(|_| Error::Input("not enough memory for denoise".into()))?; + v.resize(n, value); + Ok(v) +} + /// Something that runs the model on one tile. pub trait TileRunner: Sync { /// `input` is the tile as four planes `[R, G1, G2, B]` of `tile × tile` cells (NCHW, batch 1); the answer is @@ -112,13 +119,13 @@ pub fn denoise_bayer( .collect(); let total = tiles.len(); - let mut acc = vec![[0f32; 3]; pixels]; - let mut wsum = vec![0f32; nx * ny]; - let group = p.parallel.clamp(1, 64); + let mut acc = allocate(pixels, [0f32; 3])?; + let mut wsum = allocate(nx * ny, 0f32)?; + let group = p.parallel.clamp(1, 4); let mut done = 0; // where the time goes, printed under `LIGHTCRAFT_PROFILE` let (mut t_wait, mut t_total) = (std::time::Duration::ZERO, std::time::Duration::ZERO); - let began = std::time::Instant::now(); + let began = web_time::Instant::now(); // One group of tiles at a time: pack, run the model, check and bring each to the input's scale, in parallel. // Blending (sequential: the tiles add into one picture, in tile order so the result does not depend on timing) of @@ -128,7 +135,7 @@ pub fn denoise_bayer( chunk .par_iter() .map(|&(x0, y0, bx, ax, by, ay)| { - let mut input = vec![0f32; 4 * p.tile * p.tile]; + let mut input = allocate(4 * p.tile * p.tile, 0f32)?; if !pack_tile(mosaic, width, height, layout, x0 as isize, y0 as isize, p.tile, &mut input) { return Err(Error::Input("a tile could not be packed".into())); } @@ -164,7 +171,7 @@ pub fn denoise_bayer( || compute(chunk), || match previous.take() { Some((c, o)) => { - let started = std::time::Instant::now(); + let started = web_time::Instant::now(); let r = blend(c, o); (r, started.elapsed()) } @@ -176,12 +183,12 @@ pub fn denoise_bayer( previous = Some((chunk, outs)); } if let Some((c, o)) = previous.take() { - let started = std::time::Instant::now(); + let started = web_time::Instant::now(); blend(c, o)?; t_wait += started.elapsed(); } t_total += began.elapsed(); - let started = std::time::Instant::now(); + let started = web_time::Instant::now(); // normalise by the weights let (ox, oy) = (layout.position(0, 0).0, layout.position(0, 0).1); acc.par_chunks_mut(width).enumerate().for_each(|(y, row)| { diff --git a/crates/denoise/src/tiles.rs b/crates/denoise-core/src/tiles.rs similarity index 100% rename from crates/denoise/src/tiles.rs rename to crates/denoise-core/src/tiles.rs diff --git a/crates/denoise/Cargo.toml b/crates/denoise/Cargo.toml index 55c44a14..621fd3d6 100644 --- a/crates/denoise/Cargo.toml +++ b/crates/denoise/Cargo.toml @@ -10,11 +10,15 @@ workspace = true [features] default = [] +runtime = ["dep:faer"] +# Product-policy switch: absent by default; users may install their own model. +rawnind-model = ["lightcraft-denoise-core/rawnind-model"] [dependencies] -# Safe views over Rust-intrinsic gemm kernels. Pin the core-only release: newer backends -# generate assembly. No BLAS, cc/build scripts, nightly/AMX, or inner thread pool. -faer = { package = "faer-core", version = "=0.17.1", default-features = false, features = ["std"] } +lightcraft-denoise-core = { workspace = true } +# Safe matrix views over the pure-Rust GEMM backend; no BLAS, cc/build scripts +# or inner thread pool. Its dependencies include architecture-specific assembly. +faer = { package = "faer-core", version = "=0.17.1", default-features = false, features = ["std"], optional = true } lightcraft-raster = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/denoise/README.md b/crates/denoise/README.md index cb69e382..0397193f 100644 --- a/crates/denoise/README.md +++ b/crates/denoise/README.md @@ -3,12 +3,16 @@ L1 pure-Rust AI RAW denoise primitives and CPU inference. No weights are bundled. See [user workflow and limitations](../../docs/denoise.md). -`manifest` and `known` validate opt-in models; `onnx` reads a bounded operator subset +`lightcraft-denoise-core` provides metadata, Bayer helpers and cache formats, +re-exported here. `manifest` and `known` validate opt-in models; `onnx` reads a bounded operator subset into `net`; `cpu` executes it and `reference` supplies scalar checks. `bayer`, `tiles` and `run` pack sensor data and blend overlapping tiles while preserving clipped highlights. `product` reads/writes the disposable `.lcdn` cache. `runtime` supplies the installation self-test. Synthetic networks are generated in Rust for tests. -Run `cargo test -p lightcraft-denoise`. Real-model tests are ignored by default and +Run `cargo test -p lightcraft-denoise --features runtime`. The CPU backend is optional; +featureless and web builds keep the model metadata and cache format without faer/GEMM. +The `rawnind-model` feature enables the otherwise empty download catalog and requires +an explicit release-policy decision. Real-model tests are ignored by default and require an explicitly supplied local model. The GPU twin is `lightcraft-gpu::nn`; model management, background jobs and export integration live in `lightcraft-engine`. diff --git a/crates/denoise/src/cpu.rs b/crates/denoise/src/cpu.rs index a856fc25..34e5269e 100644 --- a/crates/denoise/src/cpu.rs +++ b/crates/denoise/src/cpu.rs @@ -13,7 +13,7 @@ const BLOCK: usize = 256; /// Cap all tensor buffers and convolution scratch for one tile together (512 MiB). const MAX_VALUES: usize = 128 * 1024 * 1024; const MAX_TILE: usize = 2048; -const MAX_IDLE: usize = 16; +const MAX_IDLE: usize = 1; type Result = std::result::Result; fn bad(why: &str) -> NetError { diff --git a/crates/denoise/src/lib.rs b/crates/denoise/src/lib.rs index c7c76b58..e72860ba 100644 --- a/crates/denoise/src/lib.rs +++ b/crates/denoise/src/lib.rs @@ -18,25 +18,20 @@ #![forbid(unsafe_code)] #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] -pub mod archive; -pub mod bayer; +#[cfg(feature = "runtime")] pub mod cpu; -pub mod known; -pub mod manifest; + pub mod net; pub mod onnx; mod onnx_proto; -pub mod product; + pub mod reference; -pub mod run; + +#[cfg(feature = "runtime")] pub mod runtime; #[doc(hidden)] pub mod synthetic; -pub mod tiles; -pub use manifest::{DenoiserManifest, Domain, Gain, ManifestError}; -pub use run::{Control, Error, Params, TileRunner, denoise_bayer}; +pub use lightcraft_denoise_core::*; -pub mod hash; -pub mod licence; pub mod synthetic_proto; diff --git a/crates/denoise/src/runtime.rs b/crates/denoise/src/runtime.rs index 4314537a..117f6775 100644 --- a/crates/denoise/src/runtime.rs +++ b/crates/denoise/src/runtime.rs @@ -408,10 +408,20 @@ mod cpu_tests { } let path = std::env::temp_dir().join(format!("lc-cpu-load-{}.onnx", std::process::id())); std::fs::write(&path, crate::synthetic::unet_onnx(64, 5, 1, 7)).unwrap(); - let mut m = crate::known::find("rawnind-bayer").unwrap().manifest; - m.tile = 64; - m.overlap = 16; - m.gain = Gain::None; + let mut m = DenoiserManifest { + id: "synthetic".into(), + name: "Synthetic".into(), + version: "1".into(), + licence: Default::default(), + source: None, + sha256: None, + size_bytes: None, + provenance: "synthetic test".into(), + domain: crate::manifest::Domain::BayerToRgb, + tile: 64, + overlap: 16, + gain: Gain::None, + }; let r = CpuRunner::load(&path, &m).unwrap(); let x = check_tile(64); assert_eq!(r.run(&x).unwrap(), r.run(&x).unwrap()); diff --git a/crates/denoise/tests/real_model.rs b/crates/denoise/tests/real_model.rs index bea7e712..203e9937 100644 --- a/crates/denoise/tests/real_model.rs +++ b/crates/denoise/tests/real_model.rs @@ -2,7 +2,7 @@ //! //! ```text //! LC_DENOISE_MODEL= LC_DENOISE_MOSAICS= LC_DENOISE_OUT= \ -//! cargo test --release -p lightcraft-denoise --test real_model -- --ignored --nocapture +//! cargo test --release -p lightcraft-denoise --features runtime --test real_model -- --ignored --nocapture //! ``` //! //! `LC_DENOISE_MOSAICS` holds `.mosaic.f32` (little-endian f32, normalised: black 0, white 1, not white @@ -11,6 +11,8 @@ //! once, default 8), `LC_DENOISE_ONLY` (one name), `LC_DENOISE_ISO` (noise strength, default 1.0; 0 adds none), //! `LC_DENOISE_FULL` (the whole frame instead of a 2048 × 1536 crop). +#![cfg(feature = "runtime")] + use std::path::{Path, PathBuf}; use std::time::Instant; diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 536df716..f11ca467 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -21,7 +21,8 @@ web-time = { workspace = true } lightcraft-preview = { workspace = true } lightcraft-codecs = { workspace = true } lightcraft-raw = { workspace = true } -lightcraft-denoise = { workspace = true } +lightcraft-denoise-core = { workspace = true } +lightcraft-denoise = { workspace = true, optional = true } lightcraft-meta = { workspace = true } miniz_oxide = { workspace = true } lightcraft-merge = { workspace = true } @@ -39,7 +40,8 @@ lightcraft-segment = { workspace = true, optional = true } [features] # AI masks with SAM 3 (Object / Describe). Off for the web and CLI builds. sam = ["dep:lightcraft-segment"] -denoise = [] +denoise = ["dep:lightcraft-denoise", "lightcraft-denoise/runtime", "lightcraft-gpu/denoise"] +rawnind-model = ["denoise", "lightcraft-denoise/rawnind-model", "lightcraft-denoise-core/rawnind-model"] [target.'cfg(not(target_arch = "wasm32"))'.dependencies] lightcraft-fetch = { workspace = true } diff --git a/crates/engine/src/cmd/denoise.rs b/crates/engine/src/cmd/denoise.rs index c5103791..e1cc0900 100644 --- a/crates/engine/src/cmd/denoise.rs +++ b/crates/engine/src/cmd/denoise.rs @@ -7,7 +7,7 @@ //! validated before they become folder names, and nothing outside a model's own folder is ever touched. use std::path::{Path, PathBuf}; -use std::sync::PoisonError; +use std::sync::{Arc, PoisonError, mpsc}; use lightcraft_catalog::PhotoId; use lightcraft_denoise::archive; @@ -18,7 +18,7 @@ use serde_json::{Value, json}; use super::{CommandSpec, always, bad, bool_or, cmd, str_param}; use crate::denoise::{Pace, RunOn, Settings, forget_gpu_set_ups, installed_models, read_capped, read_settings, write_atomic, write_settings}; -use crate::model_download::{self, State}; +use crate::model_download::State; use crate::{EngineError, Result, Session}; fn fail(what: &str, e: impl std::fmt::Display) -> EngineError { @@ -129,8 +129,16 @@ fn identify(path: &Path, c: &str, sha_known: Option<&str>) -> Result { /// Install the model in `src`. A model that passes its self-test becomes the one in use (`activate`, the default): installing /// a model is how someone says they want it. `verified` is for a download whose hash was already checked. -fn install_file(s: &mut Session, c: &str, src: &Path, acknowledged: bool, activate: bool, verified: bool) -> Result { - let dir = models_dir(s, c)?; +fn install_file( + dir: &Path, + loader: &crate::denoise::Loader, + run_on: RunOn, + accepted_at: String, + c: &str, + src: &Path, + acknowledged: bool, + verified: bool, +) -> Result<(Value, Arc)> { let ident = identify(src, c, None)?; let m = ident.manifest.clone(); manifest::validate(&m).map_err(|e| bad(c, e.to_string()))?; @@ -141,7 +149,11 @@ fn install_file(s: &mut Session, c: &str, src: &Path, acknowledged: bool, activa )); } let home = dir.join(&m.id); + if home.join("model.onnx").exists() { + return Err(bad(c, "that model is already installed; remove it before installing a replacement")); + } std::fs::create_dir_all(&home).map_err(|e| fail("could not create the model's folder", e))?; + let mut cleanup = InstallCleanup { home: home.clone(), armed: true }; let (part, final_path) = (home.join("model.onnx.part"), home.join("model.onnx")); let _ = std::fs::remove_file(&part); let placed: Result<()> = match ident.entry { @@ -162,26 +174,24 @@ fn install_file(s: &mut Session, c: &str, src: &Path, acknowledged: bool, activa return Err(e); } }; - let _ = std::fs::remove_file(&final_path); - if let Err(e) = std::fs::rename(&part, &final_path) { - let _ = std::fs::remove_file(&part); - return Err(fail("could not finish the copy", e)); - } - // a model that does not work is never left installed - let run_on = s.denoise.settings.run_on(); - let test = (s.denoise.loader)(&final_path, &m).and_then(|model| model.self_test(run_on)); - let test = match test { + // Test the staged file first. A failed replacement preserves the installed model. + let model = loader(&part, &m).map_err(|e| bad(c, e))?; + let test = match model.self_test(run_on) { Ok(t) => t, Err(e) => { - let _ = std::fs::remove_file(&final_path); - let _ = std::fs::remove_file(home.join("denoise-model.json")); - let _ = std::fs::remove_dir(&home); + let _ = std::fs::remove_file(&part); return Err(bad(c, e)); } }; + // An installed id is immutable: installing again must never destroy a usable model. + if final_path.exists() { + let _ = std::fs::remove_file(&part); + return Err(bad(c, "that model is already installed; remove it before installing a replacement")); + } + std::fs::rename(&part, &final_path).map_err(|e| fail("could not finish the copy", e))?; write_atomic(&home.join("denoise-model.json"), &serde_json::to_vec_pretty(&m).map_err(|e| fail("manifest", e))?).map_err(EngineError::Other)?; let accepted = json!({ - "acceptedAt": (s.clock)(), + "acceptedAt": accepted_at, "licence": m.licence.name, "commercial": m.licence.commercial, "fileName": ident.file_name, @@ -189,26 +199,116 @@ fn install_file(s: &mut Session, c: &str, src: &Path, acknowledged: bool, activa "selfTest": test, }); write_atomic(&home.join("installed.json"), &serde_json::to_vec_pretty(&accepted).map_err(|e| fail("record", e))?).map_err(EngineError::Other)?; - // a file that was downloaded and waited for the user's acceptance has served: the installed copy is the model now - if src.parent() == Some(dir.join(model_download::STAGING).as_path()) { - let _ = std::fs::remove_file(src); - s.denoise.downloads.staged_file_used(src); + cleanup.armed = false; + Ok((json!({"installed": row(&m, known::find(&m.id).and_then(|k| host_of(&k)), true, false, &accepted), "model": m.id}), model)) +} + +/// Removes only files created for this new installation, including on a worker panic. +struct InstallCleanup { + home: PathBuf, + armed: bool, +} +impl Drop for InstallCleanup { + fn drop(&mut self) { + if self.armed { + for name in ["model.onnx.part", "model.onnx", "denoise-model.json", "installed.json"] { + let _ = std::fs::remove_file(self.home.join(name)); + } + let _ = std::fs::remove_dir(&self.home); + } } +} + +pub(crate) struct InstallJob { + id: String, + activate: bool, + result: mpsc::Receiver), String>>, +} + +fn activate_install(s: &mut Session, result: &Value, model: Arc, activate: bool) -> Result { + let dir = models_dir(s, "denoise.models.install")?; + let id = result["installed"]["id"].as_str().ok_or_else(|| bad("denoise.models.install", "missing installed id"))?; let mut st = read_settings(&dir); if activate { - st.model = Some(m.id.clone()); + st.model = Some(id.to_string()); write_settings(&dir, &st).map_err(EngineError::Other)?; } s.denoise.touch(); s.denoise_refresh_active(true); - let host = known::find(&m.id).and_then(|k| host_of(&k)); - Ok(json!({"installed": row(&m, host, true, st.model.as_deref() == Some(m.id.as_str()), &accepted), "model": st.model})) + if let Some(a) = &s.denoise.active + && a.id == id + { + *a.model.lock().unwrap_or_else(PoisonError::into_inner) = Some(model); + } + let mut v = result.clone(); + v["installed"]["selected"] = json!(st.model.as_deref() == Some(id)); + v["model"] = json!(st.model); + Ok(v) +} + +fn start_install(s: &mut Session, src: PathBuf, id: String, activate: bool, verified: bool) -> Result { + const C: &str = "denoise.models.install"; + if s.denoise.install.is_some() { + return Err(bad(C, "another model is being installed")); + } + let dir = models_dir(s, C)?; + let loader = s.denoise.loader.clone(); + let run_on = s.denoise.settings.run_on(); + let at = (s.clock)(); + let (tx, rx) = mpsc::channel(); + std::thread::Builder::new() + .name("denoise-model-install".into()) + .stack_size(16 * 1024 * 1024) + .spawn(move || { + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _held = crate::memory::work_gate().acquire(512 * 1024 * 1024); + install_file(&dir, &loader, run_on, at, C, &src, true, verified).map_err(|e| plain(&e)) + })) + .unwrap_or_else(|_| Err("the model installation stopped unexpectedly".into())); + if verified { + let _ = std::fs::remove_file(&src); + } + let _ = tx.send(outcome); + }) + .map_err(|e| bad(C, format!("could not start installation: {e}")))?; + s.denoise.install = Some(InstallJob { id: id.clone(), activate, result: rx }); + s.denoise.downloads.set_outcome(&id, State::Installing); + Ok(json!({"started": id})) } fn install(s: &mut Session, p: &Value) -> Result { const C: &str = "denoise.models.install"; let path = str_param(p, "path").ok_or_else(|| bad(C, "missing `path`"))?; - install_file(s, C, Path::new(path), bool_or(p, "acknowledged", false), bool_or(p, "activate", true), false) + let acknowledged = bool_or(p, "acknowledged", false); + let activate = bool_or(p, "activate", true); + if !acknowledged { + return Err(bad(C, "the licence has not been accepted: show its terms first")); + } + if bool_or(p, "background", false) { + return start_install(s, PathBuf::from(path), "local-install".into(), activate, false); + } + if s.denoise.install.is_some() { + return Err(bad(C, "another model is being installed")); + } + let (result, model) = + install_file(&models_dir(s, C)?, &s.denoise.loader, s.denoise.settings.run_on(), (s.clock)(), C, Path::new(path), acknowledged, false)?; + activate_install(s, &result, model, activate) +} + +/// Read only the small adjacent manifest; hashing and execution happen in the install worker. +fn inspect(s: &mut Session, p: &Value) -> Result { + const C: &str = "denoise.models.inspect"; + let path = str_param(p, "path").ok_or_else(|| bad(C, "missing `path`"))?; + let src = Path::new(path); + if !src.extension().is_some_and(|e| e.eq_ignore_ascii_case("onnx")) { + return Err(bad(C, "choose an .onnx file with denoise-model.json beside it")); + } + let bytes = read_capped(&src.with_file_name("denoise-model.json"), MAX_MANIFEST_BYTES) + .ok_or_else(|| bad(C, "denoise-model.json is missing or too large"))?; + let m = manifest::parse(&bytes).map_err(|e| bad(C, e.to_string()))?; + let dir = models_dir(s, C)?; + Ok(json!({"kind": "model", "domain": "denoise", "path": path, "fileName": src.file_name().map(|n| n.to_string_lossy()), "model": m, + "sizeBytes": std::fs::metadata(src).ok().map(|m| m.len()), "alreadyInstalled": dir.join(&m.id).join("model.onnx").is_file()})) } /// What an error says, without the command it came from (for a line shown to the user). @@ -222,17 +322,31 @@ fn plain(e: &EngineError) -> String { /// Install every download that has arrived: each was accepted when it was started, so it is installed and chosen without /// another question. Called by `denoise.models.downloads` and by every frame's `denoise.pump`. pub(crate) fn finish_downloads(s: &mut Session) { - const C: &str = "denoise.models.download"; - for (id, path, _sha) in s.denoise.downloads.finished() { - match install_file(s, C, &path, true, true, true) { - Ok(_) => s.denoise.downloads.set_outcome(&id, State::Installed), - Err(e) => { - // a file that cannot be installed is of no use to anyone: throw it away, say why - let _ = std::fs::remove_file(&path); - s.denoise.downloads.set_outcome(&id, State::Failed(plain(&e))); - } + if let Some(job) = &s.denoise.install { + let outcome = match job.result.try_recv() { + Ok(v) => Some(v), + Err(mpsc::TryRecvError::Empty) => None, + Err(mpsc::TryRecvError::Disconnected) => Some(Err("the installation worker stopped".into())), + }; + if let Some(outcome) = outcome + && let Some(job) = s.denoise.install.take() + { + let state = match outcome { + Ok((v, model)) => match activate_install(s, &v, model, job.activate) { + Ok(_) => State::Installed, + Err(e) => State::Failed(plain(&e)), + }, + Err(e) => State::Failed(e), + }; + s.denoise.downloads.set_outcome(&job.id, state); } } + if s.denoise.install.is_none() + && let Some((id, path, _sha)) = s.denoise.downloads.finished().into_iter().next() + && let Err(e) = start_install(s, path, id.clone(), true, true) + { + s.denoise.downloads.set_outcome(&id, State::Failed(plain(&e))); + } } fn download(s: &mut Session, p: &Value) -> Result { @@ -263,6 +377,7 @@ fn download_row(id: &str, state: &State) -> Value { match state { State::Running { bytes, total } => json!({"id": id, "state": "running", "bytes": bytes, "total": total, "from": from}), State::Done { path, .. } => json!({"id": id, "state": "done", "path": path.display().to_string(), "from": from}), + State::Installing => json!({"id": id, "state": "installing", "from": from}), State::Installed => json!({"id": id, "state": "installed", "from": from}), State::Failed(why) => json!({"id": id, "state": "failed", "error": why, "from": from}), State::Cancelled => json!({"id": id, "state": "cancelled", "from": from}), @@ -517,7 +632,8 @@ fn clear(s: &mut Session, _: &Value) -> Result { pub fn specs() -> Vec { vec![ cmd!(query "denoise.models.list", "Denoise Models", [], None, "{} → {dir, productsDir, model, runtime, auto, cacheGb, threads, runOn, models: [{id, name, version, licence{name, commercial, url, notice}, provenance, source, sizeBytes, sha256, tile, known, downloadHost, installed, selected, accepted}]}", always, list), - cmd!(query "denoise.models.install", "Install Denoise Model", [], None, "{path, acknowledged: true, activate?: true} → {installed, model} — install a denoise model from a file: an .onnx with a denoise-model.json beside it, or an archive LightCraft knows (the darktable `.dtmodel`). `acknowledged` must be true: the user has been shown the model's licence and accepted it. Unless `activate` is false it becomes the model in use", always, install), + cmd!(query "denoise.models.inspect", "Inspect Denoise Model", [], None, "{path} → model terms from an adjacent manifest; never executes the file", always, inspect), + cmd!(query "denoise.models.install", "Install Denoise Model", [], None, "{path, acknowledged: true, activate?: true, background?: false} → {installed, model} or {started} — install a denoise model from a file: an .onnx with a denoise-model.json beside it, or an archive LightCraft knows (the darktable `.dtmodel`). `acknowledged` must be true: the user has been shown the model's licence and accepted it. Unless `activate` is false it becomes the model in use", always, install), cmd!(query "denoise.models.download", "Download Denoise Model", [], None, "{id, acknowledged: true} → {started, from} — fetch a model LightCraft has a pinned address for (see `downloadHost` in the list) in the background over HTTPS with lightcraft-fetch. `acknowledged` must be true: the user has been shown the model's terms and accepted them. It is checked against its size and SHA-256, installed and chosen by itself; `denoise.models.downloads` shows how far it is", always, download), cmd!(query "denoise.models.downloads", "Denoise Model Downloads", [], None, "{} → {running, downloads: [{id, state: running | done | installed | failed | cancelled, bytes, total, error, from}]} — also installs any download that has arrived", always, downloads), cmd!(query "denoise.models.downloadCancel", "Cancel Denoise Model Download", [], None, "{id} → {discarded} — stop a download, or delete a finished one that was not installed", always, download_cancel), diff --git a/crates/engine/src/denoise.rs b/crates/engine/src/denoise.rs index 37d79dcd..01bcdc78 100644 --- a/crates/engine/src/denoise.rs +++ b/crates/engine/src/denoise.rs @@ -98,6 +98,10 @@ pub(crate) trait Model: Send + Sync { /// Check it does something sensible: the test's result, or why not. Unless `run_on` is the processor the graphics /// card is set up and checked too, and the result says where the work will run. fn self_test(&self, run_on: RunOn) -> Result; + /// Conservative scratch bound for an unknown runner. + fn bytes_per_tile(&self) -> usize { + 512 * 1024 * 1024 + } /// Where the tiles run: `{kind: "gpu", adapter}`, `{kind: "cpu", reason?}` or `{kind: "pending"}` (the card is not /// set up yet), with the card's and the processor's time for the check tile (`cardMs`, `cpuMs`) once both were /// timed. Never starts anything. @@ -215,7 +219,7 @@ fn set_up_card( running.push(marker.to_path_buf()); SetUpEnded(marker.to_path_buf()) }; - let started = std::time::Instant::now(); + let started = web_time::Instant::now(); let (tx, rx) = std::sync::mpsc::channel(); std::thread::Builder::new() .name("denoise-gpu-setup".into()) @@ -241,10 +245,10 @@ fn card_tile_ms(runner: &dyn TileRunner, input: &[f32]) -> f64 { if runner.run(input).is_err() { return f64::INFINITY; } - let began = std::time::Instant::now(); + let began = web_time::Instant::now(); let mut best = f64::INFINITY; for _ in 0..3 { - let one = std::time::Instant::now(); + let one = web_time::Instant::now(); if runner.run(input).is_err() { break; } @@ -268,7 +272,7 @@ fn make_gpu(cpu: &lightcraft_denoise::runtime::CpuRunner, path: &Path, tile: usi Ok((runner, got, card_ms)) }; let on_cpu = || { - let started = std::time::Instant::now(); + let started = web_time::Instant::now(); (cpu.run(&input), started.elapsed().as_secs_f64() * 1000.0) }; let ((runner, got, card_ms), (want, cpu_ms)) = set_up_card(&path.with_file_name(GPU_SETUP_MARKER), GPU_SETUP_LIMIT, card, on_cpu)?; @@ -351,6 +355,9 @@ impl TileRunner for Network { #[cfg(feature = "denoise")] impl Model for Network { + fn bytes_per_tile(&self) -> usize { + self.cpu.bytes_per_tile() + } fn runner(&self, run_on: RunOn, threads: usize) -> (&dyn TileRunner, usize) { let threads = threads.max(1); self.last_threads.store(threads, Ordering::Relaxed); @@ -689,7 +696,7 @@ pub(crate) fn make_product(spec: &JobSpec, progress: Option<&Progress>) -> Resul if product::is_current(&spec.product, &spec.key) { return Ok(false); } - let began = std::time::Instant::now(); + let began = web_time::Instant::now(); let cancelled = || progress.is_some_and(|p| p.cancel.load(Ordering::Relaxed)); if cancelled() { return Err(MakeError::Cancelled); @@ -726,7 +733,13 @@ pub(crate) fn make_product(spec: &JobSpec, progress: Option<&Progress>) -> Resul } let decoded = began.elapsed(); let model = loaded(spec)?; - let (runner, parallel) = model.runner(spec.run_on, spec.parallel.max(1)); + // Packed input, returned RGB and the preceding group's RGB overlap while tiles run. + let scratch = + model.bytes_per_tile().saturating_add(128usize.saturating_mul(spec.manifest.tile as usize).saturating_mul(spec.manifest.tile as usize)); + let (_, budget) = crate::memory::work_gate().usage(); + let parallel = spec.parallel.clamp(1, 4).min((budget / scratch.max(1)).max(1)); + let _held = crate::memory::work_gate().acquire(mosaic.data.len().saturating_mul(20).saturating_add(scratch.saturating_mul(parallel))); + let (runner, parallel) = model.runner(spec.run_on, parallel); let params = Params { tile: spec.manifest.tile as usize, overlap: spec.manifest.overlap as usize, @@ -859,6 +872,7 @@ pub(crate) struct State { pub models_dir: Option, /// Models being downloaded at the user's request. pub downloads: Downloads, + pub(crate) install: Option, /// How a model is loaded (tests put another in). pub loader: Loader, pub(crate) active: Option, @@ -883,6 +897,7 @@ impl Default for State { State { models_dir: None, downloads: Downloads::default(), + install: None, loader: default_loader(), active: None, active_seen: None, @@ -1604,7 +1619,7 @@ mod gpu_tests { fn a_set_up_that_takes_too_long_is_given_up_on() { let dir = folder("slowsetup"); let marker = dir.join(GPU_SETUP_MARKER); - let started = std::time::Instant::now(); + let started = web_time::Instant::now(); let slow = || -> Result<(), String> { std::thread::sleep(Duration::from_millis(600)); Ok(()) diff --git a/crates/engine/src/files.rs b/crates/engine/src/files.rs index 5ef823db..2052579d 100644 --- a/crates/engine/src/files.rs +++ b/crates/engine/src/files.rs @@ -309,8 +309,11 @@ fn load_bytes_now( let tables = lightcraft_raw::profile::ProfileTables::new(&raw.color.profile, lightcraft_raw::color::illuminant_weight(&raw.color, xy)); // camera RGB → what the pipeline takes: clipped highlights rebuilt, white balance and the colour model applied, // fitted to `max_edge` and upright (the same for the plain and the denoised picture) - let finish = |mut img: Rgb32f| -> Rgb32f { + let finish = |mut img: Rgb32f| { + let mut stages = vec![("develop", t0.elapsed())]; + stages.push(("transform", t0.elapsed())); lightcraft_raw::highlight::reconstruct(&mut img, wb, HIGHLIGHT_CLIP); + stages.push(("highlights", t0.elapsed())); img.map_in_place(|p| { let c = [p[0] * wb[0], p[1] * wb[1], p[2] * wb[2]]; let rgb = [ @@ -324,21 +327,27 @@ fn load_bytes_now( }; hue_sat.as_ref().map_or(rgb, |h| h.apply(rgb)).map(|v| v.max(0.0)) }); + stages.push(("colour", t0.elapsed())); let img = fit(&img, max_edge, max_edge, Filter::Box); - img.into_oriented(raw.orientation) + stages.push(("fit", t0.elapsed())); + let img = img.into_oriented(raw.orientation); + stages.push(("orient", t0.elapsed())); + (img, stages) }; - let develop_ms = t0.elapsed(); - let img = finish(img); - let twin = twin.map(finish); + let (img, stages) = finish(img); if lightcraft_pipeline::profiling() { - let finish_ms = (t0.elapsed() - develop_ms).as_secs_f64() * 1e3; - eprintln!( - "[profile] raw source {}×{} (max {max_edge}): develop {:.1} ms, finish {finish_ms:.1} ms (after decode)", - img.width, - img.height, - develop_ms.as_secs_f64() * 1e3 - ); + let mut prev = std::time::Duration::ZERO; + let parts: Vec = stages + .iter() + .map(|(n, t)| { + let d = *t - prev; + prev = *t; + format!("{n} {:.1}", d.as_secs_f64() * 1e3) + }) + .collect(); + eprintln!("[profile] raw source {}×{} (max {max_edge}, ms after decode): {}", img.width, img.height, parts.join(", ")); } + let twin = twin.map(|picture| finish(picture).0); let (temp, tint) = xy_to_temp_tint(xy); let relative = crate::camera_preview::file_local_look(raw.format) && t.matrix_is_fallback; let camera_tone = camera_look.as_ref().map(|p| p.tone).or_else(|| raw.color.profile.tone_curve.as_ref().and_then(dng_tone_curve)); diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 5f298de4..00d8ab40 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -10,6 +10,10 @@ //! run them off the UI thread. #![forbid(unsafe_code)] +// Model/cache types remain available without linking the optional inference crate. +#[cfg(not(feature = "denoise"))] +extern crate lightcraft_denoise_core as lightcraft_denoise; + pub mod availability; mod camera_preview; pub mod camera_profiles; diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rs index 479831c9..8130b5ad 100644 --- a/crates/engine/src/media.rs +++ b/crates/engine/src/media.rs @@ -156,10 +156,9 @@ impl Twin { Twin { image, mix: std::sync::Mutex::new(None) } } - /// Bytes held, the last mix included. + /// Bytes reserved for the twin and its lazy mix, before either cache cost can grow. fn bytes(&self) -> usize { - let mixed = self.mix.lock().unwrap_or_else(std::sync::PoisonError::into_inner).as_ref().map_or(0, |m| m.1.data.len() * 12); - self.image.data.len() * 12 + mixed + self.image.data.len().saturating_mul(24) } } @@ -207,7 +206,7 @@ impl DecodedSource { mixed } - /// Bytes held: the picture, its denoised twin and the last mix. + /// Cache cost: the picture, its denoised twin and room for the lazy mix. pub fn bytes(&self) -> usize { self.image.data.len() * 12 + 64 + std::mem::size_of::() + self.denoised.as_ref().map_or(0, |t| t.bytes()) } @@ -1390,6 +1389,20 @@ mod thumbnail_hash_tests { mod tests { use super::*; + #[test] + fn denoise_mix_is_reserved_before_the_source_enters_the_cache() { + let mut plain = Rgb32f::new(8, 8); + plain.data.fill([1.0; 3]); + let mut source = DecodedSource::new(Arc::new(plain), None); + source.denoised = Some(Arc::new(Twin::new(Arc::new(Rgb32f::new(8, 8))))); + let reserved = source.bytes(); + assert!(reserved >= 3 * 8 * 8 * 12); + assert_eq!(source.image_for(0.5).data[0], [0.5; 3]); + assert_eq!(source.bytes(), reserved, "creating a mix cannot evade the LRU's recorded cost"); + let _ = source.image_for(0.25); + assert_eq!(source.bytes(), reserved, "changing Amount replaces the reserved mix"); + } + #[test] fn decoder_info_survives_render_jobs_cache_and_eviction() { let tone = lightcraft_pipeline::tone::CameraTone::new(std::array::from_fn(|i| { diff --git a/crates/engine/src/model_download.rs b/crates/engine/src/model_download.rs index f07c373d..98707f78 100644 --- a/crates/engine/src/model_download.rs +++ b/crates/engine/src/model_download.rs @@ -35,6 +35,7 @@ pub enum State { sha256: String, }, /// Installed and in use: nothing more to do but tell the user. + Installing, Installed, Failed(String), Cancelled, @@ -107,7 +108,7 @@ impl Downloads { job.cancel.store(true, Ordering::Relaxed); match job.get() { // the thread notices, deletes the partial file and marks itself cancelled - State::Running { .. } => {} + State::Running { .. } | State::Installing => {} State::Done { path, .. } => { let _ = std::fs::remove_file(path); self.jobs.remove(id); @@ -119,11 +120,6 @@ impl Downloads { true } - /// The staged file at `path` has been installed (its copy is the model now) or is no longer wanted. - pub fn staged_file_used(&mut self, path: &Path) { - self.jobs.retain(|_, j| !matches!(j.get(), State::Done { path: p, .. } if p == path)); - } - /// Downloads that have arrived and been checked, ready to install: (model id, staged file, its SHA-256). pub fn finished(&self) -> Vec<(String, PathBuf, String)> { let mut out: Vec<_> = self @@ -157,7 +153,7 @@ impl Drop for Downloads { for job in self.jobs.values() { job.cancel.store(true, Ordering::Relaxed); } - let started = std::time::Instant::now(); + let started = web_time::Instant::now(); while self.running() && started.elapsed() < Duration::from_secs(1) { std::thread::sleep(Duration::from_millis(10)); } @@ -336,7 +332,7 @@ mod tests { let mut d = Downloads::default(); d.jobs.insert("m".into(), shared.clone()); assert!(d.running()); - let started = std::time::Instant::now(); + let started = web_time::Instant::now(); drop(d); assert!(started.elapsed() < Duration::from_millis(900), "quitting must not wait for a transfer to finish"); assert_eq!(shared.get(), State::Cancelled); diff --git a/crates/engine/src/tests_denoise.rs b/crates/engine/src/tests_denoise.rs index 172f9082..ef87747f 100644 --- a/crates/engine/src/tests_denoise.rs +++ b/crates/engine/src/tests_denoise.rs @@ -81,6 +81,65 @@ fn model_files(dir: &Path, id: &str) -> PathBuf { src.join("model.onnx") } +#[test] +fn model_installation_keeps_the_frame_pump_responsive_and_reuses_the_loaded_model() { + use std::sync::{Mutex, mpsc}; + use std::time::Duration; + let mut x = setup("background-install", false); + let path = model_files(&x.dir, "stand-in"); + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + let release_rx = Mutex::new(release_rx); + let runs = x.runs.clone(); + let loads = Arc::new(AtomicUsize::new(0)); + let worker_loads = loads.clone(); + x.s.denoise.loader = Arc::new(move |_, _| { + worker_loads.fetch_add(1, Ordering::Relaxed); + entered_tx.send(()).map_err(|e| e.to_string())?; + release_rx.lock().unwrap().recv_timeout(Duration::from_secs(3)).map_err(|e| e.to_string())?; + Ok(Arc::new(Dim { runs: runs.clone() })) + }); + let info = x.s.execute("denoise.models.inspect", &json!({"path": path})).unwrap(); + assert_eq!(info["model"]["id"], "stand-in"); + assert_eq!(loads.load(Ordering::Relaxed), 0, "showing terms never executes the model"); + let r = x.s.execute("denoise.models.install", &json!({"path": path, "acknowledged": true, "background": true})).unwrap(); + assert_eq!(r["started"], "local-install"); + entered_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + // The loader is blocked: the frame pump must still return, reporting installation in progress. + let state = x.s.execute("denoise.models.downloads", &json!({})).unwrap(); + assert_eq!(state["downloads"][0]["state"], "installing"); + assert_eq!(x.s.execute("denoise.pump", &json!({"pace": "pause"})).unwrap()["active"], false); + assert!(x.s.execute("denoise.models.install", &json!({"path": path, "acknowledged": true, "background": true})).is_err()); + release_tx.send(()).unwrap(); + for _ in 0..600 { + let v = x.s.execute("denoise.models.downloads", &json!({})).unwrap(); + if v["downloads"][0]["state"] == "installed" { + break; + } + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(x.s.execute("denoise.models.downloads", &json!({})).unwrap()["downloads"][0]["state"], "installed"); + let a = x.s.denoise.active.as_ref().unwrap(); + assert!(a.model.lock().unwrap().is_some(), "the worker's tested model is reused"); + assert_eq!(loads.load(Ordering::Relaxed), 1); + drop(x.s); + let _ = std::fs::remove_dir_all(x.dir); +} + +#[test] +fn failed_model_replacement_preserves_the_installed_model() { + let mut x = setup("replacement", true); + let installed = x.dir.join("denoise-models/stand-in/model.onnx"); + let original = std::fs::read(&installed).unwrap(); + let source = model_files(&x.dir, "stand-in"); + std::fs::write(&source, "a different broken model").unwrap(); + assert!(x.s.execute("denoise.models.install", &json!({"path": source, "acknowledged": true})).is_err()); + assert_eq!(std::fs::read(installed).unwrap(), original); + assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["model"], "stand-in"); + drop(x.s); + let _ = std::fs::remove_dir_all(x.dir); +} + /// A Bayer DNG that is a smooth ramp, big enough for a 64-cell tile and a second one. fn ramp_dng(w: usize, h: usize, cfa: &str) -> Vec { let data: Vec = (0..w * h).map(|i| 600 + ((i % w) * 30 + (i / w) * 35) as u16).collect(); @@ -339,7 +398,8 @@ fn a_model_that_fails_fails_the_export_and_is_not_retried_by_the_queue() { let mut x = setup("fails", true); let id = photo(&x.s); x.s.denoise.loader = Arc::new(|_, _| Err("the model went away".into())); - // (the model chosen is read afresh: it was loaded by the install's self-test only) + // Simulate loss of the in-memory model; successful installation now retains its tested instance. + *x.s.denoise.active.as_ref().unwrap().model.lock().unwrap() = None; set_amount(&mut x.s, 80.0); let o = ExportOptions::from_json(&json!({"format": "png", "width": 100, "height": 100})); let e = export_photo(&mut x.s, id, &o, 1).err().unwrap(); diff --git a/crates/gpu/Cargo.toml b/crates/gpu/Cargo.toml index 314f69c8..b19acf54 100644 --- a/crates/gpu/Cargo.toml +++ b/crates/gpu/Cargo.toml @@ -9,7 +9,7 @@ description = "LightCraft GPU develop pipeline: wgpu compute (WGSL) twin of the workspace = true [dependencies] -lightcraft-denoise = { workspace = true } +lightcraft-denoise = { workspace = true, optional = true } lightcraft-geom = { workspace = true } lightcraft-color = { workspace = true } lightcraft-raster = { workspace = true } @@ -24,5 +24,9 @@ wgpu = { workspace = true } pollster = { workspace = true } bytemuck = { workspace = true } +[features] +default = [] +denoise = ["dep:lightcraft-denoise", "lightcraft-denoise/runtime"] + [dev-dependencies] lightcraft-scenes = { workspace = true } diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index b4e32ba9..9c234964 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -29,7 +29,7 @@ use lightcraft_raster::Rgb32f; pub mod backend; #[cfg(not(target_arch = "wasm32"))] mod ctx; -#[cfg(not(target_arch = "wasm32"))] +#[cfg(all(feature = "denoise", not(target_arch = "wasm32")))] pub mod nn; #[cfg(not(target_arch = "wasm32"))] mod params; @@ -149,7 +149,7 @@ pub fn enabled() -> bool { /// Why GPU compute that has a device of its own (denoise) is switched off by the user's settings, or `None`: the /// environment switches and the rendering preference, but not a render-device failure (the devices are separate). -#[cfg(not(target_arch = "wasm32"))] +#[cfg(all(feature = "denoise", not(target_arch = "wasm32")))] pub(crate) fn switched_off() -> Option { if env_disabled() { return Some(if backend::env_off() { "disabled by LIGHTCRAFT_GPU_BACKEND=off" } else { "disabled by LIGHTCRAFT_GPU=0" }.into()); diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml index 7836d96c..f36ac02f 100644 --- a/crates/ui-egui/Cargo.toml +++ b/crates/ui-egui/Cargo.toml @@ -25,8 +25,8 @@ serde_json = { workspace = true } log = { workspace = true } [dev-dependencies] +lightcraft-denoise = { workspace = true, features = ["runtime"] } lightcraft-codecs = { workspace = true } -lightcraft-denoise = { workspace = true } [build-dependencies] serde_json = { workspace = true } diff --git a/crates/ui-egui/locales/de-formats.json b/crates/ui-egui/locales/de-formats.json index af10202f..12672394 100644 --- a/crates/ui-egui/locales/de-formats.json +++ b/crates/ui-egui/locales/de-formats.json @@ -187,5 +187,24 @@ "Moved; {} photo(s) relinked": "Verschoben; {} Fotos neu verknüpft", "{} label (click more labels to show any of them)": "Markierung {} (weitere Markierungen auswählen, um alle passenden Fotos zu zeigen)", ", {} star{}": ", Sterne: {}{:.0}", - ", {} label": ", Markierung: {}" + ", {} label": ", Markierung: {}", + "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.": "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.", + "{:.1} MB": "{:.1} MB", + "Running on the processor: {}.": "Running on the processor: {}.", + "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.": "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.", + "Denoising {name}: {done} of {total} tiles{}": "Denoising {name}: {done} of {total} tiles{}", + "{size} GB": "{size} GB", + "{} of {} from {host}": "{} of {} from {host}", + "{} KB": "{} KB", + "{} MB": "{} MB", + "Making the denoised picture… {}%": "Making the denoised picture… {}%", + "{name} · {terms}": "{name} · {terms}", + "{n} {noun}": "{n} {noun}", + "Trained on: {provenance}": "Trained on: {provenance}", + " · {queued} waiting": " · {queued} waiting", + "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.": "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.", + "In line for AI Denoise ({ahead} ahead)…": "In line for AI Denoise ({ahead} ahead)…", + "Running on {}.": "Running on {}.", + "{file} · {size}": "{file} · {size}", + "• {assumption}": "• {assumption}" } diff --git a/crates/ui-egui/locales/de.json b/crates/ui-egui/locales/de.json index 2a472186..2465cd8e 100644 --- a/crates/ui-egui/locales/de.json +++ b/crates/ui-egui/locales/de.json @@ -1797,5 +1797,94 @@ "How to install the model": "So installierst du das Modell", "Install the SAM 3 Model": "SAM-3-Modell installieren", "This build can't download the model yet. To install it by hand, put model.safetensors, vocab.json and merges.txt from Meta's facebook/sam3 in the folder above: Object and Describe work as soon as they are there.": "Diese Version kann das Modell noch nicht herunterladen. Zum Installieren von Hand lege model.safetensors, vocab.json und merges.txt aus Metas facebook/sam3 in den obigen Ordner: Objekt und Beschreiben funktionieren, sobald sie dort liegen.", - "Open Log Folder": "Protokollordner öffnen" + "Open Log Folder": "Protokollordner öffnen", + "AI Denoise": "AI Denoise", + "AI Denoise Model": "AI Denoise Model", + "AI Denoise is off: press Use on a model.": "AI Denoise is off: press Use on a model.", + "AI Denoise needs a model. Install one in Settings.": "AI Denoise needs a model. Install one in Settings.", + "Accept & Download": "Accept & Download", + "Already installed. Remove it before installing a replacement.": "Already installed. Remove it before installing a replacement.", + "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.": "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.", + "Cache": "Cache", + "Checking and installing the model…": "Checking and installing the model…", + "Checking…": "Checking…", + "Clear cache": "Clear cache", + "Denoise Models": "Denoise Models", + "Denoise Settings": "Denoise Settings", + "Denoise all photos with an Amount": "Denoise all photos with an Amount", + "Downloaded and checked. Installing…": "Downloaded and checked. Installing…", + "Failed its last test": "Failed its last test", + "Graphics card": "Graphics card", + "I have read these terms and accept them for my own use": "I have read these terms and accept them for my own use", + "Inspect Denoise Model": "Inspect Denoise Model", + "Install Denoise Model": "Install Denoise Model", + "Install from file…": "Install from file…", + "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.": "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.", + "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).": "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).", + "LightCraft does not know this model, so it assumed:": "LightCraft does not know this model, so it assumed:", + "Limit": "Limit", + "Make it now": "Make it now", + "Make the denoised picture of the photos I am looking at": "Make the denoised picture of the photos I am looking at", + "Next up for AI Denoise…": "Next up for AI Denoise…", + "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.": "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.", + "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.": "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.", + "Open page": "Open page", + "Open the model's page": "Open the model's page", + "Opens the model's own page in your browser, to read about it or get the file yourself.": "Opens the model's own page in your browser, to read about it or get the file yourself.", + "Processor": "Processor", + "Set up AI Denoise…": "Set up AI Denoise…", + "Speed": "Speed", + "Starting…": "Starting…", + "Tested and works on this computer": "Tested and works on this computer", + "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works": "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works", + "The graphics card is set up when the next photo is made.": "The graphics card is set up when the next photo is made.", + "The model is kept on this computer only. LightCraft never uploads or shares it.": "The model is kept on this computer only. LightCraft never uploads or shares it.", + "The processor does the work.": "The processor does the work.", + "There is no model file to install": "There is no model file to install", + "This build cannot run denoise models.": "This build cannot run denoise models.", + "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)": "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)", + "This file cannot be used yet": "This file cannot be used yet", + "Tick the box to accept the model's terms first": "Tick the box to accept the model's terms first", + "Try again": "Try again", + "Try the graphics card again": "Try the graphics card again", + "Turn off": "Turn off", + "Unknown licence": "Unknown licence", + "Use": "Use", + "Waiting to start…": "Waiting to start…", + "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.": "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.", + "cached picture": "cached picture", + "cached pictures": "cached pictures", + "commercial use allowed": "commercial use allowed", + "denoiseModel.accept": "denoiseModel.accept", + "in use": "in use", + "installed": "installed", + "makes AI Denoise available": "makes AI Denoise available", + "non-commercial use only": "non-commercial use only", + "photo has its denoised picture.": "photo has its denoised picture.", + "photo waiting": "photo waiting", + "photos have their denoised picture.": "photos have their denoised picture.", + "photos waiting": "photos waiting", + "starts using it for AI Denoise": "starts using it for AI Denoise", + "terms unclear": "terms unclear", + "the graphics card": "the graphics card", + "the graphics card is not used": "the graphics card is not used", + "Cancel Denoise": "Cancel Denoise", + "Cancel Denoise Model Download": "Cancel Denoise Model Download", + "Choose Denoise Model": "Choose Denoise Model", + "Clear Denoise Cache": "Clear Denoise Cache", + "Denoise Model Downloads": "Denoise Model Downloads", + "Denoise Photos": "Denoise Photos", + "Denoise Pump": "Denoise Pump", + "Denoise Status": "Denoise Status", + "Download Denoise Model": "Download Denoise Model", + "Remove Denoise Model": "Remove Denoise Model", + "Test Denoise Model": "Test Denoise Model", + "AI Denoise is enabled on the requesting photo": "AI Denoise is enabled on the requesting photo", + "After installation, AI Denoise will turn on for the requesting photo.": "After installation, AI Denoise will turn on for the requesting photo.", + "Cancel pending action": "Cancel pending action", + "Install a denoise model to enable AI Denoise on this photo": "Install a denoise model to enable AI Denoise on this photo", + "Pending photo action cancelled": "Pending photo action cancelled", + "Pending photo action cancelled because its library or photo changed": "Pending photo action cancelled because its library or photo changed", + "ONNX model": "ONNX model", + "commercial use subject to GPL conditions": "commercial use subject to GPL conditions" } diff --git a/crates/ui-egui/locales/es-formats.json b/crates/ui-egui/locales/es-formats.json index b67d6720..54418c76 100644 --- a/crates/ui-egui/locales/es-formats.json +++ b/crates/ui-egui/locales/es-formats.json @@ -187,5 +187,24 @@ "Moved; {} photo(s) relinked": "Movida; {} foto(s) vinculada(s) de nuevo", "{} label (click more labels to show any of them)": "Etiqueta {} (hacer clic en más etiquetas para mostrar cualquiera de ellas)", ", {} star{}": ", {} estrella{}", - ", {} label": ", etiqueta {}" + ", {} label": ", etiqueta {}", + "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.": "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.", + "{:.1} MB": "{:.1} MB", + "Running on the processor: {}.": "Running on the processor: {}.", + "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.": "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.", + "Denoising {name}: {done} of {total} tiles{}": "Denoising {name}: {done} of {total} tiles{}", + "{size} GB": "{size} GB", + "{} of {} from {host}": "{} of {} from {host}", + "{} KB": "{} KB", + "{} MB": "{} MB", + "Making the denoised picture… {}%": "Making the denoised picture… {}%", + "{name} · {terms}": "{name} · {terms}", + "{n} {noun}": "{n} {noun}", + "Trained on: {provenance}": "Trained on: {provenance}", + " · {queued} waiting": " · {queued} waiting", + "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.": "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.", + "In line for AI Denoise ({ahead} ahead)…": "In line for AI Denoise ({ahead} ahead)…", + "Running on {}.": "Running on {}.", + "{file} · {size}": "{file} · {size}", + "• {assumption}": "• {assumption}" } diff --git a/crates/ui-egui/locales/es.json b/crates/ui-egui/locales/es.json index c0986c19..032412c5 100644 --- a/crates/ui-egui/locales/es.json +++ b/crates/ui-egui/locales/es.json @@ -1642,5 +1642,94 @@ "Captured from": "Capturadas desde", "Captured until": "Capturadas hasta", "Only": "Solo", - "Open Log Folder": "Abrir carpeta de registros" + "Open Log Folder": "Abrir carpeta de registros", + "AI Denoise": "AI Denoise", + "AI Denoise Model": "AI Denoise Model", + "AI Denoise is off: press Use on a model.": "AI Denoise is off: press Use on a model.", + "AI Denoise needs a model. Install one in Settings.": "AI Denoise needs a model. Install one in Settings.", + "Accept & Download": "Accept & Download", + "Already installed. Remove it before installing a replacement.": "Already installed. Remove it before installing a replacement.", + "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.": "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.", + "Cache": "Cache", + "Checking and installing the model…": "Checking and installing the model…", + "Checking…": "Checking…", + "Clear cache": "Clear cache", + "Denoise Models": "Denoise Models", + "Denoise Settings": "Denoise Settings", + "Denoise all photos with an Amount": "Denoise all photos with an Amount", + "Downloaded and checked. Installing…": "Downloaded and checked. Installing…", + "Failed its last test": "Failed its last test", + "Graphics card": "Graphics card", + "I have read these terms and accept them for my own use": "I have read these terms and accept them for my own use", + "Inspect Denoise Model": "Inspect Denoise Model", + "Install Denoise Model": "Install Denoise Model", + "Install from file…": "Install from file…", + "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.": "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.", + "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).": "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).", + "LightCraft does not know this model, so it assumed:": "LightCraft does not know this model, so it assumed:", + "Limit": "Limit", + "Make it now": "Make it now", + "Make the denoised picture of the photos I am looking at": "Make the denoised picture of the photos I am looking at", + "Next up for AI Denoise…": "Next up for AI Denoise…", + "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.": "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.", + "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.": "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.", + "Open page": "Open page", + "Open the model's page": "Open the model's page", + "Opens the model's own page in your browser, to read about it or get the file yourself.": "Opens the model's own page in your browser, to read about it or get the file yourself.", + "Processor": "Processor", + "Set up AI Denoise…": "Set up AI Denoise…", + "Speed": "Speed", + "Starting…": "Starting…", + "Tested and works on this computer": "Tested and works on this computer", + "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works": "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works", + "The graphics card is set up when the next photo is made.": "The graphics card is set up when the next photo is made.", + "The model is kept on this computer only. LightCraft never uploads or shares it.": "The model is kept on this computer only. LightCraft never uploads or shares it.", + "The processor does the work.": "The processor does the work.", + "There is no model file to install": "There is no model file to install", + "This build cannot run denoise models.": "This build cannot run denoise models.", + "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)": "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)", + "This file cannot be used yet": "This file cannot be used yet", + "Tick the box to accept the model's terms first": "Tick the box to accept the model's terms first", + "Try again": "Try again", + "Try the graphics card again": "Try the graphics card again", + "Turn off": "Turn off", + "Unknown licence": "Unknown licence", + "Use": "Use", + "Waiting to start…": "Waiting to start…", + "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.": "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.", + "cached picture": "cached picture", + "cached pictures": "cached pictures", + "commercial use allowed": "commercial use allowed", + "denoiseModel.accept": "denoiseModel.accept", + "in use": "in use", + "installed": "installed", + "makes AI Denoise available": "makes AI Denoise available", + "non-commercial use only": "non-commercial use only", + "photo has its denoised picture.": "photo has its denoised picture.", + "photo waiting": "photo waiting", + "photos have their denoised picture.": "photos have their denoised picture.", + "photos waiting": "photos waiting", + "starts using it for AI Denoise": "starts using it for AI Denoise", + "terms unclear": "terms unclear", + "the graphics card": "the graphics card", + "the graphics card is not used": "the graphics card is not used", + "Cancel Denoise": "Cancel Denoise", + "Cancel Denoise Model Download": "Cancel Denoise Model Download", + "Choose Denoise Model": "Choose Denoise Model", + "Clear Denoise Cache": "Clear Denoise Cache", + "Denoise Model Downloads": "Denoise Model Downloads", + "Denoise Photos": "Denoise Photos", + "Denoise Pump": "Denoise Pump", + "Denoise Status": "Denoise Status", + "Download Denoise Model": "Download Denoise Model", + "Remove Denoise Model": "Remove Denoise Model", + "Test Denoise Model": "Test Denoise Model", + "AI Denoise is enabled on the requesting photo": "AI Denoise is enabled on the requesting photo", + "After installation, AI Denoise will turn on for the requesting photo.": "After installation, AI Denoise will turn on for the requesting photo.", + "Cancel pending action": "Cancel pending action", + "Install a denoise model to enable AI Denoise on this photo": "Install a denoise model to enable AI Denoise on this photo", + "Pending photo action cancelled": "Pending photo action cancelled", + "Pending photo action cancelled because its library or photo changed": "Pending photo action cancelled because its library or photo changed", + "ONNX model": "ONNX model", + "commercial use subject to GPL conditions": "commercial use subject to GPL conditions" } diff --git a/crates/ui-egui/locales/ja-formats.json b/crates/ui-egui/locales/ja-formats.json index e462871c..b0e0f2f9 100644 --- a/crates/ui-egui/locales/ja-formats.json +++ b/crates/ui-egui/locales/ja-formats.json @@ -187,5 +187,24 @@ "Moved; {} photo(s) relinked": "移動しました。{}枚の写真の参照先を更新しました", "{} label (click more labels to show any of them)": "{}ラベル（ほかのラベルもクリックすると、いずれかに一致する写真を表示）", ", {} star{}": "、星{}つ{:.0}", - ", {} label": "、{}ラベル" + ", {} label": "、{}ラベル", + "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.": "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.", + "{:.1} MB": "{:.1} MB", + "Running on the processor: {}.": "Running on the processor: {}.", + "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.": "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.", + "Denoising {name}: {done} of {total} tiles{}": "Denoising {name}: {done} of {total} tiles{}", + "{size} GB": "{size} GB", + "{} of {} from {host}": "{} of {} from {host}", + "{} KB": "{} KB", + "{} MB": "{} MB", + "Making the denoised picture… {}%": "Making the denoised picture… {}%", + "{name} · {terms}": "{name} · {terms}", + "{n} {noun}": "{n} {noun}", + "Trained on: {provenance}": "Trained on: {provenance}", + " · {queued} waiting": " · {queued} waiting", + "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.": "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.", + "In line for AI Denoise ({ahead} ahead)…": "In line for AI Denoise ({ahead} ahead)…", + "Running on {}.": "Running on {}.", + "{file} · {size}": "{file} · {size}", + "• {assumption}": "• {assumption}" } diff --git a/crates/ui-egui/locales/ja.json b/crates/ui-egui/locales/ja.json index 31c61a7a..dc22711b 100644 --- a/crates/ui-egui/locales/ja.json +++ b/crates/ui-egui/locales/ja.json @@ -1642,5 +1642,94 @@ "How to install the model": "モデルのインストール方法", "Install the SAM 3 Model": "SAM 3モデルのインストール", "This build can't download the model yet. To install it by hand, put model.safetensors, vocab.json and merges.txt from Meta's facebook/sam3 in the folder above: Object and Describe work as soon as they are there.": "このビルドではまだモデルをダウンロードできません。手動でインストールするには、Meta の facebook/sam3 の model.safetensors、vocab.json、merges.txt を上のフォルダに置いてください。置いた時点で「オブジェクト」と「説明で選択」が使えるようになります。", - "Open Log Folder": "ログフォルダーを開く" + "Open Log Folder": "ログフォルダーを開く", + "AI Denoise": "AI Denoise", + "AI Denoise Model": "AI Denoise Model", + "AI Denoise is off: press Use on a model.": "AI Denoise is off: press Use on a model.", + "AI Denoise needs a model. Install one in Settings.": "AI Denoise needs a model. Install one in Settings.", + "Accept & Download": "Accept & Download", + "Already installed. Remove it before installing a replacement.": "Already installed. Remove it before installing a replacement.", + "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.": "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.", + "Cache": "Cache", + "Checking and installing the model…": "Checking and installing the model…", + "Checking…": "Checking…", + "Clear cache": "Clear cache", + "Denoise Models": "Denoise Models", + "Denoise Settings": "Denoise Settings", + "Denoise all photos with an Amount": "Denoise all photos with an Amount", + "Downloaded and checked. Installing…": "Downloaded and checked. Installing…", + "Failed its last test": "Failed its last test", + "Graphics card": "Graphics card", + "I have read these terms and accept them for my own use": "I have read these terms and accept them for my own use", + "Inspect Denoise Model": "Inspect Denoise Model", + "Install Denoise Model": "Install Denoise Model", + "Install from file…": "Install from file…", + "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.": "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.", + "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).": "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).", + "LightCraft does not know this model, so it assumed:": "LightCraft does not know this model, so it assumed:", + "Limit": "Limit", + "Make it now": "Make it now", + "Make the denoised picture of the photos I am looking at": "Make the denoised picture of the photos I am looking at", + "Next up for AI Denoise…": "Next up for AI Denoise…", + "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.": "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.", + "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.": "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.", + "Open page": "Open page", + "Open the model's page": "Open the model's page", + "Opens the model's own page in your browser, to read about it or get the file yourself.": "Opens the model's own page in your browser, to read about it or get the file yourself.", + "Processor": "Processor", + "Set up AI Denoise…": "Set up AI Denoise…", + "Speed": "Speed", + "Starting…": "Starting…", + "Tested and works on this computer": "Tested and works on this computer", + "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works": "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works", + "The graphics card is set up when the next photo is made.": "The graphics card is set up when the next photo is made.", + "The model is kept on this computer only. LightCraft never uploads or shares it.": "The model is kept on this computer only. LightCraft never uploads or shares it.", + "The processor does the work.": "The processor does the work.", + "There is no model file to install": "There is no model file to install", + "This build cannot run denoise models.": "This build cannot run denoise models.", + "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)": "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)", + "This file cannot be used yet": "This file cannot be used yet", + "Tick the box to accept the model's terms first": "Tick the box to accept the model's terms first", + "Try again": "Try again", + "Try the graphics card again": "Try the graphics card again", + "Turn off": "Turn off", + "Unknown licence": "Unknown licence", + "Use": "Use", + "Waiting to start…": "Waiting to start…", + "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.": "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.", + "cached picture": "cached picture", + "cached pictures": "cached pictures", + "commercial use allowed": "commercial use allowed", + "denoiseModel.accept": "denoiseModel.accept", + "in use": "in use", + "installed": "installed", + "makes AI Denoise available": "makes AI Denoise available", + "non-commercial use only": "non-commercial use only", + "photo has its denoised picture.": "photo has its denoised picture.", + "photo waiting": "photo waiting", + "photos have their denoised picture.": "photos have their denoised picture.", + "photos waiting": "photos waiting", + "starts using it for AI Denoise": "starts using it for AI Denoise", + "terms unclear": "terms unclear", + "the graphics card": "the graphics card", + "the graphics card is not used": "the graphics card is not used", + "Cancel Denoise": "Cancel Denoise", + "Cancel Denoise Model Download": "Cancel Denoise Model Download", + "Choose Denoise Model": "Choose Denoise Model", + "Clear Denoise Cache": "Clear Denoise Cache", + "Denoise Model Downloads": "Denoise Model Downloads", + "Denoise Photos": "Denoise Photos", + "Denoise Pump": "Denoise Pump", + "Denoise Status": "Denoise Status", + "Download Denoise Model": "Download Denoise Model", + "Remove Denoise Model": "Remove Denoise Model", + "Test Denoise Model": "Test Denoise Model", + "AI Denoise is enabled on the requesting photo": "AI Denoise is enabled on the requesting photo", + "After installation, AI Denoise will turn on for the requesting photo.": "After installation, AI Denoise will turn on for the requesting photo.", + "Cancel pending action": "Cancel pending action", + "Install a denoise model to enable AI Denoise on this photo": "Install a denoise model to enable AI Denoise on this photo", + "Pending photo action cancelled": "Pending photo action cancelled", + "Pending photo action cancelled because its library or photo changed": "Pending photo action cancelled because its library or photo changed", + "ONNX model": "ONNX model", + "commercial use subject to GPL conditions": "commercial use subject to GPL conditions" } diff --git a/crates/ui-egui/locales/pt-br-formats.json b/crates/ui-egui/locales/pt-br-formats.json index 59bec59d..ab64e082 100644 --- a/crates/ui-egui/locales/pt-br-formats.json +++ b/crates/ui-egui/locales/pt-br-formats.json @@ -187,5 +187,24 @@ "Moved; {} photo(s) relinked": "Movida; {} foto(s) vinculada(s) novamente", "{} label (click more labels to show any of them)": "Rótulo {} (clique em mais rótulos para mostrar qualquer um deles)", ", {} star{}": ", {} estrela{}", - ", {} label": ", rótulo {}" + ", {} label": ", rótulo {}", + "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.": "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.", + "{:.1} MB": "{:.1} MB", + "Running on the processor: {}.": "Running on the processor: {}.", + "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.": "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.", + "Denoising {name}: {done} of {total} tiles{}": "Denoising {name}: {done} of {total} tiles{}", + "{size} GB": "{size} GB", + "{} of {} from {host}": "{} of {} from {host}", + "{} KB": "{} KB", + "{} MB": "{} MB", + "Making the denoised picture… {}%": "Making the denoised picture… {}%", + "{name} · {terms}": "{name} · {terms}", + "{n} {noun}": "{n} {noun}", + "Trained on: {provenance}": "Trained on: {provenance}", + " · {queued} waiting": " · {queued} waiting", + "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.": "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.", + "In line for AI Denoise ({ahead} ahead)…": "In line for AI Denoise ({ahead} ahead)…", + "Running on {}.": "Running on {}.", + "{file} · {size}": "{file} · {size}", + "• {assumption}": "• {assumption}" } diff --git a/crates/ui-egui/locales/pt-br.json b/crates/ui-egui/locales/pt-br.json index 03ae0773..521da1b1 100644 --- a/crates/ui-egui/locales/pt-br.json +++ b/crates/ui-egui/locales/pt-br.json @@ -1642,5 +1642,94 @@ "≤ at most": "≤ no máximo", "≥ at least": "≥ no mínimo", "日本語": "日本語", - "繁體中文（台灣）": "繁體中文（台灣）" + "繁體中文（台灣）": "繁體中文（台灣）", + "AI Denoise": "AI Denoise", + "AI Denoise Model": "AI Denoise Model", + "AI Denoise is off: press Use on a model.": "AI Denoise is off: press Use on a model.", + "AI Denoise needs a model. Install one in Settings.": "AI Denoise needs a model. Install one in Settings.", + "Accept & Download": "Accept & Download", + "Already installed. Remove it before installing a replacement.": "Already installed. Remove it before installing a replacement.", + "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.": "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.", + "Cache": "Cache", + "Checking and installing the model…": "Checking and installing the model…", + "Checking…": "Checking…", + "Clear cache": "Clear cache", + "Denoise Models": "Denoise Models", + "Denoise Settings": "Denoise Settings", + "Denoise all photos with an Amount": "Denoise all photos with an Amount", + "Downloaded and checked. Installing…": "Downloaded and checked. Installing…", + "Failed its last test": "Failed its last test", + "Graphics card": "Graphics card", + "I have read these terms and accept them for my own use": "I have read these terms and accept them for my own use", + "Inspect Denoise Model": "Inspect Denoise Model", + "Install Denoise Model": "Install Denoise Model", + "Install from file…": "Install from file…", + "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.": "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.", + "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).": "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).", + "LightCraft does not know this model, so it assumed:": "LightCraft does not know this model, so it assumed:", + "Limit": "Limit", + "Make it now": "Make it now", + "Make the denoised picture of the photos I am looking at": "Make the denoised picture of the photos I am looking at", + "Next up for AI Denoise…": "Next up for AI Denoise…", + "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.": "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.", + "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.": "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.", + "Open page": "Open page", + "Open the model's page": "Open the model's page", + "Opens the model's own page in your browser, to read about it or get the file yourself.": "Opens the model's own page in your browser, to read about it or get the file yourself.", + "Processor": "Processor", + "Set up AI Denoise…": "Set up AI Denoise…", + "Speed": "Speed", + "Starting…": "Starting…", + "Tested and works on this computer": "Tested and works on this computer", + "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works": "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works", + "The graphics card is set up when the next photo is made.": "The graphics card is set up when the next photo is made.", + "The model is kept on this computer only. LightCraft never uploads or shares it.": "The model is kept on this computer only. LightCraft never uploads or shares it.", + "The processor does the work.": "The processor does the work.", + "There is no model file to install": "There is no model file to install", + "This build cannot run denoise models.": "This build cannot run denoise models.", + "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)": "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)", + "This file cannot be used yet": "This file cannot be used yet", + "Tick the box to accept the model's terms first": "Tick the box to accept the model's terms first", + "Try again": "Try again", + "Try the graphics card again": "Try the graphics card again", + "Turn off": "Turn off", + "Unknown licence": "Unknown licence", + "Use": "Use", + "Waiting to start…": "Waiting to start…", + "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.": "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.", + "cached picture": "cached picture", + "cached pictures": "cached pictures", + "commercial use allowed": "commercial use allowed", + "denoiseModel.accept": "denoiseModel.accept", + "in use": "in use", + "installed": "installed", + "makes AI Denoise available": "makes AI Denoise available", + "non-commercial use only": "non-commercial use only", + "photo has its denoised picture.": "photo has its denoised picture.", + "photo waiting": "photo waiting", + "photos have their denoised picture.": "photos have their denoised picture.", + "photos waiting": "photos waiting", + "starts using it for AI Denoise": "starts using it for AI Denoise", + "terms unclear": "terms unclear", + "the graphics card": "the graphics card", + "the graphics card is not used": "the graphics card is not used", + "Cancel Denoise": "Cancel Denoise", + "Cancel Denoise Model Download": "Cancel Denoise Model Download", + "Choose Denoise Model": "Choose Denoise Model", + "Clear Denoise Cache": "Clear Denoise Cache", + "Denoise Model Downloads": "Denoise Model Downloads", + "Denoise Photos": "Denoise Photos", + "Denoise Pump": "Denoise Pump", + "Denoise Status": "Denoise Status", + "Download Denoise Model": "Download Denoise Model", + "Remove Denoise Model": "Remove Denoise Model", + "Test Denoise Model": "Test Denoise Model", + "AI Denoise is enabled on the requesting photo": "AI Denoise is enabled on the requesting photo", + "After installation, AI Denoise will turn on for the requesting photo.": "After installation, AI Denoise will turn on for the requesting photo.", + "Cancel pending action": "Cancel pending action", + "Install a denoise model to enable AI Denoise on this photo": "Install a denoise model to enable AI Denoise on this photo", + "Pending photo action cancelled": "Pending photo action cancelled", + "Pending photo action cancelled because its library or photo changed": "Pending photo action cancelled because its library or photo changed", + "ONNX model": "ONNX model", + "commercial use subject to GPL conditions": "commercial use subject to GPL conditions" } diff --git a/crates/ui-egui/locales/ru-formats.json b/crates/ui-egui/locales/ru-formats.json index 2beb4a5b..f3a0f647 100644 --- a/crates/ui-egui/locales/ru-formats.json +++ b/crates/ui-egui/locales/ru-formats.json @@ -187,5 +187,24 @@ "{} — show their photos": "{} — показать фото этого человека", "· {dups} already in the library": "· уже в библиотеке: {dups}", "· {n} photo{}": "· фото: {n}{:.0}", - "… and {more} more": "… и ещё {more}" + "… and {more} more": "… и ещё {more}", + "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.": "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.", + "{:.1} MB": "{:.1} MB", + "Running on the processor: {}.": "Running on the processor: {}.", + "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.": "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.", + "Denoising {name}: {done} of {total} tiles{}": "Denoising {name}: {done} of {total} tiles{}", + "{size} GB": "{size} GB", + "{} of {} from {host}": "{} of {} from {host}", + "{} KB": "{} KB", + "{} MB": "{} MB", + "Making the denoised picture… {}%": "Making the denoised picture… {}%", + "{name} · {terms}": "{name} · {terms}", + "{n} {noun}": "{n} {noun}", + "Trained on: {provenance}": "Trained on: {provenance}", + " · {queued} waiting": " · {queued} waiting", + "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.": "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.", + "In line for AI Denoise ({ahead} ahead)…": "In line for AI Denoise ({ahead} ahead)…", + "Running on {}.": "Running on {}.", + "{file} · {size}": "{file} · {size}", + "• {assumption}": "• {assumption}" } diff --git a/crates/ui-egui/locales/ru.json b/crates/ui-egui/locales/ru.json index 530cdfcb..32ea6149 100644 --- a/crates/ui-egui/locales/ru.json +++ b/crates/ui-egui/locales/ru.json @@ -1312,5 +1312,95 @@ "of the following rules:": "из следующих условий:", "saved in": "сохранено в", "this photo": "это фото", - "− harder border · + softer border": "− жёстче край · + мягче край" + "− harder border · + softer border": "− жёстче край · + мягче край", + "AI Denoise": "AI Denoise", + "AI Denoise Model": "AI Denoise Model", + "AI Denoise is off: press Use on a model.": "AI Denoise is off: press Use on a model.", + "AI Denoise needs a model. Install one in Settings.": "AI Denoise needs a model. Install one in Settings.", + "Accept & Download": "Accept & Download", + "Already installed. Remove it before installing a replacement.": "Already installed. Remove it before installing a replacement.", + "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.": "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.", + "Cache": "Cache", + "Checking and installing the model…": "Checking and installing the model…", + "Checking…": "Checking…", + "Clear cache": "Clear cache", + "Denoise Models": "Denoise Models", + "Denoise Settings": "Denoise Settings", + "Denoise all photos with an Amount": "Denoise all photos with an Amount", + "Downloaded and checked. Installing…": "Downloaded and checked. Installing…", + "Failed its last test": "Failed its last test", + "Graphics card": "Graphics card", + "I have read these terms and accept them for my own use": "I have read these terms and accept them for my own use", + "Inspect Denoise Model": "Inspect Denoise Model", + "Install Denoise Model": "Install Denoise Model", + "Install from file…": "Install from file…", + "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.": "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.", + "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).": "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).", + "LightCraft does not know this model, so it assumed:": "LightCraft does not know this model, so it assumed:", + "Limit": "Limit", + "Make it now": "Make it now", + "Make the denoised picture of the photos I am looking at": "Make the denoised picture of the photos I am looking at", + "Models": "Models", + "Next up for AI Denoise…": "Next up for AI Denoise…", + "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.": "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.", + "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.": "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.", + "Open page": "Open page", + "Open the model's page": "Open the model's page", + "Opens the model's own page in your browser, to read about it or get the file yourself.": "Opens the model's own page in your browser, to read about it or get the file yourself.", + "Processor": "Processor", + "Set up AI Denoise…": "Set up AI Denoise…", + "Speed": "Speed", + "Starting…": "Starting…", + "Tested and works on this computer": "Tested and works on this computer", + "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works": "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works", + "The graphics card is set up when the next photo is made.": "The graphics card is set up when the next photo is made.", + "The model is kept on this computer only. LightCraft never uploads or shares it.": "The model is kept on this computer only. LightCraft never uploads or shares it.", + "The processor does the work.": "The processor does the work.", + "There is no model file to install": "There is no model file to install", + "This build cannot run denoise models.": "This build cannot run denoise models.", + "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)": "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)", + "This file cannot be used yet": "This file cannot be used yet", + "Tick the box to accept the model's terms first": "Tick the box to accept the model's terms first", + "Try again": "Try again", + "Try the graphics card again": "Try the graphics card again", + "Turn off": "Turn off", + "Unknown licence": "Unknown licence", + "Use": "Use", + "Waiting to start…": "Waiting to start…", + "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.": "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.", + "cached picture": "cached picture", + "cached pictures": "cached pictures", + "commercial use allowed": "commercial use allowed", + "denoiseModel.accept": "denoiseModel.accept", + "in use": "in use", + "installed": "installed", + "makes AI Denoise available": "makes AI Denoise available", + "non-commercial use only": "non-commercial use only", + "photo has its denoised picture.": "photo has its denoised picture.", + "photo waiting": "photo waiting", + "photos have their denoised picture.": "photos have their denoised picture.", + "photos waiting": "photos waiting", + "starts using it for AI Denoise": "starts using it for AI Denoise", + "terms unclear": "terms unclear", + "the graphics card": "the graphics card", + "the graphics card is not used": "the graphics card is not used", + "Cancel Denoise": "Cancel Denoise", + "Cancel Denoise Model Download": "Cancel Denoise Model Download", + "Choose Denoise Model": "Choose Denoise Model", + "Clear Denoise Cache": "Clear Denoise Cache", + "Denoise Model Downloads": "Denoise Model Downloads", + "Denoise Photos": "Denoise Photos", + "Denoise Pump": "Denoise Pump", + "Denoise Status": "Denoise Status", + "Download Denoise Model": "Download Denoise Model", + "Remove Denoise Model": "Remove Denoise Model", + "Test Denoise Model": "Test Denoise Model", + "AI Denoise is enabled on the requesting photo": "AI Denoise is enabled on the requesting photo", + "After installation, AI Denoise will turn on for the requesting photo.": "After installation, AI Denoise will turn on for the requesting photo.", + "Cancel pending action": "Cancel pending action", + "Install a denoise model to enable AI Denoise on this photo": "Install a denoise model to enable AI Denoise on this photo", + "Pending photo action cancelled": "Pending photo action cancelled", + "Pending photo action cancelled because its library or photo changed": "Pending photo action cancelled because its library or photo changed", + "ONNX model": "ONNX model", + "commercial use subject to GPL conditions": "commercial use subject to GPL conditions" } diff --git a/crates/ui-egui/locales/zh-hans-formats.json b/crates/ui-egui/locales/zh-hans-formats.json index eca40f8e..e32c7f66 100644 --- a/crates/ui-egui/locales/zh-hans-formats.json +++ b/crates/ui-egui/locales/zh-hans-formats.json @@ -187,5 +187,24 @@ "Moved; {} photo(s) relinked": "已移动；已重新链接 {0} 张照片", "{} label (click more labels to show any of them)": "{0}标签（单击更多标签可显示符合其中任一标签的照片）", ", {} star{}": "，{0} 星{1:.0}", - ", {} label": "，{0}标签" + ", {} label": "，{0}标签", + "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.": "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.", + "{:.1} MB": "{:.1} MB", + "Running on the processor: {}.": "Running on the processor: {}.", + "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.": "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.", + "Denoising {name}: {done} of {total} tiles{}": "Denoising {name}: {done} of {total} tiles{}", + "{size} GB": "{size} GB", + "{} of {} from {host}": "{} of {} from {host}", + "{} KB": "{} KB", + "{} MB": "{} MB", + "Making the denoised picture… {}%": "Making the denoised picture… {}%", + "{name} · {terms}": "{name} · {terms}", + "{n} {noun}": "{n} {noun}", + "Trained on: {provenance}": "Trained on: {provenance}", + " · {queued} waiting": " · {queued} waiting", + "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.": "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.", + "In line for AI Denoise ({ahead} ahead)…": "In line for AI Denoise ({ahead} ahead)…", + "Running on {}.": "Running on {}.", + "{file} · {size}": "{file} · {size}", + "• {assumption}": "• {assumption}" } diff --git a/crates/ui-egui/locales/zh-hans.json b/crates/ui-egui/locales/zh-hans.json index cea313d0..466f57b4 100644 --- a/crates/ui-egui/locales/zh-hans.json +++ b/crates/ui-egui/locales/zh-hans.json @@ -1642,5 +1642,94 @@ "How to install the model": "如何安装模型", "Install the SAM 3 Model": "安装 SAM 3 模型", "This build can't download the model yet. To install it by hand, put model.safetensors, vocab.json and merges.txt from Meta's facebook/sam3 in the folder above: Object and Describe work as soon as they are there.": "此版本尚无法下载模型。如需手动安装，请将 Meta 的 facebook/sam3 中的 model.safetensors、vocab.json 和 merges.txt 放入上方的文件夹：放入后即可使用“对象”和“描述”。", - "Open Log Folder": "打开日志文件夹" + "Open Log Folder": "打开日志文件夹", + "AI Denoise": "AI Denoise", + "AI Denoise Model": "AI Denoise Model", + "AI Denoise is off: press Use on a model.": "AI Denoise is off: press Use on a model.", + "AI Denoise needs a model. Install one in Settings.": "AI Denoise needs a model. Install one in Settings.", + "Accept & Download": "Accept & Download", + "Already installed. Remove it before installing a replacement.": "Already installed. Remove it before installing a replacement.", + "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.": "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.", + "Cache": "Cache", + "Checking and installing the model…": "Checking and installing the model…", + "Checking…": "Checking…", + "Clear cache": "Clear cache", + "Denoise Models": "Denoise Models", + "Denoise Settings": "Denoise Settings", + "Denoise all photos with an Amount": "Denoise all photos with an Amount", + "Downloaded and checked. Installing…": "Downloaded and checked. Installing…", + "Failed its last test": "Failed its last test", + "Graphics card": "Graphics card", + "I have read these terms and accept them for my own use": "I have read these terms and accept them for my own use", + "Inspect Denoise Model": "Inspect Denoise Model", + "Install Denoise Model": "Install Denoise Model", + "Install from file…": "Install from file…", + "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.": "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.", + "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).": "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).", + "LightCraft does not know this model, so it assumed:": "LightCraft does not know this model, so it assumed:", + "Limit": "Limit", + "Make it now": "Make it now", + "Make the denoised picture of the photos I am looking at": "Make the denoised picture of the photos I am looking at", + "Next up for AI Denoise…": "Next up for AI Denoise…", + "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.": "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.", + "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.": "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.", + "Open page": "Open page", + "Open the model's page": "Open the model's page", + "Opens the model's own page in your browser, to read about it or get the file yourself.": "Opens the model's own page in your browser, to read about it or get the file yourself.", + "Processor": "Processor", + "Set up AI Denoise…": "Set up AI Denoise…", + "Speed": "Speed", + "Starting…": "Starting…", + "Tested and works on this computer": "Tested and works on this computer", + "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works": "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works", + "The graphics card is set up when the next photo is made.": "The graphics card is set up when the next photo is made.", + "The model is kept on this computer only. LightCraft never uploads or shares it.": "The model is kept on this computer only. LightCraft never uploads or shares it.", + "The processor does the work.": "The processor does the work.", + "There is no model file to install": "There is no model file to install", + "This build cannot run denoise models.": "This build cannot run denoise models.", + "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)": "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)", + "This file cannot be used yet": "This file cannot be used yet", + "Tick the box to accept the model's terms first": "Tick the box to accept the model's terms first", + "Try again": "Try again", + "Try the graphics card again": "Try the graphics card again", + "Turn off": "Turn off", + "Unknown licence": "Unknown licence", + "Use": "Use", + "Waiting to start…": "Waiting to start…", + "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.": "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.", + "cached picture": "cached picture", + "cached pictures": "cached pictures", + "commercial use allowed": "commercial use allowed", + "denoiseModel.accept": "denoiseModel.accept", + "in use": "in use", + "installed": "installed", + "makes AI Denoise available": "makes AI Denoise available", + "non-commercial use only": "non-commercial use only", + "photo has its denoised picture.": "photo has its denoised picture.", + "photo waiting": "photo waiting", + "photos have their denoised picture.": "photos have their denoised picture.", + "photos waiting": "photos waiting", + "starts using it for AI Denoise": "starts using it for AI Denoise", + "terms unclear": "terms unclear", + "the graphics card": "the graphics card", + "the graphics card is not used": "the graphics card is not used", + "Cancel Denoise": "Cancel Denoise", + "Cancel Denoise Model Download": "Cancel Denoise Model Download", + "Choose Denoise Model": "Choose Denoise Model", + "Clear Denoise Cache": "Clear Denoise Cache", + "Denoise Model Downloads": "Denoise Model Downloads", + "Denoise Photos": "Denoise Photos", + "Denoise Pump": "Denoise Pump", + "Denoise Status": "Denoise Status", + "Download Denoise Model": "Download Denoise Model", + "Remove Denoise Model": "Remove Denoise Model", + "Test Denoise Model": "Test Denoise Model", + "AI Denoise is enabled on the requesting photo": "AI Denoise is enabled on the requesting photo", + "After installation, AI Denoise will turn on for the requesting photo.": "After installation, AI Denoise will turn on for the requesting photo.", + "Cancel pending action": "Cancel pending action", + "Install a denoise model to enable AI Denoise on this photo": "Install a denoise model to enable AI Denoise on this photo", + "Pending photo action cancelled": "Pending photo action cancelled", + "Pending photo action cancelled because its library or photo changed": "Pending photo action cancelled because its library or photo changed", + "ONNX model": "ONNX model", + "commercial use subject to GPL conditions": "commercial use subject to GPL conditions" } diff --git a/crates/ui-egui/locales/zh-hant-formats.json b/crates/ui-egui/locales/zh-hant-formats.json index 640e3eb7..182a7dc6 100644 --- a/crates/ui-egui/locales/zh-hant-formats.json +++ b/crates/ui-egui/locales/zh-hant-formats.json @@ -187,5 +187,24 @@ "Moved; {} photo(s) relinked": "已移動；已重新連結 {} 張照片", "{} label (click more labels to show any of them)": "{}標籤（點擊更多標籤可顯示符合其中任一標籤的照片）", ", {} star{}": "，{} 顆星{:.0}", - ", {} label": "，{}標籤" + ", {} label": "，{}標籤", + "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.": "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.", + "{:.1} MB": "{:.1} MB", + "Running on the processor: {}.": "Running on the processor: {}.", + "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.": "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.", + "Denoising {name}: {done} of {total} tiles{}": "Denoising {name}: {done} of {total} tiles{}", + "{size} GB": "{size} GB", + "{} of {} from {host}": "{} of {} from {host}", + "{} KB": "{} KB", + "{} MB": "{} MB", + "Making the denoised picture… {}%": "Making the denoised picture… {}%", + "{name} · {terms}": "{name} · {terms}", + "{n} {noun}": "{n} {noun}", + "Trained on: {provenance}": "Trained on: {provenance}", + " · {queued} waiting": " · {queued} waiting", + "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.": "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.", + "In line for AI Denoise ({ahead} ahead)…": "In line for AI Denoise ({ahead} ahead)…", + "Running on {}.": "Running on {}.", + "{file} · {size}": "{file} · {size}", + "• {assumption}": "• {assumption}" } diff --git a/crates/ui-egui/locales/zh-hant.json b/crates/ui-egui/locales/zh-hant.json index 597ecbf9..c0ab6d4a 100644 --- a/crates/ui-egui/locales/zh-hant.json +++ b/crates/ui-egui/locales/zh-hant.json @@ -1642,5 +1642,94 @@ "How to install the model": "如何安裝模型", "Install the SAM 3 Model": "安裝 SAM 3 模型", "This build can't download the model yet. To install it by hand, put model.safetensors, vocab.json and merges.txt from Meta's facebook/sam3 in the folder above: Object and Describe work as soon as they are there.": "此版本尚無法下載模型。如需手動安裝，請將 Meta 的 facebook/sam3 中的 model.safetensors、vocab.json 和 merges.txt 放入上方的資料夾：放入後即可使用「物件」與「描述」。", - "Open Log Folder": "開啟記錄資料夾" + "Open Log Folder": "開啟記錄資料夾", + "AI Denoise": "AI Denoise", + "AI Denoise Model": "AI Denoise Model", + "AI Denoise is off: press Use on a model.": "AI Denoise is off: press Use on a model.", + "AI Denoise needs a model. Install one in Settings.": "AI Denoise needs a model. Install one in Settings.", + "Accept & Download": "Accept & Download", + "Already installed. Remove it before installing a replacement.": "Already installed. Remove it before installing a replacement.", + "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.": "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.", + "Cache": "Cache", + "Checking and installing the model…": "Checking and installing the model…", + "Checking…": "Checking…", + "Clear cache": "Clear cache", + "Denoise Models": "Denoise Models", + "Denoise Settings": "Denoise Settings", + "Denoise all photos with an Amount": "Denoise all photos with an Amount", + "Downloaded and checked. Installing…": "Downloaded and checked. Installing…", + "Failed its last test": "Failed its last test", + "Graphics card": "Graphics card", + "I have read these terms and accept them for my own use": "I have read these terms and accept them for my own use", + "Inspect Denoise Model": "Inspect Denoise Model", + "Install Denoise Model": "Install Denoise Model", + "Install from file…": "Install from file…", + "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.": "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.", + "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).": "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).", + "LightCraft does not know this model, so it assumed:": "LightCraft does not know this model, so it assumed:", + "Limit": "Limit", + "Make it now": "Make it now", + "Make the denoised picture of the photos I am looking at": "Make the denoised picture of the photos I am looking at", + "Next up for AI Denoise…": "Next up for AI Denoise…", + "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.": "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.", + "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.": "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.", + "Open page": "Open page", + "Open the model's page": "Open the model's page", + "Opens the model's own page in your browser, to read about it or get the file yourself.": "Opens the model's own page in your browser, to read about it or get the file yourself.", + "Processor": "Processor", + "Set up AI Denoise…": "Set up AI Denoise…", + "Speed": "Speed", + "Starting…": "Starting…", + "Tested and works on this computer": "Tested and works on this computer", + "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works": "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works", + "The graphics card is set up when the next photo is made.": "The graphics card is set up when the next photo is made.", + "The model is kept on this computer only. LightCraft never uploads or shares it.": "The model is kept on this computer only. LightCraft never uploads or shares it.", + "The processor does the work.": "The processor does the work.", + "There is no model file to install": "There is no model file to install", + "This build cannot run denoise models.": "This build cannot run denoise models.", + "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)": "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)", + "This file cannot be used yet": "This file cannot be used yet", + "Tick the box to accept the model's terms first": "Tick the box to accept the model's terms first", + "Try again": "Try again", + "Try the graphics card again": "Try the graphics card again", + "Turn off": "Turn off", + "Unknown licence": "Unknown licence", + "Use": "Use", + "Waiting to start…": "Waiting to start…", + "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.": "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.", + "cached picture": "cached picture", + "cached pictures": "cached pictures", + "commercial use allowed": "commercial use allowed", + "denoiseModel.accept": "denoiseModel.accept", + "in use": "in use", + "installed": "installed", + "makes AI Denoise available": "makes AI Denoise available", + "non-commercial use only": "non-commercial use only", + "photo has its denoised picture.": "photo has its denoised picture.", + "photo waiting": "photo waiting", + "photos have their denoised picture.": "photos have their denoised picture.", + "photos waiting": "photos waiting", + "starts using it for AI Denoise": "starts using it for AI Denoise", + "terms unclear": "terms unclear", + "the graphics card": "the graphics card", + "the graphics card is not used": "the graphics card is not used", + "Cancel Denoise": "Cancel Denoise", + "Cancel Denoise Model Download": "Cancel Denoise Model Download", + "Choose Denoise Model": "Choose Denoise Model", + "Clear Denoise Cache": "Clear Denoise Cache", + "Denoise Model Downloads": "Denoise Model Downloads", + "Denoise Photos": "Denoise Photos", + "Denoise Pump": "Denoise Pump", + "Denoise Status": "Denoise Status", + "Download Denoise Model": "Download Denoise Model", + "Remove Denoise Model": "Remove Denoise Model", + "Test Denoise Model": "Test Denoise Model", + "AI Denoise is enabled on the requesting photo": "AI Denoise is enabled on the requesting photo", + "After installation, AI Denoise will turn on for the requesting photo.": "After installation, AI Denoise will turn on for the requesting photo.", + "Cancel pending action": "Cancel pending action", + "Install a denoise model to enable AI Denoise on this photo": "Install a denoise model to enable AI Denoise on this photo", + "Pending photo action cancelled": "Pending photo action cancelled", + "Pending photo action cancelled because its library or photo changed": "Pending photo action cancelled because its library or photo changed", + "ONNX model": "ONNX model", + "commercial use subject to GPL conditions": "commercial use subject to GPL conditions" } diff --git a/crates/ui-egui/src/headless.rs b/crates/ui-egui/src/headless.rs index 48688b6a..26f9bd47 100644 --- a/crates/ui-egui/src/headless.rs +++ b/crates/ui-egui/src/headless.rs @@ -324,7 +324,7 @@ mod tests { h.step(); let w = widgets(&mut h); assert!(w.contains("\"denoise:setup\""), "a raw photo with no model is offered the setup: {w}"); - // one click lands in Settings ▸ AI Denoise, where the model LightCraft knows is listed + // One click lands in Settings, with local installation available even without a download offer. let r = h.request("ui.clickWidget", json!({"id": "denoise:setup"}), t); assert_eq!(r["ok"], true, "{r}"); h.step(); @@ -332,7 +332,8 @@ mod tests { assert_eq!(h.app.ui.dialog, Some(Dialog::Settings { tab: "denoise".into() })); h.settle(SETTLE); let listed = h.app.session.execute("denoise.models.list", &json!({})).unwrap(); - assert!(listed["models"].as_array().unwrap().iter().any(|m| m["id"] == "rawnind-bayer" && m["installed"] == false), "{listed}"); + assert_eq!(listed["models"].as_array().unwrap().len(), lightcraft_denoise::known::all().len(), "{listed}"); + assert!(widgets(&mut h).contains("\"denoise:installFile\"")); h.app.ui.dialog = None; // a photo that is not raw: no offer h.request( diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index be2095fc..d0ccf2e9 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -107,6 +107,8 @@ pub struct Services { pub picker: Option, /// Show an open dialog for photos; returns paths. pub pick_files: Option, + /// Local ONNX model; a manifest must accompany it. + pub pick_denoise_model: Option, /// Open dialog for preset files (`.lcpreset`, `.xmp`, `.lrtemplate`, `.zip`, `.dng`, Luminar `.lmp` / `.mplumpack`). pub pick_preset_files: Option, /// Open dialog for a GPS track log (`.gpx`; Photo ▸ Auto-Tag from Tracklog…). diff --git a/crates/ui-egui/src/model_setup.rs b/crates/ui-egui/src/model_setup.rs index 238a4afe..e7094954 100644 --- a/crates/ui-egui/src/model_setup.rs +++ b/crates/ui-egui/src/model_setup.rs @@ -1,5 +1,6 @@ //! Resume a denoise request after model setup, bound to its original photo and library. +use crate::i18n::tr; use crate::{LightcraftApp, state::Dialog}; use lightcraft_catalog::{PhotoId, Source}; use lightcraft_engine::denoise::PhotoState; @@ -43,7 +44,7 @@ pub(crate) fn intercept(app: &mut LightcraftApp, command: &str, params: &Value) return Some(Err("modelSetup.cancel: use kind denoise".into())); } app.model_setup.denoise = None; - app.ui.status = "Pending photo action cancelled".into(); + app.ui.status = tr("Pending photo action cancelled").into(); return Some(Ok(json!({"cancelled": true}))); } if command == "denoise.models.downloadCancel" { @@ -85,7 +86,7 @@ pub(crate) fn intercept(app: &mut LightcraftApp, command: &str, params: &Value) app.model_setup.denoise = Some(request); app.model_setup.next_check = 0.0; app.ui.dialog = Some(Dialog::Settings { tab: "denoise".into() }); - app.ui.status = "Install a denoise model to enable AI Denoise on this photo".into(); + app.ui.status = tr("Install a denoise model to enable AI Denoise on this photo").into(); json!({"setupRequired": true, "photo": id.0}) })) } @@ -94,8 +95,8 @@ pub(crate) fn notice(app: &mut LightcraftApp, ui: &mut egui::Ui, kind: &str) { if kind != "denoise" || !app.model_setup.denoise.as_ref().is_some_and(|r| r.valid(app)) { return; } - ui.add(egui::Label::new("After installation, AI Denoise will turn on for the requesting photo.").wrap()); - let r = ui.small_button("Cancel pending action"); + ui.add(egui::Label::new(tr("After installation, AI Denoise will turn on for the requesting photo.")).wrap()); + let r = ui.small_button(tr("Cancel pending action")); crate::widgets::register(ui.ctx(), "modelSetup:cancel:denoise", r.rect); if r.clicked() { let _ = app.run("modelSetup.cancel", json!({"kind": "denoise"})); @@ -118,7 +119,7 @@ pub(crate) fn pump(app: &mut LightcraftApp, ctx: &egui::Context) { app.model_setup.next_check = now + 0.25; let Some(request) = app.model_setup.denoise.take() else { return }; if !request.valid(app) { - app.ui.status = "Pending photo action cancelled because its library or photo changed".into(); + app.ui.status = tr("Pending photo action cancelled because its library or photo changed").into(); return; } if matches!(app.session.denoise_photo_state(request.photo), PhotoState::NoModel | PhotoState::NotApplicable) { @@ -127,7 +128,7 @@ pub(crate) fn pump(app: &mut LightcraftApp, ctx: &egui::Context) { return; } match app.run("denoise.toggle", json!({"id": request.photo.0, "enabled": true})) { - Ok(_) => app.ui.status = "AI Denoise is enabled on the requesting photo".into(), + Ok(_) => app.ui.status = tr("AI Denoise is enabled on the requesting photo").into(), Err(error) => app.ui.status = error, } } diff --git a/crates/ui-egui/src/panels/denoise.rs b/crates/ui-egui/src/panels/denoise.rs index bd69b0cf..bf89acd2 100644 --- a/crates/ui-egui/src/panels/denoise.rs +++ b/crates/ui-egui/src/panels/denoise.rs @@ -5,6 +5,7 @@ //! slider mixes in. It is cached data made in the background for the photos being looked at, and never a file in the //! library. A model is fetched only when the user presses Download and accepts its terms. +use crate::i18n::{tr, tr_format}; use egui::RichText; use lightcraft_catalog::PhotoId; use lightcraft_engine::denoise::PhotoState; @@ -110,7 +111,7 @@ fn watch_downloads(app: &mut LightcraftApp, ctx: &egui::Context) { let (mut keep, mut installed) = (Vec::new(), Vec::new()); for id in std::mem::take(&mut app.caches.denoise.dl_watch) { match rows.iter().find(|r| r["id"] == id.as_str()).and_then(|r| r["state"].as_str()) { - Some("running" | "done") => keep.push(id), + Some("running" | "done" | "installing") => keep.push(id), Some("installed") => installed.push(id), _ => {} } @@ -119,7 +120,7 @@ fn watch_downloads(app: &mut LightcraftApp, ctx: &egui::Context) { for id in installed { let _ = app.session.execute("denoise.models.downloadCancel", &json!({"id": id})); app.caches.denoise.epoch += 1; - let text = "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works"; + let text = tr("The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works"); app.ui.status = text.into(); app.toast(ctx, text); } @@ -150,7 +151,7 @@ pub fn detail_status(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId, am PhotoState::NotApplicable | PhotoState::Ready => {} PhotoState::NoModel => { egui::Frame::NONE.inner_margin(pad).show(ui, |ui| { - note(ui, "AI Denoise needs a model: a one-time download, in Settings.", t.text_dim); + note(ui, "AI Denoise needs a model. Install one in Settings.", t.text_dim); ui.add_space(2.0); let r = ui.small_button(crate::i18n::tr("Set up AI Denoise…")); register(ui.ctx(), "denoise:setup", r.rect); @@ -180,15 +181,22 @@ pub fn detail_status(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId, am } PhotoState::Queued { ahead } => { egui::Frame::NONE.inner_margin(pad).show(ui, |ui| { - let text = if ahead == 0 { "Next up for AI Denoise…".to_string() } else { format!("In line for AI Denoise ({ahead} ahead)…") }; + let text = if ahead == 0 { + tr("Next up for AI Denoise…").to_string() + } else { + tr_format!("In line for AI Denoise ({ahead} ahead)…", ahead = ahead) + }; note(ui, &text, t.text_dim); }); } PhotoState::Running { done, total } => { egui::Frame::NONE.inner_margin(pad).show(ui, |ui| { let frac = if total > 0 { (done as f32 / total as f32).clamp(0.0, 1.0) } else { 0.0 }; - let text = - if total > 0 { format!("Making the denoised picture… {}%", (frac * 100.0).round() as u32) } else { "Starting…".to_string() }; + let text = if total > 0 { + tr_format!("Making the denoised picture… {}%", (frac * 100.0).round() as u32) + } else { + tr("Starting…").to_string() + }; let bar = ui.add(egui::ProgressBar::new(frac).desired_width(ui.available_width().min(240.0)).text(RichText::new(text).size(11.0))); register(ui.ctx(), "denoise:progress", bar.rect); }); @@ -235,6 +243,24 @@ pub fn settings_tab(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens) { .map(|v| v["downloads"].as_array().cloned().unwrap_or_default()) .unwrap_or_default(); heading(ui, t, "Models"); + let r = ui.add_enabled(runtime && app.services.pick_denoise_model.is_some(), egui::Button::new(tr("Install from file…"))); + register(ui.ctx(), "denoise:installFile", r.rect); + if r.clicked() + && let Some(path) = app.services.pick_denoise_model.as_mut().and_then(|f| f().into_iter().next()) + && let Ok(info) = app.run("denoise.models.inspect", json!({"path": path})) + { + app.ui.dialog = Some(crate::state::Dialog::DenoiseModel { info, accepted: false }); + } + if all.is_empty() { + hint(ui, t, "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads."); + } + if let Some(job) = downloads.iter().find(|d| d["id"] == "local-install") { + match job["state"].as_str() { + Some("installing") => hint(ui, t, "Checking and installing the model…"), + Some("failed") => hint(ui, t, job["error"].as_str().unwrap_or("Model installation failed")), + _ => {} + } + } for m in &all { let dl = downloads.iter().find(|d| d["id"] == m["id"]); model_row(app, ui, t, m, dl, runtime); @@ -290,8 +316,13 @@ fn work_section(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, list: &V .as_u64() .and_then(|p| app.session.catalog.photo(PhotoId(p)).map(|p| p.file_name.clone())) .unwrap_or_default(); - let text = - format!("Denoising {name}: {done} of {total} tiles{}", if queued > 0 { format!(" · {queued} waiting") } else { String::new() }); + let text = tr_format!( + "Denoising {name}: {done} of {total} tiles{}", + if queued > 0 { tr_format!(" · {queued} waiting", queued = queued) } else { String::new() }, + name = name, + done = done, + total = total + ); let bar = ui.add(egui::ProgressBar::new(frac).desired_width(380.0).text(RichText::new(text).size(11.5))); register(ui.ctx(), "denoise:settingsProgress", bar.rect); } @@ -302,13 +333,13 @@ fn work_section(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, list: &V } } ui.horizontal(|ui| { - let r = ui.button("Denoise all photos with an Amount"); + let r = ui.button(tr("Denoise all photos with an Amount")); register(ui.ctx(), "denoise:queueAll", r.rect); if r.clicked() { let _ = app.run("denoise.queue", json!({"scope": "withAmount"})); } if running.is_some() || queued > 0 { - let r = ui.button("Stop"); + let r = ui.button(tr("Stop")); register(ui.ctx(), "denoise:stop", r.rect); if r.clicked() { let _ = app.run("denoise.cancel", json!({})); @@ -320,7 +351,7 @@ fn work_section(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, list: &V hint( ui, t, - &format!( + &tr_format!( "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.", count(files, "cached picture", "cached pictures"), mb(Some(bytes)) @@ -330,16 +361,16 @@ fn work_section(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, list: &V ui.label(RichText::new(crate::i18n::tr("Limit")).color(t.text_label)); let current = list["cacheGb"].as_u64().unwrap_or(20) as u32; let mut choice = current; - egui::ComboBox::from_id_salt("denoise-cache-limit").selected_text(format!("{current} GB")).show_ui(ui, |ui| { + egui::ComboBox::from_id_salt("denoise-cache-limit").selected_text(tr_format!("{size} GB", size = current)).show_ui(ui, |ui| { for gb in CACHE_LIMITS { - ui.selectable_value(&mut choice, gb, format!("{gb} GB")); + ui.selectable_value(&mut choice, gb, tr_format!("{size} GB", size = gb)); } }); if choice != current { let _ = app.run("denoise.settings", json!({"cacheGb": choice})); app.caches.denoise.epoch += 1; } - let r = ui.button("Clear cache"); + let r = ui.button(tr("Clear cache")); register(ui.ctx(), "denoise:clear", r.rect); if r.clicked() { let _ = app.run("denoise.clear", json!({})); @@ -351,12 +382,12 @@ fn work_section(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, list: &V /// computer's, and only the comparison matters. fn device_line(d: &Value, run_on: &str) -> String { match d["kind"].as_str() { - _ if run_on == "cpu" => "The processor does the work.".into(), - Some("gpu") => format!("Running on {}.", d["adapter"].as_str().unwrap_or("the graphics card")), + _ if run_on == "cpu" => tr("The processor does the work.").into(), + Some("gpu") => tr_format!("Running on {}.", d["adapter"].as_str().unwrap_or(tr("the graphics card"))), Some("cpu") => { - format!("Running on the processor: {}.", d["reason"].as_str().unwrap_or("the graphics card is not used").trim_end_matches('.')) + tr_format!("Running on the processor: {}.", tr(d["reason"].as_str().unwrap_or("the graphics card is not used")).trim_end_matches('.')) } - _ => "The graphics card is set up when the next photo is made.".into(), + _ => tr("The graphics card is set up when the next photo is made.").into(), } } @@ -375,9 +406,9 @@ fn model_row(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, m: &Value, ui.label(RichText::new(size).color(t.text_dim)); } if selected { - ui.label(RichText::new("in use").color(t.accent)); + ui.label(RichText::new(tr("in use")).color(t.accent)); } else if installed { - ui.label(RichText::new("installed").color(t.text_dim)); + ui.label(RichText::new(tr("installed")).color(t.text_dim)); } }); ui.label(RichText::new(licence_line(m)).font(t.font(11.5)).color(if m["licence"]["commercial"] == "yes" { @@ -387,10 +418,10 @@ fn model_row(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, m: &Value, })); match m["accepted"]["selfTest"].as_object() { Some(test) if test.get("ok").and_then(Value::as_bool) == Some(true) => { - ui.label(RichText::new("Tested and works on this computer").font(t.font(11.5)).color(t.text_dim)); + ui.label(RichText::new(tr("Tested and works on this computer")).font(t.font(11.5)).color(t.text_dim)); } Some(_) => { - ui.label(RichText::new("Failed its last test").font(t.font(11.5)).color(t.caution)); + ui.label(RichText::new(tr("Failed its last test")).font(t.font(11.5)).color(t.caution)); } None => {} } @@ -399,15 +430,15 @@ fn model_row(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, m: &Value, let (got, total) = (d["bytes"].as_u64().unwrap_or(0), d["total"].as_u64().unwrap_or(0)); let frac = if total > 0 { (got as f32 / total as f32).clamp(0.0, 1.0) } else { 0.0 }; let text = if total > 0 && got >= total { - "Checking…".to_string() + tr("Checking…").to_string() } else { - format!("{} of {} from {host}", mb(Some(got)), mb(Some(total))) + tr_format!("{} of {} from {host}", mb(Some(got)), mb(Some(total)), host = host) }; let bar = ui.add(egui::ProgressBar::new(frac).desired_width(260.0).text(RichText::new(text).font(t.font(11.5)))); register(ui.ctx(), format!("denoise:progress:{id}"), bar.rect); } - (Some("done"), _) => { - ui.label(RichText::new("Downloaded and checked. Installing…").font(t.font(11.5)).color(t.text_dim)); + (Some("done" | "installing"), _) => { + ui.label(RichText::new(tr("Downloaded and checked. Installing…")).font(t.font(11.5)).color(t.text_dim)); } (Some("failed"), Some(d)) => { let why = sentence(d["error"].as_str().unwrap_or("The download failed")); @@ -418,21 +449,21 @@ fn model_row(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, m: &Value, }); ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { if installed { - let r = ui.button("Remove"); + let r = ui.button(tr("Remove")); register(ui.ctx(), format!("denoise:remove:{id}"), r.rect); if r.clicked() { let _ = app.run("denoise.models.remove", json!({"id": id})); app.caches.denoise.epoch += 1; } if selected { - let r = ui.button("Turn off"); + let r = ui.button(tr("Turn off")); register(ui.ctx(), format!("denoise:off:{id}"), r.rect); if r.clicked() { let _ = app.run("denoise.models.select", json!({"id": null})); app.caches.denoise.epoch += 1; } } else { - let r = ui.button("Use"); + let r = ui.button(tr("Use")); register(ui.ctx(), format!("denoise:use:{id}"), r.rect); if r.clicked() { let _ = app.run("denoise.models.select", json!({"id": id})); @@ -442,7 +473,7 @@ fn model_row(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, m: &Value, } else { match dl_state { Some("running") => { - let r = ui.button("Cancel"); + let r = ui.button(tr("Cancel")); register(ui.ctx(), format!("denoise:cancelDownload:{id}"), r.rect); if r.clicked() { let _ = app.run("denoise.models.downloadCancel", json!({"id": id})); @@ -450,12 +481,12 @@ fn model_row(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, m: &Value, } } // the engine installs it within a moment; nothing to press - Some("done") => {} + Some("done" | "installing") => {} _ => { if let Some(url) = m["source"].as_str() { let r = ui - .button("Open page") - .on_hover_text("Opens the model's own page in your browser, to read about it or get the file yourself."); + .button(tr("Open page")) + .on_hover_text(tr("Opens the model's own page in your browser, to read about it or get the file yourself.")); register(ui.ctx(), format!("denoise:get:{id}"), r.rect); if r.clicked() { open_page(app, url); @@ -463,11 +494,12 @@ fn model_row(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, m: &Value, } if !host.is_empty() { let label = if dl_state == Some("failed") { "Try again" } else { "Download" }; - let tip = format!( + let tip = tr_format!( "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.", - mb(m["sizeBytes"].as_u64()) + mb(m["sizeBytes"].as_u64()), + host = host ); - let r = ui.button(label).on_hover_text(tip); + let r = ui.button(tr(label)).on_hover_text(tip); register(ui.ctx(), format!("denoise:download:{id}"), r.rect); if r.clicked() { open_download_dialog(app, m); @@ -502,14 +534,17 @@ fn open_download_dialog(app: &mut LightcraftApp, m: &Value) { /// The dialog's OK for a denoise model: start the download and watch it. pub fn install(app: &mut LightcraftApp, info: &Value, accepted: bool) -> Result { if !accepted { - return Err("Tick the box to accept the model's terms first".into()); + return Err(tr("Tick the box to accept the model's terms first").into()); } app.caches.denoise.epoch += 1; - let id = info["download"].as_str().ok_or("There is no download for this model")?; - let r = app.run("denoise.models.download", json!({"id": id, "acknowledged": true})); + let (id, r) = if let Some(id) = info["download"].as_str() { + (id.to_string(), app.run("denoise.models.download", json!({"id": id, "acknowledged": true}))) + } else { + let path = info["path"].as_str().ok_or_else(|| tr("There is no model file to install").to_string())?; + ("local-install".into(), app.run("denoise.models.install", json!({"path": path, "acknowledged": true, "background": true}))) + }; if r.is_ok() { - app.caches.denoise.dl_watch.push(id.to_string()); - // the progress is in Settings: stay there + app.caches.denoise.dl_watch.push(id); app.ui.dialog = Some(crate::state::Dialog::Settings { tab: "denoise".into() }); } r @@ -517,7 +552,7 @@ pub fn install(app: &mut LightcraftApp, info: &Value, accepted: bool) -> Result< /// "1 photo" / "3 photos": the count with the right noun. fn count(n: u64, one: &str, many: &str) -> String { - format!("{n} {}", if n == 1 { one } else { many }) + tr_format!("{n} {noun}", n = n, noun = tr(if n == 1 { one } else { many })) } pub(super) fn window_pace(app: &LightcraftApp, ctx: &egui::Context, now: f64, watching: bool) -> &'static str { @@ -544,9 +579,9 @@ fn pace_for(focused: bool, minimized: bool, worked: f64, moved: f64, watching: b pub(super) fn mb(bytes: Option) -> String { match bytes { - Some(b) if b >= 10_000_000 => format!("{} MB", (b as f64 / 1e6).round() as u64), - Some(b) if b >= 1_000_000 => format!("{:.1} MB", b as f64 / 1e6), - Some(b) => format!("{} KB", (b as f64 / 1e3).round().max(1.0) as u64), + Some(b) if b >= 10_000_000 => tr_format!("{} MB", (b as f64 / 1e6).round() as u64), + Some(b) if b >= 1_000_000 => tr_format!("{:.1} MB", b as f64 / 1e6), + Some(b) => tr_format!("{} KB", (b as f64 / 1e3).round().max(1.0) as u64), None => String::new(), } } @@ -565,11 +600,12 @@ pub(super) fn sentence(s: &str) -> String { pub(super) fn licence_line(m: &Value) -> String { let name = m["licence"]["name"].as_str().filter(|s| !s.is_empty()).unwrap_or("Unknown licence"); let terms = match m["licence"]["commercial"].as_str() { + Some("yes") if name.starts_with("GPL") => "commercial use subject to GPL conditions", Some("yes") => "commercial use allowed", Some("no") => "non-commercial use only", _ => "terms unclear", }; - format!("{name} · {terms}") + tr_format!("{name} · {terms}", name = tr(name), terms = tr(terms)) } pub(super) fn open_page(app: &mut LightcraftApp, url: &str) { @@ -582,14 +618,15 @@ pub fn model_dialog(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, info let kind = info["kind"].as_str().unwrap_or("unsupported"); let file = info["fileName"].as_str().unwrap_or(""); if kind == "unsupported" { - ui.label(RichText::new("This file cannot be used yet").font(t.semibold(13.5)).color(t.caution)); + ui.label(RichText::new(tr("This file cannot be used yet")).font(t.semibold(13.5)).color(t.caution)); ui.add( egui::Label::new( - RichText::new(sentence(info["reason"].as_str().unwrap_or("It is not a denoise model LightCraft understands"))).color(t.text_label), + RichText::new(sentence(tr(info["reason"].as_str().unwrap_or("It is not a denoise model LightCraft understands")))) + .color(t.text_label), ) .wrap(), ); - ui.label(RichText::new(format!("{file} · {}", mb(info["sizeBytes"].as_u64()))).color(t.text_dim)); + ui.label(RichText::new(tr_format!("{file} · {size}", file = file, size = mb(info["sizeBytes"].as_u64()))).color(t.text_dim)); return; } let m = &info["model"]; @@ -598,7 +635,7 @@ pub fn model_dialog(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, info ui.label(RichText::new(mb(info["sizeBytes"].as_u64())).color(t.text_dim)); }); if info["alreadyInstalled"] == true { - ui.label(RichText::new("Already installed. Installing again is harmless.").color(t.text_dim)); + ui.label(RichText::new(tr("Already installed. Remove it before installing a replacement.")).color(t.text_dim)); } let download = info["download"].is_string(); if download { @@ -606,8 +643,12 @@ pub fn model_dialog(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, info let after = if info["domain"] == "denoise" { "starts using it for AI Denoise" } else { "makes AI Denoise available" }; ui.add( egui::Label::new( - RichText::new(format!("Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.")) - .color(t.text_label), + RichText::new(tr_format!( + "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.", + host = host, + after = tr(after) + )) + .color(t.text_label), ) .wrap(), ); @@ -620,17 +661,17 @@ pub fn model_dialog(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, info } let provenance = m["provenance"].as_str().unwrap_or(""); if !provenance.is_empty() { - ui.add(egui::Label::new(RichText::new(format!("Trained on: {provenance}")).color(t.text_dim)).wrap()); + ui.add(egui::Label::new(RichText::new(tr_format!("Trained on: {provenance}", provenance = provenance)).color(t.text_dim)).wrap()); } if kind == "draft" { ui.add_space(2.0); - ui.label(RichText::new("LightCraft does not know this model, so it assumed:").color(t.text_label)); + ui.label(RichText::new(tr("LightCraft does not know this model, so it assumed:")).color(t.text_label)); for a in info["assumptions"].as_array().into_iter().flatten().filter_map(Value::as_str) { - ui.add(egui::Label::new(RichText::new(format!("• {a}")).font(t.font(12.0)).color(t.text_dim)).wrap()); + ui.add(egui::Label::new(RichText::new(tr_format!("• {assumption}", assumption = tr(a))).font(t.font(12.0)).color(t.text_dim)).wrap()); } } if let Some(url) = m["source"].as_str().or(m["licence"]["url"].as_str()) { - let r = ui.link("Open the model's page"); + let r = ui.link(tr("Open the model's page")); register(ui.ctx(), "denoiseModel:page", r.rect); if r.clicked() { open_page(app, url); @@ -643,5 +684,5 @@ pub fn model_dialog(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, info } else { "The model is kept on this computer only. LightCraft never uploads or shares it." }; - ui.label(RichText::new(closing).font(t.font(11.5)).color(t.text_dim)); + ui.label(RichText::new(tr(closing)).font(t.font(11.5)).color(t.text_dim)); } diff --git a/crates/ui-egui/src/panels/edit.rs b/crates/ui-egui/src/panels/edit.rs index 51a54bc9..fb8d121e 100644 --- a/crates/ui-egui/src/panels/edit.rs +++ b/crates/ui-egui/src/panels/edit.rs @@ -280,7 +280,7 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) { super::denoise::detail_toggle(app, ui, id, on, applicable); control(app, ui, d, "enhance.denoise", applicable && on); super::denoise::detail_status(app, ui, id, if on { d.enhance.denoise } else { 0.0 }); - sub_title(ui, crate::i18n::tr("Manual Noise Reduction")); + sub_title(ui, crate::i18n::tr("Noise Reduction")); for c in ["detail.nrLuminance", "detail.nrDetail", "detail.nrContrast", "detail.nrColor", "detail.nrColorDetail", "detail.nrColorSmoothness"] { control(app, ui, d, c, true); diff --git a/crates/ui-egui/src/tests_model_setup.rs b/crates/ui-egui/src/tests_model_setup.rs index 892df22d..85a7ca7f 100644 --- a/crates/ui-egui/src/tests_model_setup.rs +++ b/crates/ui-egui/src/tests_model_setup.rs @@ -30,7 +30,7 @@ fn setup(name: &str) -> (LightcraftApp, PathBuf, PhotoId) { (LightcraftApp::new(s, Services::default()), dir, id) } -fn install_denoise(app: &mut LightcraftApp, dir: &Path) { +fn denoise_file(dir: &Path) -> PathBuf { let source = dir.join("source"); std::fs::create_dir_all(&source).unwrap(); let bytes = lightcraft_denoise::synthetic::smoothing_onnx(); @@ -50,7 +50,38 @@ fn install_denoise(app: &mut LightcraftApp, dir: &Path) { gain: Gain::None, }; std::fs::write(source.join("denoise-model.json"), serde_json::to_vec(&m).unwrap()).unwrap(); - app.run("denoise.models.install", json!({"path": source.join("model.onnx").to_string_lossy(), "acknowledged": true})).unwrap(); + source.join("model.onnx") +} + +fn install_denoise(app: &mut LightcraftApp, dir: &Path) { + app.run("denoise.models.install", json!({"path": denoise_file(dir), "acknowledged": true})).unwrap(); +} + +#[test] +fn accepting_a_local_model_installs_in_background_and_resumes_the_pending_action() { + let (mut app, dir, raw) = setup("local-install"); + let path = denoise_file(&dir); + app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap(); + let info = app.run("denoise.models.inspect", json!({"path": path})).unwrap(); + assert!(crate::panels::denoise::install(&mut app, &info, false).is_err()); + assert!(!dir.join("denoise/test-smoothing/model.onnx").exists()); + let result = crate::panels::denoise::install(&mut app, &info, true).unwrap(); + assert_eq!(result["started"], "local-install"); + assert_eq!(app.ui.dialog, Some(Dialog::Settings { tab: "denoise".into() })); + let ctx = egui::Context::default(); + for _ in 0..600 { + crate::panels::denoise::pump(&mut app, &ctx); + if app.session.denoise_photo_state(raw) != lightcraft_engine::denoise::PhotoState::NoModel { + break; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + resume(&mut app); + assert_eq!(amount(&app, raw), 50.0); + assert!(dir.join("denoise/test-smoothing/model.onnx").is_file()); + assert!(path.is_file(), "a user-supplied original is preserved"); + drop(app); + let _ = std::fs::remove_dir_all(dir); } fn amount(app: &LightcraftApp, id: PhotoId) -> f64 { diff --git a/docs/denoise.md b/docs/denoise.md index b8fd2901..f1196e9b 100644 --- a/docs/denoise.md +++ b/docs/denoise.md @@ -15,22 +15,29 @@ An active slider gesture finishes before the pending action resumes. ## Models and terms **No model weights are bundled.** Models live in the per-user `denoise-models` folder -(override with `LIGHTCRAFT_DENOISE_MODELS`). Settings offers a pinned, verified download -of RawNIND UtNet2 for Bayer sensors, or installation of a user-supplied ONNX file with -a `denoise-model.json` beside it. Terms must be acknowledged before installation or +(override with `LIGHTCRAFT_DENOISE_MODELS`). Settings provides **Install from file…** +for a user-supplied ONNX file with a `denoise-model.json` beside it. The default build +offers no model downloads. The separate `rawnind-model` Cargo feature enables the +pinned RawNIND offer only when a release's model policy permits it. +Terms must be acknowledged before installation or download. Downloads use the shared pure-Rust `lightcraft-fetch` client, with resume, progress, cancellation, size limits and SHA-256 verification. -RawNIND's offered ONNX weights are **GPL-3.0**, as recorded in the pinned model catalog. +RawNIND's optional ONNX weights are **GPL-3.0**, as recorded in the pinned model catalog. They are downloaded separately after consent and read by our own Rust executor. Whether releases should offer this download remains a maintainer policy decision; permissively licensed or independently trained weights remain a gap. This is not a claim that optional downloading settles licensing questions. No GPL decoder or neural -runtime code is included. Model metadata is in `crates/denoise/src/known.rs`. +runtime code is included. Model metadata is in `crates/denoise-core/src/known.rs`. Installation checks the manifest, model hash and size, then runs a synthetic self-test: finite output of the promised shape, repeatability, the declared scale and reduced noise. Unsupported operators, malformed files and a failed self-test return errors. +The UI performs copying, extraction, hashing, parsing and CPU/GPU self-tests on one +background installation worker. `denoise.models.downloads` and `denoise.pump` report +`installing`, `installed` or `failed`; an existing model is preserved on a failed replacement. +Command clients can request the same asynchronous path with `background: true`. +The synchronous command remains available for scripts that require the result immediately. The current contract is `bayerToRgb`: normalized, black-subtracted, non-white-balanced RGGB planes `[1, 4, tile, tile]` produce camera RGB `[1, 3, 2*tile, 2*tile]`. @@ -63,10 +70,16 @@ reduction and report their limitation. ## Pure-Rust CPU and GPU execution The CPU executor reads a bounded subset of ONNX into checked network data. Convolutions -use bounded im2col blocks and `faer-core 0.17.1` with its Rust-intrinsic GEMM kernels, -without BLAS, generated assembly or an inner thread pool. Tiles share the outer worker +use bounded im2col blocks and `faer-core 0.17.1` with its pure-Rust GEMM backend, +without BLAS, C dependencies or an inner thread pool. That backend includes +architecture-specific assembly in its dependencies. Tiles share the outer worker pool. A scalar implementation checks the optimized operators on synthetic networks. Tract is not a workspace dependency or a product feature. +CPU inference and the GPU model module are feature-gated; ordinary engine/web builds +do not compile the faer/GEMM backend. The engine depends on the inference crate only through `denoise = ["dep:lightcraft-denoise", …]`; +lightweight `lightcraft-denoise-core` metadata and cache formats remain available without inference. Pictures are capped at 100 megapixels; at most +four tile calls run together, chosen within the process-wide working-memory budget. +One idle CPU workspace is retained, and cache costs reserve room for the blended picture. The GPU executor uses our WGSL kernels through wgpu. It checks a real model's output against the CPU and falls back on errors, unsupported networks or a slower device. @@ -97,8 +110,8 @@ Fresh checks can be run using an already installed model and CC0 raw samples: ```text LC_DENOISE_MODEL= cargo test --release -p lightcraft-gpu --lib real_model_on_the_gpu_matches_cpu -- --ignored --nocapture -LC_DENOISE_MODEL= LC_DENOISE_RAW= cargo test --release -p lightcraft-engine --features denoise --test denoise_cpu -- --ignored --nocapture -LC_DENOISE_MODEL= LC_DENOISE_RAW= cargo test --release -p lightcraft-engine --features denoise --test denoise_real -- --ignored --nocapture +LC_DENOISE_MODEL= LC_DENOISE_RAW= cargo test --release -p lightcraft-engine --features rawnind-model --test denoise_cpu -- --ignored --nocapture +LC_DENOISE_MODEL= LC_DENOISE_RAW= cargo test --release -p lightcraft-engine --features rawnind-model --test denoise_real -- --ignored --nocapture ``` ## Commands and limits diff --git a/docs/parity.md b/docs/parity.md index a4718949..eaf9ff8e 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -26,7 +26,7 @@ and prints the summary; `cargo xtask parity --write` refreshes the summary table | C. Views & navigation (VIEW) | 15 | 1 | 1 | 0 | 8/9 (89%) | 4/4 (100%) | | D. Search & filter (FILT) | 11 | 1 | 1 | 0 | 4/4 (100%) | 4/4 (100%) | | E. Metadata (META) | 5 | 1 | 0 | 0 | 2/2 (100%) | 2/2 (100%) | -| F. Edit panel — global adjustments (EDIT) | 42 | 1 | 5 | 1 | 28/28 (100%) | 13/14 (93%) | +| F. Edit panel — global adjustments (EDIT) | 42 | 2 | 4 | 1 | 28/28 (100%) | 13/14 (93%) | | G. Profiles (PROF) | 7 | 1 | 3 | 0 | 3/4 (75%) | 3/3 (100%) | | H. Crop & rotate (CROP) | 9 | 0 | 0 | 1 | 6/6 (100%) | 3/3 (100%) | | I. Remove / healing (REM) | 7 | 1 | 2 | 2 | 4/4 (100%) | 2/3 (67%) | @@ -79,7 +79,7 @@ Take the first one nobody is working on. Object and Describe masks run SAM 3 in pure Rust (`crates/segment`); the weights (SAM License) are never bundled: the app offers a consented, verified download, but **LightCraft's CDN mirrors are not configured yet** (the built-in list is empty, so today users need their own mirror or a manual install). Subject / Sky / People could - use the same model with fixed prompts. Bayer RAW denoise now runs on pure-Rust CPU/GPU with separately installed weights; its opt-in GPL model policy, linear RGB and quality comparison remain open. Super-resolution models remain a maintainer decision. + use the same model with fixed prompts; denoise / super-resolution models remain a maintainer decision. 7. **HDR** (Q. HDR, LR-EXP-HDR), **video** (R. Video), **Classic output modules** (Map view, Book, Slideshow module, Print, publish): large, well understood, lower priority than 1–5. 8. **LR-VIEW-ZOOM** (P0): pinch zoom and two-finger pan now work, with stable image cursors between gesture events. diff --git a/xtask/src/layers.rs b/xtask/src/layers.rs index e330a8e1..376d9e3c 100644 --- a/xtask/src/layers.rs +++ b/xtask/src/layers.rs @@ -39,6 +39,7 @@ pub const TABLE: &[(&str, Class)] = &[ ("tiff", Class::Layer(0)), ("sysmem", Class::Layer(0)), ("fetch", Class::Layer(0)), + ("denoise-core", Class::Layer(1)), ("denoise", Class::Layer(1)), ("raw", Class::Layer(1)), ("codecs", Class::Layer(1)), @@ -69,6 +70,7 @@ pub const TABLE: &[(&str, Class)] = &[ pub const INTRA_LAYER_ORDER: &[&[&str]] = &[ &["geom", "color", "raster"], &["tiff", "raster"], + &["denoise-core", "denoise"], &["meta", "raw"], &["meta", "codecs"], &["meta", "develop"], From 59ab9b6e9b7199ec04fa8574b24f69fe0bcd7578 Mon Sep 17 00:00:00 2001 From: Daniel Voshart <205798369+voshart@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:31:04 -0400 Subject: [PATCH 23/46] LR-EDIT-DETAIL-DENOISE: refresh the current parity summary All seven gates pass with the hosted fonts and the earlier shared window-test correction. Generated with Codex. --- docs/parity.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/parity.md b/docs/parity.md index eaf9ff8e..53ccabc1 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -48,9 +48,9 @@ and prints the summary; `cargo xtask parity --write` refreshes the summary table | Y. Menus | 82 | 0 | 4 | 8 | 47/47 (100%) | 23/23 (100%) | | Z. Keyboard shortcuts (desktop) | 75 | 3 | 3 | 1 | 49/52 (94%) | 23/24 (96%) | | Lightroom Classic extras | 32 | 11 | 39 | 9 | — | 22/24 (92%) | -| **Total** | 393 | 36 | 86 | 36 | 193/200 (97%) | 142/152 (93%) | +| **Total** | 393 | 37 | 85 | 36 | 193/200 (97%) | 142/152 (93%) | -Weighted completion (✅ = 1, 🟡 = ½, 🚫 left out): **79.8%** of 515 in-scope rows — P0 98.2% of 200 · P1 95.7% of 152 · P2 42.0% of 162. +Weighted completion (✅ = 1, 🟡 = ½, 🚫 left out): **79.9%** of 515 in-scope rows — P0 98.2% of 200 · P1 95.7% of 152 · P2 42.3% of 162. ## Top gaps From 04e8aea57729a9ade361b0095d4c4216e5d6f367 Mon Sep 17 00:00:00 2001 From: radialmonster Date: Thu, 8 Oct 2026 21:39:09 -0400 Subject: [PATCH 24/46] LR-IMP-FORMATS: CR3 photos render in the aspect ratio set in the camera The Canon maker note's AspectInfo (tag 0x009a: aspect code, cropped width, height, left, top) holds the rectangle of the shot, measured from the recommended image crop. Use it as the CR3 default crop, so 3:2, 4:3, 16:9 and 1:1 shots develop in the shape the photographer chose. The tag is ignored when it is missing, does not fit, or the recommended crop itself is inconsistent. RENDER_CACHE_VERSION 15 -> 16. The M50 C-RAW corpus sample (raw.pixls.us 2663) was shot at 1:1 and now develops at 4000 x 4000. --- crates/engine/src/media.rs | 2 +- crates/raw/src/vendor/cr3.rs | 102 +++++++++++++++++++++++++++++---- crates/raw/tests/cr3_corpus.rs | 31 +++++++--- docs/cr3.md | 6 ++ docs/parity.md | 2 +- 5 files changed, 122 insertions(+), 21 deletions(-) diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rs index 1ed89e47..d5ae4b9e 100644 --- a/crates/engine/src/media.rs +++ b/crates/engine/src/media.rs @@ -62,7 +62,7 @@ impl SettingsHashes { } /// Bump when the pipeline's output changes, to invalidate cached thumbnails. -pub const RENDER_CACHE_VERSION: u64 = 15; +pub const RENDER_CACHE_VERSION: u64 = 16; /// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache). pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512]; diff --git a/crates/raw/src/vendor/cr3.rs b/crates/raw/src/vendor/cr3.rs index c51f8e30..b5442790 100644 --- a/crates/raw/src/vendor/cr3.rs +++ b/crates/raw/src/vendor/cr3.rs @@ -7,6 +7,15 @@ //! //! `IAD1` stores the valid sensor area separately from the recommended image crop. Both use inclusive //! sensor offsets. The CFA is anchored at the full sensor's origin; cropping shifts its phase later. +//! +//! The in-camera aspect ratio (3:2, 4:3, 16:9, 1:1 ...) is not applied to the sensor data: the raw always +//! holds the whole image. The maker note's `AspectInfo` (tag `0x009a`; ExifTool names `AspectRatio`, +//! `CroppedImageWidth`, `CroppedImageHeight`, `CroppedImageLeft`, `CroppedImageTop`) holds the shot's rectangle +//! as five 32-bit values, measured from the top-left of the recommended image crop (the "Canon image", whose +//! size is ExifTool's `CanonImageWidth` x `CanonImageHeight`). That rectangle replaces the crop. Established +//! on the CC0 PowerShot SX70 HS, EOS 250D, PowerShot G5 X Mark II and EOS M6 Mark II samples shot at 3:2, 4:3, +//! 16:9 and 1:1: the rectangle is centred, has the shot's aspect, and files shot at the sensor's own aspect +//! store the whole image. //! Unknown camera metadata versions are ignored rather than assigned a guessed white balance. use super::{black_from_columns, crx, white_from_data}; @@ -17,6 +26,7 @@ use lightcraft_tiff::{Ifd, Tiff}; const SENSOR_INFO: u16 = 0x00e0; const COLOR_DATA: u16 = 0x4001; const LENS_MODEL: u16 = 0x0095; +const ASPECT_INFO: u16 = 0x009a; /// Pick the first largest raw track. Equal-sized later tracks can be the Dual Pixel delta, which /// must never replace the main sensor image. An invalid full-size descriptor is reported, not @@ -83,16 +93,26 @@ fn sensor_crop(maker: Option<&Ifd>, width: usize, height: usize) -> Option ) } +/// The shot's aspect-ratio rectangle from `AspectInfo`, placed inside `image` (the recommended crop). `None` +/// when the tag is absent or malformed, or its rectangle doesn't fit inside `image`. +fn aspect_crop(maker: Option<&Ifd>, image: Rect) -> Option { + let data = maker?.u64s(ASPECT_INFO)?; + let [width, height, left, top] = [1usize, 2, 3, 4].map(|i| data.get(i).and_then(|&v| usize::try_from(v).ok())); + let (width, height, left, top) = (width?, height?, left?, top?); + if width < 2 || height < 2 || left.checked_add(width)? > image.width || top.checked_add(height)? > image.height { + return None; + } + Some(Rect::new(image.x.checked_add(left)?, image.y.checked_add(top)?, width, height)) +} + fn geometry(area: Option<&Cr3ImageArea>, maker: Option<&Ifd>, width: usize, height: usize) -> (Rect, Rect) { let area = area.filter(|a| usize::from(a.width) == width && usize::from(a.height) == height); let sensor = sensor_crop(maker, width, height); let active = area.and_then(|a| a.active).and_then(|b| inclusive_rect(b, width, height)).or(sensor).unwrap_or(Rect::new(0, 0, width, height)); - let crop = area.and_then(|a| inclusive_rect(a.crop, width, height)).or(sensor).and_then(|c| relative_crop(c, active)).unwrap_or(Rect::new( - 0, - 0, - active.width, - active.height, - )); + // The recommended crop, when the file gives a consistent one. `AspectInfo` is measured from its origin, so it + // is only applied on top of that; without it (the crop doesn't fit the valid area) the whole active area stays. + let recommended = area.and_then(|a| inclusive_rect(a.crop, width, height)).or(sensor).and_then(|c| relative_crop(c, active)); + let crop = recommended.map(|c| aspect_crop(maker, c).unwrap_or(c)).unwrap_or(Rect::new(0, 0, active.width, active.height)); (active, crop) } @@ -266,6 +286,10 @@ mod tests { } fn synthetic_header() -> Vec { + synthetic_header_with(None) + } + + fn synthetic_header_with(aspect: Option<[u32; 5]>) -> Vec { use lightcraft_tiff::{ByteOrder, TiffWriter, tags as t}; let tiff = |b: IfdBuilder| TiffWriter::new(ByteOrder::Little, false).write(&[b]).unwrap(); let cmt1 = tiff( @@ -275,11 +299,13 @@ mod tests { .with(t::ORIENTATION, Value::Short(vec![6])), ); let cmt2 = tiff(IfdBuilder::new().with(t::ISO_SPEED, Value::Short(vec![800]))); - let cmt3 = tiff( - IfdBuilder::new() - .with(SENSOR_INFO, Value::Short(vec![34, 64, 48, 1, 1, 6, 4, 61, 45])) - .with(LENS_MODEL, Value::Ascii("RF-S18-45mm F4.5-6.3 IS STM".into())), - ); + let mut cmt3 = IfdBuilder::new() + .with(SENSOR_INFO, Value::Short(vec![34, 64, 48, 1, 1, 6, 4, 61, 45])) + .with(LENS_MODEL, Value::Ascii("RF-S18-45mm F4.5-6.3 IS STM".into())); + if let Some(a) = aspect { + cmt3 = cmt3.with(ASPECT_INFO, Value::Long(a.to_vec())); + } + let cmt3 = tiff(cmt3); let mut canon = vec![0x85, 0xc0, 0xb6, 0x87, 0x82, 0x0f, 0x11, 0xe0, 0x81, 0x11, 0xf4, 0xce, 0x46, 0x2b, 0x6a, 0x48]; canon.extend([box_bytes(b"CMT1", &cmt1), box_bytes(b"CMT2", &cmt2), box_bytes(b"CMT3", &cmt3)].concat()); let mut coding = vec![0u8; 52]; @@ -374,6 +400,60 @@ mod tests { assert!(decode(&bytes[..4200], Mode::Header).is_err()); } + #[test] + fn aspect_info_replaces_the_crop() { + // 16:9 inside the 56 x 42 recommended crop at (2, 2): 56 x 30, 6 rows down. + let bytes = synthetic_header_with(Some([7, 56, 30, 0, 6])); + let image = decode(&bytes, Mode::Header).unwrap(); + assert_eq!(image.active_area, Rect::new(4, 2, 60, 46)); + assert_eq!(image.crop, Rect::new(2, 8, 56, 30)); + assert_eq!((image.metadata.width, image.metadata.height), (Some(56), Some(30))); + // 1:1 with a left offset. + let image = decode(&synthetic_header_with(Some([1, 42, 42, 7, 0])), Mode::Header).unwrap(); + assert_eq!(image.crop, Rect::new(9, 2, 42, 42)); + // The whole image, as stored for a shot at the sensor's own aspect, changes nothing. + let image = decode(&synthetic_header_with(Some([0, 56, 42, 0, 0])), Mode::Header).unwrap(); + assert_eq!(image.crop, Rect::new(2, 2, 56, 42)); + } + + #[test] + fn aspect_info_that_does_not_fit_is_ignored() { + for bad in [[7, 57, 30, 0, 6], [7, 56, 30, 0, 13], [7, 56, 30, 1, 6], [7, 1, 30, 0, 0], [7, 56, 0, 0, 0], [7, u32::MAX, 30, 0, 0]] { + let image = decode(&synthetic_header_with(Some(bad)), Mode::Header).unwrap(); + assert_eq!(image.crop, Rect::new(2, 2, 56, 42), "{bad:?}"); + } + } + + /// Some crop-mode files (EOS R5 Mark II 7883) record a recommended crop that reaches past their valid area. The + /// whole active area stays then, and `AspectInfo`, which is measured from the recommended crop, is not applied. + #[test] + fn aspect_info_needs_a_consistent_recommended_crop() { + let aspect = lightcraft_tiff::IfdBuilder::new().with(ASPECT_INFO, Value::Long(vec![13, 5088, 3392, 0, 0])); + let bytes = lightcraft_tiff::TiffWriter::new(lightcraft_tiff::ByteOrder::Little, false).write(&[aspect]).unwrap(); + let maker = Tiff::parse(&bytes).unwrap().ifds.remove(0); + let area = + |crop| Cr3ImageArea { width: 5376, height: 3574, crop, active: Some([132, 160, 5243, 3567]), masked_left: [0; 4], masked_top: None }; + // crop right edge 5359 > active right edge 5243: not a crop of the active area + assert_eq!(geometry(Some(&area([272, 172, 5359, 3563])), Some(&maker), 5376, 3574).1, Rect::new(0, 0, 5112, 3408)); + // consistent crop: AspectInfo applies from its origin + assert_eq!(geometry(Some(&area([140, 172, 5227, 3563])), Some(&maker), 5376, 3574).1, Rect::new(8, 12, 5088, 3392)); + } + + /// `AspectInfo` as the SX70 HS 16:9 sample stores it, byte for byte (little-endian LONG x 5), read through + /// the TIFF parser: AspectRatio 7, 5184 x 2912 at (0, 488). + #[test] + fn aspect_info_known_bytes() { + let mut tiff = b"II*".to_vec(); + tiff.extend_from_slice(&[0x9a, 0x00, 0x04, 0x00, 0x05, 0x00, 0x00, 0x00, 0x1a, 0x00, 0x00, 0x00]); + tiff.extend_from_slice(&[0, 0, 0, 0]); + tiff.extend_from_slice(&[0x07, 0, 0, 0, 0x40, 0x14, 0, 0, 0x60, 0x0b, 0, 0, 0, 0, 0, 0, 0xe8, 0x01, 0, 0]); + let parsed = Tiff::parse(&tiff).unwrap(); + let maker = parsed.ifds.first(); + assert_eq!(aspect_crop(maker, Rect::new(132, 40, 5184, 3888)), Some(Rect::new(132, 528, 5184, 2912))); + assert_eq!(aspect_crop(maker, Rect::new(0, 0, 5184, 2000)), None); + assert_eq!(aspect_crop(None, Rect::new(0, 0, 5184, 3888)), None); + } + fn prepend_tracks(bytes: &[u8], tracks: &[u8]) -> Vec { let ftyp_end = u32::from_be_bytes(bytes[..4].try_into().unwrap()) as usize; let moov_len = u32::from_be_bytes(bytes[ftyp_end..ftyp_end + 4].try_into().unwrap()) as usize; diff --git a/crates/raw/tests/cr3_corpus.rs b/crates/raw/tests/cr3_corpus.rs index 6e270576..6cd28c3b 100644 --- a/crates/raw/tests/cr3_corpus.rs +++ b/crates/raw/tests/cr3_corpus.rs @@ -15,7 +15,7 @@ fn corpus() -> PathBuf { std::env::var_os("LIGHTCRAFT_CORPUS").map(PathBuf::from).unwrap_or_else(|| Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus")) } -fn check(name: &str, hash: u64, sensor: (usize, usize)) { +fn check(name: &str, hash: u64, sensor: (usize, usize), developed: (usize, usize)) { let path = corpus().join("raw").join(name); let Ok(bytes) = std::fs::read(&path) else { eprintln!("skip: {} absent", path.display()); @@ -25,7 +25,7 @@ fn check(name: &str, hash: u64, sensor: (usize, usize)) { assert_eq!(raw.format, RawFormat::Cr3); raw.validate().unwrap(); assert_eq!((raw.width, raw.height, raw.cpp), (sensor.0, sensor.1, 1)); - assert_eq!(raw.info().developed_size(), (6000, 4000)); + assert_eq!(raw.info().developed_size(), developed); assert_eq!(raw.info(), probe_info(&bytes).unwrap()); let RawData::U16(samples) = &raw.data else { panic!("{name}: integer sensor data expected") }; let actual = @@ -39,40 +39,55 @@ fn check(name: &str, hash: u64, sensor: (usize, usize)) { fn canon_m50_lossless_sensor_is_exact() { // raw.pixls.us/getfile.php/4657; SHA256 // 1ba9ad6b51b315b88820eb0fe5fd7f52bc136ba37b1a5ca76825ebfe25b18058 - check("cr3-canon-m50-raw.cr3", 0x6226_1f0b_a81c_fcd2, (6288, 4056)); + check("cr3-canon-m50-raw.cr3", 0x6226_1f0b_a81c_fcd2, (6288, 4056), (6000, 4000)); } #[test] fn canon_r100_lossless_sensor_is_exact() { // raw.pixls.us/getfile.php/7896; SHA256 // 6d83217d58a5e6d2dabcf470430e91a16676b60531c025b81fa7453fc74b0521 - check("cr3-canon-r100-raw.cr3", 0xcbe5_299a_b9c5_2630, (6288, 4056)); + check("cr3-canon-r100-raw.cr3", 0xcbe5_299a_b9c5_2630, (6288, 4056), (6000, 4000)); } #[test] fn canon_m50_craw_sensor_is_exact() { // raw.pixls.us/getfile.php/2663; SHA256 // 15384b775867ec4c42b11882837f1e368cedc0561832ffab271221e6bb80be4c - check("cr3-canon-m50-craw.cr3", 0x9aac_bfe6_6f50_5e1b, (6288, 4056)); + // The camera was set to 1:1: the file's AspectInfo is a 4000 x 4000 square at x = 1000 inside the 6000 x 4000 + // image (the archive's "3:2" label is wrong). The sensor samples are unchanged; only the crop is. + check("cr3-canon-m50-craw.cr3", 0x9aac_bfe6_6f50_5e1b, (6288, 4056), (4000, 4000)); } #[test] fn canon_r100_craw_sensor_is_exact() { // raw.pixls.us/getfile.php/7897; SHA256 // 0b66842b2fe00329ebc05fe8ab9357ddbbe1a0a2d2e69a2893102de7fdf5c942 - check("cr3-canon-r100-craw.cr3", 0x341c_706b_37c3_8bbf, (6288, 4056)); + check("cr3-canon-r100-craw.cr3", 0x341c_706b_37c3_8bbf, (6288, 4056), (6000, 4000)); } #[test] fn canon_r8_lossless_sensor_is_exact() { // raw.pixls.us/getfile.php/6585; SHA256 // 7d5c6dbb11ff7e6ee58715d90a20f5d801e8b103f4711ff6ea329ceafcef254c - check("cr3-canon-r8-raw.cr3", 0x63c1_c6d8_d1eb_2312, (6188, 4120)); + check("cr3-canon-r8-raw.cr3", 0x63c1_c6d8_d1eb_2312, (6188, 4120), (6000, 4000)); } #[test] fn canon_r8_craw_sensor_is_exact() { // raw.pixls.us/getfile.php/6587; SHA256 // df33cf394573645ce03dea1b2e9f0b5cc2b7734e9e3391ee7114151414cf2812 - check("cr3-canon-r8-craw.cr3", 0xf836_a663_a795_a375, (6188, 4120)); + check("cr3-canon-r8-craw.cr3", 0xf836_a663_a795_a375, (6188, 4120), (6000, 4000)); +} + +#[test] +fn canon_m50_craw_crop_is_the_shot_aspect() { + // raw.pixls.us/getfile.php/2663: the 1:1 square starts 1000 px right of the recommended crop's origin, + // which is (12, 12) inside the valid sensor area. + let path = corpus().join("raw").join("cr3-canon-m50-craw.cr3"); + let Ok(bytes) = std::fs::read(&path) else { + eprintln!("skip: {} absent", path.display()); + return; + }; + let raw = decode(&bytes).unwrap(); + assert_eq!((raw.crop.x, raw.crop.y, raw.crop.width, raw.crop.height), (1012, 12, 4000, 4000)); } diff --git a/docs/cr3.md b/docs/cr3.md index 692c209d..c89d3c30 100644 --- a/docs/cr3.md +++ b/docs/cr3.md @@ -5,6 +5,12 @@ The CR3 adapter reads the CRX Bayer track, `CMP1` compression descriptor, black/white levels and the default crop come from the file. A Dual Pixel delta track is never substituted for the primary Bayer track. +The in-camera aspect ratio (3:2, 4:3, 16:9, 1:1) is not applied to the sensor data. The maker note's +`AspectInfo` (tag `0x009a`: aspect code, cropped width, height, left, top) gives the shot's rectangle inside the +recommended crop, and the decoder uses it as the default crop, so the developed image has the shape the photographer +framed (PowerShot SX70 HS, EOS 250D, PowerShot G5 X Mark II, EOS M6 Mark II and EOS M50 samples). A rectangle that +is missing or does not fit inside the crop is ignored. The embedded JPEG still shows the whole frame. + The decoder is independent Rust code shared by desktop, CLI and WebAssembly; it has no operating-system imaging dependency or external decoder. Its sources are the public prose descriptions and patents cited in diff --git a/docs/parity.md b/docs/parity.md index 90dc8a74..2eaae48b 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -137,7 +137,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins. | LR-IMP-PROFILES | Import profiles & presets | P1 | ✅ | `cmd:file.importPresets`, `cmd:preset.import`, `cmd:profile.import`, `crates/engine/src/preset_import.rs`, `crates/engine/src/preset_luminar.rs` | presets: .lcpreset, XMP, classic .lrtemplate, photos carrying edits ("DNG presets"), Luminar looks (.lmp, .mplumpack collections; sliders with a counterpart), .zip bundles, folders (folder → group), drag & drop; masks carried over; unmapped settings reported (fields that only describe the preset or photo, e.g. `Cluster`, `SortName`, `SupportsAmount2`, the as-shot white, empty Point Color slots, are not); profiles: .cube 3D LUTs; Adobe profile formats deliberately unsupported | | LR-IMP-LOCAL | Work on files in place | P0 | ✅ | `cmd:library.browse`, `cmd:photo.addToLibrary`, `cmd:library.import` (mode add), `cmd:local.addRoot`, `cmd:local.hide`, `cmd:local.restoreHidden`, `crates/engine/src/cmd/browse.rs`, `crates/ui-egui/src/panels/left.rs` (`local_section`), `crates/ui-egui/src/panels/grid.rs` (`folder_header`) | Local: browse Pictures / Desktop / Downloads / Home or any folder without adding it (breadcrumb, Include subfolders, Add N to My Photos); browsed photos stay out of All Photos, albums and counts; folders picked with Browse Folder… (or Keep in Local / Add to Local on a folder row) stay listed across restarts, and browsing a folder inside a listed one highlights it inside that tree (opened down to it, siblings reachable) instead of adding a root; edits go to XMP sidecars; importing promotes them; right-click a top-level location → Remove from Local hides the shortcut only (nothing on disk or in the catalog changes), saved with the UI state, and “Show N hidden locations” (or Browse Folder… on it) brings it back; a folder is one location however its path is spelled (`/` or `\`, trailing separator, `.`/`..`, drive-letter case; `folder_key` in `crates/catalog/src/query.rs`); browsed photos nobody changed are forgotten once their folder has not been browsed for 30 days (Settings → Performance → Local folders, 0 = never; checked when the library opens, or `cmd:library.forgetLocal` with `dryRun`; files and sidecars stay, browsing again brings them back; see `crates/catalog/src/local.rs` for what counts as a change) | | LR-IMP-SIDECAR-SPLIT | Separate XMP sidecar variants | P2 | ⬜ | `cmd:library.xmpPreferences` | sidecar naming option exists (stem/full), no split sidecars | -| LR-IMP-FORMATS | Supported formats | P0 | 🟡 | `crates/codecs/src/lib.rs`, `crates/raw/src/lib.rs` | JPEG, PNG, TIFF (1- to 64-bit integer, 16- to 64-bit float, incl. bilevel and CCITT fax), WebP, JXL, PSD, GIF, BMP; DNG (incl. lossy / Smart Preview DNG), CR2 (colour-filter layout from the file's `CR2CFAPattern` tag, issue #85), ARW (pre-2017 bodies: white balance and black level from enciphered maker-note / `SR2SubIFD` data, issue #148; downsized Sony lossless YCbCr 4:2:0 / 4:2:2 tiles decoded to linear RGB, verified on private ILCE-7M4 3:2 & 4:3 examples including 16-bit full-resolution export), NEF (uncompressed, lossless and lossy compressed 12/14-bit, `crates/raw/src/vendor/nefc.rs`), RAF (uncompressed, lossless/lossy compressed Bayer and X-Trans; `crates/raw/src/vendor/rafc.rs`, full sensor verification on 13 bodies, `docs/raf-compression.md`), RW2 / Leica RWL / Panasonic RAW in every raw format (compressed 4 and 6, the prefix-coded strips of format 8 from the GH6 on, packed 2/5/7, the 16-bit words of the 2005–2007 bodies; black levels, in-camera aspect crops and mapped-out defect pixels; 178 CC0 files from 118 bodies checked, `crates/raw/src/vendor/rw2.rs`), PEF, ORF (uncompressed; colour-filter layout from the file's Exif `CFAPattern`). 12-bit NEF black level: maker note `0x003d` is in 14-bit units (D750, D780, D850, D7500, Z 50 checked). Lossy type 2 NEF curves of version `0x44 0x40` (D780, D6, Z 5, Z 6, Z 7, Z 7_2) span a quarter of the sample range, read from the table's own point spacing. CR3 lossless CRX Bayer and version 0x100/0x200 C-RAW decodes to sensor pixels with timed white balance, black/white levels and file crop; full sensor samples match the reference on M50/R100/R8 (`crates/raw/tests/cr3_corpus.rs`, `docs/cr3.md`). Unsupported coding variants retain metadata and use the embedded JPEG when available; HEVC preview decoding remains unsupported. Missing: CR3 unverified CRX variants, compressed ORF, NEFs labelled compressed (34713) but stored uncompressed without a `0x0096` table (Z 6 packed 14-bit, D850 12-bit uncompressed), NEF "lossy after split" (preview only: imported with `preview_only` = the decoder's reason, rendered as a rendered JPEG, flagged by a grid badge and loupe/Edit/Info notices, `previewOnly` in `catalog.query`; Reload clears it once the file decodes), TIFF-based raws whose raw data sits in a private block (Phase One / Leaf IIQ, Canon EOS-1D / 1Ds and Kodak DCS TIFFs: IFD0 is only a reduced RGB image next to a much larger declared picture) are recognised from that structure and import as `preview_only` with the reason, instead of opening as a thumbnail-sized ordinary image (`thumbnail_shell` in `crates/raw/src/lib.rs`), Recognised raws LightCraft cannot decode open as `preview_only` with any usable embedded JPEG, also Canon CRW, Minolta MRW and Sigma X3F (containers recognised by magic, preview found by a structural JPEG scan), Kodak KDC (IFD0 without `ImageWidth`), Leaf MOS and Epson ERF (scan), and any decode failure of a recognised raw, not only an unsupported variant; ones with no JPEG still fail with "without an embedded preview", HEIC/AVIF decode, palette (indexed-colour) TIFF (refused by the `tiff` crate) | +| LR-IMP-FORMATS | Supported formats | P0 | 🟡 | `crates/codecs/src/lib.rs`, `crates/raw/src/lib.rs` | JPEG, PNG, TIFF (1- to 64-bit integer, 16- to 64-bit float, incl. bilevel and CCITT fax), WebP, JXL, PSD, GIF, BMP; DNG (incl. lossy / Smart Preview DNG), CR2 (colour-filter layout from the file's `CR2CFAPattern` tag, issue #85), ARW (pre-2017 bodies: white balance and black level from enciphered maker-note / `SR2SubIFD` data, issue #148; downsized Sony lossless YCbCr 4:2:0 / 4:2:2 tiles decoded to linear RGB, verified on private ILCE-7M4 3:2 & 4:3 examples including 16-bit full-resolution export), NEF (uncompressed, lossless and lossy compressed 12/14-bit, `crates/raw/src/vendor/nefc.rs`), RAF (uncompressed, lossless/lossy compressed Bayer and X-Trans; `crates/raw/src/vendor/rafc.rs`, full sensor verification on 13 bodies, `docs/raf-compression.md`), RW2 / Leica RWL / Panasonic RAW in every raw format (compressed 4 and 6, the prefix-coded strips of format 8 from the GH6 on, packed 2/5/7, the 16-bit words of the 2005–2007 bodies; black levels, in-camera aspect crops and mapped-out defect pixels; 178 CC0 files from 118 bodies checked, `crates/raw/src/vendor/rw2.rs`), PEF, ORF (uncompressed; colour-filter layout from the file's Exif `CFAPattern`). 12-bit NEF black level: maker note `0x003d` is in 14-bit units (D750, D780, D850, D7500, Z 50 checked). Lossy type 2 NEF curves of version `0x44 0x40` (D780, D6, Z 5, Z 6, Z 7, Z 7_2) span a quarter of the sample range, read from the table's own point spacing. CR3 lossless CRX Bayer and version 0x100/0x200 C-RAW decodes to sensor pixels with timed white balance, black/white levels and file crop (the shot's aspect ratio from the maker note's `AspectInfo`); full sensor samples match the reference on M50/R100/R8 (`crates/raw/tests/cr3_corpus.rs`, `docs/cr3.md`). Unsupported coding variants retain metadata and use the embedded JPEG when available; HEVC preview decoding remains unsupported. Missing: CR3 unverified CRX variants, compressed ORF, NEFs labelled compressed (34713) but stored uncompressed without a `0x0096` table (Z 6 packed 14-bit, D850 12-bit uncompressed), NEF "lossy after split" (preview only: imported with `preview_only` = the decoder's reason, rendered as a rendered JPEG, flagged by a grid badge and loupe/Edit/Info notices, `previewOnly` in `catalog.query`; Reload clears it once the file decodes), TIFF-based raws whose raw data sits in a private block (Phase One / Leaf IIQ, Canon EOS-1D / 1Ds and Kodak DCS TIFFs: IFD0 is only a reduced RGB image next to a much larger declared picture) are recognised from that structure and import as `preview_only` with the reason, instead of opening as a thumbnail-sized ordinary image (`thumbnail_shell` in `crates/raw/src/lib.rs`), Recognised raws LightCraft cannot decode open as `preview_only` with any usable embedded JPEG, also Canon CRW, Minolta MRW and Sigma X3F (containers recognised by magic, preview found by a structural JPEG scan), Kodak KDC (IFD0 without `ImageWidth`), Leaf MOS and Epson ERF (scan), and any decode failure of a recognised raw, not only an unsupported variant; ones with no JPEG still fail with "without an embedded preview", HEIC/AVIF decode, palette (indexed-colour) TIFF (refused by the `tiff` crate) | | LR-IMP-CAMERA-COVERAGE | Camera coverage, verified per model | P0 | 🟡 | `crates/raw/tests/corpus.rs`, `xtask/src/main.rs` | a supported container is not the same as every camera that writes it decoding correctly: ~55 CC0 corpus files plus 6 exact-sensor CR3 cases (plus ~60 NEFs checked by hand and 178 Panasonic / Leica RW2, RWL and RAW files from 118 bodies, checked against each file's own JPEG) against the >1,000 models Lightroom lists; per-model differences still surface (e.g. the CR2 colour-filter layout differed by model until it was read from the file's own tag, issue #85, checked on 35 CR2 bodies; pre-2017 Sony ARWs opened green until their scrambled white balance / black level were read, issue #148, checked on 16 ARWs from 15 Sony bodies; Canon PowerShot and EOS M CR2s whose `ColorData` is stored as bytes opened green until the bytes were read as 16-bit words, 25 files from 25 bodies checked). Compressed RAF sensor arrays verified on 13 Fujifilm bodies (`docs/raf-compression.md`). Grow the CC0 corpus per model and test each sample decodes with plausible colour. Nikon CropArea and untagged-preview colour-space regression tests; private Z 8 crop verified. | | LR-IMP-CULL-AT-IMPORT | Culling analysis at import | P2 | 🟡 | `cmd:photo.analyze` | run Assisted Culling on the imported photos (they're selected after an import); not automatic | | LR-IMP-MOVE | Move on import [Classic] | P1 | ✅ | `cmd:library.import` (`mode` move), `crates/engine/src/import_move.rs`, `crates/engine/src/tests_import_move.rs`, `crates/ui-egui/src/import.rs` | Transfer ▸ Move with Copy's destination, folders (day / month / one folder / custom template), rename template and example destination; XMP sidecars (both namings) move along; each source is removed only after its destination is written (hard link on the same volume, else copied, synced and compared byte for byte) and its catalog record is saved; failed, duplicate, unchecked files keep their sources, taken names get -1, -2…, never overwritten; files already in the destination/library are added in place; sources that can't be removed (read-only card) are kept and reported (`kept`). Undo removes the photos from the library but leaves the moved files at the destination. No Copy as DNG while moving | From bb7c46ae3c639fc060c81bbcfdc2689f06eb6f92 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 8 Oct 2026 23:55:26 -0400 Subject: [PATCH 25/46] Remote SAM3 worker: crate lint attributes; document the trust boundary The worker binary gets the never-crash deny list and forbid(unsafe_code) like every production crate root. docs/ai-masks.md spells out who can reach the unauthenticated loopback listener. Co-Authored-By: Claude Opus 5.5 --- crates/segment/src/bin/lightcraft-sam3-worker.rs | 5 +++++ docs/ai-masks.md | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/segment/src/bin/lightcraft-sam3-worker.rs b/crates/segment/src/bin/lightcraft-sam3-worker.rs index 9929d2eb..62c18334 100644 --- a/crates/segment/src/bin/lightcraft-sam3-worker.rs +++ b/crates/segment/src/bin/lightcraft-sam3-worker.rs @@ -1,3 +1,8 @@ +//! Serves SAM 3 on this Mac's Metal GPU to one LightCraft editor over a loopback socket (an SSH +//! tunnel); see docs/ai-masks.md → Remote Metal inference. +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + fn main() -> Result<(), Box> { let mut args = std::env::args().skip(1); let dir = args.next().ok_or("usage: lightcraft-sam3-worker MODEL_DIR [127.0.0.1:8793]")?; diff --git a/docs/ai-masks.md b/docs/ai-masks.md index 00de5016..574130ec 100644 --- a/docs/ai-masks.md +++ b/docs/ai-masks.md @@ -68,7 +68,10 @@ stay in the local library, so later adjustments and exports work without the con The worker requires Metal and accepts one editing connection at a time. It loads weights on demand and releases them when the client disconnects or remains idle for ten minutes. Both endpoints require loopback addresses: SSH provides authentication and encryption. -The listener has no independent authentication; do not expose it through a public proxy. +The listener has no independent authentication; do not expose it through a public proxy. Any +process of any user on the worker Mac can connect to it, and on the editing computer a local +process listening on the port while the tunnel is down receives the rendered model input, so use +this only on machines whose local users you trust. No local weights are required in remote mode. An unavailable worker produces an explicit error, not a silent CPU fallback. Restore the tunnel and retry. Unset From b341101ab644f77631897d67d09f348f670bf9ad Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 9 Oct 2026 00:20:28 -0400 Subject: [PATCH 26/46] Remote SAM3 worker: cargo fmt Co-Authored-By: Claude Opus 5.5 --- crates/segment/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/segment/src/lib.rs b/crates/segment/src/lib.rs index a4c3acad..6d488a91 100644 --- a/crates/segment/src/lib.rs +++ b/crates/segment/src/lib.rs @@ -32,13 +32,13 @@ mod clip; mod detector; #[cfg(not(target_arch = "wasm32"))] pub mod fetch; -#[cfg(not(target_arch = "wasm32"))] -pub mod remote; pub mod mask; #[cfg(not(target_arch = "wasm32"))] mod neck; #[cfg(not(target_arch = "wasm32"))] mod nn; +#[cfg(not(target_arch = "wasm32"))] +pub mod remote; pub mod tokenizer; #[cfg(not(target_arch = "wasm32"))] mod tracker; From 5c1dcd4f970de8e307e2bdd9f546c09fb1a38b97 Mon Sep 17 00:00:00 2001 From: onemannoplan <96282233+onemannoplan@users.noreply.github.com> Date: Thu, 8 Oct 2026 23:53:21 -0500 Subject: [PATCH 27/46] fix(macos): prevent SIGILL crash on exit by replacing viewport close with clean process exit ### Summary When selecting "Quit LightCraft" from the native macOS menu bar, the application occasionally crashes with an `EXC_BAD_INSTRUCTION` (SIGILL) error report during shutdown. ### Root Cause 1. The native menu action triggered `ctx.send_viewport_cmd(egui::ViewportCommand::Close)`. 2. Forcing an explicit viewport close via the command queue while macOS and AppKit are initiating their own termination sequence creates a race condition with `winit`. 3. This conflict during native window, view, and observer teardown leads to an illegal instruction fault. ### Solution - Replaced the manual `ViewportCommand::Close` trigger on `QUIT` with a clean `std::process::exit(0)`. - This bypasses the problematic `winit`/AppKit teardown race condition on macOS, allowing the application to shut down instantly and cleanly without generating crash reports. --- apps/lightcraft/src/native_menu.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/lightcraft/src/native_menu.rs b/apps/lightcraft/src/native_menu.rs index 647f7e27..6c6d2f2b 100644 --- a/apps/lightcraft/src/native_menu.rs +++ b/apps/lightcraft/src/native_menu.rs @@ -360,8 +360,7 @@ impl NativeMenu { lightcraft_ui_egui::i18n::set_language(app.ui.language); while let Ok(key) = self.rx.try_recv() { if key == QUIT { - ctx.send_viewport_cmd(egui::ViewportCommand::Close); - continue; + std::process::exit(0); } if key == "app.about" || key == SETTINGS { let _ = app.run(&key, serde_json::json!({})); From d9457e6b23f833924a58679a7f42505363cd7eb5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 9 Oct 2026 01:03:40 -0400 Subject: [PATCH 28/46] CI: generate each procedural demo scene once per process (FreeBSD UI tests timing out) Every demo-photo source load evaluated the procedural scene from scratch. A 24 MP demo original costs ~200 CPU-seconds (AlpineLake, the photo the Detail tests open), and #323 added ~30 loupe tests that each load it at 1:1, Before and After separately. On the 4-vCPU FreeBSD VM the ui-egui lib tests went from 1198 s to 3046 s, and the tests' 120 s/300 s waits expired (loupe windows, fixture thumbnails starved of the shared rayon pool, the demo detail snapshot). Demo scene pixels are now generated once per (scene, size) and shared: concurrent loads wait for one generation (Before/After at 1:1 no longer generate the original twice) and the most recent results are kept up to half the memory budget. Locally, pinned to 4 CPUs, the ui-egui lib suite now finishes in 155 s; without the change it had done 231 of 338 tests after 25 minutes. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/demo.rs | 140 +++++++++++++++++++++++++++++++++++++ crates/engine/src/media.rs | 2 +- 2 files changed, 141 insertions(+), 1 deletion(-) diff --git a/crates/engine/src/demo.rs b/crates/engine/src/demo.rs index 2f392cb7..dc5c3e5a 100644 --- a/crates/engine/src/demo.rs +++ b/crates/engine/src/demo.rs @@ -84,3 +84,143 @@ pub fn load(s: &mut Session) { s.selection = crate::Selection::single(*first); } } + +/// Procedural scenes are deterministic but costly to evaluate: every pixel runs the scene's noise +/// functions, so a 24 MP demo original takes minutes of CPU time on a small machine (a 4-core CI +/// VM). Each scene at each size is therefore generated once per process and shared: +/// +/// - concurrent requests wait for one generation (Before and After at 1:1 both load the +/// original; sessions side by side, e.g. parallel tests, open the same photo); +/// - the most recently used results are kept, up to [`scene_cache_limit`] bytes. They are the same +/// allocations the sessions' source caches hold, so they cost extra memory only once no session +/// holds them any more. +/// +/// Only demo photos use this; real files are decoded by their loader as before. +pub(crate) fn scene_pixels(scene: &lightcraft_scenes::Scene, max_edge: usize) -> Arc { + let key = SceneKey { kind: scene.kind, seed: scene.seed, width: scene.width, height: scene.height, max_edge }; + let slot = { + let mut c = scene_cache(); + c.tick += 1; + let tick = c.tick; + match c.entries.iter_mut().find(|e| e.key == key) { + Some(e) => { + e.used = tick; + e.slot.clone() + } + None => { + let slot = Arc::new(std::sync::OnceLock::new()); + c.entries.push(SceneEntry { key, slot: slot.clone(), used: tick }); + slot + } + } + }; + // (no lock held while generating: other scenes and sizes are served meanwhile) + let pixels = slot.get_or_init(|| Arc::new(scene.render_fit(max_edge))).clone(); + trim_scene_cache(&mut scene_cache(), scene_cache_limit()); + pixels +} + +/// Bytes of generated scenes kept for reuse: half the memory budget (768 MB by default; enough +/// for one 24 MP original and the previews around it). +fn scene_cache_limit() -> usize { + crate::memory::budget() / 2 +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +struct SceneKey { + kind: lightcraft_scenes::Kind, + seed: u32, + width: u32, + height: u32, + max_edge: usize, +} + +type SceneSlot = Arc>>; + +struct SceneEntry { + key: SceneKey, + slot: SceneSlot, + used: u64, +} + +#[derive(Default)] +struct SceneCache { + entries: Vec, + tick: u64, +} + +fn scene_cache() -> std::sync::MutexGuard<'static, SceneCache> { + static CACHE: std::sync::OnceLock> = std::sync::OnceLock::new(); + CACHE.get_or_init(Default::default).lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + +fn scene_bytes(slot: &SceneSlot) -> usize { + slot.get().map_or(0, |img| img.data.len().saturating_mul(std::mem::size_of::<[f32; 3]>())) +} + +/// Drop the least recently used generated scenes until the rest fit in `limit` bytes. Scenes still +/// being generated are kept (their waiters hold the slot anyway). +fn trim_scene_cache(c: &mut SceneCache, limit: usize) { + loop { + let total: usize = c.entries.iter().map(|e| scene_bytes(&e.slot)).fold(0, usize::saturating_add); + if total <= limit { + return; + } + let Some(i) = c.entries.iter().enumerate().filter(|(_, e)| e.slot.get().is_some()).min_by_key(|(_, e)| e.used).map(|(i, _)| i) else { + return; + }; + c.entries.swap_remove(i); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn small_scene() -> lightcraft_scenes::Scene { + lightcraft_scenes::demo_library().swap_remove(0) + } + + #[test] + fn a_scene_is_generated_once_per_size_and_shared() { + let s = small_scene(); + let a = scene_pixels(&s, 48); + let b = scene_pixels(&s, 48); + assert!(Arc::ptr_eq(&a, &b), "the second request reuses the first generation"); + assert_eq!(a.width.max(a.height), 48); + assert_eq!(a.data, s.render_fit(48).data, "the same pixels as generating it directly"); + let c = scene_pixels(&s, 40); + assert_eq!(c.width.max(c.height), 40, "another size is its own generation"); + } + + #[test] + fn concurrent_requests_wait_for_one_generation() { + let s = small_scene(); + let edge = 37; // a size no other test asks for + let got: Vec<_> = std::thread::scope(|sc| { + let hs: Vec<_> = (0..4).map(|_| sc.spawn(|| scene_pixels(&s, edge))).collect(); + hs.into_iter().map(|h| h.join().unwrap()).collect() + }); + assert!(got.windows(2).all(|w| Arc::ptr_eq(&w[0], &w[1])), "every request got the same pixels"); + } + + #[test] + fn the_cache_keeps_the_most_recent_scenes_within_its_limit() { + let key = |max_edge| SceneKey { kind: lightcraft_scenes::Kind::Dunes, seed: 1, width: 3, height: 2, max_edge }; + let entry = |max_edge, used, px: Option| { + let slot: SceneSlot = Arc::new(std::sync::OnceLock::new()); + if let Some(n) = px { + let _ = slot.set(Arc::new(lightcraft_raster::Rgb32f::new(n, 1))); + } + SceneEntry { key: key(max_edge), slot, used } + }; + let px = std::mem::size_of::<[f32; 3]>(); + let mut c = SceneCache { entries: vec![entry(1, 3, Some(10)), entry(2, 1, Some(10)), entry(3, 0, None), entry(4, 2, Some(10))], tick: 3 }; + trim_scene_cache(&mut c, 25 * px); + let mut kept: Vec = c.entries.iter().map(|e| e.key.max_edge).collect(); + kept.sort_unstable(); + assert_eq!(kept, vec![1, 3, 4], "the least recently used finished scene went; the one in progress stays"); + trim_scene_cache(&mut c, 0); + assert_eq!(c.entries.iter().map(|e| e.key.max_edge).collect::>(), vec![3], "only the scene in progress is left"); + } +} diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rs index 1ed89e47..c3b3c468 100644 --- a/crates/engine/src/media.rs +++ b/crates/engine/src/media.rs @@ -176,7 +176,7 @@ impl SourceRef { pub fn load_source(&self) -> Result { let image = match self { SourceRef::Loaded(a) => return Ok((**a).clone()), - SourceRef::Demo { scene, max_edge } => Arc::new(scene.render_fit(*max_edge)), + SourceRef::Demo { scene, max_edge } => crate::demo::scene_pixels(scene, *max_edge), SourceRef::File { path, max_edge, loader, fallback } => { let r = match loader { Some(l) => l(path, *max_edge).map(|(image, info)| DecodedSource::new(Arc::new(image), Some(info))), From 7dcfbed05c27d14e01753bc9963146f12e5a2147 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 9 Oct 2026 01:07:43 -0400 Subject: [PATCH 29/46] LR-EDIT-DETAIL-DENOISE: list settings tabs from TABS; fix the SAM 3 worker's wasm check The unknown-tab error now names every Settings tab (it missed AI Denoise after the faces merge). lightcraft-sam3-worker (#359) only builds its server off wasm32, so `cargo xtask wasm` passes again. Co-Authored-By: Claude Opus 5.5 --- crates/segment/src/bin/lightcraft-sam3-worker.rs | 5 +++++ crates/ui-egui/src/menus.rs | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/segment/src/bin/lightcraft-sam3-worker.rs b/crates/segment/src/bin/lightcraft-sam3-worker.rs index 62c18334..f8e52541 100644 --- a/crates/segment/src/bin/lightcraft-sam3-worker.rs +++ b/crates/segment/src/bin/lightcraft-sam3-worker.rs @@ -3,6 +3,7 @@ #![forbid(unsafe_code)] #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#[cfg(not(target_arch = "wasm32"))] fn main() -> Result<(), Box> { let mut args = std::env::args().skip(1); let dir = args.next().ok_or("usage: lightcraft-sam3-worker MODEL_DIR [127.0.0.1:8793]")?; @@ -11,3 +12,7 @@ fn main() -> Result<(), Box> { lightcraft_segment::remote::serve(address, std::path::Path::new(&dir), device)?; Ok(()) } + +/// The worker serves a local GPU over a socket; there is nothing to serve in a browser. +#[cfg(target_arch = "wasm32")] +fn main() {} diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index ce40ebc5..5b79548b 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -465,7 +465,8 @@ pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option { let tab = p.get("tab").and_then(Value::as_str).unwrap_or("general"); if !crate::panels::settings::TABS.iter().any(|(id, _)| *id == tab) { - return Some(Err(format!("unknown settings tab `{tab}` (general|import|performance|interface|faces)"))); + let tabs: Vec<&str> = crate::panels::settings::TABS.iter().map(|(id, _)| *id).collect(); + return Some(Err(format!("unknown settings tab `{tab}` ({})", tabs.join("|")))); } app.ui.dialog = Some(Dialog::Settings { tab: tab.into() }); Ok(Value::Null) From 989422398c3044b39b640111dc10cafebf5ba92e Mon Sep 17 00:00:00 2001 From: radialmonster Date: Fri, 9 Oct 2026 01:12:55 -0400 Subject: [PATCH 30/46] LR-IMP-FORMATS: read plain packed 12-bit PEF strips tagged as PackBits (older Pentax, Samsung GX-1L) Compression 32773 is accepted in the PEF reader only when every strip has exactly the packed size of its rows at the declared bit depth, and is then read as uncompressed MSB-first samples. Any other 32773 size stays unsupported. Uncompressed PEFs take their colour filter from the Exif CFAPattern as seen from the image area origin. A Samsung-branded TIFF whose maker note has the Pentax layout is probed as PEF. RENDER_CACHE_VERSION 16 to 17. --- crates/engine/src/media.rs | 2 +- crates/raw/src/lib.rs | 27 ++++++++++++ crates/raw/src/vendor/mod.rs | 21 ++++++++- crates/raw/src/vendor/orf.rs | 20 +-------- crates/raw/src/vendor/pef.rs | 84 +++++++++++++++++++++++++++++++++++- 5 files changed, 132 insertions(+), 22 deletions(-) diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rs index a4d2cdb7..5b892250 100644 --- a/crates/engine/src/media.rs +++ b/crates/engine/src/media.rs @@ -62,7 +62,7 @@ impl SettingsHashes { } /// Bump when the pipeline's output changes, to invalidate cached thumbnails. -pub const RENDER_CACHE_VERSION: u64 = 16; +pub const RENDER_CACHE_VERSION: u64 = 17; /// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache). pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512]; diff --git a/crates/raw/src/lib.rs b/crates/raw/src/lib.rs index d38b2fc6..3ea36e50 100644 --- a/crates/raw/src/lib.rs +++ b/crates/raw/src/lib.rs @@ -165,6 +165,10 @@ pub fn probe(bytes: &[u8]) -> Option { if make.starts_with("PENTAX") || make.starts_with("RICOH") { return Some(RawFormat::Pef); } + // a Samsung-branded body built on a Pentax design writes a Pentax-style maker note (the note's own magic) + if make.starts_with("SAMSUNG") && has_pentax_maker_note(&t, bytes) { + return Some(RawFormat::Pef); + } if make.starts_with("SAMSUNG") { return Some(RawFormat::Srw); } @@ -174,6 +178,12 @@ pub fn probe(bytes: &[u8]) -> Option { None } +/// Whether the Exif maker note starts with the signature of the Pentax layouts (`AOC\0` or `PENTAX \0`). +fn has_pentax_maker_note(t: &Tiff, bytes: &[u8]) -> bool { + let Some(e) = t.exif().and_then(|e| e.get(lightcraft_tiff::tags::MAKER_NOTE)) else { return false }; + bytes.get(e.offset as usize..).is_some_and(|n| n.starts_with(b"AOC\0") || n.starts_with(b"PENTAX \0")) +} + /// Whether some IFD is marked as raw: CFA photometric, or a raw-only compression value (99 is not a /// registered TIFF compression; Leaf MOS files use it for their tiled 16-bit lossless-JPEG raw). fn has_raw_ifd(t: &Tiff) -> bool { @@ -919,6 +929,23 @@ mod tests { assert_eq!(probe(&write(&[ifd])), Some(RawFormat::OtherTiff)); } + /// A Samsung-branded body whose maker note has a Pentax layout is read by the Pentax reader; another + /// Samsung file stays with the Samsung format. + #[test] + fn samsung_make_with_a_pentax_maker_note_is_pef() { + let with_note = |note: &[u8]| { + let mut ifd = rgb_ifd(16, 12); + ifd.set(t::MAKE, Value::Ascii("SAMSUNG TECHWIN".into())); + let mut exif = IfdBuilder::new(); + exif.set(t::MAKER_NOTE, Value::Undefined(note.to_vec())); + ifd.set_child(t::EXIF_IFD, exif); + write(&[ifd]) + }; + assert_eq!(probe(&with_note(b"AOC\0MM\0\x01\0\0\0\0\0\0")), Some(RawFormat::Pef)); + assert_eq!(probe(&with_note(b"PENTAX \0MM\0\0\0\0\0\0\0")), Some(RawFormat::Pef)); + assert_eq!(probe(&with_note(b"STMN100\0\0\0\0\0\0\0\0\0")), Some(RawFormat::Srw)); + } + #[test] fn probe_rejects_junk() { assert_eq!(probe(b""), None); diff --git a/crates/raw/src/vendor/mod.rs b/crates/raw/src/vendor/mod.rs index fbbc18eb..02165582 100644 --- a/crates/raw/src/vendor/mod.rs +++ b/crates/raw/src/vendor/mod.rs @@ -13,9 +13,28 @@ pub mod raf; mod rafc; pub mod rw2; -use crate::{BlackLevel, Rect}; +use crate::{BlackLevel, Cfa, Rect}; +use lightcraft_tiff::Tiff; use std::ops::Range; +/// Exif `CFAPattern`. +pub(crate) const EXIF_CFA_PATTERN: u16 = 0xa302; + +/// The colour-filter layout from the Exif `CFAPattern` tag (`0xa302`: two 16-bit repeat counts, found in either +/// byte order, then one byte per site: 0 = red, 1 = green, 2 = blue), when it describes a 2×2 Bayer cell. +pub(crate) fn cfa_from_exif(tiff: &Tiff) -> Option { + let &[c0, c1, r0, r1, s0, s1, s2, s3] = tiff.exif()?.bytes(EXIF_CFA_PATTERN)? else { return None }; + let two = |a: u8, b: u8| matches!((a, b), (2, 0) | (0, 2)); + let name = match [s0, s1, s2, s3] { + [0, 1, 1, 2] => "RGGB", + [2, 1, 1, 0] => "BGGR", + [1, 0, 2, 1] => "GRBG", + [1, 2, 0, 1] => "GBRG", + _ => return None, + }; + (two(c0, c1) && two(r0, r1)).then(|| Cfa::bayer_static(name)) +} + /// Black level per 2×2 CFA position (anchored at the active area origin) from masked sensor columns `cols` over /// rows `rows`. Falls back to 0 when the region is empty. pub(crate) fn black_from_columns(data: &[u16], width: usize, cols: Range, rows: Range, active: Rect) -> BlackLevel { diff --git a/crates/raw/src/vendor/orf.rs b/crates/raw/src/vendor/orf.rs index 59c5f14c..893ee64a 100644 --- a/crates/raw/src/vendor/orf.rs +++ b/crates/raw/src/vendor/orf.rs @@ -18,7 +18,7 @@ //! pattern is anchored at the sensor origin (assumed) or at the active area. Files without a usable tag fall back //! to the green diagonal found from the data, which can't tell red from blue (GRBG or RGGB is assumed). -use super::white_from_data; +use super::{cfa_from_exif, white_from_data}; use crate::tiffraw::{Packing, read_image}; use crate::unpack::unpack_msb; use crate::{BlackLevel, Cfa, ColorData, Mode, OpcodeLists, RawData, RawError, RawFormat, RawImage, Rect, Result}; @@ -35,8 +35,6 @@ pub(crate) const PREVIEW_LENGTH: u16 = 0x0102; const WB_RB: u16 = 0x0100; const BLACK: u16 = 0x0600; const CROP: [u16; 4] = [0x0612, 0x0613, 0x0614, 0x0615]; -/// Exif `CFAPattern`. -const EXIF_CFA_PATTERN: u16 = 0xa302; /// The Olympus maker note. pub(crate) fn maker_note(bytes: &[u8], tiff: &Tiff) -> Option { @@ -70,21 +68,6 @@ pub(crate) fn unpack_row_le32_msb(src: &[u8], bits: u32, out: &mut [u16]) { unpack_msb(&swapped, bits, out); } -/// The colour-filter layout from the Exif `CFAPattern` tag (`0xa302`: two 16-bit repeat counts, found in either -/// byte order, then one byte per site: 0 = red, 1 = green, 2 = blue), when it describes a 2×2 Bayer cell. -fn cfa_from_exif(tiff: &Tiff) -> Option { - let &[c0, c1, r0, r1, s0, s1, s2, s3] = tiff.exif()?.bytes(EXIF_CFA_PATTERN)? else { return None }; - let two = |a: u8, b: u8| matches!((a, b), (2, 0) | (0, 2)); - let name = match [s0, s1, s2, s3] { - [0, 1, 1, 2] => "RGGB", - [2, 1, 1, 0] => "BGGR", - [1, 0, 2, 1] => "GRBG", - [1, 2, 0, 1] => "GBRG", - _ => return None, - }; - (two(c0, c1) && two(r0, r1)).then(|| Cfa::bayer_static(name)) -} - /// The layout found from the samples, for files without the Exif tag: GRBG when the greens sit on the main /// diagonal of the 2×2 cell at the sensor origin, else RGGB. Over 32×32-pixel blocks of `a` (every fourth in both /// directions) it compares the block totals of the two sites on each diagonal: the two greens of a block see the @@ -223,6 +206,7 @@ pub(crate) fn preview(bytes: &[u8]) -> Option<&[u8]> { #[cfg(test)] mod tests { use super::*; + use crate::vendor::EXIF_CFA_PATTERN; use lightcraft_tiff::{ByteOrder, IfdBuilder, ImageData, TiffWriter}; fn orf(w: u32, h: u32, bits: u16, strip: Vec) -> Vec { diff --git a/crates/raw/src/vendor/pef.rs b/crates/raw/src/vendor/pef.rs index c9421d3b..5b80967e 100644 --- a/crates/raw/src/vendor/pef.rs +++ b/crates/raw/src/vendor/pef.rs @@ -15,12 +15,12 @@ //! (starting at 0). Found by trying candidate predictors and checking the decoded rows are continuous. //! - Maker note `0x0038`/`0x0039`: the image area's left/top and width/height (verified against the data). -use super::{black_from_columns, white_from_data}; +use super::{black_from_columns, cfa_from_exif, white_from_data}; use crate::tiffraw::{Packing, read_image_in}; use crate::{BlackLevel, Cfa, ColorData, Mode, OpcodeLists, RawData, RawError, RawFormat, RawImage, Rect, Result}; use lightcraft_geom::Orientation; -use lightcraft_tiff::image::chunk_bytes; +use lightcraft_tiff::image::{ImageInfo, chunk_bytes}; use lightcraft_tiff::{ByteOrder, Tiff, makernote, tags as t}; const CROP_ORIGIN: u16 = 0x0038; @@ -188,6 +188,19 @@ fn dark_trimmed(d: &[u16], w: usize, h: usize) -> Rect { Rect::new(left, 0, (right - left) & !1, h) } +/// Whether every strip or tile of a one-sample-per-pixel image holds exactly the bytes that its rows need when +/// the samples are packed MSB-first at the declared bit depth, rows starting on byte boundaries (TIFF 6.0 +/// section 7). A real PackBits stream of such an image has a different length (almost always shorter), so the +/// exact match is what tells plain samples under a compression tag that says otherwise. +fn is_exactly_packed(info: &ImageInfo, file_len: u64) -> bool { + let bits = u64::from(info.bits()); + if info.samples_per_pixel != 1 || info.planar != 1 || !(1..=16).contains(&bits) || info.byte_counts.len() != info.offsets.len() { + return false; + } + let chunks = info.chunks(file_len); + !chunks.is_empty() && chunks.iter().all(|c| c.len == (u64::from(c.width) * bits).div_ceil(8) * u64::from(c.height)) +} + pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result { let tiff = Tiff::parse(bytes)?; let ifd0 = &tiff.ifds[0]; @@ -226,6 +239,12 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result { let packing = if strip >= (w * hgt * 2) as u64 { Packing::Word16 } else { Packing::Msb }; read_image_in(read, bytes, &info, tiff.order, packing)? } + // Some bodies tag plain packed samples as PackBits (32773). Only a strip that holds exactly the packed + // size of its rows is read, as the uncompressed MSB-first samples it is; any other size stays unsupported. + 32773 if is_exactly_packed(&info, bytes.len() as u64) => { + let packed = ImageInfo { compression: 1, ..info.clone() }; + read_image_in(read, bytes, &packed, tiff.order, Packing::Msb)? + } c => return Err(RawError::Unsupported(format!("PEF compression {c}"))), }; let RawData::U16(ref samples) = data else { return Err(RawError::Unsupported("float PEF".into())) }; @@ -236,6 +255,12 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result { }; let cfa = match (ifd0.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), ifd0.bytes(t::CFA_PATTERN_EP)) { (Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => Cfa { width: 2, height: 2, pattern: p.to_vec() }, + // The uncompressed bodies state their pattern in the Exif `CFAPattern`, as seen from the first pixel of the + // image area (checked against each file's camera JPEG); the Huffman-compressed ones don't agree with their + // pictures (a 10 MP body says RGGB and renders correctly as BGGR), so they keep the assumed layout. + _ if matches!(info.compression, 1 | 32773) => { + cfa_from_exif(&tiff).map(|c| c.shifted(active.x, active.y)).unwrap_or_else(|| Cfa::bayer_static("BGGR")) + } _ => Cfa::bayer_static("BGGR"), }; let black = match mn.as_ref().and_then(|m| m.ifd.f64s(BLACK_POINT)).as_deref() { @@ -343,4 +368,59 @@ mod tests { assert!(decode_huffman(&src[..src.len() / 3], &h, w, hgt, 14).is_err()); assert!(table(&tb[..20], ByteOrder::Big).is_err()); } + + /// A big-endian one-strip PEF-style TIFF of `w × h` 12-bit samples with the given compression tag and strip + /// byte count, around `payload`. + fn tiny_pef(w: u16, h: u16, compression: u16, strip_len: u32, payload: &[u8]) -> Vec { + let entries: [(u16, u16, u32); 9] = [ + (0x0100, 3, w as u32 * 0x1_0000), // ImageWidth + (0x0101, 3, h as u32 * 0x1_0000), // ImageLength + (0x0102, 3, 12 * 0x1_0000), // BitsPerSample + (0x0103, 3, compression as u32 * 0x1_0000), // Compression + (0x0106, 3, 32803 * 0x1_0000), // Photometric: CFA + (0x0111, 4, 8 + 2 + 9 * 12 + 4), // StripOffsets + (0x0115, 3, 0x1_0000), // SamplesPerPixel + (0x0116, 3, h as u32 * 0x1_0000), // RowsPerStrip + (0x0117, 4, strip_len), // StripByteCounts + ]; + let mut f = b"MM\0*\0\0\0\x08".to_vec(); + f.extend_from_slice(&9u16.to_be_bytes()); + for (tag, ty, val) in entries { + f.extend_from_slice(&tag.to_be_bytes()); + f.extend_from_slice(&ty.to_be_bytes()); + f.extend_from_slice(&1u32.to_be_bytes()); + f.extend_from_slice(&val.to_be_bytes()); + } + f.extend_from_slice(&0u32.to_be_bytes()); // no next IFD + f.extend_from_slice(payload); + f + } + + /// Eight 12-bit samples, two rows of four, packed MSB-first: 0x123 0x456 0x789 0xabc / 0xdef 0x012 0x345 0x678. + const PACKED: [u8; 12] = [0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0, 0x12, 0x34, 0x56, 0x78]; + + #[test] + fn packed_samples_under_the_packbits_tag_are_read_when_the_strip_has_exactly_the_packed_size() { + let file = tiny_pef(4, 2, 32773, 12, &PACKED); + let img = decode(&file, Mode::Full).unwrap(); + let RawData::U16(v) = &img.data else { panic!("integer samples") }; + assert_eq!(v, &[0x123, 0x456, 0x789, 0xabc, 0xdef, 0x012, 0x345, 0x678]); + assert_eq!((img.width, img.height, img.bits), (4, 2, 12)); + // the same bytes tagged as plain uncompressed give the same samples + let plain = decode(&tiny_pef(4, 2, 1, 12, &PACKED), Mode::Full).unwrap(); + assert_eq!(plain.data, img.data); + // header-only mode accepts it too + assert!(decode(&file, Mode::Header).is_ok()); + } + + #[test] + fn a_packbits_tagged_strip_of_any_other_size_stays_unsupported() { + for len in [11u32, 13, 6, 24] { + let mut payload = PACKED.to_vec(); + payload.resize(24, 0x55); + let file = tiny_pef(4, 2, 32773, len, &payload); + let err = decode(&file, Mode::Full).unwrap_err(); + assert!(matches!(&err, RawError::Unsupported(m) if m.contains("32773")), "strip of {len} bytes: {err:?}"); + } + } } From 1206d3fd521e957c14b64245695e78672f0349e5 Mon Sep 17 00:00:00 2001 From: radialmonster Date: Fri, 9 Oct 2026 02:46:06 -0400 Subject: [PATCH 31/46] LR-IMP-FORMATS: decode Canon sRAW and mRAW CR2 from their YCbCr data sRAW and mRAW store subsampled YCbCr in a lossless-JPEG frame cut into slices of whole MCUs. Reuse the subsampled lossless decoder added for Sony M/S ARW with Canon's per-component sequential prediction, re-chop the stream into image planes, and convert to linear white-balanced RGB with chroma gains measured per file against the camera's own IFD2 RGB. Render cache version 17. --- crates/engine/src/media.rs | 2 +- crates/raw/src/ljpeg.rs | 97 +++-- crates/raw/src/vendor/arw.rs | 2 +- crates/raw/src/vendor/cr2.rs | 12 +- crates/raw/src/vendor/cr2/sraw.rs | 580 ++++++++++++++++++++++++++++++ 5 files changed, 661 insertions(+), 32 deletions(-) create mode 100644 crates/raw/src/vendor/cr2/sraw.rs diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rs index b0439390..4519e8f9 100644 --- a/crates/engine/src/media.rs +++ b/crates/engine/src/media.rs @@ -62,7 +62,7 @@ impl SettingsHashes { } /// Bump when the pipeline's output changes, to invalidate cached thumbnails. -pub const RENDER_CACHE_VERSION: u64 = 16; +pub const RENDER_CACHE_VERSION: u64 = 17; /// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache). pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512]; diff --git a/crates/raw/src/ljpeg.rs b/crates/raw/src/ljpeg.rs index c8087a8e..7978f933 100644 --- a/crates/raw/src/ljpeg.rs +++ b/crates/raw/src/ljpeg.rs @@ -369,12 +369,25 @@ fn check_subsampled(h: &Header) -> Result { Ok(vertical) } -/// Sony M/S tiles: horizontal prediction, no restart markers. Keep subsampled planes separate; -/// expanding them into a mosaic would silently corrupt the existing CFA callers. -pub(crate) fn frame_info_subsampled(d: &[u8]) -> Result<(usize, usize), RawError> { +/// Sony M/S tiles and Canon sRAW / mRAW: horizontal prediction, no restart markers. Keep subsampled planes +/// separate; expanding them into a mosaic would silently corrupt the existing CFA callers. +/// Returns (width, height, vertical subsampling of the chroma planes). +pub(crate) fn frame_info_subsampled(d: &[u8]) -> Result<(usize, usize, usize), RawError> { let h = parse_header(d)?; - check_subsampled(&h)?; - Ok((h.width, h.height)) + let vertical = check_subsampled(&h)?; + Ok((h.width, h.height, vertical)) +} + +/// How the first-column prediction and the running predictor of a subsampled frame are chosen. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Prediction { + /// Sony: the left neighbour in the plane; at the first column the sample above, except that a top luma + /// row of an MCU row reads the previous MCU row's top luma sample. + Geometric, + /// Canon: every component is its own one-dimensional chain in decoding order (the previous sample of the + /// same component, so the second luma row of a 4:2:0 MCU continues from the end of the first); the first + /// sample of each MCU row starts from the first sample of the previous MCU row. + Sequential, } pub(crate) struct FrameSubsampled { @@ -385,11 +398,13 @@ pub(crate) struct FrameSubsampled { pub planes: [Vec; 3], } -/// Sony's 4:2:0 / 4:2:2 LJ92 variants: T.81 MCU ordering and differences, modulo 2^16. -/// At the first column, the top luma row predicts from the previous MCU row's top +/// The 4:2:0 / 4:2:2 LJ92 variants of Sony (M/S ARW) and Canon (sRAW / mRAW): T.81 MCU ordering and differences, +/// modulo 2^16. [`Prediction`] selects the predictor each maker uses. +/// +/// Sony: at the first column, the top luma row predicts from the previous MCU row's top /// sample (two image rows above), and the bottom row from the current top sample. /// Observed black-box: using the immediately preceding image row introduces tile seams. -pub(crate) fn decode_subsampled(d: &[u8], max_samples: usize) -> Result { +pub(crate) fn decode_subsampled(d: &[u8], max_samples: usize, prediction: Prediction) -> Result { let h = parse_header(d)?; let vertical = check_subsampled(&h)?; let pixels = h.width.checked_mul(h.height).ok_or_else(|| err("frame too large"))?; @@ -410,6 +425,9 @@ pub(crate) fn decode_subsampled(d: &[u8], max_samples: usize) -> Result Result 0 { - plane[i - 1] as i32 - } else if c == 0 && dy == 0 && y >= vertical { - plane[i - vertical * width] as i32 - } else if y > 0 { - plane[i - width] as i32 - } else { - init + let pred = match prediction { + Prediction::Geometric if x > 0 => plane[i - 1] as i32, + Prediction::Geometric if c == 0 && dy == 0 && y >= vertical => plane[i - vertical * width] as i32, + Prediction::Geometric if y > 0 => plane[i - width] as i32, + Prediction::Geometric => init, + Prediction::Sequential if mx == 0 && dx == 0 && dy == 0 => row_first[c], + Prediction::Sequential => last[c], }; let ssss = tables[c].decode(&mut br)?; if ssss > 16 { return Err(err("invalid difference category")); } - plane[i] = ((pred + diff_value(&mut br, ssss)) & 0xffff) as u16; + let value = ((pred + diff_value(&mut br, ssss)) & 0xffff) as u16; + plane[i] = value; + last[c] = i32::from(value); + if mx == 0 && dx == 0 && dy == 0 { + row_first[c] = i32::from(value); + } } } } @@ -816,11 +838,13 @@ pub(crate) mod tests { /// Independent 4×4, 16-bit Sony 4:2:0 stream. Hand-specified differences in MCU order. pub(crate) fn fixture_420() -> Vec { - fixture_subsampled(4, 0x22, &[-31768, 1, 1000, 1, -16384, -16284, 1, 1, 1, 1, 1, 2, 100, 1, 1000, 1, 100, 100, 1, 1, 1, 1, 1, 2]) + fixture_subsampled(4, 4, 0x22, &[-31768, 1, 1000, 1, -16384, -16284, 1, 1, 1, 1, 1, 2, 100, 1, 1000, 1, 100, 100, 1, 1, 1, 1, 1, 2]) } - fn fixture_subsampled(height: u8, sampling: u8, differences: &[i32]) -> Vec { - let mut out = vec![0xff, 0xd8, 0xff, 0xc3, 0, 17, 16, 0, height, 0, 4, 3, 1, sampling, 0, 2, 0x11, 0, 3, 0x11, 0]; + /// A 16-bit three-component frame of `width × height` pixels with the given luma `sampling` (`0x21` / `0x22`) + /// and one shared 5-bit Huffman table, whose differences are written as given (MCU order). + pub(crate) fn fixture_subsampled(width: u8, height: u8, sampling: u8, differences: &[i32]) -> Vec { + let mut out = vec![0xff, 0xd8, 0xff, 0xc3, 0, 17, 16, 0, height, 0, width, 3, 1, sampling, 0, 2, 0x11, 0, 3, 0x11, 0]; out.extend_from_slice(&[0xff, 0xc4, 0, 36, 0]); let mut counts = [0; 16]; counts[4] = 17; @@ -844,28 +868,28 @@ pub(crate) mod tests { #[test] fn horizontal_only_subsampling_preserves_rows_and_chroma() { // Independent 4×2, 16-bit 4:2:2 stream: two luma samples, Cb, Cr per MCU. - let enc = fixture_subsampled(2, 0x21, &[-31768, 1, -16384, -16284, 1, 1, 1, 2, 100, 1, 100, 100, 1, 1, 1, 2]); - let frame = decode_subsampled(&enc, 16).unwrap(); + let enc = fixture_subsampled(4, 2, 0x21, &[-31768, 1, -16384, -16284, 1, 1, 1, 2, 100, 1, 100, 100, 1, 1, 1, 2]); + let frame = decode_subsampled(&enc, 16, Prediction::Geometric).unwrap(); assert_eq!((frame.width, frame.height, frame.vertical_subsampling), (4, 2, 1)); assert_eq!(frame.planes[0], [1000, 1001, 1002, 1003, 1100, 1101, 1102, 1103]); assert_eq!(frame.planes[1], [16384, 16385, 16484, 16485]); assert_eq!(frame.planes[2], [16484, 16486, 16584, 16586]); assert!(decode(&enc, 24).is_err()); - assert!(matches!(decode_subsampled(&enc, 15), Err(RawError::Limit(_)))); - assert!(decode_subsampled(&enc[..enc.len() - 6], 16).is_err()); + assert!(matches!(decode_subsampled(&enc, 15, Prediction::Geometric), Err(RawError::Limit(_)))); + assert!(decode_subsampled(&enc[..enc.len() - 6], 16, Prediction::Geometric).is_err()); } #[test] fn subsampled_mcu_order_preserves_each_plane() { let enc = fixture_420(); - let frame = decode_subsampled(&enc, 24).unwrap(); + let frame = decode_subsampled(&enc, 24, Prediction::Geometric).unwrap(); assert_eq!((frame.width, frame.height), (4, 4)); assert_eq!(frame.planes[0], [1000, 1001, 1002, 1003, 2000, 2001, 2002, 2003, 1100, 1101, 1102, 1103, 2100, 2101, 2102, 2103]); assert_eq!(frame.planes[1], [16384, 16385, 16484, 16485]); assert_eq!(frame.planes[2], [16484, 16486, 16584, 16586]); assert!(decode(&enc, 48).is_err()); // CFA API must keep rejecting subsampling. - assert!(matches!(decode_subsampled(&enc, 23), Err(RawError::Limit(_)))); - assert!(decode_subsampled(&enc[..enc.len() - 6], 24).is_err()); + assert!(matches!(decode_subsampled(&enc, 23, Prediction::Geometric), Err(RawError::Limit(_)))); + assert!(decode_subsampled(&enc[..enc.len() - 6], 24, Prediction::Geometric).is_err()); let mut bad = enc.clone(); bad[10] = 3; // odd frame width assert!(frame_info_subsampled(&bad).is_err()); @@ -875,6 +899,25 @@ pub(crate) mod tests { assert!(frame_info_subsampled(&bad).is_err()); } + #[test] + fn sequential_prediction_chains_each_component_in_decoding_order() { + // Canon sRAW / mRAW: the same stream as `subsampled_mcu_order_preserves_each_plane`, but each component + // continues from its previous sample in decoding order (the second luma row of an MCU follows the first), + // and a new MCU row starts from the first sample of the previous one. Worked out by hand from init 32768. + let enc = fixture_420(); + let frame = decode_subsampled(&enc, 24, Prediction::Sequential).unwrap(); + assert_eq!((frame.width, frame.height, frame.vertical_subsampling), (4, 4, 2)); + assert_eq!(frame.planes[0], [1000, 1001, 2003, 2004, 2001, 2002, 2005, 2006, 1100, 1101, 2103, 2104, 2101, 2102, 2105, 2106]); + assert_eq!(frame.planes[1], [16384, 16385, 16484, 16485]); + assert_eq!(frame.planes[2], [16484, 16486, 16584, 16586]); + // 4:2:2 has one luma row per MCU row, so both predictions agree + let enc = fixture_subsampled(4, 2, 0x21, &[-31768, 1, -16384, -16284, 1, 1, 1, 2, 100, 1, 100, 100, 1, 1, 1, 2]); + assert_eq!( + decode_subsampled(&enc, 16, Prediction::Sequential).unwrap().planes, + decode_subsampled(&enc, 16, Prediction::Geometric).unwrap().planes + ); + } + fn noise(n: usize, bits: u32, seed: u64) -> Vec { let mut s = seed; (0..n) diff --git a/crates/raw/src/vendor/arw.rs b/crates/raw/src/vendor/arw.rs index acaacf0b..156dfbc3 100644 --- a/crates/raw/src/vendor/arw.rs +++ b/crates/raw/src/vendor/arw.rs @@ -338,7 +338,7 @@ fn read_ycbcr_tiles(bytes: &[u8], info: &ImageInfo, raw: &Ifd, mode: Mode) -> Re .map(|c| { let src = chunk_bytes(bytes, c).ok_or_else(|| RawError::Corrupt("YCbCr tile outside file".into()))?; let limit = (c.width as usize).checked_mul(c.height as usize).and_then(|n| n.checked_mul(3)).ok_or(RawError::Limit("tile too large"))?; - let f = ljpeg::decode_subsampled(src, limit)?; + let f = ljpeg::decode_subsampled(src, limit, ljpeg::Prediction::Geometric)?; if f.width != c.width as usize || f.height != c.height as usize { return Err(RawError::Corrupt("Sony YCbCr tile dimensions".into())); } diff --git a/crates/raw/src/vendor/cr2.rs b/crates/raw/src/vendor/cr2.rs index d6529ece..4c7522da 100644 --- a/crates/raw/src/vendor/cr2.rs +++ b/crates/raw/src/vendor/cr2.rs @@ -17,7 +17,9 @@ //! - `ColorBalance` (maker note `0x4001`): as-shot `RGGB` levels at a model-dependent offset; we probe the known //! offsets and accept the first plausible quadruple. The array is 16-bit words; some models store it as UNDEFINED //! bytes (ColorData versions -3 and -4), which are paired up in the maker note's byte order first. -//! - sRAW / mRAW (YCbCr, subsampled) are not supported yet. +//! - sRAW / mRAW (raw IFD tag `0xc6c5` = 4) hold subsampled YCbCr instead of a mosaic: see [`sraw`]. + +mod sraw; use super::{black_from_columns, white_from_data}; use crate::{BlackLevel, Cfa, ColorData, Mode, OpcodeLists, RawData, RawError, RawFormat, RawImage, Rect, Result, ljpeg}; @@ -27,6 +29,8 @@ use lightcraft_tiff::{ByteOrder, Ifd, Tiff, Value, makernote, tags as t}; const CR2_SLICE: u16 = 0xc640; const SRAW_TYPE: u16 = 0xc6c5; +/// `SRAW_TYPE` value of the YCbCr frames of sRAW / mRAW (IFD2's preview image carries 3). +const SRAW_YCC: u32 = 4; const CR2_CFA_PATTERN: u16 = 0xc5e0; const SENSOR_INFO: u16 = 0x00e0; const COLOR_BALANCE: u16 = 0x4001; @@ -79,8 +83,10 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result { let tiff = Tiff::parse(bytes)?; let ifd0 = &tiff.ifds[0]; let raw = raw_ifd(&tiff).ok_or_else(|| RawError::Corrupt("CR2 without raw IFD".into()))?; - if raw.u32(SRAW_TYPE).is_some_and(|v| v != 1) { - return Err(RawError::Unsupported("Canon sRAW/mRAW".into())); + match raw.u32(SRAW_TYPE) { + Some(SRAW_YCC) => return sraw::decode(bytes, &tiff, raw, mode), + Some(v) if v != 1 => return Err(RawError::Unsupported(format!("Canon raw type {v}"))), + _ => {} } let off = raw.u64(t::STRIP_OFFSETS).ok_or(RawError::Tiff(lightcraft_tiff::TiffError::MissingTag(t::STRIP_OFFSETS)))?; let len = raw.u64(t::STRIP_BYTE_COUNTS).unwrap_or(bytes.len() as u64 - off.min(bytes.len() as u64)); diff --git a/crates/raw/src/vendor/cr2/sraw.rs b/crates/raw/src/vendor/cr2/sraw.rs new file mode 100644 index 00000000..df17d35b --- /dev/null +++ b/crates/raw/src/vendor/cr2/sraw.rs @@ -0,0 +1,580 @@ +//! Canon sRAW / mRAW: reduced-resolution YCbCr "raw" frames in CR2 (raw IFD tag `0xc6c5` = 4). +//! +//! What the files contain (measured on the 31 CC0 sRAW / mRAW samples of 16 models, see the evidence below): +//! +//! - The raw strip is one lossless-JPEG frame with three components. Luma is sampled 2×1 (sRAW, 4:2:2, `0x21`) or +//! 2×2 (mRAW, 4:2:0, `0x22`), both chroma components 1×1; precision 15, predictor 1, no restart markers. +//! The frame's width and height are only a factorisation of the sample count: they are not the image size. +//! - The decoded sample stream (MCU after MCU: `Y Y Cb Cr` or `Y Y Y Y Cb Cr`) is cut into vertical slices exactly +//! like the CFA data (`cr2_slice = [n, w, last]`, widths counted in samples). A slice row is a whole number of +//! MCUs, so the image is `Σw / MCU-size × 2` pixels wide, and one slice row is one pixel row (4:2:2) or two (4:2:0). +//! - Prediction: each component is a one-dimensional chain in decoding order (so the second luma row of a 4:2:0 MCU +//! continues from the end of the first), and the first sample of every frame row starts from the first sample of +//! the previous one ([`ljpeg::Prediction::Sequential`]). The Sony prediction (geometric) gives vertical stripes on +//! these files. +//! - The samples are linear. Luma is the BT.601 luma (0.299, 0.587, 0.114; also the TIFF 6.0 default +//! `YCbCrCoefficients`) of white-balanced camera RGB, about 7/8 of the black-subtracted sensor counts; chroma is +//! centred on 16384. White balance is baked in (neutral surfaces have neutral chroma); which of the maker note's +//! white-balance sets was used is not always the as-shot one, so it is not undone. +//! - Chroma is stored as a scaled colour difference, `R − Y = a·(Cr − 16384)` and `B − Y = b·(Cb − 16384)`, with +//! `G` from the luma equation. `a` and `b` are about 1.0 on bodies from 2012 on, but about 5.7 and 7.4 (the TIFF +//! 6.0 YCbCr ratio 1.402 : 1.772, scaled by about 4.1) on the 2008–2010 bodies (5D Mark II, 50D, 7D, 60D, 1D Mark IV). +//! Rather than keying a table on the model, the gains are measured per file against the 16-bit RGB image the +//! camera stores in IFD2 for these files (see [`chroma_gains`]). +//! +//! Only these files were used as evidence: the sRAW samples themselves, the same cameras' CFA files of the same +//! scenes (raw counts against the decoded samples), and the IFD2 image. The ExifTool tag documentation was used for +//! tag names. + +use super::{COLOR_BALANCE, CR2_SLICE, SENSOR_INFO, words}; +use crate::{BlackLevel, ColorData, Mode, OpcodeLists, RawData, RawError, RawFormat, RawImage, Rect, Result, ljpeg}; +use lightcraft_geom::Orientation; +use lightcraft_tiff::image::{Chunk, chunk_bytes}; +use lightcraft_tiff::{ByteOrder, Ifd, Tiff, makernote, tags as t}; +use rayon::prelude::*; + +/// Chroma samples are centred on this value. +const CHROMA_CENTRE: f64 = 16384.0; +/// BT.601 luma weights, the TIFF 6.0 default `YCbCrCoefficients`. +const KR: f64 = 0.299; +const KG: f64 = 0.587; +const KB: f64 = 0.114; +/// Decoded luma relative to black-subtracted sensor counts (white-balanced): 0.877, 0.877, 0.896, 0.880, 0.871, 0.851 +/// and 0.846 on the 7D Mark II, 5DS R, 6D Mark II, 5DS, 5D Mark II, 1D X Mark II and 50D (CFA and sRAW files of the +/// same scene), so 7/8. It sets the white level, so that a file renders about as bright as the CFA file of the same +/// scene. +const LUMA_GAIN: f32 = 0.875; +/// Full scale of a 14-bit sensor, the white level of a CFA CR2 without a saturation plateau. +const SENSOR_FULL: f32 = 16383.0; +/// Mid-tone limit (IFD2 units) of the pixels used for the chroma fit: above about a quarter of full scale the +/// camera's own RGB image rolls off. +const FIT_LIMIT: f64 = 4096.0; + +/// How the decoded frame maps to the image. +#[derive(Debug, PartialEq)] +pub(super) struct Layout { + pub width: usize, + pub height: usize, + /// Vertical subsampling of the chroma planes: 1 (4:2:2, sRAW) or 2 (4:2:0, mRAW). + pub vertical: usize, + /// Slice widths in samples. + widths: Vec, + /// Slice height in MCU rows. + rows: usize, + /// Samples per MCU. + mcu: usize, + /// Frame size from the SOF3 header. + frame_width: usize, + frame_height: usize, +} + +impl Layout { + /// The layout of a `frame_width × frame_height` frame with `vertical` chroma subsampling and `cr2_slice` tag + /// values `slices` (`[n, w, last]`). + pub(super) fn new(frame_width: usize, frame_height: usize, vertical: usize, slices: &[u64]) -> Result { + let &[n, w, last] = slices else { return Err(RawError::Corrupt("sRAW without slice layout".into())) }; + if n > 64 || w == 0 || last == 0 { + return Err(RawError::Corrupt("sRAW slice layout".into())); + } + let mcu = 2 * vertical + 2; + let mut widths = vec![w as usize; n as usize]; + widths.push(last as usize); + if widths.iter().any(|w| w % mcu != 0) { + return Err(RawError::Corrupt("sRAW slice is not a whole number of MCUs".into())); + } + let samples = + frame_width.checked_mul(frame_height).and_then(|p| p.checked_add(p / vertical)).ok_or(RawError::Limit("sRAW frame too large"))?; + let row: usize = widths.iter().sum(); + if samples == 0 || samples % row != 0 { + return Err(RawError::Corrupt(format!("sRAW slices ({row} samples per row) do not divide the frame ({samples} samples)"))); + } + let rows = samples / row; + let (width, height) = (row / mcu * 2, rows * vertical); + if width.checked_mul(height).and_then(|p| p.checked_mul(3)).is_none_or(|s| s > crate::MAX_SAMPLES) { + return Err(RawError::Limit("sRAW image too large")); + } + Ok(Layout { width, height, vertical, widths, rows, mcu, frame_width, frame_height }) + } +} + +/// Planar YCbCr at image geometry: luma `width × height`, chroma `width/2 × height/vertical`. +struct Planes { + width: usize, + height: usize, + vertical: usize, + y: Vec, + cb: Vec, + cr: Vec, +} + +impl Planes { + fn chroma_width(&self) -> usize { + self.width / 2 + } + + /// Re-chop the decoded MCU stream into image planes. + fn arrange(frame: &ljpeg::FrameSubsampled, layout: &Layout) -> Planes { + let (w, h, v, m) = (layout.width, layout.height, layout.vertical, layout.mcu); + let cw = w / 2; + let mut p = Planes { width: w, height: h, vertical: v, y: vec![0; w * h], cb: vec![0; cw * (h / v)], cr: vec![0; cw * (h / v)] }; + let mcus_per_frame_row = layout.frame_width / 2; + let mut x0 = 0; + for &sw in &layout.widths { + let per_row = sw / m; + let first = layout.rows * (x0 / m); + for r in 0..layout.rows { + for k in 0..per_row { + let q = first + r * per_row + k; + let (qy, qx) = (q / mcus_per_frame_row, q % mcus_per_frame_row); + let px = (x0 / m + k) * 2; + for dy in 0..v { + let src = (qy * v + dy) * layout.frame_width + qx * 2; + let dst = (r * v + dy) * w + px; + p.y[dst..dst + 2].copy_from_slice(&frame.planes[0][src..src + 2]); + } + p.cb[r * cw + px / 2] = frame.planes[1][qy * mcus_per_frame_row + qx]; + p.cr[r * cw + px / 2] = frame.planes[2][qy * mcus_per_frame_row + qx]; + } + } + x0 += sw; + } + p + } + + /// Mean luma / Cb / Cr over the pixel window `[x0, x1) × [y0, y1)` (non-empty, inside the image). + fn window_mean(&self, x0: usize, x1: usize, y0: usize, y1: usize) -> (f64, f64, f64) { + let (mut sy, mut ny) = (0f64, 0f64); + for y in y0..y1 { + let row = &self.y[y * self.width + x0..y * self.width + x1]; + sy += row.iter().map(|&v| f64::from(v)).sum::(); + ny += row.len() as f64; + } + let (cx0, cx1) = (x0 / 2, x1.div_ceil(2).max(x0 / 2 + 1)); + let (cy0, cy1) = (y0 / self.vertical, y1.div_ceil(self.vertical).max(y0 / self.vertical + 1)); + let (mut sb, mut sr, mut nc) = (0f64, 0f64, 0f64); + for y in cy0..cy1.min(self.height / self.vertical) { + for x in cx0..cx1.min(self.chroma_width()) { + sb += f64::from(self.cb[y * self.chroma_width() + x]); + sr += f64::from(self.cr[y * self.chroma_width() + x]); + nc += 1.0; + } + } + let nc = nc.max(1.0); + (sy / ny.max(1.0), sb / nc - CHROMA_CENTRE, sr / nc - CHROMA_CENTRE) + } + + /// Linear RGB (`width × height × 3`): `R = Y + a·Cr`, `B = Y + b·Cb`, `G` from the luma equation. + fn to_rgb(&self, a: f64, b: f64) -> Vec { + let (gr, gb) = ((KR / KG * a) as f32, (KB / KG * b) as f32); + let (a, b) = (a as f32, b as f32); + let cw = self.chroma_width(); + let centre = CHROMA_CENTRE as f32; + let mut out = vec![0u16; self.width * self.height * 3]; + out.par_chunks_mut(self.width * 3).enumerate().for_each(|(y, row)| { + let luma = &self.y[y * self.width..(y + 1) * self.width]; + let chroma = (y / self.vertical) * cw; + for (x, px) in row.as_chunks_mut::<3>().0.iter_mut().enumerate() { + let (cb, cr) = (f32::from(self.cb[chroma + x / 2]) - centre, f32::from(self.cr[chroma + x / 2]) - centre); + let l = f32::from(luma[x]); + px[0] = (l + a * cr).round().clamp(0.0, 65535.0) as u16; + px[1] = (l - gr * cr - gb * cb).round().clamp(0.0, 65535.0) as u16; + px[2] = (l + b * cb).round().clamp(0.0, 65535.0) as u16; + } + }); + out + } +} + +/// The 16-bit RGB image in IFD2 (`SRawType` 3): the camera's own rendering of the same frame at a fraction of the +/// size, white-balanced and linear below about a quarter of full scale. +struct Preview { + width: usize, + height: usize, + rgb: Vec, +} + +fn read_preview(bytes: &[u8], tiff: &Tiff) -> Option { + let info = tiff.ifds.get(2)?.image().ok()?; + let (w, h) = (info.width as usize, info.height as usize); + if info.samples_per_pixel != 3 || info.bits_per_sample != [16, 16, 16] || info.compression != 1 || info.planar != 1 || info.sample_format != 1 { + return None; + } + let n = w.checked_mul(h)?.checked_mul(3)?; + if n == 0 || n > 1 << 24 { + return None; + } + let mut raw: Vec = Vec::with_capacity(n * 2); + for c in info.chunks(bytes.len() as u64) { + raw.extend_from_slice(chunk_bytes(bytes, &c)?); + if raw.len() >= n * 2 { + break; + } + } + if raw.len() < n * 2 { + return None; + } + let order: ByteOrder = tiff.order; + let rgb = raw.as_chunks::<2>().0.iter().take(n).map(|&c| order.u16(c)).collect(); + Some(Preview { width: w, height: h, rgb }) +} + +#[derive(Default)] +struct Regression { + n: f64, + sx: f64, + st: f64, + sxx: f64, + sxt: f64, + stt: f64, +} + +impl Regression { + fn add(&mut self, x: f64, t: f64) { + self.n += 1.0; + self.sx += x; + self.st += t; + self.sxx += x * x; + self.sxt += x * t; + self.stt += t * t; + } + + /// Least-squares slope of `t` on `x` (with intercept) and its residual sum of squares. + fn slope(&self) -> Option<(f64, f64)> { + let var = self.n * self.sxx - self.sx * self.sx; + if self.n < 500.0 || var <= 1e-9 * self.n * self.n { + return None; + } + let slope = (self.n * self.sxt - self.sx * self.st) / var; + let icpt = (self.st - slope * self.sx) / self.n; + let rss = + self.stt - 2.0 * slope * self.sxt - 2.0 * icpt * self.st + slope * slope * self.sxx + 2.0 * slope * icpt * self.sx + self.n * icpt * icpt; + Some((slope, rss.max(0.0))) + } +} + +/// How the IFD2 image sits on the full-size planes: its pixel `(i, j)` covers the window starting at +/// `(x0 + i·sx, y0 + j·sy)` and `sx × sy` pixels large. +struct Grid { + x0: f64, + y0: f64, + sx: f64, + sy: f64, +} + +/// The chroma gains `(a, b)` of this file (`R − Y = a·Cr`, `B − Y = b·Cb`), by least squares of the IFD2 image +/// against the planes averaged to its size. Where IFD2 sits on the frame depends on the model (the whole frame, +/// the sensor's active area, or a whole or half-integer reduction of the frame that overhangs it by a few pixels), +/// so these are tried and the best fit is kept. `None` when the file has no usable IFD2 or the fit is not well +/// conditioned. +fn chroma_gains(planes: &Planes, preview: &Preview, active: Rect) -> Option<(f64, f64)> { + let (pw, ph) = (preview.width as f64, preview.height as f64); + let (w, h) = (planes.width as f64, planes.height as f64); + let grids = [ + Grid { x0: 0.0, y0: 0.0, sx: w / pw, sy: h / ph }, + Grid { x0: active.x as f64, y0: active.y as f64, sx: active.width as f64 / pw, sy: active.height as f64 / ph }, + Grid { x0: 0.0, y0: 0.0, sx: (2.0 * w / pw).round() / 2.0, sy: (2.0 * h / ph).round() / 2.0 }, + ]; + let mut best: Option<(f64, f64, f64)> = None; + for g in &grids { + if !(g.sx >= 1.0 && g.sy >= 1.0) { + continue; + } + let (mut fr, mut fb) = (Regression::default(), Regression::default()); + for j in 0..preview.height { + let y0 = (g.y0 + j as f64 * g.sy) as usize; + let y1 = ((g.y0 + (j + 1) as f64 * g.sy) as usize).min(planes.height); + for i in 0..preview.width { + let p = &preview.rgb[(j * preview.width + i) * 3..][..3]; + let [r, gr, b] = [f64::from(p[0]), f64::from(p[1]), f64::from(p[2])]; + if r.max(gr).max(b) >= FIT_LIMIT || r.min(gr).min(b) <= 0.0 { + continue; + } + let x0 = (g.x0 + i as f64 * g.sx) as usize; + let x1 = ((g.x0 + (i + 1) as f64 * g.sx) as usize).min(planes.width); + if x1 <= x0 || y1 <= y0 { + continue; + } + let (y, cb, cr) = planes.window_mean(x0, x1, y0, y1); + if y < 150.0 { + continue; + } + fr.add(cr, r - y); + fb.add(cb, b - y); + } + } + if let (Some((a, ra)), Some((b, rb))) = (fr.slope(), fb.slope()) { + let cost = (ra + rb) / (fr.n + fb.n); + if a.is_finite() && b.is_finite() && best.is_none_or(|(_, _, c)| cost < c) { + best = Some((a, b, cost)); + } + } + } + best.map(|(a, b, _)| (a, b)).filter(|(a, b)| (0.25..=16.0).contains(a) && (0.25..=16.0).contains(b)) +} + +/// Word index of `AverageBlackLevel` (four equal-ish values) in the maker note's `ColorData` by `ColorDataVersion` +/// (word 0), located by searching the CC0 samples for the values ExifTool reports. +fn average_black_at(version: i16) -> Option { + Some(match version { + 3 => 163, + 6 | 7 | 9 => 231, + 10 | 11 => 276, + 12 | 13 | 15 => 326, + _ => return None, + }) +} + +/// The sensor's black level in raw counts and the black pedestal carried by the luma samples, from `ColorData`. +/// The pedestal is 0 on every sample but the 40D's (`ColorDataVersion` 3), whose luma floor is 512 in the file (the +/// darkest pixels read 524–544, and the maker note repeats 512 in words 663–673). +fn colour_data_levels(words: &[u64]) -> (Option, f32) { + let version = words.first().map_or(0, |&v| v as u16 as i16); + let pedestal = if version == 3 { 512.0 } else { 0.0 }; + let black = average_black_at(version).and_then(|at| words.get(at..at + 4)).and_then(|q| { + let (lo, hi) = (q.iter().min()?, q.iter().max()?); + ((256..=4096).contains(lo) && hi - lo <= 8).then(|| q.iter().sum::() as f32 / 4.0) + }); + (black, pedestal) +} + +/// Decode a Canon sRAW / mRAW CR2 into linear white-balanced camera RGB (`cpp` 3). +pub(super) fn decode(bytes: &[u8], tiff: &Tiff, raw: &Ifd, mode: Mode) -> Result { + let ifd0 = &tiff.ifds[0]; + let off = raw.u64(t::STRIP_OFFSETS).ok_or(RawError::Tiff(lightcraft_tiff::TiffError::MissingTag(t::STRIP_OFFSETS)))?; + let len = raw.u64(t::STRIP_BYTE_COUNTS).unwrap_or(bytes.len() as u64 - off.min(bytes.len() as u64)); + let chunk = Chunk { index: 0, x: 0, y: 0, width: 0, height: 0, plane: 0, offset: off, len }; + let src = chunk_bytes(bytes, &chunk).ok_or_else(|| RawError::Corrupt("raw strip outside file".into()))?; + let (fw, fh, vertical) = ljpeg::frame_info_subsampled(src)?; + let slices = raw.u64s(CR2_SLICE).unwrap_or_default(); + let layout = Layout::new(fw, fh, vertical, &slices)?; + + let make = ifd0.string(t::MAKE).unwrap_or_default(); + let mn = + tiff.exif().and_then(|e| e.get(t::MAKER_NOTE)).and_then(|e| makernote::parse_makernote(bytes, e.offset, e.count() as u64, tiff.order, &make)); + let (w, h) = (layout.width, layout.height); + let mut active = Rect::new(0, 0, w, h); + if let Some(si) = mn.as_ref().and_then(|m| m.ifd.u64s(SENSOR_INFO)).filter(|v| v.len() >= 9) { + let (l, tp, r, b) = (si[5] as usize, si[6] as usize, si[7] as usize, si[8] as usize); + if r > l && b > tp && r < w && b < h { + active = Rect::new(l, tp, r - l + 1, b - tp + 1); + } + } + let colour_data = mn.as_ref().and_then(|m| m.ifd.value(COLOR_BALANCE).map(|v| words(v, m.order))); + let (black_counts, pedestal) = colour_data.as_deref().map_or((None, 0.0), colour_data_levels); + // 1.0 = a CFA CR2 of the same body at full 14-bit scale; see LUMA_GAIN + let white = pedestal + LUMA_GAIN * (SENSOR_FULL - black_counts.unwrap_or(0.0)); + + let data = match mode { + Mode::Header => Vec::new(), + Mode::Full => { + let samples = fw * fh + fw * fh / vertical; + let frame = ljpeg::decode_subsampled(src, samples.min(crate::MAX_SAMPLES), ljpeg::Prediction::Sequential)?; + let planes = Planes::arrange(&frame, &layout); + drop(frame); + let (a, b) = read_preview(bytes, tiff).and_then(|p| chroma_gains(&planes, &p, active)).unwrap_or((1.0, 1.0)); + planes.to_rgb(a, b) + } + }; + + let mut metadata = lightcraft_meta::from_tiff(tiff); + metadata.width = Some(active.width as u32); + metadata.height = Some(active.height as u32); + let img = RawImage { + format: RawFormat::Cr2, + width: w, + height: h, + cpp: 3, + data: RawData::U16(data), + cfa: None, + bits: 14, + black: BlackLevel::uniform(pedestal), + white: vec![white], + active_area: active, + crop: Rect::new(0, 0, active.width, active.height), + orientation: Orientation::from_exif(ifd0.u16(t::ORIENTATION).unwrap_or(1)), + color: ColorData::default(), + // the camera's white balance is already applied, as in Sony's linear YCbCr + wb_multipliers: Some([1.0; 3]), + linearized: false, + opcodes: OpcodeLists::default(), + metadata, + }; + img.validate_for(mode)?; + Ok(img) +} + +#[cfg(test)] +mod tests { + use super::*; + use lightcraft_tiff::{IfdBuilder, ImageData, TiffWriter, Value}; + + const SRAW_TYPE_TAG: u16 = 0xc6c5; + + /// The 4:2:2 stream of `ljpeg`'s hand-coded fixture (4 × 2 frame, two MCUs per frame row) decodes to luma + /// 1000 1001 / 1002 1003 / 1100 1101 / 1102 1103 in MCU order, Cb 16384 16385 16484 16485, Cr 16484 16486 + /// 16584 16586. + fn stream_422() -> Vec { + ljpeg::tests::fixture_subsampled(4, 2, 0x21, &[-31768, 1, -16384, -16284, 1, 1, 1, 2, 100, 1, 100, 100, 1, 1, 1, 2]) + } + + /// A CR2 whose raw IFD holds `stream` cut into the slices `slices` (IFD2 = a 2 × 2 16-bit RGB image when + /// `preview` is given). + fn sraw_file(stream: Vec, slices: [u16; 3], sraw_type: u32, preview: Option<[u16; 12]>) -> Vec { + let mut ifd0 = IfdBuilder::new(); + ifd0.set(t::MAKE, Value::Ascii("Canon".into())); + ifd0.set(t::MODEL, Value::Ascii("Canon EOS Test".into())); + let mut ifd2 = IfdBuilder::new(); + if let Some(p) = preview { + ifd2.set(t::COMPRESSION, Value::Short(vec![1])); + ifd2.set(t::PHOTOMETRIC, Value::Short(vec![2])); + ifd2.set(t::IMAGE_WIDTH, Value::Long(vec![2])); + ifd2.set(t::IMAGE_LENGTH, Value::Long(vec![2])); + ifd2.set(t::BITS_PER_SAMPLE, Value::Short(vec![16, 16, 16])); + ifd2.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![3])); + ifd2.set_image(ImageData::Strips { rows_per_strip: 2, strips: vec![p.iter().flat_map(|v| v.to_le_bytes()).collect()] }); + } else { + ifd2.set(1, Value::Short(vec![0])); + } + let mut ifd3 = IfdBuilder::new(); + ifd3.set(t::COMPRESSION, Value::Short(vec![6])); + ifd3.set(SRAW_TYPE_TAG, Value::Long(vec![sraw_type])); + ifd3.set(CR2_SLICE, Value::Short(slices.to_vec())); + ifd3.set_image(ImageData::Strips { rows_per_strip: 2, strips: vec![stream] }); + TiffWriter::new(ByteOrder::Little, false).write(&[ifd0, IfdBuilder::new().with(1, Value::Short(vec![0])), ifd2, ifd3]).unwrap() + } + + #[test] + fn slices_are_cut_back_into_image_columns_and_converted() { + // slices [1, 4, 4]: two slices of one MCU per row; slice 0 holds MCUs 0 and 1 (image rows 0 and 1), slice 1 + // holds MCUs 2 and 3, so the image is 4 × 2 with luma 1000 1001 1100 1101 / 1002 1003 1102 1103. + let bytes = sraw_file(stream_422(), [1, 4, 4], 4, None); + let img = crate::decode(&bytes).unwrap(); + assert_eq!((img.width, img.height, img.cpp), (4, 2, 3)); + assert!(img.cfa.is_none()); + assert_eq!(img.wb_multipliers, Some([1.0; 3])); + let RawData::U16(px) = &img.data else { panic!("integer data") }; + // no IFD2: chroma gains 1; R = Y + Cr', B = Y + Cb', G from the BT.601 luma + let expect = |y: f64, cb: f64, cr: f64| { + let (r, b) = (y + cr, y + cb); + [r, (y - KR * r - KB * b) / KG, b].map(|v| v.round() as i32) + }; + let cases = [ + ((0, 0), expect(1000.0, 0.0, 100.0)), + ((1, 0), expect(1001.0, 0.0, 100.0)), + ((2, 0), expect(1100.0, 100.0, 200.0)), + ((3, 0), expect(1101.0, 100.0, 200.0)), + ((0, 1), expect(1002.0, 1.0, 102.0)), + ((3, 1), expect(1103.0, 101.0, 202.0)), + ]; + for ((x, y), want) in cases { + let got: Vec = px[(y * 4 + x) * 3..][..3].iter().map(|&v| i32::from(v)).collect(); + assert!(got.iter().zip(want).all(|(g, w)| (g - w).abs() <= 1), "({x},{y}): {got:?} vs {want:?}"); + } + // header-only decoding reports the same image + assert_eq!(crate::probe_info(&bytes).unwrap(), img.info()); + assert!(img.develop(crate::Method::Bilinear).is_ok()); + } + + #[test] + fn a_preview_too_small_to_fit_leaves_the_gains_at_one() { + // IFD2 of 2 × 2 pixels cannot support a fit (it needs hundreds of samples): the file still decodes + let with = crate::decode(&sraw_file(stream_422(), [1, 4, 4], 4, Some([900; 12]))).unwrap(); + let without = crate::decode(&sraw_file(stream_422(), [1, 4, 4], 4, None)).unwrap(); + assert_eq!(with.data, without.data); + } + + #[test] + fn only_ycbcr_frames_take_this_path() { + // other raw types (3 is the preview image's) stay unsupported, as before + for kind in [0, 2, 3, 5] { + let bytes = sraw_file(stream_422(), [1, 4, 4], kind, None); + assert!(matches!(crate::decode(&bytes), Err(RawError::Unsupported(_))), "type {kind}"); + } + } + + #[test] + fn slice_layouts_must_fit_the_frame() { + let ok = Layout::new(4, 2, 1, &[1, 4, 4]).unwrap(); + assert_eq!((ok.width, ok.height), (4, 2)); + // 4:2:0: six samples per MCU, one slice row is two image rows + let l = Layout::new(8, 4, 2, &[1, 12, 12]).unwrap(); + assert_eq!((l.width, l.height), (8, 4)); + assert!(Layout::new(4, 2, 1, &[1, 3, 5]).is_err()); // not whole MCUs + assert!(Layout::new(4, 2, 1, &[1, 4, 8]).is_err()); // rows do not divide the frame + assert!(Layout::new(4, 2, 1, &[1, 0, 4]).is_err()); + assert!(Layout::new(4, 2, 1, &[4, 4]).is_err()); // not three values + assert!(Layout::new(4, 2, 1, &[65, 4, 4]).is_err()); + let bytes = sraw_file(stream_422(), [1, 4, 8], 4, None); + assert!(crate::decode(&bytes).is_err()); + } + + /// Textured luma and smooth chroma on `w × h` planes. + fn test_planes(w: usize, h: usize, vertical: usize) -> Planes { + let y: Vec = (0..w * h).map(|i| (600 + ((i % w) * 13 + (i / w) * 29) % 1500) as u16).collect(); + let cw = w / 2; + let ch = h / vertical; + let cb = (0..cw * ch).map(|i| (16384 + (i % cw) as i32 * 7 - (i / cw) as i32 * 3 - 150) as u16).collect(); + let cr = (0..cw * ch).map(|i| (16384 + (i / cw) as i32 * 5 + (i % cw) as i32 * 2 - 120) as u16).collect(); + Planes { width: w, height: h, vertical, y, cb, cr } + } + + /// An IFD2 image of `pw × ph` pixels at `scale` per pixel, built the way the files relate it to the planes. + fn preview_of(p: &Planes, pw: usize, ph: usize, scale: usize, a: f64, b: f64, pedestal: f64) -> Preview { + let mut rgb = Vec::new(); + for j in 0..ph { + for i in 0..pw { + let (y, cb, cr) = p.window_mean(i * scale, (i + 1) * scale, j * scale, (j + 1) * scale); + let (r, bl) = (y + a * cr, y + b * cb); + let g = (y - KR * r - KB * bl) / KG; + rgb.extend([r, g, bl].map(|v| (v + pedestal).round().clamp(0.0, 65535.0) as u16)); + } + } + Preview { width: pw, height: ph, rgb } + } + + #[test] + fn chroma_gains_are_measured_against_the_preview() { + for vertical in [1, 2] { + for (a, b) in [(1.0, 1.0), (5.7, 7.4), (1.13, 1.12)] { + let planes = test_planes(160, 96, vertical); + // IFD2 is a plain 1/4 reduction, optionally with a black pedestal (some bodies add one) + for pedestal in [0.0, 520.0] { + let preview = preview_of(&planes, 40, 24, 4, a, b, pedestal); + let (ga, gb) = + chroma_gains(&planes, &preview, Rect::new(0, 0, 160, 96)).unwrap_or_else(|| panic!("no fit for {a}, {b}, {vertical}")); + assert!((ga / a - 1.0).abs() < 0.02 && (gb / b - 1.0).abs() < 0.02, "vertical {vertical}: ({a}, {b}) measured as ({ga}, {gb})"); + } + } + } + // the preview overhangs the frame by a few pixels: the whole-frame scale (4.1) is wrong, 4 is right + let planes = test_planes(164, 96, 1); + let mut preview = preview_of(&planes, 40, 24, 4, 5.7, 7.4, 0.0); + let (ga, gb) = chroma_gains(&planes, &preview, Rect::new(0, 0, 164, 96)).unwrap(); + assert!((ga / 5.7 - 1.0).abs() < 0.02 && (gb / 7.4 - 1.0).abs() < 0.02, "({ga}, {gb})"); + // a preview of something else gives no usable fit rather than a wild gain + preview.rgb.iter_mut().enumerate().for_each(|(i, v)| *v = (1000 + (i * 7919) % 3000) as u16); + assert!( + chroma_gains(&planes, &preview, Rect::new(0, 0, 164, 96)).is_none_or(|(a, b)| (0.25..=16.0).contains(&a) && (0.25..=16.0).contains(&b)) + ); + } + + #[test] + fn colour_data_levels_by_version() { + let mut w = vec![0u64; 400]; + w[0] = 7; + w[231..235].copy_from_slice(&[2048, 2047, 2048, 2048]); + assert_eq!(colour_data_levels(&w), (Some(2047.75), 0.0)); + w[0] = 12; + assert_eq!(colour_data_levels(&w).0, None); // the black level of another version lives elsewhere + w[326..330].copy_from_slice(&[512, 512, 511, 512]); + assert_eq!(colour_data_levels(&w).0, Some(511.75)); + w[326..330].copy_from_slice(&[512, 512, 100, 512]); // implausible + assert_eq!(colour_data_levels(&w).0, None); + w[0] = 3; + assert_eq!(colour_data_levels(&w).1, 512.0); // the 40D's luma floor + w[0] = 65533; // -3 + assert_eq!(colour_data_levels(&w), (None, 0.0)); + assert_eq!(colour_data_levels(&[]), (None, 0.0)); + assert_eq!(colour_data_levels(&[7, 1, 2]), (None, 0.0)); + } +} From c7719ddcbf60882329af0eb048d3acea08f403af Mon Sep 17 00:00:00 2001 From: radialmonster Date: Fri, 9 Oct 2026 03:28:35 -0400 Subject: [PATCH 32/46] LR-IMP-FORMATS: read the PEF Exif CFAPattern from the array origin, not the image-area origin The pattern was shifted by the image-area origin for compression 1 and 32773. The *ist D (area origin 19, 11, tag RGGB) fits RGGB from the array origin in a mosaic-level fit against its camera JPEG, so the shifted reading exchanged red and blue. The six 32773 files have an even origin and keep their layout. Adds a hand-made-bytes test with an odd area origin for both compressions. --- crates/raw/src/vendor/pef.rs | 57 ++++++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/crates/raw/src/vendor/pef.rs b/crates/raw/src/vendor/pef.rs index 5b80967e..5fa7c229 100644 --- a/crates/raw/src/vendor/pef.rs +++ b/crates/raw/src/vendor/pef.rs @@ -255,12 +255,11 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result { }; let cfa = match (ifd0.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), ifd0.bytes(t::CFA_PATTERN_EP)) { (Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => Cfa { width: 2, height: 2, pattern: p.to_vec() }, - // The uncompressed bodies state their pattern in the Exif `CFAPattern`, as seen from the first pixel of the - // image area (checked against each file's camera JPEG); the Huffman-compressed ones don't agree with their - // pictures (a 10 MP body says RGGB and renders correctly as BGGR), so they keep the assumed layout. - _ if matches!(info.compression, 1 | 32773) => { - cfa_from_exif(&tiff).map(|c| c.shifted(active.x, active.y)).unwrap_or_else(|| Cfa::bayer_static("BGGR")) - } + // The uncompressed bodies state their pattern in the Exif `CFAPattern`, read from the first sample of the + // stored array, not of the image area: the *ist D (area origin 19, 11) says RGGB and measures RGGB there. A + // pattern shifted by an odd origin would exchange red and blue. (The Huffman-compressed bodies keep the + // assumed layout here.) + _ if matches!(info.compression, 1 | 32773) => cfa_from_exif(&tiff).unwrap_or_else(|| Cfa::bayer_static("BGGR")), _ => Cfa::bayer_static("BGGR"), }; let black = match mn.as_ref().and_then(|m| m.ifd.f64s(BLACK_POINT)).as_deref() { @@ -423,4 +422,50 @@ mod tests { assert!(matches!(&err, RawError::Unsupported(m) if m.contains("32773")), "strip of {len} bytes: {err:?}"); } } + + /// A 16 x 8 big-endian 12-bit PEF-style file with a 12 x 6 image area at the odd origin (3, 1) and the Exif + /// `CFAPattern` `pattern`; the samples are plain 16-bit words (compression 1) or packed 12-bit (32773). + fn pef_with_pattern(compression: u16, pattern: [u8; 4]) -> Vec { + use lightcraft_tiff::{IfdBuilder, ImageData, TiffWriter, Value}; + let px: Vec = (0..16 * 8).map(|i| ((i * 37) % 900 + 100) as u16).collect(); + let strip: Vec = if compression == 1 { + px.iter().flat_map(|v| v.to_be_bytes()).collect() + } else { + px.chunks(2).flat_map(|p| [(p[0] >> 4) as u8, ((p[0] & 15) << 4 | p[1] >> 8) as u8, p[1] as u8]).collect() + }; + let mut ifd = IfdBuilder::new(); + ifd.set(0x0100, Value::Long(vec![16])); + ifd.set(0x0101, Value::Long(vec![8])); + ifd.set(0x0102, Value::Short(vec![12])); + ifd.set(0x0103, Value::Short(vec![compression])); + ifd.set(0x0106, Value::Short(vec![32803])); + ifd.set(0x010f, Value::Ascii("PENTAX Corporation".into())); + ifd.set_image(ImageData::Strips { rows_per_strip: 8, strips: vec![strip] }); + // AOC + "MM", then one IFD: area origin and size (inline), next-IFD 0 + let mut note = b"AOCMM".to_vec(); + note.extend_from_slice(&2u16.to_be_bytes()); + for (tag, val) in [(0x0038u16, [0u8, 3, 0, 1]), (0x0039, [0, 12, 0, 6])] { + note.extend_from_slice(&tag.to_be_bytes()); + note.extend_from_slice(&3u16.to_be_bytes()); + note.extend_from_slice(&2u32.to_be_bytes()); + note.extend_from_slice(&val); + } + note.extend_from_slice(&0u32.to_be_bytes()); + let mut exif = IfdBuilder::new(); + exif.set(0xa302, Value::Undefined([&[0, 2, 0, 2][..], &pattern[..]].concat())); + exif.set(0x927c, Value::Undefined(note)); + ifd.set_child(0x8769, exif); + TiffWriter::new(ByteOrder::Big, false).write(&[ifd]).unwrap() + } + + #[test] + fn uncompressed_files_use_the_exif_pattern_from_the_array_origin_not_the_area_origin() { + for compression in [1u16, 32773] { + for (p, name) in [([0u8, 1, 1, 2], "RGGB"), ([2, 1, 1, 0], "BGGR"), ([1, 0, 2, 1], "GRBG"), ([1, 2, 0, 1], "GBRG")] { + let img = decode(&pef_with_pattern(compression, p), Mode::Full).unwrap(); + assert_eq!(img.active_area, Rect::new(3, 1, 12, 6), "compression {compression}"); + assert_eq!(img.cfa, Some(Cfa::bayer_static(name)), "compression {compression}, Exif {name}"); + } + } + } } From a45c2b661b46d3b434ba38e6ad7508b1832794c7 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 07:38:24 +0000 Subject: [PATCH 33/46] M11.1: decode EOS R7 C-RAW (odd-height CRX QP map) An odd-height adaptive QP map (EOS R7: 4732 sensor rows, 1183 QP rows) was rejected as unsupported, so R7 C-RAW files opened from their embedded JPEG. The map decodes to the exact end of its entropy stream; the last row of the second-level high-frequency bands has no partner row and its QP is used alone. Even-height maps never reach this rule, so the six verified M50/R100/R8 fixtures are unchanged. Checked on paired R7 RAW and C-RAW captures (ISO 100 and 6400): all six C-RAW files decode, developed images match their RAW counterparts visually, and ISO 6400 sensor means agree within 0.1%. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Gs6roPby7THvteijZwptQt --- crates/raw/src/vendor/crx.rs | 24 ++++++++++++++++-------- docs/cr3.md | 11 ++++++++++- docs/whats-new.md | 2 ++ 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/crates/raw/src/vendor/crx.rs b/crates/raw/src/vendor/crx.rs index dfdc73c5..2d15627f 100644 --- a/crates/raw/src/vendor/crx.rs +++ b/crates/raw/src/vendor/crx.rs @@ -9,7 +9,8 @@ //! CC0 raw.pixls.us files against an external decoder used only as a black-box oracle. //! //! Current support: version 0x100 lossless Bayer RAW and horizontal-tile C-RAW, -//! plus version 0x200 single-tile 14-bit C-RAW with adaptive QP quantization. +//! plus version 0x200 single-tile 14-bit C-RAW with adaptive QP quantization +//! (including odd-height QP maps, as on the EOS R7). //! Both ff01/ff02/ff03 and ff11/ff12/ff13 marker families are supported. //! Six full M50/R100/R8 sensor fixtures match an external decoder sample for sample. //! Unverified coding variants return explicit unsupported errors. @@ -173,9 +174,6 @@ fn supports(config: &Cr3Compression) -> Result<()> { if config.version == 0x200 && (config.levels != 3 || config.bit_depth != 14 || config.tile_width != config.width) { return Err(RawError::Unsupported("Canon CRX version 0x200 requires a single-tile 14-bit Bayer C-RAW image".into())); } - if config.version == 0x200 && !config.height.div_ceil(4).is_multiple_of(2) { - return Err(RawError::Unsupported("Canon CRX adaptive QP map with an odd height".into())); - } if config.median_bit_depth.is_some() { return Err(RawError::Unsupported("Canon CRX extended median-bit-depth header".into())); } @@ -418,7 +416,9 @@ fn adaptive_step(map: &QpMap, band: usize, x: usize, y: usize, base: u32, gain: let index = row.checked_mul(map.width).and_then(|n| n.checked_add(column)).ok_or(RawError::Limit("CRX QP map index"))?; let first = *map.data.get(index).ok_or_else(|| corrupt("QP coordinate outside map"))?; let value = if band < 7 { - let second_row = row + 1; + // An odd-height map (R7: 4732 rows give 1183 QP rows) leaves the + // last row of the coarser bands without a partner; it is used alone. + let second_row = (row + 1).min(map.height - 1); let second = *map.data.get(second_row * map.width + column).ok_or_else(|| corrupt("QP average outside map"))?; (first + second) / 2 } else { @@ -734,9 +734,6 @@ fn high_frequency_band(src: &[u8], width: usize, height: usize) -> Result Result { - if !height.is_multiple_of(2) { - return Err(RawError::Unsupported("Canon CRX adaptive QP map with an odd height".into())); - } let count = width.checked_mul(height).filter(|&n| width > 0 && n > 0 && n <= MAX_CRX_SAMPLES / 32).ok_or(RawError::Limit("CRX QP map size"))?; let mut data = Vec::new(); data.try_reserve_exact(count).map_err(|_| RawError::Limit("CRX QP allocation"))?; @@ -1006,6 +1003,17 @@ mod tests { assert!(matches!(qp_map(&[0, 0, 1, 4, 0x94], 1, 4), Err(RawError::Unsupported(_)))); } + #[test] + fn odd_height_qp_map_uses_its_last_row_alone() { + // EOS R7 C-RAW: 4732 sensor rows give a 1183-row QP map, so the last + // row of the coarser high-frequency bands has no partner row. + let map = QpMap { width: 1, height: 3, data: vec![150, 153, 151] }; + assert_eq!(adaptive_step(&map, 4, 0, 0, 0, 4).unwrap(), 7); + // Row 2 alone is QP 151; averaging with a missing row would leave the table. + assert_eq!(adaptive_step(&map, 4, 0, 1, 0, 4).unwrap(), adaptive_step(&map, 7, 0, 2, 0, 4).unwrap()); + assert!(adaptive_step(&map, 4, 0, 2, 0, 4).is_err()); + } + #[test] fn adaptive_v200_craw_reconstructs_sensor_samples() { let mut config = coding(); diff --git a/docs/cr3.md b/docs/cr3.md index 692c209d..70f81d5a 100644 --- a/docs/cr3.md +++ b/docs/cr3.md @@ -55,7 +55,16 @@ The version `0x200` QP predictor and entropy framing were checked independently. Controlled input mutations establish floor averaging of paired QP rows before the nonlinear integer table, then multiplication, division by eight, addition of the subband base and a minimum step of one. QP values 131–167 are verified; -unmeasured values and an odd QP-map height are rejected. Partial-subband and +unmeasured values are rejected. + +An odd QP-map height (EOS R7: 4732 sensor rows give 1183 QP rows) leaves the +last row of the second-level high-frequency bands without a partner row; that +row's QP is used on its own. The odd-height map decodes to the exact end of its +entropy stream. This rule is not yet checked against a black-box reference: +it was checked on paired R7 RAW and C-RAW captures of the same scene (ISO 100 +and 6400), whose developed images match visually and whose sensor means agree +within 0.1% at ISO 6400. Files with even-height maps never reach this rule, so +the six verified fixtures are unchanged. Partial-subband and rounded-plane coding, vertical C-RAW tiling and adaptive low-frequency gains remain unsupported. The decoder bounds image dimensions, sample count, tile count, marker headers and temporary allocations. The full sensor buffer is diff --git a/docs/whats-new.md b/docs/whats-new.md index ccb0be80..9b47fb84 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -49,6 +49,8 @@ are unchanged; photos imported before this change get it when imported again. - Canon CR3 raws now develop from their sensor data: lossless RAW and C-RAW, checked sample for sample on the EOS M50, R100 and R8. CR3 files the decoder can't read yet still open from their embedded JPEG, as before. +- Canon EOS R7 C-RAW files develop from their sensor data too, instead of opening from their embedded JPEG. For R7 + C-RAWs imported earlier, Photo ▸ Reload from Disk picks up the raw data. - Canon CRW, Minolta MRW, Sigma X3F, Kodak KDC, Leaf MOS and Epson ERF files that LightCraft can't decode yet now import as "preview only" with their embedded JPEG instead of failing. A raw whose data is damaged but whose preview is intact does the same. From 1399c445ef8f4ba11e26e1fd9c9bf1272c8ea757 Mon Sep 17 00:00:00 2001 From: radialmonster Date: Fri, 9 Oct 2026 04:02:20 -0400 Subject: [PATCH 34/46] LR-IMP-FORMATS: read the PEF colour-filter layout from the Exif CFAPattern for Huffman-compressed files too The Huffman-compressed Pentax files (compression 65535) assumed BGGR from the array origin. The K10D, K200D, K2000 and every body from the K-3 on are RGGB, so red and blue were exchanged before demosaicing. They now use the Exif CFAPattern from the first sample of the stored array, like the other compressions; BGGR stays the fallback for files without the tag. Extends the CFA test to compression 65535. RENDER_CACHE_VERSION 17 to 18. --- crates/engine/src/media.rs | 2 +- crates/raw/src/vendor/pef.rs | 94 ++++++++++++++++++++++-------------- 2 files changed, 58 insertions(+), 38 deletions(-) diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rs index 5b892250..ed16f8ae 100644 --- a/crates/engine/src/media.rs +++ b/crates/engine/src/media.rs @@ -62,7 +62,7 @@ impl SettingsHashes { } /// Bump when the pipeline's output changes, to invalidate cached thumbnails. -pub const RENDER_CACHE_VERSION: u64 = 17; +pub const RENDER_CACHE_VERSION: u64 = 18; /// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache). pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512]; diff --git a/crates/raw/src/vendor/pef.rs b/crates/raw/src/vendor/pef.rs index 5fa7c229..d065fc4f 100644 --- a/crates/raw/src/vendor/pef.rs +++ b/crates/raw/src/vendor/pef.rs @@ -255,12 +255,11 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result { }; let cfa = match (ifd0.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), ifd0.bytes(t::CFA_PATTERN_EP)) { (Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => Cfa { width: 2, height: 2, pattern: p.to_vec() }, - // The uncompressed bodies state their pattern in the Exif `CFAPattern`, read from the first sample of the - // stored array, not of the image area: the *ist D (area origin 19, 11) says RGGB and measures RGGB there. A - // pattern shifted by an odd origin would exchange red and blue. (The Huffman-compressed bodies keep the - // assumed layout here.) - _ if matches!(info.compression, 1 | 32773) => cfa_from_exif(&tiff).unwrap_or_else(|| Cfa::bayer_static("BGGR")), - _ => Cfa::bayer_static("BGGR"), + // Every body states its layout in the Exif `CFAPattern`, read from the first sample of the stored array, not + // of the image area: the *ist D (origin 19, 11) and the 645D (80, 59) only fit unshifted. The 12-bit 10 MP + // bodies and everything from the K-3 on say RGGB; the K-7 / K-5 / 645D generation says BGGR. Files without the + // tag keep BGGR. + _ => cfa_from_exif(&tiff).unwrap_or_else(|| Cfa::bayer_static("BGGR")), }; let black = match mn.as_ref().and_then(|m| m.ifd.f64s(BLACK_POINT)).as_deref() { Some(v @ [_, _, _, _]) => { @@ -423,49 +422,70 @@ mod tests { } } - /// A 16 x 8 big-endian 12-bit PEF-style file with a 12 x 6 image area at the odd origin (3, 1) and the Exif - /// `CFAPattern` `pattern`; the samples are plain 16-bit words (compression 1) or packed 12-bit (32773). + /// A 16 x 8 big-endian 12-bit PEF-style file with a 12 x 6 image area at the odd origin (3, 1), the Exif + /// `CFAPattern` `pattern` and an AOC maker note with the area tags (and, for 65535, a Huffman table). The samples + /// are plain 16-bit words (1), packed 12-bit (32773) or Huffman coded (65535). fn pef_with_pattern(compression: u16, pattern: [u8; 4]) -> Vec { use lightcraft_tiff::{IfdBuilder, ImageData, TiffWriter, Value}; - let px: Vec = (0..16 * 8).map(|i| ((i * 37) % 900 + 100) as u16).collect(); - let strip: Vec = if compression == 1 { - px.iter().flat_map(|v| v.to_be_bytes()).collect() - } else { - px.chunks(2).flat_map(|p| [(p[0] >> 4) as u8, ((p[0] & 15) << 4 | p[1] >> 8) as u8, p[1] as u8]).collect() + let (w, h) = (16usize, 8usize); + let px: Vec = (0..w * h).map(|i| ((i * 37) % 900 + 100) as u16).collect(); + let (tb, codes) = test_table(); + let strip: Vec = match compression { + 1 => px.iter().flat_map(|v| v.to_be_bytes()).collect(), + 32773 => px.chunks(2).flat_map(|p| [(p[0] >> 4) as u8, ((p[0] & 15) << 4 | p[1] >> 8) as u8, p[1] as u8]).collect(), + _ => encode(&px, w, &codes), }; - let mut ifd = IfdBuilder::new(); - ifd.set(0x0100, Value::Long(vec![16])); - ifd.set(0x0101, Value::Long(vec![8])); - ifd.set(0x0102, Value::Short(vec![12])); - ifd.set(0x0103, Value::Short(vec![compression])); - ifd.set(0x0106, Value::Short(vec![32803])); - ifd.set(0x010f, Value::Ascii("PENTAX Corporation".into())); - ifd.set_image(ImageData::Strips { rows_per_strip: 8, strips: vec![strip] }); - // AOC + "MM", then one IFD: area origin and size (inline), next-IFD 0 - let mut note = b"AOCMM".to_vec(); - note.extend_from_slice(&2u16.to_be_bytes()); - for (tag, val) in [(0x0038u16, [0u8, 3, 0, 1]), (0x0039, [0, 12, 0, 6])] { - note.extend_from_slice(&tag.to_be_bytes()); - note.extend_from_slice(&3u16.to_be_bytes()); - note.extend_from_slice(&2u32.to_be_bytes()); - note.extend_from_slice(&val); + let build = |table_at: u32| { + let mut ifd = IfdBuilder::new(); + ifd.set(0x0100, Value::Long(vec![w as u32])); + ifd.set(0x0101, Value::Long(vec![h as u32])); + ifd.set(0x0102, Value::Short(vec![12])); + ifd.set(0x0103, Value::Short(vec![compression])); + ifd.set(0x0106, Value::Short(vec![32803])); + ifd.set(0x010f, Value::Ascii("PENTAX Corporation".into())); + ifd.set_image(ImageData::Strips { rows_per_strip: h as u32, strips: vec![strip.clone()] }); + // AOC + "MM", then the IFD (area origin, area size and, for Huffman data, the table), next-IFD 0, the table + let mut note = b"AOCMM".to_vec(); + let mut entries = vec![(0x0038u16, 3u16, 2u32, [0u8, 3, 0, 1]), (0x0039, 3, 2, [0, 12, 0, 6])]; + if compression == 65535 { + entries.push((0x0220, 7, tb.len() as u32, table_at.to_be_bytes())); + } + note.extend_from_slice(&(entries.len() as u16).to_be_bytes()); + for (tag, ty, cnt, val) in entries { + note.extend_from_slice(&tag.to_be_bytes()); + note.extend_from_slice(&ty.to_be_bytes()); + note.extend_from_slice(&cnt.to_be_bytes()); + note.extend_from_slice(&val); + } + note.extend_from_slice(&0u32.to_be_bytes()); + if compression == 65535 { + note.extend_from_slice(&tb); + } + let mut exif = IfdBuilder::new(); + exif.set(0xa302, Value::Undefined([&[0, 2, 0, 2][..], &pattern[..]].concat())); + exif.set(0x927c, Value::Undefined(note)); + ifd.set_child(0x8769, exif); + TiffWriter::new(ByteOrder::Big, false).write(&[ifd]).unwrap() + }; + let first = build(0); + if compression != 65535 { + return first; } - note.extend_from_slice(&0u32.to_be_bytes()); - let mut exif = IfdBuilder::new(); - exif.set(0xa302, Value::Undefined([&[0, 2, 0, 2][..], &pattern[..]].concat())); - exif.set(0x927c, Value::Undefined(note)); - ifd.set_child(0x8769, exif); - TiffWriter::new(ByteOrder::Big, false).write(&[ifd]).unwrap() + let at = Tiff::parse(&first).unwrap().exif().unwrap().get(0x927c).unwrap().offset; + build((at + 6 + 2 + 3 * 12 + 4) as u32) } #[test] - fn uncompressed_files_use_the_exif_pattern_from_the_array_origin_not_the_area_origin() { - for compression in [1u16, 32773] { + fn the_pattern_is_the_exif_cfapattern_from_the_array_origin_for_every_compression() { + for compression in [1u16, 32773, 65535] { for (p, name) in [([0u8, 1, 1, 2], "RGGB"), ([2, 1, 1, 0], "BGGR"), ([1, 0, 2, 1], "GRBG"), ([1, 2, 0, 1], "GBRG")] { let img = decode(&pef_with_pattern(compression, p), Mode::Full).unwrap(); assert_eq!(img.active_area, Rect::new(3, 1, 12, 6), "compression {compression}"); assert_eq!(img.cfa, Some(Cfa::bayer_static(name)), "compression {compression}, Exif {name}"); } } + // no tag: the assumed layout stays + let img = decode(&tiny_pef(4, 2, 1, 16, &[0; 16]), Mode::Full).unwrap(); + assert_eq!(img.cfa, Some(Cfa::bayer_static("BGGR"))); } } From 3af7a0c0aeb614aea998cb70c8dca04673a209c2 Mon Sep 17 00:00:00 2001 From: radialmonster Date: Fri, 9 Oct 2026 04:18:46 -0400 Subject: [PATCH 35/46] LR-IMP-FORMATS: drop the 40D luma floor from Canon sRAW decoding The 512 floor and its ColorDataVersion 3 keying rested on one file. Versions without a known black-level position now fall back to black 0 for the white level, so the 40D sRAW renders with slightly raised blacks. --- crates/raw/src/vendor/cr2/sraw.rs | 43 ++++++++++++++----------------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/crates/raw/src/vendor/cr2/sraw.rs b/crates/raw/src/vendor/cr2/sraw.rs index df17d35b..aa5b9ab8 100644 --- a/crates/raw/src/vendor/cr2/sraw.rs +++ b/crates/raw/src/vendor/cr2/sraw.rs @@ -316,7 +316,6 @@ fn chroma_gains(planes: &Planes, preview: &Preview, active: Rect) -> Option<(f64 /// (word 0), located by searching the CC0 samples for the values ExifTool reports. fn average_black_at(version: i16) -> Option { Some(match version { - 3 => 163, 6 | 7 | 9 => 231, 10 | 11 => 276, 12 | 13 | 15 => 326, @@ -324,17 +323,13 @@ fn average_black_at(version: i16) -> Option { }) } -/// The sensor's black level in raw counts and the black pedestal carried by the luma samples, from `ColorData`. -/// The pedestal is 0 on every sample but the 40D's (`ColorDataVersion` 3), whose luma floor is 512 in the file (the -/// darkest pixels read 524–544, and the maker note repeats 512 in words 663–673). -fn colour_data_levels(words: &[u64]) -> (Option, f32) { - let version = words.first().map_or(0, |&v| v as u16 as i16); - let pedestal = if version == 3 { 512.0 } else { 0.0 }; - let black = average_black_at(version).and_then(|at| words.get(at..at + 4)).and_then(|q| { - let (lo, hi) = (q.iter().min()?, q.iter().max()?); - ((256..=4096).contains(lo) && hi - lo <= 8).then(|| q.iter().sum::() as f32 / 4.0) - }); - (black, pedestal) +/// The sensor's black level in raw counts from `ColorData` (`None` for versions whose position is not known, and +/// when the four values are not plausible and near-equal). +fn colour_data_black(words: &[u64]) -> Option { + let version = words.first().map(|&v| v as u16 as i16)?; + let q = words.get(average_black_at(version)?..)?.get(..4)?; + let (lo, hi) = (q.iter().min()?, q.iter().max()?); + ((256..=4096).contains(lo) && hi - lo <= 8).then(|| q.iter().sum::() as f32 / 4.0) } /// Decode a Canon sRAW / mRAW CR2 into linear white-balanced camera RGB (`cpp` 3). @@ -360,9 +355,9 @@ pub(super) fn decode(bytes: &[u8], tiff: &Tiff, raw: &Ifd, mode: Mode) -> Result } } let colour_data = mn.as_ref().and_then(|m| m.ifd.value(COLOR_BALANCE).map(|v| words(v, m.order))); - let (black_counts, pedestal) = colour_data.as_deref().map_or((None, 0.0), colour_data_levels); + let black_counts = colour_data.as_deref().and_then(colour_data_black); // 1.0 = a CFA CR2 of the same body at full 14-bit scale; see LUMA_GAIN - let white = pedestal + LUMA_GAIN * (SENSOR_FULL - black_counts.unwrap_or(0.0)); + let white = LUMA_GAIN * (SENSOR_FULL - black_counts.unwrap_or(0.0)); let data = match mode { Mode::Header => Vec::new(), @@ -387,7 +382,7 @@ pub(super) fn decode(bytes: &[u8], tiff: &Tiff, raw: &Ifd, mode: Mode) -> Result data: RawData::U16(data), cfa: None, bits: 14, - black: BlackLevel::uniform(pedestal), + black: BlackLevel::uniform(0.0), white: vec![white], active_area: active, crop: Rect::new(0, 0, active.width, active.height), @@ -559,22 +554,22 @@ mod tests { } #[test] - fn colour_data_levels_by_version() { + fn colour_data_black_by_version() { let mut w = vec![0u64; 400]; w[0] = 7; w[231..235].copy_from_slice(&[2048, 2047, 2048, 2048]); - assert_eq!(colour_data_levels(&w), (Some(2047.75), 0.0)); + assert_eq!(colour_data_black(&w), Some(2047.75)); w[0] = 12; - assert_eq!(colour_data_levels(&w).0, None); // the black level of another version lives elsewhere + assert_eq!(colour_data_black(&w), None); // the black level of another version lives elsewhere w[326..330].copy_from_slice(&[512, 512, 511, 512]); - assert_eq!(colour_data_levels(&w).0, Some(511.75)); + assert_eq!(colour_data_black(&w), Some(511.75)); w[326..330].copy_from_slice(&[512, 512, 100, 512]); // implausible - assert_eq!(colour_data_levels(&w).0, None); + assert_eq!(colour_data_black(&w), None); w[0] = 3; - assert_eq!(colour_data_levels(&w).1, 512.0); // the 40D's luma floor + assert_eq!(colour_data_black(&w), None); // no position known: black 0 w[0] = 65533; // -3 - assert_eq!(colour_data_levels(&w), (None, 0.0)); - assert_eq!(colour_data_levels(&[]), (None, 0.0)); - assert_eq!(colour_data_levels(&[7, 1, 2]), (None, 0.0)); + assert_eq!(colour_data_black(&w), None); + assert_eq!(colour_data_black(&[]), None); + assert_eq!(colour_data_black(&[7, 1, 2]), None); } } From 5fc85ee03404925ed522fc04867b43ed5538dc4f Mon Sep 17 00:00:00 2001 From: MarekSpodBiedry <73122968+MarekSpodBiedry@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:45:45 +0200 Subject: [PATCH 36/46] Fix #226: a refused Web Locks request no longer leaves the web app loading --- apps/lightcraft-web/src/safety.rs | 26 +++++++++++++++++++++----- docs/web.md | 4 ++-- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/apps/lightcraft-web/src/safety.rs b/apps/lightcraft-web/src/safety.rs index c0dac59c..b3aa504d 100644 --- a/apps/lightcraft-web/src/safety.rs +++ b/apps/lightcraft-web/src/safety.rs @@ -93,25 +93,41 @@ pub fn install_panic_hook() { const TAB_LOCK: &str = "lightcraft-library"; /// Hold the library lock for as long as this page lives (Web Locks API). `Some(true)`: ours; -/// `Some(false)`: another tab of this site has it; `None`: the browser can't tell (no Web Locks). +/// `Some(false)`: another tab of this site has it; `None`: the browser can't tell (no Web Locks, or +/// it refused the request). pub async fn acquire_tab_lock() -> Option { let nav = Reflect::get(&js_sys::global(), &"navigator".into()).ok()?; let locks = Reflect::get(&nav, &"locks".into()).ok().filter(|l| !l.is_undefined() && !l.is_null())?; let request: js_sys::Function = Reflect::get(&locks, &"request".into()).ok()?.dyn_into().ok()?; let opts = js_sys::Object::new(); Reflect::set(&opts, &"ifAvailable".into(), &true.into()).ok()?; - let granted = js_sys::Promise::new(&mut |resolve, _reject| { + let granted = js_sys::Promise::new(&mut |resolve, reject| { let cb = Closure::once_into_js(move |lock: JsValue| -> JsValue { let ours = !lock.is_null() && !lock.is_undefined(); let _ = resolve.call1(&JsValue::NULL, &ours.into()); // keep the lock until the page goes away: a promise that never settles if ours { js_sys::Promise::new(&mut |_, _| {}).into() } else { JsValue::NULL } }); - if let Err(e) = request.call3(&locks, &TAB_LOCK.into(), &opts, &cb) { - log::warn!("Web Locks request failed: {e:?}"); + // a refused request never calls `cb`: settle with its error instead of waiting forever + match request.call3(&locks, &TAB_LOCK.into(), &opts, &cb) { + Ok(pending) => { + let catch = Reflect::get(&pending, &"catch".into()).ok().and_then(|c| c.dyn_into::().ok()); + if let Some(catch) = catch { + let _ = catch.call1(&pending, &reject); + } + } + Err(e) => { + let _ = reject.call1(&JsValue::NULL, &e); + } } }); - JsFuture::from(granted).await.ok().map(|v| v.is_truthy()) + match JsFuture::from(granted).await { + Ok(v) => Some(v.is_truthy()), + Err(e) => { + log::warn!("Web Locks request failed, starting without the one-tab guard: {e:?}"); + None + } + } } /// Offer a blob as a download named `name`. diff --git a/docs/web.md b/docs/web.md index 6d3283c8..47a4087c 100644 --- a/docs/web.md +++ b/docs/web.md @@ -122,8 +122,8 @@ are the same in every version, so serve them with `Cache-Control: no-cache` (not a reload); a notice says why. - One tab at a time: the page holds a Web Lock (`navigator.locks`, `lightcraft-library`); a second tab or window shows "LightCraft is already open in another tab" instead of loading - its own copy (two copies would overwrite each other's saves). Browsers without Web Locks - aren't protected. + its own copy (two copies would overwrite each other's saves). Browsers without Web Locks, or + that refuse the lock request, aren't protected: the app starts without the guard. - If the stored library can't be opened, a notice says the session is temporary and nothing is saved; the stored library is left as it was. - A panic (wasm is built with `panic = "abort"`) replaces the page with a message saying the From cfca2999451e5514474ee4cbac1a4f301ebc192c Mon Sep 17 00:00:00 2001 From: radialmonster Date: Fri, 9 Oct 2026 05:29:23 -0400 Subject: [PATCH 37/46] LR-IMP-FORMATS: report Sony A200/A230/A350 packed ARW as unsupported (preview fallback) The one-strip Sony ARW layout was accepted when the raw strip was 1.00-1.25x width x height bytes. The DSLR-A200, A230 and A350 sample files carry strips of 1.06x, 1.23x and 1.04x, passed that test and decoded as colour noise. Every file that decodes correctly (108 in the test archive) has a strip of exactly width x height bytes, so the layout is now accepted only then; the rest report "Sony ARW version 1 / packed compressed variant" and open from the embedded preview like the A290/A380. Part of #461 (not a decoder). RENDER_CACHE_VERSION 16 -> 19. --- crates/engine/src/media.rs | 2 +- crates/raw/src/vendor/arw.rs | 41 +++++++++++++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rs index b0439390..5c885bd8 100644 --- a/crates/engine/src/media.rs +++ b/crates/engine/src/media.rs @@ -62,7 +62,7 @@ impl SettingsHashes { } /// Bump when the pipeline's output changes, to invalidate cached thumbnails. -pub const RENDER_CACHE_VERSION: u64 = 16; +pub const RENDER_CACHE_VERSION: u64 = 19; /// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache). pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512]; diff --git a/crates/raw/src/vendor/arw.rs b/crates/raw/src/vendor/arw.rs index acaacf0b..0c079d99 100644 --- a/crates/raw/src/vendor/arw.rs +++ b/crates/raw/src/vendor/arw.rs @@ -454,13 +454,17 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result { let linear_rgb = info.compression == 7 && info.photometric == photometric::YCBCR; let chunks = info.chunks(bytes.len() as u64); let strip_len: u64 = chunks.iter().map(|c| c.len).sum(); + // The row-per-w-bytes (ARW2) layout stores exactly one byte per pixel in a single strip; all 108 such files in the test + // archive measure exactly width x height. A200/A230/A350 files carry a strip 4-23 % larger than that (and A290/A390 one + // smaller), which is a different packing, so they are refused rather than read with the wrong layout. + let one_byte_per_sample = chunks.len() == 1 && strip_len == (w * h) as u64; let (data, out_bits) = match info.compression { 7 if linear_rgb => (RawData::U16(read_ycbcr_tiles(bytes, &info, raw, mode)?), 14), - 32767 if chunks.len() == 1 && strip_len >= (w * h) as u64 && strip_len < (w * h) as u64 * 5 / 4 && mode == Mode::Header => { + 32767 if one_byte_per_sample && mode == Mode::Header => { chunk_bytes(bytes, &chunks[0]).ok_or_else(|| RawError::Corrupt("raw strip outside file".into()))?; (RawData::U16(Vec::new()), 14) } - 32767 if chunks.len() == 1 && strip_len >= (w * h) as u64 && strip_len < (w * h) as u64 * 5 / 4 => { + 32767 if one_byte_per_sample => { let src = chunk_bytes(bytes, &chunks[0]).ok_or_else(|| RawError::Corrupt("raw strip outside file".into()))?; let curve = code_curve(&raw.u64s(TONE_CURVE).unwrap_or_else(|| vec![8000, 10400, 12900, 14100])); let mut data = vec![0u16; w * h]; @@ -473,7 +477,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result { }); (RawData::U16(data), 14) } - 32767 => return Err(RawError::Unsupported("Sony ARW version 1 / packed compressed variant".into())), + 32767 => return Err(RawError::Unsupported("Sony ARW version 1 / packed compressed variant (raw strip is not one byte per pixel)".into())), 7 if is_quad_tiled(bytes, &info) => match mode { Mode::Full => (RawData::U16(read_quad_tiles(bytes, &info)?), bits), Mode::Header => { @@ -595,6 +599,37 @@ mod tests { assert_eq!((developed.width, developed.height), (4, 4)); } + /// A 32767-compressed ARW whose single strip is `strip` bytes for a 32 x 4 image. + fn arw_with_strip(strip: usize) -> Vec { + use lightcraft_tiff::{IfdBuilder, ImageData, TiffWriter, Value}; + let mut raw = IfdBuilder::new(); + raw.set(t::MAKE, Value::Ascii("SONY".into())); + raw.set(t::MODEL, Value::Ascii("DSLR-A200".into())); + raw.set(t::IMAGE_WIDTH, Value::Long(vec![32])); + raw.set(t::IMAGE_LENGTH, Value::Long(vec![4])); + raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![12])); + raw.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![1])); + raw.set(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA])); + raw.set(t::COMPRESSION, Value::Short(vec![32767])); + raw.set(TONE_CURVE, Value::Short(vec![8000, 10400, 12900, 14100])); + raw.set_image(ImageData::Strips { rows_per_strip: 4, strips: vec![vec![0u8; strip]] }); + TiffWriter::default().write(&[raw]).unwrap() + } + + #[test] + fn strip_larger_or_smaller_than_one_byte_per_pixel_is_the_packed_variant() { + for strip in [32 * 4 + 8, 32 * 4 * 5 / 4 - 1, 32 * 4 - 1] { + for mode in [Mode::Header, Mode::Full] { + let err = decode(&arw_with_strip(strip), mode).unwrap_err(); + assert!(matches!(err, RawError::Unsupported(ref m) if m.contains("packed compressed variant")), "{strip} {mode:?}: {err:?}"); + } + } + for mode in [Mode::Header, Mode::Full] { + let r = decode(&arw_with_strip(32 * 4), mode); + assert!(!matches!(r, Err(RawError::Unsupported(ref m)) if m.contains("packed compressed variant")), "{r:?}"); + } + } + /// Encode one 16-value set as an ARW2 block (the paper's scheme, used here to test the decoder). pub(crate) fn encode_block(v: &[u16; 16]) -> [u8; 16] { let (imax, &max) = v.iter().enumerate().max_by_key(|(i, x)| (**x, usize::MAX - *i)).unwrap(); From da62ea8c499d1fff2d38e9e0c6dd9eab8b97c05e Mon Sep 17 00:00:00 2001 From: hxw <530313@qq.com> Date: Fri, 9 Oct 2026 04:34:51 -0500 Subject: [PATCH 38/46] LR-BEHAV-GPU: compile DX12 shaders with FXC, never a stray dxcompiler.dll (issue #471) wgpu's default DX12 compiler (Auto) loads whichever dxcompiler.dll the DLL search path finds first. LightCraft ships none, so it picked up another program's copy; one without dxil.dll beside it (e.g. DXC 1.7) warns that the DXIL is unsigned, wgpu treats that as a compile error, its indirect-validation pipelines fail, the device is lost and the window never opens ("Parent device is lost", exit code 1). Every wgpu instance (window and both compute devices) now uses lightcraft_gpu::backend::backend_options(): wgpu's env options with the DX12 compiler pinned to FXC (d3dcompiler_47.dll, part of Windows), unless WGPU_DX12_COMPILER asks for another. Co-Authored-By: Claude Opus 5.5 --- apps/lightcraft/src/main.rs | 14 +++++++++++++- crates/gpu/src/backend.rs | 38 +++++++++++++++++++++++++++++++++++++ crates/gpu/src/ctx.rs | 2 ++ crates/gpu/src/nn.rs | 2 ++ docs/gpu-pipeline.md | 11 +++++++++++ docs/whats-new.md | 3 +++ 6 files changed, 69 insertions(+), 1 deletion(-) diff --git a/apps/lightcraft/src/main.rs b/apps/lightcraft/src/main.rs index a0394da9..c278bc02 100644 --- a/apps/lightcraft/src/main.rs +++ b/apps/lightcraft/src/main.rs @@ -76,11 +76,13 @@ impl eframe::App for App { /// The window's renderer (egui-wgpu): eframe's defaults, with LightCraft's backend choice — DX12 /// alone on Windows unless `LIGHTCRAFT_GPU_BACKEND` / `WGPU_BACKEND` say otherwise (issue #136: -/// with Vulkan in the set, wgpu loads the Vulkan driver even when it then picks DX12). +/// with Vulkan in the set, wgpu loads the Vulkan driver even when it then picks DX12) — and DX12 +/// shaders compiled with FXC, never a stray `dxcompiler.dll` (issue #471). fn window_wgpu_options() -> eframe::egui_wgpu::WgpuConfiguration { let mut c = eframe::egui_wgpu::WgpuConfiguration::default(); if let eframe::egui_wgpu::WgpuSetup::CreateNew(n) = &mut c.wgpu_setup { n.instance_descriptor.backends = lightcraft_engine::gpu::backend::window_backends(); + n.instance_descriptor.backend_options = lightcraft_engine::gpu::backend::backend_options(); } c } @@ -898,6 +900,16 @@ mod tests { assert!(!b.is_empty()); } + /// Issue #471: the window's DX12 shaders compile with FXC — wgpu's default loaded any + /// `dxcompiler.dll` on the search path, and one without `dxil.dll` kept the window from opening. + #[test] + fn window_compiles_dx12_shaders_with_fxc() { + let eframe::egui_wgpu::WgpuSetup::CreateNew(n) = window_wgpu_options().wgpu_setup else { panic!("expected CreateNew") }; + if std::env::var_os("WGPU_DX12_COMPILER").is_none() { + assert!(matches!(n.instance_descriptor.backend_options.dx12.shader_compiler, eframe::wgpu::Dx12Compiler::Fxc)); + } + } + /// Issue #234: Windows opens links with the URL protocol handler (the default browser), not /// File Explorer, and passes the whole map link, `?`, `&` and `#` included, as one argument. #[test] diff --git a/crates/gpu/src/backend.rs b/crates/gpu/src/backend.rs index fcdea55f..041f5773 100644 --- a/crates/gpu/src/backend.rs +++ b/crates/gpu/src/backend.rs @@ -121,6 +121,30 @@ pub fn window_backends() -> Backends { resolve(choice, default, wgpu::Instance::enabled_backend_features()).unwrap_or(default) } +/// The DX12 shader compiler: FXC (`d3dcompiler_47.dll`, part of Windows) unless +/// `WGPU_DX12_COMPILER` (`wgpu`, wgpu's own variable: `fxc`, `dxc`, `auto`…) names another. +/// +/// Issue #471: wgpu's default (`Auto`) loads whichever `dxcompiler.dll` the DLL search path finds +/// first — LightCraft ships none, so it is some other program's (an SDK's, a folder on `PATH`). A +/// copy without its `dxil.dll` beside it warns that the DXIL is unsigned; wgpu takes the warning for +/// a compile error, its own validation pipelines fail, the device is lost and the window never +/// opens. FXC is always there and compiles everything LightCraft's shaders need. +pub fn dx12_compiler(env: Option<&str>) -> wgpu::Dx12Compiler { + let Some(v) = env else { return wgpu::Dx12Compiler::Fxc }; + v.parse().unwrap_or_else(|e| { + log::warn!("gpu: WGPU_DX12_COMPILER={v} ignored: {e}"); + wgpu::Dx12Compiler::Fxc + }) +} + +/// wgpu's backend options for every instance LightCraft creates — the window's and the compute +/// devices': wgpu's environment variables, with the DX12 compiler from [`dx12_compiler`]. +pub fn backend_options() -> wgpu::BackendOptions { + let mut o = wgpu::BackendOptions::from_env_or_default(); + o.dx12.shader_compiler = dx12_compiler(std::env::var("WGPU_DX12_COMPILER").ok().as_deref()); + o +} + /// `LIGHTCRAFT_GPU_BACKEND=off`. pub(crate) fn env_off() -> bool { env_choice() == BackendChoice::Off @@ -211,6 +235,20 @@ mod tests { assert!(default_window_backends("linux").contains(Backends::VULKAN | Backends::GL)); } + /// Issue #471: DX12 shaders compile with FXC, never a `dxcompiler.dll` found on the search path, + /// unless `WGPU_DX12_COMPILER` asks for it. + #[test] + fn dx12_compiles_with_fxc_unless_asked() { + assert!(matches!(dx12_compiler(None), wgpu::Dx12Compiler::Fxc)); + assert!(matches!(dx12_compiler(Some("bogus")), wgpu::Dx12Compiler::Fxc)); + assert!(matches!(dx12_compiler(Some("FXC")), wgpu::Dx12Compiler::Fxc)); + assert!(matches!(dx12_compiler(Some("dxc")), wgpu::Dx12Compiler::DynamicDxc { .. })); + assert!(matches!(dx12_compiler(Some("auto")), wgpu::Dx12Compiler::Auto)); + if std::env::var_os("WGPU_DX12_COMPILER").is_none() { + assert!(matches!(backend_options().dx12.shader_compiler, wgpu::Dx12Compiler::Fxc)); + } + } + #[test] fn resolves_against_the_build() { let compiled = Backends::DX12 | Backends::VULKAN; diff --git a/crates/gpu/src/ctx.rs b/crates/gpu/src/ctx.rs index 0f826bf4..2f4abd4d 100644 --- a/crates/gpu/src/ctx.rs +++ b/crates/gpu/src/ctx.rs @@ -282,6 +282,8 @@ impl Gpu { let mut desc = wgpu::InstanceDescriptor::new_without_display_handle(); // only these drivers are loaded (`crate::backend`: never Vulkan on Windows unless asked) desc.backends = backends; + // DX12 shaders compile with FXC (issue #471) + desc.backend_options = crate::backend::backend_options(); let instance = wgpu::Instance::new(desc); let adapter = pollster::block_on( instance.request_adapter(&wgpu::RequestAdapterOptions { power_preference: wgpu::PowerPreference::HighPerformance, ..Default::default() }), diff --git a/crates/gpu/src/nn.rs b/crates/gpu/src/nn.rs index 4231dae8..bb9d5eea 100644 --- a/crates/gpu/src/nn.rs +++ b/crates/gpu/src/nn.rs @@ -76,6 +76,8 @@ fn create_device() -> Result { fn make_device(backends: wgpu::Backends) -> Result { let mut desc = wgpu::InstanceDescriptor::new_without_display_handle(); desc.backends = backends; + // DX12 shaders compile with FXC (issue #471) + desc.backend_options = crate::backend::backend_options(); let instance = wgpu::Instance::new(desc); let wanted = std::env::var("LIGHTCRAFT_GPU_ADAPTER").ok().map(|s| s.trim().to_lowercase()).filter(|s| !s.is_empty()); let adapter = match &wanted { diff --git a/docs/gpu-pipeline.md b/docs/gpu-pipeline.md index 5eb0ce1d..c843d5a0 100644 --- a/docs/gpu-pipeline.md +++ b/docs/gpu-pipeline.md @@ -44,6 +44,17 @@ Overrides (read once at launch): the compute device always used Vulkan + DX12 (+ Metal). - `LIGHTCRAFT_GPU=0`: GPU rendering off for the process (the window is unaffected). +**DX12 shader compiler (issue #471).** Every instance (window and compute) compiles DX12 shaders with +FXC (`d3dcompiler_47.dll`, part of Windows; `lightcraft_gpu::backend::backend_options`). wgpu's +default, `Auto`, loads whichever `dxcompiler.dll` the DLL search path finds first. LightCraft ships +none, so that copy belongs to another program (a Windows SDK, a folder on `PATH`). An older copy without +`dxil.dll` beside it (e.g. DXC 1.7) warns that the DXIL is unsigned, wgpu treats the warning as a +compile error, its indirect-validation pipelines fail, the device is lost, and the window never opened +(`WGPU error: Parent device is lost`). `WGPU_DX12_COMPILER=dxc | auto | fxc` (wgpu's variable) still +chooses; with `dxc`, put `dxcompiler.dll` *and* `dxil.dll` (DXC ≥ 1.8.2502) next to `lightcraft.exe`. +FXC builds the compute kernels somewhat slower than DXC (≈ 9 s vs 7.5 s for the GPU test suite on an +RTX 4080), off the startup path. + **Off the startup path.** The compute device is created on a background thread once the window is up (`gpu::warm_up` from the first frame's settings), and never while GPU rendering is off: with Settings ▸ Performance ▸ *Use the GPU for rendering* unchecked (applied before the window opens), diff --git a/docs/whats-new.md b/docs/whats-new.md index ccb0be80..4767b59b 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -149,6 +149,9 @@ ignored before) chooses the graphics backend; `off` renders on the CPU. The GPU now starts after the window is up, and only when Settings ▸ Performance ▸ Use the GPU for rendering is on; if LightCraft ever dies while starting the GPU, the next launch starts with GPU rendering off and says how to turn it back on. +- The Windows app no longer quits at launch with "Parent device is lost" when another program's `dxcompiler.dll` is + on the DLL search path without its `dxil.dll` (issue #471): DirectX 12 shaders now always compile with the + compiler built into Windows (FXC). `WGPU_DX12_COMPILER=dxc` uses a `dxcompiler.dll` instead. - Exports and renders never write over a photo's original (issue #93): exporting into the photo's own folder with the same name and "Overwrite" (or Export with Previous repeating it), an exact output path from the control channel or MCP, a merge preview path or `lightcraft-cli render IMG.jpg -o IMG.jpg` is refused with a clear From fcef4c0fc9d88643b33c231020fe90d1a9bee5e4 Mon Sep 17 00:00:00 2001 From: rock-meister <303256+rock-meister@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:15:52 -0500 Subject: [PATCH 39/46] LRC-LIB-CATALOG-IMPORT: import catalog crops that have no HasCrop Lightroom Classic catalogs store the crop edges and straighten angle in their develop settings but, unlike XMP, never a HasCrop flag. The shared crs mapper only applies a crop when crs:HasCrop is True, so every crop in an imported catalog was dropped and reported as unmapped. Infer HasCrop in the catalog path when any crop field is present. The XMP path is unchanged: there HasCrop="False" with crop fields means the crop is switched off. Refs #483 Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/lightroom_catalog.rs | 32 +++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/crates/engine/src/lightroom_catalog.rs b/crates/engine/src/lightroom_catalog.rs index 3c510fde..def9e463 100644 --- a/crates/engine/src/lightroom_catalog.rs +++ b/crates/engine/src/lightroom_catalog.rs @@ -480,7 +480,13 @@ fn ensure_read_active(cancel: &AtomicBool) -> Result<(), String> { fn mapped_settings(text: &str, raw: bool, aspect: f64) -> Result<(Value, Vec), String> { let mut root = crate::preset_import::parse_lua(text)?; let deferred = strip_lua_sentinels(&mut root); - let (props, values) = crate::preset_import::lua_settings_props(&root)?; + let (mut props, values) = crate::preset_import::lua_settings_props(&root)?; + // A catalog's develop settings carry the crop edges and angle but no `HasCrop` (XMP has it): + // any crop field means the photo is cropped. + const CROP: [&str; 5] = ["crs:CropLeft", "crs:CropTop", "crs:CropRight", "crs:CropBottom", "crs:CropAngle"]; + if !props.contains_key("crs:HasCrop") && CROP.iter().any(|k| props.contains_key(*k)) { + props.insert("crs:HasCrop".into(), vec!["True".into()]); + } let (partial, mut unknown) = crate::crs::to_partial_report(&props, Some(&values), Some(raw), aspect); if deferred > 0 { unknown.push("Deferred Lightroom adjustments (-999999)".into()); @@ -880,6 +886,30 @@ mod tests { assert_eq!(partial["light"]["exposure"], 0.75); } + #[test] + fn catalog_crops_without_has_crop_are_imported() { + // Lightroom Classic catalogs store crop edges and angle but never `HasCrop` + let (partial, unknown) = mapped_settings( + "s = { CropAngle = 2.39897, CropConstrainAspectRatio = true, CropLeft = 0.042702, CropRight = 0.957298, CropTop = 0.1 }", + true, + 1.5, + ) + .unwrap(); + let crop = &partial["crop"]["geometry"]; + assert_eq!(crop["rect"]["x0"], 0.042702); + assert_eq!(crop["rect"]["y0"], 0.1); + assert_eq!(crop["rect"]["x1"], 0.957298); + assert_eq!(crop["rect"]["y1"], 1.0); + assert_eq!(crop["angle"], 2.39897); + assert!(!unknown.iter().any(|k| k.starts_with("CropLeft") || k.starts_with("CropAngle")), "{unknown:?}"); + // an angle alone is a straightened, uncropped frame + let (partial, _) = mapped_settings("s = { CropAngle = -1.5 }", true, 1.5).unwrap(); + assert_eq!(partial["crop"]["geometry"]["angle"], -1.5); + // no crop fields: the crop stays untouched + let (partial, _) = mapped_settings("s = { Exposure2012 = 0.5 }", true, 1.5).unwrap(); + assert!(partial.get("crop").is_none()); + } + #[test] fn deferred_auto_tone_sentinels_never_become_real_slider_values() { let mut data = sample(); From c5449926b92c6d10ccba857d89212195d497a813 Mon Sep 17 00:00:00 2001 From: rock-meister <303256+rock-meister@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:15:52 -0500 Subject: [PATCH 40/46] Contributors: add rock-meister (sync GitHub name) Co-Authored-By: Claude Opus 5.5 --- contributors/people.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contributors/people.toml b/contributors/people.toml index fbda1f24..79f20c72 100644 --- a/contributors/people.toml +++ b/contributors/people.toml @@ -22,5 +22,8 @@ real_name = "Brandon Thomas" display_name = "Brandon Thomas" sync_github_name = false +[people.rock-meister] +sync_github_name = true + [people.sanyamsingh001] display_name = "sanyamsingh001" From e5ff5461e3dc468883460612ebce8a98d42769ed Mon Sep 17 00:00:00 2001 From: radialmonster Date: Fri, 9 Oct 2026 06:33:11 -0400 Subject: [PATCH 41/46] LR-IMP-FORMATS: render cache version 21 for the Canon sRAW/mRAW decode Open PRs already use 17 to 20, so this one takes the next free number. --- crates/engine/src/media.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rs index 4519e8f9..0f584518 100644 --- a/crates/engine/src/media.rs +++ b/crates/engine/src/media.rs @@ -62,7 +62,7 @@ impl SettingsHashes { } /// Bump when the pipeline's output changes, to invalidate cached thumbnails. -pub const RENDER_CACHE_VERSION: u64 = 17; +pub const RENDER_CACHE_VERSION: u64 = 21; /// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache). pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512]; From 2ec2b5bf4a4c294276fa87847482f3f8fe043778 Mon Sep 17 00:00:00 2001 From: rock-meister <303256+rock-meister@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:49:59 -0500 Subject: [PATCH 42/46] LRC-LIB-CATALOG-IMPORT: carry catalog crops through the photo orientation crs crop edges are normalized to the stored (unrotated) image, while LightCraft's crop rectangle lives in the oriented frame. Catalog import applied them unchanged, so crops on portrait photos (EXIF 6 or 8) landed along the wrong axes. Read Lightroom's orientation code (Adobe_images.orientation: AB, BC, CD, DA and the mirrored BA, CB, DC, AD) and map the rectangle into the oriented frame with Orientation::map_norm_rect; a mirrored orientation also mirrors the straighten angle. The code comes from the catalog, so this works while the originals are still missing at import time. Refs #483 Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/lightroom_catalog.rs | 120 +++++++++++++++++++++++-- 1 file changed, 112 insertions(+), 8 deletions(-) diff --git a/crates/engine/src/lightroom_catalog.rs b/crates/engine/src/lightroom_catalog.rs index def9e463..9deedc3b 100644 --- a/crates/engine/src/lightroom_catalog.rs +++ b/crates/engine/src/lightroom_catalog.rs @@ -10,6 +10,7 @@ use std::sync::{ use crate::lightroom_sqlite::{Database, LiveTable, Value as SqlValue}; use lightcraft_catalog::{Album, Flag, Op, Photo, PhotoId, Source}; +use lightcraft_geom::{Orientation, Point, Rect}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; @@ -477,7 +478,7 @@ fn ensure_read_active(cancel: &AtomicBool) -> Result<(), String> { if cancel.load(Ordering::Relaxed) { Err("Lightroom import cancelled".into()) } else { Ok(()) } } -fn mapped_settings(text: &str, raw: bool, aspect: f64) -> Result<(Value, Vec), String> { +fn mapped_settings(text: &str, raw: bool, aspect: f64, orientation: Orientation) -> Result<(Value, Vec), String> { let mut root = crate::preset_import::parse_lua(text)?; let deferred = strip_lua_sentinels(&mut root); let (mut props, values) = crate::preset_import::lua_settings_props(&root)?; @@ -487,13 +488,56 @@ fn mapped_settings(text: &str, raw: bool, aspect: f64) -> Result<(Value, Vec 0 { unknown.push("Deferred Lightroom adjustments (-999999)".into()); } Ok((partial, unknown)) } +/// Lightroom's `Adobe_images.orientation`: where the stored image's top-left and top-right corners +/// land in the displayed photo, naming the displayed corners A, B, C, D clockwise from the top left +/// (`AB` upright, `BC` a quarter turn clockwise, `DA` a quarter turn counter-clockwise, `BA` mirrored). +fn lightroom_orientation(code: &str) -> Option { + const ALL: [Orientation; 8] = [ + Orientation::Normal, + Orientation::FlipH, + Orientation::Rotate180, + Orientation::FlipV, + Orientation::Transpose, + Orientation::Rotate90, + Orientation::Transverse, + Orientation::Rotate270, + ]; + let corner = |o: Orientation, x: f64, y: f64| match o.map(x, y, 1.0, 1.0) { + (u, v) if u < 0.5 && v < 0.5 => 'A', + (_, v) if v < 0.5 => 'B', + (u, _) if u >= 0.5 => 'C', + _ => 'D', + }; + ALL.into_iter().find(|&o| code.chars().eq([corner(o, 0.0, 0.0), corner(o, 1.0, 0.0)])) +} + +/// `crs` crop edges are normalized to the stored (unrotated) image, but LightCraft's crop is in the +/// oriented frame: carry the rectangle through `orientation`, and mirror the straighten angle when +/// the orientation mirrors. +fn orient_crop(partial: &mut Value, orientation: Orientation) { + if orientation == Orientation::Normal { + return; + } + let Some(geometry) = partial.pointer_mut("/crop/geometry").and_then(Value::as_object_mut) else { return }; + let edge = |k: &str| geometry.get("rect").and_then(|r| r.get(k)).and_then(Value::as_f64); + let (Some(x0), Some(y0), Some(x1), Some(y1)) = (edge("x0"), edge("y0"), edge("x1"), edge("y1")) else { return }; + let r = orientation.map_norm_rect(Rect::from_points(Point::new(x0, y0), Point::new(x1, y1))); + geometry.insert("rect".into(), json!({"x0": r.x0, "y0": r.y0, "x1": r.x1, "y1": r.y1})); + if orientation.to_parts().0 + && let Some(angle) = geometry.get("angle").and_then(Value::as_f64) + { + geometry.insert("angle".into(), json!(-angle)); + } +} + fn strip_lua_sentinels(value: &mut crate::preset_import::Lua) -> usize { use crate::preset_import::Lua; match value { @@ -686,7 +730,13 @@ pub(crate) fn apply_prepared(s: &mut crate::Session, data: CatalogImport, contex p.meta.keywords = src.keywords.clone(); } if !src.settings.is_empty() && !keep_develop { - match mapped_settings(&src.settings, p.kind == lightcraft_catalog::MediaKind::Raw, p.width.max(1) as f64 / p.height.max(1) as f64) { + let orientation = lightroom_orientation(text(&src.image, "orientation")).unwrap_or_default(); + match mapped_settings( + &src.settings, + p.kind == lightcraft_catalog::MediaKind::Raw, + p.width.max(1) as f64 / p.height.max(1) as f64, + orientation, + ) { Ok((partial, unknown)) => { p.develop = Arc::new(lightcraft_develop::apply_partial(&p.develop, &partial, 1.0)); p.edited = Some(now.clone()); @@ -872,7 +922,7 @@ mod tests { cyclic.collections[0].insert("parent".into(), json!(10)); assert!(apply(&mut s, cyclic, false).is_err()); assert_eq!(s.catalog.to_snapshot(), before); - assert!(mapped_settings(&format!("{}0{}", "{".repeat(1000), "}".repeat(1000)), true, 1.0).is_err()); + assert!(mapped_settings(&format!("{}0{}", "{".repeat(1000), "}".repeat(1000)), true, 1.0, Orientation::Normal).is_err()); } #[test] fn compressed_xmp_is_bounded_and_settings_are_data_only() { @@ -882,7 +932,8 @@ mod tests { assert_eq!(xmp(Some(&json!(encoded))).unwrap(), String::from_utf8_lossy(packet)); encoded[0] = 0xff; assert!(xmp(Some(&json!(encoded))).is_err()); - let (partial, _) = mapped_settings("s = { Exposure2012 = 0.75, ToneCurvePV2012 = {0,0,128,150,255,255} }", true, 1.5).unwrap(); + let (partial, _) = + mapped_settings("s = { Exposure2012 = 0.75, ToneCurvePV2012 = {0,0,128,150,255,255} }", true, 1.5, Orientation::Normal).unwrap(); assert_eq!(partial["light"]["exposure"], 0.75); } @@ -893,6 +944,7 @@ mod tests { "s = { CropAngle = 2.39897, CropConstrainAspectRatio = true, CropLeft = 0.042702, CropRight = 0.957298, CropTop = 0.1 }", true, 1.5, + Orientation::Normal, ) .unwrap(); let crop = &partial["crop"]["geometry"]; @@ -903,13 +955,65 @@ mod tests { assert_eq!(crop["angle"], 2.39897); assert!(!unknown.iter().any(|k| k.starts_with("CropLeft") || k.starts_with("CropAngle")), "{unknown:?}"); // an angle alone is a straightened, uncropped frame - let (partial, _) = mapped_settings("s = { CropAngle = -1.5 }", true, 1.5).unwrap(); + let (partial, _) = mapped_settings("s = { CropAngle = -1.5 }", true, 1.5, Orientation::Normal).unwrap(); assert_eq!(partial["crop"]["geometry"]["angle"], -1.5); // no crop fields: the crop stays untouched - let (partial, _) = mapped_settings("s = { Exposure2012 = 0.5 }", true, 1.5).unwrap(); + let (partial, _) = mapped_settings("s = { Exposure2012 = 0.5 }", true, 1.5, Orientation::Normal).unwrap(); assert!(partial.get("crop").is_none()); } + #[test] + fn lightroom_orientation_codes() { + // the three seen in a real catalog, checked against each file's EXIF orientation (1, 6, 8) + assert_eq!(lightroom_orientation("AB"), Some(Orientation::Normal)); + assert_eq!(lightroom_orientation("BC"), Some(Orientation::Rotate90)); + assert_eq!(lightroom_orientation("DA"), Some(Orientation::Rotate270)); + assert_eq!(lightroom_orientation("CD"), Some(Orientation::Rotate180)); + assert_eq!(lightroom_orientation("BA"), Some(Orientation::FlipH)); + assert_eq!(lightroom_orientation("DC"), Some(Orientation::FlipV)); + assert_eq!(lightroom_orientation("AD"), Some(Orientation::Transpose)); + assert_eq!(lightroom_orientation("CB"), Some(Orientation::Transverse)); + for code in ["", "A", "ABC", "AC", "ab", "XY"] { + assert_eq!(lightroom_orientation(code), None, "{code:?}"); + } + } + + #[test] + fn catalog_crops_follow_the_photo_orientation() { + let settings = "s = { CropLeft = 0.1, CropRight = 0.6, CropTop = 0.2, CropBottom = 0.9, CropAngle = 2 }"; + let crop = |o: Orientation| { + let (partial, _) = mapped_settings(settings, true, 1.5, o).unwrap(); + let g = &partial["crop"]["geometry"]; + let r = &g["rect"]; + [r["x0"].as_f64(), r["y0"].as_f64(), r["x1"].as_f64(), r["y1"].as_f64(), g["angle"].as_f64()].map(|v| (v.unwrap() * 1e9).round() / 1e9) + }; + // (x, y) of the stored image → (1 − y, x) a quarter turn clockwise, (y, 1 − x) counter-clockwise + assert_eq!(crop(Orientation::Normal), [0.1, 0.2, 0.6, 0.9, 2.0]); + assert_eq!(crop(Orientation::Rotate90), [0.1, 0.1, 0.8, 0.6, 2.0]); + assert_eq!(crop(Orientation::Rotate270), [0.2, 0.4, 0.9, 0.9, 2.0]); + assert_eq!(crop(Orientation::Rotate180), [0.4, 0.1, 0.9, 0.8, 2.0]); + // a mirror also mirrors the straighten angle + assert_eq!(crop(Orientation::FlipH), [0.4, 0.2, 0.9, 0.9, -2.0]); + // no crop: nothing to orient + let (partial, _) = mapped_settings("s = { Exposure2012 = 0.5 }", true, 1.5, Orientation::Rotate90).unwrap(); + assert!(partial.get("crop").is_none()); + } + + #[test] + fn imported_crops_use_the_catalog_orientation() { + let mut data = sample(); + data.photos.truncate(1); + data.photos[0].image.remove("masterImage"); + data.photos[0].image.insert("orientation".into(), json!("BC")); + data.photos[0].settings = "s = { CropLeft = 0.1, CropRight = 0.6, CropTop = 0.2, CropBottom = 0.9 }".into(); + data.photos[0].xmp = String::new(); + let mut s = crate::Session::new(); + apply(&mut s, data, false).unwrap(); + let g = s.catalog.photos().next().unwrap().develop.crop.geometry; + let r = [g.rect.x0, g.rect.y0, g.rect.x1, g.rect.y1].map(|v| (v * 1e9).round() / 1e9); + assert_eq!(r, [0.1, 0.1, 0.8, 0.6]); + } + #[test] fn deferred_auto_tone_sentinels_never_become_real_slider_values() { let mut data = sample(); @@ -926,7 +1030,7 @@ mod tests { assert_eq!(p.develop.light.shadows, 65.0); assert_eq!(p.develop.light.blacks, -22.0); assert!(report["unmapped"]["2"].as_array().unwrap().iter().any(|v| v.as_str().is_some_and(|v| v.contains("Deferred")))); - let (partial, _) = mapped_settings("s = { Exposure2012 = -5, Contrast2012 = -100 }", true, 1.0).unwrap(); + let (partial, _) = mapped_settings("s = { Exposure2012 = -5, Contrast2012 = -100 }", true, 1.0, Orientation::Normal).unwrap(); assert_eq!(partial["light"]["exposure"], -5.0); assert_eq!(partial["light"]["contrast"], -100.0); } From b9f951998fc196363607c64d61e379b526f43581 Mon Sep 17 00:00:00 2001 From: Elliot Graham Date: Fri, 9 Oct 2026 12:08:11 +0100 Subject: [PATCH 43/46] #491: Keep Local import headers within the centre pane Co-Authored-By: Codex --- crates/ui-egui/src/headless.rs | 8 ++ crates/ui-egui/src/panels/grid.rs | 129 +++++++++++------- .../ui-egui/src/panels/grid_header_tests.rs | 71 ++++++++++ 3 files changed, 161 insertions(+), 47 deletions(-) create mode 100644 crates/ui-egui/src/panels/grid_header_tests.rs diff --git a/crates/ui-egui/src/headless.rs b/crates/ui-egui/src/headless.rs index d1fae1e1..e6929269 100644 --- a/crates/ui-egui/src/headless.rs +++ b/crates/ui-egui/src/headless.rs @@ -1663,6 +1663,14 @@ mod tests { assert_eq!(h.request("ui.clickWidget", json!({"id": "button:addToLibrary"}), t)["ok"], true); h.settle(SETTLE); assert_eq!(h.app.session.catalog.photos().filter(|p| !p.local).count(), library_before + 2); + // Truncated ancestors remain clickable and still open the original, unabridged path. + let inner = dir.join("inner"); + h.request("engine.execute", json!({"command": "library.browse", "params": {"path": inner.to_string_lossy()}}), t); + h.settle(SETTLE); + let parent_index = dir.to_string_lossy().split(['/', '\\']).filter(|p| !p.is_empty()).count() - 1; + assert_eq!(h.request("ui.clickWidget", json!({"id": format!("crumb:{parent_index}")}), t)["ok"], true); + h.settle(SETTLE); + assert_eq!(h.app.session.browse.as_ref().unwrap().path, dir.to_string_lossy().replace('\\', "/")); let _ = std::fs::remove_dir_all(&dir); } diff --git a/crates/ui-egui/src/panels/grid.rs b/crates/ui-egui/src/panels/grid.rs index 579be235..1b6644e0 100644 --- a/crates/ui-egui/src/panels/grid.rs +++ b/crates/ui-egui/src/panels/grid.rs @@ -1,5 +1,9 @@ //! Photo Grid (justified rows) and Square Grid. Virtualized: only visible cells request thumbnails. +#[cfg(test)] +#[path = "grid_header_tests.rs"] +mod header_tests; + use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -137,8 +141,6 @@ fn show_inner(app: &mut LightcraftApp, ui: &mut egui::Ui) { let t = Tokens::get(ui.ctx()); let (generation, ids) = app.session.visible_shared(); // header: source title + count - let (hr, _) = ui.allocate_exact_size(vec2(ui.available_width(), 44.0), Sense::hover()); - ui.painter().rect_filled(hr, 0.0, t.canvas); let sel_n = app.session.selection.ids.len(); let chips = lightcraft_engine::filter_chips(&app.session.filter, &app.session.catalog); let total = app.session.source_total(); @@ -147,9 +149,11 @@ fn show_inner(app: &mut LightcraftApp, ui: &mut egui::Ui) { match app.session.browse.clone().filter(|_| app.session.source == lightcraft_engine::LibrarySource::Folder) { Some(b) => { let local = app.caches.grid.local(&app.session.catalog, &ids, generation); - folder_header(app, ui, hr, &b, &ids, local, &cnt) + folder_header(app, ui, &b, &ids, local, &cnt); } None => { + let (hr, _) = ui.allocate_exact_size(vec2(ui.available_width(), 44.0), Sense::hover()); + ui.painter().rect_filled(hr, 0.0, t.canvas); let full = crate::i18n::source_title(&app.session); let count_w = ui.painter().layout_no_wrap(cnt.clone(), t.font(12.5), t.text_dim).size().x; // the title gives way to the count: a long album or folder name is trimmed, in full on hover @@ -665,55 +669,86 @@ fn cell(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId, r: Rect, square /// The header of a Local folder view: the path as a breadcrumb (each part opens that folder), /// Include subfolders, Add to My Photos. -fn folder_header(app: &mut LightcraftApp, ui: &mut egui::Ui, hr: Rect, b: &lightcraft_engine::Browse, ids: &[PhotoId], local_n: usize, cnt: &str) { +fn folder_header(app: &mut LightcraftApp, ui: &mut egui::Ui, b: &lightcraft_engine::Browse, ids: &[PhotoId], local_n: usize, cnt: &str) -> Rect { let t = Tokens::get(ui.ctx()); - let mut child = - ui.new_child(egui::UiBuilder::new().max_rect(hr.shrink2(vec2(16.0, 6.0))).layout(egui::Layout::left_to_right(egui::Align::Center))); - child.spacing_mut().item_spacing.x = 4.0; - let parts: Vec<&str> = b.path.split(['/', '\\']).filter(|p| !p.is_empty()).collect(); - // the last few parts (the root side is elided) - let skip = parts.len().saturating_sub(4); - if skip > 0 { - child.label(egui::RichText::new("… ›").color(t.text_dim)); - } - for (i, part) in parts.iter().enumerate().skip(skip) { - let last = i + 1 == parts.len(); - let text = egui::RichText::new(*part).size(13.0).color(if last { t.text } else { t.text_label }); - let r = child.add(egui::Label::new(if last { text.strong() } else { text }).sense(Sense::click())); - register(child.ctx(), format!("crumb:{i}"), r.rect); - if !last { - if r.hovered() { - child.ctx().set_cursor_icon(egui::CursorIcon::PointingHand); + // The path/count and actions need independent rows. The frame reserves the actual height, + // including any wrapped actions, so the grid below never paints under the header. + let header = egui::Frame::NONE.fill(t.canvas).inner_margin(egui::Margin::symmetric(16, 6)).show(ui, |ui| { + ui.set_width(ui.available_width()); + ui.horizontal(|ui| { + ui.spacing_mut().item_spacing.x = 8.0; + let count_w = + ui.painter().layout_no_wrap(cnt.to_owned(), t.font(12.5), t.text_dim).size().x.ceil().min((ui.available_width() * 0.45).floor()); + let path_w = (ui.available_width() - count_w - 8.0).max(0.0).floor(); + ui.allocate_ui_with_layout(vec2(path_w, 24.0), egui::Layout::left_to_right(egui::Align::Center), |ui| { + folder_breadcrumbs(app, ui, &b.path); + }); + let count = ui.add_sized(vec2(count_w, 24.0), egui::Label::new(egui::RichText::new(cnt).size(12.5).color(t.text_dim)).truncate()); + register(ui.ctx(), "grid:count", count.rect); + count.on_hover_text(cnt); + }); + ui.horizontal_wrapped(|ui| { + ui.spacing_mut().item_spacing.x = 12.0; + ui.style_mut().wrap_mode = Some(egui::TextWrapMode::Wrap); + let mut sub = b.subfolders; + let c = ui.checkbox(&mut sub, crate::i18n::tr("Include subfolders")); + register(ui.ctx(), "check:includeSubfolders", c.rect); + if c.changed() { + let _ = app.run("library.browse", json!({"path": b.path, "subfolders": sub})); } - if r.clicked() { - let prefix = if b.path.starts_with('/') { format!("/{}", parts[..=i].join("/")) } else { parts[..=i].join("/") }; - let _ = app.run("library.browse", json!({"path": prefix})); - } - child.label(egui::RichText::new("›").color(t.text_dim)); - } - } - child.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { - ui.label(egui::RichText::new(cnt).size(12.5).color(t.text_dim)); - ui.add_space(10.0); - // counted once per view (cached); the ids are only gathered on a click - if local_n > 0 { - let label = crate::i18n::tr_format!("Add {local_n} to My Photos", local_n = local_n); - if crate::widgets::text_button(ui, "addToLibrary", &label, false).clicked() { - let local: Vec = ids.iter().filter(|id| app.session.catalog.photo(**id).is_some_and(|p| p.local)).map(|id| id.0).collect(); - let n = local.len(); - match app.run("photo.addToLibrary", json!({"ids": local})) { - Ok(_) => app.toast(ui.ctx(), crate::i18n::tr_format!("Added {n} photo{} to My Photos", if n == 1 { "" } else { "s" }, n = n)), - Err(e) => app.toast(ui.ctx(), e), + // Counted once per view (cached); the ids are only gathered on a click. + if local_n > 0 { + let label = crate::i18n::tr_format!("Add {local_n} to My Photos", local_n = local_n); + let button = + ui.add(egui::Button::new(egui::RichText::new(label).font(t.semibold(11.5)).color(t.text)).wrap().min_size(vec2(37.0, 24.0))); + register(ui.ctx(), "button:addToLibrary", button.rect); + if button.clicked() { + let local: Vec = ids.iter().filter(|id| app.session.catalog.photo(**id).is_some_and(|p| p.local)).map(|id| id.0).collect(); + let n = local.len(); + match app.run("photo.addToLibrary", json!({"ids": local})) { + Ok(_) => app.toast(ui.ctx(), crate::i18n::tr_format!("Added {n} photo{} to My Photos", if n == 1 { "" } else { "s" }, n = n)), + Err(e) => app.toast(ui.ctx(), e), + } } } - } - let mut sub = b.subfolders; - let c = ui.checkbox(&mut sub, crate::i18n::tr("Include subfolders")); - register(ui.ctx(), "check:includeSubfolders", c.rect); - if c.changed() { - let _ = app.run("library.browse", json!({"path": b.path, "subfolders": sub})); - } + }); }); + register(ui.ctx(), "grid:folderHeader", header.response.rect); + header.response.rect +} + +fn folder_breadcrumbs(app: &mut LightcraftApp, ui: &mut egui::Ui, path: &str) { + let t = Tokens::get(ui.ctx()); + ui.spacing_mut().item_spacing.x = 4.0; + let parts: Vec<&str> = path.split(['/', '\\']).filter(|p| !p.is_empty()).collect(); + // With all sidebars open even four ellipses and their separators can exceed the row. + // Prefer the current folder, showing more ancestors as space permits. + let visible = ((ui.available_width() / 80.0) as usize).clamp(1, 4); + let skip = parts.len().saturating_sub(visible); + if skip > 0 { + ui.label(egui::RichText::new("… ›").color(t.text_dim)).on_hover_text(path); + } + let separator_w = ui.painter().layout_no_wrap("›".into(), t.font(13.0), t.text_dim).size().x + 8.0; + for (i, part) in parts.iter().enumerate().skip(skip) { + let remaining = parts.len() - i; + let last = remaining == 1; + // Share the bounded row among the remaining crumbs, leaving room for every separator. + let width = ((ui.available_width() - separator_w * (remaining - 1) as f32) / remaining as f32).max(0.0).floor(); + let text = egui::RichText::new(*part).size(13.0).color(if last { t.text } else { t.text_label }); + let r = ui.add_sized(vec2(width, 24.0), egui::Label::new(if last { text.strong() } else { text }).truncate().sense(Sense::click())); + register(ui.ctx(), format!("crumb:{i}"), r.rect); + if !last { + if r.hovered() { + ui.ctx().set_cursor_icon(egui::CursorIcon::PointingHand); + } + if r.clicked() { + let prefix = if path.starts_with('/') { format!("/{}", parts[..=i].join("/")) } else { parts[..=i].join("/") }; + let _ = app.run("library.browse", json!({"path": prefix})); + } + ui.label(egui::RichText::new("›").size(13.0).color(t.text_dim)); + } + r.on_hover_text(path); + } } /// While photos are dragged: a badge at the pointer; the drag ends when the button is up diff --git a/crates/ui-egui/src/panels/grid_header_tests.rs b/crates/ui-egui/src/panels/grid_header_tests.rs new file mode 100644 index 00000000..c55bffb5 --- /dev/null +++ b/crates/ui-egui/src/panels/grid_header_tests.rs @@ -0,0 +1,71 @@ +use egui::{Rect, vec2}; + +use crate::{LightcraftApp, Services, headless::HeadlessView, i18n::Locale}; + +fn text_bounds(shape: &egui::Shape, out: &mut Vec) { + match shape { + egui::Shape::Text(text) => out.push(text.visual_bounding_rect()), + egui::Shape::Vec(shapes) => shapes.iter().for_each(|s| text_bounds(s, out)), + _ => {} + } +} + +#[test] +fn local_folder_header_text_and_controls_do_not_overlap() { + for locale in [Locale::En, Locale::De, Locale::PtBr] { + for width in [136.0, 200.0, 320.0, 590.0, 1100.0] { + for scale in [1.0, 2.0] { + for (path, local_n) in [ + ("/Volumes/External drive/Photography archive/Trips and holidays/A very long folder name with many photographs to import", 2408), + ("/A very long folder name with many photographs to import", 2408), + (r"C:\Users\Photographer\Pictures\A very long folder name with many photographs to import", 2408), + ("/Pictures", 0), + ] { + let mut app = LightcraftApp::new(lightcraft_engine::Session::new(), Services::default()); + let browse = lightcraft_engine::Browse { path: path.into(), subfolders: false }; + let ctx = egui::Context::default(); + crate::i18n::set_language(locale); + crate::theme::install_fonts(&ctx); + crate::theme::apply(&ctx); + let counted = crate::i18n::tr_format!("{sel_n} selected · {counted}", sel_n = 2408, counted = "2408 photos"); + let mut texts = Vec::new(); + let mut widgets = Vec::new(); + let mut header = Rect::NOTHING; + for frame in 0..3 { + let raw = HeadlessView::raw_input(vec2(width, 300.0), scale, frame as f64 / 60.0, vec![]); + let mut out = ctx.run_ui(raw, |ui| { + egui::CentralPanel::default().frame(egui::Frame::NONE).show(ui, |ui| { + header = super::folder_header(&mut app, ui, &browse, &[], local_n, &counted); + let (grid, _) = ui.allocate_exact_size(vec2(ui.available_width(), 100.0), egui::Sense::hover()); + assert!(grid.top() >= header.bottom(), "wrapped controls must reserve space above the grid"); + }); + }); + out.textures_delta.clear(); + texts.clear(); + for shape in &out.shapes { + text_bounds(&shape.shape, &mut texts); + } + widgets = crate::widgets::take_registry(&ctx); + } + crate::i18n::set_language(Locale::En); + assert!(header.left() >= 0.0 && header.right() <= width, "header must not expand past the centre pane: {header:?}"); + let check = widgets.iter().find(|(id, _)| id == "check:includeSubfolders").unwrap().1; + assert!(widgets.iter().any(|(id, _)| id.starts_with("crumb:"))); + assert_eq!(widgets.iter().any(|(id, _)| id == "button:addToLibrary"), local_n > 0); + for (id, rect) in &widgets { + assert!(header.contains_rect(*rect), "{locale:?} {width} {scale}: {id} outside header: {rect:?}"); + if id.starts_with("crumb:") { + assert!(!rect.intersects(check), "breadcrumb overlaps Include subfolders"); + } + } + for (i, a) in texts.iter().enumerate() { + assert!(header.expand(1.0).contains_rect(*a), "{locale:?} {width} {scale}: text outside header: {a:?}"); + for b in texts.iter().skip(i + 1) { + assert!(!a.intersects(*b), "{locale:?} {width} {scale}: painted text overlaps: {a:?} / {b:?}"); + } + } + } + } + } + } +} From 369c56a6ed4657a06c50b23815b722807e46f638 Mon Sep 17 00:00:00 2001 From: rock-meister <303256+rock-meister@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:26:47 -0500 Subject: [PATCH 44/46] LRC-LIB-CATALOG-IMPORT: import straightened catalog crops as Lightroom shows them With a CropAngle, Lightroom's CropLeft/CropTop and CropRight/CropBottom are the top-left and bottom-right corners of the tilted crop on the stored image, and its angle turns the other way to LightCraft's. The import treated the edges as an axis-aligned rectangle in the rotated frame and kept the angle's sign, so straightened crops came out the wrong shape and size. Carry the two corners through the orientation and then LightCraft's straighten rotation (angle = -CropAngle, mirrored orientations flip it back), using the stored image's aspect from fileWidth/fileHeight. On a real catalog every one of 223 straightened crops now matches the frame of Lightroom's export, and the content difference falls to that of unstraightened crops. Refs #483 Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/lightroom_catalog.rs | 152 +++++++++++++++++++------ 1 file changed, 117 insertions(+), 35 deletions(-) diff --git a/crates/engine/src/lightroom_catalog.rs b/crates/engine/src/lightroom_catalog.rs index 9deedc3b..6983c2f8 100644 --- a/crates/engine/src/lightroom_catalog.rs +++ b/crates/engine/src/lightroom_catalog.rs @@ -10,7 +10,7 @@ use std::sync::{ use crate::lightroom_sqlite::{Database, LiveTable, Value as SqlValue}; use lightcraft_catalog::{Album, Flag, Op, Photo, PhotoId, Source}; -use lightcraft_geom::{Orientation, Point, Rect}; +use lightcraft_geom::{Affine, Orientation, Point, Rect}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; @@ -478,7 +478,7 @@ fn ensure_read_active(cancel: &AtomicBool) -> Result<(), String> { if cancel.load(Ordering::Relaxed) { Err("Lightroom import cancelled".into()) } else { Ok(()) } } -fn mapped_settings(text: &str, raw: bool, aspect: f64, orientation: Orientation) -> Result<(Value, Vec), String> { +fn mapped_settings(text: &str, raw: bool, aspect: f64, orientation: Orientation, stored_aspect: f64) -> Result<(Value, Vec), String> { let mut root = crate::preset_import::parse_lua(text)?; let deferred = strip_lua_sentinels(&mut root); let (mut props, values) = crate::preset_import::lua_settings_props(&root)?; @@ -489,7 +489,7 @@ fn mapped_settings(text: &str, raw: bool, aspect: f64, orientation: Orientation) props.insert("crs:HasCrop".into(), vec!["True".into()]); } let (mut partial, mut unknown) = crate::crs::to_partial_report(&props, Some(&values), Some(raw), aspect); - orient_crop(&mut partial, orientation); + catalog_crop(&mut partial, orientation, stored_aspect); if deferred > 0 { unknown.push("Deferred Lightroom adjustments (-999999)".into()); } @@ -519,23 +519,34 @@ fn lightroom_orientation(code: &str) -> Option { ALL.into_iter().find(|&o| code.chars().eq([corner(o, 0.0, 0.0), corner(o, 1.0, 0.0)])) } -/// `crs` crop edges are normalized to the stored (unrotated) image, but LightCraft's crop is in the -/// oriented frame: carry the rectangle through `orientation`, and mirror the straighten angle when -/// the orientation mirrors. -fn orient_crop(partial: &mut Value, orientation: Orientation) { - if orientation == Orientation::Normal { - return; - } +/// Lightroom's crop in LightCraft's frame. `crs` gives the crop's top-left (`CropLeft`, `CropTop`) +/// and bottom-right (`CropRight`, `CropBottom`) corners on the stored image (before orientation and +/// straightening), normalized to its size, plus a straighten angle that turns the other way to +/// LightCraft's. LightCraft's crop is a rectangle in the oriented image rotated by `angle` about its +/// centre, normalized to the oriented image's size, so the corners go through the orientation and +/// then the rotation; `stored_aspect` (width / height of the stored image) matters once the +/// rectangle turns. On a real catalog this reproduced the frame aspect of all 223 straightened crops +/// in Lightroom's exports within 1%. +fn catalog_crop(partial: &mut Value, orientation: Orientation, stored_aspect: f64) { let Some(geometry) = partial.pointer_mut("/crop/geometry").and_then(Value::as_object_mut) else { return }; let edge = |k: &str| geometry.get("rect").and_then(|r| r.get(k)).and_then(Value::as_f64); let (Some(x0), Some(y0), Some(x1), Some(y1)) = (edge("x0"), edge("y0"), edge("x1"), edge("y1")) else { return }; - let r = orientation.map_norm_rect(Rect::from_points(Point::new(x0, y0), Point::new(x1, y1))); - geometry.insert("rect".into(), json!({"x0": r.x0, "y0": r.y0, "x1": r.x1, "y1": r.y1})); - if orientation.to_parts().0 - && let Some(angle) = geometry.get("angle").and_then(Value::as_f64) - { - geometry.insert("angle".into(), json!(-angle)); + let lightroom_angle = geometry.get("angle").and_then(Value::as_f64).unwrap_or(0.0); + if orientation == Orientation::Normal && lightroom_angle == 0.0 { + return; } + let w = if stored_aspect.is_finite() && stored_aspect > 0.0 { stored_aspect } else { 1.0 }; + let (ow, oh) = if orientation.swaps_axes() { (1.0, w) } else { (w, 1.0) }; + // a mirror turns the straighten the other way too + let angle = if orientation.to_parts().0 { lightroom_angle } else { -lightroom_angle }; + let to_rotated = Affine::rotate_about(angle.to_radians(), Point::new(ow / 2.0, oh / 2.0)); + let corner = |x: f64, y: f64| { + let (u, v) = orientation.map(x * w, y, w, 1.0); + to_rotated.apply(Point::new(u, v)) + }; + let r = Rect::from_points(corner(x0, y0), corner(x1, y1)); + geometry.insert("rect".into(), json!({"x0": r.x0 / ow, "y0": r.y0 / oh, "x1": r.x1 / ow, "y1": r.y1 / oh})); + geometry.insert("angle".into(), json!(angle)); } fn strip_lua_sentinels(value: &mut crate::preset_import::Lua) -> usize { @@ -731,11 +742,13 @@ pub(crate) fn apply_prepared(s: &mut crate::Session, data: CatalogImport, contex } if !src.settings.is_empty() && !keep_develop { let orientation = lightroom_orientation(text(&src.image, "orientation")).unwrap_or_default(); + let stored_aspect = number(&src.image, "fileWidth") as f64 / number(&src.image, "fileHeight") as f64; match mapped_settings( &src.settings, p.kind == lightcraft_catalog::MediaKind::Raw, p.width.max(1) as f64 / p.height.max(1) as f64, orientation, + stored_aspect, ) { Ok((partial, unknown)) => { p.develop = Arc::new(lightcraft_develop::apply_partial(&p.develop, &partial, 1.0)); @@ -844,6 +857,7 @@ pub(crate) fn apply_prepared(s: &mut crate::Session, data: CatalogImport, contex #[cfg(test)] mod tests { use super::*; + use lightcraft_geom::CropGeometry; fn row(v: Value) -> Row { serde_json::from_value(v).unwrap() } @@ -922,7 +936,7 @@ mod tests { cyclic.collections[0].insert("parent".into(), json!(10)); assert!(apply(&mut s, cyclic, false).is_err()); assert_eq!(s.catalog.to_snapshot(), before); - assert!(mapped_settings(&format!("{}0{}", "{".repeat(1000), "}".repeat(1000)), true, 1.0, Orientation::Normal).is_err()); + assert!(mapped_settings(&format!("{}0{}", "{".repeat(1000), "}".repeat(1000)), true, 1.0, Orientation::Normal, 1.5).is_err()); } #[test] fn compressed_xmp_is_bounded_and_settings_are_data_only() { @@ -933,7 +947,7 @@ mod tests { encoded[0] = 0xff; assert!(xmp(Some(&json!(encoded))).is_err()); let (partial, _) = - mapped_settings("s = { Exposure2012 = 0.75, ToneCurvePV2012 = {0,0,128,150,255,255} }", true, 1.5, Orientation::Normal).unwrap(); + mapped_settings("s = { Exposure2012 = 0.75, ToneCurvePV2012 = {0,0,128,150,255,255} }", true, 1.5, Orientation::Normal, 1.5).unwrap(); assert_eq!(partial["light"]["exposure"], 0.75); } @@ -941,10 +955,11 @@ mod tests { fn catalog_crops_without_has_crop_are_imported() { // Lightroom Classic catalogs store crop edges and angle but never `HasCrop` let (partial, unknown) = mapped_settings( - "s = { CropAngle = 2.39897, CropConstrainAspectRatio = true, CropLeft = 0.042702, CropRight = 0.957298, CropTop = 0.1 }", + "s = { CropConstrainAspectRatio = true, CropLeft = 0.042702, CropRight = 0.957298, CropTop = 0.1 }", true, 1.5, Orientation::Normal, + 1.5, ) .unwrap(); let crop = &partial["crop"]["geometry"]; @@ -952,13 +967,14 @@ mod tests { assert_eq!(crop["rect"]["y0"], 0.1); assert_eq!(crop["rect"]["x1"], 0.957298); assert_eq!(crop["rect"]["y1"], 1.0); - assert_eq!(crop["angle"], 2.39897); - assert!(!unknown.iter().any(|k| k.starts_with("CropLeft") || k.starts_with("CropAngle")), "{unknown:?}"); - // an angle alone is a straightened, uncropped frame - let (partial, _) = mapped_settings("s = { CropAngle = -1.5 }", true, 1.5, Orientation::Normal).unwrap(); - assert_eq!(partial["crop"]["geometry"]["angle"], -1.5); + assert_eq!(crop["angle"], 0.0); + assert!(!unknown.iter().any(|k| k.starts_with("CropLeft")), "{unknown:?}"); + // an angle alone still means a crop (Lightroom's angle turns the other way to LightCraft's) + let (partial, unknown) = mapped_settings("s = { CropAngle = -1.5 }", true, 1.5, Orientation::Normal, 1.5).unwrap(); + assert_eq!(partial["crop"]["geometry"]["angle"], 1.5); + assert!(!unknown.iter().any(|k| k.starts_with("CropAngle")), "{unknown:?}"); // no crop fields: the crop stays untouched - let (partial, _) = mapped_settings("s = { Exposure2012 = 0.5 }", true, 1.5, Orientation::Normal).unwrap(); + let (partial, _) = mapped_settings("s = { Exposure2012 = 0.5 }", true, 1.5, Orientation::Normal, 1.5).unwrap(); assert!(partial.get("crop").is_none()); } @@ -980,25 +996,91 @@ mod tests { #[test] fn catalog_crops_follow_the_photo_orientation() { - let settings = "s = { CropLeft = 0.1, CropRight = 0.6, CropTop = 0.2, CropBottom = 0.9, CropAngle = 2 }"; let crop = |o: Orientation| { - let (partial, _) = mapped_settings(settings, true, 1.5, o).unwrap(); + let settings = "s = { CropLeft = 0.1, CropRight = 0.6, CropTop = 0.2, CropBottom = 0.9 }"; + let (partial, _) = mapped_settings(settings, true, 1.5, o, 1.5).unwrap(); let g = &partial["crop"]["geometry"]; let r = &g["rect"]; [r["x0"].as_f64(), r["y0"].as_f64(), r["x1"].as_f64(), r["y1"].as_f64(), g["angle"].as_f64()].map(|v| (v.unwrap() * 1e9).round() / 1e9) }; // (x, y) of the stored image → (1 − y, x) a quarter turn clockwise, (y, 1 − x) counter-clockwise - assert_eq!(crop(Orientation::Normal), [0.1, 0.2, 0.6, 0.9, 2.0]); - assert_eq!(crop(Orientation::Rotate90), [0.1, 0.1, 0.8, 0.6, 2.0]); - assert_eq!(crop(Orientation::Rotate270), [0.2, 0.4, 0.9, 0.9, 2.0]); - assert_eq!(crop(Orientation::Rotate180), [0.4, 0.1, 0.9, 0.8, 2.0]); - // a mirror also mirrors the straighten angle - assert_eq!(crop(Orientation::FlipH), [0.4, 0.2, 0.9, 0.9, -2.0]); + assert_eq!(crop(Orientation::Normal), [0.1, 0.2, 0.6, 0.9, 0.0]); + assert_eq!(crop(Orientation::Rotate90), [0.1, 0.1, 0.8, 0.6, 0.0]); + assert_eq!(crop(Orientation::Rotate270), [0.2, 0.4, 0.9, 0.9, 0.0]); + assert_eq!(crop(Orientation::Rotate180), [0.4, 0.1, 0.9, 0.8, 0.0]); + assert_eq!(crop(Orientation::FlipH), [0.4, 0.2, 0.9, 0.9, 0.0]); + // a mirror turns the straighten the other way + let angle = |o: Orientation| { + let (partial, _) = mapped_settings("s = { CropAngle = 2 }", true, 1.5, o, 1.5).unwrap(); + partial["crop"]["geometry"]["angle"].as_f64().unwrap() + }; + assert_eq!(angle(Orientation::Rotate90), -2.0); + assert_eq!(angle(Orientation::FlipH), 2.0); // no crop: nothing to orient - let (partial, _) = mapped_settings("s = { Exposure2012 = 0.5 }", true, 1.5, Orientation::Rotate90).unwrap(); + let (partial, _) = mapped_settings("s = { Exposure2012 = 0.5 }", true, 1.5, Orientation::Rotate90, 1.5).unwrap(); assert!(partial.get("crop").is_none()); } + /// The `crs` corners and angle Lightroom would store for LightCraft crop `g` on a stored image + /// of aspect `w` (height 1) shown with orientation `o`. + fn lightroom_crop(g: CropGeometry, o: Orientation, w: f64) -> String { + let (ow, oh) = if o.swaps_axes() { (1.0, w) } else { (w, 1.0) }; + let to_source = Affine::rotate_about(-g.angle.to_radians(), Point::new(ow / 2.0, oh / 2.0)); + let stored = |x: f64, y: f64| { + let p = to_source.apply(Point::new(x * ow, y * oh)); + o.inverse().map(p.x, p.y, ow, oh) + }; + // the rectangle's corners that land top-left and bottom-right on the stored image + let corners = [stored(g.rect.x0, g.rect.y0), stored(g.rect.x1, g.rect.y0), stored(g.rect.x1, g.rect.y1), stored(g.rect.x0, g.rect.y1)]; + let (tl, br) = (0..4).map(|i| (corners[i], corners[(i + 2) % 4])).find(|(a, b)| a.0 < b.0 && a.1 < b.1).unwrap(); + let angle = if o.to_parts().0 { g.angle } else { -g.angle }; + format!("s = {{ CropLeft = {}, CropTop = {}, CropRight = {}, CropBottom = {}, CropAngle = {angle} }}", tl.0 / w, tl.1, br.0 / w, br.1) + } + + #[test] + fn straightened_catalog_crops_round_trip() { + let g = CropGeometry { rect: Rect::new(0.2, 0.15, 0.7, 0.8), angle: 6.5 }; + for o in [Orientation::Normal, Orientation::Rotate90, Orientation::Rotate270, Orientation::Rotate180, Orientation::FlipH] { + for angle in [6.5, -6.5] { + let g = CropGeometry { angle, ..g }; + let (partial, _) = mapped_settings(&lightroom_crop(g, o, 1.5), true, 1.5, o, 1.5).unwrap(); + let got: CropGeometry = serde_json::from_value(partial["crop"]["geometry"].clone()).unwrap(); + let close = |a: f64, b: f64| (a - b).abs() < 1e-9; + assert!( + close(got.angle, g.angle) + && [(got.rect.x0, g.rect.x0), (got.rect.y0, g.rect.y0), (got.rect.x1, g.rect.x1), (got.rect.y1, g.rect.y1)] + .iter() + .all(|&(a, b)| close(a, b)), + "{o:?} {angle}: {got:?}" + ); + } + } + } + + #[test] + fn straightened_catalog_crops_match_lightroom_exports() { + // Settings and pixel sizes from a real Lightroom Classic catalog (9504 × 6336 stored) and the + // JPEGs Lightroom exported from it at full size. + for (settings, o, export) in [ + ( + "s = { CropAngle = -9.34444, CropBottom = 0.802034, CropLeft = 0.05499, CropRight = 0.94501, CropTop = 0.197966 }", + Orientation::Normal, + (7725.0, 5150.0), + ), + ( + "s = { CropAngle = -1.48964, CropBottom = 0.962457, CropLeft = 0.010429, CropRight = 0.989571, CropTop = 0.037543 }", + Orientation::Rotate90, + (6100.0, 9150.0), + ), + ] { + let (partial, _) = mapped_settings(settings, true, 1.5, o, 9504.0 / 6336.0).unwrap(); + let r: Rect = serde_json::from_value(partial["crop"]["geometry"]["rect"].clone()).unwrap(); + let (ow, oh) = if o.swaps_axes() { (6336.0, 9504.0) } else { (9504.0, 6336.0) }; + let (w, h) = (r.width() * ow, r.height() * oh); + assert!((w - export.0).abs() <= 3.0 && (h - export.1).abs() <= 3.0, "{o:?}: {w:.1} × {h:.1}, Lightroom exported {export:?}"); + } + } + #[test] fn imported_crops_use_the_catalog_orientation() { let mut data = sample(); @@ -1030,7 +1112,7 @@ mod tests { assert_eq!(p.develop.light.shadows, 65.0); assert_eq!(p.develop.light.blacks, -22.0); assert!(report["unmapped"]["2"].as_array().unwrap().iter().any(|v| v.as_str().is_some_and(|v| v.contains("Deferred")))); - let (partial, _) = mapped_settings("s = { Exposure2012 = -5, Contrast2012 = -100 }", true, 1.0, Orientation::Normal).unwrap(); + let (partial, _) = mapped_settings("s = { Exposure2012 = -5, Contrast2012 = -100 }", true, 1.0, Orientation::Normal, 1.5).unwrap(); assert_eq!(partial["light"]["exposure"], -5.0); assert_eq!(partial["light"]["contrast"], -100.0); } From 761631c56cc99ec394f3047fc99bbc059e709283 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 9 Oct 2026 08:19:34 -0400 Subject: [PATCH 45/46] Quit from the macOS menu: save settings and the library (and honour the unsaved-changes prompt) before exiting std::process::exit skipped eframe's on_exit (ui.json, close_library) and the may_close check, so changes still in memory could be lost on Cmd+Q. Co-Authored-By: Claude Opus 5.5 --- apps/lightcraft/src/main.rs | 11 +++++++++-- apps/lightcraft/src/native_menu.rs | 13 +++++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/apps/lightcraft/src/main.rs b/apps/lightcraft/src/main.rs index a0394da9..aedbf7fb 100644 --- a/apps/lightcraft/src/main.rs +++ b/apps/lightcraft/src/main.rs @@ -52,8 +52,15 @@ struct App(LightcraftApp, PrefsWriter, #[cfg(target_os = "macos")] Option bool { lightcraft_ui_egui::i18n::set_language(app.ui.language); + let mut quit = false; while let Ok(key) = self.rx.try_recv() { if key == QUIT { - std::process::exit(0); + quit = true; + continue; } if key == "app.about" || key == SETTINGS { let _ = app.run(&key, serde_json::json!({})); @@ -374,7 +378,7 @@ impl NativeMenu { let bar = menu_bar(app); if structure_of(&bar) != self.structure { self.rebuild(app); - return; + return quit; } fn walk(nodes: &[MenuNode], items: &mut HashMap) { for n in nodes { @@ -428,6 +432,7 @@ impl NativeMenu { } self.publish_shortcuts(app); } + quit } } From 8d96dab1afe5a6d8603b4daa190823d72a143074 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 9 Oct 2026 08:24:34 -0400 Subject: [PATCH 46/46] LR-IMP-FORMATS: checked sum of the sRAW slice widths (an overflowing or zero sum is an error, not a panic) Co-Authored-By: Claude Opus 5.5 --- crates/raw/src/vendor/cr2/sraw.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/raw/src/vendor/cr2/sraw.rs b/crates/raw/src/vendor/cr2/sraw.rs index aa5b9ab8..68b40623 100644 --- a/crates/raw/src/vendor/cr2/sraw.rs +++ b/crates/raw/src/vendor/cr2/sraw.rs @@ -84,8 +84,8 @@ impl Layout { } let samples = frame_width.checked_mul(frame_height).and_then(|p| p.checked_add(p / vertical)).ok_or(RawError::Limit("sRAW frame too large"))?; - let row: usize = widths.iter().sum(); - if samples == 0 || samples % row != 0 { + let row = widths.iter().try_fold(0usize, |a, &w| a.checked_add(w)).ok_or(RawError::Limit("sRAW slice widths"))?; + if samples == 0 || row == 0 || samples % row != 0 { return Err(RawError::Corrupt(format!("sRAW slices ({row} samples per row) do not divide the frame ({samples} samples)"))); } let rows = samples / row; @@ -499,6 +499,9 @@ mod tests { assert!(Layout::new(4, 2, 1, &[1, 0, 4]).is_err()); assert!(Layout::new(4, 2, 1, &[4, 4]).is_err()); // not three values assert!(Layout::new(4, 2, 1, &[65, 4, 4]).is_err()); + // slice widths whose sum overflows (or wraps to zero) are rejected, not divided by + assert!(Layout::new(4, 2, 1, &[3, 1 << 62, 1 << 62]).is_err()); + assert!(Layout::new(4, 2, 1, &[1, u64::MAX - 3, 4]).is_err()); let bytes = sraw_file(stream_422(), [1, 4, 8], 4, None); assert!(crate::decode(&bytes).is_err()); }