From 5630734832da4b49440dec7bbeb98d560732916c Mon Sep 17 00:00:00 2001 From: hxw <530313@qq.com> Date: Fri, 9 Oct 2026 07:00:59 -0500 Subject: [PATCH] LR-BEHAV-BG-TASKS: render export batches side by side (issue #496) A batch ran its photos strictly one after another: each render is row-parallel inside, but a photo's decode, its PNG/TIFF/WebP encode and the wait on a GPU render are mostly serial, so a 32-core machine averaged ~9.5 busy cores on the CPU path and 1-2 with the GPU. run_batch now keeps export_parallelism() photos in flight (cores / 4, 1..=4; one on wasm32) on scoped lane threads plus the calling thread. Each holds an estimate of its working memory from a new export_gate (twice the memory budget, separate from the decode gate it nests over), so full-size 24 MP exports (~1.2 GB each) run two at a time by default. Lanes run at most `lanes` photos past the last one placed, which also bounds the finished files waiting for their turn. Paths, conflict handling, original protection, writes, results and progress stay in batch order (Placer), so the files are the same as one at a time; a panic in a lane becomes that photo's error. render_bench's batch scenario now goes through export_batch. 8 x 24 MP JPEGs, 32 cores + RTX 4080 Laptop: CPU 3.87 s -> 2.35 s, GPU 4.21 s -> 1.82 s. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 +- crates/engine/examples/render_bench.rs | 16 +- crates/engine/src/export.rs | 382 +++++++++++++++++++++++-- crates/engine/src/memory.rs | 20 +- crates/ui-egui/src/export_task.rs | 3 +- docs/parity.md | 2 +- 6 files changed, 384 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index e93683c6..9d2bb09a 100644 --- a/README.md +++ b/README.md @@ -269,7 +269,8 @@ lightcraft --control 7980 ~/Pictures/trip - **GPU-accelerated, CPU-exact.** The whole develop pipeline runs as wgpu compute kernels (Metal / Vulkan / DX12), checked against the CPU pipeline to within 1/255. On a 24 MP raw (Apple M4 Pro): a slider update re-renders in ~4 ms, a cold 2.5 MP loupe in ~30 ms, and a full-size export in ~0.3 s including a parallel JPEG encode. - Without a GPU the same pipeline runs on all CPU cores, redoing only the stages a slider affects. + Without a GPU the same pipeline runs on all CPU cores, redoing only the stages a slider affects. Batch exports + render several photos side by side, so one's decode and encode overlap another's render. - **Instant culling.** Opening a raw shows its embedded camera preview or cached render within ~0.1 s while the full render follows (~0.2–0.5 s for 24 MP). The next and previous photos are prepared in the background, so stepping through a shoot takes ~50 ms per photo. diff --git a/crates/engine/examples/render_bench.rs b/crates/engine/examples/render_bench.rs index 55dc2b11..6a473a2d 100644 --- a/crates/engine/examples/render_bench.rs +++ b/crates/engine/examples/render_bench.rs @@ -247,18 +247,20 @@ fn main() { } } if run("batch") && args.len() > 1 { - // full-size JPEG export of all the files given (decode + render + encode) - use lightcraft_engine::export::export_photo; + // full-size JPEG export of all the files given (decode + render + encode), as the Export + // dialog runs it: several photos side by side (`export_parallelism`), files discarded + use lightcraft_engine::export::{Destination, export_batch}; let mut session = lightcraft_engine::Session::new().with_fs(); let r = session.execute("library.import", &serde_json::json!({"paths": args})).expect("import"); let ids: Vec<_> = r["imported"].as_array().expect("ids").iter().filter_map(|v| v.as_u64()).map(lightcraft_engine::catalog::PhotoId).collect(); - let items: Vec<_> = ids.iter().enumerate().map(|(i, id)| (*id, i + 1)).collect(); let o = ExportOptions::default(); - row(&format!("batch export of {} files (decode+render+JPEG):", items.len()), &mut |g| { + let lanes = lightcraft_engine::export::export_parallelism(ids.len()); + row(&format!("batch export of {} files, {lanes} at a time (decode+render+JPEG):", ids.len()), &mut |g| { lightcraft_engine::gpu::set_enabled(g); - for &(id, seq) in &items { - drop(export_photo(&mut session, id, &o, seq).expect("export")); - session.media.forget(id); // as in a batch: every original is decoded once + let files = export_batch(&mut session, &ids, &o, &Destination::default(), &mut |_, _| Ok(()), &|_| false).expect("export"); + assert_eq!(files.len(), ids.len()); + for &id in &ids { + session.media.forget(id); // every original is decoded once per run } }); lightcraft_engine::gpu::set_enabled(true); diff --git a/crates/engine/src/export.rs b/crates/engine/src/export.rs index 33366285..1ca9f1bd 100644 --- a/crates/engine/src/export.rs +++ b/crates/engine/src/export.rs @@ -1324,6 +1324,9 @@ pub struct PreparedExport { work: Work, /// The library's originals, which [`run_batch`] never writes over (shared by a batch). guard: std::sync::Arc, + /// Estimated working memory of [`Self::run`] (bytes), held from [`crate::memory::export_gate`] + /// while it runs beside other photos of a batch. + weight: usize, } struct RenderWork { @@ -1373,25 +1376,42 @@ fn prepare_guarded( ) -> Result { let p = session.catalog.photo(id).ok_or("no such photo")?; let file_name = o.file_name_for(p, seq); - let work = if o.format.is_rendered() { + let (full_px, long) = ((p.width as usize).saturating_mul(p.height as usize), p.width.max(p.height) as usize); + let (work, weight) = if o.format.is_rendered() { let (w, h) = output_size(p, o); let meta = export_metadata(p, o); let job = session.export_job(id, w, h, o.effective_space(), o.effective_depth())?; - Work::Render(Box::new(RenderWork { job, meta, opts: o.clone() })) + let weight = render_weight(full_px, long, job.level, w.saturating_mul(h)); + (Work::Render(Box::new(RenderWork { job, meta, opts: o.clone() })), weight) } else { let lightcraft_catalog::Source::File { path } = &p.source else { return Err(format!("{} is a generated demo photo: it has no original file to export", p.file_name)); }; - Work::File { + let dng = (o.format == ExportFormat::Dng).then_some(o.dng_compression); + // the file's bytes; a DNG also decodes the raw data (16-bit samples) and writes it again + let weight = (p.file_size as usize).saturating_add(if dng.is_some() { full_px.saturating_mul(6) } else { 0 }); + let work = Work::File { path: path.clone(), read: session.media.file_bytes.clone(), packet: crate::sidecar::sidecar_packet(p, &session.catalog), - dng: (o.format == ExportFormat::Dng).then_some(o.dng_compression), + dng, label: p.file_name.clone(), size: (p.width as usize, p.height as usize), - } + }; + (work, weight) }; - Ok(PreparedExport { photo: id, file_name, work, guard }) + Ok(PreparedExport { photo: id, file_name, work, guard, weight }) +} + +/// Working memory of rendering a photo of `full_px` pixels (long edge `long`) from source level +/// `level` into `out_px` output pixels: the decoded source (linear RGB f32, 12 B/px) plus the +/// pipeline's planes and the encoded output (~36 B per output pixel). Measured: a 24 MP export +/// peaks at ~1.2 GB, one at 2048 px from the 2560 px preview at ~150 MB. +fn render_weight(full_px: usize, long: usize, level: crate::media::SourceLevel, out_px: usize) -> usize { + let edge = level.max_edge().min(long.max(1)); + let scale = edge as f64 / long.max(1) as f64; + let src_px = (full_px as f64 * scale * scale).min(usize::MAX as f64 / 64.0) as usize; + src_px.saturating_mul(12).saturating_add(out_px.saturating_mul(36)) } impl PreparedExport { @@ -1501,6 +1521,11 @@ fn sidecar_path(main: &str, ext: &str) -> String { /// policy. `progress(done, next file)` is called before each photo; returning false cancels the /// rest. Returns one JSON object per photo: `{path, width, height, bytes, sidecars}`, /// `{skipped: path}` or (unless `stop_on_error`) `{photo, file, error}`. +/// +/// Several photos render side by side ([`export_parallelism`], within the memory of +/// [`crate::memory::export_gate`]), so one photo's decode, encode or GPU wait overlaps another's +/// render (issue #496). Paths, writes, results and `progress` still follow the batch order, and the +/// files are the same as one at a time. pub fn run_batch( items: Vec, o: &ExportOptions, @@ -1510,31 +1535,226 @@ pub fn run_batch( stop_on_error: bool, progress: &mut dyn FnMut(usize, &str) -> bool, ) -> Result, String> { - use serde_json::json; - let join = |a: &str, b: &str| if a.is_empty() { b.to_string() } else { format!("{}/{b}", a.trim_end_matches('/')) }; - let dir = if o.subfolder.is_empty() { to.dir.clone() } else { join(&to.dir, &o.subfolder) }; - let single = items.len() == 1; - let mut taken = std::collections::HashSet::new(); - let mut out = Vec::new(); - for (i, item) in items.into_iter().enumerate() { - if !progress(i, &item.file_name) { - break; + let lanes = export_parallelism(items.len()); + run_batch_with(items, &mut Placer::new(o, to, write, exists, stop_on_error), progress, lanes) +} + +/// How many photos of a batch of `n` render side by side. Each render already spreads its rows +/// over every core; the others fill the cores while it decodes, encodes or waits for the GPU. +/// One on wasm32 (no threads). +pub fn export_parallelism(n: usize) -> usize { + if cfg!(target_arch = "wasm32") { + return 1; + } + let cores = std::thread::available_parallelism().map_or(1, |c| c.get()); + (cores / 4).clamp(1, 4).min(n.max(1)) +} + +/// [`run_batch`] with `lanes` photos in flight (the calling thread is one of them). +fn run_batch_with( + items: Vec, + placer: &mut Placer<'_>, + progress: &mut dyn FnMut(usize, &str) -> bool, + lanes: usize, +) -> Result, String> { + placer.single = items.len() == 1; + if lanes <= 1 || items.len() <= 1 { + for (i, item) in items.into_iter().enumerate() { + if !progress(i, &item.file_name) { + break; + } + let (photo, name, guard) = (item.photo, item.file_name.clone(), item.guard.clone()); + placer.place(photo, name, &guard, run_gated(item))?; } - let (photo, name, guard) = (item.photo, item.file_name.clone(), item.guard.clone()); - let e = match item.run() { - Ok(e) => e, - Err(err) if stop_on_error => return Err(err), - Err(err) => { - out.push(json!({"photo": photo.0, "file": name, "error": err})); + return Ok(std::mem::take(&mut placer.out)); + } + let heads: Vec<_> = items.iter().map(|i| (i.photo, i.file_name.clone(), i.guard.clone())).collect(); + let queue = Queue { + lane: std::sync::Mutex::new(Lane { items: items.into_iter().map(Some).collect(), next: 0, placed: 0, stop: false, done: Default::default() }), + cv: std::sync::Condvar::new(), + ahead: lanes, + }; + std::thread::scope(|sc| { + for k in 1..lanes { + let q = &queue; + // a lane that can't start leaves its photos to the others (and to this thread) + if std::thread::Builder::new().name(format!("export-{k}")).spawn_scoped(sc, move || q.work()).is_err() { + break; + } + } + let placed = || { + for (i, (photo, name, guard)) in heads.into_iter().enumerate() { + if !progress(i, &name) { + break; + } + let e = queue.result(i); + placer.place(photo, name, &guard, e)?; + } + Ok::<_, String>(()) + }; + let r = placed(); + queue.stop(); + r + })?; + Ok(std::mem::take(&mut placer.out)) +} + +/// Run one photo's export holding its working memory from [`crate::memory::export_gate`]; a panic +/// becomes that photo's error, not the batch's end. +fn run_gated(item: PreparedExport) -> Result { + let _held = crate::memory::export_gate().acquire(item.weight); + let what = format!("exporting {}", item.file_name); + crate::guard::catch(&what, || item.run()).and_then(|r| r) +} + +/// The photos of a batch shared by its lanes: each lane takes the next one, at most +/// [`Queue::ahead`] past the last placed. That bounds the finished files waiting in memory for +/// their turn too (their working memory is given back to the gate as soon as they are encoded). +struct Queue { + lane: std::sync::Mutex, + cv: std::sync::Condvar, + ahead: usize, +} + +struct Lane { + items: Vec>, + /// The next photo to start. + next: usize, + /// Photos handed to the placer so far. + placed: usize, + stop: bool, + done: std::collections::HashMap>, +} + +impl Lane { + fn take_next(&mut self) -> Option<(usize, PreparedExport)> { + let i = self.next; + let item = self.items.get_mut(i)?.take()?; + self.next += 1; + Some((i, item)) + } +} + +impl Queue { + fn lock(&self) -> std::sync::MutexGuard<'_, Lane> { + self.lane.lock().unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn wait<'a>(&self, g: std::sync::MutexGuard<'a, Lane>) -> std::sync::MutexGuard<'a, Lane> { + self.cv.wait(g).unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn finish(&self, i: usize, r: Result) { + self.lock().done.insert(i, r); + self.cv.notify_all(); + } + + /// A lane thread: run photos until none is left or the batch stops. + fn work(&self) { + loop { + let mut l = self.lock(); + let (i, item) = loop { + if l.stop || l.next >= l.items.len() { + return; + } + if l.next < l.placed.saturating_add(self.ahead) { + match l.take_next() { + Some(t) => break t, + None => return, + } + } + l = self.wait(l); + }; + drop(l); + self.finish(i, run_gated(item)); + } + } + + /// Photo `i`'s result, waiting for it; run here when no lane has started it yet. + fn result(&self, i: usize) -> Result { + let mut l = self.lock(); + loop { + if let Some(r) = l.done.remove(&i) { + l.placed = i + 1; + drop(l); + self.cv.notify_all(); + return r; + } + if l.next == i { + let Some((_, item)) = l.take_next() else { + return Err("export: photo missing from the batch".into()); + }; + drop(l); + self.finish(i, run_gated(item)); + l = self.lock(); continue; } + l = self.wait(l); + } + } + + fn stop(&self) { + self.lock().stop = true; + self.cv.notify_all(); + } +} + +/// Where a batch's files go: each photo's path (subfolder, conflict policy, sidecars, never an +/// original) and its write, in batch order. +struct Placer<'a> { + o: &'a ExportOptions, + to: &'a Destination, + dir: String, + single: bool, + taken: std::collections::HashSet, + out: Vec, + write: &'a mut dyn FnMut(&str, &[u8]) -> Result<(), String>, + exists: &'a dyn Fn(&str) -> bool, + stop_on_error: bool, +} + +fn join_path(a: &str, b: &str) -> String { + if a.is_empty() { b.to_string() } else { format!("{}/{b}", a.trim_end_matches('/')) } +} + +impl<'a> Placer<'a> { + fn new( + o: &'a ExportOptions, + to: &'a Destination, + write: &'a mut dyn FnMut(&str, &[u8]) -> Result<(), String>, + exists: &'a dyn Fn(&str) -> bool, + stop_on_error: bool, + ) -> Placer<'a> { + let dir = if o.subfolder.is_empty() { to.dir.clone() } else { join_path(&to.dir, &o.subfolder) }; + Placer { o, to, dir, single: false, taken: Default::default(), out: Vec::new(), write, exists, stop_on_error } + } + + /// Place photo `photo`'s export result (file `name`): write it and record the outcome. `Err` + /// only when it failed and `stop_on_error` is set. + fn place( + &mut self, + photo: lightcraft_catalog::PhotoId, + name: String, + guard: &crate::originals::OriginalGuard, + e: Result, + ) -> Result<(), String> { + use serde_json::json; + let (o, dir, exists) = (self.o, &self.dir, self.exists); + let e = match e { + Ok(e) => e, + Err(err) if self.stop_on_error => return Err(err), + Err(err) => { + self.out.push(json!({"photo": photo.0, "file": name, "error": err})); + return Ok(()); + } }; // the exported file and its sidecars let group = |main: &str| std::iter::once(main.to_string()).chain(e.sidecars.iter().map(|(x, _)| sidecar_path(main, x))).collect::>(); - let path = match to.exact.as_deref().filter(|_| single) { + let path = match self.to.exact.as_deref().filter(|_| self.single) { Some(p) => Ok(p.to_string()), None => { - let path = join(&dir, &e.file_name); + let path = join_path(dir, &e.file_name); + let taken = &self.taken; let busy = |main: &str| group(main).iter().any(|p| taken.contains(p) || exists(p)); if !busy(&path) { Ok(path) @@ -1542,12 +1762,12 @@ pub fn run_batch( match o.conflict { Conflict::Overwrite => Ok(path), Conflict::Skip => { - out.push(json!({"skipped": path})); - continue; + self.out.push(json!({"skipped": path})); + return Ok(()); } Conflict::Unique => { let (stem, ext) = e.file_name.rsplit_once('.').map_or((e.file_name.as_str(), None), |(a, b)| (a, Some(b))); - let name = |n: usize| join(&dir, &ext.map_or(format!("{stem}-{n}"), |x| format!("{stem}-{n}.{x}"))); + let name = |n: usize| join_path(dir, &ext.map_or(format!("{stem}-{n}"), |x| format!("{stem}-{n}.{x}"))); (2..1_000_000).map(name).find(|p| !busy(p)).ok_or_else(|| format!("{path}: no free file name")) } } @@ -1555,6 +1775,7 @@ pub fn run_batch( } }; let file = path.clone().unwrap_or_else(|_| name.clone()); + let write = &mut *self.write; let written = path.and_then(|path| { let files = group(&path); // never over an original, whatever the conflict policy or the exact path said @@ -1571,14 +1792,14 @@ pub fn run_batch( }); match written { Ok((path, files, sidecars)) => { - taken.extend(files); - out.push(json!({"path": path, "width": e.width, "height": e.height, "bytes": e.bytes.len(), "sidecars": sidecars})); + self.taken.extend(files); + self.out.push(json!({"path": path, "width": e.width, "height": e.height, "bytes": e.bytes.len(), "sidecars": sidecars})); } - Err(err) if stop_on_error => return Err(err), - Err(err) => out.push(json!({"photo": photo.0, "file": file, "error": err})), + Err(err) if self.stop_on_error => return Err(err), + Err(err) => self.out.push(json!({"photo": photo.0, "file": file, "error": err})), } + Ok(()) } - Ok(out) } #[cfg(test)] @@ -2225,4 +2446,103 @@ mod tests { assert!(WATERMARK_SIZE_RANGE.0 <= 0.01 && WATERMARK_SIZE_RANGE.1 >= 0.15); assert!(WATERMARK_IMAGE_WIDTH_RANGE.0 <= 0.02 && WATERMARK_IMAGE_WIDTH_RANGE.1 >= 1.0); } + + /// A batch with `lanes` photos in flight into a fake folder `out` (nothing exists there): + /// results, writes (path, bytes) and progress calls; `progress` returns false at `cancel_at`. + #[allow(clippy::type_complexity)] + fn lanes_batch( + s: &mut crate::Session, + ids: &[lightcraft_catalog::PhotoId], + o: &ExportOptions, + lanes: usize, + stop_on_error: bool, + cancel_at: usize, + ) -> (Result, String>, Vec<(String, Vec)>, Vec<(usize, String)>) { + let items = prepare_batch(s, ids, o).unwrap(); + let to = Destination { dir: "out".into(), exact: None }; + let mut written = Vec::new(); + let mut write = |p: &str, b: &[u8]| { + written.push((p.to_string(), b.to_vec())); + Ok(()) + }; + let mut seen = Vec::new(); + let mut progress = |i: usize, name: &str| { + seen.push((i, name.to_string())); + i < cancel_at + }; + let files = run_batch_with(items, &mut Placer::new(o, &to, &mut write, &|_| false, stop_on_error), &mut progress, lanes); + (files, written, seen) + } + + /// The files of a batch without their byte counts: a GPU render (when another test turns the + /// GPU off mid-run) may differ from a CPU one by an LSB, never in name, size or order. + fn shape(files: &[serde_json::Value]) -> Vec { + files.iter().map(|f| json!({"path": f["path"], "width": f["width"], "height": f["height"], "error": f["error"]})).collect() + } + + // Issue #496: photos rendered side by side give the same files, names, order and progress as + // one at a time, and a cancel stops at the same photo + #[test] + fn side_by_side_batches_match_one_at_a_time() { + let mut s = crate::Session::with_demo(); + let ids: Vec<_> = s.visible().iter().copied().take(5).collect(); + // one name for all: the Unique numbering follows the order the files are placed in + let o = ExportOptions::from_json(&json!({"format": "png", "width": 40, "naming": "same"})); + let (files, written, seen) = lanes_batch(&mut s, &ids, &o, 1, true, usize::MAX); + let files = files.unwrap(); + let paths: Vec<_> = written.iter().map(|(p, _)| p.clone()).collect(); + assert_eq!(paths, ["out/same.png", "out/same-2.png", "out/same-3.png", "out/same-4.png", "out/same-5.png"]); + let heights: std::collections::HashSet<_> = files.iter().map(|f| f["height"].as_u64()).collect(); + assert!(heights.len() > 1, "photos of different shapes show the order: {files:?}"); + for lanes in [2, 3, 8] { + let (f, w, sn) = lanes_batch(&mut s, &ids, &o, lanes, true, usize::MAX); + assert_eq!(shape(&f.unwrap()), shape(&files), "{lanes} lanes"); + assert_eq!(w.iter().map(|(p, _)| p.clone()).collect::>(), paths, "{lanes} lanes"); + assert_eq!(sn, seen, "{lanes} lanes"); + } + let (f, w, sn) = lanes_batch(&mut s, &ids, &o, 3, true, 2); + assert_eq!(shape(&f.unwrap()), shape(&files[..2])); + assert_eq!((w.len(), sn.len()), (2, 3), "photos past the cancel are never written"); + } + + // A photo that fails is reported in its place while the others are exported; with + // stop_on_error the batch ends there and nothing after it is written + #[test] + fn side_by_side_batches_report_failures_in_order() { + use lightcraft_catalog::{Op, Photo, PhotoId, Source}; + let mut s = crate::Session::with_demo(); + let mut ids: Vec<_> = s.visible().iter().copied().take(3).collect(); + let gone = + Photo::new(PhotoId(9_496), Source::File { path: "no/such/dir/gone.jpg".into() }, "gone.jpg", "JPEG", 600, 400, "2026-10-09T00:00:00"); + s.catalog.apply(Op::AddPhoto { photo: Box::new(gone) }).unwrap(); + ids.insert(1, PhotoId(9_496)); + let o = ExportOptions::from_json(&json!({"format": "jpeg", "width": 40})); + let (files, written, _) = lanes_batch(&mut s, &ids, &o, 3, false, usize::MAX); + let files = files.unwrap(); + assert_eq!(files.len(), 4); + assert_eq!((files[1]["photo"].as_u64(), files[1]["file"].as_str()), (Some(9_496), Some("gone.jpg")), "{files:?}"); + assert!(files[1]["error"].is_string()); + assert!([0, 2, 3].iter().all(|&i| files[i]["path"].is_string()), "{files:?}"); + assert_eq!(written.len(), 3); + let one = lanes_batch(&mut s, &ids, &o, 1, false, usize::MAX).0.unwrap(); + assert_eq!(shape(&files), shape(&one)); + + let (r, written, _) = lanes_batch(&mut s, &ids, &o, 3, true, usize::MAX); + assert!(r.is_err()); + assert_eq!(written.len(), 1, "only the photo before the failure"); + } + + #[test] + fn export_weights_follow_the_source_and_output_sizes() { + use crate::media::SourceLevel; + let mb = |b: usize| b >> 20; + // a full-size 24 MP export (measured peak ~1.2 GB) + assert_eq!(mb(render_weight(6000 * 4000, 6000, SourceLevel::Full, 6000 * 4000)), 1098); + // 2048 px from the 2560 px preview (measured ~150 MB) + assert_eq!(mb(render_weight(6000 * 4000, 6000, SourceLevel::Preview, 2048 * 1365)), 145); + // a photo smaller than the preview level is its own source + assert_eq!(render_weight(1000 * 500, 1000, SourceLevel::Preview, 0), 1000 * 500 * 12); + assert_eq!(render_weight(0, 0, SourceLevel::Full, 0), 0); + assert!(render_weight(usize::MAX, 1, SourceLevel::Full, usize::MAX) > 0, "no overflow"); + } } diff --git a/crates/engine/src/memory.rs b/crates/engine/src/memory.rs index d231bf65..8c421668 100644 --- a/crates/engine/src/memory.rs +++ b/crates/engine/src/memory.rs @@ -18,7 +18,9 @@ use serde::Serialize; /// - a quarter for the working memory of decodes in flight ([`work_gate`]); /// - an eighth for the GPU renderer's pool of recycled buffers (trimmed when the app is idle). /// -/// The rest is headroom for what isn't cached (renders in progress, textures, the UI). Default: +/// The rest is headroom for what isn't cached (renders in progress, textures, the UI). An export +/// batch may, in addition, hold up to twice the budget in photos rendering side by side +/// ([`export_gate`]): transient memory, given back when the batch ends. Default: /// a quarter of the machine's RAM, at most 1.5 GiB (`LIGHTCRAFT_MEMORY_MB` overrides it). pub fn default_budget() -> usize { if let Some(mb) = std::env::var("LIGHTCRAFT_MEMORY_MB").ok().and_then(|v| v.trim().parse::().ok()).filter(|m| *m >= 64) { @@ -83,6 +85,9 @@ fn apply(b: usize) { if let Some(g) = GATE.get() { g.set_limit(b / 4); } + if let Some(g) = EXPORT_GATE.get() { + g.set_limit(export_share(b)); + } lightcraft_gpu::set_pool_limit((b / 8) as u64); } @@ -174,6 +179,19 @@ pub fn work_gate() -> &'static WorkGate { static GATE: OnceLock = OnceLock::new(); +/// The process-wide gate for the photos of an export batch rendering side by side +/// ([`crate::export::run_batch`]; limit: twice the budget). Separate from [`work_gate`], which the +/// decodes inside each export take: one gate held across the other's acquire could deadlock. +pub fn export_gate() -> &'static WorkGate { + EXPORT_GATE.get_or_init(|| WorkGate::new(export_share(budget()))) +} + +fn export_share(budget: usize) -> usize { + budget.saturating_mul(2) +} + +static EXPORT_GATE: OnceLock = OnceLock::new(); + /// Entries and bytes held by one cache. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)] pub struct Usage { diff --git a/crates/ui-egui/src/export_task.rs b/crates/ui-egui/src/export_task.rs index fee92150..b012d6b5 100644 --- a/crates/ui-egui/src/export_task.rs +++ b/crates/ui-egui/src/export_task.rs @@ -1,7 +1,8 @@ //! Background export: the Export dialog, File → Export with Preset and Export with Previous hand //! their batch to a worker thread so the window stays responsive. Photos are prepared on the UI //! thread ([`lightcraft_engine::export::prepare_export`]: cheap, needs the session) and rendered, -//! encoded and written on the worker; a progress panel shows the count and a Cancel button. +//! encoded and written on the worker (several side by side: [`run_batch`]); a progress panel shows +//! the count and a Cancel button. //! //! Needs [`crate::Services::write_shared`] (a thread-safe writer); without it (web) the batch runs //! synchronously as before. diff --git a/docs/parity.md b/docs/parity.md index c06dad1b..1cb506c2 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -513,7 +513,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins. | LR-BEHAV-BATCH | Batch apply to selection | P0 | ✅ | `cmd:photo.rate`, `cmd:develop.paste`, `cmd:preset.apply`, `cmd:app.export` | | | LR-BEHAV-PREVIEW-HOVER | Hover previews | P1 | ✅ | `crates/ui-egui/src/panels/presets.rs`, `crates/ui-egui/src/panels/profiles.rs`, `crates/ui-egui/src/panels/right.rs` (`versions`), `crates/ui-egui/src/panels/detail.rs` | presets, profile menu, profile browser and versions preview in the loupe | | LR-BEHAV-PROGRESSIVE | Progressive rendering | P0 | ✅ | `crates/engine/src/media.rs`, `crates/preview/src/lib.rs` | | -| LR-BEHAV-BG-TASKS | Background tasks | P0 | ✅ | `crates/preview/src/lib.rs` (`JobPool`), `crates/ui-egui/src/export_task.rs`, `crates/ui-egui/src/import.rs` | renders off the UI thread; exports started from the UI run on a worker thread with a progress panel and Cancel (`ui.inspect` → `export`); imports show a progress window. No combined activity centre | +| LR-BEHAV-BG-TASKS | Background tasks | P0 | ✅ | `crates/preview/src/lib.rs` (`JobPool`), `crates/ui-egui/src/export_task.rs`, `crates/ui-egui/src/import.rs` | renders off the UI thread; exports started from the UI run on a worker thread with a progress panel and Cancel (`ui.inspect` → `export`), several photos of a batch side by side (`export_parallelism`, memory-gated; files, names and order as one at a time, #496); imports show a progress window. No combined activity centre | | LR-BEHAV-OFFLINE | Offline editing | P1 | ✅ | | local-first: everything works offline | | LR-BEHAV-WEB-SAFETY | Library safety in the browser build | P1 | 🟡 | `cmd:file.backupLibrary`, `cmd:file.restoreLibrary`, `apps/lightcraft-web/src/safety.rs`, `apps/lightcraft-web/src/backup.rs`, `apps/lightcraft-web/src/files.rs` | web only (experimental): Back Up Library (zip of the catalog and every stored original) and a non-destructive Restore; failed saves show as unsaved and are retried; a photo that can't be stored isn't added; one tab per library (Web Locks); notices for storage that isn't persistent and for a library that can't open; `?reset` asks first; a panic shows a message. Missing: `?safe` start, deleting originals of removed photos, zip64 (backups > 4 GB) | | LR-BEHAV-GPU | GPU acceleration | P0 | ✅ | `crates/gpu/src/render.rs`, `cmd:app.gpu`, `docs/gpu-pipeline.md` | CPU fallback per render on device limits, out of memory, device errors / loss, incomplete or black results (`perf.gpuReason`, `perf.gpuFallback`); short submissions for slow iGPUs; backend choice (DX12 only on Windows, `LIGHTCRAFT_GPU_BACKEND` / `WGPU_BACKEND`), device created after the window opens, crash sentinel (issue #136); no tiling beyond the buffer limit (CPU) |