From 5828a7d2d3180b10669c9e38c04a067536f5682a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L?= Date: Fri, 2 Oct 2026 23:16:29 +0200 Subject: [PATCH 1/2] Import: scan folders on a background thread with progress and Cancel Add Folder read every file on the UI thread before opening the review; on a NAS (~26k files) the window froze and was dropped. The scan now runs in a worker (ScanInput/scan_with) and the review opens when it finishes. Co-Authored-By: Claude Sonnet 5.5 --- crates/engine/src/import.rs | 133 +++++++++++++++++++++++++-------- crates/ui-egui/src/headless.rs | 31 +++++++- crates/ui-egui/src/import.rs | 84 ++++++++++++++++++--- crates/ui-egui/src/lib.rs | 5 ++ crates/ui-egui/src/menus.rs | 4 +- docs/parity.md | 2 +- 6 files changed, 212 insertions(+), 47 deletions(-) diff --git a/crates/engine/src/import.rs b/crates/engine/src/import.rs index 37b5bac4..dbea7fe4 100644 --- a/crates/engine/src/import.rs +++ b/crates/engine/src/import.rs @@ -261,11 +261,12 @@ pub fn expand(paths: &[String], skip: Option<&Path>) -> Vec { /// Probe files (headers + content hash) as an import would. pub(crate) fn probe_paths(s: &Session, paths: &[String]) -> Vec> { - probe_all(s, paths) + probe_all(s.media.file_probe.as_ref(), paths, &ScanProgress::default()) } -fn probe_all(s: &Session, paths: &[String]) -> Vec> { - let Some(probe) = s.media.file_probe.clone() else { +fn probe_all(probe: Option<&crate::media::FileProbe>, paths: &[String], progress: &ScanProgress) -> Vec> { + use std::sync::atomic::Ordering::Relaxed; + let Some(probe) = probe.cloned() else { return paths .iter() .map(|p| { @@ -291,7 +292,11 @@ fn probe_all(s: &Session, paths: &[String]) -> Vec> { if i >= paths.len() { break; } + if progress.cancel.load(Relaxed) { + break; + } let r = probe(&paths[i]); + progress.done.fetch_add(1, Relaxed); out.lock().unwrap_or_else(|e| e.into_inner())[i] = Some(r); } }); @@ -300,7 +305,17 @@ fn probe_all(s: &Session, paths: &[String]) -> Vec> { return results.into_iter().map(|r| r.unwrap_or_else(|| Err("not probed".into()))).collect(); } } - paths.iter().map(|p| probe(p)).collect() + paths + .iter() + .map(|p| { + if progress.cancel.load(Relaxed) { + return Err("cancelled".into()); + } + let r = probe(p); + progress.done.fetch_add(1, Relaxed); + r + }) + .collect() } /// Copy `src` into `root` (filed per `organize` by `date`) as `name` (default: its own name), @@ -331,53 +346,107 @@ fn copy_into(root: &Path, src: &str, date: &str, organize: Organize, name: Optio Ok(dst.to_string_lossy().to_string()) } +/// What a [`scan`] needs from the session, so it can run on another thread (a folder on a network +/// share can take minutes to read). +pub struct ScanInput { + probe: Option, + skip: Option, + /// path → (photo, its summary) for files already in the library. + by_path: HashMap, + by_hash: HashMap, + cache: HashMap, +} + +/// Progress and cancellation of a running [`scan_with`]. +#[derive(Default)] +pub struct ScanProgress { + /// Files to probe, set once the folders are expanded. + pub total: std::sync::atomic::AtomicUsize, + pub done: std::sync::atomic::AtomicUsize, + pub cancel: std::sync::atomic::AtomicBool, +} + +/// The result of [`scan_with`]: the candidates, and the probes to keep for the import that follows. +pub struct ScanOutput { + pub candidates: Vec, + pub probes: HashMap, +} + +impl ScanInput { + pub fn new(s: &mut Session, paths: &[String]) -> (Self, Vec) { + let skip = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone()); + let mut by_path = HashMap::new(); + let mut by_hash = HashMap::new(); + for p in s.catalog.photos() { + if let Source::File { path } = &p.source { + let c = ImportCandidate { + path: path.clone(), + format: p.format.clone(), + kind: p.kind, + width: p.width, + height: p.height, + file_size: p.file_size, + captured: p.captured.clone(), + duplicate: Some("path".into()), + existing: Some(p.id.0), + ..Default::default() + }; + by_path.insert(path.clone(), (p.id, c)); + } + if let Some(h) = &p.content_hash { + by_hash.insert(h.clone(), p.id); + } + } + let input = ScanInput { probe: s.media.file_probe.clone(), skip, by_path, by_hash, cache: std::mem::take(&mut s.import_probes) }; + (input, paths.to_vec()) + } +} + /// Find and probe what importing `paths` would add, marking duplicates (by path, or by content /// against the library and earlier candidates). Nothing is added; the probes are kept for the /// import that follows. pub fn scan(s: &mut Session, paths: &[String]) -> Vec { - let lib_dir = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone()); - let files = expand(paths, lib_dir.as_deref()); - let mut by_path: HashMap = HashMap::new(); - let mut by_hash: HashMap = HashMap::new(); - for p in s.catalog.photos() { - if let Source::File { path } = &p.source { - by_path.insert(path.clone(), p.id); - } - if let Some(h) = &p.content_hash { - by_hash.insert(h.clone(), p.id); - } - } - let todo: Vec = files.iter().filter(|f| !by_path.contains_key(f.as_str())).cloned().collect(); + let (input, paths) = ScanInput::new(s, paths); + let out = scan_with(input, &paths, &ScanProgress::default()); + s.import_probes = out.probes; + out.candidates +} + +/// [`scan`] without the session, so it can run on a worker thread. Stops early (returning what it +/// has) when `progress.cancel` is set. +pub fn scan_with(mut input: ScanInput, paths: &[String], progress: &ScanProgress) -> ScanOutput { + use std::sync::atomic::Ordering::Relaxed; + let files = expand(paths, input.skip.as_deref()); + let todo: Vec = files.iter().filter(|f| !input.by_path.contains_key(f.as_str())).cloned().collect(); + progress.total.store(todo.len(), Relaxed); // probes from a preceding `scan` are reused when the file is unchanged (same size) let cached: Vec> = todo .iter() .map(|f| { - let info = s.import_probes.remove(f)?; + let info = input.cache.remove(f)?; let size = std::fs::metadata(f).map(|m| m.len()).ok(); (size.is_none() || size == Some(info.file_size)).then_some(info) }) .collect(); let missing: Vec = todo.iter().zip(&cached).filter(|(_, c)| c.is_none()).map(|(f, _)| f.clone()).collect(); - let mut fresh = probe_all(s, &missing).into_iter(); + progress.done.store(todo.len() - missing.len(), Relaxed); + let mut fresh = probe_all(input.probe.as_ref(), &missing, progress).into_iter(); let probed: Vec> = cached.into_iter().map(|c| c.map(Ok).unwrap_or_else(|| fresh.next().unwrap_or_else(|| Err("not probed".into())))).collect(); crate::memory::release(); let mut probes: HashMap> = todo.into_iter().zip(probed).collect(); - let mut seen_hash: HashMap> = by_hash.into_iter().map(|(h, id)| (h, Some(id.0))).collect(); + let mut seen_hash: HashMap> = input.by_hash.into_iter().map(|(h, id)| (h, Some(id.0))).collect(); let mut out = Vec::with_capacity(files.len()); + let mut kept = HashMap::new(); for f in files { - let name = Path::new(&f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| f.clone()); - let mut c = ImportCandidate { path: f.clone(), name, ..Default::default() }; - if let Some(id) = by_path.get(&f) { - c.duplicate = Some("path".into()); - c.existing = Some(id.0); - if let Some(p) = s.catalog.photo(*id) { - (c.format, c.kind, c.width, c.height, c.file_size, c.captured) = - (p.format.clone(), p.kind, p.width, p.height, p.file_size, p.captured.clone()); - } + if let Some((_, c)) = input.by_path.get(&f) { + let mut c = c.clone(); + c.name = Path::new(&f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| f.clone()); out.push(c); continue; } + let name = Path::new(&f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| f.clone()); + let mut c = ImportCandidate { path: f.clone(), name, ..Default::default() }; match probes.remove(&f) { Some(Ok(info)) => { (c.format, c.kind, c.width, c.height, c.file_size, c.captured) = @@ -393,14 +462,14 @@ pub fn scan(s: &mut Session, paths: &[String]) -> Vec { } } } - s.import_probes.insert(f, info); + kept.insert(f, info); } Some(Err(e)) => c.error = Some(e), None => c.error = Some("not probed".into()), } out.push(c); } - out + ScanOutput { candidates: out, probes: kept } } /// Import files/folders. See the module docs. @@ -454,7 +523,7 @@ pub fn import_with(s: &mut Session, paths: &[String], opts: &ImportOptions) -> c } } - let probed = probe_all(s, &todo); + let probed = probe_all(s.media.file_probe.as_ref(), &todo, &ScanProgress::default()); crate::memory::release(); let now = (s.clock)(); let mut ops = promote; diff --git a/crates/ui-egui/src/headless.rs b/crates/ui-egui/src/headless.rs index c504a46f..d2669ea6 100644 --- a/crates/ui-egui/src/headless.rs +++ b/crates/ui-egui/src/headless.rs @@ -799,11 +799,36 @@ mod tests { h.app.ui.dialog = None; let r = h.request("engine.execute", json!({"command": "file.addFromDevice", "params": {"path": sub.to_string_lossy()}}), t); assert_eq!(r["ok"], true, "{r}"); + h.settle(SETTLE); let Some(crate::state::Dialog::Import { opts }) = &h.app.ui.dialog else { panic!("no import review") }; assert!(opts.copy); let _ = std::fs::remove_dir_all(&dir); } + /// The folder scan runs in the background: the request returns at once and the review opens + /// when the scan finishes (a folder on a network share must not freeze the window). + #[test] + fn add_folder_scans_in_the_background() { + let mut h = demo([1200.0, 900.0]); + let t = Duration::from_secs(10); + let dir = std::env::temp_dir().join(format!("lc-scanbg-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let img = lightcraft_raster::Rgba8::from_fn(8, 8, |x, y| [(x * 9) as u8, (y * 12) as u8, 80, 255]); + let png = lightcraft_engine::export::encode_image( + &img, + &lightcraft_engine::export::ExportOptions { format: lightcraft_engine::export::ExportFormat::Png, ..Default::default() }, + ) + .unwrap(); + std::fs::write(dir.join("a.png"), &png).unwrap(); + let r = h.request("engine.execute", json!({"command": "file.addFolder", "params": {"path": dir.to_string_lossy()}}), t); + assert_eq!(r["result"]["scanning"], true, "{r}"); + assert!(h.app.scan.is_some() || h.app.ui.dialog.is_some()); + h.settle(SETTLE); + assert!(h.app.scan.is_none()); + assert!(matches!(h.app.ui.dialog, Some(crate::state::Dialog::Import { .. }))); + let _ = std::fs::remove_dir_all(&dir); + } + /// ⇧⌘V opens Paste Selected Settings (prefilled with the copied groups); unchecking a group /// leaves it alone. ⌘F puts typing into the search field. #[test] @@ -914,9 +939,11 @@ mod tests { let n0 = h.app.session.catalog.len(); let undo0 = h.app.session.undo.len(); let r = h.request("engine.execute", json!({"command": "file.addPhotos", "params": {"paths": [dir.to_string_lossy()]}}), t); - assert_eq!(r["result"]["candidates"], 6, "{r}"); - assert_eq!(r["result"]["duplicates"], 1, "{r}"); + assert_eq!(r["result"]["scanning"], true, "{r}"); h.settle(SETTLE); + let Some(crate::state::Dialog::Import { opts }) = &h.app.ui.dialog else { panic!("no import review") }; + assert_eq!(opts.candidates.len(), 6); + assert_eq!(opts.candidates.iter().filter(|c| c.duplicate.is_some()).count(), 1); assert!(h.app.renderer.textures.keys().filter(|s| matches!(s, crate::render::Slot::Import(_))).count() >= 5, "thumbnails"); // uncheck the first photo; name a new album and keywords let r = h.request("ui.clickWidget", json!({"id": "import:0"}), t); diff --git a/crates/ui-egui/src/import.rs b/crates/ui-egui/src/import.rs index d7c5dfd3..ed0a5bef 100644 --- a/crates/ui-egui/src/import.rs +++ b/crates/ui-egui/src/import.rs @@ -5,9 +5,10 @@ //! frame, with a progress window; the whole import is one undo step. use egui::{Align2, Color32, Rect, Sense, Stroke, StrokeKind, pos2, vec2}; -use lightcraft_engine::import::ImportCandidate; +use lightcraft_engine::import::{ImportCandidate, ScanInput, ScanOutput, ScanProgress, scan_with}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; +use std::sync::atomic::Ordering; use crate::LightcraftApp; use crate::render::Slot; @@ -79,19 +80,82 @@ pub struct ImportTask { first: Option, } -/// Scan `paths` and open the review dialog. Returns the candidate counts. +/// A folder scan running on a worker thread (a network share can take minutes to read; the +/// window must keep answering meanwhile). +pub struct ScanTask { + progress: std::sync::Arc, + rx: std::sync::mpsc::Receiver, + /// Open the review with "copy into the library" checked (a camera / card). + pub copy: bool, +} + +/// Scan `paths` in the background, then open the review dialog (see [`poll_scan`]). pub fn open(app: &mut LightcraftApp, paths: Vec) -> Result { - let r = app.session.execute("library.importPreview", &json!({"paths": paths})).map_err(|e| e.to_string())?; - let candidates: Vec = serde_json::from_value(r["candidates"].clone()).unwrap_or_default(); - if candidates.is_empty() { - app.toast(&egui::Context::default(), "No photos found"); - return Ok(json!({"candidates": 0})); + if app.scan.is_some() { + return Err("a scan is already running".into()); + } + let (input, paths) = ScanInput::new(&mut app.session, &paths); + let progress = std::sync::Arc::new(ScanProgress::default()); + let (tx, rx) = std::sync::mpsc::channel(); + let p = progress.clone(); + let job = move || { + let _ = tx.send(scan_with(input, &paths, &p)); + }; + #[cfg(not(target_arch = "wasm32"))] + std::thread::spawn(job); + #[cfg(target_arch = "wasm32")] + job(); + app.scan = Some(ScanTask { progress, rx, copy: false }); + Ok(json!({"scanning": true})) +} + +/// Collect a finished scan and open the review (called every frame). +pub fn poll_scan(app: &mut LightcraftApp, ctx: &egui::Context) { + let Some(task) = app.scan.as_ref() else { return }; + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + let out = match task.rx.try_recv() { + Ok(o) => o, + Err(std::sync::mpsc::TryRecvError::Empty) => return, + Err(_) => { + app.scan = None; + app.toast(ctx, "Scan failed"); + return; + } + }; + let task = app.scan.take().expect("scan task"); + if task.progress.cancel.load(Ordering::Relaxed) { + return; + } + app.session.import_probes = out.probes; + if out.candidates.is_empty() { + app.toast(ctx, "No photos found"); + return; } app.renderer.forget_imports(); - let d = ImportDialog::new(candidates); - let out = json!({"candidates": d.candidates.len(), "duplicates": r["duplicates"]}); + let mut d = ImportDialog::new(out.candidates); + d.copy = task.copy; app.ui.dialog = Some(crate::state::Dialog::Import { opts: Box::new(d) }); - Ok(out) +} + +/// The progress window while a folder is being scanned. +pub fn scan_progress(app: &mut LightcraftApp, ctx: &egui::Context) { + let Some(task) = &app.scan else { return }; + let t = Tokens::get(ctx); + let total = task.progress.total.load(Ordering::Relaxed); + let done = task.progress.done.load(Ordering::Relaxed); + let text = if total == 0 { "Looking for photos…".to_string() } else { format!("Reading photos… {done} of {total}") }; + let mut cancel = false; + egui::Window::new("Scanning").title_bar(false).resizable(false).anchor(Align2::CENTER_BOTTOM, [0.0, -80.0]).fixed_size([340.0, 80.0]).show( + ctx, + |ui| { + ui.label(egui::RichText::new(text).color(t.text)); + ui.add(egui::ProgressBar::new(done as f32 / total.max(1) as f32).desired_width(320.0)); + cancel = ui.button("Cancel").clicked(); + }, + ); + if cancel { + task.progress.cancel.store(true, Ordering::Relaxed); + } } /// Start importing the dialog's checked files (the dialog's OK / `ui.dialog.confirm`). diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 56341733..6a67bd16 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -125,6 +125,8 @@ pub struct LightcraftApp { pub merge: merge::MergeState, /// An import in progress (the import review dialog's batches). pub import: Option, + /// A folder scan in progress (feeds the import review). + pub scan: Option, /// A background export in progress. pub export: Option, /// The files of the last finished background export (`ui.inspect` → `export.last`). @@ -172,6 +174,7 @@ impl LightcraftApp { loupe_shown: None, merge: merge::MergeState::default(), import: None, + scan: None, export: None, last_export_result: None, hover_preview: None, @@ -404,6 +407,7 @@ impl LightcraftApp { } self.renderer.poll(ctx, &mut self.session); merge::poll(self, ctx); + import::poll_scan(self, ctx); import::tick(self, ctx); self.preview_build_status(ctx); self.slideshow_tick(ctx); @@ -565,6 +569,7 @@ impl LightcraftApp { panels::second::show(self, &ctx); panels::dialogs::show(self, &ctx); import::progress(self, &ctx); + import::scan_progress(self, &ctx); export_task::poll(self, &ctx); panels::grid::drag_feedback(self, &ctx); panels::toast(self, &ctx); diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index 5112b531..90b714cd 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -879,8 +879,8 @@ pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option Date: Sat, 3 Oct 2026 18:25:05 +0200 Subject: [PATCH 2/2] Import: update copy-import test for the background folder scan Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/headless.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/ui-egui/src/headless.rs b/crates/ui-egui/src/headless.rs index d2669ea6..bf8d442d 100644 --- a/crates/ui-egui/src/headless.rs +++ b/crates/ui-egui/src/headless.rs @@ -990,8 +990,10 @@ mod tests { let mut h = Headless::new(app, [1300.0, 900.0], 1.0); let t = Duration::from_secs(10); let r = h.request("engine.execute", json!({"command": "file.addPhotos", "params": {"paths": [src.to_string_lossy()]}}), t); - assert_eq!(r["result"]["candidates"], 10, "{r}"); + assert_eq!(r["result"]["scanning"], true, "{r}"); h.settle(SETTLE); + let Some(crate::state::Dialog::Import { opts }) = &h.app.ui.dialog else { panic!("no import review") }; + assert_eq!(opts.candidates.len(), 10); for id in ["button:importCopy", "button:importDest"] { let r = h.request("ui.clickWidget", json!({"id": id}), t); assert_eq!(r["ok"], true, "{id}: {r}");