shifathosn
/
lightcraft
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2 from GlemNL/fix/nas-folder-import 

Import: scan folders in the background (fixes Add Folder on network shares)
This commit is contained in:
Brandon Thomas 2026-10-03 18:15:21 -04:00 committed by GitHub
parent 7bf24ff856 97ba282044
commit d473a4d9bf
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 215 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

133
crates/engine/src/import.rs
View File

@ -261,11 +261,12 @@ pub fn expand(paths: &[String], skip: Option<&Path>) -> Vec<String> {
/// Probe files (headers + content hash) as an import would.
pub(crate) fn probe_paths(s: &Session, paths: &[String]) -> Vec<Result<ProbeInfo, String>> {
probe_all(s, paths)
probe_all(s.media.file_probe.as_ref(), paths, &ScanProgress::default())
}
fn probe_all(s: &Session, paths: &[String]) -> Vec<Result<ProbeInfo, String>> {
let Some(probe) = s.media.file_probe.clone() else {
fn probe_all(probe: Option<&crate::media::FileProbe>, paths: &[String], progress: &ScanProgress) -> Vec<Result<ProbeInfo, String>> {
use std::sync::atomic::Ordering::Relaxed;
let Some(probe) = probe.cloned() else {
return paths
.iter()
.map(|p| {
@ -291,7 +292,11 @@ fn probe_all(s: &Session, paths: &[String]) -> Vec<Result<ProbeInfo, String>> {
if i >= paths.len() {
break;
}
if progress.cancel.load(Relaxed) {
break;
}
let r = probe(&paths[i]);
progress.done.fetch_add(1, Relaxed);
out.lock().unwrap_or_else(|e| e.into_inner())[i] = Some(r);
}
});
@ -300,7 +305,17 @@ fn probe_all(s: &Session, paths: &[String]) -> Vec<Result<ProbeInfo, String>> {
return results.into_iter().map(|r| r.unwrap_or_else(|| Err("not probed".into()))).collect();
}
}
paths.iter().map(|p| probe(p)).collect()
paths
.iter()
.map(|p| {
if progress.cancel.load(Relaxed) {
return Err("cancelled".into());
}
let r = probe(p);
progress.done.fetch_add(1, Relaxed);
r
})
.collect()
}
/// Copy `src` into `root` (filed per `organize` by `date`) as `name` (default: its own name),
@ -331,53 +346,107 @@ fn copy_into(root: &Path, src: &str, date: &str, organize: Organize, name: Optio
Ok(dst.to_string_lossy().to_string())
}
/// What a [`scan`] needs from the session, so it can run on another thread (a folder on a network
/// share can take minutes to read).
pub struct ScanInput {
probe: Option<crate::media::FileProbe>,
skip: Option<PathBuf>,
/// path → (photo, its summary) for files already in the library.
by_path: HashMap<String, (PhotoId, ImportCandidate)>,
by_hash: HashMap<String, PhotoId>,
cache: HashMap<String, ProbeInfo>,
}
/// Progress and cancellation of a running [`scan_with`].
#[derive(Default)]
pub struct ScanProgress {
/// Files to probe, set once the folders are expanded.
pub total: std::sync::atomic::AtomicUsize,
pub done: std::sync::atomic::AtomicUsize,
pub cancel: std::sync::atomic::AtomicBool,
}
/// The result of [`scan_with`]: the candidates, and the probes to keep for the import that follows.
pub struct ScanOutput {
pub candidates: Vec<ImportCandidate>,
pub probes: HashMap<String, ProbeInfo>,
}
impl ScanInput {
pub fn new(s: &mut Session, paths: &[String]) -> (Self, Vec<String>) {
let skip = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone());
let mut by_path = HashMap::new();
let mut by_hash = HashMap::new();
for p in s.catalog.photos() {
if let Source::File { path } = &p.source {
let c = ImportCandidate {
path: path.clone(),
format: p.format.clone(),
kind: p.kind,
width: p.width,
height: p.height,
file_size: p.file_size,
captured: p.captured.clone(),
duplicate: Some("path".into()),
existing: Some(p.id.0),
..Default::default()
};
by_path.insert(path.clone(), (p.id, c));
}
if let Some(h) = &p.content_hash {
by_hash.insert(h.clone(), p.id);
}
}
let input = ScanInput { probe: s.media.file_probe.clone(), skip, by_path, by_hash, cache: std::mem::take(&mut s.import_probes) };
(input, paths.to_vec())
}
}
/// Find and probe what importing `paths` would add, marking duplicates (by path, or by content
/// against the library and earlier candidates). Nothing is added; the probes are kept for the
/// import that follows.
pub fn scan(s: &mut Session, paths: &[String]) -> Vec<ImportCandidate> {
let lib_dir = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone());
let files = expand(paths, lib_dir.as_deref());
let mut by_path: HashMap<String, PhotoId> = HashMap::new();
let mut by_hash: HashMap<String, PhotoId> = HashMap::new();
for p in s.catalog.photos() {
if let Source::File { path } = &p.source {
by_path.insert(path.clone(), p.id);
}
if let Some(h) = &p.content_hash {
by_hash.insert(h.clone(), p.id);
}
}
let todo: Vec<String> = files.iter().filter(|f| !by_path.contains_key(f.as_str())).cloned().collect();
let (input, paths) = ScanInput::new(s, paths);
let out = scan_with(input, &paths, &ScanProgress::default());
s.import_probes = out.probes;
out.candidates
}
/// [`scan`] without the session, so it can run on a worker thread. Stops early (returning what it
/// has) when `progress.cancel` is set.
pub fn scan_with(mut input: ScanInput, paths: &[String], progress: &ScanProgress) -> ScanOutput {
use std::sync::atomic::Ordering::Relaxed;
let files = expand(paths, input.skip.as_deref());
let todo: Vec<String> = files.iter().filter(|f| !input.by_path.contains_key(f.as_str())).cloned().collect();
progress.total.store(todo.len(), Relaxed);
// probes from a preceding `scan` are reused when the file is unchanged (same size)
let cached: Vec<Option<ProbeInfo>> = todo
.iter()
.map(|f| {
let info = s.import_probes.remove(f)?;
let info = input.cache.remove(f)?;
let size = std::fs::metadata(f).map(|m| m.len()).ok();
(size.is_none() || size == Some(info.file_size)).then_some(info)
})
.collect();
let missing: Vec<String> = todo.iter().zip(&cached).filter(|(_, c)| c.is_none()).map(|(f, _)| f.clone()).collect();
let mut fresh = probe_all(s, &missing).into_iter();
progress.done.store(todo.len() - missing.len(), Relaxed);
let mut fresh = probe_all(input.probe.as_ref(), &missing, progress).into_iter();
let probed: Vec<Result<ProbeInfo, String>> =
cached.into_iter().map(|c| c.map(Ok).unwrap_or_else(|| fresh.next().unwrap_or_else(|| Err("not probed".into())))).collect();
crate::memory::release();
let mut probes: HashMap<String, Result<ProbeInfo, String>> = todo.into_iter().zip(probed).collect();
let mut seen_hash: HashMap<String, Option<u64>> = by_hash.into_iter().map(|(h, id)| (h, Some(id.0))).collect();
let mut seen_hash: HashMap<String, Option<u64>> = input.by_hash.into_iter().map(|(h, id)| (h, Some(id.0))).collect();
let mut out = Vec::with_capacity(files.len());
let mut kept = HashMap::new();
for f in files {
let name = Path::new(&f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| f.clone());
let mut c = ImportCandidate { path: f.clone(), name, ..Default::default() };
if let Some(id) = by_path.get(&f) {
c.duplicate = Some("path".into());
c.existing = Some(id.0);
if let Some(p) = s.catalog.photo(*id) {
(c.format, c.kind, c.width, c.height, c.file_size, c.captured) =
(p.format.clone(), p.kind, p.width, p.height, p.file_size, p.captured.clone());
}
if let Some((_, c)) = input.by_path.get(&f) {
let mut c = c.clone();
c.name = Path::new(&f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| f.clone());
out.push(c);
continue;
}
let name = Path::new(&f).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| f.clone());
let mut c = ImportCandidate { path: f.clone(), name, ..Default::default() };
match probes.remove(&f) {
Some(Ok(info)) => {
(c.format, c.kind, c.width, c.height, c.file_size, c.captured) =
@ -393,14 +462,14 @@ pub fn scan(s: &mut Session, paths: &[String]) -> Vec<ImportCandidate> {
}
}
}
s.import_probes.insert(f, info);
kept.insert(f, info);
}
Some(Err(e)) => c.error = Some(e),
None => c.error = Some("not probed".into()),
}
out.push(c);
}
out
ScanOutput { candidates: out, probes: kept }
}
/// Import files/folders. See the module docs.
@ -454,7 +523,7 @@ pub fn import_with(s: &mut Session, paths: &[String], opts: &ImportOptions) -> c
}
}
let probed = probe_all(s, &todo);
let probed = probe_all(s.media.file_probe.as_ref(), &todo, &ScanProgress::default());
crate::memory::release();
let now = (s.clock)();
let mut ops = promote;

35
crates/ui-egui/src/headless.rs
View File

@ -799,11 +799,36 @@ mod tests {
h.app.ui.dialog = None;
let r = h.request("engine.execute", json!({"command": "file.addFromDevice", "params": {"path": sub.to_string_lossy()}}), t);
assert_eq!(r["ok"], true, "{r}");
h.settle(SETTLE);
let Some(crate::state::Dialog::Import { opts }) = &h.app.ui.dialog else { panic!("no import review") };
assert!(opts.copy);
let _ = std::fs::remove_dir_all(&dir);
}
/// The folder scan runs in the background: the request returns at once and the review opens
/// when the scan finishes (a folder on a network share must not freeze the window).
#[test]
fn add_folder_scans_in_the_background() {
let mut h = demo([1200.0, 900.0]);
let t = Duration::from_secs(10);
let dir = std::env::temp_dir().join(format!("lc-scanbg-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let img = lightcraft_raster::Rgba8::from_fn(8, 8, |x, y| [(x * 9) as u8, (y * 12) as u8, 80, 255]);
let png = lightcraft_engine::export::encode_image(
&img,
&lightcraft_engine::export::ExportOptions { format: lightcraft_engine::export::ExportFormat::Png, ..Default::default() },
)
.unwrap();
std::fs::write(dir.join("a.png"), &png).unwrap();
let r = h.request("engine.execute", json!({"command": "file.addFolder", "params": {"path": dir.to_string_lossy()}}), t);
assert_eq!(r["result"]["scanning"], true, "{r}");
assert!(h.app.scan.is_some() || h.app.ui.dialog.is_some());
h.settle(SETTLE);
assert!(h.app.scan.is_none());
assert!(matches!(h.app.ui.dialog, Some(crate::state::Dialog::Import { .. })));
let _ = std::fs::remove_dir_all(&dir);
}
/// ⇧⌘V opens Paste Selected Settings (prefilled with the copied groups); unchecking a group
/// leaves it alone. ⌘F puts typing into the search field.
#[test]
@ -914,9 +939,11 @@ mod tests {
let n0 = h.app.session.catalog.len();
let undo0 = h.app.session.undo.len();
let r = h.request("engine.execute", json!({"command": "file.addPhotos", "params": {"paths": [dir.to_string_lossy()]}}), t);
assert_eq!(r["result"]["candidates"], 6, "{r}");
assert_eq!(r["result"]["duplicates"], 1, "{r}");
assert_eq!(r["result"]["scanning"], true, "{r}");
h.settle(SETTLE);
let Some(crate::state::Dialog::Import { opts }) = &h.app.ui.dialog else { panic!("no import review") };
assert_eq!(opts.candidates.len(), 6);
assert_eq!(opts.candidates.iter().filter(|c| c.duplicate.is_some()).count(), 1);
assert!(h.app.renderer.textures.keys().filter(|s| matches!(s, crate::render::Slot::Import(_))).count() >= 5, "thumbnails");
// uncheck the first photo; name a new album and keywords
let r = h.request("ui.clickWidget", json!({"id": "import:0"}), t);
@ -963,8 +990,10 @@ mod tests {
let mut h = Headless::new(app, [1300.0, 900.0], 1.0);
let t = Duration::from_secs(10);
let r = h.request("engine.execute", json!({"command": "file.addPhotos", "params": {"paths": [src.to_string_lossy()]}}), t);
assert_eq!(r["result"]["candidates"], 10, "{r}");
assert_eq!(r["result"]["scanning"], true, "{r}");
h.settle(SETTLE);
let Some(crate::state::Dialog::Import { opts }) = &h.app.ui.dialog else { panic!("no import review") };
assert_eq!(opts.candidates.len(), 10);
for id in ["button:importCopy", "button:importDest"] {
let r = h.request("ui.clickWidget", json!({"id": id}), t);
assert_eq!(r["ok"], true, "{id}: {r}");

84
crates/ui-egui/src/import.rs
View File

@ -5,9 +5,10 @@
//! frame, with a progress window; the whole import is one undo step.
use egui::{Align2, Color32, Rect, Sense, Stroke, StrokeKind, pos2, vec2};
use lightcraft_engine::import::ImportCandidate;
use lightcraft_engine::import::{ImportCandidate, ScanInput, ScanOutput, ScanProgress, scan_with};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use std::sync::atomic::Ordering;
use crate::LightcraftApp;
use crate::render::Slot;
@ -79,19 +80,82 @@ pub struct ImportTask {
first: Option<u64>,
}
/// Scan `paths` and open the review dialog. Returns the candidate counts.
/// A folder scan running on a worker thread (a network share can take minutes to read; the
/// window must keep answering meanwhile).
pub struct ScanTask {
progress: std::sync::Arc<ScanProgress>,
rx: std::sync::mpsc::Receiver<ScanOutput>,
/// Open the review with "copy into the library" checked (a camera / card).
pub copy: bool,
}
/// Scan `paths` in the background, then open the review dialog (see [`poll_scan`]).
pub fn open(app: &mut LightcraftApp, paths: Vec<String>) -> Result<Value, String> {
let r = app.session.execute("library.importPreview", &json!({"paths": paths})).map_err(|e| e.to_string())?;
let candidates: Vec<ImportCandidate> = serde_json::from_value(r["candidates"].clone()).unwrap_or_default();
if candidates.is_empty() {
app.toast(&egui::Context::default(), "No photos found");
return Ok(json!({"candidates": 0}));
if app.scan.is_some() {
return Err("a scan is already running".into());
}
let (input, paths) = ScanInput::new(&mut app.session, &paths);
let progress = std::sync::Arc::new(ScanProgress::default());
let (tx, rx) = std::sync::mpsc::channel();
let p = progress.clone();
let job = move || {
let _ = tx.send(scan_with(input, &paths, &p));
};
#[cfg(not(target_arch = "wasm32"))]
std::thread::spawn(job);
#[cfg(target_arch = "wasm32")]
job();
app.scan = Some(ScanTask { progress, rx, copy: false });
Ok(json!({"scanning": true}))
}
/// Collect a finished scan and open the review (called every frame).
pub fn poll_scan(app: &mut LightcraftApp, ctx: &egui::Context) {
let Some(task) = app.scan.as_ref() else { return };
ctx.request_repaint_after(std::time::Duration::from_millis(100));
let out = match task.rx.try_recv() {
Ok(o) => o,
Err(std::sync::mpsc::TryRecvError::Empty) => return,
Err(_) => {
app.scan = None;
app.toast(ctx, "Scan failed");
return;
}
};
let task = app.scan.take().expect("scan task");
if task.progress.cancel.load(Ordering::Relaxed) {
return;
}
app.session.import_probes = out.probes;
if out.candidates.is_empty() {
app.toast(ctx, "No photos found");
return;
}
app.renderer.forget_imports();
let d = ImportDialog::new(candidates);
let out = json!({"candidates": d.candidates.len(), "duplicates": r["duplicates"]});
let mut d = ImportDialog::new(out.candidates);
d.copy = task.copy;
app.ui.dialog = Some(crate::state::Dialog::Import { opts: Box::new(d) });
Ok(out)
}
/// The progress window while a folder is being scanned.
pub fn scan_progress(app: &mut LightcraftApp, ctx: &egui::Context) {
let Some(task) = &app.scan else { return };
let t = Tokens::get(ctx);
let total = task.progress.total.load(Ordering::Relaxed);
let done = task.progress.done.load(Ordering::Relaxed);
let text = if total == 0 { "Looking for photos…".to_string() } else { format!("Reading photos… {done} of {total}") };
let mut cancel = false;
egui::Window::new("Scanning").title_bar(false).resizable(false).anchor(Align2::CENTER_BOTTOM, [0.0, -80.0]).fixed_size([340.0, 80.0]).show(
ctx,
|ui| {
ui.label(egui::RichText::new(text).color(t.text));
ui.add(egui::ProgressBar::new(done as f32 / total.max(1) as f32).desired_width(320.0));
cancel = ui.button("Cancel").clicked();
},
);
if cancel {
task.progress.cancel.store(true, Ordering::Relaxed);
}
}
/// Start importing the dialog's checked files (the dialog's OK / `ui.dialog.confirm`).

5
crates/ui-egui/src/lib.rs
View File

@ -125,6 +125,8 @@ pub struct LightcraftApp {
pub merge: merge::MergeState,
/// An import in progress (the import review dialog's batches).
pub import: Option<import::ImportTask>,
/// A folder scan in progress (feeds the import review).
pub scan: Option<import::ScanTask>,
/// A background export in progress.
pub export: Option<export_task::ExportTask>,
/// The files of the last finished background export (`ui.inspect` → `export.last`).
@ -172,6 +174,7 @@ impl LightcraftApp {
loupe_shown: None,
merge: merge::MergeState::default(),
import: None,
scan: None,
export: None,
last_export_result: None,
hover_preview: None,
@ -404,6 +407,7 @@ impl LightcraftApp {
}
self.renderer.poll(ctx, &mut self.session);
merge::poll(self, ctx);
import::poll_scan(self, ctx);
import::tick(self, ctx);
self.preview_build_status(ctx);
self.slideshow_tick(ctx);
@ -565,6 +569,7 @@ impl LightcraftApp {
panels::second::show(self, &ctx);
panels::dialogs::show(self, &ctx);
import::progress(self, &ctx);
import::scan_progress(self, &ctx);
export_task::poll(self, &ctx);
panels::grid::drag_feedback(self, &ctx);
panels::toast(self, &ctx);

4
crates/ui-egui/src/menus.rs
View File

@ -879,8 +879,8 @@ pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option<Re
},
};
let r = crate::import::open(app, vec![path]);
if let Some(crate::state::Dialog::Import { opts }) = &mut app.ui.dialog {
opts.copy = true;
if let Some(t) = &mut app.scan {
t.copy = true;
}
r
}

2
docs/parity.md
View File

@ -98,7 +98,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins.
| Id | Feature | Tier | Status | Evidence | Missing / notes |
|---|---|---|---|---|---|
| LR-IMP-ADD-DIALOG | Add photos/folders | P0 | ✅ | `cmd:file.addPhotos`, `cmd:library.importPreview`, `cmd:library.import` (`mode`, `album`, `albumName`, `preset`, `keywords`), `crates/ui-egui/src/import.rs` | review dialog: candidate grid with thumbnails and checkboxes (duplicates by path/content marked and unchecked), destination, album (existing/new), preset, keywords; batched import with a progress window; one undo step. Drag-and-drop still imports directly |
| LR-IMP-ADD-DIALOG | Add photos/folders | P0 | ✅ | `cmd:file.addPhotos`, `cmd:library.importPreview`, `cmd:library.import` (`mode`, `album`, `albumName`, `preset`, `keywords`), `crates/ui-egui/src/import.rs` | review dialog: candidate grid with thumbnails and checkboxes (duplicates by path/content marked and unchecked), destination, album (existing/new), preset, keywords; the folder scan runs on a background thread with a progress window and Cancel (a NAS folder no longer freezes the window); batched import with a progress window; one undo step. Drag-and-drop still imports directly |
| LR-IMP-DRAGDROP | Drop files/folders to import | P0 | ✅ | `crates/ui-egui/src/lib.rs` (dropped files → `cmd:library.import`) | dropping onto a specific album not supported |
| LR-IMP-DUPES | Skip duplicates by content | P1 | ✅ | `crates/engine/src/import.rs`, `crates/engine/src/tests_import.rs` | |
| LR-IMP-DEVICE | Import from camera/card | P1 | ✅ | `cmd:library.devices`, `cmd:file.addFromDevice`, `crates/engine/src/devices.rs` | mounted volumes with a DCIM folder (macOS /Volumes, Linux /media and /run/media, Windows drive letters); File → Add from Device → the import review, copying into the library by default; no PTP/MTP (cameras that don't mount as a disk) |