diff --git a/crates/engine/src/cmd/browse.rs b/crates/engine/src/cmd/browse.rs index a8a192f9..9be18351 100644 --- a/crates/engine/src/cmd/browse.rs +++ b/crates/engine/src/cmd/browse.rs @@ -107,7 +107,7 @@ impl Session { let smart = self.media.smart_dir.clone(); let plan = self .catalog - .forget_local_plan(&now, days, &|p| keep.contains(&p.id) || smart.as_ref().is_some_and(|d| d.join(crate::smart::file_name(p)).exists())); + .forget_local_plan(&now, days, &|p| keep.contains(&p.id) || smart.as_ref().is_some_and(|d| crate::smart::file_path(d, p).exists())); if dry_run { return plan; } diff --git a/crates/engine/src/cmd/previews.rs b/crates/engine/src/cmd/previews.rs index 372a539d..b46c1352 100644 --- a/crates/engine/src/cmd/previews.rs +++ b/crates/engine/src/cmd/previews.rs @@ -243,6 +243,9 @@ fn smart_run( break; } let _one = SMART_WRITE.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + // Read either the new shard or a pre-upgrade flat file, without making new work + // depend on having the original online. + let path = crate::smart::existing_file_path(dir, path); if discard { if std::fs::remove_file(&path).is_ok() { n.removed += 1; @@ -266,7 +269,14 @@ fn smart_run( .ok_or_else(|| "nothing to build from".to_string()) .and_then(|s| s.load_source()) .and_then(|src| crate::smart::encode(&src.image, src.info_or(Default::default()).camera_tone.as_ref())) - .and_then(|b| lightcraft_catalog::safe_file::write_atomic(&path, &b).map_err(|e| format!("{}: {e}", path.display()))); + .and_then(|b| { + let parent = path.parent().ok_or_else(|| format!("{}: no parent folder", path.display()))?; + // Only create the leaf shard: do not recreate a missing external drive. + if !parent.is_dir() { + std::fs::create_dir(parent).map_err(|e| format!("{}: {e}", parent.display()))?; + } + lightcraft_catalog::safe_file::write_atomic(&path, &b).map_err(|e| format!("{}: {e}", path.display())) + }); match r { Ok(()) => { n.built += 1; @@ -321,7 +331,7 @@ fn smart(s: &mut Session, p: &Value) -> Result { if !matches!(ph.source, lightcraft_catalog::Source::File { .. }) { continue; } - let path = dir.join(crate::smart::file_name(&ph)); + let path = crate::smart::sharded_file_path(&dir, &ph); let source = (!discard).then(|| s.media.source_ref(&ph, crate::media::SourceLevel::Preview)); jobs.push((id, path, source, ph.relative_wb())); } @@ -367,7 +377,7 @@ pub(crate) fn refresh_stale_smart_previews(s: &mut Session) -> Option = s.catalog.photos().filter(|p| matches!(p.source, lightcraft_catalog::Source::File { .. }) && p.relative_wb()).collect(); let jobs: Vec = - photos.iter().map(|p| (p.id, dir.join(crate::smart::file_name(p)), Some(s.media.origin_ref(&p.source, max_edge)), true)).collect(); + photos.iter().map(|p| (p.id, crate::smart::sharded_file_path(&dir, p), Some(s.media.origin_ref(&p.source, max_edge)), true)).collect(); std::thread::Builder::new() .name("lc-smart-look".into()) .spawn(move || refresh_stale(&dir, jobs)) @@ -378,7 +388,11 @@ pub(crate) fn refresh_stale_smart_previews(s: &mut Session) -> Option) { - let stale: Vec = jobs.into_iter().filter(|(_, path, _, _)| crate::smart::is_stale(path)).collect(); + let stale: Vec = jobs + .into_iter() + .map(|(id, path, source, look)| (id, crate::smart::existing_file_path(dir, path), source, look)) + .filter(|(_, path, _, _)| crate::smart::is_stale(path)) + .collect(); if !stale.is_empty() { // `custom`: the folder is there already, and must not be recreated if it went away since match smart_run(dir, true, false, stale, None) { @@ -461,7 +475,7 @@ fn smart_location(s: &mut Session, p: &Value) -> Result { /// Whether photo `id` has a smart preview. pub fn has_smart_preview(s: &Session, id: lightcraft_catalog::PhotoId) -> bool { match (&s.media.smart_dir, s.catalog.photo(id)) { - (Some(dir), Some(p)) => crate::smart::is_valid(&dir.join(crate::smart::file_name(p))), + (Some(dir), Some(p)) => crate::smart::is_valid(&crate::smart::file_path(dir, p)), _ => false, } } diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rs index ea61c347..fb42d7b9 100644 --- a/crates/engine/src/media.rs +++ b/crates/engine/src/media.rs @@ -635,7 +635,9 @@ impl MediaCache { // be offline is tried first and its smart preview is the render worker's fallback. #[cfg(not(target_arch = "wasm32"))] if let (Source::File { path }, Some(dir)) = (&p.source, &self.smart_dir) { - let sp = dir.join(crate::smart::file_name(p)); + let sharded = crate::smart::sharded_file_path(dir, p); + let legacy = dir.join(crate::smart::file_name(p)); + let sp = if self.availability.exists(&sharded.to_string_lossy()) == Some(true) { sharded } else { legacy }; // (a proxy cut short fails to load, like the offline original; the worker's fallback // in `SourceRef::load` only uses a proxy that passes `smart::is_valid`) if self.availability.is_offline(path) && self.availability.exists(&sp.to_string_lossy()) == Some(true) { diff --git a/crates/engine/src/smart.rs b/crates/engine/src/smart.rs index cda0baa6..532454a1 100644 --- a/crates/engine/src/smart.rs +++ b/crates/engine/src/smart.rs @@ -21,6 +21,30 @@ pub fn file_name(p: &Photo) -> String { format!("{:032x}.lcsp", h.0) } +/// The default sharded location of a proxy. A two-character hex prefix bounds each +/// folder to about 1/256 of the library's smart previews. +pub fn sharded_file_path(dir: &Path, p: &Photo) -> PathBuf { + let name = file_name(p); + dir.join(&name[..2]).join(name) +} + +/// Read an existing proxy in either layout. New proxies use the sharded location; previews +/// made by older releases still load and can be moved with smartPreviewsLocation. +pub fn file_path(dir: &Path, p: &Photo) -> PathBuf { + existing_file_path(dir, sharded_file_path(dir, p)) +} + +/// Resolve a sharded path to an older flat file when present. Call only off the UI thread: +/// checking a removable drive synchronously could stall drawing. +pub fn existing_file_path(dir: &Path, sharded: PathBuf) -> PathBuf { + if sharded.exists() { + return sharded; + } + let Some(name) = sharded.file_name() else { return sharded }; + let legacy = dir.join(name); + if legacy.exists() { legacy } else { sharded } +} + /// The most header [`is_valid`] reads (the JSON line before the JPEG). const HEADER_MAX: u64 = 4096; @@ -104,12 +128,45 @@ pub fn check_writable(dir: &Path) -> Result<(), String> { Ok(()) } +/// The owned proxy files in both layouts. Do not follow symlinked directories or +/// traverse unrelated folders inside a user-selected smart preview directory. +fn files(dir: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() }; + let mut found = Vec::new(); + for e in entries.flatten() { + let Ok(kind) = e.file_type() else { continue }; + if kind.is_file() && e.path().extension().is_some_and(|ext| ext == "lcsp") { + found.push(e.path()); // legacy layout + } else if kind.is_dir() { + let shard_name = e.file_name(); + let Some(shard) = shard_name.to_str() else { continue }; + if shard.len() != 2 || !shard.bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()) { + continue; + } + let Ok(children) = std::fs::read_dir(e.path()) else { continue }; + for child in children.flatten() { + let name = child.file_name(); + let Some(name) = name.to_str() else { continue }; + let Some(stem) = name.strip_suffix(".lcsp") else { continue }; + if stem.len() == 32 + && stem.starts_with(shard) + && stem.bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()) + && child.file_type().is_ok_and(|kind| kind.is_file()) + { + found.push(child.path()); + } + } + } + } + found +} + /// The number of smart previews in `dir` and their total size in bytes. pub fn stats(dir: &Path) -> (usize, u64) { - let Ok(rd) = std::fs::read_dir(dir) else { return (0, 0) }; - rd.flatten() - .filter(|e| e.path().extension().is_some_and(|x| x == "lcsp")) - .fold((0, 0), |(n, b), e| (n + 1, b + e.metadata().map_or(0, |m| m.len()))) + files(dir).into_iter().fold((0usize, 0u64), |(count, bytes), path| { + let size = std::fs::metadata(path).map_or(0, |m| m.len()); + (count.saturating_add(1), bytes.saturating_add(size)) + }) } /// What to do with the smart previews in the old folder when the location changes. @@ -140,24 +197,30 @@ pub fn migrate(from: &Path, to: &Path, what: Existing) -> (usize, Vec) { if what == Existing::Leave { return (done, failed); } - let Ok(rd) = std::fs::read_dir(from) else { return (done, failed) }; - for e in rd.flatten().filter(|e| e.path().extension().is_some_and(|x| x == "lcsp")) { - let src = e.path(); + for src in files(from) { + let Some(name) = src.file_name() else { continue }; + let dst = name.to_str().and_then(|n| n.strip_suffix(".lcsp")).filter(|stem| { + stem.len() == 32 && stem.bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()) + }).map_or_else(|| to.join(name), |stem| to.join(&stem[..2]).join(name)); let r = match what { Existing::Discard => std::fs::remove_file(&src), _ => { - let dst = to.join(e.file_name()); - if is_valid(&dst) { - // named by content: the complete copy already there is the same preview - std::fs::remove_file(&src) - } else { - // across drives: copied atomically (never a partial proxy at the new place) - std::fs::rename(&src, &dst).or_else(|_| { - std::fs::read(&src) - .and_then(|b| lightcraft_catalog::safe_file::write_atomic(&dst, &b)) - .and_then(|()| std::fs::remove_file(&src)) - }) - } + let parent = dst.parent().unwrap_or(to); + // Create only the shard, never missing parent directories on an unplugged drive. + let prepare = if parent.is_dir() { Ok(()) } else { std::fs::create_dir(parent) }; + prepare.and_then(|()| { + if is_valid(&dst) { + // named by content: the complete copy already there is the same preview + std::fs::remove_file(&src) + } else { + // across drives: copied atomically (never a partial proxy at the new place) + std::fs::rename(&src, &dst).or_else(|_| { + std::fs::read(&src) + .and_then(|b| lightcraft_catalog::safe_file::write_atomic(&dst, &b)) + .and_then(|()| std::fs::remove_file(&src)) + }) + } + }) } }; match r { @@ -288,6 +351,30 @@ mod tests { let _ = std::fs::remove_dir_all(&base); } + #[test] + fn sharded_and_legacy_proxies_are_counted_and_moved() { + let base = temp("shards"); + let from = base.join("from"); + let to = base.join("to"); + std::fs::create_dir_all(from.join("a1")).unwrap(); + std::fs::create_dir(&to).unwrap(); + let old = "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2.lcsp"; + let new = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1.lcsp"; + std::fs::write(from.join(old), b"legacy").unwrap(); + std::fs::write(from.join("a1").join(new), b"sharded").unwrap(); + std::fs::write(from.join("a1").join("notes.txt"), b"user file").unwrap(); + std::fs::write(from.join("notes.lcsp.txt"), b"keep").unwrap(); + assert_eq!(stats(&from), (2, 13)); + let (moved, failed) = migrate(&from, &to, Existing::Move); + assert_eq!((moved, failed), (2, Vec::::new())); + assert_eq!(stats(&from), (0, 0)); + assert_eq!(stats(&to), (2, 13)); + assert_eq!(std::fs::read(to.join("b2").join(old)).unwrap(), b"legacy"); + assert_eq!(std::fs::read(to.join("a1").join(new)).unwrap(), b"sharded"); + assert!(from.join("a1").join("notes.txt").exists()); + let _ = std::fs::remove_dir_all(&base); + } + #[test] fn damaged_proxies_are_invalid_and_never_kept_over_good_ones() { let base = temp("valid"); diff --git a/crates/engine/src/tests_import.rs b/crates/engine/src/tests_import.rs index 5f40163d..7aff5fd4 100644 --- a/crates/engine/src/tests_import.rs +++ b/crates/engine/src/tests_import.rs @@ -462,14 +462,14 @@ fn damaged_smart_previews_are_rebuilt_and_failed_writes_leave_none() { s.execute("library.import", &json!({"paths": [src.join("a.png").to_string_lossy()]})).unwrap(); let id = s.active().unwrap(); let dir = s.media.smart_dir.clone().unwrap(); - let file = dir.join(crate::smart::file_name(s.catalog.photo(id).unwrap())); + let file = crate::smart::sharded_file_path(&dir, s.catalog.photo(id).unwrap()); { let _fault = lightcraft_catalog::safe_file::fail_writes_after(64); let r = s.execute("library.smartPreviews", &json!({})).unwrap(); assert_eq!((r["built"].as_u64(), r["failed"].as_array().map(Vec::len)), (Some(0), Some(1)), "{r}"); } assert!(!file.exists(), "no partial proxy"); - assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 0, "no temp file left"); + assert_eq!(std::fs::read_dir(file.parent().unwrap()).unwrap().count(), 0, "no temp file left in shard"); assert_eq!(s.execute("photo.smartPreview", &json!({})).unwrap()["smartPreview"], false); let r = s.execute("library.smartPreviews", &json!({})).unwrap(); @@ -795,6 +795,13 @@ fn smart_previews_folder_is_chosen_per_library() { assert_eq!(loc["path"], lib.join("Smart Previews").to_string_lossy().as_ref()); s.execute("library.smartPreviews", &json!({})).unwrap(); assert_eq!(crate::smart::stats(&lib.join("Smart Previews")).0, 1); + // A flat proxy from older builds remains discoverable and migrates to the new sharded layout. + let dir = lib.join("Smart Previews"); + let photo = s.catalog.photo(id).unwrap(); + let shard = crate::smart::sharded_file_path(&dir, photo); + let flat = dir.join(crate::smart::file_name(photo)); + std::fs::rename(&shard, &flat).unwrap(); + assert_eq!(s.execute("photo.smartPreview", &json!({})).unwrap()["smartPreview"], true); // relative paths and missing drives are refused, and nothing is created on this drive assert!(s.execute("library.smartPreviewsLocation", &json!({"path": "rel/dir"})).is_err()); @@ -1300,8 +1307,8 @@ fn smart_previews_from_an_older_look_fit_are_refreshed() { assert!(s.catalog.photo(id).unwrap().relative_wb() && !s.catalog.photo(png).unwrap().relative_wb()); s.execute("library.select", &json!({"ids": [id.0, png.0], "active": id.0})).unwrap(); let dir = s.media.smart_dir.clone().unwrap(); - let file = dir.join(crate::smart::file_name(s.catalog.photo(id).unwrap())); - let png_file = dir.join(crate::smart::file_name(s.catalog.photo(png).unwrap())); + let file = crate::smart::sharded_file_path(&dir, s.catalog.photo(id).unwrap()); + let png_file = crate::smart::sharded_file_path(&dir, s.catalog.photo(png).unwrap()); let (old, new) = (test_tone(1.5), test_tone(3.5)); let calls = std::sync::Arc::new(AtomicUsize::new(0)); let plain = s.media.file_loader.clone().unwrap(); diff --git a/crates/ui-egui/src/panels/right.rs b/crates/ui-egui/src/panels/right.rs index 5ca4de41..4dca2c7a 100644 --- a/crates/ui-egui/src/panels/right.rs +++ b/crates/ui-egui/src/panels/right.rs @@ -699,7 +699,11 @@ fn info(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) { .media .smart_dir .as_ref() - .is_some_and(|d| avail.exists(&d.join(lightcraft_engine::smart::file_name(&p)).to_string_lossy()) == Some(true)); + .is_some_and(|d| { + let shard = lightcraft_engine::smart::sharded_file_path(d, &p); + let legacy = d.join(lightcraft_engine::smart::file_name(&p)); + avail.exists(&shard.to_string_lossy()) == Some(true) || avail.exists(&legacy.to_string_lossy()) == Some(true) + }); if !online || smart { let text = match (online, smart) { (false, true) => "Original offline ยท editing the smart preview",