LRC-LIB-CATALOG-IMPORT: address SQLite, worker & archive review
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Adrian D'souza
parent
4a427894ed
commit
866d02e577
|
|
@ -29,7 +29,6 @@ serde = { workspace = true }
|
|||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
log = { workspace = true }
|
||||
sqlite-core = "=0.12.0"
|
||||
ab_glyph = { workspace = true }
|
||||
lightcraft-segment = { workspace = true, optional = true }
|
||||
|
||||
|
|
|
|||
|
|
@ -610,6 +610,78 @@ impl Prepared {
|
|||
pub fn is_empty(&self) -> bool {
|
||||
self.items.is_empty()
|
||||
}
|
||||
|
||||
/// Recheck Add-only entries against the owner catalog immediately before commit.
|
||||
/// Preparation may have run while another import added the same path or content.
|
||||
pub fn revalidate_add(&mut self, s: &Session) {
|
||||
let mut paths = HashMap::new();
|
||||
let mut hashes = HashMap::new();
|
||||
for photo in s.catalog.photos() {
|
||||
if let Source::File { path } = &photo.source {
|
||||
paths.insert(normalize_import_path(path), photo.id);
|
||||
}
|
||||
if let Some(hash) = &photo.content_hash {
|
||||
hashes.insert(hash.clone(), photo.id);
|
||||
}
|
||||
}
|
||||
let mut items = Vec::with_capacity(self.items.len());
|
||||
for item in std::mem::take(&mut self.items) {
|
||||
let PreparedItem::Ready(ready) = item else {
|
||||
items.push(item);
|
||||
continue;
|
||||
};
|
||||
let ReadyFile { path, stored, info, sidecar, placed } = *ready;
|
||||
let existing_path = paths.get(&normalize_import_path(&stored)).copied().or_else(|| paths.get(&normalize_import_path(&path)).copied());
|
||||
let existing_hash = info.content_hash.as_ref().and_then(|hash| hashes.get(hash).copied());
|
||||
if let Some(existing) = existing_path.or(existing_hash) {
|
||||
if let Some(placed) = &placed {
|
||||
crate::import_move::rollback(placed);
|
||||
}
|
||||
items.push(PreparedItem::Duplicate {
|
||||
path,
|
||||
existing: Some(existing),
|
||||
reason: if existing_path.is_some() { "path" } else { "content" },
|
||||
hash: info.content_hash,
|
||||
});
|
||||
} else {
|
||||
items.push(PreparedItem::Ready(Box::new(ReadyFile { path, stored, info, sidecar, placed })));
|
||||
}
|
||||
}
|
||||
self.items = items;
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_import_path(path: &str) -> String {
|
||||
let path = path.replace('\\', "/");
|
||||
if cfg!(windows) { path.to_lowercase() } else { path }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod prepared_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn revalidate_add_only_converts_matching_ready_item() {
|
||||
let mut s = Session::new();
|
||||
let existing_path = std::env::temp_dir().join("lightcraft-revalidate-existing.jpg");
|
||||
let other_path = std::env::temp_dir().join("lightcraft-revalidate-other.jpg");
|
||||
let existing = Photo::new(PhotoId(1), Source::File { path: existing_path.to_string_lossy().into() }, "existing.jpg", "JPG", 1, 1, "now");
|
||||
s.commit("existing", Op::AddPhoto { photo: Box::new(existing) }).unwrap();
|
||||
let ready = |path: &Path| ReadyFile {
|
||||
path: path.to_string_lossy().into(),
|
||||
stored: path.to_string_lossy().into(),
|
||||
info: ProbeInfo { format: "JPG".into(), ..ProbeInfo::default() },
|
||||
sidecar: None,
|
||||
placed: None,
|
||||
};
|
||||
let mut prepared = Prepared {
|
||||
scanned: 2,
|
||||
items: vec![PreparedItem::Ready(Box::new(ready(&existing_path))), PreparedItem::Ready(Box::new(ready(&other_path)))],
|
||||
};
|
||||
prepared.revalidate_add(&s);
|
||||
assert!(matches!(prepared.items.first(), Some(PreparedItem::Duplicate { reason: "path", .. })));
|
||||
assert!(matches!(prepared.items.get(1), Some(PreparedItem::Ready(_))));
|
||||
}
|
||||
}
|
||||
|
||||
impl ImportJob {
|
||||
|
|
|
|||
|
|
@ -25,7 +25,10 @@ pub mod guard;
|
|||
pub mod import;
|
||||
mod import_move;
|
||||
pub mod library;
|
||||
mod lightroom_archive;
|
||||
pub mod lightroom_catalog;
|
||||
pub mod lightroom_job;
|
||||
mod lightroom_sqlite;
|
||||
pub mod media;
|
||||
pub mod memory;
|
||||
pub mod merge;
|
||||
|
|
@ -162,6 +165,8 @@ pub struct Session {
|
|||
pub active_spot: Option<usize>,
|
||||
/// The persistent library this session writes to (`None` = in-memory only).
|
||||
pub library: Option<library::Library>,
|
||||
/// Identity of current library opening; refreshed only after a successful open.
|
||||
pub(crate) library_identity: Arc<()>,
|
||||
/// XMP sidecar preferences (persisted with the library).
|
||||
pub xmp: sidecar::XmpPrefs,
|
||||
/// Parameters of the last export (`app.export` params, minus targets), persisted in prefs.json.
|
||||
|
|
@ -249,6 +254,7 @@ impl Session {
|
|||
segmenter: segment::Segmenter::default(),
|
||||
active_spot: None,
|
||||
library: None,
|
||||
library_identity: Arc::new(()),
|
||||
xmp: sidecar::XmpPrefs::default(),
|
||||
last_export: None,
|
||||
export_presets: Vec::new(),
|
||||
|
|
|
|||
|
|
@ -399,6 +399,7 @@ impl Session {
|
|||
lib.forgot_local = Some(plan);
|
||||
}
|
||||
}
|
||||
self.library_identity = std::sync::Arc::new(());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -491,12 +492,16 @@ impl Session {
|
|||
lib.retry_at = None;
|
||||
lib.last_error = None;
|
||||
}
|
||||
match (persisted, snapshot) {
|
||||
let result = match (persisted, snapshot) {
|
||||
(_, Ok(())) => Ok(()),
|
||||
// neither the log nor the snapshot took the queued ops
|
||||
(Err(EngineError::NotSaved(e)), Err(s)) if lib.journal.seq() == before => Err(EngineError::NotSaved(format!("{e}; snapshot: {s}"))),
|
||||
(_, Err(s)) => Err(s.into()),
|
||||
};
|
||||
if result.is_ok() {
|
||||
self.library_identity = std::sync::Arc::new(());
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn view_json(&self) -> Vec<u8> {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,654 @@
|
|||
//! Bounded, deduplicated persistence for Lightroom import source data.
|
||||
//!
|
||||
//! Archives use a deterministic JSON representation wrapped in zlib. The JSON
|
||||
//! writer streams through a bounded compressor, so an import cannot first build
|
||||
//! an unbounded archive buffer. Blob columns are omitted by the catalog reader;
|
||||
//! decoded XMP,
|
||||
//! settings, history, snapshots, identities, paths, collections & membership
|
||||
//! remain available for recovery.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::fs::{self, File};
|
||||
use std::io::{self, Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use miniz_oxide::deflate::{
|
||||
core::{CompressorOxide, create_comp_flags_from_zip_params},
|
||||
stream::deflate,
|
||||
};
|
||||
use miniz_oxide::{MZFlush, MZStatus};
|
||||
use serde::ser::{Serialize, SerializeMap, SerializeSeq, SerializeStruct, Serializer};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::lightroom_catalog::{CatalogImport, CatalogPhoto};
|
||||
|
||||
/// Maximum size of one complete archive on disk, including format header.
|
||||
pub const MAX_ARCHIVE_BYTES: u64 = 32 * 1024 * 1024;
|
||||
/// Maximum total size retained in managed archives.
|
||||
pub const MAX_TOTAL_ARCHIVE_BYTES: u64 = 128 * 1024 * 1024;
|
||||
/// Maximum number of managed archives retained.
|
||||
pub const MAX_MANAGED_ARCHIVES: usize = 8;
|
||||
|
||||
const FORMAT_VERSION: u8 = 1;
|
||||
const HEADER: &[u8] = b"LC-LRARCH\0\x01Z";
|
||||
const NAME_PREFIX: &str = "lightroom-import-";
|
||||
const NAME_SUFFIX: &str = ".lca";
|
||||
const MAX_SCAN: usize = 1024;
|
||||
const IO_BUFFER: usize = 16 * 1024;
|
||||
|
||||
/// Store compact source data below `directory` and return its managed archive.
|
||||
///
|
||||
/// The operation is safe to call before catalog mutation. It rejects a
|
||||
/// symlink directory, writes only `lightroom-import-<number>.lca`, compares the
|
||||
/// complete bounded byte stream with recent managed archives, and retains at
|
||||
/// most [`MAX_MANAGED_ARCHIVES`] and [`MAX_TOTAL_ARCHIVE_BYTES`]. Legacy
|
||||
/// `lightroom-import-*.json` files are deliberately ignored and preserved for
|
||||
/// an explicit caller warning.
|
||||
pub fn store(directory: &Path, data: &CatalogImport) -> Result<PathBuf, String> {
|
||||
prepare_directory(directory)?;
|
||||
let mut managed = managed_files(directory)?;
|
||||
managed.sort_by_key(|entry| entry.index);
|
||||
|
||||
let temporary = temporary_path(directory)?;
|
||||
write_archive(&temporary, data).map_err(|error| {
|
||||
let _ = remove_managed_file(&temporary);
|
||||
error
|
||||
})?;
|
||||
|
||||
let recent = managed.iter().rev().take(MAX_MANAGED_ARCHIVES);
|
||||
for entry in recent {
|
||||
let same = match same_bounded(&temporary, &entry.path) {
|
||||
Ok(same) => same,
|
||||
Err(error) => {
|
||||
let _ = remove_managed_file(&temporary);
|
||||
return Err(error.to_string());
|
||||
}
|
||||
};
|
||||
if same {
|
||||
remove_managed_file(&temporary).map_err(|e| e.to_string())?;
|
||||
let matched = entry.path.clone();
|
||||
let mut all = managed.clone();
|
||||
prune_existing(directory, &mut all, Some(&matched))?;
|
||||
return Ok(matched);
|
||||
}
|
||||
}
|
||||
|
||||
let index = managed.last().map_or(0, |entry| entry.index.saturating_add(1));
|
||||
if managed.last().is_some_and(|entry| entry.index == u64::MAX) {
|
||||
let _ = remove_managed_file(&temporary);
|
||||
return Err("managed Lightroom archive index is exhausted".into());
|
||||
}
|
||||
let destination = match next_destination(directory, index) {
|
||||
Ok(path) => path,
|
||||
Err(error) => {
|
||||
let _ = remove_managed_file(&temporary);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let size = match fs::metadata(&temporary) {
|
||||
Ok(meta) => meta.len(),
|
||||
Err(error) => {
|
||||
let _ = remove_managed_file(&temporary);
|
||||
return Err(error.to_string());
|
||||
}
|
||||
};
|
||||
let new_entry = Managed { index, path: destination.clone(), size };
|
||||
let mut after = managed;
|
||||
after.push(new_entry);
|
||||
after.sort_by_key(|entry| entry.index);
|
||||
let retirees = match retirement_set(&after) {
|
||||
Ok(retirees) => retirees,
|
||||
Err(error) => {
|
||||
let _ = remove_managed_file(&temporary);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let quarantined = match quarantine(&retirees) {
|
||||
Ok(quarantined) => quarantined,
|
||||
Err(error) => {
|
||||
let _ = remove_managed_file(&temporary);
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(error) = publish_new(&temporary, &destination) {
|
||||
rollback_quarantine(&quarantined);
|
||||
let _ = remove_managed_file(&temporary);
|
||||
return Err(error.to_string());
|
||||
}
|
||||
if let Err(error) = remove_managed_file(&temporary) {
|
||||
return Err(format!("archive published but temporary cleanup failed: {error}"));
|
||||
}
|
||||
for (old, held) in quarantined {
|
||||
if let Err(error) = remove_managed_file(&held) {
|
||||
return Err(format!("archive published but could not retire {}: {error}", old.display()));
|
||||
}
|
||||
}
|
||||
Ok(destination)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct Managed {
|
||||
index: u64,
|
||||
path: PathBuf,
|
||||
size: u64,
|
||||
}
|
||||
|
||||
fn prepare_directory(directory: &Path) -> Result<(), String> {
|
||||
match fs::symlink_metadata(directory) {
|
||||
Ok(meta) => {
|
||||
if meta.file_type().is_symlink() || !meta.is_dir() {
|
||||
return Err(format!("archive directory is not a real directory: {}", directory.display()));
|
||||
}
|
||||
}
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => {
|
||||
fs::create_dir_all(directory).map_err(|e| e.to_string())?;
|
||||
let meta = fs::symlink_metadata(directory).map_err(|e| e.to_string())?;
|
||||
if meta.file_type().is_symlink() || !meta.is_dir() {
|
||||
return Err(format!("archive directory is not a real directory: {}", directory.display()));
|
||||
}
|
||||
}
|
||||
Err(error) => return Err(error.to_string()),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn managed_files(directory: &Path) -> Result<Vec<Managed>, String> {
|
||||
let mut files = Vec::new();
|
||||
for (seen, result) in fs::read_dir(directory).map_err(|e| e.to_string())?.enumerate() {
|
||||
if seen >= MAX_SCAN {
|
||||
return Err(format!("too many entries in archive directory (limit {MAX_SCAN})"));
|
||||
}
|
||||
let entry = result.map_err(|e| e.to_string())?;
|
||||
let Some(index) = parse_managed_name(&entry.file_name()) else { continue };
|
||||
let meta = fs::symlink_metadata(entry.path()).map_err(|e| e.to_string())?;
|
||||
if meta.file_type().is_symlink() || !meta.is_file() {
|
||||
continue;
|
||||
}
|
||||
files.push(Managed { index, path: entry.path(), size: meta.len() });
|
||||
}
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
fn parse_managed_name(name: &std::ffi::OsStr) -> Option<u64> {
|
||||
let name = name.to_str()?;
|
||||
let digits = name.strip_prefix(NAME_PREFIX)?.strip_suffix(NAME_SUFFIX)?;
|
||||
if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) {
|
||||
return None;
|
||||
}
|
||||
digits.parse().ok()
|
||||
}
|
||||
|
||||
fn temporary_path(directory: &Path) -> Result<PathBuf, String> {
|
||||
let pid = std::process::id();
|
||||
for slot in 0..MAX_SCAN {
|
||||
let path = directory.join(format!(".lightroom-import-{pid}-{slot}.tmp"));
|
||||
match fs::symlink_metadata(&path) {
|
||||
Ok(_) => continue,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(path),
|
||||
Err(error) => return Err(error.to_string()),
|
||||
}
|
||||
}
|
||||
Err("no safe temporary archive name available".into())
|
||||
}
|
||||
|
||||
fn next_destination(directory: &Path, mut index: u64) -> Result<PathBuf, String> {
|
||||
for _ in 0..MAX_SCAN {
|
||||
let path = directory.join(format!("{NAME_PREFIX}{index}{NAME_SUFFIX}"));
|
||||
match fs::symlink_metadata(&path) {
|
||||
Ok(_) => index = index.checked_add(1).ok_or("managed archive index is exhausted")?,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(path),
|
||||
Err(error) => return Err(error.to_string()),
|
||||
}
|
||||
}
|
||||
Err("no safe managed archive name available".into())
|
||||
}
|
||||
|
||||
fn write_archive(path: &Path, data: &CatalogImport) -> Result<(), String> {
|
||||
lightcraft_catalog::safe_file::write_atomic_with(path, &mut |raw| {
|
||||
let mut bounded = BoundedWriter { inner: raw, total: 0 };
|
||||
bounded.write_all(HEADER)?;
|
||||
let mut compressor = ZlibWriter::new(&mut bounded);
|
||||
serde_json::to_writer(&mut compressor, &ArchiveRef(data)).map_err(json_io)?;
|
||||
compressor.finish()?;
|
||||
Ok(())
|
||||
})
|
||||
.map(|_| ())
|
||||
.map_err(|error| format!("could not write Lightroom archive: {error}"))
|
||||
}
|
||||
|
||||
fn json_io(error: serde_json::Error) -> io::Error {
|
||||
io::Error::new(io::ErrorKind::InvalidData, error.to_string())
|
||||
}
|
||||
|
||||
struct BoundedWriter<'a> {
|
||||
inner: &'a mut dyn Write,
|
||||
total: u64,
|
||||
}
|
||||
|
||||
impl Write for BoundedWriter<'_> {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
let next = self.total.checked_add(bytes.len() as u64).ok_or_else(limit_error)?;
|
||||
if next > MAX_ARCHIVE_BYTES {
|
||||
return Err(limit_error());
|
||||
}
|
||||
self.inner.write_all(bytes)?;
|
||||
self.total = next;
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.inner.flush()
|
||||
}
|
||||
}
|
||||
|
||||
fn limit_error() -> io::Error {
|
||||
io::Error::new(io::ErrorKind::InvalidData, "Lightroom archive exceeds 32 MiB")
|
||||
}
|
||||
|
||||
struct ZlibWriter<'a, 'b> {
|
||||
sink: &'a mut BoundedWriter<'b>,
|
||||
compressor: CompressorOxide,
|
||||
plain: u64,
|
||||
}
|
||||
|
||||
impl<'a, 'b> ZlibWriter<'a, 'b> {
|
||||
fn new(sink: &'a mut BoundedWriter<'b>) -> ZlibWriter<'a, 'b> {
|
||||
let flags = create_comp_flags_from_zip_params(6, 15, 0);
|
||||
ZlibWriter { sink, compressor: CompressorOxide::new(flags), plain: 0 }
|
||||
}
|
||||
|
||||
fn run(&mut self, input: &[u8], flush: MZFlush) -> io::Result<()> {
|
||||
let mut rest = input;
|
||||
loop {
|
||||
let mut output = [0u8; IO_BUFFER];
|
||||
let result = deflate(&mut self.compressor, rest, &mut output, flush);
|
||||
self.sink.write_all(&output[..result.bytes_written])?;
|
||||
match result.status {
|
||||
Ok(MZStatus::StreamEnd) => return Ok(()),
|
||||
Ok(MZStatus::Ok) => {
|
||||
let consumed = result.bytes_consumed;
|
||||
if consumed > rest.len() {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid compressor progress"));
|
||||
}
|
||||
rest = &rest[consumed..];
|
||||
if rest.is_empty() && flush == MZFlush::None {
|
||||
return Ok(());
|
||||
}
|
||||
if consumed == 0 && result.bytes_written == 0 {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "compressor made no progress"));
|
||||
}
|
||||
}
|
||||
Ok(status) => {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, format!("unsupported compressor status: {status:?}")));
|
||||
}
|
||||
Err(error) => return Err(io::Error::new(io::ErrorKind::InvalidData, format!("compression failed: {error:?}"))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn finish(&mut self) -> io::Result<()> {
|
||||
self.run(&[], MZFlush::Finish)
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for ZlibWriter<'_, '_> {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
if bytes.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let next = self.plain.checked_add(bytes.len() as u64).ok_or_else(limit_error)?;
|
||||
if next > MAX_ARCHIVE_BYTES {
|
||||
return Err(limit_error());
|
||||
}
|
||||
self.run(bytes, MZFlush::None)?;
|
||||
self.plain = next;
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn same_bounded(left: &Path, right: &Path) -> io::Result<bool> {
|
||||
let left_meta = fs::symlink_metadata(left)?;
|
||||
let right_meta = fs::symlink_metadata(right)?;
|
||||
if left_meta.file_type().is_symlink() || right_meta.file_type().is_symlink() || !left_meta.is_file() || !right_meta.is_file() {
|
||||
return Ok(false);
|
||||
}
|
||||
let left_len = left_meta.len();
|
||||
let right_len = right_meta.len();
|
||||
if left_len != right_len || left_len > MAX_ARCHIVE_BYTES {
|
||||
return Ok(false);
|
||||
}
|
||||
let mut left = File::open(left)?;
|
||||
let mut right = File::open(right)?;
|
||||
let mut left_buf = [0u8; IO_BUFFER];
|
||||
let mut right_buf = [0u8; IO_BUFFER];
|
||||
loop {
|
||||
let left_n = left.read(&mut left_buf)?;
|
||||
let right_n = right.read(&mut right_buf)?;
|
||||
if left_n != right_n {
|
||||
return Ok(false);
|
||||
}
|
||||
if left_n == 0 {
|
||||
return Ok(true);
|
||||
}
|
||||
if left_buf[..left_n] != right_buf[..right_n] {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn retirement_set(entries: &[Managed]) -> Result<Vec<Managed>, String> {
|
||||
retirement_set_protected(entries, None)
|
||||
}
|
||||
|
||||
fn retirement_set_protected(entries: &[Managed], protected: Option<&Path>) -> Result<Vec<Managed>, String> {
|
||||
let mut ordered = entries.to_vec();
|
||||
ordered.sort_by_key(|entry| entry.index);
|
||||
let mut total = ordered.iter().try_fold(0u64, |sum, entry| sum.checked_add(entry.size)).ok_or("archive size overflow")?;
|
||||
let mut retirees = Vec::new();
|
||||
while ordered.len() > MAX_MANAGED_ARCHIVES || total > MAX_TOTAL_ARCHIVE_BYTES {
|
||||
let Some(position) = ordered.iter().position(|entry| protected.map_or(true, |path| entry.path != path)) else {
|
||||
return Err("cannot satisfy archive retention limit".into());
|
||||
};
|
||||
let oldest = ordered.remove(position);
|
||||
total = total.saturating_sub(oldest.size);
|
||||
retirees.push(oldest);
|
||||
}
|
||||
Ok(retirees)
|
||||
}
|
||||
|
||||
fn prune_existing(directory: &Path, entries: &mut [Managed], protected: Option<&Path>) -> Result<(), String> {
|
||||
let retirees = retirement_set_protected(entries, protected)?;
|
||||
let quarantined = quarantine(&retirees)?;
|
||||
for (old, held) in quarantined {
|
||||
if let Err(error) = remove_managed_file(&held) {
|
||||
return Err(format!("could not retire {} from {}: {error}", old.display(), directory.display()));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn quarantine(entries: &[Managed]) -> Result<Vec<(PathBuf, PathBuf)>, String> {
|
||||
let mut moved = Vec::new();
|
||||
let pid = std::process::id();
|
||||
for (slot, entry) in entries.iter().enumerate() {
|
||||
let held = entry.path.with_file_name(format!(".lightroom-retire-{pid}-{slot}.tmp"));
|
||||
match fs::symlink_metadata(&held) {
|
||||
Ok(_) => {
|
||||
rollback_quarantine(&moved);
|
||||
return Err(format!("retention temporary path already exists: {}", held.display()));
|
||||
}
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(error) => {
|
||||
rollback_quarantine(&moved);
|
||||
return Err(error.to_string());
|
||||
}
|
||||
}
|
||||
let meta = fs::symlink_metadata(&entry.path).map_err(|error| {
|
||||
rollback_quarantine(&moved);
|
||||
error.to_string()
|
||||
})?;
|
||||
if meta.file_type().is_symlink() || !meta.is_file() {
|
||||
rollback_quarantine(&moved);
|
||||
return Err(format!("managed archive changed into non-file: {}", entry.path.display()));
|
||||
}
|
||||
if let Err(error) = fs::rename(&entry.path, &held) {
|
||||
rollback_quarantine(&moved);
|
||||
return Err(error.to_string());
|
||||
}
|
||||
moved.push((entry.path.clone(), held));
|
||||
}
|
||||
Ok(moved)
|
||||
}
|
||||
|
||||
fn rollback_quarantine(moved: &[(PathBuf, PathBuf)]) {
|
||||
for (original, held) in moved.iter().rev() {
|
||||
let _ = fs::rename(held, original);
|
||||
}
|
||||
}
|
||||
|
||||
fn publish_new(temporary: &Path, destination: &Path) -> io::Result<()> {
|
||||
match fs::hard_link(temporary, destination) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(error) => Err(io::Error::new(error.kind(), format!("cannot atomically publish archive: {error}"))),
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_managed_file(path: &Path) -> io::Result<()> {
|
||||
let meta = fs::symlink_metadata(path)?;
|
||||
if meta.file_type().is_symlink() || !meta.is_file() {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidInput, "refusing to remove non-regular archive path"));
|
||||
}
|
||||
fs::remove_file(path)
|
||||
}
|
||||
|
||||
struct ArchiveRef<'a>(&'a CatalogImport);
|
||||
|
||||
impl Serialize for ArchiveRef<'_> {
|
||||
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
let mut out = serializer.serialize_struct("CatalogImportArchive", 7)?;
|
||||
out.serialize_field("version", &FORMAT_VERSION)?;
|
||||
out.serialize_field("source", &self.0.source)?;
|
||||
out.serialize_field("photos", &PhotoSeq(&self.0.photos))?;
|
||||
out.serialize_field("collections", &RowSeq(&self.0.collections))?;
|
||||
out.serialize_field("members", &RowSeq(&self.0.members))?;
|
||||
out.serialize_field("collection_content", &RowSeq(&self.0.collection_content))?;
|
||||
out.serialize_field("warnings", &self.0.warnings)?;
|
||||
out.end()
|
||||
}
|
||||
}
|
||||
|
||||
struct PhotoSeq<'a>(&'a [CatalogPhoto]);
|
||||
impl Serialize for PhotoSeq<'_> {
|
||||
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
let mut out = serializer.serialize_seq(Some(self.0.len()))?;
|
||||
for photo in self.0 {
|
||||
out.serialize_element(&PhotoRef(photo))?;
|
||||
}
|
||||
out.end()
|
||||
}
|
||||
}
|
||||
|
||||
struct PhotoRef<'a>(&'a CatalogPhoto);
|
||||
impl Serialize for PhotoRef<'_> {
|
||||
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
let p = self.0;
|
||||
let mut out = serializer.serialize_struct("CatalogPhotoArchive", 8)?;
|
||||
out.serialize_field("source_id", &p.source_id)?;
|
||||
out.serialize_field("uuid", &p.uuid)?;
|
||||
out.serialize_field("path", &p.path)?;
|
||||
out.serialize_field("image", &RowRef(&p.image))?;
|
||||
out.serialize_field("settings", &p.settings)?;
|
||||
out.serialize_field("xmp", &p.xmp)?;
|
||||
out.serialize_field("keywords", &p.keywords)?;
|
||||
out.serialize_field("history", &RowSeq(&p.history))?;
|
||||
out.serialize_field("snapshots", &RowSeq(&p.snapshots))?;
|
||||
out.end()
|
||||
}
|
||||
}
|
||||
|
||||
struct RowSeq<'a>(&'a [BTreeMap<String, Value>]);
|
||||
impl Serialize for RowSeq<'_> {
|
||||
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
let mut out = serializer.serialize_seq(Some(self.0.len()))?;
|
||||
for row in self.0 {
|
||||
out.serialize_element(&RowRef(row))?;
|
||||
}
|
||||
out.end()
|
||||
}
|
||||
}
|
||||
|
||||
struct RowRef<'a>(&'a BTreeMap<String, Value>);
|
||||
impl Serialize for RowRef<'_> {
|
||||
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
let mut out = serializer.serialize_map(None)?;
|
||||
for (key, value) in self.0 {
|
||||
out.serialize_entry(key, value)?;
|
||||
}
|
||||
out.end()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::lightroom_catalog::{CatalogImport, CatalogPhoto};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
static NEXT: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
fn tempdir() -> PathBuf {
|
||||
let path = std::env::temp_dir().join(format!("lightcraft-lr-archive-{}-{}", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed)));
|
||||
fs::create_dir_all(&path).unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
fn data(source: String) -> CatalogImport {
|
||||
CatalogImport {
|
||||
source,
|
||||
photos: Vec::new(),
|
||||
collections: Vec::new(),
|
||||
members: Vec::new(),
|
||||
collection_content: Vec::new(),
|
||||
warnings: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn rich_data() -> CatalogImport {
|
||||
let photo = CatalogPhoto {
|
||||
source_id: 7,
|
||||
uuid: "uuid-7".into(),
|
||||
path: "/photos/one.raw".into(),
|
||||
image: BTreeMap::from([("rating".into(), serde_json::json!(5)), ("opaque_blob".into(), serde_json::json!([1, 2, 3, 4]))]),
|
||||
settings: "s = { Exposure2012 = 0.75 }".into(),
|
||||
xmp: "<xmp>decoded</xmp>".into(),
|
||||
keywords: vec!["Trips|Goa".into()],
|
||||
history: vec![BTreeMap::from([("text".into(), serde_json::json!("step")), ("curve".into(), serde_json::json!([0, 128, 255]))])],
|
||||
snapshots: vec![BTreeMap::from([("name".into(), serde_json::json!("before"))])],
|
||||
};
|
||||
CatalogImport {
|
||||
source: "source.lrcat".into(),
|
||||
photos: vec![photo],
|
||||
collections: vec![BTreeMap::from([("name".into(), serde_json::json!("Keepers"))])],
|
||||
members: vec![BTreeMap::from([("image".into(), serde_json::json!(7))])],
|
||||
collection_content: vec![BTreeMap::from([("rule".into(), serde_json::json!("rating > 3"))])],
|
||||
warnings: vec!["one warning".into()],
|
||||
}
|
||||
}
|
||||
|
||||
fn managed_count(path: &Path) -> usize {
|
||||
fs::read_dir(path).unwrap().filter_map(Result::ok).filter(|entry| parse_managed_name(&entry.file_name()).is_some()).count()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deduplicates_identical_deterministic_archives() {
|
||||
let path = tempdir();
|
||||
let input = data("catalog.lrcat".into());
|
||||
let first = store(&path, &input).unwrap();
|
||||
let second = store(&path, &input).unwrap();
|
||||
assert_eq!(first, second);
|
||||
assert_eq!(managed_count(&path), 1);
|
||||
fs::remove_dir_all(path).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dedup_returns_archive_that_retention_keeps() {
|
||||
let path = tempdir();
|
||||
let input = data("catalog.lrcat".into());
|
||||
let first = store(&path, &input).unwrap();
|
||||
for index in 1..MAX_MANAGED_ARCHIVES {
|
||||
let filler = path.join(format!("lightroom-import-{index}.lca"));
|
||||
File::create(filler).unwrap().set_len(19 * 1024 * 1024).unwrap();
|
||||
}
|
||||
let reused = store(&path, &input).unwrap();
|
||||
assert_eq!(reused, first);
|
||||
assert!(reused.is_file());
|
||||
assert!(managed_count(&path) <= MAX_MANAGED_ARCHIVES);
|
||||
fs::remove_dir_all(path).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn caps_managed_archive_count_and_preserves_unrelated_files() {
|
||||
let path = tempdir();
|
||||
let human = path.join("notes.txt");
|
||||
let legacy = path.join("lightroom-import-0.json");
|
||||
fs::write(&human, b"keep").unwrap();
|
||||
fs::write(&legacy, b"legacy").unwrap();
|
||||
for index in 0..(MAX_MANAGED_ARCHIVES + 3) {
|
||||
let input = data(format!("catalog-{index}.lrcat"));
|
||||
store(&path, &input).unwrap();
|
||||
}
|
||||
assert!(managed_count(&path) <= MAX_MANAGED_ARCHIVES);
|
||||
assert_eq!(fs::read(&human).unwrap(), b"keep");
|
||||
assert_eq!(fs::read(&legacy).unwrap(), b"legacy");
|
||||
fs::remove_dir_all(path).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_oversize_before_creating_archive() {
|
||||
let path = tempdir();
|
||||
let input = data("x".repeat((MAX_ARCHIVE_BYTES as usize) + 1));
|
||||
assert!(store(&path, &input).is_err());
|
||||
assert_eq!(managed_count(&path), 0);
|
||||
fs::remove_dir_all(path).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_roundtrips_zlib_payload_and_preserves_numeric_arrays() {
|
||||
let path = tempdir();
|
||||
let archive = store(&path, &rich_data()).unwrap();
|
||||
let bytes = fs::read(archive).unwrap();
|
||||
assert!(bytes.starts_with(HEADER));
|
||||
let json = miniz_oxide::inflate::decompress_to_vec_zlib_with_limit(&bytes[HEADER.len()..], MAX_ARCHIVE_BYTES as usize).unwrap();
|
||||
let decoded: Value = serde_json::from_slice(&json).unwrap();
|
||||
assert_eq!(decoded["version"], serde_json::json!(FORMAT_VERSION));
|
||||
assert_eq!(decoded["source"], serde_json::json!("source.lrcat"));
|
||||
assert_eq!(decoded["photos"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(decoded["photos"][0]["source_id"], serde_json::json!(7));
|
||||
assert_eq!(decoded["photos"][0]["uuid"], serde_json::json!("uuid-7"));
|
||||
assert_eq!(decoded["photos"][0]["path"], serde_json::json!("/photos/one.raw"));
|
||||
assert_eq!(decoded["photos"][0]["settings"], serde_json::json!("s = { Exposure2012 = 0.75 }"));
|
||||
assert_eq!(decoded["photos"][0]["xmp"], serde_json::json!("<xmp>decoded</xmp>"));
|
||||
assert_eq!(decoded["photos"][0]["keywords"], serde_json::json!(["Trips|Goa"]));
|
||||
assert_eq!(decoded["photos"][0]["image"]["rating"], serde_json::json!(5));
|
||||
assert_eq!(decoded["photos"][0]["image"]["opaque_blob"], serde_json::json!([1, 2, 3, 4]));
|
||||
assert_eq!(decoded["photos"][0]["history"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(decoded["photos"][0]["history"][0]["curve"], serde_json::json!([0, 128, 255]));
|
||||
assert_eq!(decoded["photos"][0]["snapshots"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(decoded["collections"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(decoded["collections"][0]["name"], serde_json::json!("Keepers"));
|
||||
assert_eq!(decoded["members"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(decoded["members"][0]["image"], serde_json::json!(7));
|
||||
assert_eq!(decoded["collection_content"].as_array().unwrap().len(), 1);
|
||||
assert_eq!(decoded["warnings"], serde_json::json!(["one warning"]));
|
||||
fs::remove_dir_all(path).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retention_enforces_aggregate_byte_cap_with_small_synthetic_entries() {
|
||||
let mut entries: Vec<_> = (0..MAX_MANAGED_ARCHIVES as u64)
|
||||
.map(|index| Managed { index, path: PathBuf::from(format!("lightroom-import-{index}.lca")), size: 20 * 1024 * 1024 })
|
||||
.collect();
|
||||
entries.push(Managed { index: 8, path: PathBuf::from("lightroom-import-8.lca"), size: 20 * 1024 * 1024 });
|
||||
let retirees = retirement_set(&entries).unwrap();
|
||||
assert_eq!(retirees.len(), 3);
|
||||
assert_eq!(retirees.iter().map(|entry| entry.index).collect::<Vec<_>>(), vec![0, 1, 2]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retention_failure_preserves_old_archive() {
|
||||
let path = tempdir();
|
||||
let old = path.join("lightroom-import-0.lca");
|
||||
let held = path.join(format!(".lightroom-retire-{}-0.tmp", std::process::id()));
|
||||
fs::write(&old, b"old archive").unwrap();
|
||||
fs::write(&held, b"occupied").unwrap();
|
||||
let entry = Managed { index: 0, path: old.clone(), size: 11 };
|
||||
assert!(quarantine(&[entry]).is_err());
|
||||
assert_eq!(fs::read(&old).unwrap(), b"old archive");
|
||||
fs::remove_dir_all(path).unwrap();
|
||||
}
|
||||
}
|
||||
|
|
@ -3,12 +3,15 @@
|
|||
//! Raw settings are archived before mutation; mapped edits are approximations, not Adobe renders.
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
};
|
||||
|
||||
use crate::lightroom_sqlite::{Database, LiveTable, Value as SqlValue};
|
||||
use lightcraft_catalog::{Album, Flag, Op, Photo, PhotoId, Source};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Value, json};
|
||||
use sqlite_core::{Database, Value as SqlValue};
|
||||
|
||||
type Row = BTreeMap<String, Value>;
|
||||
const COMMAND: &str = "library.importLightroom";
|
||||
|
|
@ -296,6 +299,10 @@ fn text<'a>(r: &'a Row, k: &str) -> &'a str {
|
|||
fn number(r: &Row, k: &str) -> i64 {
|
||||
r.get(k).and_then(Value::as_f64).filter(|n| n.is_finite()).unwrap_or(0.0) as i64
|
||||
}
|
||||
|
||||
pub(crate) fn number_for_job(r: &Row, k: &str) -> i64 {
|
||||
number(r, k)
|
||||
}
|
||||
fn key(path: &str) -> String {
|
||||
let p = path.replace('\\', "/");
|
||||
if cfg!(windows) { p.to_lowercase() } else { p }
|
||||
|
|
@ -309,7 +316,7 @@ fn source_path(p: &Photo) -> Option<&str> {
|
|||
|
||||
struct Reader {
|
||||
db: Database,
|
||||
tables: Vec<sqlite_core::attribution::LiveTable>,
|
||||
tables: Vec<LiveTable>,
|
||||
}
|
||||
impl Reader {
|
||||
fn table(&self, name: &str, required: bool) -> Result<Vec<Row>, String> {
|
||||
|
|
@ -328,17 +335,64 @@ impl Reader {
|
|||
.iter()
|
||||
.cloned()
|
||||
.zip(r.values.into_iter().map(|v| match v {
|
||||
SqlValue::Null => Value::Null,
|
||||
SqlValue::Integer(n) => json!(n),
|
||||
SqlValue::Real(n) => json!(n),
|
||||
SqlValue::Text(s) => json!(s),
|
||||
// Opaque data is retained in the source catalog; the XMP packet is decoded below.
|
||||
SqlValue::Blob(b) => json!(b),
|
||||
SqlValue::Null => Some(Value::Null),
|
||||
SqlValue::Integer(n) => Some(json!(n)),
|
||||
SqlValue::Real(n) => Some(json!(n)),
|
||||
SqlValue::Text(s) => Some(json!(s)),
|
||||
// Opaque Lightroom payloads never become JSON byte arrays.
|
||||
SqlValue::Blob(_) => None,
|
||||
}))
|
||||
.filter_map(|(column, value)| value.map(|value| (column, value)))
|
||||
.collect()
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn xmp_table(&self) -> Result<(Vec<Row>, Vec<String>), String> {
|
||||
let Some(t) = self.tables.iter().find(|t| t.name == "Adobe_AdditionalMetadata") else { return Ok((Vec::new(), Vec::new())) };
|
||||
let columns = t.column_names.as_ref().ok_or_else(|| "cannot read Adobe_AdditionalMetadata columns".to_string())?;
|
||||
let rows = self.db.read_table(t.rootpage, columns.len()).map_err(|e| format!("Adobe_AdditionalMetadata: {e:?}"))?;
|
||||
if rows.len() > 1_000_000 {
|
||||
return Err("Adobe_AdditionalMetadata: more than one million records".into());
|
||||
}
|
||||
let mut warnings = Vec::new();
|
||||
let mut out = Vec::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
let image = row
|
||||
.values
|
||||
.iter()
|
||||
.zip(columns)
|
||||
.find_map(|(value, column)| {
|
||||
(column == "image").then(|| match value {
|
||||
SqlValue::Integer(n) => *n,
|
||||
_ => 0,
|
||||
})
|
||||
})
|
||||
.unwrap_or(0);
|
||||
let mut mapped = Row::new();
|
||||
for (column, value) in columns.iter().zip(row.values) {
|
||||
let value = match value {
|
||||
SqlValue::Null => Some(Value::Null),
|
||||
SqlValue::Integer(n) => Some(json!(n)),
|
||||
SqlValue::Real(n) => Some(json!(n)),
|
||||
SqlValue::Text(s) => Some(json!(s)),
|
||||
SqlValue::Blob(bytes) if column == "xmp" => match decode_xmp_bytes(&bytes) {
|
||||
Ok(packet) => Some(Value::String(packet)),
|
||||
Err(error) => {
|
||||
warnings.push(format!("image {image}: {error}"));
|
||||
None
|
||||
}
|
||||
},
|
||||
SqlValue::Blob(_) => None,
|
||||
};
|
||||
if let Some(value) = value {
|
||||
mapped.insert(column.clone(), value);
|
||||
}
|
||||
}
|
||||
out.push(mapped);
|
||||
}
|
||||
Ok((out, warnings))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
|
|
@ -364,10 +418,38 @@ pub struct CatalogImport {
|
|||
pub warnings: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Default, Serialize, Deserialize)]
|
||||
struct ImportIndex {
|
||||
photos: BTreeMap<String, u64>,
|
||||
collections: BTreeMap<String, u64>,
|
||||
#[derive(Clone, Default, Serialize, Deserialize)]
|
||||
pub(crate) struct ImportIndex {
|
||||
pub(crate) photos: BTreeMap<String, u64>,
|
||||
pub(crate) collections: BTreeMap<String, u64>,
|
||||
}
|
||||
|
||||
pub(crate) fn load_index(path: Option<&Path>) -> crate::Result<ImportIndex> {
|
||||
match path {
|
||||
Some(path) => {
|
||||
let file = match std::fs::File::open(path) {
|
||||
Ok(file) => file,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(ImportIndex::default()),
|
||||
Err(e) => return Err(error(e.to_string())),
|
||||
};
|
||||
if file.metadata().map_err(|e| error(e.to_string()))?.len() > 16 * 1024 * 1024 {
|
||||
return Err(error("Lightroom import index exceeds 16 MiB"));
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
use std::io::Read;
|
||||
file.take(16 * 1024 * 1024 + 1).read_to_end(&mut bytes).map_err(|e| error(e.to_string()))?;
|
||||
if bytes.len() > 16 * 1024 * 1024 {
|
||||
return Err(error("Lightroom import index exceeds 16 MiB"));
|
||||
}
|
||||
serde_json::from_slice(&bytes).map_err(|e| error(format!("invalid Lightroom import index: {e}")))
|
||||
}
|
||||
None => Ok(ImportIndex::default()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct AppliedImport {
|
||||
pub(crate) report: Value,
|
||||
pub(crate) index: ImportIndex,
|
||||
}
|
||||
|
||||
fn identity(data: &CatalogImport, photo: &CatalogPhoto) -> String {
|
||||
|
|
@ -406,6 +488,10 @@ fn validate(data: &CatalogImport) -> Result<(), String> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn validate_for_job(data: &CatalogImport) -> Result<(), String> {
|
||||
validate(data)
|
||||
}
|
||||
|
||||
fn bytes(path: &Path) -> Result<Vec<u8>, String> {
|
||||
use std::io::Read;
|
||||
let f = std::fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?;
|
||||
|
|
@ -440,6 +526,19 @@ fn xmp(v: Option<&Value>) -> Result<String, String> {
|
|||
}
|
||||
}
|
||||
|
||||
fn decode_xmp_bytes(bytes: &[u8]) -> Result<String, String> {
|
||||
let size = bytes.get(..4).and_then(|s| <[u8; 4]>::try_from(s).ok()).map(u32::from_be_bytes).ok_or("truncated XMP header")?;
|
||||
if size > 16 << 20 {
|
||||
return Err("XMP packet exceeds 16 MiB".into());
|
||||
}
|
||||
let decoded = miniz_oxide::inflate::decompress_to_vec_zlib_with_limit(bytes.get(4..).ok_or("truncated XMP")?, 16 << 20)
|
||||
.map_err(|_| "invalid compressed XMP")?;
|
||||
if decoded.len() != size as usize {
|
||||
return Err("XMP packet length mismatch".into());
|
||||
}
|
||||
String::from_utf8(decoded).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
fn be_word(bytes: &[u8], at: usize) -> Result<u32, String> {
|
||||
let word = bytes.get(at..at + 4).and_then(|s| <[u8; 4]>::try_from(s).ok()).ok_or("truncated WAL")?;
|
||||
Ok(u32::from_be_bytes(word))
|
||||
|
|
@ -500,6 +599,17 @@ fn committed_wal_len(main: &[u8], wal: &[u8]) -> Result<usize, String> {
|
|||
|
||||
/// Inspect `.lrcat` plus committed WAL pages without modifying either source file.
|
||||
pub fn read(path: &Path) -> Result<CatalogImport, String> {
|
||||
let cancel = AtomicBool::new(false);
|
||||
let total = AtomicUsize::new(0);
|
||||
let done = AtomicUsize::new(0);
|
||||
read_with_progress(path, &cancel, &total, &done)
|
||||
}
|
||||
|
||||
pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize, done: &AtomicUsize) -> Result<CatalogImport, String> {
|
||||
// 13 catalog tables plus one validation stage; ImportJob adds one unit per original path.
|
||||
total.store(14, Ordering::Relaxed);
|
||||
done.store(0, Ordering::Relaxed);
|
||||
ensure_read_active(cancel)?;
|
||||
let before = std::fs::metadata(path).map_err(|e| e.to_string())?;
|
||||
let wal_path = PathBuf::from(format!("{}-wal", path.display()));
|
||||
let wal_before = std::fs::metadata(&wal_path).ok().map(|m| (m.len(), m.modified().ok()));
|
||||
|
|
@ -510,22 +620,33 @@ pub fn read(path: &Path) -> Result<CatalogImport, String> {
|
|||
Err(e) => return Err(e.to_string()),
|
||||
};
|
||||
let wal_len = committed_wal_len(&main, &wal)?;
|
||||
ensure_read_active(cancel)?;
|
||||
let db = Database::open_with_wal(main, &wal[..wal_len]).map_err(|e| format!("invalid SQLite catalog: {e:?}"))?;
|
||||
let after = std::fs::metadata(path).map_err(|e| e.to_string())?;
|
||||
let wal_after = std::fs::metadata(&wal_path).ok().map(|m| (m.len(), m.modified().ok()));
|
||||
if before.len() != after.len() || before.modified().ok() != after.modified().ok() || wal_before != wal_after {
|
||||
return Err("catalog changed during reading; close Lightroom and retry".into());
|
||||
}
|
||||
let reader = Reader { tables: db.live_tables(), db };
|
||||
let roots: HashMap<_, _> = reader.table("AgLibraryRootFolder", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();
|
||||
let folders: HashMap<_, _> = reader.table("AgLibraryFolder", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();
|
||||
let files: HashMap<_, _> = reader.table("AgLibraryFile", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();
|
||||
let reader = Reader { tables: db.live_tables()?, db };
|
||||
let mut table = |name: &str, required: bool| -> Result<Vec<Row>, String> {
|
||||
ensure_read_active(cancel)?;
|
||||
let rows = reader.table(name, required)?;
|
||||
done.fetch_add(1, Ordering::Relaxed);
|
||||
Ok(rows)
|
||||
};
|
||||
let roots: HashMap<_, _> = table("AgLibraryRootFolder", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();
|
||||
let folders: HashMap<_, _> = table("AgLibraryFolder", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();
|
||||
let files: HashMap<_, _> = table("AgLibraryFile", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();
|
||||
let develops: HashMap<_, _> =
|
||||
reader.table("Adobe_imageDevelopSettings", false)?.into_iter().map(|r| (number(&r, "image"), text(&r, "text").to_string())).collect();
|
||||
let metadata: HashMap<_, _> = reader.table("Adobe_AdditionalMetadata", false)?.into_iter().map(|r| (number(&r, "image"), r)).collect();
|
||||
let keyword_rows: HashMap<_, _> = reader.table("AgLibraryKeyword", false)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();
|
||||
table("Adobe_imageDevelopSettings", false)?.into_iter().map(|r| (number(&r, "image"), text(&r, "text").to_string())).collect();
|
||||
ensure_read_active(cancel)?;
|
||||
let (metadata_rows, mut metadata_warnings) = reader.xmp_table()?;
|
||||
done.fetch_add(1, Ordering::Relaxed);
|
||||
let metadata: HashMap<_, _> = metadata_rows.into_iter().map(|r| (number(&r, "image"), r)).collect();
|
||||
let keyword_rows: HashMap<_, _> = table("AgLibraryKeyword", false)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect();
|
||||
let mut keywords: HashMap<i64, Vec<String>> = HashMap::new();
|
||||
for r in reader.table("AgLibraryKeywordImage", false)? {
|
||||
for r in table("AgLibraryKeywordImage", false)? {
|
||||
ensure_read_active(cancel)?;
|
||||
let mut parts = Vec::new();
|
||||
let mut at = number(&r, "tag");
|
||||
let mut seen = HashSet::new();
|
||||
|
|
@ -552,17 +673,18 @@ pub fn read(path: &Path) -> Result<CatalogImport, String> {
|
|||
}
|
||||
out
|
||||
};
|
||||
let mut history = group(reader.table("Adobe_libraryImageDevelopHistoryStep", false)?);
|
||||
let mut snapshots = group(reader.table("Adobe_libraryImageDevelopSnapshot", false)?);
|
||||
let mut history = group(table("Adobe_libraryImageDevelopHistoryStep", false)?);
|
||||
let mut snapshots = group(table("Adobe_libraryImageDevelopSnapshot", false)?);
|
||||
let mut out = CatalogImport {
|
||||
source: path.to_string_lossy().into(),
|
||||
photos: Vec::new(),
|
||||
collections: reader.table("AgLibraryCollection", false)?,
|
||||
members: reader.table("AgLibraryCollectionImage", false)?,
|
||||
collection_content: reader.table("AgLibraryCollectionContent", false)?,
|
||||
warnings: Vec::new(),
|
||||
collections: table("AgLibraryCollection", false)?,
|
||||
members: table("AgLibraryCollectionImage", false)?,
|
||||
collection_content: table("AgLibraryCollectionContent", false)?,
|
||||
warnings: std::mem::take(&mut metadata_warnings),
|
||||
};
|
||||
for image in reader.table("Adobe_images", true)? {
|
||||
for image in table("Adobe_images", true)? {
|
||||
ensure_read_active(cancel)?;
|
||||
let id = number(&image, "id_local");
|
||||
let file = files.get(&number(&image, "rootFile")).ok_or_else(|| format!("image {id}: missing file record"))?;
|
||||
let folder = folders.get(&number(file, "folder")).ok_or_else(|| format!("image {id}: missing folder record"))?;
|
||||
|
|
@ -593,9 +715,14 @@ pub fn read(path: &Path) -> Result<CatalogImport, String> {
|
|||
snapshots: snapshots.remove(&id).unwrap_or_default(),
|
||||
});
|
||||
}
|
||||
ensure_read_active(cancel)?;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn ensure_read_active(cancel: &AtomicBool) -> Result<(), String> {
|
||||
if cancel.load(Ordering::Relaxed) { Err("Lightroom import cancelled".into()) } else { Ok(()) }
|
||||
}
|
||||
|
||||
fn mapped_settings(text: &str, raw: bool, aspect: f64) -> Result<(Value, Vec<String>), String> {
|
||||
let mut root = crate::preset_import::parse_lua(text)?;
|
||||
let deferred = strip_lua_sentinels(&mut root);
|
||||
|
|
@ -651,39 +778,60 @@ fn strip_xmp_sentinels(value: &mut Value) {
|
|||
/// Import into the current library. Existing edited records are preserved unless explicitly
|
||||
/// requested; missing originals remain catalogued for later relinking. The source catalog is read-only.
|
||||
pub fn import(s: &mut crate::Session, path: &Path, update_existing: bool) -> crate::Result<Value> {
|
||||
let data = read(path).map_err(error)?;
|
||||
apply(s, data, update_existing)
|
||||
let mut job = crate::lightroom_job::LightroomJob::new(s, path.to_path_buf(), update_existing)?;
|
||||
let cancel = AtomicBool::new(false);
|
||||
let prepared = job.prepare(&cancel)?;
|
||||
let completion = crate::lightroom_job::commit_prepared(s, prepared)?;
|
||||
if s.library.as_ref().is_some_and(|library| library.on_disk) {
|
||||
s.persist()?;
|
||||
}
|
||||
completion.finalization.finish()?;
|
||||
Ok(completion.report)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn apply(s: &mut crate::Session, data: CatalogImport, update_existing: bool) -> crate::Result<Value> {
|
||||
validate(&data).map_err(error)?;
|
||||
let mut warnings = data.warnings.clone();
|
||||
let archive = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.join("Interop"));
|
||||
let index_path = archive.as_ref().map(|p| p.join("lightroom-index.json"));
|
||||
let mut index: ImportIndex = match index_path.as_ref().map(std::fs::read) {
|
||||
Some(Ok(b)) => serde_json::from_slice(&b).map_err(|e| error(format!("invalid Lightroom import index: {e}")))?,
|
||||
Some(Err(e)) if e.kind() != std::io::ErrorKind::NotFound => return Err(error(e.to_string())),
|
||||
_ => ImportIndex::default(),
|
||||
};
|
||||
// Preserve source data before catalog changes. A unique archive is never overwritten.
|
||||
let archive_path = if let Some(dir) = archive {
|
||||
std::fs::create_dir_all(&dir).map_err(|e| error(e.to_string()))?;
|
||||
let bytes = serde_json::to_vec(&data).map_err(|e| error(e.to_string()))?;
|
||||
let mut names = (0..10000).map(|n| dir.join(format!("lightroom-import-{n}.json")));
|
||||
Some(lightcraft_catalog::safe_file::write_new_unique(&mut names, &bytes).map_err(|e| error(e.to_string()))?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let archive_dir = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.join("Interop"));
|
||||
let index_path = archive_dir.as_ref().map(|p| p.join("lightroom-index.json"));
|
||||
let mut index = load_index(index_path.as_deref())?;
|
||||
let archive_path = archive_dir.as_ref().map(|dir| crate::lightroom_archive::store(dir, &data).map_err(error)).transpose()?;
|
||||
let undo0 = s.undo.len();
|
||||
let existing: HashMap<_, _> =
|
||||
s.catalog.photos().filter(|p| p.copy_of.is_none()).filter_map(|p| source_path(p).map(|path| (key(path), p.id))).collect();
|
||||
let undo0 = s.undo.len();
|
||||
let paths: Vec<_> = data
|
||||
.photos
|
||||
.iter()
|
||||
.filter(|p| number(&p.image, "masterImage") == 0 && !existing.contains_key(&key(&p.path)) && Path::new(&p.path).is_file())
|
||||
.map(|p| p.path.clone())
|
||||
.collect();
|
||||
let missing_sources: HashSet<String> =
|
||||
data.photos.iter().filter(|p| number(&p.image, "masterImage") == 0).filter(|p| !Path::new(&p.path).is_file()).map(|p| key(&p.path)).collect();
|
||||
let files = crate::import::import(s, &paths, crate::import::ImportMode::Add)?;
|
||||
let applied =
|
||||
apply_prepared(s, data, ApplyPrepared { update_existing, files, index: &mut index, archive_path, missing_sources: &missing_sources, undo0 })?;
|
||||
if let Some(path) = index_path {
|
||||
s.persist()?;
|
||||
let bytes = serde_json::to_vec(&applied.index).map_err(|e| error(e.to_string()))?;
|
||||
lightcraft_catalog::safe_file::write_atomic(&path, &bytes).map_err(|e| error(e.to_string()))?;
|
||||
}
|
||||
Ok(applied.report)
|
||||
}
|
||||
|
||||
/// Apply prepared normal-file results and Lightroom metadata in one owner-thread operation.
|
||||
/// This function does no filesystem work; callers finish its returned index separately.
|
||||
pub(crate) struct ApplyPrepared<'a> {
|
||||
pub(crate) update_existing: bool,
|
||||
pub(crate) files: crate::import::ImportReport,
|
||||
pub(crate) index: &'a mut ImportIndex,
|
||||
pub(crate) archive_path: Option<PathBuf>,
|
||||
pub(crate) missing_sources: &'a HashSet<String>,
|
||||
pub(crate) undo0: usize,
|
||||
}
|
||||
|
||||
pub(crate) fn apply_prepared(s: &mut crate::Session, data: CatalogImport, context: ApplyPrepared<'_>) -> crate::Result<AppliedImport> {
|
||||
let ApplyPrepared { update_existing, files, index, archive_path, missing_sources, undo0 } = context;
|
||||
validate(&data).map_err(error)?;
|
||||
let mut warnings = data.warnings.clone();
|
||||
let duplicates: HashMap<_, _> = files.duplicates.iter().filter_map(|d| d.existing.map(|id| (key(&d.path), PhotoId(id)))).collect();
|
||||
let mut by_path: HashMap<_, _> =
|
||||
s.catalog.photos().filter(|p| p.copy_of.is_none()).filter_map(|p| source_path(p).map(|path| (key(path), p.id))).collect();
|
||||
|
|
@ -729,7 +877,7 @@ fn apply(s: &mut crate::Session, data: CatalogImport, update_existing: bool) ->
|
|||
} else {
|
||||
let id = s.catalog.alloc_photo_id();
|
||||
let name = Path::new(&src.path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| src.path.clone());
|
||||
if !Path::new(&src.path).is_file() {
|
||||
if missing_sources.contains(&key(&src.path)) {
|
||||
missing.push(src.path.clone());
|
||||
}
|
||||
let mut p = Photo::new(
|
||||
|
|
@ -866,12 +1014,8 @@ fn apply(s: &mut crate::Session, data: CatalogImport, update_existing: bool) ->
|
|||
s.commit("Import Lightroom Catalog", Op::Batch { ops })?;
|
||||
}
|
||||
s.merge_undo(s.undo.len().saturating_sub(undo0), "Import Lightroom Catalog");
|
||||
if let Some(path) = index_path {
|
||||
s.persist()?;
|
||||
let bytes = serde_json::to_vec(&index).map_err(|e| error(e.to_string()))?;
|
||||
lightcraft_catalog::safe_file::write_atomic(&path, &bytes).map_err(|e| error(e.to_string()))?;
|
||||
}
|
||||
Ok(
|
||||
json!({"source":data.source,"photos":ids.len(),"imported":files.imported.len(),"collections":albums.len(),"preservedExistingEdits":preserved,"missing":missing,"failed":files.failed,"warnings":warnings,"unmapped":unmapped,"archive":archive_path,"mapping":ids}),
|
||||
)
|
||||
Ok(AppliedImport {
|
||||
report: json!({"source":data.source,"photos":ids.len(),"imported":files.imported.len(),"collections":albums.len(),"preservedExistingEdits":preserved,"missing":missing,"failed":files.failed,"warnings":warnings,"unmapped":unmapped,"archive":archive_path,"mapping":ids}),
|
||||
index: index.clone(),
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,386 @@
|
|||
//! Detached Lightroom catalog preparation and owner-thread commit.
|
||||
//!
|
||||
//! `LightroomJob::new` takes only an in-memory session snapshot. `prepare` owns all catalog
|
||||
//! reading, probing and archive staging, while `commit_prepared` only applies catalog operations
|
||||
//! to the session that created the job. Archive index bytes are returned for a worker to finish
|
||||
//! after the owner has accepted the catalog operation.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, AtomicUsize, Ordering},
|
||||
};
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::import::{ImportJob, ImportMode, ImportOptions, Prepared};
|
||||
use crate::lightroom_catalog::{self, CatalogImport, ImportIndex};
|
||||
|
||||
const COMMAND: &str = "library.importLightroom";
|
||||
const MAX_INDEX_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
fn error(message: impl Into<String>) -> crate::EngineError {
|
||||
crate::EngineError::BadParams { cmd: COMMAND.into(), msg: message.into() }
|
||||
}
|
||||
|
||||
/// Memory identity of the library which owns a prepared completion.
|
||||
///
|
||||
/// Each successful open/close refreshes session identity, including reopening the same path.
|
||||
/// The owner check deliberately does not compare catalog revisions: ordinary edits while preparation
|
||||
/// runs must survive and are merged by the owner commit.
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct LightroomLibraryToken {
|
||||
library_dir: Option<PathBuf>,
|
||||
identity: Arc<()>,
|
||||
}
|
||||
|
||||
impl LightroomLibraryToken {
|
||||
fn capture(s: &crate::Session) -> Self {
|
||||
Self {
|
||||
library_dir: s.library.as_ref().filter(|library| library.on_disk).map(|library| library.dir.clone()),
|
||||
identity: Arc::clone(&s.library_identity),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `s` still refers to the library which started preparation.
|
||||
pub fn matches(&self, s: &crate::Session) -> bool {
|
||||
Arc::ptr_eq(&self.identity, &s.library_identity)
|
||||
&& self.library_dir == s.library.as_ref().filter(|library| library.on_disk).map(|library| library.dir.clone())
|
||||
}
|
||||
|
||||
pub fn library_dir(&self) -> Option<&Path> {
|
||||
self.library_dir.as_deref()
|
||||
}
|
||||
}
|
||||
|
||||
/// Filesystem-free result of catalog preparation.
|
||||
pub struct PreparedLightroom {
|
||||
pub(crate) data: CatalogImport,
|
||||
pub(crate) normal: Prepared,
|
||||
pub(crate) index: ImportIndex,
|
||||
pub(crate) archive_path: Option<PathBuf>,
|
||||
pub(crate) token: LightroomLibraryToken,
|
||||
pub(crate) missing_sources: HashSet<String>,
|
||||
pub(crate) update_existing: bool,
|
||||
pub(crate) now: String,
|
||||
cancelled: bool,
|
||||
}
|
||||
|
||||
impl PreparedLightroom {
|
||||
pub fn source(&self) -> &str {
|
||||
&self.data.source
|
||||
}
|
||||
|
||||
pub fn photo_count(&self) -> usize {
|
||||
self.data.photos.len()
|
||||
}
|
||||
|
||||
pub fn token(&self) -> &LightroomLibraryToken {
|
||||
&self.token
|
||||
}
|
||||
|
||||
pub fn is_cancelled(&self) -> bool {
|
||||
self.cancelled
|
||||
}
|
||||
|
||||
/// Roll back any normal-import filesystem placement held by this prepared value.
|
||||
pub fn rollback(&self) {
|
||||
self.normal.rollback();
|
||||
}
|
||||
}
|
||||
|
||||
/// Index write returned by owner commit. `finish` is intended for a worker after the owner has
|
||||
/// returned, and performs the only archive-index filesystem operation in this pipeline.
|
||||
pub struct LightroomArchiveFinalization {
|
||||
path: Option<PathBuf>,
|
||||
bytes: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
impl LightroomArchiveFinalization {
|
||||
pub fn finish(self) -> crate::Result<()> {
|
||||
let (Some(path), Some(bytes)) = (self.path, self.bytes) else { return Ok(()) };
|
||||
if bytes.len() > MAX_INDEX_BYTES {
|
||||
return Err(error("Lightroom import index exceeds 16 MiB"));
|
||||
}
|
||||
lightcraft_catalog::safe_file::write_atomic(&path, &bytes).map_err(|e| error(e.to_string()))
|
||||
}
|
||||
|
||||
pub fn path(&self) -> Option<&Path> {
|
||||
self.path.as_deref()
|
||||
}
|
||||
}
|
||||
|
||||
/// Owner-thread catalog result. The report is ready for the UI immediately; index finalization
|
||||
/// can remain on the worker while its task stays alive.
|
||||
pub struct LightroomCompletion {
|
||||
pub report: Value,
|
||||
pub finalization: LightroomArchiveFinalization,
|
||||
}
|
||||
|
||||
/// Alias used by hosts that call the result a commit.
|
||||
pub type LightroomCommit = LightroomCompletion;
|
||||
|
||||
/// A Lightroom import detached from the owner thread.
|
||||
pub struct LightroomJob {
|
||||
path: PathBuf,
|
||||
update_existing: bool,
|
||||
import: ImportJob,
|
||||
token: LightroomLibraryToken,
|
||||
archive_dir: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl LightroomJob {
|
||||
/// Snapshot session state only. This constructor performs no filesystem work.
|
||||
pub fn new(s: &mut crate::Session, path: PathBuf, update_existing: bool) -> crate::Result<Self> {
|
||||
let opts = ImportOptions { mode: ImportMode::Add, ..ImportOptions::default() };
|
||||
let import = ImportJob::new(s, opts)?;
|
||||
let archive_dir = s.library.as_ref().filter(|library| library.on_disk).map(|library| library.dir.join("Interop"));
|
||||
Ok(Self { path, update_existing, import, token: LightroomLibraryToken::capture(s), archive_dir })
|
||||
}
|
||||
|
||||
pub fn total(&self) -> usize {
|
||||
self.import.total.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn done(&self) -> usize {
|
||||
self.import.done.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn progress(&self) -> (usize, usize) {
|
||||
(self.done(), self.total())
|
||||
}
|
||||
|
||||
pub fn token(&self) -> &LightroomLibraryToken {
|
||||
&self.token
|
||||
}
|
||||
|
||||
pub fn total_atomic(&self) -> std::sync::Arc<AtomicUsize> {
|
||||
std::sync::Arc::clone(&self.import.total)
|
||||
}
|
||||
|
||||
pub fn done_atomic(&self) -> std::sync::Arc<AtomicUsize> {
|
||||
std::sync::Arc::clone(&self.import.done)
|
||||
}
|
||||
|
||||
/// Finish an owner commit's deferred archive-index write on a worker.
|
||||
pub fn finish_archive(finalization: LightroomArchiveFinalization) -> crate::Result<()> {
|
||||
finalization.finish()
|
||||
}
|
||||
|
||||
/// Read/probe/stage all source data off the owner thread.
|
||||
pub fn prepare(&mut self, cancel: &AtomicBool) -> crate::Result<PreparedLightroom> {
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
return Err(error("Lightroom import cancelled"));
|
||||
}
|
||||
self.import.total.store(1, Ordering::Relaxed);
|
||||
self.import.done.store(0, Ordering::Relaxed);
|
||||
let data = lightroom_catalog::read_with_progress(&self.path, cancel, &self.import.total, &self.import.done).map_err(error)?;
|
||||
lightroom_catalog::validate_for_job(&data).map_err(error)?;
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
return Err(error("Lightroom import cancelled"));
|
||||
}
|
||||
self.import.done.fetch_add(1, Ordering::Relaxed);
|
||||
let paths: Vec<String> = data
|
||||
.photos
|
||||
.iter()
|
||||
.filter(|photo| lightroom_catalog::number_for_job(&photo.image, "masterImage") == 0)
|
||||
.map(|photo| photo.path.clone())
|
||||
.collect();
|
||||
let missing_sources = data
|
||||
.photos
|
||||
.iter()
|
||||
.filter(|photo| lightroom_catalog::number_for_job(&photo.image, "masterImage") == 0)
|
||||
.filter(|photo| !Path::new(&photo.path).is_file())
|
||||
.map(|photo| normalize(&photo.path))
|
||||
.collect();
|
||||
let normal = self.import.prepare(&paths, cancel);
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
normal.rollback();
|
||||
return Err(error("Lightroom import cancelled"));
|
||||
}
|
||||
let archive_path = self.archive_dir.as_ref().map(|dir| crate::lightroom_archive::store(dir, &data).map_err(error)).transpose()?;
|
||||
let index_path = self.archive_dir.as_ref().map(|dir| dir.join("lightroom-index.json"));
|
||||
let index = lightroom_catalog::load_index(index_path.as_deref())?;
|
||||
Ok(PreparedLightroom {
|
||||
data,
|
||||
normal,
|
||||
index,
|
||||
archive_path,
|
||||
token: self.token.clone(),
|
||||
missing_sources,
|
||||
update_existing: self.update_existing,
|
||||
now: self.import.now().to_string(),
|
||||
cancelled: cancel.load(Ordering::Relaxed),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply prepared data on the owner thread. No source, archive or index filesystem work occurs.
|
||||
pub fn commit_prepared(s: &mut crate::Session, mut prepared: PreparedLightroom) -> crate::Result<LightroomCompletion> {
|
||||
if prepared.cancelled {
|
||||
prepared.normal.rollback();
|
||||
return Err(error("Lightroom import cancelled"));
|
||||
}
|
||||
if !prepared.token.matches(s) {
|
||||
prepared.normal.rollback();
|
||||
return Err(error("Lightroom import belongs to a different open library"));
|
||||
}
|
||||
// A command-scoped Lightroom import must never rewrite source XMP sidecars. Direct depth-0
|
||||
// callers do not set this transient flag, so they cannot affect a later command.
|
||||
if s.depth > 0 {
|
||||
s.skip_auto_write = true;
|
||||
}
|
||||
|
||||
let undo0 = s.undo.len();
|
||||
prepared.normal.revalidate_add(s);
|
||||
let options = ImportOptions { mode: ImportMode::Add, ..ImportOptions::default() };
|
||||
let now = prepared.now.clone();
|
||||
let files = crate::import::commit_prepared(s, &options, &now, prepared.normal)?;
|
||||
let applied = lightroom_catalog::apply_prepared(
|
||||
s,
|
||||
prepared.data,
|
||||
lightroom_catalog::ApplyPrepared {
|
||||
update_existing: prepared.update_existing,
|
||||
files,
|
||||
index: &mut prepared.index,
|
||||
archive_path: prepared.archive_path,
|
||||
missing_sources: &prepared.missing_sources,
|
||||
undo0,
|
||||
},
|
||||
)?;
|
||||
let path = prepared.token.library_dir().map(|dir| dir.join("Interop/lightroom-index.json"));
|
||||
let bytes = if path.is_some() { Some(serde_json::to_vec(&applied.index).map_err(|e| error(e.to_string()))?) } else { None };
|
||||
Ok(LightroomCompletion { report: applied.report, finalization: LightroomArchiveFinalization { path, bytes } })
|
||||
}
|
||||
|
||||
fn normalize(path: &str) -> String {
|
||||
let path = path.replace('\\', "/");
|
||||
if cfg!(windows) { path.to_lowercase() } else { path }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn empty_data() -> CatalogImport {
|
||||
CatalogImport {
|
||||
source: "fixture.lrcat".into(),
|
||||
photos: Vec::new(),
|
||||
collections: Vec::new(),
|
||||
members: Vec::new(),
|
||||
collection_content: Vec::new(),
|
||||
warnings: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn cancelled_prepared(s: &crate::Session) -> PreparedLightroom {
|
||||
PreparedLightroom {
|
||||
data: empty_data(),
|
||||
normal: Prepared::default(),
|
||||
index: ImportIndex::default(),
|
||||
archive_path: None,
|
||||
token: LightroomLibraryToken::capture(s),
|
||||
missing_sources: HashSet::new(),
|
||||
update_existing: false,
|
||||
now: "2026-01-01T00:00:00".into(),
|
||||
cancelled: true,
|
||||
}
|
||||
}
|
||||
|
||||
fn prepared_for(s: &crate::Session, path: &str, rating: i64) -> PreparedLightroom {
|
||||
let mut image = std::collections::BTreeMap::new();
|
||||
image.insert("id_local".into(), serde_json::json!(1));
|
||||
image.insert("rating".into(), serde_json::json!(rating));
|
||||
image.insert("pick".into(), serde_json::json!(0));
|
||||
image.insert("fileFormat".into(), serde_json::json!("JPG"));
|
||||
image.insert("fileWidth".into(), serde_json::json!(1));
|
||||
image.insert("fileHeight".into(), serde_json::json!(1));
|
||||
PreparedLightroom {
|
||||
data: CatalogImport {
|
||||
source: "fixture.lrcat".into(),
|
||||
photos: vec![lightroom_catalog::CatalogPhoto {
|
||||
source_id: 1,
|
||||
uuid: "fixture-photo".into(),
|
||||
path: path.into(),
|
||||
image,
|
||||
settings: String::new(),
|
||||
xmp: String::new(),
|
||||
keywords: Vec::new(),
|
||||
history: Vec::new(),
|
||||
snapshots: Vec::new(),
|
||||
}],
|
||||
collections: Vec::new(),
|
||||
members: Vec::new(),
|
||||
collection_content: Vec::new(),
|
||||
warnings: Vec::new(),
|
||||
},
|
||||
normal: Prepared::default(),
|
||||
index: ImportIndex::default(),
|
||||
archive_path: None,
|
||||
token: LightroomLibraryToken::capture(s),
|
||||
missing_sources: [normalize(path)].into_iter().collect(),
|
||||
update_existing: false,
|
||||
now: "2026-01-01T00:00:00".into(),
|
||||
cancelled: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancellation_before_commit_leaves_owner_catalog_unchanged() {
|
||||
let mut session = crate::Session::new();
|
||||
let before = session.catalog.to_snapshot();
|
||||
let prepared = cancelled_prepared(&session);
|
||||
assert!(commit_prepared(&mut session, prepared).is_err());
|
||||
assert_eq!(session.catalog.to_snapshot(), before);
|
||||
assert!(session.undo.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn library_identity_rejects_late_completion_after_reopen() {
|
||||
let mut session = crate::Session::new();
|
||||
let token = LightroomLibraryToken::capture(&session);
|
||||
assert!(token.matches(&session));
|
||||
session.library_identity = Arc::new(());
|
||||
assert!(!token.matches(&session));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepared_reimport_preserves_current_edit_and_uses_one_undo_per_import() {
|
||||
let mut session = crate::Session::new();
|
||||
let path = "/missing/lightroom-job-edit.jpg";
|
||||
let prepared = prepared_for(&session, path, 4);
|
||||
let first = commit_prepared(&mut session, prepared).unwrap();
|
||||
assert_eq!(first.report["mapping"]["1"].as_u64(), Some(1));
|
||||
assert_eq!(session.undo.len(), 1);
|
||||
let id = lightcraft_catalog::PhotoId(1);
|
||||
session.set_develop(id, lightcraft_develop::DevelopSettings::default(), "Personal").unwrap();
|
||||
let before = session.catalog.photo(id).map(|photo| photo.develop.clone());
|
||||
let prepared = prepared_for(&session, path, 5);
|
||||
commit_prepared(&mut session, prepared).unwrap();
|
||||
assert_eq!(session.catalog.photo(id).map(|photo| photo.develop.clone()), before);
|
||||
assert_eq!(session.catalog.photo(id).map(|photo| photo.rating), Some(5));
|
||||
assert_eq!(session.undo.len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn execute_fn_import_does_not_rewrite_source_sidecar() {
|
||||
let dir = std::env::temp_dir().join(format!("lightcraft-lr-job-sidecar-{}", std::process::id()));
|
||||
let path = dir.join("source.jpg");
|
||||
let sidecar = dir.join("source.xmp");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
std::fs::write(&path, b"fixture").unwrap();
|
||||
std::fs::write(&sidecar, b"original sidecar").unwrap();
|
||||
let original = std::fs::read(&sidecar).unwrap();
|
||||
let mut session = crate::Session::new();
|
||||
session.xmp.auto_write = true;
|
||||
let result = session.execute_fn("library.importLightroom", |s| {
|
||||
let prepared = prepared_for(s, &path.to_string_lossy(), 5);
|
||||
let completion = commit_prepared(s, prepared)?;
|
||||
Ok(completion.report)
|
||||
});
|
||||
assert!(result.is_ok());
|
||||
assert_eq!(std::fs::read(&sidecar).unwrap(), original);
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -13,6 +13,7 @@ pub mod headless;
|
|||
pub mod i18n;
|
||||
pub mod icons;
|
||||
pub mod import;
|
||||
pub mod lightroom_import;
|
||||
pub mod links;
|
||||
pub mod menubar;
|
||||
pub mod menus;
|
||||
|
|
@ -187,6 +188,10 @@ pub struct LightcraftApp {
|
|||
pub import: Option<import::ImportTask>,
|
||||
/// A folder scan in progress (feeds the import review).
|
||||
pub scan: Option<import::ScanTask>,
|
||||
/// A Lightroom catalog inspect/import in progress.
|
||||
pub lightroom: Option<lightroom_import::LightroomTask>,
|
||||
/// Last terminal Lightroom result, exposed by the command's status/wait response.
|
||||
pub lightroom_last: Option<Value>,
|
||||
/// A background export in progress.
|
||||
pub export: Option<export_task::ExportTask>,
|
||||
/// Background file-system work of other commands (Find Missing Photos, auto import…).
|
||||
|
|
@ -246,6 +251,8 @@ impl LightcraftApp {
|
|||
merge: merge::MergeState::default(),
|
||||
import: None,
|
||||
scan: None,
|
||||
lightroom: None,
|
||||
lightroom_last: None,
|
||||
export: None,
|
||||
tasks: Default::default(),
|
||||
last_export_result: None,
|
||||
|
|
@ -621,6 +628,7 @@ impl LightcraftApp {
|
|||
merge::poll(self, ctx);
|
||||
import::poll_scan(self, ctx);
|
||||
import::tick(self, ctx);
|
||||
lightroom_import::tick(self, ctx);
|
||||
tasks::poll(self, ctx);
|
||||
self.preview_build_status(ctx);
|
||||
self.save_status(ctx);
|
||||
|
|
@ -642,7 +650,7 @@ impl LightcraftApp {
|
|||
if let Some(folder) = self.session.import_defaults.auto_folder.clone() {
|
||||
const LABEL: &str = "Auto Import";
|
||||
let now = ctx.input(|i| i.time);
|
||||
if now - self.ui.auto_import_at >= 3.0 && self.import.is_none() && !self.tasks.is_running(LABEL) {
|
||||
if now - self.ui.auto_import_at >= 3.0 && self.import.is_none() && self.lightroom.is_none() && !self.tasks.is_running(LABEL) {
|
||||
self.ui.auto_import_at = now;
|
||||
let work = move || lightcraft_engine::cmd::library::list_auto_import_folder(&folder);
|
||||
let done = |app: &mut LightcraftApp, _ctx: &egui::Context, listing: Result<Vec<(String, u64)>, String>| {
|
||||
|
|
@ -683,10 +691,12 @@ impl LightcraftApp {
|
|||
let _ = self.run("file.importPresets", serde_json::json!({"paths": presets}));
|
||||
}
|
||||
// read and added on a worker thread (dropped folders can be large, or on a slow drive)
|
||||
if !photos.is_empty()
|
||||
&& let Err(e) = import::start_paths(self, photos)
|
||||
{
|
||||
self.toast(ctx, e);
|
||||
if !photos.is_empty() {
|
||||
if self.lightroom.is_some() {
|
||||
self.toast(ctx, "Finish Lightroom catalog import before adding photos");
|
||||
} else if let Err(e) = import::start_paths(self, photos) {
|
||||
self.toast(ctx, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -823,6 +833,7 @@ impl LightcraftApp {
|
|||
panels::library_problem::show(self, &ctx);
|
||||
import::progress(self, &ctx);
|
||||
import::scan_progress(self, &ctx);
|
||||
lightroom_import::progress(self, &ctx);
|
||||
export_task::poll(self, &ctx);
|
||||
panels::grid::drag_feedback(self, &ctx);
|
||||
panels::toast(self, &ctx);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,456 @@
|
|||
//! Background Lightroom catalog inspection and import.
|
||||
//!
|
||||
//! Reading a catalog, probing its originals and staging its archive can take minutes on a
|
||||
//! removable or network drive. This task keeps that work off the owner thread, commits prepared
|
||||
//! data only on the owner thread, then finishes the archive index on a worker.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, mpsc};
|
||||
|
||||
use egui::Align2;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::LightcraftApp;
|
||||
use crate::theme::Tokens;
|
||||
use crate::widgets::register;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum Kind {
|
||||
Inspect,
|
||||
Import,
|
||||
}
|
||||
|
||||
enum Message {
|
||||
Inspected(Result<Value, String>),
|
||||
Prepared(Result<lightcraft_engine::lightroom_job::PreparedLightroom, String>),
|
||||
Finalized(Result<(), String>),
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum Phase {
|
||||
Reading,
|
||||
Committing,
|
||||
Finalizing,
|
||||
}
|
||||
|
||||
/// A Lightroom task in flight. The receiver is polled from the egui owner thread.
|
||||
pub struct LightroomTask {
|
||||
kind: Kind,
|
||||
path: PathBuf,
|
||||
cancel: Arc<AtomicBool>,
|
||||
total: Arc<AtomicUsize>,
|
||||
done: Arc<AtomicUsize>,
|
||||
rx: mpsc::Receiver<Message>,
|
||||
phase: Phase,
|
||||
cancelled: bool,
|
||||
}
|
||||
|
||||
impl LightroomTask {
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn status(&self) -> Value {
|
||||
let kind = match self.kind {
|
||||
Kind::Inspect => "inspect",
|
||||
Kind::Import => "import",
|
||||
};
|
||||
let phase = match self.phase {
|
||||
Phase::Reading => "reading",
|
||||
Phase::Committing => "committing",
|
||||
Phase::Finalizing => "finalizing",
|
||||
};
|
||||
json!({
|
||||
"running": true,
|
||||
"kind": kind,
|
||||
"path": self.path,
|
||||
"phase": phase,
|
||||
"done": self.done.load(Ordering::Relaxed),
|
||||
"total": self.total.load(Ordering::Relaxed),
|
||||
"cancelled": self.cancelled || self.cancel.load(Ordering::Relaxed),
|
||||
})
|
||||
}
|
||||
|
||||
fn cancel(&mut self) {
|
||||
self.cancelled = true;
|
||||
self.cancel.store(true, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for LightroomTask {
|
||||
fn drop(&mut self) {
|
||||
self.cancel.store(true, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
fn unsupported_wasm() -> Result<Value, String> {
|
||||
Err("Lightroom catalog import is unavailable in browser builds; open the catalog in the native LightCraft app".into())
|
||||
}
|
||||
|
||||
/// Whether a Lightroom task currently owns the catalog transition.
|
||||
pub fn is_running(app: &LightcraftApp) -> bool {
|
||||
app.lightroom.is_some()
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn busy(app: &LightcraftApp) -> Result<(), String> {
|
||||
if app.lightroom.is_some() {
|
||||
return Err("a Lightroom catalog task is already running".into());
|
||||
}
|
||||
if app.import.is_some() || app.scan.is_some() {
|
||||
return Err("an import is running".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn spawn_inspect(
|
||||
path: PathBuf,
|
||||
cancel: Arc<AtomicBool>,
|
||||
total: Arc<AtomicUsize>,
|
||||
done: Arc<AtomicUsize>,
|
||||
tx: mpsc::Sender<Message>,
|
||||
ctx: egui::Context,
|
||||
) -> Result<(), String> {
|
||||
std::thread::Builder::new()
|
||||
.name("lc-lightroom-inspect".into())
|
||||
.spawn(move || {
|
||||
let result = lightcraft_engine::guard::catch("Lightroom inspect", || {
|
||||
lightcraft_engine::lightroom_catalog::read_with_progress(&path, &cancel, &total, &done)
|
||||
})
|
||||
.and_then(|r| r.map(inspect_report));
|
||||
let _ = tx.send(Message::Inspected(result));
|
||||
ctx.request_repaint();
|
||||
})
|
||||
.map(|_| ())
|
||||
.map_err(|e| format!("could not start Lightroom inspect: {e}"))
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn spawn_prepare(
|
||||
mut job: lightcraft_engine::lightroom_job::LightroomJob,
|
||||
cancel: Arc<AtomicBool>,
|
||||
tx: mpsc::Sender<Message>,
|
||||
ctx: egui::Context,
|
||||
) -> Result<(), String> {
|
||||
std::thread::Builder::new()
|
||||
.name("lc-lightroom-import".into())
|
||||
.spawn(move || {
|
||||
let result = lightcraft_engine::guard::catch("Lightroom import", || job.prepare(&cancel)).and_then(|r| r.map_err(|e| e.to_string()));
|
||||
if let Err(mpsc::SendError(Message::Prepared(Ok(prepared)))) = tx.send(Message::Prepared(result)) {
|
||||
// The owner dropped the task (for example while closing): staged copies/moves
|
||||
// must be put back even though no UI receiver remains to accept the result.
|
||||
prepared.rollback();
|
||||
}
|
||||
ctx.request_repaint();
|
||||
})
|
||||
.map(|_| ())
|
||||
.map_err(|e| format!("could not start Lightroom import: {e}"))
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn spawn_finalize(
|
||||
finalization: lightcraft_engine::lightroom_job::LightroomArchiveFinalization,
|
||||
tx: mpsc::Sender<Message>,
|
||||
ctx: egui::Context,
|
||||
) -> Result<(), String> {
|
||||
std::thread::Builder::new()
|
||||
.name("lc-lightroom-finalize".into())
|
||||
.spawn(move || {
|
||||
let result = lightcraft_engine::guard::catch("Lightroom archive finalization", || finalization.finish())
|
||||
.and_then(|r| r.map_err(|e| e.to_string()));
|
||||
let _ = tx.send(Message::Finalized(result));
|
||||
ctx.request_repaint();
|
||||
})
|
||||
.map(|_| ())
|
||||
.map_err(|e| format!("could not start Lightroom archive finalization: {e}"))
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
fn spawn_finalize(
|
||||
_finalization: lightcraft_engine::lightroom_job::LightroomArchiveFinalization,
|
||||
_tx: mpsc::Sender<Message>,
|
||||
_ctx: egui::Context,
|
||||
) -> Result<(), String> {
|
||||
unsupported_wasm().map(|_| ())
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn inspect_report(data: lightcraft_engine::lightroom_catalog::CatalogImport) -> Value {
|
||||
json!({
|
||||
"photos": data.photos.len(),
|
||||
"collections": data.collections.iter().filter(|r| r.get("systemOnly").and_then(Value::as_f64).unwrap_or(0.0) == 0.0).count(),
|
||||
"missing": data.photos.iter().filter(|p| !std::path::Path::new(&p.path).is_file()).map(|p| &p.path).collect::<Vec<_>>(),
|
||||
"warnings": data.warnings,
|
||||
})
|
||||
}
|
||||
|
||||
fn terminal(app: &mut LightcraftApp, ctx: &egui::Context, kind: Kind, value: Value) {
|
||||
let message = if let Some(e) = value.get("error").and_then(Value::as_str) {
|
||||
let warning = value.get("indexWarning").and_then(Value::as_str).map_or(String::new(), |w| format!("; index warning: {w}"));
|
||||
format!("Lightroom {} failed: {e}{warning}", if matches!(kind, Kind::Inspect) { "inspection" } else { "import" })
|
||||
} else if value.get("cancelled").and_then(Value::as_bool).unwrap_or(false) {
|
||||
format!("Lightroom {} cancelled", if matches!(kind, Kind::Inspect) { "inspection" } else { "import" })
|
||||
} else if matches!(kind, Kind::Inspect) {
|
||||
let photos = value["photos"].as_u64().unwrap_or(0);
|
||||
let collections = value["collections"].as_u64().unwrap_or(0);
|
||||
let missing = value["missing"].as_array().map_or(0, Vec::len);
|
||||
format!("Lightroom catalog inspected: {photos} photos, {collections} collections, {missing} missing")
|
||||
} else {
|
||||
let report = value.get("report").unwrap_or(&value);
|
||||
let imported = report["imported"].as_u64().unwrap_or(0);
|
||||
let failed = report["failed"].as_array().map_or(0, Vec::len);
|
||||
let warning = value.get("indexWarning").and_then(Value::as_str);
|
||||
if let Some(warning) = warning {
|
||||
format!("Lightroom import complete: {imported} photos added; index warning: {warning}")
|
||||
} else if failed == 0 {
|
||||
format!("Lightroom import complete: {imported} photos added")
|
||||
} else {
|
||||
format!("Lightroom import complete: {imported} photos added, {failed} failed")
|
||||
}
|
||||
};
|
||||
app.lightroom_last = Some(value);
|
||||
app.lightroom = None;
|
||||
app.toast_for(ctx, message, 6.0);
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn start_inspect(app: &mut LightcraftApp, path: PathBuf, ctx: &egui::Context) -> Result<Value, String> {
|
||||
busy(app)?;
|
||||
let cancel = Arc::new(AtomicBool::new(false));
|
||||
let total = Arc::new(AtomicUsize::new(0));
|
||||
let done = Arc::new(AtomicUsize::new(0));
|
||||
let (tx, rx) = mpsc::channel();
|
||||
spawn_inspect(path.clone(), cancel.clone(), total.clone(), done.clone(), tx, ctx.clone())?;
|
||||
app.lightroom_last = None;
|
||||
app.lightroom = Some(LightroomTask { kind: Kind::Inspect, path: path.clone(), cancel, total, done, rx, phase: Phase::Reading, cancelled: false });
|
||||
Ok(json!({"running": true, "kind": "inspect", "path": path}))
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn start_import(app: &mut LightcraftApp, path: PathBuf, update_existing: bool, ctx: &egui::Context) -> Result<Value, String> {
|
||||
busy(app)?;
|
||||
let job = lightcraft_engine::lightroom_job::LightroomJob::new(&mut app.session, path.clone(), update_existing).map_err(|e| e.to_string())?;
|
||||
let cancel = Arc::new(AtomicBool::new(false));
|
||||
let total = job.total_atomic();
|
||||
let done = job.done_atomic();
|
||||
let (tx, rx) = mpsc::channel();
|
||||
spawn_prepare(job, cancel.clone(), tx, ctx.clone())?;
|
||||
app.lightroom_last = None;
|
||||
app.lightroom = Some(LightroomTask { kind: Kind::Import, path: path.clone(), cancel, total, done, rx, phase: Phase::Reading, cancelled: false });
|
||||
Ok(json!({"running": true, "kind": "import", "path": path}))
|
||||
}
|
||||
|
||||
/// Run Lightroom UI commands before the generic engine dispatcher.
|
||||
pub fn command(app: &mut LightcraftApp, id: &str, p: &Value, ctx: &egui::Context) -> Result<Value, String> {
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
{
|
||||
let _ = (app, id, p, ctx);
|
||||
return unsupported_wasm();
|
||||
}
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
{
|
||||
let path = p.get("path").and_then(Value::as_str).map(PathBuf::from);
|
||||
let wait = p.get("wait").and_then(Value::as_bool).unwrap_or(false);
|
||||
let status = p.get("status").and_then(Value::as_bool).unwrap_or(false) || path.is_none();
|
||||
if status {
|
||||
if p.get("cancel").and_then(Value::as_bool).unwrap_or(false) {
|
||||
if let Some(task) = app.lightroom.as_mut() {
|
||||
task.cancel();
|
||||
}
|
||||
}
|
||||
if wait && app.lightroom.is_some() {
|
||||
return wait_for(app, ctx, std::time::Duration::from_secs(600));
|
||||
}
|
||||
return Ok(app.lightroom.as_ref().map(LightroomTask::status).or_else(|| app.lightroom_last.clone()).unwrap_or(Value::Null));
|
||||
}
|
||||
let Some(path) = path else { return Err("missing `path`".into()) };
|
||||
let result = if id == "library.inspectLightroom" {
|
||||
start_inspect(app, path, ctx)
|
||||
} else {
|
||||
start_import(app, path, p.get("updateExisting").and_then(Value::as_bool).unwrap_or(false), ctx)
|
||||
}?;
|
||||
if wait {
|
||||
return wait_for(app, ctx, std::time::Duration::from_secs(600));
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
/// Poll owner-thread work, commit a prepared import, and finish its archive index.
|
||||
pub fn tick(app: &mut LightcraftApp, ctx: &egui::Context) {
|
||||
let Some(mut task) = app.lightroom.take() else { return };
|
||||
let message = match task.rx.try_recv() {
|
||||
Ok(m) => m,
|
||||
Err(mpsc::TryRecvError::Empty) => {
|
||||
app.lightroom = Some(task);
|
||||
return;
|
||||
}
|
||||
Err(mpsc::TryRecvError::Disconnected) => {
|
||||
terminal(app, ctx, task.kind, json!({"error": "Lightroom worker stopped unexpectedly"}));
|
||||
return;
|
||||
}
|
||||
};
|
||||
match message {
|
||||
Message::Inspected(result) => match result {
|
||||
Ok(report) if !task.cancel.load(Ordering::Relaxed) => terminal(app, ctx, task.kind, report),
|
||||
Ok(_) => terminal(app, ctx, task.kind, json!({"cancelled": true})),
|
||||
Err(e) if task.cancel.load(Ordering::Relaxed) || e.contains("cancelled") => terminal(app, ctx, task.kind, json!({"cancelled": true})),
|
||||
Err(e) => terminal(app, ctx, task.kind, json!({"error": e})),
|
||||
},
|
||||
Message::Prepared(result) => {
|
||||
let prepared = match result {
|
||||
Ok(p) => p,
|
||||
Err(e) if task.cancel.load(Ordering::Relaxed) || e.contains("cancelled") => {
|
||||
terminal(app, ctx, task.kind, json!({"cancelled": true}));
|
||||
return;
|
||||
}
|
||||
Err(e) => {
|
||||
terminal(app, ctx, task.kind, json!({"error": e}));
|
||||
return;
|
||||
}
|
||||
};
|
||||
if task.cancel.load(Ordering::Relaxed) || prepared.is_cancelled() || !prepared.token().matches(&app.session) {
|
||||
prepared.rollback();
|
||||
terminal(
|
||||
app,
|
||||
ctx,
|
||||
task.kind,
|
||||
if task.cancel.load(Ordering::Relaxed) {
|
||||
json!({"cancelled": true})
|
||||
} else {
|
||||
json!({"error": "Lightroom import belongs to a different open library"})
|
||||
},
|
||||
);
|
||||
return;
|
||||
}
|
||||
task.phase = Phase::Committing;
|
||||
let mut finalization = None;
|
||||
let mut report = None;
|
||||
let committed = app.session.execute_fn("library.importLightroom", |s| {
|
||||
let completion = lightcraft_engine::lightroom_job::commit_prepared(s, prepared)?;
|
||||
report = Some(completion.report.clone());
|
||||
finalization = Some(completion.finalization);
|
||||
Ok(completion.report)
|
||||
});
|
||||
let report = report.unwrap_or(Value::Null);
|
||||
task.phase = Phase::Finalizing;
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let commit_error = committed.as_ref().err().map(ToString::to_string);
|
||||
let Some(finalization) = finalization else {
|
||||
terminal(app, ctx, task.kind, json!({"error": commit_error.clone().unwrap_or_else(|| "Lightroom import did not commit".into())}));
|
||||
return;
|
||||
};
|
||||
if let Err(e) = spawn_finalize(finalization, tx, ctx.clone()) {
|
||||
let mut result = json!({"report": report, "indexWarning": e});
|
||||
if let Some(error) = commit_error {
|
||||
result["error"] = json!(error);
|
||||
}
|
||||
terminal(app, ctx, task.kind, result);
|
||||
return;
|
||||
}
|
||||
task.rx = rx;
|
||||
task.cancelled = false;
|
||||
let mut pending = json!({"running": true, "kind": "import", "report": report});
|
||||
if let Some(error) = commit_error {
|
||||
pending["error"] = json!(error);
|
||||
}
|
||||
app.lightroom_last = Some(pending);
|
||||
app.lightroom = Some(task);
|
||||
}
|
||||
Message::Finalized(result) => {
|
||||
let pending = app.lightroom_last.take().unwrap_or(Value::Null);
|
||||
let report = pending.get("report").cloned().unwrap_or(Value::Null);
|
||||
match result {
|
||||
Ok(()) if pending.get("error").is_none() => terminal(app, ctx, task.kind, report),
|
||||
Ok(()) => terminal(app, ctx, task.kind, json!({"report": report, "error": pending["error"].clone()})),
|
||||
Err(e) => {
|
||||
let mut value = json!({"report": report, "indexWarning": e});
|
||||
if let Some(error) = pending.get("error") {
|
||||
value["error"] = error.clone();
|
||||
}
|
||||
terminal(app, ctx, task.kind, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ctx.request_repaint();
|
||||
}
|
||||
|
||||
/// Wait for a task in control/headless mode while still applying owner-thread completion.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub fn wait_for(app: &mut LightcraftApp, ctx: &egui::Context, timeout: std::time::Duration) -> Result<Value, String> {
|
||||
let start = std::time::Instant::now();
|
||||
while app.lightroom.is_some() {
|
||||
tick(app, ctx);
|
||||
if start.elapsed() >= timeout {
|
||||
return Err("timed out waiting for Lightroom task".into());
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(2));
|
||||
}
|
||||
Ok(app.lightroom_last.clone().unwrap_or(Value::Null))
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub fn wait_for(_app: &mut LightcraftApp, _ctx: &egui::Context, _timeout: std::time::Duration) -> Result<Value, String> {
|
||||
unsupported_wasm()
|
||||
}
|
||||
|
||||
/// The progress window for inspect/import, with cancellation.
|
||||
pub fn progress(app: &mut LightcraftApp, ctx: &egui::Context) {
|
||||
let Some(task) = app.lightroom.as_ref() else { return };
|
||||
let total = task.total.load(Ordering::Relaxed);
|
||||
let done = task.done.load(Ordering::Relaxed);
|
||||
let title = match task.kind {
|
||||
Kind::Inspect => "Inspecting Lightroom Catalog",
|
||||
Kind::Import => "Importing Lightroom Catalog",
|
||||
};
|
||||
let text = match task.phase {
|
||||
Phase::Reading if total == 0 => "Reading Lightroom catalog…".to_string(),
|
||||
Phase::Reading => format!("Reading Lightroom catalog… {done} of {total}"),
|
||||
Phase::Committing => "Adding Lightroom photos…".to_string(),
|
||||
Phase::Finalizing => "Saving Lightroom import index…".to_string(),
|
||||
};
|
||||
let mut cancel = false;
|
||||
let t = Tokens::get(ctx);
|
||||
egui::Window::new(title).title_bar(false).resizable(false).anchor(Align2::CENTER_BOTTOM, [0.0, -80.0]).fixed_size([360.0, 92.0]).show(
|
||||
ctx,
|
||||
|ui| {
|
||||
ui.label(egui::RichText::new(text).color(t.text));
|
||||
ui.add(egui::ProgressBar::new(done as f32 / total.max(1) as f32).desired_width(340.0));
|
||||
let r = ui.add_enabled(task.phase == Phase::Reading && !task.cancel.load(Ordering::Relaxed), egui::Button::new("Cancel"));
|
||||
register(ui.ctx(), "button:lightroomCancel", r.rect);
|
||||
cancel = r.clicked();
|
||||
},
|
||||
);
|
||||
if cancel {
|
||||
if let Some(task) = app.lightroom.as_mut() {
|
||||
task.cancel();
|
||||
}
|
||||
ctx.request_repaint();
|
||||
}
|
||||
if app.lightroom.is_some() {
|
||||
ctx.request_repaint_after(std::time::Duration::from_millis(100));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::inspect_report;
|
||||
use lightcraft_engine::lightroom_catalog::CatalogImport;
|
||||
|
||||
#[test]
|
||||
fn inspect_report_keeps_empty_catalog_shape() {
|
||||
let report = inspect_report(CatalogImport {
|
||||
source: "synthetic.lrcat".into(),
|
||||
photos: Vec::new(),
|
||||
collections: Vec::new(),
|
||||
members: Vec::new(),
|
||||
collection_content: Vec::new(),
|
||||
warnings: vec!["warning".into()],
|
||||
});
|
||||
assert_eq!(report["photos"], 0);
|
||||
assert_eq!(report["collections"], 0);
|
||||
assert_eq!(report["missing"], serde_json::json!([]));
|
||||
assert_eq!(report["warnings"], serde_json::json!(["warning"]));
|
||||
}
|
||||
}
|
||||
|
|
@ -236,6 +236,10 @@ pub fn parse_rgb(v: &Value) -> Option<[u8; 3]> {
|
|||
|
||||
/// Handle UI commands; `None` means "not a UI command — send it to the engine".
|
||||
pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option<Result<Value, String>> {
|
||||
if matches!(id, "library.inspectLightroom" | "library.importLightroom") {
|
||||
let ctx = egui::Context::default();
|
||||
return Some(crate::lightroom_import::command(app, id, p, &ctx));
|
||||
}
|
||||
if let Some(language) = language_from_command(id) {
|
||||
app.ui.language = language;
|
||||
// Immediately, not on the next frame: the reply and anything else run this frame
|
||||
|
|
@ -948,11 +952,17 @@ pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option<Re
|
|||
Ok(Value::Null)
|
||||
}
|
||||
"library.browse" if !cfg!(target_arch = "wasm32") => {
|
||||
if crate::lightroom_import::is_running(app) {
|
||||
return Some(Err("finish Lightroom catalog import before browsing folders".into()));
|
||||
}
|
||||
// listed and read in the background (see `import::browse`)
|
||||
let path = p.get("path").and_then(Value::as_str)?;
|
||||
crate::import::browse(app, path, p.get("subfolders").and_then(Value::as_bool))
|
||||
}
|
||||
"file.addPhotos" => {
|
||||
if crate::lightroom_import::is_running(app) {
|
||||
return Some(Err("finish Lightroom catalog import before adding photos".into()));
|
||||
}
|
||||
let paths = match p.get("paths").and_then(Value::as_array) {
|
||||
Some(a) => a.iter().filter_map(Value::as_str).map(str::to_string).collect(),
|
||||
None => app.services.pick_files.as_mut().map(|f| f()).unwrap_or_default(),
|
||||
|
|
@ -1278,7 +1288,7 @@ pub fn ui_enabled(app: &LightcraftApp, id: &str) -> bool {
|
|||
"file.exportCurvePresets" => !app.session.curve_presets.is_empty(),
|
||||
"view.compare" => app.session.catalog.len() > 1,
|
||||
"view.fullScreenPreview" | "view.infoOverlay" | "view.navigator" => app.session.active().is_some() || app.ui.fullscreen,
|
||||
"app.openLibrary" | "file.addFolder" => app.services.pick_folder.is_some(),
|
||||
"app.openLibrary" | "file.addFolder" => app.services.pick_folder.is_some() && !crate::lightroom_import::is_running(app),
|
||||
"file.backupLibrary" => app.services.backup_library.is_some(),
|
||||
"file.restoreLibrary" => app.services.restore_library.is_some(),
|
||||
"compare.swap" | "compare.makeSelect" => app.ui.view == ViewMode::Compare,
|
||||
|
|
|
|||
|
|
@ -530,6 +530,9 @@ fn interface_tab(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens) {
|
|||
/// `app.openLibrary {path?}`: close the current library and open (or create) the one at `path`,
|
||||
/// or a folder chosen in a dialog. Remembered as the library to open at launch.
|
||||
pub fn open_library(app: &mut LightcraftApp, p: &Value) -> Result<Value, String> {
|
||||
if crate::lightroom_import::is_running(app) {
|
||||
return Err("wait for Lightroom catalog import to finish before switching libraries".into());
|
||||
}
|
||||
let path = match p.get("path").and_then(Value::as_str) {
|
||||
Some(x) => x.to_string(),
|
||||
None => match app.services.pick_folder.as_mut() {
|
||||
|
|
@ -542,6 +545,7 @@ pub fn open_library(app: &mut LightcraftApp, p: &Value) -> Result<Value, String>
|
|||
};
|
||||
app.session.close_library().map_err(|e| e.to_string())?;
|
||||
app.session.open_library(&path, false).map_err(|e| e.to_string())?;
|
||||
app.lightroom_last = None;
|
||||
app.renderer.forget_all();
|
||||
app.ui.compare = None;
|
||||
app.ui.settings.library_path = path.clone();
|
||||
|
|
|
|||
|
|
@ -19,10 +19,15 @@ identity prevents reimport from duplicating virtual copies and collections. One
|
|||
catalog changes; recovery archives remain available.
|
||||
|
||||
Before changing records, source image records, decoded XMP, verbatim develop settings, history,
|
||||
snapshots and collection content are saved to `Interop/lightroom-import-N.json` in the LightCraft
|
||||
library. `Interop/lightroom-index.json` records imported identities. Source catalogs and originals
|
||||
snapshots and collection content are saved to compressed `Interop/lightroom-import-N.lca` archives
|
||||
in the LightCraft library. Unchanged source data reuses its existing archive. Opaque SQL BLOBs are
|
||||
skipped; decoded XMP and structured recovery data remain. Each archive is capped at 32 MiB
|
||||
(uncompressed and on disk), with at most eight archives and 128 MiB total retained. Oldest managed
|
||||
archives are retired; legacy `.json` archives remain untouched. `Interop/lightroom-index.json` records imported identities. Source catalogs and originals
|
||||
are never overwritten. Native import is bounded to a 1 GiB database/WAL, 16 MiB XMP packets and
|
||||
one million rows per table. The SQLite reader is Apache-2.0 `sqlite-core`, written in Rust.
|
||||
one million rows per table. The in-tree, read-only Rust SQLite B-tree reader follows SQLite’s public file-format specification;
|
||||
it introduces no dependency or C runtime. Table payloads are capped at 256 MiB. Unsupported
|
||||
WITHOUT ROWID and virtual tables produce explicit errors.
|
||||
|
||||
Develop settings reuse the existing XMP/preset mapper. Supported sliders, curves and supported
|
||||
mask structures remain editable, but this is approximate rendering: camera profiles, Adobe AI
|
||||
|
|
@ -34,5 +39,15 @@ Lightroom's `-999999` deferred-adjustment sentinel is omitted from both catalog
|
|||
it is reported and archived, never clamped into a real slider value. Deferred Adobe Auto Tone
|
||||
is not evaluated by the importer; LightCraft's Auto control remains available after migration.
|
||||
|
||||
Native desktop inspection/import runs on a cancellable background worker with progress. Only
|
||||
prepared catalog operations commit on the owner thread; archive index writing stays on a worker.
|
||||
Late results from a closed or switched library are rejected, including reopening the same path.
|
||||
Ordinary edits during preparation remain intact. CLI commands complete synchronously using the
|
||||
same preparation/commit pipeline; browser builds reject native catalog import.
|
||||
|
||||
Archive version 1 begins with the 12 bytes `LC-LRARCH\0\x01Z`, followed by zlib-compressed JSON.
|
||||
Recovery readers must cap decompression at 32 MiB. JSON retains source identities, photo paths,
|
||||
settings, decoded XMP, keywords, history, snapshots, collection definitions and membership.
|
||||
|
||||
The importer uses shared Rust code on macOS, Windows and Linux. If originals move between
|
||||
computers or volumes, use LightCraft's missing-photo relinking tools to update their paths.
|
||||
|
|
|
|||
|
|
@ -716,7 +716,7 @@ From `08-lightroom-classic-extras.md` (Classic-only features) and part 2 of `06-
|
|||
| LRC-LIB-SURVEY | Survey view | P2 | ✅ | `cmd:view.survey` | N |
|
||||
| LRC-LIB-REFVIEW | Reference view | P2 | ✅ | `cmd:view.reference`, `cmd:photo.setReference`, `crates/ui-egui/src/panels/compare.rs` (`show_reference`) | View ▸ Reference View (⇧R): the reference photo beside the active one (edited with the Edit panel); photo menu ▸ Set as Reference Photo |
|
||||
| LRC-LIB-CATALOG-PANEL | Catalog sets | P1 | ✅ | `cmd:library.source` (`missing`) | all, recently added, picks, recently deleted, missing photos (when any) |
|
||||
| LRC-LIB-CATALOG-IMPORT | Native Lightroom Classic catalog import | P1 | 🟡 | `cmd:library.inspectLightroom`, `cmd:library.importLightroom`, `crates/engine/src/lightroom_catalog.rs`, `docs/lightroom-catalog-import.md` | read-only `.lrcat` plus committed WAL pages, in-process Rust SQLite reader; originals in place, ratings/flags/labels, XMP, hierarchical keywords, collection sets/regular collections, virtual copies, mapped develop settings; stable reimport identities, undo, missing-file placeholders, existing edits preserved. Verbatim settings/history/snapshots/collection rules archived. Adobe rendering, AI data and native history replay remain incomplete; no writes to Lightroom catalogs |
|
||||
| LRC-LIB-CATALOG-IMPORT | Native Lightroom Classic catalog import | P1 | 🟡 | `cmd:library.inspectLightroom`, `cmd:library.importLightroom`, `crates/engine/src/lightroom_catalog.rs`, `docs/lightroom-catalog-import.md` | read-only `.lrcat` plus committed WAL pages, in-tree read-only Rust SQLite B-tree reader; background inspection/import with progress/cancellation, per-open library identity checks; originals in place, ratings/flags/labels, XMP, hierarchical keywords, collection sets/regular collections, virtual copies, mapped develop settings; stable reimport identities, undo, missing-file placeholders, existing edits preserved. Settings/history/snapshots/collection rules retained in deduplicated compressed archives (32 MiB each, eight archives/128 MiB aggregate); opaque SQL BLOBs skipped. Adobe rendering, AI data and native history replay remain incomplete; no writes to Lightroom catalogs |
|
||||
| LRC-LIB-CATALOG-LOCK | Catalog open in one program at a time | P1 | ✅ | `cmd:app.openLibrary`, `crates/catalog/src/lock.rs`, `crates/engine/src/library.rs` (`open_library`) | exclusive OS lock on `catalog.lock` for the session (app and `lightcraft-cli --library`); a second opener is refused with who has it (`catalog.lock.owner`: program, process, computer); released by the OS on a crash, so never stale; file systems without locks open unprotected with a warning; no read-only mode; the browser build (OPFS) has no guard across tabs yet. A library that can't be opened at launch (locked, unreadable, newer format, drive missing) shows a window — Try Again, Choose Another Library…, Continue Without Saving (then a banner; nothing written), Quit — instead of a silent demo session (`crates/ui-egui/src/panels/library_problem.rs`) |
|
||||
| LRC-LIB-FOLDERS | Disk folder tree | P1 | ✅ | `cmd:library.browse`, `cmd:folder.rename`, `cmd:folder.move`, `crates/ui-egui/src/panels/left.rs` (`local_section`, `folder_tree`) | standard places + the browsed folder + Browse Folder…, each an expandable tree of subfolders, breadcrumb navigation; context menu: Rename Folder…, Move Folder To…, Show in Finder (on disk, photos relinked; one undo step each: undo / redo move the folder back and forth, refused when the destination is taken) |
|
||||
| LRC-LIB-COLLECTIONS | Collections & sets | P1 | ✅ | `cmd:album.create`, `cmd:album.createSmart`, `cmd:album.toggleTarget`, `cmd:album.setTarget`, `cmd:album.clearQuick` | albums + folders (sets), smart albums (rule editor; rules incl. File Path — the whole string, either separator, case-insensitive), Quick Collection and target album (B in the grids adds / removes the selection; album menu ▸ Set as Target Album; the target is marked +) |
|
||||
|
|
|
|||
Loading…
Reference in New Issue