From 866d02e577cb28b3f3b86da52a4174098348ec14 Mon Sep 17 00:00:00 2001 From: Adrian D'souza Date: Thu, 8 Oct 2026 12:41:14 +0530 Subject: [PATCH] LRC-LIB-CATALOG-IMPORT: address SQLite, worker & archive review Co-authored-by: Codex --- crates/engine/Cargo.toml | 1 - crates/engine/src/import.rs | 72 ++ crates/engine/src/lib.rs | 6 + crates/engine/src/library.rs | 7 +- crates/engine/src/lightroom_archive.rs | 654 +++++++++++++++ crates/engine/src/lightroom_catalog.rs | 260 ++++-- crates/engine/src/lightroom_job.rs | 386 +++++++++ crates/engine/src/lightroom_sqlite.rs | 1071 ++++++++++++++++++++++++ crates/ui-egui/src/lib.rs | 21 +- crates/ui-egui/src/lightroom_import.rs | 456 ++++++++++ crates/ui-egui/src/menus.rs | 12 +- crates/ui-egui/src/panels/settings.rs | 4 + docs/lightroom-catalog-import.md | 21 +- docs/parity.md | 2 +- 14 files changed, 2903 insertions(+), 70 deletions(-) create mode 100644 crates/engine/src/lightroom_archive.rs create mode 100644 crates/engine/src/lightroom_job.rs create mode 100644 crates/engine/src/lightroom_sqlite.rs create mode 100644 crates/ui-egui/src/lightroom_import.rs diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index f59523f0..c2d4e8a8 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -29,7 +29,6 @@ serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } log = { workspace = true } -sqlite-core = "=0.12.0" ab_glyph = { workspace = true } lightcraft-segment = { workspace = true, optional = true } diff --git a/crates/engine/src/import.rs b/crates/engine/src/import.rs index 4895eee7..1e94226e 100644 --- a/crates/engine/src/import.rs +++ b/crates/engine/src/import.rs @@ -610,6 +610,78 @@ impl Prepared { pub fn is_empty(&self) -> bool { self.items.is_empty() } + + /// Recheck Add-only entries against the owner catalog immediately before commit. + /// Preparation may have run while another import added the same path or content. + pub fn revalidate_add(&mut self, s: &Session) { + let mut paths = HashMap::new(); + let mut hashes = HashMap::new(); + for photo in s.catalog.photos() { + if let Source::File { path } = &photo.source { + paths.insert(normalize_import_path(path), photo.id); + } + if let Some(hash) = &photo.content_hash { + hashes.insert(hash.clone(), photo.id); + } + } + let mut items = Vec::with_capacity(self.items.len()); + for item in std::mem::take(&mut self.items) { + let PreparedItem::Ready(ready) = item else { + items.push(item); + continue; + }; + let ReadyFile { path, stored, info, sidecar, placed } = *ready; + let existing_path = paths.get(&normalize_import_path(&stored)).copied().or_else(|| paths.get(&normalize_import_path(&path)).copied()); + let existing_hash = info.content_hash.as_ref().and_then(|hash| hashes.get(hash).copied()); + if let Some(existing) = existing_path.or(existing_hash) { + if let Some(placed) = &placed { + crate::import_move::rollback(placed); + } + items.push(PreparedItem::Duplicate { + path, + existing: Some(existing), + reason: if existing_path.is_some() { "path" } else { "content" }, + hash: info.content_hash, + }); + } else { + items.push(PreparedItem::Ready(Box::new(ReadyFile { path, stored, info, sidecar, placed }))); + } + } + self.items = items; + } +} + +fn normalize_import_path(path: &str) -> String { + let path = path.replace('\\', "/"); + if cfg!(windows) { path.to_lowercase() } else { path } +} + +#[cfg(test)] +mod prepared_tests { + use super::*; + + #[test] + fn revalidate_add_only_converts_matching_ready_item() { + let mut s = Session::new(); + let existing_path = std::env::temp_dir().join("lightcraft-revalidate-existing.jpg"); + let other_path = std::env::temp_dir().join("lightcraft-revalidate-other.jpg"); + let existing = Photo::new(PhotoId(1), Source::File { path: existing_path.to_string_lossy().into() }, "existing.jpg", "JPG", 1, 1, "now"); + s.commit("existing", Op::AddPhoto { photo: Box::new(existing) }).unwrap(); + let ready = |path: &Path| ReadyFile { + path: path.to_string_lossy().into(), + stored: path.to_string_lossy().into(), + info: ProbeInfo { format: "JPG".into(), ..ProbeInfo::default() }, + sidecar: None, + placed: None, + }; + let mut prepared = Prepared { + scanned: 2, + items: vec![PreparedItem::Ready(Box::new(ready(&existing_path))), PreparedItem::Ready(Box::new(ready(&other_path)))], + }; + prepared.revalidate_add(&s); + assert!(matches!(prepared.items.first(), Some(PreparedItem::Duplicate { reason: "path", .. }))); + assert!(matches!(prepared.items.get(1), Some(PreparedItem::Ready(_)))); + } } impl ImportJob { diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 4ca8b5bc..122bde8f 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -25,7 +25,10 @@ pub mod guard; pub mod import; mod import_move; pub mod library; +mod lightroom_archive; pub mod lightroom_catalog; +pub mod lightroom_job; +mod lightroom_sqlite; pub mod media; pub mod memory; pub mod merge; @@ -162,6 +165,8 @@ pub struct Session { pub active_spot: Option, /// The persistent library this session writes to (`None` = in-memory only). pub library: Option, + /// Identity of current library opening; refreshed only after a successful open. + pub(crate) library_identity: Arc<()>, /// XMP sidecar preferences (persisted with the library). pub xmp: sidecar::XmpPrefs, /// Parameters of the last export (`app.export` params, minus targets), persisted in prefs.json. @@ -249,6 +254,7 @@ impl Session { segmenter: segment::Segmenter::default(), active_spot: None, library: None, + library_identity: Arc::new(()), xmp: sidecar::XmpPrefs::default(), last_export: None, export_presets: Vec::new(), diff --git a/crates/engine/src/library.rs b/crates/engine/src/library.rs index 6a63e378..a3f5be33 100644 --- a/crates/engine/src/library.rs +++ b/crates/engine/src/library.rs @@ -399,6 +399,7 @@ impl Session { lib.forgot_local = Some(plan); } } + self.library_identity = std::sync::Arc::new(()); Ok(()) } @@ -491,12 +492,16 @@ impl Session { lib.retry_at = None; lib.last_error = None; } - match (persisted, snapshot) { + let result = match (persisted, snapshot) { (_, Ok(())) => Ok(()), // neither the log nor the snapshot took the queued ops (Err(EngineError::NotSaved(e)), Err(s)) if lib.journal.seq() == before => Err(EngineError::NotSaved(format!("{e}; snapshot: {s}"))), (_, Err(s)) => Err(s.into()), + }; + if result.is_ok() { + self.library_identity = std::sync::Arc::new(()); } + result } fn view_json(&self) -> Vec { diff --git a/crates/engine/src/lightroom_archive.rs b/crates/engine/src/lightroom_archive.rs new file mode 100644 index 00000000..ecb53e25 --- /dev/null +++ b/crates/engine/src/lightroom_archive.rs @@ -0,0 +1,654 @@ +//! Bounded, deduplicated persistence for Lightroom import source data. +//! +//! Archives use a deterministic JSON representation wrapped in zlib. The JSON +//! writer streams through a bounded compressor, so an import cannot first build +//! an unbounded archive buffer. Blob columns are omitted by the catalog reader; +//! decoded XMP, +//! settings, history, snapshots, identities, paths, collections & membership +//! remain available for recovery. + +use std::collections::BTreeMap; +use std::fs::{self, File}; +use std::io::{self, Read, Write}; +use std::path::{Path, PathBuf}; + +use miniz_oxide::deflate::{ + core::{CompressorOxide, create_comp_flags_from_zip_params}, + stream::deflate, +}; +use miniz_oxide::{MZFlush, MZStatus}; +use serde::ser::{Serialize, SerializeMap, SerializeSeq, SerializeStruct, Serializer}; +use serde_json::Value; + +use crate::lightroom_catalog::{CatalogImport, CatalogPhoto}; + +/// Maximum size of one complete archive on disk, including format header. +pub const MAX_ARCHIVE_BYTES: u64 = 32 * 1024 * 1024; +/// Maximum total size retained in managed archives. +pub const MAX_TOTAL_ARCHIVE_BYTES: u64 = 128 * 1024 * 1024; +/// Maximum number of managed archives retained. +pub const MAX_MANAGED_ARCHIVES: usize = 8; + +const FORMAT_VERSION: u8 = 1; +const HEADER: &[u8] = b"LC-LRARCH\0\x01Z"; +const NAME_PREFIX: &str = "lightroom-import-"; +const NAME_SUFFIX: &str = ".lca"; +const MAX_SCAN: usize = 1024; +const IO_BUFFER: usize = 16 * 1024; + +/// Store compact source data below `directory` and return its managed archive. +/// +/// The operation is safe to call before catalog mutation. It rejects a +/// symlink directory, writes only `lightroom-import-.lca`, compares the +/// complete bounded byte stream with recent managed archives, and retains at +/// most [`MAX_MANAGED_ARCHIVES`] and [`MAX_TOTAL_ARCHIVE_BYTES`]. Legacy +/// `lightroom-import-*.json` files are deliberately ignored and preserved for +/// an explicit caller warning. +pub fn store(directory: &Path, data: &CatalogImport) -> Result { + prepare_directory(directory)?; + let mut managed = managed_files(directory)?; + managed.sort_by_key(|entry| entry.index); + + let temporary = temporary_path(directory)?; + write_archive(&temporary, data).map_err(|error| { + let _ = remove_managed_file(&temporary); + error + })?; + + let recent = managed.iter().rev().take(MAX_MANAGED_ARCHIVES); + for entry in recent { + let same = match same_bounded(&temporary, &entry.path) { + Ok(same) => same, + Err(error) => { + let _ = remove_managed_file(&temporary); + return Err(error.to_string()); + } + }; + if same { + remove_managed_file(&temporary).map_err(|e| e.to_string())?; + let matched = entry.path.clone(); + let mut all = managed.clone(); + prune_existing(directory, &mut all, Some(&matched))?; + return Ok(matched); + } + } + + let index = managed.last().map_or(0, |entry| entry.index.saturating_add(1)); + if managed.last().is_some_and(|entry| entry.index == u64::MAX) { + let _ = remove_managed_file(&temporary); + return Err("managed Lightroom archive index is exhausted".into()); + } + let destination = match next_destination(directory, index) { + Ok(path) => path, + Err(error) => { + let _ = remove_managed_file(&temporary); + return Err(error); + } + }; + let size = match fs::metadata(&temporary) { + Ok(meta) => meta.len(), + Err(error) => { + let _ = remove_managed_file(&temporary); + return Err(error.to_string()); + } + }; + let new_entry = Managed { index, path: destination.clone(), size }; + let mut after = managed; + after.push(new_entry); + after.sort_by_key(|entry| entry.index); + let retirees = match retirement_set(&after) { + Ok(retirees) => retirees, + Err(error) => { + let _ = remove_managed_file(&temporary); + return Err(error); + } + }; + let quarantined = match quarantine(&retirees) { + Ok(quarantined) => quarantined, + Err(error) => { + let _ = remove_managed_file(&temporary); + return Err(error); + } + }; + + if let Err(error) = publish_new(&temporary, &destination) { + rollback_quarantine(&quarantined); + let _ = remove_managed_file(&temporary); + return Err(error.to_string()); + } + if let Err(error) = remove_managed_file(&temporary) { + return Err(format!("archive published but temporary cleanup failed: {error}")); + } + for (old, held) in quarantined { + if let Err(error) = remove_managed_file(&held) { + return Err(format!("archive published but could not retire {}: {error}", old.display())); + } + } + Ok(destination) +} + +#[derive(Clone, Debug)] +struct Managed { + index: u64, + path: PathBuf, + size: u64, +} + +fn prepare_directory(directory: &Path) -> Result<(), String> { + match fs::symlink_metadata(directory) { + Ok(meta) => { + if meta.file_type().is_symlink() || !meta.is_dir() { + return Err(format!("archive directory is not a real directory: {}", directory.display())); + } + } + Err(error) if error.kind() == io::ErrorKind::NotFound => { + fs::create_dir_all(directory).map_err(|e| e.to_string())?; + let meta = fs::symlink_metadata(directory).map_err(|e| e.to_string())?; + if meta.file_type().is_symlink() || !meta.is_dir() { + return Err(format!("archive directory is not a real directory: {}", directory.display())); + } + } + Err(error) => return Err(error.to_string()), + } + Ok(()) +} + +fn managed_files(directory: &Path) -> Result, String> { + let mut files = Vec::new(); + for (seen, result) in fs::read_dir(directory).map_err(|e| e.to_string())?.enumerate() { + if seen >= MAX_SCAN { + return Err(format!("too many entries in archive directory (limit {MAX_SCAN})")); + } + let entry = result.map_err(|e| e.to_string())?; + let Some(index) = parse_managed_name(&entry.file_name()) else { continue }; + let meta = fs::symlink_metadata(entry.path()).map_err(|e| e.to_string())?; + if meta.file_type().is_symlink() || !meta.is_file() { + continue; + } + files.push(Managed { index, path: entry.path(), size: meta.len() }); + } + Ok(files) +} + +fn parse_managed_name(name: &std::ffi::OsStr) -> Option { + let name = name.to_str()?; + let digits = name.strip_prefix(NAME_PREFIX)?.strip_suffix(NAME_SUFFIX)?; + if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + digits.parse().ok() +} + +fn temporary_path(directory: &Path) -> Result { + let pid = std::process::id(); + for slot in 0..MAX_SCAN { + let path = directory.join(format!(".lightroom-import-{pid}-{slot}.tmp")); + match fs::symlink_metadata(&path) { + Ok(_) => continue, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(path), + Err(error) => return Err(error.to_string()), + } + } + Err("no safe temporary archive name available".into()) +} + +fn next_destination(directory: &Path, mut index: u64) -> Result { + for _ in 0..MAX_SCAN { + let path = directory.join(format!("{NAME_PREFIX}{index}{NAME_SUFFIX}")); + match fs::symlink_metadata(&path) { + Ok(_) => index = index.checked_add(1).ok_or("managed archive index is exhausted")?, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(path), + Err(error) => return Err(error.to_string()), + } + } + Err("no safe managed archive name available".into()) +} + +fn write_archive(path: &Path, data: &CatalogImport) -> Result<(), String> { + lightcraft_catalog::safe_file::write_atomic_with(path, &mut |raw| { + let mut bounded = BoundedWriter { inner: raw, total: 0 }; + bounded.write_all(HEADER)?; + let mut compressor = ZlibWriter::new(&mut bounded); + serde_json::to_writer(&mut compressor, &ArchiveRef(data)).map_err(json_io)?; + compressor.finish()?; + Ok(()) + }) + .map(|_| ()) + .map_err(|error| format!("could not write Lightroom archive: {error}")) +} + +fn json_io(error: serde_json::Error) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, error.to_string()) +} + +struct BoundedWriter<'a> { + inner: &'a mut dyn Write, + total: u64, +} + +impl Write for BoundedWriter<'_> { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let next = self.total.checked_add(bytes.len() as u64).ok_or_else(limit_error)?; + if next > MAX_ARCHIVE_BYTES { + return Err(limit_error()); + } + self.inner.write_all(bytes)?; + self.total = next; + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +fn limit_error() -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, "Lightroom archive exceeds 32 MiB") +} + +struct ZlibWriter<'a, 'b> { + sink: &'a mut BoundedWriter<'b>, + compressor: CompressorOxide, + plain: u64, +} + +impl<'a, 'b> ZlibWriter<'a, 'b> { + fn new(sink: &'a mut BoundedWriter<'b>) -> ZlibWriter<'a, 'b> { + let flags = create_comp_flags_from_zip_params(6, 15, 0); + ZlibWriter { sink, compressor: CompressorOxide::new(flags), plain: 0 } + } + + fn run(&mut self, input: &[u8], flush: MZFlush) -> io::Result<()> { + let mut rest = input; + loop { + let mut output = [0u8; IO_BUFFER]; + let result = deflate(&mut self.compressor, rest, &mut output, flush); + self.sink.write_all(&output[..result.bytes_written])?; + match result.status { + Ok(MZStatus::StreamEnd) => return Ok(()), + Ok(MZStatus::Ok) => { + let consumed = result.bytes_consumed; + if consumed > rest.len() { + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid compressor progress")); + } + rest = &rest[consumed..]; + if rest.is_empty() && flush == MZFlush::None { + return Ok(()); + } + if consumed == 0 && result.bytes_written == 0 { + return Err(io::Error::new(io::ErrorKind::InvalidData, "compressor made no progress")); + } + } + Ok(status) => { + return Err(io::Error::new(io::ErrorKind::InvalidData, format!("unsupported compressor status: {status:?}"))); + } + Err(error) => return Err(io::Error::new(io::ErrorKind::InvalidData, format!("compression failed: {error:?}"))), + } + } + } + + fn finish(&mut self) -> io::Result<()> { + self.run(&[], MZFlush::Finish) + } +} + +impl Write for ZlibWriter<'_, '_> { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if bytes.is_empty() { + return Ok(0); + } + let next = self.plain.checked_add(bytes.len() as u64).ok_or_else(limit_error)?; + if next > MAX_ARCHIVE_BYTES { + return Err(limit_error()); + } + self.run(bytes, MZFlush::None)?; + self.plain = next; + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +fn same_bounded(left: &Path, right: &Path) -> io::Result { + let left_meta = fs::symlink_metadata(left)?; + let right_meta = fs::symlink_metadata(right)?; + if left_meta.file_type().is_symlink() || right_meta.file_type().is_symlink() || !left_meta.is_file() || !right_meta.is_file() { + return Ok(false); + } + let left_len = left_meta.len(); + let right_len = right_meta.len(); + if left_len != right_len || left_len > MAX_ARCHIVE_BYTES { + return Ok(false); + } + let mut left = File::open(left)?; + let mut right = File::open(right)?; + let mut left_buf = [0u8; IO_BUFFER]; + let mut right_buf = [0u8; IO_BUFFER]; + loop { + let left_n = left.read(&mut left_buf)?; + let right_n = right.read(&mut right_buf)?; + if left_n != right_n { + return Ok(false); + } + if left_n == 0 { + return Ok(true); + } + if left_buf[..left_n] != right_buf[..right_n] { + return Ok(false); + } + } +} + +fn retirement_set(entries: &[Managed]) -> Result, String> { + retirement_set_protected(entries, None) +} + +fn retirement_set_protected(entries: &[Managed], protected: Option<&Path>) -> Result, String> { + let mut ordered = entries.to_vec(); + ordered.sort_by_key(|entry| entry.index); + let mut total = ordered.iter().try_fold(0u64, |sum, entry| sum.checked_add(entry.size)).ok_or("archive size overflow")?; + let mut retirees = Vec::new(); + while ordered.len() > MAX_MANAGED_ARCHIVES || total > MAX_TOTAL_ARCHIVE_BYTES { + let Some(position) = ordered.iter().position(|entry| protected.map_or(true, |path| entry.path != path)) else { + return Err("cannot satisfy archive retention limit".into()); + }; + let oldest = ordered.remove(position); + total = total.saturating_sub(oldest.size); + retirees.push(oldest); + } + Ok(retirees) +} + +fn prune_existing(directory: &Path, entries: &mut [Managed], protected: Option<&Path>) -> Result<(), String> { + let retirees = retirement_set_protected(entries, protected)?; + let quarantined = quarantine(&retirees)?; + for (old, held) in quarantined { + if let Err(error) = remove_managed_file(&held) { + return Err(format!("could not retire {} from {}: {error}", old.display(), directory.display())); + } + } + Ok(()) +} + +fn quarantine(entries: &[Managed]) -> Result, String> { + let mut moved = Vec::new(); + let pid = std::process::id(); + for (slot, entry) in entries.iter().enumerate() { + let held = entry.path.with_file_name(format!(".lightroom-retire-{pid}-{slot}.tmp")); + match fs::symlink_metadata(&held) { + Ok(_) => { + rollback_quarantine(&moved); + return Err(format!("retention temporary path already exists: {}", held.display())); + } + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => { + rollback_quarantine(&moved); + return Err(error.to_string()); + } + } + let meta = fs::symlink_metadata(&entry.path).map_err(|error| { + rollback_quarantine(&moved); + error.to_string() + })?; + if meta.file_type().is_symlink() || !meta.is_file() { + rollback_quarantine(&moved); + return Err(format!("managed archive changed into non-file: {}", entry.path.display())); + } + if let Err(error) = fs::rename(&entry.path, &held) { + rollback_quarantine(&moved); + return Err(error.to_string()); + } + moved.push((entry.path.clone(), held)); + } + Ok(moved) +} + +fn rollback_quarantine(moved: &[(PathBuf, PathBuf)]) { + for (original, held) in moved.iter().rev() { + let _ = fs::rename(held, original); + } +} + +fn publish_new(temporary: &Path, destination: &Path) -> io::Result<()> { + match fs::hard_link(temporary, destination) { + Ok(()) => Ok(()), + Err(error) => Err(io::Error::new(error.kind(), format!("cannot atomically publish archive: {error}"))), + } +} + +fn remove_managed_file(path: &Path) -> io::Result<()> { + let meta = fs::symlink_metadata(path)?; + if meta.file_type().is_symlink() || !meta.is_file() { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "refusing to remove non-regular archive path")); + } + fs::remove_file(path) +} + +struct ArchiveRef<'a>(&'a CatalogImport); + +impl Serialize for ArchiveRef<'_> { + fn serialize(&self, serializer: S) -> Result { + let mut out = serializer.serialize_struct("CatalogImportArchive", 7)?; + out.serialize_field("version", &FORMAT_VERSION)?; + out.serialize_field("source", &self.0.source)?; + out.serialize_field("photos", &PhotoSeq(&self.0.photos))?; + out.serialize_field("collections", &RowSeq(&self.0.collections))?; + out.serialize_field("members", &RowSeq(&self.0.members))?; + out.serialize_field("collection_content", &RowSeq(&self.0.collection_content))?; + out.serialize_field("warnings", &self.0.warnings)?; + out.end() + } +} + +struct PhotoSeq<'a>(&'a [CatalogPhoto]); +impl Serialize for PhotoSeq<'_> { + fn serialize(&self, serializer: S) -> Result { + let mut out = serializer.serialize_seq(Some(self.0.len()))?; + for photo in self.0 { + out.serialize_element(&PhotoRef(photo))?; + } + out.end() + } +} + +struct PhotoRef<'a>(&'a CatalogPhoto); +impl Serialize for PhotoRef<'_> { + fn serialize(&self, serializer: S) -> Result { + let p = self.0; + let mut out = serializer.serialize_struct("CatalogPhotoArchive", 8)?; + out.serialize_field("source_id", &p.source_id)?; + out.serialize_field("uuid", &p.uuid)?; + out.serialize_field("path", &p.path)?; + out.serialize_field("image", &RowRef(&p.image))?; + out.serialize_field("settings", &p.settings)?; + out.serialize_field("xmp", &p.xmp)?; + out.serialize_field("keywords", &p.keywords)?; + out.serialize_field("history", &RowSeq(&p.history))?; + out.serialize_field("snapshots", &RowSeq(&p.snapshots))?; + out.end() + } +} + +struct RowSeq<'a>(&'a [BTreeMap]); +impl Serialize for RowSeq<'_> { + fn serialize(&self, serializer: S) -> Result { + let mut out = serializer.serialize_seq(Some(self.0.len()))?; + for row in self.0 { + out.serialize_element(&RowRef(row))?; + } + out.end() + } +} + +struct RowRef<'a>(&'a BTreeMap); +impl Serialize for RowRef<'_> { + fn serialize(&self, serializer: S) -> Result { + let mut out = serializer.serialize_map(None)?; + for (key, value) in self.0 { + out.serialize_entry(key, value)?; + } + out.end() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::lightroom_catalog::{CatalogImport, CatalogPhoto}; + use std::sync::atomic::{AtomicU64, Ordering}; + + static NEXT: AtomicU64 = AtomicU64::new(0); + + fn tempdir() -> PathBuf { + let path = std::env::temp_dir().join(format!("lightcraft-lr-archive-{}-{}", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed))); + fs::create_dir_all(&path).unwrap(); + path + } + + fn data(source: String) -> CatalogImport { + CatalogImport { + source, + photos: Vec::new(), + collections: Vec::new(), + members: Vec::new(), + collection_content: Vec::new(), + warnings: Vec::new(), + } + } + + fn rich_data() -> CatalogImport { + let photo = CatalogPhoto { + source_id: 7, + uuid: "uuid-7".into(), + path: "/photos/one.raw".into(), + image: BTreeMap::from([("rating".into(), serde_json::json!(5)), ("opaque_blob".into(), serde_json::json!([1, 2, 3, 4]))]), + settings: "s = { Exposure2012 = 0.75 }".into(), + xmp: "decoded".into(), + keywords: vec!["Trips|Goa".into()], + history: vec![BTreeMap::from([("text".into(), serde_json::json!("step")), ("curve".into(), serde_json::json!([0, 128, 255]))])], + snapshots: vec![BTreeMap::from([("name".into(), serde_json::json!("before"))])], + }; + CatalogImport { + source: "source.lrcat".into(), + photos: vec![photo], + collections: vec![BTreeMap::from([("name".into(), serde_json::json!("Keepers"))])], + members: vec![BTreeMap::from([("image".into(), serde_json::json!(7))])], + collection_content: vec![BTreeMap::from([("rule".into(), serde_json::json!("rating > 3"))])], + warnings: vec!["one warning".into()], + } + } + + fn managed_count(path: &Path) -> usize { + fs::read_dir(path).unwrap().filter_map(Result::ok).filter(|entry| parse_managed_name(&entry.file_name()).is_some()).count() + } + + #[test] + fn deduplicates_identical_deterministic_archives() { + let path = tempdir(); + let input = data("catalog.lrcat".into()); + let first = store(&path, &input).unwrap(); + let second = store(&path, &input).unwrap(); + assert_eq!(first, second); + assert_eq!(managed_count(&path), 1); + fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn dedup_returns_archive_that_retention_keeps() { + let path = tempdir(); + let input = data("catalog.lrcat".into()); + let first = store(&path, &input).unwrap(); + for index in 1..MAX_MANAGED_ARCHIVES { + let filler = path.join(format!("lightroom-import-{index}.lca")); + File::create(filler).unwrap().set_len(19 * 1024 * 1024).unwrap(); + } + let reused = store(&path, &input).unwrap(); + assert_eq!(reused, first); + assert!(reused.is_file()); + assert!(managed_count(&path) <= MAX_MANAGED_ARCHIVES); + fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn caps_managed_archive_count_and_preserves_unrelated_files() { + let path = tempdir(); + let human = path.join("notes.txt"); + let legacy = path.join("lightroom-import-0.json"); + fs::write(&human, b"keep").unwrap(); + fs::write(&legacy, b"legacy").unwrap(); + for index in 0..(MAX_MANAGED_ARCHIVES + 3) { + let input = data(format!("catalog-{index}.lrcat")); + store(&path, &input).unwrap(); + } + assert!(managed_count(&path) <= MAX_MANAGED_ARCHIVES); + assert_eq!(fs::read(&human).unwrap(), b"keep"); + assert_eq!(fs::read(&legacy).unwrap(), b"legacy"); + fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn rejects_oversize_before_creating_archive() { + let path = tempdir(); + let input = data("x".repeat((MAX_ARCHIVE_BYTES as usize) + 1)); + assert!(store(&path, &input).is_err()); + assert_eq!(managed_count(&path), 0); + fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn archive_roundtrips_zlib_payload_and_preserves_numeric_arrays() { + let path = tempdir(); + let archive = store(&path, &rich_data()).unwrap(); + let bytes = fs::read(archive).unwrap(); + assert!(bytes.starts_with(HEADER)); + let json = miniz_oxide::inflate::decompress_to_vec_zlib_with_limit(&bytes[HEADER.len()..], MAX_ARCHIVE_BYTES as usize).unwrap(); + let decoded: Value = serde_json::from_slice(&json).unwrap(); + assert_eq!(decoded["version"], serde_json::json!(FORMAT_VERSION)); + assert_eq!(decoded["source"], serde_json::json!("source.lrcat")); + assert_eq!(decoded["photos"].as_array().unwrap().len(), 1); + assert_eq!(decoded["photos"][0]["source_id"], serde_json::json!(7)); + assert_eq!(decoded["photos"][0]["uuid"], serde_json::json!("uuid-7")); + assert_eq!(decoded["photos"][0]["path"], serde_json::json!("/photos/one.raw")); + assert_eq!(decoded["photos"][0]["settings"], serde_json::json!("s = { Exposure2012 = 0.75 }")); + assert_eq!(decoded["photos"][0]["xmp"], serde_json::json!("decoded")); + assert_eq!(decoded["photos"][0]["keywords"], serde_json::json!(["Trips|Goa"])); + assert_eq!(decoded["photos"][0]["image"]["rating"], serde_json::json!(5)); + assert_eq!(decoded["photos"][0]["image"]["opaque_blob"], serde_json::json!([1, 2, 3, 4])); + assert_eq!(decoded["photos"][0]["history"].as_array().unwrap().len(), 1); + assert_eq!(decoded["photos"][0]["history"][0]["curve"], serde_json::json!([0, 128, 255])); + assert_eq!(decoded["photos"][0]["snapshots"].as_array().unwrap().len(), 1); + assert_eq!(decoded["collections"].as_array().unwrap().len(), 1); + assert_eq!(decoded["collections"][0]["name"], serde_json::json!("Keepers")); + assert_eq!(decoded["members"].as_array().unwrap().len(), 1); + assert_eq!(decoded["members"][0]["image"], serde_json::json!(7)); + assert_eq!(decoded["collection_content"].as_array().unwrap().len(), 1); + assert_eq!(decoded["warnings"], serde_json::json!(["one warning"])); + fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn retention_enforces_aggregate_byte_cap_with_small_synthetic_entries() { + let mut entries: Vec<_> = (0..MAX_MANAGED_ARCHIVES as u64) + .map(|index| Managed { index, path: PathBuf::from(format!("lightroom-import-{index}.lca")), size: 20 * 1024 * 1024 }) + .collect(); + entries.push(Managed { index: 8, path: PathBuf::from("lightroom-import-8.lca"), size: 20 * 1024 * 1024 }); + let retirees = retirement_set(&entries).unwrap(); + assert_eq!(retirees.len(), 3); + assert_eq!(retirees.iter().map(|entry| entry.index).collect::>(), vec![0, 1, 2]); + } + + #[test] + fn retention_failure_preserves_old_archive() { + let path = tempdir(); + let old = path.join("lightroom-import-0.lca"); + let held = path.join(format!(".lightroom-retire-{}-0.tmp", std::process::id())); + fs::write(&old, b"old archive").unwrap(); + fs::write(&held, b"occupied").unwrap(); + let entry = Managed { index: 0, path: old.clone(), size: 11 }; + assert!(quarantine(&[entry]).is_err()); + assert_eq!(fs::read(&old).unwrap(), b"old archive"); + fs::remove_dir_all(path).unwrap(); + } +} diff --git a/crates/engine/src/lightroom_catalog.rs b/crates/engine/src/lightroom_catalog.rs index 5ce8d577..47f22247 100644 --- a/crates/engine/src/lightroom_catalog.rs +++ b/crates/engine/src/lightroom_catalog.rs @@ -3,12 +3,15 @@ //! Raw settings are archived before mutation; mapped edits are approximations, not Adobe renders. use std::collections::{BTreeMap, HashMap, HashSet}; use std::path::{Path, PathBuf}; -use std::sync::Arc; +use std::sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, +}; +use crate::lightroom_sqlite::{Database, LiveTable, Value as SqlValue}; use lightcraft_catalog::{Album, Flag, Op, Photo, PhotoId, Source}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; -use sqlite_core::{Database, Value as SqlValue}; type Row = BTreeMap; const COMMAND: &str = "library.importLightroom"; @@ -296,6 +299,10 @@ fn text<'a>(r: &'a Row, k: &str) -> &'a str { fn number(r: &Row, k: &str) -> i64 { r.get(k).and_then(Value::as_f64).filter(|n| n.is_finite()).unwrap_or(0.0) as i64 } + +pub(crate) fn number_for_job(r: &Row, k: &str) -> i64 { + number(r, k) +} fn key(path: &str) -> String { let p = path.replace('\\', "/"); if cfg!(windows) { p.to_lowercase() } else { p } @@ -309,7 +316,7 @@ fn source_path(p: &Photo) -> Option<&str> { struct Reader { db: Database, - tables: Vec, + tables: Vec, } impl Reader { fn table(&self, name: &str, required: bool) -> Result, String> { @@ -328,17 +335,64 @@ impl Reader { .iter() .cloned() .zip(r.values.into_iter().map(|v| match v { - SqlValue::Null => Value::Null, - SqlValue::Integer(n) => json!(n), - SqlValue::Real(n) => json!(n), - SqlValue::Text(s) => json!(s), - // Opaque data is retained in the source catalog; the XMP packet is decoded below. - SqlValue::Blob(b) => json!(b), + SqlValue::Null => Some(Value::Null), + SqlValue::Integer(n) => Some(json!(n)), + SqlValue::Real(n) => Some(json!(n)), + SqlValue::Text(s) => Some(json!(s)), + // Opaque Lightroom payloads never become JSON byte arrays. + SqlValue::Blob(_) => None, })) + .filter_map(|(column, value)| value.map(|value| (column, value))) .collect() }) .collect()) } + + fn xmp_table(&self) -> Result<(Vec, Vec), String> { + let Some(t) = self.tables.iter().find(|t| t.name == "Adobe_AdditionalMetadata") else { return Ok((Vec::new(), Vec::new())) }; + let columns = t.column_names.as_ref().ok_or_else(|| "cannot read Adobe_AdditionalMetadata columns".to_string())?; + let rows = self.db.read_table(t.rootpage, columns.len()).map_err(|e| format!("Adobe_AdditionalMetadata: {e:?}"))?; + if rows.len() > 1_000_000 { + return Err("Adobe_AdditionalMetadata: more than one million records".into()); + } + let mut warnings = Vec::new(); + let mut out = Vec::with_capacity(rows.len()); + for row in rows { + let image = row + .values + .iter() + .zip(columns) + .find_map(|(value, column)| { + (column == "image").then(|| match value { + SqlValue::Integer(n) => *n, + _ => 0, + }) + }) + .unwrap_or(0); + let mut mapped = Row::new(); + for (column, value) in columns.iter().zip(row.values) { + let value = match value { + SqlValue::Null => Some(Value::Null), + SqlValue::Integer(n) => Some(json!(n)), + SqlValue::Real(n) => Some(json!(n)), + SqlValue::Text(s) => Some(json!(s)), + SqlValue::Blob(bytes) if column == "xmp" => match decode_xmp_bytes(&bytes) { + Ok(packet) => Some(Value::String(packet)), + Err(error) => { + warnings.push(format!("image {image}: {error}")); + None + } + }, + SqlValue::Blob(_) => None, + }; + if let Some(value) = value { + mapped.insert(column.clone(), value); + } + } + out.push(mapped); + } + Ok((out, warnings)) + } } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -364,10 +418,38 @@ pub struct CatalogImport { pub warnings: Vec, } -#[derive(Default, Serialize, Deserialize)] -struct ImportIndex { - photos: BTreeMap, - collections: BTreeMap, +#[derive(Clone, Default, Serialize, Deserialize)] +pub(crate) struct ImportIndex { + pub(crate) photos: BTreeMap, + pub(crate) collections: BTreeMap, +} + +pub(crate) fn load_index(path: Option<&Path>) -> crate::Result { + match path { + Some(path) => { + let file = match std::fs::File::open(path) { + Ok(file) => file, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(ImportIndex::default()), + Err(e) => return Err(error(e.to_string())), + }; + if file.metadata().map_err(|e| error(e.to_string()))?.len() > 16 * 1024 * 1024 { + return Err(error("Lightroom import index exceeds 16 MiB")); + } + let mut bytes = Vec::new(); + use std::io::Read; + file.take(16 * 1024 * 1024 + 1).read_to_end(&mut bytes).map_err(|e| error(e.to_string()))?; + if bytes.len() > 16 * 1024 * 1024 { + return Err(error("Lightroom import index exceeds 16 MiB")); + } + serde_json::from_slice(&bytes).map_err(|e| error(format!("invalid Lightroom import index: {e}"))) + } + None => Ok(ImportIndex::default()), + } +} + +pub(crate) struct AppliedImport { + pub(crate) report: Value, + pub(crate) index: ImportIndex, } fn identity(data: &CatalogImport, photo: &CatalogPhoto) -> String { @@ -406,6 +488,10 @@ fn validate(data: &CatalogImport) -> Result<(), String> { Ok(()) } +pub(crate) fn validate_for_job(data: &CatalogImport) -> Result<(), String> { + validate(data) +} + fn bytes(path: &Path) -> Result, String> { use std::io::Read; let f = std::fs::File::open(path).map_err(|e| format!("{}: {e}", path.display()))?; @@ -440,6 +526,19 @@ fn xmp(v: Option<&Value>) -> Result { } } +fn decode_xmp_bytes(bytes: &[u8]) -> Result { + let size = bytes.get(..4).and_then(|s| <[u8; 4]>::try_from(s).ok()).map(u32::from_be_bytes).ok_or("truncated XMP header")?; + if size > 16 << 20 { + return Err("XMP packet exceeds 16 MiB".into()); + } + let decoded = miniz_oxide::inflate::decompress_to_vec_zlib_with_limit(bytes.get(4..).ok_or("truncated XMP")?, 16 << 20) + .map_err(|_| "invalid compressed XMP")?; + if decoded.len() != size as usize { + return Err("XMP packet length mismatch".into()); + } + String::from_utf8(decoded).map_err(|e| e.to_string()) +} + fn be_word(bytes: &[u8], at: usize) -> Result { let word = bytes.get(at..at + 4).and_then(|s| <[u8; 4]>::try_from(s).ok()).ok_or("truncated WAL")?; Ok(u32::from_be_bytes(word)) @@ -500,6 +599,17 @@ fn committed_wal_len(main: &[u8], wal: &[u8]) -> Result { /// Inspect `.lrcat` plus committed WAL pages without modifying either source file. pub fn read(path: &Path) -> Result { + let cancel = AtomicBool::new(false); + let total = AtomicUsize::new(0); + let done = AtomicUsize::new(0); + read_with_progress(path, &cancel, &total, &done) +} + +pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize, done: &AtomicUsize) -> Result { + // 13 catalog tables plus one validation stage; ImportJob adds one unit per original path. + total.store(14, Ordering::Relaxed); + done.store(0, Ordering::Relaxed); + ensure_read_active(cancel)?; let before = std::fs::metadata(path).map_err(|e| e.to_string())?; let wal_path = PathBuf::from(format!("{}-wal", path.display())); let wal_before = std::fs::metadata(&wal_path).ok().map(|m| (m.len(), m.modified().ok())); @@ -510,22 +620,33 @@ pub fn read(path: &Path) -> Result { Err(e) => return Err(e.to_string()), }; let wal_len = committed_wal_len(&main, &wal)?; + ensure_read_active(cancel)?; let db = Database::open_with_wal(main, &wal[..wal_len]).map_err(|e| format!("invalid SQLite catalog: {e:?}"))?; let after = std::fs::metadata(path).map_err(|e| e.to_string())?; let wal_after = std::fs::metadata(&wal_path).ok().map(|m| (m.len(), m.modified().ok())); if before.len() != after.len() || before.modified().ok() != after.modified().ok() || wal_before != wal_after { return Err("catalog changed during reading; close Lightroom and retry".into()); } - let reader = Reader { tables: db.live_tables(), db }; - let roots: HashMap<_, _> = reader.table("AgLibraryRootFolder", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect(); - let folders: HashMap<_, _> = reader.table("AgLibraryFolder", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect(); - let files: HashMap<_, _> = reader.table("AgLibraryFile", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect(); + let reader = Reader { tables: db.live_tables()?, db }; + let mut table = |name: &str, required: bool| -> Result, String> { + ensure_read_active(cancel)?; + let rows = reader.table(name, required)?; + done.fetch_add(1, Ordering::Relaxed); + Ok(rows) + }; + let roots: HashMap<_, _> = table("AgLibraryRootFolder", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect(); + let folders: HashMap<_, _> = table("AgLibraryFolder", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect(); + let files: HashMap<_, _> = table("AgLibraryFile", true)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect(); let develops: HashMap<_, _> = - reader.table("Adobe_imageDevelopSettings", false)?.into_iter().map(|r| (number(&r, "image"), text(&r, "text").to_string())).collect(); - let metadata: HashMap<_, _> = reader.table("Adobe_AdditionalMetadata", false)?.into_iter().map(|r| (number(&r, "image"), r)).collect(); - let keyword_rows: HashMap<_, _> = reader.table("AgLibraryKeyword", false)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect(); + table("Adobe_imageDevelopSettings", false)?.into_iter().map(|r| (number(&r, "image"), text(&r, "text").to_string())).collect(); + ensure_read_active(cancel)?; + let (metadata_rows, mut metadata_warnings) = reader.xmp_table()?; + done.fetch_add(1, Ordering::Relaxed); + let metadata: HashMap<_, _> = metadata_rows.into_iter().map(|r| (number(&r, "image"), r)).collect(); + let keyword_rows: HashMap<_, _> = table("AgLibraryKeyword", false)?.into_iter().map(|r| (number(&r, "id_local"), r)).collect(); let mut keywords: HashMap> = HashMap::new(); - for r in reader.table("AgLibraryKeywordImage", false)? { + for r in table("AgLibraryKeywordImage", false)? { + ensure_read_active(cancel)?; let mut parts = Vec::new(); let mut at = number(&r, "tag"); let mut seen = HashSet::new(); @@ -552,17 +673,18 @@ pub fn read(path: &Path) -> Result { } out }; - let mut history = group(reader.table("Adobe_libraryImageDevelopHistoryStep", false)?); - let mut snapshots = group(reader.table("Adobe_libraryImageDevelopSnapshot", false)?); + let mut history = group(table("Adobe_libraryImageDevelopHistoryStep", false)?); + let mut snapshots = group(table("Adobe_libraryImageDevelopSnapshot", false)?); let mut out = CatalogImport { source: path.to_string_lossy().into(), photos: Vec::new(), - collections: reader.table("AgLibraryCollection", false)?, - members: reader.table("AgLibraryCollectionImage", false)?, - collection_content: reader.table("AgLibraryCollectionContent", false)?, - warnings: Vec::new(), + collections: table("AgLibraryCollection", false)?, + members: table("AgLibraryCollectionImage", false)?, + collection_content: table("AgLibraryCollectionContent", false)?, + warnings: std::mem::take(&mut metadata_warnings), }; - for image in reader.table("Adobe_images", true)? { + for image in table("Adobe_images", true)? { + ensure_read_active(cancel)?; let id = number(&image, "id_local"); let file = files.get(&number(&image, "rootFile")).ok_or_else(|| format!("image {id}: missing file record"))?; let folder = folders.get(&number(file, "folder")).ok_or_else(|| format!("image {id}: missing folder record"))?; @@ -593,9 +715,14 @@ pub fn read(path: &Path) -> Result { snapshots: snapshots.remove(&id).unwrap_or_default(), }); } + ensure_read_active(cancel)?; Ok(out) } +fn ensure_read_active(cancel: &AtomicBool) -> Result<(), String> { + if cancel.load(Ordering::Relaxed) { Err("Lightroom import cancelled".into()) } else { Ok(()) } +} + fn mapped_settings(text: &str, raw: bool, aspect: f64) -> Result<(Value, Vec), String> { let mut root = crate::preset_import::parse_lua(text)?; let deferred = strip_lua_sentinels(&mut root); @@ -651,39 +778,60 @@ fn strip_xmp_sentinels(value: &mut Value) { /// Import into the current library. Existing edited records are preserved unless explicitly /// requested; missing originals remain catalogued for later relinking. The source catalog is read-only. pub fn import(s: &mut crate::Session, path: &Path, update_existing: bool) -> crate::Result { - let data = read(path).map_err(error)?; - apply(s, data, update_existing) + let mut job = crate::lightroom_job::LightroomJob::new(s, path.to_path_buf(), update_existing)?; + let cancel = AtomicBool::new(false); + let prepared = job.prepare(&cancel)?; + let completion = crate::lightroom_job::commit_prepared(s, prepared)?; + if s.library.as_ref().is_some_and(|library| library.on_disk) { + s.persist()?; + } + completion.finalization.finish()?; + Ok(completion.report) } +#[cfg(test)] fn apply(s: &mut crate::Session, data: CatalogImport, update_existing: bool) -> crate::Result { - validate(&data).map_err(error)?; - let mut warnings = data.warnings.clone(); - let archive = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.join("Interop")); - let index_path = archive.as_ref().map(|p| p.join("lightroom-index.json")); - let mut index: ImportIndex = match index_path.as_ref().map(std::fs::read) { - Some(Ok(b)) => serde_json::from_slice(&b).map_err(|e| error(format!("invalid Lightroom import index: {e}")))?, - Some(Err(e)) if e.kind() != std::io::ErrorKind::NotFound => return Err(error(e.to_string())), - _ => ImportIndex::default(), - }; - // Preserve source data before catalog changes. A unique archive is never overwritten. - let archive_path = if let Some(dir) = archive { - std::fs::create_dir_all(&dir).map_err(|e| error(e.to_string()))?; - let bytes = serde_json::to_vec(&data).map_err(|e| error(e.to_string()))?; - let mut names = (0..10000).map(|n| dir.join(format!("lightroom-import-{n}.json"))); - Some(lightcraft_catalog::safe_file::write_new_unique(&mut names, &bytes).map_err(|e| error(e.to_string()))?) - } else { - None - }; + let archive_dir = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.join("Interop")); + let index_path = archive_dir.as_ref().map(|p| p.join("lightroom-index.json")); + let mut index = load_index(index_path.as_deref())?; + let archive_path = archive_dir.as_ref().map(|dir| crate::lightroom_archive::store(dir, &data).map_err(error)).transpose()?; + let undo0 = s.undo.len(); let existing: HashMap<_, _> = s.catalog.photos().filter(|p| p.copy_of.is_none()).filter_map(|p| source_path(p).map(|path| (key(path), p.id))).collect(); - let undo0 = s.undo.len(); let paths: Vec<_> = data .photos .iter() .filter(|p| number(&p.image, "masterImage") == 0 && !existing.contains_key(&key(&p.path)) && Path::new(&p.path).is_file()) .map(|p| p.path.clone()) .collect(); + let missing_sources: HashSet = + data.photos.iter().filter(|p| number(&p.image, "masterImage") == 0).filter(|p| !Path::new(&p.path).is_file()).map(|p| key(&p.path)).collect(); let files = crate::import::import(s, &paths, crate::import::ImportMode::Add)?; + let applied = + apply_prepared(s, data, ApplyPrepared { update_existing, files, index: &mut index, archive_path, missing_sources: &missing_sources, undo0 })?; + if let Some(path) = index_path { + s.persist()?; + let bytes = serde_json::to_vec(&applied.index).map_err(|e| error(e.to_string()))?; + lightcraft_catalog::safe_file::write_atomic(&path, &bytes).map_err(|e| error(e.to_string()))?; + } + Ok(applied.report) +} + +/// Apply prepared normal-file results and Lightroom metadata in one owner-thread operation. +/// This function does no filesystem work; callers finish its returned index separately. +pub(crate) struct ApplyPrepared<'a> { + pub(crate) update_existing: bool, + pub(crate) files: crate::import::ImportReport, + pub(crate) index: &'a mut ImportIndex, + pub(crate) archive_path: Option, + pub(crate) missing_sources: &'a HashSet, + pub(crate) undo0: usize, +} + +pub(crate) fn apply_prepared(s: &mut crate::Session, data: CatalogImport, context: ApplyPrepared<'_>) -> crate::Result { + let ApplyPrepared { update_existing, files, index, archive_path, missing_sources, undo0 } = context; + validate(&data).map_err(error)?; + let mut warnings = data.warnings.clone(); let duplicates: HashMap<_, _> = files.duplicates.iter().filter_map(|d| d.existing.map(|id| (key(&d.path), PhotoId(id)))).collect(); let mut by_path: HashMap<_, _> = s.catalog.photos().filter(|p| p.copy_of.is_none()).filter_map(|p| source_path(p).map(|path| (key(path), p.id))).collect(); @@ -729,7 +877,7 @@ fn apply(s: &mut crate::Session, data: CatalogImport, update_existing: bool) -> } else { let id = s.catalog.alloc_photo_id(); let name = Path::new(&src.path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| src.path.clone()); - if !Path::new(&src.path).is_file() { + if missing_sources.contains(&key(&src.path)) { missing.push(src.path.clone()); } let mut p = Photo::new( @@ -866,12 +1014,8 @@ fn apply(s: &mut crate::Session, data: CatalogImport, update_existing: bool) -> s.commit("Import Lightroom Catalog", Op::Batch { ops })?; } s.merge_undo(s.undo.len().saturating_sub(undo0), "Import Lightroom Catalog"); - if let Some(path) = index_path { - s.persist()?; - let bytes = serde_json::to_vec(&index).map_err(|e| error(e.to_string()))?; - lightcraft_catalog::safe_file::write_atomic(&path, &bytes).map_err(|e| error(e.to_string()))?; - } - Ok( - json!({"source":data.source,"photos":ids.len(),"imported":files.imported.len(),"collections":albums.len(),"preservedExistingEdits":preserved,"missing":missing,"failed":files.failed,"warnings":warnings,"unmapped":unmapped,"archive":archive_path,"mapping":ids}), - ) + Ok(AppliedImport { + report: json!({"source":data.source,"photos":ids.len(),"imported":files.imported.len(),"collections":albums.len(),"preservedExistingEdits":preserved,"missing":missing,"failed":files.failed,"warnings":warnings,"unmapped":unmapped,"archive":archive_path,"mapping":ids}), + index: index.clone(), + }) } diff --git a/crates/engine/src/lightroom_job.rs b/crates/engine/src/lightroom_job.rs new file mode 100644 index 00000000..ab383aa1 --- /dev/null +++ b/crates/engine/src/lightroom_job.rs @@ -0,0 +1,386 @@ +//! Detached Lightroom catalog preparation and owner-thread commit. +//! +//! `LightroomJob::new` takes only an in-memory session snapshot. `prepare` owns all catalog +//! reading, probing and archive staging, while `commit_prepared` only applies catalog operations +//! to the session that created the job. Archive index bytes are returned for a worker to finish +//! after the owner has accepted the catalog operation. + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, +}; + +use serde_json::Value; + +use crate::import::{ImportJob, ImportMode, ImportOptions, Prepared}; +use crate::lightroom_catalog::{self, CatalogImport, ImportIndex}; + +const COMMAND: &str = "library.importLightroom"; +const MAX_INDEX_BYTES: usize = 16 * 1024 * 1024; + +fn error(message: impl Into) -> crate::EngineError { + crate::EngineError::BadParams { cmd: COMMAND.into(), msg: message.into() } +} + +/// Memory identity of the library which owns a prepared completion. +/// +/// Each successful open/close refreshes session identity, including reopening the same path. +/// The owner check deliberately does not compare catalog revisions: ordinary edits while preparation +/// runs must survive and are merged by the owner commit. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct LightroomLibraryToken { + library_dir: Option, + identity: Arc<()>, +} + +impl LightroomLibraryToken { + fn capture(s: &crate::Session) -> Self { + Self { + library_dir: s.library.as_ref().filter(|library| library.on_disk).map(|library| library.dir.clone()), + identity: Arc::clone(&s.library_identity), + } + } + + /// Whether `s` still refers to the library which started preparation. + pub fn matches(&self, s: &crate::Session) -> bool { + Arc::ptr_eq(&self.identity, &s.library_identity) + && self.library_dir == s.library.as_ref().filter(|library| library.on_disk).map(|library| library.dir.clone()) + } + + pub fn library_dir(&self) -> Option<&Path> { + self.library_dir.as_deref() + } +} + +/// Filesystem-free result of catalog preparation. +pub struct PreparedLightroom { + pub(crate) data: CatalogImport, + pub(crate) normal: Prepared, + pub(crate) index: ImportIndex, + pub(crate) archive_path: Option, + pub(crate) token: LightroomLibraryToken, + pub(crate) missing_sources: HashSet, + pub(crate) update_existing: bool, + pub(crate) now: String, + cancelled: bool, +} + +impl PreparedLightroom { + pub fn source(&self) -> &str { + &self.data.source + } + + pub fn photo_count(&self) -> usize { + self.data.photos.len() + } + + pub fn token(&self) -> &LightroomLibraryToken { + &self.token + } + + pub fn is_cancelled(&self) -> bool { + self.cancelled + } + + /// Roll back any normal-import filesystem placement held by this prepared value. + pub fn rollback(&self) { + self.normal.rollback(); + } +} + +/// Index write returned by owner commit. `finish` is intended for a worker after the owner has +/// returned, and performs the only archive-index filesystem operation in this pipeline. +pub struct LightroomArchiveFinalization { + path: Option, + bytes: Option>, +} + +impl LightroomArchiveFinalization { + pub fn finish(self) -> crate::Result<()> { + let (Some(path), Some(bytes)) = (self.path, self.bytes) else { return Ok(()) }; + if bytes.len() > MAX_INDEX_BYTES { + return Err(error("Lightroom import index exceeds 16 MiB")); + } + lightcraft_catalog::safe_file::write_atomic(&path, &bytes).map_err(|e| error(e.to_string())) + } + + pub fn path(&self) -> Option<&Path> { + self.path.as_deref() + } +} + +/// Owner-thread catalog result. The report is ready for the UI immediately; index finalization +/// can remain on the worker while its task stays alive. +pub struct LightroomCompletion { + pub report: Value, + pub finalization: LightroomArchiveFinalization, +} + +/// Alias used by hosts that call the result a commit. +pub type LightroomCommit = LightroomCompletion; + +/// A Lightroom import detached from the owner thread. +pub struct LightroomJob { + path: PathBuf, + update_existing: bool, + import: ImportJob, + token: LightroomLibraryToken, + archive_dir: Option, +} + +impl LightroomJob { + /// Snapshot session state only. This constructor performs no filesystem work. + pub fn new(s: &mut crate::Session, path: PathBuf, update_existing: bool) -> crate::Result { + let opts = ImportOptions { mode: ImportMode::Add, ..ImportOptions::default() }; + let import = ImportJob::new(s, opts)?; + let archive_dir = s.library.as_ref().filter(|library| library.on_disk).map(|library| library.dir.join("Interop")); + Ok(Self { path, update_existing, import, token: LightroomLibraryToken::capture(s), archive_dir }) + } + + pub fn total(&self) -> usize { + self.import.total.load(Ordering::Relaxed) + } + + pub fn done(&self) -> usize { + self.import.done.load(Ordering::Relaxed) + } + + pub fn progress(&self) -> (usize, usize) { + (self.done(), self.total()) + } + + pub fn token(&self) -> &LightroomLibraryToken { + &self.token + } + + pub fn total_atomic(&self) -> std::sync::Arc { + std::sync::Arc::clone(&self.import.total) + } + + pub fn done_atomic(&self) -> std::sync::Arc { + std::sync::Arc::clone(&self.import.done) + } + + /// Finish an owner commit's deferred archive-index write on a worker. + pub fn finish_archive(finalization: LightroomArchiveFinalization) -> crate::Result<()> { + finalization.finish() + } + + /// Read/probe/stage all source data off the owner thread. + pub fn prepare(&mut self, cancel: &AtomicBool) -> crate::Result { + if cancel.load(Ordering::Relaxed) { + return Err(error("Lightroom import cancelled")); + } + self.import.total.store(1, Ordering::Relaxed); + self.import.done.store(0, Ordering::Relaxed); + let data = lightroom_catalog::read_with_progress(&self.path, cancel, &self.import.total, &self.import.done).map_err(error)?; + lightroom_catalog::validate_for_job(&data).map_err(error)?; + if cancel.load(Ordering::Relaxed) { + return Err(error("Lightroom import cancelled")); + } + self.import.done.fetch_add(1, Ordering::Relaxed); + let paths: Vec = data + .photos + .iter() + .filter(|photo| lightroom_catalog::number_for_job(&photo.image, "masterImage") == 0) + .map(|photo| photo.path.clone()) + .collect(); + let missing_sources = data + .photos + .iter() + .filter(|photo| lightroom_catalog::number_for_job(&photo.image, "masterImage") == 0) + .filter(|photo| !Path::new(&photo.path).is_file()) + .map(|photo| normalize(&photo.path)) + .collect(); + let normal = self.import.prepare(&paths, cancel); + if cancel.load(Ordering::Relaxed) { + normal.rollback(); + return Err(error("Lightroom import cancelled")); + } + let archive_path = self.archive_dir.as_ref().map(|dir| crate::lightroom_archive::store(dir, &data).map_err(error)).transpose()?; + let index_path = self.archive_dir.as_ref().map(|dir| dir.join("lightroom-index.json")); + let index = lightroom_catalog::load_index(index_path.as_deref())?; + Ok(PreparedLightroom { + data, + normal, + index, + archive_path, + token: self.token.clone(), + missing_sources, + update_existing: self.update_existing, + now: self.import.now().to_string(), + cancelled: cancel.load(Ordering::Relaxed), + }) + } +} + +/// Apply prepared data on the owner thread. No source, archive or index filesystem work occurs. +pub fn commit_prepared(s: &mut crate::Session, mut prepared: PreparedLightroom) -> crate::Result { + if prepared.cancelled { + prepared.normal.rollback(); + return Err(error("Lightroom import cancelled")); + } + if !prepared.token.matches(s) { + prepared.normal.rollback(); + return Err(error("Lightroom import belongs to a different open library")); + } + // A command-scoped Lightroom import must never rewrite source XMP sidecars. Direct depth-0 + // callers do not set this transient flag, so they cannot affect a later command. + if s.depth > 0 { + s.skip_auto_write = true; + } + + let undo0 = s.undo.len(); + prepared.normal.revalidate_add(s); + let options = ImportOptions { mode: ImportMode::Add, ..ImportOptions::default() }; + let now = prepared.now.clone(); + let files = crate::import::commit_prepared(s, &options, &now, prepared.normal)?; + let applied = lightroom_catalog::apply_prepared( + s, + prepared.data, + lightroom_catalog::ApplyPrepared { + update_existing: prepared.update_existing, + files, + index: &mut prepared.index, + archive_path: prepared.archive_path, + missing_sources: &prepared.missing_sources, + undo0, + }, + )?; + let path = prepared.token.library_dir().map(|dir| dir.join("Interop/lightroom-index.json")); + let bytes = if path.is_some() { Some(serde_json::to_vec(&applied.index).map_err(|e| error(e.to_string()))?) } else { None }; + Ok(LightroomCompletion { report: applied.report, finalization: LightroomArchiveFinalization { path, bytes } }) +} + +fn normalize(path: &str) -> String { + let path = path.replace('\\', "/"); + if cfg!(windows) { path.to_lowercase() } else { path } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn empty_data() -> CatalogImport { + CatalogImport { + source: "fixture.lrcat".into(), + photos: Vec::new(), + collections: Vec::new(), + members: Vec::new(), + collection_content: Vec::new(), + warnings: Vec::new(), + } + } + + fn cancelled_prepared(s: &crate::Session) -> PreparedLightroom { + PreparedLightroom { + data: empty_data(), + normal: Prepared::default(), + index: ImportIndex::default(), + archive_path: None, + token: LightroomLibraryToken::capture(s), + missing_sources: HashSet::new(), + update_existing: false, + now: "2026-01-01T00:00:00".into(), + cancelled: true, + } + } + + fn prepared_for(s: &crate::Session, path: &str, rating: i64) -> PreparedLightroom { + let mut image = std::collections::BTreeMap::new(); + image.insert("id_local".into(), serde_json::json!(1)); + image.insert("rating".into(), serde_json::json!(rating)); + image.insert("pick".into(), serde_json::json!(0)); + image.insert("fileFormat".into(), serde_json::json!("JPG")); + image.insert("fileWidth".into(), serde_json::json!(1)); + image.insert("fileHeight".into(), serde_json::json!(1)); + PreparedLightroom { + data: CatalogImport { + source: "fixture.lrcat".into(), + photos: vec![lightroom_catalog::CatalogPhoto { + source_id: 1, + uuid: "fixture-photo".into(), + path: path.into(), + image, + settings: String::new(), + xmp: String::new(), + keywords: Vec::new(), + history: Vec::new(), + snapshots: Vec::new(), + }], + collections: Vec::new(), + members: Vec::new(), + collection_content: Vec::new(), + warnings: Vec::new(), + }, + normal: Prepared::default(), + index: ImportIndex::default(), + archive_path: None, + token: LightroomLibraryToken::capture(s), + missing_sources: [normalize(path)].into_iter().collect(), + update_existing: false, + now: "2026-01-01T00:00:00".into(), + cancelled: false, + } + } + + #[test] + fn cancellation_before_commit_leaves_owner_catalog_unchanged() { + let mut session = crate::Session::new(); + let before = session.catalog.to_snapshot(); + let prepared = cancelled_prepared(&session); + assert!(commit_prepared(&mut session, prepared).is_err()); + assert_eq!(session.catalog.to_snapshot(), before); + assert!(session.undo.is_empty()); + } + + #[test] + fn library_identity_rejects_late_completion_after_reopen() { + let mut session = crate::Session::new(); + let token = LightroomLibraryToken::capture(&session); + assert!(token.matches(&session)); + session.library_identity = Arc::new(()); + assert!(!token.matches(&session)); + } + + #[test] + fn prepared_reimport_preserves_current_edit_and_uses_one_undo_per_import() { + let mut session = crate::Session::new(); + let path = "/missing/lightroom-job-edit.jpg"; + let prepared = prepared_for(&session, path, 4); + let first = commit_prepared(&mut session, prepared).unwrap(); + assert_eq!(first.report["mapping"]["1"].as_u64(), Some(1)); + assert_eq!(session.undo.len(), 1); + let id = lightcraft_catalog::PhotoId(1); + session.set_develop(id, lightcraft_develop::DevelopSettings::default(), "Personal").unwrap(); + let before = session.catalog.photo(id).map(|photo| photo.develop.clone()); + let prepared = prepared_for(&session, path, 5); + commit_prepared(&mut session, prepared).unwrap(); + assert_eq!(session.catalog.photo(id).map(|photo| photo.develop.clone()), before); + assert_eq!(session.catalog.photo(id).map(|photo| photo.rating), Some(5)); + assert_eq!(session.undo.len(), 3); + } + + #[test] + fn execute_fn_import_does_not_rewrite_source_sidecar() { + let dir = std::env::temp_dir().join(format!("lightcraft-lr-job-sidecar-{}", std::process::id())); + let path = dir.join("source.jpg"); + let sidecar = dir.join("source.xmp"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(&path, b"fixture").unwrap(); + std::fs::write(&sidecar, b"original sidecar").unwrap(); + let original = std::fs::read(&sidecar).unwrap(); + let mut session = crate::Session::new(); + session.xmp.auto_write = true; + let result = session.execute_fn("library.importLightroom", |s| { + let prepared = prepared_for(s, &path.to_string_lossy(), 5); + let completion = commit_prepared(s, prepared)?; + Ok(completion.report) + }); + assert!(result.is_ok()); + assert_eq!(std::fs::read(&sidecar).unwrap(), original); + std::fs::remove_dir_all(dir).unwrap(); + } +} diff --git a/crates/engine/src/lightroom_sqlite.rs b/crates/engine/src/lightroom_sqlite.rs new file mode 100644 index 00000000..be7f4cb9 --- /dev/null +++ b/crates/engine/src/lightroom_sqlite.rs @@ -0,0 +1,1071 @@ +//! Small, read-only SQLite reader used by Lightroom catalog import. +//! +//! This is a clean-room implementation based on SQLite's public file-format +//! description: (sections 1.6, 1.7, +//! 2.1, 2.2, 2.3, 2.4 and 2.5). It is deliberately a page reader, rather +//! than a SQL engine. It never writes its input and has no SQLite runtime +//! dependency. + +use std::collections::{HashMap, HashSet}; + +const MAX_PAGE_ID: u32 = 16_777_216; +const MAX_COLUMNS: usize = 16_384; +const MAX_ROWS: usize = 1_000_000; +const MAX_CELLS: usize = 4_000_000; +const MAX_PAYLOAD: usize = 64 * 1024 * 1024; +const MAX_TABLE_PAYLOAD: usize = 256 * 1024 * 1024; +const MAX_OVERFLOW_PAGES: usize = 1_048_576; +const MAX_BTREE_DEPTH: usize = 64; + +/// Values supported by SQLite table records. +#[derive(Clone, Debug, PartialEq)] +pub enum Value { + Null, + Integer(i64), + Real(f64), + Text(String), + Blob(Vec), +} + +/// A row returned by [`Database::read_table`]. +#[derive(Clone, Debug, PartialEq)] +pub struct Record { + pub values: Vec, +} + +/// A table discovered in `sqlite_schema`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LiveTable { + pub name: String, + pub rootpage: u32, + pub column_names: Option>, +} + +/// Read-only SQLite database image, with a committed WAL overlay if supplied. +pub struct Database { + main: Vec, + overlay: HashMap>, + page_size: usize, + reserved: usize, + encoding: TextEncoding, + page_count: u32, + tables: Vec, + rowid_aliases: HashMap, +} + +#[derive(Clone, Copy)] +enum TextEncoding { + Utf8, + Utf16Le, + Utf16Be, +} + +struct WalkState { + seen: HashSet, + out: Vec, + payload_total: usize, +} + +struct WalOverlay { + page_size: usize, + frame_count: usize, + pages: HashMap>, + page_count: Option, +} + +struct SchemaInfo { + tables: Vec, + aliases: HashMap, +} + +impl Database { + /// Open a main database image and an already checksum-validated WAL prefix. + /// + /// The WAL argument must include its 32-byte header and only complete + /// frames through a commit. Frames are overlaid by page number, with the + /// final commit's database-size field defining the visible page count. + pub fn open_with_wal(main: Vec, committed_wal: &[u8]) -> Result { + let main_header = main.get(..100).ok_or("truncated SQLite header")?; + if main_header.get(..16) != Some(b"SQLite format 3\0") { + return Err("invalid SQLite header signature".into()); + } + let page_size = sqlite_page_size(be_u16(main_header, 16)?)?; + validate_header_fields(main_header, page_size)?; + if main.len() < page_size || main.len() % page_size != 0 { + return Err("SQLite image is not a whole number of pages".into()); + } + let main_pages = u32::try_from(main.len() / page_size).map_err(|_| "SQLite image is too large")?; + if main_pages == 0 || main_pages > MAX_PAGE_ID { + return Err("invalid SQLite image page count".into()); + } + + let mut overlay = HashMap::new(); + let mut wal_page_count = None; + if !committed_wal.is_empty() { + let parsed = parse_wal(committed_wal, page_size)?; + if parsed.page_size != page_size { + return Err("WAL page size mismatch".into()); + } + if parsed.frame_count > MAX_CELLS { + return Err("WAL contains too many frames".into()); + } + overlay = parsed.pages; + wal_page_count = parsed.page_count; + } + let effective_header = overlay.get(&1).map(Vec::as_slice).unwrap_or(main_header); + if effective_header.get(..16) != Some(b"SQLite format 3\0") { + return Err("invalid SQLite header page in WAL".into()); + } + let effective_size = sqlite_page_size(be_u16(effective_header, 16)?)?; + if effective_size != page_size { + return Err("WAL page 1 changes SQLite page size".into()); + } + let reserved = validate_header_fields(effective_header, page_size)?; + let header_pages = be_u32(effective_header, 28)?; + let header_valid = be_u32(effective_header, 24)? != 0 && be_u32(effective_header, 24)? == be_u32(effective_header, 92)?; + if header_valid && header_pages > main_pages && wal_page_count.is_none() { + return Err("SQLite header page count exceeds image".into()); + } + let mut page_count = if header_valid && header_pages != 0 { header_pages } else { main_pages }; + if let Some(count) = wal_page_count { + page_count = count; + } + if page_count == 0 || page_count > MAX_PAGE_ID { + return Err("invalid visible SQLite page count".into()); + } + let encoding = match be_u32(effective_header, 56)? { + 0 | 1 => TextEncoding::Utf8, + 2 => TextEncoding::Utf16Le, + 3 => TextEncoding::Utf16Be, + _ => return Err("unsupported SQLite text encoding".into()), + }; + + let mut db = Self { main, overlay, page_size, reserved, encoding, page_count, tables: Vec::new(), rowid_aliases: HashMap::new() }; + let schema = db.parse_schema()?; + db.tables = schema.tables; + db.rowid_aliases = schema.aliases; + Ok(db) + } + + /// Return tables from `sqlite_schema` in schema order. + pub fn live_tables(&self) -> Result, String> { + Ok(self.tables.clone()) + } + + /// Read a table B-tree, decoding up to `ncols` values per record. + pub fn read_table(&self, rootpage: u32, ncols: usize) -> Result, String> { + if !(1..=MAX_PAGE_ID).contains(&rootpage) || rootpage > self.page_count { + return Err("table root page is outside database".into()); + } + if ncols > MAX_COLUMNS { + return Err("table has too many columns".into()); + } + let alias = self.rowid_aliases.get(&rootpage).copied(); + let mut state = WalkState { seen: HashSet::new(), out: Vec::new(), payload_total: 0 }; + self.visit_table(rootpage, ncols, alias, 0, &mut state)?; + Ok(state.out) + } + + fn page(&self, page_id: u32) -> Result<&[u8], String> { + if page_id == 0 || page_id > self.page_count { + return Err(format!("page {page_id} is outside database")); + } + if let Some(page) = self.overlay.get(&page_id) { + return Ok(page); + } + let id = usize::try_from(page_id).map_err(|_| "page number does not fit host")?; + let start = id.checked_sub(1).and_then(|n| n.checked_mul(self.page_size)).ok_or("page offset overflow")?; + let end = start.checked_add(self.page_size).ok_or("page offset overflow")?; + self.main.get(start..end).ok_or_else(|| format!("page {page_id} is truncated")) + } + + fn visit_table(&self, page_id: u32, ncols: usize, alias: Option, depth: usize, state: &mut WalkState) -> Result<(), String> { + if depth > MAX_BTREE_DEPTH { + return Err("SQLite B-tree is too deep".into()); + } + if !state.seen.insert(page_id) { + return Err("SQLite B-tree contains a cycle or shared page".into()); + } + let page = self.page(page_id)?; + let usable = self.page_size.checked_sub(self.reserved).ok_or("invalid usable page size")?; + let base = if page_id == 1 { 100 } else { 0 }; + let content = page.get(..usable).ok_or("truncated usable SQLite page")?; + let kind = *content.get(base).ok_or("truncated B-tree header")?; + let cells = usize::from(be_u16_at(content, base + 3)?); + if cells > MAX_CELLS || state.out.len().saturating_add(cells) > MAX_ROWS.saturating_add(MAX_CELLS) { + return Err("SQLite B-tree contains too many cells".into()); + } + match kind { + 0x0d => { + let ptr_start = base.checked_add(8).ok_or("B-tree header overflow")?; + let ptr_bytes = cells.checked_mul(2).ok_or("B-tree pointer-array overflow")?; + let ptr_end = ptr_start.checked_add(ptr_bytes).ok_or("B-tree pointer-array overflow")?; + if ptr_end > usable { + return Err("truncated B-tree pointer array".into()); + } + for i in 0..cells { + let p = ptr_start.checked_add(i.checked_mul(2).ok_or("cell offset overflow")?).ok_or("cell offset overflow")?; + let cell = usize::from(be_u16_at(content, p)?); + if cell < ptr_end || cell >= usable { + return Err("invalid B-tree cell pointer".into()); + } + let mut at = cell; + let payload_len = usize_from_u64(read_varint(content, &mut at)?, "payload length")?; + if payload_len > MAX_PAYLOAD { + return Err("SQLite record payload is too large".into()); + } + let rowid = i64::from_be_bytes(read_varint(content, &mut at)?.to_be_bytes()); + state.payload_total = state.payload_total.checked_add(payload_len).ok_or("SQLite table payload total overflow")?; + if state.payload_total > MAX_TABLE_PAYLOAD { + return Err("SQLite table payload exceeds bounded import size".into()); + } + let payload = self.payload(page, at, payload_len)?; + let mut values = decode_record(&payload, ncols, self.encoding)?; + if let Some(index) = alias { + if index >= values.len() { + return Err("INTEGER PRIMARY KEY column is missing from record".into()); + } + if matches!(values.get(index), Some(Value::Null)) { + if let Some(value) = values.get_mut(index) { + *value = Value::Integer(rowid); + } + } + } + if state.out.len() >= MAX_ROWS { + return Err("SQLite table contains more than one million records".into()); + } + state.out.push(Record { values }); + } + Ok(()) + } + 0x05 => { + let ptr_start = base.checked_add(12).ok_or("B-tree header overflow")?; + let ptr_bytes = cells.checked_mul(2).ok_or("B-tree pointer-array overflow")?; + let ptr_end = ptr_start.checked_add(ptr_bytes).ok_or("B-tree pointer-array overflow")?; + if ptr_end > usable { + return Err("truncated interior B-tree pointer array".into()); + } + for i in 0..cells { + let p = ptr_start.checked_add(i.checked_mul(2).ok_or("cell offset overflow")?).ok_or("cell offset overflow")?; + let cell = usize::from(be_u16_at(content, p)?); + if cell < ptr_end || cell >= usable { + return Err("invalid interior B-tree cell pointer".into()); + } + let child = be_u32_at(content, cell)?; + if child == 0 { + return Err("interior B-tree cell has null child".into()); + } + let mut key_at = cell.checked_add(4).ok_or("interior cell overflow")?; + let _ = read_varint(content, &mut key_at)?; + self.visit_table(child, ncols, alias, depth + 1, state)?; + } + let right = be_u32_at(content, base + 8)?; + if right == 0 { + return Err("interior B-tree has null right child".into()); + } + self.visit_table(right, ncols, alias, depth + 1, state) + } + 0x02 | 0x0a => Err("SQLite index B-trees are unsupported".into()), + other => Err(format!("unsupported SQLite B-tree page type 0x{other:02x}")), + } + } + + fn payload(&self, page: &[u8], cell_at: usize, payload_len: usize) -> Result, String> { + let usable = self.page_size.checked_sub(self.reserved).ok_or("invalid usable page size")?; + if usable < 8 { + return Err("SQLite usable page size is too small".into()); + } + let max_local = usable.saturating_sub(35); + let min_local = ((usable - 12).saturating_mul(32) / 255).saturating_sub(23); + let local = if payload_len <= max_local { + payload_len + } else { + let span = usable - 4; + let candidate = min_local.saturating_add((payload_len - min_local) % span); + if candidate > max_local { min_local } else { candidate } + }; + let local_end = cell_at.checked_add(local).ok_or("SQLite cell payload overflow")?; + if local_end > usable { + return Err("truncated SQLite cell payload".into()); + } + let mut out = Vec::with_capacity(payload_len); + out.extend_from_slice(page.get(cell_at..local_end).ok_or("truncated SQLite cell payload")?); + let mut remaining = payload_len.saturating_sub(local); + if remaining == 0 { + return Ok(out); + } + let overflow_at = local_end; + match overflow_at.checked_add(4) { + Some(end) if end <= usable => {} + _ => return Err("SQLite overflow pointer is in reserved page bytes".into()), + } + let first = be_u32_at(page, overflow_at)?; + let mut next = first; + let mut seen = HashSet::new(); + while remaining > 0 { + if next == 0 || !seen.insert(next) || seen.len() > MAX_OVERFLOW_PAGES { + return Err("invalid or cyclic SQLite overflow chain".into()); + } + let overflow = self.page(next)?; + let take = remaining.min(usable - 4); + let end = 4usize.checked_add(take).ok_or("overflow payload offset")?; + out.extend_from_slice(overflow.get(4..end).ok_or("truncated SQLite overflow page")?); + remaining -= take; + if remaining > 0 { + next = be_u32_at(overflow, 0)?; + } + } + Ok(out) + } + + fn parse_schema(&self) -> Result { + let mut tables = Vec::new(); + let mut aliases = HashMap::new(); + let mut state = WalkState { seen: HashSet::new(), out: Vec::new(), payload_total: 0 }; + self.visit_table(1, 5, None, 0, &mut state)?; + for record in state.out { + if record.values.len() < 5 { + return Err("sqlite_schema row has too few columns".into()); + } + let kind = match record.values.first() { + Some(Value::Text(s)) => s.as_str(), + _ => return Err("sqlite_schema row has invalid type".into()), + }; + let name = match record.values.get(1) { + Some(Value::Text(s)) => s.clone(), + _ => return Err("sqlite_schema row has invalid name".into()), + }; + if kind.eq_ignore_ascii_case("virtual") { + return Err(format!("virtual table {name} is unsupported")); + } + if !kind.eq_ignore_ascii_case("table") { + continue; + } + let rootpage = match record.values.get(3) { + Some(Value::Integer(n)) if *n > 0 => u32::try_from(*n).map_err(|_| "sqlite_schema root page is invalid")?, + _ => return Err(format!("table {name} has invalid root page")), + }; + let sql = match record.values.get(4) { + Some(Value::Text(s)) => s, + _ => return Err(format!("table {name} has no CREATE TABLE SQL")), + }; + let (column_names, alias) = parse_create_table(sql)?; + if column_names.len() > MAX_COLUMNS { + return Err(format!("table {name} has too many columns")); + } + if tables.iter().any(|table: &LiveTable| table.rootpage == rootpage) { + return Err("duplicate SQLite table root page".into()); + } + if let Some(index) = alias { + aliases.insert(rootpage, index); + } + tables.push(LiveTable { name, rootpage, column_names: Some(column_names) }); + } + Ok(SchemaInfo { tables, aliases }) + } +} + +fn validate_header_fields(header: &[u8], page_size: usize) -> Result { + let write_version = *header.get(18).ok_or("truncated SQLite header")?; + let read_version = *header.get(19).ok_or("truncated SQLite header")?; + if !matches!(write_version, 1 | 2) || !matches!(read_version, 1 | 2) { + return Err("unsupported SQLite file format version".into()); + } + if *header.get(21).ok_or("truncated SQLite header")? != 64 + || *header.get(22).ok_or("truncated SQLite header")? != 32 + || *header.get(23).ok_or("truncated SQLite header")? != 32 + { + return Err("invalid SQLite payload fractions".into()); + } + let reserved = usize::from(*header.get(20).ok_or("truncated SQLite header")?); + let usable = page_size.checked_sub(reserved).ok_or("invalid SQLite reserved-byte count")?; + if usable < 480 { + return Err("SQLite usable page size is below 480 bytes".into()); + } + let schema_format = be_u32(header, 44)?; + if !(1..=4).contains(&schema_format) { + return Err("unsupported SQLite schema format".into()); + } + Ok(reserved) +} + +fn parse_wal(wal: &[u8], page_size: usize) -> Result { + let header = wal.get(..32).ok_or("truncated WAL header")?; + let magic = be_u32(header, 0)?; + if magic != 0x377f0682 && magic != 0x377f0683 { + return Err("invalid WAL magic".into()); + } + if be_u32(header, 4)? != 3_007_000 || be_u32(header, 8)? != u32::try_from(page_size).map_err(|_| "page size overflow")? { + return Err("invalid WAL page size or format version".into()); + } + let stride = page_size.checked_add(24).ok_or("WAL frame size overflow")?; + let body = wal.get(32..).ok_or("truncated WAL body")?; + if body.len() % stride != 0 { + return Err("WAL ends with a partial frame".into()); + } + let salt = (be_u32(header, 16)?, be_u32(header, 20)?); + let frame_count = body.len() / stride; + if frame_count > MAX_CELLS { + return Err("WAL has too many frames".into()); + } + let mut overlay = HashMap::new(); + let mut page_count = None; + for frame_no in 0..frame_count { + let start = frame_no.checked_mul(stride).ok_or("WAL frame offset overflow")?; + let end = start.checked_add(stride).ok_or("WAL frame offset overflow")?; + let frame = body.get(start..end).ok_or("truncated WAL frame")?; + let page_id = be_u32(frame, 0)?; + if page_id == 0 || page_id > MAX_PAGE_ID || salt != (be_u32(frame, 8)?, be_u32(frame, 12)?) { + return Err("invalid WAL frame page or salt".into()); + } + let commit = be_u32(frame, 4)?; + if commit != 0 { + if commit > MAX_PAGE_ID { + return Err("WAL commit database size is too large".into()); + } + page_count = Some(commit); + } + overlay.insert(page_id, frame.get(24..).ok_or("truncated WAL page")?.to_vec()); + } + Ok(WalOverlay { page_size, frame_count, pages: overlay, page_count }) +} + +fn decode_record(payload: &[u8], ncols: usize, encoding: TextEncoding) -> Result, String> { + let mut at = 0; + let header_len = usize_from_u64(read_varint(payload, &mut at)?, "record header length")?; + if header_len < at || header_len > payload.len() { + return Err("invalid SQLite record header length".into()); + } + let mut serials = Vec::new(); + while at < header_len { + if serials.len() >= ncols { + return Err("SQLite record has more columns than table schema".into()); + } + serials.push(read_varint(payload, &mut at)?); + } + if at != header_len { + return Err("SQLite record header is malformed".into()); + } + let mut body = header_len; + let mut values = Vec::with_capacity(ncols); + for serial in serials { + let (value, size) = serial_value(payload, &mut body, serial, encoding)?; + values.push(value); + if values.len() > MAX_COLUMNS { + return Err("SQLite record has too many columns".into()); + } + let _ = size; + } + if body > payload.len() { + return Err("SQLite record body is truncated".into()); + } + values.resize(ncols, Value::Null); + Ok(values) +} + +fn serial_value(payload: &[u8], body: &mut usize, serial: u64, encoding: TextEncoding) -> Result<(Value, usize), String> { + let (kind, len) = match serial { + 0 => (0, 0), + 1 => (1, 1), + 2 => (2, 2), + 3 => (3, 3), + 4 => (4, 4), + 5 => (5, 6), + 6 => (6, 8), + 7 => (7, 8), + 8 => (8, 0), + 9 => (9, 0), + 10 | 11 => return Err("reserved SQLite serial type".into()), + n if n >= 12 => { + let len = usize_from_u64((n - if n % 2 == 0 { 12 } else { 13 }) / 2, "SQLite field length")?; + (if n % 2 == 0 { 12 } else { 13 }, len) + } + _ => return Err("invalid SQLite serial type".into()), + }; + let end = body.checked_add(len).ok_or("SQLite record body offset overflow")?; + let bytes = payload.get(*body..end).ok_or("SQLite record body is truncated")?; + let value = match kind { + 0 => Value::Null, + 1 => Value::Integer(i64::from(i8::from_be_bytes([*bytes.first().ok_or("truncated integer")?]))), + 2 => Value::Integer(i64::from(i16::from_be_bytes(bytes.try_into().map_err(|_| "truncated integer")?))), + 3 => { + let first = *bytes.first().ok_or("truncated integer")?; + let second = *bytes.get(1).ok_or("truncated integer")?; + let third = *bytes.get(2).ok_or("truncated integer")?; + let n = (u32::from(first) << 16) | (u32::from(second) << 8) | u32::from(third); + let signed = if n & 0x80_0000 != 0 { n | 0xff00_0000 } else { n }; + Value::Integer(i64::from(i32::from_be_bytes(signed.to_be_bytes()))) + } + 4 => Value::Integer(i64::from(i32::from_be_bytes(bytes.try_into().map_err(|_| "truncated integer")?))), + 5 => { + let n = bytes.iter().fold(0u64, |acc, b| (acc << 8) | u64::from(*b)); + let signed = if n & (1 << 47) != 0 { n | 0xffff_0000_0000_0000 } else { n }; + Value::Integer(i64::from_be_bytes(signed.to_be_bytes())) + } + 6 => Value::Integer(i64::from_be_bytes(bytes.try_into().map_err(|_| "truncated integer")?)), + 7 => { + let n = f64::from_be_bytes(bytes.try_into().map_err(|_| "truncated real")?); + if !n.is_finite() { + return Err("non-finite SQLite real value".into()); + } + Value::Real(n) + } + 8 => Value::Integer(0), + 9 => Value::Integer(1), + 12 => Value::Blob(bytes.to_vec()), + 13 => Value::Text(decode_text(bytes, encoding)?), + _ => return Err("invalid SQLite field type".into()), + }; + *body = end; + Ok((value, len)) +} + +fn decode_text(bytes: &[u8], encoding: TextEncoding) -> Result { + match encoding { + TextEncoding::Utf8 => String::from_utf8(bytes.to_vec()).map_err(|_| "invalid UTF-8 SQLite text".into()), + TextEncoding::Utf16Le | TextEncoding::Utf16Be => { + if bytes.len() % 2 != 0 { + return Err("odd-length UTF-16 SQLite text".into()); + } + let units: Result, String> = bytes + .as_chunks::<2>() + .0 + .iter() + .map(|pair| { + let first = *pair.first().ok_or("truncated UTF-16 SQLite text")?; + let second = *pair.get(1).ok_or("truncated UTF-16 SQLite text")?; + Ok(match encoding { + TextEncoding::Utf16Le => u16::from_le_bytes([first, second]), + TextEncoding::Utf16Be => u16::from_be_bytes([first, second]), + TextEncoding::Utf8 => return Err("invalid UTF-16 encoding state".into()), + }) + }) + .collect(); + String::from_utf16(&units?).map_err(|_| "invalid UTF-16 SQLite text".into()) + } + } +} + +fn parse_create_table(sql: &str) -> Result<(Vec, Option), String> { + let tokens = tokenize_sql(sql)?; + if !tokens.first().is_some_and(|t| t.word_is("CREATE")) { + return Err("malformed CREATE TABLE SQL".into()); + } + let mut kind_at = 1usize; + if tokens.get(kind_at).is_some_and(|t| t.word_is("TEMPORARY") || t.word_is("TEMP")) { + kind_at += 1; + } + if tokens.get(kind_at).is_some_and(|t| t.word_is("VIRTUAL")) { + return Err("virtual tables are unsupported".into()); + } + if !tokens.get(kind_at).is_some_and(|t| t.word_is("TABLE")) { + return Err("malformed CREATE TABLE SQL".into()); + } + let open = tokens.iter().position(|t| t.is_symbol(b'(')).ok_or("CREATE TABLE has no column list")?; + let mut depth = 0usize; + let mut close = None; + for (i, token) in tokens.iter().enumerate().skip(open) { + if token.is_symbol(b'(') { + depth += 1; + } else if token.is_symbol(b')') { + depth = depth.checked_sub(1).ok_or("unbalanced CREATE TABLE parentheses")?; + if depth == 0 { + close = Some(i); + break; + } + } + } + let close = close.ok_or("unterminated CREATE TABLE column list")?; + if tokens + .get(close + 1..) + .ok_or("invalid CREATE TABLE suffix")? + .windows(2) + .any(|w| w.first().is_some_and(|t| t.word_is("WITHOUT")) && w.get(1).is_some_and(|t| t.word_is("ROWID"))) + { + return Err("WITHOUT ROWID tables are unsupported".into()); + } + let body = tokens.get(open + 1..close).ok_or("invalid CREATE TABLE column list")?; + let mut columns = Vec::new(); + let mut alias = None; + let mut integer_columns = HashSet::new(); + let mut table_primary = None; + let mut start = 0usize; + let mut nested = 0usize; + for i in 0..=body.len() { + let comma = i == body.len() || (nested == 0 && body.get(i).is_some_and(|t| t.is_symbol(b','))); + if comma { + let part = body.get(start..i).ok_or("invalid CREATE TABLE column range")?; + if part.is_empty() { + return Err("empty CREATE TABLE column definition".into()); + } + if is_table_constraint(part) { + if table_primary.is_none() { + table_primary = table_primary_column(part); + } + } else { + let name = part.first().and_then(Token::identifier).ok_or("column name is not an identifier")?; + if columns.iter().any(|old| old == name) { + return Err("duplicate SQLite column name".into()); + } + let index = columns.len(); + if exact_integer_decl(part) { + integer_columns.insert(name.to_ascii_lowercase()); + } + if integer_primary_key_alias(part) { + alias = Some(index); + } + columns.push(name.to_string()); + } + start = i + 1; + } else if body.get(i).is_some_and(|token| token.is_symbol(b'(')) { + nested += 1; + } else if body.get(i).is_some_and(|token| token.is_symbol(b')')) { + nested = nested.checked_sub(1).ok_or("unbalanced CREATE TABLE column definition")?; + } + } + if columns.is_empty() { + return Err("CREATE TABLE has no columns".into()); + } + if alias.is_none() { + if let Some(primary) = table_primary { + let key = primary.to_ascii_lowercase(); + if integer_columns.contains(&key) { + alias = columns.iter().position(|column| column.eq_ignore_ascii_case(&primary)); + } + } + } + Ok((columns, alias)) +} + +fn is_table_constraint(part: &[Token]) -> bool { + part.first().is_some_and(|token| ["CONSTRAINT", "PRIMARY", "UNIQUE", "CHECK", "FOREIGN"].iter().any(|word| token.word_is(word))) +} + +fn exact_integer_decl(part: &[Token]) -> bool { + if !part.get(1).is_some_and(|token| token.word_is("INTEGER")) { + return false; + } + match part.get(2) { + None => true, + Some(token) if token.is_symbol(b'(') => false, + Some(token) => ["PRIMARY", "NOT", "NULL", "UNIQUE", "CHECK", "DEFAULT", "COLLATE", "REFERENCES", "CONSTRAINT", "GENERATED"] + .iter() + .any(|word| token.word_is(word)), + } +} + +fn table_primary_column(part: &[Token]) -> Option { + let primary = part.iter().position(|token| token.word_is("PRIMARY"))?; + if !part.get(primary + 1).is_some_and(|token| token.word_is("KEY")) { + return None; + } + let open = primary + 2; + if !part.get(open).is_some_and(|token| token.is_symbol(b'(')) { + return None; + } + let close = (open + 1..part.len()).find(|index| part.get(*index).is_some_and(|token| token.is_symbol(b')')))?; + let terms = part.get(open + 1..close)?; + if terms.len() == 1 || (terms.len() == 2 && terms.get(1).is_some_and(|token| token.word_is("ASC") || token.word_is("DESC"))) { + return terms.first().and_then(Token::identifier).map(str::to_owned); + } + None +} + +fn integer_primary_key_alias(part: &[Token]) -> bool { + if !exact_integer_decl(part) { + return false; + } + let mut primary = None; + let mut depth = 0usize; + for i in 2..part.len() { + if part.get(i).is_some_and(|token| token.is_symbol(b'(')) { + depth += 1; + } else if part.get(i).is_some_and(|token| token.is_symbol(b')')) { + depth = depth.saturating_sub(1); + } else if depth == 0 && part.get(i).is_some_and(|token| token.word_is("PRIMARY")) && part.get(i + 1).is_some_and(|t| t.word_is("KEY")) { + primary = Some(i); + break; + } + } + let Some(i) = primary else { return false }; + !part.get(i + 2).is_some_and(|token| token.word_is("DESC")) && !part.get(i + 2).is_some_and(|token| token.is_symbol(b'(')) +} + +#[derive(Clone, Debug)] +enum Token { + Word(String), + Quoted(String), + StringLiteral, + Symbol(u8), +} + +impl Token { + fn is_symbol(&self, symbol: u8) -> bool { + matches!(self, Token::Symbol(value) if *value == symbol) + } + + fn word_is(&self, wanted: &str) -> bool { + matches!(self, Token::Word(word) if word.eq_ignore_ascii_case(wanted)) + } + + fn identifier(&self) -> Option<&str> { + match self { + Token::Word(word) | Token::Quoted(word) => Some(word), + _ => None, + } + } +} + +fn tokenize_sql(sql: &str) -> Result, String> { + let bytes = sql.as_bytes(); + let mut out = Vec::new(); + let mut at = 0usize; + while at < bytes.len() { + if bytes.get(at).is_some_and(u8::is_ascii_whitespace) { + at += 1; + } else if bytes.get(at) == Some(&b'-') && bytes.get(at + 1) == Some(&b'-') { + at += 2; + while at < bytes.len() && bytes.get(at) != Some(&b'\n') { + at += 1; + } + } else if bytes.get(at) == Some(&b'/') && bytes.get(at + 1) == Some(&b'*') { + at += 2; + let mut closed = false; + while at + 1 < bytes.len() { + if bytes.get(at) == Some(&b'*') && bytes.get(at + 1) == Some(&b'/') { + at += 2; + closed = true; + break; + } + at += 1; + } + if !closed { + return Err("unterminated SQL comment".into()); + } + } else if matches!(bytes.get(at), Some(b'(' | b')' | b',' | b'.' | b';')) { + out.push(Token::Symbol(*bytes.get(at).ok_or("truncated SQL punctuation")?)); + at += 1; + } else if matches!(bytes.get(at), Some(b'"' | b'`' | b'[')) { + let quote = *bytes.get(at).ok_or("truncated SQL quote")?; + let end_quote = if quote == b'[' { b']' } else { quote }; + at += 1; + let mut value = Vec::new(); + let mut closed = false; + while at < bytes.len() { + if bytes.get(at) == Some(&end_quote) { + if bytes.get(at + 1) == Some(&end_quote) { + value.push(end_quote); + at += 2; + } else { + at += 1; + closed = true; + break; + } + } else { + value.push(*bytes.get(at).ok_or("truncated SQL identifier")?); + at += 1; + } + } + if !closed { + return Err("unterminated quoted SQL identifier".into()); + } + out.push(Token::Quoted(String::from_utf8(value).map_err(|_| "invalid SQL identifier encoding")?)); + } else if bytes.get(at) == Some(&b'\'') { + at += 1; + let mut closed = false; + while at < bytes.len() { + if bytes.get(at) == Some(&b'\'') { + if bytes.get(at + 1) == Some(&b'\'') { + at += 2; + } else { + at += 1; + closed = true; + break; + } + } else { + at += 1; + } + } + if !closed { + return Err("unterminated SQL string".into()); + } + out.push(Token::StringLiteral); + } else { + let start = at; + while at < bytes.len() + && !bytes.get(at).is_some_and(u8::is_ascii_whitespace) + && !matches!(bytes.get(at), Some(b'(' | b')' | b',' | b'.' | b';' | b'\'' | b'"' | b'`' | b'[')) + { + at += 1; + } + if start == at { + return Err("unsupported SQL punctuation".into()); + } + out.push(Token::Word(sql.get(start..at).ok_or("invalid SQL token boundary")?.to_string())); + } + } + Ok(out) +} + +fn sqlite_page_size(value: u16) -> Result { + let size = if value == 1 { 65_536 } else { usize::from(value) }; + if !(512..=65_536).contains(&size) || !size.is_power_of_two() { + return Err("invalid SQLite page size".into()); + } + Ok(size) +} + +fn read_varint(bytes: &[u8], at: &mut usize) -> Result { + let mut value = 0u64; + for i in 0..9 { + let byte = *bytes.get(*at).ok_or("truncated SQLite varint")?; + *at = (*at).checked_add(1).ok_or("SQLite varint offset overflow")?; + if i == 8 { + value = (value << 8) | u64::from(byte); + return Ok(value); + } + value = (value << 7) | u64::from(byte & 0x7f); + if byte & 0x80 == 0 { + return Ok(value); + } + } + Err("invalid SQLite varint".into()) +} + +fn usize_from_u64(value: u64, label: &str) -> Result { + usize::try_from(value).map_err(|_| format!("{label} exceeds host size")) +} + +fn be_u16(bytes: &[u8], at: usize) -> Result { + let end = at.checked_add(2).ok_or("SQLite integer offset overflow")?; + bytes.get(at..end).and_then(|s| <[u8; 2]>::try_from(s).ok()).map(u16::from_be_bytes).ok_or("truncated SQLite integer".into()) +} + +fn be_u16_at(bytes: &[u8], at: usize) -> Result { + be_u16(bytes, at) +} + +fn be_u32(bytes: &[u8], at: usize) -> Result { + let end = at.checked_add(4).ok_or("SQLite integer offset overflow")?; + bytes.get(at..end).and_then(|s| <[u8; 4]>::try_from(s).ok()).map(u32::from_be_bytes).ok_or("truncated SQLite integer".into()) +} + +fn be_u32_at(bytes: &[u8], at: usize) -> Result { + be_u32(bytes, at) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn varint(mut value: u64) -> Vec { + let mut out = vec![u8::try_from(value & 0x7f).unwrap_or(0)]; + value >>= 7; + while value != 0 { + out.push(u8::try_from(value & 0x7f).unwrap_or(0) | 0x80); + value >>= 7; + } + out.reverse(); + out + } + + fn leaf(page: &mut [u8], base: usize, rowid: u64, payload: &[u8]) { + page[base] = 0x0d; + page[base + 3..base + 5].copy_from_slice(&1u16.to_be_bytes()); + let mut cell = varint(payload.len() as u64); + cell.extend(varint(rowid)); + cell.extend(payload); + let start = page.len() - cell.len(); + page[start..].copy_from_slice(&cell); + page[base + 8..base + 10].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes()); + page[base + 5..base + 7].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes()); + } + + #[test] + fn schema_names_and_integer_primary_key_alias_are_decoded() { + let sql = r#"CREATE TABLE "t" ("id_local" INTEGER PRIMARY KEY, "odd,name" TEXT, PRIMARY KEY ("id_local"))"#; + let (names, alias) = parse_create_table(sql).unwrap(); + assert_eq!(names, vec!["id_local", "odd,name"]); + assert_eq!(alias, Some(0)); + let (_, no_alias) = parse_create_table("CREATE TABLE t (id INTEGER PRIMARY KEY DESC, value TEXT)").unwrap(); + assert_eq!(no_alias, None); + let (table_names, table_alias) = parse_create_table("CREATE TABLE t (id INTEGER, value TEXT, PRIMARY KEY(id))").unwrap(); + assert_eq!(table_names, vec!["id", "value"]); + assert_eq!(table_alias, Some(0)); + } + + #[test] + fn synthetic_btree_substitutes_rowid_and_reads_schema() { + let mut main = vec![0u8; 2 * 512]; + main[..16].copy_from_slice(b"SQLite format 3\0"); + main[16..18].copy_from_slice(&512u16.to_be_bytes()); + main[18] = 1; + main[19] = 1; + main[21] = 64; + main[22] = 32; + main[23] = 32; + main[28..32].copy_from_slice(&2u32.to_be_bytes()); + main[44..48].copy_from_slice(&4u32.to_be_bytes()); + main[56..60].copy_from_slice(&1u32.to_be_bytes()); + let sql = "CREATE TABLE \"t\" (\"id_local\" INTEGER PRIMARY KEY, \"name\" TEXT)"; + // rootpage is an integer field, so replace fourth text value with a compact record. + let mut header = vec![23, 15, 15, 1, u8::try_from(13 + 2 * sql.len()).unwrap_or(0)]; + let mut body = b"tablett".to_vec(); + body.push(2); + body.extend(sql.as_bytes()); + let mut record = varint((1 + header.len()) as u64); + record.append(&mut header); + record.extend(body); + let schema = record; + leaf(&mut main[512..], 0, 1, &[3, 0, 15, b'x']); + leaf(&mut main[..512], 100, 1, &schema); + let db = Database::open_with_wal(main, &[]).unwrap(); + let tables = db.live_tables().unwrap(); + assert!(tables.iter().any(|table| table.name == "t")); + let rows = db.read_table(2, 2).unwrap(); + assert_eq!(rows.first().and_then(|row| row.values.first()), Some(&Value::Integer(1))); + } + + #[test] + fn interior_table_pointer_array_uses_cell_offsets() { + let mut main = vec![0u8; 4 * 512]; + let mut interior = vec![0u8; 512]; + interior[0] = 0x05; + interior[3..5].copy_from_slice(&1u16.to_be_bytes()); + interior[8..12].copy_from_slice(&4u32.to_be_bytes()); + interior[12..14].copy_from_slice(&500u16.to_be_bytes()); + interior[500..504].copy_from_slice(&3u32.to_be_bytes()); + interior[504] = 1; + main[512..1024].copy_from_slice(&interior); + leaf(&mut main[1024..1536], 0, 1, &[2, 0]); + leaf(&mut main[1536..], 0, 2, &[2, 0]); + let db = Database { + main, + overlay: HashMap::new(), + page_size: 512, + reserved: 0, + encoding: TextEncoding::Utf8, + page_count: 4, + tables: Vec::new(), + rowid_aliases: HashMap::new(), + }; + assert_eq!(db.read_table(2, 1).unwrap().len(), 2); + } + + #[test] + fn negative_rowid_varint_is_signed() { + let mut main = vec![0u8; 512]; + let page = &mut main[..]; + page[100] = 0x0d; + page[103..105].copy_from_slice(&1u16.to_be_bytes()); + let start = 500usize; + page[start] = 2; + for offset in 1..10 { + if let Some(byte) = page.get_mut(start + offset) { + *byte = 0xff; + } + } + page[start + 10..start + 12].copy_from_slice(&[2, 0]); + page[108..110].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes()); + page[105..107].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes()); + let mut aliases = HashMap::new(); + aliases.insert(1, 0); + let db = Database { + main, + overlay: HashMap::new(), + page_size: 512, + reserved: 0, + encoding: TextEncoding::Utf8, + page_count: 1, + tables: Vec::new(), + rowid_aliases: aliases, + }; + let rows = db.read_table(1, 1).unwrap(); + assert_eq!(rows.first().and_then(|row| row.values.first()), Some(&Value::Integer(-1))); + } + + #[test] + fn reserved_page_trailer_cannot_supply_cells() { + let mut main = vec![0u8; 512]; + main[100] = 0x0d; + main[103..105].copy_from_slice(&1u16.to_be_bytes()); + main[108..110].copy_from_slice(&490u16.to_be_bytes()); + let db = Database { + main, + overlay: HashMap::new(), + page_size: 512, + reserved: 32, + encoding: TextEncoding::Utf8, + page_count: 1, + tables: Vec::new(), + rowid_aliases: HashMap::new(), + }; + assert!(db.read_table(1, 1).is_err()); + } + + #[test] + fn utf16_and_overflow_payloads_are_supported() { + let mut payload = vec![2, 17, b'A', 0]; + assert_eq!(decode_record(&payload, 1, TextEncoding::Utf16Le).unwrap(), vec![Value::Text("A".into())]); + payload.clear(); + payload.extend([2, 17, 0, b'A']); + assert_eq!(decode_record(&payload, 1, TextEncoding::Utf16Be).unwrap(), vec![Value::Text("A".into())]); + + let mut main = vec![0u8; 2 * 512]; + let db = Database { + main: std::mem::take(&mut main), + overlay: HashMap::new(), + page_size: 512, + reserved: 0, + encoding: TextEncoding::Utf8, + page_count: 2, + tables: Vec::new(), + rowid_aliases: HashMap::new(), + }; + let mut cell_page = vec![b'x'; 512]; + cell_page[100..104].copy_from_slice(&2u32.to_be_bytes()); + let inline_page = vec![b'i'; 512]; + assert_eq!(db.payload(&inline_page, 0, 110).unwrap(), vec![b'i'; 110]); + let overflow_page = vec![b'y'; 512]; + let mut db = db; + db.main[512..].copy_from_slice(&overflow_page); + let mut with_overflow = db.overlay; + with_overflow.insert(2, overflow_page); + db.overlay = with_overflow; + let bytes = db.payload(&cell_page, 0, 600).unwrap(); + assert_eq!(bytes.len(), 600); + } + + #[test] + fn malformed_varint_and_wal_cycle_fail_cleanly() { + let mut at = 0; + assert!(read_varint(&[0x80; 8], &mut at).is_err()); + let mut wal = vec![0u8; 32 + 24 + 512]; + wal[..4].copy_from_slice(&0x377f0682u32.to_be_bytes()); + wal[4..8].copy_from_slice(&3_007_000u32.to_be_bytes()); + wal[8..12].copy_from_slice(&512u32.to_be_bytes()); + wal[16..20].copy_from_slice(&7u32.to_be_bytes()); + wal[20..24].copy_from_slice(&8u32.to_be_bytes()); + wal[32..36].copy_from_slice(&1u32.to_be_bytes()); + wal[36..40].copy_from_slice(&1u32.to_be_bytes()); + wal[40..44].copy_from_slice(&7u32.to_be_bytes()); + wal[44..48].copy_from_slice(&8u32.to_be_bytes()); + let parsed = parse_wal(&wal, 512).unwrap(); + assert_eq!(parsed.frame_count, 1); + assert_eq!(parsed.page_count, Some(1)); + assert_eq!(parsed.pages.get(&1).map(Vec::len), Some(512)); + let mut page = vec![0u8; 512]; + page[16..18].copy_from_slice(&512u16.to_be_bytes()); + let db = Database { + main: vec![0; 512], + overlay: HashMap::new(), + page_size: 512, + reserved: 0, + encoding: TextEncoding::Utf8, + page_count: 1, + tables: Vec::new(), + rowid_aliases: HashMap::new(), + }; + assert!(db.payload(&page, 0, 600).is_err()); + } +} diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 1b7b99a4..1252fdb3 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -13,6 +13,7 @@ pub mod headless; pub mod i18n; pub mod icons; pub mod import; +pub mod lightroom_import; pub mod links; pub mod menubar; pub mod menus; @@ -187,6 +188,10 @@ pub struct LightcraftApp { pub import: Option, /// A folder scan in progress (feeds the import review). pub scan: Option, + /// A Lightroom catalog inspect/import in progress. + pub lightroom: Option, + /// Last terminal Lightroom result, exposed by the command's status/wait response. + pub lightroom_last: Option, /// A background export in progress. pub export: Option, /// Background file-system work of other commands (Find Missing Photos, auto import…). @@ -246,6 +251,8 @@ impl LightcraftApp { merge: merge::MergeState::default(), import: None, scan: None, + lightroom: None, + lightroom_last: None, export: None, tasks: Default::default(), last_export_result: None, @@ -621,6 +628,7 @@ impl LightcraftApp { merge::poll(self, ctx); import::poll_scan(self, ctx); import::tick(self, ctx); + lightroom_import::tick(self, ctx); tasks::poll(self, ctx); self.preview_build_status(ctx); self.save_status(ctx); @@ -642,7 +650,7 @@ impl LightcraftApp { if let Some(folder) = self.session.import_defaults.auto_folder.clone() { const LABEL: &str = "Auto Import"; let now = ctx.input(|i| i.time); - if now - self.ui.auto_import_at >= 3.0 && self.import.is_none() && !self.tasks.is_running(LABEL) { + if now - self.ui.auto_import_at >= 3.0 && self.import.is_none() && self.lightroom.is_none() && !self.tasks.is_running(LABEL) { self.ui.auto_import_at = now; let work = move || lightcraft_engine::cmd::library::list_auto_import_folder(&folder); let done = |app: &mut LightcraftApp, _ctx: &egui::Context, listing: Result, String>| { @@ -683,10 +691,12 @@ impl LightcraftApp { let _ = self.run("file.importPresets", serde_json::json!({"paths": presets})); } // read and added on a worker thread (dropped folders can be large, or on a slow drive) - if !photos.is_empty() - && let Err(e) = import::start_paths(self, photos) - { - self.toast(ctx, e); + if !photos.is_empty() { + if self.lightroom.is_some() { + self.toast(ctx, "Finish Lightroom catalog import before adding photos"); + } else if let Err(e) = import::start_paths(self, photos) { + self.toast(ctx, e); + } } } } @@ -823,6 +833,7 @@ impl LightcraftApp { panels::library_problem::show(self, &ctx); import::progress(self, &ctx); import::scan_progress(self, &ctx); + lightroom_import::progress(self, &ctx); export_task::poll(self, &ctx); panels::grid::drag_feedback(self, &ctx); panels::toast(self, &ctx); diff --git a/crates/ui-egui/src/lightroom_import.rs b/crates/ui-egui/src/lightroom_import.rs new file mode 100644 index 00000000..6b2e0a2d --- /dev/null +++ b/crates/ui-egui/src/lightroom_import.rs @@ -0,0 +1,456 @@ +//! Background Lightroom catalog inspection and import. +//! +//! Reading a catalog, probing its originals and staging its archive can take minutes on a +//! removable or network drive. This task keeps that work off the owner thread, commits prepared +//! data only on the owner thread, then finishes the archive index on a worker. + +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, mpsc}; + +use egui::Align2; +use serde_json::{Value, json}; + +use crate::LightcraftApp; +use crate::theme::Tokens; +use crate::widgets::register; + +#[derive(Clone, Copy)] +enum Kind { + Inspect, + Import, +} + +enum Message { + Inspected(Result), + Prepared(Result), + Finalized(Result<(), String>), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Phase { + Reading, + Committing, + Finalizing, +} + +/// A Lightroom task in flight. The receiver is polled from the egui owner thread. +pub struct LightroomTask { + kind: Kind, + path: PathBuf, + cancel: Arc, + total: Arc, + done: Arc, + rx: mpsc::Receiver, + phase: Phase, + cancelled: bool, +} + +impl LightroomTask { + #[cfg(not(target_arch = "wasm32"))] + fn status(&self) -> Value { + let kind = match self.kind { + Kind::Inspect => "inspect", + Kind::Import => "import", + }; + let phase = match self.phase { + Phase::Reading => "reading", + Phase::Committing => "committing", + Phase::Finalizing => "finalizing", + }; + json!({ + "running": true, + "kind": kind, + "path": self.path, + "phase": phase, + "done": self.done.load(Ordering::Relaxed), + "total": self.total.load(Ordering::Relaxed), + "cancelled": self.cancelled || self.cancel.load(Ordering::Relaxed), + }) + } + + fn cancel(&mut self) { + self.cancelled = true; + self.cancel.store(true, Ordering::Relaxed); + } +} + +impl Drop for LightroomTask { + fn drop(&mut self) { + self.cancel.store(true, Ordering::Relaxed); + } +} + +#[cfg(target_arch = "wasm32")] +fn unsupported_wasm() -> Result { + Err("Lightroom catalog import is unavailable in browser builds; open the catalog in the native LightCraft app".into()) +} + +/// Whether a Lightroom task currently owns the catalog transition. +pub fn is_running(app: &LightcraftApp) -> bool { + app.lightroom.is_some() +} + +#[cfg(not(target_arch = "wasm32"))] +fn busy(app: &LightcraftApp) -> Result<(), String> { + if app.lightroom.is_some() { + return Err("a Lightroom catalog task is already running".into()); + } + if app.import.is_some() || app.scan.is_some() { + return Err("an import is running".into()); + } + Ok(()) +} + +#[cfg(not(target_arch = "wasm32"))] +fn spawn_inspect( + path: PathBuf, + cancel: Arc, + total: Arc, + done: Arc, + tx: mpsc::Sender, + ctx: egui::Context, +) -> Result<(), String> { + std::thread::Builder::new() + .name("lc-lightroom-inspect".into()) + .spawn(move || { + let result = lightcraft_engine::guard::catch("Lightroom inspect", || { + lightcraft_engine::lightroom_catalog::read_with_progress(&path, &cancel, &total, &done) + }) + .and_then(|r| r.map(inspect_report)); + let _ = tx.send(Message::Inspected(result)); + ctx.request_repaint(); + }) + .map(|_| ()) + .map_err(|e| format!("could not start Lightroom inspect: {e}")) +} + +#[cfg(not(target_arch = "wasm32"))] +fn spawn_prepare( + mut job: lightcraft_engine::lightroom_job::LightroomJob, + cancel: Arc, + tx: mpsc::Sender, + ctx: egui::Context, +) -> Result<(), String> { + std::thread::Builder::new() + .name("lc-lightroom-import".into()) + .spawn(move || { + let result = lightcraft_engine::guard::catch("Lightroom import", || job.prepare(&cancel)).and_then(|r| r.map_err(|e| e.to_string())); + if let Err(mpsc::SendError(Message::Prepared(Ok(prepared)))) = tx.send(Message::Prepared(result)) { + // The owner dropped the task (for example while closing): staged copies/moves + // must be put back even though no UI receiver remains to accept the result. + prepared.rollback(); + } + ctx.request_repaint(); + }) + .map(|_| ()) + .map_err(|e| format!("could not start Lightroom import: {e}")) +} + +#[cfg(not(target_arch = "wasm32"))] +fn spawn_finalize( + finalization: lightcraft_engine::lightroom_job::LightroomArchiveFinalization, + tx: mpsc::Sender, + ctx: egui::Context, +) -> Result<(), String> { + std::thread::Builder::new() + .name("lc-lightroom-finalize".into()) + .spawn(move || { + let result = lightcraft_engine::guard::catch("Lightroom archive finalization", || finalization.finish()) + .and_then(|r| r.map_err(|e| e.to_string())); + let _ = tx.send(Message::Finalized(result)); + ctx.request_repaint(); + }) + .map(|_| ()) + .map_err(|e| format!("could not start Lightroom archive finalization: {e}")) +} + +#[cfg(target_arch = "wasm32")] +fn spawn_finalize( + _finalization: lightcraft_engine::lightroom_job::LightroomArchiveFinalization, + _tx: mpsc::Sender, + _ctx: egui::Context, +) -> Result<(), String> { + unsupported_wasm().map(|_| ()) +} + +#[cfg(not(target_arch = "wasm32"))] +fn inspect_report(data: lightcraft_engine::lightroom_catalog::CatalogImport) -> Value { + json!({ + "photos": data.photos.len(), + "collections": data.collections.iter().filter(|r| r.get("systemOnly").and_then(Value::as_f64).unwrap_or(0.0) == 0.0).count(), + "missing": data.photos.iter().filter(|p| !std::path::Path::new(&p.path).is_file()).map(|p| &p.path).collect::>(), + "warnings": data.warnings, + }) +} + +fn terminal(app: &mut LightcraftApp, ctx: &egui::Context, kind: Kind, value: Value) { + let message = if let Some(e) = value.get("error").and_then(Value::as_str) { + let warning = value.get("indexWarning").and_then(Value::as_str).map_or(String::new(), |w| format!("; index warning: {w}")); + format!("Lightroom {} failed: {e}{warning}", if matches!(kind, Kind::Inspect) { "inspection" } else { "import" }) + } else if value.get("cancelled").and_then(Value::as_bool).unwrap_or(false) { + format!("Lightroom {} cancelled", if matches!(kind, Kind::Inspect) { "inspection" } else { "import" }) + } else if matches!(kind, Kind::Inspect) { + let photos = value["photos"].as_u64().unwrap_or(0); + let collections = value["collections"].as_u64().unwrap_or(0); + let missing = value["missing"].as_array().map_or(0, Vec::len); + format!("Lightroom catalog inspected: {photos} photos, {collections} collections, {missing} missing") + } else { + let report = value.get("report").unwrap_or(&value); + let imported = report["imported"].as_u64().unwrap_or(0); + let failed = report["failed"].as_array().map_or(0, Vec::len); + let warning = value.get("indexWarning").and_then(Value::as_str); + if let Some(warning) = warning { + format!("Lightroom import complete: {imported} photos added; index warning: {warning}") + } else if failed == 0 { + format!("Lightroom import complete: {imported} photos added") + } else { + format!("Lightroom import complete: {imported} photos added, {failed} failed") + } + }; + app.lightroom_last = Some(value); + app.lightroom = None; + app.toast_for(ctx, message, 6.0); +} + +#[cfg(not(target_arch = "wasm32"))] +fn start_inspect(app: &mut LightcraftApp, path: PathBuf, ctx: &egui::Context) -> Result { + busy(app)?; + let cancel = Arc::new(AtomicBool::new(false)); + let total = Arc::new(AtomicUsize::new(0)); + let done = Arc::new(AtomicUsize::new(0)); + let (tx, rx) = mpsc::channel(); + spawn_inspect(path.clone(), cancel.clone(), total.clone(), done.clone(), tx, ctx.clone())?; + app.lightroom_last = None; + app.lightroom = Some(LightroomTask { kind: Kind::Inspect, path: path.clone(), cancel, total, done, rx, phase: Phase::Reading, cancelled: false }); + Ok(json!({"running": true, "kind": "inspect", "path": path})) +} + +#[cfg(not(target_arch = "wasm32"))] +fn start_import(app: &mut LightcraftApp, path: PathBuf, update_existing: bool, ctx: &egui::Context) -> Result { + busy(app)?; + let job = lightcraft_engine::lightroom_job::LightroomJob::new(&mut app.session, path.clone(), update_existing).map_err(|e| e.to_string())?; + let cancel = Arc::new(AtomicBool::new(false)); + let total = job.total_atomic(); + let done = job.done_atomic(); + let (tx, rx) = mpsc::channel(); + spawn_prepare(job, cancel.clone(), tx, ctx.clone())?; + app.lightroom_last = None; + app.lightroom = Some(LightroomTask { kind: Kind::Import, path: path.clone(), cancel, total, done, rx, phase: Phase::Reading, cancelled: false }); + Ok(json!({"running": true, "kind": "import", "path": path})) +} + +/// Run Lightroom UI commands before the generic engine dispatcher. +pub fn command(app: &mut LightcraftApp, id: &str, p: &Value, ctx: &egui::Context) -> Result { + #[cfg(target_arch = "wasm32")] + { + let _ = (app, id, p, ctx); + return unsupported_wasm(); + } + #[cfg(not(target_arch = "wasm32"))] + { + let path = p.get("path").and_then(Value::as_str).map(PathBuf::from); + let wait = p.get("wait").and_then(Value::as_bool).unwrap_or(false); + let status = p.get("status").and_then(Value::as_bool).unwrap_or(false) || path.is_none(); + if status { + if p.get("cancel").and_then(Value::as_bool).unwrap_or(false) { + if let Some(task) = app.lightroom.as_mut() { + task.cancel(); + } + } + if wait && app.lightroom.is_some() { + return wait_for(app, ctx, std::time::Duration::from_secs(600)); + } + return Ok(app.lightroom.as_ref().map(LightroomTask::status).or_else(|| app.lightroom_last.clone()).unwrap_or(Value::Null)); + } + let Some(path) = path else { return Err("missing `path`".into()) }; + let result = if id == "library.inspectLightroom" { + start_inspect(app, path, ctx) + } else { + start_import(app, path, p.get("updateExisting").and_then(Value::as_bool).unwrap_or(false), ctx) + }?; + if wait { + return wait_for(app, ctx, std::time::Duration::from_secs(600)); + } + Ok(result) + } +} + +/// Poll owner-thread work, commit a prepared import, and finish its archive index. +pub fn tick(app: &mut LightcraftApp, ctx: &egui::Context) { + let Some(mut task) = app.lightroom.take() else { return }; + let message = match task.rx.try_recv() { + Ok(m) => m, + Err(mpsc::TryRecvError::Empty) => { + app.lightroom = Some(task); + return; + } + Err(mpsc::TryRecvError::Disconnected) => { + terminal(app, ctx, task.kind, json!({"error": "Lightroom worker stopped unexpectedly"})); + return; + } + }; + match message { + Message::Inspected(result) => match result { + Ok(report) if !task.cancel.load(Ordering::Relaxed) => terminal(app, ctx, task.kind, report), + Ok(_) => terminal(app, ctx, task.kind, json!({"cancelled": true})), + Err(e) if task.cancel.load(Ordering::Relaxed) || e.contains("cancelled") => terminal(app, ctx, task.kind, json!({"cancelled": true})), + Err(e) => terminal(app, ctx, task.kind, json!({"error": e})), + }, + Message::Prepared(result) => { + let prepared = match result { + Ok(p) => p, + Err(e) if task.cancel.load(Ordering::Relaxed) || e.contains("cancelled") => { + terminal(app, ctx, task.kind, json!({"cancelled": true})); + return; + } + Err(e) => { + terminal(app, ctx, task.kind, json!({"error": e})); + return; + } + }; + if task.cancel.load(Ordering::Relaxed) || prepared.is_cancelled() || !prepared.token().matches(&app.session) { + prepared.rollback(); + terminal( + app, + ctx, + task.kind, + if task.cancel.load(Ordering::Relaxed) { + json!({"cancelled": true}) + } else { + json!({"error": "Lightroom import belongs to a different open library"}) + }, + ); + return; + } + task.phase = Phase::Committing; + let mut finalization = None; + let mut report = None; + let committed = app.session.execute_fn("library.importLightroom", |s| { + let completion = lightcraft_engine::lightroom_job::commit_prepared(s, prepared)?; + report = Some(completion.report.clone()); + finalization = Some(completion.finalization); + Ok(completion.report) + }); + let report = report.unwrap_or(Value::Null); + task.phase = Phase::Finalizing; + let (tx, rx) = mpsc::channel(); + let commit_error = committed.as_ref().err().map(ToString::to_string); + let Some(finalization) = finalization else { + terminal(app, ctx, task.kind, json!({"error": commit_error.clone().unwrap_or_else(|| "Lightroom import did not commit".into())})); + return; + }; + if let Err(e) = spawn_finalize(finalization, tx, ctx.clone()) { + let mut result = json!({"report": report, "indexWarning": e}); + if let Some(error) = commit_error { + result["error"] = json!(error); + } + terminal(app, ctx, task.kind, result); + return; + } + task.rx = rx; + task.cancelled = false; + let mut pending = json!({"running": true, "kind": "import", "report": report}); + if let Some(error) = commit_error { + pending["error"] = json!(error); + } + app.lightroom_last = Some(pending); + app.lightroom = Some(task); + } + Message::Finalized(result) => { + let pending = app.lightroom_last.take().unwrap_or(Value::Null); + let report = pending.get("report").cloned().unwrap_or(Value::Null); + match result { + Ok(()) if pending.get("error").is_none() => terminal(app, ctx, task.kind, report), + Ok(()) => terminal(app, ctx, task.kind, json!({"report": report, "error": pending["error"].clone()})), + Err(e) => { + let mut value = json!({"report": report, "indexWarning": e}); + if let Some(error) = pending.get("error") { + value["error"] = error.clone(); + } + terminal(app, ctx, task.kind, value); + } + } + } + } + ctx.request_repaint(); +} + +/// Wait for a task in control/headless mode while still applying owner-thread completion. +#[cfg(not(target_arch = "wasm32"))] +pub fn wait_for(app: &mut LightcraftApp, ctx: &egui::Context, timeout: std::time::Duration) -> Result { + let start = std::time::Instant::now(); + while app.lightroom.is_some() { + tick(app, ctx); + if start.elapsed() >= timeout { + return Err("timed out waiting for Lightroom task".into()); + } + std::thread::sleep(std::time::Duration::from_millis(2)); + } + Ok(app.lightroom_last.clone().unwrap_or(Value::Null)) +} + +#[cfg(target_arch = "wasm32")] +pub fn wait_for(_app: &mut LightcraftApp, _ctx: &egui::Context, _timeout: std::time::Duration) -> Result { + unsupported_wasm() +} + +/// The progress window for inspect/import, with cancellation. +pub fn progress(app: &mut LightcraftApp, ctx: &egui::Context) { + let Some(task) = app.lightroom.as_ref() else { return }; + let total = task.total.load(Ordering::Relaxed); + let done = task.done.load(Ordering::Relaxed); + let title = match task.kind { + Kind::Inspect => "Inspecting Lightroom Catalog", + Kind::Import => "Importing Lightroom Catalog", + }; + let text = match task.phase { + Phase::Reading if total == 0 => "Reading Lightroom catalog…".to_string(), + Phase::Reading => format!("Reading Lightroom catalog… {done} of {total}"), + Phase::Committing => "Adding Lightroom photos…".to_string(), + Phase::Finalizing => "Saving Lightroom import index…".to_string(), + }; + let mut cancel = false; + let t = Tokens::get(ctx); + egui::Window::new(title).title_bar(false).resizable(false).anchor(Align2::CENTER_BOTTOM, [0.0, -80.0]).fixed_size([360.0, 92.0]).show( + ctx, + |ui| { + ui.label(egui::RichText::new(text).color(t.text)); + ui.add(egui::ProgressBar::new(done as f32 / total.max(1) as f32).desired_width(340.0)); + let r = ui.add_enabled(task.phase == Phase::Reading && !task.cancel.load(Ordering::Relaxed), egui::Button::new("Cancel")); + register(ui.ctx(), "button:lightroomCancel", r.rect); + cancel = r.clicked(); + }, + ); + if cancel { + if let Some(task) = app.lightroom.as_mut() { + task.cancel(); + } + ctx.request_repaint(); + } + if app.lightroom.is_some() { + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } +} + +#[cfg(test)] +mod tests { + use super::inspect_report; + use lightcraft_engine::lightroom_catalog::CatalogImport; + + #[test] + fn inspect_report_keeps_empty_catalog_shape() { + let report = inspect_report(CatalogImport { + source: "synthetic.lrcat".into(), + photos: Vec::new(), + collections: Vec::new(), + members: Vec::new(), + collection_content: Vec::new(), + warnings: vec!["warning".into()], + }); + assert_eq!(report["photos"], 0); + assert_eq!(report["collections"], 0); + assert_eq!(report["missing"], serde_json::json!([])); + assert_eq!(report["warnings"], serde_json::json!(["warning"])); + } +} diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index 5c841d9e..b04ba2bd 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -236,6 +236,10 @@ pub fn parse_rgb(v: &Value) -> Option<[u8; 3]> { /// Handle UI commands; `None` means "not a UI command — send it to the engine". pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option> { + if matches!(id, "library.inspectLightroom" | "library.importLightroom") { + let ctx = egui::Context::default(); + return Some(crate::lightroom_import::command(app, id, p, &ctx)); + } if let Some(language) = language_from_command(id) { app.ui.language = language; // Immediately, not on the next frame: the reply and anything else run this frame @@ -948,11 +952,17 @@ pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option { + if crate::lightroom_import::is_running(app) { + return Some(Err("finish Lightroom catalog import before browsing folders".into())); + } // listed and read in the background (see `import::browse`) let path = p.get("path").and_then(Value::as_str)?; crate::import::browse(app, path, p.get("subfolders").and_then(Value::as_bool)) } "file.addPhotos" => { + if crate::lightroom_import::is_running(app) { + return Some(Err("finish Lightroom catalog import before adding photos".into())); + } let paths = match p.get("paths").and_then(Value::as_array) { Some(a) => a.iter().filter_map(Value::as_str).map(str::to_string).collect(), None => app.services.pick_files.as_mut().map(|f| f()).unwrap_or_default(), @@ -1278,7 +1288,7 @@ pub fn ui_enabled(app: &LightcraftApp, id: &str) -> bool { "file.exportCurvePresets" => !app.session.curve_presets.is_empty(), "view.compare" => app.session.catalog.len() > 1, "view.fullScreenPreview" | "view.infoOverlay" | "view.navigator" => app.session.active().is_some() || app.ui.fullscreen, - "app.openLibrary" | "file.addFolder" => app.services.pick_folder.is_some(), + "app.openLibrary" | "file.addFolder" => app.services.pick_folder.is_some() && !crate::lightroom_import::is_running(app), "file.backupLibrary" => app.services.backup_library.is_some(), "file.restoreLibrary" => app.services.restore_library.is_some(), "compare.swap" | "compare.makeSelect" => app.ui.view == ViewMode::Compare, diff --git a/crates/ui-egui/src/panels/settings.rs b/crates/ui-egui/src/panels/settings.rs index e191668d..745c06ad 100644 --- a/crates/ui-egui/src/panels/settings.rs +++ b/crates/ui-egui/src/panels/settings.rs @@ -530,6 +530,9 @@ fn interface_tab(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens) { /// `app.openLibrary {path?}`: close the current library and open (or create) the one at `path`, /// or a folder chosen in a dialog. Remembered as the library to open at launch. pub fn open_library(app: &mut LightcraftApp, p: &Value) -> Result { + if crate::lightroom_import::is_running(app) { + return Err("wait for Lightroom catalog import to finish before switching libraries".into()); + } let path = match p.get("path").and_then(Value::as_str) { Some(x) => x.to_string(), None => match app.services.pick_folder.as_mut() { @@ -542,6 +545,7 @@ pub fn open_library(app: &mut LightcraftApp, p: &Value) -> Result }; app.session.close_library().map_err(|e| e.to_string())?; app.session.open_library(&path, false).map_err(|e| e.to_string())?; + app.lightroom_last = None; app.renderer.forget_all(); app.ui.compare = None; app.ui.settings.library_path = path.clone(); diff --git a/docs/lightroom-catalog-import.md b/docs/lightroom-catalog-import.md index a5412da1..694441aa 100644 --- a/docs/lightroom-catalog-import.md +++ b/docs/lightroom-catalog-import.md @@ -19,10 +19,15 @@ identity prevents reimport from duplicating virtual copies and collections. One catalog changes; recovery archives remain available. Before changing records, source image records, decoded XMP, verbatim develop settings, history, -snapshots and collection content are saved to `Interop/lightroom-import-N.json` in the LightCraft -library. `Interop/lightroom-index.json` records imported identities. Source catalogs and originals +snapshots and collection content are saved to compressed `Interop/lightroom-import-N.lca` archives +in the LightCraft library. Unchanged source data reuses its existing archive. Opaque SQL BLOBs are +skipped; decoded XMP and structured recovery data remain. Each archive is capped at 32 MiB +(uncompressed and on disk), with at most eight archives and 128 MiB total retained. Oldest managed +archives are retired; legacy `.json` archives remain untouched. `Interop/lightroom-index.json` records imported identities. Source catalogs and originals are never overwritten. Native import is bounded to a 1 GiB database/WAL, 16 MiB XMP packets and -one million rows per table. The SQLite reader is Apache-2.0 `sqlite-core`, written in Rust. +one million rows per table. The in-tree, read-only Rust SQLite B-tree reader follows SQLite’s public file-format specification; +it introduces no dependency or C runtime. Table payloads are capped at 256 MiB. Unsupported +WITHOUT ROWID and virtual tables produce explicit errors. Develop settings reuse the existing XMP/preset mapper. Supported sliders, curves and supported mask structures remain editable, but this is approximate rendering: camera profiles, Adobe AI @@ -34,5 +39,15 @@ Lightroom's `-999999` deferred-adjustment sentinel is omitted from both catalog it is reported and archived, never clamped into a real slider value. Deferred Adobe Auto Tone is not evaluated by the importer; LightCraft's Auto control remains available after migration. +Native desktop inspection/import runs on a cancellable background worker with progress. Only +prepared catalog operations commit on the owner thread; archive index writing stays on a worker. +Late results from a closed or switched library are rejected, including reopening the same path. +Ordinary edits during preparation remain intact. CLI commands complete synchronously using the +same preparation/commit pipeline; browser builds reject native catalog import. + +Archive version 1 begins with the 12 bytes `LC-LRARCH\0\x01Z`, followed by zlib-compressed JSON. +Recovery readers must cap decompression at 32 MiB. JSON retains source identities, photo paths, +settings, decoded XMP, keywords, history, snapshots, collection definitions and membership. + The importer uses shared Rust code on macOS, Windows and Linux. If originals move between computers or volumes, use LightCraft's missing-photo relinking tools to update their paths. diff --git a/docs/parity.md b/docs/parity.md index 3aee3d8e..79ae9f5e 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -716,7 +716,7 @@ From `08-lightroom-classic-extras.md` (Classic-only features) and part 2 of `06- | LRC-LIB-SURVEY | Survey view | P2 | ✅ | `cmd:view.survey` | N | | LRC-LIB-REFVIEW | Reference view | P2 | ✅ | `cmd:view.reference`, `cmd:photo.setReference`, `crates/ui-egui/src/panels/compare.rs` (`show_reference`) | View ▸ Reference View (⇧R): the reference photo beside the active one (edited with the Edit panel); photo menu ▸ Set as Reference Photo | | LRC-LIB-CATALOG-PANEL | Catalog sets | P1 | ✅ | `cmd:library.source` (`missing`) | all, recently added, picks, recently deleted, missing photos (when any) | -| LRC-LIB-CATALOG-IMPORT | Native Lightroom Classic catalog import | P1 | 🟡 | `cmd:library.inspectLightroom`, `cmd:library.importLightroom`, `crates/engine/src/lightroom_catalog.rs`, `docs/lightroom-catalog-import.md` | read-only `.lrcat` plus committed WAL pages, in-process Rust SQLite reader; originals in place, ratings/flags/labels, XMP, hierarchical keywords, collection sets/regular collections, virtual copies, mapped develop settings; stable reimport identities, undo, missing-file placeholders, existing edits preserved. Verbatim settings/history/snapshots/collection rules archived. Adobe rendering, AI data and native history replay remain incomplete; no writes to Lightroom catalogs | +| LRC-LIB-CATALOG-IMPORT | Native Lightroom Classic catalog import | P1 | 🟡 | `cmd:library.inspectLightroom`, `cmd:library.importLightroom`, `crates/engine/src/lightroom_catalog.rs`, `docs/lightroom-catalog-import.md` | read-only `.lrcat` plus committed WAL pages, in-tree read-only Rust SQLite B-tree reader; background inspection/import with progress/cancellation, per-open library identity checks; originals in place, ratings/flags/labels, XMP, hierarchical keywords, collection sets/regular collections, virtual copies, mapped develop settings; stable reimport identities, undo, missing-file placeholders, existing edits preserved. Settings/history/snapshots/collection rules retained in deduplicated compressed archives (32 MiB each, eight archives/128 MiB aggregate); opaque SQL BLOBs skipped. Adobe rendering, AI data and native history replay remain incomplete; no writes to Lightroom catalogs | | LRC-LIB-CATALOG-LOCK | Catalog open in one program at a time | P1 | ✅ | `cmd:app.openLibrary`, `crates/catalog/src/lock.rs`, `crates/engine/src/library.rs` (`open_library`) | exclusive OS lock on `catalog.lock` for the session (app and `lightcraft-cli --library`); a second opener is refused with who has it (`catalog.lock.owner`: program, process, computer); released by the OS on a crash, so never stale; file systems without locks open unprotected with a warning; no read-only mode; the browser build (OPFS) has no guard across tabs yet. A library that can't be opened at launch (locked, unreadable, newer format, drive missing) shows a window — Try Again, Choose Another Library…, Continue Without Saving (then a banner; nothing written), Quit — instead of a silent demo session (`crates/ui-egui/src/panels/library_problem.rs`) | | LRC-LIB-FOLDERS | Disk folder tree | P1 | ✅ | `cmd:library.browse`, `cmd:folder.rename`, `cmd:folder.move`, `crates/ui-egui/src/panels/left.rs` (`local_section`, `folder_tree`) | standard places + the browsed folder + Browse Folder…, each an expandable tree of subfolders, breadcrumb navigation; context menu: Rename Folder…, Move Folder To…, Show in Finder (on disk, photos relinked; one undo step each: undo / redo move the folder back and forth, refused when the destination is taken) | | LRC-LIB-COLLECTIONS | Collections & sets | P1 | ✅ | `cmd:album.create`, `cmd:album.createSmart`, `cmd:album.toggleTarget`, `cmd:album.setTarget`, `cmd:album.clearQuick` | albums + folders (sets), smart albums (rule editor; rules incl. File Path — the whole string, either separator, case-insensitive), Quick Collection and target album (B in the grids adds / removes the selection; album menu ▸ Set as Target Album; the target is marked +) |