From b273f8e6087ae9f20ebfc2f4fb3708fd263f0171 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:53:36 +0200 Subject: [PATCH] #798: read large Lightroom catalogs by page Generated with Codex. --- crates/engine/src/lightroom_catalog.rs | 37 ++++- crates/engine/src/lightroom_sqlite.rs | 220 ++++++++++++++++++++++++- docs/gaps.md | 2 +- docs/lightroom-catalog-import.md | 13 +- docs/parity-checklist.md | 2 +- 5 files changed, 257 insertions(+), 17 deletions(-) diff --git a/crates/engine/src/lightroom_catalog.rs b/crates/engine/src/lightroom_catalog.rs index 6c1c7724..a727491f 100644 --- a/crates/engine/src/lightroom_catalog.rs +++ b/crates/engine/src/lightroom_catalog.rs @@ -8,7 +8,7 @@ use std::sync::{ atomic::{AtomicBool, AtomicUsize, Ordering}, }; -use crate::lightroom_sqlite::{Database, LiveTable, Value as SqlValue}; +use crate::lightroom_sqlite::{Database, FileSource, LiveTable, Value as SqlValue}; use lightcraft_catalog::{Album, Flag, Op, Photo, PhotoId, Source}; use lightcraft_geom::{Affine, Orientation, Point, Rect}; use serde::{Deserialize, Serialize}; @@ -352,15 +352,15 @@ pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize, let before = std::fs::metadata(path).map_err(|e| e.to_string())?; let wal_path = PathBuf::from(format!("{}-wal", path.display())); let wal_before = std::fs::metadata(&wal_path).ok().map(|m| (m.len(), m.modified().ok())); - let main = bytes(path)?; + let main = FileSource::open(path)?; let wal = match std::fs::metadata(&wal_path) { Ok(_) => bytes(&wal_path)?, Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(), Err(e) => return Err(e.to_string()), }; - let wal_len = committed_wal_len(&main, &wal)?; + let wal_len = committed_wal_len(main.header(), &wal)?; ensure_read_active(cancel)?; - let db = Database::open_with_wal(main, &wal[..wal_len]).map_err(|e| format!("invalid SQLite catalog: {e:?}"))?; + let db = Database::open_file_with_wal(main, &wal[..wal_len]).map_err(|e| format!("invalid SQLite catalog: {e:?}"))?; let after = std::fs::metadata(path).map_err(|e| e.to_string())?; let wal_after = std::fs::metadata(&wal_path).ok().map(|m| (m.len(), m.modified().ok())); if before.len() != after.len() || before.modified().ok() != after.modified().ok() || wal_before != wal_after { @@ -471,6 +471,11 @@ pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize, }); } ensure_read_active(cancel)?; + reader.db.ensure_unchanged()?; + let wal_after = std::fs::metadata(&wal_path).ok().map(|m| (m.len(), m.modified().ok())); + if wal_before != wal_after { + return Err("catalog changed during reading; close Lightroom and retry".into()); + } Ok(out) } @@ -1302,6 +1307,30 @@ mod tests { assert_eq!(catalog.photos[0].uuid, "fixture-image"); assert_eq!(number(&catalog.photos[0].image, "rating"), 5); assert!(catalog.photos[0].path.ends_with("test.ARW")); + // Put the required image table beyond the old 1 GiB cutoff without allocating it. + // A main-file-sized allocation would defeat the purpose of the page-backed reader. + // The large fixture relies on Unix sparse-file behavior; small parity runs everywhere. + #[cfg(unix)] + { + let far_offset = 1u64 << 30; + schema.last_mut().unwrap()[3] = I((far_offset / 2048 + 1) as i64); + leaf(&mut bytes[..2048], 100, &schema); + use std::io::{Seek, SeekFrom, Write}; + let mut sparse = std::fs::OpenOptions::new().write(true).truncate(true).open(&path).unwrap(); + sparse.write_all(&bytes).unwrap(); + sparse.set_len(far_offset + 2048).unwrap(); + sparse.seek(SeekFrom::Start(far_offset)).unwrap(); + sparse.write_all(&bytes[4 * 2048..5 * 2048]).unwrap(); + drop(sparse); + let large = read(&path).unwrap(); + assert_eq!(serde_json::to_value(&large.photos).unwrap(), serde_json::to_value(&catalog.photos).unwrap()); + let mut session = crate::Session::new(); + session.execute("library.importLightroom", &json!({"path": path.to_string_lossy()})).unwrap(); + assert_eq!(session.catalog.len(), 1); + assert_eq!(session.catalog.photos().next().unwrap().rating, 5); + session.execute("edit.undo", &json!({})).unwrap(); + assert!(session.catalog.is_empty()); + } std::fs::write(&path, b"bad catalog").unwrap(); assert!(read(&path).is_err()); std::fs::remove_file(path).unwrap(); diff --git a/crates/engine/src/lightroom_sqlite.rs b/crates/engine/src/lightroom_sqlite.rs index 900fe4a2..f00cb111 100644 --- a/crates/engine/src/lightroom_sqlite.rs +++ b/crates/engine/src/lightroom_sqlite.rs @@ -6,7 +6,79 @@ //! than a SQL engine. It never writes its input and has no SQLite runtime //! dependency. +use std::borrow::Cow; use std::collections::{HashMap, HashSet}; +use std::fs::{File, Metadata}; +use std::io::{Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +const MAX_DATABASE_BYTES: u64 = 16 << 30; + +/// A bounded, read-only page source. Only referenced pages are loaded, never the whole file. +pub struct FileSource { + file: Mutex, + path: PathBuf, + before: Metadata, + header: Vec, +} + +impl FileSource { + pub fn open(path: &Path) -> Result { + if !std::fs::metadata(path).map_err(|e| e.to_string())?.is_file() { + return Err("SQLite catalog must be a regular file".into()); + } + let mut file = File::open(path).map_err(|e| e.to_string())?; + let before = file.metadata().map_err(|e| e.to_string())?; + if !before.is_file() { + return Err("SQLite catalog must be a regular file".into()); + } + if before.len() > MAX_DATABASE_BYTES { + return Err("catalog exceeds 16 GiB import limit".into()); + } + let mut header = vec![0; 100]; + file.read_exact(&mut header).map_err(|e| format!("cannot read SQLite header: {e}"))?; + let source = Self { file: Mutex::new(file), path: path.into(), before, header }; + source.ensure_unchanged()?; + Ok(source) + } + + pub fn header(&self) -> &[u8] { + &self.header + } + + pub fn ensure_unchanged(&self) -> Result<(), String> { + let file = self.file.lock().map_err(|_| "SQLite page source lock poisoned")?; + let handle = file.metadata().map_err(|e| e.to_string())?; + let path = std::fs::metadata(&self.path).map_err(|_| "catalog changed during reading; close Lightroom and retry")?; + if !same_metadata(&self.before, &handle) || !same_metadata(&self.before, &path) { + return Err("catalog changed during reading; close Lightroom and retry".into()); + } + Ok(()) + } + + fn page(&self, page_id: u32, size: usize) -> Result, String> { + let start = u64::from(page_id).checked_sub(1).and_then(|n| n.checked_mul(size as u64)).ok_or("page offset overflow")?; + if start.checked_add(size as u64).is_none_or(|end| end > self.before.len()) { + return Err(format!("page {page_id} is truncated")); + } + let mut out = vec![0; size]; + let mut file = self.file.lock().map_err(|_| "SQLite page source lock poisoned")?; + file.seek(SeekFrom::Start(start)).and_then(|_| file.read_exact(&mut out)).map_err(|e| format!("cannot read SQLite page {page_id}: {e}"))?; + Ok(out) + } +} + +fn same_metadata(a: &Metadata, b: &Metadata) -> bool { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + if a.dev() != b.dev() || a.ino() != b.ino() { + return false; + } + } + a.len() == b.len() && a.modified().ok() == b.modified().ok() && a.created().ok() == b.created().ok() +} const MAX_PAGE_ID: u32 = 16_777_216; const MAX_COLUMNS: usize = 16_384; @@ -44,6 +116,7 @@ pub struct LiveTable { /// Read-only SQLite database image, with a committed WAL overlay if supplied. pub struct Database { main: Vec, + file: Option, overlay: HashMap>, page_size: usize, reserved: usize, @@ -84,17 +157,34 @@ impl Database { /// The WAL argument must include its 32-byte header and only complete /// frames through a commit. Frames are overlaid by page number, with the /// final commit's database-size field defining the visible page count. + #[cfg(test)] pub fn open_with_wal(main: Vec, committed_wal: &[u8]) -> Result { + Self::open(main, None, committed_wal) + } + + pub fn open_file_with_wal(source: FileSource, committed_wal: &[u8]) -> Result { + Self::open(source.header().to_vec(), Some(source), committed_wal) + } + + pub fn ensure_unchanged(&self) -> Result<(), String> { + if let Some(file) = &self.file { + file.ensure_unchanged()?; + } + Ok(()) + } + + fn open(main: Vec, file: Option, committed_wal: &[u8]) -> Result { let main_header = main.get(..100).ok_or("truncated SQLite header")?; if main_header.get(..16) != Some(b"SQLite format 3\0") { return Err("invalid SQLite header signature".into()); } let page_size = sqlite_page_size(be_u16(main_header, 16)?)?; validate_header_fields(main_header, page_size)?; - if main.len() < page_size || !main.len().is_multiple_of(page_size) { + let main_len = file.as_ref().map_or(main.len() as u64, |source| source.before.len()); + if main_len < page_size as u64 || !main_len.is_multiple_of(page_size as u64) { return Err("SQLite image is not a whole number of pages".into()); } - let main_pages = u32::try_from(main.len() / page_size).map_err(|_| "SQLite image is too large")?; + let main_pages = u32::try_from(main_len / page_size as u64).map_err(|_| "SQLite image is too large")?; if main_pages == 0 || main_pages > MAX_PAGE_ID { return Err("invalid SQLite image page count".into()); } @@ -140,7 +230,7 @@ impl Database { _ => return Err("unsupported SQLite text encoding".into()), }; - let mut db = Self { main, overlay, page_size, reserved, encoding, page_count, tables: Vec::new(), rowid_aliases: HashMap::new() }; + let mut db = Self { main, file, overlay, page_size, reserved, encoding, page_count, tables: Vec::new(), rowid_aliases: HashMap::new() }; let schema = db.parse_schema()?; db.tables = schema.tables; db.rowid_aliases = schema.aliases; @@ -166,17 +256,20 @@ impl Database { Ok(state.out) } - fn page(&self, page_id: u32) -> Result<&[u8], String> { + fn page(&self, page_id: u32) -> Result, String> { if page_id == 0 || page_id > self.page_count { return Err(format!("page {page_id} is outside database")); } if let Some(page) = self.overlay.get(&page_id) { - return Ok(page); + return Ok(Cow::Borrowed(page)); + } + if let Some(file) = &self.file { + return file.page(page_id, self.page_size).map(Cow::Owned); } let id = usize::try_from(page_id).map_err(|_| "page number does not fit host")?; let start = id.checked_sub(1).and_then(|n| n.checked_mul(self.page_size)).ok_or("page offset overflow")?; let end = start.checked_add(self.page_size).ok_or("page offset overflow")?; - self.main.get(start..end).ok_or_else(|| format!("page {page_id} is truncated")) + self.main.get(start..end).map(Cow::Borrowed).ok_or_else(|| format!("page {page_id} is truncated")) } fn visit_table(&self, page_id: u32, ncols: usize, alias: Option, depth: usize, state: &mut WalkState) -> Result<(), String> { @@ -219,7 +312,7 @@ impl Database { if state.payload_total > MAX_TABLE_PAYLOAD { return Err("SQLite table payload exceeds bounded import size".into()); } - let payload = self.payload(page, at, payload_len)?; + let payload = self.payload(&page, at, payload_len)?; let mut values = decode_record(&payload, ncols, self.encoding)?; if let Some(index) = alias { if index >= values.len() { @@ -325,7 +418,7 @@ impl Database { out.extend_from_slice(overflow.get(4..end).ok_or("truncated SQLite overflow page")?); remaining -= take; if remaining > 0 { - next = be_u32_at(overflow, 0)?; + next = be_u32_at(&overflow, 0)?; } } Ok(out) @@ -871,6 +964,72 @@ fn be_u32_at(bytes: &[u8], at: usize) -> Result { mod tests { use super::*; + struct Scratch(PathBuf); + impl Scratch { + fn new(bytes: &[u8]) -> Self { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let n = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("lightcraft-page-source-{}-{n}.lrcat", std::process::id())); + std::fs::write(&path, bytes).unwrap(); + Self(path) + } + } + impl Drop for Scratch { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } + } + + fn file_backed(mut db: Database) -> (Scratch, Database) { + let scratch = Scratch::new(&db.main); + db.file = Some(FileSource::open(&scratch.0).unwrap()); + db.main.truncate(100); + (scratch, db) + } + + #[test] + fn file_source_bounds_and_mutation_detection() { + let scratch = Scratch::new(&[0; 512]); + let source = FileSource::open(&scratch.0).unwrap(); + assert_eq!(source.page(1, 512).unwrap().len(), 512); + assert!(source.page(2, 512).unwrap_err().contains("truncated")); + File::options().write(true).open(&scratch.0).unwrap().set_len(100).unwrap(); + assert!(source.page(1, 512).is_err()); + assert!(source.ensure_unchanged().unwrap_err().contains("changed during reading")); + drop(source); + // Unix set_len creates a hole. Do not reserve a 16 GiB normal file on Windows. + #[cfg(unix)] + { + File::options().write(true).open(&scratch.0).unwrap().set_len(MAX_DATABASE_BYTES + 1).unwrap(); + assert_eq!(FileSource::open(&scratch.0).err().unwrap(), "catalog exceeds 16 GiB import limit"); + } + } + + #[cfg(unix)] + #[test] + fn file_source_detects_same_sized_path_replacement() { + let original = Scratch::new(&[0; 512]); + let replacement = Scratch::new(&[0; 512]); + let source = FileSource::open(&original.0).unwrap(); + std::fs::rename(&replacement.0, &original.0).unwrap(); + assert!(source.ensure_unchanged().unwrap_err().contains("changed during reading")); + } + + #[test] + fn file_source_detects_same_length_content_changes() { + let scratch = Scratch::new(&[0; 512]); + let source = FileSource::open(&scratch.0).unwrap(); + std::fs::write(&scratch.0, [1; 512]).unwrap(); + // Make the timestamp change deterministic even on filesystems with coarse timestamps. + File::options() + .write(true) + .open(&scratch.0) + .unwrap() + .set_modified(source.before.modified().unwrap() + std::time::Duration::from_secs(2)) + .unwrap(); + assert!(source.ensure_unchanged().unwrap_err().contains("changed during reading")); + } + fn varint(mut value: u64) -> Vec { let mut out = vec![u8::try_from(value & 0x7f).unwrap_or(0)]; value >>= 7; @@ -996,6 +1155,40 @@ mod tests { assert!(tables.iter().any(|table| table.name == "t")); let rows = db.read_table(2, 2).unwrap(); assert_eq!(rows.first().and_then(|row| row.values.first()), Some(&Value::Integer(1))); + let scratch = Scratch::new(&db.main); + let file_db = Database::open_file_with_wal(FileSource::open(&scratch.0).unwrap(), &[]).unwrap(); + assert_eq!(file_db.live_tables().unwrap(), tables); + assert_eq!(file_db.read_table(2, 2).unwrap(), rows); + assert_eq!(file_db.main.len(), 100); + file_db.ensure_unchanged().unwrap(); + // The same committed WAL page overrides the lazily read main page. + let mut wal = vec![0; 32 + 24 + 512]; + wal[..4].copy_from_slice(&0x377f0682u32.to_be_bytes()); + wal[4..8].copy_from_slice(&3_007_000u32.to_be_bytes()); + wal[8..12].copy_from_slice(&512u32.to_be_bytes()); + wal[32..36].copy_from_slice(&2u32.to_be_bytes()); + wal[36..40].copy_from_slice(&2u32.to_be_bytes()); + leaf(&mut wal[56..], 0, 2, &[3, 0, 15, b'y']); + let memory_wal = Database::open_with_wal(db.main.clone(), &wal).unwrap(); + let file_wal = Database::open_file_with_wal(FileSource::open(&scratch.0).unwrap(), &wal).unwrap(); + assert_eq!(memory_wal.read_table(2, 2).unwrap(), file_wal.read_table(2, 2).unwrap()); + assert_eq!(file_wal.read_table(2, 2).unwrap()[0].values[0], Value::Integer(2)); + let mut malformed = db.main.clone(); + malformed.push(0); + let partial = Scratch::new(&malformed); + assert_eq!( + Database::open_file_with_wal(FileSource::open(&partial.0).unwrap(), &[]).err().unwrap(), + "SQLite image is not a whole number of pages" + ); + let mut malformed = db.main.clone(); + malformed[24..28].copy_from_slice(&1u32.to_be_bytes()); + malformed[92..96].copy_from_slice(&1u32.to_be_bytes()); + malformed[28..32].copy_from_slice(&3u32.to_be_bytes()); + let oversized = Scratch::new(&malformed); + assert_eq!( + Database::open_file_with_wal(FileSource::open(&oversized.0).unwrap(), &[]).err().unwrap(), + "SQLite header page count exceeds image" + ); } #[test] @@ -1012,6 +1205,7 @@ mod tests { leaf(&mut main[1024..1536], 0, 1, &[2, 0]); leaf(&mut main[1536..], 0, 2, &[2, 0]); let db = Database { + file: None, main, overlay: HashMap::new(), page_size: 512, @@ -1043,6 +1237,7 @@ mod tests { let mut aliases = HashMap::new(); aliases.insert(1, 0); let db = Database { + file: None, main, overlay: HashMap::new(), page_size: 512, @@ -1063,6 +1258,7 @@ mod tests { main[103..105].copy_from_slice(&1u16.to_be_bytes()); main[108..110].copy_from_slice(&490u16.to_be_bytes()); let db = Database { + file: None, main, overlay: HashMap::new(), page_size: 512, @@ -1073,6 +1269,8 @@ mod tests { rowid_aliases: HashMap::new(), }; assert!(db.read_table(1, 1).is_err()); + let (_scratch, db) = file_backed(db); + assert!(db.read_table(1, 1).is_err()); } #[test] @@ -1085,6 +1283,7 @@ mod tests { let mut main = vec![0u8; 2 * 512]; let db = Database { + file: None, main: std::mem::take(&mut main), overlay: HashMap::new(), page_size: 512, @@ -1111,6 +1310,10 @@ mod tests { assert_eq!(bytes.len(), 600); assert_eq!(&bytes[..92], &[b'x'; 92]); assert_eq!(&bytes[92..], &[b'y'; 508]); + let (_scratch, mut db) = file_backed(db); + db.overlay.clear(); + let bytes = db.payload(&cell_page, 0, 600).unwrap(); + assert_eq!(&bytes[92..], &[b'y'; 508]); } #[test] @@ -1134,6 +1337,7 @@ mod tests { let mut page = vec![0u8; 512]; page[16..18].copy_from_slice(&512u16.to_be_bytes()); let db = Database { + file: None, main: vec![0; 512], overlay: HashMap::new(), page_size: 512, diff --git a/docs/gaps.md b/docs/gaps.md index 64b1cb8a..98cc1754 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -27,7 +27,7 @@ Hours are Opus 5.5 agent wall-clock hours (see [target-app-parity.md](target-app | 10 B | **Per-model verification thin, plus cheap per-body decode fixes**: 234 of 1,446 Adobe models verified (16%), ~40% of photographers on a verified body; Nikon 5 bodies verified, Canon 11. Cheap fixes: Z 6II packed 14-bit NEF (#769, PR #671), Sony A100 / F828 regression (#702), DSLR-A200…A390 / A850 12-bit ARW (#535) | measured: [raw/coverage.tsv](raw/coverage.tsv) | per-body bugs (levels, crops, WB) surface only from users | 25–45 | [raw-parity](raw-parity.md#by-maker-measured) | | 11 | **Enhance: Raw Details, Super Resolution; AI denoise without shippable weights or X-Trans** | ENH rows 0%; denoise Bayer only, RawNIND weights GPL and opt-in | Fujifilm users get no AI denoise; nobody gets it out of the box | 30–60 + model decision | [raw-parity](raw-parity.md#demosaic-levels-crops-and-lens-metadata) | | 12 | **Content-aware / generative remove, detect distractions, reflections removal** | REM rows 75%; patch synthesis missing | cleanup work goes back to Photoshop | 25–45 (+ generative model) | [target-app-parity](target-app-parity.md#ai) | -| 13 | **Lightroom catalog migration is one-way and approximate**: unsupported Adobe profiles / AI settings archived, history kept as data | lightroom-catalog-import.md | switching users lose some looks | 8–12 | [file-format-parity](file-format-parity.md) | +| 13 | **Lightroom catalog migration is one-way and approximate**: unsupported Adobe profiles / AI settings archived, history kept as data; main-file pages read on demand up to 16 GiB, with dense-table and WAL bounds retained (#798) | lightroom-catalog-import.md | switching users lose some looks | 8–12 | [file-format-parity](file-format-parity.md) | | 14 | **HDR display** (EDR / Windows HDR); HLG; HDR settings from `crs` XMP | HDR rows 70%; LR-VIEW-HDR-DISPLAY ⬜ | HDR editing works blind on HDR monitors | 12–25 | [hardware-parity](hardware-parity.md) | | 15 | **Print module**: custom packages, printer colour management (contact-sheet PDF only) | LRC-PRINT rows | Classic users who print | 25–40 | [target-app-parity](target-app-parity.md#features) | | 16 | **Map view, Book, Slideshow module, publish services** | LRC-MAP / BOOK / SS rows ⬜ | Classic users; publish services matter for Flickr / SmugMug workflows | 60–100 | [target-app-parity](target-app-parity.md#features) | diff --git a/docs/lightroom-catalog-import.md b/docs/lightroom-catalog-import.md index 0c18a652..3ae13c75 100644 --- a/docs/lightroom-catalog-import.md +++ b/docs/lightroom-catalog-import.md @@ -28,9 +28,16 @@ retained. Oldest managed archives are retired; legacy `.json` archives remain un `Interop/lightroom-index.json` records imported identities. Source catalogs and originals are never overwritten. -Native import is bounded to a 1 GiB database/WAL, 16 MiB XMP packets and one million rows per -table. The in-tree, read-only Rust SQLite B-tree reader follows SQLite’s public file-format -specification; it introduces no dependency or C runtime. Table payloads are capped at 256 MiB. +Native import reads referenced database pages on demand (at most 64 KiB per page), rather than +loading the whole `.lrcat`. Database files are capped at 16 GiB and 16,777,216 pages; the WAL +remains capped at 1 GiB. Tables still have a 256 MiB encoded-payload budget and one million rows, +individual records are capped at 64 MiB, and XMP packets at 16 MiB. These limits can still reject +large catalogs with dense tables; file size alone no longer determines the memory allocation. +The open file and source path are checked again after all tables are read. Close Lightroom before +importing: metadata checks detect ordinary concurrent changes, but are not SQLite snapshot isolation. + +The in-tree, read-only Rust SQLite B-tree reader follows SQLite's public file-format +specification; it introduces no dependency or C runtime. Optional history, keyword and collection tables that exceed limits are skipped with warnings; required photo tables still report errors. Unrelated unsupported schemas are ignored. A needed WITHOUT ROWID or virtual table produces an explicit error, handled as a warning when optional. diff --git a/docs/parity-checklist.md b/docs/parity-checklist.md index 09941108..2b6cf481 100644 --- a/docs/parity-checklist.md +++ b/docs/parity-checklist.md @@ -679,7 +679,7 @@ From `08-lightroom-classic-extras.md` (Classic-only features) and part 2 of `06- | LRC-LIB-SURVEY | Survey view | P2 | ✅ | `cmd:view.survey` | N | | LRC-LIB-REFVIEW | Reference view | P2 | ✅ | `cmd:view.reference`, `cmd:photo.setReference`, `crates/ui-egui/src/panels/compare.rs` (`show_reference`) | View ▸ Reference View (⇧R): the reference photo beside the active one (edited with the Edit panel); photo menu ▸ Set as Reference Photo | | LRC-LIB-CATALOG-PANEL | Catalog sets | P1 | ✅ | `cmd:library.source` (`missing`) | all, recently added, picks, recently deleted, missing photos (when any) | -| LRC-LIB-CATALOG-IMPORT | Native Lightroom Classic catalog import | P1 | 🟡 | `cmd:library.inspectLightroom`, `cmd:library.importLightroom`, `crates/engine/src/lightroom_catalog.rs`, `docs/lightroom-catalog-import.md` | read-only `.lrcat` plus committed WAL pages, in-tree read-only Rust SQLite B-tree reader; background inspection/import with progress/cancellation, per-open library identity checks; originals in place, ratings/flags/labels, XMP, hierarchical keywords, collection sets/regular collections, virtual copies, mapped develop settings; stable reimport identities, undo, missing-file placeholders, existing edits preserved. Best-effort deduplicated recovery archives retain settings/history/snapshots/collection rules (32 MiB each, eight archives/128 MiB aggregate); archive limits/write failures and optional-table limits warn without blocking photo import; unrelated unsupported tables and opaque SQL BLOBs skipped. Custom white balance on relative-WB raws (`RAW` in Lightroom's terms) stays As Shot and is reported, since the catalog keeps no as-shot white to shift from (issue #510). Adobe rendering, AI data and native history replay remain incomplete; no writes to Lightroom catalogs | +| LRC-LIB-CATALOG-IMPORT | Native Lightroom Classic catalog import | P1 | 🟡 | `cmd:library.inspectLightroom`, `cmd:library.importLightroom`, `crates/engine/src/lightroom_catalog.rs`, `docs/lightroom-catalog-import.md` | read-only `.lrcat` plus committed WAL pages, in-tree read-only Rust SQLite B-tree reader; on-demand main-file pages up to 16 GiB with existing table/record bounds (WAL still capped at 1 GiB, issue #798); background inspection/import with progress/cancellation, per-open library identity checks; originals in place, ratings/flags/labels, XMP, hierarchical keywords, collection sets/regular collections, virtual copies, mapped develop settings; stable reimport identities, undo, missing-file placeholders, existing edits preserved. Best-effort deduplicated recovery archives retain settings/history/snapshots/collection rules (32 MiB each, eight archives/128 MiB aggregate); archive limits/write failures and optional-table limits warn without blocking photo import; unrelated unsupported tables and opaque SQL BLOBs skipped. Custom white balance on relative-WB raws (`RAW` in Lightroom's terms) stays As Shot and is reported, since the catalog keeps no as-shot white to shift from (issue #510). Adobe rendering, AI data and native history replay remain incomplete; no writes to Lightroom catalogs | | LRC-LIB-CATALOG-LOCK | Catalog open in one program at a time | P1 | ✅ | `cmd:app.openLibrary`, `crates/catalog/src/lock.rs`, `crates/engine/src/library.rs` (`open_library`) | exclusive OS lock on `catalog.lock` for the session (app and `lightcraft-cli --library`); a second opener is refused with who has it (`catalog.lock.owner`: program, process, computer); released by the OS on a crash, so never stale; file systems without locks open unprotected with a warning (the app's notices, `library.info` → `settingsWarnings`, and the CLI's stderr; issue #171); no read-only mode; the browser build (OPFS) has no guard across tabs yet. A library that can't be opened at launch (locked, unreadable, newer format, drive missing) shows a window — Try Again, Choose Another Library…, Continue Without Saving (then a banner; nothing written), Quit — instead of a silent demo session (`crates/ui-egui/src/panels/library_problem.rs`) | | LRC-LIB-FOLDERS | Disk folder tree | P1 | ✅ | `cmd:library.browse`, `cmd:folder.rename`, `cmd:folder.move`, `crates/ui-egui/src/panels/left.rs` (`local_section`, `folder_tree`) | standard places + the browsed folder + Browse Folder…, each an expandable tree of subfolders, breadcrumb navigation; context menu: Rename Folder…, Move Folder To…, Show in Finder (on disk, photos relinked; one undo step each: undo / redo move the folder back and forth, refused when the destination is taken) | | LRC-LIB-LIBFOLDERS | Library folders | P1 | ✅ | `cmd:library.folders`, `cmd:library.source` (`libraryFolder`), `cmd:library.filter` (`libraryFolder`), `cmd:library.removeFolder`, `cmd:folder.rename`, `cmd:folder.move`, `cmd:folder.label`, `cmd:folder.scanChanges`, `cmd:folder.synchronize`, `crates/catalog/src/folders.rs`, `crates/engine/src/sync.rs`, `crates/ui-egui/src/sync.rs`, `crates/ui-egui/src/panels/left.rs` (`folders_section`, `folder_menu_for_library`, `row_menu`), `crates/ui-egui/src/tests_panels.rs` | sidebar Folders: the folders the library's photos were imported from, with photo counts (subfolders included), as another way to explore the library next to By Date and Keywords; Local stays the way to browse any folder on disk. Each disk is a row of its own and starts open: a drive letter, a Windows share (`\\server\share`, `\\?\UNC\…`, `\\wsl$\…`), a macOS share or disk (`/Volumes/tokyo`), a Linux mount (`/media`, `/run/media`, `/mnt`, gvfs `smb-share:…`, `/net/`), NFS (`/Network/Servers/`), else “This Computer” — told apart from the stored paths alone, so an offline share lists like a mounted one and the sidebar never waits on a network. Only folders that hold imported photos are listed (browsed, deleted and demo photos don't count); every folder is a row of its own (`Users` → `me` → `Pictures`), and a folder that holds nothing itself and leads to one folder starts open; every level indents one step more than its parent however deep it goes: the whole sidebar (Albums, Local, By Date, Folders, Keywords) scrolls sideways when its widest row is wider than the panel (rows are measured one frame behind; Create Album and the counts stay at the visible edge); a name that does not fit ends in an ellipsis at the count (the full name on hover; a row asks for room for at most 140 pt of its name, so depth, not one long name, makes the sidebar scroll), as does the grid title; selection and hover bars end at the panel's edge; click makes the folder the source, like an album or a Local folder (its photos and those of the folders inside it fill the grid, titled “photos/travel”; the filter bar narrows it further; `library.source` `libraryFolder` + `path`; kept with the library's view), and its row is always opened into view; the shown folder follows a rename or move and their undo (engine-tested; the sidebar test covers rename), is dropped back to All Photos when it is removed or no longer holds a photo, and a smart album made from the view keeps it, also when its rules are edited (the editor has no folder field). Only a folder that shows exactly its own photos can be shown (not the startup disk, not `/Volumes`-like folders above disks, not an empty one). A row that would cover other disks' photos too (the startup disk, `/Volumes`, `/mnt`…) only opens. Right-click (row or triangle): Rename Folder…, Move Folder To…, Show in Finder (on disk, photos relinked, as in Local), Set Color Label ▸ (the five colours with their custom names, None, Edit Label Names…; `folder.label`, one undo step; the row shows the label's dot before its count and says it with its name; kept in the catalog as a folder record under the folder's identity, so it follows Rename / Move and their undo and redo, the folders inside going along), Synchronize Folder… (the dialog opens at once and scans the folder and the folders inside it on a worker thread — the look for new files beside parallel checks of the library's photos — then offers Import new photos (checked; files only browsed in Local count as new), Relink photos whose file was renamed or moved within the folder (checked; matched by content — a Duplicate's too — or, for a photo with no hash on record such as one from a Lightroom catalog, by file size and dimensions when exactly one file agrees), Remove missing photos from the library (to Recently Deleted) and Read metadata updates from XMP sidecars (saved after the photo came in or was last edited here, and saying something the library doesn't; the sidecar wins), with counts and skipped duplicates / unreadable files; `folder.scanChanges` / `folder.synchronize`, which acts on exactly the scan shown — a scan whose folder's photos changed meanwhile is refused and made again in the background — and is one undo step, also with a full history; a folder that isn't there (an unplugged disk) is reported as such, not as every photo missing; a whole disk or a folder holding disks needs `disk: true`; Esc stops the scan; Synchronize hands everything to a worker thread — the import, the checks that moved and missing files are still as found, the sidecar reads — and commits its steps between frames, shown in the activity stack with ✕ to stop, so no file is touched on the UI thread) and Remove from Library… (asks; moves the folder's photos to Recently Deleted, one undo step, no file touched); a disk row (not the startup disk) offers Set Color Label ▸, Synchronize Folder… and Remove Disk from Library… (the command needs `disk: true` for a whole disk or a folder spanning several). Limits: names are compared as written (macOS `/Volumes/tokyo` vs `/Volumes/TOKYO` are two disks; `/Volumes/Macintosh HD/…` and `/…` two rows); no disk capacity / online status; no “include subfolders” switch (a folder always includes the ones inside it); creating a folder and moving photos between folders are not here; folders can't be filtered or sorted by label, and a label stays with the old path when the folder is moved outside LightCraft (Find Missing relinks photos, not folder records); Synchronize has no “show import dialog first” or “show missing photos” option, and only knows when a photo came in or was last developed here: a rating, label or metadata change made here (auto-write off) after another app saved the sidecar is listed as an update, and reading it restores the sidecar's older value (reading is never the default); the walk for new files isn't cancelled midway and its depth/entry limits are only logged |