diff --git a/Cargo.lock b/Cargo.lock index 75fef728..d8f36eab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2008,7 +2008,9 @@ dependencies = [ name = "lightcraft-catalog" version = "0.1.0" dependencies = [ + "crc32fast", "lightcraft-develop", + "log", "proptest", "serde", "serde_json", @@ -2083,6 +2085,7 @@ dependencies = [ "lightcraft-geom", "lightcraft-meta", "lightcraft-pipeline", + "lightcraft-preview", "lightcraft-raster", "lightcraft-raw", "lightcraft-scenes", @@ -2138,6 +2141,18 @@ dependencies = [ "serde", ] +[[package]] +name = "lightcraft-preview" +version = "0.1.0" +dependencies = [ + "jpeg-encoder", + "lightcraft-raster", + "log", + "siphasher", + "zune-core", + "zune-jpeg", +] + [[package]] name = "lightcraft-raster" version = "0.1.0" @@ -2196,6 +2211,7 @@ dependencies = [ "lightcraft-engine", "lightcraft-geom", "lightcraft-pipeline", + "lightcraft-preview", "lightcraft-raster", "log", "serde", diff --git a/Cargo.toml b/Cargo.toml index 66bcbb60..86780d7a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ lightcraft-develop = { path = "crates/develop" } lightcraft-scenes = { path = "crates/scenes" } lightcraft-pipeline = { path = "crates/pipeline" } lightcraft-catalog = { path = "crates/catalog" } +lightcraft-preview = { path = "crates/preview" } lightcraft-engine = { path = "crates/engine" } lightcraft-ui-egui = { path = "crates/ui-egui" } lightcraft-mcp = { path = "crates/mcp" } @@ -34,6 +35,8 @@ log = "0.4" rayon = "1" proptest = "1" bytemuck = { version = "1", features = ["derive"] } +crc32fast = "1" +siphasher = "1" # formats quick-xml = "0.38" diff --git a/README.md b/README.md index 9fdb5c17..1cf51747 100644 --- a/README.md +++ b/README.md @@ -235,8 +235,9 @@ LightCraft is young and moving fast — see the [roadmap](ROADMAP.md) for estima ```sh git clone https://github.com/storytold/lightcraft && cd lightcraft -cargo run --release -p lightcraft # opens with a procedurally generated demo library -cargo run --release -p lightcraft -- ~/Pictures/trip # or point it at your photos (folders are scanned) +cargo run --release -p lightcraft # opens your library (~/Pictures/LightCraft Library; a new one starts with demo photos) +cargo run --release -p lightcraft -- ~/Pictures/trip # import your photos (folders are scanned, duplicates skipped) +cargo run --release -p lightcraft -- --memory # a throwaway in-memory demo session (writes nothing) cargo run --release -p lightcraft -- --control 7980 # with the automation channel cargo xtask web --serve # the same app in the browser: http://127.0.0.1:8080/ cargo run --release -p lightcraft-cli -- render photo.jpg -o out.jpg --set light.exposure=0.5 diff --git a/apps/lightcraft/examples/gen_photos.rs b/apps/lightcraft/examples/gen_photos.rs new file mode 100644 index 00000000..ee7ba9a5 --- /dev/null +++ b/apps/lightcraft/examples/gen_photos.rs @@ -0,0 +1,45 @@ +//! Generate a folder of procedural test photos (JPEG) for import/startup benchmarks. +//! +//! `cargo run --release -p lightcraft --example gen_photos -- OUT_DIR [COUNT=200] [LONG_EDGE=2000]` +//! +//! Each photo is a `lightcraft-scenes` demo scene rendered through the pipeline with a varied +//! look (exposure, temperature, vibrance), so every file has distinct bytes. The output is our own +//! procedural content; keep it out of the repository (write to a scratch or gitignored folder). + +use lightcraft_engine::Session; +use serde_json::json; + +fn main() { + let mut args = std::env::args().skip(1); + let out = std::path::PathBuf::from(args.next().expect("usage: gen_photos OUT_DIR [COUNT] [LONG_EDGE]")); + let count: usize = args.next().and_then(|c| c.parse().ok()).unwrap_or(200); + let edge: usize = args.next().and_then(|c| c.parse().ok()).unwrap_or(2000); + std::fs::create_dir_all(&out).expect("create output folder"); + let mut s = Session::with_demo(); + let ids = s.visible_cloned(); + let t0 = std::time::Instant::now(); + for i in 0..count { + let id = ids[i % ids.len()]; + let round = i / ids.len(); + s.execute("library.select", &json!({"ids": [id.0]})).expect("select"); + s.execute("develop.reset", &json!({})).expect("reset"); + let v = json!({ + "light.exposure": (round as f64 * 0.37).sin() * 0.8, + "wb.temp": 6500.0 + (round as f64 * 1.3).cos() * 1500.0, + "color.vibrance": (round * 13 % 50) as f64, + }); + let _ = s.execute("develop.set", &json!({"values": v})); + let img = s.render_now(id, edge, edge).expect("render"); + let bytes = lightcraft_codecs::encode_jpeg( + &lightcraft_codecs::EncodeImage::rgba8(&img.image), + 88, + lightcraft_codecs::ChromaSubsampling::S420, + &lightcraft_codecs::EncodeMeta::default(), + ) + .expect("encode"); + let dir = out.join(format!("roll-{:02}", i / 50 + 1)); + std::fs::create_dir_all(&dir).expect("mkdir"); + std::fs::write(dir.join(format!("LCG{:05}.jpg", i + 1)), bytes).expect("write"); + } + eprintln!("wrote {count} photos ({edge} px) to {} in {:.1} s", out.display(), t0.elapsed().as_secs_f64()); +} diff --git a/apps/lightcraft/src/main.rs b/apps/lightcraft/src/main.rs index 6fbd9339..5b39ee4e 100644 --- a/apps/lightcraft/src/main.rs +++ b/apps/lightcraft/src/main.rs @@ -1,6 +1,12 @@ //! LightCraft desktop app. //! -//! Usage: `lightcraft [--control ] [--demo] [files or folders…]` +//! Usage: `lightcraft [--library DIR | --memory] [--no-demo] [--control ] [files or folders…]` +//! +//! The library (catalog, presets, thumbnail cache) lives in `--library DIR`, else +//! `$LIGHTCRAFT_LIBRARY`, else `~/Pictures/LightCraft Library`; a new library starts with the +//! procedural demo photos unless `--no-demo` or files are given. Files and folders on the command +//! line are imported (duplicates are skipped). `--memory` runs an in-memory session that writes +//! nothing (demo photos unless files are given; used by the README showcase scripts). //! //! `--control ` (or `LIGHTCRAFT_CONTROL_PORT`) starts a localhost JSON-lines control server: //! `{"id":1,"method":"ui.inspect","params":{}}` → `{"id":1,"ok":true,"result":…}`. @@ -25,6 +31,9 @@ impl eframe::App for App { } fn on_exit(&mut self) { save_prefs(&self.0); + if let Err(e) = self.0.session.close_library() { + eprintln!("lightcraft: saving the library failed: {e}"); + } } } @@ -88,40 +97,55 @@ fn services() -> Services { } } -/// Expand folders (recursively) into photo files. -fn collect(path: &std::path::Path, out: &mut Vec) { - if path.is_dir() { - if let Ok(rd) = std::fs::read_dir(path) { - let mut v: Vec<_> = rd.flatten().map(|e| e.path()).collect(); - v.sort(); - for p in v { - collect(&p, out); - } +/// The persistent library session (or an in-memory one with `--memory` / if the library can't open). +fn open_session(in_memory: bool, dir: Option, seed_demo: bool) -> Session { + let fallback = || if seed_demo { Session::with_demo() } else { Session::new() }.with_fs().with_system_clock(); + if in_memory { + return fallback(); + } + let Some(dir) = dir else { + eprintln!("lightcraft: no library location (set --library or LIGHTCRAFT_LIBRARY); running in memory"); + return fallback(); + }; + let t0 = std::time::Instant::now(); + let mut s = Session::new().with_fs().with_system_clock(); + match s.open_library(&dir, seed_demo) { + Ok(r) => { + let (replayed, torn) = (r.replayed, r.torn_bytes); + eprintln!( + "lightcraft: library {}: {} photos, {replayed} log records replayed{} ({:.1} ms)", + dir.display(), + s.catalog.len(), + if torn > 0 { ", torn tail repaired" } else { "" }, + t0.elapsed().as_secs_f64() * 1000.0 + ); + s + } + Err(e) => { + eprintln!("lightcraft: can't open library {}: {e}; running in memory", dir.display()); + fallback() } - } else if path.extension().is_some_and(|e| { - matches!( - e.to_string_lossy().to_lowercase().as_str(), - "jpg" | "jpeg" | "png" | "tif" | "tiff" | "webp" | "dng" | "cr2" | "nef" | "arw" | "psd" | "jxl" | "gif" | "bmp" - ) - }) { - out.push(path.to_string_lossy().to_string()); } } fn main() -> eframe::Result { let mut control_port: Option = std::env::var("LIGHTCRAFT_CONTROL_PORT").ok().and_then(|p| p.parse().ok()); let mut files = Vec::new(); - let mut demo = true; + let mut seed_demo = true; + let mut in_memory = false; + let mut library_dir = lightcraft_engine::library::default_dir(); let mut args = std::env::args().skip(1); while let Some(a) = args.next() { match a.as_str() { "--control" => control_port = args.next().and_then(|p| p.parse().ok()), - "--no-demo" => demo = false, + "--library" => library_dir = args.next().map(std::path::PathBuf::from), + "--no-demo" => seed_demo = false, + "--memory" | "--demo" => in_memory = true, "--version" => { println!("lightcraft {}", env!("CARGO_PKG_VERSION")); return Ok(()); } - _ => collect(std::path::Path::new(&a), &mut files), + _ => files.push(a), } } let options = eframe::NativeOptions { @@ -139,7 +163,7 @@ fn main() -> eframe::Result { "LightCraft", options, Box::new(move |cc| { - let session = if demo && files.is_empty() { Session::with_demo() } else { Session::new() }.with_fs(); + let session = open_session(in_memory, library_dir, seed_demo && files.is_empty()); let mut app = LightcraftApp::new(session, services()); load_prefs(&mut app); app.integrated_titlebar = cfg!(target_os = "macos"); diff --git a/crates/catalog/Cargo.toml b/crates/catalog/Cargo.toml index c5f6dc86..fb772703 100644 --- a/crates/catalog/Cargo.toml +++ b/crates/catalog/Cargo.toml @@ -13,6 +13,8 @@ lightcraft-develop = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } +crc32fast = { workspace = true } +log = { workspace = true } [dev-dependencies] proptest = { workspace = true } diff --git a/crates/catalog/src/journal.rs b/crates/catalog/src/journal.rs new file mode 100644 index 00000000..f7089c06 --- /dev/null +++ b/crates/catalog/src/journal.rs @@ -0,0 +1,240 @@ +//! Crash-safe catalog persistence: an append-only op log plus periodic snapshots. +//! +//! Files (in a [`Store`]): +//! - `catalog.snap` — `{"format":"lightcraft-catalog","version":1,"seq":N,"catalog":{…}}`, the +//! state after op `N`; replaced atomically (temp + fsync + rename). +//! - `catalog.log` — JSON lines, one per op applied after the snapshot: +//! `{"seq":N,"crc":C,"op":{…}}` where `C` is the CRC-32 of the op's JSON text. Appends are +//! fsynced before [`Journal::append`] returns. +//! +//! Loading = snapshot + replay of the records with `seq > snapshot seq`. Recovery rules: +//! - a **torn final record** (crash mid-append: truncated line or bad CRC at the end) is dropped +//! and the file is cut back to the last good record, so later appends start on a clean line; +//! - records already covered by the snapshot (crash between writing the snapshot and resetting +//! the log) are skipped; +//! - a bad record **followed by good ones** is real damage: replay stops there, the log is kept +//! as `catalog.log.damaged-`, and a fresh snapshot is written so the library stays usable. + +use crate::store::Store; +use crate::{Catalog, CatalogError, Op, Result}; + +pub const SNAPSHOT: &str = "catalog.snap"; +pub const LOG: &str = "catalog.log"; +const FORMAT: &str = "lightcraft-catalog"; +const VERSION: u32 = 1; + +/// When [`Journal::wants_snapshot`] says it's time to compact the log. +#[derive(Clone, Copy, Debug)] +pub struct SnapshotPolicy { + pub max_records: u64, + pub max_bytes: u64, +} + +impl Default for SnapshotPolicy { + fn default() -> Self { + SnapshotPolicy { max_records: 2000, max_bytes: 16 << 20 } + } +} + +/// What happened while loading. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct LoadReport { + /// `seq` of the snapshot that was loaded (0 = none). + pub snapshot_seq: u64, + /// Log records applied on top of the snapshot. + pub replayed: usize, + /// Records skipped because the snapshot already contained them. + pub stale: usize, + /// Records whose op failed to apply (should never happen; reported, not fatal). + pub failed: usize, + /// Bytes of a torn final record that were dropped. + pub torn_bytes: u64, + /// The damaged log was preserved under this name. + pub damaged: Option, + /// No catalog files existed (a new library). + pub created: bool, +} + +pub struct Journal { + store: Box, + /// `seq` of the last durable op. + seq: u64, + snapshot_seq: u64, + log_records: u64, + log_bytes: u64, + pub policy: SnapshotPolicy, +} + +fn io(e: std::io::Error) -> CatalogError { + CatalogError::Io(e.to_string()) +} + +/// One log line (without the newline) for op number `seq`. +pub fn encode_record(seq: u64, op: &Op) -> String { + let body = serde_json::to_string(op).unwrap_or_default(); + let crc = crc32fast::hash(body.as_bytes()); + format!("{{\"seq\":{seq},\"crc\":{crc},\"op\":{body}}}") +} + +/// Parse one log line (without the newline). `None` if it's malformed or the CRC doesn't match. +pub fn decode_record(line: &str) -> Option<(u64, Op)> { + let rest = line.strip_prefix("{\"seq\":")?; + let (seq, rest) = rest.split_once(",\"crc\":")?; + let (crc, rest) = rest.split_once(",\"op\":")?; + let body = rest.strip_suffix('}')?; + let seq: u64 = seq.parse().ok()?; + let crc: u32 = crc.parse().ok()?; + if crc32fast::hash(body.as_bytes()) != crc { + return None; + } + serde_json::from_str(body).ok().map(|op| (seq, op)) +} + +#[derive(serde::Deserialize)] +struct SnapFile { + format: String, + version: u32, + seq: u64, + catalog: Catalog, +} + +impl Journal { + /// Open (or create) the catalog in `store`: load the snapshot, replay the log, repair a torn + /// tail. Fails only if the snapshot itself is unreadable (then nothing is modified). + pub fn open(mut store: Box) -> Result<(Journal, Catalog, LoadReport)> { + let mut report = LoadReport::default(); + let snap = store.read(SNAPSHOT).map_err(io)?; + let log = store.read(LOG).map_err(io)?; + report.created = snap.is_none() && log.is_none(); + let (mut catalog, snapshot_seq) = match snap { + Some(bytes) => { + let s: SnapFile = serde_json::from_slice(&bytes).map_err(|e| CatalogError::Corrupt(format!("{SNAPSHOT}: {e}")))?; + if s.format != FORMAT || s.version > VERSION { + return Err(CatalogError::Corrupt(format!("{SNAPSHOT}: unsupported format {} v{}", s.format, s.version))); + } + (s.catalog, s.seq) + } + None => (Catalog::new(), 0), + }; + report.snapshot_seq = snapshot_seq; + let mut j = Journal { store, seq: snapshot_seq, snapshot_seq, log_records: 0, log_bytes: 0, policy: SnapshotPolicy::default() }; + let log = log.unwrap_or_default(); + + // Scan records; `good_end` is the byte offset just past the last good record. + let mut pos = 0usize; + let mut good_end = 0usize; + let mut damaged_at: Option = None; + let mut needs_newline = false; + while pos < log.len() { + let (line_end, next, has_nl) = match log[pos..].iter().position(|b| *b == b'\n') { + Some(i) => (pos + i, pos + i + 1, true), + None => (log.len(), log.len(), false), + }; + let line = std::str::from_utf8(&log[pos..line_end]).ok().map(|l| l.trim_end_matches('\r')); + if line.is_some_and(|l| l.trim().is_empty()) { + pos = next; + good_end = next; + continue; + } + match line.and_then(decode_record) { + Some((seq, op)) if seq <= j.seq => { + // already in the snapshot + let _ = op; + report.stale += 1; + } + Some((seq, op)) if seq == j.seq + 1 => { + if catalog.apply(op).is_err() { + report.failed += 1; + } else { + report.replayed += 1; + } + j.seq = seq; + } + _ => { + // bad record (or a gap): torn tail if nothing good follows, else damage + let rest_has_good = log[next..].split(|b| *b == b'\n').any(|l| std::str::from_utf8(l).ok().and_then(decode_record).is_some()); + if rest_has_good { + damaged_at = Some(pos); + } + break; + } + } + j.log_records += 1; + pos = next; + good_end = next; + needs_newline = !has_nl; + } + + if let Some(at) = damaged_at { + let name = format!("{LOG}.damaged-{}", j.seq); + j.store.write_atomic(&name, &log).map_err(io)?; + log::warn!("catalog log damaged at byte {at}; kept as {name}; state recovered up to op {}", j.seq); + report.damaged = Some(name); + j.snapshot(&catalog)?; + } else { + if good_end < log.len() { + report.torn_bytes = (log.len() - good_end) as u64; + log::warn!("catalog log: dropped a torn final record ({} bytes)", report.torn_bytes); + j.store.truncate(LOG, good_end as u64).map_err(io)?; + } else if needs_newline { + j.store.append(LOG, b"\n").map_err(io)?; + } + j.log_bytes = good_end as u64; + } + catalog.revision = 0; + Ok((j, catalog, report)) + } + + /// Append ops (already applied to the live catalog) durably, in order. + pub fn append(&mut self, ops: &[Op]) -> Result<()> { + if ops.is_empty() { + return Ok(()); + } + let mut buf = String::new(); + let mut seq = self.seq; + for op in ops { + seq += 1; + buf.push_str(&encode_record(seq, op)); + buf.push('\n'); + } + self.store.append(LOG, buf.as_bytes()).map_err(io)?; + self.seq = seq; + self.log_records += ops.len() as u64; + self.log_bytes += buf.len() as u64; + Ok(()) + } + + /// Write a snapshot of `catalog` (which must reflect every appended op) and reset the log. + pub fn snapshot(&mut self, catalog: &Catalog) -> Result<()> { + let body = serde_json::to_string(catalog).map_err(|e| CatalogError::Invalid(e.to_string()))?; + let file = format!("{{\"format\":\"{FORMAT}\",\"version\":{VERSION},\"seq\":{},\"catalog\":{body}}}\n", self.seq); + self.store.write_atomic(SNAPSHOT, file.as_bytes()).map_err(io)?; + // A crash here leaves old records in the log; they are skipped by seq on load. + self.store.write_atomic(LOG, b"").map_err(io)?; + self.snapshot_seq = self.seq; + self.log_records = 0; + self.log_bytes = 0; + Ok(()) + } + + /// The log is long enough to be worth compacting. + pub fn wants_snapshot(&self) -> bool { + self.log_records >= self.policy.max_records || self.log_bytes >= self.policy.max_bytes + } + + pub fn seq(&self) -> u64 { + self.seq + } + pub fn snapshot_seq(&self) -> u64 { + self.snapshot_seq + } + pub fn log_records(&self) -> u64 { + self.log_records + } + pub fn log_bytes(&self) -> u64 { + self.log_bytes + } + pub fn describe(&self) -> String { + self.store.describe() + } +} diff --git a/crates/catalog/src/lib.rs b/crates/catalog/src/lib.rs index 1381edbc..4b03795c 100644 --- a/crates/catalog/src/lib.rs +++ b/crates/catalog/src/lib.rs @@ -2,21 +2,25 @@ //! //! State changes only through [`Op`]s. [`Catalog::apply`] returns the inverse op, which gives: //! - **persistence**: ops are appended to a log (JSON lines) and replayed on load after the last -//! snapshot — crash-safe and diff-friendly; +//! snapshot — crash-safe and diff-friendly (see [`journal`]); //! - **undo/redo**: the engine keeps inverse ops; //! - **determinism**: replaying the log reproduces the state exactly (property-tested). #![forbid(unsafe_code)] +pub mod journal; pub mod model; pub mod query; +pub mod store; use std::collections::BTreeMap; use std::sync::Arc; +pub use journal::{Journal, LoadReport, SnapshotPolicy}; use lightcraft_develop::DevelopSettings; pub use model::*; pub use query::{DateGroup, Filter, RatingOp, Sort, SortKey}; use serde::{Deserialize, Serialize}; +pub use store::{FsStore, MemStore, Store}; #[derive(Debug, thiserror::Error, PartialEq)] pub enum CatalogError { @@ -28,10 +32,15 @@ pub enum CatalogError { Invalid(String), #[error("corrupt catalog data: {0}")] Corrupt(String), + #[error("catalog storage: {0}")] + Io(String), } pub type Result = std::result::Result; +/// Maximum History entries kept per photo. +pub const HISTORY_LIMIT: usize = 200; + /// Every catalog mutation. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(tag = "op", rename_all = "camelCase")] @@ -76,6 +85,12 @@ pub enum Op { id: PhotoId, history: Vec, }, + /// Append one History entry (dropping the oldest beyond [`HISTORY_LIMIT`]). Logged instead of + /// a full `SetHistory` so each edit costs one step in the op log. + PushHistory { + id: PhotoId, + step: HistoryStep, + }, AddAlbum { album: Album, }, @@ -227,6 +242,16 @@ impl Catalog { let p = self.photo_mut(id)?; Op::SetHistory { id, history: std::mem::replace(&mut p.history, history) } } + Op::PushHistory { id, step } => { + let p = self.photo_mut(id)?; + let old = p.history.clone(); + p.history.push(step); + if p.history.len() > HISTORY_LIMIT { + let n = p.history.len() - HISTORY_LIMIT; + p.history.drain(..n); + } + Op::SetHistory { id, history: old } + } Op::AddAlbum { album } => { if self.albums.contains_key(&album.id) { return Err(CatalogError::Invalid(format!("album {:?} exists", album.id))); @@ -352,3 +377,5 @@ impl Catalog { #[cfg(test)] mod tests; +#[cfg(test)] +mod tests_journal; diff --git a/crates/catalog/src/model.rs b/crates/catalog/src/model.rs index b3c0d4fa..4a2d6a07 100644 --- a/crates/catalog/src/model.rs +++ b/crates/catalog/src/model.rs @@ -143,6 +143,9 @@ pub struct Photo { /// Raw files: the camera's as-shot white balance (Kelvin, tint). #[serde(default)] pub as_shot_wb: Option<(f64, f64)>, + /// Hash of the file's bytes (hex), for duplicate detection and preview-cache keys. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content_hash: Option, } impl Photo { @@ -169,6 +172,7 @@ impl Photo { deleted: false, duration: None, as_shot_wb: None, + content_hash: None, } } pub fn is_edited(&self) -> bool { diff --git a/crates/catalog/src/store.rs b/crates/catalog/src/store.rs new file mode 100644 index 00000000..98e690b9 --- /dev/null +++ b/crates/catalog/src/store.rs @@ -0,0 +1,148 @@ +//! Byte storage for the catalog files (`catalog.snap`, `catalog.log`). +//! +//! [`FsStore`] is the native implementation (a library directory); [`MemStore`] keeps files in +//! memory (tests, ephemeral sessions) and lets tests simulate crashes by editing the bytes. A web +//! host can implement [`Store`] over OPFS. + +use std::collections::BTreeMap; +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +/// A flat namespace of named files with the few operations the journal needs. +pub trait Store: Send { + /// The whole file, or `None` if it doesn't exist. + fn read(&mut self, name: &str) -> io::Result>>; + /// Replace a file atomically: after a crash either the old or the new content is visible, + /// never a mix (native: write temp + fsync + rename + fsync directory). + fn write_atomic(&mut self, name: &str, data: &[u8]) -> io::Result<()>; + /// Append and make durable (fsync) before returning. + fn append(&mut self, name: &str, data: &[u8]) -> io::Result<()>; + /// Cut a file to `len` bytes (drop a torn tail before appending again). + fn truncate(&mut self, name: &str, len: u64) -> io::Result<()>; + /// Human-readable location (diagnostics). + fn describe(&self) -> String; +} + +/// Files in a directory. +pub struct FsStore { + dir: PathBuf, + /// Open append handle (kept between appends; dropped on rewrite/truncate). + appender: Option<(String, std::fs::File)>, +} + +impl FsStore { + /// Use `dir` (created if missing). + pub fn open(dir: impl AsRef) -> io::Result { + std::fs::create_dir_all(dir.as_ref())?; + Ok(FsStore { dir: dir.as_ref().to_path_buf(), appender: None }) + } + pub fn dir(&self) -> &Path { + &self.dir + } + fn path(&self, name: &str) -> PathBuf { + self.dir.join(name) + } + fn sync_dir(&self) { + // Makes the rename durable on POSIX; opening a directory fails on Windows (where + // `rename` is already journaled by NTFS), so errors are ignored. + if let Ok(d) = std::fs::File::open(&self.dir) { + let _ = d.sync_all(); + } + } +} + +impl Store for FsStore { + fn read(&mut self, name: &str) -> io::Result>> { + match std::fs::read(self.path(name)) { + Ok(b) => Ok(Some(b)), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e), + } + } + + fn write_atomic(&mut self, name: &str, data: &[u8]) -> io::Result<()> { + use std::io::Write; + if self.appender.as_ref().is_some_and(|(n, _)| n == name) { + self.appender = None; + } + let tmp = self.path(&format!("{name}.tmp")); + { + let mut f = std::fs::File::create(&tmp)?; + f.write_all(data)?; + f.sync_all()?; + } + std::fs::rename(&tmp, self.path(name))?; + self.sync_dir(); + Ok(()) + } + + fn append(&mut self, name: &str, data: &[u8]) -> io::Result<()> { + use std::io::Write; + if self.appender.as_ref().is_none_or(|(n, _)| n != name) { + let existed = self.path(name).exists(); + let f = std::fs::OpenOptions::new().create(true).append(true).open(self.path(name))?; + if !existed { + self.sync_dir(); + } + self.appender = Some((name.to_string(), f)); + } + let (_, f) = self.appender.as_mut().expect("appender just opened"); + f.write_all(data)?; + f.sync_data() + } + + fn truncate(&mut self, name: &str, len: u64) -> io::Result<()> { + if self.appender.as_ref().is_some_and(|(n, _)| n == name) { + self.appender = None; + } + let f = std::fs::OpenOptions::new().write(true).open(self.path(name))?; + f.set_len(len)?; + f.sync_all() + } + + fn describe(&self) -> String { + self.dir.display().to_string() + } +} + +/// In-memory files. Clones share the same files (so a test can "reopen" or corrupt them). +#[derive(Clone, Default)] +pub struct MemStore { + pub files: Arc>>>, +} + +impl MemStore { + pub fn new() -> MemStore { + MemStore::default() + } + pub fn get(&self, name: &str) -> Option> { + self.files.lock().unwrap_or_else(|e| e.into_inner()).get(name).cloned() + } + pub fn set(&self, name: &str, data: Vec) { + self.files.lock().unwrap_or_else(|e| e.into_inner()).insert(name.to_string(), data); + } +} + +impl Store for MemStore { + fn read(&mut self, name: &str) -> io::Result>> { + Ok(self.get(name)) + } + fn write_atomic(&mut self, name: &str, data: &[u8]) -> io::Result<()> { + self.set(name, data.to_vec()); + Ok(()) + } + fn append(&mut self, name: &str, data: &[u8]) -> io::Result<()> { + self.files.lock().unwrap_or_else(|e| e.into_inner()).entry(name.to_string()).or_default().extend_from_slice(data); + Ok(()) + } + fn truncate(&mut self, name: &str, len: u64) -> io::Result<()> { + if let Some(f) = self.files.lock().unwrap_or_else(|e| e.into_inner()).get_mut(name) { + f.truncate(len as usize); + } + Ok(()) + } + fn describe(&self) -> String { + "memory".into() + } +} diff --git a/crates/catalog/src/tests_journal.rs b/crates/catalog/src/tests_journal.rs new file mode 100644 index 00000000..2cbe8098 --- /dev/null +++ b/crates/catalog/src/tests_journal.rs @@ -0,0 +1,230 @@ +//! Journal (op log + snapshot) tests: replay(log) == state, torn tails, stale logs, damage. + +use std::sync::Arc; + +use lightcraft_develop::DevelopSettings; +use proptest::prelude::*; + +use crate::journal::{LOG, SNAPSHOT, decode_record, encode_record}; +use crate::*; + +fn open(m: &MemStore) -> (Journal, Catalog, LoadReport) { + Journal::open(Box::new(m.clone())).unwrap() +} + +/// Interpret a random action against the live catalog as an op (may be invalid: then the live +/// apply fails and, like in the engine, nothing is logged). +fn op_for(c: &mut Catalog, kind: u8, a: u8, b: u8) -> Op { + let photos: Vec = c.photos().map(|p| p.id).collect(); + let albums: Vec = c.albums().map(|a| a.id).collect(); + let pid = photos.get(a as usize % photos.len().max(1)).copied().unwrap_or(PhotoId(99)); + let aid = albums.get(a as usize % albums.len().max(1)).copied().unwrap_or(AlbumId(99)); + match kind % 10 { + 0 | 1 => { + let id = c.alloc_photo_id(); + let mut p = Photo::new(id, Source::File { path: format!("/p/{}.jpg", id.0) }, &format!("{}.jpg", id.0), "JPEG", 60, 40, "2026-01-01"); + p.content_hash = Some(format!("{:032x}", id.0 * 7919)); + Op::AddPhoto { photo: Box::new(p) } + } + 2 => Op::SetRating { id: pid, rating: b % 7 }, + 3 => Op::SetFlag { id: pid, flag: [Flag::None, Flag::Pick, Flag::Reject][b as usize % 3] }, + 4 => { + let mut d = DevelopSettings::default(); + d.light.exposure = b as f64 / 37.0 - 2.0; + d.light.contrast = -(a as f64) * 0.37; + let settings = Arc::new(d); + Op::Batch { + ops: vec![ + Op::SetDevelop { id: pid, settings: settings.clone(), label: "Exposure".into(), edited: Some(format!("t{b}")) }, + Op::PushHistory { id: pid, step: HistoryStep { label: "Exposure".into(), settings } }, + ], + } + } + 5 => { + let id = c.alloc_album_id(); + Op::AddAlbum { album: Album { id, name: format!("A{b}"), parent: None, folder: b.is_multiple_of(4), photos: vec![], cover: None } } + } + 6 => Op::SetAlbumPhotos { id: aid, photos: photos.iter().copied().filter(|p| (p.0 + b as u64).is_multiple_of(3)).collect() }, + 7 => Op::RenameAlbum { id: aid, name: format!("Renamed {b} ✓") }, + 8 => Op::SetDeleted { id: pid, deleted: b.is_multiple_of(2) }, + _ => c.delete_permanently_ops(pid), + } +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(64))] + + /// replay(snapshot + log) == live state, with snapshots at random points and reopening + /// (resuming appends) in between. + #[test] + fn replay_equals_state(actions in proptest::collection::vec((0u8..20, 0u8..255, 0u8..255, 0u8..12), 1..80)) { + let m = MemStore::new(); + let (mut j, mut live, r) = open(&m); + prop_assert!(r.created); + for (kind, a, b, ctl) in actions { + let op = op_for(&mut live, kind, a, b); + if live.apply(op.clone()).is_ok() { + j.append(&[op]).unwrap(); + } + match ctl { + 0 => j.snapshot(&live).unwrap(), + 1 => { + let (j2, c2, _) = open(&m); + prop_assert_eq!(c2.to_snapshot(), live.to_snapshot()); + j = j2; + } + _ => {} + } + } + let (_, loaded, r) = open(&m); + prop_assert_eq!(r.failed, 0); + prop_assert_eq!(loaded.to_snapshot(), live.to_snapshot()); + // allocation continues where it left off + prop_assert_eq!(loaded.clone().alloc_photo_id(), live.clone().alloc_photo_id()); + } +} + +fn sample_catalog_with_log() -> (MemStore, Catalog, Catalog) { + let m = MemStore::new(); + let (mut j, mut c, _) = open(&m); + for k in 0..6u8 { + let op = op_for(&mut c, k, k, k * 3); + if c.apply(op.clone()).is_ok() { + j.append(&[op]).unwrap(); + } + } + let before_last = c.clone(); + let op = Op::SetRating { id: c.photos().next().unwrap().id, rating: 5 }; + c.apply(op.clone()).unwrap(); + j.append(&[op]).unwrap(); + (m, before_last, c) +} + +/// A crash mid-append: every possible truncation of the last record still loads (to the state +/// before that op), the torn bytes are cut off, and appending afterwards works. +#[test] +fn torn_last_record_is_tolerated() { + let (m, before_last, full) = sample_catalog_with_log(); + let log = m.get(LOG).unwrap(); + let last_start = log[..log.len() - 1].iter().rposition(|b| *b == b'\n').unwrap() + 1; + for cut in last_start..log.len() { + let t = MemStore::new(); + t.set(LOG, log[..cut].to_vec()); + let (mut j, mut c, r) = open(&t); + // the full record minus only its newline is complete and kept + let expect = if cut == log.len() - 1 { &full } else { &before_last }; + assert_eq!(c.to_snapshot(), expect.to_snapshot(), "cut at {cut}"); + assert!(r.damaged.is_none()); + if cut < log.len() - 1 { + assert_eq!(r.torn_bytes as usize, cut - last_start, "cut at {cut}"); + } + // keeps working after recovery + let id = c.photos().next().unwrap().id; + let op = Op::SetFlag { id, flag: Flag::Reject }; + c.apply(op.clone()).unwrap(); + j.append(&[op]).unwrap(); + let (_, c2, r2) = open(&t); + assert_eq!(c2.to_snapshot(), c.to_snapshot(), "cut at {cut}"); + assert_eq!(r2.torn_bytes, 0); + } + // garbage bytes (a partially flushed page) at the end are a torn tail too + let t = MemStore::new(); + let mut g = log.clone(); + g.extend_from_slice(&[0, 0, 0, b'{', 0xff]); + t.set(LOG, g); + let (_, c, r) = open(&t); + assert_eq!(c.to_snapshot(), full.to_snapshot()); + assert_eq!(r.torn_bytes, 5); +} + +/// Crash after the snapshot was written but before the log was reset: the stale records are +/// skipped by sequence number. +#[test] +fn stale_log_after_snapshot_is_skipped() { + let (m, _, full) = sample_catalog_with_log(); + let old_log = m.get(LOG).unwrap(); + let (mut j, c, _) = open(&m); + j.snapshot(&c).unwrap(); + assert!(m.get(LOG).unwrap().is_empty()); + m.set(LOG, old_log); + let (_, c2, r) = open(&m); + assert_eq!(c2.to_snapshot(), full.to_snapshot()); + assert!(r.stale > 0 && r.replayed == 0); +} + +/// Damage in the middle of the log (not a torn tail): recover what precedes it, keep the +/// damaged file, and write a consistent snapshot. +#[test] +fn mid_log_damage_is_preserved_and_recovered() { + let (m, _, _) = sample_catalog_with_log(); + let log = m.get(LOG).unwrap(); + let text = String::from_utf8(log).unwrap(); + let mut lines: Vec = text.lines().map(str::to_string).collect(); + assert!(lines.len() > 4); + lines[2] = lines[2].replacen("\"op\":{\"op\":\"", "\"op\":{\"op\":\"x", 1); + let bad = format!("{}\n", lines.join("\n")).into_bytes(); + m.set(LOG, bad.clone()); + let (_, c, r) = open(&m); + let name = r.damaged.clone().expect("damage reported"); + assert_eq!(m.get(&name).unwrap(), bad); + assert_eq!(r.replayed, 2); + // the recovered state was snapshotted: reopening is clean and identical + let (_, c2, r2) = open(&m); + assert_eq!(c2.to_snapshot(), c.to_snapshot()); + assert!(r2.damaged.is_none()); +} + +#[test] +fn record_codec() { + let op = Op::RenameAlbum { id: AlbumId(3), name: "Line\nbreak \"quoted\" }".into() }; + let line = encode_record(42, &op); + assert!(!line.contains('\n')); + assert_eq!(decode_record(&line), Some((42, op))); + assert_eq!(decode_record(&line.replace("Line", "Lime")), None, "crc mismatch"); + assert_eq!(decode_record(&line[..line.len() - 1]), None); + // log lines stay plain JSON + let v: serde_json::Value = serde_json::from_str(&line).unwrap(); + assert_eq!(v["op"]["op"], "renameAlbum"); +} + +#[test] +fn push_history_is_bounded_and_invertible() { + let mut c = Catalog::new(); + let id = c.alloc_photo_id(); + c.apply(Op::AddPhoto { photo: Box::new(Photo::new(id, Source::Demo { scene: 1 }, "a", "JPEG", 1, 1, "t")) }).unwrap(); + let step = |i: usize| HistoryStep { label: format!("s{i}"), settings: Arc::new(DevelopSettings::default()) }; + for i in 0..HISTORY_LIMIT + 5 { + c.apply(Op::PushHistory { id, step: step(i) }).unwrap(); + } + let h = &c.photo(id).unwrap().history; + assert_eq!(h.len(), HISTORY_LIMIT); + assert_eq!(h[0].label, "s5"); + let before = c.to_snapshot(); + let inv = c.apply(Op::PushHistory { id, step: step(999) }).unwrap(); + c.apply(inv).unwrap(); + assert_eq!(c.to_snapshot(), before); +} + +#[test] +fn fs_store_roundtrip() { + let dir = std::env::temp_dir().join(format!("lc-journal-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let (mut j, mut c, r) = Journal::open(Box::new(FsStore::open(&dir).unwrap())).unwrap(); + assert!(r.created); + let id = c.alloc_photo_id(); + let add = Op::AddPhoto { photo: Box::new(Photo::new(id, Source::File { path: "/a.jpg".into() }, "a.jpg", "JPEG", 1, 1, "t")) }; + c.apply(add.clone()).unwrap(); + c.apply(Op::SetRating { id, rating: 4 }).unwrap(); + j.append(&[add, Op::SetRating { id, rating: 4 }]).unwrap(); + drop(j); + let (mut j, c2, r) = Journal::open(Box::new(FsStore::open(&dir).unwrap())).unwrap(); + assert_eq!((r.replayed, r.created), (2, false)); + assert_eq!(c2.to_snapshot(), c.to_snapshot()); + j.snapshot(&c2).unwrap(); + assert_eq!(std::fs::metadata(dir.join(LOG)).unwrap().len(), 0); + assert!(!dir.join(format!("{SNAPSHOT}.tmp")).exists()); + let (_, c3, r) = Journal::open(Box::new(FsStore::open(&dir).unwrap())).unwrap(); + assert_eq!((r.snapshot_seq, r.replayed), (2, 0)); + assert_eq!(c3.to_snapshot(), c.to_snapshot()); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index f292de82..39a9e2f5 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -16,6 +16,7 @@ lightcraft-develop = { workspace = true } lightcraft-scenes = { workspace = true } lightcraft-pipeline = { workspace = true } lightcraft-catalog = { workspace = true } +lightcraft-preview = { workspace = true } lightcraft-codecs = { workspace = true } lightcraft-raw = { workspace = true } lightcraft-meta = { workspace = true } diff --git a/crates/engine/src/cmd/library.rs b/crates/engine/src/cmd/library.rs index 0a8f6d46..5030286e 100644 --- a/crates/engine/src/cmd/library.rs +++ b/crates/engine/src/cmd/library.rs @@ -364,23 +364,94 @@ pub fn specs() -> Vec { ok() }), // ---- import - cmd!("library.import", "Add Photos…", ["File"], Some("Cmd+Shift+I"), "{paths: [path], album?: albumId}", always, |s, p| { - let paths: Vec = p - .get("paths") - .and_then(Value::as_array) - .map(|a| a.iter().filter_map(Value::as_str).map(str::to_string).collect()) - .unwrap_or_default(); - if paths.is_empty() { - return Err(bad("library.import", "no paths")); + cmd!( + "library.import", + "Add Photos…", + ["File"], + Some("Cmd+Shift+I"), + "{paths: [file or folder (recursive)], mode?: add|copy (copy into the library's Originals/), album?: albumId} → {imported, duplicates, failed}", + always, + |s, p| { + let paths: Vec = p + .get("paths") + .and_then(Value::as_array) + .map(|a| a.iter().filter_map(Value::as_str).map(str::to_string).collect()) + .unwrap_or_default(); + if paths.is_empty() { + return Err(bad("library.import", "no paths")); + } + let mode = match str_param(p, "mode").unwrap_or("add") { + "add" => crate::import::ImportMode::Add, + "copy" => crate::import::ImportMode::Copy, + other => return Err(bad("library.import", format!("unknown mode `{other}` (add|copy)"))), + }; + let report = crate::import::import(s, &paths, mode)?; + if let Some(a) = p.get("album").and_then(Value::as_u64) + && !report.imported.is_empty() + { + s.execute("album.addPhotos", &json!({"id": a, "ids": report.imported}))?; + } + if let Some(f) = report.imported.first() { + s.selection = Selection::single(PhotoId(*f)); + } + Ok(serde_json::to_value(&report).unwrap_or_default()) } - let imported = crate::media::import_paths(s, &paths)?; - if let Some(a) = p.get("album").and_then(Value::as_u64) { - s.execute("album.addPhotos", &json!({"id": a, "ids": imported.iter().map(|i| i.0).collect::>()}))?; - } - if let Some(f) = imported.first() { - s.selection = Selection::single(*f); - } - Ok(json!({"imported": imported.iter().map(|i| i.0).collect::>()})) + ), + // ---- persistence + cmd!(query "library.info", "Library Info", [], None, "{}", always, |s, _| { + let photos = s.catalog.len(); + let albums = s.catalog.albums().count(); + let (rendered_n, rendered_bytes) = s.media.rendered.mem_usage(); + let (sources_n, sources_bytes) = s.media.source_usage(); + let disk = s.media.rendered.disk().map(|d| { + use std::sync::atomic::Ordering::Relaxed; + json!({ + "path": d.dir().display().to_string(), + "bytes": d.size(), + "hits": d.hits.load(Relaxed), + "misses": d.misses.load(Relaxed), + "writes": d.writes.load(Relaxed), + }) + }); + let cache = json!({ + "renderedInMemory": rendered_n, "renderedBytes": rendered_bytes, + "sourcesInMemory": sources_n, "sourceBytes": sources_bytes, + "disk": disk, + }); + let Some(lib) = &s.library else { + return Ok(json!({"persistent": false, "photos": photos, "albums": albums, "cache": cache})); + }; + let j = lib.journal(); + Ok(json!({ + "persistent": true, + "path": lib.dir.display().to_string(), + "photos": photos, + "albums": albums, + "seq": j.seq(), + "snapshotSeq": j.snapshot_seq(), + "logRecords": j.log_records(), + "logBytes": j.log_bytes(), + "lastError": lib.last_error, + "cache": cache, + "load": { + "created": lib.report.created, + "replayed": lib.report.replayed, + "tornBytes": lib.report.torn_bytes, + "damaged": lib.report.damaged, + }, + })) + }), + cmd!("library.compact", "Optimize Library", ["File"], None, "{}", has_library, |s, _| { + s.compact_library()?; + ok() + }), + cmd!("library.clearPreviews", "Clear Preview Cache", ["File"], None, "{}", always, |s, _| { + s.media.rendered.clear(); + ok() }), ] } + +fn has_library(s: &Session) -> std::result::Result<(), String> { + if s.library.is_some() { Ok(()) } else { Err("no library is open (in-memory session)".into()) } +} diff --git a/crates/engine/src/files.rs b/crates/engine/src/files.rs index 0e79c9c9..f4a4a08a 100644 --- a/crates/engine/src/files.rs +++ b/crates/engine/src/files.rs @@ -40,6 +40,7 @@ fn ext_upper(name: &str) -> String { /// Probe a file's bytes: kind, dimensions (oriented), metadata. pub fn probe_bytes(name: &str, bytes: &[u8]) -> Result { + let content_hash = Some(lightcraft_preview::hash_bytes(bytes).to_string()); let m = lightcraft_meta::extract(bytes); let (meta, captured) = meta_of(&m); if lightcraft_raw::probe(bytes).is_some() { @@ -62,6 +63,7 @@ pub fn probe_bytes(name: &str, bytes: &[u8]) -> Result { captured, meta, as_shot_wb, + content_hash, }); } let fmt = lightcraft_codecs::sniff(bytes).ok_or("unrecognized file format")?; @@ -79,7 +81,17 @@ pub fn probe_bytes(name: &str, bytes: &[u8]) -> Result { "" => format!("{fmt:?}").to_uppercase(), e => e.to_string(), }; - Ok(ProbeInfo { width: w, height: h, format, kind: MediaKind::Image, file_size: bytes.len() as u64, captured, meta, as_shot_wb: None }) + Ok(ProbeInfo { + width: w, + height: h, + format, + kind: MediaKind::Image, + file_size: bytes.len() as u64, + captured, + meta, + as_shot_wb: None, + content_hash, + }) } /// Decode a file into a linear Rec.2020 image no larger than `max_edge`, oriented. diff --git a/crates/engine/src/import.rs b/crates/engine/src/import.rs new file mode 100644 index 00000000..8432537c --- /dev/null +++ b/crates/engine/src/import.rs @@ -0,0 +1,267 @@ +//! Import: files and folders (recursive) → photos. +//! +//! 1. Expand folders recursively into supported files (hidden entries and the library's own +//! folder are skipped), sorted for a stable order. +//! 2. Skip paths already in the catalog. +//! 3. Probe the rest — dimensions, metadata, and the **content hash** of the bytes — in parallel +//! on native targets. +//! 4. Skip **duplicates by content** (same bytes already in the library, or twice in this batch). +//! 5. *Add* in place (the photo points at the original file) or *copy* into the library's +//! `Originals/YYYY/YYYY-MM-DD/` folder (names made unique) and point at the copy. +//! 6. Commit all new photos as one undoable op. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use lightcraft_catalog::{Op, Photo, PhotoId, Source}; +use serde::Serialize; + +use crate::Session; +use crate::media::ProbeInfo; + +/// File extensions LightCraft imports (lower case). +pub const EXTENSIONS: &[&str] = &[ + "jpg", "jpeg", "png", "tif", "tiff", "webp", "dng", "cr2", "cr3", "nef", "arw", "raf", "orf", "rw2", "pef", "psd", "jxl", "gif", "bmp", "heic", + "avif", +]; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum ImportMode { + /// Reference the files where they are. + #[default] + Add, + /// Copy the files into the library's `Originals/` folder. + Copy, +} + +#[derive(Clone, Debug, Default, Serialize)] +pub struct Duplicate { + pub path: String, + /// The photo that already has these bytes (or this path). + pub existing: Option, + /// `"path"` (already imported from there) or `"content"` (same bytes elsewhere). + pub reason: &'static str, +} + +#[derive(Clone, Debug, Default, Serialize)] +pub struct ImportReport { + pub imported: Vec, + pub duplicates: Vec, + /// (path, error) + pub failed: Vec<(String, String)>, + /// Files found after expanding folders. + pub scanned: usize, +} + +pub fn is_supported(path: &Path) -> bool { + path.extension().is_some_and(|e| EXTENSIONS.contains(&e.to_string_lossy().to_lowercase().as_str())) +} + +/// Expand files and folders (recursively) into supported files. `skip` (e.g. the library folder) +/// is never descended into. +pub fn expand(paths: &[String], skip: Option<&Path>) -> Vec { + fn walk(p: &Path, skip: Option<&Path>, out: &mut Vec, top: bool) { + let hidden = p.file_name().is_some_and(|n| n.to_string_lossy().starts_with('.')); + if (hidden && !top) || skip.is_some_and(|s| p == s) { + return; + } + if p.is_dir() { + let Ok(rd) = std::fs::read_dir(p) else { return }; + let mut v: Vec = rd.flatten().map(|e| e.path()).collect(); + v.sort(); + for c in v { + walk(&c, skip, out, false); + } + } else if top || is_supported(p) { + // explicitly named files are attempted even with an unknown extension (sniffed) + out.push(p.to_string_lossy().to_string()); + } + } + let mut out = Vec::new(); + for p in paths { + walk(Path::new(p), skip, &mut out, true); + } + let mut seen = std::collections::HashSet::new(); + out.retain(|p| seen.insert(p.clone())); + out +} + +fn probe_all(s: &Session, paths: &[String]) -> Vec> { + let Some(probe) = s.media.file_probe.clone() else { + return paths + .iter() + .map(|p| { + Ok(ProbeInfo { + format: Path::new(p).extension().map(|e| e.to_string_lossy().to_uppercase()).unwrap_or_default(), + ..Default::default() + }) + }) + .collect(); + }; + #[cfg(not(target_arch = "wasm32"))] + { + let n = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4).clamp(1, 8).min(paths.len().max(1)); + if n > 1 { + let mut results: Vec>> = vec![None; paths.len()]; + let next = std::sync::atomic::AtomicUsize::new(0); + let out = std::sync::Mutex::new(&mut results); + std::thread::scope(|sc| { + for _ in 0..n { + sc.spawn(|| { + loop { + let i = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if i >= paths.len() { + break; + } + let r = probe(&paths[i]); + out.lock().unwrap_or_else(|e| e.into_inner())[i] = Some(r); + } + }); + } + }); + return results.into_iter().map(|r| r.unwrap_or_else(|| Err("not probed".into()))).collect(); + } + } + paths.iter().map(|p| probe(p)).collect() +} + +/// `Originals/YYYY/YYYY-MM-DD/name`, made unique. +fn copy_into_library(lib: &Path, src: &str, date: &str) -> Result { + let day = date.get(..10).filter(|d| d.len() == 10).unwrap_or("undated"); + let year = day.get(..4).unwrap_or("undated"); + let dir = lib.join("Originals").join(year).join(day); + std::fs::create_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))?; + let name = Path::new(src).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "photo".into()); + let (stem, ext) = match name.rsplit_once('.') { + Some((s, e)) => (s.to_string(), format!(".{e}")), + None => (name.clone(), String::new()), + }; + let mut dst = dir.join(&name); + let mut i = 1; + while dst.exists() { + dst = dir.join(format!("{stem}-{i}{ext}")); + i += 1; + } + std::fs::copy(src, &dst).map_err(|e| format!("copy {src}: {e}"))?; + Ok(dst.to_string_lossy().to_string()) +} + +/// Import files/folders. See the module docs. +pub fn import(s: &mut Session, paths: &[String], mode: ImportMode) -> crate::Result { + let lib_dir = s.library.as_ref().map(|l| l.dir.clone()); + if mode == ImportMode::Copy && lib_dir.is_none() { + return Err(crate::EngineError::Other("copying into the library needs an open library".into())); + } + let files = expand(paths, lib_dir.as_deref()); + let mut report = ImportReport { scanned: files.len(), ..Default::default() }; + + // existing paths and content hashes + let mut by_path: HashMap<&str, PhotoId> = HashMap::new(); + let mut by_hash: HashMap = HashMap::new(); + for p in s.catalog.photos() { + if let Source::File { path } = &p.source { + by_path.insert(path.as_str(), p.id); + } + if let Some(h) = &p.content_hash { + by_hash.insert(h.clone(), p.id); + } + } + let mut todo = Vec::new(); + for f in files { + match by_path.get(f.as_str()) { + Some(id) => report.duplicates.push(Duplicate { path: f, existing: Some(id.0), reason: "path" }), + None => todo.push(f), + } + } + + let probed = probe_all(s, &todo); + let now = (s.clock)(); + let mut ops = Vec::new(); + for (path, info) in todo.into_iter().zip(probed) { + let info = match info { + Ok(i) => i, + Err(e) => { + log::warn!("import {path}: {e}"); + report.failed.push((path, e)); + continue; + } + }; + if let Some(h) = &info.content_hash + && let Some(id) = by_hash.get(h) + { + report.duplicates.push(Duplicate { path, existing: Some(id.0), reason: "content" }); + continue; + } + let stored = match (mode, &lib_dir) { + (ImportMode::Copy, Some(lib)) if !Path::new(&path).starts_with(lib) => { + match copy_into_library(lib, &path, info.captured.as_deref().unwrap_or(&now)) { + Ok(p) => p, + Err(e) => { + report.failed.push((path, e)); + continue; + } + } + } + _ => path.clone(), + }; + let id = s.catalog.alloc_photo_id(); + if let Some(h) = &info.content_hash { + by_hash.insert(h.clone(), id); + } + let name = Path::new(&path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.clone()); + let mut p = Photo::new(id, Source::File { path: stored }, &name, &info.format, info.width, info.height, &now); + p.kind = info.kind; + p.file_size = info.file_size; + p.captured = info.captured; + p.meta = info.meta; + p.as_shot_wb = info.as_shot_wb; + p.content_hash = info.content_hash; + if let Some((t, tint)) = info.as_shot_wb { + p.develop = std::sync::Arc::new(lightcraft_develop::DevelopSettings::for_raw(t, tint)); + } + report.imported.push(id.0); + ops.push(Op::AddPhoto { photo: Box::new(p) }); + } + if !ops.is_empty() { + s.commit(&format!("Add {} Photo{}", ops.len(), if ops.len() == 1 { "" } else { "s" }), Op::Batch { ops })?; + } + Ok(report) +} + +/// The current local time as ISO 8601 (`YYYY-MM-DDTHH:MM:SS`, UTC on targets without a clock +/// offset); for [`Session::clock`] on native hosts. +pub fn system_clock() -> String { + #[cfg(not(target_arch = "wasm32"))] + { + let secs = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs() as i64).unwrap_or(0); + civil(secs) + } + #[cfg(target_arch = "wasm32")] + { + "2026-01-01T00:00:00".to_string() + } +} + +/// Unix seconds → `YYYY-MM-DDTHH:MM:SS` (UTC), proleptic Gregorian (H. Hinnant's algorithm). +pub fn civil(secs: i64) -> String { + let days = secs.div_euclid(86_400); + let rem = secs.rem_euclid(86_400); + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = yoe + era * 400 + i64::from(m <= 2); + format!("{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}", rem / 3600, rem % 3600 / 60, rem % 60) +} + +impl Session { + /// Use the system clock for import/edit times (native hosts). + pub fn with_system_clock(mut self) -> Self { + self.clock = Box::new(system_clock); + self + } +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index d740ac29..01ae5da9 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -14,6 +14,8 @@ pub mod cmd; pub mod demo; pub mod export; pub mod files; +pub mod import; +pub mod library; pub mod media; pub mod presets; mod view; @@ -86,6 +88,8 @@ pub struct Session { depth: u32, /// Selected mask (Masking panel), by mask id. pub active_mask: Option, + /// The persistent library this session writes to (`None` = in-memory only). + pub library: Option, } impl Default for Session { @@ -116,6 +120,7 @@ impl Session { clock: Box::new(|| "2026-09-30T12:00:00".to_string()), depth: 0, active_mask: None, + library: None, } } @@ -141,6 +146,11 @@ impl Session { self.journal.drain(..1000); } } + if self.depth == 0 && self.library.is_some() { + // Make the command durable before reporting success (on failure the ops stay pending, + // are retried after the next command, and `library.info` reports the error). + let _ = self.persist(); + } r } @@ -217,15 +227,11 @@ impl Session { /// The op that sets a photo's develop settings and appends a History entry (for batches). pub fn develop_op(&self, id: PhotoId, settings: DevelopSettings, label: &str) -> Option { - let p = self.catalog.photo(id)?; + self.catalog.photo(id)?; let settings = Arc::new(settings); - let mut history = p.history.clone(); - history.push(lightcraft_catalog::HistoryStep { label: label.into(), settings: settings.clone() }); - if history.len() > 200 { - history.remove(0); - } + let step = lightcraft_catalog::HistoryStep { label: label.into(), settings: settings.clone() }; Some(Op::Batch { - ops: vec![Op::SetDevelop { id, settings, label: label.into(), edited: Some((self.clock)()) }, Op::SetHistory { id, history }], + ops: vec![Op::SetDevelop { id, settings, label: label.into(), edited: Some((self.clock)()) }, Op::PushHistory { id, step }], }) } @@ -301,3 +307,7 @@ impl Session { #[cfg(test)] mod tests; +#[cfg(test)] +mod tests_import; +#[cfg(test)] +mod tests_library; diff --git a/crates/engine/src/library.rs b/crates/engine/src/library.rs new file mode 100644 index 00000000..1ec5c4f4 --- /dev/null +++ b/crates/engine/src/library.rs @@ -0,0 +1,203 @@ +//! A persistent library: a directory holding the catalog (op log + snapshot), user presets, the +//! last view state and the preview cache. +//! +//! ```text +//! LightCraft Library/ +//! catalog.snap catalog.log (lightcraft-catalog journal) +//! presets.json view.json (user presets + favourites; last source/filter/sort/selection) +//! thumbs/ (rendered thumbnail cache, safe to delete) +//! Originals/ (photos imported with "copy into library") +//! ``` +//! +//! Every top-level [`Session::execute`] persists the ops it produced (fsynced) before returning, +//! so a crash loses at most the command in flight. The log is compacted into a snapshot when it +//! grows (see [`lightcraft_catalog::SnapshotPolicy`]) and on [`Session::close_library`]. + +use std::path::{Path, PathBuf}; + +use lightcraft_catalog::{FsStore, Journal, LoadReport, Store}; +use lightcraft_develop::Preset; +use serde::{Deserialize, Serialize}; + +use crate::{EngineError, LibrarySource, Result, Selection, Session}; + +/// Library directory name inside the user's Pictures folder. +pub const DEFAULT_NAME: &str = "LightCraft Library"; + +/// The default library location: `$LIGHTCRAFT_LIBRARY` if set, else `~/Pictures/LightCraft Library` +/// (`%USERPROFILE%\Pictures\LightCraft Library` on Windows). +pub fn default_dir() -> Option { + if let Some(p) = std::env::var_os("LIGHTCRAFT_LIBRARY").filter(|p| !p.is_empty()) { + return Some(PathBuf::from(p)); + } + let home = if cfg!(windows) { std::env::var_os("USERPROFILE") } else { std::env::var_os("HOME") }?; + Some(PathBuf::from(home).join("Pictures").join(DEFAULT_NAME)) +} + +pub struct Library { + pub dir: PathBuf, + journal: Journal, + /// What happened when the catalog was loaded. + pub report: LoadReport, + /// Last persistence error (shown by the UI; ops stay pending and are retried). + pub last_error: Option, + presets_written: String, +} + +#[derive(Default, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +struct PresetsFile { + user: Vec, + favorites: Vec, +} + +#[derive(Default, Serialize, Deserialize)] +#[serde(default)] +struct ViewFile { + source: LibrarySource, + filter: lightcraft_catalog::Filter, + sort: lightcraft_catalog::Sort, + selection: Selection, +} + +impl Library { + pub fn journal(&self) -> &Journal { + &self.journal + } + pub fn thumbs_dir(&self) -> PathBuf { + self.dir.join("thumbs") + } + pub fn originals_dir(&self) -> PathBuf { + self.dir.join("Originals") + } +} + +fn presets_json(presets: &[Preset]) -> String { + let f = PresetsFile { + user: presets.iter().filter(|p| !p.builtin).cloned().collect(), + favorites: presets.iter().filter(|p| p.builtin && p.favorite).map(|p| p.id.clone()).collect(), + }; + serde_json::to_string_pretty(&f).unwrap_or_default() +} + +fn write_atomic(dir: &Path, name: &str, data: &[u8]) -> std::io::Result<()> { + FsStore::open(dir)?.write_atomic(name, data) +} + +impl Session { + /// Open (or create) the library at `dir` into this session, replacing its catalog. With + /// `seed_demo`, a newly created library starts with the procedural demo photos. + pub fn open_library(&mut self, dir: impl AsRef, seed_demo: bool) -> Result<&LoadReport> { + let dir = dir.as_ref().to_path_buf(); + let store = FsStore::open(&dir).map_err(|e| EngineError::Other(format!("can't open library {}: {e}", dir.display())))?; + let (mut journal, catalog, report) = Journal::open(Box::new(store))?; + self.catalog = catalog; + self.undo.clear(); + self.redo.clear(); + self.interaction = None; + self.pending_log.clear(); + self.selection = Selection::default(); + self.source = LibrarySource::All; + if report.created && seed_demo { + crate::demo::load(self); + journal.snapshot(&self.catalog)?; + } + // presets + if let Ok(bytes) = std::fs::read(dir.join("presets.json")) + && let Ok(f) = serde_json::from_slice::(&bytes) + { + for p in &mut self.presets { + p.favorite = p.builtin && f.favorites.contains(&p.id) || (!p.builtin && p.favorite); + } + for u in f.user { + if !self.presets.iter().any(|p| p.id == u.id) { + self.presets.push(u); + } + } + } + // view state + if let Ok(bytes) = std::fs::read(dir.join("view.json")) + && let Ok(v) = serde_json::from_slice::(&bytes) + { + self.source = v.source; + self.filter = v.filter; + self.sort = v.sort; + self.selection = v.selection; + self.selection.ids.retain(|id| self.catalog.photo(*id).is_some()); + self.selection.active = self.selection.active.filter(|id| self.catalog.photo(*id).is_some()); + } + if self.selection.active.is_none() + && let Some(first) = self.visible_cloned().first() + { + self.selection = Selection::single(*first); + } + let presets_written = presets_json(&self.presets); + self.media.attach_disk_cache(&dir.join("thumbs")); + self.library = Some(Library { dir, journal, report, last_error: None, presets_written }); + Ok(&self.library.as_ref().expect("just set").report) + } + + /// Write pending ops to the log (fsynced), compact when due, and save changed presets. + /// Called after every top-level command; cheap when nothing changed. + pub fn persist(&mut self) -> Result<()> { + let Some(lib) = self.library.as_mut() else { return Ok(()) }; + if !self.pending_log.is_empty() { + if let Err(e) = lib.journal.append(&self.pending_log) { + lib.last_error = Some(e.to_string()); + log::error!("library: {e}"); + return Err(e.into()); + } + self.pending_log.clear(); + lib.last_error = None; + } + // Never snapshot mid-interaction: the catalog then holds an uncommitted preview value. + if lib.journal.wants_snapshot() && self.interaction.is_none() { + lib.journal.snapshot(&self.catalog)?; + } + let presets = presets_json(&self.presets); + if presets != lib.presets_written { + if let Err(e) = write_atomic(&lib.dir, "presets.json", presets.as_bytes()) { + log::error!("library: presets: {e}"); + } else { + lib.presets_written = presets; + } + } + Ok(()) + } + + /// Persist if commands left ops pending (cheap; frontends call it once per frame for state + /// changed outside [`Session::execute`]). + pub fn persist_if_dirty(&mut self) { + if self.library.is_some() && !self.pending_log.is_empty() { + let _ = self.persist(); + } + } + + /// Flush everything and write a snapshot (on quit). Ends an open interaction first. + pub fn close_library(&mut self) -> Result<()> { + if self.library.is_none() { + return Ok(()); + } + let _ = self.end_interaction(); + self.persist()?; + let view = ViewFile { source: self.source, filter: self.filter.clone(), sort: self.sort, selection: self.selection.clone() }; + let Some(lib) = self.library.as_mut() else { return Ok(()) }; + lib.journal.snapshot(&self.catalog)?; + if let Ok(v) = serde_json::to_vec_pretty(&view) { + let _ = write_atomic(&lib.dir, "view.json", &v); + } + Ok(()) + } + + /// Compact the log into a snapshot now. + pub fn compact_library(&mut self) -> Result<()> { + self.persist()?; + if self.interaction.is_some() { + return Err(EngineError::Other("can't compact during an interaction".into())); + } + if let Some(lib) = self.library.as_mut() { + lib.journal.snapshot(&self.catalog)?; + } + Ok(()) + } +} diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rs index 21100410..2d56b459 100644 --- a/crates/engine/src/media.rs +++ b/crates/engine/src/media.rs @@ -1,19 +1,36 @@ //! Source proxies and render jobs. //! -//! Sources are decoded (or generated) once per level and cached: `Thumb` (≤ 512 px long edge) for -//! the grid and filmstrip, `Preview` (≤ 2560 px) for the loupe. A [`RenderJob`] is self-contained -//! and `Send`: frontends run it on a worker thread; if the source wasn't cached yet the job loads it -//! and hands it back in the [`RenderResult`] so the cache can keep it. +//! Sources are decoded (or generated) once per level and cached in memory LRUs: `Thumb` (≤ 512 px +//! long edge) for the grid and filmstrip, `Preview` (≤ 2560 px) for the loupe. A [`RenderJob`] is +//! self-contained and `Send`: frontends run it on a worker thread; if the source wasn't cached yet +//! the job loads it and hands it back in the [`RenderResult`] so the cache can keep it. +//! +//! Rendered thumbnails are cached too ([`lightcraft_preview::PreviewCache`]: memory LRU, plus a +//! disk cache in the library's `thumbs/` folder), keyed by the photo's content hash, the develop +//! settings hash, the size and [`RENDER_CACHE_VERSION`] — so reopening a library shows its grid +//! without decoding a single original, and an edit simply produces a new key. -use std::collections::HashMap; use std::sync::Arc; use lightcraft_catalog::{MediaKind, Photo, PhotoId, Source}; use lightcraft_develop::DevelopSettings; use lightcraft_pipeline::{RenderRequest, Rendered, SourceInfo}; -use lightcraft_raster::Rgb32f; +use lightcraft_preview::{Hash128, Hasher128, Lru, PreviewCache}; +use lightcraft_raster::{Histogram, Rgb32f}; use serde::{Deserialize, Serialize}; +/// Bump when the pipeline's output changes, to invalidate cached thumbnails. +pub const RENDER_CACHE_VERSION: u64 = 1; + +/// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache). +pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512]; + +/// Memory budgets. +const THUMB_SOURCE_BYTES: usize = 384 << 20; +const RENDERED_MEM_BYTES: usize = 128 << 20; +/// Disk budget for the thumbnail cache. +pub const DISK_CACHE_BYTES: u64 = 2 << 30; + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum SourceLevel { @@ -57,19 +74,40 @@ impl SourceRef { } } -#[derive(Default)] pub struct MediaCache { - thumbs: HashMap>, + /// Decoded thumbnail-level sources (LRU by bytes). + thumbs: Lru>, previews: Vec<(PhotoId, Arc)>, /// How many previews to keep (LRU). pub preview_capacity: usize, pub file_loader: Option, pub file_probe: Option, scenes: Vec, + /// Rendered thumbnails (memory, plus disk once a library is attached). + pub rendered: Arc, +} + +impl Default for MediaCache { + fn default() -> Self { + MediaCache { + thumbs: Lru::new(THUMB_SOURCE_BYTES), + previews: Vec::new(), + preview_capacity: 0, + file_loader: None, + file_probe: None, + scenes: Vec::new(), + rendered: Arc::new(PreviewCache::memory(RENDERED_MEM_BYTES)), + } + } } impl MediaCache { - pub fn get(&self, id: PhotoId, level: SourceLevel) -> Option> { + /// Keep rendered thumbnails on disk in `dir` as well. + pub fn attach_disk_cache(&mut self, dir: &std::path::Path) { + self.rendered = Arc::new(PreviewCache::with_disk(RENDERED_MEM_BYTES, dir, DISK_CACHE_BYTES)); + } + + pub fn get(&mut self, id: PhotoId, level: SourceLevel) -> Option> { match level { SourceLevel::Thumb => self.thumbs.get(&id).cloned(), SourceLevel::Preview => self.previews.iter().find(|(p, _)| *p == id).map(|(_, a)| a.clone()), @@ -79,7 +117,8 @@ impl MediaCache { pub fn insert(&mut self, id: PhotoId, level: SourceLevel, img: Arc) { match level { SourceLevel::Thumb => { - self.thumbs.insert(id, img); + let cost = img.width * img.height * 12 + 64; + self.thumbs.insert(id, img, cost); } SourceLevel::Preview => { self.previews.retain(|(p, _)| *p != id); @@ -92,6 +131,17 @@ impl MediaCache { } } + /// Forget a photo's decoded sources (e.g. after its file changed). + pub fn forget(&mut self, id: PhotoId) { + self.thumbs.remove(&id); + self.previews.retain(|(p, _)| *p != id); + } + + /// (decoded thumbnail sources, bytes). + pub fn source_usage(&self) -> (usize, usize) { + (self.thumbs.len(), self.thumbs.cost()) + } + pub fn source_ref(&mut self, p: &Photo, level: SourceLevel) -> SourceRef { if let Some(a) = self.get(p.id, level) { return SourceRef::Loaded(a); @@ -122,6 +172,8 @@ pub struct RenderJob { pub request: RenderRequest, /// Identifies the result for caching: hash of settings + request. pub key: u64, + /// Rendered-thumbnail cache and this job's key in it. + pub cache: Option<(Arc, Hash128)>, } pub struct RenderResult { @@ -135,10 +187,20 @@ pub struct RenderResult { impl RenderJob { pub fn run(self) -> RenderResult { + if let Some((cache, key)) = &self.cache + && let Some(img) = cache.get(*key) + { + let image = Arc::unwrap_or_clone(img); + let histogram = Histogram::of_srgb8(&image); + return RenderResult { photo: self.photo, level: self.level, key: self.key, rendered: Ok(Rendered { image, histogram }), loaded: None }; + } let was_loaded = matches!(self.source, SourceRef::Loaded(_)); match self.source.load() { Ok(src) => { let rendered = lightcraft_pipeline::render(&src, &self.info, &self.settings, &self.request); + if let Some((cache, key)) = &self.cache { + cache.put(*key, Arc::new(rendered.image.clone())); + } RenderResult { photo: self.photo, level: self.level, key: self.key, rendered: Ok(rendered), loaded: (!was_loaded).then_some(src) } } Err(e) => RenderResult { photo: self.photo, level: self.level, key: self.key, rendered: Err(e), loaded: None }, @@ -146,6 +208,15 @@ impl RenderJob { } } +/// What identifies a photo's pixels for caching: its content hash, else its source. +pub fn content_key(p: &Photo) -> String { + match (&p.content_hash, &p.source) { + (Some(h), _) => h.clone(), + (None, Source::Demo { scene }) => format!("demo:{scene}"), + (None, Source::File { path }) => format!("file:{path}:{}", p.file_size), + } +} + pub fn source_info(p: &Photo) -> SourceInfo { // Procedural demo scenes are scene-referred HDR (like raw files): use the filmic tone map. if matches!(p.source, Source::Demo { .. }) { @@ -157,6 +228,25 @@ pub fn source_info(p: &Photo) -> SourceInfo { impl crate::Session { /// Build a render job for `id` fitting `max_w × max_h`. `before` renders the unedited look. pub fn render_job(&mut self, id: PhotoId, max_w: usize, max_h: usize, before: bool, apply_crop: bool) -> Option { + self.build_job(id, max_w, max_h, before, apply_crop, None) + } + + /// A grid/filmstrip thumbnail job: the long edge is rounded up to one of [`THUMB_SIZES`] + /// (≥ `long_edge`) and the result comes from / goes to the thumbnail cache (memory + disk). + pub fn thumb_job(&mut self, id: PhotoId, long_edge: usize) -> Option { + let b = THUMB_SIZES.iter().copied().find(|s| *s >= long_edge).unwrap_or(THUMB_SIZES[THUMB_SIZES.len() - 1]); + self.build_job(id, b, b, false, true, Some(b)) + } + + fn build_job( + &mut self, + id: PhotoId, + max_w: usize, + max_h: usize, + before: bool, + apply_crop: bool, + thumb_bucket: Option, + ) -> Option { let p = self.catalog.photo(id)?.clone(); let level = SourceLevel::for_size(max_w.max(max_h)); let source = self.media.source_ref(&p, level); @@ -167,7 +257,11 @@ impl crate::Session { ^ ((max_h as u64) << 20) ^ (apply_crop as u64) ^ ((level == SourceLevel::Preview) as u64) << 60; - Some(RenderJob { photo: id, level, source, info: source_info(&p), settings, request, key }) + let cache = thumb_bucket.map(|b| { + let k = Hasher128::new().str(&content_key(&p)).u64(settings.hash64()).u64(b as u64).u64(RENDER_CACHE_VERSION).finish(); + (self.media.rendered.clone(), k) + }); + Some(RenderJob { photo: id, level, source, info: source_info(&p), settings, request, key, cache }) } /// Accept a finished job's loaded source into the cache. @@ -205,49 +299,8 @@ pub struct ProbeInfo { pub captured: Option, pub meta: lightcraft_catalog::Meta, pub as_shot_wb: Option<(f64, f64)>, + /// Hash of the file's bytes (hex), for duplicate detection. + pub content_hash: Option, } pub type FileProbe = Arc Result + Send + Sync>; - -/// Register files as photos (duplicates by path are skipped). Returns the new ids. -pub fn import_paths(s: &mut crate::Session, paths: &[String]) -> crate::Result> { - let now = (s.clock)(); - let mut ops = Vec::new(); - let mut ids = Vec::new(); - for path in paths { - let exists = s.catalog.photos().any(|p| matches!(&p.source, Source::File { path: q } if q == path)); - if exists { - continue; - } - let info = match &s.media.file_probe { - Some(probe) => match probe(path) { - Ok(i) => i, - Err(e) => { - log::warn!("import {path}: {e}"); - continue; - } - }, - None => ProbeInfo { - format: std::path::Path::new(path).extension().map(|e| e.to_string_lossy().to_uppercase()).unwrap_or_default(), - ..Default::default() - }, - }; - let id = s.catalog.alloc_photo_id(); - let name = std::path::Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.clone()); - let mut p = Photo::new(id, Source::File { path: path.clone() }, &name, &info.format, info.width, info.height, &now); - p.kind = info.kind; - p.file_size = info.file_size; - p.captured = info.captured; - p.meta = info.meta; - p.as_shot_wb = info.as_shot_wb; - if let Some((t, tint)) = info.as_shot_wb { - p.develop = std::sync::Arc::new(lightcraft_develop::DevelopSettings::for_raw(t, tint)); - } - ids.push(id); - ops.push(lightcraft_catalog::Op::AddPhoto { photo: Box::new(p) }); - } - if !ops.is_empty() { - s.commit("Add Photos", lightcraft_catalog::Op::Batch { ops })?; - } - Ok(ids) -} diff --git a/crates/engine/src/tests_import.rs b/crates/engine/src/tests_import.rs new file mode 100644 index 00000000..6e1289e3 --- /dev/null +++ b/crates/engine/src/tests_import.rs @@ -0,0 +1,103 @@ +//! Import: recursive folders, duplicate detection by content, add vs copy into the library. + +use std::path::Path; + +use serde_json::{Value, json}; + +use crate::Session; + +fn temp_dir(tag: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!("lc-import-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +/// A small procedural PNG (distinct per seed). +fn write_png(path: &Path, seed: u8) { + let (w, h) = (48usize, 32usize); + let data: Vec<[u8; 4]> = (0..w * h).map(|i| [(i % w * 5) as u8, (i / w * 7) as u8, seed, 255]).collect(); + let img = lightcraft_raster::Rgba8 { width: w, height: h, data }; + let bytes = lightcraft_codecs::encode_png(&lightcraft_codecs::EncodeImage::rgba8(&img), &lightcraft_codecs::EncodeMeta::default()).unwrap(); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, bytes).unwrap(); +} + +fn ids(v: &Value, key: &str) -> usize { + v[key].as_array().map(Vec::len).unwrap_or(0) +} + +#[test] +fn recursive_import_with_duplicates() { + let src = temp_dir("src"); + write_png(&src.join("a.png"), 1); + write_png(&src.join("trip/b.png"), 2); + write_png(&src.join("trip/day2/c.PNG"), 3); + std::fs::copy(src.join("a.png"), src.join("trip/a-copy.png")).unwrap(); // same bytes + write_png(&src.join(".hidden/x.png"), 9); + std::fs::write(src.join("notes.txt"), "not a photo").unwrap(); + std::fs::write(src.join("trip/broken.jpg"), "garbage").unwrap(); + + let mut s = Session::new().with_fs(); + let r = s.execute("library.import", &json!({"paths": [src.to_string_lossy()]})).unwrap(); + assert_eq!(r["scanned"], 5, "{r}"); // a, b, c, a-copy, broken (hidden + txt skipped) + assert_eq!(ids(&r, "imported"), 3, "{r}"); + assert_eq!(ids(&r, "duplicates"), 1, "{r}"); + assert_eq!(r["duplicates"][0]["reason"], "content"); + assert_eq!(ids(&r, "failed"), 1, "{r}"); + assert_eq!(s.catalog.len(), 3); + assert!(s.catalog.photos().all(|p| p.content_hash.as_ref().is_some_and(|h| h.len() == 32) && p.width == 48)); + + // importing again: everything is a duplicate (by path), one undo step for the first import + let r2 = s.execute("library.import", &json!({"paths": [src.to_string_lossy()]})).unwrap(); + assert_eq!(ids(&r2, "imported"), 0); + assert_eq!(r2["duplicates"].as_array().unwrap().iter().filter(|d| d["reason"] == "path").count(), 3); + s.execute("edit.undo", &json!({})).unwrap(); + assert_eq!(s.catalog.len(), 0); + let _ = std::fs::remove_dir_all(&src); +} + +#[test] +fn copy_into_library_and_persist() { + let src = temp_dir("copysrc"); + let lib = temp_dir("copylib"); + write_png(&src.join("one.png"), 4); + write_png(&src.join("sub/one.png"), 5); // same name, different content + let mut s = Session::new().with_fs(); + assert!(s.execute("library.import", &json!({"paths": [src.to_string_lossy()], "mode": "copy"})).is_err(), "copy needs a library"); + s.open_library(&lib, false).unwrap(); + let r = s.execute("library.import", &json!({"paths": [src.to_string_lossy()], "mode": "copy"})).unwrap(); + assert_eq!(ids(&r, "imported"), 2, "{r}"); + let paths: Vec = s + .catalog + .photos() + .map(|p| match &p.source { + lightcraft_catalog::Source::File { path } => path.clone(), + _ => panic!(), + }) + .collect(); + for p in &paths { + assert!(Path::new(p).starts_with(lib.join("Originals")), "{p}"); + assert!(Path::new(p).exists()); + } + assert_ne!(paths[0], paths[1], "unique names"); + // originals deleted from the source: the library copies remain usable after a restart + std::fs::remove_dir_all(&src).unwrap(); + drop(s); + let mut s = Session::new().with_fs(); + s.open_library(&lib, false).unwrap(); + assert_eq!(s.catalog.len(), 2); + let id = s.catalog.photos().next().unwrap().id; + assert!(s.render_now(id, 32, 32).is_ok()); + // the library folder itself is never re-imported + let r = s.execute("library.import", &json!({"paths": [lib.to_string_lossy()]})).unwrap(); + assert_eq!(r["scanned"], 0, "{r}"); + let _ = std::fs::remove_dir_all(&lib); +} + +#[test] +fn civil_dates() { + assert_eq!(crate::import::civil(0), "1970-01-01T00:00:00"); + assert_eq!(crate::import::civil(951_782_400), "2000-02-29T00:00:00"); + assert_eq!(crate::import::civil(1_790_000_000), "2026-09-21T14:13:20"); +} diff --git a/crates/engine/src/tests_library.rs b/crates/engine/src/tests_library.rs new file mode 100644 index 00000000..e3e75753 --- /dev/null +++ b/crates/engine/src/tests_library.rs @@ -0,0 +1,128 @@ +//! Persistent library: commands survive a restart (with and without a clean close). + +use serde_json::json; + +use crate::Session; + +fn temp_dir(tag: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!("lc-engine-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + d +} + +fn open(dir: &std::path::Path, seed: bool) -> Session { + let mut s = Session::new(); + s.open_library(dir, seed).unwrap(); + s +} + +#[test] +fn edits_survive_restart_without_close() { + let dir = temp_dir("crash"); + let mut s = open(&dir, true); + assert!(s.library.as_ref().unwrap().report.created); + assert!(s.catalog.len() > 5, "seeded demo"); + let id = s.selection.active.unwrap(); + s.execute("photo.rate", &json!({"rating": 5})).unwrap(); + s.execute("photo.flag", &json!({"flag": "reject"})).unwrap(); + s.execute("develop.set", &json!({"control": "light.exposure", "value": 1.25})).unwrap(); + // a slider drag: one undo step, logged once at the end + s.execute("develop.beginInteraction", &json!({"label": "Contrast"})).unwrap(); + for v in [5, 10, 20, 33] { + s.execute("develop.set", &json!({"control": "light.contrast", "value": v})).unwrap(); + } + s.execute("develop.endInteraction", &json!({})).unwrap(); + s.execute("album.create", &json!({"name": "Keepers", "addSelected": true})).unwrap(); + s.execute("preset.create", &json!({"name": "Bright"})).unwrap(); + s.execute("edit.undo", &json!({})).ok(); + s.execute("edit.redo", &json!({})).ok(); + let expect = s.catalog.to_snapshot(); + let info = s.execute("library.info", &json!({})).unwrap(); + assert!(info["logRecords"].as_u64().unwrap() >= 5, "{info}"); + drop(s); // no close: like a crash + + let s2 = open(&dir, true); + let r = &s2.library.as_ref().unwrap().report; + assert!(!r.created && r.replayed >= 5, "{r:?}"); + assert_eq!(s2.catalog.to_snapshot(), expect); + let p = s2.catalog.photo(id).unwrap(); + assert_eq!((p.rating, p.develop.light.exposure, p.develop.light.contrast), (5, 1.25, 33.0)); + assert!(s2.catalog.albums().any(|a| a.name == "Keepers" && a.photos == vec![id])); + assert!(s2.presets.iter().any(|p| p.name == "Bright" && !p.builtin)); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn close_writes_snapshot_and_view_state() { + let dir = temp_dir("close"); + let mut s = open(&dir, true); + let ids = s.visible_cloned(); + s.execute("library.select", &json!({"ids": [ids[3].0]})).unwrap(); + s.execute("photo.rate", &json!({"rating": 2})).unwrap(); + let expect = s.catalog.to_snapshot(); + s.close_library().unwrap(); + assert_eq!(std::fs::metadata(dir.join("catalog.log")).unwrap().len(), 0); + + let s2 = open(&dir, true); + let r = &s2.library.as_ref().unwrap().report; + assert_eq!(r.replayed, 0); + assert_eq!(s2.catalog.to_snapshot(), expect); + assert_eq!(s2.selection.active, Some(ids[3])); + let _ = std::fs::remove_dir_all(&dir); +} + +/// Thumbnails rendered in one session are served from the disk cache in the next, without +/// loading the source; an edit changes the key. +#[test] +fn thumbnails_come_from_the_disk_cache_after_restart() { + let dir = temp_dir("thumbs"); + let mut s = open(&dir, true); + let ids: Vec<_> = s.visible_cloned().into_iter().take(3).collect(); + for id in &ids { + let job = s.thumb_job(*id, 230).unwrap(); + assert_eq!(job.request.max_w, 256, "bucketed"); + let r = job.run(); + assert!(r.loaded.is_some(), "first render decodes the source"); + s.accept(&r); + let img = r.rendered.unwrap().image; + assert_eq!(img.width.max(img.height), 256); + } + drop(s); + + let mut s = open(&dir, true); + for id in &ids { + let r = s.thumb_job(*id, 240).unwrap().run(); + assert!(r.loaded.is_none(), "served from cache"); + let img = r.rendered.unwrap().image; + assert_eq!(img.width.max(img.height), 256); + } + let info = s.execute("library.info", &json!({})).unwrap(); + assert_eq!(info["cache"]["disk"]["hits"], 3, "{info}"); + // an edit is a new key: rendered (from the source), then cached + s.execute("library.select", &json!({"ids": [ids[0].0]})).unwrap(); + s.execute("develop.set", &json!({"control": "light.exposure", "value": 0.5})).unwrap(); + let r = s.thumb_job(ids[0], 230).unwrap().run(); + assert!(r.loaded.is_some()); + // exact-size renders (loupe, before/after, exports) bypass the thumbnail cache + assert!(s.render_job(ids[0], 230, 230, true, true).unwrap().cache.is_none()); + assert!(s.render_job(ids[0], 230, 230, false, true).unwrap().cache.is_none()); + s.execute("library.clearPreviews", &json!({})).unwrap(); + assert!(s.thumb_job(ids[1], 230).unwrap().run().loaded.is_some()); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn new_library_without_seed_is_empty_and_compacts() { + let dir = temp_dir("empty"); + let mut s = open(&dir, false); + assert!(s.catalog.is_empty()); + s.execute("album.create", &json!({"name": "A"})).unwrap(); + s.execute("library.compact", &json!({})).unwrap(); + let info = s.execute("library.info", &json!({})).unwrap(); + assert_eq!(info["logRecords"], 0); + assert_eq!(info["snapshotSeq"], info["seq"]); + let s2 = open(&dir, true); + assert_eq!(s2.catalog.len(), 0, "an existing library is never seeded"); + assert_eq!(s2.catalog.albums().count(), 1); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/preview/Cargo.toml b/crates/preview/Cargo.toml new file mode 100644 index 00000000..12e21b16 --- /dev/null +++ b/crates/preview/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "lightcraft-preview" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "LightCraft preview cache: content hashing, memory LRU, disk thumbnail cache, prioritized job pool (L3)" + +[lints] +workspace = true + +[dependencies] +lightcraft-raster = { workspace = true } +siphasher = { workspace = true } +jpeg-encoder = { workspace = true } +zune-jpeg = { workspace = true } +zune-core = { workspace = true } +log = { workspace = true } diff --git a/crates/preview/src/disk.rs b/crates/preview/src/disk.rs new file mode 100644 index 00000000..ccf0d480 --- /dev/null +++ b/crates/preview/src/disk.rs @@ -0,0 +1,176 @@ +//! Rendered thumbnails on disk: `/<2 hex>/<32 hex>.jpg`, bounded in bytes. +//! +//! The cache is disposable: files are written via temp + rename (readers never see partial +//! files) but not fsynced; unreadable files are deleted and re-rendered. When the total size +//! exceeds the budget, the least recently used files (by modification time, refreshed on read) +//! are removed down to 80 % of the budget. + +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU64, Ordering}; + +use lightcraft_raster::Rgba8; + +use crate::Hash128; + +const QUALITY: u8 = 90; + +pub struct DiskCache { + dir: PathBuf, + budget: u64, + /// Bytes on disk (`None` until first scanned). + total: Mutex>, + pub hits: AtomicU64, + pub misses: AtomicU64, + pub writes: AtomicU64, +} + +impl DiskCache { + pub fn new(dir: &Path, budget: u64) -> DiskCache { + DiskCache { + dir: dir.to_path_buf(), + budget, + total: Mutex::new(None), + hits: AtomicU64::new(0), + misses: AtomicU64::new(0), + writes: AtomicU64::new(0), + } + } + + pub fn dir(&self) -> &Path { + &self.dir + } + + fn path(&self, key: Hash128) -> PathBuf { + let hex = key.to_string(); + self.dir.join(&hex[..2]).join(format!("{hex}.jpg")) + } + + pub fn get(&self, key: Hash128) -> Option { + let p = self.path(key); + let Ok(bytes) = std::fs::read(&p) else { + self.misses.fetch_add(1, Ordering::Relaxed); + return None; + }; + match decode(&bytes) { + Some(img) => { + self.hits.fetch_add(1, Ordering::Relaxed); + // refresh recency for pruning + if let Ok(f) = std::fs::File::options().write(true).open(&p) { + let _ = f.set_modified(std::time::SystemTime::now()); + } + Some(img) + } + None => { + log::warn!("preview cache: dropping unreadable {}", p.display()); + let _ = std::fs::remove_file(&p); + self.misses.fetch_add(1, Ordering::Relaxed); + None + } + } + } + + pub fn put(&self, key: Hash128, img: &Rgba8) { + let Some(bytes) = encode(img) else { return }; + let p = self.path(key); + let Some(parent) = p.parent() else { return }; + if std::fs::create_dir_all(parent).is_err() { + return; + } + // know the current total before adding (the first scan must not count this file twice) + let _ = self.size(); + let tmp = p.with_extension(format!("tmp{:?}", std::thread::current().id()).replace(['(', ')'], "")); + if std::fs::write(&tmp, &bytes).is_err() || std::fs::rename(&tmp, &p).is_err() { + let _ = std::fs::remove_file(&tmp); + return; + } + self.writes.fetch_add(1, Ordering::Relaxed); + let over = { + let mut t = self.total.lock().unwrap_or_else(|e| e.into_inner()); + let total = t.get_or_insert(0); + *total += bytes.len() as u64; + *total > self.budget + }; + if over { + self.prune(); + } + } + + /// All cache files: (path, size, modified). + fn scan(&self) -> Vec<(PathBuf, u64, std::time::SystemTime)> { + let mut out = Vec::new(); + let Ok(rd) = std::fs::read_dir(&self.dir) else { return out }; + for sub in rd.flatten() { + let Ok(files) = std::fs::read_dir(sub.path()) else { continue }; + for f in files.flatten() { + if let Ok(m) = f.metadata() + && m.is_file() + { + out.push((f.path(), m.len(), m.modified().unwrap_or(std::time::UNIX_EPOCH))); + } + } + } + out + } + + /// Remove least recently used files until the cache is at most 80 % of its budget. + pub fn prune(&self) { + let mut files = self.scan(); + let mut total: u64 = files.iter().map(|f| f.1).sum(); + let target = self.budget / 5 * 4; + files.sort_by_key(|f| f.2); + for (p, len, _) in files { + if total <= target { + break; + } + if std::fs::remove_file(&p).is_ok() { + total -= len; + } + } + *self.total.lock().unwrap_or_else(|e| e.into_inner()) = Some(total); + } + + /// Bytes on disk (scans if not yet known). + pub fn size(&self) -> u64 { + let mut t = self.total.lock().unwrap_or_else(|e| e.into_inner()); + *t.get_or_insert_with(|| self.scan().iter().map(|f| f.1).sum()) + } + + pub fn clear(&self) { + for (p, _, _) in self.scan() { + let _ = std::fs::remove_file(p); + } + *self.total.lock().unwrap_or_else(|e| e.into_inner()) = Some(0); + } +} + +fn encode(img: &Rgba8) -> Option> { + if img.width == 0 || img.height == 0 || img.width > u16::MAX as usize || img.height > u16::MAX as usize { + return None; + } + let mut rgb = Vec::with_capacity(img.width * img.height * 3); + for p in &img.data { + rgb.extend_from_slice(&p[..3]); + } + let mut out = Vec::new(); + let mut enc = jpeg_encoder::Encoder::new(&mut out, QUALITY); + enc.set_sampling_factor(jpeg_encoder::SamplingFactor::R_4_2_0); + enc.encode(&rgb, img.width as u16, img.height as u16, jpeg_encoder::ColorType::Rgb).ok()?; + Some(out) +} + +fn decode(bytes: &[u8]) -> Option { + use zune_core::bytestream::ZCursor; + use zune_core::colorspace::ColorSpace; + use zune_core::options::DecoderOptions; + let opts = DecoderOptions::default().jpeg_set_out_colorspace(ColorSpace::RGB); + let mut d = zune_jpeg::JpegDecoder::new_with_options(ZCursor::new(bytes), opts); + let px = d.decode().ok()?; + let info = d.info()?; + let (w, h) = (info.width as usize, info.height as usize); + if px.len() < w * h * 3 { + return None; + } + let data = px.chunks_exact(3).take(w * h).map(|c| [c[0], c[1], c[2], 255]).collect(); + Some(Rgba8 { width: w, height: h, data }) +} diff --git a/crates/preview/src/hash.rs b/crates/preview/src/hash.rs new file mode 100644 index 00000000..e3ab51fe --- /dev/null +++ b/crates/preview/src/hash.rs @@ -0,0 +1,61 @@ +//! 128-bit content hashing (SipHash-1-3, fixed key). Not cryptographic — it identifies files +//! for duplicate detection and keys caches; collisions are astronomically unlikely by accident. + +use std::fmt; +use std::hash::Hasher; + +use siphasher::sip128::{Hasher128 as _, SipHasher13}; + +/// A 128-bit hash, shown as 32 hex digits. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Hash128(pub u128); + +impl fmt::Display for Hash128 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:032x}", self.0) + } +} + +impl Hash128 { + pub fn parse(s: &str) -> Option { + (s.len() == 32).then(|| u128::from_str_radix(s, 16).ok().map(Hash128)).flatten() + } +} + +/// Streaming hasher. +#[derive(Clone, Debug)] +pub struct Hasher128(SipHasher13); + +impl Default for Hasher128 { + fn default() -> Self { + Hasher128(SipHasher13::new_with_keys(0x4c69_6768_7443_7261, 0x6674_2043_6f6e_7465)) + } +} + +impl Hasher128 { + pub fn new() -> Self { + Self::default() + } + pub fn update(&mut self, bytes: &[u8]) -> &mut Self { + self.0.write(bytes); + self + } + /// Length-prefixed string (so `("ab","c")` and `("a","bc")` differ). + pub fn str(&mut self, s: &str) -> &mut Self { + self.0.write(&(s.len() as u64).to_le_bytes()); + self.0.write(s.as_bytes()); + self + } + pub fn u64(&mut self, v: u64) -> &mut Self { + self.0.write(&v.to_le_bytes()); + self + } + pub fn finish(&self) -> Hash128 { + Hash128(self.0.finish128().as_u128()) + } +} + +/// Hash of a whole byte string. +pub fn hash_bytes(bytes: &[u8]) -> Hash128 { + Hasher128::new().update(bytes).finish() +} diff --git a/crates/preview/src/lib.rs b/crates/preview/src/lib.rs new file mode 100644 index 00000000..478c784d --- /dev/null +++ b/crates/preview/src/lib.rs @@ -0,0 +1,86 @@ +//! Previews: everything that makes a big library show up instantly. +//! +//! - [`hash`]: 128-bit content hashes (file bytes → duplicate detection and cache keys). +//! - [`Lru`]: a cost-bounded memory LRU (decoded sources, rendered thumbnails). +//! - [`DiskCache`]: rendered thumbnails as JPEG files keyed by `(content hash, settings hash, +//! size, renderer version)`, bounded in bytes (least recently used files are pruned). +//! - [`PreviewCache`]: memory LRU in front of the disk cache; shared by render workers. +//! - [`JobPool`]: worker threads with per-slot de-duplication and priorities (visible thumbnails +//! before off-screen prefetch); runs jobs inline on wasm. +//! +//! No UI dependencies (L3): the egui frontend, the CLI and the MCP server share it. +#![forbid(unsafe_code)] + +pub mod disk; +pub mod hash; +pub mod lru; +pub mod pool; + +use std::path::Path; +use std::sync::{Arc, Mutex}; + +pub use disk::DiskCache; +pub use hash::{Hash128, Hasher128, hash_bytes}; +use lightcraft_raster::Rgba8; +pub use lru::Lru; +pub use pool::JobPool; + +/// Rendered-thumbnail cache: memory LRU over an optional disk cache. +pub struct PreviewCache { + mem: Mutex>>, + disk: Option, +} + +impl PreviewCache { + /// Memory-only cache of at most `mem_bytes`. + pub fn memory(mem_bytes: usize) -> PreviewCache { + PreviewCache { mem: Mutex::new(Lru::new(mem_bytes)), disk: None } + } + + /// Memory LRU + disk cache in `dir` (at most `disk_bytes`). + pub fn with_disk(mem_bytes: usize, dir: &Path, disk_bytes: u64) -> PreviewCache { + PreviewCache { mem: Mutex::new(Lru::new(mem_bytes)), disk: Some(DiskCache::new(dir, disk_bytes)) } + } + + pub fn disk(&self) -> Option<&DiskCache> { + self.disk.as_ref() + } + + pub fn get(&self, key: Hash128) -> Option> { + if let Some(v) = self.mem.lock().unwrap_or_else(|e| e.into_inner()).get(&key) { + return Some(v.clone()); + } + let img = Arc::new(self.disk.as_ref()?.get(key)?); + self.mem.lock().unwrap_or_else(|e| e.into_inner()).insert(key, img.clone(), cost(&img)); + Some(img) + } + + pub fn put(&self, key: Hash128, img: Arc) { + if let Some(d) = &self.disk { + d.put(key, &img); + } + let c = cost(&img); + self.mem.lock().unwrap_or_else(|e| e.into_inner()).insert(key, img, c); + } + + /// Drop everything (memory and disk). + pub fn clear(&self) { + self.mem.lock().unwrap_or_else(|e| e.into_inner()).clear(); + if let Some(d) = &self.disk { + d.clear(); + } + } + + /// (entries, bytes) held in memory. + pub fn mem_usage(&self) -> (usize, usize) { + let m = self.mem.lock().unwrap_or_else(|e| e.into_inner()); + (m.len(), m.cost()) + } +} + +fn cost(img: &Rgba8) -> usize { + img.width * img.height * 4 + 64 +} + +#[cfg(test)] +mod tests; diff --git a/crates/preview/src/lru.rs b/crates/preview/src/lru.rs new file mode 100644 index 00000000..1be8bca4 --- /dev/null +++ b/crates/preview/src/lru.rs @@ -0,0 +1,97 @@ +//! A cost-bounded least-recently-used map. + +use std::collections::{BTreeMap, HashMap}; +use std::hash::Hash; + +pub struct Lru { + map: HashMap, + /// Use tick → key, oldest first. + order: BTreeMap, + tick: u64, + cost: usize, + budget: usize, +} + +impl Lru { + /// An LRU holding at most `budget` cost units (e.g. bytes). The most recent entry is always + /// kept, even if it alone exceeds the budget. + pub fn new(budget: usize) -> Self { + Lru { map: HashMap::new(), order: BTreeMap::new(), tick: 0, cost: 0, budget } + } + + fn touch(&mut self, k: &K) { + self.tick += 1; + let t = self.tick; + if let Some(e) = self.map.get_mut(k) { + self.order.remove(&e.2); + e.2 = t; + self.order.insert(t, k.clone()); + } + } + + pub fn get(&mut self, k: &K) -> Option<&V> { + if !self.map.contains_key(k) { + return None; + } + self.touch(k); + self.map.get(k).map(|e| &e.0) + } + + pub fn peek(&self, k: &K) -> Option<&V> { + self.map.get(k).map(|e| &e.0) + } + + pub fn contains(&self, k: &K) -> bool { + self.map.contains_key(k) + } + + pub fn insert(&mut self, k: K, v: V, cost: usize) { + self.remove(&k); + self.tick += 1; + self.order.insert(self.tick, k.clone()); + self.map.insert(k, (v, cost, self.tick)); + self.cost += cost; + while self.cost > self.budget && self.map.len() > 1 { + let Some((_, old)) = self.order.pop_first() else { break }; + if let Some((_, c, _)) = self.map.remove(&old) { + self.cost -= c; + } + } + } + + pub fn remove(&mut self, k: &K) -> Option { + let (v, c, t) = self.map.remove(k)?; + self.order.remove(&t); + self.cost -= c; + Some(v) + } + + pub fn retain(&mut self, mut f: impl FnMut(&K) -> bool) { + let drop: Vec = self.map.keys().filter(|k| !f(k)).cloned().collect(); + for k in drop { + self.remove(&k); + } + } + + pub fn clear(&mut self) { + self.map.clear(); + self.order.clear(); + self.cost = 0; + } + + pub fn len(&self) -> usize { + self.map.len() + } + pub fn is_empty(&self) -> bool { + self.map.is_empty() + } + pub fn cost(&self) -> usize { + self.cost + } + pub fn budget(&self) -> usize { + self.budget + } + pub fn set_budget(&mut self, budget: usize) { + self.budget = budget; + } +} diff --git a/crates/preview/src/pool.rs b/crates/preview/src/pool.rs new file mode 100644 index 00000000..be1bbc64 --- /dev/null +++ b/crates/preview/src/pool.rs @@ -0,0 +1,179 @@ +//! A prioritized job pool with per-slot de-duplication. +//! +//! Each job targets a *slot* (a grid cell, the loupe…). Submitting for a slot replaces a job for +//! that slot that hasn't started yet, so scrolling or dragging never builds a backlog. Workers +//! take the highest priority first (newest first among equals). Frontends re-prioritize or drop +//! queued jobs when the view changes (e.g. thumbnails scrolled out of view). +//! +//! Native: a fixed set of worker threads, started on first use. wasm32 (no threads): the host +//! calls [`JobPool::run_inline`] once per frame. + +use std::hash::Hash; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{Receiver, Sender, channel}; +use std::sync::{Arc, Condvar, Mutex}; + +pub type Job = Box R + Send>; + +struct Entry { + slot: S, + key: u64, + priority: u32, + seq: u64, + job: Job, +} + +struct Shared { + queue: Mutex>>, + cv: Condvar, + shutdown: AtomicBool, +} + +/// A finished job. +pub struct Done { + pub slot: S, + /// The key the job was submitted with (to drop stale results). + pub key: u64, + pub result: R, + /// Run time in milliseconds (0 on wasm). + pub ms: f64, +} + +pub struct JobPool { + shared: Arc>, + tx: Sender>, + rx: Receiver>, + threads: usize, + started: bool, + seq: u64, +} + +impl JobPool { + /// A pool with `threads` workers (none on wasm; `0` = jobs only run via [`JobPool::run_inline`]). + pub fn new(threads: usize) -> Self { + let (tx, rx) = channel(); + JobPool { + shared: Arc::new(Shared { queue: Mutex::new(Vec::new()), cv: Condvar::new(), shutdown: AtomicBool::new(false) }), + tx, + rx, + threads, + started: false, + seq: 0, + } + } + + /// Workers for this machine: cores − 1, between 2 and 8. + pub fn default_threads() -> usize { + #[cfg(not(target_arch = "wasm32"))] + { + std::thread::available_parallelism().map(|n| n.get().saturating_sub(1)).unwrap_or(4).clamp(2, 8) + } + #[cfg(target_arch = "wasm32")] + { + 1 + } + } + + fn start(&mut self) { + if self.started { + return; + } + self.started = true; + #[cfg(not(target_arch = "wasm32"))] + for i in 0..self.threads { + let shared = self.shared.clone(); + let tx = self.tx.clone(); + let _ = std::thread::Builder::new().name(format!("lc-job-{i}")).spawn(move || { + while let Some(e) = take(&shared, true) { + let t0 = std::time::Instant::now(); + let result = (e.job)(); + let done = Done { slot: e.slot, key: e.key, result, ms: t0.elapsed().as_secs_f64() * 1000.0 }; + if tx.send(done).is_err() { + break; + } + } + }); + } + } + + /// Queue `job` for `slot`, replacing a queued (not yet running) job for the same slot. + pub fn submit(&mut self, slot: S, key: u64, priority: u32, job: Job) { + self.start(); + self.seq += 1; + let (lock, cv) = (&self.shared.queue, &self.shared.cv); + let mut q = lock.lock().unwrap_or_else(|e| e.into_inner()); + q.retain(|e| e.slot != slot); + q.push(Entry { slot, key, priority, seq: self.seq, job }); + cv.notify_one(); + } + + /// Change queued jobs' priorities: `f(slot, priority)` returns the new priority, or `None` to + /// drop the job. Returns the dropped slots. + pub fn reprioritize(&self, mut f: impl FnMut(&S, u32) -> Option) -> Vec { + let mut dropped = Vec::new(); + let mut q = self.shared.queue.lock().unwrap_or_else(|e| e.into_inner()); + q.retain_mut(|e| match f(&e.slot, e.priority) { + Some(p) => { + e.priority = p; + true + } + None => { + dropped.push(e.slot); + false + } + }); + dropped + } + + /// Queued (not started) jobs. + pub fn queued(&self) -> usize { + self.shared.queue.lock().map(|q| q.len()).unwrap_or(0) + } + + /// Is a job for `slot` queued? + pub fn is_queued(&self, slot: S) -> bool { + self.shared.queue.lock().map(|q| q.iter().any(|e| e.slot == slot)).unwrap_or(false) + } + + /// A finished job, if any. + pub fn try_recv(&self) -> Option> { + self.rx.try_recv().ok() + } + + /// Run up to `n` jobs on the calling thread (wasm, tests). Returns how many ran. + pub fn run_inline(&mut self, n: usize) -> usize { + let mut ran = 0; + while ran < n { + let Some(e) = take(&self.shared, false) else { break }; + let result = (e.job)(); + let _ = self.tx.send(Done { slot: e.slot, key: e.key, result, ms: 0.0 }); + ran += 1; + } + ran + } +} + +/// Pop the best job (highest priority, newest first). Blocks when `wait` until one is queued or +/// the pool shuts down. +fn take(shared: &Shared, wait: bool) -> Option> { + let mut q = shared.queue.lock().unwrap_or_else(|e| e.into_inner()); + loop { + if shared.shutdown.load(Ordering::Relaxed) { + return None; + } + if let Some(i) = q.iter().enumerate().max_by_key(|(_, e)| (e.priority, e.seq)).map(|(i, _)| i) { + return Some(q.swap_remove(i)); + } + if !wait { + return None; + } + q = shared.cv.wait(q).unwrap_or_else(|e| e.into_inner()); + } +} + +impl Drop for JobPool { + fn drop(&mut self) { + self.shared.shutdown.store(true, Ordering::Relaxed); + self.shared.cv.notify_all(); + } +} diff --git a/crates/preview/src/tests.rs b/crates/preview/src/tests.rs new file mode 100644 index 00000000..038f935a --- /dev/null +++ b/crates/preview/src/tests.rs @@ -0,0 +1,156 @@ +use std::sync::Arc; + +use lightcraft_raster::Rgba8; + +use super::*; + +fn temp_dir(tag: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!("lc-preview-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + d +} + +fn gradient(w: usize, h: usize, seed: u8) -> Rgba8 { + let data = (0..w * h).map(|i| [((i % w) * 255 / w) as u8, ((i / w) * 255 / h) as u8, seed, 255]).collect(); + Rgba8 { width: w, height: h, data } +} + +#[test] +fn hashes_are_stable_and_distinct() { + let a = hash_bytes(b"hello"); + assert_eq!(a, hash_bytes(b"hello")); + assert_ne!(a, hash_bytes(b"hellp")); + assert_eq!(Hash128::parse(&a.to_string()), Some(a)); + assert_ne!(Hasher128::new().str("ab").str("c").finish(), Hasher128::new().str("a").str("bc").finish()); + // streaming == one shot + assert_eq!(Hasher128::new().update(b"hel").update(b"lo").finish(), a); +} + +#[test] +fn lru_evicts_least_recent_by_cost() { + let mut l = Lru::new(10); + l.insert(1, "a", 4); + l.insert(2, "b", 4); + assert_eq!(l.get(&1), Some(&"a")); // 1 is now most recent + l.insert(3, "c", 4); // over budget: evicts 2 + assert!(l.contains(&1) && !l.contains(&2) && l.contains(&3)); + assert_eq!(l.cost(), 8); + l.insert(4, "huge", 50); // alone over budget: kept, everything else evicted + assert_eq!((l.len(), l.cost()), (1, 50)); + l.insert(4, "small", 1); + assert_eq!((l.len(), l.cost()), (1, 1)); + l.retain(|k| *k != 4); + assert!(l.is_empty()); +} + +#[test] +fn disk_cache_roundtrip_and_prune() { + let dir = temp_dir("disk"); + let d = DiskCache::new(&dir, 40_000); + let img = gradient(160, 100, 7); + let k = hash_bytes(b"k1"); + assert!(d.get(k).is_none()); + d.put(k, &img); + let back = d.get(k).expect("hit"); + assert_eq!((back.width, back.height), (160, 100)); + let err: i64 = img.data.iter().zip(&back.data).map(|(a, b)| (a[0] as i64 - b[0] as i64).abs() + (a[1] as i64 - b[1] as i64).abs()).max().unwrap(); + assert!(err < 24, "JPEG error {err}"); + assert!(back.data.iter().all(|p| p[3] == 255)); + // fill past the budget: the oldest entries go, the newest stay + let keys: Vec<_> = (0..40u8).map(|i| hash_bytes(&[i, 1])).collect(); + for (i, k) in keys.iter().enumerate() { + d.put(*k, &gradient(160, 100, i as u8)); + } + assert!(d.size() <= 40_000, "{}", d.size()); + assert!(d.get(*keys.last().unwrap()).is_some()); + assert!(d.get(keys[0]).is_none()); + // a corrupt file is a miss and gets removed + let k2 = hash_bytes(b"corrupt"); + d.put(k2, &img); + let hex = k2.to_string(); + let p = dir.join(&hex[..2]).join(format!("{hex}.jpg")); + std::fs::write(&p, b"not a jpeg").unwrap(); + assert!(d.get(k2).is_none()); + assert!(!p.exists()); + d.clear(); + assert_eq!(d.size(), 0); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn preview_cache_memory_then_disk() { + let dir = temp_dir("pc"); + let k = hash_bytes(b"x"); + { + let c = PreviewCache::with_disk(1 << 20, &dir, 1 << 24); + c.put(k, Arc::new(gradient(64, 48, 1))); + assert_eq!(c.mem_usage().0, 1); + assert!(c.get(k).is_some()); + } + // a new process: memory is empty, disk serves it + let c = PreviewCache::with_disk(1 << 20, &dir, 1 << 24); + assert_eq!(c.mem_usage().0, 0); + assert_eq!(c.get(k).map(|i| i.width), Some(64)); + assert_eq!(c.mem_usage().0, 1); + assert_eq!(c.disk().unwrap().hits.load(std::sync::atomic::Ordering::Relaxed), 1); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn pool_priorities_dedupe_and_drop() { + let mut p: JobPool = JobPool::new(1); + // One worker, held busy by a gate job while the others queue up. + let gate = Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new())); + let g = gate.clone(); + p.submit( + 999, + 0, + 1000, + Box::new(move || { + let (m, cv) = &*g; + let mut open = m.lock().unwrap(); + while !*open { + open = cv.wait(open).unwrap(); + } + 999 + }), + ); + // wait until the worker took the gate job + while p.queued() > 0 { + std::thread::yield_now(); + } + p.submit(1, 1, 5, Box::new(|| 1)); // off-screen thumb + p.submit(2, 1, 10, Box::new(|| 2)); // visible thumb + p.submit(3, 1, 100, Box::new(|| 3)); // loupe + p.submit(2, 2, 10, Box::new(|| 22)); // newer request for slot 2 replaces the queued one + p.submit(4, 1, 5, Box::new(|| 4)); + assert_eq!(p.queued(), 4); + let dropped = p.reprioritize(|s, pr| if *s == 4 { None } else { Some(pr) }); + assert_eq!(dropped, vec![4]); + { + let (m, cv) = &*gate; + *m.lock().unwrap() = true; + cv.notify_all(); + } + let mut order = Vec::new(); + while order.len() < 4 { + if let Some(d) = p.try_recv() { + order.push((d.slot, d.key, d.result)); + } else { + std::thread::yield_now(); + } + } + assert_eq!(order, vec![(999, 0, 999), (3, 1, 3), (2, 2, 22), (1, 1, 1)]); +} + +#[test] +fn pool_runs_inline() { + // No workers: run_inline drains the queue on the caller's thread (the wasm path). + let mut p: JobPool = JobPool::new(0); + p.submit(1, 0, 1, Box::new(|| 1)); + p.submit(2, 0, 9, Box::new(|| 2)); + assert_eq!(p.run_inline(1), 1); + assert_eq!(p.try_recv().map(|d| d.result), Some(2)); + assert_eq!(p.run_inline(10), 1); + assert_eq!(p.try_recv().map(|d| d.result), Some(1)); +} diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml index 7c38bd3b..499dbfdd 100644 --- a/crates/ui-egui/Cargo.toml +++ b/crates/ui-egui/Cargo.toml @@ -17,6 +17,7 @@ lightcraft-pipeline = { workspace = true } lightcraft-catalog = { workspace = true } web-time = { workspace = true } lightcraft-engine = { workspace = true } +lightcraft-preview = { workspace = true } egui = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/ui-egui/src/control.rs b/crates/ui-egui/src/control.rs index 4a7fad6a..97e130f9 100644 --- a/crates/ui-egui/src/control.rs +++ b/crates/ui-egui/src/control.rs @@ -77,7 +77,7 @@ pub fn inspect(app: &LightcraftApp, ctx: &egui::Context) -> Value { "selection": app.session.selection.ids.iter().map(|p| p.0).collect::>(), "activeMask": app.session.active_mask, "widgetCount": app.widgets.len(), - "perf": {"frameMs": app.perf.frame_ms, "fps": app.perf.fps, "lastRenderMs": app.renderer.last_main_ms, "renderQueue": app.renderer.queued(), "rendersDone": app.renderer.completed}, + "perf": {"frameMs": app.perf.frame_ms, "fps": app.perf.fps, "lastRenderMs": app.renderer.last_main_ms, "renderQueue": app.renderer.queued(), "rendersInFlight": app.renderer.in_flight(), "rendersDone": app.renderer.completed, "thumbTextures": app.renderer.thumb_textures()}, "status": app.ui.status, }) } diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index f470f843..0244b4df 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -201,6 +201,7 @@ impl LightcraftApp { ctx.request_repaint(); } self.renderer.poll(ctx, &mut self.session); + self.session.persist_if_dirty(); self.collect_screenshots(ctx); self.issue_screenshots(ctx); if self.fonts_ready { diff --git a/crates/ui-egui/src/panels/detail.rs b/crates/ui-egui/src/panels/detail.rs index 2b10eca8..361c14f8 100644 --- a/crates/ui-egui/src/panels/detail.rs +++ b/crates/ui-egui/src/panels/detail.rs @@ -626,7 +626,7 @@ fn filmstrip(app: &mut LightcraftApp, ui: &mut egui::Ui, r: Rect) { p.text(pos2(cr.right() - 8.0, cr.top() + 10.0), Align2::RIGHT_CENTER, &ph.format, t.semibold(8.5), t.text_dim); } let img_area = Rect::from_min_max(cr.min + vec2(10.0, 22.0), cr.max - vec2(10.0, 8.0)); - if let Some(job) = app.session.render_job(*id, (256.0 * ppp.min(2.0) / 2.0) as usize * 2, 256, false, true) { + if let Some(job) = app.session.thumb_job(*id, (256.0 * ppp.min(2.0) / 2.0) as usize * 2) { app.renderer.request(Slot::Thumb(*id), job, 8); } if let Some(tex) = app.renderer.textures.get(&Slot::Thumb(*id)) { diff --git a/crates/ui-egui/src/panels/grid.rs b/crates/ui-egui/src/panels/grid.rs index 3c4bc15c..6312c0cb 100644 --- a/crates/ui-egui/src/panels/grid.rs +++ b/crates/ui-egui/src/panels/grid.rs @@ -144,7 +144,7 @@ fn cell(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId, r: Rect, square }; // thumbnail let size = thumb_px(img_rect.width().max(img_rect.height()), ppp); - if let Some(job) = app.session.render_job(id, size, size, false, true) { + if let Some(job) = app.session.thumb_job(id, size) { app.renderer.request(Slot::Thumb(id), job, if onscreen { 10 } else { 5 }); } if let Some(tex) = app.renderer.textures.get(&Slot::Thumb(id)) { diff --git a/crates/ui-egui/src/render.rs b/crates/ui-egui/src/render.rs index 90df55e6..ecfe694b 100644 --- a/crates/ui-egui/src/render.rs +++ b/crates/ui-egui/src/render.rs @@ -1,16 +1,17 @@ -//! Background rendering: a small worker pool runs engine [`RenderJob`]s; results become textures. +//! Background rendering: engine [`RenderJob`]s run on a [`JobPool`]; results become textures. //! //! Each visible thing that needs pixels is a *slot* (a grid thumbnail, the loupe, the "before" //! image…). Requests are deduplicated per slot (a newer request replaces a queued older one) and -//! prioritised (loupe first, then visible thumbnails). On wasm the jobs run inline, one per frame. +//! prioritised (loupe first, then on-screen thumbnails, then prefetch); thumbnails scrolled far +//! out of view are dropped from the queue. Thumbnail jobs hit the engine's preview cache (memory + +//! disk) before rendering. On wasm the jobs run inline, one per frame. use std::collections::HashMap; -use std::sync::mpsc::{Receiver, Sender, channel}; -use std::sync::{Arc, Condvar, Mutex}; use lightcraft_catalog::PhotoId; use lightcraft_engine::Session; use lightcraft_engine::media::{RenderJob, RenderResult}; +use lightcraft_preview::JobPool; use lightcraft_raster::Histogram; #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] @@ -30,15 +31,11 @@ pub struct Tex { pub ms: f64, } -type Queue = Arc<(Mutex>, Condvar)>; - pub struct Renderer { - queue: Queue, - results_tx: Sender<(Slot, RenderResult, f64)>, - results: Receiver<(Slot, RenderResult, f64)>, - pending: HashMap, + pool: JobPool, + /// Slot → (key, priority) of the request in flight (queued or running). + pending: HashMap, pub textures: HashMap, - started: bool, pub last_main_ms: f64, /// Jobs finished since start (for inspect/perf). pub completed: u64, @@ -46,104 +43,64 @@ pub struct Renderer { impl Default for Renderer { fn default() -> Self { - let (tx, rx) = channel(); - Renderer { - queue: Arc::new((Mutex::new(Vec::new()), Condvar::new())), - results_tx: tx, - results: rx, - pending: HashMap::new(), - textures: HashMap::new(), - started: false, - last_main_ms: 0.0, - completed: 0, - } + let threads = if cfg!(target_arch = "wasm32") { 0 } else { JobPool::::default_threads().min(6) }; + Renderer { pool: JobPool::new(threads), pending: HashMap::new(), textures: HashMap::new(), last_main_ms: 0.0, completed: 0 } } } -fn now() -> f64 { - crate::now_ms() -} - impl Renderer { - fn start(&mut self) { - if self.started { - return; - } - self.started = true; - #[cfg(not(target_arch = "wasm32"))] - { - let n = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4).clamp(2, 6); - for i in 0..n { - let q = self.queue.clone(); - let tx = self.results_tx.clone(); - let _ = std::thread::Builder::new().name(format!("lc-render-{i}")).spawn(move || { - loop { - let job = { - let (lock, cv) = &*q; - let mut g = lock.lock().unwrap_or_else(|e| e.into_inner()); - while g.is_empty() { - g = cv.wait(g).unwrap_or_else(|e| e.into_inner()); - } - // highest priority, newest first - let best = g.iter().enumerate().max_by_key(|(i, (_, p, _))| (*p, *i)).map(|(i, _)| i).unwrap_or(0); - g.remove(best) - }; - let t0 = now(); - let (slot, _, job) = job; - let r = job.run(); - if tx.send((slot, r, now() - t0)).is_err() { - break; - } - } - }); - } - } - } - /// Is the slot's current texture (or pending request) already for `key`? pub fn is_current(&self, slot: Slot, key: u64) -> bool { - self.textures.get(&slot).is_some_and(|t| t.key == key) || self.pending.get(&slot) == Some(&key) + self.textures.get(&slot).is_some_and(|t| t.key == key) || self.pending.get(&slot).is_some_and(|p| p.0 == key) } - /// Request a render for `slot` (no-op if already current or pending). + /// Request a render for `slot` (no-op if already current or pending at the same priority). pub fn request(&mut self, slot: Slot, job: RenderJob, priority: u32) { - if self.is_current(slot, job.key) { + if self.textures.get(&slot).is_some_and(|t| t.key == job.key) { return; } - self.start(); - self.pending.insert(slot, job.key); - let (lock, cv) = &*self.queue; - let mut g = lock.lock().unwrap_or_else(|e| e.into_inner()); - g.retain(|(s, _, _)| *s != slot); - g.push((slot, priority, job)); - cv.notify_one(); + if let Some(&(key, prio)) = self.pending.get(&slot) + && key == job.key + && (prio == priority || !self.pool.is_queued(slot)) + { + return; + } + self.pending.insert(slot, (job.key, priority)); + let key = job.key; + self.pool.submit(slot, key, priority, Box::new(move || job.run())); } pub fn queued(&self) -> usize { - self.queue.0.lock().map(|g| g.len()).unwrap_or(0) + self.pool.queued() + } + + /// Requests queued or running. + pub fn in_flight(&self) -> usize { + self.pending.len() + } + + /// Thumbnail textures currently loaded. + pub fn thumb_textures(&self) -> usize { + self.textures.keys().filter(|s| matches!(s, Slot::Thumb(_))).count() } /// Collect finished jobs into textures. Returns true if anything changed. pub fn poll(&mut self, ctx: &egui::Context, session: &mut Session) -> bool { + // wasm: run one job per frame on this thread, timed with the host clock #[cfg(target_arch = "wasm32")] - { - // inline: run one job per frame - let job = { - let mut g = self.queue.0.lock().unwrap_or_else(|e| e.into_inner()); - let best = g.iter().enumerate().max_by_key(|(i, (_, p, _))| (*p, *i)).map(|(i, _)| i); - best.map(|b| g.remove(b)) - }; - if let Some((slot, _, job)) = job { - let t0 = now(); - let r = job.run(); - let _ = self.results_tx.send((slot, r, now() - t0)); - } - } + let inline_ms = { + let t0 = crate::now_ms(); + self.pool.run_inline(1); + crate::now_ms() - t0 + }; + #[cfg(not(target_arch = "wasm32"))] + let inline_ms = 0.0; let mut changed = false; - while let Ok((slot, r, ms)) = self.results.try_recv() { + while let Some(done) = self.pool.try_recv() { + let (slot, r, ms) = (done.slot, done.result, if done.ms > 0.0 { done.ms } else { inline_ms }); session.accept(&r); self.completed += 1; - if self.pending.get(&slot) == Some(&r.key) { + if self.pending.get(&slot).is_some_and(|p| p.0 == r.key) { self.pending.remove(&slot); } let Ok(rendered) = r.rendered else { @@ -180,8 +137,16 @@ impl Renderer { changed } - /// Drop thumbnails that aren't in `keep` (bounded memory for huge libraries). + /// Drop thumbnails that aren't in `keep` (bounded memory for huge libraries), and queued + /// thumbnail jobs for photos that scrolled out of `keep`. pub fn evict_thumbs(&mut self, keep: &std::collections::HashSet, max: usize) { + let dropped = self.pool.reprioritize(|s, p| match s { + Slot::Thumb(id) if !keep.contains(id) && p <= 10 => None, + _ => Some(p), + }); + for s in dropped { + self.pending.remove(&s); + } let thumbs = self.textures.keys().filter(|s| matches!(s, Slot::Thumb(_))).count(); if thumbs <= max { return; diff --git a/docs/showcase/README.md b/docs/showcase/README.md index b5e4b167..9d69e7ae 100644 --- a/docs/showcase/README.md +++ b/docs/showcase/README.md @@ -4,11 +4,11 @@ The screenshots in `docs/images/` are captured by driving LightCraft over its co ```sh # public-domain set (downloaded into the gitignored corpus/, see assets/ATTRIBUTION.md for sources) -cargo run --release -p lightcraft -- --control 7980 corpus/images/pd & +cargo run --release -p lightcraft -- --memory --control 7980 corpus/images/pd & python3 docs/showcase/run.py docs/showcase/pd-edits.jsonl # procedural demo library -cargo run --release -p lightcraft -- --control 7980 & +cargo run --release -p lightcraft -- --memory --control 7980 & python3 docs/showcase/run.py docs/showcase/demo.jsonl # compress for the repo (macOS) diff --git a/docs/showcase/demo.jsonl b/docs/showcase/demo.jsonl index ab9b1031..dfb44c38 100644 --- a/docs/showcase/demo.jsonl +++ b/docs/showcase/demo.jsonl @@ -1,4 +1,4 @@ -# Demo-library showcase: run LightCraft with `--control 7980` (no files → procedural demo library) +# Demo-library showcase: run LightCraft with `--memory --control 7980` (no files → procedural demo library) {"method":"ui.resize","params":{"width":1600,"height":1000}} {"method":"engine.execute","params":{"command":"view.photoGrid"}} {"method":"ui.set","params":{"leftPanel":true,"right":"edit","thumbSize":230,"openSections":["light","color"],"presets":false}}