Merge branch 'worktree-agent-a51e3311b37df5c4e'
This commit is contained in:
Brandon Thomas
commit
fa77faeb6b
|
|
@ -2008,7 +2008,9 @@ dependencies = [
|
|||
name = "lightcraft-catalog"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"lightcraft-develop",
|
||||
"log",
|
||||
"proptest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
|
@ -2083,6 +2085,7 @@ dependencies = [
|
|||
"lightcraft-geom",
|
||||
"lightcraft-meta",
|
||||
"lightcraft-pipeline",
|
||||
"lightcraft-preview",
|
||||
"lightcraft-raster",
|
||||
"lightcraft-raw",
|
||||
"lightcraft-scenes",
|
||||
|
|
@ -2138,6 +2141,18 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-preview"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"jpeg-encoder",
|
||||
"lightcraft-raster",
|
||||
"log",
|
||||
"siphasher",
|
||||
"zune-core",
|
||||
"zune-jpeg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lightcraft-raster"
|
||||
version = "0.1.0"
|
||||
|
|
@ -2196,6 +2211,7 @@ dependencies = [
|
|||
"lightcraft-engine",
|
||||
"lightcraft-geom",
|
||||
"lightcraft-pipeline",
|
||||
"lightcraft-preview",
|
||||
"lightcraft-raster",
|
||||
"log",
|
||||
"serde",
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ lightcraft-develop = { path = "crates/develop" }
|
|||
lightcraft-scenes = { path = "crates/scenes" }
|
||||
lightcraft-pipeline = { path = "crates/pipeline" }
|
||||
lightcraft-catalog = { path = "crates/catalog" }
|
||||
lightcraft-preview = { path = "crates/preview" }
|
||||
lightcraft-engine = { path = "crates/engine" }
|
||||
lightcraft-ui-egui = { path = "crates/ui-egui" }
|
||||
lightcraft-mcp = { path = "crates/mcp" }
|
||||
|
|
@ -34,6 +35,8 @@ log = "0.4"
|
|||
rayon = "1"
|
||||
proptest = "1"
|
||||
bytemuck = { version = "1", features = ["derive"] }
|
||||
crc32fast = "1"
|
||||
siphasher = "1"
|
||||
|
||||
# formats
|
||||
quick-xml = "0.38"
|
||||
|
|
|
|||
|
|
@ -235,8 +235,9 @@ LightCraft is young and moving fast — see the [roadmap](ROADMAP.md) for estima
|
|||
|
||||
```sh
|
||||
git clone https://github.com/storytold/lightcraft && cd lightcraft
|
||||
cargo run --release -p lightcraft # opens with a procedurally generated demo library
|
||||
cargo run --release -p lightcraft -- ~/Pictures/trip # or point it at your photos (folders are scanned)
|
||||
cargo run --release -p lightcraft # opens your library (~/Pictures/LightCraft Library; a new one starts with demo photos)
|
||||
cargo run --release -p lightcraft -- ~/Pictures/trip # import your photos (folders are scanned, duplicates skipped)
|
||||
cargo run --release -p lightcraft -- --memory # a throwaway in-memory demo session (writes nothing)
|
||||
cargo run --release -p lightcraft -- --control 7980 # with the automation channel
|
||||
cargo xtask web --serve # the same app in the browser: http://127.0.0.1:8080/
|
||||
cargo run --release -p lightcraft-cli -- render photo.jpg -o out.jpg --set light.exposure=0.5
|
||||
|
|
|
|||
|
|
@ -0,0 +1,45 @@
|
|||
//! Generate a folder of procedural test photos (JPEG) for import/startup benchmarks.
|
||||
//!
|
||||
//! `cargo run --release -p lightcraft --example gen_photos -- OUT_DIR [COUNT=200] [LONG_EDGE=2000]`
|
||||
//!
|
||||
//! Each photo is a `lightcraft-scenes` demo scene rendered through the pipeline with a varied
|
||||
//! look (exposure, temperature, vibrance), so every file has distinct bytes. The output is our own
|
||||
//! procedural content; keep it out of the repository (write to a scratch or gitignored folder).
|
||||
|
||||
use lightcraft_engine::Session;
|
||||
use serde_json::json;
|
||||
|
||||
fn main() {
|
||||
let mut args = std::env::args().skip(1);
|
||||
let out = std::path::PathBuf::from(args.next().expect("usage: gen_photos OUT_DIR [COUNT] [LONG_EDGE]"));
|
||||
let count: usize = args.next().and_then(|c| c.parse().ok()).unwrap_or(200);
|
||||
let edge: usize = args.next().and_then(|c| c.parse().ok()).unwrap_or(2000);
|
||||
std::fs::create_dir_all(&out).expect("create output folder");
|
||||
let mut s = Session::with_demo();
|
||||
let ids = s.visible_cloned();
|
||||
let t0 = std::time::Instant::now();
|
||||
for i in 0..count {
|
||||
let id = ids[i % ids.len()];
|
||||
let round = i / ids.len();
|
||||
s.execute("library.select", &json!({"ids": [id.0]})).expect("select");
|
||||
s.execute("develop.reset", &json!({})).expect("reset");
|
||||
let v = json!({
|
||||
"light.exposure": (round as f64 * 0.37).sin() * 0.8,
|
||||
"wb.temp": 6500.0 + (round as f64 * 1.3).cos() * 1500.0,
|
||||
"color.vibrance": (round * 13 % 50) as f64,
|
||||
});
|
||||
let _ = s.execute("develop.set", &json!({"values": v}));
|
||||
let img = s.render_now(id, edge, edge).expect("render");
|
||||
let bytes = lightcraft_codecs::encode_jpeg(
|
||||
&lightcraft_codecs::EncodeImage::rgba8(&img.image),
|
||||
88,
|
||||
lightcraft_codecs::ChromaSubsampling::S420,
|
||||
&lightcraft_codecs::EncodeMeta::default(),
|
||||
)
|
||||
.expect("encode");
|
||||
let dir = out.join(format!("roll-{:02}", i / 50 + 1));
|
||||
std::fs::create_dir_all(&dir).expect("mkdir");
|
||||
std::fs::write(dir.join(format!("LCG{:05}.jpg", i + 1)), bytes).expect("write");
|
||||
}
|
||||
eprintln!("wrote {count} photos ({edge} px) to {} in {:.1} s", out.display(), t0.elapsed().as_secs_f64());
|
||||
}
|
||||
|
|
@ -1,6 +1,12 @@
|
|||
//! LightCraft desktop app.
|
||||
//!
|
||||
//! Usage: `lightcraft [--control <port>] [--demo] [files or folders…]`
|
||||
//! Usage: `lightcraft [--library DIR | --memory] [--no-demo] [--control <port>] [files or folders…]`
|
||||
//!
|
||||
//! The library (catalog, presets, thumbnail cache) lives in `--library DIR`, else
|
||||
//! `$LIGHTCRAFT_LIBRARY`, else `~/Pictures/LightCraft Library`; a new library starts with the
|
||||
//! procedural demo photos unless `--no-demo` or files are given. Files and folders on the command
|
||||
//! line are imported (duplicates are skipped). `--memory` runs an in-memory session that writes
|
||||
//! nothing (demo photos unless files are given; used by the README showcase scripts).
|
||||
//!
|
||||
//! `--control <port>` (or `LIGHTCRAFT_CONTROL_PORT`) starts a localhost JSON-lines control server:
|
||||
//! `{"id":1,"method":"ui.inspect","params":{}}` → `{"id":1,"ok":true,"result":…}`.
|
||||
|
|
@ -25,6 +31,9 @@ impl eframe::App for App {
|
|||
}
|
||||
fn on_exit(&mut self) {
|
||||
save_prefs(&self.0);
|
||||
if let Err(e) = self.0.session.close_library() {
|
||||
eprintln!("lightcraft: saving the library failed: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -88,40 +97,55 @@ fn services() -> Services {
|
|||
}
|
||||
}
|
||||
|
||||
/// Expand folders (recursively) into photo files.
|
||||
fn collect(path: &std::path::Path, out: &mut Vec<String>) {
|
||||
if path.is_dir() {
|
||||
if let Ok(rd) = std::fs::read_dir(path) {
|
||||
let mut v: Vec<_> = rd.flatten().map(|e| e.path()).collect();
|
||||
v.sort();
|
||||
for p in v {
|
||||
collect(&p, out);
|
||||
}
|
||||
/// The persistent library session (or an in-memory one with `--memory` / if the library can't open).
|
||||
fn open_session(in_memory: bool, dir: Option<std::path::PathBuf>, seed_demo: bool) -> Session {
|
||||
let fallback = || if seed_demo { Session::with_demo() } else { Session::new() }.with_fs().with_system_clock();
|
||||
if in_memory {
|
||||
return fallback();
|
||||
}
|
||||
let Some(dir) = dir else {
|
||||
eprintln!("lightcraft: no library location (set --library or LIGHTCRAFT_LIBRARY); running in memory");
|
||||
return fallback();
|
||||
};
|
||||
let t0 = std::time::Instant::now();
|
||||
let mut s = Session::new().with_fs().with_system_clock();
|
||||
match s.open_library(&dir, seed_demo) {
|
||||
Ok(r) => {
|
||||
let (replayed, torn) = (r.replayed, r.torn_bytes);
|
||||
eprintln!(
|
||||
"lightcraft: library {}: {} photos, {replayed} log records replayed{} ({:.1} ms)",
|
||||
dir.display(),
|
||||
s.catalog.len(),
|
||||
if torn > 0 { ", torn tail repaired" } else { "" },
|
||||
t0.elapsed().as_secs_f64() * 1000.0
|
||||
);
|
||||
s
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("lightcraft: can't open library {}: {e}; running in memory", dir.display());
|
||||
fallback()
|
||||
}
|
||||
} else if path.extension().is_some_and(|e| {
|
||||
matches!(
|
||||
e.to_string_lossy().to_lowercase().as_str(),
|
||||
"jpg" | "jpeg" | "png" | "tif" | "tiff" | "webp" | "dng" | "cr2" | "nef" | "arw" | "psd" | "jxl" | "gif" | "bmp"
|
||||
)
|
||||
}) {
|
||||
out.push(path.to_string_lossy().to_string());
|
||||
}
|
||||
}
|
||||
|
||||
fn main() -> eframe::Result {
|
||||
let mut control_port: Option<u16> = std::env::var("LIGHTCRAFT_CONTROL_PORT").ok().and_then(|p| p.parse().ok());
|
||||
let mut files = Vec::new();
|
||||
let mut demo = true;
|
||||
let mut seed_demo = true;
|
||||
let mut in_memory = false;
|
||||
let mut library_dir = lightcraft_engine::library::default_dir();
|
||||
let mut args = std::env::args().skip(1);
|
||||
while let Some(a) = args.next() {
|
||||
match a.as_str() {
|
||||
"--control" => control_port = args.next().and_then(|p| p.parse().ok()),
|
||||
"--no-demo" => demo = false,
|
||||
"--library" => library_dir = args.next().map(std::path::PathBuf::from),
|
||||
"--no-demo" => seed_demo = false,
|
||||
"--memory" | "--demo" => in_memory = true,
|
||||
"--version" => {
|
||||
println!("lightcraft {}", env!("CARGO_PKG_VERSION"));
|
||||
return Ok(());
|
||||
}
|
||||
_ => collect(std::path::Path::new(&a), &mut files),
|
||||
_ => files.push(a),
|
||||
}
|
||||
}
|
||||
let options = eframe::NativeOptions {
|
||||
|
|
@ -139,7 +163,7 @@ fn main() -> eframe::Result {
|
|||
"LightCraft",
|
||||
options,
|
||||
Box::new(move |cc| {
|
||||
let session = if demo && files.is_empty() { Session::with_demo() } else { Session::new() }.with_fs();
|
||||
let session = open_session(in_memory, library_dir, seed_demo && files.is_empty());
|
||||
let mut app = LightcraftApp::new(session, services());
|
||||
load_prefs(&mut app);
|
||||
app.integrated_titlebar = cfg!(target_os = "macos");
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@ lightcraft-develop = { workspace = true }
|
|||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
crc32fast = { workspace = true }
|
||||
log = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
proptest = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -0,0 +1,240 @@
|
|||
//! Crash-safe catalog persistence: an append-only op log plus periodic snapshots.
|
||||
//!
|
||||
//! Files (in a [`Store`]):
|
||||
//! - `catalog.snap` — `{"format":"lightcraft-catalog","version":1,"seq":N,"catalog":{…}}`, the
|
||||
//! state after op `N`; replaced atomically (temp + fsync + rename).
|
||||
//! - `catalog.log` — JSON lines, one per op applied after the snapshot:
|
||||
//! `{"seq":N,"crc":C,"op":{…}}` where `C` is the CRC-32 of the op's JSON text. Appends are
|
||||
//! fsynced before [`Journal::append`] returns.
|
||||
//!
|
||||
//! Loading = snapshot + replay of the records with `seq > snapshot seq`. Recovery rules:
|
||||
//! - a **torn final record** (crash mid-append: truncated line or bad CRC at the end) is dropped
|
||||
//! and the file is cut back to the last good record, so later appends start on a clean line;
|
||||
//! - records already covered by the snapshot (crash between writing the snapshot and resetting
|
||||
//! the log) are skipped;
|
||||
//! - a bad record **followed by good ones** is real damage: replay stops there, the log is kept
|
||||
//! as `catalog.log.damaged-<seq>`, and a fresh snapshot is written so the library stays usable.
|
||||
|
||||
use crate::store::Store;
|
||||
use crate::{Catalog, CatalogError, Op, Result};
|
||||
|
||||
pub const SNAPSHOT: &str = "catalog.snap";
|
||||
pub const LOG: &str = "catalog.log";
|
||||
const FORMAT: &str = "lightcraft-catalog";
|
||||
const VERSION: u32 = 1;
|
||||
|
||||
/// When [`Journal::wants_snapshot`] says it's time to compact the log.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct SnapshotPolicy {
|
||||
pub max_records: u64,
|
||||
pub max_bytes: u64,
|
||||
}
|
||||
|
||||
impl Default for SnapshotPolicy {
|
||||
fn default() -> Self {
|
||||
SnapshotPolicy { max_records: 2000, max_bytes: 16 << 20 }
|
||||
}
|
||||
}
|
||||
|
||||
/// What happened while loading.
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct LoadReport {
|
||||
/// `seq` of the snapshot that was loaded (0 = none).
|
||||
pub snapshot_seq: u64,
|
||||
/// Log records applied on top of the snapshot.
|
||||
pub replayed: usize,
|
||||
/// Records skipped because the snapshot already contained them.
|
||||
pub stale: usize,
|
||||
/// Records whose op failed to apply (should never happen; reported, not fatal).
|
||||
pub failed: usize,
|
||||
/// Bytes of a torn final record that were dropped.
|
||||
pub torn_bytes: u64,
|
||||
/// The damaged log was preserved under this name.
|
||||
pub damaged: Option<String>,
|
||||
/// No catalog files existed (a new library).
|
||||
pub created: bool,
|
||||
}
|
||||
|
||||
pub struct Journal {
|
||||
store: Box<dyn Store>,
|
||||
/// `seq` of the last durable op.
|
||||
seq: u64,
|
||||
snapshot_seq: u64,
|
||||
log_records: u64,
|
||||
log_bytes: u64,
|
||||
pub policy: SnapshotPolicy,
|
||||
}
|
||||
|
||||
fn io(e: std::io::Error) -> CatalogError {
|
||||
CatalogError::Io(e.to_string())
|
||||
}
|
||||
|
||||
/// One log line (without the newline) for op number `seq`.
|
||||
pub fn encode_record(seq: u64, op: &Op) -> String {
|
||||
let body = serde_json::to_string(op).unwrap_or_default();
|
||||
let crc = crc32fast::hash(body.as_bytes());
|
||||
format!("{{\"seq\":{seq},\"crc\":{crc},\"op\":{body}}}")
|
||||
}
|
||||
|
||||
/// Parse one log line (without the newline). `None` if it's malformed or the CRC doesn't match.
|
||||
pub fn decode_record(line: &str) -> Option<(u64, Op)> {
|
||||
let rest = line.strip_prefix("{\"seq\":")?;
|
||||
let (seq, rest) = rest.split_once(",\"crc\":")?;
|
||||
let (crc, rest) = rest.split_once(",\"op\":")?;
|
||||
let body = rest.strip_suffix('}')?;
|
||||
let seq: u64 = seq.parse().ok()?;
|
||||
let crc: u32 = crc.parse().ok()?;
|
||||
if crc32fast::hash(body.as_bytes()) != crc {
|
||||
return None;
|
||||
}
|
||||
serde_json::from_str(body).ok().map(|op| (seq, op))
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct SnapFile {
|
||||
format: String,
|
||||
version: u32,
|
||||
seq: u64,
|
||||
catalog: Catalog,
|
||||
}
|
||||
|
||||
impl Journal {
|
||||
/// Open (or create) the catalog in `store`: load the snapshot, replay the log, repair a torn
|
||||
/// tail. Fails only if the snapshot itself is unreadable (then nothing is modified).
|
||||
pub fn open(mut store: Box<dyn Store>) -> Result<(Journal, Catalog, LoadReport)> {
|
||||
let mut report = LoadReport::default();
|
||||
let snap = store.read(SNAPSHOT).map_err(io)?;
|
||||
let log = store.read(LOG).map_err(io)?;
|
||||
report.created = snap.is_none() && log.is_none();
|
||||
let (mut catalog, snapshot_seq) = match snap {
|
||||
Some(bytes) => {
|
||||
let s: SnapFile = serde_json::from_slice(&bytes).map_err(|e| CatalogError::Corrupt(format!("{SNAPSHOT}: {e}")))?;
|
||||
if s.format != FORMAT || s.version > VERSION {
|
||||
return Err(CatalogError::Corrupt(format!("{SNAPSHOT}: unsupported format {} v{}", s.format, s.version)));
|
||||
}
|
||||
(s.catalog, s.seq)
|
||||
}
|
||||
None => (Catalog::new(), 0),
|
||||
};
|
||||
report.snapshot_seq = snapshot_seq;
|
||||
let mut j = Journal { store, seq: snapshot_seq, snapshot_seq, log_records: 0, log_bytes: 0, policy: SnapshotPolicy::default() };
|
||||
let log = log.unwrap_or_default();
|
||||
|
||||
// Scan records; `good_end` is the byte offset just past the last good record.
|
||||
let mut pos = 0usize;
|
||||
let mut good_end = 0usize;
|
||||
let mut damaged_at: Option<usize> = None;
|
||||
let mut needs_newline = false;
|
||||
while pos < log.len() {
|
||||
let (line_end, next, has_nl) = match log[pos..].iter().position(|b| *b == b'\n') {
|
||||
Some(i) => (pos + i, pos + i + 1, true),
|
||||
None => (log.len(), log.len(), false),
|
||||
};
|
||||
let line = std::str::from_utf8(&log[pos..line_end]).ok().map(|l| l.trim_end_matches('\r'));
|
||||
if line.is_some_and(|l| l.trim().is_empty()) {
|
||||
pos = next;
|
||||
good_end = next;
|
||||
continue;
|
||||
}
|
||||
match line.and_then(decode_record) {
|
||||
Some((seq, op)) if seq <= j.seq => {
|
||||
// already in the snapshot
|
||||
let _ = op;
|
||||
report.stale += 1;
|
||||
}
|
||||
Some((seq, op)) if seq == j.seq + 1 => {
|
||||
if catalog.apply(op).is_err() {
|
||||
report.failed += 1;
|
||||
} else {
|
||||
report.replayed += 1;
|
||||
}
|
||||
j.seq = seq;
|
||||
}
|
||||
_ => {
|
||||
// bad record (or a gap): torn tail if nothing good follows, else damage
|
||||
let rest_has_good = log[next..].split(|b| *b == b'\n').any(|l| std::str::from_utf8(l).ok().and_then(decode_record).is_some());
|
||||
if rest_has_good {
|
||||
damaged_at = Some(pos);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
j.log_records += 1;
|
||||
pos = next;
|
||||
good_end = next;
|
||||
needs_newline = !has_nl;
|
||||
}
|
||||
|
||||
if let Some(at) = damaged_at {
|
||||
let name = format!("{LOG}.damaged-{}", j.seq);
|
||||
j.store.write_atomic(&name, &log).map_err(io)?;
|
||||
log::warn!("catalog log damaged at byte {at}; kept as {name}; state recovered up to op {}", j.seq);
|
||||
report.damaged = Some(name);
|
||||
j.snapshot(&catalog)?;
|
||||
} else {
|
||||
if good_end < log.len() {
|
||||
report.torn_bytes = (log.len() - good_end) as u64;
|
||||
log::warn!("catalog log: dropped a torn final record ({} bytes)", report.torn_bytes);
|
||||
j.store.truncate(LOG, good_end as u64).map_err(io)?;
|
||||
} else if needs_newline {
|
||||
j.store.append(LOG, b"\n").map_err(io)?;
|
||||
}
|
||||
j.log_bytes = good_end as u64;
|
||||
}
|
||||
catalog.revision = 0;
|
||||
Ok((j, catalog, report))
|
||||
}
|
||||
|
||||
/// Append ops (already applied to the live catalog) durably, in order.
|
||||
pub fn append(&mut self, ops: &[Op]) -> Result<()> {
|
||||
if ops.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut buf = String::new();
|
||||
let mut seq = self.seq;
|
||||
for op in ops {
|
||||
seq += 1;
|
||||
buf.push_str(&encode_record(seq, op));
|
||||
buf.push('\n');
|
||||
}
|
||||
self.store.append(LOG, buf.as_bytes()).map_err(io)?;
|
||||
self.seq = seq;
|
||||
self.log_records += ops.len() as u64;
|
||||
self.log_bytes += buf.len() as u64;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Write a snapshot of `catalog` (which must reflect every appended op) and reset the log.
|
||||
pub fn snapshot(&mut self, catalog: &Catalog) -> Result<()> {
|
||||
let body = serde_json::to_string(catalog).map_err(|e| CatalogError::Invalid(e.to_string()))?;
|
||||
let file = format!("{{\"format\":\"{FORMAT}\",\"version\":{VERSION},\"seq\":{},\"catalog\":{body}}}\n", self.seq);
|
||||
self.store.write_atomic(SNAPSHOT, file.as_bytes()).map_err(io)?;
|
||||
// A crash here leaves old records in the log; they are skipped by seq on load.
|
||||
self.store.write_atomic(LOG, b"").map_err(io)?;
|
||||
self.snapshot_seq = self.seq;
|
||||
self.log_records = 0;
|
||||
self.log_bytes = 0;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The log is long enough to be worth compacting.
|
||||
pub fn wants_snapshot(&self) -> bool {
|
||||
self.log_records >= self.policy.max_records || self.log_bytes >= self.policy.max_bytes
|
||||
}
|
||||
|
||||
pub fn seq(&self) -> u64 {
|
||||
self.seq
|
||||
}
|
||||
pub fn snapshot_seq(&self) -> u64 {
|
||||
self.snapshot_seq
|
||||
}
|
||||
pub fn log_records(&self) -> u64 {
|
||||
self.log_records
|
||||
}
|
||||
pub fn log_bytes(&self) -> u64 {
|
||||
self.log_bytes
|
||||
}
|
||||
pub fn describe(&self) -> String {
|
||||
self.store.describe()
|
||||
}
|
||||
}
|
||||
|
|
@ -2,21 +2,25 @@
|
|||
//!
|
||||
//! State changes only through [`Op`]s. [`Catalog::apply`] returns the inverse op, which gives:
|
||||
//! - **persistence**: ops are appended to a log (JSON lines) and replayed on load after the last
|
||||
//! snapshot — crash-safe and diff-friendly;
|
||||
//! snapshot — crash-safe and diff-friendly (see [`journal`]);
|
||||
//! - **undo/redo**: the engine keeps inverse ops;
|
||||
//! - **determinism**: replaying the log reproduces the state exactly (property-tested).
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
pub mod journal;
|
||||
pub mod model;
|
||||
pub mod query;
|
||||
pub mod store;
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
pub use journal::{Journal, LoadReport, SnapshotPolicy};
|
||||
use lightcraft_develop::DevelopSettings;
|
||||
pub use model::*;
|
||||
pub use query::{DateGroup, Filter, RatingOp, Sort, SortKey};
|
||||
use serde::{Deserialize, Serialize};
|
||||
pub use store::{FsStore, MemStore, Store};
|
||||
|
||||
#[derive(Debug, thiserror::Error, PartialEq)]
|
||||
pub enum CatalogError {
|
||||
|
|
@ -28,10 +32,15 @@ pub enum CatalogError {
|
|||
Invalid(String),
|
||||
#[error("corrupt catalog data: {0}")]
|
||||
Corrupt(String),
|
||||
#[error("catalog storage: {0}")]
|
||||
Io(String),
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, CatalogError>;
|
||||
|
||||
/// Maximum History entries kept per photo.
|
||||
pub const HISTORY_LIMIT: usize = 200;
|
||||
|
||||
/// Every catalog mutation.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(tag = "op", rename_all = "camelCase")]
|
||||
|
|
@ -76,6 +85,12 @@ pub enum Op {
|
|||
id: PhotoId,
|
||||
history: Vec<HistoryStep>,
|
||||
},
|
||||
/// Append one History entry (dropping the oldest beyond [`HISTORY_LIMIT`]). Logged instead of
|
||||
/// a full `SetHistory` so each edit costs one step in the op log.
|
||||
PushHistory {
|
||||
id: PhotoId,
|
||||
step: HistoryStep,
|
||||
},
|
||||
AddAlbum {
|
||||
album: Album,
|
||||
},
|
||||
|
|
@ -227,6 +242,16 @@ impl Catalog {
|
|||
let p = self.photo_mut(id)?;
|
||||
Op::SetHistory { id, history: std::mem::replace(&mut p.history, history) }
|
||||
}
|
||||
Op::PushHistory { id, step } => {
|
||||
let p = self.photo_mut(id)?;
|
||||
let old = p.history.clone();
|
||||
p.history.push(step);
|
||||
if p.history.len() > HISTORY_LIMIT {
|
||||
let n = p.history.len() - HISTORY_LIMIT;
|
||||
p.history.drain(..n);
|
||||
}
|
||||
Op::SetHistory { id, history: old }
|
||||
}
|
||||
Op::AddAlbum { album } => {
|
||||
if self.albums.contains_key(&album.id) {
|
||||
return Err(CatalogError::Invalid(format!("album {:?} exists", album.id)));
|
||||
|
|
@ -352,3 +377,5 @@ impl Catalog {
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
#[cfg(test)]
|
||||
mod tests_journal;
|
||||
|
|
|
|||
|
|
@ -143,6 +143,9 @@ pub struct Photo {
|
|||
/// Raw files: the camera's as-shot white balance (Kelvin, tint).
|
||||
#[serde(default)]
|
||||
pub as_shot_wb: Option<(f64, f64)>,
|
||||
/// Hash of the file's bytes (hex), for duplicate detection and preview-cache keys.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub content_hash: Option<String>,
|
||||
}
|
||||
|
||||
impl Photo {
|
||||
|
|
@ -169,6 +172,7 @@ impl Photo {
|
|||
deleted: false,
|
||||
duration: None,
|
||||
as_shot_wb: None,
|
||||
content_hash: None,
|
||||
}
|
||||
}
|
||||
pub fn is_edited(&self) -> bool {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,148 @@
|
|||
//! Byte storage for the catalog files (`catalog.snap`, `catalog.log`).
|
||||
//!
|
||||
//! [`FsStore`] is the native implementation (a library directory); [`MemStore`] keeps files in
|
||||
//! memory (tests, ephemeral sessions) and lets tests simulate crashes by editing the bytes. A web
|
||||
//! host can implement [`Store`] over OPFS.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
/// A flat namespace of named files with the few operations the journal needs.
|
||||
pub trait Store: Send {
|
||||
/// The whole file, or `None` if it doesn't exist.
|
||||
fn read(&mut self, name: &str) -> io::Result<Option<Vec<u8>>>;
|
||||
/// Replace a file atomically: after a crash either the old or the new content is visible,
|
||||
/// never a mix (native: write temp + fsync + rename + fsync directory).
|
||||
fn write_atomic(&mut self, name: &str, data: &[u8]) -> io::Result<()>;
|
||||
/// Append and make durable (fsync) before returning.
|
||||
fn append(&mut self, name: &str, data: &[u8]) -> io::Result<()>;
|
||||
/// Cut a file to `len` bytes (drop a torn tail before appending again).
|
||||
fn truncate(&mut self, name: &str, len: u64) -> io::Result<()>;
|
||||
/// Human-readable location (diagnostics).
|
||||
fn describe(&self) -> String;
|
||||
}
|
||||
|
||||
/// Files in a directory.
|
||||
pub struct FsStore {
|
||||
dir: PathBuf,
|
||||
/// Open append handle (kept between appends; dropped on rewrite/truncate).
|
||||
appender: Option<(String, std::fs::File)>,
|
||||
}
|
||||
|
||||
impl FsStore {
|
||||
/// Use `dir` (created if missing).
|
||||
pub fn open(dir: impl AsRef<Path>) -> io::Result<FsStore> {
|
||||
std::fs::create_dir_all(dir.as_ref())?;
|
||||
Ok(FsStore { dir: dir.as_ref().to_path_buf(), appender: None })
|
||||
}
|
||||
pub fn dir(&self) -> &Path {
|
||||
&self.dir
|
||||
}
|
||||
fn path(&self, name: &str) -> PathBuf {
|
||||
self.dir.join(name)
|
||||
}
|
||||
fn sync_dir(&self) {
|
||||
// Makes the rename durable on POSIX; opening a directory fails on Windows (where
|
||||
// `rename` is already journaled by NTFS), so errors are ignored.
|
||||
if let Ok(d) = std::fs::File::open(&self.dir) {
|
||||
let _ = d.sync_all();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Store for FsStore {
|
||||
fn read(&mut self, name: &str) -> io::Result<Option<Vec<u8>>> {
|
||||
match std::fs::read(self.path(name)) {
|
||||
Ok(b) => Ok(Some(b)),
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
fn write_atomic(&mut self, name: &str, data: &[u8]) -> io::Result<()> {
|
||||
use std::io::Write;
|
||||
if self.appender.as_ref().is_some_and(|(n, _)| n == name) {
|
||||
self.appender = None;
|
||||
}
|
||||
let tmp = self.path(&format!("{name}.tmp"));
|
||||
{
|
||||
let mut f = std::fs::File::create(&tmp)?;
|
||||
f.write_all(data)?;
|
||||
f.sync_all()?;
|
||||
}
|
||||
std::fs::rename(&tmp, self.path(name))?;
|
||||
self.sync_dir();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn append(&mut self, name: &str, data: &[u8]) -> io::Result<()> {
|
||||
use std::io::Write;
|
||||
if self.appender.as_ref().is_none_or(|(n, _)| n != name) {
|
||||
let existed = self.path(name).exists();
|
||||
let f = std::fs::OpenOptions::new().create(true).append(true).open(self.path(name))?;
|
||||
if !existed {
|
||||
self.sync_dir();
|
||||
}
|
||||
self.appender = Some((name.to_string(), f));
|
||||
}
|
||||
let (_, f) = self.appender.as_mut().expect("appender just opened");
|
||||
f.write_all(data)?;
|
||||
f.sync_data()
|
||||
}
|
||||
|
||||
fn truncate(&mut self, name: &str, len: u64) -> io::Result<()> {
|
||||
if self.appender.as_ref().is_some_and(|(n, _)| n == name) {
|
||||
self.appender = None;
|
||||
}
|
||||
let f = std::fs::OpenOptions::new().write(true).open(self.path(name))?;
|
||||
f.set_len(len)?;
|
||||
f.sync_all()
|
||||
}
|
||||
|
||||
fn describe(&self) -> String {
|
||||
self.dir.display().to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// In-memory files. Clones share the same files (so a test can "reopen" or corrupt them).
|
||||
#[derive(Clone, Default)]
|
||||
pub struct MemStore {
|
||||
pub files: Arc<Mutex<BTreeMap<String, Vec<u8>>>>,
|
||||
}
|
||||
|
||||
impl MemStore {
|
||||
pub fn new() -> MemStore {
|
||||
MemStore::default()
|
||||
}
|
||||
pub fn get(&self, name: &str) -> Option<Vec<u8>> {
|
||||
self.files.lock().unwrap_or_else(|e| e.into_inner()).get(name).cloned()
|
||||
}
|
||||
pub fn set(&self, name: &str, data: Vec<u8>) {
|
||||
self.files.lock().unwrap_or_else(|e| e.into_inner()).insert(name.to_string(), data);
|
||||
}
|
||||
}
|
||||
|
||||
impl Store for MemStore {
|
||||
fn read(&mut self, name: &str) -> io::Result<Option<Vec<u8>>> {
|
||||
Ok(self.get(name))
|
||||
}
|
||||
fn write_atomic(&mut self, name: &str, data: &[u8]) -> io::Result<()> {
|
||||
self.set(name, data.to_vec());
|
||||
Ok(())
|
||||
}
|
||||
fn append(&mut self, name: &str, data: &[u8]) -> io::Result<()> {
|
||||
self.files.lock().unwrap_or_else(|e| e.into_inner()).entry(name.to_string()).or_default().extend_from_slice(data);
|
||||
Ok(())
|
||||
}
|
||||
fn truncate(&mut self, name: &str, len: u64) -> io::Result<()> {
|
||||
if let Some(f) = self.files.lock().unwrap_or_else(|e| e.into_inner()).get_mut(name) {
|
||||
f.truncate(len as usize);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn describe(&self) -> String {
|
||||
"memory".into()
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,230 @@
|
|||
//! Journal (op log + snapshot) tests: replay(log) == state, torn tails, stale logs, damage.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use lightcraft_develop::DevelopSettings;
|
||||
use proptest::prelude::*;
|
||||
|
||||
use crate::journal::{LOG, SNAPSHOT, decode_record, encode_record};
|
||||
use crate::*;
|
||||
|
||||
fn open(m: &MemStore) -> (Journal, Catalog, LoadReport) {
|
||||
Journal::open(Box::new(m.clone())).unwrap()
|
||||
}
|
||||
|
||||
/// Interpret a random action against the live catalog as an op (may be invalid: then the live
|
||||
/// apply fails and, like in the engine, nothing is logged).
|
||||
fn op_for(c: &mut Catalog, kind: u8, a: u8, b: u8) -> Op {
|
||||
let photos: Vec<PhotoId> = c.photos().map(|p| p.id).collect();
|
||||
let albums: Vec<AlbumId> = c.albums().map(|a| a.id).collect();
|
||||
let pid = photos.get(a as usize % photos.len().max(1)).copied().unwrap_or(PhotoId(99));
|
||||
let aid = albums.get(a as usize % albums.len().max(1)).copied().unwrap_or(AlbumId(99));
|
||||
match kind % 10 {
|
||||
0 | 1 => {
|
||||
let id = c.alloc_photo_id();
|
||||
let mut p = Photo::new(id, Source::File { path: format!("/p/{}.jpg", id.0) }, &format!("{}.jpg", id.0), "JPEG", 60, 40, "2026-01-01");
|
||||
p.content_hash = Some(format!("{:032x}", id.0 * 7919));
|
||||
Op::AddPhoto { photo: Box::new(p) }
|
||||
}
|
||||
2 => Op::SetRating { id: pid, rating: b % 7 },
|
||||
3 => Op::SetFlag { id: pid, flag: [Flag::None, Flag::Pick, Flag::Reject][b as usize % 3] },
|
||||
4 => {
|
||||
let mut d = DevelopSettings::default();
|
||||
d.light.exposure = b as f64 / 37.0 - 2.0;
|
||||
d.light.contrast = -(a as f64) * 0.37;
|
||||
let settings = Arc::new(d);
|
||||
Op::Batch {
|
||||
ops: vec![
|
||||
Op::SetDevelop { id: pid, settings: settings.clone(), label: "Exposure".into(), edited: Some(format!("t{b}")) },
|
||||
Op::PushHistory { id: pid, step: HistoryStep { label: "Exposure".into(), settings } },
|
||||
],
|
||||
}
|
||||
}
|
||||
5 => {
|
||||
let id = c.alloc_album_id();
|
||||
Op::AddAlbum { album: Album { id, name: format!("A{b}"), parent: None, folder: b.is_multiple_of(4), photos: vec![], cover: None } }
|
||||
}
|
||||
6 => Op::SetAlbumPhotos { id: aid, photos: photos.iter().copied().filter(|p| (p.0 + b as u64).is_multiple_of(3)).collect() },
|
||||
7 => Op::RenameAlbum { id: aid, name: format!("Renamed {b} ✓") },
|
||||
8 => Op::SetDeleted { id: pid, deleted: b.is_multiple_of(2) },
|
||||
_ => c.delete_permanently_ops(pid),
|
||||
}
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#![proptest_config(ProptestConfig::with_cases(64))]
|
||||
|
||||
/// replay(snapshot + log) == live state, with snapshots at random points and reopening
|
||||
/// (resuming appends) in between.
|
||||
#[test]
|
||||
fn replay_equals_state(actions in proptest::collection::vec((0u8..20, 0u8..255, 0u8..255, 0u8..12), 1..80)) {
|
||||
let m = MemStore::new();
|
||||
let (mut j, mut live, r) = open(&m);
|
||||
prop_assert!(r.created);
|
||||
for (kind, a, b, ctl) in actions {
|
||||
let op = op_for(&mut live, kind, a, b);
|
||||
if live.apply(op.clone()).is_ok() {
|
||||
j.append(&[op]).unwrap();
|
||||
}
|
||||
match ctl {
|
||||
0 => j.snapshot(&live).unwrap(),
|
||||
1 => {
|
||||
let (j2, c2, _) = open(&m);
|
||||
prop_assert_eq!(c2.to_snapshot(), live.to_snapshot());
|
||||
j = j2;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let (_, loaded, r) = open(&m);
|
||||
prop_assert_eq!(r.failed, 0);
|
||||
prop_assert_eq!(loaded.to_snapshot(), live.to_snapshot());
|
||||
// allocation continues where it left off
|
||||
prop_assert_eq!(loaded.clone().alloc_photo_id(), live.clone().alloc_photo_id());
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_catalog_with_log() -> (MemStore, Catalog, Catalog) {
|
||||
let m = MemStore::new();
|
||||
let (mut j, mut c, _) = open(&m);
|
||||
for k in 0..6u8 {
|
||||
let op = op_for(&mut c, k, k, k * 3);
|
||||
if c.apply(op.clone()).is_ok() {
|
||||
j.append(&[op]).unwrap();
|
||||
}
|
||||
}
|
||||
let before_last = c.clone();
|
||||
let op = Op::SetRating { id: c.photos().next().unwrap().id, rating: 5 };
|
||||
c.apply(op.clone()).unwrap();
|
||||
j.append(&[op]).unwrap();
|
||||
(m, before_last, c)
|
||||
}
|
||||
|
||||
/// A crash mid-append: every possible truncation of the last record still loads (to the state
|
||||
/// before that op), the torn bytes are cut off, and appending afterwards works.
|
||||
#[test]
|
||||
fn torn_last_record_is_tolerated() {
|
||||
let (m, before_last, full) = sample_catalog_with_log();
|
||||
let log = m.get(LOG).unwrap();
|
||||
let last_start = log[..log.len() - 1].iter().rposition(|b| *b == b'\n').unwrap() + 1;
|
||||
for cut in last_start..log.len() {
|
||||
let t = MemStore::new();
|
||||
t.set(LOG, log[..cut].to_vec());
|
||||
let (mut j, mut c, r) = open(&t);
|
||||
// the full record minus only its newline is complete and kept
|
||||
let expect = if cut == log.len() - 1 { &full } else { &before_last };
|
||||
assert_eq!(c.to_snapshot(), expect.to_snapshot(), "cut at {cut}");
|
||||
assert!(r.damaged.is_none());
|
||||
if cut < log.len() - 1 {
|
||||
assert_eq!(r.torn_bytes as usize, cut - last_start, "cut at {cut}");
|
||||
}
|
||||
// keeps working after recovery
|
||||
let id = c.photos().next().unwrap().id;
|
||||
let op = Op::SetFlag { id, flag: Flag::Reject };
|
||||
c.apply(op.clone()).unwrap();
|
||||
j.append(&[op]).unwrap();
|
||||
let (_, c2, r2) = open(&t);
|
||||
assert_eq!(c2.to_snapshot(), c.to_snapshot(), "cut at {cut}");
|
||||
assert_eq!(r2.torn_bytes, 0);
|
||||
}
|
||||
// garbage bytes (a partially flushed page) at the end are a torn tail too
|
||||
let t = MemStore::new();
|
||||
let mut g = log.clone();
|
||||
g.extend_from_slice(&[0, 0, 0, b'{', 0xff]);
|
||||
t.set(LOG, g);
|
||||
let (_, c, r) = open(&t);
|
||||
assert_eq!(c.to_snapshot(), full.to_snapshot());
|
||||
assert_eq!(r.torn_bytes, 5);
|
||||
}
|
||||
|
||||
/// Crash after the snapshot was written but before the log was reset: the stale records are
|
||||
/// skipped by sequence number.
|
||||
#[test]
|
||||
fn stale_log_after_snapshot_is_skipped() {
|
||||
let (m, _, full) = sample_catalog_with_log();
|
||||
let old_log = m.get(LOG).unwrap();
|
||||
let (mut j, c, _) = open(&m);
|
||||
j.snapshot(&c).unwrap();
|
||||
assert!(m.get(LOG).unwrap().is_empty());
|
||||
m.set(LOG, old_log);
|
||||
let (_, c2, r) = open(&m);
|
||||
assert_eq!(c2.to_snapshot(), full.to_snapshot());
|
||||
assert!(r.stale > 0 && r.replayed == 0);
|
||||
}
|
||||
|
||||
/// Damage in the middle of the log (not a torn tail): recover what precedes it, keep the
|
||||
/// damaged file, and write a consistent snapshot.
|
||||
#[test]
|
||||
fn mid_log_damage_is_preserved_and_recovered() {
|
||||
let (m, _, _) = sample_catalog_with_log();
|
||||
let log = m.get(LOG).unwrap();
|
||||
let text = String::from_utf8(log).unwrap();
|
||||
let mut lines: Vec<String> = text.lines().map(str::to_string).collect();
|
||||
assert!(lines.len() > 4);
|
||||
lines[2] = lines[2].replacen("\"op\":{\"op\":\"", "\"op\":{\"op\":\"x", 1);
|
||||
let bad = format!("{}\n", lines.join("\n")).into_bytes();
|
||||
m.set(LOG, bad.clone());
|
||||
let (_, c, r) = open(&m);
|
||||
let name = r.damaged.clone().expect("damage reported");
|
||||
assert_eq!(m.get(&name).unwrap(), bad);
|
||||
assert_eq!(r.replayed, 2);
|
||||
// the recovered state was snapshotted: reopening is clean and identical
|
||||
let (_, c2, r2) = open(&m);
|
||||
assert_eq!(c2.to_snapshot(), c.to_snapshot());
|
||||
assert!(r2.damaged.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_codec() {
|
||||
let op = Op::RenameAlbum { id: AlbumId(3), name: "Line\nbreak \"quoted\" }".into() };
|
||||
let line = encode_record(42, &op);
|
||||
assert!(!line.contains('\n'));
|
||||
assert_eq!(decode_record(&line), Some((42, op)));
|
||||
assert_eq!(decode_record(&line.replace("Line", "Lime")), None, "crc mismatch");
|
||||
assert_eq!(decode_record(&line[..line.len() - 1]), None);
|
||||
// log lines stay plain JSON
|
||||
let v: serde_json::Value = serde_json::from_str(&line).unwrap();
|
||||
assert_eq!(v["op"]["op"], "renameAlbum");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn push_history_is_bounded_and_invertible() {
|
||||
let mut c = Catalog::new();
|
||||
let id = c.alloc_photo_id();
|
||||
c.apply(Op::AddPhoto { photo: Box::new(Photo::new(id, Source::Demo { scene: 1 }, "a", "JPEG", 1, 1, "t")) }).unwrap();
|
||||
let step = |i: usize| HistoryStep { label: format!("s{i}"), settings: Arc::new(DevelopSettings::default()) };
|
||||
for i in 0..HISTORY_LIMIT + 5 {
|
||||
c.apply(Op::PushHistory { id, step: step(i) }).unwrap();
|
||||
}
|
||||
let h = &c.photo(id).unwrap().history;
|
||||
assert_eq!(h.len(), HISTORY_LIMIT);
|
||||
assert_eq!(h[0].label, "s5");
|
||||
let before = c.to_snapshot();
|
||||
let inv = c.apply(Op::PushHistory { id, step: step(999) }).unwrap();
|
||||
c.apply(inv).unwrap();
|
||||
assert_eq!(c.to_snapshot(), before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fs_store_roundtrip() {
|
||||
let dir = std::env::temp_dir().join(format!("lc-journal-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
let (mut j, mut c, r) = Journal::open(Box::new(FsStore::open(&dir).unwrap())).unwrap();
|
||||
assert!(r.created);
|
||||
let id = c.alloc_photo_id();
|
||||
let add = Op::AddPhoto { photo: Box::new(Photo::new(id, Source::File { path: "/a.jpg".into() }, "a.jpg", "JPEG", 1, 1, "t")) };
|
||||
c.apply(add.clone()).unwrap();
|
||||
c.apply(Op::SetRating { id, rating: 4 }).unwrap();
|
||||
j.append(&[add, Op::SetRating { id, rating: 4 }]).unwrap();
|
||||
drop(j);
|
||||
let (mut j, c2, r) = Journal::open(Box::new(FsStore::open(&dir).unwrap())).unwrap();
|
||||
assert_eq!((r.replayed, r.created), (2, false));
|
||||
assert_eq!(c2.to_snapshot(), c.to_snapshot());
|
||||
j.snapshot(&c2).unwrap();
|
||||
assert_eq!(std::fs::metadata(dir.join(LOG)).unwrap().len(), 0);
|
||||
assert!(!dir.join(format!("{SNAPSHOT}.tmp")).exists());
|
||||
let (_, c3, r) = Journal::open(Box::new(FsStore::open(&dir).unwrap())).unwrap();
|
||||
assert_eq!((r.snapshot_seq, r.replayed), (2, 0));
|
||||
assert_eq!(c3.to_snapshot(), c.to_snapshot());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
|
@ -16,6 +16,7 @@ lightcraft-develop = { workspace = true }
|
|||
lightcraft-scenes = { workspace = true }
|
||||
lightcraft-pipeline = { workspace = true }
|
||||
lightcraft-catalog = { workspace = true }
|
||||
lightcraft-preview = { workspace = true }
|
||||
lightcraft-codecs = { workspace = true }
|
||||
lightcraft-raw = { workspace = true }
|
||||
lightcraft-meta = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -364,23 +364,94 @@ pub fn specs() -> Vec<CommandSpec> {
|
|||
ok()
|
||||
}),
|
||||
// ---- import
|
||||
cmd!("library.import", "Add Photos…", ["File"], Some("Cmd+Shift+I"), "{paths: [path], album?: albumId}", always, |s, p| {
|
||||
let paths: Vec<String> = p
|
||||
.get("paths")
|
||||
.and_then(Value::as_array)
|
||||
.map(|a| a.iter().filter_map(Value::as_str).map(str::to_string).collect())
|
||||
.unwrap_or_default();
|
||||
if paths.is_empty() {
|
||||
return Err(bad("library.import", "no paths"));
|
||||
cmd!(
|
||||
"library.import",
|
||||
"Add Photos…",
|
||||
["File"],
|
||||
Some("Cmd+Shift+I"),
|
||||
"{paths: [file or folder (recursive)], mode?: add|copy (copy into the library's Originals/), album?: albumId} → {imported, duplicates, failed}",
|
||||
always,
|
||||
|s, p| {
|
||||
let paths: Vec<String> = p
|
||||
.get("paths")
|
||||
.and_then(Value::as_array)
|
||||
.map(|a| a.iter().filter_map(Value::as_str).map(str::to_string).collect())
|
||||
.unwrap_or_default();
|
||||
if paths.is_empty() {
|
||||
return Err(bad("library.import", "no paths"));
|
||||
}
|
||||
let mode = match str_param(p, "mode").unwrap_or("add") {
|
||||
"add" => crate::import::ImportMode::Add,
|
||||
"copy" => crate::import::ImportMode::Copy,
|
||||
other => return Err(bad("library.import", format!("unknown mode `{other}` (add|copy)"))),
|
||||
};
|
||||
let report = crate::import::import(s, &paths, mode)?;
|
||||
if let Some(a) = p.get("album").and_then(Value::as_u64)
|
||||
&& !report.imported.is_empty()
|
||||
{
|
||||
s.execute("album.addPhotos", &json!({"id": a, "ids": report.imported}))?;
|
||||
}
|
||||
if let Some(f) = report.imported.first() {
|
||||
s.selection = Selection::single(PhotoId(*f));
|
||||
}
|
||||
Ok(serde_json::to_value(&report).unwrap_or_default())
|
||||
}
|
||||
let imported = crate::media::import_paths(s, &paths)?;
|
||||
if let Some(a) = p.get("album").and_then(Value::as_u64) {
|
||||
s.execute("album.addPhotos", &json!({"id": a, "ids": imported.iter().map(|i| i.0).collect::<Vec<_>>()}))?;
|
||||
}
|
||||
if let Some(f) = imported.first() {
|
||||
s.selection = Selection::single(*f);
|
||||
}
|
||||
Ok(json!({"imported": imported.iter().map(|i| i.0).collect::<Vec<_>>()}))
|
||||
),
|
||||
// ---- persistence
|
||||
cmd!(query "library.info", "Library Info", [], None, "{}", always, |s, _| {
|
||||
let photos = s.catalog.len();
|
||||
let albums = s.catalog.albums().count();
|
||||
let (rendered_n, rendered_bytes) = s.media.rendered.mem_usage();
|
||||
let (sources_n, sources_bytes) = s.media.source_usage();
|
||||
let disk = s.media.rendered.disk().map(|d| {
|
||||
use std::sync::atomic::Ordering::Relaxed;
|
||||
json!({
|
||||
"path": d.dir().display().to_string(),
|
||||
"bytes": d.size(),
|
||||
"hits": d.hits.load(Relaxed),
|
||||
"misses": d.misses.load(Relaxed),
|
||||
"writes": d.writes.load(Relaxed),
|
||||
})
|
||||
});
|
||||
let cache = json!({
|
||||
"renderedInMemory": rendered_n, "renderedBytes": rendered_bytes,
|
||||
"sourcesInMemory": sources_n, "sourceBytes": sources_bytes,
|
||||
"disk": disk,
|
||||
});
|
||||
let Some(lib) = &s.library else {
|
||||
return Ok(json!({"persistent": false, "photos": photos, "albums": albums, "cache": cache}));
|
||||
};
|
||||
let j = lib.journal();
|
||||
Ok(json!({
|
||||
"persistent": true,
|
||||
"path": lib.dir.display().to_string(),
|
||||
"photos": photos,
|
||||
"albums": albums,
|
||||
"seq": j.seq(),
|
||||
"snapshotSeq": j.snapshot_seq(),
|
||||
"logRecords": j.log_records(),
|
||||
"logBytes": j.log_bytes(),
|
||||
"lastError": lib.last_error,
|
||||
"cache": cache,
|
||||
"load": {
|
||||
"created": lib.report.created,
|
||||
"replayed": lib.report.replayed,
|
||||
"tornBytes": lib.report.torn_bytes,
|
||||
"damaged": lib.report.damaged,
|
||||
},
|
||||
}))
|
||||
}),
|
||||
cmd!("library.compact", "Optimize Library", ["File"], None, "{}", has_library, |s, _| {
|
||||
s.compact_library()?;
|
||||
ok()
|
||||
}),
|
||||
cmd!("library.clearPreviews", "Clear Preview Cache", ["File"], None, "{}", always, |s, _| {
|
||||
s.media.rendered.clear();
|
||||
ok()
|
||||
}),
|
||||
]
|
||||
}
|
||||
|
||||
fn has_library(s: &Session) -> std::result::Result<(), String> {
|
||||
if s.library.is_some() { Ok(()) } else { Err("no library is open (in-memory session)".into()) }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ fn ext_upper(name: &str) -> String {
|
|||
|
||||
/// Probe a file's bytes: kind, dimensions (oriented), metadata.
|
||||
pub fn probe_bytes(name: &str, bytes: &[u8]) -> Result<ProbeInfo, String> {
|
||||
let content_hash = Some(lightcraft_preview::hash_bytes(bytes).to_string());
|
||||
let m = lightcraft_meta::extract(bytes);
|
||||
let (meta, captured) = meta_of(&m);
|
||||
if lightcraft_raw::probe(bytes).is_some() {
|
||||
|
|
@ -62,6 +63,7 @@ pub fn probe_bytes(name: &str, bytes: &[u8]) -> Result<ProbeInfo, String> {
|
|||
captured,
|
||||
meta,
|
||||
as_shot_wb,
|
||||
content_hash,
|
||||
});
|
||||
}
|
||||
let fmt = lightcraft_codecs::sniff(bytes).ok_or("unrecognized file format")?;
|
||||
|
|
@ -79,7 +81,17 @@ pub fn probe_bytes(name: &str, bytes: &[u8]) -> Result<ProbeInfo, String> {
|
|||
"" => format!("{fmt:?}").to_uppercase(),
|
||||
e => e.to_string(),
|
||||
};
|
||||
Ok(ProbeInfo { width: w, height: h, format, kind: MediaKind::Image, file_size: bytes.len() as u64, captured, meta, as_shot_wb: None })
|
||||
Ok(ProbeInfo {
|
||||
width: w,
|
||||
height: h,
|
||||
format,
|
||||
kind: MediaKind::Image,
|
||||
file_size: bytes.len() as u64,
|
||||
captured,
|
||||
meta,
|
||||
as_shot_wb: None,
|
||||
content_hash,
|
||||
})
|
||||
}
|
||||
|
||||
/// Decode a file into a linear Rec.2020 image no larger than `max_edge`, oriented.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,267 @@
|
|||
//! Import: files and folders (recursive) → photos.
|
||||
//!
|
||||
//! 1. Expand folders recursively into supported files (hidden entries and the library's own
|
||||
//! folder are skipped), sorted for a stable order.
|
||||
//! 2. Skip paths already in the catalog.
|
||||
//! 3. Probe the rest — dimensions, metadata, and the **content hash** of the bytes — in parallel
|
||||
//! on native targets.
|
||||
//! 4. Skip **duplicates by content** (same bytes already in the library, or twice in this batch).
|
||||
//! 5. *Add* in place (the photo points at the original file) or *copy* into the library's
|
||||
//! `Originals/YYYY/YYYY-MM-DD/` folder (names made unique) and point at the copy.
|
||||
//! 6. Commit all new photos as one undoable op.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use lightcraft_catalog::{Op, Photo, PhotoId, Source};
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::Session;
|
||||
use crate::media::ProbeInfo;
|
||||
|
||||
/// File extensions LightCraft imports (lower case).
|
||||
pub const EXTENSIONS: &[&str] = &[
|
||||
"jpg", "jpeg", "png", "tif", "tiff", "webp", "dng", "cr2", "cr3", "nef", "arw", "raf", "orf", "rw2", "pef", "psd", "jxl", "gif", "bmp", "heic",
|
||||
"avif",
|
||||
];
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum ImportMode {
|
||||
/// Reference the files where they are.
|
||||
#[default]
|
||||
Add,
|
||||
/// Copy the files into the library's `Originals/` folder.
|
||||
Copy,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize)]
|
||||
pub struct Duplicate {
|
||||
pub path: String,
|
||||
/// The photo that already has these bytes (or this path).
|
||||
pub existing: Option<u64>,
|
||||
/// `"path"` (already imported from there) or `"content"` (same bytes elsewhere).
|
||||
pub reason: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize)]
|
||||
pub struct ImportReport {
|
||||
pub imported: Vec<u64>,
|
||||
pub duplicates: Vec<Duplicate>,
|
||||
/// (path, error)
|
||||
pub failed: Vec<(String, String)>,
|
||||
/// Files found after expanding folders.
|
||||
pub scanned: usize,
|
||||
}
|
||||
|
||||
pub fn is_supported(path: &Path) -> bool {
|
||||
path.extension().is_some_and(|e| EXTENSIONS.contains(&e.to_string_lossy().to_lowercase().as_str()))
|
||||
}
|
||||
|
||||
/// Expand files and folders (recursively) into supported files. `skip` (e.g. the library folder)
|
||||
/// is never descended into.
|
||||
pub fn expand(paths: &[String], skip: Option<&Path>) -> Vec<String> {
|
||||
fn walk(p: &Path, skip: Option<&Path>, out: &mut Vec<String>, top: bool) {
|
||||
let hidden = p.file_name().is_some_and(|n| n.to_string_lossy().starts_with('.'));
|
||||
if (hidden && !top) || skip.is_some_and(|s| p == s) {
|
||||
return;
|
||||
}
|
||||
if p.is_dir() {
|
||||
let Ok(rd) = std::fs::read_dir(p) else { return };
|
||||
let mut v: Vec<PathBuf> = rd.flatten().map(|e| e.path()).collect();
|
||||
v.sort();
|
||||
for c in v {
|
||||
walk(&c, skip, out, false);
|
||||
}
|
||||
} else if top || is_supported(p) {
|
||||
// explicitly named files are attempted even with an unknown extension (sniffed)
|
||||
out.push(p.to_string_lossy().to_string());
|
||||
}
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
for p in paths {
|
||||
walk(Path::new(p), skip, &mut out, true);
|
||||
}
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
out.retain(|p| seen.insert(p.clone()));
|
||||
out
|
||||
}
|
||||
|
||||
fn probe_all(s: &Session, paths: &[String]) -> Vec<Result<ProbeInfo, String>> {
|
||||
let Some(probe) = s.media.file_probe.clone() else {
|
||||
return paths
|
||||
.iter()
|
||||
.map(|p| {
|
||||
Ok(ProbeInfo {
|
||||
format: Path::new(p).extension().map(|e| e.to_string_lossy().to_uppercase()).unwrap_or_default(),
|
||||
..Default::default()
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
};
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
{
|
||||
let n = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4).clamp(1, 8).min(paths.len().max(1));
|
||||
if n > 1 {
|
||||
let mut results: Vec<Option<Result<ProbeInfo, String>>> = vec![None; paths.len()];
|
||||
let next = std::sync::atomic::AtomicUsize::new(0);
|
||||
let out = std::sync::Mutex::new(&mut results);
|
||||
std::thread::scope(|sc| {
|
||||
for _ in 0..n {
|
||||
sc.spawn(|| {
|
||||
loop {
|
||||
let i = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
if i >= paths.len() {
|
||||
break;
|
||||
}
|
||||
let r = probe(&paths[i]);
|
||||
out.lock().unwrap_or_else(|e| e.into_inner())[i] = Some(r);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
return results.into_iter().map(|r| r.unwrap_or_else(|| Err("not probed".into()))).collect();
|
||||
}
|
||||
}
|
||||
paths.iter().map(|p| probe(p)).collect()
|
||||
}
|
||||
|
||||
/// `Originals/YYYY/YYYY-MM-DD/name`, made unique.
|
||||
fn copy_into_library(lib: &Path, src: &str, date: &str) -> Result<String, String> {
|
||||
let day = date.get(..10).filter(|d| d.len() == 10).unwrap_or("undated");
|
||||
let year = day.get(..4).unwrap_or("undated");
|
||||
let dir = lib.join("Originals").join(year).join(day);
|
||||
std::fs::create_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))?;
|
||||
let name = Path::new(src).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "photo".into());
|
||||
let (stem, ext) = match name.rsplit_once('.') {
|
||||
Some((s, e)) => (s.to_string(), format!(".{e}")),
|
||||
None => (name.clone(), String::new()),
|
||||
};
|
||||
let mut dst = dir.join(&name);
|
||||
let mut i = 1;
|
||||
while dst.exists() {
|
||||
dst = dir.join(format!("{stem}-{i}{ext}"));
|
||||
i += 1;
|
||||
}
|
||||
std::fs::copy(src, &dst).map_err(|e| format!("copy {src}: {e}"))?;
|
||||
Ok(dst.to_string_lossy().to_string())
|
||||
}
|
||||
|
||||
/// Import files/folders. See the module docs.
|
||||
pub fn import(s: &mut Session, paths: &[String], mode: ImportMode) -> crate::Result<ImportReport> {
|
||||
let lib_dir = s.library.as_ref().map(|l| l.dir.clone());
|
||||
if mode == ImportMode::Copy && lib_dir.is_none() {
|
||||
return Err(crate::EngineError::Other("copying into the library needs an open library".into()));
|
||||
}
|
||||
let files = expand(paths, lib_dir.as_deref());
|
||||
let mut report = ImportReport { scanned: files.len(), ..Default::default() };
|
||||
|
||||
// existing paths and content hashes
|
||||
let mut by_path: HashMap<&str, PhotoId> = HashMap::new();
|
||||
let mut by_hash: HashMap<String, PhotoId> = HashMap::new();
|
||||
for p in s.catalog.photos() {
|
||||
if let Source::File { path } = &p.source {
|
||||
by_path.insert(path.as_str(), p.id);
|
||||
}
|
||||
if let Some(h) = &p.content_hash {
|
||||
by_hash.insert(h.clone(), p.id);
|
||||
}
|
||||
}
|
||||
let mut todo = Vec::new();
|
||||
for f in files {
|
||||
match by_path.get(f.as_str()) {
|
||||
Some(id) => report.duplicates.push(Duplicate { path: f, existing: Some(id.0), reason: "path" }),
|
||||
None => todo.push(f),
|
||||
}
|
||||
}
|
||||
|
||||
let probed = probe_all(s, &todo);
|
||||
let now = (s.clock)();
|
||||
let mut ops = Vec::new();
|
||||
for (path, info) in todo.into_iter().zip(probed) {
|
||||
let info = match info {
|
||||
Ok(i) => i,
|
||||
Err(e) => {
|
||||
log::warn!("import {path}: {e}");
|
||||
report.failed.push((path, e));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if let Some(h) = &info.content_hash
|
||||
&& let Some(id) = by_hash.get(h)
|
||||
{
|
||||
report.duplicates.push(Duplicate { path, existing: Some(id.0), reason: "content" });
|
||||
continue;
|
||||
}
|
||||
let stored = match (mode, &lib_dir) {
|
||||
(ImportMode::Copy, Some(lib)) if !Path::new(&path).starts_with(lib) => {
|
||||
match copy_into_library(lib, &path, info.captured.as_deref().unwrap_or(&now)) {
|
||||
Ok(p) => p,
|
||||
Err(e) => {
|
||||
report.failed.push((path, e));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => path.clone(),
|
||||
};
|
||||
let id = s.catalog.alloc_photo_id();
|
||||
if let Some(h) = &info.content_hash {
|
||||
by_hash.insert(h.clone(), id);
|
||||
}
|
||||
let name = Path::new(&path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.clone());
|
||||
let mut p = Photo::new(id, Source::File { path: stored }, &name, &info.format, info.width, info.height, &now);
|
||||
p.kind = info.kind;
|
||||
p.file_size = info.file_size;
|
||||
p.captured = info.captured;
|
||||
p.meta = info.meta;
|
||||
p.as_shot_wb = info.as_shot_wb;
|
||||
p.content_hash = info.content_hash;
|
||||
if let Some((t, tint)) = info.as_shot_wb {
|
||||
p.develop = std::sync::Arc::new(lightcraft_develop::DevelopSettings::for_raw(t, tint));
|
||||
}
|
||||
report.imported.push(id.0);
|
||||
ops.push(Op::AddPhoto { photo: Box::new(p) });
|
||||
}
|
||||
if !ops.is_empty() {
|
||||
s.commit(&format!("Add {} Photo{}", ops.len(), if ops.len() == 1 { "" } else { "s" }), Op::Batch { ops })?;
|
||||
}
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
/// The current local time as ISO 8601 (`YYYY-MM-DDTHH:MM:SS`, UTC on targets without a clock
|
||||
/// offset); for [`Session::clock`] on native hosts.
|
||||
pub fn system_clock() -> String {
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
{
|
||||
let secs = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs() as i64).unwrap_or(0);
|
||||
civil(secs)
|
||||
}
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
{
|
||||
"2026-01-01T00:00:00".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
/// Unix seconds → `YYYY-MM-DDTHH:MM:SS` (UTC), proleptic Gregorian (H. Hinnant's algorithm).
|
||||
pub fn civil(secs: i64) -> String {
|
||||
let days = secs.div_euclid(86_400);
|
||||
let rem = secs.rem_euclid(86_400);
|
||||
let z = days + 719_468;
|
||||
let era = z.div_euclid(146_097);
|
||||
let doe = z - era * 146_097;
|
||||
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
|
||||
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
|
||||
let mp = (5 * doy + 2) / 153;
|
||||
let d = doy - (153 * mp + 2) / 5 + 1;
|
||||
let m = if mp < 10 { mp + 3 } else { mp - 9 };
|
||||
let y = yoe + era * 400 + i64::from(m <= 2);
|
||||
format!("{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}", rem / 3600, rem % 3600 / 60, rem % 60)
|
||||
}
|
||||
|
||||
impl Session {
|
||||
/// Use the system clock for import/edit times (native hosts).
|
||||
pub fn with_system_clock(mut self) -> Self {
|
||||
self.clock = Box::new(system_clock);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
|
@ -14,6 +14,8 @@ pub mod cmd;
|
|||
pub mod demo;
|
||||
pub mod export;
|
||||
pub mod files;
|
||||
pub mod import;
|
||||
pub mod library;
|
||||
pub mod media;
|
||||
pub mod presets;
|
||||
mod view;
|
||||
|
|
@ -86,6 +88,8 @@ pub struct Session {
|
|||
depth: u32,
|
||||
/// Selected mask (Masking panel), by mask id.
|
||||
pub active_mask: Option<u32>,
|
||||
/// The persistent library this session writes to (`None` = in-memory only).
|
||||
pub library: Option<library::Library>,
|
||||
}
|
||||
|
||||
impl Default for Session {
|
||||
|
|
@ -116,6 +120,7 @@ impl Session {
|
|||
clock: Box::new(|| "2026-09-30T12:00:00".to_string()),
|
||||
depth: 0,
|
||||
active_mask: None,
|
||||
library: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -141,6 +146,11 @@ impl Session {
|
|||
self.journal.drain(..1000);
|
||||
}
|
||||
}
|
||||
if self.depth == 0 && self.library.is_some() {
|
||||
// Make the command durable before reporting success (on failure the ops stay pending,
|
||||
// are retried after the next command, and `library.info` reports the error).
|
||||
let _ = self.persist();
|
||||
}
|
||||
r
|
||||
}
|
||||
|
||||
|
|
@ -217,15 +227,11 @@ impl Session {
|
|||
|
||||
/// The op that sets a photo's develop settings and appends a History entry (for batches).
|
||||
pub fn develop_op(&self, id: PhotoId, settings: DevelopSettings, label: &str) -> Option<Op> {
|
||||
let p = self.catalog.photo(id)?;
|
||||
self.catalog.photo(id)?;
|
||||
let settings = Arc::new(settings);
|
||||
let mut history = p.history.clone();
|
||||
history.push(lightcraft_catalog::HistoryStep { label: label.into(), settings: settings.clone() });
|
||||
if history.len() > 200 {
|
||||
history.remove(0);
|
||||
}
|
||||
let step = lightcraft_catalog::HistoryStep { label: label.into(), settings: settings.clone() };
|
||||
Some(Op::Batch {
|
||||
ops: vec![Op::SetDevelop { id, settings, label: label.into(), edited: Some((self.clock)()) }, Op::SetHistory { id, history }],
|
||||
ops: vec![Op::SetDevelop { id, settings, label: label.into(), edited: Some((self.clock)()) }, Op::PushHistory { id, step }],
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -301,3 +307,7 @@ impl Session {
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
#[cfg(test)]
|
||||
mod tests_import;
|
||||
#[cfg(test)]
|
||||
mod tests_library;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,203 @@
|
|||
//! A persistent library: a directory holding the catalog (op log + snapshot), user presets, the
|
||||
//! last view state and the preview cache.
|
||||
//!
|
||||
//! ```text
|
||||
//! LightCraft Library/
|
||||
//! catalog.snap catalog.log (lightcraft-catalog journal)
|
||||
//! presets.json view.json (user presets + favourites; last source/filter/sort/selection)
|
||||
//! thumbs/ (rendered thumbnail cache, safe to delete)
|
||||
//! Originals/ (photos imported with "copy into library")
|
||||
//! ```
|
||||
//!
|
||||
//! Every top-level [`Session::execute`] persists the ops it produced (fsynced) before returning,
|
||||
//! so a crash loses at most the command in flight. The log is compacted into a snapshot when it
|
||||
//! grows (see [`lightcraft_catalog::SnapshotPolicy`]) and on [`Session::close_library`].
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use lightcraft_catalog::{FsStore, Journal, LoadReport, Store};
|
||||
use lightcraft_develop::Preset;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{EngineError, LibrarySource, Result, Selection, Session};
|
||||
|
||||
/// Library directory name inside the user's Pictures folder.
|
||||
pub const DEFAULT_NAME: &str = "LightCraft Library";
|
||||
|
||||
/// The default library location: `$LIGHTCRAFT_LIBRARY` if set, else `~/Pictures/LightCraft Library`
|
||||
/// (`%USERPROFILE%\Pictures\LightCraft Library` on Windows).
|
||||
pub fn default_dir() -> Option<PathBuf> {
|
||||
if let Some(p) = std::env::var_os("LIGHTCRAFT_LIBRARY").filter(|p| !p.is_empty()) {
|
||||
return Some(PathBuf::from(p));
|
||||
}
|
||||
let home = if cfg!(windows) { std::env::var_os("USERPROFILE") } else { std::env::var_os("HOME") }?;
|
||||
Some(PathBuf::from(home).join("Pictures").join(DEFAULT_NAME))
|
||||
}
|
||||
|
||||
pub struct Library {
|
||||
pub dir: PathBuf,
|
||||
journal: Journal,
|
||||
/// What happened when the catalog was loaded.
|
||||
pub report: LoadReport,
|
||||
/// Last persistence error (shown by the UI; ops stay pending and are retried).
|
||||
pub last_error: Option<String>,
|
||||
presets_written: String,
|
||||
}
|
||||
|
||||
#[derive(Default, Serialize, Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
struct PresetsFile {
|
||||
user: Vec<Preset>,
|
||||
favorites: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Default, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
struct ViewFile {
|
||||
source: LibrarySource,
|
||||
filter: lightcraft_catalog::Filter,
|
||||
sort: lightcraft_catalog::Sort,
|
||||
selection: Selection,
|
||||
}
|
||||
|
||||
impl Library {
|
||||
pub fn journal(&self) -> &Journal {
|
||||
&self.journal
|
||||
}
|
||||
pub fn thumbs_dir(&self) -> PathBuf {
|
||||
self.dir.join("thumbs")
|
||||
}
|
||||
pub fn originals_dir(&self) -> PathBuf {
|
||||
self.dir.join("Originals")
|
||||
}
|
||||
}
|
||||
|
||||
fn presets_json(presets: &[Preset]) -> String {
|
||||
let f = PresetsFile {
|
||||
user: presets.iter().filter(|p| !p.builtin).cloned().collect(),
|
||||
favorites: presets.iter().filter(|p| p.builtin && p.favorite).map(|p| p.id.clone()).collect(),
|
||||
};
|
||||
serde_json::to_string_pretty(&f).unwrap_or_default()
|
||||
}
|
||||
|
||||
fn write_atomic(dir: &Path, name: &str, data: &[u8]) -> std::io::Result<()> {
|
||||
FsStore::open(dir)?.write_atomic(name, data)
|
||||
}
|
||||
|
||||
impl Session {
|
||||
/// Open (or create) the library at `dir` into this session, replacing its catalog. With
|
||||
/// `seed_demo`, a newly created library starts with the procedural demo photos.
|
||||
pub fn open_library(&mut self, dir: impl AsRef<Path>, seed_demo: bool) -> Result<&LoadReport> {
|
||||
let dir = dir.as_ref().to_path_buf();
|
||||
let store = FsStore::open(&dir).map_err(|e| EngineError::Other(format!("can't open library {}: {e}", dir.display())))?;
|
||||
let (mut journal, catalog, report) = Journal::open(Box::new(store))?;
|
||||
self.catalog = catalog;
|
||||
self.undo.clear();
|
||||
self.redo.clear();
|
||||
self.interaction = None;
|
||||
self.pending_log.clear();
|
||||
self.selection = Selection::default();
|
||||
self.source = LibrarySource::All;
|
||||
if report.created && seed_demo {
|
||||
crate::demo::load(self);
|
||||
journal.snapshot(&self.catalog)?;
|
||||
}
|
||||
// presets
|
||||
if let Ok(bytes) = std::fs::read(dir.join("presets.json"))
|
||||
&& let Ok(f) = serde_json::from_slice::<PresetsFile>(&bytes)
|
||||
{
|
||||
for p in &mut self.presets {
|
||||
p.favorite = p.builtin && f.favorites.contains(&p.id) || (!p.builtin && p.favorite);
|
||||
}
|
||||
for u in f.user {
|
||||
if !self.presets.iter().any(|p| p.id == u.id) {
|
||||
self.presets.push(u);
|
||||
}
|
||||
}
|
||||
}
|
||||
// view state
|
||||
if let Ok(bytes) = std::fs::read(dir.join("view.json"))
|
||||
&& let Ok(v) = serde_json::from_slice::<ViewFile>(&bytes)
|
||||
{
|
||||
self.source = v.source;
|
||||
self.filter = v.filter;
|
||||
self.sort = v.sort;
|
||||
self.selection = v.selection;
|
||||
self.selection.ids.retain(|id| self.catalog.photo(*id).is_some());
|
||||
self.selection.active = self.selection.active.filter(|id| self.catalog.photo(*id).is_some());
|
||||
}
|
||||
if self.selection.active.is_none()
|
||||
&& let Some(first) = self.visible_cloned().first()
|
||||
{
|
||||
self.selection = Selection::single(*first);
|
||||
}
|
||||
let presets_written = presets_json(&self.presets);
|
||||
self.media.attach_disk_cache(&dir.join("thumbs"));
|
||||
self.library = Some(Library { dir, journal, report, last_error: None, presets_written });
|
||||
Ok(&self.library.as_ref().expect("just set").report)
|
||||
}
|
||||
|
||||
/// Write pending ops to the log (fsynced), compact when due, and save changed presets.
|
||||
/// Called after every top-level command; cheap when nothing changed.
|
||||
pub fn persist(&mut self) -> Result<()> {
|
||||
let Some(lib) = self.library.as_mut() else { return Ok(()) };
|
||||
if !self.pending_log.is_empty() {
|
||||
if let Err(e) = lib.journal.append(&self.pending_log) {
|
||||
lib.last_error = Some(e.to_string());
|
||||
log::error!("library: {e}");
|
||||
return Err(e.into());
|
||||
}
|
||||
self.pending_log.clear();
|
||||
lib.last_error = None;
|
||||
}
|
||||
// Never snapshot mid-interaction: the catalog then holds an uncommitted preview value.
|
||||
if lib.journal.wants_snapshot() && self.interaction.is_none() {
|
||||
lib.journal.snapshot(&self.catalog)?;
|
||||
}
|
||||
let presets = presets_json(&self.presets);
|
||||
if presets != lib.presets_written {
|
||||
if let Err(e) = write_atomic(&lib.dir, "presets.json", presets.as_bytes()) {
|
||||
log::error!("library: presets: {e}");
|
||||
} else {
|
||||
lib.presets_written = presets;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Persist if commands left ops pending (cheap; frontends call it once per frame for state
|
||||
/// changed outside [`Session::execute`]).
|
||||
pub fn persist_if_dirty(&mut self) {
|
||||
if self.library.is_some() && !self.pending_log.is_empty() {
|
||||
let _ = self.persist();
|
||||
}
|
||||
}
|
||||
|
||||
/// Flush everything and write a snapshot (on quit). Ends an open interaction first.
|
||||
pub fn close_library(&mut self) -> Result<()> {
|
||||
if self.library.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
let _ = self.end_interaction();
|
||||
self.persist()?;
|
||||
let view = ViewFile { source: self.source, filter: self.filter.clone(), sort: self.sort, selection: self.selection.clone() };
|
||||
let Some(lib) = self.library.as_mut() else { return Ok(()) };
|
||||
lib.journal.snapshot(&self.catalog)?;
|
||||
if let Ok(v) = serde_json::to_vec_pretty(&view) {
|
||||
let _ = write_atomic(&lib.dir, "view.json", &v);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Compact the log into a snapshot now.
|
||||
pub fn compact_library(&mut self) -> Result<()> {
|
||||
self.persist()?;
|
||||
if self.interaction.is_some() {
|
||||
return Err(EngineError::Other("can't compact during an interaction".into()));
|
||||
}
|
||||
if let Some(lib) = self.library.as_mut() {
|
||||
lib.journal.snapshot(&self.catalog)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
@ -1,19 +1,36 @@
|
|||
//! Source proxies and render jobs.
|
||||
//!
|
||||
//! Sources are decoded (or generated) once per level and cached: `Thumb` (≤ 512 px long edge) for
|
||||
//! the grid and filmstrip, `Preview` (≤ 2560 px) for the loupe. A [`RenderJob`] is self-contained
|
||||
//! and `Send`: frontends run it on a worker thread; if the source wasn't cached yet the job loads it
|
||||
//! and hands it back in the [`RenderResult`] so the cache can keep it.
|
||||
//! Sources are decoded (or generated) once per level and cached in memory LRUs: `Thumb` (≤ 512 px
|
||||
//! long edge) for the grid and filmstrip, `Preview` (≤ 2560 px) for the loupe. A [`RenderJob`] is
|
||||
//! self-contained and `Send`: frontends run it on a worker thread; if the source wasn't cached yet
|
||||
//! the job loads it and hands it back in the [`RenderResult`] so the cache can keep it.
|
||||
//!
|
||||
//! Rendered thumbnails are cached too ([`lightcraft_preview::PreviewCache`]: memory LRU, plus a
|
||||
//! disk cache in the library's `thumbs/` folder), keyed by the photo's content hash, the develop
|
||||
//! settings hash, the size and [`RENDER_CACHE_VERSION`] — so reopening a library shows its grid
|
||||
//! without decoding a single original, and an edit simply produces a new key.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use lightcraft_catalog::{MediaKind, Photo, PhotoId, Source};
|
||||
use lightcraft_develop::DevelopSettings;
|
||||
use lightcraft_pipeline::{RenderRequest, Rendered, SourceInfo};
|
||||
use lightcraft_raster::Rgb32f;
|
||||
use lightcraft_preview::{Hash128, Hasher128, Lru, PreviewCache};
|
||||
use lightcraft_raster::{Histogram, Rgb32f};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Bump when the pipeline's output changes, to invalidate cached thumbnails.
|
||||
pub const RENDER_CACHE_VERSION: u64 = 1;
|
||||
|
||||
/// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache).
|
||||
pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512];
|
||||
|
||||
/// Memory budgets.
|
||||
const THUMB_SOURCE_BYTES: usize = 384 << 20;
|
||||
const RENDERED_MEM_BYTES: usize = 128 << 20;
|
||||
/// Disk budget for the thumbnail cache.
|
||||
pub const DISK_CACHE_BYTES: u64 = 2 << 30;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum SourceLevel {
|
||||
|
|
@ -57,19 +74,40 @@ impl SourceRef {
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct MediaCache {
|
||||
thumbs: HashMap<PhotoId, Arc<Rgb32f>>,
|
||||
/// Decoded thumbnail-level sources (LRU by bytes).
|
||||
thumbs: Lru<PhotoId, Arc<Rgb32f>>,
|
||||
previews: Vec<(PhotoId, Arc<Rgb32f>)>,
|
||||
/// How many previews to keep (LRU).
|
||||
pub preview_capacity: usize,
|
||||
pub file_loader: Option<FileLoader>,
|
||||
pub file_probe: Option<FileProbe>,
|
||||
scenes: Vec<lightcraft_scenes::Scene>,
|
||||
/// Rendered thumbnails (memory, plus disk once a library is attached).
|
||||
pub rendered: Arc<PreviewCache>,
|
||||
}
|
||||
|
||||
impl Default for MediaCache {
|
||||
fn default() -> Self {
|
||||
MediaCache {
|
||||
thumbs: Lru::new(THUMB_SOURCE_BYTES),
|
||||
previews: Vec::new(),
|
||||
preview_capacity: 0,
|
||||
file_loader: None,
|
||||
file_probe: None,
|
||||
scenes: Vec::new(),
|
||||
rendered: Arc::new(PreviewCache::memory(RENDERED_MEM_BYTES)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl MediaCache {
|
||||
pub fn get(&self, id: PhotoId, level: SourceLevel) -> Option<Arc<Rgb32f>> {
|
||||
/// Keep rendered thumbnails on disk in `dir` as well.
|
||||
pub fn attach_disk_cache(&mut self, dir: &std::path::Path) {
|
||||
self.rendered = Arc::new(PreviewCache::with_disk(RENDERED_MEM_BYTES, dir, DISK_CACHE_BYTES));
|
||||
}
|
||||
|
||||
pub fn get(&mut self, id: PhotoId, level: SourceLevel) -> Option<Arc<Rgb32f>> {
|
||||
match level {
|
||||
SourceLevel::Thumb => self.thumbs.get(&id).cloned(),
|
||||
SourceLevel::Preview => self.previews.iter().find(|(p, _)| *p == id).map(|(_, a)| a.clone()),
|
||||
|
|
@ -79,7 +117,8 @@ impl MediaCache {
|
|||
pub fn insert(&mut self, id: PhotoId, level: SourceLevel, img: Arc<Rgb32f>) {
|
||||
match level {
|
||||
SourceLevel::Thumb => {
|
||||
self.thumbs.insert(id, img);
|
||||
let cost = img.width * img.height * 12 + 64;
|
||||
self.thumbs.insert(id, img, cost);
|
||||
}
|
||||
SourceLevel::Preview => {
|
||||
self.previews.retain(|(p, _)| *p != id);
|
||||
|
|
@ -92,6 +131,17 @@ impl MediaCache {
|
|||
}
|
||||
}
|
||||
|
||||
/// Forget a photo's decoded sources (e.g. after its file changed).
|
||||
pub fn forget(&mut self, id: PhotoId) {
|
||||
self.thumbs.remove(&id);
|
||||
self.previews.retain(|(p, _)| *p != id);
|
||||
}
|
||||
|
||||
/// (decoded thumbnail sources, bytes).
|
||||
pub fn source_usage(&self) -> (usize, usize) {
|
||||
(self.thumbs.len(), self.thumbs.cost())
|
||||
}
|
||||
|
||||
pub fn source_ref(&mut self, p: &Photo, level: SourceLevel) -> SourceRef {
|
||||
if let Some(a) = self.get(p.id, level) {
|
||||
return SourceRef::Loaded(a);
|
||||
|
|
@ -122,6 +172,8 @@ pub struct RenderJob {
|
|||
pub request: RenderRequest,
|
||||
/// Identifies the result for caching: hash of settings + request.
|
||||
pub key: u64,
|
||||
/// Rendered-thumbnail cache and this job's key in it.
|
||||
pub cache: Option<(Arc<PreviewCache>, Hash128)>,
|
||||
}
|
||||
|
||||
pub struct RenderResult {
|
||||
|
|
@ -135,10 +187,20 @@ pub struct RenderResult {
|
|||
|
||||
impl RenderJob {
|
||||
pub fn run(self) -> RenderResult {
|
||||
if let Some((cache, key)) = &self.cache
|
||||
&& let Some(img) = cache.get(*key)
|
||||
{
|
||||
let image = Arc::unwrap_or_clone(img);
|
||||
let histogram = Histogram::of_srgb8(&image);
|
||||
return RenderResult { photo: self.photo, level: self.level, key: self.key, rendered: Ok(Rendered { image, histogram }), loaded: None };
|
||||
}
|
||||
let was_loaded = matches!(self.source, SourceRef::Loaded(_));
|
||||
match self.source.load() {
|
||||
Ok(src) => {
|
||||
let rendered = lightcraft_pipeline::render(&src, &self.info, &self.settings, &self.request);
|
||||
if let Some((cache, key)) = &self.cache {
|
||||
cache.put(*key, Arc::new(rendered.image.clone()));
|
||||
}
|
||||
RenderResult { photo: self.photo, level: self.level, key: self.key, rendered: Ok(rendered), loaded: (!was_loaded).then_some(src) }
|
||||
}
|
||||
Err(e) => RenderResult { photo: self.photo, level: self.level, key: self.key, rendered: Err(e), loaded: None },
|
||||
|
|
@ -146,6 +208,15 @@ impl RenderJob {
|
|||
}
|
||||
}
|
||||
|
||||
/// What identifies a photo's pixels for caching: its content hash, else its source.
|
||||
pub fn content_key(p: &Photo) -> String {
|
||||
match (&p.content_hash, &p.source) {
|
||||
(Some(h), _) => h.clone(),
|
||||
(None, Source::Demo { scene }) => format!("demo:{scene}"),
|
||||
(None, Source::File { path }) => format!("file:{path}:{}", p.file_size),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn source_info(p: &Photo) -> SourceInfo {
|
||||
// Procedural demo scenes are scene-referred HDR (like raw files): use the filmic tone map.
|
||||
if matches!(p.source, Source::Demo { .. }) {
|
||||
|
|
@ -157,6 +228,25 @@ pub fn source_info(p: &Photo) -> SourceInfo {
|
|||
impl crate::Session {
|
||||
/// Build a render job for `id` fitting `max_w × max_h`. `before` renders the unedited look.
|
||||
pub fn render_job(&mut self, id: PhotoId, max_w: usize, max_h: usize, before: bool, apply_crop: bool) -> Option<RenderJob> {
|
||||
self.build_job(id, max_w, max_h, before, apply_crop, None)
|
||||
}
|
||||
|
||||
/// A grid/filmstrip thumbnail job: the long edge is rounded up to one of [`THUMB_SIZES`]
|
||||
/// (≥ `long_edge`) and the result comes from / goes to the thumbnail cache (memory + disk).
|
||||
pub fn thumb_job(&mut self, id: PhotoId, long_edge: usize) -> Option<RenderJob> {
|
||||
let b = THUMB_SIZES.iter().copied().find(|s| *s >= long_edge).unwrap_or(THUMB_SIZES[THUMB_SIZES.len() - 1]);
|
||||
self.build_job(id, b, b, false, true, Some(b))
|
||||
}
|
||||
|
||||
fn build_job(
|
||||
&mut self,
|
||||
id: PhotoId,
|
||||
max_w: usize,
|
||||
max_h: usize,
|
||||
before: bool,
|
||||
apply_crop: bool,
|
||||
thumb_bucket: Option<usize>,
|
||||
) -> Option<RenderJob> {
|
||||
let p = self.catalog.photo(id)?.clone();
|
||||
let level = SourceLevel::for_size(max_w.max(max_h));
|
||||
let source = self.media.source_ref(&p, level);
|
||||
|
|
@ -167,7 +257,11 @@ impl crate::Session {
|
|||
^ ((max_h as u64) << 20)
|
||||
^ (apply_crop as u64)
|
||||
^ ((level == SourceLevel::Preview) as u64) << 60;
|
||||
Some(RenderJob { photo: id, level, source, info: source_info(&p), settings, request, key })
|
||||
let cache = thumb_bucket.map(|b| {
|
||||
let k = Hasher128::new().str(&content_key(&p)).u64(settings.hash64()).u64(b as u64).u64(RENDER_CACHE_VERSION).finish();
|
||||
(self.media.rendered.clone(), k)
|
||||
});
|
||||
Some(RenderJob { photo: id, level, source, info: source_info(&p), settings, request, key, cache })
|
||||
}
|
||||
|
||||
/// Accept a finished job's loaded source into the cache.
|
||||
|
|
@ -205,49 +299,8 @@ pub struct ProbeInfo {
|
|||
pub captured: Option<String>,
|
||||
pub meta: lightcraft_catalog::Meta,
|
||||
pub as_shot_wb: Option<(f64, f64)>,
|
||||
/// Hash of the file's bytes (hex), for duplicate detection.
|
||||
pub content_hash: Option<String>,
|
||||
}
|
||||
|
||||
pub type FileProbe = Arc<dyn Fn(&str) -> Result<ProbeInfo, String> + Send + Sync>;
|
||||
|
||||
/// Register files as photos (duplicates by path are skipped). Returns the new ids.
|
||||
pub fn import_paths(s: &mut crate::Session, paths: &[String]) -> crate::Result<Vec<PhotoId>> {
|
||||
let now = (s.clock)();
|
||||
let mut ops = Vec::new();
|
||||
let mut ids = Vec::new();
|
||||
for path in paths {
|
||||
let exists = s.catalog.photos().any(|p| matches!(&p.source, Source::File { path: q } if q == path));
|
||||
if exists {
|
||||
continue;
|
||||
}
|
||||
let info = match &s.media.file_probe {
|
||||
Some(probe) => match probe(path) {
|
||||
Ok(i) => i,
|
||||
Err(e) => {
|
||||
log::warn!("import {path}: {e}");
|
||||
continue;
|
||||
}
|
||||
},
|
||||
None => ProbeInfo {
|
||||
format: std::path::Path::new(path).extension().map(|e| e.to_string_lossy().to_uppercase()).unwrap_or_default(),
|
||||
..Default::default()
|
||||
},
|
||||
};
|
||||
let id = s.catalog.alloc_photo_id();
|
||||
let name = std::path::Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.clone());
|
||||
let mut p = Photo::new(id, Source::File { path: path.clone() }, &name, &info.format, info.width, info.height, &now);
|
||||
p.kind = info.kind;
|
||||
p.file_size = info.file_size;
|
||||
p.captured = info.captured;
|
||||
p.meta = info.meta;
|
||||
p.as_shot_wb = info.as_shot_wb;
|
||||
if let Some((t, tint)) = info.as_shot_wb {
|
||||
p.develop = std::sync::Arc::new(lightcraft_develop::DevelopSettings::for_raw(t, tint));
|
||||
}
|
||||
ids.push(id);
|
||||
ops.push(lightcraft_catalog::Op::AddPhoto { photo: Box::new(p) });
|
||||
}
|
||||
if !ops.is_empty() {
|
||||
s.commit("Add Photos", lightcraft_catalog::Op::Batch { ops })?;
|
||||
}
|
||||
Ok(ids)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,103 @@
|
|||
//! Import: recursive folders, duplicate detection by content, add vs copy into the library.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::Session;
|
||||
|
||||
fn temp_dir(tag: &str) -> std::path::PathBuf {
|
||||
let d = std::env::temp_dir().join(format!("lc-import-{tag}-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
std::fs::create_dir_all(&d).unwrap();
|
||||
d
|
||||
}
|
||||
|
||||
/// A small procedural PNG (distinct per seed).
|
||||
fn write_png(path: &Path, seed: u8) {
|
||||
let (w, h) = (48usize, 32usize);
|
||||
let data: Vec<[u8; 4]> = (0..w * h).map(|i| [(i % w * 5) as u8, (i / w * 7) as u8, seed, 255]).collect();
|
||||
let img = lightcraft_raster::Rgba8 { width: w, height: h, data };
|
||||
let bytes = lightcraft_codecs::encode_png(&lightcraft_codecs::EncodeImage::rgba8(&img), &lightcraft_codecs::EncodeMeta::default()).unwrap();
|
||||
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
std::fs::write(path, bytes).unwrap();
|
||||
}
|
||||
|
||||
fn ids(v: &Value, key: &str) -> usize {
|
||||
v[key].as_array().map(Vec::len).unwrap_or(0)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recursive_import_with_duplicates() {
|
||||
let src = temp_dir("src");
|
||||
write_png(&src.join("a.png"), 1);
|
||||
write_png(&src.join("trip/b.png"), 2);
|
||||
write_png(&src.join("trip/day2/c.PNG"), 3);
|
||||
std::fs::copy(src.join("a.png"), src.join("trip/a-copy.png")).unwrap(); // same bytes
|
||||
write_png(&src.join(".hidden/x.png"), 9);
|
||||
std::fs::write(src.join("notes.txt"), "not a photo").unwrap();
|
||||
std::fs::write(src.join("trip/broken.jpg"), "garbage").unwrap();
|
||||
|
||||
let mut s = Session::new().with_fs();
|
||||
let r = s.execute("library.import", &json!({"paths": [src.to_string_lossy()]})).unwrap();
|
||||
assert_eq!(r["scanned"], 5, "{r}"); // a, b, c, a-copy, broken (hidden + txt skipped)
|
||||
assert_eq!(ids(&r, "imported"), 3, "{r}");
|
||||
assert_eq!(ids(&r, "duplicates"), 1, "{r}");
|
||||
assert_eq!(r["duplicates"][0]["reason"], "content");
|
||||
assert_eq!(ids(&r, "failed"), 1, "{r}");
|
||||
assert_eq!(s.catalog.len(), 3);
|
||||
assert!(s.catalog.photos().all(|p| p.content_hash.as_ref().is_some_and(|h| h.len() == 32) && p.width == 48));
|
||||
|
||||
// importing again: everything is a duplicate (by path), one undo step for the first import
|
||||
let r2 = s.execute("library.import", &json!({"paths": [src.to_string_lossy()]})).unwrap();
|
||||
assert_eq!(ids(&r2, "imported"), 0);
|
||||
assert_eq!(r2["duplicates"].as_array().unwrap().iter().filter(|d| d["reason"] == "path").count(), 3);
|
||||
s.execute("edit.undo", &json!({})).unwrap();
|
||||
assert_eq!(s.catalog.len(), 0);
|
||||
let _ = std::fs::remove_dir_all(&src);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn copy_into_library_and_persist() {
|
||||
let src = temp_dir("copysrc");
|
||||
let lib = temp_dir("copylib");
|
||||
write_png(&src.join("one.png"), 4);
|
||||
write_png(&src.join("sub/one.png"), 5); // same name, different content
|
||||
let mut s = Session::new().with_fs();
|
||||
assert!(s.execute("library.import", &json!({"paths": [src.to_string_lossy()], "mode": "copy"})).is_err(), "copy needs a library");
|
||||
s.open_library(&lib, false).unwrap();
|
||||
let r = s.execute("library.import", &json!({"paths": [src.to_string_lossy()], "mode": "copy"})).unwrap();
|
||||
assert_eq!(ids(&r, "imported"), 2, "{r}");
|
||||
let paths: Vec<String> = s
|
||||
.catalog
|
||||
.photos()
|
||||
.map(|p| match &p.source {
|
||||
lightcraft_catalog::Source::File { path } => path.clone(),
|
||||
_ => panic!(),
|
||||
})
|
||||
.collect();
|
||||
for p in &paths {
|
||||
assert!(Path::new(p).starts_with(lib.join("Originals")), "{p}");
|
||||
assert!(Path::new(p).exists());
|
||||
}
|
||||
assert_ne!(paths[0], paths[1], "unique names");
|
||||
// originals deleted from the source: the library copies remain usable after a restart
|
||||
std::fs::remove_dir_all(&src).unwrap();
|
||||
drop(s);
|
||||
let mut s = Session::new().with_fs();
|
||||
s.open_library(&lib, false).unwrap();
|
||||
assert_eq!(s.catalog.len(), 2);
|
||||
let id = s.catalog.photos().next().unwrap().id;
|
||||
assert!(s.render_now(id, 32, 32).is_ok());
|
||||
// the library folder itself is never re-imported
|
||||
let r = s.execute("library.import", &json!({"paths": [lib.to_string_lossy()]})).unwrap();
|
||||
assert_eq!(r["scanned"], 0, "{r}");
|
||||
let _ = std::fs::remove_dir_all(&lib);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn civil_dates() {
|
||||
assert_eq!(crate::import::civil(0), "1970-01-01T00:00:00");
|
||||
assert_eq!(crate::import::civil(951_782_400), "2000-02-29T00:00:00");
|
||||
assert_eq!(crate::import::civil(1_790_000_000), "2026-09-21T14:13:20");
|
||||
}
|
||||
|
|
@ -0,0 +1,128 @@
|
|||
//! Persistent library: commands survive a restart (with and without a clean close).
|
||||
|
||||
use serde_json::json;
|
||||
|
||||
use crate::Session;
|
||||
|
||||
fn temp_dir(tag: &str) -> std::path::PathBuf {
|
||||
let d = std::env::temp_dir().join(format!("lc-engine-{tag}-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
d
|
||||
}
|
||||
|
||||
fn open(dir: &std::path::Path, seed: bool) -> Session {
|
||||
let mut s = Session::new();
|
||||
s.open_library(dir, seed).unwrap();
|
||||
s
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn edits_survive_restart_without_close() {
|
||||
let dir = temp_dir("crash");
|
||||
let mut s = open(&dir, true);
|
||||
assert!(s.library.as_ref().unwrap().report.created);
|
||||
assert!(s.catalog.len() > 5, "seeded demo");
|
||||
let id = s.selection.active.unwrap();
|
||||
s.execute("photo.rate", &json!({"rating": 5})).unwrap();
|
||||
s.execute("photo.flag", &json!({"flag": "reject"})).unwrap();
|
||||
s.execute("develop.set", &json!({"control": "light.exposure", "value": 1.25})).unwrap();
|
||||
// a slider drag: one undo step, logged once at the end
|
||||
s.execute("develop.beginInteraction", &json!({"label": "Contrast"})).unwrap();
|
||||
for v in [5, 10, 20, 33] {
|
||||
s.execute("develop.set", &json!({"control": "light.contrast", "value": v})).unwrap();
|
||||
}
|
||||
s.execute("develop.endInteraction", &json!({})).unwrap();
|
||||
s.execute("album.create", &json!({"name": "Keepers", "addSelected": true})).unwrap();
|
||||
s.execute("preset.create", &json!({"name": "Bright"})).unwrap();
|
||||
s.execute("edit.undo", &json!({})).ok();
|
||||
s.execute("edit.redo", &json!({})).ok();
|
||||
let expect = s.catalog.to_snapshot();
|
||||
let info = s.execute("library.info", &json!({})).unwrap();
|
||||
assert!(info["logRecords"].as_u64().unwrap() >= 5, "{info}");
|
||||
drop(s); // no close: like a crash
|
||||
|
||||
let s2 = open(&dir, true);
|
||||
let r = &s2.library.as_ref().unwrap().report;
|
||||
assert!(!r.created && r.replayed >= 5, "{r:?}");
|
||||
assert_eq!(s2.catalog.to_snapshot(), expect);
|
||||
let p = s2.catalog.photo(id).unwrap();
|
||||
assert_eq!((p.rating, p.develop.light.exposure, p.develop.light.contrast), (5, 1.25, 33.0));
|
||||
assert!(s2.catalog.albums().any(|a| a.name == "Keepers" && a.photos == vec![id]));
|
||||
assert!(s2.presets.iter().any(|p| p.name == "Bright" && !p.builtin));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn close_writes_snapshot_and_view_state() {
|
||||
let dir = temp_dir("close");
|
||||
let mut s = open(&dir, true);
|
||||
let ids = s.visible_cloned();
|
||||
s.execute("library.select", &json!({"ids": [ids[3].0]})).unwrap();
|
||||
s.execute("photo.rate", &json!({"rating": 2})).unwrap();
|
||||
let expect = s.catalog.to_snapshot();
|
||||
s.close_library().unwrap();
|
||||
assert_eq!(std::fs::metadata(dir.join("catalog.log")).unwrap().len(), 0);
|
||||
|
||||
let s2 = open(&dir, true);
|
||||
let r = &s2.library.as_ref().unwrap().report;
|
||||
assert_eq!(r.replayed, 0);
|
||||
assert_eq!(s2.catalog.to_snapshot(), expect);
|
||||
assert_eq!(s2.selection.active, Some(ids[3]));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// Thumbnails rendered in one session are served from the disk cache in the next, without
|
||||
/// loading the source; an edit changes the key.
|
||||
#[test]
|
||||
fn thumbnails_come_from_the_disk_cache_after_restart() {
|
||||
let dir = temp_dir("thumbs");
|
||||
let mut s = open(&dir, true);
|
||||
let ids: Vec<_> = s.visible_cloned().into_iter().take(3).collect();
|
||||
for id in &ids {
|
||||
let job = s.thumb_job(*id, 230).unwrap();
|
||||
assert_eq!(job.request.max_w, 256, "bucketed");
|
||||
let r = job.run();
|
||||
assert!(r.loaded.is_some(), "first render decodes the source");
|
||||
s.accept(&r);
|
||||
let img = r.rendered.unwrap().image;
|
||||
assert_eq!(img.width.max(img.height), 256);
|
||||
}
|
||||
drop(s);
|
||||
|
||||
let mut s = open(&dir, true);
|
||||
for id in &ids {
|
||||
let r = s.thumb_job(*id, 240).unwrap().run();
|
||||
assert!(r.loaded.is_none(), "served from cache");
|
||||
let img = r.rendered.unwrap().image;
|
||||
assert_eq!(img.width.max(img.height), 256);
|
||||
}
|
||||
let info = s.execute("library.info", &json!({})).unwrap();
|
||||
assert_eq!(info["cache"]["disk"]["hits"], 3, "{info}");
|
||||
// an edit is a new key: rendered (from the source), then cached
|
||||
s.execute("library.select", &json!({"ids": [ids[0].0]})).unwrap();
|
||||
s.execute("develop.set", &json!({"control": "light.exposure", "value": 0.5})).unwrap();
|
||||
let r = s.thumb_job(ids[0], 230).unwrap().run();
|
||||
assert!(r.loaded.is_some());
|
||||
// exact-size renders (loupe, before/after, exports) bypass the thumbnail cache
|
||||
assert!(s.render_job(ids[0], 230, 230, true, true).unwrap().cache.is_none());
|
||||
assert!(s.render_job(ids[0], 230, 230, false, true).unwrap().cache.is_none());
|
||||
s.execute("library.clearPreviews", &json!({})).unwrap();
|
||||
assert!(s.thumb_job(ids[1], 230).unwrap().run().loaded.is_some());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_library_without_seed_is_empty_and_compacts() {
|
||||
let dir = temp_dir("empty");
|
||||
let mut s = open(&dir, false);
|
||||
assert!(s.catalog.is_empty());
|
||||
s.execute("album.create", &json!({"name": "A"})).unwrap();
|
||||
s.execute("library.compact", &json!({})).unwrap();
|
||||
let info = s.execute("library.info", &json!({})).unwrap();
|
||||
assert_eq!(info["logRecords"], 0);
|
||||
assert_eq!(info["snapshotSeq"], info["seq"]);
|
||||
let s2 = open(&dir, true);
|
||||
assert_eq!(s2.catalog.len(), 0, "an existing library is never seeded");
|
||||
assert_eq!(s2.catalog.albums().count(), 1);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
[package]
|
||||
name = "lightcraft-preview"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
description = "LightCraft preview cache: content hashing, memory LRU, disk thumbnail cache, prioritized job pool (L3)"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
lightcraft-raster = { workspace = true }
|
||||
siphasher = { workspace = true }
|
||||
jpeg-encoder = { workspace = true }
|
||||
zune-jpeg = { workspace = true }
|
||||
zune-core = { workspace = true }
|
||||
log = { workspace = true }
|
||||
|
|
@ -0,0 +1,176 @@
|
|||
//! Rendered thumbnails on disk: `<dir>/<2 hex>/<32 hex>.jpg`, bounded in bytes.
|
||||
//!
|
||||
//! The cache is disposable: files are written via temp + rename (readers never see partial
|
||||
//! files) but not fsynced; unreadable files are deleted and re-rendered. When the total size
|
||||
//! exceeds the budget, the least recently used files (by modification time, refreshed on read)
|
||||
//! are removed down to 80 % of the budget.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use lightcraft_raster::Rgba8;
|
||||
|
||||
use crate::Hash128;
|
||||
|
||||
const QUALITY: u8 = 90;
|
||||
|
||||
pub struct DiskCache {
|
||||
dir: PathBuf,
|
||||
budget: u64,
|
||||
/// Bytes on disk (`None` until first scanned).
|
||||
total: Mutex<Option<u64>>,
|
||||
pub hits: AtomicU64,
|
||||
pub misses: AtomicU64,
|
||||
pub writes: AtomicU64,
|
||||
}
|
||||
|
||||
impl DiskCache {
|
||||
pub fn new(dir: &Path, budget: u64) -> DiskCache {
|
||||
DiskCache {
|
||||
dir: dir.to_path_buf(),
|
||||
budget,
|
||||
total: Mutex::new(None),
|
||||
hits: AtomicU64::new(0),
|
||||
misses: AtomicU64::new(0),
|
||||
writes: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn dir(&self) -> &Path {
|
||||
&self.dir
|
||||
}
|
||||
|
||||
fn path(&self, key: Hash128) -> PathBuf {
|
||||
let hex = key.to_string();
|
||||
self.dir.join(&hex[..2]).join(format!("{hex}.jpg"))
|
||||
}
|
||||
|
||||
pub fn get(&self, key: Hash128) -> Option<Rgba8> {
|
||||
let p = self.path(key);
|
||||
let Ok(bytes) = std::fs::read(&p) else {
|
||||
self.misses.fetch_add(1, Ordering::Relaxed);
|
||||
return None;
|
||||
};
|
||||
match decode(&bytes) {
|
||||
Some(img) => {
|
||||
self.hits.fetch_add(1, Ordering::Relaxed);
|
||||
// refresh recency for pruning
|
||||
if let Ok(f) = std::fs::File::options().write(true).open(&p) {
|
||||
let _ = f.set_modified(std::time::SystemTime::now());
|
||||
}
|
||||
Some(img)
|
||||
}
|
||||
None => {
|
||||
log::warn!("preview cache: dropping unreadable {}", p.display());
|
||||
let _ = std::fs::remove_file(&p);
|
||||
self.misses.fetch_add(1, Ordering::Relaxed);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn put(&self, key: Hash128, img: &Rgba8) {
|
||||
let Some(bytes) = encode(img) else { return };
|
||||
let p = self.path(key);
|
||||
let Some(parent) = p.parent() else { return };
|
||||
if std::fs::create_dir_all(parent).is_err() {
|
||||
return;
|
||||
}
|
||||
// know the current total before adding (the first scan must not count this file twice)
|
||||
let _ = self.size();
|
||||
let tmp = p.with_extension(format!("tmp{:?}", std::thread::current().id()).replace(['(', ')'], ""));
|
||||
if std::fs::write(&tmp, &bytes).is_err() || std::fs::rename(&tmp, &p).is_err() {
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
return;
|
||||
}
|
||||
self.writes.fetch_add(1, Ordering::Relaxed);
|
||||
let over = {
|
||||
let mut t = self.total.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let total = t.get_or_insert(0);
|
||||
*total += bytes.len() as u64;
|
||||
*total > self.budget
|
||||
};
|
||||
if over {
|
||||
self.prune();
|
||||
}
|
||||
}
|
||||
|
||||
/// All cache files: (path, size, modified).
|
||||
fn scan(&self) -> Vec<(PathBuf, u64, std::time::SystemTime)> {
|
||||
let mut out = Vec::new();
|
||||
let Ok(rd) = std::fs::read_dir(&self.dir) else { return out };
|
||||
for sub in rd.flatten() {
|
||||
let Ok(files) = std::fs::read_dir(sub.path()) else { continue };
|
||||
for f in files.flatten() {
|
||||
if let Ok(m) = f.metadata()
|
||||
&& m.is_file()
|
||||
{
|
||||
out.push((f.path(), m.len(), m.modified().unwrap_or(std::time::UNIX_EPOCH)));
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Remove least recently used files until the cache is at most 80 % of its budget.
|
||||
pub fn prune(&self) {
|
||||
let mut files = self.scan();
|
||||
let mut total: u64 = files.iter().map(|f| f.1).sum();
|
||||
let target = self.budget / 5 * 4;
|
||||
files.sort_by_key(|f| f.2);
|
||||
for (p, len, _) in files {
|
||||
if total <= target {
|
||||
break;
|
||||
}
|
||||
if std::fs::remove_file(&p).is_ok() {
|
||||
total -= len;
|
||||
}
|
||||
}
|
||||
*self.total.lock().unwrap_or_else(|e| e.into_inner()) = Some(total);
|
||||
}
|
||||
|
||||
/// Bytes on disk (scans if not yet known).
|
||||
pub fn size(&self) -> u64 {
|
||||
let mut t = self.total.lock().unwrap_or_else(|e| e.into_inner());
|
||||
*t.get_or_insert_with(|| self.scan().iter().map(|f| f.1).sum())
|
||||
}
|
||||
|
||||
pub fn clear(&self) {
|
||||
for (p, _, _) in self.scan() {
|
||||
let _ = std::fs::remove_file(p);
|
||||
}
|
||||
*self.total.lock().unwrap_or_else(|e| e.into_inner()) = Some(0);
|
||||
}
|
||||
}
|
||||
|
||||
fn encode(img: &Rgba8) -> Option<Vec<u8>> {
|
||||
if img.width == 0 || img.height == 0 || img.width > u16::MAX as usize || img.height > u16::MAX as usize {
|
||||
return None;
|
||||
}
|
||||
let mut rgb = Vec::with_capacity(img.width * img.height * 3);
|
||||
for p in &img.data {
|
||||
rgb.extend_from_slice(&p[..3]);
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
let mut enc = jpeg_encoder::Encoder::new(&mut out, QUALITY);
|
||||
enc.set_sampling_factor(jpeg_encoder::SamplingFactor::R_4_2_0);
|
||||
enc.encode(&rgb, img.width as u16, img.height as u16, jpeg_encoder::ColorType::Rgb).ok()?;
|
||||
Some(out)
|
||||
}
|
||||
|
||||
fn decode(bytes: &[u8]) -> Option<Rgba8> {
|
||||
use zune_core::bytestream::ZCursor;
|
||||
use zune_core::colorspace::ColorSpace;
|
||||
use zune_core::options::DecoderOptions;
|
||||
let opts = DecoderOptions::default().jpeg_set_out_colorspace(ColorSpace::RGB);
|
||||
let mut d = zune_jpeg::JpegDecoder::new_with_options(ZCursor::new(bytes), opts);
|
||||
let px = d.decode().ok()?;
|
||||
let info = d.info()?;
|
||||
let (w, h) = (info.width as usize, info.height as usize);
|
||||
if px.len() < w * h * 3 {
|
||||
return None;
|
||||
}
|
||||
let data = px.chunks_exact(3).take(w * h).map(|c| [c[0], c[1], c[2], 255]).collect();
|
||||
Some(Rgba8 { width: w, height: h, data })
|
||||
}
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
//! 128-bit content hashing (SipHash-1-3, fixed key). Not cryptographic — it identifies files
|
||||
//! for duplicate detection and keys caches; collisions are astronomically unlikely by accident.
|
||||
|
||||
use std::fmt;
|
||||
use std::hash::Hasher;
|
||||
|
||||
use siphasher::sip128::{Hasher128 as _, SipHasher13};
|
||||
|
||||
/// A 128-bit hash, shown as 32 hex digits.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||||
pub struct Hash128(pub u128);
|
||||
|
||||
impl fmt::Display for Hash128 {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{:032x}", self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl Hash128 {
|
||||
pub fn parse(s: &str) -> Option<Hash128> {
|
||||
(s.len() == 32).then(|| u128::from_str_radix(s, 16).ok().map(Hash128)).flatten()
|
||||
}
|
||||
}
|
||||
|
||||
/// Streaming hasher.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Hasher128(SipHasher13);
|
||||
|
||||
impl Default for Hasher128 {
|
||||
fn default() -> Self {
|
||||
Hasher128(SipHasher13::new_with_keys(0x4c69_6768_7443_7261, 0x6674_2043_6f6e_7465))
|
||||
}
|
||||
}
|
||||
|
||||
impl Hasher128 {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
pub fn update(&mut self, bytes: &[u8]) -> &mut Self {
|
||||
self.0.write(bytes);
|
||||
self
|
||||
}
|
||||
/// Length-prefixed string (so `("ab","c")` and `("a","bc")` differ).
|
||||
pub fn str(&mut self, s: &str) -> &mut Self {
|
||||
self.0.write(&(s.len() as u64).to_le_bytes());
|
||||
self.0.write(s.as_bytes());
|
||||
self
|
||||
}
|
||||
pub fn u64(&mut self, v: u64) -> &mut Self {
|
||||
self.0.write(&v.to_le_bytes());
|
||||
self
|
||||
}
|
||||
pub fn finish(&self) -> Hash128 {
|
||||
Hash128(self.0.finish128().as_u128())
|
||||
}
|
||||
}
|
||||
|
||||
/// Hash of a whole byte string.
|
||||
pub fn hash_bytes(bytes: &[u8]) -> Hash128 {
|
||||
Hasher128::new().update(bytes).finish()
|
||||
}
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
//! Previews: everything that makes a big library show up instantly.
|
||||
//!
|
||||
//! - [`hash`]: 128-bit content hashes (file bytes → duplicate detection and cache keys).
|
||||
//! - [`Lru`]: a cost-bounded memory LRU (decoded sources, rendered thumbnails).
|
||||
//! - [`DiskCache`]: rendered thumbnails as JPEG files keyed by `(content hash, settings hash,
|
||||
//! size, renderer version)`, bounded in bytes (least recently used files are pruned).
|
||||
//! - [`PreviewCache`]: memory LRU in front of the disk cache; shared by render workers.
|
||||
//! - [`JobPool`]: worker threads with per-slot de-duplication and priorities (visible thumbnails
|
||||
//! before off-screen prefetch); runs jobs inline on wasm.
|
||||
//!
|
||||
//! No UI dependencies (L3): the egui frontend, the CLI and the MCP server share it.
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
pub mod disk;
|
||||
pub mod hash;
|
||||
pub mod lru;
|
||||
pub mod pool;
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
pub use disk::DiskCache;
|
||||
pub use hash::{Hash128, Hasher128, hash_bytes};
|
||||
use lightcraft_raster::Rgba8;
|
||||
pub use lru::Lru;
|
||||
pub use pool::JobPool;
|
||||
|
||||
/// Rendered-thumbnail cache: memory LRU over an optional disk cache.
|
||||
pub struct PreviewCache {
|
||||
mem: Mutex<Lru<Hash128, Arc<Rgba8>>>,
|
||||
disk: Option<DiskCache>,
|
||||
}
|
||||
|
||||
impl PreviewCache {
|
||||
/// Memory-only cache of at most `mem_bytes`.
|
||||
pub fn memory(mem_bytes: usize) -> PreviewCache {
|
||||
PreviewCache { mem: Mutex::new(Lru::new(mem_bytes)), disk: None }
|
||||
}
|
||||
|
||||
/// Memory LRU + disk cache in `dir` (at most `disk_bytes`).
|
||||
pub fn with_disk(mem_bytes: usize, dir: &Path, disk_bytes: u64) -> PreviewCache {
|
||||
PreviewCache { mem: Mutex::new(Lru::new(mem_bytes)), disk: Some(DiskCache::new(dir, disk_bytes)) }
|
||||
}
|
||||
|
||||
pub fn disk(&self) -> Option<&DiskCache> {
|
||||
self.disk.as_ref()
|
||||
}
|
||||
|
||||
pub fn get(&self, key: Hash128) -> Option<Arc<Rgba8>> {
|
||||
if let Some(v) = self.mem.lock().unwrap_or_else(|e| e.into_inner()).get(&key) {
|
||||
return Some(v.clone());
|
||||
}
|
||||
let img = Arc::new(self.disk.as_ref()?.get(key)?);
|
||||
self.mem.lock().unwrap_or_else(|e| e.into_inner()).insert(key, img.clone(), cost(&img));
|
||||
Some(img)
|
||||
}
|
||||
|
||||
pub fn put(&self, key: Hash128, img: Arc<Rgba8>) {
|
||||
if let Some(d) = &self.disk {
|
||||
d.put(key, &img);
|
||||
}
|
||||
let c = cost(&img);
|
||||
self.mem.lock().unwrap_or_else(|e| e.into_inner()).insert(key, img, c);
|
||||
}
|
||||
|
||||
/// Drop everything (memory and disk).
|
||||
pub fn clear(&self) {
|
||||
self.mem.lock().unwrap_or_else(|e| e.into_inner()).clear();
|
||||
if let Some(d) = &self.disk {
|
||||
d.clear();
|
||||
}
|
||||
}
|
||||
|
||||
/// (entries, bytes) held in memory.
|
||||
pub fn mem_usage(&self) -> (usize, usize) {
|
||||
let m = self.mem.lock().unwrap_or_else(|e| e.into_inner());
|
||||
(m.len(), m.cost())
|
||||
}
|
||||
}
|
||||
|
||||
fn cost(img: &Rgba8) -> usize {
|
||||
img.width * img.height * 4 + 64
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
//! A cost-bounded least-recently-used map.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::hash::Hash;
|
||||
|
||||
pub struct Lru<K, V> {
|
||||
map: HashMap<K, (V, usize, u64)>,
|
||||
/// Use tick → key, oldest first.
|
||||
order: BTreeMap<u64, K>,
|
||||
tick: u64,
|
||||
cost: usize,
|
||||
budget: usize,
|
||||
}
|
||||
|
||||
impl<K: Eq + Hash + Clone, V> Lru<K, V> {
|
||||
/// An LRU holding at most `budget` cost units (e.g. bytes). The most recent entry is always
|
||||
/// kept, even if it alone exceeds the budget.
|
||||
pub fn new(budget: usize) -> Self {
|
||||
Lru { map: HashMap::new(), order: BTreeMap::new(), tick: 0, cost: 0, budget }
|
||||
}
|
||||
|
||||
fn touch(&mut self, k: &K) {
|
||||
self.tick += 1;
|
||||
let t = self.tick;
|
||||
if let Some(e) = self.map.get_mut(k) {
|
||||
self.order.remove(&e.2);
|
||||
e.2 = t;
|
||||
self.order.insert(t, k.clone());
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get(&mut self, k: &K) -> Option<&V> {
|
||||
if !self.map.contains_key(k) {
|
||||
return None;
|
||||
}
|
||||
self.touch(k);
|
||||
self.map.get(k).map(|e| &e.0)
|
||||
}
|
||||
|
||||
pub fn peek(&self, k: &K) -> Option<&V> {
|
||||
self.map.get(k).map(|e| &e.0)
|
||||
}
|
||||
|
||||
pub fn contains(&self, k: &K) -> bool {
|
||||
self.map.contains_key(k)
|
||||
}
|
||||
|
||||
pub fn insert(&mut self, k: K, v: V, cost: usize) {
|
||||
self.remove(&k);
|
||||
self.tick += 1;
|
||||
self.order.insert(self.tick, k.clone());
|
||||
self.map.insert(k, (v, cost, self.tick));
|
||||
self.cost += cost;
|
||||
while self.cost > self.budget && self.map.len() > 1 {
|
||||
let Some((_, old)) = self.order.pop_first() else { break };
|
||||
if let Some((_, c, _)) = self.map.remove(&old) {
|
||||
self.cost -= c;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn remove(&mut self, k: &K) -> Option<V> {
|
||||
let (v, c, t) = self.map.remove(k)?;
|
||||
self.order.remove(&t);
|
||||
self.cost -= c;
|
||||
Some(v)
|
||||
}
|
||||
|
||||
pub fn retain(&mut self, mut f: impl FnMut(&K) -> bool) {
|
||||
let drop: Vec<K> = self.map.keys().filter(|k| !f(k)).cloned().collect();
|
||||
for k in drop {
|
||||
self.remove(&k);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn clear(&mut self) {
|
||||
self.map.clear();
|
||||
self.order.clear();
|
||||
self.cost = 0;
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.map.len()
|
||||
}
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.map.is_empty()
|
||||
}
|
||||
pub fn cost(&self) -> usize {
|
||||
self.cost
|
||||
}
|
||||
pub fn budget(&self) -> usize {
|
||||
self.budget
|
||||
}
|
||||
pub fn set_budget(&mut self, budget: usize) {
|
||||
self.budget = budget;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,179 @@
|
|||
//! A prioritized job pool with per-slot de-duplication.
|
||||
//!
|
||||
//! Each job targets a *slot* (a grid cell, the loupe…). Submitting for a slot replaces a job for
|
||||
//! that slot that hasn't started yet, so scrolling or dragging never builds a backlog. Workers
|
||||
//! take the highest priority first (newest first among equals). Frontends re-prioritize or drop
|
||||
//! queued jobs when the view changes (e.g. thumbnails scrolled out of view).
|
||||
//!
|
||||
//! Native: a fixed set of worker threads, started on first use. wasm32 (no threads): the host
|
||||
//! calls [`JobPool::run_inline`] once per frame.
|
||||
|
||||
use std::hash::Hash;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::mpsc::{Receiver, Sender, channel};
|
||||
use std::sync::{Arc, Condvar, Mutex};
|
||||
|
||||
pub type Job<R> = Box<dyn FnOnce() -> R + Send>;
|
||||
|
||||
struct Entry<S, R> {
|
||||
slot: S,
|
||||
key: u64,
|
||||
priority: u32,
|
||||
seq: u64,
|
||||
job: Job<R>,
|
||||
}
|
||||
|
||||
struct Shared<S, R> {
|
||||
queue: Mutex<Vec<Entry<S, R>>>,
|
||||
cv: Condvar,
|
||||
shutdown: AtomicBool,
|
||||
}
|
||||
|
||||
/// A finished job.
|
||||
pub struct Done<S, R> {
|
||||
pub slot: S,
|
||||
/// The key the job was submitted with (to drop stale results).
|
||||
pub key: u64,
|
||||
pub result: R,
|
||||
/// Run time in milliseconds (0 on wasm).
|
||||
pub ms: f64,
|
||||
}
|
||||
|
||||
pub struct JobPool<S, R> {
|
||||
shared: Arc<Shared<S, R>>,
|
||||
tx: Sender<Done<S, R>>,
|
||||
rx: Receiver<Done<S, R>>,
|
||||
threads: usize,
|
||||
started: bool,
|
||||
seq: u64,
|
||||
}
|
||||
|
||||
impl<S: Copy + Eq + Hash + Send + 'static, R: Send + 'static> JobPool<S, R> {
|
||||
/// A pool with `threads` workers (none on wasm; `0` = jobs only run via [`JobPool::run_inline`]).
|
||||
pub fn new(threads: usize) -> Self {
|
||||
let (tx, rx) = channel();
|
||||
JobPool {
|
||||
shared: Arc::new(Shared { queue: Mutex::new(Vec::new()), cv: Condvar::new(), shutdown: AtomicBool::new(false) }),
|
||||
tx,
|
||||
rx,
|
||||
threads,
|
||||
started: false,
|
||||
seq: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Workers for this machine: cores − 1, between 2 and 8.
|
||||
pub fn default_threads() -> usize {
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
{
|
||||
std::thread::available_parallelism().map(|n| n.get().saturating_sub(1)).unwrap_or(4).clamp(2, 8)
|
||||
}
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
{
|
||||
1
|
||||
}
|
||||
}
|
||||
|
||||
fn start(&mut self) {
|
||||
if self.started {
|
||||
return;
|
||||
}
|
||||
self.started = true;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
for i in 0..self.threads {
|
||||
let shared = self.shared.clone();
|
||||
let tx = self.tx.clone();
|
||||
let _ = std::thread::Builder::new().name(format!("lc-job-{i}")).spawn(move || {
|
||||
while let Some(e) = take(&shared, true) {
|
||||
let t0 = std::time::Instant::now();
|
||||
let result = (e.job)();
|
||||
let done = Done { slot: e.slot, key: e.key, result, ms: t0.elapsed().as_secs_f64() * 1000.0 };
|
||||
if tx.send(done).is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Queue `job` for `slot`, replacing a queued (not yet running) job for the same slot.
|
||||
pub fn submit(&mut self, slot: S, key: u64, priority: u32, job: Job<R>) {
|
||||
self.start();
|
||||
self.seq += 1;
|
||||
let (lock, cv) = (&self.shared.queue, &self.shared.cv);
|
||||
let mut q = lock.lock().unwrap_or_else(|e| e.into_inner());
|
||||
q.retain(|e| e.slot != slot);
|
||||
q.push(Entry { slot, key, priority, seq: self.seq, job });
|
||||
cv.notify_one();
|
||||
}
|
||||
|
||||
/// Change queued jobs' priorities: `f(slot, priority)` returns the new priority, or `None` to
|
||||
/// drop the job. Returns the dropped slots.
|
||||
pub fn reprioritize(&self, mut f: impl FnMut(&S, u32) -> Option<u32>) -> Vec<S> {
|
||||
let mut dropped = Vec::new();
|
||||
let mut q = self.shared.queue.lock().unwrap_or_else(|e| e.into_inner());
|
||||
q.retain_mut(|e| match f(&e.slot, e.priority) {
|
||||
Some(p) => {
|
||||
e.priority = p;
|
||||
true
|
||||
}
|
||||
None => {
|
||||
dropped.push(e.slot);
|
||||
false
|
||||
}
|
||||
});
|
||||
dropped
|
||||
}
|
||||
|
||||
/// Queued (not started) jobs.
|
||||
pub fn queued(&self) -> usize {
|
||||
self.shared.queue.lock().map(|q| q.len()).unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Is a job for `slot` queued?
|
||||
pub fn is_queued(&self, slot: S) -> bool {
|
||||
self.shared.queue.lock().map(|q| q.iter().any(|e| e.slot == slot)).unwrap_or(false)
|
||||
}
|
||||
|
||||
/// A finished job, if any.
|
||||
pub fn try_recv(&self) -> Option<Done<S, R>> {
|
||||
self.rx.try_recv().ok()
|
||||
}
|
||||
|
||||
/// Run up to `n` jobs on the calling thread (wasm, tests). Returns how many ran.
|
||||
pub fn run_inline(&mut self, n: usize) -> usize {
|
||||
let mut ran = 0;
|
||||
while ran < n {
|
||||
let Some(e) = take(&self.shared, false) else { break };
|
||||
let result = (e.job)();
|
||||
let _ = self.tx.send(Done { slot: e.slot, key: e.key, result, ms: 0.0 });
|
||||
ran += 1;
|
||||
}
|
||||
ran
|
||||
}
|
||||
}
|
||||
|
||||
/// Pop the best job (highest priority, newest first). Blocks when `wait` until one is queued or
|
||||
/// the pool shuts down.
|
||||
fn take<S, R>(shared: &Shared<S, R>, wait: bool) -> Option<Entry<S, R>> {
|
||||
let mut q = shared.queue.lock().unwrap_or_else(|e| e.into_inner());
|
||||
loop {
|
||||
if shared.shutdown.load(Ordering::Relaxed) {
|
||||
return None;
|
||||
}
|
||||
if let Some(i) = q.iter().enumerate().max_by_key(|(_, e)| (e.priority, e.seq)).map(|(i, _)| i) {
|
||||
return Some(q.swap_remove(i));
|
||||
}
|
||||
if !wait {
|
||||
return None;
|
||||
}
|
||||
q = shared.cv.wait(q).unwrap_or_else(|e| e.into_inner());
|
||||
}
|
||||
}
|
||||
|
||||
impl<S, R> Drop for JobPool<S, R> {
|
||||
fn drop(&mut self) {
|
||||
self.shared.shutdown.store(true, Ordering::Relaxed);
|
||||
self.shared.cv.notify_all();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,156 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use lightcraft_raster::Rgba8;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn temp_dir(tag: &str) -> std::path::PathBuf {
|
||||
let d = std::env::temp_dir().join(format!("lc-preview-{tag}-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
d
|
||||
}
|
||||
|
||||
fn gradient(w: usize, h: usize, seed: u8) -> Rgba8 {
|
||||
let data = (0..w * h).map(|i| [((i % w) * 255 / w) as u8, ((i / w) * 255 / h) as u8, seed, 255]).collect();
|
||||
Rgba8 { width: w, height: h, data }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hashes_are_stable_and_distinct() {
|
||||
let a = hash_bytes(b"hello");
|
||||
assert_eq!(a, hash_bytes(b"hello"));
|
||||
assert_ne!(a, hash_bytes(b"hellp"));
|
||||
assert_eq!(Hash128::parse(&a.to_string()), Some(a));
|
||||
assert_ne!(Hasher128::new().str("ab").str("c").finish(), Hasher128::new().str("a").str("bc").finish());
|
||||
// streaming == one shot
|
||||
assert_eq!(Hasher128::new().update(b"hel").update(b"lo").finish(), a);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lru_evicts_least_recent_by_cost() {
|
||||
let mut l = Lru::new(10);
|
||||
l.insert(1, "a", 4);
|
||||
l.insert(2, "b", 4);
|
||||
assert_eq!(l.get(&1), Some(&"a")); // 1 is now most recent
|
||||
l.insert(3, "c", 4); // over budget: evicts 2
|
||||
assert!(l.contains(&1) && !l.contains(&2) && l.contains(&3));
|
||||
assert_eq!(l.cost(), 8);
|
||||
l.insert(4, "huge", 50); // alone over budget: kept, everything else evicted
|
||||
assert_eq!((l.len(), l.cost()), (1, 50));
|
||||
l.insert(4, "small", 1);
|
||||
assert_eq!((l.len(), l.cost()), (1, 1));
|
||||
l.retain(|k| *k != 4);
|
||||
assert!(l.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disk_cache_roundtrip_and_prune() {
|
||||
let dir = temp_dir("disk");
|
||||
let d = DiskCache::new(&dir, 40_000);
|
||||
let img = gradient(160, 100, 7);
|
||||
let k = hash_bytes(b"k1");
|
||||
assert!(d.get(k).is_none());
|
||||
d.put(k, &img);
|
||||
let back = d.get(k).expect("hit");
|
||||
assert_eq!((back.width, back.height), (160, 100));
|
||||
let err: i64 = img.data.iter().zip(&back.data).map(|(a, b)| (a[0] as i64 - b[0] as i64).abs() + (a[1] as i64 - b[1] as i64).abs()).max().unwrap();
|
||||
assert!(err < 24, "JPEG error {err}");
|
||||
assert!(back.data.iter().all(|p| p[3] == 255));
|
||||
// fill past the budget: the oldest entries go, the newest stay
|
||||
let keys: Vec<_> = (0..40u8).map(|i| hash_bytes(&[i, 1])).collect();
|
||||
for (i, k) in keys.iter().enumerate() {
|
||||
d.put(*k, &gradient(160, 100, i as u8));
|
||||
}
|
||||
assert!(d.size() <= 40_000, "{}", d.size());
|
||||
assert!(d.get(*keys.last().unwrap()).is_some());
|
||||
assert!(d.get(keys[0]).is_none());
|
||||
// a corrupt file is a miss and gets removed
|
||||
let k2 = hash_bytes(b"corrupt");
|
||||
d.put(k2, &img);
|
||||
let hex = k2.to_string();
|
||||
let p = dir.join(&hex[..2]).join(format!("{hex}.jpg"));
|
||||
std::fs::write(&p, b"not a jpeg").unwrap();
|
||||
assert!(d.get(k2).is_none());
|
||||
assert!(!p.exists());
|
||||
d.clear();
|
||||
assert_eq!(d.size(), 0);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preview_cache_memory_then_disk() {
|
||||
let dir = temp_dir("pc");
|
||||
let k = hash_bytes(b"x");
|
||||
{
|
||||
let c = PreviewCache::with_disk(1 << 20, &dir, 1 << 24);
|
||||
c.put(k, Arc::new(gradient(64, 48, 1)));
|
||||
assert_eq!(c.mem_usage().0, 1);
|
||||
assert!(c.get(k).is_some());
|
||||
}
|
||||
// a new process: memory is empty, disk serves it
|
||||
let c = PreviewCache::with_disk(1 << 20, &dir, 1 << 24);
|
||||
assert_eq!(c.mem_usage().0, 0);
|
||||
assert_eq!(c.get(k).map(|i| i.width), Some(64));
|
||||
assert_eq!(c.mem_usage().0, 1);
|
||||
assert_eq!(c.disk().unwrap().hits.load(std::sync::atomic::Ordering::Relaxed), 1);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_priorities_dedupe_and_drop() {
|
||||
let mut p: JobPool<u32, u32> = JobPool::new(1);
|
||||
// One worker, held busy by a gate job while the others queue up.
|
||||
let gate = Arc::new((std::sync::Mutex::new(false), std::sync::Condvar::new()));
|
||||
let g = gate.clone();
|
||||
p.submit(
|
||||
999,
|
||||
0,
|
||||
1000,
|
||||
Box::new(move || {
|
||||
let (m, cv) = &*g;
|
||||
let mut open = m.lock().unwrap();
|
||||
while !*open {
|
||||
open = cv.wait(open).unwrap();
|
||||
}
|
||||
999
|
||||
}),
|
||||
);
|
||||
// wait until the worker took the gate job
|
||||
while p.queued() > 0 {
|
||||
std::thread::yield_now();
|
||||
}
|
||||
p.submit(1, 1, 5, Box::new(|| 1)); // off-screen thumb
|
||||
p.submit(2, 1, 10, Box::new(|| 2)); // visible thumb
|
||||
p.submit(3, 1, 100, Box::new(|| 3)); // loupe
|
||||
p.submit(2, 2, 10, Box::new(|| 22)); // newer request for slot 2 replaces the queued one
|
||||
p.submit(4, 1, 5, Box::new(|| 4));
|
||||
assert_eq!(p.queued(), 4);
|
||||
let dropped = p.reprioritize(|s, pr| if *s == 4 { None } else { Some(pr) });
|
||||
assert_eq!(dropped, vec![4]);
|
||||
{
|
||||
let (m, cv) = &*gate;
|
||||
*m.lock().unwrap() = true;
|
||||
cv.notify_all();
|
||||
}
|
||||
let mut order = Vec::new();
|
||||
while order.len() < 4 {
|
||||
if let Some(d) = p.try_recv() {
|
||||
order.push((d.slot, d.key, d.result));
|
||||
} else {
|
||||
std::thread::yield_now();
|
||||
}
|
||||
}
|
||||
assert_eq!(order, vec![(999, 0, 999), (3, 1, 3), (2, 2, 22), (1, 1, 1)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_runs_inline() {
|
||||
// No workers: run_inline drains the queue on the caller's thread (the wasm path).
|
||||
let mut p: JobPool<u8, u8> = JobPool::new(0);
|
||||
p.submit(1, 0, 1, Box::new(|| 1));
|
||||
p.submit(2, 0, 9, Box::new(|| 2));
|
||||
assert_eq!(p.run_inline(1), 1);
|
||||
assert_eq!(p.try_recv().map(|d| d.result), Some(2));
|
||||
assert_eq!(p.run_inline(10), 1);
|
||||
assert_eq!(p.try_recv().map(|d| d.result), Some(1));
|
||||
}
|
||||
|
|
@ -17,6 +17,7 @@ lightcraft-pipeline = { workspace = true }
|
|||
lightcraft-catalog = { workspace = true }
|
||||
web-time = { workspace = true }
|
||||
lightcraft-engine = { workspace = true }
|
||||
lightcraft-preview = { workspace = true }
|
||||
egui = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ pub fn inspect(app: &LightcraftApp, ctx: &egui::Context) -> Value {
|
|||
"selection": app.session.selection.ids.iter().map(|p| p.0).collect::<Vec<_>>(),
|
||||
"activeMask": app.session.active_mask,
|
||||
"widgetCount": app.widgets.len(),
|
||||
"perf": {"frameMs": app.perf.frame_ms, "fps": app.perf.fps, "lastRenderMs": app.renderer.last_main_ms, "renderQueue": app.renderer.queued(), "rendersDone": app.renderer.completed},
|
||||
"perf": {"frameMs": app.perf.frame_ms, "fps": app.perf.fps, "lastRenderMs": app.renderer.last_main_ms, "renderQueue": app.renderer.queued(), "rendersInFlight": app.renderer.in_flight(), "rendersDone": app.renderer.completed, "thumbTextures": app.renderer.thumb_textures()},
|
||||
"status": app.ui.status,
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -201,6 +201,7 @@ impl LightcraftApp {
|
|||
ctx.request_repaint();
|
||||
}
|
||||
self.renderer.poll(ctx, &mut self.session);
|
||||
self.session.persist_if_dirty();
|
||||
self.collect_screenshots(ctx);
|
||||
self.issue_screenshots(ctx);
|
||||
if self.fonts_ready {
|
||||
|
|
|
|||
|
|
@ -626,7 +626,7 @@ fn filmstrip(app: &mut LightcraftApp, ui: &mut egui::Ui, r: Rect) {
|
|||
p.text(pos2(cr.right() - 8.0, cr.top() + 10.0), Align2::RIGHT_CENTER, &ph.format, t.semibold(8.5), t.text_dim);
|
||||
}
|
||||
let img_area = Rect::from_min_max(cr.min + vec2(10.0, 22.0), cr.max - vec2(10.0, 8.0));
|
||||
if let Some(job) = app.session.render_job(*id, (256.0 * ppp.min(2.0) / 2.0) as usize * 2, 256, false, true) {
|
||||
if let Some(job) = app.session.thumb_job(*id, (256.0 * ppp.min(2.0) / 2.0) as usize * 2) {
|
||||
app.renderer.request(Slot::Thumb(*id), job, 8);
|
||||
}
|
||||
if let Some(tex) = app.renderer.textures.get(&Slot::Thumb(*id)) {
|
||||
|
|
|
|||
|
|
@ -144,7 +144,7 @@ fn cell(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId, r: Rect, square
|
|||
};
|
||||
// thumbnail
|
||||
let size = thumb_px(img_rect.width().max(img_rect.height()), ppp);
|
||||
if let Some(job) = app.session.render_job(id, size, size, false, true) {
|
||||
if let Some(job) = app.session.thumb_job(id, size) {
|
||||
app.renderer.request(Slot::Thumb(id), job, if onscreen { 10 } else { 5 });
|
||||
}
|
||||
if let Some(tex) = app.renderer.textures.get(&Slot::Thumb(id)) {
|
||||
|
|
|
|||
|
|
@ -1,16 +1,17 @@
|
|||
//! Background rendering: a small worker pool runs engine [`RenderJob`]s; results become textures.
|
||||
//! Background rendering: engine [`RenderJob`]s run on a [`JobPool`]; results become textures.
|
||||
//!
|
||||
//! Each visible thing that needs pixels is a *slot* (a grid thumbnail, the loupe, the "before"
|
||||
//! image…). Requests are deduplicated per slot (a newer request replaces a queued older one) and
|
||||
//! prioritised (loupe first, then visible thumbnails). On wasm the jobs run inline, one per frame.
|
||||
//! prioritised (loupe first, then on-screen thumbnails, then prefetch); thumbnails scrolled far
|
||||
//! out of view are dropped from the queue. Thumbnail jobs hit the engine's preview cache (memory +
|
||||
//! disk) before rendering. On wasm the jobs run inline, one per frame.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::mpsc::{Receiver, Sender, channel};
|
||||
use std::sync::{Arc, Condvar, Mutex};
|
||||
|
||||
use lightcraft_catalog::PhotoId;
|
||||
use lightcraft_engine::Session;
|
||||
use lightcraft_engine::media::{RenderJob, RenderResult};
|
||||
use lightcraft_preview::JobPool;
|
||||
use lightcraft_raster::Histogram;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
|
|
@ -30,15 +31,11 @@ pub struct Tex {
|
|||
pub ms: f64,
|
||||
}
|
||||
|
||||
type Queue = Arc<(Mutex<Vec<(Slot, u32, RenderJob)>>, Condvar)>;
|
||||
|
||||
pub struct Renderer {
|
||||
queue: Queue,
|
||||
results_tx: Sender<(Slot, RenderResult, f64)>,
|
||||
results: Receiver<(Slot, RenderResult, f64)>,
|
||||
pending: HashMap<Slot, u64>,
|
||||
pool: JobPool<Slot, RenderResult>,
|
||||
/// Slot → (key, priority) of the request in flight (queued or running).
|
||||
pending: HashMap<Slot, (u64, u32)>,
|
||||
pub textures: HashMap<Slot, Tex>,
|
||||
started: bool,
|
||||
pub last_main_ms: f64,
|
||||
/// Jobs finished since start (for inspect/perf).
|
||||
pub completed: u64,
|
||||
|
|
@ -46,104 +43,64 @@ pub struct Renderer {
|
|||
|
||||
impl Default for Renderer {
|
||||
fn default() -> Self {
|
||||
let (tx, rx) = channel();
|
||||
Renderer {
|
||||
queue: Arc::new((Mutex::new(Vec::new()), Condvar::new())),
|
||||
results_tx: tx,
|
||||
results: rx,
|
||||
pending: HashMap::new(),
|
||||
textures: HashMap::new(),
|
||||
started: false,
|
||||
last_main_ms: 0.0,
|
||||
completed: 0,
|
||||
}
|
||||
let threads = if cfg!(target_arch = "wasm32") { 0 } else { JobPool::<Slot, RenderResult>::default_threads().min(6) };
|
||||
Renderer { pool: JobPool::new(threads), pending: HashMap::new(), textures: HashMap::new(), last_main_ms: 0.0, completed: 0 }
|
||||
}
|
||||
}
|
||||
|
||||
fn now() -> f64 {
|
||||
crate::now_ms()
|
||||
}
|
||||
|
||||
impl Renderer {
|
||||
fn start(&mut self) {
|
||||
if self.started {
|
||||
return;
|
||||
}
|
||||
self.started = true;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
{
|
||||
let n = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4).clamp(2, 6);
|
||||
for i in 0..n {
|
||||
let q = self.queue.clone();
|
||||
let tx = self.results_tx.clone();
|
||||
let _ = std::thread::Builder::new().name(format!("lc-render-{i}")).spawn(move || {
|
||||
loop {
|
||||
let job = {
|
||||
let (lock, cv) = &*q;
|
||||
let mut g = lock.lock().unwrap_or_else(|e| e.into_inner());
|
||||
while g.is_empty() {
|
||||
g = cv.wait(g).unwrap_or_else(|e| e.into_inner());
|
||||
}
|
||||
// highest priority, newest first
|
||||
let best = g.iter().enumerate().max_by_key(|(i, (_, p, _))| (*p, *i)).map(|(i, _)| i).unwrap_or(0);
|
||||
g.remove(best)
|
||||
};
|
||||
let t0 = now();
|
||||
let (slot, _, job) = job;
|
||||
let r = job.run();
|
||||
if tx.send((slot, r, now() - t0)).is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Is the slot's current texture (or pending request) already for `key`?
|
||||
pub fn is_current(&self, slot: Slot, key: u64) -> bool {
|
||||
self.textures.get(&slot).is_some_and(|t| t.key == key) || self.pending.get(&slot) == Some(&key)
|
||||
self.textures.get(&slot).is_some_and(|t| t.key == key) || self.pending.get(&slot).is_some_and(|p| p.0 == key)
|
||||
}
|
||||
|
||||
/// Request a render for `slot` (no-op if already current or pending).
|
||||
/// Request a render for `slot` (no-op if already current or pending at the same priority).
|
||||
pub fn request(&mut self, slot: Slot, job: RenderJob, priority: u32) {
|
||||
if self.is_current(slot, job.key) {
|
||||
if self.textures.get(&slot).is_some_and(|t| t.key == job.key) {
|
||||
return;
|
||||
}
|
||||
self.start();
|
||||
self.pending.insert(slot, job.key);
|
||||
let (lock, cv) = &*self.queue;
|
||||
let mut g = lock.lock().unwrap_or_else(|e| e.into_inner());
|
||||
g.retain(|(s, _, _)| *s != slot);
|
||||
g.push((slot, priority, job));
|
||||
cv.notify_one();
|
||||
if let Some(&(key, prio)) = self.pending.get(&slot)
|
||||
&& key == job.key
|
||||
&& (prio == priority || !self.pool.is_queued(slot))
|
||||
{
|
||||
return;
|
||||
}
|
||||
self.pending.insert(slot, (job.key, priority));
|
||||
let key = job.key;
|
||||
self.pool.submit(slot, key, priority, Box::new(move || job.run()));
|
||||
}
|
||||
|
||||
pub fn queued(&self) -> usize {
|
||||
self.queue.0.lock().map(|g| g.len()).unwrap_or(0)
|
||||
self.pool.queued()
|
||||
}
|
||||
|
||||
/// Requests queued or running.
|
||||
pub fn in_flight(&self) -> usize {
|
||||
self.pending.len()
|
||||
}
|
||||
|
||||
/// Thumbnail textures currently loaded.
|
||||
pub fn thumb_textures(&self) -> usize {
|
||||
self.textures.keys().filter(|s| matches!(s, Slot::Thumb(_))).count()
|
||||
}
|
||||
|
||||
/// Collect finished jobs into textures. Returns true if anything changed.
|
||||
pub fn poll(&mut self, ctx: &egui::Context, session: &mut Session) -> bool {
|
||||
// wasm: run one job per frame on this thread, timed with the host clock
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
{
|
||||
// inline: run one job per frame
|
||||
let job = {
|
||||
let mut g = self.queue.0.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let best = g.iter().enumerate().max_by_key(|(i, (_, p, _))| (*p, *i)).map(|(i, _)| i);
|
||||
best.map(|b| g.remove(b))
|
||||
};
|
||||
if let Some((slot, _, job)) = job {
|
||||
let t0 = now();
|
||||
let r = job.run();
|
||||
let _ = self.results_tx.send((slot, r, now() - t0));
|
||||
}
|
||||
}
|
||||
let inline_ms = {
|
||||
let t0 = crate::now_ms();
|
||||
self.pool.run_inline(1);
|
||||
crate::now_ms() - t0
|
||||
};
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
let inline_ms = 0.0;
|
||||
let mut changed = false;
|
||||
while let Ok((slot, r, ms)) = self.results.try_recv() {
|
||||
while let Some(done) = self.pool.try_recv() {
|
||||
let (slot, r, ms) = (done.slot, done.result, if done.ms > 0.0 { done.ms } else { inline_ms });
|
||||
session.accept(&r);
|
||||
self.completed += 1;
|
||||
if self.pending.get(&slot) == Some(&r.key) {
|
||||
if self.pending.get(&slot).is_some_and(|p| p.0 == r.key) {
|
||||
self.pending.remove(&slot);
|
||||
}
|
||||
let Ok(rendered) = r.rendered else {
|
||||
|
|
@ -180,8 +137,16 @@ impl Renderer {
|
|||
changed
|
||||
}
|
||||
|
||||
/// Drop thumbnails that aren't in `keep` (bounded memory for huge libraries).
|
||||
/// Drop thumbnails that aren't in `keep` (bounded memory for huge libraries), and queued
|
||||
/// thumbnail jobs for photos that scrolled out of `keep`.
|
||||
pub fn evict_thumbs(&mut self, keep: &std::collections::HashSet<PhotoId>, max: usize) {
|
||||
let dropped = self.pool.reprioritize(|s, p| match s {
|
||||
Slot::Thumb(id) if !keep.contains(id) && p <= 10 => None,
|
||||
_ => Some(p),
|
||||
});
|
||||
for s in dropped {
|
||||
self.pending.remove(&s);
|
||||
}
|
||||
let thumbs = self.textures.keys().filter(|s| matches!(s, Slot::Thumb(_))).count();
|
||||
if thumbs <= max {
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -4,11 +4,11 @@ The screenshots in `docs/images/` are captured by driving LightCraft over its co
|
|||
|
||||
```sh
|
||||
# public-domain set (downloaded into the gitignored corpus/, see assets/ATTRIBUTION.md for sources)
|
||||
cargo run --release -p lightcraft -- --control 7980 corpus/images/pd &
|
||||
cargo run --release -p lightcraft -- --memory --control 7980 corpus/images/pd &
|
||||
python3 docs/showcase/run.py docs/showcase/pd-edits.jsonl
|
||||
|
||||
# procedural demo library
|
||||
cargo run --release -p lightcraft -- --control 7980 &
|
||||
cargo run --release -p lightcraft -- --memory --control 7980 &
|
||||
python3 docs/showcase/run.py docs/showcase/demo.jsonl
|
||||
|
||||
# compress for the repo (macOS)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Demo-library showcase: run LightCraft with `--control 7980` (no files → procedural demo library)
|
||||
# Demo-library showcase: run LightCraft with `--memory --control 7980` (no files → procedural demo library)
|
||||
{"method":"ui.resize","params":{"width":1600,"height":1000}}
|
||||
{"method":"engine.execute","params":{"command":"view.photoGrid"}}
|
||||
{"method":"ui.set","params":{"leftPanel":true,"right":"edit","thumbSize":230,"openSections":["light","color"],"presets":false}}
|
||||
|
|
|
|||
Loading…
Reference in New Issue