From ec78cb823606b6a8fb2cb2c51b5e66b13e39f09b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 00:05:38 -0400 Subject: [PATCH] =?UTF-8?q?Perf:=20big=20libraries=20=E2=80=94=20keyword?= =?UTF-8?q?=20tree=205.6=C3=97=20faster,=20per-frame=20library=20queries?= =?UTF-8?q?=20cached?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 100k-photo scaling check (ignored test `scale_100k_library_queries`) showed the left panel rebuilding the keyword tree every frame (323 ms), the grid recomputing date headers (57 ms) and the Keywords panel its suggestions (44 ms) every frame. - keyword_tree builds with hash maps instead of linear sibling scans: 323 → 57 ms at 100k. - LightcraftApp::caches keeps the keyword tree, date runs and keyword suggestions per catalog revision (and inputs), so each is computed once per change instead of every frame. Co-Authored-By: Claude Opus 5.5 --- crates/catalog/src/keywords.rs | 76 +++++++++++++++++---------- crates/engine/src/tests_library.rs | 82 ++++++++++++++++++++++++++++++ crates/ui-egui/src/lib.rs | 61 ++++++++++++++++++++++ crates/ui-egui/src/panels/grid.rs | 2 +- crates/ui-egui/src/panels/left.rs | 2 +- crates/ui-egui/src/panels/right.rs | 2 +- 6 files changed, 195 insertions(+), 30 deletions(-) diff --git a/crates/catalog/src/keywords.rs b/crates/catalog/src/keywords.rs index 4fc1b9c6..dde02015 100644 --- a/crates/catalog/src/keywords.rs +++ b/crates/catalog/src/keywords.rs @@ -51,38 +51,60 @@ pub struct KeywordNode { impl Catalog { /// The keyword tree: every keyword level with photo counts, sorted by name (case-insensitive). pub fn keyword_tree(&self) -> Vec { - fn insert(nodes: &mut Vec, parts: &[&str], prefix: &str, photo: u64, seen: &mut std::collections::HashSet<(String, u64)>) { - let Some((first, rest)) = parts.split_first() else { return }; - let path = if prefix.is_empty() { first.to_string() } else { format!("{prefix}|{first}") }; - let i = match nodes.iter().position(|n| n.name.eq_ignore_ascii_case(first)) { - Some(i) => i, - None => { - nodes.push(KeywordNode { name: first.to_string(), path: path.clone(), count: 0, children: Vec::new() }); - nodes.len() - 1 - } - }; - if seen.insert((path.to_lowercase(), photo)) { - nodes[i].count += 1; - } - let p = nodes[i].path.clone(); - insert(&mut nodes[i].children, rest, &p, photo, seen); + use std::collections::{HashMap, HashSet}; + // every keyword level (`travel`, `travel|italy`…) by its lower-case path: the name and + // path as first written, and the photos counted (once per photo, even when two of its + // keywords share a parent) + struct Level { + name: String, + path: String, + count: usize, + parent: Option, } - fn sort(nodes: &mut [KeywordNode]) { - nodes.sort_by_key(|n| n.name.to_lowercase()); - for n in nodes { - sort(&mut n.children); - } - } - let mut roots = Vec::new(); - let mut seen = std::collections::HashSet::new(); + let mut levels: HashMap = HashMap::new(); + let mut this_photo: HashSet = HashSet::new(); for p in self.photos().filter(|p| p.in_library()) { + this_photo.clear(); for k in &p.meta.keywords { - let parts: Vec<&str> = k.split(SEP).map(str::trim).filter(|s| !s.is_empty()).collect(); - insert(&mut roots, &parts, "", p.id.0, &mut seen); + let mut path = String::new(); + let mut lower = String::new(); + let mut parent: Option = None; + for part in k.split(SEP).map(str::trim).filter(|s| !s.is_empty()) { + if !path.is_empty() { + path.push(SEP); + lower.push(SEP); + } + path.push_str(part); + lower.push_str(&part.to_lowercase()); + let l = levels.entry(lower.clone()).or_insert_with(|| Level { + name: part.to_string(), + path: path.clone(), + count: 0, + parent: parent.clone(), + }); + if this_photo.insert(lower.clone()) { + l.count += 1; + } + parent = Some(lower.clone()); + } } } - sort(&mut roots); - roots + let mut kids: HashMap, Vec> = HashMap::new(); + for (lower, l) in &levels { + kids.entry(l.parent.clone()).or_default().push(lower.clone()); + } + fn build(at: Option, levels: &HashMap, kids: &mut HashMap, Vec>) -> Vec { + let Some(mut list) = kids.remove(&at) else { return Vec::new() }; + list.sort_by_key(|l| levels[l].name.to_lowercase()); + list.into_iter() + .map(|l| { + let lv = &levels[&l]; + let children = build(Some(l.clone()), levels, kids); + KeywordNode { name: lv.name.clone(), path: lv.path.clone(), count: lv.count, children } + }) + .collect() + } + build(None, &levels, &mut kids) } /// `SetMeta` ops for every photo whose keywords `f` changes. diff --git a/crates/engine/src/tests_library.rs b/crates/engine/src/tests_library.rs index 67bc9564..6b8ee8d9 100644 --- a/crates/engine/src/tests_library.rs +++ b/crates/engine/src/tests_library.rs @@ -237,3 +237,85 @@ fn quick_collection_and_target_album() { s.execute("album.clearQuick", &serde_json::json!({})).unwrap(); assert_eq!(s.catalog.album_count(quick), 0); } + +/// Scaling check (ignored: `cargo test --release -p lightcraft-engine -- --ignored scale --nocapture`): +/// the per-frame / per-click library queries on a 100k-photo catalog. +#[test] +#[ignore] +fn scale_100k_library_queries() { + use lightcraft_catalog::{Op, Photo, PhotoId, Source}; + use std::time::Instant; + let mut s = crate::Session::new(); + let n = 100_000u64; + let mut ops = Vec::with_capacity(n as usize); + for i in 0..n { + let mut p = Photo::new( + PhotoId(i + 1), + Source::Demo { scene: (i % 20) as u32 }, + &format!("IMG_{i:06}.jpg"), + "JPEG", + 6000, + 4000, + "2026-01-01T00:00:00", + ); + p.captured = Some(format!("20{:02}-{:02}-{:02}T{:02}:00:00", 10 + i % 16, 1 + i % 12, 1 + i % 28, i % 24)); + p.rating = (i % 6) as u8; + p.meta.keywords = vec![format!("kw{}", i % 500), "travel|italy".into()]; + p.meta.camera = format!("Model {}", i % 30); + ops.push(Op::AddPhoto { photo: Box::new(p) }); + } + let t = Instant::now(); + s.commit("Add", Op::Batch { ops }).unwrap(); + eprintln!("add 100k: {:.0} ms", t.elapsed().as_secs_f64() * 1e3); + let time = |what: &str, f: &mut dyn FnMut()| { + let t = Instant::now(); + f(); + let ms = t.elapsed().as_secs_f64() * 1e3; + eprintln!("{what}: {ms:.1} ms"); + ms + }; + let mut worst: Vec<(String, f64)> = Vec::new(); + let mut m = |w: &str, f: &mut dyn FnMut()| worst.push((w.to_string(), time(w, f))); + m("visible (all)", &mut || { + let _ = s.visible_cloned(); + }); + m("visible (cached)", &mut || { + let _ = s.visible_cloned(); + }); + m("filter rating ≥ 4", &mut || { + s.execute("library.filter", &serde_json::json!({"rating": 4})).unwrap(); + let _ = s.visible_cloned(); + }); + m("filter ruleSet keyword + date", &mut || { + s.execute("library.filter", &serde_json::json!({"rating": 0, "ruleSet": {"rules": [{"field": "keywords", "op": "contains", "value": "kw42"}, {"field": "captureDate", "op": "is", "value": "2015"}]}})).unwrap(); + let _ = s.visible_cloned(); + }); + s.execute("library.filter", &serde_json::json!({"ruleSet": null})).unwrap(); + m("date groups", &mut || { + let _ = s.execute("library.groups", &serde_json::json!({})).unwrap(); + }); + m("keyword tree", &mut || { + let _ = s.execute("keyword.list", &serde_json::json!({})).unwrap(); + }); + m("keyword suggestions", &mut || { + let _ = s.catalog.keyword_suggestions(&["kw1".to_string()], "kw", 12); + }); + let alb = s.execute("album.createSmart", &serde_json::json!({"name": "S", "rules": {"rating": 5}})).unwrap()["id"].as_u64().unwrap(); + m("smart album count", &mut || { + let _ = s.catalog.album_count(lightcraft_catalog::AlbumId(alb)); + }); + m("albums.list", &mut || { + let _ = s.execute("albums.list", &serde_json::json!({})).unwrap(); + }); + m("select all", &mut || { + s.execute("library.selectAll", &serde_json::json!({})).unwrap(); + }); + m("rate 100k selected", &mut || { + s.execute("photo.rate", &serde_json::json!({"rating": 3})).unwrap(); + }); + m("undo", &mut || { + s.execute("edit.undo", &serde_json::json!({})).unwrap(); + }); + worst.sort_by(|a, b| b.1.total_cmp(&a.1)); + eprintln!("slowest: {:?}", &worst[..3]); +} diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 5de32bc3..35bfba5a 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -80,6 +80,9 @@ pub struct Perf { } pub struct LightcraftApp { + /// Per-catalog-revision caches of library-wide results the panels show every frame + /// (expensive on big libraries). + pub caches: Caches, pub session: Session, pub ui: UiState, pub services: Services, @@ -147,6 +150,7 @@ impl LightcraftApp { services, renderer: render::Renderer::default(), perf: Perf::default(), + caches: Caches::default(), integrated_titlebar: false, native_menu: false, native_shortcuts: Default::default(), @@ -620,3 +624,60 @@ mod drop_tests { } } } + +/// Results recomputed only when the catalog (or their inputs) change. +#[derive(Default)] +pub struct Caches { + keyword_tree: Option<(u64, std::sync::Arc>)>, + date_runs: Option<(u64, std::sync::Arc>)>, + suggestions: Option<(u64, std::sync::Arc>)>, +} + +fn key_of(parts: impl std::hash::Hash) -> u64 { + use std::hash::{BuildHasher, BuildHasherDefault, DefaultHasher}; + BuildHasherDefault::::default().hash_one(parts) +} + +impl Caches { + /// The library's keyword tree. + pub fn keyword_tree(&mut self, cat: &lightcraft_catalog::Catalog) -> std::sync::Arc> { + match &self.keyword_tree { + Some((r, t)) if *r == cat.revision => t.clone(), + _ => { + let t = std::sync::Arc::new(cat.keyword_tree()); + self.keyword_tree = Some((cat.revision, t.clone())); + t + } + } + } + /// Date headers for `ids` in the grid. + pub fn date_runs( + &mut self, + cat: &lightcraft_catalog::Catalog, + ids: &[lightcraft_catalog::PhotoId], + key: lightcraft_catalog::SortKey, + by: lightcraft_catalog::GroupBy, + ) -> std::sync::Arc> { + let k = key_of((cat.revision, ids, format!("{key:?}{by:?}"))); + match &self.date_runs { + Some((h, r)) if *h == k => r.clone(), + _ => { + let r = std::sync::Arc::new(cat.date_runs(ids, key, by)); + self.date_runs = Some((k, r.clone())); + r + } + } + } + /// Keyword suggestions for a photo with `current` keywords and the typed `prefix`. + pub fn suggestions(&mut self, cat: &lightcraft_catalog::Catalog, current: &[String], prefix: &str, n: usize) -> std::sync::Arc> { + let k = key_of((cat.revision, current, prefix, n)); + match &self.suggestions { + Some((h, v)) if *h == k => v.clone(), + _ => { + let v = std::sync::Arc::new(cat.keyword_suggestions(current, prefix, n)); + self.suggestions = Some((k, v.clone())); + v + } + } + } +} diff --git a/crates/ui-egui/src/panels/grid.rs b/crates/ui-egui/src/panels/grid.rs index 8555786d..384c980b 100644 --- a/crates/ui-egui/src/panels/grid.rs +++ b/crates/ui-egui/src/panels/grid.rs @@ -82,7 +82,7 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui) { } else { app.session.sort.key }; - app.session.catalog.date_runs(&ids, key, by) + (*app.caches.date_runs(&app.session.catalog, &ids, key, by)).clone() }; let spans: Vec<(usize, usize)> = runs.iter().map(|r| (r.start, r.count)).collect(); let lay = layout(&aspects, &spans, avail_w, target, square); diff --git a/crates/ui-egui/src/panels/left.rs b/crates/ui-egui/src/panels/left.rs index 140186f8..73030077 100644 --- a/crates/ui-egui/src/panels/left.rs +++ b/crates/ui-egui/src/panels/left.rs @@ -449,7 +449,7 @@ fn is_within(app: &LightcraftApp, id: lightcraft_catalog::AlbumId, ancestor: lig /// context menu renames, merges or deletes the keyword across the library. fn keywords_section(app: &mut LightcraftApp, ui: &mut egui::Ui) { let t = Tokens::get(ui.ctx()); - let tree = app.session.catalog.keyword_tree(); + let tree = app.caches.keyword_tree(&app.session.catalog); if tree.is_empty() { return; } diff --git a/crates/ui-egui/src/panels/right.rs b/crates/ui-egui/src/panels/right.rs index aa6a9473..94574dd8 100644 --- a/crates/ui-egui/src/panels/right.rs +++ b/crates/ui-egui/src/panels/right.rs @@ -632,7 +632,7 @@ fn keywords(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) { // photo's keywords, else the most used ones let typed = ui.data(|d| d.get_temp::(kid).unwrap_or_default()); let last = typed.rsplit(',').next().unwrap_or("").trim().to_string(); - let suggestions = app.session.catalog.keyword_suggestions(&p.meta.keywords, &last, 12); + let suggestions = (*app.caches.suggestions(&app.session.catalog, &p.meta.keywords, &last, 12)).clone(); if !suggestions.is_empty() { ui.label(egui::RichText::new("Suggestions").color(t.text_dim)); ui.horizontal_wrapped(|ui| {