Perf: big libraries — keyword tree 5.6× faster, per-frame library queries cached
A 100k-photo scaling check (ignored test `scale_100k_library_queries`) showed the left panel rebuilding the keyword tree every frame (323 ms), the grid recomputing date headers (57 ms) and the Keywords panel its suggestions (44 ms) every frame. - keyword_tree builds with hash maps instead of linear sibling scans: 323 → 57 ms at 100k. - LightcraftApp::caches keeps the keyword tree, date runs and keyword suggestions per catalog revision (and inputs), so each is computed once per change instead of every frame. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Brandon Thomas
parent
1fe5dea3ab
commit
ec78cb8236
|
|
@ -51,38 +51,60 @@ pub struct KeywordNode {
|
|||
impl Catalog {
|
||||
/// The keyword tree: every keyword level with photo counts, sorted by name (case-insensitive).
|
||||
pub fn keyword_tree(&self) -> Vec<KeywordNode> {
|
||||
fn insert(nodes: &mut Vec<KeywordNode>, parts: &[&str], prefix: &str, photo: u64, seen: &mut std::collections::HashSet<(String, u64)>) {
|
||||
let Some((first, rest)) = parts.split_first() else { return };
|
||||
let path = if prefix.is_empty() { first.to_string() } else { format!("{prefix}|{first}") };
|
||||
let i = match nodes.iter().position(|n| n.name.eq_ignore_ascii_case(first)) {
|
||||
Some(i) => i,
|
||||
None => {
|
||||
nodes.push(KeywordNode { name: first.to_string(), path: path.clone(), count: 0, children: Vec::new() });
|
||||
nodes.len() - 1
|
||||
}
|
||||
};
|
||||
if seen.insert((path.to_lowercase(), photo)) {
|
||||
nodes[i].count += 1;
|
||||
}
|
||||
let p = nodes[i].path.clone();
|
||||
insert(&mut nodes[i].children, rest, &p, photo, seen);
|
||||
use std::collections::{HashMap, HashSet};
|
||||
// every keyword level (`travel`, `travel|italy`…) by its lower-case path: the name and
|
||||
// path as first written, and the photos counted (once per photo, even when two of its
|
||||
// keywords share a parent)
|
||||
struct Level {
|
||||
name: String,
|
||||
path: String,
|
||||
count: usize,
|
||||
parent: Option<String>,
|
||||
}
|
||||
fn sort(nodes: &mut [KeywordNode]) {
|
||||
nodes.sort_by_key(|n| n.name.to_lowercase());
|
||||
for n in nodes {
|
||||
sort(&mut n.children);
|
||||
}
|
||||
}
|
||||
let mut roots = Vec::new();
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
let mut levels: HashMap<String, Level> = HashMap::new();
|
||||
let mut this_photo: HashSet<String> = HashSet::new();
|
||||
for p in self.photos().filter(|p| p.in_library()) {
|
||||
this_photo.clear();
|
||||
for k in &p.meta.keywords {
|
||||
let parts: Vec<&str> = k.split(SEP).map(str::trim).filter(|s| !s.is_empty()).collect();
|
||||
insert(&mut roots, &parts, "", p.id.0, &mut seen);
|
||||
let mut path = String::new();
|
||||
let mut lower = String::new();
|
||||
let mut parent: Option<String> = None;
|
||||
for part in k.split(SEP).map(str::trim).filter(|s| !s.is_empty()) {
|
||||
if !path.is_empty() {
|
||||
path.push(SEP);
|
||||
lower.push(SEP);
|
||||
}
|
||||
path.push_str(part);
|
||||
lower.push_str(&part.to_lowercase());
|
||||
let l = levels.entry(lower.clone()).or_insert_with(|| Level {
|
||||
name: part.to_string(),
|
||||
path: path.clone(),
|
||||
count: 0,
|
||||
parent: parent.clone(),
|
||||
});
|
||||
if this_photo.insert(lower.clone()) {
|
||||
l.count += 1;
|
||||
}
|
||||
parent = Some(lower.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
sort(&mut roots);
|
||||
roots
|
||||
let mut kids: HashMap<Option<String>, Vec<String>> = HashMap::new();
|
||||
for (lower, l) in &levels {
|
||||
kids.entry(l.parent.clone()).or_default().push(lower.clone());
|
||||
}
|
||||
fn build(at: Option<String>, levels: &HashMap<String, Level>, kids: &mut HashMap<Option<String>, Vec<String>>) -> Vec<KeywordNode> {
|
||||
let Some(mut list) = kids.remove(&at) else { return Vec::new() };
|
||||
list.sort_by_key(|l| levels[l].name.to_lowercase());
|
||||
list.into_iter()
|
||||
.map(|l| {
|
||||
let lv = &levels[&l];
|
||||
let children = build(Some(l.clone()), levels, kids);
|
||||
KeywordNode { name: lv.name.clone(), path: lv.path.clone(), count: lv.count, children }
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
build(None, &levels, &mut kids)
|
||||
}
|
||||
|
||||
/// `SetMeta` ops for every photo whose keywords `f` changes.
|
||||
|
|
|
|||
|
|
@ -237,3 +237,85 @@ fn quick_collection_and_target_album() {
|
|||
s.execute("album.clearQuick", &serde_json::json!({})).unwrap();
|
||||
assert_eq!(s.catalog.album_count(quick), 0);
|
||||
}
|
||||
|
||||
/// Scaling check (ignored: `cargo test --release -p lightcraft-engine -- --ignored scale --nocapture`):
|
||||
/// the per-frame / per-click library queries on a 100k-photo catalog.
|
||||
#[test]
|
||||
#[ignore]
|
||||
fn scale_100k_library_queries() {
|
||||
use lightcraft_catalog::{Op, Photo, PhotoId, Source};
|
||||
use std::time::Instant;
|
||||
let mut s = crate::Session::new();
|
||||
let n = 100_000u64;
|
||||
let mut ops = Vec::with_capacity(n as usize);
|
||||
for i in 0..n {
|
||||
let mut p = Photo::new(
|
||||
PhotoId(i + 1),
|
||||
Source::Demo { scene: (i % 20) as u32 },
|
||||
&format!("IMG_{i:06}.jpg"),
|
||||
"JPEG",
|
||||
6000,
|
||||
4000,
|
||||
"2026-01-01T00:00:00",
|
||||
);
|
||||
p.captured = Some(format!("20{:02}-{:02}-{:02}T{:02}:00:00", 10 + i % 16, 1 + i % 12, 1 + i % 28, i % 24));
|
||||
p.rating = (i % 6) as u8;
|
||||
p.meta.keywords = vec![format!("kw{}", i % 500), "travel|italy".into()];
|
||||
p.meta.camera = format!("Model {}", i % 30);
|
||||
ops.push(Op::AddPhoto { photo: Box::new(p) });
|
||||
}
|
||||
let t = Instant::now();
|
||||
s.commit("Add", Op::Batch { ops }).unwrap();
|
||||
eprintln!("add 100k: {:.0} ms", t.elapsed().as_secs_f64() * 1e3);
|
||||
let time = |what: &str, f: &mut dyn FnMut()| {
|
||||
let t = Instant::now();
|
||||
f();
|
||||
let ms = t.elapsed().as_secs_f64() * 1e3;
|
||||
eprintln!("{what}: {ms:.1} ms");
|
||||
ms
|
||||
};
|
||||
let mut worst: Vec<(String, f64)> = Vec::new();
|
||||
let mut m = |w: &str, f: &mut dyn FnMut()| worst.push((w.to_string(), time(w, f)));
|
||||
m("visible (all)", &mut || {
|
||||
let _ = s.visible_cloned();
|
||||
});
|
||||
m("visible (cached)", &mut || {
|
||||
let _ = s.visible_cloned();
|
||||
});
|
||||
m("filter rating ≥ 4", &mut || {
|
||||
s.execute("library.filter", &serde_json::json!({"rating": 4})).unwrap();
|
||||
let _ = s.visible_cloned();
|
||||
});
|
||||
m("filter ruleSet keyword + date", &mut || {
|
||||
s.execute("library.filter", &serde_json::json!({"rating": 0, "ruleSet": {"rules": [{"field": "keywords", "op": "contains", "value": "kw42"}, {"field": "captureDate", "op": "is", "value": "2015"}]}})).unwrap();
|
||||
let _ = s.visible_cloned();
|
||||
});
|
||||
s.execute("library.filter", &serde_json::json!({"ruleSet": null})).unwrap();
|
||||
m("date groups", &mut || {
|
||||
let _ = s.execute("library.groups", &serde_json::json!({})).unwrap();
|
||||
});
|
||||
m("keyword tree", &mut || {
|
||||
let _ = s.execute("keyword.list", &serde_json::json!({})).unwrap();
|
||||
});
|
||||
m("keyword suggestions", &mut || {
|
||||
let _ = s.catalog.keyword_suggestions(&["kw1".to_string()], "kw", 12);
|
||||
});
|
||||
let alb = s.execute("album.createSmart", &serde_json::json!({"name": "S", "rules": {"rating": 5}})).unwrap()["id"].as_u64().unwrap();
|
||||
m("smart album count", &mut || {
|
||||
let _ = s.catalog.album_count(lightcraft_catalog::AlbumId(alb));
|
||||
});
|
||||
m("albums.list", &mut || {
|
||||
let _ = s.execute("albums.list", &serde_json::json!({})).unwrap();
|
||||
});
|
||||
m("select all", &mut || {
|
||||
s.execute("library.selectAll", &serde_json::json!({})).unwrap();
|
||||
});
|
||||
m("rate 100k selected", &mut || {
|
||||
s.execute("photo.rate", &serde_json::json!({"rating": 3})).unwrap();
|
||||
});
|
||||
m("undo", &mut || {
|
||||
s.execute("edit.undo", &serde_json::json!({})).unwrap();
|
||||
});
|
||||
worst.sort_by(|a, b| b.1.total_cmp(&a.1));
|
||||
eprintln!("slowest: {:?}", &worst[..3]);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -80,6 +80,9 @@ pub struct Perf {
|
|||
}
|
||||
|
||||
pub struct LightcraftApp {
|
||||
/// Per-catalog-revision caches of library-wide results the panels show every frame
|
||||
/// (expensive on big libraries).
|
||||
pub caches: Caches,
|
||||
pub session: Session,
|
||||
pub ui: UiState,
|
||||
pub services: Services,
|
||||
|
|
@ -147,6 +150,7 @@ impl LightcraftApp {
|
|||
services,
|
||||
renderer: render::Renderer::default(),
|
||||
perf: Perf::default(),
|
||||
caches: Caches::default(),
|
||||
integrated_titlebar: false,
|
||||
native_menu: false,
|
||||
native_shortcuts: Default::default(),
|
||||
|
|
@ -620,3 +624,60 @@ mod drop_tests {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Results recomputed only when the catalog (or their inputs) change.
|
||||
#[derive(Default)]
|
||||
pub struct Caches {
|
||||
keyword_tree: Option<(u64, std::sync::Arc<Vec<lightcraft_catalog::KeywordNode>>)>,
|
||||
date_runs: Option<(u64, std::sync::Arc<Vec<lightcraft_catalog::DateRun>>)>,
|
||||
suggestions: Option<(u64, std::sync::Arc<Vec<String>>)>,
|
||||
}
|
||||
|
||||
fn key_of(parts: impl std::hash::Hash) -> u64 {
|
||||
use std::hash::{BuildHasher, BuildHasherDefault, DefaultHasher};
|
||||
BuildHasherDefault::<DefaultHasher>::default().hash_one(parts)
|
||||
}
|
||||
|
||||
impl Caches {
|
||||
/// The library's keyword tree.
|
||||
pub fn keyword_tree(&mut self, cat: &lightcraft_catalog::Catalog) -> std::sync::Arc<Vec<lightcraft_catalog::KeywordNode>> {
|
||||
match &self.keyword_tree {
|
||||
Some((r, t)) if *r == cat.revision => t.clone(),
|
||||
_ => {
|
||||
let t = std::sync::Arc::new(cat.keyword_tree());
|
||||
self.keyword_tree = Some((cat.revision, t.clone()));
|
||||
t
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Date headers for `ids` in the grid.
|
||||
pub fn date_runs(
|
||||
&mut self,
|
||||
cat: &lightcraft_catalog::Catalog,
|
||||
ids: &[lightcraft_catalog::PhotoId],
|
||||
key: lightcraft_catalog::SortKey,
|
||||
by: lightcraft_catalog::GroupBy,
|
||||
) -> std::sync::Arc<Vec<lightcraft_catalog::DateRun>> {
|
||||
let k = key_of((cat.revision, ids, format!("{key:?}{by:?}")));
|
||||
match &self.date_runs {
|
||||
Some((h, r)) if *h == k => r.clone(),
|
||||
_ => {
|
||||
let r = std::sync::Arc::new(cat.date_runs(ids, key, by));
|
||||
self.date_runs = Some((k, r.clone()));
|
||||
r
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Keyword suggestions for a photo with `current` keywords and the typed `prefix`.
|
||||
pub fn suggestions(&mut self, cat: &lightcraft_catalog::Catalog, current: &[String], prefix: &str, n: usize) -> std::sync::Arc<Vec<String>> {
|
||||
let k = key_of((cat.revision, current, prefix, n));
|
||||
match &self.suggestions {
|
||||
Some((h, v)) if *h == k => v.clone(),
|
||||
_ => {
|
||||
let v = std::sync::Arc::new(cat.keyword_suggestions(current, prefix, n));
|
||||
self.suggestions = Some((k, v.clone()));
|
||||
v
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -82,7 +82,7 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui) {
|
|||
} else {
|
||||
app.session.sort.key
|
||||
};
|
||||
app.session.catalog.date_runs(&ids, key, by)
|
||||
(*app.caches.date_runs(&app.session.catalog, &ids, key, by)).clone()
|
||||
};
|
||||
let spans: Vec<(usize, usize)> = runs.iter().map(|r| (r.start, r.count)).collect();
|
||||
let lay = layout(&aspects, &spans, avail_w, target, square);
|
||||
|
|
|
|||
|
|
@ -449,7 +449,7 @@ fn is_within(app: &LightcraftApp, id: lightcraft_catalog::AlbumId, ancestor: lig
|
|||
/// context menu renames, merges or deletes the keyword across the library.
|
||||
fn keywords_section(app: &mut LightcraftApp, ui: &mut egui::Ui) {
|
||||
let t = Tokens::get(ui.ctx());
|
||||
let tree = app.session.catalog.keyword_tree();
|
||||
let tree = app.caches.keyword_tree(&app.session.catalog);
|
||||
if tree.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -632,7 +632,7 @@ fn keywords(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) {
|
|||
// photo's keywords, else the most used ones
|
||||
let typed = ui.data(|d| d.get_temp::<String>(kid).unwrap_or_default());
|
||||
let last = typed.rsplit(',').next().unwrap_or("").trim().to_string();
|
||||
let suggestions = app.session.catalog.keyword_suggestions(&p.meta.keywords, &last, 12);
|
||||
let suggestions = (*app.caches.suggestions(&app.session.catalog, &p.meta.keywords, &last, 12)).clone();
|
||||
if !suggestions.is_empty() {
|
||||
ui.label(egui::RichText::new("Suggestions").color(t.text_dim));
|
||||
ui.horizontal_wrapped(|ui| {
|
||||
|
|
|
|||
Loading…
Reference in New Issue