diff --git a/AGENTS.md b/AGENTS.md index 24e9f144..f46b6a18 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,6 +59,8 @@ and often so a crash loses minutes, not hours. `apps/lightcraft-cli/tests/`). New features need at least one test that would fail without them. - **Run the app after every user-visible change**: launch with `--control 7980`, drive it with a JSON-lines script (`docs/showcase/run.py file.jsonl`), take `ui.screenshot`, and look at it. Check `ui.inspect` → `perf`. +- **Benchmarks:** `cargo xtask bench` (24 MP raw from corpus; CPU and GPU columns) appends to `target/bench/history.jsonl` + and flags CPU-time regressions > 20 % vs the previous run (`--strict` to fail). Run it before and after perf work. - **Measure, don't guess**: `LIGHTCRAFT_PROFILE=1` prints per-stage pipeline timings to stderr; time CLI renders with `/usr/bin/time`. Record numbers in `plan/STATUS.md` → Metrics. Budgets: slider update ≤ 16 ms (draft) / loupe ≤ 60 ms on ~2.5 MP; export ≤ 1 s per 24 MP JPEG. diff --git a/CLAUDE.md b/CLAUDE.md index 40f492c7..f46b6a18 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,6 +42,10 @@ and often so a crash loses minutes, not hours. ## Running and looking at the app - `cargo run --release -p lightcraft -- --control 7980` opens the desktop app with the JSON-lines control server (see `docs/control-protocol.md`). - For UI work, **look at the result**: drive via the control channel and take `ui.screenshot`, compare with `plan/lightroom/screenshots/`. +- **Unattended (display asleep/locked, CI, nightly runs): prefer headless snapshots** — no window needed: + `lightcraft-cli snapshot --demo --script tour.jsonl -o out.png --size 1600x1000` (control-protocol requests, + one per line; see `docs/control-protocol.md` → Headless rendering). In a running app use + `ui.screenshot {"path": …, "headless": true}`; windowed screenshots fall back to headless after 2 s. - MCP: `lightcraft-cli mcp` (headless, `--demo` for the procedural library) or `lightcraft-cli mcp --connect` (drives the running app). See `docs/mcp.md`. Quick non-UI checks: `lightcraft-cli render in.jpg -o out.jpg --set light.exposure=1`. - Export goes through `lightcraft_engine::export` (one encoder for app, CLI, MCP and web); UI-only commands live in @@ -55,6 +59,8 @@ and often so a crash loses minutes, not hours. `apps/lightcraft-cli/tests/`). New features need at least one test that would fail without them. - **Run the app after every user-visible change**: launch with `--control 7980`, drive it with a JSON-lines script (`docs/showcase/run.py file.jsonl`), take `ui.screenshot`, and look at it. Check `ui.inspect` → `perf`. +- **Benchmarks:** `cargo xtask bench` (24 MP raw from corpus; CPU and GPU columns) appends to `target/bench/history.jsonl` + and flags CPU-time regressions > 20 % vs the previous run (`--strict` to fail). Run it before and after perf work. - **Measure, don't guess**: `LIGHTCRAFT_PROFILE=1` prints per-stage pipeline timings to stderr; time CLI renders with `/usr/bin/time`. Record numbers in `plan/STATUS.md` → Metrics. Budgets: slider update ≤ 16 ms (draft) / loupe ≤ 60 ms on ~2.5 MP; export ≤ 1 s per 24 MP JPEG. diff --git a/xtask/src/bench.rs b/xtask/src/bench.rs new file mode 100644 index 00000000..7f94f37f --- /dev/null +++ b/xtask/src/bench.rs @@ -0,0 +1,160 @@ +//! `cargo xtask bench [FILE] [--strict] [--threshold PCT]`: run the end-to-end render benchmark, record +//! the results in `target/bench/history.jsonl` (one JSON object per run, with git revision and load +//! average) and compare with the previous run. +//! +//! Comparisons use **process CPU time**, which reflects the work done even on a busy shared machine; +//! wall-clock is recorded for reference. `--strict` exits non-zero when a CPU metric regressed by more +//! than the threshold (default 20 %). + +use std::collections::BTreeMap; +use std::path::Path; +use std::process::Command; + +/// Parse `render_bench` output lines such as +/// `loupe draft 1152×768 cold: cpu: 379.5 ms wall, 139.5 ms cpu | gpu: 19.9 ms wall, 17.2 ms cpu` +/// or `export JPEG encode: 63.9 ms wall, 278.0 ms cpu` into `name/backend/{wall,cpu}` → ms. +pub fn parse(output: &str) -> BTreeMap { + let mut out = BTreeMap::new(); + for line in output.lines() { + let Some((name, rest)) = line.split_once(':') else { continue }; + if !rest.contains(" ms ") { + continue; + } + let name = name.trim(); + for seg in rest.split('|') { + let seg = seg.trim(); + let (backend, body) = match seg.split_once(':') { + Some((b, body)) if matches!(b.trim(), "cpu" | "gpu") => (b.trim(), body), + _ => ("cpu", seg), + }; + for part in body.split(',') { + let toks: Vec<&str> = part.split_whitespace().collect(); + if let [v, "ms", kind] = toks[..] + && let Ok(v) = v.parse::() + && matches!(kind, "wall" | "cpu") + { + out.insert(format!("{name}/{backend}/{kind}"), v); + } + } + } + } + out +} + +/// CPU-time metrics that got slower than `threshold` (fraction) vs `prev`: (key, prev, now). +pub fn regressions(prev: &BTreeMap, now: &BTreeMap, threshold: f64) -> Vec<(String, f64, f64)> { + now.iter() + .filter(|(k, _)| k.ends_with("/cpu")) + .filter_map(|(k, &v)| prev.get(k).map(|&p| (k.clone(), p, v))) + .filter(|(_, p, v)| *p > 1.0 && *v > p * (1.0 + threshold)) + .collect() +} + +fn load_avg() -> Option { + let out = Command::new("uptime").output().ok()?; + let s = String::from_utf8_lossy(&out.stdout); + let tail = s.rsplit_once("load average")?.1; + tail.trim_start_matches(['s', ':', ' ']).split([',', ' ']).find(|t| !t.is_empty())?.parse().ok() +} + +fn git_rev(root: &Path) -> String { + Command::new("git") + .args(["rev-parse", "--short", "HEAD"]) + .current_dir(root) + .output() + .ok() + .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()) + .unwrap_or_default() +} + +pub fn run(root: &Path, args: &[&str]) -> Result<(), String> { + let strict = args.contains(&"--strict"); + let threshold = + args.iter().position(|a| *a == "--threshold").and_then(|i| args.get(i + 1)).and_then(|v| v.parse::().ok()).unwrap_or(20.0) / 100.0; + let file = args.iter().find(|a| !a.starts_with("--") && a.parse::().is_err()).map(|s| s.to_string()); + let file = file.or_else(|| { + let default = root.join("corpus/raw/arw-sony-a7m3-compressed.arw"); + default.exists().then(|| default.to_string_lossy().to_string()) + }); + let mut cmd = crate::cargo(); + cmd.args(["run", "--release", "-q", "-p", "lightcraft-engine", "--example", "render_bench", "--"]); + if let Some(f) = &file { + cmd.arg(f); + } + eprintln!("$ cargo run --release -p lightcraft-engine --example render_bench -- {}", file.as_deref().unwrap_or("(procedural 24 MP source)")); + let out = cmd.output().map_err(|e| format!("render_bench: {e}"))?; + let text = String::from_utf8_lossy(&out.stdout).to_string(); + print!("{text}"); + if !out.status.success() { + return Err(format!("render_bench failed:\n{}", String::from_utf8_lossy(&out.stderr))); + } + let metrics = parse(&text); + if metrics.is_empty() { + return Err("no metrics parsed from render_bench output".into()); + } + let dir = root.join("target/bench"); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + let hist = dir.join("history.jsonl"); + let prev: Option> = std::fs::read_to_string(&hist) + .ok() + .and_then(|s| s.lines().rev().find(|l| !l.trim().is_empty()).map(str::to_string)) + .and_then(|l| serde_json::from_str::(&l).ok()) + .and_then(|v| serde_json::from_value(v["metrics"].clone()).ok()); + let rec = serde_json::json!({ + "rev": git_rev(root), + "time": std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0), + "load": load_avg(), + "input": file, + "metrics": metrics, + }); + let mut line = rec.to_string(); + line.push('\n'); + use std::io::Write; + std::fs::OpenOptions::new().create(true).append(true).open(&hist).and_then(|mut f| f.write_all(line.as_bytes())).map_err(|e| e.to_string())?; + eprintln!("bench: {} metrics recorded in {}", metrics.len(), hist.display()); + let Some(prev) = prev else { + eprintln!("bench: first run — nothing to compare"); + return Ok(()); + }; + let reg = regressions(&prev, &metrics, threshold); + if reg.is_empty() { + eprintln!("bench: no CPU-time regressions > {:.0}% vs previous run", threshold * 100.0); + return Ok(()); + } + for (k, p, v) in ® { + eprintln!("bench: REGRESSION {k}: {p:.1} → {v:.1} ms (+{:.0}%)", (v / p - 1.0) * 100.0); + } + if strict { Err(format!("{} CPU-time regression(s)", reg.len())) } else { Ok(()) } +} + +#[cfg(test)] +mod tests { + use super::*; + + const OUT: &str = "gpu: Apple M4 Pro (Metal) (device + kernels: 322 ms) +gpu vs cpu, loupe 1920×1280 typical: max 1 LSB, mean 0.0001 LSB +loupe draft 1152×768 cold: cpu: 379.5 ms wall, 139.5 ms cpu | gpu: 19.9 ms wall, 17.2 ms cpu +export JPEG encode: 63.9 ms wall, 278.0 ms cpu +"; + + #[test] + fn parses_cpu_gpu_and_plain_lines() { + let m = parse(OUT); + assert_eq!(m["loupe draft 1152×768 cold/cpu/wall"], 379.5); + assert_eq!(m["loupe draft 1152×768 cold/gpu/cpu"], 17.2); + assert_eq!(m["export JPEG encode/cpu/cpu"], 278.0); + assert_eq!(m.len(), 6, "{m:?}"); + } + + #[test] + fn flags_only_cpu_time_regressions_over_threshold() { + let prev = parse(OUT); + let mut now = prev.clone(); + *now.get_mut("export JPEG encode/cpu/cpu").unwrap() = 400.0; // +44 % + *now.get_mut("export JPEG encode/cpu/wall").unwrap() = 400.0; // wall ignored + *now.get_mut("loupe draft 1152×768 cold/gpu/cpu").unwrap() = 18.0; // +5 % + let r = regressions(&prev, &now, 0.2); + assert_eq!(r.len(), 1); + assert_eq!(r[0].0, "export JPEG encode/cpu/cpu"); + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index c44460cf..fb19fc0b 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -4,6 +4,7 @@ //! invoked through `std::process::Command`. mod assets; +mod bench; mod layers; mod stats; mod web; @@ -16,6 +17,9 @@ usage: cargo xtask commands: assets every image/icon/font/media file is attributed in assets/ATTRIBUTION.md; no Adobe assets + bench [FILE] [--strict] [--threshold PCT] + run the render benchmark, append to target/bench/history.jsonl, compare CPU time with + the previous run (default input: corpus/raw/arw-sony-a7m3-compressed.arw) layers enforce the crate dependency layering (plan/architecture.md §3) wasm cargo check --target wasm32-unknown-unknown for the wasm-safe crates (+ the web app) web [--serve [port]] [--dev] @@ -33,6 +37,7 @@ fn main() -> ExitCode { let result = match args.first().map(String::as_str) { Some("layers") => cmd_layers(), Some("assets") => assets::run(&root()), + Some("bench") => bench::run(&root(), &rest), Some("wasm") => cmd_wasm(), Some("web") => web::run(&rest), Some("ci") => cmd_ci(),