M5.7: cargo xtask bench — render benchmark history + CPU-time regression check
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Brandon Thomas
parent
b13274f711
commit
e0feb47ab8
|
|
@ -59,6 +59,8 @@ and often so a crash loses minutes, not hours.
|
|||
`apps/lightcraft-cli/tests/`). New features need at least one test that would fail without them.
|
||||
- **Run the app after every user-visible change**: launch with `--control 7980`, drive it with a JSON-lines script
|
||||
(`docs/showcase/run.py file.jsonl`), take `ui.screenshot`, and look at it. Check `ui.inspect` → `perf`.
|
||||
- **Benchmarks:** `cargo xtask bench` (24 MP raw from corpus; CPU and GPU columns) appends to `target/bench/history.jsonl`
|
||||
and flags CPU-time regressions > 20 % vs the previous run (`--strict` to fail). Run it before and after perf work.
|
||||
- **Measure, don't guess**: `LIGHTCRAFT_PROFILE=1` prints per-stage pipeline timings to stderr; time CLI renders
|
||||
with `/usr/bin/time`. Record numbers in `plan/STATUS.md` → Metrics. Budgets: slider update ≤ 16 ms (draft) / loupe
|
||||
≤ 60 ms on ~2.5 MP; export ≤ 1 s per 24 MP JPEG.
|
||||
|
|
|
|||
|
|
@ -42,6 +42,10 @@ and often so a crash loses minutes, not hours.
|
|||
## Running and looking at the app
|
||||
- `cargo run --release -p lightcraft -- --control 7980` opens the desktop app with the JSON-lines control server (see `docs/control-protocol.md`).
|
||||
- For UI work, **look at the result**: drive via the control channel and take `ui.screenshot`, compare with `plan/lightroom/screenshots/`.
|
||||
- **Unattended (display asleep/locked, CI, nightly runs): prefer headless snapshots** — no window needed:
|
||||
`lightcraft-cli snapshot --demo --script tour.jsonl -o out.png --size 1600x1000` (control-protocol requests,
|
||||
one per line; see `docs/control-protocol.md` → Headless rendering). In a running app use
|
||||
`ui.screenshot {"path": …, "headless": true}`; windowed screenshots fall back to headless after 2 s.
|
||||
- MCP: `lightcraft-cli mcp` (headless, `--demo` for the procedural library) or `lightcraft-cli mcp --connect` (drives
|
||||
the running app). See `docs/mcp.md`. Quick non-UI checks: `lightcraft-cli render in.jpg -o out.jpg --set light.exposure=1`.
|
||||
- Export goes through `lightcraft_engine::export` (one encoder for app, CLI, MCP and web); UI-only commands live in
|
||||
|
|
@ -55,6 +59,8 @@ and often so a crash loses minutes, not hours.
|
|||
`apps/lightcraft-cli/tests/`). New features need at least one test that would fail without them.
|
||||
- **Run the app after every user-visible change**: launch with `--control 7980`, drive it with a JSON-lines script
|
||||
(`docs/showcase/run.py file.jsonl`), take `ui.screenshot`, and look at it. Check `ui.inspect` → `perf`.
|
||||
- **Benchmarks:** `cargo xtask bench` (24 MP raw from corpus; CPU and GPU columns) appends to `target/bench/history.jsonl`
|
||||
and flags CPU-time regressions > 20 % vs the previous run (`--strict` to fail). Run it before and after perf work.
|
||||
- **Measure, don't guess**: `LIGHTCRAFT_PROFILE=1` prints per-stage pipeline timings to stderr; time CLI renders
|
||||
with `/usr/bin/time`. Record numbers in `plan/STATUS.md` → Metrics. Budgets: slider update ≤ 16 ms (draft) / loupe
|
||||
≤ 60 ms on ~2.5 MP; export ≤ 1 s per 24 MP JPEG.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,160 @@
|
|||
//! `cargo xtask bench [FILE] [--strict] [--threshold PCT]`: run the end-to-end render benchmark, record
|
||||
//! the results in `target/bench/history.jsonl` (one JSON object per run, with git revision and load
|
||||
//! average) and compare with the previous run.
|
||||
//!
|
||||
//! Comparisons use **process CPU time**, which reflects the work done even on a busy shared machine;
|
||||
//! wall-clock is recorded for reference. `--strict` exits non-zero when a CPU metric regressed by more
|
||||
//! than the threshold (default 20 %).
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
|
||||
/// Parse `render_bench` output lines such as
|
||||
/// `loupe draft 1152×768 cold: cpu: 379.5 ms wall, 139.5 ms cpu | gpu: 19.9 ms wall, 17.2 ms cpu`
|
||||
/// or `export JPEG encode: 63.9 ms wall, 278.0 ms cpu` into `name/backend/{wall,cpu}` → ms.
|
||||
pub fn parse(output: &str) -> BTreeMap<String, f64> {
|
||||
let mut out = BTreeMap::new();
|
||||
for line in output.lines() {
|
||||
let Some((name, rest)) = line.split_once(':') else { continue };
|
||||
if !rest.contains(" ms ") {
|
||||
continue;
|
||||
}
|
||||
let name = name.trim();
|
||||
for seg in rest.split('|') {
|
||||
let seg = seg.trim();
|
||||
let (backend, body) = match seg.split_once(':') {
|
||||
Some((b, body)) if matches!(b.trim(), "cpu" | "gpu") => (b.trim(), body),
|
||||
_ => ("cpu", seg),
|
||||
};
|
||||
for part in body.split(',') {
|
||||
let toks: Vec<&str> = part.split_whitespace().collect();
|
||||
if let [v, "ms", kind] = toks[..]
|
||||
&& let Ok(v) = v.parse::<f64>()
|
||||
&& matches!(kind, "wall" | "cpu")
|
||||
{
|
||||
out.insert(format!("{name}/{backend}/{kind}"), v);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// CPU-time metrics that got slower than `threshold` (fraction) vs `prev`: (key, prev, now).
|
||||
pub fn regressions(prev: &BTreeMap<String, f64>, now: &BTreeMap<String, f64>, threshold: f64) -> Vec<(String, f64, f64)> {
|
||||
now.iter()
|
||||
.filter(|(k, _)| k.ends_with("/cpu"))
|
||||
.filter_map(|(k, &v)| prev.get(k).map(|&p| (k.clone(), p, v)))
|
||||
.filter(|(_, p, v)| *p > 1.0 && *v > p * (1.0 + threshold))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn load_avg() -> Option<f64> {
|
||||
let out = Command::new("uptime").output().ok()?;
|
||||
let s = String::from_utf8_lossy(&out.stdout);
|
||||
let tail = s.rsplit_once("load average")?.1;
|
||||
tail.trim_start_matches(['s', ':', ' ']).split([',', ' ']).find(|t| !t.is_empty())?.parse().ok()
|
||||
}
|
||||
|
||||
fn git_rev(root: &Path) -> String {
|
||||
Command::new("git")
|
||||
.args(["rev-parse", "--short", "HEAD"])
|
||||
.current_dir(root)
|
||||
.output()
|
||||
.ok()
|
||||
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub fn run(root: &Path, args: &[&str]) -> Result<(), String> {
|
||||
let strict = args.contains(&"--strict");
|
||||
let threshold =
|
||||
args.iter().position(|a| *a == "--threshold").and_then(|i| args.get(i + 1)).and_then(|v| v.parse::<f64>().ok()).unwrap_or(20.0) / 100.0;
|
||||
let file = args.iter().find(|a| !a.starts_with("--") && a.parse::<f64>().is_err()).map(|s| s.to_string());
|
||||
let file = file.or_else(|| {
|
||||
let default = root.join("corpus/raw/arw-sony-a7m3-compressed.arw");
|
||||
default.exists().then(|| default.to_string_lossy().to_string())
|
||||
});
|
||||
let mut cmd = crate::cargo();
|
||||
cmd.args(["run", "--release", "-q", "-p", "lightcraft-engine", "--example", "render_bench", "--"]);
|
||||
if let Some(f) = &file {
|
||||
cmd.arg(f);
|
||||
}
|
||||
eprintln!("$ cargo run --release -p lightcraft-engine --example render_bench -- {}", file.as_deref().unwrap_or("(procedural 24 MP source)"));
|
||||
let out = cmd.output().map_err(|e| format!("render_bench: {e}"))?;
|
||||
let text = String::from_utf8_lossy(&out.stdout).to_string();
|
||||
print!("{text}");
|
||||
if !out.status.success() {
|
||||
return Err(format!("render_bench failed:\n{}", String::from_utf8_lossy(&out.stderr)));
|
||||
}
|
||||
let metrics = parse(&text);
|
||||
if metrics.is_empty() {
|
||||
return Err("no metrics parsed from render_bench output".into());
|
||||
}
|
||||
let dir = root.join("target/bench");
|
||||
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
|
||||
let hist = dir.join("history.jsonl");
|
||||
let prev: Option<BTreeMap<String, f64>> = std::fs::read_to_string(&hist)
|
||||
.ok()
|
||||
.and_then(|s| s.lines().rev().find(|l| !l.trim().is_empty()).map(str::to_string))
|
||||
.and_then(|l| serde_json::from_str::<serde_json::Value>(&l).ok())
|
||||
.and_then(|v| serde_json::from_value(v["metrics"].clone()).ok());
|
||||
let rec = serde_json::json!({
|
||||
"rev": git_rev(root),
|
||||
"time": std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0),
|
||||
"load": load_avg(),
|
||||
"input": file,
|
||||
"metrics": metrics,
|
||||
});
|
||||
let mut line = rec.to_string();
|
||||
line.push('\n');
|
||||
use std::io::Write;
|
||||
std::fs::OpenOptions::new().create(true).append(true).open(&hist).and_then(|mut f| f.write_all(line.as_bytes())).map_err(|e| e.to_string())?;
|
||||
eprintln!("bench: {} metrics recorded in {}", metrics.len(), hist.display());
|
||||
let Some(prev) = prev else {
|
||||
eprintln!("bench: first run — nothing to compare");
|
||||
return Ok(());
|
||||
};
|
||||
let reg = regressions(&prev, &metrics, threshold);
|
||||
if reg.is_empty() {
|
||||
eprintln!("bench: no CPU-time regressions > {:.0}% vs previous run", threshold * 100.0);
|
||||
return Ok(());
|
||||
}
|
||||
for (k, p, v) in ® {
|
||||
eprintln!("bench: REGRESSION {k}: {p:.1} → {v:.1} ms (+{:.0}%)", (v / p - 1.0) * 100.0);
|
||||
}
|
||||
if strict { Err(format!("{} CPU-time regression(s)", reg.len())) } else { Ok(()) }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const OUT: &str = "gpu: Apple M4 Pro (Metal) (device + kernels: 322 ms)
|
||||
gpu vs cpu, loupe 1920×1280 typical: max 1 LSB, mean 0.0001 LSB
|
||||
loupe draft 1152×768 cold: cpu: 379.5 ms wall, 139.5 ms cpu | gpu: 19.9 ms wall, 17.2 ms cpu
|
||||
export JPEG encode: 63.9 ms wall, 278.0 ms cpu
|
||||
";
|
||||
|
||||
#[test]
|
||||
fn parses_cpu_gpu_and_plain_lines() {
|
||||
let m = parse(OUT);
|
||||
assert_eq!(m["loupe draft 1152×768 cold/cpu/wall"], 379.5);
|
||||
assert_eq!(m["loupe draft 1152×768 cold/gpu/cpu"], 17.2);
|
||||
assert_eq!(m["export JPEG encode/cpu/cpu"], 278.0);
|
||||
assert_eq!(m.len(), 6, "{m:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flags_only_cpu_time_regressions_over_threshold() {
|
||||
let prev = parse(OUT);
|
||||
let mut now = prev.clone();
|
||||
*now.get_mut("export JPEG encode/cpu/cpu").unwrap() = 400.0; // +44 %
|
||||
*now.get_mut("export JPEG encode/cpu/wall").unwrap() = 400.0; // wall ignored
|
||||
*now.get_mut("loupe draft 1152×768 cold/gpu/cpu").unwrap() = 18.0; // +5 %
|
||||
let r = regressions(&prev, &now, 0.2);
|
||||
assert_eq!(r.len(), 1);
|
||||
assert_eq!(r[0].0, "export JPEG encode/cpu/cpu");
|
||||
}
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@
|
|||
//! invoked through `std::process::Command`.
|
||||
|
||||
mod assets;
|
||||
mod bench;
|
||||
mod layers;
|
||||
mod stats;
|
||||
mod web;
|
||||
|
|
@ -16,6 +17,9 @@ usage: cargo xtask <command>
|
|||
|
||||
commands:
|
||||
assets every image/icon/font/media file is attributed in assets/ATTRIBUTION.md; no Adobe assets
|
||||
bench [FILE] [--strict] [--threshold PCT]
|
||||
run the render benchmark, append to target/bench/history.jsonl, compare CPU time with
|
||||
the previous run (default input: corpus/raw/arw-sony-a7m3-compressed.arw)
|
||||
layers enforce the crate dependency layering (plan/architecture.md §3)
|
||||
wasm cargo check --target wasm32-unknown-unknown for the wasm-safe crates (+ the web app)
|
||||
web [--serve [port]] [--dev]
|
||||
|
|
@ -33,6 +37,7 @@ fn main() -> ExitCode {
|
|||
let result = match args.first().map(String::as_str) {
|
||||
Some("layers") => cmd_layers(),
|
||||
Some("assets") => assets::run(&root()),
|
||||
Some("bench") => bench::run(&root(), &rest),
|
||||
Some("wasm") => cmd_wasm(),
|
||||
Some("web") => web::run(&rest),
|
||||
Some("ci") => cmd_ci(),
|
||||
|
|
|
|||
Loading…
Reference in New Issue