From 0f955e6c82d0b57b73d97dd8768c07b86b2df860 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 5 Oct 2026 10:53:02 -0400 Subject: [PATCH] Scorecard: cargo xtask perf + scorecard, budgets, checklists, nightly perf (#221) (#239) (#239) --- .github/workflows/ci.yml | 6 + .github/workflows/perf-nightly.yml | 91 ++ AGENTS.md | 17 +- Cargo.lock | 15 + crates/testkit/Cargo.toml | 5 + crates/testkit/src/lib.rs | 4 + crates/testkit/src/perf.rs | 194 +++ crates/ui-egui/Cargo.toml | 2 + crates/ui-egui/examples/fx_bench.rs | 24 +- crates/ui-egui/examples/interactive_bench.rs | 29 +- crates/ui-egui/examples/large_image_bench.rs | 30 +- crates/ui-egui/examples/perf_scenarios.rs | 638 ++++++++++ crates/ui-egui/examples/type_bench.rs | 14 + docs/development.md | 54 + docs/releasing.md | 8 +- docs/roadmap.md | 9 + docs/scorecard.md | 279 +++++ perf/baseline.json | 299 +++++ perf/budgets.toml | 411 +++++++ scorecard/automation.toml | 82 ++ scorecard/bugs.toml | 230 ++++ scorecard/distribution.toml | 70 ++ scorecard/file_compat.toml | 195 ++++ scorecard/reliability.toml | 51 + scorecard/tools.toml | 216 ++++ scorecard/type.toml | 68 ++ scorecard/ui.toml | 167 +++ xtask/Cargo.toml | 3 + xtask/src/main.rs | 9 + xtask/src/perf.rs | 1104 ++++++++++++++++++ xtask/src/scorecard.rs | 821 +++++++++++++ 31 files changed, 5134 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/perf-nightly.yml create mode 100644 crates/testkit/src/perf.rs create mode 100644 crates/ui-egui/examples/perf_scenarios.rs create mode 100644 docs/scorecard.md create mode 100644 perf/baseline.json create mode 100644 perf/budgets.toml create mode 100644 scorecard/automation.toml create mode 100644 scorecard/bugs.toml create mode 100644 scorecard/distribution.toml create mode 100644 scorecard/file_compat.toml create mode 100644 scorecard/reliability.toml create mode 100644 scorecard/tools.toml create mode 100644 scorecard/type.toml create mode 100644 scorecard/ui.toml create mode 100644 xtask/src/perf.rs create mode 100644 xtask/src/scorecard.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10171a6e..be4e9b4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,6 +60,12 @@ jobs: - name: Dependency layering run: cargo xtask layers + # docs/scorecard.md must match its sources (checklists, budgets, baseline, corpus floors, + # prefs audit). Static: it reads committed files only, no benchmarks run here. + - name: Scorecard up to date + if: runner.os == 'Linux' + run: cargo xtask scorecard --check + wasm: name: wasm32 check runs-on: ubuntu-latest diff --git a/.github/workflows/perf-nightly.yml b/.github/workflows/perf-nightly.yml new file mode 100644 index 00000000..10d22292 --- /dev/null +++ b/.github/workflows/perf-nightly.yml @@ -0,0 +1,91 @@ +name: Perf (nightly) + +# The performance scorecard (#221): `cargo xtask perf` on a fixed macOS runner (Apple silicon, +# Metal), every night and on demand. It never runs on pull requests, so it can't block one. +# It fails when an enforced budget in perf/budgets.toml breaks or a scenario's p50 regresses +# more than 15 % against perf/baseline.json from the same machine class (other classes are +# never compared). The table goes to the job summary; results.json is uploaded as an artifact. +# +# To give this runner class its own baseline: run it by hand with "update_baseline", download +# the `perf-baseline` artifact and commit it as perf/baseline.json in a PR. + +on: + schedule: + - cron: "23 3 * * *" + workflow_dispatch: + inputs: + quick: + description: "Quick subset (small documents, no budgets)" + type: boolean + default: false + update_baseline: + description: "Also write perf/baseline.json from this run (uploaded as an artifact, not committed)" + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: perf-nightly + cancel-in-progress: false + +env: + CARGO_TERM_COLOR: always + +jobs: + perf: + name: perf (macOS, Apple GPU) + # Pinned image, not -latest: the machine class must stay fixed for comparisons. + runs-on: macos-15 + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + with: + key: perf-release + + - name: Run benchmarks + id: perf + env: + QUICK: ${{ inputs.quick }} + UPDATE: ${{ inputs.update_baseline }} + run: | + args=() + if [ "$QUICK" = "true" ]; then args+=(--quick); fi + if [ "$UPDATE" = "true" ]; then args+=(--update-baseline); fi + cargo xtask perf "${args[@]}" + + - name: Job summary + if: always() + run: | + if [ -f target/perf/summary.md ]; then + cat target/perf/summary.md >> "$GITHUB_STEP_SUMMARY" + else + echo "No perf summary: the run failed before writing one (see the log)." >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: perf-results + path: | + target/perf/*.json + target/perf/summary.md + if-no-files-found: warn + retention-days: 30 + + - name: Upload baseline + if: always() && inputs.update_baseline + uses: actions/upload-artifact@v4 + with: + name: perf-baseline + path: perf/baseline.json + if-no-files-found: warn + retention-days: 30 diff --git a/AGENTS.md b/AGENTS.md index dc326207..d4cb7420 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,7 @@ PhotoCraft is an open-source, native, Photoshop-comparable image editor written | `docs/ui-design.md` | Design tokens, themes, widgets, and how to match Photoshop's look | | `docs/roadmap.md` | Honest parity assessment (where we're lacking, where we're going), milestones, **current focus** | | `docs/parity.md` | Generated list of every Photoshop menu item, live or missing | +| `docs/scorecard.md` | Generated scorecard: performance budgets and numbers, corpus floors, per-area checklists (tools, files, UI, type, automation, reliability, distribution), settings that do nothing | | `crates//README.md` (where present) | Public API of that crate | ## 2. Workspace map @@ -32,7 +33,7 @@ apps/ photocraft desktop app (eframe/wgpu), TCP control server photocraft-cli headless CLI (convert/info/run/batch/commands/mcp) photocraft-web the same app in the browser (trunk + wasm-bindgen) -xtask/ cargo xtask layers | wasm | ci | stats | corpus | parity +xtask/ cargo xtask layers | wasm | ci | stats | corpus | parity | perf | scorecard ``` **Layering is enforced** by `cargo xtask layers`. A crate may depend only on lower layers. `psd`, `codecs` and `cms` depend on nothing in the workspace. Nothing below `ui-egui` may use egui, eframe, winit or rfd. A new crate must be registered in `xtask/src/layers.rs`. @@ -71,9 +72,12 @@ Priorities: important infrastructure first, then low-hanging parity, then the lo where PhotoCraft is lacking and the priority order of where we're going. `docs/parity.md` (menu wiring) is not a measure of behaviour. When your work moves a measured number (PSD oracle, round trips, workflow tests, performance), update that section with the dated figure. -1. `docs/roadmap.md` → **Current focus**. -2. `cargo xtask parity` → `docs/parity.md` lists every missing menu item, grouped by menu. Low-hanging fruit is usually a missing command whose algorithm already exists in `algo`, `paint`, `vector` or `text`. -3. `log/devlog.md` → the "Still open" bullets of recent entries. +1. **Check `docs/scorecard.md`** before picking work: each area's `missing` and `partial` rows, + the performance scenarios that are over budget or not measurable yet, and the count of + settings that do nothing. Its numbers are measured; prefer them to estimates. +2. `docs/roadmap.md` → **Current focus**. +3. `cargo xtask parity` → `docs/parity.md` lists every missing menu item, grouped by menu. Low-hanging fruit is usually a missing command whose algorithm already exists in `algo`, `paint`, `vector` or `text`. +4. `log/devlog.md` → the "Still open" bullets of recent entries. When parity rises, raise `FLOOR` in `crates/ui-egui/src/parity.rs` (never lower it). @@ -86,6 +90,10 @@ cargo xtask layers cargo xtask wasm # if you touched L0–L6 cargo xtask parity # if you added commands; commit the regenerated docs/parity.md cargo test -p photocraft-engine --test panic_hunt -- --ignored # if you added/changed commands: no panic on adversarial input (Rule 9) +cargo xtask scorecard # if you moved a number: flip the checklist row in scorecard/*.toml, raise a + # corpus floor, fix a dead preference, or meet a budget (then set enforce = true + # in perf/budgets.toml); commit the regenerated docs/scorecard.md (CI checks it) +cargo xtask perf --quick # if you touched a hot path; `cargo xtask perf --update-baseline` publishes a full run ``` Commands must **never panic** on bad input (Rule 9): every `run` closure and the code it calls returns `Err`, not a panic, for any params or document state. New commands come with a graceful-failure test (empty/out-of-range/wrong-type params → `Err`, not a crash). @@ -105,6 +113,7 @@ Then append a terse entry to `log/devlog.md` (what landed, numbers, what's still - `docs/roadmap.md`: milestones M0–M12, status and the current focus. - `docs/parity.md`: generated Photoshop menu coverage. +- `docs/scorecard.md`: generated scorecard (sources: `scorecard/*.toml`, `perf/budgets.toml`, `perf/baseline.json`, corpus floors, prefs audit). - `docs/releasing.md`: cutting a release (`cargo xtask version`, the `release` branch), signing secrets, packaging scripts in `packaging/`. - `../craftrules/release/playbook.md`: how every storytold app builds signed release binaries (the canonical recipe; `docs/release-playbook.md` just points there); `docs/releasing.md` is PhotoCraft's specifics. - `plan/` (local, gitignored): research, parity plan, execution plan, estimates. diff --git a/Cargo.lock b/Cargo.lock index 2dd6dd91..2e4071ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2621,6 +2621,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "memory-stats" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c73f5c649995a115e1a0220b35e4df0a1294500477f97a91d0660fb5abeb574a" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -3667,9 +3677,11 @@ dependencies = [ name = "photocraft-testkit" version = "0.2.0" dependencies = [ + "memory-stats", "photocraft-color", "photocraft-geom", "photocraft-raster", + "serde_json", ] [[package]] @@ -3708,6 +3720,7 @@ dependencies = [ "photocraft-io", "photocraft-plugins", "photocraft-raster", + "photocraft-testkit", "photocraft-text", "photocraft-vector", "rayon", @@ -6258,7 +6271,9 @@ checksum = "ec7a2a501ed189703dba8b08142f057e887dfc4b2cc4db2d343ac6376ba3e0b9" name = "xtask" version = "0.2.0" dependencies = [ + "serde", "serde_json", + "toml", ] [[package]] diff --git a/crates/testkit/Cargo.toml b/crates/testkit/Cargo.toml index cf54ef16..0d69300d 100644 --- a/crates/testkit/Cargo.toml +++ b/crates/testkit/Cargo.toml @@ -11,3 +11,8 @@ workspace = true photocraft-geom = { workspace = true } photocraft-color = { workspace = true } photocraft-raster = { workspace = true } +serde_json = { workspace = true } + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +# In-process resident memory for the benchmarks (`perf`; MIT OR Apache-2.0). +memory-stats = "1.2" diff --git a/crates/testkit/src/lib.rs b/crates/testkit/src/lib.rs index 5851b6ff..36cd4f6b 100644 --- a/crates/testkit/src/lib.rs +++ b/crates/testkit/src/lib.rs @@ -1 +1,5 @@ //! photocraft-testkit +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +#[cfg(not(target_arch = "wasm32"))] +pub mod perf; diff --git a/crates/testkit/src/perf.rs b/crates/testkit/src/perf.rs new file mode 100644 index 00000000..6bdf68ab --- /dev/null +++ b/crates/testkit/src/perf.rs @@ -0,0 +1,194 @@ +//! Benchmark helpers shared by the `--json` benches that `cargo xtask perf` runs (#221). +//! +//! - [`RssSampler`]: peak resident memory measured in-process. A background thread polls the +//! process's resident set size every few milliseconds; [`RssSampler::reset`] starts a new +//! window, so each scenario gets its own peak. Polling can miss a spike shorter than the +//! interval; on Linux [`process_peak_rss_bytes`] also reads the kernel's exact high-water mark. +//! - [`Summary`]: p50 / p95 / max of a scenario's samples (nearest-rank percentiles). +//! - [`row`] / [`report`]: the JSON shape `xtask perf` reads. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::thread::JoinHandle; +use std::time::Duration; + +use serde_json::{Value, json}; + +/// Current resident set size of this process, in bytes. +pub fn current_rss_bytes() -> Option { + memory_stats::memory_stats().map(|m| m.physical_mem as u64) +} + +/// The process's peak resident set size so far: the kernel's high-water mark on Linux +/// (`VmHWM`), `None` elsewhere (use an [`RssSampler`]). +pub fn process_peak_rss_bytes() -> Option { + let status = std::fs::read_to_string("/proc/self/status").ok()?; + parse_vm_hwm(&status) +} + +fn parse_vm_hwm(status: &str) -> Option { + let line = status.lines().find(|l| l.starts_with("VmHWM:"))?; + let kb: u64 = line.split_whitespace().nth(1)?.parse().ok()?; + kb.checked_mul(1024) +} + +/// Polls the resident set size on a background thread and keeps the maximum. +pub struct RssSampler { + peak: Arc, + stop: Arc, + thread: Option>, +} + +impl RssSampler { + /// Starts sampling every `interval` (2–5 ms is cheap: one syscall per sample). + pub fn start(interval: Duration) -> Self { + let peak = Arc::new(AtomicU64::new(current_rss_bytes().unwrap_or(0))); + let stop = Arc::new(AtomicBool::new(false)); + let (p, s) = (Arc::clone(&peak), Arc::clone(&stop)); + let thread = std::thread::Builder::new() + .name("rss-sampler".into()) + .spawn(move || { + while !s.load(Ordering::Relaxed) { + if let Some(v) = current_rss_bytes() { + p.fetch_max(v, Ordering::Relaxed); + } + std::thread::sleep(interval); + } + }) + .ok(); + Self { peak, stop, thread } + } + + /// Starts a new window: the peak becomes the current resident size. + pub fn reset(&self) { + self.peak.store(current_rss_bytes().unwrap_or(0), Ordering::Relaxed); + } + + /// The peak resident size since the last [`reset`](Self::reset) (or start), including now. + pub fn peak(&self) -> Option { + if let Some(v) = current_rss_bytes() { + self.peak.fetch_max(v, Ordering::Relaxed); + } + Some(self.peak.load(Ordering::Relaxed)).filter(|v| *v > 0) + } +} + +impl Drop for RssSampler { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + if let Some(t) = self.thread.take() { + // A sampler thread that panicked has nothing to report; don't cascade. + let _ = t.join(); + } + } +} + +/// p50 / p95 / max of a set of timings (ms). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Summary { + pub n: usize, + pub p50: f64, + pub p95: f64, + pub max: f64, +} + +/// Nearest-rank percentile `q` (0..=1) of sorted, non-empty `v`. +fn rank(v: &[f64], q: f64) -> Option { + let n = v.len(); + let k = ((q * n as f64).ceil() as usize).clamp(1, n.max(1)); + v.get(k - 1).copied() +} + +impl Summary { + /// `None` when there are no finite samples. + pub fn of(samples: &[f64]) -> Option { + let mut v: Vec = samples.iter().copied().filter(|x| x.is_finite()).collect(); + v.sort_by(f64::total_cmp); + Some(Self { n: v.len(), p50: rank(&v, 0.5)?, p95: rank(&v, 0.95)?, max: *v.last()? }) + } +} + +/// One scenario row: `name`, sample count, p50/p95/max, the raw samples, peak RSS and GPU bytes. +pub fn row(name: &str, samples: &[f64], peak_rss: Option, gpu_bytes: Option) -> Value { + let s = Summary::of(samples); + json!({ + "name": name, + "n": s.map_or(0, |s| s.n), + "p50_ms": s.map(|s| s.p50), + "p95_ms": s.map(|s| s.p95), + "max_ms": s.map(|s| s.max), + "samples_ms": samples, + "peak_rss_bytes": peak_rss, + "gpu_bytes": gpu_bytes, + }) +} + +/// A bench's whole report: its name, free-form context (document size, adapter…), the rows and +/// the process's peak RSS (the kernel's high-water mark where there is one, else the highest +/// sampled peak). +pub fn report(bench: &str, context: Value, rows: Vec, sampler: Option<&RssSampler>) -> Value { + let sampled = rows.iter().filter_map(|r| r["peak_rss_bytes"].as_u64()).chain(sampler.and_then(RssSampler::peak)).max(); + let process_peak = process_peak_rss_bytes().or(sampled); + json!({ + "bench": bench, + "schema": 1, + "context": context, + "rows": rows, + "process_peak_rss_bytes": process_peak, + }) +} + +/// Writes `report` to `path` (pretty JSON). Errors are returned, not panicked. +pub fn write_report(path: &str, report: &Value) -> Result<(), String> { + let text = serde_json::to_string_pretty(report).map_err(|e| format!("encode {path}: {e}"))?; + std::fs::write(path, text).map_err(|e| format!("write {path}: {e}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn summary_nearest_rank() { + let v: Vec = (1..=20).map(f64::from).collect(); + let s = Summary::of(&v).unwrap(); + assert_eq!((s.n, s.p50, s.p95, s.max), (20, 10.0, 19.0, 20.0)); + let s = Summary::of(&[3.0]).unwrap(); + assert_eq!((s.p50, s.p95, s.max), (3.0, 3.0, 3.0)); + assert!(Summary::of(&[]).is_none()); + assert!(Summary::of(&[f64::NAN]).is_none()); + } + + #[test] + fn vm_hwm_parses() { + assert_eq!(parse_vm_hwm("Name:\tx\nVmHWM:\t 2048 kB\nVmRSS: 1 kB\n"), Some(2048 * 1024)); + assert_eq!(parse_vm_hwm("VmHWM: lots\n"), None); + assert_eq!(parse_vm_hwm(""), None); + } + + #[test] + fn sampler_reports_a_peak() { + let s = RssSampler::start(Duration::from_millis(1)); + s.reset(); + let big = vec![1u8; 32 << 20]; + std::hint::black_box(&big); + let p = s.peak(); + drop(big); + // Platforms without a memory API report nothing rather than a made-up number. + if current_rss_bytes().is_some() { + assert!(p.unwrap() > 0); + } + } + + #[test] + fn row_shape() { + let r = row("x", &[2.0, 1.0, 3.0], Some(10), None); + assert_eq!(r["p50_ms"], 2.0); + assert_eq!(r["max_ms"], 3.0); + assert_eq!(r["n"], 3); + assert!(r["gpu_bytes"].is_null()); + let empty = row("y", &[], None, None); + assert!(empty["p95_ms"].is_null()); + } +} diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml index a4b8cba5..d3a01737 100644 --- a/crates/ui-egui/Cargo.toml +++ b/crates/ui-egui/Cargo.toml @@ -37,3 +37,5 @@ photocraft-io = { workspace = true } # Only for examples/interactive_bench.rs (JPEG encode, .pcraft saves). photocraft-codecs = { workspace = true } photocraft-format = { workspace = true } +# Benchmark helpers for the `--json` benches `cargo xtask perf` runs (in-process RSS, percentiles). +photocraft-testkit = { workspace = true } diff --git a/crates/ui-egui/examples/fx_bench.rs b/crates/ui-egui/examples/fx_bench.rs index 91c00db2..aa2bf61b 100644 --- a/crates/ui-egui/examples/fx_bench.rs +++ b/crates/ui-egui/examples/fx_bench.rs @@ -130,6 +130,7 @@ struct Gpu { device: wgpu::Device, queue: wgpu::Queue, comp: photocraft_gpu::Compositor, + adapter: String, } fn gpu() -> Option { @@ -142,7 +143,7 @@ fn gpu() -> Option { let limits = photocraft_ui_egui::gpu_canvas::device_limits(&adapter); let (device, queue) = block_on(adapter.request_device(&wgpu::DeviceDescriptor { required_limits: limits, ..Default::default() })).ok()?; let comp = photocraft_gpu::Compositor::new(&device); - Some(Gpu { device, queue, comp }) + Some(Gpu { device, queue, comp, adapter: adapter.get_info().name }) } fn gpu_time(g: &mut Gpu, doc: &Document, region: Rect) -> Result { @@ -227,6 +228,7 @@ fn main() { let full = doc.bounds(); println!("document {}x{} ({} layers)", doc.size.width, doc.size.height, doc.walk().len()); let mut g = gpu(); + let adapter = g.as_ref().map(|g| g.adapter.clone()); let mut gt = |doc: &Document, r: Rect| match g.as_mut() { Some(g) => gpu_time(g, doc, r), None => Err("no adapter".into()), @@ -246,8 +248,13 @@ fn main() { }; println!("{what:<42} {mp:6.2} MP cpu {c:>22} gpu {g} (min / median)"); }; + // `--json out.json` (`cargo xtask perf`): one row per case and compositor, with samples and + // the peak RSS while the case ran. + let rss = photocraft_testkit::perf::RssSampler::start(std::time::Duration::from_millis(2)); + let mut json_rows: Vec = Vec::new(); // Each case: `edit` mutates the document, then both compositors refresh `region`. let mut case = |doc: &mut Document, what: &str, n: usize, edit: &mut dyn FnMut(&mut Document, usize) -> Rect, cpu_too: bool| { + rss.reset(); let (mut cs, mut gs, mut gerr) = (Vec::new(), Vec::new(), None); let mut region = full; for i in 0..n { @@ -260,6 +267,13 @@ fn main() { Err(e) => gerr = Some(e), } } + let peak = rss.peak(); + if !gs.is_empty() { + json_rows.push(photocraft_testkit::perf::row(&format!("{what} (GPU)"), &gs, peak, None)); + } + if !cs.is_empty() { + json_rows.push(photocraft_testkit::perf::row(&format!("{what} (CPU)"), &cs, peak, None)); + } show( what, region, @@ -350,4 +364,12 @@ fn main() { }, true, ); + if let Some(out) = arg(&args, "--json") { + let context = json!({"width": w, "height": h, "gpu_adapter": adapter}); + let report = photocraft_testkit::perf::report("fx_bench", context, json_rows, Some(&rss)); + if let Err(e) = photocraft_testkit::perf::write_report(&out, &report) { + eprintln!("{e}"); + std::process::exit(1); + } + } } diff --git a/crates/ui-egui/examples/interactive_bench.rs b/crates/ui-egui/examples/interactive_bench.rs index 8681c90a..b8a15fd8 100644 --- a/crates/ui-egui/examples/interactive_bench.rs +++ b/crates/ui-egui/examples/interactive_bench.rs @@ -83,6 +83,10 @@ struct Bench { /// state later rows need). only: Option, rows: Vec<(String, f64, f64, f64, f64)>, + /// Per row (same order as `rows`): the timed samples and the peak RSS while they ran, for + /// `--json` (`cargo xtask perf`). + extra: Vec<(Vec, Option)>, + rss: photocraft_testkit::perf::RssSampler, } impl Bench { @@ -116,8 +120,10 @@ impl Bench { f(self); return; } + self.rss.reset(); let cold = f(self); let mut v: Vec = (0..self.reps).map(|_| f(self)).collect(); + self.extra.push((v.clone(), self.rss.peak())); v.sort_by(f64::total_cmp); let med = v[v.len() / 2]; println!("{name:<44} {med:>9.1} ms (min {:>8.1}, max {:>8.1}, cold {:>8.1})", v[0], v[v.len() - 1], cold); @@ -161,7 +167,14 @@ fn main() { let (w, h) = arg(&args, "--size").and_then(|s| s.split_once('x').map(|(a, b)| (a.parse().unwrap_or(7360), b.parse().unwrap_or(4912)))).unwrap_or((7360, 4912)); let reps: usize = arg(&args, "--reps").and_then(|v| v.parse().ok()).unwrap_or(5).max(1); - let mut b = Bench { gpu: if args.iter().any(|a| a == "--cpu") { None } else { gpu() }, reps, only: arg(&args, "--only"), rows: Vec::new() }; + let mut b = Bench { + gpu: if args.iter().any(|a| a == "--cpu") { None } else { gpu() }, + reps, + only: arg(&args, "--only"), + rows: Vec::new(), + extra: Vec::new(), + rss: photocraft_testkit::perf::RssSampler::start(std::time::Duration::from_millis(2)), + }; println!("document {w}×{h} ({:.1} MP), 8-bit RGB, {} reps, {}", (w * h) as f64 / 1e6, reps, if b.gpu.is_some() { "GPU canvas" } else { "CPU canvas" }); // ---- open / save / export ---------------------------------------------------------------- @@ -399,8 +412,18 @@ fn main() { }); if let Some(out) = arg(&args, "--json") { - let rows: Vec = - b.rows.iter().map(|(n, med, min, max, cold)| json!({"name": n, "median_ms": med, "min_ms": min, "max_ms": max, "cold_ms": cold})).collect(); + let rows: Vec = b + .rows + .iter() + .zip(&b.extra) + .map(|((n, med, min, max, cold), (samples, rss))| { + let mut r = json!({"name": n, "median_ms": med, "min_ms": min, "max_ms": max, "cold_ms": cold}); + // For `cargo xtask perf`: raw samples (it computes p50 / p95) and peak RSS. + r["samples_ms"] = json!(samples); + r["peak_rss_bytes"] = json!(rss); + r + }) + .collect(); std::fs::write(&out, serde_json::to_string_pretty(&json!({"width": w, "height": h, "gpu": b.gpu.is_some(), "rows": rows})).unwrap()) .expect("write json"); } diff --git a/crates/ui-egui/examples/large_image_bench.rs b/crates/ui-egui/examples/large_image_bench.rs index 3b08640f..5ad50da1 100644 --- a/crates/ui-egui/examples/large_image_bench.rs +++ b/crates/ui-egui/examples/large_image_bench.rs @@ -99,7 +99,10 @@ fn main() { let jpeg = photocraft_codecs::encode(&img, photocraft_codecs::Format::Jpeg, &Default::default()).expect("jpeg"); std::fs::write(&src, jpeg).expect("write cache"); eprintln!("generated {} in {:.0} ms (run again to measure)", src.display(), ms(t)); - return; + // `--json` (`cargo xtask perf`) measures in the same run. + if arg(&args, "--json").is_none() { + return; + } } let gpu = if args.iter().any(|a| a == "--cpu") { None } else { canvas() }; let gpu = gpu.as_ref(); @@ -119,23 +122,37 @@ fn main() { let t_open = ms(t); let t = Instant::now(); let path = refresh(gpu, &s, true); + let t_first = ms(t); println!("open (decode + document) {t_open:>9.0} ms"); - println!("first refresh ({path:<8}) {:>9.0} ms", ms(t)); + println!("first refresh ({path:<8}) {t_first:>9.0} ms"); + // `--json out.json` (`cargo xtask perf`): open and first refresh (every op includes them), + // the canvas texture size and the process's peak RSS. + let json_out = arg(&args, "--json"); + let mut json_rows = vec![ + photocraft_testkit::perf::row("open (decode + document)", &[t_open], None, None), + photocraft_testkit::perf::row("first refresh", &[t_first], None, None), + ]; if let Some((g, _)) = gpu && let Some((format, bytes)) = g.texture_info(s.active().expect("doc").doc.id.0) { println!("canvas texture {format:?}, {} MB with mips", bytes >> 20); + if let Some(r) = json_rows.get_mut(1) { + r["gpu_bytes"] = json!(bytes); + } } let doc = || s.active().expect("doc").doc.clone(); match op.as_str() { "open" => {} "refresh" => { + let mut v = Vec::new(); for _ in 0..3 { let t = Instant::now(); let path = refresh(gpu, &s, true); + v.push(ms(t)); println!("full refresh ({path:<8}) {:>9.0} ms", ms(t)); } + json_rows.push(photocraft_testkit::perf::row("full refresh", &v, None, None)); } "thumbs" => { let d = doc(); @@ -181,4 +198,13 @@ fn main() { } other => eprintln!("unknown --op {other}"), } + if let Some(out) = json_out { + let peak = photocraft_testkit::perf::process_peak_rss_bytes().or_else(photocraft_testkit::perf::current_rss_bytes); + let context = json!({"width": w, "height": h, "op": op, "gpu_adapter": gpu.map(|(_, rs)| rs.adapter.get_info().name)}); + let mut report = photocraft_testkit::perf::report("large_image_bench", context, json_rows, None); + report["process_peak_rss_bytes"] = json!(peak); + if let Err(e) = photocraft_testkit::perf::write_report(&out, &report) { + eprintln!("{e}"); + } + } } diff --git a/crates/ui-egui/examples/perf_scenarios.rs b/crates/ui-egui/examples/perf_scenarios.rs new file mode 100644 index 00000000..2c5b16d4 --- /dev/null +++ b/crates/ui-egui/examples/perf_scenarios.rs @@ -0,0 +1,638 @@ +//! Scorecard performance scenarios (#221): the measurable rows of the targets in #209, #210 and +//! #211, each timed through the command path the UI, CLI and MCP share (`Session::execute`) plus +//! the canvas refresh the app does after it (the app's own `GpuCanvas`, on a device created the +//! way the app creates it; the CPU compositor without an adapter). +//! +//! ```sh +//! cargo run --release -p photocraft-ui-egui --example perf_scenarios -- [--quick] [--json out.json] [--only text] [--reps N] [--cpu] +//! ``` +//! +//! `cargo xtask perf` runs this and maps the rows to scenario ids in `perf/budgets.toml`, so +//! keep row names stable. `--quick` uses small synthetic documents (seconds, for smoke runs); +//! its numbers are only comparable with other `--quick` runs. Every row records p50 / p95 / max +//! over its samples, the peak resident memory measured in-process while it ran, and the GPU +//! bytes the canvas holds for the document (texture + compositor) when there is a GPU. + +use std::time::Instant; + +use eframe::egui_wgpu::RenderState; +use photocraft_engine::Session; +use photocraft_testkit::perf::{RssSampler, report, row, write_report}; +use photocraft_ui_egui::gpu_canvas::GpuCanvas; +use serde_json::{Value, json}; + +type Res = Result; + +fn arg(args: &[String], name: &str) -> Option { + args.iter().position(|a| a == name).and_then(|i| args.get(i + 1).cloned()) +} + +fn ms(t: Instant) -> f64 { + t.elapsed().as_secs_f64() * 1000.0 +} + +fn exec(s: &mut Session, id: &str, p: Value) -> Res { + s.execute(id, p).map_err(|e| format!("{id}: {e}")) +} + +fn active_layer(s: &Session) -> Res { + s.active().and_then(|d| d.active_layer).map(|l| l.0).ok_or_else(|| "no active layer".to_string()) +} + +/// A photo-like RGB image (gradients, soft blobs, fine noise) with a high-contrast disc as the +/// subject, encoded as JPEG so it opens like a real photo. +fn photo_jpeg(w: u32, h: u32) -> Res> { + use rayon::prelude::*; + let mut px = vec![0u8; w as usize * h as usize * 3]; + let (cx, cy, r) = (w as f32 * 0.55, h as f32 * 0.5, w.min(h) as f32 / 3.0); + px.par_chunks_mut(w as usize * 3).enumerate().for_each(|(y, row)| { + for x in 0..w as usize { + let (fx, fy) = (x as f32 / w as f32, y as f32 / h as f32); + let n = ((x.wrapping_mul(73_856_093) ^ y.wrapping_mul(19_349_663)) % 997) as f32 / 997.0 - 0.5; + let blob = (-(((fx - 0.4) * 3.0).powi(2) + ((fy - 0.55) * 4.0).powi(2))).exp(); + let inside = ((x as f32 - cx).powi(2) + (y as f32 - cy).powi(2)).sqrt() < r; + let (r0, g0, b0) = if inside { (0.85, 0.3 + 0.2 * (fx * 40.0).sin(), 0.15) } else { (0.35, 0.45, 0.55) }; + let rgb = [r0 + 0.1 * blob + n * 0.08, g0 + 0.1 * blob + n * 0.08, b0 + 0.2 * fy + n * 0.08]; + for (c, v) in rgb.into_iter().enumerate() { + row[x * 3 + c] = (v.clamp(0.0, 1.0) * 255.0 + 0.5) as u8; + } + } + }); + let img = + photocraft_codecs::Image::from_raw(w, h, photocraft_codecs::ChannelLayout::Rgb, photocraft_codecs::SampleType::U8, px).map_err(|e| e.to_string())?; + photocraft_codecs::encode(&img, photocraft_codecs::Format::Jpeg, &Default::default()).map_err(|e| e.to_string()) +} + +fn open_photo(w: u32, h: u32) -> Res { + let jpeg = photo_jpeg(w, h)?; + let doc = photocraft_io::import("photo.jpg", &jpeg).map_err(|e| e.to_string())?.document; + let mut s = Session::new(); + s.open_document(doc, Some("photo.jpg".into())); + Ok(s) +} + +/// How far an edit's composite change reaches beyond its damage (layer effects); mirrors +/// `canvas::effect_reach`. +fn reach(layers: &[photocraft_doc::Layer]) -> i32 { + layers + .iter() + .map(|l| { + let own = if photocraft_compose::effects::has_effects(l) { photocraft_compose::effects::margin(l) } else { 0 }; + let kids = match &l.content { + photocraft_doc::LayerContent::Group(g) => reach(&g.children), + _ => 0, + }; + own + kids + }) + .max() + .unwrap_or(0) +} + +struct Bench { + gpu: Option<(GpuCanvas, RenderState)>, + sampler: RssSampler, + reps: usize, + only: Option, + rows: Vec, + /// Rows that couldn't run: (name, error). + errors: Vec<(String, String)>, + /// The row being timed, so a panic inside it is reported against that row. + current: Option, +} + +impl Bench { + fn wanted(&self, name: &str) -> bool { + self.only.as_ref().is_none_or(|o| name.contains(o.as_str())) + } + + /// The canvas refresh after an edit (`canvas::ensure_gpu`): the damage rect grown by the + /// effect reach when the session reports one, else everything; waits for the GPU. + fn refresh(&self, s: &Session, full: bool) -> Res { + let st = s.active().ok_or("no document")?; + let doc = &st.doc; + let damage = if full { None } else { st.last_damage.map(|r| r.inflate(reach(&doc.layers)).intersect(&doc.bounds())) }; + let t = Instant::now(); + match &self.gpu { + Some((g, rs)) => { + g.refresh(doc.id.0, doc, damage, None); + let _ = rs.device.poll(eframe::wgpu::PollType::Wait { submission_index: None, timeout: None }); + } + None => { + let r = damage.unwrap_or(doc.bounds()).intersect(&doc.bounds()); + if !r.is_empty() { + std::hint::black_box(photocraft_compose::render(doc, r)); + } + } + } + Ok(ms(t)) + } + + fn gpu_bytes(&self, s: &Session) -> Option { + let (g, _) = self.gpu.as_ref()?; + let tex = s.active().and_then(|d| g.texture_info(d.doc.id.0)).map_or(0, |(_, b)| b); + let (resident, fx) = g.compositor_bytes().unwrap_or((0, 0)); + Some(tex + resident + fx as u64) + } + + /// Times `f` `reps` times after one untimed warm-up run; `f` returns its own time in ms. + fn time(&mut self, name: &str, s: &mut Session, reps: usize, warm: bool, mut f: impl FnMut(&Self, &mut Session, usize) -> Res) { + if !self.wanted(name) { + return; + } + self.current = Some(name.to_string()); + self.sampler.reset(); + let mut samples = Vec::new(); + let mut run = || -> Res<()> { + if warm { + f(self, s, 0)?; + } + for i in 0..reps.max(1) { + samples.push(f(self, s, i + 1)?); + } + Ok(()) + }; + let result = run(); + self.record(name, s, &samples, result); + } + + fn record(&mut self, name: &str, s: &Session, samples: &[f64], result: Res<()>) { + match result { + Ok(()) => { + let r = row(name, samples, self.sampler.peak(), self.gpu_bytes(s)); + println!( + "{name:<52} p50 {:>9.2} ms p95 {:>9.2} ms max {:>9.2} ms (n {}, peak RSS {} MB)", + r["p50_ms"].as_f64().unwrap_or(f64::NAN), + r["p95_ms"].as_f64().unwrap_or(f64::NAN), + r["max_ms"].as_f64().unwrap_or(f64::NAN), + samples.len(), + r["peak_rss_bytes"].as_u64().unwrap_or(0) >> 20, + ); + self.rows.push(r); + } + Err(e) => { + println!("{name:<52} ERROR {e}"); + self.errors.push((name.to_string(), e)); + } + } + } +} + +/// A headless GPU canvas on a device created like the app's. +fn canvas() -> Option<(GpuCanvas, RenderState)> { + let setup = photocraft_ui_egui::gpu_canvas::wgpu_setup(); + let rs = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| egui_kittest::wgpu::create_render_state(setup, Default::default()))).ok()?; + eprintln!("adapter: {}", rs.adapter.get_info().name); + Some((GpuCanvas::new(&rs), rs)) +} + +/// Sizes per mode. +struct Sizes { + /// The layered document of #209 (20 MP). + layered: (u32, u32), + /// A layer of 12 MP to duplicate and paste. + big_layer: (u32, u32), + /// Layers in the "150-layer document". + many_layers: usize, + /// The 16-bit document (15000 × 10000). + huge16: (u32, u32), + /// The 32-bit document with 20 layers. + float32: (u32, u32), + /// 10 MP for Content-Aware Fill and Spot Healing. + ten_mp: (u32, u32), + /// 24 MP for the selection algorithms and Content-Aware Scale. + big_photo: (u32, u32), + /// A4 at 300 ppi. + a4: (u32, u32), +} + +const FULL: Sizes = Sizes { + layered: (5472, 3648), + big_layer: (4243, 2828), + many_layers: 150, + huge16: (15000, 10000), + float32: (4000, 3000), + ten_mp: (3872, 2592), + big_photo: (6000, 4000), + a4: (2480, 3508), +}; + +const QUICK: Sizes = Sizes { + layered: (1600, 1066), + big_layer: (1200, 800), + many_layers: 40, + huge16: (2400, 1600), + float32: (1200, 900), + ten_mp: (1200, 800), + big_photo: (1600, 1066), + a4: (1240, 1754), +}; + +/// The layered document of #209: a background, six pixel layers with a drop shadow and a stroke, +/// four type layers with a drop shadow, and a Levels adjustment on top. Returns the session and +/// the id of the styled layer the edits target. +fn layered_doc(w: u32, h: u32) -> Res<(Session, u64)> { + let mut s = Session::new(); + exec(&mut s, "file.new", json!({"width": w, "height": h, "background": "white"}))?; + let mut target = 0; + for i in 0..6u32 { + exec(&mut s, "layer.new.layer", json!({"name": format!("panel {i}")}))?; + let (x, y) = ((i % 3) * w / 3 + w / 24, (i / 3) * h / 2 + h / 16); + exec(&mut s, "select.rect", json!({"x": x, "y": y, "width": w / 4, "height": h / 3, "ellipse": i % 2 == 1}))?; + exec(&mut s, "edit.fill", json!({"color": format!("#{:02x}6040", 40 + i * 30)}))?; + exec(&mut s, "select.deselect", json!({}))?; + exec(&mut s, "layer.layerStyle.dropShadow", json!({"distance": 12, "size": 16}))?; + exec(&mut s, "layer.layerStyle.stroke", json!({"size": 3, "color": "#202020", "add": true}))?; + if i == 1 { + target = active_layer(&s)?; + } + } + for i in 0..4u32 { + exec(&mut s, "type.create", json!({"x": w / 10 + i * w / 5, "y": h * 9 / 10, "text": format!("Headline {i}"), "size": (h / 30).max(12)}))?; + exec(&mut s, "layer.layerStyle.dropShadow", json!({"distance": 6, "size": 8}))?; + } + exec(&mut s, "layer.newAdjustmentLayer.levels", json!({"inBlack": 8, "inWhite": 245, "gamma": 1.05}))?; + exec(&mut s, "layer.select", json!({"layer": target}))?; + Ok((s, target)) +} + +/// A document with `n` layers: small painted rects, every fifth a type layer, every seventh +/// styled. +fn many_layers_doc(w: u32, h: u32, n: usize) -> Res<(Session, u64)> { + let mut s = Session::new(); + exec(&mut s, "file.new", json!({"width": w, "height": h, "background": "white"}))?; + let mut mid = 0; + for i in 0..n as u32 { + let (x, y) = ((i * 397) % (w - w / 8), (i * 211) % (h - h / 8)); + if i % 5 == 4 { + exec(&mut s, "type.create", json!({"x": x, "y": y + 40, "text": format!("Label {i}"), "size": 28}))?; + } else { + exec(&mut s, "layer.new.layer", json!({"name": format!("item {i}")}))?; + exec(&mut s, "select.rect", json!({"x": x, "y": y, "width": w / 10, "height": h / 10}))?; + exec(&mut s, "edit.fill", json!({"color": format!("#{:02x}{:02x}80", (i * 37) % 256, (i * 91) % 256)}))?; + exec(&mut s, "select.deselect", json!({}))?; + } + if i % 7 == 3 { + exec(&mut s, "layer.layerStyle.dropShadow", json!({"distance": 6, "size": 8}))?; + } + if i as usize == n / 2 { + mid = active_layer(&s)?; + } + } + exec(&mut s, "layer.select", json!({"layer": mid}))?; + Ok((s, mid)) +} + +fn main() { + let args: Vec = std::env::args().collect(); + let quick = args.iter().any(|a| a == "--quick"); + let sz = if quick { &QUICK } else { &FULL }; + let reps: usize = arg(&args, "--reps").and_then(|v| v.parse().ok()).unwrap_or(if quick { 7 } else { 15 }).max(1); + let gpu = if args.iter().any(|a| a == "--cpu") { None } else { canvas() }; + let adapter = gpu.as_ref().map(|(_, rs)| rs.adapter.get_info().name); + let mut b = Bench { + gpu, + sampler: RssSampler::start(std::time::Duration::from_millis(2)), + reps, + only: arg(&args, "--only"), + rows: Vec::new(), + errors: Vec::new(), + current: None, + }; + println!("perf_scenarios: {} mode, {} reps, {}", if quick { "quick" } else { "full" }, reps, adapter.as_deref().unwrap_or("CPU canvas")); + let t_all = Instant::now(); + + // A panic in one group (a crash the scenario found) is reported against the row that was + // running, and the other groups still run. + type Group = fn(&mut Bench, &Sizes); + let groups: [(&str, Group); 5] = [ + ("layered", layered_scenarios), + ("many layers", many_layer_scenarios), + ("bit depths", depth_scenarios), + ("long operations", job_scenarios), + ("algorithms", algorithm_scenarios), + ]; + let use_gpu = b.gpu.is_some(); + for (i, (group, run)) in groups.into_iter().enumerate() { + b.current = None; + // Each group starts on a fresh canvas device (one document open, as in the app), so + // GPU memory or an error left by one group never carries into the next. + if use_gpu && i > 0 { + b.gpu = None; + b.gpu = canvas(); + } + if let Err(e) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| run(&mut b, sz))) { + let msg = e.downcast_ref::().cloned().or_else(|| e.downcast_ref::<&str>().map(|s| s.to_string())).unwrap_or_else(|| "panic".into()); + let msg = format!("panicked: {}", msg.lines().filter(|l| !l.trim().is_empty()).collect::>().join(" / ")); + let row = b.current.take().unwrap_or_else(|| group.to_string()); + println!("{row:<52} ERROR {msg}"); + b.errors.push((row, msg)); + } + } + + println!("total {:.1} s", t_all.elapsed().as_secs_f64()); + if let Some(out) = arg(&args, "--json") { + let context = json!({ + "mode": if quick { "quick" } else { "full" }, + "reps": reps, + "gpu_adapter": adapter, + "errors": b.errors.iter().map(|(n, e)| json!({"name": n, "error": e})).collect::>(), + }); + let rep = report("perf_scenarios", context, std::mem::take(&mut b.rows), Some(&b.sampler)); + if let Err(e) = write_report(&out, &rep) { + eprintln!("{e}"); + std::process::exit(1); + } + } +} + +/// #209 rows on the layered 20 MP document. +fn layered_scenarios(b: &mut Bench, sz: &Sizes) { + let (w, h) = sz.layered; + let (mut s, target) = match layered_doc(w, h) { + Ok(v) => v, + Err(e) => { + b.errors.push(("layered document".into(), e)); + return; + } + }; + let _ = b.refresh(&s, true); + let reps = b.reps; + b.time("move styled layer 10 px + refresh", &mut s, reps, true, |b, s, i| { + let d = if i % 2 == 0 { 10 } else { -10 }; + let t = Instant::now(); + exec(s, "layer.translate", json!({"layer": target, "dx": d, "dy": d / 2}))?; + Ok(ms(t) + b.refresh(s, false)?) + }); + b.time("set layer opacity + refresh", &mut s, reps, true, |b, s, i| { + let t = Instant::now(); + exec(s, "layer.setProps", json!({"layer": target, "opacity": if i % 2 == 0 { 0.6 } else { 0.9 }}))?; + Ok(ms(t) + b.refresh(s, false)?) + }); + b.time("set layer fill + refresh", &mut s, reps, true, |b, s, i| { + let t = Instant::now(); + exec(s, "layer.setProps", json!({"layer": target, "fill": if i % 2 == 0 { 0.5 } else { 1.0 }}))?; + Ok(ms(t) + b.refresh(s, false)?) + }); + b.time("set blend mode + refresh", &mut s, reps, true, |b, s, i| { + let t = Instant::now(); + exec(s, "layer.setProps", json!({"layer": target, "blend": if i % 2 == 0 { "Multiply" } else { "Normal" }}))?; + Ok(ms(t) + b.refresh(s, false)?) + }); + b.time("toggle visibility + refresh", &mut s, reps, true, |b, s, i| { + let t = Instant::now(); + exec(s, "layer.setProps", json!({"layer": target, "visible": i % 2 == 0}))?; + Ok(ms(t) + b.refresh(s, false)?) + }); + let _ = exec(&mut s, "layer.setProps", json!({"layer": target, "visible": true})); + b.time("undo move + refresh", &mut s, reps, true, |b, s, _| { + exec(s, "layer.translate", json!({"layer": target, "dx": 7, "dy": 3}))?; + b.refresh(s, false)?; + let t = Instant::now(); + exec(s, "edit.undo", json!({}))?; + Ok(ms(t) + b.refresh(s, false)?) + }); + b.time("redo opacity change + refresh", &mut s, reps, true, |b, s, i| { + exec(s, "layer.setProps", json!({"layer": target, "opacity": 0.5 + 0.02 * i as f64}))?; + exec(s, "edit.undo", json!({}))?; + b.refresh(s, false)?; + let t = Instant::now(); + exec(s, "edit.redo", json!({}))?; + Ok(ms(t) + b.refresh(s, false)?) + }); + + // A 12 MP layer, duplicated and pasted: the first time (a fresh layer, cold caches) and + // again (the same source). + let (lw, lh) = sz.big_layer; + let fresh_layer = |s: &mut Session| -> Res { + exec(s, "layer.new.layer", json!({"name": "12 MP"}))?; + exec(s, "select.rect", json!({"x": (w - lw) / 2, "y": (h - lh) / 2, "width": lw, "height": lh}))?; + exec(s, "edit.fill", json!({"color": "#3070a0"}))?; + exec(s, "select.deselect", json!({}))?; + active_layer(s) + }; + let reps_big = reps.min(5); + let mut src = 0; + b.time("duplicate 12 MP layer (first) + refresh", &mut s, reps_big, false, |b, s, _| { + src = fresh_layer(s)?; + b.refresh(s, false)?; + let t = Instant::now(); + exec(s, "layer.duplicate", json!({"layer": src}))?; + Ok(ms(t) + b.refresh(s, false)?) + }); + b.time("duplicate 12 MP layer (repeat) + refresh", &mut s, reps_big, false, |b, s, _| { + let t = Instant::now(); + exec(s, "layer.duplicate", json!({"layer": src}))?; + let v = ms(t) + b.refresh(s, false)?; + exec(s, "edit.undo", json!({}))?; + b.refresh(s, false)?; + Ok(v) + }); + b.time("paste 12 MP layer (first) + refresh", &mut s, reps_big, false, |b, s, _| { + let id = fresh_layer(s)?; + exec(s, "layer.select", json!({"layer": id}))?; + exec(s, "select.all", json!({}))?; + exec(s, "edit.copy", json!({}))?; + exec(s, "select.deselect", json!({}))?; + b.refresh(s, false)?; + let t = Instant::now(); + exec(s, "edit.paste", json!({}))?; + Ok(ms(t) + b.refresh(s, false)?) + }); + b.time("paste 12 MP layer (repeat) + refresh", &mut s, reps_big, false, |b, s, _| { + let t = Instant::now(); + exec(s, "edit.paste", json!({}))?; + let v = ms(t) + b.refresh(s, false)?; + exec(s, "edit.undo", json!({}))?; + b.refresh(s, false)?; + Ok(v) + }); +} + +/// #209: arrow-key nudge in the 150-layer document. +fn many_layer_scenarios(b: &mut Bench, sz: &Sizes) { + // Row names stay stable across modes: the layer count is in the mode, not the key. + let name = "nudge 1 px in the many-layer document + refresh".to_string(); + if !b.wanted(&name) { + return; + } + // The document's first refresh is part of this row (a crash there is reported against it). + b.current = Some(name.clone()); + let (w, h) = sz.layered; + let (mut s, mid) = match many_layers_doc(w, h, sz.many_layers) { + Ok(v) => v, + Err(e) => { + b.errors.push((name, e)); + return; + } + }; + let _ = b.refresh(&s, true); + let reps = b.reps; + b.time(&name, &mut s, reps, true, |b, s, i| { + let t = Instant::now(); + exec(s, "layer.translate", json!({"layer": mid, "dx": if i % 2 == 0 { 1 } else { -1 }, "dy": 0}))?; + Ok(ms(t) + b.refresh(s, false)?) + }); +} + +/// #209: edits on the 15000 × 10000 16-bit document and the 32-bit document with 20 layers. +fn depth_scenarios(b: &mut Bench, sz: &Sizes) { + let reps = b.reps; + for (label, (w, h), depth, layers) in [("16-bit 15000x10000", sz.huge16, 16, 1u32), ("32-bit 20 layers", sz.float32, 32, 20)] { + let dab = format!("brush dab 60 px on {label} + refresh"); + let mv = format!("move layer on {label} + refresh"); + if !b.wanted(&dab) && !b.wanted(&mv) { + continue; + } + let built = (|| -> Res<(Session, u64)> { + let mut s = Session::new(); + exec(&mut s, "file.new", json!({"width": w, "height": h, "depth": depth, "background": "white"}))?; + let mut first = 0; + for i in 0..layers { + exec(&mut s, "layer.new.layer", json!({"name": format!("layer {i}")}))?; + let (x, y) = ((i * 173) % (w * 3 / 4), (i * 97) % (h * 3 / 4)); + exec(&mut s, "select.rect", json!({"x": x, "y": y, "width": w / 4, "height": h / 4}))?; + exec(&mut s, "edit.fill", json!({"color": format!("#{:02x}8060", (i * 41) % 256)}))?; + exec(&mut s, "select.deselect", json!({}))?; + if i == 0 { + first = active_layer(&s)?; + } + } + Ok((s, first)) + })(); + let (mut s, layer) = match built { + Ok(v) => v, + Err(e) => { + b.errors.push((dab, e)); + continue; + } + }; + let _ = b.refresh(&s, true); + let _ = exec(&mut s, "layer.select", json!({"layer": layer})); + b.time(&dab, &mut s, reps, true, |b, s, i| { + let (x, y) = (200 + (i as u32 * 97) % (w - 400), 200 + (i as u32 * 61) % (h - 400)); + let t = Instant::now(); + exec(s, "paint.stroke", json!({"points": [[x, y]], "size": 60, "hardness": 0.8, "color": "#c03020"}))?; + Ok(ms(t) + b.refresh(s, false)?) + }); + b.time(&mv, &mut s, reps, true, |b, s, i| { + let t = Instant::now(); + exec(s, "layer.translate", json!({"layer": layer, "dx": if i % 2 == 0 { 10 } else { -10 }, "dy": 0}))?; + Ok(ms(t) + b.refresh(s, false)?) + }); + } +} + +/// #210 rows that are measurable before the job system exists. +fn job_scenarios(b: &mut Bench, sz: &Sizes) { + let reps = b.reps.min(5); + let open = "open 20 MP layered PSD (decode + first composite)"; + if b.wanted(open) { + let (w, h) = sz.layered; + match layered_doc(w, h).and_then(|(s, _)| { + let d = s.active().ok_or("no document")?.doc.clone(); + photocraft_io::export(&d, "layered.psd", &Default::default()).map(|o| o.bytes).map_err(|e| e.to_string()) + }) { + Ok(psd) => { + let mut holder = Session::new(); + b.time(open, &mut holder, reps, true, |b, s, _| { + let t = Instant::now(); + let d = photocraft_io::import("layered.psd", &psd).map_err(|e| e.to_string())?.document; + *s = Session::new(); + s.open_document(d, Some("layered.psd".into())); + Ok(ms(t) + b.refresh(s, true)?) + }); + } + Err(e) => b.errors.push((open.into(), e)), + } + } + let caf = "Content-Aware Fill 10 MP (200 px hole)"; + let heal = "Spot Healing stroke 10 MP"; + if b.wanted(caf) || b.wanted(heal) { + let (w, h) = sz.ten_mp; + match open_photo(w, h) { + Ok(mut s) => { + b.time(caf, &mut s, reps, false, |_, s, i| { + exec(s, "select.rect", json!({"x": w / 3 + i as u32 * 40, "y": h / 3, "width": 200, "height": 200, "ellipse": true}))?; + let t = Instant::now(); + exec(s, "edit.contentAwareFill", json!({"seed": i}))?; + let v = ms(t); + exec(s, "edit.undo", json!({}))?; + exec(s, "select.deselect", json!({}))?; + Ok(v) + }); + b.time(heal, &mut s, reps, false, |b, s, i| { + let (x, y) = (w / 4 + i as u32 * 60, h / 4); + let pts: Vec = (0..10).map(|k| json!([x + k * 12, y + k * 4])).collect(); + let t = Instant::now(); + exec(s, "paint.spotHealing", json!({"points": pts, "size": 60}))?; + Ok(ms(t) + b.refresh(s, false)?) + }); + } + Err(e) => b.errors.push((caf.into(), e)), + } + } +} + +/// #211 rows: selection algorithms, Content-Aware Scale and Select Subject on 24 MP, and a brush +/// dab on an A4 300 ppi CMYK document. +fn algorithm_scenarios(b: &mut Bench, sz: &Sizes) { + let reps = b.reps.min(5); + let names = [ + "Select > Modify > Smooth r 50 on 24 MP", + "Select > Modify > Feather r 50 on 24 MP", + "Select Subject 24 MP (repeat)", + "Content-Aware Scale 24 MP to 90 % width", + ]; + if names.iter().any(|n| b.wanted(n)) { + let (w, h) = sz.big_photo; + match open_photo(w, h) { + Ok(mut s) => { + for (name, op) in [(names[0], "select.modify.smooth"), (names[1], "select.modify.feather")] { + b.time(name, &mut s, reps, false, |_, s, _| { + exec(s, "select.rect", json!({"x": w / 6, "y": h / 6, "width": w * 2 / 3, "height": h * 2 / 3, "ellipse": true}))?; + let t = Instant::now(); + exec(s, op, json!({"radius": 50}))?; + let v = ms(t); + exec(s, "select.deselect", json!({}))?; + Ok(v) + }); + } + b.time(names[2], &mut s, reps, true, |_, s, _| { + let t = Instant::now(); + exec(s, "select.subject", json!({}))?; + let v = ms(t); + exec(s, "select.deselect", json!({}))?; + Ok(v) + }); + // One run: today this takes far longer than its budget. + b.time(names[3], &mut s, 1, false, |_, s, _| { + let t = Instant::now(); + exec(s, "edit.contentAwareScale", json!({"scaleX": 90, "scaleY": 100}))?; + let v = ms(t); + exec(s, "edit.undo", json!({}))?; + Ok(v) + }); + } + Err(e) => b.errors.push((names[0].into(), e)), + } + } + let cmyk = "brush dab on A4 300 ppi CMYK + refresh"; + if b.wanted(cmyk) { + let (w, h) = sz.a4; + let mut s = Session::new(); + let setup = exec(&mut s, "file.new", json!({"width": w, "height": h, "mode": "cmyk", "resolution": 300, "background": "white"})) + .and_then(|_| exec(&mut s, "layer.new.layer", json!({"name": "paint"}))); + if let Err(e) = setup { + b.errors.push((cmyk.into(), e)); + return; + } + let _ = b.refresh(&s, true); + let reps = b.reps; + b.time(cmyk, &mut s, reps, true, |b, s, i| { + let (x, y) = (200 + (i as u32 * 97) % (w - 400), 200 + (i as u32 * 61) % (h - 400)); + let t = Instant::now(); + exec(s, "paint.stroke", json!({"points": [[x, y]], "size": 40, "hardness": 0.8, "color": "#c03020"}))?; + Ok(ms(t) + b.refresh(s, false)?) + }); + } +} diff --git a/crates/ui-egui/examples/type_bench.rs b/crates/ui-egui/examples/type_bench.rs index ecbdcc44..9b9239d3 100644 --- a/crates/ui-egui/examples/type_bench.rs +++ b/crates/ui-egui/examples/type_bench.rs @@ -81,6 +81,20 @@ fn main() { while size(&app) != start && undos < steps + 1 && app.session.undo() { undos += 1; } + // `--json out.json` (`cargo xtask perf`): the per-step samples. + if let Some(out) = arg(&args, "--json") { + use photocraft_testkit::perf::{process_peak_rss_bytes, report, row, write_report}; + let rss = photocraft_testkit::perf::current_rss_bytes().max(process_peak_rss_bytes()); + let rows = vec![ + row("font size step: command", &cmd, rss, None), + row("font size step: canvas refresh", &canvas, rss, None), + row("font size step: total", &total, rss, None), + ]; + let rep = report("type_bench", json!({"width": w, "height": h, "layers": n, "steps": steps, "effects": effects}), rows, None); + if let Err(e) = write_report(&out, &rep) { + eprintln!("{e}"); + } + } println!( "size step (median of {steps}): command {:.2} ms, canvas refresh {:.2} ms, total {:.2} ms; undo steps for the drag: {undos}", median(cmd), diff --git a/docs/development.md b/docs/development.md index 04817d6e..9341be8c 100644 --- a/docs/development.md +++ b/docs/development.md @@ -185,6 +185,60 @@ the live command registry (`menus::is_live`) and rewrites [`docs/parity.md`](par - `ui.inspect` returns `perf` timings (UI ms per frame, composite ms, upload ms). - Never scan full surfaces per frame. Cache per document revision (`PhotocraftApp::cached_bounds`). An uncached `content_bounds()` on a 36 MP layer once cost 77 ms per frame. +## Scorecard and performance budgets + +[`docs/scorecard.md`](scorecard.md) says, with numbers, where PhotoCraft stands per area. It is +generated; never edit it by hand. + +```sh +cargo xtask scorecard # regenerate docs/scorecard.md +cargo xtask scorecard --check # what CI runs: fails if the committed file is stale +cargo xtask perf # full benchmark run (tens of minutes, release build) +cargo xtask perf --quick # small synthetic documents (about a minute once built) +cargo xtask perf --update-baseline # also write perf/baseline.json from this run +``` + +**Sources.** The scorecard reads only committed files, so it is deterministic: +- `scorecard/*.toml`: one checklist per area (tools, file compatibility, UI, type, automation, + reliability, distribution, open bugs). Each item has an `id`, a short `target`, a `status` + (`done`, `partial` or `missing`), the `issue` and a `note` with the evidence (file:line). + Statuses must be true on `main`: verify against the code, and say `partial` with a note when + unsure. Flip a row in the PR that changes it. +- `perf/budgets.toml`: the benches `xtask perf` runs and the scenarios P1…Pn, each mapped to one + bench row, with a budget (targets from #209, #210, #211). `enforce = false` marks a target the + code doesn't meet yet (reported as over budget; only regressions fail); flip it to `true` in the + PR that meets it. Scenarios the code can't support yet carry `not_measurable` and a reason, never + a number. +- `perf/baseline.json`: the numbers the scorecard shows, written only by + `cargo xtask perf --update-baseline`, with the machine (CPU, RAM, GPU adapter, OS), its machine + class, commit, date and load average. +- `crates/io/tests/corpus.rs`: the corpus floors (every `Source { .. }` constant). +- The prefs audit: `Preferences` fields that no code reads (target 0, #204), plus counts taken + from the tree (never-crash attribute coverage, `docs/parity.md`). + +**Perf runs.** `xtask perf` builds the benches in release (`perf_scenarios`, `interactive_bench`, +`fx_bench`, `type_bench`, `large_image_bench`, and `layout_bench` once it exists), runs each with +`--json`, and merges the reports into `target/perf/results.json` keyed by scenario id, with p50, +p95 and max (nearest rank over the samples), peak RSS measured in-process (`photocraft-testkit`'s +`perf::RssSampler`), GPU bytes held by the canvas, and the load average before and after each +bench. `target/perf/summary.md` is the Markdown table. It exits non-zero when an enforced budget +breaks, when a scenario's p50 regresses more than 15 % (`--threshold`, or `regression_pct`) +against a baseline from the same machine class and mode, or when a bench fails. Runs from +different machine classes are never compared, and `--quick` numbers are only compared with a +quick baseline. Machines that run other work give noisy numbers: check the load average in the +summary before trusting a regression, and record it next to any number you quote. + +**Adding a scenario.** Add a row to a bench (keep its name stable: it is the key), give the +bench `--json` support through `photocraft_testkit::perf::{row, report, write_report}`, then add a +`[[scenario]]` to `perf/budgets.toml` and run `cargo xtask scorecard`. + +**CI.** `ci.yml` runs `cargo xtask scorecard --check`. `perf-nightly.yml` runs `cargo xtask perf` +on a fixed macOS runner every night (and on demand), posts the table as the job summary, uploads +`results.json`, and fails on a broken budget or regression. It never runs on pull requests: the +release build of the benches alone takes longer than PR CI should. To give the nightly runner +its own baseline, run the workflow by hand with `update_baseline` and commit the `perf-baseline` +artifact as `perf/baseline.json`. + ## Web build `apps/photocraft-web` runs the same `PhotocraftApp` in the browser through eframe's web runner. The renderer is wgpu: WebGPU where the browser has it, WebGL2 otherwise. It is Rust only. The only JavaScript is the glue that wasm-bindgen generates. diff --git a/docs/releasing.md b/docs/releasing.md index 9692ccff..69f632fe 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -25,7 +25,13 @@ and ids stay lowercase (`photocraft---.`, `ai.stor every artifact and `SHA256SUMS.txt`. The notes are generated from the merged PRs. 4. **Check it.** Download an installer or two and look at the job summaries. Any `::warning::` there means a signing secret was missing and that artifact is unsigned. -5. **Publish** the draft in the GitHub UI. Publishing creates the `v0.2.0` tag. Versions with a +5. **Add the scorecard deltas** to the draft's notes: what moved in + [`docs/scorecard.md`](scorecard.md) since the previous release. Compare the summary table + and the performance table of both tags (`git diff v v -- docs/scorecard.md`) + and list each area's done / partial / missing change, the scenarios that newly meet their + budget or got slower, corpus floors raised, and the change in settings that do nothing. + Quote numbers with the baseline's machine and load average. +6. **Publish** the draft in the GitHub UI. Publishing creates the `v0.2.0` tag. Versions with a pre-release suffix (`-rc.1`) are marked as pre-releases. Pushing to `release` again before you publish rebuilds the same draft and replaces its assets. diff --git a/docs/roadmap.md b/docs/roadmap.md index 80b2eca1..f54d8a91 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -34,6 +34,15 @@ This is the reference answer to "how close are we to Photoshop parity, really". before picking work. Update it (with dated measurements) when the numbers move; don't restate the menu-parity number in its place. +**Measured numbers live in the [scorecard](scorecard.md)** (`cargo xtask scorecard`, kept current +by CI): performance scenarios against their budgets, corpus floors, per-area checklists with +done / partial / missing counts, and the number of settings that do nothing. Where the scorecard +and the estimates below disagree, the scorecard's numbers supersede them. As of 2026-10-05 it +counts 67 of 129 preferences that nothing reads; 3 of the 25 budgeted performance scenarios meet +their #209–#211 targets (first baseline, taken on a heavily loaded machine), and the 150-layer +nudge scenario crashes the GPU compositor; the open area issues' checklists (#203–#220) are +almost entirely missing or partial. + **Bottom line.** Two days after 0.2.0 we had merged ~96 PRs and closed ~48 issues, but **real Photoshop parity is still well below 50%**. The biggest gaps are AI, missing tools, professional workflow depth and the plug-in ecosystem. Most fixes since 0.2.0 have passed our tests but have diff --git a/docs/scorecard.md b/docs/scorecard.md new file mode 100644 index 00000000..16fcee14 --- /dev/null +++ b/docs/scorecard.md @@ -0,0 +1,279 @@ +# PhotoCraft scorecard + +Generated by `cargo xtask scorecard` from `perf/budgets.toml`, `perf/baseline.json`, `crates/io/tests/corpus.rs`, `scorecard/*.toml` and the source tree. Do not edit by hand: change the sources and regenerate (CI runs `cargo xtask scorecard --check`). Checklist statuses are verified against the code; numbers here supersede estimates elsewhere. + +## Summary + +| Area | Done | Partial | Missing | Key numbers | +|---|---:|---:|---:|---| +| [Performance](#performance) | 3 | 22 | 10 | 44 scenarios: 3 within budget, 22 over budget or failing, 9 tracked only, 0 not run, 10 not measurable yet (baseline 2026-10-05) | +| [File compatibility](#file-compatibility) | 0 | 6 | 21 | io corpus: oracle ≥ 145/170, round trip ≥ 169/170; psd-tools corpus: oracle ≥ 228/309, round trip ≥ 307/309 | +| [Tools](#tools) | 0 | 16 | 14 | | +| [Workspace and UI](#workspace-and-ui) | 0 | 3 | 20 | settings that do nothing: 66 of 129 (target 0) | +| [Type](#type) | 0 | 3 | 6 | | +| [Automation](#automation) | 1 | 3 | 7 | 626 of 626 menu items live (100.0%) | +| [Reliability](#reliability) | 2 | 1 | 4 | never-crash lints on 25 of 25 crates; panic_hunt present | +| [Distribution](#distribution) | 4 | 2 | 4 | | +| [Open bugs](#open-bugs) | 0 | 3 | 29 | | + +## Performance + +Budgets: `perf/budgets.toml` (targets from #209, #210, #211). Numbers: the committed baseline (`perf/baseline.json`, full run), updated only by `cargo xtask perf --update-baseline`. The nightly `perf-nightly` workflow runs `cargo xtask perf` on a fixed macOS runner and fails on a broken enforced budget or a p50 regression over 15 % against a baseline from the same machine class. "Over budget" rows are targets the code doesn't meet yet: they are reported, and only their regressions fail. + +Baseline: 2026-10-05, commit `f5d2bcb54a`, Apple M4 Pro (48 GB), GPU Apple M4 Pro, macOS 15.6; load average 224.90 / 226.31 / 196.03 (1/5/15 min) at the start of the run. + +| Id | Scenario | Target | p50 ms | p95 ms | max ms | Peak RSS | GPU | Status | Issue | +|---|---|---|---:|---:|---:|---:|---:|---|---| +| P1 | Move a layer with layer styles (per drag update) | ≤ 25 ms p95 | 321 | 535 | 535 | 825 MB | 409 MB | over budget (target not met yet) | [#209](https://github.com/storytold/photocraft/issues/209) | +| P2 | Change layer opacity | ≤ 20 ms p95 | 276 | 443 | 443 | 639 MB | 409 MB | over budget (target not met yet) | [#209](https://github.com/storytold/photocraft/issues/209) | +| P3 | Change layer fill | ≤ 20 ms p95 | 279 | 467 | 467 | 506 MB | 409 MB | over budget (target not met yet) | [#209](https://github.com/storytold/photocraft/issues/209) | +| P4 | Change blend mode | ≤ 20 ms p95 | 206 | 423 | 423 | 579 MB | 409 MB | over budget (target not met yet) | [#209](https://github.com/storytold/photocraft/issues/209) | +| P5 | Toggle visibility | ≤ 10 ms p95 | 178 | 425 | 425 | 668 MB | 409 MB | over budget (target not met yet) | [#209](https://github.com/storytold/photocraft/issues/209) | +| P6 | Undo a move | ≤ 25 ms p95 | 256 | 530 | 530 | 650 MB | 409 MB | over budget (target not met yet) | [#209](https://github.com/storytold/photocraft/issues/209) | +| P7 | Redo a property change | ≤ 25 ms p95 | 144 | 212 | 212 | 669 MB | 409 MB | over budget (target not met yet) | [#209](https://github.com/storytold/photocraft/issues/209) | +| P8 | Duplicate a 12 MP layer (first) | ≤ 100 ms p95 | 508 | 699 | 699 | 907 MB | 919 MB | over budget (target not met yet) | [#209](https://github.com/storytold/photocraft/issues/209) | +| P9 | Duplicate a 12 MP layer (repeat) | ≤ 20 ms p95 | 406 | 627 | 627 | 749 MB | 919 MB | over budget (target not met yet) | [#209](https://github.com/storytold/photocraft/issues/209) | +| P10 | Paste a 12 MP layer (first) | ≤ 100 ms p95 | 1232 | 3136 | 3136 | 1.3 GB | 1.4 GB | over budget (target not met yet) | [#209](https://github.com/storytold/photocraft/issues/209) | +| P11 | Paste a 12 MP layer (repeat) | ≤ 20 ms p95 | 445 | 1174 | 1174 | 958 MB | 1.4 GB | over budget (target not met yet) | [#209](https://github.com/storytold/photocraft/issues/209) | +| P12 | Arrow-key nudge in a 150-layer document | ≤ 30 ms p95 | – | – | – | – | – | **missing**: panicked: wgpu error: Validation Error / Caused by: / In Queue::write_buffer / Buffer with 'pc_compose_uniforms' label is invalid | [#209](https://github.com/storytold/photocraft/issues/209) | +| P13 | Brush dab on a 15000×10000 16-bit document | ≤ 80 ms p95 | 6.42 | 146 | 146 | 839 MB | 2.0 GB | over budget (target not met yet) | [#209](https://github.com/storytold/photocraft/issues/209) | +| P14 | Move a layer on a 15000×10000 16-bit document | ≤ 80 ms p95 | 2698 | 4318 | 4318 | 1.5 GB | 2.0 GB | over budget (target not met yet) | [#209](https://github.com/storytold/photocraft/issues/209) | +| P16 | Brush dab on a 32-bit document with 20 layers | ≤ 100 ms p95 | 21.5 | 161 | 161 | 793 MB | 1.6 GB | over budget (target not met yet) | [#209](https://github.com/storytold/photocraft/issues/209) | +| P17 | Move a layer on a 32-bit document with 20 layers | ≤ 100 ms p95 | 386 | 681 | 681 | 1.2 GB | 1.6 GB | over budget (target not met yet) | [#209](https://github.com/storytold/photocraft/issues/209) | +| P18 | Open a 20 MP layered PSD: first pixels (today: fully opened) | ≤ 1000 ms p95 | 1753 | 2794 | 2794 | 2.2 GB | 1.9 GB | over budget (target not met yet) | [#210](https://github.com/storytold/photocraft/issues/210) | +| P19 | Open a 20 MP layered PSD: editable | ≤ 3000 ms p95 | 1753 | 2794 | 2794 | 2.2 GB | 1.9 GB | pass | [#210](https://github.com/storytold/photocraft/issues/210) | +| P21 | Content-Aware Fill on 10 MP | ≤ 1500 ms p95 | 218 | 256 | 256 | 1.8 GB | 1.8 GB | pass | [#210](https://github.com/storytold/photocraft/issues/210) | +| P22 | Spot Healing stroke on 10 MP | ≤ 500 ms p95 | 157 | 247 | 247 | 1.8 GB | 1.9 GB | pass | [#210](https://github.com/storytold/photocraft/issues/210) | +| P26 | Select › Modify › Smooth on 24 MP (radius 50) | ≤ 100 ms p95 | 19296 | 22051 | 22051 | 1.9 GB | – | over budget (target not met yet) | [#211](https://github.com/storytold/photocraft/issues/211) | +| P27 | Select › Modify › Feather on 24 MP (radius 50) | ≤ 100 ms p95 | 42668 | 52100 | 52100 | 650 MB | – | over budget (target not met yet) | [#211](https://github.com/storytold/photocraft/issues/211) | +| P28 | Content-Aware Scale 24 MP (to 90 % width) | ≤ 3000 ms p95 | 954192 | 954192 | 954192 | 2.1 GB | – | over budget (target not met yet) | [#211](https://github.com/storytold/photocraft/issues/211) | +| P29 | Select Subject on 24 MP, repeated | ≤ 100 ms p95 | 559 | 693 | 693 | 1.4 GB | – | over budget (target not met yet) | [#211](https://github.com/storytold/photocraft/issues/211) | +| P33 | Brush dab on A4 300 ppi CMYK (dab to screen) | ≤ 6 ms p95 | 34.7 | 235 | 235 | 775 MB | 87 MB | over budget (target not met yet) | [#211](https://github.com/storytold/photocraft/issues/211) | +| P40 | Brush dab 60 px on 36 MP | track only | 1.45 | 3.19 | 3.19 | 1.3 GB | – | tracked | | +| P41 | Gaussian Blur r 40 on 36 MP | track only | 2963 | 3069 | 3069 | 1.9 GB | – | tracked | | +| P42 | Curves dialog change, layer preview, 36 MP | track only | 38.7 | 158 | 158 | 1.1 GB | – | tracked | | +| P43 | Save .psd, 36 MP, 3 layers | track only | 3047 | 3446 | 3446 | 1.2 GB | – | tracked | | +| P44 | Open .psd, 36 MP, 3 layers | track only | 3535 | 4547 | 4547 | 1.6 GB | – | tracked | | +| P45 | Full refresh with layer effects, warm caches, 36 MP (GPU) | track only | 105 | 160 | 160 | 714 MB | – | tracked | | +| P46 | Brush dab on a layer with effects, 36 MP (GPU) | track only | 30.6 | 83.3 | 83.3 | 863 MB | – | tracked | | +| P47 | Font-size drag step, 50 type layers | track only | 23.3 | 60.5 | 71.2 | 600 MB | – | tracked | | +| P48 | Full refresh of a 14000×14000 document | track only | 8715 | 9897 | 9897 | – | – | tracked | | + +**Not measurable yet** (the code can't do it yet; no number is recorded): + +| Id | Scenario | Target | Why | Issue | +|---|---|---|---|---| +| P15 | Zoom / pan frame time on a 15000×10000 16-bit document | ≤ 16 ms per frame | no headless canvas frame harness yet (zoom and pan sample the GPU texture in the egui paint callback) | [#209](https://github.com/storytold/photocraft/issues/209) | +| P20 | Open a 2 GB PSD / 4–5 GB PSB | opens with progress, editable, never blank | no progress or background open (#210) and no generated large-file fixtures (#216) | [#210](https://github.com/storytold/photocraft/issues/210) | +| P23 | Warp / Puppet / Perspective preview at 4K | ≤ 16 ms per frame, commit in the background | the previews run inside the UI tools; no headless frame harness, no background commit | [#210](https://github.com/storytold/photocraft/issues/210) | +| P24 | Import 500 brushes | no UI pause over 100 ms | import runs synchronously on the UI thread (no job system, #210) | [#210](https://github.com/storytold/photocraft/issues/210) | +| P25 | Cancel any long job | takes effect within 200 ms, document unchanged | no operation can be cancelled yet (#210) | [#210](https://github.com/storytold/photocraft/issues/210) | +| P30 | Liquify 800 px brush on 3500×2200 at full resolution | ≤ 1 ms per update | the Liquify proxy is capped at 1600 px and the stroke path lives in the UI tool | [#211](https://github.com/storytold/photocraft/issues/211) | +| P31 | Camera Raw exposure change (proxy) | ≤ 10 ms | the proxy is built inside the dialog; no headless entry point yet | [#211](https://github.com/storytold/photocraft/issues/211) | +| P32 | Brush release on a long stroke | no hitch at mouse-up | the live renderer and the release re-render are UI-side; needs a frame harness | [#211](https://github.com/storytold/photocraft/issues/211) | +| P34 | Brush preview memory, small dab on an 8K viewport | ≤ 2 MiB | the live brush preview buffers are UI-side; no headless counter yet | [#211](https://github.com/storytold/photocraft/issues/211) | +| P35 | Marquee drag on 24 MP with Alt/Shift | ≤ 17 ms p95 | Shift/Alt marquee modifiers don't exist yet (#208) and the drag preview is UI-side | [#211](https://github.com/storytold/photocraft/issues/211) | + +## File compatibility + +Corpus floors (`crates/io/tests/corpus.rs`): the enforced minimums; the corpus test fails below them when the corpus is present. + +| Corpus | Directory | Files | Oracle pass floor | Round-trip floor | +|---|---|---:|---:|---:| +| io corpus | `corpus/psd` | 170 | 145 | 169 | +| psd-tools corpus | `corpus/psd-tools` | 309 | 228 | 307 | + +| Id | Target | Status | Issue | Note | +|---|---|---|---|---| +| FILE-215-1 | Smart Objects export as embedded Smart Objects (SoLd + lnk2), not flat pixels | missing | [#215](https://github.com/storytold/photocraft/issues/215) | smart_cmds.rs:415 detach_psd drops SoLd; psd_export.rs:276 re-emits imported raw only | +| FILE-215-2 | Smart Filters survive PSD save and re-open (filterFX descriptor read and written) | missing | [#215](https://github.com/storytold/photocraft/issues/215) | psd_import.rs:259 smart_filters: Vec::new(); no filterFX in crates | +| FILE-215-3 | Filter mask can be painted directly, with undo | missing | [#215](https://github.com/storytold/photocraft/issues/215) | smart_cmds.rs:382 mask only from the selection | +| FILE-215-4 | Adjustments (Hue/Sat + Colorize, Curves, Levels, Shadows/Highlights) apply as Smart Filters | missing | [#215](https://github.com/storytold/photocraft/issues/215) | add_smart_filter callers: filters, lens, distort, plug-ins only | +| FILE-215-5 | Smart Filter rows: collapse, mask thumb, blend icon, drag reorder, context menu, live preview | partial | [#215](https://github.com/storytold/photocraft/issues/215) | smart_ui.rs:31 rows + visibility + editor; no collapse, drag or context menu | +| FILE-215-6 | Smart Object corner skew and perspective (quad or homography transform) | missing | [#215](https://github.com/storytold/photocraft/issues/215) | doc lib.rs:312 SmartObject.transform is an Affine | +| FILE-215-7 | Layered TIFF read and write (tag 37724), honouring the ask-before-saving pref | missing | [#215](https://github.com/storytold/photocraft/issues/215) | No 37724 / ImageSourceData in crates | +| FILE-215-8 | Large and BigTIFF import with banded decode across all IFDs | missing | [#215](https://github.com/storytold/photocraft/issues/215) | codecs tiff.rs:3 'Only the first IFD is read' | +| FILE-215-9 | SVG opens and places as shape layers or a vector Smart Object | missing | [#215](https://github.com/storytold/photocraft/issues/215) | No SVG codec or import | +| FILE-215-10 | Multi-page PDF save, PDF presentation, layer comps to PDF, vector/text output, PDF import | missing | [#215](https://github.com/storytold/photocraft/issues/215) | Only print_cmds.rs:133 print_pdf (raster print page) | +| FILE-215-11 | RAW: lossy DNG, highlight modes, .xmp sidecars, RAW prefs wired, per-camera corpus | missing | [#215](https://github.com/storytold/photocraft/issues/215) | raw sensor.rs:210 lossy DNG unsupported; raw_defaults unread; no xmp | +| FILE-215-12 | Camera Raw dialog shows a histogram; settings persist as a PSD Smart Filter | partial | [#215](https://github.com/storytold/photocraft/issues/215) | No histogram; lens_cmds.rs:268 stores an in-app Smart Filter, PSD drops it | +| FILE-215-13 | Video layer frames persist in .pcraft; PSD video layers mapped | partial | [#215](https://github.com/storytold/photocraft/issues/215) | format convert.rs:89 + tests/roundtrip.rs:282 persist frames; no PSD video mapping | +| FILE-215-14 | User swatch model with ACO/ASE import and export | missing | [#215](https://github.com/storytold/photocraft/issues/215) | panels.rs:954 const SWATCHES: [[u8; 3]; 40] | +| FILE-215-15 | Layer Style contour picker, curve editor, presets, .shc import/export | missing | [#215](https://github.com/storytold/photocraft/issues/215) | doc effects.rs:57 Contour model only | +| FILE-215-16 | Advanced Blending (knockout, blend as group, mask hides effects) end to end | missing | [#215](https://github.com/storytold/photocraft/issues/215) | No knockout / blend_interior / transparency_shapes in doc, compose or io | +| FILE-215-17 | CMYK/RGB documents preview spot inks; spot plates export (TIFF extra samples, PDF) | missing | [#215](https://github.com/storytold/photocraft/issues/215) | compose multichannel.rs:1 spot preview for Multichannel only | +| FILE-215-18 | Save for Web per-slice settings remembered; Export As remembers and offers nearest scaling | partial | [#215](https://github.com/storytold/photocraft/issues/215) | file_ui.rs:188 last_web in-session only; export_dialog.rs:16 fixed defaults | +| FILE-215-19 | Save As JPEG opens a JPEG Options dialog (quality 0-12, format options) | missing | [#215](https://github.com/storytold/photocraft/issues/215) | ExportSettings has jpeg_quality 1-100 only (lib.rs:118) | +| FILE-215-20 | Artboard tool with handles and + buttons, Duplicate Artboard beside, vector PDF pages | partial | [#215](https://github.com/storytold/photocraft/issues/215) | artboard_ui.rs:1 names/properties only; no tool, + buttons or duplicate | +| FILE-215-21 | Local libraries of colours, patterns, images and layers, as linked items or copies | missing | [#215](https://github.com/storytold/photocraft/issues/215) | Only preset libraries (preset_store.rs) | +| FILE-216-1 | Every corpus fetched and enforced in CI, cached by pin | missing | [#216](https://github.com/storytold/photocraft/issues/216) | No corpus step in ci.yml; in progress in #190 | +| FILE-216-2 | Large-file fixtures: 20 MP, 150-layer, generated 2 GB PSD and 4-5 GB PSB stress tests | missing | [#216](https://github.com/storytold/photocraft/issues/216) | Only banded.rs covers a >8 MP synthetic export | +| FILE-216-3 | TIFF, RAW and EXR corpora read by tests; per-camera RAW coverage reported | missing | [#216](https://github.com/storytold/photocraft/issues/216) | corpus/tiff, raw and exr only listed by xtask corpus; no test reads them | +| FILE-216-4 | Exports re-open in a third-party oracle editor | missing | [#216](https://github.com/storytold/photocraft/issues/216) | The round trip is our export -> our import | +| FILE-216-5 | Save-path tests: layered TIFF, PDF, SVG, ACO/ASE, .shc, Smart Filters, atomic-save failure | missing | [#216](https://github.com/storytold/photocraft/issues/216) | Features absent; no atomic-save failure-injection test | +| FILE-216-6 | 30-50 end-to-end workflow acceptance tasks driven through MCP or commands | partial | [#216](https://github.com/storytold/photocraft/issues/216) | automation/tests/agent_tasks.rs: 10 MCP tasks | + +## Tools + +| Id | Target | Status | Issue | Note | +|---|---|---|---|---| +| TOOL-213-1 | Pencil tool: aliased 1 px painting, square pixel-aligned cursor | partial | [#213](https://github.com/storytold/photocraft/issues/213) | paint.pencil exists (brush_cmds.rs:322) with an aliasing test; no Tool::Pencil (state.rs:7) | +| TOOL-213-2 | Mixer Brush tool on the toolbar | partial | [#213](https://github.com/storytold/photocraft/issues/213) | paint.mixerBrush (brush_cmds.rs:337) + panel options; no Tool variant | +| TOOL-213-3 | Patch tool with a live preview of the patched area | missing | [#213](https://github.com/storytold/photocraft/issues/213) | No patch command or tool | +| TOOL-213-4 | Content-Aware Move tool reusing inpaint completion | missing | [#213](https://github.com/storytold/photocraft/issues/213) | inpaint::complete only used by spot healing (retouch_cmds.rs:386) | +| TOOL-213-5 | Painting a type/shape/smart/fill layer offers to rasterize, then continues the stroke | missing | [#213](https://github.com/storytold/photocraft/issues/213) | Rasterize only in the layer menus (layer_menu_ui.rs:40) | +| TOOL-213-6 | Fill dialog: contents, blend, opacity, preserve transparency; remembered; Shift+Backspace | partial | [#213](https://github.com/storytold/photocraft/issues/213) | edit.fill: color or pattern + opacity, Shift+F5 (commands.rs:263-267); no blend/history/CA/persist | +| TOOL-213-7 | Live stroke preview for every brush-based retouch tool | missing | [#213](https://github.com/storytold/photocraft/issues/213) | Live stroke only for Brush and Eraser (canvas.rs:1684) | +| TOOL-213-8 | Clone Source: W/H link, drawn overlay (opacity/blend/clip/auto-hide), Alt target cursor | partial | [#213](https://github.com/storytold/photocraft/issues/213) | Panel W/H + overlay settings (preset_panels.rs:946-1015); no link, overlay never drawn | +| TOOL-213-9 | Eyedropper comparison ring (new colour above, current below) while sampling | missing | [#213](https://github.com/storytold/photocraft/issues/213) | sample_eyedropper (canvas.rs:1552) draws no ring | +| TOOL-213-10 | Ctrl+Alt drag resizes brush size/hardness with an on-canvas readout | missing | [#213](https://github.com/storytold/photocraft/issues/213) | Only [ ] resize (shortcuts.rs:270) | +| TOOL-213-11 | Editable pressure curve, global and per control | missing | [#213](https://github.com/storytold/photocraft/issues/213) | No pressure curve in engine, paint or ui-egui | +| TOOL-213-12 | Import an image file as a brush tip | missing | [#213](https://github.com/storytold/photocraft/issues/213) | Only brush.defineFromSelection and brush.presets.importAbr | +| TOOL-213-13 | Brush regression tests: no seams, hard edge, zero-scatter defaults, ABR never black | partial | [#213](https://github.com/storytold/photocraft/issues/213) | Missing-pattern ABR test (abr_map/tests.rs:181), hardness test; no seam/black tests | +| TOOL-213-14 | Tablet: no OS arrow cursor over the canvas on macOS, Windows and Linux | partial | [#213](https://github.com/storytold/photocraft/issues/213) | Brush tools set CursorIcon::None (canvas.rs:1259-1275); not verified per OS | +| TOOL-214-1 | Select and Mask workspace: view modes, Refine Edge brush, 6 outputs | partial | [#214](https://github.com/storytold/photocraft/issues/214) | Opens the generic select.refineEdge dialog (menus.rs:308) | +| TOOL-214-2 | Marquee: Shift square, Alt from centre, live W x H readout | partial | [#214](https://github.com/storytold/photocraft/issues/214) | marquee_end supports shift (chrome_ui.rs:167) but canvas.rs passes false (#208) | +| TOOL-214-3 | Color Range test: Sampled Colors at Fuzziness 0 selects the exact match only | partial | [#214](https://github.com/storytold/photocraft/issues/214) | Fuzziness-0 tests cover midtones/shadows only (selection_cmds.rs:444-447) | +| TOOL-214-4 | Move tool: box-select layers, Auto-Select hover outline, double-click fit, drag to a tab | missing | [#214](https://github.com/storytold/photocraft/issues/214) | move_mods.rs covers Shift/Alt/arrows only | +| TOOL-214-5 | Right-click menu during Free Transform (scale, rotate, skew, warp, flip, rotate 90/180) | missing | [#214](https://github.com/storytold/photocraft/issues/214) | No context menu in canvas.rs or transform_tool.rs | +| TOOL-214-6 | Puppet Warp mesh follows the object outline (constrained triangulation) | missing | [#214](https://github.com/storytold/photocraft/issues/214) | algo/puppet.rs:136 builds a block-grid mesh | +| TOOL-214-7 | Content-Aware Scale through the Free Transform box with a live proxy preview | partial | [#214](https://github.com/storytold/photocraft/issues/214) | edit.contentAwareScale (edit_menu_cmds.rs:899) behind a parameter dialog | +| TOOL-214-8 | A measured ruler angle prefills Image Rotation > Arbitrary (0.01 deg) | missing | [#214](https://github.com/storytold/photocraft/issues/214) | Arbitrary rotation is a generic dialog; ruler only feeds straighten | +| TOOL-214-9 | Crop: W x H x Resolution with units, saved crop presets, Straighten in the options bar | partial | [#214](https://github.com/storytold/photocraft/issues/214) | Ratio dropdown and fields only (panels.rs:551-579) | +| TOOL-214-10 | Pen: Alt breaks a handle, Shift 45 deg, Ctrl direct-selects, Backspace, close-drag handle | missing | [#214](https://github.com/storytold/photocraft/issues/214) | pen_down gets no modifiers (canvas.rs:1632) | +| TOOL-214-11 | Stroke Path dialog: pick any brush-based tool, Simulate Pressure | partial | [#214](https://github.com/storytold/photocraft/issues/214) | path.stroke brush, pencil, eraser (vector_cmds.rs:1170); brush-only button, no dialog | +| TOOL-214-12 | Shapes: on-canvas corner radius handles, shape feather, selection to shape with tolerance | partial | [#214](https://github.com/storytold/photocraft/issues/214) | select.convertToShape with tolerance (vector_cmds.rs:1256); radius as a field only | +| TOOL-214-13 | Mask Properties: Density and Feather controls | partial | [#214](https://github.com/storytold/photocraft/issues/214) | Vector-mask density/feather in the engine (vector_cmds.rs:948); no UI controls | +| TOOL-214-14 | Vanishing Point window: draw and edit planes, paste, clone in perspective | partial | [#214](https://github.com/storytold/photocraft/issues/214) | filter.vanishingPoint (vp_cmds.rs:15) behind a parameter dialog | +| TOOL-214-15 | Content-Aware Fill workspace: paint the sampling area, preview, rotate/scale/mirror, new layer | partial | [#214](https://github.com/storytold/photocraft/issues/214) | Engine sampling auto, rect, custom (edit_menu_cmds.rs:324); generic dialog | +| TOOL-214-16 | Liquify: redo (Cmd+Shift+Z) | missing | [#214](https://github.com/storytold/photocraft/issues/214) | liquify_ui.rs has undo only | + +## Workspace and UI + +**Settings that do nothing: 66 of 129** (target 0, #204). Counted by the prefs audit (`xtask/src/scorecard.rs`): a `Preferences` field counts as read when code outside the prefs model (`prefs.rs` except its `impl Session` blocks; tests excluded; the schema-driven dialog never names a field) accesses `section.field`, or `.section` and `.field` in the same file, or calls a section method that reads `self.field`. It is a heuristic lower bound: a generic name can look read when it isn't. + +Unread: `general.color_picker`, `general.beep_when_done`, `general.export_clipboard`, `general.resize_image_during_place`, `general.always_create_smart_objects_when_placing`, `general.animated_zoom`, `general.zoom_resizes_windows`, `general.use_legacy_free_transform`, `general.auto_show_home_screen`, `interface.ui_font_size`, `interface.show_channels_in_color`, `interface.dynamic_color_sliders`, `interface.show_tooltips`, `workspace.auto_collapse_icon_panels`, `workspace.auto_show_hidden_panels`, `workspace.open_documents_as_tabs`, `workspace.enable_floating_document_window_docking`, `workspace.large_tabs`, `workspace.enable_narrow_options_bar`, `tools.show_tooltips`, `tools.zoom_clicked_point_to_center`, `tools.enable_flick_panning`, `tools.vary_round_brush_hardness_on_hud`, `tools.show_transformation_values`, `tools.overscroll`, `tools.double_click_layer_mask_launches_select_and_mask`, `file_handling.image_previews`, `file_handling.lowercase_extension`, `file_handling.save_in_background`, `file_handling.ignore_exif_profile_tag`, `file_handling.ask_before_saving_layered_tiff`, `file_handling.maximize_psd_compatibility`, `file_handling.recent_file_count`, `file_handling.recent_files`, `performance.cache_levels`, `performance.legacy_compositing`, `scratch_disks.disks`, `cursors.brush_preview_color`, `units_and_rulers.type_units`, `units_and_rulers.column_width`, `units_and_rulers.gutter`, `units_and_rulers.print_resolution`, `units_and_rulers.screen_resolution`, `plug_ins.show_extension_panels`, `plug_ins.allow_scripts_to_connect`, `plug_ins.generator_enabled`, `type_.smart_quotes`, `type_.missing_glyph_protection`, `type_.show_font_names_in_english`, `type_.use_esc_to_commit`, `type_.text_engine`, `type_.font_preview`, `type_.fill_new_type_layers_with_placeholder`, `type_.recent_fonts`, `enhanced_controls.scrubby_slider_acceleration`, `enhanced_controls.touch_gestures`, `enhanced_controls.zoom_with_trackpad_pinch`, `enhanced_controls.rotate_view_with_trackpad`, `raw_defaults.color_space`, `raw_defaults.bit_depth`, `raw_defaults.resolution`, `raw_defaults.sharpen_for`, `raw_defaults.open_as_smart_object`, `raw_defaults.apply_auto_tone`, `integrations.allow_agent_control`, `integrations.control_port`. + +| Id | Target | Status | Issue | Note | +|---|---|---|---|---| +| UI-217-1 | Tool options, last Export As / Save for Web / Fill settings and font recents persist | missing | [#217](https://github.com/storytold/photocraft/issues/217) | No tool_options/export fields in Preferences (prefs.rs:694); export_dialog.rs:21 png/85 | +| UI-217-2 | Window size and dock width remembered across restarts | missing | [#217](https://github.com/storytold/photocraft/issues/217) | main.rs:108 fixed 1440x900; dock.rs:529 snapshot omits the width | +| UI-217-3 | File > Open starts in the last-used folder | missing | [#217](https://github.com/storytold/photocraft/issues/217) | services.rs:110 FileDialog::new() without set_directory | +| UI-217-4 | Preferences all take effect; recent files persist and honour recent_file_count (#204) | missing | [#217](https://github.com/storytold/photocraft/issues/217) | See the prefs audit and BUG-204-* | +| UI-217-5 | Panel groups can float (menu + drag out of the dock) and are restored at launch | missing | [#217](https://github.com/storytold/photocraft/issues/217) | No floating groups in dock.rs/panels.rs | +| UI-217-6 | Document tabs: overflow list, context menu (Close/Others/All/Reveal), drag to reorder | missing | [#217](https://github.com/storytold/photocraft/issues/217) | canvas.rs:674-770 tabs click/close only | +| UI-217-7 | Command palette shows recently run commands on an empty query | missing | [#217](https://github.com/storytold/photocraft/issues/217) | palette.rs:83-108 fuzzy hits only | +| UI-217-8 | Tool-specific canvas right-click menu; right-click spelling suggestions in text fields | partial | [#217](https://github.com/storytold/photocraft/issues/217) | paint_mouse.rs:67 right-click opens the brush picker only | +| UI-217-9 | UI scale presets 100/115/125/150/200 % through zoom_factor, remembered | missing | [#217](https://github.com/storytold/photocraft/issues/217) | prefs.rs:61 UiScale unused; input_tests.rs:71 asserts zoom_factor stays 1.0 | +| UI-217-10 | Appearance: animation toggle, canvas pattern toggle per theme, start-screen recents | missing | [#217](https://github.com/storytold/photocraft/issues/217) | No such prefs; start screen (canvas.rs:825) has no recents | +| UI-217-11 | Soft dark and soft light themes with picker previews; outlined knobs in light themes | missing | [#217](https://github.com/storytold/photocraft/issues/217) | theme.rs:28 five themes | +| UI-217-12 | Save custom New Document presets; a real Recent category | missing | [#217](https://github.com/storytold/photocraft/issues/217) | new_doc_ui.rs:17 'Recent' is a static list | +| UI-217-13 | Numeric fields accept arithmetic and units; scrubby labels; double-click resets a slider | partial | [#217](https://github.com/storytold/photocraft/issues/217) | widgets.rs:131 DragValue scrubs; no expressions, label scrub or reset | +| UI-217-14 | Layers panel: wheel steps blend modes, drop on New/Trash duplicates/deletes, checker behind art | missing | [#217](https://github.com/storytold/photocraft/issues/217) | panels.rs:1108 plain dropdown; trash is click-only (panels.rs:1213) | +| UI-217-15 | Curves: smoothing / pencil mode, black/grey/white eyedroppers, tooltips | missing | [#217](https://github.com/storytold/photocraft/issues/217) | adjust_editors.rs:397 has none of them | +| UI-217-16 | Hue/Saturation: Saturation and Lightness sliders get hue-following colour tracks | missing | [#217](https://github.com/storytold/photocraft/issues/217) | Only the Hue slider has a gradient | +| UI-217-17 | Color panel: wheel plus RGB/HSB/CMYK/Lab/Gray/Web slider modes; resizes with its group | missing | [#217](https://github.com/storytold/photocraft/issues/217) | panels.rs:1837 single field + hex/RGB readout | +| UI-217-18 | Alt+wheel zooms in gentle ~5 % steps; zoom controls uncramped | missing | [#217](https://github.com/storytold/photocraft/issues/217) | canvas.rs:1107-1125 no Alt handling | +| UI-217-19 | Shortcut hints rendered per platform; Reveal label per OS | missing | [#217](https://github.com/storytold/photocraft/issues/217) | ~33 hard-coded modifier glyph strings in ui-egui; menu_catalog.rs:341 one label | +| UI-217-20 | 'Show rulers in new documents' preference; guides stored as f64 | missing | [#217](https://github.com/storytold/photocraft/issues/217) | No rulers pref; doc lib.rs:523 guides are Vec | +| UI-217-21 | One Preset Manager for every preset kind with folders, search, reorder, load/save sets | partial | [#217](https://github.com/storytold/photocraft/issues/217) | edit_menu_cmds.rs:714 covers brushes, custom shapes, patterns with reorder | +| UI-217-22 | Stamp Visible into a new layer keeping originals; Alt+Merge keeps originals | missing | [#217](https://github.com/storytold/photocraft/issues/217) | Only layer.mergeVisible (edit_cmds.rs:416) | +| UI-217-23 | Layer clipboard: copy/paste whole layers (masks, styles, type) between documents | missing | [#217](https://github.com/storytold/photocraft/issues/217) | Only copy/paste layer style (extra_cmds.rs:592) | + +## Type + +| Id | Target | Status | Issue | Note | +|---|---|---|---|---| +| TYPE-218-1 | Type on a path: path shape, glyphs along the curve, caret follows, PSD round trip | missing | [#218](https://github.com/storytold/photocraft/issues/218) | doc text.rs:209 TextShape has Point and Box only | +| TYPE-218-2 | Paragraph direction (Auto/LTR/RTL) in the panel and options bar; visual RTL caret | missing | [#218](https://github.com/storytold/photocraft/issues/218) | Engine has TextDirection (text.rs:155); no UI | +| TYPE-218-3 | Thai/Khmer/Indic fallback fonts; golden shaping tests for complex scripts and RTL | missing | [#218](https://github.com/storytold/photocraft/issues/218) | fonts.rs:28 fallback covers Arabic/Hebrew; no per-script golden tests (only a bidi test) | +| TYPE-218-4 | Variable font axis sliders from the font's axes; opsz follows the point size | partial | [#218](https://github.com/storytold/photocraft/issues/218) | type_cmds.rs:184 accepts variations; FaceInfo.axes unused in the UI | +| TYPE-218-5 | OpenType UI: Character panel buttons, Numerals and Stylistic Sets submenus | partial | [#218](https://github.com/storytold/photocraft/issues/218) | Type > OpenType toggles (type_extra_cmds.rs:512); no panel buttons or submenus | +| TYPE-218-6 | Manual pair kerning (Alt+arrows), optical kerning rendered, both round-trip PSD (#206) | missing | [#218](https://github.com/storytold/photocraft/issues/218) | Optical renders as Metrics (text.rs:21); Alt+arrow moves by word | +| TYPE-218-7 | Warp Text dialog previews live and opens with the layer's current warp | missing | [#218](https://github.com/storytold/photocraft/issues/218) | view_cmds.rs:834 always seeds arc, bend 50 | +| TYPE-218-8 | Text editing: select all, double/triple/quadruple-click selection, paste, with tests | partial | [#218](https://github.com/storytold/photocraft/issues/218) | Cmd+A, paste and double-click exist; no triple/quadruple click, no tests | +| TYPE-218-9 | Offer 'Rasterize and continue?' for Distort/Perspective/Warp on live type | missing | [#218](https://github.com/storytold/photocraft/issues/218) | No such prompt | + +## Automation + +Menu wiring (`docs/parity.md`): 626 of 626 menu items live (100.0%). This counts dispatch, not behaviour. + +| Id | Target | Status | Issue | Note | +|---|---|---|---|---| +| AUTO-1 | Every menu item dispatches a command id (UI, CLI, control channel and MCP share them) | done | | docs/parity.md: all menu items live; menus.rs is_live | +| AUTO-219-1 | Record actions in a document-independent form (implicit layer, merged type keystrokes) | missing | [#219](https://github.com/storytold/photocraft/issues/219) | actions.rs:47 copies raw journal entries with explicit layer ids | +| AUTO-219-2 | Actions panel: step and dialog toggles, reorder, delete a step, Stop, sets, .atn import | missing | [#219](https://github.com/storytold/photocraft/issues/219) | Panel deletes whole actions only (actions.rs:146) | +| AUTO-219-3 | actions.record/stop/play/list commands for agents and the CLI | missing | [#219](https://github.com/storytold/photocraft/issues/219) | Recording is UI-only (actions.rs:37-60) | +| AUTO-219-4 | Replay tests for crop, filters, selections and styled text on a second document | missing | [#219](https://github.com/storytold/photocraft/issues/219) | One replay test (actions.rs:171: new layer, rect select, fill) | +| AUTO-219-5 | Batch: subfolders, open documents, naming template, progress/cancel, error log, stop | partial | [#219](https://github.com/storytold/photocraft/issues/219) | file_cmds.rs:590-631 folder/array input, per-file errors; no recursion, template or cancel | +| AUTO-219-6 | Plug-in ABI v2: panel kind, progress/cancel calls, read other layers, Last Filter repeats | missing | [#219](https://github.com/storytold/photocraft/issues/219) | runtime.rs:15 ABI_VERSION = 1; manifest.rs:18 Kind::Filter only | +| AUTO-219-7 | Local ONNX models behind a flag, managed in Preferences; subject select, background removal | missing | [#219](https://github.com/storytold/photocraft/issues/219) | No ONNX runtime dependency | +| AUTO-219-8 | xtask check that every panel button routes through a command id | missing | [#219](https://github.com/storytold/photocraft/issues/219) | No such xtask check | +| AUTO-219-9 | Headless commands for in-canvas type editing, Liquify/Puppet strokes, transform handles | partial | [#219](https://github.com/storytold/photocraft/issues/219) | filter.liquify strokes, edit.puppetWarp pins, type.edit, edit.transform; no caret API | +| AUTO-219-10 | Agent acceptance suite grown to 30-50 workflows | partial | [#219](https://github.com/storytold/photocraft/issues/219) | automation/tests/agent_tasks.rs runs 10 workflows over MCP | + +## Reliability + +Never-crash attribute on 25 of 25 crate roots. + +| Id | Target | Status | Issue | Note | +|---|---|---|---|---| +| REL-1 | panic_hunt fuzzes every command with adversarial params | done | | engine/tests/panic_hunt.rs: every command, own thread, 4 s timeout; #[ignore], run in the gate | +| REL-2 | Every workspace crate carries the never-crash deny(clippy::unwrap_used, ...) attribute | done | | Counted automatically above (crate roots under crates/ and apps/); testkit gained it in #221 | +| REL-3 | The app catches escaped panics around command dispatch and import/export | partial | | engine lib.rs:336 dispatch; crash_guard.rs:21 import/export; the dispatch guard is untested | +| REL-4 | Atomic save (temp, fsync, rename) for every document write | missing | [#203](https://github.com/storytold/photocraft/issues/203) | Documents use fs::write in place; prefs/presets/bundles use tmp + rename without fsync | +| REL-212-1 | Fallible allocation for paste/erase/fill/transform/save/import, clear out-of-memory error | missing | [#212](https://github.com/storytold/photocraft/issues/212) | try_reserve only in io/flat.rs:221; codec max_alloc limits cover decode only | +| REL-212-2 | Export switches to PSB when the estimated file size exceeds 2 GB | missing | [#212](https://github.com/storytold/photocraft/issues/212) | io/psd_export.rs:558 switches only when a side exceeds 30000 px | +| REL-212-3 | Tests: an allocation-limit injection makes each command Err with the document unchanged | missing | [#212](https://github.com/storytold/photocraft/issues/212) | Only codec/raw max_alloc tests (codecs/tests/limits.rs:48) | + +## Distribution + +| Id | Target | Status | Issue | Note | +|---|---|---|---|---| +| DIST-1 | macOS universal build (aarch64 + x86_64), signed and notarized | done | | release.yml:66,75,93 (macos-15); 0.2.0 shipped notarized | +| DIST-2 | Windows x64 and x86 builds | partial | | release.yml:113-114; signing skipped with a warning until the signing material exists | +| DIST-3 | Linux x86_64 and aarch64: AppImage, deb, rpm, tar.gz | done | | release.yml:162-163,186 | +| DIST-4 | Flatpak single-file bundles, x86_64 and aarch64 | done | | release.yml:204-216 (#163) | +| DIST-5 | Web build (wasm32, trunk) | done | | release.yml:255 | +| DIST-220-1 | Help > Check for Updates plus an opt-out periodic release-feed check on all desktops | missing | [#220](https://github.com/storytold/photocraft/issues/220) | No update check in crates | +| DIST-220-2 | Windows ARM64 (aarch64-pc-windows-msvc) in the release matrix, signed like x64 | missing | [#220](https://github.com/storytold/photocraft/issues/220) | release.yml:113-114 only x86_64 and i686 msvc | +| DIST-220-3 | In-app bug report with reviewable diagnostics that opens a prefilled issue | missing | [#220](https://github.com/storytold/photocraft/issues/220) | No bug report dialog in crates | +| DIST-220-4 | Translation coverage per language reported by xtask; RTL layout for Arabic and Hebrew | missing | [#220](https://github.com/storytold/photocraft/issues/220) | No i18n in crates or xtask (waits on #169) | +| DIST-220-5 | Linux AppImage x86_64 + aarch64 ship; Flatpak kept; Flathub later (#173) | partial | [#220](https://github.com/storytold/photocraft/issues/220) | AppImage and Flatpak ship (release.yml:162-216); Flathub not done | + +## Open bugs + +| Id | Target | Status | Issue | Note | +|---|---|---|---|---| +| BUG-203-1 | Every document write goes through one atomic_write (temp, fsync, rename) | missing | [#203](https://github.com/storytold/photocraft/issues/203) | services.rs:125 std::fs::write; file_cmds.rs:91 fs::write; automation/files.rs:67 in place | +| BUG-203-2 | Temp file in the same dir, sync_all, rename over target, fsync dir on Unix | missing | [#203](https://github.com/storytold/photocraft/issues/203) | No sync_all anywhere; services.rs:71 write_atomic (prefs only) does tmp + rename, no fsync | +| BUG-203-3 | Windows: retry the rename with backoff on PermissionDenied | missing | [#203](https://github.com/storytold/photocraft/issues/203) | No retry around fs::rename (services.rs:77, preset_store.rs:189, format/store.rs:457) | +| BUG-203-4 | Unwritable directory errors clearly, never falls back to writing in place | missing | [#203](https://github.com/storytold/photocraft/issues/203) | No atomic path exists for documents; all document writes are in place | +| BUG-203-5 | On any failure the original file is untouched and the temp file removed | missing | [#203](https://github.com/storytold/photocraft/issues/203) | services.rs:76 write_atomic leaves .tmp behind if rename fails; documents write in place | +| BUG-203-6 | Failure-injection tests: mid-write, rename and read-only dir errors keep original bytes | missing | [#203](https://github.com/storytold/photocraft/issues/203) | No such tests under crates/ or apps/ | +| BUG-204-1 | Interface prefs (UI scale, font size, channels in colour, sliders, tooltips) wired or hidden | missing | [#204](https://github.com/storytold/photocraft/issues/204) | prefs.rs:196-202; no reader outside prefs.rs (see the prefs audit) | +| BUG-204-2 | workspace.large_tabs, animated_zoom, auto_show_home_screen wired or hidden | missing | [#204](https://github.com/storytold/photocraft/issues/204) | prefs.rs:163,166,229; no reader outside prefs.rs | +| BUG-204-3 | RAW prefs and ask_before_saving_layered_tiff read, or hidden | missing | [#204](https://github.com/storytold/photocraft/issues/204) | raw_defaults (prefs.rs:712), ask_before_saving_layered_tiff (prefs.rs:312) have no readers | +| BUG-204-4 | Recent files persisted through prefs across restarts | missing | [#204](https://github.com/storytold/photocraft/issues/204) | ui-egui lib.rs:461 push_recent only touches app.ui.recent_files | +| BUG-204-5 | Recent-files list honours recent_file_count (default 20) | missing | [#204](https://github.com/storytold/photocraft/issues/204) | lib.rs:465 truncate(10) hard-coded; menus.rs:1065 test asserts 'capped at 10' | +| BUG-204-6 | Test: every Prefs field is read somewhere or on a hidden allowlist | partial | [#204](https://github.com/storytold/photocraft/issues/204) | The scorecard's prefs audit counts unread fields; no engine test or allowlist yet | +| BUG-205-1 | Free Transform with a selection moves only the selected part of a linked mask | missing | [#205](https://github.com/storytold/photocraft/issues/205) | transform_cmds.rs:127-130 warps the whole linked mask even when the selection split pixels | +| BUG-205-2 | Live transform preview draws the layer with its mask applied | missing | [#205](https://github.com/storytold/photocraft/issues/205) | transform_tool.rs:173 preview_image uses the raw surface | +| BUG-205-3 | edit.transform honours target mask: transforms only a targeted, unlinked mask | missing | [#205](https://github.com/storytold/photocraft/issues/205) | transform_cmds.rs:214-270 never reads a target param | +| BUG-205-4 | Test: a linked raster mask with no selection moves with the pixels | partial | [#205](https://github.com/storytold/photocraft/issues/205) | Implemented at transform_cmds.rs:127; no raster-mask test | +| BUG-205-5 | Tests: linked mask + selection stays aligned; unlinked stays; targeted mask only | missing | [#205](https://github.com/storytold/photocraft/issues/205) | Unlinked case honoured (m.linked check, l.128) but untested; the other two not implemented | +| BUG-206-1 | CharStyle models per-character manual kerning separately from the auto-kern mode | missing | [#206](https://github.com/storytold/photocraft/issues/206) | doc/text.rs:76 only `kerning: Kerning` enum (Metrics/Optical/Off) | +| BUG-206-2 | PSD import keeps Optical and manual Kerning values | missing | [#206](https://github.com/storytold/photocraft/issues/206) | text/psd.rs:264 maps anything not Off to Metrics; Kerning value ignored | +| BUG-206-3 | PSD export writes the real AutoKerning mode and Kerning values | missing | [#206](https://github.com/storytold/photocraft/issues/206) | text/psd.rs:380-381 AutoKerning bool only; Kerning hard-coded 0 | +| BUG-206-4 | Alt+Left/Right at a collapsed caret adjusts pair kerning; Character panel field | missing | [#206](https://github.com/storytold/photocraft/issues/206) | type_tool.rs:896-912 only a mode combo | +| BUG-206-5 | Tests: PSD round trip of manual and Optical kerning, a real file in the oracle corpus | missing | [#206](https://github.com/storytold/photocraft/issues/206) | No kerning round-trip test | +| BUG-207-1 | Retouch tools respect the paint target (mask, alpha channel, Quick Mask) | missing | [#207](https://github.com/storytold/photocraft/issues/207) | retouch_cmds.rs:83 run_stroke always uses l.surface_mut(); no paint_target() call | +| BUG-207-2 | Sample All Layers applies to Blur, Sharpen and Smudge | missing | [#207](https://github.com/storytold/photocraft/issues/207) | Checkbox at retouch_ui.rs:171-190 is not passed to the stroke commands | +| BUG-207-3 | Smudge mixes premultiplied RGBA (no dark fringe at transparent edges) | missing | [#207](https://github.com/storytold/photocraft/issues/207) | No premultiply in paint/src/retouch.rs | +| BUG-207-4 | edit.transform transforms a targeted alpha channel | missing | [#207](https://github.com/storytold/photocraft/issues/207) | transform_cmds.rs transform() has no target handling | +| BUG-207-5 | Tests: per-tool mask/channel targeting, Sample All Layers, no smudge fringe | missing | [#207](https://github.com/storytold/photocraft/issues/207) | Only smudge_drags_colour_along_the_stroke (retouch.rs:407) | +| BUG-208-1 | Shift while dragging a Rect/Elliptical Marquee constrains to a square/circle | missing | [#208](https://github.com/storytold/photocraft/issues/208) | canvas.rs:1498 and :1769 pass shift=false to marquee_end | +| BUG-208-2 | Alt while dragging a marquee draws from the centre | missing | [#208](https://github.com/storytold/photocraft/issues/208) | marquee_end (chrome_ui.rs:157) has no centre mode | +| BUG-208-3 | W x H px readout beside the cursor while dragging a marquee | missing | [#208](https://github.com/storytold/photocraft/issues/208) | No size readout in canvas.rs or tool_feedback.rs | +| BUG-208-4 | Add/subtract/intersect from modifiers pressed before the drag, kept by a test | partial | [#208](https://github.com/storytold/photocraft/issues/208) | Mods captured at press (canvas.rs:1679, used :1779); only a badge test (tool_feedback.rs:195) | +| BUG-208-5 | Kittest: Shift drag gives a square, Alt drag is centred, readout matches the size | missing | [#208](https://github.com/storytold/photocraft/issues/208) | No marquee drag tests | diff --git a/perf/baseline.json b/perf/baseline.json new file mode 100644 index 00000000..d964da06 --- /dev/null +++ b/perf/baseline.json @@ -0,0 +1,299 @@ +{ + "machine": { + "arch": "aarch64", + "class": "macos-aarch64-apple-m4-pro-apple-m4-pro", + "cores": 14, + "cpu": "Apple M4 Pro", + "gpu_adapter": "Apple M4 Pro", + "os": "macos", + "os_version": "macOS 15.6", + "ram_bytes": 51539607552 + }, + "machine_class": "macos-aarch64-apple-m4-pro-apple-m4-pro", + "modes": { + "full": { + "commit": "f5d2bcb54abd0f45e50f5822e869ea5da9b06b91", + "date": "2026-10-05T14:17:51Z", + "load_avg_end": [ + 289.86, + 276.99, + 270.86 + ], + "load_avg_start": [ + 224.9, + 226.31, + 196.03 + ], + "scenarios": { + "P1": { + "gpu_bytes": 429194378, + "max_ms": 535.0277920000001, + "n": 15, + "p50_ms": 321.068333, + "p95_ms": 535.0277920000001, + "peak_rss_bytes": 865124352 + }, + "P10": { + "gpu_bytes": 1498741898, + "max_ms": 3135.981125, + "n": 5, + "p50_ms": 1232.297418, + "p95_ms": 3135.981125, + "peak_rss_bytes": 1390755840 + }, + "P11": { + "gpu_bytes": 1498741898, + "max_ms": 1174.1605009999998, + "n": 5, + "p50_ms": 445.372791, + "p95_ms": 1174.1605009999998, + "peak_rss_bytes": 1005027328 + }, + "P13": { + "gpu_bytes": 2115958908, + "max_ms": 145.70308400000002, + "n": 15, + "p50_ms": 6.421208999999999, + "p95_ms": 145.70308400000002, + "peak_rss_bytes": 880541696 + }, + "P14": { + "gpu_bytes": 2115958908, + "max_ms": 4318.054, + "n": 15, + "p50_ms": 2698.3841249999996, + "p95_ms": 4318.054, + "peak_rss_bytes": 1557413888 + }, + "P16": { + "gpu_bytes": 1750668536, + "max_ms": 160.640333, + "n": 15, + "p50_ms": 21.536042, + "p95_ms": 160.640333, + "peak_rss_bytes": 831586304 + }, + "P17": { + "gpu_bytes": 1750668536, + "max_ms": 681.0902500000001, + "n": 15, + "p50_ms": 385.865959, + "p95_ms": 681.0902500000001, + "peak_rss_bytes": 1300987904 + }, + "P18": { + "gpu_bytes": 2042852008, + "max_ms": 2794.2514169999995, + "n": 5, + "p50_ms": 1752.639541, + "p95_ms": 2794.2514169999995, + "peak_rss_bytes": 2316730368 + }, + "P19": { + "gpu_bytes": 2042852008, + "max_ms": 2794.2514169999995, + "n": 5, + "p50_ms": 1752.639541, + "p95_ms": 2794.2514169999995, + "peak_rss_bytes": 2316730368 + }, + "P2": { + "gpu_bytes": 429194378, + "max_ms": 443.161208, + "n": 15, + "p50_ms": 276.31104200000004, + "p95_ms": 443.161208, + "peak_rss_bytes": 670842880 + }, + "P21": { + "gpu_bytes": 1936389156, + "max_ms": 255.857875, + "n": 5, + "p50_ms": 218.395042, + "p95_ms": 255.857875, + "peak_rss_bytes": 1913143296 + }, + "P22": { + "gpu_bytes": 2036052728, + "max_ms": 246.851125, + "n": 5, + "p50_ms": 157.088292, + "p95_ms": 246.851125, + "peak_rss_bytes": 1959444480 + }, + "P26": { + "gpu_bytes": 0, + "max_ms": 22050.895333, + "n": 5, + "p50_ms": 19295.759749999997, + "p95_ms": 22050.895333, + "peak_rss_bytes": 1998864384 + }, + "P27": { + "gpu_bytes": 0, + "max_ms": 52099.803917000005, + "n": 5, + "p50_ms": 42667.745542, + "p95_ms": 52099.803917000005, + "peak_rss_bytes": 682262528 + }, + "P28": { + "gpu_bytes": 0, + "max_ms": 954192.103709, + "n": 1, + "p50_ms": 954192.103709, + "p95_ms": 954192.103709, + "peak_rss_bytes": 2214494208 + }, + "P29": { + "gpu_bytes": 0, + "max_ms": 692.8735409999999, + "n": 5, + "p50_ms": 559.221084, + "p95_ms": 692.8735409999999, + "peak_rss_bytes": 1474887680 + }, + "P3": { + "gpu_bytes": 429194378, + "max_ms": 466.9662080000001, + "n": 15, + "p50_ms": 279.122417, + "p95_ms": 466.9662080000001, + "peak_rss_bytes": 531382272 + }, + "P33": { + "gpu_bytes": 92272684, + "max_ms": 234.867418, + "n": 15, + "p50_ms": 34.655457999999996, + "p95_ms": 234.867418, + "peak_rss_bytes": 813088768 + }, + "P4": { + "gpu_bytes": 429194378, + "max_ms": 422.791916, + "n": 15, + "p50_ms": 205.565041, + "p95_ms": 422.791916, + "peak_rss_bytes": 607289344 + }, + "P40": { + "gpu_bytes": null, + "max_ms": 3.1855830000000003, + "n": 5, + "p50_ms": 1.451375, + "p95_ms": 3.1855830000000003, + "peak_rss_bytes": 1445150720 + }, + "P41": { + "gpu_bytes": null, + "max_ms": 3069.238875, + "n": 5, + "p50_ms": 2962.736792, + "p95_ms": 3069.238875, + "peak_rss_bytes": 2064908288 + }, + "P42": { + "gpu_bytes": null, + "max_ms": 158.405874, + "n": 5, + "p50_ms": 38.686333000000005, + "p95_ms": 158.405874, + "peak_rss_bytes": 1216888832 + }, + "P43": { + "gpu_bytes": null, + "max_ms": 3446.197583, + "n": 5, + "p50_ms": 3047.266834, + "p95_ms": 3446.197583, + "peak_rss_bytes": 1325137920 + }, + "P44": { + "gpu_bytes": null, + "max_ms": 4546.568376, + "n": 5, + "p50_ms": 3534.9131660000003, + "p95_ms": 4546.568376, + "peak_rss_bytes": 1764016128 + }, + "P45": { + "gpu_bytes": null, + "max_ms": 160.32670900000002, + "n": 3, + "p50_ms": 105.263791, + "p95_ms": 160.32670900000002, + "peak_rss_bytes": 749600768 + }, + "P46": { + "gpu_bytes": null, + "max_ms": 83.28816599999999, + "n": 5, + "p50_ms": 30.59725, + "p95_ms": 83.28816599999999, + "peak_rss_bytes": 905560064 + }, + "P47": { + "gpu_bytes": null, + "max_ms": 71.153042, + "n": 30, + "p50_ms": 23.329875, + "p95_ms": 60.499292, + "peak_rss_bytes": 630161408 + }, + "P48": { + "gpu_bytes": null, + "max_ms": 9896.922, + "n": 3, + "p50_ms": 8714.82725, + "p95_ms": 9896.922, + "peak_rss_bytes": null + }, + "P5": { + "gpu_bytes": 429194378, + "max_ms": 424.806751, + "n": 15, + "p50_ms": 177.937708, + "p95_ms": 424.806751, + "peak_rss_bytes": 700907520 + }, + "P6": { + "gpu_bytes": 429194378, + "max_ms": 530.064251, + "n": 15, + "p50_ms": 255.51825, + "p95_ms": 530.064251, + "peak_rss_bytes": 682541056 + }, + "P7": { + "gpu_bytes": 429194378, + "max_ms": 211.910958, + "n": 15, + "p50_ms": 144.162875, + "p95_ms": 211.910958, + "peak_rss_bytes": 701497344 + }, + "P8": { + "gpu_bytes": 963968138, + "max_ms": 699.380084, + "n": 5, + "p50_ms": 508.15320899999995, + "p95_ms": 699.380084, + "peak_rss_bytes": 951123968 + }, + "P9": { + "gpu_bytes": 963968138, + "max_ms": 626.701542, + "n": 5, + "p50_ms": 405.743043, + "p95_ms": 626.701542, + "peak_rss_bytes": 785924096 + } + }, + "failed": { + "P12": "panicked: wgpu error: Validation Error / Caused by: / In Queue::write_buffer / Buffer with 'pc_compose_uniforms' label is invalid" + } + } + }, + "schema": 1 +} diff --git a/perf/budgets.toml b/perf/budgets.toml new file mode 100644 index 00000000..3466fa87 --- /dev/null +++ b/perf/budgets.toml @@ -0,0 +1,411 @@ +# Performance budgets for `cargo xtask perf` and the scorecard (#221). +# +# Targets come from #209 (interactive edits), #210 (long operations) and #211 (slow algorithms). +# Each [[scenario]] is measured by one row of one bench (`bench` + `row`, the row's exact name +# in the bench's `--json` report), judged on `metric` (p50 | p95 | max; default p95) against +# `budget_ms`. `enforce = false` marks a target the code doesn't meet yet: it is reported as +# "over budget" but only a regression fails the run; flip it to true in the PR that meets it. +# Scenarios the code can't support yet carry `not_measurable` (why) instead of a bench: never +# invent a number for them. +# +# Benches run in release with `--json ` appended. `quick_args` = also runs in +# `cargo xtask perf --quick` (small synthetic documents; budgets don't apply there). + +[settings] +# A scenario regresses when its p50 grows by more than this share of the baseline… +regression_pct = 15.0 +# …and by more than this many milliseconds (timer noise on very fast rows). +regression_min_ms = 1.0 + +[[bench]] +name = "perf_scenarios" +package = "photocraft-ui-egui" +quick_args = ["--quick"] +note = "the #209/#210/#211 rows: layered 20 MP, 150 layers, 16/32-bit, open, retouch, selection" + +[[bench]] +name = "interactive_bench" +package = "photocraft-ui-egui" +args = ["--reps", "5"] +note = "36 MP photo: dabs, filters, adjustment previews, saves" + +[[bench]] +name = "fx_bench" +package = "photocraft-ui-egui" +args = ["--no-cpu", "--reps", "5"] +note = "36 MP document with layer effects, GPU compositor" + +[[bench]] +name = "type_bench" +package = "photocraft-ui-egui" +quick_args = ["--size", "2000x1500", "--layers", "20", "--steps", "15"] +note = "font-size drag on 50 type layers" + +[[bench]] +name = "large_image_bench" +package = "photocraft-ui-egui" +args = ["--size", "14000x14000", "--op", "refresh"] +note = "196 MP: open and full refresh" + +# Hook: the layout-document bench lands with perf/layout-documents (#125/#128). It runs (and its +# rows are kept in results.json) as soon as the example exists; add scenarios for its rows then. +[[bench]] +name = "layout_bench" +package = "photocraft-ui-egui" +requires = "crates/ui-egui/examples/layout_bench.rs" +quick_args = [] +note = "layout PSDs with many styled layers (#125/#128)" + +# ---- #209: interactive edits on layered documents (20 MP, styled layers) --------------------- + +[[scenario]] +id = "P1" +title = "Move a layer with layer styles (per drag update)" +issue = 209 +bench = "perf_scenarios" +row = "move styled layer 10 px + refresh" +budget_ms = 25 +enforce = false # over budget in the 2026-10-05 baseline + +[[scenario]] +id = "P2" +title = "Change layer opacity" +issue = 209 +bench = "perf_scenarios" +row = "set layer opacity + refresh" +budget_ms = 20 +enforce = false # over budget in the 2026-10-05 baseline + +[[scenario]] +id = "P3" +title = "Change layer fill" +issue = 209 +bench = "perf_scenarios" +row = "set layer fill + refresh" +budget_ms = 20 +enforce = false # over budget in the 2026-10-05 baseline + +[[scenario]] +id = "P4" +title = "Change blend mode" +issue = 209 +bench = "perf_scenarios" +row = "set blend mode + refresh" +budget_ms = 20 +enforce = false # over budget in the 2026-10-05 baseline + +[[scenario]] +id = "P5" +title = "Toggle visibility" +issue = 209 +bench = "perf_scenarios" +row = "toggle visibility + refresh" +budget_ms = 10 +enforce = false # over budget in the 2026-10-05 baseline + +[[scenario]] +id = "P6" +title = "Undo a move" +issue = 209 +bench = "perf_scenarios" +row = "undo move + refresh" +budget_ms = 25 +enforce = false # over budget in the 2026-10-05 baseline + +[[scenario]] +id = "P7" +title = "Redo a property change" +issue = 209 +bench = "perf_scenarios" +row = "redo opacity change + refresh" +budget_ms = 25 +enforce = false # over budget in the 2026-10-05 baseline + +[[scenario]] +id = "P8" +title = "Duplicate a 12 MP layer (first)" +issue = 209 +bench = "perf_scenarios" +row = "duplicate 12 MP layer (first) + refresh" +budget_ms = 100 +enforce = false # over budget in the 2026-10-05 baseline + +[[scenario]] +id = "P9" +title = "Duplicate a 12 MP layer (repeat)" +issue = 209 +bench = "perf_scenarios" +row = "duplicate 12 MP layer (repeat) + refresh" +budget_ms = 20 +enforce = false # over budget in the 2026-10-05 baseline + +[[scenario]] +id = "P10" +title = "Paste a 12 MP layer (first)" +issue = 209 +bench = "perf_scenarios" +row = "paste 12 MP layer (first) + refresh" +budget_ms = 100 +enforce = false # over budget in the 2026-10-05 baseline + +[[scenario]] +id = "P11" +title = "Paste a 12 MP layer (repeat)" +issue = 209 +bench = "perf_scenarios" +row = "paste 12 MP layer (repeat) + refresh" +budget_ms = 20 +enforce = false # over budget in the 2026-10-05 baseline + +[[scenario]] +id = "P12" +title = "Arrow-key nudge in a 150-layer document" +issue = 209 +bench = "perf_scenarios" +row = "nudge 1 px in the many-layer document + refresh" +budget_ms = 30 +enforce = false # over budget in the 2026-10-05 baseline + +[[scenario]] +id = "P13" +title = "Brush dab on a 15000×10000 16-bit document" +issue = 209 +bench = "perf_scenarios" +row = "brush dab 60 px on 16-bit 15000x10000 + refresh" +budget_ms = 80 +enforce = false # over budget in the 2026-10-05 baseline + +[[scenario]] +id = "P14" +title = "Move a layer on a 15000×10000 16-bit document" +issue = 209 +bench = "perf_scenarios" +row = "move layer on 16-bit 15000x10000 + refresh" +budget_ms = 80 +enforce = false # over budget in the 2026-10-05 baseline + +[[scenario]] +id = "P15" +title = "Zoom / pan frame time on a 15000×10000 16-bit document" +issue = 209 +target = "≤ 16 ms per frame" +not_measurable = "no headless canvas frame harness yet (zoom and pan sample the GPU texture in the egui paint callback)" + +[[scenario]] +id = "P16" +title = "Brush dab on a 32-bit document with 20 layers" +issue = 209 +bench = "perf_scenarios" +row = "brush dab 60 px on 32-bit 20 layers + refresh" +budget_ms = 100 +enforce = false # over budget in the 2026-10-05 baseline + +[[scenario]] +id = "P17" +title = "Move a layer on a 32-bit document with 20 layers" +issue = 209 +bench = "perf_scenarios" +row = "move layer on 32-bit 20 layers + refresh" +budget_ms = 100 +enforce = false # over budget in the 2026-10-05 baseline + +# ---- #210: long operations --------------------------------------------------------------------- + +[[scenario]] +id = "P18" +title = "Open a 20 MP layered PSD: first pixels (today: fully opened)" +issue = 210 +bench = "perf_scenarios" +row = "open 20 MP layered PSD (decode + first composite)" +budget_ms = 1000 +enforce = false # over budget in the 2026-10-05 baseline + +[[scenario]] +id = "P19" +title = "Open a 20 MP layered PSD: editable" +issue = 210 +bench = "perf_scenarios" +row = "open 20 MP layered PSD (decode + first composite)" +budget_ms = 3000 + +[[scenario]] +id = "P20" +title = "Open a 2 GB PSD / 4–5 GB PSB" +issue = 210 +target = "opens with progress, editable, never blank" +not_measurable = "no progress or background open (#210) and no generated large-file fixtures (#216)" + +[[scenario]] +id = "P21" +title = "Content-Aware Fill on 10 MP" +issue = 210 +bench = "perf_scenarios" +row = "Content-Aware Fill 10 MP (200 px hole)" +budget_ms = 1500 + +[[scenario]] +id = "P22" +title = "Spot Healing stroke on 10 MP" +issue = 210 +bench = "perf_scenarios" +row = "Spot Healing stroke 10 MP" +budget_ms = 500 + +[[scenario]] +id = "P23" +title = "Warp / Puppet / Perspective preview at 4K" +issue = 210 +target = "≤ 16 ms per frame, commit in the background" +not_measurable = "the previews run inside the UI tools; no headless frame harness, no background commit" + +[[scenario]] +id = "P24" +title = "Import 500 brushes" +issue = 210 +target = "no UI pause over 100 ms" +not_measurable = "import runs synchronously on the UI thread (no job system, #210)" + +[[scenario]] +id = "P25" +title = "Cancel any long job" +issue = 210 +target = "takes effect within 200 ms, document unchanged" +not_measurable = "no operation can be cancelled yet (#210)" + +# ---- #211: slow algorithms --------------------------------------------------------------------- + +[[scenario]] +id = "P26" +title = "Select › Modify › Smooth on 24 MP (radius 50)" +issue = 211 +bench = "perf_scenarios" +row = "Select > Modify > Smooth r 50 on 24 MP" +budget_ms = 100 +enforce = false # over budget in the 2026-10-05 baseline + +[[scenario]] +id = "P27" +title = "Select › Modify › Feather on 24 MP (radius 50)" +issue = 211 +bench = "perf_scenarios" +row = "Select > Modify > Feather r 50 on 24 MP" +budget_ms = 100 +enforce = false # over budget in the 2026-10-05 baseline + +[[scenario]] +id = "P28" +title = "Content-Aware Scale 24 MP (to 90 % width)" +issue = 211 +bench = "perf_scenarios" +row = "Content-Aware Scale 24 MP to 90 % width" +budget_ms = 3000 +enforce = false # over budget in the 2026-10-05 baseline + +[[scenario]] +id = "P29" +title = "Select Subject on 24 MP, repeated" +issue = 211 +bench = "perf_scenarios" +row = "Select Subject 24 MP (repeat)" +budget_ms = 100 +enforce = false # over budget in the 2026-10-05 baseline + +[[scenario]] +id = "P30" +title = "Liquify 800 px brush on 3500×2200 at full resolution" +issue = 211 +target = "≤ 1 ms per update" +not_measurable = "the Liquify proxy is capped at 1600 px and the stroke path lives in the UI tool" + +[[scenario]] +id = "P31" +title = "Camera Raw exposure change (proxy)" +issue = 211 +target = "≤ 10 ms" +not_measurable = "the proxy is built inside the dialog; no headless entry point yet" + +[[scenario]] +id = "P32" +title = "Brush release on a long stroke" +issue = 211 +target = "no hitch at mouse-up" +not_measurable = "the live renderer and the release re-render are UI-side; needs a frame harness" + +[[scenario]] +id = "P33" +title = "Brush dab on A4 300 ppi CMYK (dab to screen)" +issue = 211 +bench = "perf_scenarios" +row = "brush dab on A4 300 ppi CMYK + refresh" +budget_ms = 6 +enforce = false # over budget in the 2026-10-05 baseline + +[[scenario]] +id = "P34" +title = "Brush preview memory, small dab on an 8K viewport" +issue = 211 +target = "≤ 2 MiB" +not_measurable = "the live brush preview buffers are UI-side; no headless counter yet" + +[[scenario]] +id = "P35" +title = "Marquee drag on 24 MP with Alt/Shift" +issue = 211 +target = "≤ 17 ms p95" +not_measurable = "Shift/Alt marquee modifiers don't exist yet (#208) and the drag preview is UI-side" + +# ---- tracked (no budget yet): regressions only ------------------------------------------------- + +[[scenario]] +id = "P40" +title = "Brush dab 60 px on 36 MP" +bench = "interactive_bench" +row = "brush dab 60 px (command + refresh)" + +[[scenario]] +id = "P41" +title = "Gaussian Blur r 40 on 36 MP" +bench = "interactive_bench" +row = "Gaussian Blur r 40 (photo layer) + refresh" + +[[scenario]] +id = "P42" +title = "Curves dialog change, layer preview, 36 MP" +bench = "interactive_bench" +row = "Curves dialog change, layer preview" + +[[scenario]] +id = "P43" +title = "Save .psd, 36 MP, 3 layers" +bench = "interactive_bench" +row = "save .psd" + +[[scenario]] +id = "P44" +title = "Open .psd, 36 MP, 3 layers" +bench = "interactive_bench" +row = "open .psd (3 layers) + refresh" + +[[scenario]] +id = "P45" +title = "Full refresh with layer effects, warm caches, 36 MP (GPU)" +bench = "fx_bench" +row = "full refresh, warm caches (GPU)" + +[[scenario]] +id = "P46" +title = "Brush dab on a layer with effects, 36 MP (GPU)" +bench = "fx_bench" +row = "dab on the effect layer (damage + reach) (GPU)" + +[[scenario]] +id = "P47" +title = "Font-size drag step, 50 type layers" +bench = "type_bench" +row = "font size step: total" + +[[scenario]] +id = "P48" +title = "Full refresh of a 14000×14000 document" +bench = "large_image_bench" +row = "full refresh" diff --git a/scorecard/automation.toml b/scorecard/automation.toml new file mode 100644 index 00000000..530e0128 --- /dev/null +++ b/scorecard/automation.toml @@ -0,0 +1,82 @@ +# Scorecard checklist: automation (#219). The command count and the menu wiring number come from +# docs/parity.md. +# Statuses must be true on main. Verified against 4b53f62 on 2026-10-05. +area = "automation" +title = "Automation" +order = 6 + +[[item]] +id = "AUTO-1" +target = "Every menu item dispatches a command id (UI, CLI, control channel and MCP share them)" +status = "done" +note = "docs/parity.md: all menu items live; menus.rs is_live" + +[[item]] +id = "AUTO-219-1" +target = "Record actions in a document-independent form (implicit layer, merged type keystrokes)" +status = "missing" +issue = 219 +note = "actions.rs:47 copies raw journal entries with explicit layer ids" + +[[item]] +id = "AUTO-219-2" +target = "Actions panel: step and dialog toggles, reorder, delete a step, Stop, sets, .atn import" +status = "missing" +issue = 219 +note = "Panel deletes whole actions only (actions.rs:146)" + +[[item]] +id = "AUTO-219-3" +target = "actions.record/stop/play/list commands for agents and the CLI" +status = "missing" +issue = 219 +note = "Recording is UI-only (actions.rs:37-60)" + +[[item]] +id = "AUTO-219-4" +target = "Replay tests for crop, filters, selections and styled text on a second document" +status = "missing" +issue = 219 +note = "One replay test (actions.rs:171: new layer, rect select, fill)" + +[[item]] +id = "AUTO-219-5" +target = "Batch: subfolders, open documents, naming template, progress/cancel, error log, stop" +status = "partial" +issue = 219 +note = "file_cmds.rs:590-631 folder/array input, per-file errors; no recursion, template or cancel" + +[[item]] +id = "AUTO-219-6" +target = "Plug-in ABI v2: panel kind, progress/cancel calls, read other layers, Last Filter repeats" +status = "missing" +issue = 219 +note = "runtime.rs:15 ABI_VERSION = 1; manifest.rs:18 Kind::Filter only" + +[[item]] +id = "AUTO-219-7" +target = "Local ONNX models behind a flag, managed in Preferences; subject select, background removal" +status = "missing" +issue = 219 +note = "No ONNX runtime dependency" + +[[item]] +id = "AUTO-219-8" +target = "xtask check that every panel button routes through a command id" +status = "missing" +issue = 219 +note = "No such xtask check" + +[[item]] +id = "AUTO-219-9" +target = "Headless commands for in-canvas type editing, Liquify/Puppet strokes, transform handles" +status = "partial" +issue = 219 +note = "filter.liquify strokes, edit.puppetWarp pins, type.edit, edit.transform; no caret API" + +[[item]] +id = "AUTO-219-10" +target = "Agent acceptance suite grown to 30-50 workflows" +status = "partial" +issue = 219 +note = "automation/tests/agent_tasks.rs runs 10 workflows over MCP" diff --git a/scorecard/bugs.toml b/scorecard/bugs.toml new file mode 100644 index 00000000..54baabb5 --- /dev/null +++ b/scorecard/bugs.toml @@ -0,0 +1,230 @@ +# Scorecard checklist: open bugs (#203–#208). +# Each item: id, short target, status (done | partial | missing), issue, note (evidence). +# Statuses must be true on main. Verified against 4b53f62 on 2026-10-05. +area = "bugs" +title = "Open bugs" +order = 9 + +[[item]] +id = "BUG-203-1" +target = "Every document write goes through one atomic_write (temp, fsync, rename)" +status = "missing" +issue = 203 +note = "services.rs:125 std::fs::write; file_cmds.rs:91 fs::write; automation/files.rs:67 in place" + +[[item]] +id = "BUG-203-2" +target = "Temp file in the same dir, sync_all, rename over target, fsync dir on Unix" +status = "missing" +issue = 203 +note = "No sync_all anywhere; services.rs:71 write_atomic (prefs only) does tmp + rename, no fsync" + +[[item]] +id = "BUG-203-3" +target = "Windows: retry the rename with backoff on PermissionDenied" +status = "missing" +issue = 203 +note = "No retry around fs::rename (services.rs:77, preset_store.rs:189, format/store.rs:457)" + +[[item]] +id = "BUG-203-4" +target = "Unwritable directory errors clearly, never falls back to writing in place" +status = "missing" +issue = 203 +note = "No atomic path exists for documents; all document writes are in place" + +[[item]] +id = "BUG-203-5" +target = "On any failure the original file is untouched and the temp file removed" +status = "missing" +issue = 203 +note = "services.rs:76 write_atomic leaves .tmp behind if rename fails; documents write in place" + +[[item]] +id = "BUG-203-6" +target = "Failure-injection tests: mid-write, rename and read-only dir errors keep original bytes" +status = "missing" +issue = 203 +note = "No such tests under crates/ or apps/" + +[[item]] +id = "BUG-204-1" +target = "Interface prefs (UI scale, font size, channels in colour, sliders, tooltips) wired or hidden" +status = "missing" +issue = 204 +note = "prefs.rs:196-202; no reader outside prefs.rs (see the prefs audit)" + +[[item]] +id = "BUG-204-2" +target = "workspace.large_tabs, animated_zoom, auto_show_home_screen wired or hidden" +status = "missing" +issue = 204 +note = "prefs.rs:163,166,229; no reader outside prefs.rs" + +[[item]] +id = "BUG-204-3" +target = "RAW prefs and ask_before_saving_layered_tiff read, or hidden" +status = "missing" +issue = 204 +note = "raw_defaults (prefs.rs:712), ask_before_saving_layered_tiff (prefs.rs:312) have no readers" + +[[item]] +id = "BUG-204-4" +target = "Recent files persisted through prefs across restarts" +status = "missing" +issue = 204 +note = "ui-egui lib.rs:461 push_recent only touches app.ui.recent_files" + +[[item]] +id = "BUG-204-5" +target = "Recent-files list honours recent_file_count (default 20)" +status = "missing" +issue = 204 +note = "lib.rs:465 truncate(10) hard-coded; menus.rs:1065 test asserts 'capped at 10'" + +[[item]] +id = "BUG-204-6" +target = "Test: every Prefs field is read somewhere or on a hidden allowlist" +status = "partial" +issue = 204 +note = "The scorecard's prefs audit counts unread fields; no engine test or allowlist yet" + +[[item]] +id = "BUG-205-1" +target = "Free Transform with a selection moves only the selected part of a linked mask" +status = "missing" +issue = 205 +note = "transform_cmds.rs:127-130 warps the whole linked mask even when the selection split pixels" + +[[item]] +id = "BUG-205-2" +target = "Live transform preview draws the layer with its mask applied" +status = "missing" +issue = 205 +note = "transform_tool.rs:173 preview_image uses the raw surface" + +[[item]] +id = "BUG-205-3" +target = "edit.transform honours target mask: transforms only a targeted, unlinked mask" +status = "missing" +issue = 205 +note = "transform_cmds.rs:214-270 never reads a target param" + +[[item]] +id = "BUG-205-4" +target = "Test: a linked raster mask with no selection moves with the pixels" +status = "partial" +issue = 205 +note = "Implemented at transform_cmds.rs:127; no raster-mask test" + +[[item]] +id = "BUG-205-5" +target = "Tests: linked mask + selection stays aligned; unlinked stays; targeted mask only" +status = "missing" +issue = 205 +note = "Unlinked case honoured (m.linked check, l.128) but untested; the other two not implemented" + +[[item]] +id = "BUG-206-1" +target = "CharStyle models per-character manual kerning separately from the auto-kern mode" +status = "missing" +issue = 206 +note = "doc/text.rs:76 only `kerning: Kerning` enum (Metrics/Optical/Off)" + +[[item]] +id = "BUG-206-2" +target = "PSD import keeps Optical and manual Kerning values" +status = "missing" +issue = 206 +note = "text/psd.rs:264 maps anything not Off to Metrics; Kerning value ignored" + +[[item]] +id = "BUG-206-3" +target = "PSD export writes the real AutoKerning mode and Kerning values" +status = "missing" +issue = 206 +note = "text/psd.rs:380-381 AutoKerning bool only; Kerning hard-coded 0" + +[[item]] +id = "BUG-206-4" +target = "Alt+Left/Right at a collapsed caret adjusts pair kerning; Character panel field" +status = "missing" +issue = 206 +note = "type_tool.rs:896-912 only a mode combo" + +[[item]] +id = "BUG-206-5" +target = "Tests: PSD round trip of manual and Optical kerning, a real file in the oracle corpus" +status = "missing" +issue = 206 +note = "No kerning round-trip test" + +[[item]] +id = "BUG-207-1" +target = "Retouch tools respect the paint target (mask, alpha channel, Quick Mask)" +status = "missing" +issue = 207 +note = "retouch_cmds.rs:83 run_stroke always uses l.surface_mut(); no paint_target() call" + +[[item]] +id = "BUG-207-2" +target = "Sample All Layers applies to Blur, Sharpen and Smudge" +status = "missing" +issue = 207 +note = "Checkbox at retouch_ui.rs:171-190 is not passed to the stroke commands" + +[[item]] +id = "BUG-207-3" +target = "Smudge mixes premultiplied RGBA (no dark fringe at transparent edges)" +status = "missing" +issue = 207 +note = "No premultiply in paint/src/retouch.rs" + +[[item]] +id = "BUG-207-4" +target = "edit.transform transforms a targeted alpha channel" +status = "missing" +issue = 207 +note = "transform_cmds.rs transform() has no target handling" + +[[item]] +id = "BUG-207-5" +target = "Tests: per-tool mask/channel targeting, Sample All Layers, no smudge fringe" +status = "missing" +issue = 207 +note = "Only smudge_drags_colour_along_the_stroke (retouch.rs:407)" + +[[item]] +id = "BUG-208-1" +target = "Shift while dragging a Rect/Elliptical Marquee constrains to a square/circle" +status = "missing" +issue = 208 +note = "canvas.rs:1498 and :1769 pass shift=false to marquee_end" + +[[item]] +id = "BUG-208-2" +target = "Alt while dragging a marquee draws from the centre" +status = "missing" +issue = 208 +note = "marquee_end (chrome_ui.rs:157) has no centre mode" + +[[item]] +id = "BUG-208-3" +target = "W x H px readout beside the cursor while dragging a marquee" +status = "missing" +issue = 208 +note = "No size readout in canvas.rs or tool_feedback.rs" + +[[item]] +id = "BUG-208-4" +target = "Add/subtract/intersect from modifiers pressed before the drag, kept by a test" +status = "partial" +issue = 208 +note = "Mods captured at press (canvas.rs:1679, used :1779); only a badge test (tool_feedback.rs:195)" + +[[item]] +id = "BUG-208-5" +target = "Kittest: Shift drag gives a square, Alt drag is centred, readout matches the size" +status = "missing" +issue = 208 +note = "No marquee drag tests" diff --git a/scorecard/distribution.toml b/scorecard/distribution.toml new file mode 100644 index 00000000..6eaea804 --- /dev/null +++ b/scorecard/distribution.toml @@ -0,0 +1,70 @@ +# Scorecard checklist: distribution (#220) and what ships today (.github/workflows/release.yml). +# Statuses must be true on main. Verified against 4b53f62 on 2026-10-05. +area = "distribution" +title = "Distribution" +order = 8 + +[[item]] +id = "DIST-1" +target = "macOS universal build (aarch64 + x86_64), signed and notarized" +status = "done" +note = "release.yml:66,75,93 (macos-15); 0.2.0 shipped notarized" + +[[item]] +id = "DIST-2" +target = "Windows x64 and x86 builds" +status = "partial" +note = "release.yml:113-114; signing skipped with a warning until the signing material exists" + +[[item]] +id = "DIST-3" +target = "Linux x86_64 and aarch64: AppImage, deb, rpm, tar.gz" +status = "done" +note = "release.yml:162-163,186" + +[[item]] +id = "DIST-4" +target = "Flatpak single-file bundles, x86_64 and aarch64" +status = "done" +note = "release.yml:204-216 (#163)" + +[[item]] +id = "DIST-5" +target = "Web build (wasm32, trunk)" +status = "done" +note = "release.yml:255" + +[[item]] +id = "DIST-220-1" +target = "Help > Check for Updates plus an opt-out periodic release-feed check on all desktops" +status = "missing" +issue = 220 +note = "No update check in crates" + +[[item]] +id = "DIST-220-2" +target = "Windows ARM64 (aarch64-pc-windows-msvc) in the release matrix, signed like x64" +status = "missing" +issue = 220 +note = "release.yml:113-114 only x86_64 and i686 msvc" + +[[item]] +id = "DIST-220-3" +target = "In-app bug report with reviewable diagnostics that opens a prefilled issue" +status = "missing" +issue = 220 +note = "No bug report dialog in crates" + +[[item]] +id = "DIST-220-4" +target = "Translation coverage per language reported by xtask; RTL layout for Arabic and Hebrew" +status = "missing" +issue = 220 +note = "No i18n in crates or xtask (waits on #169)" + +[[item]] +id = "DIST-220-5" +target = "Linux AppImage x86_64 + aarch64 ship; Flatpak kept; Flathub later (#173)" +status = "partial" +issue = 220 +note = "AppImage and Flatpak ship (release.yml:162-216); Flathub not done" diff --git a/scorecard/file_compat.toml b/scorecard/file_compat.toml new file mode 100644 index 00000000..826ccb7a --- /dev/null +++ b/scorecard/file_compat.toml @@ -0,0 +1,195 @@ +# Scorecard checklist: file compatibility (#215 formats, #216 measurement). +# Corpus numbers come from crates/io/tests/corpus.rs, not from here. +# Statuses must be true on main. Verified against 4b53f62 on 2026-10-05. +area = "file" +title = "File compatibility" +order = 2 + +[[item]] +id = "FILE-215-1" +target = "Smart Objects export as embedded Smart Objects (SoLd + lnk2), not flat pixels" +status = "missing" +issue = 215 +note = "smart_cmds.rs:415 detach_psd drops SoLd; psd_export.rs:276 re-emits imported raw only" + +[[item]] +id = "FILE-215-2" +target = "Smart Filters survive PSD save and re-open (filterFX descriptor read and written)" +status = "missing" +issue = 215 +note = "psd_import.rs:259 smart_filters: Vec::new(); no filterFX in crates" + +[[item]] +id = "FILE-215-3" +target = "Filter mask can be painted directly, with undo" +status = "missing" +issue = 215 +note = "smart_cmds.rs:382 mask only from the selection" + +[[item]] +id = "FILE-215-4" +target = "Adjustments (Hue/Sat + Colorize, Curves, Levels, Shadows/Highlights) apply as Smart Filters" +status = "missing" +issue = 215 +note = "add_smart_filter callers: filters, lens, distort, plug-ins only" + +[[item]] +id = "FILE-215-5" +target = "Smart Filter rows: collapse, mask thumb, blend icon, drag reorder, context menu, live preview" +status = "partial" +issue = 215 +note = "smart_ui.rs:31 rows + visibility + editor; no collapse, drag or context menu" + +[[item]] +id = "FILE-215-6" +target = "Smart Object corner skew and perspective (quad or homography transform)" +status = "missing" +issue = 215 +note = "doc lib.rs:312 SmartObject.transform is an Affine" + +[[item]] +id = "FILE-215-7" +target = "Layered TIFF read and write (tag 37724), honouring the ask-before-saving pref" +status = "missing" +issue = 215 +note = "No 37724 / ImageSourceData in crates" + +[[item]] +id = "FILE-215-8" +target = "Large and BigTIFF import with banded decode across all IFDs" +status = "missing" +issue = 215 +note = "codecs tiff.rs:3 'Only the first IFD is read'" + +[[item]] +id = "FILE-215-9" +target = "SVG opens and places as shape layers or a vector Smart Object" +status = "missing" +issue = 215 +note = "No SVG codec or import" + +[[item]] +id = "FILE-215-10" +target = "Multi-page PDF save, PDF presentation, layer comps to PDF, vector/text output, PDF import" +status = "missing" +issue = 215 +note = "Only print_cmds.rs:133 print_pdf (raster print page)" + +[[item]] +id = "FILE-215-11" +target = "RAW: lossy DNG, highlight modes, .xmp sidecars, RAW prefs wired, per-camera corpus" +status = "missing" +issue = 215 +note = "raw sensor.rs:210 lossy DNG unsupported; raw_defaults unread; no xmp" + +[[item]] +id = "FILE-215-12" +target = "Camera Raw dialog shows a histogram; settings persist as a PSD Smart Filter" +status = "partial" +issue = 215 +note = "No histogram; lens_cmds.rs:268 stores an in-app Smart Filter, PSD drops it" + +[[item]] +id = "FILE-215-13" +target = "Video layer frames persist in .pcraft; PSD video layers mapped" +status = "partial" +issue = 215 +note = "format convert.rs:89 + tests/roundtrip.rs:282 persist frames; no PSD video mapping" + +[[item]] +id = "FILE-215-14" +target = "User swatch model with ACO/ASE import and export" +status = "missing" +issue = 215 +note = "panels.rs:954 const SWATCHES: [[u8; 3]; 40]" + +[[item]] +id = "FILE-215-15" +target = "Layer Style contour picker, curve editor, presets, .shc import/export" +status = "missing" +issue = 215 +note = "doc effects.rs:57 Contour model only" + +[[item]] +id = "FILE-215-16" +target = "Advanced Blending (knockout, blend as group, mask hides effects) end to end" +status = "missing" +issue = 215 +note = "No knockout / blend_interior / transparency_shapes in doc, compose or io" + +[[item]] +id = "FILE-215-17" +target = "CMYK/RGB documents preview spot inks; spot plates export (TIFF extra samples, PDF)" +status = "missing" +issue = 215 +note = "compose multichannel.rs:1 spot preview for Multichannel only" + +[[item]] +id = "FILE-215-18" +target = "Save for Web per-slice settings remembered; Export As remembers and offers nearest scaling" +status = "partial" +issue = 215 +note = "file_ui.rs:188 last_web in-session only; export_dialog.rs:16 fixed defaults" + +[[item]] +id = "FILE-215-19" +target = "Save As JPEG opens a JPEG Options dialog (quality 0-12, format options)" +status = "missing" +issue = 215 +note = "ExportSettings has jpeg_quality 1-100 only (lib.rs:118)" + +[[item]] +id = "FILE-215-20" +target = "Artboard tool with handles and + buttons, Duplicate Artboard beside, vector PDF pages" +status = "partial" +issue = 215 +note = "artboard_ui.rs:1 names/properties only; no tool, + buttons or duplicate" + +[[item]] +id = "FILE-215-21" +target = "Local libraries of colours, patterns, images and layers, as linked items or copies" +status = "missing" +issue = 215 +note = "Only preset libraries (preset_store.rs)" + +[[item]] +id = "FILE-216-1" +target = "Every corpus fetched and enforced in CI, cached by pin" +status = "missing" +issue = 216 +note = "No corpus step in ci.yml; in progress in #190" + +[[item]] +id = "FILE-216-2" +target = "Large-file fixtures: 20 MP, 150-layer, generated 2 GB PSD and 4-5 GB PSB stress tests" +status = "missing" +issue = 216 +note = "Only banded.rs covers a >8 MP synthetic export" + +[[item]] +id = "FILE-216-3" +target = "TIFF, RAW and EXR corpora read by tests; per-camera RAW coverage reported" +status = "missing" +issue = 216 +note = "corpus/tiff, raw and exr only listed by xtask corpus; no test reads them" + +[[item]] +id = "FILE-216-4" +target = "Exports re-open in a third-party oracle editor" +status = "missing" +issue = 216 +note = "The round trip is our export -> our import" + +[[item]] +id = "FILE-216-5" +target = "Save-path tests: layered TIFF, PDF, SVG, ACO/ASE, .shc, Smart Filters, atomic-save failure" +status = "missing" +issue = 216 +note = "Features absent; no atomic-save failure-injection test" + +[[item]] +id = "FILE-216-6" +target = "30-50 end-to-end workflow acceptance tasks driven through MCP or commands" +status = "partial" +issue = 216 +note = "automation/tests/agent_tasks.rs: 10 MCP tasks" diff --git a/scorecard/reliability.toml b/scorecard/reliability.toml new file mode 100644 index 00000000..b36e492b --- /dev/null +++ b/scorecard/reliability.toml @@ -0,0 +1,51 @@ +# Scorecard checklist: reliability (#212, #203, never-crash). +# Statuses must be true on main. Verified against 4b53f62 on 2026-10-05. +area = "reliability" +title = "Reliability" +order = 7 + +[[item]] +id = "REL-1" +target = "panic_hunt fuzzes every command with adversarial params" +status = "done" +note = "engine/tests/panic_hunt.rs: every command, own thread, 4 s timeout; #[ignore], run in the gate" + +[[item]] +id = "REL-2" +target = "Every workspace crate carries the never-crash deny(clippy::unwrap_used, ...) attribute" +status = "done" +note = "Counted automatically above (crate roots under crates/ and apps/); testkit gained it in #221" + +[[item]] +id = "REL-3" +target = "The app catches escaped panics around command dispatch and import/export" +status = "partial" +note = "engine lib.rs:336 dispatch; crash_guard.rs:21 import/export; the dispatch guard is untested" + +[[item]] +id = "REL-4" +target = "Atomic save (temp, fsync, rename) for every document write" +status = "missing" +issue = 203 +note = "Documents use fs::write in place; prefs/presets/bundles use tmp + rename without fsync" + +[[item]] +id = "REL-212-1" +target = "Fallible allocation for paste/erase/fill/transform/save/import, clear out-of-memory error" +status = "missing" +issue = 212 +note = "try_reserve only in io/flat.rs:221; codec max_alloc limits cover decode only" + +[[item]] +id = "REL-212-2" +target = "Export switches to PSB when the estimated file size exceeds 2 GB" +status = "missing" +issue = 212 +note = "io/psd_export.rs:558 switches only when a side exceeds 30000 px" + +[[item]] +id = "REL-212-3" +target = "Tests: an allocation-limit injection makes each command Err with the document unchanged" +status = "missing" +issue = 212 +note = "Only codec/raw max_alloc tests (codecs/tests/limits.rs:48)" diff --git a/scorecard/tools.toml b/scorecard/tools.toml new file mode 100644 index 00000000..4ce8ba21 --- /dev/null +++ b/scorecard/tools.toml @@ -0,0 +1,216 @@ +# Scorecard checklist: tools (#213 painting and retouching, #214 selection, transform, drawing). +# An item is done when it has a command, a UI and tests. +# Statuses must be true on main. Verified against 4b53f62 on 2026-10-05. +area = "tools" +title = "Tools" +order = 3 + +[[item]] +id = "TOOL-213-1" +target = "Pencil tool: aliased 1 px painting, square pixel-aligned cursor" +status = "partial" +issue = 213 +note = "paint.pencil exists (brush_cmds.rs:322) with an aliasing test; no Tool::Pencil (state.rs:7)" + +[[item]] +id = "TOOL-213-2" +target = "Mixer Brush tool on the toolbar" +status = "partial" +issue = 213 +note = "paint.mixerBrush (brush_cmds.rs:337) + panel options; no Tool variant" + +[[item]] +id = "TOOL-213-3" +target = "Patch tool with a live preview of the patched area" +status = "missing" +issue = 213 +note = "No patch command or tool" + +[[item]] +id = "TOOL-213-4" +target = "Content-Aware Move tool reusing inpaint completion" +status = "missing" +issue = 213 +note = "inpaint::complete only used by spot healing (retouch_cmds.rs:386)" + +[[item]] +id = "TOOL-213-5" +target = "Painting a type/shape/smart/fill layer offers to rasterize, then continues the stroke" +status = "missing" +issue = 213 +note = "Rasterize only in the layer menus (layer_menu_ui.rs:40)" + +[[item]] +id = "TOOL-213-6" +target = "Fill dialog: contents, blend, opacity, preserve transparency; remembered; Shift+Backspace" +status = "partial" +issue = 213 +note = "edit.fill: color or pattern + opacity, Shift+F5 (commands.rs:263-267); no blend/history/CA/persist" + +[[item]] +id = "TOOL-213-7" +target = "Live stroke preview for every brush-based retouch tool" +status = "missing" +issue = 213 +note = "Live stroke only for Brush and Eraser (canvas.rs:1684)" + +[[item]] +id = "TOOL-213-8" +target = "Clone Source: W/H link, drawn overlay (opacity/blend/clip/auto-hide), Alt target cursor" +status = "partial" +issue = 213 +note = "Panel W/H + overlay settings (preset_panels.rs:946-1015); no link, overlay never drawn" + +[[item]] +id = "TOOL-213-9" +target = "Eyedropper comparison ring (new colour above, current below) while sampling" +status = "missing" +issue = 213 +note = "sample_eyedropper (canvas.rs:1552) draws no ring" + +[[item]] +id = "TOOL-213-10" +target = "Ctrl+Alt drag resizes brush size/hardness with an on-canvas readout" +status = "missing" +issue = 213 +note = "Only [ ] resize (shortcuts.rs:270)" + +[[item]] +id = "TOOL-213-11" +target = "Editable pressure curve, global and per control" +status = "missing" +issue = 213 +note = "No pressure curve in engine, paint or ui-egui" + +[[item]] +id = "TOOL-213-12" +target = "Import an image file as a brush tip" +status = "missing" +issue = 213 +note = "Only brush.defineFromSelection and brush.presets.importAbr" + +[[item]] +id = "TOOL-213-13" +target = "Brush regression tests: no seams, hard edge, zero-scatter defaults, ABR never black" +status = "partial" +issue = 213 +note = "Missing-pattern ABR test (abr_map/tests.rs:181), hardness test; no seam/black tests" + +[[item]] +id = "TOOL-213-14" +target = "Tablet: no OS arrow cursor over the canvas on macOS, Windows and Linux" +status = "partial" +issue = 213 +note = "Brush tools set CursorIcon::None (canvas.rs:1259-1275); not verified per OS" + +[[item]] +id = "TOOL-214-1" +target = "Select and Mask workspace: view modes, Refine Edge brush, 6 outputs" +status = "partial" +issue = 214 +note = "Opens the generic select.refineEdge dialog (menus.rs:308)" + +[[item]] +id = "TOOL-214-2" +target = "Marquee: Shift square, Alt from centre, live W x H readout" +status = "partial" +issue = 214 +note = "marquee_end supports shift (chrome_ui.rs:167) but canvas.rs passes false (#208)" + +[[item]] +id = "TOOL-214-3" +target = "Color Range test: Sampled Colors at Fuzziness 0 selects the exact match only" +status = "partial" +issue = 214 +note = "Fuzziness-0 tests cover midtones/shadows only (selection_cmds.rs:444-447)" + +[[item]] +id = "TOOL-214-4" +target = "Move tool: box-select layers, Auto-Select hover outline, double-click fit, drag to a tab" +status = "missing" +issue = 214 +note = "move_mods.rs covers Shift/Alt/arrows only" + +[[item]] +id = "TOOL-214-5" +target = "Right-click menu during Free Transform (scale, rotate, skew, warp, flip, rotate 90/180)" +status = "missing" +issue = 214 +note = "No context menu in canvas.rs or transform_tool.rs" + +[[item]] +id = "TOOL-214-6" +target = "Puppet Warp mesh follows the object outline (constrained triangulation)" +status = "missing" +issue = 214 +note = "algo/puppet.rs:136 builds a block-grid mesh" + +[[item]] +id = "TOOL-214-7" +target = "Content-Aware Scale through the Free Transform box with a live proxy preview" +status = "partial" +issue = 214 +note = "edit.contentAwareScale (edit_menu_cmds.rs:899) behind a parameter dialog" + +[[item]] +id = "TOOL-214-8" +target = "A measured ruler angle prefills Image Rotation > Arbitrary (0.01 deg)" +status = "missing" +issue = 214 +note = "Arbitrary rotation is a generic dialog; ruler only feeds straighten" + +[[item]] +id = "TOOL-214-9" +target = "Crop: W x H x Resolution with units, saved crop presets, Straighten in the options bar" +status = "partial" +issue = 214 +note = "Ratio dropdown and fields only (panels.rs:551-579)" + +[[item]] +id = "TOOL-214-10" +target = "Pen: Alt breaks a handle, Shift 45 deg, Ctrl direct-selects, Backspace, close-drag handle" +status = "missing" +issue = 214 +note = "pen_down gets no modifiers (canvas.rs:1632)" + +[[item]] +id = "TOOL-214-11" +target = "Stroke Path dialog: pick any brush-based tool, Simulate Pressure" +status = "partial" +issue = 214 +note = "path.stroke brush, pencil, eraser (vector_cmds.rs:1170); brush-only button, no dialog" + +[[item]] +id = "TOOL-214-12" +target = "Shapes: on-canvas corner radius handles, shape feather, selection to shape with tolerance" +status = "partial" +issue = 214 +note = "select.convertToShape with tolerance (vector_cmds.rs:1256); radius as a field only" + +[[item]] +id = "TOOL-214-13" +target = "Mask Properties: Density and Feather controls" +status = "partial" +issue = 214 +note = "Vector-mask density/feather in the engine (vector_cmds.rs:948); no UI controls" + +[[item]] +id = "TOOL-214-14" +target = "Vanishing Point window: draw and edit planes, paste, clone in perspective" +status = "partial" +issue = 214 +note = "filter.vanishingPoint (vp_cmds.rs:15) behind a parameter dialog" + +[[item]] +id = "TOOL-214-15" +target = "Content-Aware Fill workspace: paint the sampling area, preview, rotate/scale/mirror, new layer" +status = "partial" +issue = 214 +note = "Engine sampling auto, rect, custom (edit_menu_cmds.rs:324); generic dialog" + +[[item]] +id = "TOOL-214-16" +target = "Liquify: redo (Cmd+Shift+Z)" +status = "missing" +issue = 214 +note = "liquify_ui.rs has undo only" diff --git a/scorecard/type.toml b/scorecard/type.toml new file mode 100644 index 00000000..407897e7 --- /dev/null +++ b/scorecard/type.toml @@ -0,0 +1,68 @@ +# Scorecard checklist: type (#218, #206). +# Statuses must be true on main. Verified against 4b53f62 on 2026-10-05. +area = "type" +title = "Type" +order = 5 + +[[item]] +id = "TYPE-218-1" +target = "Type on a path: path shape, glyphs along the curve, caret follows, PSD round trip" +status = "missing" +issue = 218 +note = "doc text.rs:209 TextShape has Point and Box only" + +[[item]] +id = "TYPE-218-2" +target = "Paragraph direction (Auto/LTR/RTL) in the panel and options bar; visual RTL caret" +status = "missing" +issue = 218 +note = "Engine has TextDirection (text.rs:155); no UI" + +[[item]] +id = "TYPE-218-3" +target = "Thai/Khmer/Indic fallback fonts; golden shaping tests for complex scripts and RTL" +status = "missing" +issue = 218 +note = "fonts.rs:28 fallback covers Arabic/Hebrew; no per-script golden tests (only a bidi test)" + +[[item]] +id = "TYPE-218-4" +target = "Variable font axis sliders from the font's axes; opsz follows the point size" +status = "partial" +issue = 218 +note = "type_cmds.rs:184 accepts variations; FaceInfo.axes unused in the UI" + +[[item]] +id = "TYPE-218-5" +target = "OpenType UI: Character panel buttons, Numerals and Stylistic Sets submenus" +status = "partial" +issue = 218 +note = "Type > OpenType toggles (type_extra_cmds.rs:512); no panel buttons or submenus" + +[[item]] +id = "TYPE-218-6" +target = "Manual pair kerning (Alt+arrows), optical kerning rendered, both round-trip PSD (#206)" +status = "missing" +issue = 218 +note = "Optical renders as Metrics (text.rs:21); Alt+arrow moves by word" + +[[item]] +id = "TYPE-218-7" +target = "Warp Text dialog previews live and opens with the layer's current warp" +status = "missing" +issue = 218 +note = "view_cmds.rs:834 always seeds arc, bend 50" + +[[item]] +id = "TYPE-218-8" +target = "Text editing: select all, double/triple/quadruple-click selection, paste, with tests" +status = "partial" +issue = 218 +note = "Cmd+A, paste and double-click exist; no triple/quadruple click, no tests" + +[[item]] +id = "TYPE-218-9" +target = "Offer 'Rasterize and continue?' for Distort/Perspective/Warp on live type" +status = "missing" +issue = 218 +note = "No such prompt" diff --git a/scorecard/ui.toml b/scorecard/ui.toml new file mode 100644 index 00000000..b73d4990 --- /dev/null +++ b/scorecard/ui.toml @@ -0,0 +1,167 @@ +# Scorecard checklist: workspace and UI (#217). The "settings that do nothing" count comes from +# the prefs audit (#204), not from here. +# Statuses must be true on main. Verified against 4b53f62 on 2026-10-05. +area = "ui" +title = "Workspace and UI" +order = 4 + +[[item]] +id = "UI-217-1" +target = "Tool options, last Export As / Save for Web / Fill settings and font recents persist" +status = "missing" +issue = 217 +note = "No tool_options/export fields in Preferences (prefs.rs:694); export_dialog.rs:21 png/85" + +[[item]] +id = "UI-217-2" +target = "Window size and dock width remembered across restarts" +status = "missing" +issue = 217 +note = "main.rs:108 fixed 1440x900; dock.rs:529 snapshot omits the width" + +[[item]] +id = "UI-217-3" +target = "File > Open starts in the last-used folder" +status = "missing" +issue = 217 +note = "services.rs:110 FileDialog::new() without set_directory" + +[[item]] +id = "UI-217-4" +target = "Preferences all take effect; recent files persist and honour recent_file_count (#204)" +status = "missing" +issue = 217 +note = "See the prefs audit and BUG-204-*" + +[[item]] +id = "UI-217-5" +target = "Panel groups can float (menu + drag out of the dock) and are restored at launch" +status = "missing" +issue = 217 +note = "No floating groups in dock.rs/panels.rs" + +[[item]] +id = "UI-217-6" +target = "Document tabs: overflow list, context menu (Close/Others/All/Reveal), drag to reorder" +status = "missing" +issue = 217 +note = "canvas.rs:674-770 tabs click/close only" + +[[item]] +id = "UI-217-7" +target = "Command palette shows recently run commands on an empty query" +status = "missing" +issue = 217 +note = "palette.rs:83-108 fuzzy hits only" + +[[item]] +id = "UI-217-8" +target = "Tool-specific canvas right-click menu; right-click spelling suggestions in text fields" +status = "partial" +issue = 217 +note = "paint_mouse.rs:67 right-click opens the brush picker only" + +[[item]] +id = "UI-217-9" +target = "UI scale presets 100/115/125/150/200 % through zoom_factor, remembered" +status = "missing" +issue = 217 +note = "prefs.rs:61 UiScale unused; input_tests.rs:71 asserts zoom_factor stays 1.0" + +[[item]] +id = "UI-217-10" +target = "Appearance: animation toggle, canvas pattern toggle per theme, start-screen recents" +status = "missing" +issue = 217 +note = "No such prefs; start screen (canvas.rs:825) has no recents" + +[[item]] +id = "UI-217-11" +target = "Soft dark and soft light themes with picker previews; outlined knobs in light themes" +status = "missing" +issue = 217 +note = "theme.rs:28 five themes" + +[[item]] +id = "UI-217-12" +target = "Save custom New Document presets; a real Recent category" +status = "missing" +issue = 217 +note = "new_doc_ui.rs:17 'Recent' is a static list" + +[[item]] +id = "UI-217-13" +target = "Numeric fields accept arithmetic and units; scrubby labels; double-click resets a slider" +status = "partial" +issue = 217 +note = "widgets.rs:131 DragValue scrubs; no expressions, label scrub or reset" + +[[item]] +id = "UI-217-14" +target = "Layers panel: wheel steps blend modes, drop on New/Trash duplicates/deletes, checker behind art" +status = "missing" +issue = 217 +note = "panels.rs:1108 plain dropdown; trash is click-only (panels.rs:1213)" + +[[item]] +id = "UI-217-15" +target = "Curves: smoothing / pencil mode, black/grey/white eyedroppers, tooltips" +status = "missing" +issue = 217 +note = "adjust_editors.rs:397 has none of them" + +[[item]] +id = "UI-217-16" +target = "Hue/Saturation: Saturation and Lightness sliders get hue-following colour tracks" +status = "missing" +issue = 217 +note = "Only the Hue slider has a gradient" + +[[item]] +id = "UI-217-17" +target = "Color panel: wheel plus RGB/HSB/CMYK/Lab/Gray/Web slider modes; resizes with its group" +status = "missing" +issue = 217 +note = "panels.rs:1837 single field + hex/RGB readout" + +[[item]] +id = "UI-217-18" +target = "Alt+wheel zooms in gentle ~5 % steps; zoom controls uncramped" +status = "missing" +issue = 217 +note = "canvas.rs:1107-1125 no Alt handling" + +[[item]] +id = "UI-217-19" +target = "Shortcut hints rendered per platform; Reveal label per OS" +status = "missing" +issue = 217 +note = "~33 hard-coded modifier glyph strings in ui-egui; menu_catalog.rs:341 one label" + +[[item]] +id = "UI-217-20" +target = "'Show rulers in new documents' preference; guides stored as f64" +status = "missing" +issue = 217 +note = "No rulers pref; doc lib.rs:523 guides are Vec" + +[[item]] +id = "UI-217-21" +target = "One Preset Manager for every preset kind with folders, search, reorder, load/save sets" +status = "partial" +issue = 217 +note = "edit_menu_cmds.rs:714 covers brushes, custom shapes, patterns with reorder" + +[[item]] +id = "UI-217-22" +target = "Stamp Visible into a new layer keeping originals; Alt+Merge keeps originals" +status = "missing" +issue = 217 +note = "Only layer.mergeVisible (edit_cmds.rs:416)" + +[[item]] +id = "UI-217-23" +target = "Layer clipboard: copy/paste whole layers (masks, styles, type) between documents" +status = "missing" +issue = 217 +note = "Only copy/paste layer style (extra_cmds.rs:592)" diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 6d935ae8..25305d54 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -10,3 +10,6 @@ workspace = true [dependencies] serde_json = { workspace = true } +serde = { workspace = true } +# perf/budgets.toml and scorecard/*.toml (MIT OR Apache-2.0). +toml = "1" diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 423793e1..1d23e5c6 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -5,7 +5,9 @@ mod ico; mod layers; +mod perf; mod psd_tools; +mod scorecard; mod sha256; mod stats; mod version; @@ -26,6 +28,11 @@ commands: into corpus/psd-tools stats [--exact] count tests and lines per crate (--exact: ask the test harness via `-- --list`) parity Photoshop menu parity; rewrites docs/parity.md + perf [--quick] [--update-baseline] [--threshold PCT] [--bench NAME]... [--skip-build] [--reuse] + run the release benches, merge them by scenario id into target/perf/results.json, + check perf/budgets.toml and perf/baseline.json (non-zero on a broken budget or regression) + scorecard [--check] + regenerate docs/scorecard.md (--check: fail if it is stale) version [set X.Y.Z[-pre]] print the workspace version, or set it (Cargo.toml + Cargo.lock) ico ... @@ -43,6 +50,8 @@ fn main() -> ExitCode { Some("corpus") => cmd_corpus(rest.contains(&"--download")), Some("stats") => stats::run(&root(), rest.contains(&"--exact")), Some("parity") => cmd_parity(), + Some("perf") => perf::run(&root(), &rest), + Some("scorecard") => scorecard::run(&root(), &rest), Some("version") => version::run(&root(), &rest), Some("ico") => ico::run(&rest), Some("-h" | "--help" | "help") | None => { diff --git a/xtask/src/perf.rs b/xtask/src/perf.rs new file mode 100644 index 00000000..ac9fa993 --- /dev/null +++ b/xtask/src/perf.rs @@ -0,0 +1,1104 @@ +//! `cargo xtask perf`: run the release benches, merge their `--json` reports into one result keyed +//! by scenario id, and check it against `perf/budgets.toml` and `perf/baseline.json` (#221). +//! +//! ```text +//! cargo xtask perf [--quick] [--update-baseline] [--threshold PCT] [--bench NAME]... [--skip-build] [--reuse] +//! ``` +//! +//! - `--quick`: only the benches with `quick_args` (small synthetic documents; minutes, not tens +//! of minutes). Quick numbers are compared only with a quick baseline, and budgets don't apply +//! (they are set for the full-size documents). +//! - `--update-baseline`: write this run into `perf/baseline.json` (the only way it changes). +//! - `--threshold PCT`: the regression threshold (default `settings.regression_pct`, 15 %). +//! - `--bench NAME`: run only these benches (repeatable). +//! - `--skip-build`: don't run `cargo build` first. `--reuse`: don't run the benches either; +//! re-evaluate the reports already in `target/perf/`. +//! +//! Output: `target/perf/results.json` (everything), `target/perf/summary.md` (the table the +//! nightly job posts as its summary) and one `target/perf/.json` per bench. +//! +//! Exit status: non-zero when an enforced budget breaks (full mode), a scenario's p50 regresses +//! more than the threshold against a baseline from the same machine class and mode, or a bench +//! fails. Runs from different machine classes are never compared. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::Instant; + +use serde::Deserialize; +use serde_json::{Value, json}; + +pub const BUDGETS: &str = "perf/budgets.toml"; +pub const BASELINE: &str = "perf/baseline.json"; +const MISSING_PREFIX: &str = "**missing**: "; + +// ---- budgets.toml ---------------------------------------------------------------------------- + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Budgets { + #[serde(default)] + pub settings: Settings, + #[serde(default)] + pub bench: Vec, + #[serde(default)] + pub scenario: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct Settings { + /// A scenario regresses when its p50 grows by more than this share of the baseline… + pub regression_pct: f64, + /// …and by more than this many milliseconds (timer noise on sub-millisecond rows). + pub regression_min_ms: f64, +} + +impl Default for Settings { + fn default() -> Self { + Self { regression_pct: 15.0, regression_min_ms: 1.0 } + } +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BenchSpec { + pub name: String, + pub package: String, + /// Example name; defaults to `name`. + pub example: Option, + /// Arguments of a full run (`--json ` is appended). + #[serde(default)] + pub args: Vec, + /// Arguments of a `--quick` run; absent = the bench is skipped in quick mode. + pub quick_args: Option>, + /// Source file that must exist for the bench to run (a bench that lives on another branch): + /// skipped, not failed, while it is missing. + pub requires: Option, + #[serde(default)] + pub note: String, +} + +impl BenchSpec { + pub fn example(&self) -> &str { + self.example.as_deref().unwrap_or(&self.name) + } +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum Metric { + P50, + #[default] + P95, + Max, +} + +impl Metric { + fn label(self) -> &'static str { + match self { + Metric::P50 => "p50", + Metric::P95 => "p95", + Metric::Max => "max", + } + } +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Scenario { + pub id: String, + pub title: String, + pub issue: Option, + /// Bench and row name that measure it; both absent when `not_measurable` is set. + pub bench: Option, + pub row: Option, + #[serde(default)] + pub metric: Metric, + pub budget_ms: Option, + /// Free-form target when it isn't a plain time budget. + pub target: Option, + /// A broken budget fails the run. Off for targets the code doesn't meet yet: they are + /// reported ("over budget") but only regressions fail. + #[serde(default = "yes")] + pub enforce: bool, + /// Why this can't be measured yet (the code doesn't support it). + pub not_measurable: Option, +} + +fn yes() -> bool { + true +} + +impl Scenario { + pub fn target_text(&self) -> String { + if let Some(t) = &self.target { + return t.clone(); + } + match self.budget_ms { + Some(b) if b.fract() == 0.0 => format!("≤ {b:.0} ms {}", self.metric.label()), + Some(b) => format!("≤ {b} ms {}", self.metric.label()), + None => "track only".into(), + } + } +} + +pub fn parse_budgets(text: &str) -> Result { + let b: Budgets = toml::from_str(text).map_err(|e| format!("{BUDGETS}: {e}"))?; + let mut ids = std::collections::BTreeSet::new(); + for s in &b.scenario { + if !ids.insert(s.id.as_str()) { + return Err(format!("{BUDGETS}: duplicate scenario id {}", s.id)); + } + match (&s.bench, &s.row, &s.not_measurable) { + (Some(bench), Some(_), None) => { + if !b.bench.iter().any(|x| &x.name == bench) { + return Err(format!("{BUDGETS}: scenario {} uses undefined bench `{bench}`", s.id)); + } + } + (None, None, Some(_)) => {} + _ => return Err(format!("{BUDGETS}: scenario {} needs either bench + row, or not_measurable", s.id)), + } + if s.budget_ms.is_some_and(|v| !(v.is_finite() && v > 0.0)) { + return Err(format!("{BUDGETS}: scenario {} has an invalid budget_ms", s.id)); + } + } + if !(b.settings.regression_pct.is_finite() && b.settings.regression_pct > 0.0) { + return Err(format!("{BUDGETS}: settings.regression_pct must be > 0")); + } + Ok(b) +} + +// ---- measurements ---------------------------------------------------------------------------- + +/// One scenario's numbers. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Stats { + pub n: usize, + pub p50: Option, + pub p95: Option, + pub max: Option, + pub peak_rss: Option, + pub gpu_bytes: Option, +} + +impl Stats { + pub fn metric(&self, m: Metric) -> Option { + match m { + Metric::P50 => self.p50, + Metric::P95 => self.p95, + Metric::Max => self.max, + } + } + + pub fn to_json(&self) -> Value { + json!({"n": self.n, "p50_ms": self.p50, "p95_ms": self.p95, "max_ms": self.max, "peak_rss_bytes": self.peak_rss, "gpu_bytes": self.gpu_bytes}) + } + + pub fn from_json(v: &Value) -> Self { + Self { + n: v.get("n").and_then(Value::as_u64).unwrap_or(0) as usize, + p50: v.get("p50_ms").and_then(Value::as_f64), + p95: v.get("p95_ms").and_then(Value::as_f64), + max: v.get("max_ms").and_then(Value::as_f64), + peak_rss: v.get("peak_rss_bytes").and_then(Value::as_u64), + gpu_bytes: v.get("gpu_bytes").and_then(Value::as_u64), + } + } +} + +/// Nearest-rank percentile of sorted, non-empty `v`. +fn rank(v: &[f64], q: f64) -> Option { + let n = v.len(); + if n == 0 { + return None; + } + let k = ((q * n as f64).ceil() as usize).clamp(1, n); + v.get(k - 1).copied() +} + +/// A bench row → stats. Rows carry `samples_ms` (preferred: percentiles are recomputed here), +/// or `p50_ms`/`p95_ms`/`max_ms`, or the older `median_ms`/`max_ms`. A p95 is never invented: +/// with neither samples nor a reported p95 it stays empty. +pub fn row_stats(row: &Value) -> Stats { + let mut samples: Vec = + row.get("samples_ms").and_then(Value::as_array).map(|a| a.iter().filter_map(Value::as_f64).filter(|x| x.is_finite()).collect()).unwrap_or_default(); + samples.sort_by(f64::total_cmp); + let f = |k: &str| row.get(k).and_then(Value::as_f64); + let mut s = Stats { + n: samples.len(), + p50: f("p50_ms").or_else(|| f("median_ms")), + p95: f("p95_ms"), + max: f("max_ms"), + peak_rss: row.get("peak_rss_bytes").and_then(Value::as_u64), + gpu_bytes: row.get("gpu_bytes").and_then(Value::as_u64), + }; + if !samples.is_empty() { + s.p50 = rank(&samples, 0.5); + s.p95 = rank(&samples, 0.95); + s.max = samples.last().copied(); + } else if let Some(n) = row.get("n").and_then(Value::as_u64) { + s.n = n as usize; + } + s +} + +/// The row named `name` in a bench report (`{"rows": [{"name": …}, …]}`). +pub fn find_row<'a>(report: &'a Value, name: &str) -> Option<&'a Value> { + report.get("rows")?.as_array()?.iter().find(|r| r.get("name").and_then(Value::as_str) == Some(name)) +} + +/// `cur` regressed against `base` by more than `pct` % and more than `min_ms`. +pub fn regressed(cur: f64, base: f64, pct: f64, min_ms: f64) -> bool { + base > 0.0 && cur.is_finite() && cur > base * (1.0 + pct / 100.0) && cur - base > min_ms +} + +pub fn change_pct(cur: f64, base: f64) -> Option { + (base > 0.0 && cur.is_finite()).then(|| (cur / base - 1.0) * 100.0) +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Status { + Pass, + /// Over budget, not enforced (target not met yet). + Over, + /// Over an enforced budget. + Broken, + Regressed, + /// Measured, no budget, no regression. + Tracked, + /// The bench ran but the row is missing or errored. + Missing(String), + NotRun, + NotMeasurable(String), +} + +impl Status { + pub fn label(&self) -> String { + match self { + Status::Pass => "pass".into(), + Status::Over => "over budget (target not met yet)".into(), + Status::Broken => "**BUDGET BROKEN**".into(), + Status::Regressed => "**REGRESSED**".into(), + Status::Tracked => "tracked".into(), + Status::Missing(e) => format!("{MISSING_PREFIX}{e}"), + Status::NotRun => "not run".into(), + Status::NotMeasurable(_) => "not measurable yet".into(), + } + } + pub fn fails(&self) -> bool { + matches!(self, Status::Broken | Status::Regressed | Status::Missing(_)) + } +} + +/// One evaluated scenario. +#[derive(Clone, Debug)] +pub struct Outcome { + pub id: String, + pub title: String, + pub issue: Option, + pub target: String, + pub stats: Option, + pub baseline_p50: Option, + pub change_pct: Option, + pub status: Status, +} + +/// Budget and regression verdict for one scenario. `budgets_apply` is false in quick mode. +pub fn verdict(sc: &Scenario, stats: &Stats, base: Option<&Stats>, settings: &Settings, threshold: f64, budgets_apply: bool) -> Status { + if let (Some(cur), Some(b)) = (stats.p50, base.and_then(|b| b.p50)) + && regressed(cur, b, threshold, settings.regression_min_ms) + { + return Status::Regressed; + } + match (budgets_apply, sc.budget_ms, stats.metric(sc.metric)) { + (true, Some(budget), Some(v)) if v > budget => { + if sc.enforce { + Status::Broken + } else { + Status::Over + } + } + (true, Some(_), Some(_)) => Status::Pass, + (true, Some(_), None) => Status::Missing(format!("no {} in the bench row", sc.metric.label())), + _ => Status::Tracked, + } +} + +/// Evaluates every scenario against the bench reports (`bench name → report`, `None` when the +/// bench didn't run in this mode) and the baseline's scenarios for this mode. +pub fn evaluate( + budgets: &Budgets, + reports: &BTreeMap>, + baseline: Option<&BTreeMap>, + threshold: f64, + budgets_apply: bool, +) -> Vec { + budgets + .scenario + .iter() + .map(|sc| { + let base = baseline.and_then(|b| b.get(&sc.id)); + let mut o = Outcome { + id: sc.id.clone(), + title: sc.title.clone(), + issue: sc.issue, + target: sc.target_text(), + stats: None, + baseline_p50: base.and_then(|b| b.p50), + change_pct: None, + status: Status::NotRun, + }; + if let Some(why) = &sc.not_measurable { + o.status = Status::NotMeasurable(why.clone()); + return o; + } + let (Some(bench), Some(row)) = (&sc.bench, &sc.row) else { return o }; + match reports.get(bench) { + None => {} + Some(Err(e)) => o.status = Status::Missing(format!("bench `{bench}` failed: {e}")), + Some(Ok(rep)) => match find_row(rep, row) { + None => { + let err = rep + .get("context") + .and_then(|c| c.get("errors")) + .and_then(Value::as_array) + .and_then(|a| a.iter().find(|e| e.get("name").and_then(Value::as_str) == Some(row.as_str()))) + .and_then(|e| e.get("error")) + .and_then(Value::as_str) + .map_or_else(|| format!("row `{row}` not in `{bench}`"), str::to_string); + o.status = Status::Missing(err); + } + Some(r) => { + let st = row_stats(r); + o.change_pct = st.p50.zip(o.baseline_p50).and_then(|(c, b)| change_pct(c, b)); + o.status = verdict(sc, &st, base, &budgets.settings, threshold, budgets_apply); + o.stats = Some(st); + } + }, + } + o + }) + .collect() +} + +// ---- formatting ------------------------------------------------------------------------------ + +pub fn fmt_num(v: f64) -> String { + if v >= 100.0 { + format!("{v:.0}") + } else if v >= 10.0 { + format!("{v:.1}") + } else { + format!("{v:.2}") + } +} + +fn fmt_ms(v: Option) -> String { + v.map_or_else(|| "–".into(), fmt_num) +} + +pub fn fmt_bytes(v: Option) -> String { + match v { + None | Some(0) => "–".into(), + Some(b) if b >= 1 << 30 => format!("{:.1} GB", b as f64 / (1u64 << 30) as f64), + Some(b) => format!("{} MB", b >> 20), + } +} + +pub fn issue_link(n: Option) -> String { + n.map_or_else(String::new, |n| format!("[#{n}](https://github.com/storytold/photocraft/issues/{n})")) +} + +/// The scenario table (Markdown); `with_baseline` adds the change against the baseline. +/// Not-measurable rows go to a separate list after it. +pub fn render_table(outcomes: &[Outcome], with_baseline: bool) -> String { + let mut s = if with_baseline { + String::from( + "| Id | Scenario | Target | p50 ms | p95 ms | max ms | Peak RSS | GPU | vs baseline | Status | Issue |\n|---|---|---|---:|---:|---:|---:|---:|---:|---|---|\n", + ) + } else { + String::from( + "| Id | Scenario | Target | p50 ms | p95 ms | max ms | Peak RSS | GPU | Status | Issue |\n|---|---|---|---:|---:|---:|---:|---:|---|---|\n", + ) + }; + for o in outcomes.iter().filter(|o| !matches!(o.status, Status::NotMeasurable(_))) { + let st = o.stats.clone().unwrap_or_default(); + let vs = if with_baseline { format!(" {} |", o.change_pct.map_or_else(|| "–".into(), |c| format!("{c:+.0} %"))) } else { String::new() }; + s.push_str(&format!( + "| {} | {} | {} | {} | {} | {} | {} | {} |{} {} | {} |\n", + o.id, + o.title, + o.target, + fmt_ms(st.p50), + fmt_ms(st.p95), + fmt_ms(st.max), + fmt_bytes(st.peak_rss), + fmt_bytes(st.gpu_bytes), + vs, + o.status.label(), + issue_link(o.issue), + )); + } + let nm: Vec<&Outcome> = outcomes.iter().filter(|o| matches!(o.status, Status::NotMeasurable(_))).collect(); + if !nm.is_empty() { + s.push_str( + "\n**Not measurable yet** (the code can't do it yet; no number is recorded):\n\n| Id | Scenario | Target | Why | Issue |\n|---|---|---|---|---|\n", + ); + for o in nm { + let why = match &o.status { + Status::NotMeasurable(w) => w.as_str(), + _ => "", + }; + s.push_str(&format!("| {} | {} | {} | {} | {} |\n", o.id, o.title, o.target, why, issue_link(o.issue))); + } + } + s +} + +/// Counts for the scorecard summary: (within budget, over budget or failing, tracked only, not measurable). +pub fn counts(outcomes: &[Outcome]) -> (usize, usize, usize, usize) { + let mut c = (0, 0, 0, 0); + for o in outcomes { + match o.status { + Status::Pass => c.0 += 1, + Status::Over | Status::Broken | Status::Regressed | Status::Missing(_) => c.1 += 1, + Status::NotMeasurable(_) => c.3 += 1, + _ => c.2 += 1, + } + } + c +} + +// ---- machine --------------------------------------------------------------------------------- + +/// Lower-case, `-`-separated, alphanumeric only. +pub fn slug(s: &str) -> String { + let mut out = String::new(); + for ch in s.chars() { + if ch.is_ascii_alphanumeric() { + out.push(ch.to_ascii_lowercase()); + } else if !out.ends_with('-') && !out.is_empty() { + out.push('-'); + } + } + out.trim_end_matches('-').to_string() +} + +/// Runs are compared only within one class: OS, architecture, CPU model and GPU adapter. +pub fn machine_class(os: &str, arch: &str, cpu: &str, gpu: &str) -> String { + [os, arch, cpu, gpu].iter().map(|p| slug(p)).filter(|p| !p.is_empty()).collect::>().join("-") +} + +/// The first three numbers in `/proc/loadavg` or `sysctl -n vm.loadavg` output. +pub fn parse_loadavg(s: &str) -> Option<[f64; 3]> { + let v: Vec = s.split_whitespace().filter_map(|t| t.trim_matches(|c| c == '{' || c == '}').parse().ok()).take(3).collect(); + Some([*v.first()?, *v.get(1)?, *v.get(2)?]) +} + +fn output(cmd: &str, args: &[&str]) -> Option { + let o = Command::new(cmd).args(args).output().ok()?; + o.status.success().then(|| String::from_utf8_lossy(&o.stdout).trim().to_string()).filter(|s| !s.is_empty()) +} + +pub fn load_avg() -> Option<[f64; 3]> { + if let Ok(s) = std::fs::read_to_string("/proc/loadavg") { + return parse_loadavg(&s); + } + output("sysctl", &["-n", "vm.loadavg"]).and_then(|s| parse_loadavg(&s)) +} + +fn proc_field(file: &str, key: &str) -> Option { + let s = std::fs::read_to_string(file).ok()?; + s.lines().find(|l| l.starts_with(key)).and_then(|l| l.split_once(':').or_else(|| l.split_once('='))).map(|(_, v)| v.trim().trim_matches('"').to_string()) +} + +pub fn machine() -> Value { + let (os, arch) = (std::env::consts::OS, std::env::consts::ARCH); + let cores = std::thread::available_parallelism().map(|n| n.get()).ok(); + let (cpu, ram, os_version) = match os { + "macos" => ( + output("sysctl", &["-n", "machdep.cpu.brand_string"]), + output("sysctl", &["-n", "hw.memsize"]).and_then(|s| s.parse::().ok()), + output("sw_vers", &["-productVersion"]).map(|v| format!("macOS {v}")), + ), + "linux" => ( + proc_field("/proc/cpuinfo", "model name").or_else(|| proc_field("/proc/cpuinfo", "CPU part").map(|p| format!("ARM part {p}"))), + proc_field("/proc/meminfo", "MemTotal") + .and_then(|v| v.split_whitespace().next().and_then(|k| k.parse::().ok())) + .and_then(|k| k.checked_mul(1024)), + proc_field("/etc/os-release", "PRETTY_NAME"), + ), + _ => (std::env::var("PROCESSOR_IDENTIFIER").ok(), None, output("cmd", &["/c", "ver"])), + }; + json!({"os": os, "os_version": os_version, "arch": arch, "cpu": cpu, "cores": cores, "ram_bytes": ram}) +} + +/// `YYYY-MM-DDTHH:MM:SSZ` for a Unix time (civil-from-days, proleptic Gregorian). +pub fn iso_utc(secs: u64) -> String { + let days = (secs / 86_400) as i64; + let rem = secs % 86_400; + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097); + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = yoe + era * 400 + i64::from(m <= 2); + format!("{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}Z", rem / 3600, rem % 3600 / 60, rem % 60) +} + +fn now_iso() -> String { + iso_utc(std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0)) +} + +fn git(root: &Path, args: &[&str]) -> Option { + let o = Command::new("git").current_dir(root).args(args).output().ok()?; + o.status.success().then(|| String::from_utf8_lossy(&o.stdout).trim().to_string()) +} + +// ---- baseline -------------------------------------------------------------------------------- + +/// The baseline's scenarios for `mode`, if it was recorded on `class`. `Err` explains a skip. +pub fn baseline_for(baseline: &Value, class: &str, mode: &str) -> Result, String> { + let bc = baseline.get("machine_class").and_then(Value::as_str).unwrap_or(""); + if bc != class { + return Err(format!("baseline is from machine class `{bc}`, this is `{class}`: regression check skipped")); + } + let sc = baseline + .get("modes") + .and_then(|m| m.get(mode)) + .and_then(|m| m.get("scenarios")) + .and_then(Value::as_object) + .ok_or_else(|| format!("baseline has no `{mode}` run: regression check skipped"))?; + Ok(sc.iter().map(|(k, v)| (k.clone(), Stats::from_json(v))).collect()) +} + +/// `old` with this run's mode replaced (or a fresh baseline when the machine class changed). +pub fn updated_baseline(old: Option<&Value>, results: &Value) -> Value { + let class = results.get("machine").and_then(|m| m.get("class")).cloned().unwrap_or(Value::Null); + let mode = results.get("mode").and_then(Value::as_str).unwrap_or("full").to_string(); + let mut modes = old.filter(|o| o.get("machine_class") == Some(&class)).and_then(|o| o.get("modes")).and_then(Value::as_object).cloned().unwrap_or_default(); + let scenarios: serde_json::Map = results + .get("scenarios") + .and_then(Value::as_object) + .map(|m| m.iter().filter(|(_, v)| v.get("p50_ms").is_some_and(|x| !x.is_null())).map(|(k, v)| (k.clone(), Stats::from_json(v).to_json())).collect()) + .unwrap_or_default(); + // Scenarios whose bench row failed (a crash the bench found): kept so the scorecard says so. + let failed: serde_json::Map = results + .get("scenarios") + .and_then(Value::as_object) + .map(|m| m.iter().filter_map(|(k, v)| v.get("status")?.as_str()?.strip_prefix(MISSING_PREFIX).map(|e| (k.clone(), json!(e)))).collect()) + .unwrap_or_default(); + modes.insert( + mode, + json!({ + "date": results.get("date"), + "commit": results.get("commit"), + "load_avg_start": results.get("load_avg_start"), + "load_avg_end": results.get("load_avg_end"), + "scenarios": scenarios, + "failed": failed, + }), + ); + json!({"schema": 1, "machine_class": class, "machine": results.get("machine"), "modes": modes}) +} + +// ---- running --------------------------------------------------------------------------------- + +struct Opts { + quick: bool, + update_baseline: bool, + threshold: Option, + benches: Vec, + skip_build: bool, + reuse: bool, +} + +fn parse_opts(rest: &[&str]) -> Result { + let mut o = Opts { quick: false, update_baseline: false, threshold: None, benches: Vec::new(), skip_build: false, reuse: false }; + let mut it = rest.iter(); + while let Some(a) = it.next() { + match *a { + "--quick" => o.quick = true, + "--update-baseline" => o.update_baseline = true, + "--skip-build" => o.skip_build = true, + "--reuse" => { + o.reuse = true; + o.skip_build = true; + } + "--threshold" => { + let v = it.next().and_then(|v| v.parse::().ok()).filter(|v| v.is_finite() && *v > 0.0).ok_or("--threshold needs a percentage > 0")?; + o.threshold = Some(v); + } + "--bench" => o.benches.push(it.next().ok_or("--bench needs a name")?.to_string()), + other => return Err(format!("perf: unknown option `{other}`")), + } + } + Ok(o) +} + +fn target_dir() -> Result { + let m = crate::metadata()?; + m.get("target_directory").and_then(Value::as_str).map(PathBuf::from).ok_or_else(|| "cargo metadata: no target_directory".into()) +} + +fn write(path: &Path, text: &str) -> Result<(), String> { + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir).map_err(|e| format!("create {}: {e}", dir.display()))?; + } + std::fs::write(path, text).map_err(|e| format!("write {}: {e}", path.display())) +} + +pub fn run(root: &Path, rest: &[&str]) -> Result<(), String> { + let o = parse_opts(rest)?; + let text = std::fs::read_to_string(root.join(BUDGETS)).map_err(|e| format!("read {BUDGETS}: {e}"))?; + let budgets = parse_budgets(&text)?; + let mode = if o.quick { "quick" } else { "full" }; + let threshold = o.threshold.unwrap_or(budgets.settings.regression_pct); + let target = target_dir()?; + let out_dir = target.join("perf"); + + // Which benches run in this mode. + let mut plan: Vec<(&BenchSpec, Vec)> = Vec::new(); + let mut skipped: Vec<(String, String)> = Vec::new(); + for b in &budgets.bench { + if !o.benches.is_empty() && !o.benches.contains(&b.name) { + continue; + } + let args = if o.quick { + match &b.quick_args { + Some(a) => a.clone(), + None => continue, + } + } else { + b.args.clone() + }; + if let Some(req) = &b.requires + && !root.join(req).exists() + { + skipped.push((b.name.clone(), format!("{req} not on this branch yet"))); + continue; + } + plan.push((b, args)); + } + + if !o.skip_build { + let mut by_pkg: BTreeMap<&str, Vec<&str>> = BTreeMap::new(); + for (b, _) in &plan { + by_pkg.entry(b.package.as_str()).or_default().push(b.example()); + } + for (pkg, examples) in by_pkg { + let mut c = crate::cargo(); + c.args(["build", "--release", "-p", pkg]); + for e in &examples { + c.args(["--example", e]); + } + crate::run(c, &format!("cargo build --release -p {pkg} --example {}", examples.join(" --example ")))?; + } + } + + let load_start = load_avg(); + let mut reports: BTreeMap> = BTreeMap::new(); + let mut bench_meta = serde_json::Map::new(); + for (b, args) in &plan { + let json_path = out_dir.join(format!("{}.json", b.name)); + let before = load_avg(); + let t = Instant::now(); + let result = if o.reuse { + std::fs::read_to_string(&json_path).map_err(|e| format!("--reuse: {}: {e}", json_path.display())) + } else { + let _ = std::fs::remove_file(&json_path); + if let Some(dir) = json_path.parent() { + std::fs::create_dir_all(dir).map_err(|e| format!("create {}: {e}", dir.display()))?; + } + let exe = target.join("release").join("examples").join(format!("{}{}", b.example(), std::env::consts::EXE_SUFFIX)); + let mut c = Command::new(&exe); + c.current_dir(root).args(args).arg("--json").arg(&json_path); + let shown = format!("{} {} --json {}", b.example(), args.join(" "), json_path.display()); + crate::run(c, &shown).and_then(|()| std::fs::read_to_string(&json_path).map_err(|e| format!("{}: no JSON report ({e})", b.name))) + }; + let parsed = result.and_then(|t| serde_json::from_str::(&t).map_err(|e| format!("{}: bad JSON: {e}", b.name))); + let after = load_avg(); + bench_meta.insert( + b.name.clone(), + json!({ + "status": if parsed.is_ok() { "ok" } else { "error" }, + "note": b.note, + "error": parsed.as_ref().err(), + "seconds": t.elapsed().as_secs_f64(), + "args": args, + "load_avg_before": before, + "load_avg_after": after, + "context": parsed.as_ref().ok().and_then(|r| r.get("context")).cloned(), + "process_peak_rss_bytes": parsed.as_ref().ok().and_then(|r| r.get("process_peak_rss_bytes")).cloned(), + // Every row, mapped to a scenario or not (new benches show up here first). + "rows": parsed.as_ref().ok().and_then(|r| r.get("rows")).and_then(Value::as_array).map(|rows| { + rows.iter() + .map(|r| { + let mut v = row_stats(r).to_json(); + if let Some(o) = v.as_object_mut() { + o.insert("name".into(), r.get("name").cloned().unwrap_or(Value::Null)); + } + v + }) + .collect::>() + }), + }), + ); + reports.insert(b.name.clone(), parsed); + } + for (name, why) in &skipped { + bench_meta.insert(name.clone(), json!({"status": "skipped", "error": why})); + } + let load_end = load_avg(); + + let mut m = machine(); + let gpu = reports + .values() + .filter_map(|r| r.as_ref().ok()) + .find_map(|r| r.get("context").and_then(|c| c.get("gpu_adapter")).and_then(Value::as_str).map(str::to_string)); + let gpu_name = gpu.clone().unwrap_or_else(|| "cpu-canvas".into()); + let class = machine_class( + m.get("os").and_then(Value::as_str).unwrap_or(""), + m.get("arch").and_then(Value::as_str).unwrap_or(""), + m.get("cpu").and_then(Value::as_str).unwrap_or(""), + &gpu_name, + ); + if let Some(obj) = m.as_object_mut() { + obj.insert("gpu_adapter".into(), json!(gpu)); + obj.insert("class".into(), json!(class)); + } + + let baseline_value: Option = std::fs::read_to_string(root.join(BASELINE)).ok().and_then(|t| serde_json::from_str(&t).ok()); + let (baseline, baseline_note) = match baseline_value.as_ref().map(|b| baseline_for(b, &class, mode)) { + Some(Ok(b)) => (Some(b), None), + Some(Err(e)) => (None, Some(e)), + None => (None, Some(format!("no {BASELINE}: regression check skipped"))), + }; + let outcomes = evaluate(&budgets, &reports, baseline.as_ref(), threshold, !o.quick); + + let scenarios: serde_json::Map = budgets + .scenario + .iter() + .zip(&outcomes) + .map(|(sc, oc)| { + let mut v = oc.stats.as_ref().map(Stats::to_json).unwrap_or_else(|| json!({})); + if let Some(obj) = v.as_object_mut() { + obj.insert("title".into(), json!(sc.title)); + obj.insert("issue".into(), json!(sc.issue)); + obj.insert("bench".into(), json!(sc.bench)); + obj.insert("row".into(), json!(sc.row)); + obj.insert("metric".into(), json!(sc.metric.label())); + obj.insert("budget_ms".into(), json!(sc.budget_ms)); + obj.insert("target".into(), json!(sc.target_text())); + obj.insert("enforce".into(), json!(sc.enforce)); + obj.insert("baseline_p50_ms".into(), json!(oc.baseline_p50)); + obj.insert("change_pct".into(), json!(oc.change_pct)); + obj.insert("status".into(), json!(oc.status.label())); + if let Status::NotMeasurable(w) = &oc.status { + obj.insert("not_measurable".into(), json!(w)); + } + } + (sc.id.clone(), v) + }) + .collect(); + let failures: Vec = outcomes.iter().filter(|o| o.status.fails()).map(|o| format!("{} {}: {}", o.id, o.title, o.status.label())).collect(); + let bench_failures: Vec = reports.iter().filter_map(|(n, r)| r.as_ref().err().map(|e| format!("bench {n}: {e}"))).collect(); + let commit = git(root, &["rev-parse", "HEAD"]); + let dirty = git(root, &["status", "--porcelain"]).is_some_and(|s| !s.is_empty()); + let results = json!({ + "schema": 1, + "mode": mode, + "date": now_iso(), + "commit": commit, + "dirty": dirty, + "machine": m, + "load_avg_start": load_start, + "load_avg_end": load_end, + "threshold_pct": threshold, + "baseline_note": baseline_note, + "benches": bench_meta, + "scenarios": scenarios, + "failures": failures.iter().chain(&bench_failures).collect::>(), + }); + let text = serde_json::to_string_pretty(&results).map_err(|e| format!("encode results: {e}"))?; + write(&out_dir.join("results.json"), &text)?; + + let summary = render_summary(&results, &outcomes, baseline_note.as_deref()); + write(&out_dir.join("summary.md"), &summary)?; + println!("\n{summary}"); + println!("wrote {}", out_dir.join("results.json").display()); + + if o.update_baseline { + let nb = updated_baseline(baseline_value.as_ref(), &results); + let text = serde_json::to_string_pretty(&nb).map_err(|e| format!("encode baseline: {e}"))?; + write(&root.join(BASELINE), &format!("{text}\n"))?; + println!("updated {BASELINE} ({mode}, machine class {class})"); + } + let all: Vec<&String> = failures.iter().chain(&bench_failures).collect(); + if all.is_empty() { Ok(()) } else { Err(format!("{} perf failure(s):\n {}", all.len(), all.iter().map(|s| s.as_str()).collect::>().join("\n "))) } +} + +/// The committed baseline's full-mode numbers judged against the budgets (what the scorecard +/// shows: deterministic, no run needed). Scenarios without a baseline number are "not run". +pub fn outcomes_from_baseline(budgets: &Budgets, baseline: Option<&Value>) -> Vec { + let mode = baseline.and_then(|b| b.get("modes")?.get("full")); + let full: Option> = + mode.and_then(|m| m.get("scenarios")?.as_object().cloned()).map(|m| m.iter().map(|(k, v)| (k.clone(), Stats::from_json(v))).collect()); + let failed = |id: &str| mode.and_then(|m| m.get("failed")?.get(id)?.as_str()).map(str::to_string); + budgets + .scenario + .iter() + .map(|sc| { + let stats = full.as_ref().and_then(|f| f.get(&sc.id)).cloned(); + let status = match (&sc.not_measurable, &stats) { + (Some(w), _) => Status::NotMeasurable(w.clone()), + (None, Some(st)) => verdict(sc, st, None, &budgets.settings, budgets.settings.regression_pct, true), + (None, None) => failed(&sc.id).map_or(Status::NotRun, Status::Missing), + }; + Outcome { + id: sc.id.clone(), + title: sc.title.clone(), + issue: sc.issue, + target: sc.target_text(), + stats, + baseline_p50: None, + change_pct: None, + status, + } + }) + .collect() +} + +pub fn fmt_load(v: Option<&Value>) -> String { + v.and_then(Value::as_array).map_or_else(|| "n/a".into(), |a| a.iter().filter_map(Value::as_f64).map(|x| format!("{x:.2}")).collect::>().join(" / ")) +} + +/// The Markdown summary of one run: context lines and the scenario table. +pub fn render_summary(results: &Value, outcomes: &[Outcome], baseline_note: Option<&str>) -> String { + let m = results.get("machine").cloned().unwrap_or(Value::Null); + let s = |v: Option<&Value>| v.and_then(Value::as_str).unwrap_or("unknown").to_string(); + let ram = m.get("ram_bytes").and_then(Value::as_u64).map_or_else(|| "?".into(), |b| format!("{} GB", b >> 30)); + let mut out = format!( + "## Performance ({} run)\n\n- Machine: {} ({} cores, {ram}), GPU {}, {} {}; class `{}`\n- Commit {}{}, {}\n- Load average (1/5/15 min): {} at start, {} at end\n", + s(results.get("mode")), + s(m.get("cpu")), + m.get("cores").and_then(Value::as_u64).unwrap_or(0), + s(m.get("gpu_adapter")), + s(m.get("os_version")), + s(m.get("arch")), + s(m.get("class")), + s(results.get("commit")), + if results.get("dirty").and_then(Value::as_bool) == Some(true) { " (dirty)" } else { "" }, + s(results.get("date")), + fmt_load(results.get("load_avg_start")), + fmt_load(results.get("load_avg_end")), + ); + if let Some(n) = baseline_note { + out.push_str(&format!("- {n}\n")); + } + if results.get("mode").and_then(Value::as_str) == Some("quick") { + out.push_str("- Quick mode: small documents, budgets not applied (they are set for full-size documents).\n"); + } + if let Some(b) = results.get("benches").and_then(Value::as_object) { + for (name, v) in b { + let st = v.get("status").and_then(Value::as_str).unwrap_or("?"); + let secs = v.get("seconds").and_then(Value::as_f64).map_or_else(String::new, |x| format!(", {x:.0} s")); + let err = v.get("error").and_then(Value::as_str).map_or_else(String::new, |e| format!(": {e}")); + out.push_str(&format!("- Bench `{name}`: {st}{secs}, load {}{err}\n", fmt_load(v.get("load_avg_before")))); + } + } + out.push('\n'); + out.push_str(&render_table(outcomes, true)); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + const TOML: &str = r#" +[settings] +regression_pct = 15.0 +regression_min_ms = 0.5 + +[[bench]] +name = "b" +package = "p" +quick_args = ["--quick"] + +[[scenario]] +id = "P1" +title = "move" +issue = 209 +bench = "b" +row = "move row" +budget_ms = 25 + +[[scenario]] +id = "P2" +title = "opacity" +bench = "b" +row = "opacity row" +budget_ms = 20 +enforce = false + +[[scenario]] +id = "P3" +title = "zoom" +target = "≤ 16 ms per frame" +not_measurable = "needs a frame harness" +"#; + + fn report(rows: Value) -> Value { + json!({"bench": "b", "rows": rows}) + } + + #[test] + fn budgets_parse_and_validate() { + let b = parse_budgets(TOML).unwrap(); + assert_eq!(b.scenario.len(), 3); + assert_eq!(b.scenario[0].metric, Metric::P95); + assert!(b.scenario[0].enforce); + assert!(!b.scenario[1].enforce); + assert_eq!(b.scenario[0].target_text(), "≤ 25 ms p95"); + assert_eq!(b.scenario[2].target_text(), "≤ 16 ms per frame"); + assert_eq!(b.settings.regression_min_ms, 0.5); + // Duplicate ids, undefined benches, half-specified rows and bad budgets are rejected. + assert!(parse_budgets(&format!("{TOML}\n[[scenario]]\nid = \"P1\"\ntitle = \"x\"\nnot_measurable = \"y\"\n")).is_err()); + assert!(parse_budgets("[[scenario]]\nid = \"P1\"\ntitle = \"x\"\nbench = \"nope\"\nrow = \"r\"\n").is_err()); + assert!(parse_budgets("[[bench]]\nname = \"b\"\npackage = \"p\"\n[[scenario]]\nid = \"P1\"\ntitle = \"x\"\nbench = \"b\"\n").is_err()); + assert!( + parse_budgets("[[bench]]\nname = \"b\"\npackage = \"p\"\n[[scenario]]\nid = \"P1\"\ntitle = \"x\"\nbench = \"b\"\nrow = \"r\"\nbudget_ms = -1\n") + .is_err() + ); + assert!(parse_budgets("[settings]\nregression_pct = 0\n").is_err()); + assert!(parse_budgets("bogus = 1\n").is_err()); + assert!(parse_budgets("not toml [").is_err()); + } + + #[test] + fn row_stats_prefers_samples_and_never_invents_p95() { + let s = row_stats(&json!({"name": "x", "samples_ms": [5.0, 1.0, 3.0, 2.0, 4.0], "p50_ms": 99.0})); + assert_eq!((s.n, s.p50, s.p95, s.max), (5, Some(3.0), Some(5.0), Some(5.0))); + // Older rows: median + max only. p95 stays empty. + let s = row_stats(&json!({"name": "x", "median_ms": 4.0, "min_ms": 1.0, "max_ms": 9.0})); + assert_eq!((s.p50, s.p95, s.max), (Some(4.0), None, Some(9.0))); + let s = row_stats(&json!({"name": "x", "samples_ms": ["bad", null]})); + assert_eq!(s.p50, None); + } + + #[test] + fn regression_detection() { + assert!(!regressed(11.4, 10.0, 15.0, 0.5)); + assert!(regressed(11.6, 10.0, 15.0, 0.5)); + // Below the absolute noise floor: no regression. + assert!(!regressed(0.30, 0.20, 15.0, 0.5)); + assert!(!regressed(5.0, 0.0, 15.0, 0.5)); + assert!(!regressed(f64::NAN, 1.0, 15.0, 0.0)); + assert_eq!(change_pct(12.0, 10.0).map(|v| v.round()), Some(20.0)); + assert_eq!(change_pct(1.0, 0.0), None); + } + + #[test] + fn evaluate_budgets_regressions_and_missing_rows() { + let b = parse_budgets(TOML).unwrap(); + let mut reports = BTreeMap::new(); + reports.insert( + "b".to_string(), + Ok(report(json!([{"name": "move row", "samples_ms": [10.0, 12.0, 30.0]}, {"name": "opacity row", "samples_ms": [40.0]}]))), + ); + let out = evaluate(&b, &reports, None, 15.0, true); + assert_eq!(out[0].status, Status::Broken, "p95 30 > 25 and enforced"); + assert_eq!(out[1].status, Status::Over, "not enforced: reported, doesn't fail"); + assert!(matches!(out[2].status, Status::NotMeasurable(_))); + assert!(out[0].status.fails() && !out[1].status.fails() && !out[2].status.fails()); + + // Quick mode: no budgets, only regressions. + let mut base = BTreeMap::new(); + base.insert("P1".to_string(), Stats { p50: Some(10.0), ..Default::default() }); + base.insert("P2".to_string(), Stats { p50: Some(30.0), ..Default::default() }); + let out = evaluate(&b, &reports, Some(&base), 15.0, false); + assert_eq!(out[0].status, Status::Regressed, "p50 12 vs 10 is +20 %"); + assert_eq!(out[1].status, Status::Regressed, "p50 40 vs 30"); + let out = evaluate(&b, &reports, Some(&base), 50.0, false); + assert_eq!(out[0].status, Status::Tracked, "threshold is configurable"); + + // A row the bench didn't produce fails, with the bench's own error when it has one. + let mut reports = BTreeMap::new(); + reports.insert("b".to_string(), Ok(json!({"rows": [], "context": {"errors": [{"name": "move row", "error": "boom"}]}}))); + let out = evaluate(&b, &reports, None, 15.0, true); + assert_eq!(out[0].status, Status::Missing("boom".into())); + assert!(matches!(&out[1].status, Status::Missing(e) if e.contains("not in"))); + // A bench that didn't run in this mode: not run, not a failure. + let out = evaluate(&b, &BTreeMap::new(), None, 15.0, true); + assert_eq!(out[0].status, Status::NotRun); + assert!(!out[0].status.fails()); + } + + #[test] + fn baselines_never_cross_machine_classes() { + let results = json!({"mode": "full", "date": "d", "commit": "c", "machine": {"class": "macos-aarch64-m4"}, "scenarios": {"P1": {"p50_ms": 3.0, "p95_ms": 4.0, "max_ms": 5.0, "n": 3}, "P3": {"status": "not measurable yet"}}}); + let b = updated_baseline(None, &results); + let got = baseline_for(&b, "macos-aarch64-m4", "full").unwrap(); + assert_eq!(got.get("P1").and_then(|s| s.p50), Some(3.0)); + assert!(!got.contains_key("P3"), "rows without numbers are not baselined"); + assert!(baseline_for(&b, "linux-x86-64-epyc", "full").is_err()); + assert!(baseline_for(&b, "macos-aarch64-m4", "quick").is_err()); + // Same class: the other mode is kept. New class: replaced. + let quick = json!({"mode": "quick", "machine": {"class": "macos-aarch64-m4"}, "scenarios": {"P1": {"p50_ms": 1.0}}}); + let b2 = updated_baseline(Some(&b), &quick); + assert!(baseline_for(&b2, "macos-aarch64-m4", "full").is_ok()); + assert!(baseline_for(&b2, "macos-aarch64-m4", "quick").is_ok()); + // A crashed row is kept as a failure, and the scorecard shows it. + let crashed = json!({"mode": "full", "machine": {"class": "c"}, "scenarios": {"P1": {"status": "**missing**: panicked: boom"}}}); + let bc = updated_baseline(None, &crashed); + let b = parse_budgets(TOML).unwrap(); + let o = outcomes_from_baseline(&b, Some(&bc)); + assert_eq!(o[0].status, Status::Missing("panicked: boom".into())); + assert_eq!(o[1].status, Status::NotRun); + let other = json!({"mode": "quick", "machine": {"class": "other"}, "scenarios": {}}); + let b3 = updated_baseline(Some(&b2), &other); + assert!(baseline_for(&b3, "other", "full").is_err()); + } + + #[test] + fn machine_helpers() { + assert_eq!(slug("Apple M4 Pro (Metal)"), "apple-m4-pro-metal"); + assert_eq!(machine_class("macos", "aarch64", "Apple M4 Pro", "Apple M4 Pro"), "macos-aarch64-apple-m4-pro-apple-m4-pro"); + assert_eq!(parse_loadavg("{ 2.10 1.50 0.75 }"), Some([2.1, 1.5, 0.75])); + assert_eq!(parse_loadavg("0.52 0.58 0.59 1/467 12345\n"), Some([0.52, 0.58, 0.59])); + assert_eq!(parse_loadavg("garbage"), None); + assert_eq!(iso_utc(0), "1970-01-01T00:00:00Z"); + assert_eq!(iso_utc(1_791_158_400), "2026-10-05T00:00:00Z"); + assert_eq!(iso_utc(951_782_400), "2000-02-29T00:00:00Z"); + } + + #[test] + fn table_lists_not_measurable_rows_separately() { + let b = parse_budgets(TOML).unwrap(); + let mut reports = BTreeMap::new(); + reports.insert("b".to_string(), Ok(report(json!([{"name": "move row", "samples_ms": [10.0], "peak_rss_bytes": 3u64 << 30}])))); + let t = render_table(&evaluate(&b, &reports, None, 15.0, true), true); + assert!(t.contains("| P1 | move | ≤ 25 ms p95 | 10.0 | 10.0 | 10.0 | 3.0 GB |")); + assert!(t.contains("**Not measurable yet**")); + assert!(t.contains("| P3 | zoom | ≤ 16 ms per frame | needs a frame harness | |")); + assert_eq!(counts(&evaluate(&b, &reports, None, 15.0, true)), (1, 1, 0, 1), "the row the bench lacks counts as failing"); + } + + #[test] + fn options() { + assert!(parse_opts(&["--bogus"]).is_err()); + assert!(parse_opts(&["--threshold", "x"]).is_err()); + assert!(parse_opts(&["--threshold", "-3"]).is_err()); + let o = parse_opts(&["--quick", "--threshold", "20", "--bench", "a", "--reuse"]).unwrap(); + assert!(o.quick && o.reuse && o.skip_build); + assert_eq!(o.threshold, Some(20.0)); + assert_eq!(o.benches, ["a"]); + } +} diff --git a/xtask/src/scorecard.rs b/xtask/src/scorecard.rs new file mode 100644 index 00000000..a976b0ca --- /dev/null +++ b/xtask/src/scorecard.rs @@ -0,0 +1,821 @@ +//! `cargo xtask scorecard [--check]`: generate `docs/scorecard.md` (#221). +//! +//! Sources, all committed, so the output is deterministic and CI can check it: +//! - `perf/budgets.toml` + `perf/baseline.json`: the performance table (the baseline's full-run +//! numbers against the budgets). A fresh `cargo xtask perf` result in `target/` is not used: +//! publish it with `cargo xtask perf --update-baseline`. +//! - `crates/io/tests/corpus.rs`: the corpus floors (`Source { .. }` constants). +//! - `scorecard/*.toml`: per-area checklists (id, target, status, issue, note). +//! - the prefs audit: `Preferences` fields that no code outside `prefs.rs` / `prefs_ui.rs` reads. +//! - counted from the tree: never-crash attribute coverage, `docs/parity.md`'s live count. +//! +//! `--check` fails when `docs/scorecard.md` differs from what would be generated. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use serde::Deserialize; +use serde_json::Value; + +use crate::perf; + +pub const OUT: &str = "docs/scorecard.md"; +const CORPUS_RS: &str = "crates/io/tests/corpus.rs"; +const PREFS_RS: &str = "crates/engine/src/prefs.rs"; + +// ---- checklists ------------------------------------------------------------------------------ + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum ItemStatus { + Done, + Partial, + Missing, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Item { + pub id: String, + pub target: String, + pub status: ItemStatus, + pub issue: Option, + pub note: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Checklist { + pub area: String, + pub title: String, + #[serde(default)] + pub order: u32, + #[serde(default)] + pub item: Vec, +} + +impl Checklist { + pub fn counts(&self) -> (usize, usize, usize) { + let n = |s| self.item.iter().filter(|i| i.status == s).count(); + (n(ItemStatus::Done), n(ItemStatus::Partial), n(ItemStatus::Missing)) + } +} + +pub fn parse_checklist(name: &str, text: &str) -> Result { + let c: Checklist = toml::from_str(text).map_err(|e| format!("{name}: {e}"))?; + for i in &c.item { + if i.id.trim().is_empty() || i.target.trim().is_empty() { + return Err(format!("{name}: every item needs an id and a target")); + } + if i.target.contains('|') || i.note.as_deref().is_some_and(|n| n.contains('|')) { + return Err(format!("{name}: {}: `|` breaks the Markdown table", i.id)); + } + } + Ok(c) +} + +fn load_checklists(root: &Path) -> Result, String> { + let dir = root.join("scorecard"); + let mut files: Vec = std::fs::read_dir(&dir) + .map_err(|e| format!("read {}: {e}", dir.display()))? + .flatten() + .map(|e| e.path()) + .filter(|p| p.extension().is_some_and(|x| x == "toml")) + .collect(); + files.sort(); + let mut out = Vec::new(); + let mut ids = BTreeSet::new(); + for f in files { + let name = format!("scorecard/{}", f.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default()); + let text = std::fs::read_to_string(&f).map_err(|e| format!("read {name}: {e}"))?; + let c = parse_checklist(&name, &text)?; + for i in &c.item { + if !ids.insert(i.id.clone()) { + return Err(format!("{name}: duplicate item id {}", i.id)); + } + } + out.push(c); + } + out.sort_by(|a, b| a.order.cmp(&b.order).then_with(|| a.area.cmp(&b.area))); + Ok(out) +} + +// ---- corpus floors --------------------------------------------------------------------------- + +#[derive(Debug, Default, PartialEq)] +pub struct Corpus { + pub label: String, + pub dir: String, + pub files: Option, + pub pass_floor: Option, + pub roundtrip_floor: Option, + pub doc: String, +} + +/// Every `const X: Source = Source { .. };` in `corpus.rs`, with its doc comment. Fields are +/// read by name, so new fields or new sources (more corpora) need no change here. +pub fn parse_corpus(src: &str) -> Vec { + let lines: Vec<&str> = src.lines().collect(); + let mut out = Vec::new(); + for (i, line) in lines.iter().enumerate() { + let t = line.trim_start(); + if !(t.starts_with("const ") && t.contains(": Source")) { + continue; + } + // The literal may continue on the next lines. + let mut lit = String::new(); + for l in lines.iter().skip(i) { + lit.push_str(l.trim()); + lit.push(' '); + if l.contains("};") { + break; + } + } + let Some(body) = lit.split_once("Source {").map(|(_, b)| b.split('}').next().unwrap_or("")) else { continue }; + let mut c = Corpus::default(); + for field in body.split(',') { + let Some((k, v)) = field.split_once(':') else { continue }; + let v = v.trim().trim_matches('"').to_string(); + match k.trim() { + "label" => c.label = v, + "default_dir" => c.dir = v, + "pass_floor" => c.pass_floor = v.parse().ok(), + "roundtrip_floor" => c.roundtrip_floor = v.parse().ok(), + _ => {} + } + } + let mut doc: Vec<&str> = + lines.iter().take(i).rev().take_while(|l| l.trim_start().starts_with("///")).map(|l| l.trim_start().trim_start_matches('/').trim()).collect(); + doc.reverse(); + c.doc = doc.join(" "); + c.files = c + .doc + .split_whitespace() + .collect::>() + .windows(2) + .find(|w| w.get(1).is_some_and(|x| x.starts_with("files"))) + .and_then(|w| w.first()?.trim_start_matches('(').parse().ok()); + out.push(c); + } + out +} + +// ---- prefs audit ----------------------------------------------------------------------------- + +/// A preference: `section.field` (`section` is the `Preferences` field holding a section +/// struct), or a top-level `Preferences` field when `section` is `None`. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct PrefField { + pub section: Option, + pub field: String, + /// Methods of the section type (in `prefs.rs`) that read this field through `self.field`: + /// calling `section.method()` reads it too. + pub helpers: BTreeSet, +} + +impl PrefField { + pub fn path(&self) -> String { + match &self.section { + Some(s) => format!("{s}.{}", self.field), + None => self.field.clone(), + } + } +} + +/// `pub struct` name → its `pub` fields `(name, type)`, for structs written at column 0. +fn structs(src: &str) -> BTreeMap> { + let mut out = BTreeMap::new(); + let mut cur: Option<(String, Vec<(String, String)>)> = None; + for line in src.lines() { + if let Some(rest) = line.strip_prefix("pub struct ") + && let Some(name) = rest.strip_suffix(" {") + { + cur = Some((name.trim().to_string(), Vec::new())); + continue; + } + if line == "}" { + if let Some((n, f)) = cur.take() { + out.insert(n, f); + } + continue; + } + if let Some((_, fields)) = cur.as_mut() + && let Some(rest) = line.trim_start().strip_prefix("pub ") + && let Some((name, ty)) = rest.split_once(':') + && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') + { + fields.push((name.to_string(), ty.trim().trim_end_matches(',').trim().to_string())); + } + } + out +} + +/// Top-level blocks (`header {` at column 0 up to the next `}` at column 0): `(header, body)`. +fn blocks<'a>(src: &'a str, starts: &str) -> Vec<(&'a str, String)> { + let mut out = Vec::new(); + let mut cur: Option<(&str, String)> = None; + for line in src.lines() { + if cur.is_none() && line.starts_with(starts) && line.ends_with('{') { + cur = Some((line.trim_end_matches('{').trim(), String::new())); + continue; + } + if line == "}" { + if let Some(b) = cur.take() { + out.push(b); + } + continue; + } + if let Some((_, body)) = cur.as_mut() { + body.push_str(line); + body.push('\n'); + } + } + out +} + +/// Methods of `impl ` blocks → the `self.` fields each one reads. +fn helper_methods(src: &str, ty: &str) -> BTreeMap> { + let mut out: BTreeMap> = BTreeMap::new(); + for (_, body) in blocks(src, "impl ").into_iter().filter(|(h, _)| *h == format!("impl {ty}")) { + let mut name: Option = None; + for line in body.lines() { + let t = line.trim_start(); + if let Some(rest) = t.strip_prefix("pub fn ").or_else(|| t.strip_prefix("fn ")) { + name = Some(rest.chars().take_while(|c| c.is_ascii_alphanumeric() || *c == '_').collect()); + } + if let Some(n) = &name { + let a = accesses(line); + out.entry(n.clone()).or_default().extend(a.pairs.iter().filter_map(|p| p.strip_prefix("self.")).map(str::to_string)); + } + } + } + out +} + +/// The leaves of `pub struct Preferences` in `prefs.rs`, with the section helper methods that +/// read each one. +pub fn pref_fields(src: &str) -> Vec { + let s = structs(src); + let mut out = Vec::new(); + for (name, ty) in s.get("Preferences").cloned().unwrap_or_default() { + match s.get(&ty) { + Some(sub) => { + let helpers = helper_methods(src, &ty); + out.extend(sub.iter().map(|(f, _)| PrefField { + section: Some(name.clone()), + field: f.clone(), + helpers: helpers.iter().filter(|(_, used)| used.contains(f)).map(|(m, _)| m.clone()).collect(), + })) + } + None => out.push(PrefField { section: None, field: name, helpers: BTreeSet::new() }), + } + } + out +} + +/// The engine code inside `prefs.rs`: its `impl Session` blocks (e.g. `apply_prefs`, which +/// pushes preferences into live state). Everything else there is the model itself. +pub fn session_code(src: &str) -> String { + blocks(src, "impl Session").into_iter().map(|(_, b)| b).collect() +} + +fn ident_char(c: u8) -> bool { + c.is_ascii_alphanumeric() || c == b'_' +} + +/// Field accesses in one source file: every `.ident` and every `a.b` pair. +#[derive(Default)] +pub struct Accesses { + singles: BTreeSet, + pairs: BTreeSet, + mentions_prefs: bool, +} + +pub fn accesses(src: &str) -> Accesses { + let b = src.as_bytes(); + let mut a = Accesses { mentions_prefs: src.contains("prefs"), ..Default::default() }; + for (i, &c) in b.iter().enumerate() { + if c != b'.' { + continue; + } + let end = b.iter().skip(i + 1).take_while(|&&x| ident_char(x)).count(); + let Some(name) = src.get(i + 1..i + 1 + end).filter(|n| n.bytes().next().is_some_and(|x| x.is_ascii_alphabetic() || x == b'_')) else { continue }; + a.singles.insert(name.to_string()); + let start = b.iter().take(i).rev().take_while(|&&x| ident_char(x)).count(); + if let Some(prev) = src.get(i - start..i).filter(|p| !p.is_empty()) { + a.pairs.insert(format!("{prev}.{name}")); + } + } + a +} + +/// Whether some file reads `f`: `section.field` appears, or both `.section` and `.field` appear +/// in the same file (a section bound to a variable first). Top-level fields: `.field` in a file +/// that mentions `prefs`. Generic names (`enabled`) can be counted as read by accident, never +/// the other way round, so the count is a lower bound. +pub fn is_read(f: &PrefField, files: &[Accesses]) -> bool { + let reads = |a: &Accesses, s: &str, name: &str| a.pairs.contains(&format!("{s}.{name}")) || (a.singles.contains(s) && a.singles.contains(name)); + files.iter().any(|a| match &f.section { + Some(s) => reads(a, s, &f.field) || f.helpers.iter().any(|m| reads(a, s, m)), + None => a.mentions_prefs && a.singles.contains(&f.field), + }) +} + +/// `src` without its top-level `#[cfg(test)]` items (a `mod tests { .. }` block, a `mod x;` +/// line or a test helper function), so unit tests don't count as reads. +pub fn strip_tests(src: &str) -> String { + let mut out = String::new(); + let mut lines = src.lines(); + while let Some(line) = lines.next() { + if line != "#[cfg(test)]" { + out.push_str(line); + out.push('\n'); + continue; + } + // Further attributes, then the item: one line (`mod x;`) or a block up to `}` at column 0. + for item in lines.by_ref() { + if item.starts_with("#[") { + continue; + } + if !item.trim_end().ends_with(';') { + for l in lines.by_ref() { + if l == "}" { + break; + } + } + } + break; + } + } + out +} + +/// Source files the audit scans: `.rs` under `crates/` and `apps/`, minus the prefs model +/// (`prefs.rs` and its tests), tests, examples, benches and fuzz targets, with `#[cfg(test)]` +/// items removed. `prefs_ui.rs` is scanned: the dialog is schema-driven and never names a field, +/// so its field accesses are the appliers (canvas colours, autosave, history log). +fn audit_sources(root: &Path) -> Vec<(PathBuf, String)> { + fn walk(dir: &Path, out: &mut Vec) { + let Ok(rd) = std::fs::read_dir(dir) else { return }; + for e in rd.flatten() { + let p = e.path(); + let name = e.file_name().to_string_lossy().into_owned(); + if p.is_dir() { + if !matches!(name.as_str(), "tests" | "examples" | "benches" | "fuzz" | "target") && !name.starts_with('.') { + walk(&p, out); + } + } else if name.ends_with(".rs") && name != "tests.rs" { + out.push(p); + } + } + } + let mut files = Vec::new(); + walk(&root.join("crates"), &mut files); + walk(&root.join("apps"), &mut files); + files.sort(); + let model = root.join(PREFS_RS); + let prefs_dir = root.join("crates/engine/src/prefs"); + files + .into_iter() + .filter(|p| *p != model && !p.starts_with(&prefs_dir)) + .filter_map(|p| { + let s = std::fs::read_to_string(&p).ok()?; + Some((p, strip_tests(&s.replace("\r\n", "\n")))) + }) + .collect() +} + +/// Preferences that nothing reads (sorted paths). +pub fn prefs_audit(root: &Path) -> Result<(usize, Vec), String> { + let src = std::fs::read_to_string(root.join(PREFS_RS)).map_err(|e| format!("read {PREFS_RS}: {e}"))?; + let fields = pref_fields(&src); + if fields.is_empty() { + return Err(format!("{PREFS_RS}: no `pub struct Preferences` fields found (did the struct move?)")); + } + let mut files: Vec = audit_sources(root).iter().map(|(_, s)| accesses(s)).collect(); + files.push(accesses(&session_code(&strip_tests(&src)))); + let unread: Vec = fields.iter().filter(|f| !is_read(f, &files)).map(PrefField::path).collect(); + Ok((fields.len(), unread)) +} + +// ---- counted from the tree ------------------------------------------------------------------- + +const NEVER_CRASH: &str = "deny(clippy::unwrap_used"; + +/// Crate roots (`crates/*/src/lib.rs`; an app's `lib.rs`, else its `main.rs`) and the ones +/// missing the never-crash attribute. +fn never_crash(root: &Path) -> (usize, Vec) { + let mut roots = Vec::new(); + for (dir, lib_only) in [("crates", true), ("apps", false)] { + let Ok(rd) = std::fs::read_dir(root.join(dir)) else { continue }; + let mut pkgs: Vec = rd.flatten().map(|e| e.path()).filter(|p| p.join("Cargo.toml").exists()).collect(); + pkgs.sort(); + for p in pkgs { + let lib = p.join("src/lib.rs"); + let main = p.join("src/main.rs"); + let r = if lib.exists() { + lib + } else if !lib_only && main.exists() { + main + } else { + continue; + }; + roots.push(r); + } + } + let missing = roots + .iter() + .filter(|r| !std::fs::read_to_string(r).is_ok_and(|s| s.contains(NEVER_CRASH))) + .map(|r| r.strip_prefix(root).unwrap_or(r).to_string_lossy().replace('\\', "/")) + .collect(); + (roots.len(), missing) +} + +/// `**626 of 626 menu items live (100.0%).**` from `docs/parity.md`. +fn parity_line(root: &Path) -> Option { + let s = std::fs::read_to_string(root.join("docs/parity.md")).ok()?; + s.lines().find(|l| l.contains("menu items live")).map(|l| l.trim().trim_matches('*').trim_end_matches('.').to_string()) +} + +// ---- rendering ------------------------------------------------------------------------------- + +fn status_word(s: ItemStatus) -> &'static str { + match s { + ItemStatus::Done => "done", + ItemStatus::Partial => "partial", + ItemStatus::Missing => "missing", + } +} + +pub fn render_checklist(c: &Checklist) -> String { + let mut s = String::from("| Id | Target | Status | Issue | Note |\n|---|---|---|---|---|\n"); + for i in &c.item { + s.push_str(&format!("| {} | {} | {} | {} | {} |\n", i.id, i.target, status_word(i.status), perf::issue_link(i.issue), i.note.as_deref().unwrap_or(""))); + } + s +} + +/// Everything `render` needs, gathered from the tree. +pub struct Inputs { + pub budgets: perf::Budgets, + pub baseline: Option, + pub corpora: Vec, + pub checklists: Vec, + pub prefs: (usize, Vec), + pub never_crash: (usize, Vec), + pub parity: Option, + pub panic_hunt: bool, +} + +pub fn gather(root: &Path) -> Result { + let budgets = perf::parse_budgets(&std::fs::read_to_string(root.join(perf::BUDGETS)).map_err(|e| format!("read {}: {e}", perf::BUDGETS))?)?; + let baseline = match std::fs::read_to_string(root.join(perf::BASELINE)) { + Ok(t) => Some(serde_json::from_str(&t).map_err(|e| format!("{}: {e}", perf::BASELINE))?), + Err(_) => None, + }; + let corpora = parse_corpus(&std::fs::read_to_string(root.join(CORPUS_RS)).map_err(|e| format!("read {CORPUS_RS}: {e}"))?); + Ok(Inputs { + budgets, + baseline, + corpora, + checklists: load_checklists(root)?, + prefs: prefs_audit(root)?, + never_crash: never_crash(root), + parity: parity_line(root), + panic_hunt: root.join("crates/engine/tests/panic_hunt.rs").exists(), + }) +} + +fn checklist_row(c: &Checklist, key: &str) -> String { + let (d, p, m) = c.counts(); + format!("| [{}](#{}) | {d} | {p} | {m} | {key} |\n", c.title, anchor(&c.title)) +} + +/// GitHub's heading anchor: lower-case, spaces to `-`, punctuation dropped. +pub fn anchor(title: &str) -> String { + title + .to_lowercase() + .chars() + .filter_map(|c| { + if c == ' ' { + Some('-') + } else if c.is_alphanumeric() || c == '-' { + Some(c) + } else { + None + } + }) + .collect() +} + +pub fn render(inp: &Inputs) -> String { + let outcomes = perf::outcomes_from_baseline(&inp.budgets, inp.baseline.as_ref()); + let (pass, over, tracked, nm) = perf::counts(&outcomes); + let not_run = outcomes.iter().filter(|o| o.status == perf::Status::NotRun).count(); + let full = inp.baseline.as_ref().and_then(|b| b.get("modes")?.get("full")); + let machine = inp.baseline.as_ref().and_then(|b| b.get("machine")); + let ms = |k: &str| machine.and_then(|m| m.get(k)).and_then(Value::as_str).unwrap_or("unknown").to_string(); + let base_date = full.and_then(|f| f.get("date")).and_then(Value::as_str).map(|d| d.get(..10).unwrap_or(d).to_string()); + let (unread_n, total_prefs) = (inp.prefs.1.len(), inp.prefs.0); + let (roots, missing_nc) = (&inp.never_crash.0, &inp.never_crash.1); + + let mut s = String::from( + "# PhotoCraft scorecard\n\nGenerated by `cargo xtask scorecard` from `perf/budgets.toml`, `perf/baseline.json`, \ +`crates/io/tests/corpus.rs`, `scorecard/*.toml` and the source tree. Do not edit by hand: change the sources and \ +regenerate (CI runs `cargo xtask scorecard --check`). Checklist statuses are verified against the code; numbers \ +here supersede estimates elsewhere.\n\n## Summary\n\n| Area | Done | Partial | Missing | Key numbers |\n|---|---:|---:|---:|---|\n", + ); + let perf_key = match &base_date { + Some(d) => format!( + "{} scenarios: {pass} within budget, {over} over budget or failing, {tracked} tracked only, {not_run} not run, {nm} not measurable yet (baseline {d})", + outcomes.len() + ), + None => format!("{} scenarios, not run yet (no baseline); {nm} not measurable yet", outcomes.len()), + }; + s.push_str(&format!("| [Performance](#performance) | {pass} | {over} | {} | {perf_key} |\n", nm + not_run)); + let corpus_key = inp + .corpora + .iter() + .map(|c| { + let of = c.files.map_or_else(String::new, |f| format!("/{f}")); + format!("{}: oracle ≥ {}{of}, round trip ≥ {}{of}", c.label, c.pass_floor.unwrap_or(0), c.roundtrip_floor.unwrap_or(0)) + }) + .collect::>() + .join("; "); + for c in &inp.checklists { + let key = match c.area.as_str() { + "file" => corpus_key.clone(), + "ui" => format!("settings that do nothing: {unread_n} of {total_prefs} (target 0)"), + "automation" => inp.parity.clone().unwrap_or_default(), + "reliability" => format!( + "never-crash lints on {} of {roots} crates; panic_hunt {}", + roots - missing_nc.len(), + if inp.panic_hunt { "present" } else { "missing" } + ), + _ => String::new(), + }; + s.push_str(&checklist_row(c, &key)); + } + + // Performance. + s.push_str("\n## Performance\n\n"); + s.push_str(&format!( + "Budgets: `perf/budgets.toml` (targets from #209, #210, #211). Numbers: the committed baseline \ +(`perf/baseline.json`, full run), updated only by `cargo xtask perf --update-baseline`. The nightly \ +`perf-nightly` workflow runs `cargo xtask perf` on a fixed macOS runner and fails on a broken enforced \ +budget or a p50 regression over {:.0} % against a baseline from the same machine class. \"Over budget\" \ +rows are targets the code doesn't meet yet: they are reported, and only their regressions fail.\n\n", + inp.budgets.settings.regression_pct + )); + match (&base_date, full) { + (Some(d), Some(f)) => { + let load = perf::fmt_load(f.get("load_avg_start")); + let commit = f.get("commit").and_then(Value::as_str).map_or_else(|| "unknown".into(), |c| c.get(..10).unwrap_or(c).to_string()); + let ram = machine.and_then(|m| m.get("ram_bytes")).and_then(Value::as_u64).map_or_else(|| "?".into(), |b| format!("{} GB", b >> 30)); + s.push_str(&format!( + "Baseline: {d}, commit `{commit}`, {} ({ram}), GPU {}, {}; load average {load} (1/5/15 min) at the start of the run.\n\n", + ms("cpu"), + ms("gpu_adapter"), + ms("os_version") + )); + } + _ => s.push_str("Baseline: **not run** (no `perf/baseline.json` full run yet).\n\n"), + } + s.push_str(&perf::render_table(&outcomes, false)); + + // Corpora. + for c in &inp.checklists { + s.push_str(&format!("\n## {}\n\n", c.title)); + match c.area.as_str() { + "file" => { + s.push_str("Corpus floors (`crates/io/tests/corpus.rs`): the enforced minimums; the corpus test fails below them when the corpus is present.\n\n| Corpus | Directory | Files | Oracle pass floor | Round-trip floor |\n|---|---|---:|---:|---:|\n"); + for k in &inp.corpora { + let n = |v: Option| v.map_or_else(|| "–".into(), |x| x.to_string()); + s.push_str(&format!("| {} | `{}` | {} | {} | {} |\n", k.label, k.dir, n(k.files), n(k.pass_floor), n(k.roundtrip_floor))); + } + s.push('\n'); + } + "ui" => { + s.push_str(&format!( + "**Settings that do nothing: {unread_n} of {total_prefs}** (target 0, #204). Counted by the prefs audit \ +(`xtask/src/scorecard.rs`): a `Preferences` field counts as read when code outside the prefs model (`prefs.rs` \ +except its `impl Session` blocks; tests excluded; the schema-driven dialog never names a field) accesses \ +`section.field`, or `.section` and `.field` in the same file, or calls a section method that reads `self.field`. \ +It is a heuristic lower bound: a generic name can look read when it isn't.\n\n" + )); + if !inp.prefs.1.is_empty() { + s.push_str(&format!("Unread: {}.\n\n", inp.prefs.1.iter().map(|p| format!("`{p}`")).collect::>().join(", "))); + } + } + "automation" => { + if let Some(p) = &inp.parity { + s.push_str(&format!("Menu wiring (`docs/parity.md`): {p}. This counts dispatch, not behaviour.\n\n")); + } + } + "reliability" => { + s.push_str(&format!("Never-crash attribute on {} of {roots} crate roots", roots - missing_nc.len())); + if missing_nc.is_empty() { + s.push_str(".\n\n"); + } else { + s.push_str(&format!("; missing: {}.\n\n", missing_nc.iter().map(|m| format!("`{m}`")).collect::>().join(", "))); + } + } + _ => {} + } + s.push_str(&render_checklist(c)); + } + s +} + +pub fn run(root: &Path, rest: &[&str]) -> Result<(), String> { + let check = rest.contains(&"--check"); + if let Some(bad) = rest.iter().find(|a| **a != "--check") { + return Err(format!("scorecard: unknown option `{bad}`")); + } + let inp = gather(root)?; + let text = render(&inp); + let path = root.join(OUT); + if check { + let current = std::fs::read_to_string(&path).unwrap_or_default().replace("\r\n", "\n"); + if current != text { + return Err(format!("{OUT} is stale: run `cargo xtask scorecard` and commit the result")); + } + println!("{OUT} is up to date"); + return Ok(()); + } + std::fs::write(&path, &text).map_err(|e| format!("write {OUT}: {e}"))?; + println!("wrote {OUT}: settings that do nothing {} of {}", inp.prefs.1.len(), inp.prefs.0); + if let Ok(t) = std::fs::read_to_string(crate::root().join("target/perf/results.json")) + && let Ok(v) = serde_json::from_str::(&t) + { + println!( + "note: target/perf/results.json ({} run, {}) is not in the scorecard; publish it with `cargo xtask perf --update-baseline`", + v.get("mode").and_then(Value::as_str).unwrap_or("?"), + v.get("date").and_then(Value::as_str).unwrap_or("?") + ); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + const PREFS: &str = r#" +pub struct General { + pub beep_when_done: bool, + /// doc + pub animated_zoom: bool, +} + +pub struct Interface { + pub ui_scale: UiScale, + pub grid_colors: Colors, + pub grid_size: Size, +} + +impl Interface { + pub fn colors(&self) -> [u8; 2] { + match self.grid_colors { + _ => [0, 1], + } + } + fn size(&self) -> f32 { + self.grid_size.px() + } +} + +impl Default for Interface { + fn default() -> Self { + Self { ui_scale: 1, grid_colors: x, grid_size: y } + } +} + +impl Session { + pub fn apply_prefs(&mut self) { + let n = self.prefs().general.beep_when_done; + } +} + +#[derive(Default)] +pub struct Preferences { + pub general: General, + pub interface: Interface, + pub shortcuts: BTreeMap, + pub workspace_locked: bool, +} +"#; + + #[test] + fn pref_fields_walk_sections() { + let fields = pref_fields(PREFS); + let f: Vec = fields.iter().map(PrefField::path).collect(); + assert_eq!( + f, + [ + "general.beep_when_done", + "general.animated_zoom", + "interface.ui_scale", + "interface.grid_colors", + "interface.grid_size", + "shortcuts", + "workspace_locked" + ] + ); + // Section helper methods: `colors()` reads grid_colors, `size()` reads grid_size. + let helpers: Vec> = fields.iter().map(|f| f.helpers.iter().cloned().collect()).collect(); + assert_eq!(helpers[3], ["colors"]); + assert_eq!(helpers[4], ["size"]); + assert!(helpers[2].is_empty()); + assert!(pref_fields("pub struct Other {\n pub x: u8,\n}\n").is_empty()); + // Only `impl Session` counts as engine code inside prefs.rs. + let code = session_code(PREFS); + assert!(code.contains("general.beep_when_done") && !code.contains("grid_colors")); + } + + #[test] + fn prefs_audit_rules() { + let fields = pref_fields(PREFS); + let files = [ + accesses(&session_code(PREFS)), + // Reads through a section helper. + accesses("let [a, b] = app.session.prefs().interface.colors();"), + // Section bound first, field read later in the same file. + accesses("let i = &prefs.interface; let s = i.ui_scale;"), + // A top-level field read in a file that mentions prefs. + accesses("let p = session.prefs(); p.shortcuts.get(id)"), + // Not prefs: `.workspace_locked` in a file that never mentions prefs doesn't count. + accesses("state.workspace_locked = true;"), + ]; + let unread: Vec = fields.iter().filter(|f| !is_read(f, &files)).map(PrefField::path).collect(); + assert_eq!(unread, ["general.animated_zoom", "interface.grid_size", "workspace_locked"]); + // A word inside an identifier or a string is not an access. + let a = accesses("let animated_zoom = 1; \"general.animated_zoom\".len(); x.animated_zoomed"); + assert!(a.pairs.contains("general.animated_zoom"), "string paths look like accesses; documented lower bound"); + assert!(a.singles.contains("animated_zoom"), "and so do their parts"); + assert!(!a.singles.contains("general"), "`let animated_zoom` and `\"general` are not accesses"); + assert!(a.singles.contains("animated_zoomed")); + } + + #[test] + fn test_items_are_stripped() { + let src = "fn a() { p.general.x }\n#[cfg(test)]\nfn helper() {\n p.general.y\n}\n\nfn b() { p.general.z }\n#[cfg(test)]\nmod more;\n#[cfg(test)]\n#[allow(clippy::unreachable)]\nmod tests {\n fn t() { p.general.w }\n}\n"; + let s = strip_tests(src); + assert!(s.contains("general.x") && s.contains("general.z")); + assert!(!s.contains("general.y") && !s.contains("general.w") && !s.contains("mod more")); + } + + #[test] + fn corpus_floors_parse() { + let src = r#" +/// Hand-picked mix in `corpus/psd` (170 files: 121 vs the merged image + 12 vs the thumbnail). +const MIXED: Source = Source { label: "io corpus", env: "PHOTOCRAFT_CORPUS", default_dir: "corpus/psd", pass_floor: 133, roundtrip_floor: 169 }; + +/// The full psd-tools test set (309 files at the pinned commit): +/// 202 vs the merged image. +const PSD_TOOLS: Source = + Source { label: "psd-tools corpus", env: "X", default_dir: "corpus/psd-tools", pass_floor: 212, roundtrip_floor: 307, extra: 5 }; +"#; + let c = parse_corpus(src); + assert_eq!(c.len(), 2); + assert_eq!( + (c[0].label.as_str(), c[0].dir.as_str(), c[0].files, c[0].pass_floor, c[0].roundtrip_floor), + ("io corpus", "corpus/psd", Some(170), Some(133), Some(169)) + ); + assert_eq!((c[1].files, c[1].pass_floor, c[1].roundtrip_floor), (Some(309), Some(212), Some(307))); + assert!(parse_corpus("const X: u32 = 1;").is_empty()); + } + + #[test] + fn checklists_validate() { + let ok = "area = \"tools\"\ntitle = \"Tools\"\norder = 3\n[[item]]\nid = \"T-1\"\ntarget = \"Pencil\"\nstatus = \"partial\"\nissue = 213\n"; + let c = parse_checklist("t.toml", ok).unwrap(); + assert_eq!(c.counts(), (0, 1, 0)); + assert!(render_checklist(&c).contains("| T-1 | Pencil | partial | [#213](https://github.com/storytold/photocraft/issues/213) | |")); + assert!(parse_checklist("t.toml", &ok.replace("partial", "almost")).is_err(), "status must be done|partial|missing"); + assert!(parse_checklist("t.toml", &ok.replace("Pencil", "a | b")).is_err()); + assert!(parse_checklist("t.toml", &format!("{ok}bogus = 1\n")).is_err()); + } + + #[test] + fn anchors() { + assert_eq!(anchor("Workspace and UI"), "workspace-and-ui"); + assert_eq!(anchor("File compatibility"), "file-compatibility"); + } + + #[test] + fn staleness_check_and_determinism() { + let root = crate::root(); + let inp = gather(&root).unwrap_or_else(|e| panic!("{e}")); + let a = render(&inp); + assert_eq!(a, render(&gather(&root).unwrap()), "rendering is deterministic"); + // The committed file is current (what `--check` enforces in CI). + let committed = std::fs::read_to_string(root.join(OUT)).unwrap_or_default().replace("\r\n", "\n"); + assert!(committed == a, "{OUT} is stale: run `cargo xtask scorecard`"); + // Every scenario and every checklist area shows up. + for sc in &inp.budgets.scenario { + assert!(a.contains(&format!("| {} |", sc.id)), "{} missing from the scorecard", sc.id); + } + } +}