vectorcraft/crates/trace/tests/quality/main.rs

//! Image Trace quality harness: traces images drawn from known vector art with every preset and
//! measures the result (see `docs/image-trace-quality.md`, which this writes).
//!
//! ```text
//! TRACE_QUALITY_REPORT=report.md cargo test --release -p vectorcraft-trace --test quality -- --ignored --nocapture
//! ```
//!
//! `TRACE_QUALITY_MP` (default `1,4,16`) sets the megapixel sizes of the cost table.
//!
//! A yardstick: `TRACE_QUALITY_IMAGES=dir` also writes the drawings there as PNG files, and
//! `TRACE_QUALITY_YARDSTICK=dir` measures the SVG files of the same names found there (what another
//! tracer made of those PNG files, run by hand as a black box) the same way.
// Integration tests: unwrapping and panicking on failure is fine here, unlike in shipped code (AGENTS.md › Robustness).
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
// The peak-memory measure needs a counting global allocator.
#![allow(unsafe_code)]

mod inputs;
mod measure;
mod raster;

use std::alloc::{GlobalAlloc, Layout, System};
use std::fmt::Write as _;
use std::sync::atomic::{AtomicUsize, Ordering::Relaxed};
use std::time::Instant;

use inputs::{CLASSES, Class, Input, VARIANTS, Variant, inputs};
use measure::{Metrics, measure};
use vectorcraft_trace::{Raster, TraceParams, TraceResult, TracedPath, presets, trace_within};

/// The engine's limit (`MAX_TRACE_ANCHORS`): a trace past it is refused.
const MAX_ANCHORS: usize = 2_000_000;

struct Counting;
static LIVE: AtomicUsize = AtomicUsize::new(0);
static PEAK: AtomicUsize = AtomicUsize::new(0);

fn grew(by: usize) {
    PEAK.fetch_max(LIVE.fetch_add(by, Relaxed) + by, Relaxed);
}

unsafe impl GlobalAlloc for Counting {
    unsafe fn alloc(&self, l: Layout) -> *mut u8 {
        grew(l.size());
        unsafe { System.alloc(l) }
    }
    unsafe fn alloc_zeroed(&self, l: Layout) -> *mut u8 {
        grew(l.size());
        unsafe { System.alloc_zeroed(l) }
    }
    unsafe fn dealloc(&self, p: *mut u8, l: Layout) {
        LIVE.fetch_sub(l.size(), Relaxed);
        unsafe { System.dealloc(p, l) }
    }
    unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 {
        LIVE.fetch_sub(l.size(), Relaxed);
        grew(new);
        unsafe { System.realloc(p, l, new) }
    }
}

#[global_allocator]
static ALLOC: Counting = Counting;

/// Whether `input` is what `preset` is meant for.
fn target(preset: &str, input: &Input) -> bool {
    let classes: &[Class] = match preset {
        "High Fidelity Photo" | "Low Fidelity Photo" | "Shades of Gray" => &[Class::Tone],
        // Three colours cannot show the badge's five.
        "3 Colors" => return input.name == "two touching halves",
        "6 Colors" | "16 Colors" | "Flat Logo" => &[Class::Flat],
        "Sketched Art" => &[Class::Shape, Class::Lines],
        "Silhouettes" => &[Class::Shape],
        "Line Art" => &[Class::Lines],
        "Technical Drawing" => &[Class::Lines, Class::Poly],
        _ => &[Class::Shape, Class::Poly],
    };
    classes.contains(&input.class)
}

/// One trace: which preset and input, and what was measured (`None`: the trace was refused).
struct Run {
    preset: usize,
    /// The input is one the preset is meant for.
    target: bool,
    class: Class,
    variant: Variant,
    metrics: Option<Metrics>,
}

/// Every (size, variant) an input is traced at: clean at three sizes, the others at the middle one.
const DRAWINGS: [(u32, Variant); 7] = [
    (128, Variant::Clean),
    (256, Variant::Clean),
    (512, Variant::Clean),
    (256, Variant::Hard),
    (256, Variant::Blur),
    (256, Variant::Noise),
    (256, Variant::Scan),
];

type Drawing<'a> = (&'a Input, u32, Variant, Raster);

fn draw_all(inputs: &[Input]) -> Vec<Drawing<'_>> {
    inputs.iter().flat_map(|input| DRAWINGS.map(|(size, variant)| (input, size, variant, input.draw(size, variant)))).collect()
}

/// The file name (without extension) of a drawing given to or made by another tracer.
fn stem(input: &Input, size: u32, variant: Variant) -> String {
    format!("{}-{size}-{}", input.name.replace(' ', "-"), variant.name())
}

/// The value of attribute `name` of an SVG element's text.
fn attr<'a>(element: &'a str, name: &str) -> Option<&'a str> {
    let key = format!("{name}=\"");
    let at = element.match_indices(&key).find(|(i, _)| element[..*i].ends_with(char::is_whitespace))?.0 + key.len();
    Some(&element[at..at + element[at..].find('"')?])
}

/// The filled paths of an SVG file as a trace in black. Only what the yardstick's files use is
/// read: `path` elements with a `d`, and `matrix`, `translate` or `scale` transforms on them.
fn svg_trace(svg: &str) -> TraceResult {
    let paths = svg.split("<path").skip(1).filter_map(|rest| {
        let element = &rest[..rest.find('>')?];
        let mut path = kurbo::BezPath::from_svg(attr(element, "d")?).ok()?;
        if let Some(t) = attr(element, "transform") {
            let (kind, args) = t.trim_end_matches(')').split_once('(').unwrap();
            let v: Vec<f64> = args.split([',', ' ']).filter(|a| !a.is_empty()).map(|a| a.parse().unwrap()).collect();
            path.apply_affine(match (kind, v.as_slice()) {
                ("matrix", &[a, b, c, d, e, f]) => kurbo::Affine::new([a, b, c, d, e, f]),
                ("translate", &[x, y]) => kurbo::Affine::translate((x, y)),
                ("scale", &[x, y]) => kurbo::Affine::scale_non_uniform(x, y),
                ("scale", &[k]) => kurbo::Affine::scale(k),
                _ => panic!("unread transform {t}"),
            });
        }
        Some(TracedPath { path: vectorcraft_geom::PathData::from_bezpath(&path), color: [0; 3], pixels: 0, stroke: None })
    });
    TraceResult { paths: paths.collect(), palette: vec![[0; 3]] }
}

fn run_all(presets: &[(&'static str, TraceParams)], drawn: &[Drawing]) -> Vec<Run> {
    std::thread::scope(|s| {
        let workers: Vec<_> = presets
            .iter()
            .enumerate()
            .map(|(pi, (name, params))| {
                s.spawn(move || {
                    drawn
                        .iter()
                        .map(|(input, size, variant, src)| {
                            let metrics = trace_within(src, params, MAX_ANCHORS).ok().map(|res| measure(input, *variant, *size, src, params, &res));
                            Run { preset: pi, target: target(name, input), class: input.class, variant: *variant, metrics }
                        })
                        .collect::<Vec<_>>()
                })
            })
            .collect();
        workers.into_iter().flat_map(|w| w.join().unwrap()).collect()
    })
}

/// The mean (or, with `max`, the largest) of `field` over `runs`, to `digits` places; `–` when it
/// applies to none of them.
fn stat(runs: &[&Metrics], field: impl Fn(&Metrics) -> f64, max: bool, digits: usize) -> String {
    let v: Vec<f64> = runs.iter().map(|m| field(m)).filter(|v| !v.is_nan()).collect();
    if v.is_empty() {
        return "–".into();
    }
    let x = if max { v.iter().copied().fold(f64::NEG_INFINITY, f64::max) } else { v.iter().sum::<f64>() / v.len() as f64 };
    // `+ 0.0` turns a negative zero into zero.
    if x.is_finite() { format!("{:.digits$}", x + 0.0) } else { "all".into() }
}

const HEADER: &str = "| ΔE | edge ΔE | off mean | off max | missed max | paths | anchors | × art | lines % | corners % | kinks | odd colours | odd px | gap px | overlap px | specks | outside px |\n|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|\n";

fn row(runs: &[&Run]) -> String {
    let m: Vec<&Metrics> = runs.iter().filter_map(|r| r.metrics.as_ref()).collect();
    if m.is_empty() {
        return format!("| refused |{}", " |".repeat(16));
    }
    let cells = [
        stat(&m, |m| m.delta, false, 2),
        stat(&m, |m| m.edge_delta, false, 2),
        stat(&m, |m| m.geo_mean, false, 3),
        stat(&m, |m| m.geo_max, true, 2),
        stat(&m, |m| m.miss_max, true, 2),
        stat(&m, |m| m.paths, false, 1),
        stat(&m, |m| m.anchors, false, 0),
        stat(&m, |m| m.anchor_ratio, false, 2),
        stat(&m, |m| m.lines, false, 0),
        stat(&m, |m| m.corners, false, 0),
        stat(&m, |m| m.kinks, false, 1),
        stat(&m, |m| m.slivers, false, 1),
        stat(&m, |m| m.sliver_area, false, 0),
        stat(&m, |m| m.gap, false, 1),
        stat(&m, |m| m.overlap, false, 1),
        stat(&m, |m| m.specks, false, 1),
        stat(&m, |m| m.outside, true, 1),
    ];
    format!("| {} |", cells.join(" | "))
}

/// A cost measurement: seconds, paths, anchors and peak megabytes above the image itself.
fn cost(img: &Raster, params: &TraceParams) -> String {
    let base = LIVE.load(Relaxed);
    PEAK.store(base, Relaxed);
    let start = Instant::now();
    let res = trace_within(img, params, MAX_ANCHORS);
    let secs = start.elapsed().as_secs_f64();
    let mb = (PEAK.load(Relaxed) - base) as f64 / f64::from(1 << 20);
    match res {
        Ok(res) => format!("{secs:.2} s, {} paths, {} anchors, {mb:.0} MB", res.paths.len(), res.anchor_count()),
        Err(e) => {
            let why = e.to_string();
            let why = if why.contains("anchor points") { "over 2,000,000 anchors" } else { "not flat art" };
            format!("refused after {secs:.2} s ({why}), {mb:.0} MB")
        }
    }
}

fn report() -> String {
    let presets = presets();
    let inputs = inputs();
    let drawn = draw_all(&inputs);
    let one_colour = |d: &&Drawing| matches!(d.0.class, Class::Shape | Class::Poly | Class::Lines);
    if let Ok(dir) = std::env::var("TRACE_QUALITY_IMAGES") {
        for (input, size, variant, img) in &drawn {
            std::fs::write(format!("{dir}/{}.png", stem(input, *size, *variant)), img.encode_png()).unwrap();
        }
    }
    let runs = run_all(&presets, &drawn);
    let pick = |preset: usize, f: &dyn Fn(&Run) -> bool| runs.iter().filter(|r| r.preset == preset && f(r)).collect::<Vec<_>>();
    let mut md = String::new();

    md += "## 1. Each preset on the inputs it is meant for\n\nAll sizes and variants (clean, aliased, blurred, noisy, scanned) of those inputs.\n\n| preset | inputs ";
    md += HEADER.replacen("\n|", "\n|:--|:--|", 1).as_str();
    for (pi, (name, _)) in presets.iter().enumerate() {
        let on: Vec<&str> = inputs.iter().filter(|i| target(name, i)).map(|i| i.name).collect();
        let _ = writeln!(md, "| {name} | {} {}", on.join(", "), row(&pick(pi, &|r| r.target)));
    }

    md += "\n## 2. Each preset on each kind of input (clean, three sizes)\n\n| preset | inputs ";
    md += HEADER.replacen("\n|", "\n|:--|:--|", 1).as_str();
    for (pi, (name, _)) in presets.iter().enumerate() {
        for class in CLASSES {
            let _ = writeln!(md, "| {name} | {} {}", class.name(), row(&pick(pi, &|r| r.class == class && r.variant == Variant::Clean)));
        }
    }

    md += "\n## 3. Each preset on its inputs, by variant (256 px)\n\n| preset | variant ";
    md += HEADER.replacen("\n|", "\n|:--|:--|", 1).as_str();
    for (pi, (name, _)) in presets.iter().enumerate() {
        for variant in VARIANTS {
            let _ = writeln!(md, "| {name} | {} {}", variant.name(), row(&pick(pi, &|r| r.target && r.variant == variant)));
        }
    }

    if let Ok(dir) = std::env::var("TRACE_QUALITY_YARDSTICK") {
        // Measured as a black-and-white trace without a white background.
        let params = vectorcraft_trace::preset("Black and White Logo").unwrap();
        let theirs: Vec<Run> = drawn
            .iter()
            .filter(one_colour)
            .filter_map(|(input, size, variant, src)| {
                let svg = std::fs::read_to_string(format!("{dir}/{}.svg", stem(input, *size, *variant))).ok()?;
                let metrics = Some(measure(input, *variant, *size, src, &params, &svg_trace(&svg)));
                Some(Run { preset: usize::MAX, target: true, class: input.class, variant: *variant, metrics })
            })
            .collect();
        let _ = write!(md, "\n## 4. Yardstick\n\n{} traces by another tracer, measured the same way.\n\n| inputs | variant ", theirs.len());
        md += HEADER.replacen("\n|", "\n|:--|:--|", 1).as_str();
        for class in [Class::Shape, Class::Poly, Class::Lines] {
            for variant in VARIANTS {
                let rows: Vec<&Run> = theirs.iter().filter(|r| r.class == class && r.variant == variant).collect();
                let _ = writeln!(md, "| {} | {} {}", class.name(), variant.name(), row(&rows));
            }
        }
    }

    // Cost, one trace at a time so that the memory peak is that trace's own.
    let sizes: Vec<u32> = std::env::var("TRACE_QUALITY_MP")
        .unwrap_or_else(|_| "1,4,16".into())
        .split(',')
        .filter_map(|s| s.trim().parse::<f64>().ok())
        .map(|mp| (mp.sqrt() * 1024.0).round() as u32)
        .collect();
    let (photo, badge) = (inputs.iter().find(|i| i.name == "photo-like").unwrap(), inputs.iter().find(|i| i.name == "four-colour badge").unwrap());
    md += "\n## 5. Cost\n\nOne trace at a time on this machine (all cores); memory is the peak allocated above the image itself. A trace of more than 2,000,000 anchors is refused, as in the app.\n\n| preset | image | megapixels | result |\n|:--|:--|--:|:--|\n";
    for size in sizes {
        let n = size as usize;
        let images = [("photo-like", photo.draw(size, Variant::Clean)), ("four-colour badge", raster::render(&badge.scaled(size), badge.bg, n, n))];
        for (name, params) in &presets {
            for (image, img) in &images {
                let _ = writeln!(md, "| {name} | {image} | {:.0} | {} |", f64::from(size).powi(2) / f64::from(1 << 20), cost(img, params));
            }
        }
    }
    md
}

#[test]
#[ignore = "slow: measures every preset and writes the quality report (run with --release)"]
fn quality_report() {
    let md = report();
    println!("{md}");
    if let Ok(path) = std::env::var("TRACE_QUALITY_REPORT") {
        std::fs::write(path, md).unwrap();
    }
}

/// The harness measures what it says: its rasteriser gives a disc its area, an image is at no
/// distance from itself, and a disc traced in black and white lies on the disc.
#[test]
fn harness_measures_a_disc() {
    let disc = kurbo::Shape::to_path(&kurbo::Circle::new((40.3, 39.6), 25.0), 1e-3);
    let area: f32 = raster::coverage(&disc, 80, 80).iter().sum();
    assert!((f64::from(area) - std::f64::consts::PI * 625.0).abs() < 2.0, "{area}");

    let inputs = inputs();
    let circle = &inputs[0];
    let src = circle.draw(128, Variant::Clean);
    assert_eq!(measure::fidelity(&src, &src), (0.0, 0.0));
    let params = vectorcraft_trace::preset("Black and White Logo").unwrap();
    let res = trace_within(&src, &params, MAX_ANCHORS).unwrap();
    let m = measure(circle, Variant::Clean, 128, &src, &params, &res);
    assert_eq!(m.paths, 1.0);
    assert!(m.geo_mean < 0.06 && m.geo_max < 0.3 && m.miss_max < 0.3, "{m:?}");
    assert!(m.delta < 0.2 && m.kinks == 0.0 && m.gap.is_nan(), "{m:?}");
}