From b26fbb6247977158fb7022c303e0bba25333667d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 03:44:03 -0400 Subject: [PATCH 1/9] M10.2: VP9 inter frames (motion search, MV prediction, loop filter, forward probability updates, rate control) effectcraft-vp9enc gains inter frames: single LAST reference, 16x16/8x8 blocks, candidate + diamond search refined to quarter samples with the regular 8-tap filters, NEARESTMV/NEARMV/ZEROMV/NEWMV with spec MV prediction, skip blocks, the in-loop deblocking filter, per-frame forward probability updates, a key frame interval and target-bitrate rate control. Decoded output matches the encoder's reconstruction exactly through FilmCraft's decoder and ffmpeg. WebM export now encodes sequentially with a key frame every two seconds. Co-Authored-By: Claude Opus 5.5 --- crates/export/src/lib.rs | 2 +- crates/export/src/webm.rs | 72 ++- crates/export/tests/vp9_roundtrip.rs | 181 +++++- crates/vp9enc/Cargo.toml | 2 +- crates/vp9enc/README.md | 52 +- crates/vp9enc/src/bool.rs | 25 - crates/vp9enc/src/lf.rs | 124 ++++ crates/vp9enc/src/lib.rs | 934 ++++++++------------------- crates/vp9enc/src/mc.rs | 207 ++++++ crates/vp9enc/src/probs.rs | 426 ++++++++++++ crates/vp9enc/src/tables.rs | 73 +++ crates/vp9enc/src/tile.rs | 894 +++++++++++++++++++++++++ crates/vp9enc/src/transform.rs | 249 +++++++ docs/parity.md | 2 +- 14 files changed, 2511 insertions(+), 732 deletions(-) create mode 100644 crates/vp9enc/src/lf.rs create mode 100644 crates/vp9enc/src/mc.rs create mode 100644 crates/vp9enc/src/probs.rs create mode 100644 crates/vp9enc/src/tile.rs create mode 100644 crates/vp9enc/src/transform.rs diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index 941a7723..c9196625 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -7,7 +7,7 @@ //! | PNG / JPEG / TIFF sequence | one file per frame | 8-bit | PNG, TIFF | — | //! | OpenEXR sequence | one file per frame | 32-bit float, linear light, premultiplied | yes | — | //! | Animated GIF | GIF89a | 256-colour palette per frame (NeuQuant) | 1-bit | — | -//! | WebM | WebM (Matroska) | VP9 profile 0 intra frames (`effectcraft-vp9enc`), 8-bit 4:2:0 | VP9 alpha (BlockAdditional) | Opus (`effectcraft-opusenc`), 48 kHz stereo | +//! | WebM | WebM (Matroska) | VP9 profile 0 (`effectcraft-vp9enc`: key + inter frames, a key frame every 2 s, loop filter), 8-bit 4:2:0 | VP9 alpha (BlockAdditional) | Opus (`effectcraft-opusenc`), 48 kHz stereo | //! | WAV / AIFF | RIFF WAVE / AIFF | — | — | 16-bit PCM stereo (audio only) | //! //! Frames are rendered in parallel batches (rayon; one batch ≈ one frame per core) and handed to diff --git a/crates/export/src/webm.rs b/crates/export/src/webm.rs index eb99cd22..65ca1189 100644 --- a/crates/export/src/webm.rs +++ b/crates/export/src/webm.rs @@ -5,8 +5,10 @@ //! (size patched at the end), Info, Tracks and one Cluster per render batch. Video frames are //! SimpleBlocks; with alpha (Channels: RGB + Alpha) each frame is a BlockGroup whose //! BlockAdditional (BlockAddID 1) carries a second VP9 frame coding the alpha channel as luma, -//! with `AlphaMode` 1 on the track, as WebM defines. VP9 comes from `effectcraft-vp9enc` (every -//! frame a key frame), Opus from `effectcraft-opusenc` (CELT, 48 kHz stereo, 20 ms packets, +//! with `AlphaMode` 1 on the track, as WebM defines. VP9 comes from `effectcraft-vp9enc` (key +//! frames every two seconds, inter frames between them: SimpleBlocks carry the key flag, inter +//! BlockGroups a ReferenceBlock; the alpha stream's key frames line up with the colour stream's), +//! Opus from `effectcraft-opusenc` (CELT, 48 kHz stereo, 20 ms packets, //! `CodecDelay` = pre-skip). WAV (RIFF, 16-bit PCM) and AIFF (big-endian 16-bit, 80-bit //! extended sample rate) follow their published file layouts. @@ -82,6 +84,10 @@ fn block(track: u64, rel: i16, flags: u8, data: &[u8]) -> Vec { struct Frame { ms: i64, track: u64, + /// A key frame (VP9) or any audio packet. + key: bool, + /// Timestamp of the previous video frame (ReferenceBlock of inter frames). + prev_ms: Option, data: Vec, alpha: Option>, } @@ -94,10 +100,15 @@ fn cluster(frames: &mut [Frame]) -> Vec { for f in frames.iter() { let rel = (f.ms - base).clamp(i16::MIN as i64, i16::MAX as i64) as i16; match &f.alpha { - None => el(&mut body, 0xA3, &block(f.track, rel, 0x80, &f.data)), + None => el(&mut body, 0xA3, &block(f.track, rel, if f.key { 0x80 } else { 0x00 }, &f.data)), Some(a) => { let mut g = vec![]; el(&mut g, 0xA1, &block(f.track, rel, 0x00, &f.data)); + if let (false, Some(p)) = (f.key, f.prev_ms) { + // ReferenceBlock: a signed timestamp relative to this block (an inter frame). + let d = (p - f.ms).clamp(i16::MIN as i64, -1) as i16; + el(&mut g, 0xFB, &d.to_be_bytes()); + } let mut more = vec![]; el_uint(&mut more, 0xEE, 1); el(&mut more, 0xA5, a); @@ -117,7 +128,19 @@ pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re let rate = job.settings.rate(comp); let alpha = job.output.channels == Channels::Rgba; let quality = job.output.quality.clamp(1, 100); - let cfg = effectcraft_vp9enc::EncoderConfig { width: w, height: h, quality, full_range: false }; + let fps = rate.as_f64().max(1.0); + let cfg = effectcraft_vp9enc::EncoderConfig { + width: w, + height: h, + quality, + full_range: false, + keyframe_interval: (fps * 2.0).round().max(1.0) as u32, + target_kbps: None, + frame_rate: fps, + loop_filter: true, + }; + let mut color_enc = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone()); + let mut alpha_enc = alpha.then(|| effectcraft_vp9enc::Vp9Encoder::new(cfg.clone())); let with_audio = wants_audio(job); let mut opus = with_audio.then(|| effectcraft_opusenc::OpusEncoder::new(2, OPUS_BITRATE)); let total = st.total; @@ -209,7 +232,7 @@ pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re while pcm_left.len() >= fs { let pkt = enc.encode_float(&pcm_left[..fs]); pcm_left.drain(..fs); - frames.push(Frame { ms: packets as i64 * 20, track: 2, data: pkt, alpha: None }); + frames.push(Frame { ms: packets as i64 * 20, track: 2, key: true, prev_ms: None, data: pkt, alpha: None }); packets += 1; } }; @@ -218,23 +241,28 @@ pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re let mut i = 0; while i < total { let end = (i + batch).min(total); - let encoded: Vec<(Vec, Option>)> = (i..end) - .into_par_iter() - .map(|k| { - let px = rgba8(&render_frame(job, comp, k), comp, if alpha { Channels::Rgba } else { Channels::Rgb }, w, h); - let mut enc = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone()); - let (y, u, vv) = effectcraft_vp9enc::rgba_to_yuv420(&px, w, h, false); - let color = enc.encode_yuv420(&y, &u, &vv); - let a = alpha.then(|| { - let mut ea = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone()); - let (y, u, vv) = effectcraft_vp9enc::alpha_to_yuv420(&px, w, h); - ea.encode_yuv420(&y, &u, &vv) - }); - (color, a) - }) - .collect(); - let mut frames: Vec = - encoded.into_iter().enumerate().map(|(j, (data, alpha))| Frame { ms: frame_ms(i + j as u64), track: 1, data, alpha }).collect(); + // Frames render in parallel; the VP9 streams (inter-coded) encode in order, colour and + // alpha side by side. + let pixels: Vec> = + (i..end).into_par_iter().map(|k| rgba8(&render_frame(job, comp, k), comp, if alpha { Channels::Rgba } else { Channels::Rgb }, w, h)).collect(); + let mut frames: Vec = Vec::with_capacity(pixels.len()); + for (j, px) in pixels.iter().enumerate() { + let (color, a) = rayon::join( + || { + let (y, u, vv) = effectcraft_vp9enc::rgba_to_yuv420(px, w, h, false); + color_enc.encode_frame(&y, &u, &vv, false) + }, + || { + alpha_enc.as_mut().map(|ea| { + let (y, u, vv) = effectcraft_vp9enc::alpha_to_yuv420(px, w, h); + ea.encode_frame(&y, &u, &vv, false).data + }) + }, + ); + let k = i + j as u64; + frames.push(Frame { ms: frame_ms(k), track: 1, key: color.key, prev_ms: (k > 0).then(|| frame_ms(k - 1)), data: color.data, alpha: a }); + } + drop(pixels); let t_end = if end >= total { span_end } else { job.settings.frame_time(comp, end) }; audio_until(t_end, &mut frames, end >= total); let c = cluster(&mut frames); diff --git a/crates/export/tests/vp9_roundtrip.rs b/crates/export/tests/vp9_roundtrip.rs index 8156d850..5bf46e0a 100644 --- a/crates/export/tests/vp9_roundtrip.rs +++ b/crates/export/tests/vp9_roundtrip.rs @@ -61,7 +61,7 @@ fn lossless_is_exact_for_odd_sizes() { for (w, h) in [(1, 1), (33, 17), (64, 64), (200, 120), (7, 70)] { for kind in 0..3 { let (y, u, v) = source(w, h, kind); - let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false }); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false, ..Default::default() }); let f = e.encode_yuv420(&y, &u, &v); let p = decode(&f); assert_eq!((p.width, p.height), (w as u32, h as u32)); @@ -80,7 +80,7 @@ fn lossy_quality_and_size() { let (y, u, v) = source(w, h, kind); let mut last = 0usize; for q in [30u8, 60, 80, 95] { - let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: q, full_range: false }); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: q, full_range: false, ..Default::default() }); let f = e.encode_yuv420(&y, &u, &v); let p = decode(&f); let got = crop(&plane_u8(&p.y), p.y_stride, w, h); @@ -103,7 +103,7 @@ fn wide_frames_use_tile_columns() { // 4160 px > 64 superblocks: two tile columns are required. let (w, h) = (4160, 16); let (y, u, v) = source(w, h, 0); - let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 90, full_range: false }); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 90, full_range: false, ..Default::default() }); let f = e.encode_yuv420(&y, &u, &v); let p = decode(&f); assert!(psnr(&crop(&plane_u8(&p.y), p.y_stride, w, h), &y) > 35.0); @@ -113,7 +113,7 @@ fn wide_frames_use_tile_columns() { fn ffmpeg_decodes_our_stream() { let (w, h) = (96, 64); let (y, u, v) = source(w, h, 0); - let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false }); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false, ..Default::default() }); let frame = e.encode_yuv420(&y, &u, &v); // IVF container (one frame). let mut ivf = Vec::new(); @@ -147,3 +147,176 @@ fn ffmpeg_decodes_our_stream() { assert_eq!(&out.stdout[..w * h], &y[..], "ffmpeg luma matches (lossless)"); assert_eq!(&out.stdout[w * h..w * h + u.len()], &u[..]); } + +// ---------------------------------------------------------------- inter frames + +/// Frame `t` of a moving sequence: a diagonal gradient with a sinusoidal texture and bright +/// squares, all translating by a fractional amount per frame (`kind` 1 adds noise that changes +/// every frame). +fn moving(w: usize, h: usize, t: usize, kind: u32) -> (Vec, Vec, Vec) { + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let (dx, dy) = (1.75 * t as f64, 0.6 * t as f64); + let mut s = 1234u32 + t as u32 * 77; + let mut rnd = || { + s = s.wrapping_mul(1664525).wrapping_add(1013904223); + (s >> 28) as f64 + }; + let lum = |x: f64, y: f64| -> f64 { + let g = 40.0 + 120.0 * (x / w as f64) + 50.0 * (y / h as f64); + let tex = 18.0 * ((x * 0.31).sin() * (y * 0.23).cos()); + let sq = if (x - 20.0).rem_euclid(64.0) < 18.0 && (y - 10.0).rem_euclid(48.0) < 14.0 { 45.0 } else { 0.0 }; + g + tex + sq + }; + let mut y = vec![0u8; w * h]; + for yy in 0..h { + for x in 0..w { + let n = if kind == 1 { rnd() - 7.5 } else { 0.0 }; + y[yy * w + x] = (lum(x as f64 - dx, yy as f64 - dy) + n).round().clamp(0.0, 255.0) as u8; + } + } + let u: Vec = (0..cw * ch).map(|i| (100.0 + 40.0 * (((i % cw) as f64 * 2.0 - dx) / w as f64)).round() as u8).collect(); + let v: Vec = (0..cw * ch).map(|i| (150.0 - 30.0 * (((i / cw) as f64 * 2.0 - dy) / h as f64)).round() as u8).collect(); + (y, u, v) +} + +fn planes_of(p: &filmcraft_vp9::Picture, w: usize, h: usize) -> [Vec; 3] { + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + [crop(&plane_u8(&p.y), p.y_stride, w, h), crop(&plane_u8(&p.u), p.uv_stride, cw, ch), crop(&plane_u8(&p.v), p.uv_stride, cw, ch)] +} + +/// Encode `n` frames and decode them with one decoder: every decoded frame must equal the +/// encoder's reconstruction. Returns (total bytes, mean luma PSNR vs the source). +fn run_sequence(cfg: EncoderConfig, n: usize, kind: u32) -> (usize, f64) { + let (w, h) = (cfg.width as usize, cfg.height as usize); + let mut e = Vp9Encoder::new(cfg); + let mut d = Decoder::new(); + let (mut bytes, mut db) = (0usize, 0.0); + for t in 0..n { + let (y, u, v) = moving(w, h, t, kind); + let f = e.encode_frame(&y, &u, &v, false); + bytes += f.data.len(); + let kind_s = if f.key { "key" } else { "inter" }; + let mut pics = d.decode(&f.data, t as i64).unwrap_or_else(|err| panic!("{w}x{h} frame {t} ({kind_s}): {err:?}")); + assert_eq!(pics.len(), 1); + let got = planes_of(&pics.remove(0), w, h); + let rec = e.reconstruction().expect("reconstruction"); + for k in 0..3 { + if got[k] != rec[k] { + let first = got[k].iter().zip(&rec[k]).position(|(a, b)| a != b).unwrap_or(0); + let pw = if k == 0 { w } else { w.div_ceil(2) }; + panic!("{w}x{h} frame {t} ({kind_s}, q{} lf{}): plane {k} differs first at ({}, {})", f.q_idx, f.filter_level, first % pw, first / pw); + } + } + db += psnr(&got[0], &y); + } + (bytes, db / n as f64) +} + +#[test] +fn inter_frames_decode_exactly() { + for (w, h) in [(64, 64), (96, 72), (33, 17), (130, 66)] { + for (q, kind) in [(30u8, 0u32), (70, 1), (92, 0), (100, 0)] { + let cfg = EncoderConfig { width: w as u32, height: h as u32, quality: q, keyframe_interval: 6, ..Default::default() }; + run_sequence(cfg, 8, kind); + } + } +} + +#[test] +fn inter_frames_with_tile_columns_decode_exactly() { + let cfg = EncoderConfig { width: 4160, height: 24, quality: 80, keyframe_interval: 0, ..Default::default() }; + run_sequence(cfg, 3, 0); +} + +#[test] +fn inter_psnr_rises_with_bitrate() { + let (w, h) = (160, 96); + let mut last = (0usize, 0.0f64); + let mut rows = vec![]; + for kbps in [60u32, 150, 400, 1200] { + let cfg = EncoderConfig { width: w, height: h, quality: 60, target_kbps: Some(kbps), frame_rate: 30.0, keyframe_interval: 30, ..Default::default() }; + let (bytes, db) = run_sequence(cfg, 20, 0); + rows.push(format!("{kbps} kbps target: {:.0} kbps actual, {db:.2} dB", bytes as f64 * 8.0 * 30.0 / 20.0 / 1000.0)); + assert!(bytes > last.0 && db > last.1, "{rows:#?}"); + last = (bytes, db); + } + eprintln!("{rows:#?}"); +} + +#[test] +fn inter_frames_are_much_smaller_than_intra() { + let (w, h) = (192u32, 108u32); + let mut report = vec![]; + for q in [50u8, 75, 90] { + let intra = EncoderConfig { width: w, height: h, quality: q, keyframe_interval: 1, ..Default::default() }; + let inter = EncoderConfig { keyframe_interval: 0, ..intra.clone() }; + let (bi, pi) = run_sequence(intra, 24, 0); + let (bp, pp) = run_sequence(inter, 24, 0); + report.push(format!("q{q}: intra {bi} B {pi:.2} dB, inter {bp} B {pp:.2} dB, ratio {:.2}", bi as f64 / bp as f64)); + assert!(pp > pi - 1.0, "{report:#?}"); + assert!(bi as f64 >= 3.0 * bp as f64, "{report:#?}"); + } + eprintln!("{report:#?}"); +} + +#[test] +fn ffmpeg_decodes_inter_frames_like_the_encoder() { + let (w, h) = (96usize, 64usize); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 75, keyframe_interval: 5, ..Default::default() }); + let n = 7; + let mut ivf = Vec::new(); + ivf.extend_from_slice(b"DKIF"); + ivf.extend_from_slice(&0u16.to_le_bytes()); + ivf.extend_from_slice(&32u16.to_le_bytes()); + ivf.extend_from_slice(b"VP90"); + ivf.extend_from_slice(&(w as u16).to_le_bytes()); + ivf.extend_from_slice(&(h as u16).to_le_bytes()); + ivf.extend_from_slice(&30u32.to_le_bytes()); + ivf.extend_from_slice(&1u32.to_le_bytes()); + ivf.extend_from_slice(&(n as u32).to_le_bytes()); + ivf.extend_from_slice(&0u32.to_le_bytes()); + let mut recon = vec![]; + for t in 0..n { + let (y, u, v) = moving(w, h, t, 0); + let f = e.encode_frame(&y, &u, &v, false); + ivf.extend_from_slice(&(f.data.len() as u32).to_le_bytes()); + ivf.extend_from_slice(&(t as u64).to_le_bytes()); + ivf.extend_from_slice(&f.data); + recon.push(e.reconstruction().expect("recon")); + } + let dir = std::env::temp_dir().join(format!("effectcraft-vp9-inter-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("a.ivf"); + std::fs::write(&path, &ivf).unwrap(); + let out = std::process::Command::new("ffmpeg") + .args(["-hide_banner", "-loglevel", "error", "-i"]) + .arg(&path) + .args(["-f", "rawvideo", "-pix_fmt", "yuv420p", "-"]) + .output(); + let Ok(out) = out else { + eprintln!("ffmpeg not found: skipping oracle"); + return; + }; + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + let fsz = w * h + 2 * w.div_ceil(2) * h.div_ceil(2); + assert_eq!(out.stdout.len(), fsz * n); + for (t, r) in recon.iter().enumerate() { + let frame = &out.stdout[t * fsz..(t + 1) * fsz]; + assert_eq!(&frame[..w * h], &r[0][..], "ffmpeg luma of frame {t}"); + assert_eq!(&frame[w * h..w * h + r[1].len()], &r[1][..], "ffmpeg U of frame {t}"); + } +} + +/// Encoder speed on 720p (run with `--ignored --nocapture`). +#[test] +#[ignore] +fn bench_720p() { + let (w, h) = (1280usize, 720usize); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 80, keyframe_interval: 0, ..Default::default() }); + for t in 0..6 { + let (y, u, v) = moving(w, h, t, 0); + let t0 = std::time::Instant::now(); + let f = e.encode_frame(&y, &u, &v, false); + eprintln!("frame {t} ({}): {} B in {:.0} ms", if f.key { "key" } else { "inter" }, f.data.len(), t0.elapsed().as_secs_f64() * 1000.0); + } +} diff --git a/crates/vp9enc/Cargo.toml b/crates/vp9enc/Cargo.toml index 62144b25..fe2e767c 100644 --- a/crates/vp9enc/Cargo.toml +++ b/crates/vp9enc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-vp9enc" -description = "VP9 intra-frame encoder (profile 0, 8-bit 4:2:0) for WebM export, from the VP9 Bitstream & Decoding Process Specification" +description = "VP9 encoder (profile 0, 8-bit 4:2:0; intra + inter frames, loop filter, rate control) for WebM export, from the VP9 Bitstream & Decoding Process Specification" version.workspace = true edition.workspace = true license.workspace = true diff --git a/crates/vp9enc/README.md b/crates/vp9enc/README.md index c8f781b8..b9f94fae 100644 --- a/crates/vp9enc/README.md +++ b/crates/vp9enc/README.md @@ -1,15 +1,47 @@ # effectcraft-vp9enc -A small VP9 encoder for WebM export: every frame is a key frame (profile 0, 8-bit 4:2:0, -BT.709), coded with 8×8 intra blocks (DC / V / H / TM prediction chosen per block), 4×4 -DCT/ADST transforms with default probabilities, or the Walsh–Hadamard transform at quality 100 -(mathematically lossless). +A small VP9 encoder for WebM export (profile 0, 8-bit 4:2:0, BT.709). + +* **Key frames**: 8×8 intra blocks (DC / V / H / TM prediction chosen per block), 4×4 DCT/ADST + transforms, or the Walsh–Hadamard transform at quality 100 (mathematically lossless). +* **Inter frames**: one reference (the previous frame, LAST), 16×16 or 8×8 blocks chosen by a + cost comparison; motion search from the predicted vectors (NearestMv / NearMv / last frame's + co-located vector) with a whole-sample cross/diamond search refined to half and quarter + samples with the regular 8-tap interpolation filters; NEARESTMV / NEARMV / ZEROMV / NEWMV + chosen by bit cost; blocks with no coefficients worth coding are skipped; 4×4 DCT residual + with a rate–distortion check per transform block. Inter frames are coded error resilient + (`error_resilient_mode` = 1: no dependence on earlier probability contexts or motion vectors), + so `UsePrevFrameMvs` is never needed. No high-precision (1/8) vectors, no compound prediction, + no intra blocks inside inter frames. +* **Loop filter**: the in-loop deblocking filter (§8.8) on every lossy frame; the level is the + best of three trials against the source (sharpness 0, no mode/reference deltas). +* **Probabilities**: per-frame forward updates in the compressed header (`diff_update_prob` / + `update_mv_prob`) of the coefficient, skip, inter-mode, is-inter, reference, partition and + motion-vector probabilities. Tiles are recorded as symbols first, the frame's own counts pick + the updates that pay for their signalling, then the symbols are entropy coded with the updated + probabilities. There is no backward adaptation (`refresh_frame_context` = 0). +* **Rate control**: constant quality (quality → `base_q_idx`) or a target bitrate: a per-frame + q index from a bits ≈ complexity / step model per frame type, key frames getting a larger share + of each key-frame interval, with the running over/undershoot paid back over about a second. +* **Key frame interval** is configurable (1 = every frame a key frame, the old intra-only mode). + +The encoder reconstructs exactly what a decoder reconstructs (inter prediction with the +decoder's vector clamping and edge-clamped reference fetches, integer inverse transforms, the +loop filter) and exposes it (`Vp9Encoder::reconstruction`); the tests require FilmCraft's VP9 +decoder (and ffmpeg, when installed) to output exactly that reconstruction, frame by frame, for +moving synthetic sequences at several sizes and qualities, with tile columns, and lossless. + +On a moving-gradient sequence (192×108, 24 frames) inter coding is 3.9–7× smaller than all-intra +at the same quality setting with equal or better PSNR. Written from the **VP9 Bitstream & Decoding Process Specification, version 0.6 (31 March 2016)**: the encoder emits the uncompressed and compressed headers (§6.2, §6.3), partition, mode -info and token syntax (§6.4) and reconstructs with the specification's intra prediction (§8.5.1) -and integer inverse transforms (§8.7) so it stays in step with any conformant decoder. The -boolean encoder is the inverse of the §9.2 decoder in the form given in RFC 6386 §7.3. Spec -tables (§10) were transcribed from FilmCraft's first-party decoder `filmcraft-vp9` (MIT OR -Apache-2.0), which is also the decoder the tests round-trip through; ffmpeg is used only as an -external test oracle. No libvpx or other third-party encoder/decoder source was consulted. +info, motion vector and token syntax (§6.4), derives motion vector candidates with the §6.4 +`find_mv_refs` / `find_best_ref_mvs` processes, and reconstructs with the specification's intra +and inter prediction (§8.5), integer inverse transforms (§8.7) and loop filter (§8.8) so it stays +in step with any conformant decoder. The boolean encoder is the inverse of the §9.2 decoder in +the form given in RFC 6386 §7.3. Spec tables (§10) are the specification's; the key-frame tables +were transcribed from FilmCraft's first-party decoder `filmcraft-vp9` (MIT OR Apache-2.0), which +is also the decoder the tests round-trip through and whose behaviour (loop filter edge rules, +vector candidate order) the encoder was checked against; ffmpeg is used only as an external test +oracle. No libvpx or other third-party encoder/decoder source was consulted. diff --git a/crates/vp9enc/src/bool.rs b/crates/vp9enc/src/bool.rs index 45db92dd..0cd91b0e 100644 --- a/crates/vp9enc/src/bool.rs +++ b/crates/vp9enc/src/bool.rs @@ -64,31 +64,6 @@ impl BoolEncoder { } } - /// Write `value` with a tree (leaves stored negated, as in the decoder's tables). - pub fn tree(&mut self, tree: &[i8], probs: &[u8], value: u8) { - // Find the path to the leaf. - fn path(tree: &[i8], node: usize, value: u8, out: &mut Vec<(usize, bool)>) -> bool { - for b in 0..2 { - let t = tree[node + b]; - out.push((node >> 1, b == 1)); - if t <= 0 { - if (-t) as u8 == value { - return true; - } - } else if path(tree, t as usize, value, out) { - return true; - } - out.pop(); - } - false - } - let mut p = Vec::with_capacity(8); - path(tree, 0, value, &mut p); - for (i, bit) in p { - self.write(bit, probs[i]); - } - } - /// Flush (RFC 6386 §7.3) and return the bytes. pub fn finish(mut self) -> Vec { let mut c = self.bit_count; diff --git a/crates/vp9enc/src/lf.rs b/crates/vp9enc/src/lf.rs new file mode 100644 index 00000000..d781144f --- /dev/null +++ b/crates/vp9enc/src/lf.rs @@ -0,0 +1,124 @@ +//! The in-loop deblocking filter (spec 8.8), as the decoder applies it to a frame coded with 4×4 +//! transforms, 8×8 / 16×16 blocks, no segmentation and no reference/mode deltas: superblocks in +//! raster order, for each plane the vertical edges (pass 0) then the horizontal edges (pass 1). +//! Transform edges inside a skipped inter block are left alone; every 4×4 edge uses the 4-tap +//! filter except edges on a 32-sample boundary, which get the 8-tap (flat) filter. + +use crate::{BLOCK_16X16, Mi, Plane}; + +#[derive(Clone, Copy)] +struct Limits { + limit: i32, + blimit: i32, + thresh: i32, +} + +/// Filter the reconstructed planes (Y, U, V, or a prefix of them) in place at `level` (0 = off; +/// sharpness 0). +pub fn filter_frame(planes: &mut [Plane], mi: &[Mi], mi_rows: usize, mi_cols: usize, level: u8) { + if level == 0 { + return; + } + let l = level as i32; + let limit = l.max(1); + let lim = Limits { limit, blimit: 2 * (l + 2) + limit, thresh: l >> 4 }; + for sbr in 0..mi_rows.div_ceil(8) { + for sbc in 0..mi_cols.div_ceil(8) { + for (plane, p) in planes.iter_mut().enumerate() { + for pass in 0..2 { + sb_plane(p, mi, mi_rows, mi_cols, plane, pass, sbr * 8, sbc * 8, lim); + } + } + } + } +} + +#[allow(clippy::too_many_arguments)] +fn sb_plane(p: &mut Plane, mi: &[Mi], mi_rows: usize, mi_cols: usize, plane: usize, pass: usize, row: usize, col: usize, lim: Limits) { + let (sx, sy) = if plane > 0 { (1, 1) } else { (0, 0) }; + let (sub, edge_len) = if pass == 0 { (sx, 64 >> sy) } else { (sy, 64 >> sx) }; + let stride = p.w; + for edge in 0..(16usize >> sub) { + let mut i = 0; + while i < edge_len { + let (x, y) = if pass == 0 { (col * 8 + edge * (4 << sx), row * 8 + (i << sy)) } else { (col * 8 + (i << sx), row * 8 + edge * (4 << sy)) }; + if x >= 8 * mi_cols || y >= 8 * mi_rows || (pass == 0 && x == 0) || (pass == 1 && y == 0) { + i += 8; + continue; + } + let lc = ((x >> 3) >> sx) << sx; + let lr = ((y >> 3) >> sy) << sy; + let m = mi[lr * mi_cols + lc]; + let size = if sub == 0 { m.size } else { m.size.max(BLOCK_16X16) }; + let n8 = if size >= BLOCK_16X16 { 2 } else { 1 }; + let is_block_edge = if pass == 0 { x & (8 * n8 - 1) == 0 } else { y & (8 * n8 - 1) == 0 }; + let tx_ok = !m.inter || !m.skip; + if !is_block_edge && !tx_ok { + i += 8; + continue; + } + let wide = edge % 8 == 0; + let n = if pass == 0 { (8 * mi_rows - y).div_ceil(1 << sy) } else { (8 * mi_cols - x).div_ceil(1 << sx) }.min(8).min(edge_len - i); + // A transform edge is not filtered in the last odd chroma column. + let odd_last = pass == 1 && sx == 1 && mi_cols & 1 == 1 && edge & 1 == 1; + let (px, py) = (x >> sx, y >> sy); + for l in 0..n { + let is_tx_edge = tx_ok && !(odd_last && x + (l << sx) + 8 >= mi_cols * 8); + if !(is_block_edge || is_tx_edge) { + continue; + } + let (pos, step) = if pass == 0 { ((py + l) * stride + px, 1) } else { (py * stride + px + l, stride) }; + filter_lane(&mut p.rec, pos, step, wide, lim); + } + i += 8; + } + } +} + +/// Filter mask, narrow (4-tap) and 8-tap flat filters of spec 8.8.5 across one edge position; +/// `pos` is q0, samples across the edge are `step` apart. +fn filter_lane(d: &mut [u8], pos: usize, step: usize, wide: bool, lim: Limits) { + let at = |k: isize| d[(pos as isize + k * step as isize) as usize] as i32; + let (p3, p2, p1, p0) = (at(-4), at(-3), at(-2), at(-1)); + let (q0, q1, q2, q3) = (at(0), at(1), at(2), at(3)); + let Limits { limit, blimit, thresh } = lim; + let mask = (p3 - p2).abs() <= limit + && (p2 - p1).abs() <= limit + && (p1 - p0).abs() <= limit + && (q1 - q0).abs() <= limit + && (q2 - q1).abs() <= limit + && (q3 - q2).abs() <= limit + && (p0 - q0).abs() * 2 + (p1 - q1).abs() / 2 <= blimit; + if !mask { + return; + } + let flat = + wide && (p1 - p0).abs() <= 1 && (q1 - q0).abs() <= 1 && (p2 - p0).abs() <= 1 && (q2 - q0).abs() <= 1 && (p3 - p0).abs() <= 1 && (q3 - q0).abs() <= 1; + let mut set = |k: isize, v: i32| d[(pos as isize + k * step as isize) as usize] = v as u8; + if flat { + // 8-tap: positions p2..q2, each the rounded mean of a 7-wide clamped window plus itself. + let s = [p3, p2, p1, p0, q0, q1, q2, q3]; + let px = |k: isize| s[(k.clamp(-4, 3) + 4) as usize]; + for k in -3isize..3 { + let mut sum = px(k); + for j in -3..=3 { + sum += px(k + j); + } + set(k, (sum + 4) >> 3); + } + return; + } + let c = |v: i32| v.clamp(-128, 127); + let hev = (p1 - p0).abs() > thresh || (q1 - q0).abs() > thresh; + let (ps1, ps0, qs0, qs1) = (p1 - 128, p0 - 128, q0 - 128, q1 - 128); + let filter = c(if hev { c(ps1 - qs1) } else { 0 } + 3 * (qs0 - ps0)); + let f1 = c(filter + 4) >> 3; + let f2 = c(filter + 3) >> 3; + set(0, c(qs0 - f1) + 128); + set(-1, c(ps0 + f2) + 128); + if !hev { + let f = (f1 + 1) >> 1; + set(1, c(qs1 - f) + 128); + set(-2, c(ps1 + f) + 128); + } +} diff --git a/crates/vp9enc/src/lib.rs b/crates/vp9enc/src/lib.rs index 02cbb0e2..8f72cb7b 100644 --- a/crates/vp9enc/src/lib.rs +++ b/crates/vp9enc/src/lib.rs @@ -1,290 +1,89 @@ -//! A VP9 intra-frame encoder (profile 0, 8-bit 4:2:0), written from the *VP9 Bitstream & -//! Decoding Process Specification* v0.6 by mirroring its decoding process in reverse (see -//! README.md). Every frame is a key frame: one tile row (tile columns as the frame width -//! requires), 64×64 superblocks split to 8×8 blocks, 4×4 transforms (the Walsh–Hadamard -//! transform at quality 100, which is lossless), intra modes DC / V / H / TM chosen per block by -//! prediction error, default probabilities (no probability updates). The encoder reconstructs -//! exactly what a decoder reconstructs (the specification's integer inverse transforms and -//! intra edge rules), so prediction never drifts. +//! A VP9 encoder (profile 0, 8-bit 4:2:0), written from the *VP9 Bitstream & Decoding Process +//! Specification* v0.6 by mirroring its decoding process in reverse (see README.md). +//! +//! * **Key frames**: 64×64 superblocks split to 8×8 intra blocks (DC / V / H / TM chosen per +//! block), 4×4 DCT/ADST transforms (the Walsh–Hadamard transform at quality 100: lossless). +//! * **Inter frames** (error resilient, single reference: the previous frame as LAST): 16×16 or +//! 8×8 blocks with motion vectors found by a candidate + diamond search refined to quarter +//! samples with the regular 8-tap filters; NEARESTMV / NEARMV / ZEROMV / NEWMV chosen by cost, +//! motion vector prediction exactly as the decoder derives it; skipped blocks; 4×4 DCT residual. +//! * **In-loop deblocking** (8.8) on every lossy frame; the level is picked per frame by trial. +//! * **Probabilities**: per-frame forward updates (compressed header) of the coefficient, skip, +//! inter mode, reference, partition and motion vector probabilities, chosen from the frame's +//! own symbol counts (no backward adaptation: `refresh_frame_context` = 0). +//! * **Rate control**: constant quality (quality → base_q_idx), or a target bitrate with a +//! per-frame q_idx model. +//! +//! The encoder reconstructs exactly what a decoder reconstructs (prediction, integer inverse +//! transforms, loop filter), so prediction never drifts; [`Vp9Encoder::reconstruction`] returns +//! it. mod bool; +mod lf; +mod mc; +mod probs; mod tables; +mod tile; +mod transform; -use bool::BoolEncoder; -use tables::*; +pub use mc::Mv; +use probs::Probs; +use tables::{AC_QLOOKUP, DC_QLOOKUP}; +use transform::Quant; -const DC_PRED: u8 = 0; -const V_PRED: u8 = 1; -const H_PRED: u8 = 2; -const TM_PRED: u8 = 9; -const MODES: [u8; 4] = [DC_PRED, V_PRED, H_PRED, TM_PRED]; - -const DCT_DCT: u8 = 0; -const ADST_DCT: u8 = 1; -const DCT_ADST: u8 = 2; -const ADST_ADST: u8 = 3; - -/// mode2txfm_map for the modes used here. -fn mode_tx_type(mode: u8) -> u8 { - match mode { - V_PRED => ADST_DCT, - H_PRED => DCT_ADST, - TM_PRED => ADST_ADST, - _ => DCT_DCT, - } -} - -const PARTITION_TREE: [i8; 6] = [0, 2, -1, 4, -2, -3]; -const INTRA_MODE_TREE: [i8; 18] = [0, 2, -9, 4, -1, 6, 8, 12, -2, 10, -4, -5, -3, 14, -8, 16, -6, -7]; - -/// extra_bits[token] = (cat, numExtra, base). -const EXTRA_BITS: [(u8, u8, u16); 11] = - [(0, 0, 0), (0, 0, 1), (0, 0, 2), (0, 0, 3), (0, 0, 4), (1, 1, 5), (2, 2, 7), (3, 3, 11), (4, 4, 19), (5, 5, 35), (6, 14, 67)]; -const CAT_PROBS: [&[u8]; 7] = [ - &[], - &[159], - &[165, 145], - &[173, 148, 140], - &[176, 155, 140, 135], - &[180, 157, 141, 134, 130], - &[254, 254, 254, 252, 249, 243, 230, 196, 177, 153, 140, 133, 130, 129], -]; +pub(crate) const BLOCK_8X8: u8 = 3; +pub(crate) const BLOCK_16X16: u8 = 6; /// Encoder settings. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq)] pub struct EncoderConfig { pub width: u32, pub height: u32, - /// 1..=100; 100 is lossless. + /// 1..=100; 100 is lossless. With a target bitrate: the quality of the first frame. pub quality: u8, /// Full-range (0–255) rather than limited (16–235) YUV; only signalled. pub full_range: bool, + /// A key frame every this many frames (1 = every frame is a key frame; 0 = only the first). + pub keyframe_interval: u32, + /// Target bitrate in kbit/s (`None`: constant quality). + pub target_kbps: Option, + /// Frames per second (rate control). + pub frame_rate: f64, + /// The in-loop deblocking filter. + pub loop_filter: bool, } -/// Probabilities of the full token tree (9.3.2) for model probability `p`. -fn pareto(p: u8) -> [u8; 8] { - let p = p.max(1) as usize; - let x = (p - 1) / 2; - let mut t = [0u8; 8]; - for n in 0..8 { - t[n] = if p & 1 == 1 { PARETO_TABLE[x * 8 + n] } else { ((PARETO_TABLE[x * 8 + n] as u32 + PARETO_TABLE[(x + 1) * 8 + n] as u32) >> 1) as u8 }; - } - t -} - -fn scan(tx_type: u8) -> &'static [u16; 16] { - match tx_type { - ADST_DCT => &ROW_SCAN_4X4, - DCT_ADST => &COL_SCAN_4X4, - _ => &DEFAULT_SCAN_4X4, +impl Default for EncoderConfig { + fn default() -> Self { + EncoderConfig { width: 0, height: 0, quality: 80, full_range: false, keyframe_interval: 60, target_kbps: None, frame_rate: 30.0, loop_filter: true } } } -/// Coefficient context neighbours (9.3.2) of each scan position. -fn neighbors(tx_type: u8) -> [(usize, usize); 16] { - let sc = scan(tx_type); - let mut nb = [(0, 0); 16]; - for (c, &pos) in sc.iter().enumerate().skip(1) { - let pos = pos as usize; - let (i, j) = (pos / 4, pos % 4); - nb[c] = if i > 0 && j > 0 { - let a = (i - 1) * 4 + j; - let b = i * 4 + j - 1; - match tx_type { - DCT_ADST => (a, a), - ADST_DCT => (b, b), - _ => (a, b), - } - } else if i > 0 { - ((i - 1) * 4 + j, (i - 1) * 4 + j) - } else { - (j - 1, j - 1) - }; - } - nb +/// One coded frame. +#[derive(Clone, Debug)] +pub struct EncodedFrame { + pub data: Vec, + pub key: bool, + pub q_idx: u8, + pub filter_level: u8, } -// ---------------------------------------------------------------- transforms (8.7) - -fn r14(v: i32) -> i32 { - (v + (1 << 13)) >> 14 -} - -fn idct4(x: [i32; 4]) -> [i32; 4] { - let v0 = r14(x[0] * 11585 - x[2] * 11585); - let v1 = r14(x[0] * 11585 + x[2] * 11585); - let v2 = r14(x[1] * 6270 - x[3] * 15137); - let v3 = r14(x[1] * 15137 + x[3] * 6270); - [v1 + v3, v0 + v2, v0 - v2, v1 - v3] -} - -fn iadst4(x: [i32; 4]) -> [i32; 4] { - let (s19, s29, s39, s49) = (5283i64, 9929i64, 13377i64, 15212i64); - let x: [i64; 4] = x.map(|v| v as i64); - let s0 = s19 * x[0]; - let s1 = s29 * x[0]; - let s2 = s39 * x[1]; - let s3 = s49 * x[2]; - let s4 = s19 * x[2]; - let s5 = s29 * x[3]; - let s6 = s49 * x[3]; - let s7 = s39 * (x[0] - x[2] + x[3]); - let x0 = s0 + s3 + s5; - let x1 = s1 - s4 - s6; - let x2 = s7; - let x3 = s2; - let r = |v: i64| ((v + (1 << 13)) >> 14) as i32; - [r(x0 + x3), r(x1 + x3), r(x2), r(x0 + x1 - x3)] -} - -/// The decoder's 4×4 inverse transform of dequantised coefficients (row-major) to a residual. -fn inverse4(coefs: &[i32; 16], tx_type: u8) -> [i32; 16] { - let row_adst = matches!(tx_type, DCT_ADST | ADST_ADST); - let col_adst = matches!(tx_type, ADST_DCT | ADST_ADST); - let mut t = [[0i32; 4]; 4]; - for i in 0..4 { - let r = [coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]]; - t[i] = if row_adst { iadst4(r) } else { idct4(r) }; - } - let mut out = [0i32; 16]; - for j in 0..4 { - let c = [t[0][j], t[1][j], t[2][j], t[3][j]]; - let c = if col_adst { iadst4(c) } else { idct4(c) }; - for i in 0..4 { - out[i * 4 + j] = (c[i] + 8) >> 4; - } - } - out -} - -/// Inverse Walsh–Hadamard (8.7.1.10) on one line. -fn iwht(x: [i32; 4], shift: u32) -> [i32; 4] { - let mut a = x[0] >> shift; - let mut c = x[1] >> shift; - let mut d = x[2] >> shift; - let mut b = x[3] >> shift; - a += c; - d -= b; - let e = (a - d) >> 1; - b = e - b; - c = e - c; - a -= b; - d += c; - [a, b, c, d] -} - -/// The exact inverse of [`iwht`] (with shift 0): inputs that produce outputs `y`. -fn fwht(y: [i32; 4]) -> [i32; 4] { - let (aa, bb, cc, dd) = (y[0], y[1], y[2], y[3]); - let d1 = dd - cc; - let a1 = aa + bb; - let e = (a1 - d1) >> 1; - let b0 = e - bb; - let c0 = e - cc; - let a0 = a1 - c0; - let d0 = d1 + b0; - [a0, c0, d0, b0] -} - -fn inverse_wht(coefs: &[i32; 16]) -> [i32; 16] { - let mut t = [[0i32; 4]; 4]; - for i in 0..4 { - t[i] = iwht([coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]], 2); - } - let mut out = [0i32; 16]; - for j in 0..4 { - let r = iwht([t[0][j], t[1][j], t[2][j], t[3][j]], 0); - for i in 0..4 { - out[i * 4 + j] = r[i]; - } - } - out -} - -/// Lossless forward transform: levels (dequantised by 4) whose inverse is exactly `res`. -fn forward_wht(res: &[i32; 16]) -> [i32; 16] { - let mut t = [[0i32; 4]; 4]; - for j in 0..4 { - let col = fwht([res[j], res[4 + j], res[8 + j], res[12 + j]]); - for i in 0..4 { - t[i][j] = col[i]; - } - } - let mut lv = [0i32; 16]; - for i in 0..4 { - let r = fwht(t[i]); - lv[i * 4..i * 4 + 4].copy_from_slice(&r); - } - lv -} - -/// Forward transform matrices: the inverses of the decoder's (linear part of the) inverse -/// transforms, so `inverse4(forward(res)) ≈ res`. -fn forward_matrices() -> &'static [[[f64; 16]; 16]; 4] { - use std::sync::OnceLock; - static M: OnceLock<[[[f64; 16]; 16]; 4]> = OnceLock::new(); - M.get_or_init(|| { - let mut out = [[[0.0; 16]; 16]; 4]; - for (ty, o) in out.iter_mut().enumerate() { - // Columns of the inverse: response to a scaled unit coefficient. - let k = 1 << 16; - let mut inv = [[0.0f64; 16]; 16]; - for c in 0..16 { - let mut e = [0i32; 16]; - e[c] = k; - let r = inverse4(&e, ty as u8); - for p in 0..16 { - inv[p][c] = r[p] as f64 / k as f64; - } - } - *o = invert16(inv); - } - out - }) -} - -fn invert16(m: [[f64; 16]; 16]) -> [[f64; 16]; 16] { - let mut a = m; - let mut inv = [[0.0; 16]; 16]; - for (i, r) in inv.iter_mut().enumerate() { - r[i] = 1.0; - } - for col in 0..16 { - let piv = (col..16).max_by(|&x, &y| a[x][col].abs().total_cmp(&a[y][col].abs())).unwrap_or(col); - a.swap(col, piv); - inv.swap(col, piv); - let d = a[col][col]; - for j in 0..16 { - a[col][j] /= d; - inv[col][j] /= d; - } - for r in 0..16 { - if r != col { - let f = a[r][col]; - if f != 0.0 { - for j in 0..16 { - a[r][j] -= f * a[col][j]; - inv[r][j] -= f * inv[col][j]; - } - } - } - } - } - inv -} - -// ---------------------------------------------------------------- planes and prediction - -struct Plane { - w: usize, - /// Last valid column / row for prediction edges (the 8-aligned mode-info grid). - max_x: usize, - max_y: usize, - src: Vec, - rec: Vec, +/// A plane being coded: source and reconstruction at superblock-aligned size. +pub(crate) struct Plane { + /// Stride (superblock-aligned width). + pub w: usize, + /// Last valid column / row for intra prediction edges (the 8-aligned mode-info grid). + pub max_x: usize, + pub max_y: usize, + /// Visible size. + pub vis_w: usize, + pub vis_h: usize, + pub src: Vec, + pub rec: Vec, } impl Plane { + #[allow(clippy::too_many_arguments)] fn new(data: &[u8], vw: usize, vh: usize, w: usize, h: usize, max_x: usize, max_y: usize) -> Plane { let mut src = vec![0u8; w * h]; for y in 0..h { @@ -294,103 +93,141 @@ impl Plane { src[y * w + x] = data.get(sy * vw + sx).copied().unwrap_or(128); } } - Plane { w, max_x, max_y, src, rec: vec![0; w * h] } + Plane { w, max_x, max_y, vis_w: vw, vis_h: vh, src, rec: vec![0; w * h] } } } -/// Intra prediction (8.5.1) of a square block for DC / V / H / TM. -fn predict(p: &Plane, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> [i32; 64] { - let size = 1usize << log2; - let base = 128i32; - let buf = &p.rec; - let w = p.w; - let mut above = [0i32; 8]; - let mut left = [0i32; 8]; - if have_above { - for i in 0..size { - above[i] = buf[(y - 1) * w + (x + i).min(p.max_x)] as i32; - } - } else { - above[..size].fill(base - 1); - } - let top_left = if have_above && have_left { - buf[(y - 1) * w + (x - 1).min(p.max_x)] as i32 - } else if have_above { - base + 1 - } else { - base - 1 - }; - if have_left { - for i in 0..size { - left[i] = buf[(y + i).min(p.max_y) * w + x - 1] as i32; - } - } else { - left[..size].fill(base + 1); - } - let mut out = [0i32; 64]; - for i in 0..size { - for j in 0..size { - out[i * size + j] = match mode { - V_PRED => above[j], - H_PRED => left[i], - TM_PRED => (above[j] + left[i] - top_left).clamp(0, 255), - _ => 0, - }; - } - } - if mode == DC_PRED { - let v = match (have_left, have_above) { - (true, true) => (left[..size].iter().sum::() + above[..size].iter().sum::() + size as i32) >> (log2 + 1), - (true, false) => (left[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, - (false, true) => (above[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, - (false, false) => base, - }; - out[..size * size].fill(v); - } - out +/// Mode info of one 8×8 position. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct Mi { + pub inter: bool, + /// Intra mode (0..=9) or inter mode (NEARESTMV..=NEWMV). + pub mode: u8, + pub mv: Mv, + /// Skip as the loop filter sees it (inter blocks without coefficients). + pub skip: bool, + pub size: u8, } -// ---------------------------------------------------------------- the encoder +/// Rate control state (target bitrate mode). +#[derive(Clone, Debug)] +struct RateControl { + /// bits ≈ complexity / step: per frame type, from the last frame of that type. + key_complexity: Option, + inter_complexity: Option, + /// Bits spent minus bits budgeted so far. + debt: f64, + last_inter_q: Option, +} pub struct Vp9Encoder { cfg: EncoderConfig, - qidx: u8, + base_qidx: u8, lossless: bool, + frames: u64, + since_key: u32, + /// The reconstruction of the last frame (the LAST reference): planes at their strides. + reference: Option<[Plane; 3]>, + mvs: Vec, + rc: RateControl, } -/// Per-tile coding state. -struct Tile<'a> { - enc: &'a Vp9Encoder, - bc: BoolEncoder, - planes: &'a mut [Plane; 3], - mi_cols: usize, - mi_rows: usize, - col_start: usize, - above_nz: [Vec; 3], - left_nz: [Vec; 3], - above_part: Vec, - left_part: Vec, - /// y mode of every 8×8 block (for mode contexts). - modes: &'a mut [u8], +fn quality_to_qidx(q: u8) -> u8 { + // 99 ≈ 3, 90 ≈ 30, 50 ≈ 150, 1 ≈ 255. + ((100 - q.clamp(1, 99) as i32) * 255 / 85).clamp(1, 255) as u8 } impl Vp9Encoder { pub fn new(cfg: EncoderConfig) -> Self { let q = cfg.quality.clamp(1, 100); - let lossless = q >= 100; - // Quality → base_q_idx: 99 ≈ 3, 90 ≈ 30, 50 ≈ 150, 1 ≈ 255. - let qidx = if lossless { 0 } else { ((100 - q as i32) * 255 / 85).clamp(1, 255) as u8 }; - Vp9Encoder { cfg, qidx, lossless } + let lossless = q >= 100 && cfg.target_kbps.is_none(); + let base_qidx = if lossless { 0 } else { quality_to_qidx(q) }; + Vp9Encoder { + cfg, + base_qidx, + lossless, + frames: 0, + since_key: 0, + reference: None, + mvs: vec![], + rc: RateControl { key_complexity: None, inter_complexity: None, debt: 0.0, last_inter_q: None }, + } } pub fn config(&self) -> &EncoderConfig { &self.cfg } - /// Encode one key frame from planar 8-bit 4:2:0 (`y`: width×height, `u`/`v`: + /// Encode one frame from planar 8-bit 4:2:0 (`y`: width×height, `u`/`v`: /// ((w+1)/2)×((h+1)/2), tightly packed). pub fn encode_yuv420(&mut self, y: &[u8], u: &[u8], v: &[u8]) -> Vec { + self.encode_frame(y, u, v, false).data + } + + /// The decoder's reconstruction of the last coded frame (visible area, tightly packed Y, U, + /// V). + pub fn reconstruction(&self) -> Option<[Vec; 3]> { + let r = self.reference.as_ref()?; + Some(std::array::from_fn(|i| { + let p = &r[i]; + (0..p.vis_h).flat_map(|y| p.rec[y * p.w..y * p.w + p.vis_w].iter().copied()).collect() + })) + } + + fn is_key_due(&self, force: bool) -> bool { + force || self.reference.is_none() || self.cfg.keyframe_interval == 1 || (self.cfg.keyframe_interval > 1 && self.since_key >= self.cfg.keyframe_interval) + } + + /// The q index of the next frame. + fn pick_qidx(&mut self, key: bool) -> u8 { + if self.lossless { + return 0; + } + let Some(kbps) = self.cfg.target_kbps else { return self.base_qidx }; + let per_frame = kbps as f64 * 1000.0 / self.cfg.frame_rate.max(1.0); + let interval = if self.cfg.keyframe_interval == 0 { 300.0 } else { self.cfg.keyframe_interval as f64 }; + // Key frames get a larger share of the group's budget. + let boost = (interval / 6.0).clamp(1.0, 5.0); + let group = per_frame * interval; + let key_share = group * boost / (boost + interval - 1.0); + let inter_share = (group - key_share) / (interval - 1.0).max(1.0); + let mut target = if key { key_share } else { inter_share }; + // Pay back (or spend) the running difference over about a second. + let horizon = self.cfg.frame_rate.max(1.0); + target = (target - self.rc.debt / horizon).max(target * 0.25); + let complexity = if key { self.rc.key_complexity } else { self.rc.inter_complexity.or(self.rc.key_complexity.map(|c| c / (boost * 2.0))) }; + let Some(cx) = complexity else { return self.base_qidx }; + let step = (cx / target).max(4.0); + let mut q = (0..=255u8).min_by_key(|&q| ((AC_QLOOKUP[q as usize] as f64 - step).abs() * 16.0) as i64).unwrap_or(self.base_qidx); + if !key && let Some(last) = self.rc.last_inter_q { + q = q.clamp(last.saturating_sub(24), last.saturating_add(24)); + } + q.max(1) + } + + fn rc_update(&mut self, key: bool, qidx: u8, bytes: usize) { + let Some(kbps) = self.cfg.target_kbps else { return }; + let bits = bytes as f64 * 8.0; + let cx = bits * AC_QLOOKUP[qidx as usize] as f64; + if key { + self.rc.key_complexity = Some(cx); + } else { + self.rc.inter_complexity = Some(match self.rc.inter_complexity { + Some(c) => c * 0.5 + cx * 0.5, + None => cx, + }); + self.rc.last_inter_q = Some(qidx); + } + let per_frame = kbps as f64 * 1000.0 / self.cfg.frame_rate.max(1.0); + self.rc.debt += bits - per_frame; + } + + /// Encode one frame; `force_key` starts a new key frame. + pub fn encode_frame(&mut self, y: &[u8], u: &[u8], v: &[u8], force_key: bool) -> EncodedFrame { let (w, h) = (self.cfg.width.max(1) as usize, self.cfg.height.max(1) as usize); + let key = self.is_key_due(force_key); + let qidx = self.pick_qidx(key); + let lossless = qidx == 0; let mi_cols = w.div_ceil(8); let mi_rows = h.div_ceil(8); let sb_cols = mi_cols.div_ceil(8); @@ -413,71 +250,77 @@ impl Vp9Encoder { } max_log2 -= 1; let tile_cols = 1usize << min_log2; - let mut modes = vec![0u8; mi_cols * mi_rows]; - let mut tiles = vec![]; + let quant = Quant { dcq: DC_QLOOKUP[qidx as usize], acq: AC_QLOOKUP[qidx as usize], lossless, round_dc: 0.5, round_ac: 0.38 }; + let step = AC_QLOOKUP[qidx as usize] as f64 / 8.0; + let lambda_sse = 0.35 * step * step; + let refs = if key { + None + } else { + self.reference.as_ref().map(|r| std::array::from_fn::<_, 3, _>(|i| mc::RefPlane { data: &r[i].rec, stride: r[i].w, w: r[i].vis_w, h: r[i].vis_h })) + }; + let prev = (!key && self.mvs.len() == mi_cols * mi_rows).then_some(&self.mvs[..]); + let phases = refs.as_ref().map(|r| mc::PhasePlanes::new(&r[0], 48)); + let fp = tile::FrameParams { key, quant, mi_cols, mi_rows, refs, phases, prev_mvs: prev, lambda_sad: lambda_sse.sqrt(), lambda_sse }; + let mut mi = vec![Mi::default(); mi_cols * mi_rows]; + let mut writers = vec![]; for t in 0..tile_cols { let start = (((t * sb_cols) >> min_log2) * 8).min(mi_cols); let end = ((((t + 1) * sb_cols) >> min_log2) * 8).min(mi_cols); - let mut tile = Tile { - enc: self, - bc: BoolEncoder::new(), - planes: &mut planes, - mi_cols, - mi_rows, - col_start: start, - above_nz: [vec![0; mi_cols * 2 + 16], vec![0; mi_cols * 2 + 16], vec![0; mi_cols * 2 + 16]], - left_nz: [vec![0; 16], vec![0; 16], vec![0; 16]], - above_part: vec![0; mi_cols + 8], - left_part: vec![0; 8], - modes: &mut modes, - }; - tile.bc.write(false, 128); - for r in (0..mi_rows).step_by(8) { - for p in 0..3 { - tile.left_nz[p].fill(0); - } - tile.left_part.fill(0); - for c in (start..end).step_by(8) { - tile.partition(r, c, 12); - } - } - tiles.push(tile.bc.finish()); + let mut tile = tile::Tile::new(&fp, &mut planes, &mut mi, start, end); + tile.code(); + writers.push(tile.w); } - // Compressed header. - let mut ch_enc = BoolEncoder::new(); - ch_enc.write(false, 128); - if !self.lossless { - ch_enc.literal(0, 2); // tx_mode ONLY_4X4 + // Forward probability updates from this frame's counts. + let defaults = Probs::defaults(); + let mut counts = vec![[0u32; 2]; probs::NPROBS]; + for wr in &writers { + wr.count(&mut counts); } - ch_enc.literal(0, 1); // no coefficient probability updates (TX_4X4) - for _ in 0..3 { - ch_enc.write(false, 252); // skip probabilities unchanged - } - let compressed = ch_enc.finish(); + let wanted = probs::plan_updates(&defaults, &counts, !key); + let (compressed, used) = probs::compressed_header(&defaults, &wanted, lossless, !key); + let tiles: Vec> = writers.iter().map(|wr| wr.emit(&used)).collect(); + drop(fp); + // Loop filter: the level with the least error against the source. + let level = if lossless || !self.cfg.loop_filter { 0 } else { choose_level(&mut planes, &mi, mi_rows, mi_cols, qidx, key) }; + lf::filter_frame(&mut planes, &mi, mi_rows, mi_cols, level); // Uncompressed header. let mut bw = BitWriter::default(); bw.put(2, 2); // frame_marker bw.put(0, 1); // profile low bit bw.put(0, 1); // profile high bit bw.put(0, 1); // show_existing_frame - bw.put(0, 1); // frame_type = KEY_FRAME + bw.put(!key as u32, 1); // frame_type bw.put(1, 1); // show_frame - bw.put(0, 1); // error_resilient_mode - bw.put(0x49, 8); - bw.put(0x83, 8); - bw.put(0x42, 8); - bw.put(2, 3); // color_space CS_BT_709 - bw.put(self.cfg.full_range as u32, 1); - bw.put(w as u32 - 1, 16); - bw.put(h as u32 - 1, 16); - bw.put(0, 1); // render_and_frame_size_different - bw.put(0, 1); // refresh_frame_context - bw.put(1, 1); // frame_parallel_decoding_mode + if key { + bw.put(0, 1); // error_resilient_mode + bw.put(0x49, 8); + bw.put(0x83, 8); + bw.put(0x42, 8); + bw.put(2, 3); // color_space CS_BT_709 + bw.put(self.cfg.full_range as u32, 1); + bw.put(w as u32 - 1, 16); + bw.put(h as u32 - 1, 16); + bw.put(0, 1); // render_and_frame_size_different + bw.put(0, 1); // refresh_frame_context + bw.put(1, 1); // frame_parallel_decoding_mode + } else { + bw.put(1, 1); // error_resilient_mode (no reliance on earlier contexts or vectors) + bw.put(0x01, 8); // refresh_frame_flags: slot 0 becomes the new LAST + for _ in 0..3 { + bw.put(0, 3); // ref_frame_idx: slot 0 + bw.put(0, 1); // sign bias + } + bw.put(1, 1); // found_ref: the size of LAST + bw.put(0, 1); // render_and_frame_size_different + bw.put(0, 1); // allow_high_precision_mv + bw.put(0, 1); // is_filter_switchable + bw.put(1, 2); // raw_interpolation_filter → EIGHTTAP (regular) + } bw.put(0, 2); // frame_context_idx - bw.put(0, 6); // loop_filter_level + bw.put(level as u32, 6); // loop_filter_level bw.put(0, 3); // sharpness bw.put(0, 1); // mode_ref_delta_enabled - bw.put(self.qidx as u32, 8); + bw.put(qidx as u32, 8); bw.put(0, 1); // delta_q_y_dc bw.put(0, 1); // delta_q_uv_dc bw.put(0, 1); // delta_q_uv_ac @@ -496,10 +339,46 @@ impl Vp9Encoder { } out.extend_from_slice(&t); } - out + // State for the next frame. + self.mvs = mi.iter().map(|m| if m.inter { m.mv } else { Mv::ZERO }).collect(); + self.reference = Some(planes); + self.frames += 1; + self.since_key = if key { 1 } else { self.since_key + 1 }; + self.rc_update(key, qidx, out.len()); + EncodedFrame { data: out, key, q_idx: qidx, filter_level: level } } } +/// Try a few loop filter levels on the luma reconstruction; the one closest to the source +/// (squared error over the visible area) wins (0 when filtering doesn't help). +fn choose_level(planes: &mut [Plane; 3], mi: &[Mi], mi_rows: usize, mi_cols: usize, qidx: u8, key: bool) -> u8 { + let guess = ((qidx as f64 * if key { 0.25 } else { 0.2 }).round() as u8).clamp(1, 63); + let sse = |p: &Plane| -> u64 { + let mut s = 0u64; + for y in 0..p.vis_h { + for x in 0..p.vis_w { + let d = p.src[y * p.w + x] as i64 - p.rec[y * p.w + x] as i64; + s += (d * d) as u64; + } + } + s + }; + let mut best = (sse(&planes[0]), 0u8); + let saved = planes[0].rec.clone(); + for level in [guess / 2, guess, (guess as u16 * 3 / 2).min(63) as u8] { + if level == 0 { + continue; + } + lf::filter_frame(&mut planes[..1], mi, mi_rows, mi_cols, level); + let s = sse(&planes[0]); + planes[0].rec.copy_from_slice(&saved); + if s < best.0 { + best = (s, level); + } + } + best.1 +} + #[derive(Default)] struct BitWriter { out: Vec, @@ -527,254 +406,6 @@ impl BitWriter { } } -const BLOCK_8X8: u8 = 3; -const NUM_8X8: [usize; 13] = [1, 1, 1, 1, 1, 2, 2, 2, 4, 4, 4, 8, 8]; -const MI_WIDTH_LOG2: [usize; 13] = [0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3]; - -impl Tile<'_> { - fn partition(&mut self, r: usize, c: usize, bsize: u8) { - if r >= self.mi_rows || c >= self.mi_cols { - return; - } - let num8 = NUM_8X8[bsize as usize]; - let half = num8 >> 1; - let has_rows = (r + half) < self.mi_rows; - let has_cols = (c + half) < self.mi_cols; - let bsl = MI_WIDTH_LOG2[bsize as usize]; - let boffset = 3 - bsl; - let mut above = 0u8; - let mut left = 0u8; - for i in 0..num8 { - above |= self.above_part[c + i]; - left |= self.left_part[(r & 7) + i]; - } - let above = (above >> boffset) & 1; - let left = (left >> boffset) & 1; - let ctx = bsl * 4 + left as usize * 2 + above as usize; - let probs = &KF_PARTITION_PROBS[ctx * 3..ctx * 3 + 3]; - if bsize == BLOCK_8X8 { - // PARTITION_NONE. - self.bc.write(false, probs[0]); - self.block(r, c); - self.above_part[c] = 15 >> 1; - self.left_part[r & 7] = 15 >> 1; - return; - } - // PARTITION_SPLIT. - if has_rows && has_cols { - self.bc.tree(&PARTITION_TREE, probs, 3); - } else if has_cols { - self.bc.write(true, probs[1]); - } else if has_rows { - self.bc.write(true, probs[2]); - } - let sub = bsize - 3; - self.partition(r, c, sub); - self.partition(r, c + half, sub); - self.partition(r + half, c, sub); - self.partition(r + half, c + half, sub); - } - - fn block(&mut self, r: usize, c: usize) { - let avail_u = r > 0; - let avail_l = c > self.col_start; - // Mode decision on the 8×8 luma block and the 4×4 chroma blocks. - let y_mode = self.choose(0, c * 8, r * 8, 3, avail_l, avail_u); - let uv_mode = { - let mut best = (i64::MAX, DC_PRED); - for m in MODES { - let cost: i64 = (1..3).map(|p| self.cost(p, c * 4, r * 4, 2, m, avail_l, avail_u)).sum(); - if cost < best.0 { - best = (cost, m); - } - } - best.1 - }; - // skip = 0 (context: neighbours' skip, always 0 here); tx_size implied (4×4). - self.bc.write(false, DEFAULT_SKIP_PROB[0]); - let above_mode = if avail_u { self.modes[(r - 1) * self.mi_cols + c] } else { DC_PRED }; - let left_mode = if avail_l { self.modes[r * self.mi_cols + c - 1] } else { DC_PRED }; - let o = (above_mode as usize * 10 + left_mode as usize) * 9; - self.bc.tree(&INTRA_MODE_TREE, &KF_Y_MODE_PROBS[o..o + 9], y_mode); - let o = y_mode as usize * 9; - self.bc.tree(&INTRA_MODE_TREE, &KF_UV_MODE_PROBS[o..o + 9], uv_mode); - self.modes[r * self.mi_cols + c] = y_mode; - // Residual: luma 2×2 transform blocks, then one 4×4 block per chroma plane. - for (dy, dx) in [(0, 0), (0, 1), (1, 0), (1, 1)] { - let tx = if self.enc.lossless { DCT_DCT } else { mode_tx_type(y_mode) }; - self.tx_block(0, c * 8 + dx * 4, r * 8 + dy * 4, y_mode, tx, avail_l || dx > 0, avail_u || dy > 0); - } - for p in 1..3 { - self.tx_block(p, c * 4, r * 4, uv_mode, DCT_DCT, avail_l, avail_u); - } - } - - fn cost(&self, plane: usize, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> i64 { - let p = &self.planes[plane]; - let pred = predict(p, x, y, log2, mode, have_left, have_above); - let size = 1 << log2; - let mut s = 0i64; - for i in 0..size { - for j in 0..size { - s += (p.src[(y + i) * p.w + x + j] as i32 - pred[i * size + j]).abs() as i64; - } - } - s - } - - fn choose(&self, plane: usize, x: usize, y: usize, log2: usize, have_left: bool, have_above: bool) -> u8 { - let mut best = (i64::MAX, DC_PRED); - for m in MODES { - let cost = self.cost(plane, x, y, log2, m, have_left, have_above); - if cost < best.0 { - best = (cost, m); - } - } - best.1 - } - - #[allow(clippy::too_many_arguments)] - fn tx_block(&mut self, plane: usize, x: usize, y: usize, mode: u8, tx_type: u8, have_left: bool, have_above: bool) { - let ss = (plane > 0) as usize; - let max_x = (self.mi_cols * 8) >> ss; - let max_y = (self.mi_rows * 8) >> ss; - let (x4, y4) = (x >> 2, y >> 2); - let ly = y4 & (15 >> ss); - if x >= max_x || y >= max_y { - self.above_nz[plane][x4] = 0; - self.left_nz[plane][ly] = 0; - return; - } - let p = &self.planes[plane]; - let pred = predict(p, x, y, 2, mode, have_left, have_above); - let mut res = [0i32; 16]; - for i in 0..4 { - for j in 0..4 { - res[i * 4 + j] = p.src[(y + i) * p.w + x + j] as i32 - pred[i * 4 + j]; - } - } - let (dcq, acq) = (DC_QLOOKUP[self.enc.qidx as usize], AC_QLOOKUP[self.enc.qidx as usize]); - // Quantised levels (raster order). - let levels: [i32; 16] = if self.enc.lossless { - forward_wht(&res) - } else { - let f = &forward_matrices()[tx_type as usize]; - let mut lv = [0i32; 16]; - for k in 0..16 { - let coef: f64 = (0..16).map(|q| f[k][q] * res[q] as f64).sum(); - let qs = if k == 0 { dcq } else { acq } as f64; - let a = coef.abs() / qs; - // Rounding with a small dead zone for AC coefficients. - let l = if k == 0 { (a + 0.5).floor() } else { (a + 0.38).floor() }; - lv[k] = if coef < 0.0 { -(l as i32) } else { l as i32 }; - } - lv - }; - let sc = scan(tx_type); - let eob = (0..16).rev().find(|&i| levels[sc[i] as usize] != 0).map_or(0, |i| i + 1); - // Tokens. - let ctx0 = (self.above_nz[plane][x4] + self.left_nz[plane][ly]) as usize; - self.tokens(plane, tx_type, &levels, eob, ctx0); - // Reconstruct exactly as the decoder does. - let mut deq = [0i32; 16]; - for k in 0..16 { - deq[k] = levels[k] * if k == 0 { dcq } else { acq }; - } - let resid = if eob == 0 { - [0i32; 16] - } else if self.enc.lossless { - inverse_wht(&deq) - } else { - inverse4(&deq, tx_type) - }; - let p = &mut self.planes[plane]; - for i in 0..4 { - for j in 0..4 { - p.rec[(y + i) * p.w + x + j] = (pred[i * 4 + j] + resid[i * 4 + j]).clamp(0, 255) as u8; - } - } - let nz = (eob > 0) as u8; - self.above_nz[plane][x4] = nz; - self.left_nz[plane][ly] = nz; - } - - fn tokens(&mut self, plane: usize, tx_type: u8, levels: &[i32; 16], eob: usize, ctx0: usize) { - let sc = scan(tx_type); - let nb = neighbors(tx_type); - let ptype = (plane > 0) as usize; - // coef_probs[TX_4X4][ptype][intra]: 6 bands × 6 contexts × 3. - let base = (ptype * 2) * 6 * 6 * 3; - let mut tc = [0u8; 16]; - let mut check_eob = true; - let mut c = 0; - while c < 16 { - let pos = sc[c] as usize; - let band = COEFBAND_4X4[c] as usize; - let ctx = if c == 0 { ctx0 } else { ((1 + tc[nb[c].0] + tc[nb[c].1]) >> 1) as usize }; - let o = base + (band * 6 + ctx) * 3; - let p = [DEFAULT_COEF_PROBS[o], DEFAULT_COEF_PROBS[o + 1], DEFAULT_COEF_PROBS[o + 2]]; - if check_eob { - let more = c < eob; - self.bc.write(more, p[0]); - if !more { - break; - } - } - let v = levels[pos].unsigned_abs(); - if v == 0 { - self.bc.write(false, p[1]); - tc[pos] = 0; - check_eob = false; - c += 1; - continue; - } - self.bc.write(true, p[1]); - check_eob = true; - let token = match v { - 1 => 1, - 2 => 2, - 3 => 3, - 4 => 4, - 5..=6 => 5, - 7..=10 => 6, - 11..=18 => 7, - 19..=34 => 8, - 35..=66 => 9, - _ => 10, - }; - if token == 1 { - self.bc.write(false, p[2]); - } else { - self.bc.write(true, p[2]); - let pp = pareto(p[2]); - let path: &[(usize, bool)] = match token { - 2 => &[(0, false), (1, false)], - 3 => &[(0, false), (1, true), (2, false)], - 4 => &[(0, false), (1, true), (2, true)], - 5 => &[(0, true), (3, false), (4, false)], - 6 => &[(0, true), (3, false), (4, true)], - 7 => &[(0, true), (3, true), (5, false), (6, false)], - 8 => &[(0, true), (3, true), (5, false), (6, true)], - 9 => &[(0, true), (3, true), (5, true), (7, false)], - _ => &[(0, true), (3, true), (5, true), (7, true)], - }; - for &(n, b) in path { - self.bc.write(b, pp[n]); - } - let (cat, num_extra, base_v) = EXTRA_BITS[token]; - let extra = (v - base_v as u32).min((1 << num_extra) - 1); - let cp = CAT_PROBS[cat as usize]; - for e in 0..num_extra as usize { - self.bc.write((extra >> (num_extra as usize - 1 - e)) & 1 != 0, cp[e]); - } - } - tc[pos] = ENERGY_CLASS[token]; - self.bc.write(levels[pos] < 0, 128); - c += 1; - } - } -} - // ---------------------------------------------------------------- colour conversion /// BT.709 RGB(A) 8-bit → planar YUV 4:2:0 (limited or full range). @@ -822,36 +453,3 @@ pub fn alpha_to_yuv420(rgba: &[u8], w: u32, h: u32) -> (Vec, Vec, Vec = rgba.chunks_exact(4).take(w * h).map(|p| p[3]).collect(); (y, vec![128; cw * ch], vec![128; cw * ch]) } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn wht_is_exactly_invertible() { - let mut s = 7u32; - for _ in 0..2000 { - let mut r = [0i32; 16]; - for v in &mut r { - s = s.wrapping_mul(1664525).wrapping_add(1013904223); - *v = (s >> 16) as i32 % 511 - 255; - } - let lv = forward_wht(&r); - let deq = lv.map(|l| l * 4); - assert_eq!(inverse_wht(&deq), r); - } - } - - #[test] - fn forward_transforms_invert_the_decoder() { - for ty in 0..4u8 { - let f = &forward_matrices()[ty as usize]; - let r: [i32; 16] = std::array::from_fn(|i| (i as i32 * 37 % 61) - 30); - let c: [i32; 16] = std::array::from_fn(|k| (0..16).map(|q| f[k][q] * r[q] as f64).sum::().round() as i32); - let back = inverse4(&c, ty); - for i in 0..16 { - assert!((back[i] - r[i]).abs() <= 1, "type {ty}: {back:?} vs {r:?}"); - } - } - } -} diff --git a/crates/vp9enc/src/mc.rs b/crates/vp9enc/src/mc.rs new file mode 100644 index 00000000..779a1ea5 --- /dev/null +++ b/crates/vp9enc/src/mc.rs @@ -0,0 +1,207 @@ +//! Inter prediction exactly as the decoder forms it: motion vector clamping (spec 8.5.2.2) and +//! the unscaled block inter prediction process (8.5.2.4) with the regular 8-tap filters and +//! reference samples clamped to the reference frame's visible area. + +use crate::tables::SUBPEL_REGULAR; + +/// A motion vector in 1/8 luma samples. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Mv { + pub row: i32, + pub col: i32, +} + +impl Mv { + pub const ZERO: Mv = Mv { row: 0, col: 0 }; + pub fn new(row: i32, col: i32) -> Mv { + Mv { row, col } + } + pub fn minus(self, o: Mv) -> Mv { + Mv::new(self.row - o.row, self.col - o.col) + } +} + +/// A reference plane: `w`×`h` visible samples at `stride`. +pub struct RefPlane<'a> { + pub data: &'a [u8], + pub stride: usize, + pub w: usize, + pub h: usize, +} + +/// The block's motion vector for `plane` in 1/16 samples of that plane, clamped as the +/// decoder clamps it. `n8`: block size in 8×8 units. +pub fn clamp_for_plane(mv: Mv, r: usize, c: usize, n8: usize, mi_rows: usize, mi_cols: usize, chroma: bool) -> (i32, i32) { + let s = chroma as i32; + let (r, c, b) = (r as i32, c as i32, n8 as i32); + let to_top = (-(r * 8 * 16)) >> s; + let to_bottom = (((mi_rows as i32 - b - r) * 8) * 16) >> s; + let to_left = (-(c * 8 * 16)) >> s; + let to_right = (((mi_cols as i32 - b - c) * 8) * 16) >> s; + let spel = (4 + ((b * 8) >> s)) << 4; + let row = ((2 * mv.row) >> s).clamp(to_top - spel, (to_bottom + spel - 16).max(to_top - spel)); + let col = ((2 * mv.col) >> s).clamp(to_left - spel, (to_right + spel - 16).max(to_left - spel)); + (row, col) +} + +/// Predict a `w`×`h` block whose top-left sample is at (`x16`, `y16`) (1/16 samples) into `out` +/// (stride `w`). +pub fn predict(r: &RefPlane, x16: i32, y16: i32, w: usize, h: usize, out: &mut [u8]) { + let (x0, y0) = (x16 >> 4, y16 >> 4); + let (fx, fy) = ((x16 & 15) as usize, (y16 & 15) as usize); + let (lx, ly) = (r.w as i32 - 1, r.h as i32 - 1); + let inside = x0 - 3 >= 0 && y0 - 3 >= 0 && x0 + w as i32 + 4 <= lx && y0 + h as i32 + 4 <= ly; + let at = |x: i32, y: i32| -> i32 { + if inside { r.data[y as usize * r.stride + x as usize] as i32 } else { r.data[y.clamp(0, ly) as usize * r.stride + x.clamp(0, lx) as usize] as i32 } + }; + let fh = &SUBPEL_REGULAR[fx]; + let fv = &SUBPEL_REGULAR[fy]; + if fx == 0 && fy == 0 { + for i in 0..h { + for j in 0..w { + out[i * w + j] = at(x0 + j as i32, y0 + i as i32) as u8; + } + } + return; + } + // Horizontal pass over h + 7 rows (from y0 - 3), then vertical. + let mut tmp = [0u8; 23 * 16]; + for i in 0..h + 7 { + let y = y0 - 3 + i as i32; + for j in 0..w { + let x = x0 + j as i32 - 3; + let mut acc = 64; + for t in 0..8 { + acc += fh[t] * at(x + t as i32, y); + } + tmp[i * w + j] = (acc >> 7).clamp(0, 255) as u8; + } + } + for i in 0..h { + for j in 0..w { + let mut acc = 64; + for t in 0..8 { + acc += fv[t] * tmp[(i + t) * w + j] as i32; + } + out[i * w + j] = (acc >> 7).clamp(0, 255) as u8; + } + } +} + +/// The reference luma pre-interpolated at the 16 quarter-sample phases the encoder uses (motion +/// vectors are even in 1/8 units: fractions 0, 4, 8, 12 in 1/16), over the visible area plus a +/// border. Each phase plane holds exactly what [`predict`] returns at that phase (the decoder's +/// separable filter with edge clamping), so motion search reads predictions directly. +pub struct PhasePlanes { + border: usize, + stride: usize, + rows: usize, + planes: Vec>, +} + +impl PhasePlanes { + pub fn new(r: &RefPlane, border: usize) -> PhasePlanes { + const M: usize = 4; + let (pw, ph) = (r.w + 2 * border, r.h + 2 * border); + // Edge-replicated source with M more samples on every side for the filter taps. + let rs = pw + 2 * M; + let rh = ph + 2 * M; + let off = (border + M) as i32; + let mut raw = vec![0u8; rs * rh]; + for y in 0..rh { + let sy = (y as i32 - off).clamp(0, r.h as i32 - 1) as usize; + let src = &r.data[sy * r.stride..sy * r.stride + r.w]; + for x in 0..rs { + raw[y * rs + x] = src[(x as i32 - off).clamp(0, r.w as i32 - 1) as usize]; + } + } + // Horizontal phases over every raw row (inner columns). + let hpass = |fx: usize| -> Vec { + let f = &SUBPEL_REGULAR[fx]; + let mut out = vec![0u8; pw * rh]; + for y in 0..rh { + let row = &raw[y * rs..y * rs + rs]; + for x in 0..pw { + let s = &row[x + M - 3..x + M + 5]; + let acc: i32 = 64 + (0..8).map(|t| f[t] * s[t] as i32).sum::(); + out[y * pw + x] = (acc >> 7).clamp(0, 255) as u8; + } + } + out + }; + let unfiltered: Vec = (0..rh).flat_map(|y| raw[y * rs + M..y * rs + M + pw].iter().copied()).collect(); + let h: Vec> = (0..4).map(|k| if k == 0 { unfiltered.clone() } else { hpass(k * 4) }).collect(); + let mut planes = Vec::with_capacity(16); + for fy in 0..4 { + for hp in &h { + if fy == 0 { + planes.push(hp[M * pw..(M + ph) * pw].to_vec()); + continue; + } + let f = &SUBPEL_REGULAR[fy * 4]; + let mut out = vec![0u8; pw * ph]; + for y in 0..ph { + for x in 0..pw { + let mut acc = 64; + for t in 0..8 { + acc += f[t] * hp[(y + M + t - 3) * pw + x] as i32; + } + out[y * pw + x] = (acc >> 7).clamp(0, 255) as u8; + } + } + planes.push(out); + } + } + PhasePlanes { border, stride: pw, rows: ph, planes } + } + + /// The prediction of the `n`×`n` block at (`x16`, `y16`) (fractions multiples of 4) as rows of + /// a phase plane: (plane, offset of the first sample, stride). `None` beyond the border. + pub fn block(&self, x16: i32, y16: i32, n: usize) -> Option<(&[u8], usize, usize)> { + let (x0, y0) = ((x16 >> 4) + self.border as i32, (y16 >> 4) + self.border as i32); + let (fx, fy) = ((x16 & 15) as usize, (y16 & 15) as usize); + if fx % 4 != 0 || fy % 4 != 0 || x0 < 0 || y0 < 0 || x0 as usize + n > self.stride || y0 as usize + n > self.rows { + return None; + } + Some((&self.planes[(fy / 4) * 4 + fx / 4], y0 as usize * self.stride + x0 as usize, self.stride)) + } +} + +#[cfg(test)] +mod tests { + #[test] + fn phase_planes_match_predict() { + let (w, h) = (37usize, 29usize); + let data: Vec = (0..w * h).map(|i| ((i * 7919) % 251) as u8).collect(); + let r = RefPlane { data: &data, stride: w, w, h }; + let pp = PhasePlanes::new(&r, 24); + let mut out = [0u8; 64]; + for y16 in (-20 * 16..(h as i32 + 10) * 16).step_by(4 * 7) { + for x16 in (-20 * 16..(w as i32 + 10) * 16).step_by(4 * 5) { + let Some((p, o, s)) = pp.block(x16, y16, 8) else { continue }; + predict(&r, x16, y16, 8, 8, &mut out); + for i in 0..8 { + assert_eq!(&p[o + i * s..o + i * s + 8], &out[i * 8..i * 8 + 8], "({x16}, {y16}) row {i}"); + } + } + } + } + + use super::*; + + #[test] + fn whole_sample_and_linear_ramps() { + let (w, h) = (40usize, 30usize); + let data: Vec = (0..w * h).map(|i| ((i % w) * 4 + (i / w)) as u8).collect(); + let r = RefPlane { data: &data, stride: w, w, h }; + let mut out = [0u8; 64]; + predict(&r, 10 << 4, 8 << 4, 8, 8, &mut out); + assert_eq!(out[0], data[8 * w + 10]); + // Half a sample right on a linear ramp: exact midpoint. + predict(&r, (10 << 4) + 8, 8 << 4, 8, 8, &mut out); + assert_eq!(out[0], data[8 * w + 10] + 2); + // Off the left edge: clamped to column 0. + predict(&r, -(20 << 4), 0, 4, 4, &mut out); + assert_eq!(out[0], data[0]); + } +} diff --git a/crates/vp9enc/src/probs.rs b/crates/vp9enc/src/probs.rs new file mode 100644 index 00000000..8f55bbba --- /dev/null +++ b/crates/vp9enc/src/probs.rs @@ -0,0 +1,426 @@ +//! Probabilities: the frame's adaptable probability table (one flat array laid out in the order +//! the compressed header updates them), a symbol recorder that codes every boolean decision +//! against either a fixed probability or an entry of that table, bit-cost estimates, and the +//! per-frame forward updates (spec 6.3: `diff_update_prob` with `decode_term_subexp` / +//! `inv_remap_prob`, and `update_mv_prob`). +//! +//! Tiles are first recorded, then the encoder counts how often each table entry coded a 0 or a +//! 1, picks updates that pay for themselves, writes them in the compressed header and finally +//! entropy codes the recorded symbols with the updated probabilities. + +use crate::bool::BoolEncoder; +use crate::tables::*; + +/// Coefficient probabilities `[ptype][ref][band][ctx][3]` for 4×4 transforms (band 0 uses +/// contexts 0..3 only). +pub const COEF: usize = 0; +pub const SKIP: usize = 432; +pub const INTER_MODE: usize = SKIP + 3; +pub const IS_INTER: usize = INTER_MODE + 21; +pub const SINGLE_REF: usize = IS_INTER + 4; +pub const PARTITION: usize = SINGLE_REF + 10; +pub const MV_JOINT: usize = PARTITION + 48; +pub const MV_SIGN: usize = MV_JOINT + 3; +pub const MV_CLASS: usize = MV_SIGN + 2; +pub const MV_CLASS0_BIT: usize = MV_CLASS + 20; +pub const MV_BITS: usize = MV_CLASS0_BIT + 2; +pub const MV_CLASS0_FR: usize = MV_BITS + 20; +pub const MV_FR: usize = MV_CLASS0_FR + 12; +pub const NPROBS: usize = MV_FR + 6; + +/// Index of the first of the three coefficient probabilities of a context. +pub fn coef_index(ptype: usize, inter: bool, band: usize, ctx: usize) -> usize { + COEF + (((ptype * 2 + inter as usize) * 6 + band) * 6 + ctx) * 3 +} + +#[derive(Clone)] +pub struct Probs(pub [u8; NPROBS]); + +impl Probs { + /// The default probabilities (setup_past_independence / key frames). + pub fn defaults() -> Probs { + let mut p = [128u8; NPROBS]; + p[COEF..COEF + 432].copy_from_slice(&DEFAULT_COEF_PROBS); + p[SKIP..SKIP + 3].copy_from_slice(&DEFAULT_SKIP_PROB); + p[INTER_MODE..INTER_MODE + 21].copy_from_slice(&DEFAULT_INTER_MODE_PROBS); + p[IS_INTER..IS_INTER + 4].copy_from_slice(&DEFAULT_IS_INTER_PROB); + p[SINGLE_REF..SINGLE_REF + 10].copy_from_slice(&DEFAULT_SINGLE_REF_PROB); + p[PARTITION..PARTITION + 48].copy_from_slice(&DEFAULT_PARTITION_PROBS); + p[MV_JOINT..MV_JOINT + 3].copy_from_slice(&DEFAULT_MV_JOINT_PROBS); + p[MV_SIGN..MV_SIGN + 2].copy_from_slice(&DEFAULT_MV_SIGN_PROB); + p[MV_CLASS..MV_CLASS + 20].copy_from_slice(&DEFAULT_MV_CLASS_PROBS); + p[MV_CLASS0_BIT..MV_CLASS0_BIT + 2].copy_from_slice(&DEFAULT_MV_CLASS0_BIT_PROB); + p[MV_BITS..MV_BITS + 20].copy_from_slice(&DEFAULT_MV_BITS_PROB); + p[MV_CLASS0_FR..MV_CLASS0_FR + 12].copy_from_slice(&DEFAULT_MV_CLASS0_FR_PROBS); + p[MV_FR..MV_FR + 6].copy_from_slice(&DEFAULT_MV_FR_PROBS); + Probs(p) + } +} + +/// Probabilities of the full token tree (9.3.2) for model probability `p`. +pub fn pareto(p: u8) -> [u8; 8] { + let p = p.max(1) as usize; + let x = (p - 1) / 2; + let mut t = [0u8; 8]; + for n in 0..8 { + t[n] = if p & 1 == 1 { PARETO_TABLE[x * 8 + n] } else { ((PARETO_TABLE[x * 8 + n] as u32 + PARETO_TABLE[(x + 1) * 8 + n] as u32) >> 1) as u8 }; + } + t +} + +/// The probability a recorded symbol is coded with. +#[derive(Clone, Copy, Debug)] +pub enum P { + Fixed(u8), + /// An entry of the frame's [`Probs`]. + Ctx(u16), + /// Node `n` of the Pareto token tree derived from the coefficient probability at the index. + Pareto(u16, u8), +} + +#[derive(Clone, Copy)] +struct Sym { + bit: bool, + p: P, +} + +/// Records the boolean decisions of a tile. +#[derive(Default)] +pub struct Writer { + syms: Vec, +} + +impl Writer { + #[inline] + pub fn put(&mut self, bit: bool, p: P) { + self.syms.push(Sym { bit, p }); + } + #[inline] + pub fn fixed(&mut self, bit: bool, prob: u8) { + self.put(bit, P::Fixed(prob)); + } + #[inline] + pub fn ctx(&mut self, bit: bool, idx: usize) { + self.put(bit, P::Ctx(idx as u16)); + } + /// Write `value` with `tree` (leaves stored negated); node `i` uses `prob(i)`. + pub fn tree(&mut self, tree: &[i8], value: u8, prob: impl Fn(usize) -> P) { + for (node, bit) in tree_path(tree, value) { + self.put(bit, prob(node)); + } + } + + /// Add the 0/1 counts of every table entry this tile used. + pub fn count(&self, counts: &mut [[u32; 2]]) { + for s in &self.syms { + if let P::Ctx(i) = s.p { + counts[i as usize][s.bit as usize] += 1; + } + } + } + + /// Entropy code the recorded symbols. + pub fn emit(&self, probs: &Probs) -> Vec { + let mut bc = BoolEncoder::new(); + bc.write(false, 128); // marker bit + for s in &self.syms { + let p = match s.p { + P::Fixed(p) => p, + P::Ctx(i) => probs.0[i as usize], + P::Pareto(i, n) => pareto(probs.0[i as usize])[n as usize], + }; + bc.write(s.bit, p); + } + bc.finish() + } +} + +/// The (node, bit) path to `value` in a tree (node = tree index / 2). +pub fn tree_path(tree: &[i8], value: u8) -> impl Iterator { + struct Path { + steps: [(u8, bool); 16], + len: usize, + } + fn walk(tree: &[i8], node: usize, value: u8, out: &mut Path) -> bool { + for b in 0..2 { + let t = tree[node + b]; + out.steps[out.len] = ((node >> 1) as u8, b == 1); + out.len += 1; + if t <= 0 { + if (-t) as u8 == value { + return true; + } + } else if walk(tree, t as usize, value, out) { + return true; + } + out.len -= 1; + } + false + } + let mut p = Path { steps: [(0, false); 16], len: 0 }; + walk(tree, 0, value, &mut p); + let Path { steps, len } = p; + steps.into_iter().take(len).map(|(n, b)| (n as usize, b)) +} + +// ---------------------------------------------------------------- costs + +/// Cost in 1/256 bit of coding a 0 (`[p][0]`) or a 1 (`[p][1]`) with probability `p`. +pub fn cost_table() -> &'static [[u32; 2]; 256] { + use std::sync::OnceLock; + static T: OnceLock<[[u32; 2]; 256]> = OnceLock::new(); + T.get_or_init(|| { + let mut t = [[0u32; 2]; 256]; + for (p, e) in t.iter_mut().enumerate() { + let q = (p.max(1) as f64) / 256.0; + e[0] = (-q.log2() * 256.0).round() as u32; + e[1] = (-(1.0 - q).log2() * 256.0).round() as u32; + } + t + }) +} + +#[inline] +pub fn cost(bit: bool, p: u8) -> u32 { + cost_table()[p as usize][bit as usize] +} + +/// Cost of `value` in a tree with probabilities `probs` (node i → probs[i]). +pub fn tree_cost(tree: &[i8], probs: &[u8], value: u8) -> u32 { + tree_path(tree, value).map(|(n, b)| cost(b, probs[n])).sum() +} + +// ---------------------------------------------------------------- forward updates + +fn inv_recenter_nonneg(v: i32, m: i32) -> i32 { + if v > 2 * m { + v + } else if v & 1 != 0 { + m - ((v + 1) >> 1) + } else { + m + (v >> 1) + } +} + +/// `inv_remap_prob` (6.3.5). +fn inv_remap_prob(delta: usize, prob: u8) -> u8 { + let v = inv_map_table()[delta.min(254)] as i32; + let m = prob as i32 - 1; + let r = if (m << 1) <= 255 { 1 + inv_recenter_nonneg(v, m) } else { 255 - inv_recenter_nonneg(v, 255 - 1 - m) }; + r.clamp(1, 255) as u8 +} + +/// The `decode_term_subexp` value that moves `old` to `new`. +fn delta_for(old: u8, new: u8) -> Option { + (0..255).find(|&d| inv_remap_prob(d, old) == new) +} + +/// Bits of `decode_term_subexp` for `d`. +fn subexp_bits(d: usize) -> u32 { + match d { + 0..16 => 5, + 16..32 => 6, + 32..64 => 8, + 64..129 => 10, + _ => 11, + } +} + +fn write_subexp(bc: &mut BoolEncoder, d: usize) { + let d = d as u32; + if d < 16 { + bc.literal(0, 1); + bc.literal(d, 4); + } else if d < 32 { + bc.literal(2, 2); + bc.literal(d - 16, 4); + } else if d < 64 { + bc.literal(6, 3); + bc.literal(d - 32, 5); + } else { + bc.literal(7, 3); + if d < 129 { + bc.literal(d - 64, 7); + } else { + let v = (d + 1) >> 1; + bc.literal(v, 7); + bc.literal((d + 1) & 1, 1); + } + } +} + +fn counts_cost(c: [u32; 2], p: u8) -> u64 { + c[0] as u64 * cost(false, p) as u64 + c[1] as u64 * cost(true, p) as u64 +} + +/// Whether table entry `i` is an MV probability (7-bit `update_mv_prob` rather than +/// `diff_update_prob`). +fn is_mv(i: usize) -> bool { + i >= MV_JOINT +} + +/// Choose updated probabilities: an entry changes when the bits saved on this frame's symbols +/// exceed the cost of signalling the change. `inter`: the inter-frame tables are updatable. +pub fn plan_updates(old: &Probs, counts: &[[u32; 2]], inter: bool) -> Probs { + let mut new = old.clone(); + let flag = cost(true, 252) as i64 - cost(false, 252) as i64; + let limit = if inter { NPROBS } else { INTER_MODE }; + for i in 0..limit { + let c = counts[i]; + let n = c[0] + c[1]; + if n == 0 { + continue; + } + let o = old.0[i]; + let ideal = ((c[0] as f64 * 256.0 / n as f64).round() as i32).clamp(1, 255) as u8; + let cands: Vec = if is_mv(i) { vec![ideal | 1, ideal.saturating_sub(1).max(1) | 1] } else { vec![ideal] }; + let base = counts_cost(c, o) as i64; + let mut best = (0i64, o); + for p in cands { + if p == o { + continue; + } + let signal = if is_mv(i) { + 7 * 256 + } else { + match delta_for(o, p) { + Some(d) => subexp_bits(d) as i64 * 256, + None => continue, + } + }; + let gain = base - counts_cost(c, p) as i64 - signal - flag; + if gain > best.0 { + best = (gain, p); + } + } + new.0[i] = best.1; + } + new +} + +/// `diff_update_prob` for one entry. +fn diff_update(bc: &mut BoolEncoder, old: &Probs, new: &Probs, i: usize) { + let (o, n) = (old.0[i], new.0[i]); + match (o != n).then(|| delta_for(o, n)).flatten() { + Some(d) => { + bc.write(true, 252); + write_subexp(bc, d); + } + None => bc.write(false, 252), + } +} + +/// `update_mv_prob` for one entry. +fn mv_update(bc: &mut BoolEncoder, old: &Probs, new: &Probs, i: usize) { + let n = new.0[i]; + if n != old.0[i] && n & 1 == 1 { + bc.write(true, 252); + bc.literal(n as u32 >> 1, 7); + } else { + bc.write(false, 252); + } +} + +/// The compressed header (6.3) of a frame coded with ONLY_4X4 transforms, single LAST +/// references and a fixed interpolation filter, moving the probabilities from `old` to `new`. +/// Entries `new` cannot signal are left at `old` (and must be coded with `old`): the caller +/// codes the tiles with the probabilities this returns. +pub fn compressed_header(old: &Probs, new: &Probs, lossless: bool, inter: bool) -> (Vec, Probs) { + let mut bc = BoolEncoder::new(); + bc.write(false, 128); + if !lossless { + bc.literal(0, 2); // tx_mode = ONLY_4X4 + } + // read_coef_probs: TX_4X4 only. + let coef_changed = (COEF..COEF + 432).any(|i| new.0[i] != old.0[i]); + bc.literal(coef_changed as u32, 1); + if coef_changed { + for i in 0..2 { + for j in 0..2 { + for k in 0..6 { + for l in 0..if k == 0 { 3 } else { 6 } { + for m in 0..3 { + diff_update(&mut bc, old, new, coef_index(i, j == 1, k, l) + m); + } + } + } + } + } + } + for i in 0..3 { + diff_update(&mut bc, old, new, SKIP + i); + } + if inter { + for i in 0..21 { + diff_update(&mut bc, old, new, INTER_MODE + i); + } + for i in 0..4 { + diff_update(&mut bc, old, new, IS_INTER + i); + } + // reference_mode is SINGLE_REFERENCE (no compound: equal sign biases). + for i in 0..10 { + diff_update(&mut bc, old, new, SINGLE_REF + i); + } + for _ in 0..36 { + bc.write(false, 252); // y mode probabilities (no intra blocks in inter frames) + } + for i in 0..48 { + diff_update(&mut bc, old, new, PARTITION + i); + } + for j in 0..3 { + mv_update(&mut bc, old, new, MV_JOINT + j); + } + for i in 0..2 { + mv_update(&mut bc, old, new, MV_SIGN + i); + for j in 0..10 { + mv_update(&mut bc, old, new, MV_CLASS + i * 10 + j); + } + mv_update(&mut bc, old, new, MV_CLASS0_BIT + i); + for j in 0..10 { + mv_update(&mut bc, old, new, MV_BITS + i * 10 + j); + } + } + for i in 0..2 { + for j in 0..6 { + mv_update(&mut bc, old, new, MV_CLASS0_FR + i * 6 + j); + } + for j in 0..3 { + mv_update(&mut bc, old, new, MV_FR + i * 3 + j); + } + } + } + // What the decoder ends up with. + let mut used = old.clone(); + for i in 0..NPROBS { + let ok = if is_mv(i) { new.0[i] & 1 == 1 } else { delta_for(old.0[i], new.0[i]).is_some() }; + if ok && (inter || i < INTER_MODE) { + used.0[i] = new.0[i]; + } + } + (bc.finish(), used) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn remap_reaches_every_probability() { + for old in 1..=255u8 { + for new in 1..=255u8 { + if new != old { + assert!(delta_for(old, new).is_some(), "{old} -> {new}"); + } + } + } + } + + #[test] + fn inv_map_table_shape() { + let t = inv_map_table(); + assert_eq!(&t[..4], &[7, 20, 33, 46]); + assert_eq!(t[19], 254); + assert_eq!(&t[20..27], &[1, 2, 3, 4, 5, 6, 8]); + assert_eq!(&t[252..], &[252, 253, 253]); + } +} diff --git a/crates/vp9enc/src/tables.rs b/crates/vp9enc/src/tables.rs index dc8027a6..ffb26641 100644 --- a/crates/vp9enc/src/tables.rs +++ b/crates/vp9enc/src/tables.rs @@ -128,3 +128,76 @@ pub const DEFAULT_COEF_PROBS: [u8; 432] = [ 79, 1, 20, 39, 99, 167, 239, 47, 141, 224, 10, 104, 178, 2, 73, 133, 1, 44, 85, 1, 22, 47, 127, 145, 243, 71, 129, 228, 17, 93, 177, 3, 61, 124, 1, 41, 84, 1, 21, 52, 157, 78, 244, 140, 72, 231, 69, 58, 184, 31, 44, 137, 14, 38, 105, 8, 23, 61, ]; + +// ---------------------------------------------------------------- inter frames (spec section 10.5) + +/// `inv_map_table` (used by `inv_remap_prob`): the 20 values 7 + 13k first, then every other value +/// 1..=253 in increasing order, then 253 again. +pub fn inv_map_table() -> &'static [u8; 255] { + use std::sync::OnceLock; + static T: OnceLock<[u8; 255]> = OnceLock::new(); + T.get_or_init(|| { + let mut t = [253u8; 255]; + let mut n = 0; + for k in 0..20 { + t[n] = 7 + 13 * k; + n += 1; + } + for v in 1..=253u8 { + if v < 7 || (v - 7) % 13 != 0 { + t[n] = v; + n += 1; + } + } + t + }) +} + +/// The regular 8-tap sub-sample interpolation filters (`subpel_filters[EIGHTTAP]`), 16 positions. +pub const SUBPEL_REGULAR: [[i32; 8]; 16] = [ + [0, 0, 0, 128, 0, 0, 0, 0], + [0, 1, -5, 126, 8, -3, 1, 0], + [-1, 3, -10, 122, 18, -6, 2, 0], + [-1, 4, -13, 118, 27, -9, 3, -1], + [-1, 4, -16, 112, 37, -11, 4, -1], + [-1, 5, -18, 105, 48, -14, 4, -1], + [-1, 5, -19, 97, 58, -16, 5, -1], + [-1, 6, -19, 88, 68, -18, 5, -1], + [-1, 6, -19, 78, 78, -19, 6, -1], + [-1, 5, -18, 68, 88, -19, 6, -1], + [-1, 5, -16, 58, 97, -19, 5, -1], + [-1, 4, -14, 48, 105, -18, 5, -1], + [-1, 4, -11, 37, 112, -16, 4, -1], + [-1, 3, -9, 27, 118, -13, 4, -1], + [0, 2, -6, 18, 122, -10, 3, -1], + [0, 1, -3, 8, 126, -5, 1, 0], +]; + +/// `mv_ref_blocks` (row, column offsets) for BLOCK_8X8 and BLOCK_16X16. +pub const MV_REF_8X8: [(i32, i32); 8] = [(-1, 0), (0, -1), (-1, -1), (-2, 0), (0, -2), (-2, -1), (-1, -2), (-2, -2)]; +pub const MV_REF_16X16: [(i32, i32); 8] = [(-1, 0), (0, -1), (-1, 1), (1, -1), (-1, -1), (-3, 0), (0, -3), (-3, -3)]; + +/// `mode_2_counter` indexed by y mode (intra modes 0..=9, then NEARESTMV, NEARMV, ZEROMV, NEWMV). +pub const MODE_2_COUNTER: [u8; 14] = [9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 0, 0, 3, 1]; + +/// `counter_to_context`; INVALID_CASE entries are 9. +pub const COUNTER_TO_CONTEXT: [u8; 19] = [2, 3, 4, 1, 3, 9, 0, 9, 9, 5, 5, 9, 5, 9, 9, 9, 9, 9, 6]; + +pub const DEFAULT_PARTITION_PROBS: [u8; 48] = [ + 199, 122, 141, 147, 63, 159, 148, 133, 118, 121, 104, 114, 174, 73, 87, 92, 41, 83, 82, 99, 50, 53, 39, 39, 177, 58, 59, 68, 26, 63, 52, 79, 25, 17, 14, + 12, 222, 34, 30, 72, 16, 44, 58, 32, 12, 10, 7, 6, +]; + +pub const DEFAULT_IS_INTER_PROB: [u8; 4] = [9, 102, 187, 225]; + +pub const DEFAULT_SINGLE_REF_PROB: [u8; 10] = [33, 16, 77, 74, 142, 142, 172, 170, 238, 247]; + +pub const DEFAULT_INTER_MODE_PROBS: [u8; 21] = [2, 173, 34, 7, 145, 85, 7, 166, 63, 7, 94, 66, 8, 64, 46, 17, 81, 31, 25, 29, 30]; + +pub const DEFAULT_MV_JOINT_PROBS: [u8; 3] = [32, 64, 96]; +pub const DEFAULT_MV_SIGN_PROB: [u8; 2] = [128, 128]; +pub const DEFAULT_MV_CLASS_PROBS: [u8; 20] = [224, 144, 192, 168, 192, 176, 192, 198, 198, 245, 216, 128, 176, 160, 176, 176, 192, 198, 198, 208]; +pub const DEFAULT_MV_CLASS0_BIT_PROB: [u8; 2] = [216, 208]; +pub const DEFAULT_MV_BITS_PROB: [u8; 20] = [136, 140, 148, 160, 176, 192, 224, 234, 234, 240, 136, 140, 148, 160, 176, 192, 224, 234, 234, 240]; +pub const DEFAULT_MV_CLASS0_FR_PROBS: [u8; 12] = [128, 128, 64, 96, 112, 64, 128, 128, 64, 96, 112, 64]; +pub const DEFAULT_MV_FR_PROBS: [u8; 6] = [64, 96, 64, 64, 96, 64]; diff --git a/crates/vp9enc/src/tile.rs b/crates/vp9enc/src/tile.rs new file mode 100644 index 00000000..fe3a1a49 --- /dev/null +++ b/crates/vp9enc/src/tile.rs @@ -0,0 +1,894 @@ +//! Tile coding: partitions, intra blocks (key frames) and inter blocks (motion search, mode +//! decision, motion vector prediction, residual), recorded as symbols (see [`crate::probs`]). +//! +//! Mode info, motion vector candidate lists and contexts follow the decoding process of the +//! VP9 specification (6.4, 6.4.x `find_mv_refs` / `find_best_ref_mvs`) for the subset of the +//! syntax this encoder emits: 8×8 and 16×16 blocks, single LAST references, no +//! `UsePrevFrameMvs` (inter frames are error resilient), no high-precision vectors. + +use crate::mc::{Mv, PhasePlanes, RefPlane, clamp_for_plane, predict}; +use crate::probs::{self, P, Writer, cost, tree_cost}; +use crate::tables::*; +use crate::transform::{ADST_ADST, ADST_DCT, DCT_ADST, DCT_DCT, Quant}; +use crate::{BLOCK_8X8, BLOCK_16X16, Mi, Plane}; + +pub const DC_PRED: u8 = 0; +pub const V_PRED: u8 = 1; +pub const H_PRED: u8 = 2; +pub const TM_PRED: u8 = 9; +const MODES: [u8; 4] = [DC_PRED, V_PRED, H_PRED, TM_PRED]; +pub const NEARESTMV: u8 = 10; +pub const NEARMV: u8 = 11; +pub const ZEROMV: u8 = 12; +pub const NEWMV: u8 = 13; + +const PARTITION_TREE: [i8; 6] = [0, 2, -1, 4, -2, -3]; +const INTRA_MODE_TREE: [i8; 18] = [0, 2, -9, 4, -1, 6, 8, 12, -2, 10, -4, -5, -3, 14, -8, 16, -6, -7]; +/// Leaves are `mode - NEARESTMV`. +const INTER_MODE_TREE: [i8; 6] = [-2, 2, 0, 4, -1, -3]; +const MV_JOINT_TREE: [i8; 6] = [0, 2, -1, 4, -2, -3]; +const MV_CLASS_TREE: [i8; 20] = [0, 2, -1, 4, 6, 8, -2, -3, 10, 12, -4, -5, -6, 14, 16, 18, -7, -8, -9, -10]; +const MV_FR_TREE: [i8; 6] = [0, 2, -1, 4, -2, -3]; + +/// extra_bits[token] = (cat, numExtra, base). +const EXTRA_BITS: [(u8, u8, u16); 11] = + [(0, 0, 0), (0, 0, 1), (0, 0, 2), (0, 0, 3), (0, 0, 4), (1, 1, 5), (2, 2, 7), (3, 3, 11), (4, 4, 19), (5, 5, 35), (6, 14, 67)]; +const CAT_PROBS: [&[u8]; 7] = [ + &[], + &[159], + &[165, 145], + &[173, 148, 140], + &[176, 155, 140, 135], + &[180, 157, 141, 134, 130], + &[254, 254, 254, 252, 249, 243, 230, 196, 177, 153, 140, 133, 130, 129], +]; + +/// mode2txfm_map for the intra modes used here (inter blocks use DCT_DCT). +fn mode_tx_type(mode: u8) -> u8 { + match mode { + V_PRED => ADST_DCT, + H_PRED => DCT_ADST, + TM_PRED => ADST_ADST, + _ => DCT_DCT, + } +} + +fn scan(tx_type: u8) -> &'static [u16; 16] { + match tx_type { + ADST_DCT => &ROW_SCAN_4X4, + DCT_ADST => &COL_SCAN_4X4, + _ => &DEFAULT_SCAN_4X4, + } +} + +/// Coefficient context neighbours (9.3.2) of each scan position. +fn neighbors(tx_type: u8) -> [(usize, usize); 16] { + let sc = scan(tx_type); + let mut nb = [(0, 0); 16]; + for (c, &pos) in sc.iter().enumerate().skip(1) { + let pos = pos as usize; + let (i, j) = (pos / 4, pos % 4); + nb[c] = if i > 0 && j > 0 { + let a = (i - 1) * 4 + j; + let b = i * 4 + j - 1; + match tx_type { + DCT_ADST => (a, a), + ADST_DCT => (b, b), + _ => (a, b), + } + } else if i > 0 { + ((i - 1) * 4 + j, (i - 1) * 4 + j) + } else { + (j - 1, j - 1) + }; + } + nb +} + +fn eob_of(levels: &[i32; 16], tx_type: u8) -> usize { + let sc = scan(tx_type); + (0..16).rev().find(|&i| levels[sc[i] as usize] != 0).map_or(0, |i| i + 1) +} + +/// Rough coefficient bit cost (1/256 bit) for rate-distortion decisions. +fn levels_cost(levels: &[i32; 16], eob: usize) -> u32 { + if eob == 0 { + return 0; + } + let mut bits = 256; + for &l in levels { + let a = l.unsigned_abs(); + bits += if a == 0 { 160 } else { 640 + 384 * (32 - a.leading_zeros()) }; + } + bits +} + +/// Intra prediction (8.5.1) of a square block for DC / V / H / TM. +fn predict_intra(p: &Plane, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> [i32; 64] { + let size = 1usize << log2; + let base = 128i32; + let buf = &p.rec; + let w = p.w; + let mut above = [0i32; 8]; + let mut left = [0i32; 8]; + if have_above { + for i in 0..size { + above[i] = buf[(y - 1) * w + (x + i).min(p.max_x)] as i32; + } + } else { + above[..size].fill(base - 1); + } + let top_left = if have_above && have_left { + buf[(y - 1) * w + (x - 1).min(p.max_x)] as i32 + } else if have_above { + base + 1 + } else { + base - 1 + }; + if have_left { + for i in 0..size { + left[i] = buf[(y + i).min(p.max_y) * w + x - 1] as i32; + } + } else { + left[..size].fill(base + 1); + } + let mut out = [0i32; 64]; + for i in 0..size { + for j in 0..size { + out[i * size + j] = match mode { + V_PRED => above[j], + H_PRED => left[i], + TM_PRED => (above[j] + left[i] - top_left).clamp(0, 255), + _ => 0, + }; + } + } + if mode == DC_PRED { + let v = match (have_left, have_above) { + (true, true) => (left[..size].iter().sum::() + above[..size].iter().sum::() + size as i32) >> (log2 + 1), + (true, false) => (left[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, + (false, true) => (above[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, + (false, false) => base, + }; + out[..size * size].fill(v); + } + out +} + +// ---------------------------------------------------------------- motion vector coding + +/// `mv_class` of an offset (|component| - 1). +fn mv_class(offset: u32) -> u32 { + if offset < 16 { 0 } else { (31 - offset.leading_zeros()) - 3 } +} + +fn mv_component_cost(i: usize, v: i32) -> u32 { + let d = &probs_defaults().0; + let offset = v.unsigned_abs() - 1; + let c = mv_class(offset); + let mut bits = cost(v < 0, d[probs::MV_SIGN + i]); + bits += tree_cost(&MV_CLASS_TREE, &d[probs::MV_CLASS + i * 10..], c as u8); + let fr = ((offset >> 1) & 3) as u8; + if c == 0 { + let b = offset >> 3; + bits += cost(b == 1, d[probs::MV_CLASS0_BIT + i]); + bits += tree_cost(&MV_FR_TREE, &d[probs::MV_CLASS0_FR + i * 6 + b as usize * 3..], fr); + } else { + let dd = (offset - (1 << (c + 3))) >> 3; + for b in 0..c { + bits += cost((dd >> b) & 1 == 1, d[probs::MV_BITS + i * 10 + b as usize]); + } + bits += tree_cost(&MV_FR_TREE, &d[probs::MV_FR + i * 3..], fr); + } + bits +} + +const MV_TABLE: i32 = 8192; + +/// Component costs for |v| ≤ MV_TABLE (index v + MV_TABLE) and the four joint costs. +fn mv_tables() -> &'static ([Vec; 2], [u32; 4]) { + use std::sync::OnceLock; + static T: OnceLock<([Vec; 2], [u32; 4])> = OnceLock::new(); + T.get_or_init(|| { + let comp = std::array::from_fn(|i| (-MV_TABLE..=MV_TABLE).map(|v| if v == 0 { 0 } else { mv_component_cost(i, v) }).collect()); + let d = &probs_defaults().0; + let joints = std::array::from_fn(|j| tree_cost(&MV_JOINT_TREE, &d[probs::MV_JOINT..], j as u8)); + (comp, joints) + }) +} + +/// Bits (1/256) of a NEWMV difference with the default probabilities. +pub fn mv_cost(diff: Mv) -> u32 { + let (comp, joints) = mv_tables(); + let joint = (diff.row != 0) as usize * 2 + (diff.col != 0) as usize; + let c = |i: usize, v: i32| if v.abs() <= MV_TABLE { comp[i][(v + MV_TABLE) as usize] } else { mv_component_cost(i, v) }; + joints[joint] + if diff.row != 0 { c(0, diff.row) } else { 0 } + if diff.col != 0 { c(1, diff.col) } else { 0 } +} + +fn probs_defaults() -> &'static probs::Probs { + use std::sync::OnceLock; + static D: OnceLock = OnceLock::new(); + D.get_or_init(probs::Probs::defaults) +} + +// ---------------------------------------------------------------- the tile + +/// Per-frame coding parameters shared by the tiles. +pub struct FrameParams<'a> { + pub key: bool, + pub quant: Quant, + pub mi_cols: usize, + pub mi_rows: usize, + /// Reference planes (inter frames). + pub refs: Option<[RefPlane<'a>; 3]>, + /// The reference luma at every quarter-sample phase (motion search). + pub phases: Option, + /// Encoder-side motion hints from the previous frame (by MI position). + pub prev_mvs: Option<&'a [Mv]>, + /// Lagrangian multipliers: per bit, in SAD and in squared-error units. + pub lambda_sad: f64, + pub lambda_sse: f64, +} + +pub struct Tile<'a> { + pub f: &'a FrameParams<'a>, + pub w: Writer, + pub planes: &'a mut [Plane; 3], + pub mi: &'a mut [Mi], + pub col_start: usize, + pub col_end: usize, + above_nz: [Vec; 3], + left_nz: [Vec; 3], + above_part: Vec, + left_part: Vec, + /// Motion vectors of the four 8×8 blocks evaluated for the current 16×16 area. + pending: [(usize, usize, Mv); 4], +} + +impl<'a> Tile<'a> { + pub fn new(f: &'a FrameParams<'a>, planes: &'a mut [Plane; 3], mi: &'a mut [Mi], col_start: usize, col_end: usize) -> Tile<'a> { + let n = f.mi_cols * 2 + 16; + Tile { + f, + w: Writer::default(), + planes, + mi, + col_start, + col_end, + above_nz: [vec![0; n], vec![0; n], vec![0; n]], + left_nz: [vec![0; 16], vec![0; 16], vec![0; 16]], + above_part: vec![0; f.mi_cols + 8], + left_part: vec![0; 8], + pending: [(usize::MAX, 0, Mv::ZERO); 4], + } + } + + /// Code the tile's superblocks. + pub fn code(&mut self) { + for r in (0..self.f.mi_rows).step_by(8) { + for p in 0..3 { + self.left_nz[p].fill(0); + } + self.left_part.fill(0); + for c in (self.col_start..self.col_end).step_by(8) { + self.partition(r, c, 12); + } + } + } + + fn mi_at(&self, r: usize, c: usize) -> Mi { + self.mi[r * self.f.mi_cols + c] + } + + fn set_mi(&mut self, r: usize, c: usize, n8: usize, m: Mi) { + for dr in 0..n8 { + for dc in 0..n8 { + if r + dr < self.f.mi_rows && c + dc < self.f.mi_cols { + self.mi[(r + dr) * self.f.mi_cols + c + dc] = m; + } + } + } + } + + fn partition(&mut self, r: usize, c: usize, bsize: u8) { + let (mi_rows, mi_cols) = (self.f.mi_rows, self.f.mi_cols); + if r >= mi_rows || c >= mi_cols { + return; + } + let num8 = [1usize, 1, 1, 1, 1, 2, 2, 2, 4, 4, 4, 8, 8][bsize as usize]; + let half = num8 >> 1; + let has_rows = (r + half) < mi_rows; + let has_cols = (c + half) < mi_cols; + let bsl = [0usize, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3][bsize as usize]; + let boffset = 3 - bsl; + let mut above = 0u8; + let mut left = 0u8; + for i in 0..num8 { + above |= self.above_part[c + i]; + left |= self.left_part[(r & 7) + i]; + } + let ctx = bsl * 4 + (((left >> boffset) & 1) as usize) * 2 + ((above >> boffset) & 1) as usize; + let key = self.f.key; + let prob = |i: usize| if key { P::Fixed(KF_PARTITION_PROBS[ctx * 3 + i]) } else { P::Ctx((probs::PARTITION + ctx * 3 + i) as u16) }; + if bsize == BLOCK_8X8 { + self.w.put(false, prob(0)); // PARTITION_NONE + if key { + self.intra_block(r, c); + } else { + let mv = self.pending.iter().find(|p| p.0 == r && p.1 == c).map(|p| p.2); + let mv = mv.unwrap_or_else(|| self.search(r, c, BLOCK_8X8).0); + self.inter_block(r, c, BLOCK_8X8, mv); + } + self.above_part[c] = 15 >> 1; + self.left_part[r & 7] = 15 >> 1; + return; + } + if !key && bsize == BLOCK_16X16 && has_rows && has_cols { + if let Some(mv) = self.decide16(r, c) { + self.w.put(false, prob(0)); // PARTITION_NONE + self.inter_block(r, c, BLOCK_16X16, mv); + for i in 0..2 { + self.above_part[c + i] = 15 >> 2; + self.left_part[(r & 7) + i] = 15 >> 2; + } + return; + } + } else if !key && bsize == BLOCK_16X16 { + self.pending = [(usize::MAX, 0, Mv::ZERO); 4]; + } + // PARTITION_SPLIT. + if has_rows && has_cols { + self.w.tree(&PARTITION_TREE, 3, prob); + } else if has_cols { + self.w.put(true, prob(1)); + } else if has_rows { + self.w.put(true, prob(2)); + } + let sub = bsize - 3; + self.partition(r, c, sub); + self.partition(r, c + half, sub); + self.partition(r + half, c, sub); + self.partition(r + half, c + half, sub); + } + + // ------------------------------------------------------------ intra (key frames) + + fn intra_block(&mut self, r: usize, c: usize) { + let avail_u = r > 0; + let avail_l = c > self.col_start; + let y_mode = self.choose_intra(0, c * 8, r * 8, 3, avail_l, avail_u); + let uv_mode = { + let mut best = (i64::MAX, DC_PRED); + for m in MODES { + let cost: i64 = (1..3).map(|p| self.intra_cost(p, c * 4, r * 4, 2, m, avail_l, avail_u)).sum(); + if cost < best.0 { + best = (cost, m); + } + } + best.1 + }; + // skip = 0 (neighbours never skip in key frames: context 0); tx_size implied (4×4). + self.w.ctx(false, probs::SKIP); + let above_mode = if avail_u { self.mi_at(r - 1, c).mode } else { DC_PRED }; + let left_mode = if avail_l { self.mi_at(r, c - 1).mode } else { DC_PRED }; + let o = (above_mode as usize * 10 + left_mode as usize) * 9; + self.w.tree(&INTRA_MODE_TREE, y_mode, |i| P::Fixed(KF_Y_MODE_PROBS[o + i])); + let o = y_mode as usize * 9; + self.w.tree(&INTRA_MODE_TREE, uv_mode, |i| P::Fixed(KF_UV_MODE_PROBS[o + i])); + self.set_mi(r, c, 1, Mi { inter: false, mode: y_mode, mv: Mv::ZERO, skip: false, size: BLOCK_8X8 }); + // Residual: luma 2×2 transform blocks, then one 4×4 block per chroma plane. + for (dy, dx) in [(0, 0), (0, 1), (1, 0), (1, 1)] { + let tx = if self.f.quant.lossless { DCT_DCT } else { mode_tx_type(y_mode) }; + self.intra_tx(0, c * 8 + dx * 4, r * 8 + dy * 4, y_mode, tx, avail_l || dx > 0, avail_u || dy > 0); + } + for p in 1..3 { + self.intra_tx(p, c * 4, r * 4, uv_mode, DCT_DCT, avail_l, avail_u); + } + } + + #[allow(clippy::too_many_arguments)] + fn intra_cost(&self, plane: usize, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> i64 { + let p = &self.planes[plane]; + let pred = predict_intra(p, x, y, log2, mode, have_left, have_above); + let size = 1 << log2; + let mut s = 0i64; + for i in 0..size { + for j in 0..size { + s += (p.src[(y + i) * p.w + x + j] as i32 - pred[i * size + j]).abs() as i64; + } + } + s + } + + fn choose_intra(&self, plane: usize, x: usize, y: usize, log2: usize, have_left: bool, have_above: bool) -> u8 { + let mut best = (i64::MAX, DC_PRED); + for m in MODES { + let cost = self.intra_cost(plane, x, y, log2, m, have_left, have_above); + if cost < best.0 { + best = (cost, m); + } + } + best.1 + } + + #[allow(clippy::too_many_arguments)] + fn intra_tx(&mut self, plane: usize, x: usize, y: usize, mode: u8, tx_type: u8, have_left: bool, have_above: bool) { + let ss = (plane > 0) as usize; + let max_x = (self.f.mi_cols * 8) >> ss; + let max_y = (self.f.mi_rows * 8) >> ss; + let (x4, y4) = (x >> 2, y >> 2); + let ly = y4 & (15 >> ss); + if x >= max_x || y >= max_y { + self.above_nz[plane][x4] = 0; + self.left_nz[plane][ly] = 0; + return; + } + let p = &self.planes[plane]; + let pred = predict_intra(p, x, y, 2, mode, have_left, have_above); + let mut res = [0i32; 16]; + for i in 0..4 { + for j in 0..4 { + res[i * 4 + j] = p.src[(y + i) * p.w + x + j] as i32 - pred[i * 4 + j]; + } + } + let q = Quant { round_dc: 0.5, round_ac: 0.38, ..self.f.quant }; + let levels = q.levels(&res, tx_type); + let eob = eob_of(&levels, tx_type); + let ctx0 = (self.above_nz[plane][x4] + self.left_nz[plane][ly]) as usize; + self.tokens(plane, false, tx_type, &levels, eob, ctx0); + let resid = q.residual(&levels, tx_type, eob); + let p = &mut self.planes[plane]; + for i in 0..4 { + for j in 0..4 { + p.rec[(y + i) * p.w + x + j] = (pred[i * 4 + j] + resid[i * 4 + j]).clamp(0, 255) as u8; + } + } + let nz = (eob > 0) as u8; + self.above_nz[plane][x4] = nz; + self.left_nz[plane][ly] = nz; + } + + fn tokens(&mut self, plane: usize, inter: bool, tx_type: u8, levels: &[i32; 16], eob: usize, ctx0: usize) { + let sc = scan(tx_type); + let nb = neighbors(tx_type); + let ptype = (plane > 0) as usize; + let mut tc = [0u8; 16]; + let mut check_eob = true; + let mut c = 0; + while c < 16 { + let pos = sc[c] as usize; + let band = COEFBAND_4X4[c] as usize; + let ctx = if c == 0 { ctx0 } else { ((1 + tc[nb[c].0] + tc[nb[c].1]) >> 1) as usize }; + let o = probs::coef_index(ptype, inter, band, ctx); + if check_eob { + let more = c < eob; + self.w.ctx(more, o); + if !more { + break; + } + } + let v = levels[pos].unsigned_abs(); + if v == 0 { + self.w.ctx(false, o + 1); + tc[pos] = 0; + check_eob = false; + c += 1; + continue; + } + self.w.ctx(true, o + 1); + check_eob = true; + let token = match v { + 1 => 1, + 2 => 2, + 3 => 3, + 4 => 4, + 5..=6 => 5, + 7..=10 => 6, + 11..=18 => 7, + 19..=34 => 8, + 35..=66 => 9, + _ => 10, + }; + if token == 1 { + self.w.ctx(false, o + 2); + } else { + self.w.ctx(true, o + 2); + let path: &[(u8, bool)] = match token { + 2 => &[(0, false), (1, false)], + 3 => &[(0, false), (1, true), (2, false)], + 4 => &[(0, false), (1, true), (2, true)], + 5 => &[(0, true), (3, false), (4, false)], + 6 => &[(0, true), (3, false), (4, true)], + 7 => &[(0, true), (3, true), (5, false), (6, false)], + 8 => &[(0, true), (3, true), (5, false), (6, true)], + 9 => &[(0, true), (3, true), (5, true), (7, false)], + _ => &[(0, true), (3, true), (5, true), (7, true)], + }; + for &(n, b) in path { + self.w.put(b, P::Pareto((o + 2) as u16, n)); + } + let (cat, num_extra, base_v) = EXTRA_BITS[token]; + let extra = (v - base_v as u32).min((1 << num_extra) - 1); + let cp = CAT_PROBS[cat as usize]; + for e in 0..num_extra as usize { + self.w.fixed((extra >> (num_extra as usize - 1 - e)) & 1 != 0, cp[e]); + } + } + tc[pos] = ENERGY_CLASS[token]; + self.w.fixed(levels[pos] < 0, 128); + c += 1; + } + } + + // ------------------------------------------------------------ inter + + fn is_inside(&self, r: i32, c: i32) -> bool { + r >= 0 && (r as usize) < self.f.mi_rows && c >= self.col_start as i32 && (c as usize) < self.col_end + } + + /// `find_mv_refs` + `find_best_ref_mvs` for LAST_FRAME: (NearestMv, NearMv, mode context). + fn mv_refs(&self, r: usize, c: usize, bsize: u8) -> (Mv, Mv, usize) { + let search = if bsize == BLOCK_16X16 { &MV_REF_16X16 } else { &MV_REF_8X8 }; + let mut list = [Mv::ZERO; 2]; + let mut count = 0; + let add = |mv: Mv, list: &mut [Mv; 2], count: &mut usize| { + if *count >= 2 || (*count > 0 && mv == list[0]) { + return; + } + list[*count] = mv; + *count += 1; + }; + let mut counter = 0usize; + for (i, &(dr, dc)) in search.iter().enumerate() { + let (cr, cc) = (r as i32 + dr, c as i32 + dc); + if !self.is_inside(cr, cc) { + continue; + } + let m = self.mi_at(cr as usize, cc as usize); + if i < 2 { + counter += MODE_2_COUNTER[m.mode as usize] as usize; + } + if m.inter { + add(m.mv, &mut list, &mut count); + } + } + let ctx = COUNTER_TO_CONTEXT[counter.min(18)] as usize; + let n8 = if bsize == BLOCK_16X16 { 2 } else { 1 }; + let clamp = |mv: Mv, border: i32| -> Mv { + let (r, c, b) = (r as i32, c as i32, n8); + let to_top = -(r * 64); + let to_bottom = (self.f.mi_rows as i32 - b - r) * 64; + let to_left = -(c * 64); + let to_right = (self.f.mi_cols as i32 - b - c) * 64; + Mv::new( + mv.row.clamp(to_top - border, (to_bottom + border).max(to_top - border)), + mv.col.clamp(to_left - border, (to_right + border).max(to_left - border)), + ) + }; + for mv in list.iter_mut() { + *mv = clamp(*mv, 128); + // No high precision: odd components move toward zero. + let even = |v: i32| if v & 1 != 0 { v + if v > 0 { -1 } else { 1 } } else { v }; + *mv = clamp(Mv::new(even(mv.row), even(mv.col)), (160 - 4) << 3); + } + (list[0], list[1], ctx) + } + + /// The allowed motion vector range of a block (no decoder clamping inside it). + fn mv_range(&self, r: usize, c: usize, n: i32) -> (i32, i32, i32, i32) { + let n8 = n / 8; + let (r, c) = (r as i32, c as i32); + let lo_c = -(c * 64) - (4 + n) * 8; + let hi_c = (self.f.mi_cols as i32 - n8 - c) * 64 + (4 + n) * 8 - 8; + let lo_r = -(r * 64) - (4 + n) * 8; + let hi_r = (self.f.mi_rows as i32 - n8 - r) * 64 + (4 + n) * 8 - 8; + let lim = 8 * 1000; + (lo_r.max(-lim), hi_r.min(lim), lo_c.max(-lim), hi_c.min(lim)) + } + + /// SAD of the luma prediction at `mv` against the source. + fn sad(&self, r: usize, c: usize, n: usize, mv: Mv) -> u32 { + let p = &self.planes[0]; + let (x16, y16) = (((c * 8) << 4) as i32 + 2 * mv.col, ((r * 8) << 4) as i32 + 2 * mv.row); + let mut s = 0u32; + if let Some((pl, o, st)) = self.f.phases.as_ref().and_then(|ph| ph.block(x16, y16, n)) { + for i in 0..n { + let a = &p.src[(r * 8 + i) * p.w + c * 8..][..n]; + let b = &pl[o + i * st..o + i * st + n]; + s += a.iter().zip(b).map(|(a, b)| (*a as i32 - *b as i32).unsigned_abs()).sum::(); + } + return s; + } + let refs = self.f.refs.as_ref().expect("reference"); + let mut pred = [0u8; 256]; + predict(&refs[0], x16, y16, n, n, &mut pred); + for i in 0..n { + let a = &p.src[(r * 8 + i) * p.w + c * 8..][..n]; + s += a.iter().zip(&pred[i * n..i * n + n]).map(|(a, b)| (*a as i32 - *b as i32).unsigned_abs()).sum::(); + } + s + } + + /// Motion search for the block at (`r`, `c`): the best vector and its cost + /// (SAD + λ·bits). + fn search(&self, r: usize, c: usize, bsize: u8) -> (Mv, f64) { + let n = if bsize == BLOCK_16X16 { 16 } else { 8 }; + let (nearest, near, mctx) = self.mv_refs(r, c, bsize); + let (lo_r, hi_r, lo_c, hi_c) = self.mv_range(r, c, n as i32); + let inside = |mv: Mv| mv.row >= lo_r && mv.row <= hi_r && mv.col >= lo_c && mv.col <= hi_c; + let lambda = self.f.lambda_sad / 256.0; + let d = &probs_defaults().0; + let mb: [u32; 4] = std::array::from_fn(|m| tree_cost(&INTER_MODE_TREE, &d[probs::INTER_MODE + mctx * 3..], m as u8)); + let mode_bits = |m: u8| mb[(m - NEARESTMV) as usize]; + let cost = |mv: Mv| -> f64 { + let bits = if mv == nearest { + mode_bits(NEARESTMV) + } else if mv == Mv::ZERO { + mode_bits(ZEROMV) + } else if mv == near { + mode_bits(NEARMV) + } else { + mode_bits(NEWMV) + mv_cost(mv.minus(nearest)) + }; + self.sad(r, c, n, mv) as f64 + lambda * bits as f64 + }; + let mut best = (Mv::ZERO, cost(Mv::ZERO)); + let prev = self.f.prev_mvs.map(|p| p[r * self.f.mi_cols + c]); + for cand in [Some(nearest), Some(near), prev].into_iter().flatten() { + if cand != Mv::ZERO && inside(cand) { + let j = cost(cand); + if j < best.1 { + best = (cand, j); + } + } + } + // A perfect match needs no search. + let good_enough = lambda * 256.0 * 0.5; + if best.1 > good_enough { + // Whole-sample search from the best candidate: a coarse cross, then a small diamond. + let round8 = |v: i32| (v + 4).div_euclid(8) * 8; + let mut cur = Mv::new(round8(best.0.row), round8(best.0.col)); + let mut cur_j = if inside(cur) { cost(cur) } else { f64::MAX }; + if cur_j < best.1 { + best = (cur, cur_j); + } else { + cur = best.0; + cur_j = best.1; + } + for (step, max_iters) in [(16, 1), (8, 2), (4, 3), (2, 4), (1, 8)] { + let s = step * 8; + for _ in 0..max_iters { + let mut moved = false; + let center = cur; + for (dr, dc) in [(-s, 0), (s, 0), (0, -s), (0, s)] { + let cand = Mv::new(center.row + dr, center.col + dc); + if !inside(cand) { + continue; + } + let j = cost(cand); + if j < cur_j { + cur = cand; + cur_j = j; + moved = true; + } + } + if !moved { + break; + } + } + } + if cur_j < best.1 { + best = (cur, cur_j); + } + } + // Sub-sample refinement: half then quarter samples (1/8 units, even: no high precision). + for s in [4, 2] { + let center = best.0; + for (dr, dc) in [(-s, 0), (s, 0), (0, -s), (0, s), (-s, -s), (-s, s), (s, -s), (s, s)] { + let cand = Mv::new(center.row + dr, center.col + dc); + if !inside(cand) { + continue; + } + let j = cost(cand); + if j < best.1 { + best = (cand, j); + } + } + } + best + } + + /// Choose between one 16×16 block and four 8×8 blocks at (`r`, `c`): `Some(mv)` for 16×16. + fn decide16(&mut self, r: usize, c: usize) -> Option { + let (mv16, j16) = self.search(r, c, BLOCK_16X16); + let lambda = self.f.lambda_sad; + // A near-perfect 16×16 prediction needs no split. + if j16 < 2.0 * 256.0 / 256.0 * 16.0 { + return Some(mv16); + } + let mut j8 = lambda * 4.0; // partition split + three more blocks of mode info (≈ 4 bits) + for (k, (dr, dc)) in [(0, 0), (0, 1), (1, 0), (1, 1)].into_iter().enumerate() { + let (rr, cc) = (r + dr, c + dc); + let (mv, j) = self.search(rr, cc, BLOCK_8X8); + j8 += j; + self.pending[k] = (rr, cc, mv); + // Tentative mode info so the next 8×8 blocks see this one as a candidate. + self.set_mi(rr, cc, 1, Mi { inter: true, mode: NEWMV, mv, skip: false, size: BLOCK_8X8 }); + } + if j16 <= j8 { + self.pending = [(usize::MAX, 0, Mv::ZERO); 4]; + Some(mv16) + } else { + None + } + } + + /// Code an inter block with motion vector `mv`. + fn inter_block(&mut self, r: usize, c: usize, bsize: u8, mv: Mv) { + let (mi_rows, mi_cols) = (self.f.mi_rows, self.f.mi_cols); + let n8 = if bsize == BLOCK_16X16 { 2 } else { 1 }; + let n = 8 * n8; + let (nearest, near, mctx) = self.mv_refs(r, c, bsize); + let d = &probs_defaults().0; + let mode_bits = |m: u8| tree_cost(&INTER_MODE_TREE, &d[probs::INTER_MODE + mctx * 3..], m - NEARESTMV); + let mut mode = (NEWMV, mode_bits(NEWMV) + mv_cost(mv.minus(nearest))); + for (m, ok) in [(NEARESTMV, mv == nearest), (NEARMV, mv == near), (ZEROMV, mv == Mv::ZERO)] { + if ok && mode_bits(m) < mode.1 { + mode = (m, mode_bits(m)); + } + } + let mode = mode.0; + // Prediction of every plane (written straight into the reconstruction). + let refs = self.f.refs.as_ref().expect("reference"); + for plane in 0..3 { + let ch = plane > 0; + let (row16, col16) = clamp_for_plane(mv, r, c, n8, mi_rows, mi_cols, ch); + let bn = if ch { n / 2 } else { n }; + let (bx, by) = if ch { (c * 4, r * 4) } else { (c * 8, r * 8) }; + let mut pred = [0u8; 256]; + predict(&refs[plane], ((bx as i32) << 4) + col16, ((by as i32) << 4) + row16, bn, bn, &mut pred); + let p = &mut self.planes[plane]; + for i in 0..bn { + p.rec[(by + i) * p.w + bx..][..bn].copy_from_slice(&pred[i * bn..i * bn + bn]); + } + } + // Residual: quantise every 4×4, dropping blocks whose coefficients don't pay for themselves. + let q = Quant { round_dc: 0.5, round_ac: 0.42, ..self.f.quant }; + // Weight of the rate term when dropping a transform block's coefficients (low: keep quality on par + // with key frames at the same q index). + let zl = 0.07; + let tx = DCT_DCT; + let mut blocks: Vec<(usize, usize, usize, [i32; 16], usize)> = Vec::with_capacity(24); + let mut any = false; + for plane in 0..3 { + let ch = plane > 0; + let bn = if ch { n / 2 } else { n }; + let (bx, by) = if ch { (c * 4, r * 4) } else { (c * 8, r * 8) }; + let p = &self.planes[plane]; + for ty in (0..bn).step_by(4) { + for tx4 in (0..bn).step_by(4) { + let (x, y) = (bx + tx4, by + ty); + let mut res = [0i32; 16]; + for i in 0..4 { + for j in 0..4 { + let k = (y + i) * p.w + x + j; + res[i * 4 + j] = p.src[k] as i32 - p.rec[k] as i32; + } + } + let mut levels = q.levels(&res, tx); + let mut eob = eob_of(&levels, tx); + if eob > 0 && !q.lossless { + let back = q.residual(&levels, tx, eob); + let d_skip: i64 = res.iter().map(|v| (*v as i64) * (*v as i64)).sum(); + let d_code: i64 = res.iter().zip(&back).map(|(a, b)| ((a - b) as i64).pow(2)).sum(); + if d_skip as f64 <= d_code as f64 + zl * self.f.lambda_sse * levels_cost(&levels, eob) as f64 / 256.0 { + levels = [0; 16]; + eob = 0; + } + } + any |= eob > 0; + blocks.push((plane, x, y, levels, eob)); + } + } + } + let skip = !any; + // Mode info. + let avail_u = r > 0; + let avail_l = c > self.col_start; + let above = avail_u.then(|| self.mi_at(r - 1, c)); + let left = avail_l.then(|| self.mi_at(r, c - 1)); + let skip_ctx = above.is_some_and(|m| m.skip) as usize + left.is_some_and(|m| m.skip) as usize; + self.w.ctx(skip, probs::SKIP + skip_ctx); + // is_inter context: neighbours are inter (no intra blocks in inter frames). + let intra = |m: Option| m.is_some_and(|m| !m.inter); + let inter_ctx = match (above, left) { + (Some(_), Some(_)) => { + if intra(above) && intra(left) { + 3 + } else { + (intra(above) || intra(left)) as usize + } + } + (Some(_), None) => 2 * intra(above) as usize, + (None, Some(_)) => 2 * intra(left) as usize, + (None, None) => 0, + }; + self.w.ctx(true, probs::IS_INTER + inter_ctx); + // single_ref_p1 = 0 (LAST_FRAME); context for single-reference LAST neighbours. + let ref_ctx = match (above, left) { + (Some(a), Some(l)) => match (a.inter, l.inter) { + (false, false) => 2, + (false, true) => 4, + (true, false) => 4, + (true, true) => 4, + }, + (Some(m), None) | (None, Some(m)) => { + if m.inter { + 4 + } else { + 2 + } + } + (None, None) => 2, + }; + self.w.ctx(false, probs::SINGLE_REF + ref_ctx * 2); + self.w.tree(&INTER_MODE_TREE, mode - NEARESTMV, |i| P::Ctx((probs::INTER_MODE + mctx * 3 + i) as u16)); + if mode == NEWMV { + self.write_mv(mv.minus(nearest)); + } + self.set_mi(r, c, n8, Mi { inter: true, mode, mv, skip, size: bsize }); + // Tokens and reconstruction. + for (plane, x, y, levels, eob) in blocks { + let ss = (plane > 0) as usize; + let (x4, ly) = (x >> 2, (y >> 2) & (15 >> ss)); + if !skip { + let ctx0 = (self.above_nz[plane][x4] + self.left_nz[plane][ly]) as usize; + self.tokens(plane, true, tx, &levels, eob, ctx0); + if eob > 0 { + let resid = q.residual(&levels, tx, eob); + let p = &mut self.planes[plane]; + for i in 0..4 { + for j in 0..4 { + let k = (y + i) * p.w + x + j; + p.rec[k] = (p.rec[k] as i32 + resid[i * 4 + j]).clamp(0, 255) as u8; + } + } + } + } + let nz = (!skip && eob > 0) as u8; + self.above_nz[plane][x4] = nz; + self.left_nz[plane][ly] = nz; + } + } + + fn write_mv(&mut self, diff: Mv) { + let joint = (diff.row != 0) as u8 * 2 + (diff.col != 0) as u8; + self.w.tree(&MV_JOINT_TREE, joint, |i| P::Ctx((probs::MV_JOINT + i) as u16)); + if diff.row != 0 { + self.write_mv_component(0, diff.row); + } + if diff.col != 0 { + self.write_mv_component(1, diff.col); + } + } + + fn write_mv_component(&mut self, i: usize, v: i32) { + debug_assert!(v != 0 && v & 1 == 0, "vectors are even without high precision"); + let offset = v.unsigned_abs() - 1; + let c = mv_class(offset); + self.w.ctx(v < 0, probs::MV_SIGN + i); + self.w.tree(&MV_CLASS_TREE, c as u8, |k| P::Ctx((probs::MV_CLASS + i * 10 + k) as u16)); + let fr = ((offset >> 1) & 3) as u8; + if c == 0 { + let b = (offset >> 3) as usize; + self.w.ctx(b == 1, probs::MV_CLASS0_BIT + i); + self.w.tree(&MV_FR_TREE, fr, |k| P::Ctx((probs::MV_CLASS0_FR + i * 6 + b * 3 + k) as u16)); + } else { + let dd = (offset - (1 << (c + 3))) >> 3; + for b in 0..c { + self.w.ctx((dd >> b) & 1 == 1, probs::MV_BITS + i * 10 + b as usize); + } + self.w.tree(&MV_FR_TREE, fr, |k| P::Ctx((probs::MV_FR + i * 3 + k) as u16)); + } + } +} diff --git a/crates/vp9enc/src/transform.rs b/crates/vp9enc/src/transform.rs new file mode 100644 index 00000000..e6a685d5 --- /dev/null +++ b/crates/vp9enc/src/transform.rs @@ -0,0 +1,249 @@ +//! 4×4 transforms: the decoder's integer inverse DCT / ADST (spec 8.7.1) and Walsh–Hadamard +//! transform (8.7.1.10), and their forward counterparts (the exact inverse matrices of the +//! decoder's linear transforms, so `inverse4(forward(res)) ≈ res`; the WHT inverse is exact). + +pub const DCT_DCT: u8 = 0; +pub const ADST_DCT: u8 = 1; +pub const DCT_ADST: u8 = 2; +pub const ADST_ADST: u8 = 3; + +fn r14(v: i32) -> i32 { + (v + (1 << 13)) >> 14 +} + +fn idct4(x: [i32; 4]) -> [i32; 4] { + let v0 = r14(x[0] * 11585 - x[2] * 11585); + let v1 = r14(x[0] * 11585 + x[2] * 11585); + let v2 = r14(x[1] * 6270 - x[3] * 15137); + let v3 = r14(x[1] * 15137 + x[3] * 6270); + [v1 + v3, v0 + v2, v0 - v2, v1 - v3] +} + +fn iadst4(x: [i32; 4]) -> [i32; 4] { + let (s19, s29, s39, s49) = (5283i64, 9929i64, 13377i64, 15212i64); + let x: [i64; 4] = x.map(|v| v as i64); + let s0 = s19 * x[0]; + let s1 = s29 * x[0]; + let s2 = s39 * x[1]; + let s3 = s49 * x[2]; + let s4 = s19 * x[2]; + let s5 = s29 * x[3]; + let s6 = s49 * x[3]; + let s7 = s39 * (x[0] - x[2] + x[3]); + let x0 = s0 + s3 + s5; + let x1 = s1 - s4 - s6; + let x2 = s7; + let x3 = s2; + let r = |v: i64| ((v + (1 << 13)) >> 14) as i32; + [r(x0 + x3), r(x1 + x3), r(x2), r(x0 + x1 - x3)] +} + +/// The decoder's 4×4 inverse transform of dequantised coefficients (row-major) to a residual. +pub fn inverse4(coefs: &[i32; 16], tx_type: u8) -> [i32; 16] { + let row_adst = matches!(tx_type, DCT_ADST | ADST_ADST); + let col_adst = matches!(tx_type, ADST_DCT | ADST_ADST); + let mut t = [[0i32; 4]; 4]; + for i in 0..4 { + let r = [coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]]; + t[i] = if row_adst { iadst4(r) } else { idct4(r) }; + } + let mut out = [0i32; 16]; + for j in 0..4 { + let c = [t[0][j], t[1][j], t[2][j], t[3][j]]; + let c = if col_adst { iadst4(c) } else { idct4(c) }; + for i in 0..4 { + out[i * 4 + j] = (c[i] + 8) >> 4; + } + } + out +} + +/// Inverse Walsh–Hadamard (8.7.1.10) on one line. +fn iwht(x: [i32; 4], shift: u32) -> [i32; 4] { + let mut a = x[0] >> shift; + let mut c = x[1] >> shift; + let mut d = x[2] >> shift; + let mut b = x[3] >> shift; + a += c; + d -= b; + let e = (a - d) >> 1; + b = e - b; + c = e - c; + a -= b; + d += c; + [a, b, c, d] +} + +/// The exact inverse of [`iwht`] (with shift 0): inputs that produce outputs `y`. +fn fwht(y: [i32; 4]) -> [i32; 4] { + let (aa, bb, cc, dd) = (y[0], y[1], y[2], y[3]); + let d1 = dd - cc; + let a1 = aa + bb; + let e = (a1 - d1) >> 1; + let b0 = e - bb; + let c0 = e - cc; + let a0 = a1 - c0; + let d0 = d1 + b0; + [a0, c0, d0, b0] +} + +pub fn inverse_wht(coefs: &[i32; 16]) -> [i32; 16] { + let mut t = [[0i32; 4]; 4]; + for i in 0..4 { + t[i] = iwht([coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]], 2); + } + let mut out = [0i32; 16]; + for j in 0..4 { + let r = iwht([t[0][j], t[1][j], t[2][j], t[3][j]], 0); + for i in 0..4 { + out[i * 4 + j] = r[i]; + } + } + out +} + +/// Lossless forward transform: levels (dequantised by 4) whose inverse is exactly `res`. +pub fn forward_wht(res: &[i32; 16]) -> [i32; 16] { + let mut t = [[0i32; 4]; 4]; + for j in 0..4 { + let col = fwht([res[j], res[4 + j], res[8 + j], res[12 + j]]); + for i in 0..4 { + t[i][j] = col[i]; + } + } + let mut lv = [0i32; 16]; + for i in 0..4 { + let r = fwht(t[i]); + lv[i * 4..i * 4 + 4].copy_from_slice(&r); + } + lv +} + +/// Forward transform matrices: the inverses of the decoder's (linear part of the) inverse +/// transforms. +pub fn forward_matrices() -> &'static [[[f64; 16]; 16]; 4] { + use std::sync::OnceLock; + static M: OnceLock<[[[f64; 16]; 16]; 4]> = OnceLock::new(); + M.get_or_init(|| { + let mut out = [[[0.0; 16]; 16]; 4]; + for (ty, o) in out.iter_mut().enumerate() { + // Columns of the inverse: response to a scaled unit coefficient. + let k = 1 << 16; + let mut inv = [[0.0f64; 16]; 16]; + for c in 0..16 { + let mut e = [0i32; 16]; + e[c] = k; + let r = inverse4(&e, ty as u8); + for p in 0..16 { + inv[p][c] = r[p] as f64 / k as f64; + } + } + *o = invert16(inv); + } + out + }) +} + +fn invert16(m: [[f64; 16]; 16]) -> [[f64; 16]; 16] { + let mut a = m; + let mut inv = [[0.0; 16]; 16]; + for (i, r) in inv.iter_mut().enumerate() { + r[i] = 1.0; + } + for col in 0..16 { + let piv = (col..16).max_by(|&x, &y| a[x][col].abs().total_cmp(&a[y][col].abs())).unwrap_or(col); + a.swap(col, piv); + inv.swap(col, piv); + let d = a[col][col]; + for j in 0..16 { + a[col][j] /= d; + inv[col][j] /= d; + } + for r in 0..16 { + if r != col { + let f = a[r][col]; + if f != 0.0 { + for j in 0..16 { + a[r][j] -= f * a[col][j]; + inv[r][j] -= f * inv[col][j]; + } + } + } + } + } + inv +} + +/// Quantisation settings of one block. +#[derive(Clone, Copy)] +pub struct Quant { + pub dcq: i32, + pub acq: i32, + pub lossless: bool, + /// Rounding offsets (fractions of a step) for DC and AC; smaller = wider dead zone. + pub round_dc: f64, + pub round_ac: f64, +} + +impl Quant { + /// Quantised levels (raster order) of a residual. + pub fn levels(&self, res: &[i32; 16], tx_type: u8) -> [i32; 16] { + if self.lossless { + return forward_wht(res); + } + let f = &forward_matrices()[tx_type as usize]; + let mut lv = [0i32; 16]; + for k in 0..16 { + let coef: f64 = (0..16).map(|q| f[k][q] * res[q] as f64).sum(); + let (qs, rnd) = if k == 0 { (self.dcq, self.round_dc) } else { (self.acq, self.round_ac) }; + let l = (coef.abs() / qs as f64 + rnd).floor() as i32; + lv[k] = if coef < 0.0 { -l } else { l }; + } + lv + } + + /// The decoder's residual for `levels` (all zero when `eob` is 0). + pub fn residual(&self, levels: &[i32; 16], tx_type: u8, eob: usize) -> [i32; 16] { + if eob == 0 { + return [0; 16]; + } + let mut deq = [0i32; 16]; + for k in 0..16 { + deq[k] = levels[k] * if k == 0 { self.dcq } else { self.acq }; + } + if self.lossless { inverse_wht(&deq) } else { inverse4(&deq, tx_type) } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wht_is_exactly_invertible() { + let mut s = 7u32; + for _ in 0..2000 { + let mut r = [0i32; 16]; + for v in &mut r { + s = s.wrapping_mul(1664525).wrapping_add(1013904223); + *v = (s >> 16) as i32 % 511 - 255; + } + let lv = forward_wht(&r); + let deq = lv.map(|l| l * 4); + assert_eq!(inverse_wht(&deq), r); + } + } + + #[test] + fn forward_transforms_invert_the_decoder() { + for ty in 0..4u8 { + let f = &forward_matrices()[ty as usize]; + let r: [i32; 16] = std::array::from_fn(|i| (i as i32 * 37 % 61) - 30); + let c: [i32; 16] = std::array::from_fn(|k| (0..16).map(|q| f[k][q] * r[q] as f64).sum::().round() as i32); + let back = inverse4(&c, ty); + for i in 0..16 { + assert!((back[i] - r[i]).abs() <= 1, "type {ty}: {back:?} vs {r:?}"); + } + } + } +} diff --git a/docs/parity.md b/docs/parity.md index 38df7882..8fc67b9a 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -110,7 +110,7 @@ the disk cache in the formats wave; the 3D Camera Tracker in M12.6.) | Area | Weighted parity | Remaining (agent-hours) | Biggest gaps | |---|---|---|---| | Layers | 88% | 4.8 | frame blending, collapse transformations, slip edit | -| Output | 90% | 3.0 | multiple output modules, pre-render (WebM with VP9 alpha + Opus and WAV/AIFF audio-only landed: VP9 is intra-only, Opus CELT-only) | +| Output | 90% | 3.0 | multiple output modules, pre-render (WebM with VP9 alpha + Opus and WAV/AIFF audio-only landed: VP9 key + inter frames with motion search, loop filter and rate control; Opus CELT-only) | | Audio | 85% | 0.5 | audio to keyframes | | Import | 88% | 2.0 | AI/EPS/PDF vector footage, PSD smart objects and 3D layers (PSD as footage/composition/retain layer sizes, SVG footage and Create Shapes from Vector Layer landed) | | Automation | ≈ 90% | 0.8 | scripting covers the documented core object model (AUT-2, M14.4: `app`, project items, comps, layers, properties and keyframes, text documents, markers, render queue, Script Console, `effectcraft-cli script`, MCP `run_script`); still missing: ScriptUI panels/dialogs, `.jsxbin`, sockets | From 89c9b1b6eefb7a652a6ef1b2561d236b2520227b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 04:07:20 -0400 Subject: [PATCH 2/9] =?UTF-8?q?M13.1:=20branching=20history,=20puppet=20pi?= =?UTF-8?q?n=20recording,=20WebAssembly=20plug-in=20API,=20ScriptUI=20mode?= =?UTF-8?q?l=20and=20hosts,=20File=20=E2=96=B8=20Scripts,=20deterministic?= =?UTF-8?q?=20render=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - engine: History keeps undone lines as branches; edit.history.list/goto; History panel tree; MCP history tool - puppet.recordPin / puppet.recordOptions; viewer Cmd-drag records; Record Options form - effects::plugin (EffectPlugin trait, register_plugin, API v1); effectcraft-plugin (wasmi, sandboxed, fuel-limited); effect.plugins.load/list - ScriptUI: engine scriptui model + layout + scriptui.* commands; script crate scriptui.js and script-host threads (dialogs block show()) - File ▸ Scripts: install script / ScriptUI panel, sample scripts, window.scriptPanel - host test: bit-identical renders across runs and thread counts Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 151 ++++- Cargo.toml | 3 + apps/effectcraft-web/src/store.rs | 12 + crates/automation/src/tools.rs | 19 + crates/effects/Cargo.toml | 1 + crates/effects/src/lib.rs | 28 +- crates/effects/src/plugin.rs | 338 +++++++++++ .../Create Null at Selected Layers.jsx | 41 ++ crates/engine/scripts/Rename Layers.jsx | 71 +++ crates/engine/scripts/Render Queue Batch.jsx | 52 ++ .../scripts/ScriptUI Panels/Layer Tools.jsx | 99 +++ .../scripts/Sort Layers by In Point.jsx | 37 ++ crates/engine/src/commands/edit.rs | 54 +- crates/engine/src/commands/effect.rs | 81 ++- crates/engine/src/commands/file_more.rs | 35 +- crates/engine/src/commands/mod.rs | 2 + crates/engine/src/commands/puppet.rs | 157 ++++- crates/engine/src/commands/scripts.rs | 222 +++++++ crates/engine/src/config.rs | 22 +- crates/engine/src/history.rs | 346 +++++++++++ crates/engine/src/lib.rs | 35 +- crates/engine/src/mask_track.rs | 5 +- crates/engine/src/menus.rs | 4 +- crates/engine/src/scriptui.rs | 572 ++++++++++++++++++ crates/engine/src/session_settings.rs | 6 +- crates/engine/src/tests_history.rs | 135 +++++ crates/engine/src/tests_paint.rs | 53 ++ crates/engine/src/tracking.rs | 5 +- crates/host/Cargo.toml | 6 + crates/host/src/lib.rs | 2 + crates/host/tests/determinism.rs | 74 +++ crates/plugin/Cargo.toml | 25 + crates/plugin/src/lib.rs | 66 ++ crates/plugin/src/tests.rs | 164 +++++ crates/plugin/src/wasm.rs | 151 +++++ crates/script/src/lib.rs | 15 +- crates/script/src/model.rs | 4 +- crates/script/src/runtime.rs | 170 ++++-- crates/script/src/scriptui.js | 544 +++++++++++++++++ crates/script/src/tests_ui.rs | 299 +++++++++ crates/script/src/ui.rs | 339 +++++++++++ crates/ui-egui/src/header.rs | 16 +- crates/ui-egui/src/panels/effects_presets.rs | 4 +- crates/ui-egui/src/panels/forms.rs | 13 + crates/ui-egui/src/panels/misc.rs | 49 +- crates/ui-egui/src/panels/viewer.rs | 39 +- docs/architecture.md | 7 +- xtask/src/main.rs | 1 + 48 files changed, 4440 insertions(+), 134 deletions(-) create mode 100644 crates/effects/src/plugin.rs create mode 100644 crates/engine/scripts/Create Null at Selected Layers.jsx create mode 100644 crates/engine/scripts/Rename Layers.jsx create mode 100644 crates/engine/scripts/Render Queue Batch.jsx create mode 100644 crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx create mode 100644 crates/engine/scripts/Sort Layers by In Point.jsx create mode 100644 crates/engine/src/commands/scripts.rs create mode 100644 crates/engine/src/history.rs create mode 100644 crates/engine/src/scriptui.rs create mode 100644 crates/engine/src/tests_history.rs create mode 100644 crates/host/tests/determinism.rs create mode 100644 crates/plugin/Cargo.toml create mode 100644 crates/plugin/src/lib.rs create mode 100644 crates/plugin/src/tests.rs create mode 100644 crates/plugin/src/wasm.rs create mode 100644 crates/script/src/scriptui.js create mode 100644 crates/script/src/tests_ui.rs create mode 100644 crates/script/src/ui.rs diff --git a/Cargo.lock b/Cargo.lock index c7934100..3aba6695 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1197,6 +1197,7 @@ dependencies = [ "effectcraft-project", "effectcraft-raster", "effectcraft-track", + "log", "rayon", "serde", "serde_json", @@ -1309,12 +1310,14 @@ dependencies = [ "effectcraft-export", "effectcraft-expr", "effectcraft-media", + "effectcraft-plugin", "effectcraft-project", "effectcraft-psd", "effectcraft-raster", "effectcraft-script", "effectcraft-svg", "effectcraft-time", + "rayon", "serde_json", ] @@ -1398,6 +1401,18 @@ dependencies = [ "tiny-skia", ] +[[package]] +name = "effectcraft-plugin" +version = "0.1.0" +dependencies = [ + "effectcraft-effects", + "effectcraft-keyframe", + "effectcraft-project", + "effectcraft-raster", + "serde_json", + "wasmi", +] + [[package]] name = "effectcraft-project" version = "0.1.0" @@ -2102,6 +2117,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foldhash" version = "0.2.0" @@ -2276,7 +2297,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed4a1bb24121291d27230c1b1b44e07d6a9b28cefdb32fe1581dfb84e14f940a" dependencies = [ "bytemuck", - "foldhash", + "foldhash 0.2.0", "hashbrown 0.17.1", "log", "peniko", @@ -2416,6 +2437,15 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + [[package]] name = "hashbrown" version = "0.16.1" @@ -2424,7 +2454,7 @@ checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.2.0", ] [[package]] @@ -2435,7 +2465,7 @@ checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.2.0", ] [[package]] @@ -2753,6 +2783,12 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "lebe" version = "0.5.3" @@ -4432,6 +4468,12 @@ dependencies = [ "serde", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + [[package]] name = "spirv" version = "0.4.0+sdk-1.4.341.0" @@ -4459,6 +4501,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6637bab7722d379c8b41ba849228d680cc12d0a45ba1fa2b48f2a30577a06731" +[[package]] +name = "string-interner" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23de088478b31c349c9ba67816fa55d9355232d63c3afea8bf513e31f0f1d2c0" +dependencies = [ + "hashbrown 0.15.5", + "serde", +] + [[package]] name = "strum" version = "0.28.0" @@ -5127,6 +5179,99 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.261.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2608e8bb6d67fd68f5a8d0eb1363d6e7bcbc1f8ded5a0bd3a1e382462b876b22" +dependencies = [ + "leb128fmt", + "wasmparser 0.261.0", +] + +[[package]] +name = "wasmi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78693fcdd618e0fc34af59c6b8efa9ac5d58c68df940beff4bedddb6acfe7c27" +dependencies = [ + "spin", + "wasmi_collections", + "wasmi_core", + "wasmi_ir", + "wasmparser 0.228.0", + "wat", +] + +[[package]] +name = "wasmi_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a8be2aa467cf2d29e96ff759472c36eeb44a3c81c67fc9cb76c9a24c519c557" +dependencies = [ + "string-interner", +] + +[[package]] +name = "wasmi_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69372d5fda3ea3d1e0aa6603c7888110e0187e88ea17cd8fc2e2df0a0e1f37fa" +dependencies = [ + "libm", +] + +[[package]] +name = "wasmi_ir" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f17b774caa13c618c7244f1ee51fe23c5e7b8538a471fa46d9949779758aed6" +dependencies = [ + "wasmi_core", +] + +[[package]] +name = "wasmparser" +version = "0.228.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4abf1132c1fdf747d56bbc1bb52152400c70f336870f968b85e89ea422198ae3" +dependencies = [ + "bitflags 2.13.2", + "indexmap", +] + +[[package]] +name = "wasmparser" +version = "0.261.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f20f20e44f7e8aeb6744823ea9d869ede51e51be4fdaedede2852282e54d2d8" +dependencies = [ + "bitflags 2.13.2", + "indexmap", +] + +[[package]] +name = "wast" +version = "261.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "776443145731a4062e5b0d392892a2005909b6ab72d9fdc3cad53dd1a714e44a" +dependencies = [ + "bumpalo", + "leb128fmt", + "memchr", + "unicode-width", + "wasm-encoder", +] + +[[package]] +name = "wat" +version = "1.261.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7b4d1a49ea73a8f3326e74e3a05db667001b16bd1035ed3356fc1a0ed05ca7f" +dependencies = [ + "wast", +] + [[package]] name = "wayland-backend" version = "0.3.17" diff --git a/Cargo.toml b/Cargo.toml index 22709f60..51a2c8ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ effectcraft-ui-egui = { path = "crates/ui-egui" } effectcraft-automation = { path = "crates/automation" } effectcraft-opusenc = { path = "crates/opusenc" } effectcraft-script = { path = "crates/script" } +effectcraft-plugin = { path = "crates/plugin" } # foundation thiserror = "2" @@ -52,6 +53,8 @@ proptest = "1" # scripting (expressions) boa_engine = { version = "0.22", default-features = false } +# WebAssembly effect plug-ins (sandboxed interpreter; MIT/Apache-2.0, pure Rust) +wasmi = { version = "2", default-features = false, features = ["std", "validate", "wat", "deterministic", "portable-dispatch"] } # geometry / fonts kurbo = { version = "0.13", features = ["serde"] } diff --git a/apps/effectcraft-web/src/store.rs b/apps/effectcraft-web/src/store.rs index 0e4ea029..471d96b5 100644 --- a/apps/effectcraft-web/src/store.rs +++ b/apps/effectcraft-web/src/store.rs @@ -216,6 +216,18 @@ impl ConfigStore for WebConfig { self.store.changed(); Ok(()) } + fn list(&self, dir: &str) -> Vec { + let pre = format!("{CONFIG}{}/", dir.trim_end_matches('/')); + let mut v: Vec = self + .store + .lock() + .list(&pre) + .into_iter() + .filter_map(|(k, ..)| k.strip_prefix(&pre).filter(|n| !n.is_empty() && !n.contains('/')).map(str::to_string)) + .collect(); + v.sort(); + v + } fn dir(&self) -> Option { Some(PathBuf::from("/")) } diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 739c5524..f9034a38 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -315,6 +315,18 @@ fn redo(b: &mut Backend, a: &Value) -> Result { history(b, a, "edit.redo") } +fn history_tool(b: &mut Backend, a: &Value) -> Result { + if let Some(g) = get(a, "goto") { + let p = match g { + Value::String(id) if id.parse::().is_err() => json!({"id": id}), + Value::String(n) => json!({"index": n.parse::().unwrap_or(0)}), + v => json!({"index": v}), + }; + b.exec("edit.history.goto", p)?; + } + json_reply(b.exec("edit.history.list", json!({}))?) +} + // ------------------------------------------------------------------ bridge-only tools fn screenshot(b: &mut Backend, a: &Value) -> Result { @@ -535,6 +547,13 @@ static TOOLS: &[ToolDef] = &[ schema: || schema(json!({"steps": {"type": "integer", "minimum": 1}}), &[]), run: redo, }, + ToolDef { + name: "history", + description: "The branching undo history (History panel): without arguments lists every state {index, id, label, parent, depth (0 = working line, >0 = branch), current, future}; undoing and then editing keeps the undone states as a branch. With `goto` (an index or id from the listing) jumps to that state, on any branch, without losing the others.", + bridge_only: false, + schema: || schema(json!({"goto": {"type": ["integer", "string"], "description": "State index or id to jump to (edit.history.goto)."}}), &[]), + run: history_tool, + }, ToolDef { name: "screenshot", description: "Screenshot of the live app window, or one panel (`Composition`, `Timeline`, `EffectControls`, `Project`, ...) or element id, as PNG (longest side `max_side`, default 1600). Use it to check what the UI looks like after an action.", diff --git a/crates/effects/Cargo.toml b/crates/effects/Cargo.toml index 965cdbd2..9b065360 100644 --- a/crates/effects/Cargo.toml +++ b/crates/effects/Cargo.toml @@ -19,4 +19,5 @@ effectcraft-track = { workspace = true } rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +log = { workspace = true } diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index c4a47508..663b4bf7 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -41,6 +41,7 @@ pub mod ocio_config; pub mod paint; mod perspective; mod perspective2; +pub mod plugin; pub mod psim; pub mod puppet; pub mod roto; @@ -423,14 +424,37 @@ pub fn registry() -> &'static [EffectSpec] { }) } +/// A built-in effect or a registered plug-in by id. pub fn find(id: &str) -> Option<&'static EffectSpec> { - registry().iter().find(|s| s.id == id) + registry().iter().find(|s| s.id == id).or_else(|| plugin::plugins().into_iter().find(|s| s.id == id)) +} + +/// Every effect: the built-ins ([`registry`]) and the registered plug-ins +/// ([`plugin::register_plugin`]), sorted by category then name. +pub fn all() -> Vec<&'static EffectSpec> { + let plugins = plugin::plugins(); + let mut v: Vec<&'static EffectSpec> = registry().iter().collect(); + if !plugins.is_empty() { + v.extend(plugins); + v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); + } + v +} + +/// Effects & Presets categories: [`CATEGORIES`] then the plug-ins' own, alphabetically. +pub fn categories() -> Vec<&'static str> { + let mut v: Vec<&'static str> = CATEGORIES.to_vec(); + let mut extra: Vec<&'static str> = plugin::plugins().iter().map(|s| s.category).filter(|c| !CATEGORIES.contains(c)).collect(); + extra.sort_unstable(); + extra.dedup(); + v.extend(extra); + v } /// Find by id or (case-insensitive) display name. pub fn lookup(name_or_id: &str) -> Option<&'static EffectSpec> { find(name_or_id) - .or_else(|| registry().iter().find(|s| s.name.eq_ignore_ascii_case(name_or_id))) + .or_else(|| all().into_iter().find(|s| s.name.eq_ignore_ascii_case(name_or_id))) .or_else(|| { // After Effects' third-party display names we register under a generic name. keylight::KEYLIGHT_ALIASES.iter().any(|a| a.eq_ignore_ascii_case(name_or_id)).then(|| find("ec.keying.keylight")).flatten() diff --git a/crates/effects/src/plugin.rs b/crates/effects/src/plugin.rs new file mode 100644 index 00000000..242f52b9 --- /dev/null +++ b/crates/effects/src/plugin.rs @@ -0,0 +1,338 @@ +//! The effect plug-in API, version [`PLUGIN_API_VERSION`]. +//! +//! A plug-in is anything implementing [`EffectPlugin`]: a [`PluginManifest`] (stable id, +//! display name, Effects & Presets category, typed parameters) and a `render` that processes a +//! [`PluginFrame`] in place. [`register_plugin`] turns it into an ordinary [`EffectSpec`]: +//! instances are property groups like every built-in effect, so plug-in parameters animate, +//! take expressions, show in Effect Controls and are addressable by agents, and the effect is +//! applied by id (`effect.apply {"effect": "org.example.posterize"}`) like a built-in. +//! +//! Plug-ins come from Rust code (implement the trait and register it at start-up) or from +//! sandboxed WebAssembly modules (`effectcraft-plugin`, see `docs/plugins.md`), which implement +//! the same contract through a small C-like ABI. +//! +//! The contract (stable within an API version): +//! * pixels are premultiplied RGBA `f32` (0–1, may exceed 1 in 32 bpc projects), row-major, +//! in the layer's buffer, which may be scaled (preview resolution) and padded; +//! * parameters are flattened to `f64`s in declaration order: slider / angle / checkbox (0 or 1) +//! / popup (0-based index) take one value, point two (**frame pixels**, already mapped to the +//! buffer), colour four (RGBA 0–1); +//! * `time` is layer time in seconds; `scale` is the buffer's pixels per layer pixel (scale +//! pixel distances like blur radii by it); +//! * rendering must be deterministic (same inputs, same output) and may run on several threads +//! at once. + +use std::sync::{Arc, OnceLock, RwLock}; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use serde::{Deserialize, Serialize}; + +use crate::{Buf, EffectCtx, EffectSpec, ParamSpec, RenderFn}; + +/// The plug-in API version this build implements. +pub const PLUGIN_API_VERSION: u32 = 1; + +/// The most plug-in effects one process can register. +pub const MAX_PLUGINS: usize = 64; + +/// A plug-in parameter's kind, default and range. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +pub enum PluginParamKind { + Slider { + default: f64, + #[serde(default)] + min: f64, + #[serde(default = "hundred")] + max: f64, + /// Slider range (defaults to min..max). + #[serde(default, rename = "sliderMin")] + slider_min: Option, + #[serde(default, rename = "sliderMax")] + slider_max: Option, + #[serde(default = "two")] + decimals: u8, + }, + Angle { + #[serde(default)] + default: f64, + }, + Checkbox { + #[serde(default)] + default: bool, + }, + Popup { + options: Vec, + /// 0-based index. + #[serde(default)] + default: u32, + }, + /// Default as fractions of the layer size (`[0.5, 0.5]` = centre). + Point { + #[serde(default = "centre")] + default: [f64; 2], + }, + /// RGBA, 0–1. + Color { + #[serde(default = "white")] + default: [f64; 4], + }, +} + +fn hundred() -> f64 { + 100.0 +} +fn two() -> u8 { + 2 +} +fn centre() -> [f64; 2] { + [0.5, 0.5] +} +fn white() -> [f64; 4] { + [1.0; 4] +} + +impl PluginParamKind { + /// How many `f64`s the parameter flattens to. + pub fn width(&self) -> usize { + match self { + PluginParamKind::Point { .. } => 2, + PluginParamKind::Color { .. } => 4, + _ => 1, + } + } +} + +/// One parameter of a plug-in effect. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct PluginParam { + /// Stable id (the property's match name; letters, digits, `_`). + pub id: String, + pub name: String, + #[serde(flatten)] + pub kind: PluginParamKind, +} + +/// What a plug-in declares about itself. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct PluginManifest { + /// The plug-in API version it was written for (must be [`PLUGIN_API_VERSION`]). + pub api: u32, + /// Stable, reverse-domain effect id (`org.example.posterize`); `ec.` is reserved. + pub id: String, + pub name: String, + /// Effects & Presets category: one of [`crate::CATEGORIES`] or a new one. + pub category: String, + #[serde(default)] + pub version: String, + #[serde(default)] + pub author: String, + #[serde(default)] + pub description: String, + #[serde(default)] + pub params: Vec, +} + +impl PluginManifest { + /// Check the manifest against this API version. + pub fn validate(&self) -> Result<(), String> { + if self.api != PLUGIN_API_VERSION { + return Err(format!("plug-in `{}` targets plug-in API {} but this build implements API {PLUGIN_API_VERSION}", self.id, self.api)); + } + let ok_id = |s: &str| !s.is_empty() && s.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')); + if !ok_id(&self.id) || self.id.starts_with("ec.") { + return Err(format!("bad plug-in id `{}` (reverse-domain, letters, digits, `.`, `_`, `-`; `ec.` is reserved)", self.id)); + } + if self.name.trim().is_empty() || self.category.trim().is_empty() { + return Err(format!("plug-in `{}` needs a name and a category", self.id)); + } + let mut seen = std::collections::HashSet::new(); + for p in &self.params { + if p.id.is_empty() || !p.id.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') || !seen.insert(p.id.as_str()) { + return Err(format!("plug-in `{}`: bad or duplicate parameter id `{}`", self.id, p.id)); + } + if let PluginParamKind::Popup { options, default } = &p.kind + && (options.is_empty() || *default as usize >= options.len()) + { + return Err(format!("plug-in `{}`: popup `{}` needs options and a default index in range", self.id, p.id)); + } + } + Ok(()) + } + + /// Total flattened parameter count. + pub fn flat_len(&self) -> usize { + self.params.iter().map(|p| p.kind.width()).sum() + } +} + +/// The pixels a plug-in renders, in place. +pub struct PluginFrame<'a> { + pub width: u32, + pub height: u32, + /// Premultiplied RGBA, row-major, `width * height` pixels. + pub pixels: &'a mut [[f32; 4]], + /// Buffer pixels per layer pixel (preview resolution). + pub scale: f64, +} + +/// Parameter values for one render, flattened (see the module docs) with typed accessors. +pub struct PluginParams<'a> { + pub manifest: &'a PluginManifest, + pub flat: &'a [f64], +} + +impl PluginParams<'_> { + fn slot(&self, id: &str) -> Option<&[f64]> { + let mut at = 0; + for p in &self.manifest.params { + let w = p.kind.width(); + if p.id == id { + return self.flat.get(at..at + w); + } + at += w; + } + None + } + /// Slider / angle / popup index / checkbox (0 or 1). + pub fn f(&self, id: &str) -> f64 { + self.slot(id).and_then(|s| s.first().copied()).unwrap_or(0.0) + } + pub fn b(&self, id: &str) -> bool { + self.f(id) != 0.0 + } + /// Point in frame pixels. + pub fn point(&self, id: &str) -> [f64; 2] { + self.slot(id).map(|s| [s[0], s[1]]).unwrap_or([0.0; 2]) + } + pub fn color(&self, id: &str) -> [f64; 4] { + self.slot(id).map(|s| [s[0], s[1], s[2], s[3]]).unwrap_or([1.0; 4]) + } +} + +/// An effect plug-in. +pub trait EffectPlugin: Send + Sync { + fn manifest(&self) -> &PluginManifest; + /// Process `frame` in place. `time` is layer time in seconds. An error leaves the frame as + /// it was (the error is logged once). + fn render(&self, frame: &mut PluginFrame, params: &PluginParams, time: f64) -> Result<(), String>; + /// Where the plug-in came from (a file path, `built-in`…), for listings. + fn source(&self) -> String { + "built-in".into() + } +} + +static SLOTS: [OnceLock>; MAX_PLUGINS] = [const { OnceLock::new() }; MAX_PLUGINS]; + +/// Registered plug-in effects, in registration order. +fn plugin_specs() -> &'static RwLock> { + static P: OnceLock>> = OnceLock::new(); + P.get_or_init(|| RwLock::new(vec![])) +} + +/// The registered plug-in effects. +pub fn plugins() -> Vec<&'static EffectSpec> { + plugin_specs().read().map(|v| v.clone()).unwrap_or_default() +} + +/// The plug-in behind a registered effect id. +pub fn plugin(id: &str) -> Option> { + SLOTS.iter().filter_map(|s| s.get()).find(|p| p.manifest().id == id).cloned() +} + +fn leak(s: &str) -> &'static str { + Box::leak(s.to_string().into_boxed_str()) +} + +fn param_spec(p: &PluginParam) -> ParamSpec { + let (default, ui) = match &p.kind { + PluginParamKind::Slider { default, min, max, slider_min, slider_max, decimals } => ( + Value::Scalar(*default), + ParamUi::Slider { min: *min, max: *max, slider_min: slider_min.unwrap_or(*min), slider_max: slider_max.unwrap_or(*max), decimals: *decimals }, + ), + PluginParamKind::Angle { default } => (Value::Scalar(*default), ParamUi::Angle), + PluginParamKind::Checkbox { default } => (Value::Bool(*default), ParamUi::Checkbox), + PluginParamKind::Popup { options, default } => (Value::Enum(*default), ParamUi::Popup { options: options.clone() }), + PluginParamKind::Point { default } => (Value::Vec2(*default), ParamUi::Point), + PluginParamKind::Color { default } => (Value::Color(*default), ParamUi::Color), + }; + ParamSpec { id: leak(&p.id), name: leak(&p.name), default, ui } +} + +/// Register a plug-in effect: it appears in Effects & Presets under its category and is +/// applied by id like a built-in. Fails for a bad manifest, a taken id or when +/// [`MAX_PLUGINS`] are registered. +pub fn register_plugin(plugin: Arc) -> Result<&'static EffectSpec, String> { + let m = plugin.manifest().clone(); + m.validate()?; + let mut specs = plugin_specs().write().map_err(|_| "plug-in registry poisoned".to_string())?; + if crate::registry().iter().any(|s| s.id == m.id) || specs.iter().any(|s| s.id == m.id) { + return Err(format!("an effect with id `{}` is already registered", m.id)); + } + let slot = SLOTS.iter().position(|s| s.get().is_none()).ok_or_else(|| format!("too many plug-ins (at most {MAX_PLUGINS})"))?; + SLOTS[slot].set(plugin).map_err(|_| "plug-in slot taken".to_string())?; + let spec: &'static EffectSpec = Box::leak(Box::new(EffectSpec { + id: leak(&m.id), + name: leak(&m.name), + category: leak(&m.category), + params: m.params.iter().map(param_spec).collect(), + render: TRAMPOLINES[slot], + gpu: false, + float: true, + })); + specs.push(spec); + Ok(spec) +} + +/// Each slot needs its own `fn` (effect render functions are plain function pointers). +fn trampoline(ctx: &EffectCtx, buf: Buf) -> Buf { + run_slot(I, ctx, buf) +} + +macro_rules! trampolines { + ($($i:literal)*) => { [$(trampoline::<$i> as RenderFn),*] }; +} + +const TRAMPOLINES: [RenderFn; MAX_PLUGINS] = trampolines!( + 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 + 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 +); + +/// Flatten an instance's evaluated parameters for the plug-in (points mapped to frame pixels). +pub fn flatten(m: &PluginManifest, ctx: &EffectCtx, buf: &Buf) -> Vec { + let mut out = Vec::with_capacity(m.flat_len()); + for p in &m.params { + let v = ctx.params.get(&p.id); + match &p.kind { + PluginParamKind::Point { .. } => { + let q = v.map(Value::as_vec2).unwrap_or([0.0; 2]); + let (x, y) = buf.to_px(q); + out.extend([x, y]); + } + PluginParamKind::Color { .. } => { + let c = v.map(Value::as_color).unwrap_or([1.0; 4]); + out.extend(c.iter().map(|x| *x as f64)); + } + PluginParamKind::Checkbox { .. } => out.push(if v.map(Value::as_bool).unwrap_or(false) { 1.0 } else { 0.0 }), + PluginParamKind::Popup { .. } => out.push(v.map(Value::as_enum).unwrap_or(0) as f64), + _ => out.push(v.map(Value::as_f64).unwrap_or(0.0)), + } + } + out +} + +fn run_slot(slot: usize, ctx: &EffectCtx, mut buf: Buf) -> Buf { + let Some(plugin) = SLOTS[slot].get() else { return buf }; + let m = plugin.manifest(); + let flat = flatten(m, ctx, &buf); + let params = PluginParams { manifest: m, flat: &flat }; + let (width, height) = (buf.img.width, buf.img.height); + let before = buf.img.data.clone(); + let mut frame = PluginFrame { width, height, pixels: &mut buf.img.data, scale: buf.scale }; + if let Err(e) = plugin.render(&mut frame, ¶ms, ctx.time) { + log::warn!("plug-in `{}`: {e}", m.id); + buf.img.data = before; + } + buf +} diff --git a/crates/engine/scripts/Create Null at Selected Layers.jsx b/crates/engine/scripts/Create Null at Selected Layers.jsx new file mode 100644 index 00000000..44442c05 --- /dev/null +++ b/crates/engine/scripts/Create Null at Selected Layers.jsx @@ -0,0 +1,41 @@ +// Create Null at Selected Layers — an EffectCraft sample script (original work, MIT OR +// Apache-2.0). +// +// Adds a null at the centroid of the selected layers' positions (at the current time) and +// parents the layers that have no parent to it (set PARENT below to false to skip that). +// One undo step. + +(function createNullAtCentroid() { + var PARENT = true; + var comp = app.project.activeItem; + if (!(comp instanceof CompItem)) { + alert("Create Null: open a composition first."); + return; + } + var layers = comp.selectedLayers; + if (layers.length === 0) { + alert("Create Null: select one or more layers."); + return; + } + var t = comp.time; + var sum = [0, 0, 0]; + for (var i = 0; i < layers.length; i++) { + var p = layers[i].transform.position.valueAtTime(t, false); + sum[0] += p[0]; + sum[1] += p[1]; + sum[2] += p.length > 2 ? p[2] : 0; + } + var c = [sum[0] / layers.length, sum[1] / layers.length, sum[2] / layers.length]; + + app.beginUndoGroup("Create Null at Selected Layers"); + var nul = comp.layers.addNull(); + nul.name = "Centroid"; + nul.transform.position.setValue(c); + if (PARENT) { + for (var k = 0; k < layers.length; k++) { + if (!layers[k].parent) layers[k].setParentWithJump(nul); + } + } + app.endUndoGroup(); + writeLn("Null at [" + c[0].toFixed(1) + ", " + c[1].toFixed(1) + "]"); +})(); diff --git a/crates/engine/scripts/Rename Layers.jsx b/crates/engine/scripts/Rename Layers.jsx new file mode 100644 index 00000000..6de9d4d4 --- /dev/null +++ b/crates/engine/scripts/Rename Layers.jsx @@ -0,0 +1,71 @@ +// Rename Layers — an EffectCraft sample script (original work, MIT OR Apache-2.0). +// +// Renames the selected layers of the active composition (all layers when none are selected): +// find & replace, then an optional prefix, suffix and running number. One undo step. + +(function renameLayers() { + var comp = app.project.activeItem; + if (!(comp instanceof CompItem)) { + alert("Rename Layers: open a composition first."); + return; + } + var layers = comp.selectedLayers; + if (layers.length === 0) { + layers = []; + for (var i = 1; i <= comp.numLayers; i++) layers.push(comp.layer(i)); + } + + var dlg = new Window("dialog", "Rename Layers"); + dlg.orientation = "column"; + dlg.alignChildren = ["fill", "top"]; + + var find = dlg.add("panel", undefined, "Find and Replace"); + find.alignChildren = ["fill", "top"]; + var row1 = find.add("group"); + row1.add("statictext", undefined, "Find:"); + var findText = row1.add("edittext", undefined, ""); + findText.characters = 20; + var row2 = find.add("group"); + row2.add("statictext", undefined, "Replace:"); + var replaceText = row2.add("edittext", undefined, ""); + replaceText.characters = 20; + + var extra = dlg.add("panel", undefined, "Add"); + extra.alignChildren = ["fill", "top"]; + var row3 = extra.add("group"); + row3.add("statictext", undefined, "Prefix:"); + var prefix = row3.add("edittext", undefined, ""); + prefix.characters = 12; + row3.add("statictext", undefined, "Suffix:"); + var suffix = row3.add("edittext", undefined, ""); + suffix.characters = 12; + var number = extra.add("checkbox", undefined, "Number the layers"); + var row4 = extra.add("group"); + row4.add("statictext", undefined, "Start at:"); + var start = row4.add("edittext", undefined, "1"); + start.characters = 5; + start.enabled = false; + number.onClick = function () { + start.enabled = number.value; + }; + + var buttons = dlg.add("group"); + buttons.alignment = "right"; + buttons.add("button", undefined, "Cancel", { name: "cancel" }); + buttons.add("button", undefined, "OK", { name: "ok" }); + + if (dlg.show() !== 1) return; + + app.beginUndoGroup("Rename Layers"); + var n = parseInt(start.text, 10); + if (isNaN(n)) n = 1; + for (var k = 0; k < layers.length; k++) { + var name = layers[k].name; + if (findText.text !== "") name = name.split(findText.text).join(replaceText.text); + name = prefix.text + name + suffix.text; + if (number.value) name += " " + (n + k); + layers[k].name = name; + } + app.endUndoGroup(); + writeLn("Renamed " + layers.length + " layer(s)."); +})(); diff --git a/crates/engine/scripts/Render Queue Batch.jsx b/crates/engine/scripts/Render Queue Batch.jsx new file mode 100644 index 00000000..1dc5ca4c --- /dev/null +++ b/crates/engine/scripts/Render Queue Batch.jsx @@ -0,0 +1,52 @@ +// Render Queue Batch — an EffectCraft sample script (original work, MIT OR Apache-2.0). +// +// Adds the compositions selected in the Project panel (every composition when none are +// selected) to the Render Queue, each with an output file named after the comp in a folder you +// choose, and optionally starts rendering. + +(function renderQueueBatch() { + var comps = []; + var sel = app.project.selection; + for (var i = 0; i < sel.length; i++) if (sel[i] instanceof CompItem) comps.push(sel[i]); + if (comps.length === 0) { + for (var j = 1; j <= app.project.numItems; j++) { + var it = app.project.item(j); + if (it instanceof CompItem) comps.push(it); + } + } + if (comps.length === 0) { + alert("Render Queue Batch: the project has no compositions."); + return; + } + + var dlg = new Window("dialog", "Render Queue Batch"); + dlg.alignChildren = ["fill", "top"]; + dlg.add("statictext", undefined, comps.length + " composition(s) will be queued."); + var row = dlg.add("group"); + row.add("statictext", undefined, "Output folder:"); + var folder = row.add("edittext", undefined, Folder.desktop ? Folder.desktop.fsName : ""); + folder.characters = 30; + var fmtRow = dlg.add("group"); + fmtRow.add("statictext", undefined, "Format:"); + var format = fmtRow.add("dropdownlist", undefined, ["H.264 (.mp4)", "PNG Sequence"]); + format.selection = 0; + var go = dlg.add("checkbox", undefined, "Start rendering"); + var buttons = dlg.add("group"); + buttons.alignment = "right"; + buttons.add("button", undefined, "Cancel", { name: "cancel" }); + buttons.add("button", undefined, "Queue", { name: "ok" }); + if (dlg.show() !== 1) return; + + var png = format.selection && format.selection.index === 1; + app.beginUndoGroup("Render Queue Batch"); + for (var k = 0; k < comps.length; k++) { + var rq = app.project.renderQueue.items.add(comps[k]); + var name = comps[k].name.replace(/[\\\/:*?"<>|]/g, "_"); + var om = rq.outputModule(1); + if (png) om.format = "png"; + om.file = new File(folder.text + "/" + name + (png ? "_[#####].png" : ".mp4")); + } + app.endUndoGroup(); + writeLn("Queued " + comps.length + " composition(s)."); + if (go.value) app.project.renderQueue.render(); +})(); diff --git a/crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx b/crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx new file mode 100644 index 00000000..28991849 --- /dev/null +++ b/crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx @@ -0,0 +1,99 @@ +// Layer Tools — an EffectCraft sample ScriptUI panel (original work, MIT OR Apache-2.0). +// +// Installed in the ScriptUI Panels folder it docks like any panel (Window ▸ Layer Tools.jsx); +// run from File ▸ Scripts it opens as a floating palette. Buttons act on the selected layers of +// the active composition. + +(function layerTools(thisObj) { + var ui = thisObj instanceof Panel ? thisObj : new Window("palette", "Layer Tools", undefined, { resizeable: true }); + ui.orientation = "column"; + ui.alignChildren = ["fill", "top"]; + ui.spacing = 6; + + function comp() { + var c = app.project.activeItem; + if (!(c instanceof CompItem)) { + status.text = "Open a composition first."; + return null; + } + return c; + } + function selected(c) { + var l = c.selectedLayers; + if (l.length === 0) status.text = "Select some layers."; + return l; + } + + var labelRow = ui.add("group"); + labelRow.add("statictext", undefined, "Opacity:"); + var opacity = labelRow.add("slider", undefined, 100, 0, 100); + opacity.preferredSize.width = 120; + var opacityValue = labelRow.add("statictext", undefined, "100%"); + opacityValue.characters = 5; + opacity.onChanging = function () { + opacityValue.text = Math.round(opacity.value) + "%"; + }; + opacity.onChange = function () { + var c = comp(); + if (!c) return; + var layers = selected(c); + app.beginUndoGroup("Set Opacity"); + for (var i = 0; i < layers.length; i++) layers[i].transform.opacity.setValue(Math.round(opacity.value)); + app.endUndoGroup(); + status.text = "Opacity " + Math.round(opacity.value) + "% on " + layers.length + " layer(s)."; + }; + + var shy = ui.add("checkbox", undefined, "Shy"); + shy.onClick = function () { + var c = comp(); + if (!c) return; + var layers = selected(c); + app.beginUndoGroup("Shy Layers"); + for (var i = 0; i < layers.length; i++) layers[i].shy = shy.value; + app.endUndoGroup(); + }; + + var nameRow = ui.add("group"); + nameRow.alignChildren = ["fill", "center"]; + var newName = nameRow.add("edittext", undefined, "Layer"); + newName.characters = 14; + var renameBtn = nameRow.add("button", undefined, "Rename"); + renameBtn.onClick = function () { + var c = comp(); + if (!c) return; + var layers = selected(c); + app.beginUndoGroup("Rename Layers"); + for (var i = 0; i < layers.length; i++) layers[i].name = layers.length > 1 ? newName.text + " " + (i + 1) : newName.text; + app.endUndoGroup(); + status.text = "Renamed " + layers.length + " layer(s)."; + }; + + var nullBtn = ui.add("button", undefined, "Null at Centroid"); + nullBtn.onClick = function () { + var c = comp(); + if (!c) return; + var layers = selected(c); + if (layers.length === 0) return; + var x = 0, y = 0; + for (var i = 0; i < layers.length; i++) { + var p = layers[i].transform.position.value; + x += p[0]; + y += p[1]; + } + app.beginUndoGroup("Null at Centroid"); + var n = c.layers.addNull(); + n.transform.position.setValue([x / layers.length, y / layers.length]); + app.endUndoGroup(); + status.text = "Added " + n.name + "."; + }; + + var status = ui.add("statictext", undefined, "Select layers, then pick a tool."); + status.characters = 30; + + if (ui instanceof Window) { + ui.center(); + ui.show(); + } else { + ui.layout.layout(true); + } +})(this); diff --git a/crates/engine/scripts/Sort Layers by In Point.jsx b/crates/engine/scripts/Sort Layers by In Point.jsx new file mode 100644 index 00000000..c59473d7 --- /dev/null +++ b/crates/engine/scripts/Sort Layers by In Point.jsx @@ -0,0 +1,37 @@ +// Sort Layers by In Point — an EffectCraft sample script (original work, MIT OR Apache-2.0). +// +// Reorders the selected layers of the active composition (all layers when none are selected) +// so the layer that starts first is on top. Layers with the same in point keep their order. +// One undo step. + +(function sortLayersByInPoint() { + var comp = app.project.activeItem; + if (!(comp instanceof CompItem)) { + alert("Sort Layers by In Point: open a composition first."); + return; + } + var layers = comp.selectedLayers; + if (layers.length === 0) { + layers = []; + for (var i = 1; i <= comp.numLayers; i++) layers.push(comp.layer(i)); + } + if (layers.length < 2) return; + + // The slots the layers occupy now (stack indices, top first). + var slots = layers.map(function (l) { return l.index; }).sort(function (a, b) { return a - b; }); + var order = layers + .map(function (l, k) { return { layer: l, inPoint: l.inPoint, k: k }; }) + .sort(function (a, b) { return a.inPoint - b.inPoint || a.k - b.k; }); + + app.beginUndoGroup("Sort Layers by In Point"); + // Place each layer, earliest first, in the next slot from the top. + for (var j = 0; j < order.length; j++) { + var target = slots[j]; + var layer = order[j].layer; + if (layer.index === target) continue; + var anchor = comp.layer(target); + layer.moveBefore(anchor); + } + app.endUndoGroup(); + writeLn("Sorted " + layers.length + " layer(s) by in point."); +})(); diff --git a/crates/engine/src/commands/edit.rs b/crates/engine/src/commands/edit.rs index 03cd8087..a76c2a27 100644 --- a/crates/engine/src/commands/edit.rs +++ b/crates/engine/src/commands/edit.rs @@ -584,8 +584,7 @@ fn edit_original(s: &mut Session, _: &Value) -> Result { } fn purge(s: &mut Session, _: &Value) -> Result { - s.history.undo.clear(); - s.history.redo.clear(); + s.history.clear(); s.toast("Purged undo history"); Ok(Value::Null) } @@ -630,9 +629,60 @@ pub fn specs() -> Vec { cmd!("edit.purge", "Purge", [], None, "{what?: all|memoryAndDisk|memory|disk|3d|image|snapshot}", always_ok, purge_caches), crate::query!("cache.diskStats", "Disk Cache Statistics", "{}", disk_stats), cmd!("edit.editOriginal", "Edit Original...", ["Edit"], Some("Cmd+E"), "{}", has_selected_footage, edit_original), + crate::query!( + "edit.history.list", + "History", + "{} → {states: [{index, id, label, parent, depth, current, line, future}], current, branches}", + history_list + ), + cmd!( + "edit.history.goto", + "Go to History State", + [], + None, + "{index? (from edit.history.list) | id? | steps? (negative = back along the line)}", + always_ok, + history_goto + ), ] } +// ---------------------------------------------------------------- History panel + +/// `edit.history.list`: every state of the (branching) undo history. +fn history_list(s: &mut Session, _: &Value) -> Result { + let nodes = s.history_tree(); + let current = nodes.iter().position(|n| n.current); + Ok(json!({"states": nodes, "current": current, "branches": s.history.branches.len()})) +} + +/// `edit.history.goto`: jump to a state by `index` (from `edit.history.list`), `id`, or +/// `steps` along the working line (negative = back, like Undo). +fn history_goto(s: &mut Session, p: &Value) -> Result { + let c = "edit.history.goto"; + if let Some(n) = p.get("steps").and_then(Value::as_i64) { + for _ in 0..n.unsigned_abs() { + let moved = if n < 0 { s.undo() } else { s.redo() }; + if !moved { + break; + } + } + } else { + let nodes = s.history_tree(); + let id = match (p.get("id").and_then(Value::as_str), p.get("index").and_then(Value::as_u64)) { + (Some(id), _) => id.to_string(), + (None, Some(i)) => nodes.get(i as usize).map(|n| n.id.clone()).ok_or_else(|| super::bad(c, format!("no state {i} (0..{})", nodes.len())))?, + _ => return Err(super::bad(c, "give `index`, `id` or `steps`")), + }; + if !s.goto_history(&id) { + return Err(super::bad(c, format!("no history state `{id}`"))); + } + } + let nodes = s.history_tree(); + let cur = nodes.iter().find(|n| n.current).cloned(); + Ok(json!({"current": cur.as_ref().map(|n| n.index), "label": cur.map(|n| n.label)})) +} + fn layers_or_keys(s: &Session) -> std::result::Result<(), String> { if !s.state.selected_keys.is_empty() || !s.state.selected_vertices.is_empty() { super::has_comp(s) } else { has_layers(s) } } diff --git a/crates/engine/src/commands/effect.rs b/crates/engine/src/commands/effect.rs index b0fb8c31..6ce73ac3 100644 --- a/crates/engine/src/commands/effect.rs +++ b/crates/engine/src/commands/effect.rs @@ -7,6 +7,68 @@ use serde_json::{Value, json}; use super::{CommandSpec, b_p, bad, has_layers, layer_mut, layers_p, str_p}; use crate::{EngineError, Result, Session, cmd}; +/// `effect.plugins.list`: the registered plug-in effects (WebAssembly or Rust). +fn plugins_list(s: &mut Session, _: &Value) -> Result { + use effectcraft_effects::plugin; + let list: Vec = plugin::plugins() + .into_iter() + .map(|spec| { + let p = plugin::plugin(spec.id); + let m = p.as_ref().map(|p| p.manifest().clone()); + json!({ + "id": spec.id, + "name": spec.name, + "category": spec.category, + "version": m.as_ref().map(|m| m.version.clone()), + "author": m.as_ref().map(|m| m.author.clone()), + "description": m.as_ref().map(|m| m.description.clone()), + "source": p.map(|p| p.source()), + "params": spec.params.iter().map(|p| p.id).collect::>(), + }) + }) + .collect(); + Ok(json!({"api": plugin::PLUGIN_API_VERSION, "wasm": s.plugin_loader.is_some(), "plugins": list})) +} + +/// `effect.plugins.load`: load WebAssembly effect plug-ins (a file, or every `.wasm` in a +/// folder) through the host's loader ([`Session::plugin_loader`]). +fn plugins_load(s: &mut Session, p: &Value) -> Result { + let c = "effect.plugins.load"; + let loader = s.plugin_loader.ok_or_else(|| EngineError::Other("WebAssembly effect plug-ins are not available in this build".into()))?; + let files: Vec = match (str_p(p, "path"), str_p(p, "folder")) { + (Some(path), _) => vec![path.to_string()], + (None, Some(dir)) => { + let mut v: Vec = std::fs::read_dir(dir) + .map_err(|e| bad(c, format!("cannot list {dir}: {e}")))? + .flatten() + .map(|e| e.path()) + .filter(|p| p.extension().is_some_and(|x| x.eq_ignore_ascii_case("wasm"))) + .map(|p| p.to_string_lossy().to_string()) + .collect(); + v.sort(); + v + } + _ => return Err(bad(c, "give `path` or `folder`")), + }; + let (mut loaded, mut errors) = (vec![], vec![]); + for f in &files { + let r = s.services.read_file(f).map_err(|e| format!("cannot read {f}: {e}")).and_then(|bytes| loader(&bytes, f)); + match r { + Ok(v) => loaded.push(v), + Err(e) => errors.push(json!({"path": f, "error": e})), + } + } + if str_p(p, "path").is_some() + && let Some(e) = errors.first() + { + return Err(EngineError::Other(format!("{}: {}", files[0], e["error"].as_str().unwrap_or("failed")))); + } + if !loaded.is_empty() { + s.toast(format!("Loaded {} effect plug-in(s)", loaded.len())); + } + Ok(json!({"loaded": loaded, "errors": errors})) +} + fn apply(s: &mut Session, p: &Value) -> Result { let name = str_p(p, "effect").ok_or_else(|| bad("effect.apply", "missing `effect` (id or name)"))?; let spec = effectcraft_effects::lookup(name).ok_or_else(|| bad("effect.apply", format!("unknown effect `{name}`")))?; @@ -284,8 +346,8 @@ fn last(s: &mut Session, p: &Value) -> Result { fn list(_: &mut Session, p: &Value) -> Result { let filter = str_p(p, "filter").map(str::to_ascii_lowercase); - let v: Vec = effectcraft_effects::registry() - .iter() + let v: Vec = effectcraft_effects::all() + .into_iter() .filter(|e| filter.as_ref().is_none_or(|f| e.name.to_ascii_lowercase().contains(f) || e.id.contains(f.as_str()))) .map(|e| json!({"id": e.id, "name": e.name, "category": e.category, "params": e.params.iter().map(|p| json!({"id": p.id, "name": p.name, "default": p.default.to_json()})).collect::>()})) .collect(); @@ -305,5 +367,20 @@ pub fn specs() -> Vec { cmd!("effect.paste", "Paste Effects", [], None, "{layers?} — adds the copied effects to the layers", has_layers, paste), cmd!("effect.reset", "Reset Effect", [], None, "{layer?, effect}", has_layers, reset), crate::query!("effect.list", "List Effects", "{filter?}", list), + crate::query!( + "effect.plugins.list", + "List Effect Plug-ins", + "{} → {api, wasm, plugins: [{id, name, category, version, author, source, params}]}", + plugins_list + ), + cmd!( + "effect.plugins.load", + "Load Effect Plug-in...", + [], + None, + "{path (.wasm / .wat plug-in, API v1) | folder (loads every .wasm in it)}", + super::always, + plugins_load + ), ] } diff --git a/crates/engine/src/commands/file_more.rs b/crates/engine/src/commands/file_more.rs index 6908bcd5..9175af59 100644 --- a/crates/engine/src/commands/file_more.rs +++ b/crates/engine/src/commands/file_more.rs @@ -364,6 +364,14 @@ fn collect_files(s: &mut Session, p: &Value) -> Result { /// array or JSON lines of `{"command": id, "params": {…}}` (also accepts the control channel's /// `{"method": "engine.execute", "params": {"id", "params"}}`). fn run_script(s: &mut Session, p: &Value) -> Result { + // File ▸ Scripts ▸ . + if let Some(name) = str_p(p, "name") + && p.get("path").is_none() + && p.get("steps").is_none() + { + let name = name.to_string(); + return super::scripts::run_named(s, &name); + } if let Some(path) = str_p(p, "path") && p.get("steps").is_none() && is_js_script(path) @@ -375,13 +383,7 @@ fn run_script(s: &mut Session, p: &Value) -> Result { let code = String::from_utf8_lossy(&bytes).to_string(); let name = std::path::Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.to_string()); let out = run_js(s, &code, path, false)?; - if let Some(e) = out.get("error").filter(|e| !e.is_null()) { - let line = e.get("line").and_then(Value::as_u64).map(|l| format!(" (line {l})")).unwrap_or_default(); - let msg = e.get("message").and_then(Value::as_str).unwrap_or("script error"); - s.events.push(crate::Event::Toast { message: format!("{name}{line}: {msg}"), error: true }); - return Err(EngineError::Other(format!("{name}{line}: {msg}"))); - } - return Ok(out); + return report_js(s, &name, out); } let steps: Vec = match (p.get("steps"), str_p(p, "path")) { (Some(Value::Array(a)), _) => a.clone(), @@ -426,6 +428,23 @@ fn is_js_script(path: &str) -> bool { l.ends_with(".jsx") || l.ends_with(".js") || l.ends_with(".jsxbin") } +/// A script error becomes a toast and a command error. +fn report_js(s: &mut Session, name: &str, out: Value) -> Result { + if let Some(e) = out.get("error").filter(|e| !e.is_null()) { + let line = e.get("line").and_then(Value::as_u64).map(|l| format!(" (line {l})")).unwrap_or_default(); + let msg = e.get("message").and_then(Value::as_str).unwrap_or("script error"); + s.events.push(crate::Event::Toast { message: format!("{name}{line}: {msg}"), error: true }); + return Err(EngineError::Other(format!("{name}{line}: {msg}"))); + } + Ok(out) +} + +/// Run a script's code under its file name (installed / sample scripts, ScriptUI panels). +pub(crate) fn run_js_named(s: &mut Session, code: &str, name: &str) -> Result { + let out = run_js(s, code, name, false)?; + report_js(s, name, out) +} + /// Run JavaScript through the session's scripting engine. fn run_js(s: &mut Session, code: &str, name: &str, console: bool) -> Result { let runner = s.script.ok_or_else(|| EngineError::Other("JavaScript scripting is not available in this build".into()))?; @@ -809,7 +828,7 @@ pub fn specs() -> Vec { "Run Script File...", ["File", "Scripts"], None, - "{path (.jsx/.js JavaScript, or a .jsonl/.json command script) | steps: [{command, params}]}", + "{path (.jsx/.js JavaScript, or a .jsonl/.json command script) | steps: [{command, params}] | name (an installed or sample script, see file.scripts.list)}", always, run_script ), diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 4112cf6c..16afc125 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -40,6 +40,7 @@ mod query; mod render_queue; pub(crate) mod rig3d; pub mod roto_cmds; +pub mod scripts; mod settings; mod shape_stroke; mod stubs; @@ -158,6 +159,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(essential::specs()); v.extend(expr_tools::specs()); v.extend(proxy::specs()); + v.extend(scripts::specs()); v.extend(stubs::specs()); v }) diff --git a/crates/engine/src/commands/puppet.rs b/crates/engine/src/commands/puppet.rs index 4ada1dcd..890e1091 100644 --- a/crates/engine/src/commands/puppet.rs +++ b/crates/engine/src/commands/puppet.rs @@ -7,7 +7,7 @@ use effectcraft_effects::puppet::{self, MeshOpts, PinKind}; use effectcraft_effects::{Buf, Params}; -use effectcraft_keyframe::Value as KV; +use effectcraft_keyframe::{Keyframe, Value as KV}; use effectcraft_project::build::Ids; use effectcraft_project::{ItemId, LayerId, Node, PropGroup, Uid}; use effectcraft_render::{EvalCtx, RenderOpts, Renderer}; @@ -309,6 +309,122 @@ fn mesh_opts(s: &mut Session, p: &Value) -> Result { Ok(Value::Null) } +/// Parse recorded samples `[[t, x, y]…]` (t = seconds since the drag began, layer space). +fn samples(p: &Value, cmd: &str) -> Result> { + let arr = p + .get("samples") + .and_then(Value::as_array) + .ok_or_else(|| bad(cmd, "missing `samples`: [[t, x, y]…] (t = seconds since the drag began, layer space)"))?; + let mut rec = Vec::with_capacity(arr.len()); + for (i, v) in arr.iter().enumerate() { + let a: Vec = v.as_array().map(|a| a.iter().filter_map(Value::as_f64).collect()).unwrap_or_default(); + if a.len() < 3 || a.iter().any(|x| !x.is_finite()) { + return Err(bad(cmd, format!("sample {i}: expected [t, x, y]"))); + } + rec.push((a[0], [a[1], a[2]])); + } + if rec.is_empty() { + return Err(bad(cmd, "need at least one sample")); + } + rec.sort_by(|a, b| a.0.total_cmp(&b.0)); + Ok(rec) +} + +/// Puppet tool Record mode (⌘/Ctrl-drag a pin): the pin's drag, sampled in real time, becomes +/// Position keyframes at the comp frame rate from the current time on, replacing the keys in +/// the recorded span. Record Options: `speed` (%: 100 plays back at the speed of the drag, +/// 200 twice as slow) and `smoothing` (removes extraneous keys like the Smoother; 0 = a key +/// on every frame). The current time stays where recording began. +fn record_pin(s: &mut Session, p: &Value) -> Result { + let cmd = "puppet.recordPin"; + let (cid, lid) = layer_p(s, p, cmd)?; + let key = p.get("pin").cloned().ok_or_else(|| bad(cmd, "missing `pin` (uid or name)"))?; + let uid = pin_uid(s, cid, lid, &key, cmd)?; + let rec = samples(p, cmd)?; + let opts = &s.state.puppet; + let speed = f_p(p, "speed").unwrap_or(opts.record_speed).clamp(1.0, 10_000.0) / 100.0; + // Smoothing 10 (the default) keeps the path within 1 layer pixel of the drag. + let smoothing = f_p(p, "smoothing").unwrap_or(opts.record_smoothing).max(0.0) / 10.0; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let fr = comp.frame_rate; + let fd = fr.frame_duration().seconds(); + let start = match p.get("start").and_then(Value::as_f64) { + Some(t) => fr.snap_nearest(Tick::from_seconds_f64(t)), + None => s.time_of(cid), + }; + let f0 = fr.frame_at(start); + let end_frame = fr.frame_at(comp.duration - comp.frame_duration()); + let t_rec0 = rec[0].0; + let span = (rec[rec.len() - 1].0 - t_rec0) * speed; + let nframes = (span / fd).round().max(0.0) as i64; + let at = |secs: f64| { + let i = rec.partition_point(|r| r.0 <= secs).clamp(1, rec.len().max(2) - 1); + if rec.len() == 1 { + return rec[0].1; + } + let (a, b) = (rec[i - 1], rec[i]); + let f = if b.0 > a.0 { ((secs - a.0) / (b.0 - a.0)).clamp(0.0, 1.0) } else { 1.0 }; + [a.1[0] + (b.1[0] - a.1[0]) * f, a.1[1] + (b.1[1] - a.1[1]) * f] + }; + let n = s.edit("Record Puppet Pin", None, |proj, st| { + let l = layer_mut(proj, cid, lid)?; + let lc = l.clone(); + let g = l.props.find_group_mut(uid).ok_or_else(|| bad(cmd, "pin gone"))?; + let kind = PinKind::from_index(g.get("kind").map(|k| k.value.as_enum()).unwrap_or(0)); + if !kind.moves() { + return Err(bad(cmd, "only Position and Advanced pins record motion")); + } + let pos = g.get_mut("position").ok_or_else(|| bad(cmd, "the pin has no Position"))?; + let mut new: Vec = vec![]; + for k in 0..=nframes { + let f = f0 + k; + if f > end_frame { + break; + } + let q = at(t_rec0 + k as f64 * fd / speed); + new.push(Keyframe::new(lc.layer_time(fr.tick_of(f)), KV::Vec2(q))); + } + let (Some(t0), Some(t1)) = (new.first().map(|k| k.time), new.last().map(|k| k.time)) else { return Ok(0) }; + let mut keys: Vec = pos.keys.iter().filter(|k| k.time < t0 || k.time > t1).cloned().collect(); + keys.extend(new); + keys.sort_by_key(|k| k.time); + let (a, b) = (keys.iter().position(|k| k.time == t0).unwrap_or(0), keys.iter().position(|k| k.time == t1).unwrap_or(0)); + if smoothing > 0.0 && b > a + 1 { + let samples: Vec<(Tick, Vec)> = keys[a..=b].iter().map(|k| (k.time, k.value.components())).collect(); + keys = super::anim_tools::smooth_keys(&keys, a, b, &samples, smoothing, true); + } + pos.keys = keys; + pos.value = KV::Vec2(rec[0].1); + let puid = pos.uid; + st.selected_props = vec![(lid, uid)]; + st.selected_keys = pos.keys.iter().filter(|k| k.time >= t0 && k.time <= t1).map(|k| crate::KeyRef { layer: lid, prop: puid, time: k.time }).collect(); + Ok(st.selected_keys.len()) + })?; + // The current time returns to where recording began. + s.state.times.insert(cid, start); + let end = fr.tick_of((f0 + nframes).min(end_frame)); + Ok(json!({"keys": n, "start": start.seconds(), "end": end.seconds(), "frames": nframes + 1})) +} + +/// Record Options (the Puppet tool's Record Options… dialog): `speed`, `smoothing`, +/// `useDraftDeformation`, `showMesh`. Returns the options. +fn record_options(s: &mut Session, p: &Value) -> Result { + let o = &mut s.state.puppet; + if let Some(v) = f_p(p, "speed") { + o.record_speed = v.clamp(1.0, 10_000.0); + } + if let Some(v) = f_p(p, "smoothing") { + o.record_smoothing = v.clamp(0.0, 100.0); + } + if let Some(v) = p.get("useDraftDeformation").and_then(Value::as_bool) { + o.record_draft = v; + } + if let Some(v) = p.get("showMesh").and_then(Value::as_bool) { + o.record_show_mesh = v; + } + Ok(json!({"speed": o.record_speed, "smoothing": o.record_smoothing, "useDraftDeformation": o.record_draft, "showMesh": o.record_show_mesh})) +} + /// Meshes, pins and the deformed mesh at the current time (layer space). fn info(s: &mut Session, p: &Value) -> Result { let (cid, lid) = layer_p(s, p, "puppet.info")?; @@ -364,6 +480,24 @@ pub fn specs() -> Vec { mesh_opts ), crate::query!("puppet.info", "Puppet Mesh Info", "{layer?, time? | frame?}", info), + cmd!( + "puppet.recordPin", + "Record Puppet Pin", + [], + None, + "{layer?, pin: uid | name, samples: [[t, x, y]…] (t = seconds since the drag began, layer space), start? (s, default current time), speed? (%, Record Options), smoothing? (Record Options)}", + has_comp, + record_pin + ), + cmd!( + "puppet.recordOptions", + "Record Options...", + [], + None, + "{speed? (%, 100), smoothing? (0-100), useDraftDeformation?, showMesh?}", + super::always, + record_options + ), ] } @@ -376,11 +510,30 @@ pub struct PuppetOptions { pub triangles: f64, /// Show the mesh in the viewer. pub show_mesh: bool, + /// Record Options ▸ Speed (%): 100 plays a recording back at the speed of the drag. + pub record_speed: f64, + /// Record Options ▸ Smoothing: removes extraneous keys from recorded motion (0 = a key per + /// frame). + pub record_smoothing: f64, + /// Record Options ▸ Use Draft Deformation: the viewer shows the draft (unexpanded) mesh + /// while recording. + pub record_draft: bool, + /// Record Options ▸ Show Mesh while recording. + pub record_show_mesh: bool, } impl Default for PuppetOptions { fn default() -> Self { let d = MeshOpts::default(); - PuppetOptions { density: d.density, expansion: d.expansion, triangles: d.triangles, show_mesh: false } + PuppetOptions { + density: d.density, + expansion: d.expansion, + triangles: d.triangles, + show_mesh: false, + record_speed: 100.0, + record_smoothing: 10.0, + record_draft: false, + record_show_mesh: false, + } } } diff --git a/crates/engine/src/commands/scripts.rs b/crates/engine/src/commands/scripts.rs new file mode 100644 index 00000000..628533af --- /dev/null +++ b/crates/engine/src/commands/scripts.rs @@ -0,0 +1,222 @@ +//! File ▸ Scripts and ScriptUI. +//! +//! * **Installed scripts** live in the settings store under `Scripts/` (File ▸ Scripts ▸ Install +//! Script File…) and `Scripts/ScriptUI Panels/` (Install ScriptUI Panel…), next to the +//! settings on the desktop, so dropping `.jsx` files in those folders works too. File ▸ Scripts +//! lists them with the bundled sample scripts ([`SAMPLES`]) and runs them by name; ScriptUI +//! panels appear at the bottom of the Window menu and open as dockable panels +//! (`window.scriptPanel`), running with `this` = the panel as in After Effects. +//! * **ScriptUI windows** (`scriptui.*`): list the open script windows, read their control +//! trees, click buttons and set values like a user, close them (see [`crate::scriptui`]). + +use serde::Serialize; +use serde_json::{Value, json}; + +use super::{CommandSpec, always, bad, str_p}; +use crate::{EngineError, Event, Result, Session, cmd, scriptui}; + +/// The settings-store folders of installed scripts. +pub const SCRIPTS_DIR: &str = "Scripts"; +pub const PANELS_DIR: &str = "Scripts/ScriptUI Panels"; + +/// Sample scripts that ship with EffectCraft (original work): (file name, ScriptUI panel, code). +pub const SAMPLES: &[(&str, bool, &str)] = &[ + ("Create Null at Selected Layers.jsx", false, include_str!("../../scripts/Create Null at Selected Layers.jsx")), + ("Rename Layers.jsx", false, include_str!("../../scripts/Rename Layers.jsx")), + ("Render Queue Batch.jsx", false, include_str!("../../scripts/Render Queue Batch.jsx")), + ("Sort Layers by In Point.jsx", false, include_str!("../../scripts/Sort Layers by In Point.jsx")), + ("Layer Tools.jsx", true, include_str!("../../scripts/ScriptUI Panels/Layer Tools.jsx")), +]; + +/// A script File ▸ Scripts (or the Window menu, for panels) offers. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct ScriptEntry { + /// File name (`Rename Layers.jsx`): what the menus show and commands take. + pub name: String, + /// In the ScriptUI Panels folder: opens as a dockable panel. + pub panel: bool, + /// `installed` or `sample`. + pub source: &'static str, +} + +fn is_script(name: &str) -> bool { + let l = name.to_ascii_lowercase(); + l.ends_with(".jsx") || l.ends_with(".js") +} + +/// Installed scripts and panels, then the samples not shadowed by an installed one, by name. +pub fn scripts(s: &Session) -> Vec { + let mut v: Vec = vec![]; + if let Some(cfg) = &s.config { + for (dir, panel) in [(SCRIPTS_DIR, false), (PANELS_DIR, true)] { + for n in cfg.list(dir).into_iter().filter(|n| is_script(n)) { + v.push(ScriptEntry { name: n, panel, source: "installed" }); + } + } + } + for (name, panel, _) in SAMPLES { + if !v.iter().any(|e| e.name == *name && e.panel == *panel) { + v.push(ScriptEntry { name: name.to_string(), panel: *panel, source: "sample" }); + } + } + v.sort_by(|a, b| a.panel.cmp(&b.panel).then(a.name.to_lowercase().cmp(&b.name.to_lowercase()))); + v +} + +/// The code of an installed or sample script / panel. +pub fn script_code(s: &Session, name: &str, panel: Option) -> Option<(bool, String)> { + if let Some(cfg) = &s.config { + for (dir, is_panel) in [(SCRIPTS_DIR, false), (PANELS_DIR, true)] { + if panel.is_some_and(|p| p != is_panel) { + continue; + } + if let Some(code) = cfg.read(&format!("{dir}/{name}")) { + return Some((is_panel, code)); + } + } + } + SAMPLES.iter().find(|(n, p, _)| *n == name && panel.is_none_or(|w| w == *p)).map(|(_, p, c)| (*p, c.to_string())) +} + +fn install(s: &mut Session, p: &Value, panel: bool) -> Result { + let c = if panel { "file.installScriptUIPanel" } else { "file.installScript" }; + let path = str_p(p, "path").ok_or_else(|| bad(c, "missing `path` (a .jsx / .js script)"))?; + let name = std::path::Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); + if !is_script(&name) { + return Err(bad(c, "scripts are .jsx or .js files (.jsxbin is not supported)")); + } + let bytes = s.services.read_file(path).map_err(|e| EngineError::Other(format!("cannot read {path}: {e}")))?; + let code = String::from_utf8(bytes).map_err(|_| bad(c, "the script is not UTF-8 text"))?; + let cfg = s.config.clone().ok_or_else(|| EngineError::Other("this session has no settings folder to install scripts into".into()))?; + let dir = if panel { PANELS_DIR } else { SCRIPTS_DIR }; + cfg.write(&format!("{dir}/{name}"), &code).map_err(|e| EngineError::Other(format!("cannot install {name}: {e}")))?; + // After Effects asks for a restart; our menus pick it up at once. + s.toast(if panel { + format!("Installed ScriptUI panel {name}: find it at the bottom of the Window menu") + } else { + format!("Installed script {name}: find it in File ▸ Scripts") + }); + Ok(json!({"name": name, "panel": panel})) +} + +fn list(s: &mut Session, _: &Value) -> Result { + Ok(json!(scripts(s))) +} + +fn uninstall(s: &mut Session, p: &Value) -> Result { + let c = "file.uninstallScript"; + let name = str_p(p, "name").ok_or_else(|| bad(c, "missing `name`"))?; + let cfg = s.config.clone().ok_or_else(|| bad(c, "no settings folder"))?; + let mut removed = false; + for dir in [SCRIPTS_DIR, PANELS_DIR] { + let key = format!("{dir}/{name}"); + if cfg.read(&key).is_some() { + cfg.remove(&key).map_err(|e| EngineError::Other(e.to_string()))?; + removed = true; + } + } + if removed { Ok(json!({"removed": name})) } else { Err(bad(c, format!("`{name}` is not installed"))) } +} + +/// Run an installed or sample script by name (File ▸ Scripts ▸ ). A ScriptUI panel run +/// this way gets `this` = a floating palette-like panel window. +pub(crate) fn run_named(s: &mut Session, name: &str) -> Result { + let (panel, code) = script_code(s, name, None).ok_or_else(|| bad("file.runScript", format!("no installed or sample script `{name}`")))?; + if panel { + return open_panel(s, &json!({"name": name})); + } + super::file_more::run_js_named(s, &code, name) +} + +/// The code a ScriptUI panel runs with: `this` is a dockable Panel titled after the script. +pub fn panel_wrapper(title: &str, code: &str) -> String { + // On one line, so line numbers in errors stay those of the file. + format!("(function () {{ {code}\n}}).call(__uiDockPanel({}));", Value::String(title.to_string())) +} + +/// Window ▸ : run the panel script (or bring its panel forward). +fn open_panel(s: &mut Session, p: &Value) -> Result { + let c = "window.scriptPanel"; + let name = str_p(p, "name").ok_or_else(|| bad(c, "missing `name` (a script in the ScriptUI Panels folder)"))?.to_string(); + let title = name.trim_end_matches(".jsx").trim_end_matches(".js").to_string(); + if let Some(w) = s.script_ui.windows.iter().find(|w| w.kind == scriptui::WindowKind::Panel && w.script == name) { + let id = w.id; + s.events.push(Event::Frontend { command: c.into(), params: json!({"window": id}) }); + return Ok(json!({"window": id, "open": true})); + } + let (_, code) = script_code(s, &name, Some(true)).or_else(|| script_code(s, &name, None)).ok_or_else(|| bad(c, format!("no ScriptUI panel `{name}`")))?; + let out = super::file_more::run_js_named(s, &panel_wrapper(&title, &code), &name)?; + let id = s.script_ui.windows.iter().rev().find(|w| w.kind == scriptui::WindowKind::Panel && w.script == name).map(|w| w.id); + if let Some(id) = id { + s.events.push(Event::Frontend { command: c.into(), params: json!({"window": id}) }); + } + Ok(json!({"window": id, "result": out})) +} + +pub fn specs() -> Vec { + vec![ + cmd!( + "file.installScript", + "Install Script File...", + ["File", "Scripts"], + None, + "{path (.jsx / .js)} → copies it to the Scripts folder; it appears in File ▸ Scripts", + always, + |s, p| install(s, p, false) + ), + cmd!( + "file.installScriptUIPanel", + "Install ScriptUI Panel...", + ["File", "Scripts"], + None, + "{path (.jsx / .js)} → copies it to the ScriptUI Panels folder; it appears in the Window menu", + always, + |s, p| install(s, p, true) + ), + cmd!("file.uninstallScript", "Uninstall Script", [], None, "{name}", always, uninstall), + crate::query!("file.scripts.list", "List Scripts", "{} → [{name, panel, source: installed|sample}]", list), + cmd!( + "window.scriptPanel", + "ScriptUI Panel", + [], + None, + "{name (a script in the ScriptUI Panels folder, e.g. `Layer Tools.jsx`)} → opens it as a dockable panel", + always, + open_panel + ), + crate::query!("scriptui.list", "List Script Windows", "{} → [{window, title, kind: dialog|palette|window|panel, script, modal, size}]", scriptui::list), + crate::query!( + "scriptui.get", + "Script Window Controls", + "{window?: id | title} → {id, title, kind, root: {id, type, name, text, value, checked, items, selection, bounds, enabled, children…}}", + scriptui::get + ), + cmd!( + "scriptui.click", + "Click Script Window Control", + [], + None, + "{window?: id | title, widget: id | \"#id\" | properties.name | text} (buttons, checkboxes, radio buttons, tabs)", + always, + scriptui::click + ), + cmd!( + "scriptui.set", + "Set Script Window Control", + [], + None, + "{window?, widget, value: text | number | bool | item index | item text} (edit text, sliders, checkboxes, lists) → fires onChanging / onChange", + always, + scriptui::set + ), + cmd!( + "scriptui.close", + "Close Script Window", + [], + None, + "{window?, result? (a dialog's show() returns it; default 2 = Cancel)}", + always, + scriptui::close + ), + ] +} diff --git a/crates/engine/src/config.rs b/crates/engine/src/config.rs index ae7cfe09..0608dc6c 100644 --- a/crates/engine/src/config.rs +++ b/crates/engine/src/config.rs @@ -16,6 +16,12 @@ pub trait ConfigStore: Send + Sync { fn read(&self, name: &str) -> Option; fn write(&self, name: &str, data: &str) -> std::io::Result<()>; fn remove(&self, name: &str) -> std::io::Result<()>; + /// Names of the entries directly in `dir` (a `/`-separated name prefix such as `Scripts`), + /// sorted. Stores that can't list return nothing. + fn list(&self, dir: &str) -> Vec { + let _ = dir; + vec![] + } /// The folder next to the configuration (default Auto-Save folder for untitled projects), /// in the namespace of [`ConfigStore::files`]; `None` when there is none. fn dir(&self) -> Option { @@ -86,6 +92,11 @@ impl ConfigStore for MemoryConfig { } Ok(()) } + fn list(&self, dir: &str) -> Vec { + let pre = format!("{}/", dir.trim_end_matches('/')); + let Ok(m) = self.files.lock() else { return vec![] }; + m.keys().filter_map(|k| k.strip_prefix(&pre)).filter(|n| !n.is_empty() && !n.contains('/')).map(str::to_string).collect() + } } /// Files in one directory, written atomically. @@ -104,8 +115,15 @@ impl ConfigStore for DirConfig { std::fs::read_to_string(self.dir.join(name)).ok() } fn write(&self, name: &str, data: &str) -> std::io::Result<()> { - std::fs::create_dir_all(&self.dir)?; - atomic_write(&self.dir.join(name), data.as_bytes()) + let path = self.dir.join(name); + std::fs::create_dir_all(path.parent().unwrap_or(&self.dir))?; + atomic_write(&path, data.as_bytes()) + } + fn list(&self, dir: &str) -> Vec { + let Ok(rd) = std::fs::read_dir(self.dir.join(dir)) else { return vec![] }; + let mut v: Vec = rd.flatten().filter(|e| e.path().is_file()).map(|e| e.file_name().to_string_lossy().to_string()).collect(); + v.sort(); + v } fn remove(&self, name: &str) -> std::io::Result<()> { match std::fs::remove_file(self.dir.join(name)) { diff --git a/crates/engine/src/history.rs b/crates/engine/src/history.rs new file mode 100644 index 00000000..36630969 --- /dev/null +++ b/crates/engine/src/history.rs @@ -0,0 +1,346 @@ +//! Branching undo history (the History panel; better than After Effects, whose undo is a +//! straight line). +//! +//! [`History`] keeps After Effects' linear model as its working line — `undo` (the states +//! before each step, oldest first) and `redo` (undone states, next first) — and adds +//! **branches**: when you undo and then do something new, the undone states are not thrown +//! away but kept as a [`Branch`] hanging off the state they grew from. [`History::tree`] lists +//! every state as a tree and [`Session::goto_history`] jumps to any of them, on any branch: the +//! path to the target becomes the working line, and the line you left becomes a branch. +//! +//! States are whole-project snapshots (`Arc`), identified by their address while the +//! history holds them ([`HistoryNode::id`]), so ids stay valid across undo, redo and jumps. + +use std::collections::HashMap; +use std::sync::Arc; + +use effectcraft_project::Project; +use serde::Serialize; + +use crate::Session; + +/// Label of the oldest state in the history (the project as opened, or the oldest state kept). +pub const ROOT_LABEL: &str = "Original"; + +/// Undo history of whole-project snapshots. +#[derive(Clone, Default)] +pub struct History { + /// The states before each step on the working line, oldest first: (step label, state). + pub undo: Vec<(String, Arc)>, + /// Undone steps, the next one last: (step label, the state it leads to). + pub redo: Vec<(String, Arc)>, + /// Key of the last merged step: a continuous gesture with the same key folds into one step. + pub merge_key: Option, + /// Alternative futures kept when a new step was made after undoing. + pub branches: Vec, +} + +/// A line of states that grows from another state of the history. +#[derive(Clone)] +pub struct Branch { + /// The state the branch grows from. + pub parent: Arc, + /// The branch's states in order: (label of the step that made it, state). + pub steps: Vec<(String, Arc)>, +} + +/// One state of the history tree (`edit.history.list`). +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct HistoryNode { + /// Position in the listing (depth-first, the working line's continuation first). + pub index: usize, + /// Stable id of the state while the history keeps it. + pub id: String, + /// The step that made this state ([`ROOT_LABEL`] for the oldest). + pub label: String, + /// Index of the state it was made from. + pub parent: Option, + /// Branch nesting (0 = the working line). + pub depth: usize, + /// The current state. + pub current: bool, + /// On the working line (Undo / Redo walk it). + pub line: bool, + /// On the working line after the current state (Redo reaches it). + pub future: bool, +} + +fn key(p: &Arc) -> usize { + Arc::as_ptr(p) as usize +} + +/// Id of a state (its address while the history holds it). +pub fn state_id(p: &Arc) -> String { + format!("s{:x}", key(p)) +} + +/// The whole tree as an arena (internal). +struct Arena { + label: Vec, + state: Vec>, + parent: Vec>, + children: Vec>, + /// Index of each working-line state, oldest first, and the current state's position in it. + line: Vec, + current: usize, +} + +impl Arena { + fn build(h: &History, current: &Arc) -> Arena { + let mut a = Arena { label: vec![], state: vec![], parent: vec![], children: vec![], line: vec![], current: 0 }; + let add = |a: &mut Arena, label: String, st: Arc, parent: Option| { + let i = a.state.len(); + a.label.push(label); + a.state.push(st); + a.parent.push(parent); + a.children.push(vec![]); + if let Some(p) = parent { + a.children[p].push(i); + } + i + }; + // The working line: undo states, the current state, then the redo states. + let mut prev: Option = None; + let mut next_label = ROOT_LABEL.to_string(); + for (label, st) in &h.undo { + let i = add(&mut a, std::mem::take(&mut next_label), st.clone(), prev); + a.line.push(i); + prev = Some(i); + next_label = label.clone(); + } + let cur = add(&mut a, next_label, current.clone(), prev); + a.line.push(cur); + a.current = cur; + prev = Some(cur); + for (label, st) in h.redo.iter().rev() { + let i = add(&mut a, label.clone(), st.clone(), prev); + a.line.push(i); + prev = Some(i); + } + // Branches, attached to whichever state they grow from (branches of branches need the + // parent branch first, hence the passes). Orphans (their state was dropped) are skipped. + let mut index: HashMap = a.state.iter().enumerate().map(|(i, s)| (key(s), i)).collect(); + let mut pending: Vec<&Branch> = h.branches.iter().collect(); + loop { + let before = pending.len(); + pending.retain(|b| { + let Some(&p) = index.get(&key(&b.parent)) else { return true }; + let mut prev = p; + for (label, st) in &b.steps { + if index.contains_key(&key(st)) { + // Already in the tree (shared state): continue from it. + prev = index[&key(st)]; + continue; + } + let i = add(&mut a, label.clone(), st.clone(), Some(prev)); + index.insert(key(st), i); + prev = i; + } + false + }); + if pending.is_empty() || pending.len() == before { + break; + } + } + a + } + + fn on_line(&self) -> Vec { + let mut v = vec![false; self.state.len()]; + for &i in &self.line { + v[i] = true; + } + v + } + + /// Depth-first listing: the working line's child first, then branches in creation order. + fn listing(&self) -> Vec { + let on_line = self.on_line(); + let pos_in_line: HashMap = self.line.iter().enumerate().map(|(k, &i)| (i, k)).collect(); + let cur_pos = pos_in_line[&self.current]; + let mut out: Vec = vec![]; + let mut order: HashMap = HashMap::new(); + // (node, depth) + let mut stack: Vec<(usize, usize)> = vec![(self.line[0], 0)]; + while let Some((n, depth)) = stack.pop() { + order.insert(n, out.len()); + let line = on_line[n]; + out.push(HistoryNode { + index: out.len(), + id: state_id(&self.state[n]), + label: self.label[n].clone(), + parent: self.parent[n].and_then(|p| order.get(&p).copied()), + depth, + current: n == self.current, + line, + future: line && pos_in_line[&n] > cur_pos, + }); + // Push in reverse so the first child pops first; the line child continues at the + // same depth, branch children one deeper; a branch's own continuation (its first + // child) stays at the branch's depth. + for (k, &c) in self.children[n].iter().enumerate().rev() { + let cont = if line { on_line[c] } else { k == 0 }; + stack.push((c, if cont { depth } else { depth + 1 })); + } + } + out + } +} + +impl History { + /// Record a new step made from `before`: the redo line (if any) is kept as a branch. + pub fn record(&mut self, label: &str, before: Arc, levels: usize) { + self.abandon_redo(&before); + self.undo.push((label.to_string(), before.clone())); + self.merge_key = None; + self.trim(levels, &before); + } + + /// Keep the undone states as a branch growing from `from` (the state a new step starts from). + pub fn abandon_redo(&mut self, from: &Arc) { + if self.redo.is_empty() { + return; + } + let steps: Vec<(String, Arc)> = self.redo.drain(..).rev().collect(); + self.branches.push(Branch { parent: from.clone(), steps }); + } + + /// Apply the undo-levels limit: drop the oldest working-line states, the branches that grew + /// from them, and the oldest branches when branches hold more states than `levels`. + pub fn trim(&mut self, levels: usize, current: &Arc) { + let levels = levels.max(1); + if self.undo.len() > levels { + let extra = self.undo.len() - levels; + self.undo.drain(..extra); + } + let mut total: usize = self.branches.iter().map(|b| b.steps.len()).sum(); + while total > levels && !self.branches.is_empty() { + total -= self.branches.remove(0).steps.len(); + } + self.prune(current); + } + + /// Drop branches whose parent state is no longer in the history (`current` = the current + /// state, which the history doesn't store). + pub fn prune(&mut self, current: &Arc) { + if self.branches.is_empty() { + return; + } + let mut live: std::collections::HashSet = self.undo.iter().chain(self.redo.iter()).map(|(_, p)| key(p)).collect(); + live.insert(key(current)); + let mut keep = vec![false; self.branches.len()]; + loop { + let mut changed = false; + for (i, b) in self.branches.iter().enumerate() { + if !keep[i] && live.contains(&key(&b.parent)) { + keep[i] = true; + live.extend(b.steps.iter().map(|(_, p)| key(p))); + changed = true; + } + } + if !changed { + break; + } + } + let mut k = keep.into_iter(); + self.branches.retain(|_| k.next().unwrap_or(false)); + } + + /// Drop everything (Edit ▸ Purge ▸ Undo, New / Open Project). + pub fn clear(&mut self) { + *self = History::default(); + } + + /// The history as a tree, given the current state. + pub fn tree(&self, current: &Arc) -> Vec { + Arena::build(self, current).listing() + } + + /// Number of states on branches (not on the working line). + pub fn branch_states(&self) -> usize { + self.branches.iter().map(|b| b.steps.len()).sum() + } +} + +impl Session { + /// The undo history as a tree (the History panel, `edit.history.list`). + pub fn history_tree(&self) -> Vec { + self.history.tree(&self.project) + } + + /// Jump to any state of the history (`id` from [`HistoryNode::id`]), including states on + /// other branches: the path to it becomes the working line (its undone continuation the + /// redo line), and the line left behind is kept as a branch. Returns false for an unknown + /// id. + pub fn goto_history(&mut self, id: &str) -> bool { + let a = Arena::build(&self.history, &self.project); + let Some(target) = a.state.iter().position(|s| state_id(s) == id) else { return false }; + if target == a.current { + return true; + } + let on_line = a.on_line(); + // Path root → target. + let mut path = vec![target]; + while let Some(p) = a.parent[*path.last().unwrap_or(&target)] { + path.push(p); + } + path.reverse(); + // Continue past the target: the working line where it runs on, else each branch's own + // continuation (its first child). + let mut new_line = path.clone(); + let mut n = target; + loop { + let kids = &a.children[n]; + let next = if on_line[n] { kids.iter().copied().find(|&c| on_line[c]) } else { kids.first().copied() }; + match next { + Some(c) => { + new_line.push(c); + n = c; + } + None => break, + } + } + let mut in_new = vec![false; a.state.len()]; + for &i in &new_line { + in_new[i] = true; + } + // Every state off the new line goes into branches: a chain per first child, the other + // children starting branches of their own. + fn chain(a: &Arena, start: usize, parent: usize, in_new: &[bool], out: &mut Vec) { + let mut steps = vec![]; + let mut n = start; + let mut extra: Vec<(usize, usize)> = vec![]; + loop { + steps.push((a.label[n].clone(), a.state[n].clone())); + let kids: Vec = a.children[n].iter().copied().filter(|&c| !in_new[c]).collect(); + let Some((&first, rest)) = kids.split_first() else { break }; + extra.extend(rest.iter().map(|&c| (c, n))); + n = first; + } + out.push(Branch { parent: a.state[parent].clone(), steps }); + for (c, p) in extra { + chain(a, c, p, in_new, out); + } + } + let mut branches = vec![]; + for &n in &new_line { + for &c in &a.children[n] { + if !in_new[c] { + chain(&a, c, n, &in_new, &mut branches); + } + } + } + let k = new_line.iter().position(|&i| i == target).unwrap_or(0); + let undo: Vec<(String, Arc)> = (0..k).map(|i| (a.label[new_line[i + 1]].clone(), a.state[new_line[i]].clone())).collect(); + let redo: Vec<(String, Arc)> = new_line[k + 1..].iter().rev().map(|&i| (a.label[i].clone(), a.state[i].clone())).collect(); + self.project = a.state[target].clone(); + self.history.undo = undo; + self.history.redo = redo; + self.history.branches = branches; + self.history.merge_key = None; + self.sanitize_state(); + self.bump(); + true + } +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 9333329a..ad6ae4fb 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -13,6 +13,7 @@ pub mod camera_track; pub mod commands; pub mod config; pub mod demo; +pub mod history; pub mod links; pub mod mask_track; pub mod menus; @@ -21,6 +22,7 @@ pub mod prefs; pub mod psd_import; pub mod render_queue; pub mod roto; +pub mod scriptui; mod session_settings; pub mod shortcuts; pub mod tracking; @@ -47,6 +49,7 @@ pub use effectcraft_render as render; pub use effectcraft_text as text; pub use effectcraft_time as time; pub use effectcraft_track as track; +pub use history::{Branch, History, HistoryNode}; pub use render_queue::{ExportJob, ExportResult, Exporter, JobState}; #[derive(Debug, thiserror::Error)] @@ -90,15 +93,6 @@ pub trait Importer: Send + Sync { fn probe(&self, path: &str) -> std::result::Result; } -/// Undo history of whole-project snapshots. -#[derive(Clone, Default)] -pub struct History { - pub undo: Vec<(String, Arc)>, - pub redo: Vec<(String, Arc)>, - /// Key of the last merged step: a continuous gesture with the same key folds into one step. - pub merge_key: Option, -} - /// A keyframe reference: layer, property uid, key time (layer time). #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct KeyRef { @@ -320,6 +314,11 @@ pub struct Session { /// The JavaScript scripting engine (set by the host that links `effectcraft-script`): /// `script.run`, File ▸ Scripts ▸ Run Script File… (`.jsx`/`.js`) and the Script Console. pub script: Option, + /// Loads WebAssembly effect plug-ins (set by the host that links `effectcraft-plugin` with + /// its runtime): `effect.plugins.load`. + pub plugin_loader: Option, + /// ScriptUI windows and panels opened by scripts (see [`scriptui`]). + pub script_ui: scriptui::ScriptUi, /// Runs renders and analyses off the UI thread where there are no threads (the web app's /// Web Workers, see [`offload`]); `None` = threads (desktop) or inline (wasm32). pub offload: Option>, @@ -341,6 +340,10 @@ pub struct ScriptRequest<'a> { /// column, file} | null}`. pub type ScriptRunner = fn(&mut Session, &ScriptRequest) -> Value; +/// Loads one WebAssembly effect plug-in (`bytes`, `source` path) into the effect registry and +/// describes it (`{id, name, category, version, api, params}`). +pub type PluginLoader = fn(&[u8], &str) -> std::result::Result; + impl Default for Session { fn default() -> Self { Session { @@ -379,6 +382,8 @@ impl Default for Session { autosave: autosave::AutoSaveState::default(), snapshot: None, script: None, + plugin_loader: None, + script_ui: scriptui::ScriptUi::default(), offload: None, offloaded: vec![], } @@ -453,15 +458,13 @@ impl Session { } let same = merge.is_some() && merge.map(str::to_string) == self.history.merge_key; if !same { - self.history.undo.push((label.to_string(), before)); + // Undone steps aren't lost: they stay in the History panel as a branch. let levels = self.prefs.general.undo_levels.max(1) as usize; - if self.history.undo.len() > levels { - let extra = self.history.undo.len() - levels; - self.history.undo.drain(..extra); - } + self.history.record(label, before, levels); + } else { + self.history.abandon_redo(&before); } self.history.merge_key = merge.map(str::to_string); - self.history.redo.clear(); self.project = Arc::new(p); self.state = st; self.bump(); @@ -748,6 +751,8 @@ pub fn text_families() -> Vec { effectcraft_text::families().into_iter().map(|(f, _)| f).collect() } #[cfg(test)] +mod tests_history; +#[cfg(test)] mod tests_paint; #[cfg(test)] mod tests_roto; diff --git a/crates/engine/src/mask_track.rs b/crates/engine/src/mask_track.rs index 00315198..7d41fa71 100644 --- a/crates/engine/src/mask_track.rs +++ b/crates/engine/src/mask_track.rs @@ -247,9 +247,8 @@ impl Session { return self.offload_analysis(crate::offload::WorkerJob::MaskTrack { comp, layer, mask, direction, path, method, times }); } let wait = wait || cfg!(target_arch = "wasm32"); - self.history.undo.push(("Track Mask".into(), self.project.clone())); - self.history.redo.clear(); - self.history.merge_key = None; + let levels = self.prefs.general.undo_levels as usize; + self.history.record("Track Mask", self.project.clone(), levels); let shared = Arc::new(MaskTrackShared::default()); let (comp, layer) = (work.comp, work.layer); let thread = if wait { diff --git a/crates/engine/src/menus.rs b/crates/engine/src/menus.rs index 51fa4e02..7f14a988 100644 --- a/crates/engine/src/menus.rs +++ b/crates/engine/src/menus.rs @@ -280,7 +280,7 @@ pub fn parse(text: &str, mac: bool) -> Result, String> { /// Effect ▸ category submenus from the effect registry. fn effect_categories() -> Vec { let mut cats: Vec<(&str, Vec)> = vec![]; - for e in crate::effects::registry() { + for e in crate::effects::all() { let entry = MenuNode::Item(MenuEntry { label: e.name.into(), command: "effect.apply".into(), params: serde_json::json!({"effect": e.id}), shortcut: None }); match cats.iter_mut().find(|(c, _)| *c == e.category) { @@ -371,6 +371,8 @@ File Find Missing Footage | file.findMissing {"what":"footage"} --- Scripts + Install Script File... | file.installScript + Install ScriptUI Panel... | file.installScriptUIPanel Run Script File... | file.runScript --- Create Proxy diff --git a/crates/engine/src/scriptui.rs b/crates/engine/src/scriptui.rs new file mode 100644 index 00000000..2a76ba04 --- /dev/null +++ b/crates/engine/src/scriptui.rs @@ -0,0 +1,572 @@ +//! ScriptUI: the windows, dialogs and dockable panels scripts build with After Effects' +//! ScriptUI object model (`new Window("dialog", "Title")`, `win.add("button", undefined, "OK")`, +//! `orientation`, `alignChildren`, `onClick`/`onChange`, `show()`, `close()`, +//! `layout.layout()`…). +//! +//! The scripting engine (`effectcraft-script`) owns the live JavaScript objects; after every +//! script step it publishes a serde description of each open window here ([`ScriptWindow`], +//! a [`Widget`] tree with computed [`Widget::bounds`]). Frontends draw that description (the egui +//! UI renders dialogs, palettes and dockable panels from it) and report what the user does as +//! [`ScriptUiEvent`]s through the `scriptui.*` commands, which hand them to the script's +//! handlers ([`ScriptUi::dispatch`]). Agents use the same commands: list the open script +//! windows, read their widget trees, click buttons and set values. +//! +//! [`layout`] is ScriptUI's automatic layout (orientation, alignChildren / alignment, spacing, +//! margins, preferredSize), so every frontend places widgets the same way. + +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::{EngineError, Result, Session}; + +/// What kind of window a script made. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum WindowKind { + /// Modal: the script waits in `show()` until it closes. + #[default] + Dialog, + /// Floating, non-modal. + Palette, + /// A non-modal document window. + Window, + /// A dockable panel (a script from the ScriptUI Panels folder, opened from the Window menu). + Panel, +} + +/// ScriptUI control types (the `type` passed to `add()`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum WidgetKind { + /// The window itself (the tree's root). + #[default] + Window, + Panel, + Group, + Button, + IconButton, + StaticText, + EditText, + Checkbox, + RadioButton, + Slider, + Scrollbar, + Progressbar, + DropDownList, + ListBox, + TabbedPanel, + Tab, + Image, + /// A type we don't draw (it still lays out as an empty box). + #[serde(other)] + Unknown, +} + +impl WidgetKind { + pub fn is_container(self) -> bool { + matches!(self, WidgetKind::Window | WidgetKind::Panel | WidgetKind::Group | WidgetKind::TabbedPanel | WidgetKind::Tab) + } +} + +/// One control (or container) of a script window. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct Widget { + /// Unique within its window (the window itself is 0). + pub id: u32, + #[serde(rename = "type")] + pub kind: WidgetKind, + /// `properties.name` (agents can address controls by it). + pub name: String, + pub text: String, + /// Slider / scrollbar / progressbar value. + pub value: f64, + pub min: f64, + pub max: f64, + /// Checkbox / radio button state. + pub checked: bool, + /// Drop-down list / list box items. + pub items: Vec, + /// Selected item indices. + pub selection: Vec, + /// `column` (default for windows and panels), `row` (groups) or `stack`. + pub orientation: String, + /// `[horizontal, vertical]`: left / center / right / fill, top / center / bottom / fill. + pub align_children: Vec, + /// This control's own alignment in its parent (overrides the parent's alignChildren). + pub alignment: Vec, + pub spacing: Option, + /// `[left, top, right, bottom]`. + pub margins: Option<[f64; 4]>, + pub preferred_size: Option<[f64; 2]>, + /// Bounds set by the script (`[left, top, right, bottom]` relative to the parent). + pub fixed_bounds: Option<[f64; 4]>, + /// Laid-out bounds `[x, y, width, height]` relative to the window's content. + pub bounds: [f64; 4], + pub enabled: bool, + pub visible: bool, + pub help_tip: String, + pub multiline: bool, + pub read_only: bool, + /// Event handlers the script attached (`onClick`, `onChange`, `onChanging`…). + pub handlers: Vec, + /// The active tab of a tabbed panel (child index). + pub active_tab: usize, + pub children: Vec, +} + +impl Widget { + pub fn find(&self, id: u32) -> Option<&Widget> { + if self.id == id { + return Some(self); + } + self.children.iter().find_map(|c| c.find(id)) + } + pub fn find_mut(&mut self, id: u32) -> Option<&mut Widget> { + if self.id == id { + return Some(self); + } + self.children.iter_mut().find_map(|c| c.find_mut(id)) + } + /// Every control, depth first. + pub fn walk<'a>(&'a self, out: &mut Vec<&'a Widget>) { + out.push(self); + for c in &self.children { + c.walk(out); + } + } + /// A control by id (number or `"#12"`), `properties.name` or text. + pub fn lookup(&self, key: &Value) -> Option<&Widget> { + let mut all = vec![]; + self.walk(&mut all); + match key { + Value::Number(n) => n.as_u64().and_then(|n| self.find(n as u32)), + Value::String(s) => { + if let Some(n) = s.strip_prefix('#').and_then(|n| n.parse::().ok()) { + return self.find(n); + } + all.iter().find(|w| !w.name.is_empty() && w.name == *s).or_else(|| all.iter().find(|w| w.id != 0 && w.text == *s)).copied() + } + _ => None, + } + } +} + +/// An open script window. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct ScriptWindow { + /// Unique in the process. + pub id: u32, + /// The script context that owns it (its handlers run there). + pub host: u32, + pub kind: WindowKind, + pub title: String, + /// The script that made it. + pub script: String, + /// Shown and not closed. + pub visible: bool, + /// The script is waiting in this dialog's `show()`. + pub modal: bool, + /// The window and its controls (`root.bounds` is the content size). + pub root: Widget, +} + +/// A user (or agent) action on a script window control. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScriptUiEvent { + pub window: u32, + pub widget: u32, + /// `click`, `change`, `changing` or `close`. + pub kind: String, + /// The new value: text, number, bool, or selected index(es); for `close`, the result. + pub value: Value, +} + +/// Hands an event to the script that owns the window; returns what the handler reported +/// (`{ok, output, error}`) and updates [`Session::script_ui`]. +pub type ScriptUiDispatch = fn(&mut Session, &ScriptUiEvent) -> std::result::Result; + +/// The open script windows. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(default)] +pub struct ScriptUi { + pub windows: Vec, + /// Bumped whenever a window changes. + pub revision: u64, + /// The scripting engine's event hand-off (set with [`Session::script`]). + #[serde(skip)] + pub dispatch: Option, +} + +impl ScriptUi { + pub fn window(&self, id: u32) -> Option<&ScriptWindow> { + self.windows.iter().find(|w| w.id == id) + } + + /// Replace the windows a script host published (laying them out). + pub fn publish(&mut self, host: u32, mut windows: Vec) { + for w in &mut windows { + w.host = host; + layout(w); + } + windows.retain(|w| w.visible); + let mut out: Vec = vec![]; + // Keep the order windows first appeared in. + for old in self.windows.drain(..) { + if old.host != host { + out.push(old); + } else if let Some(i) = windows.iter().position(|w| w.id == old.id) { + out.push(windows.remove(i)); + } + } + out.extend(windows); + self.windows = out; + self.revision += 1; + } +} + +// ---------------------------------------------------------------- layout + +/// Average character width and line height of the default dialog font (points). +const CHAR_W: f64 = 7.0; +const LINE_H: f64 = 16.0; + +fn text_w(s: &str) -> f64 { + s.lines().map(|l| l.chars().count()).max().unwrap_or(0) as f64 * CHAR_W +} + +fn lines(s: &str) -> f64 { + s.lines().count().max(1) as f64 +} + +fn orientation(w: &Widget) -> &str { + match w.orientation.as_str() { + "" => match w.kind { + WidgetKind::Group => "row", + WidgetKind::TabbedPanel => "stack", + _ => "column", + }, + o => o, + } +} + +fn margins(w: &Widget) -> [f64; 4] { + w.margins.unwrap_or(match w.kind { + WidgetKind::Window => [15.0; 4], + WidgetKind::Panel => [10.0, 15.0, 10.0, 10.0], + WidgetKind::TabbedPanel => [0.0, 24.0, 0.0, 0.0], + WidgetKind::Tab => [10.0; 4], + _ => [0.0; 4], + }) +} + +fn spacing(w: &Widget) -> f64 { + w.spacing.unwrap_or(10.0) +} + +fn align_children(w: &Widget) -> (String, String) { + let d = match w.kind { + WidgetKind::Group => ("center", "center"), + _ => ("center", "top"), + }; + let a = |i: usize, d: &str| w.align_children.get(i).filter(|s| !s.is_empty()).cloned().unwrap_or_else(|| d.to_string()); + // A single value applies to the axis it names. + if w.align_children.len() == 1 { + let v = w.align_children[0].clone(); + return match v.as_str() { + "top" | "bottom" => (d.0.into(), v), + "fill" if orientation(w) == "row" => (d.0.into(), v), + _ => (v, d.1.into()), + }; + } + (a(0, d.0), a(1, d.1)) +} + +/// Preferred size of a control (its own content; containers include their children). +pub fn preferred(w: &Widget) -> [f64; 2] { + let ps = w.preferred_size.unwrap_or([0.0; 2]); + let content = if w.kind.is_container() { + let kids: Vec<[f64; 2]> = w.children.iter().filter(|c| c.visible).map(preferred).collect(); + let m = margins(w); + let sp = spacing(w); + let n = kids.len() as f64; + let (cw, ch) = match orientation(w) { + "row" => (kids.iter().map(|k| k[0]).sum::() + sp * (n - 1.0).max(0.0), kids.iter().map(|k| k[1]).fold(0.0, f64::max)), + "stack" => (kids.iter().map(|k| k[0]).fold(0.0, f64::max), kids.iter().map(|k| k[1]).fold(0.0, f64::max)), + _ => (kids.iter().map(|k| k[0]).fold(0.0, f64::max), kids.iter().map(|k| k[1]).sum::() + sp * (n - 1.0).max(0.0)), + }; + let title_w = if matches!(w.kind, WidgetKind::Panel) { text_w(&w.text) + 20.0 } else { 0.0 }; + [(cw + m[0] + m[2]).max(title_w), ch + m[1] + m[3]] + } else { + match w.kind { + WidgetKind::Button | WidgetKind::IconButton => [(text_w(&w.text) + 24.0).max(80.0), 25.0], + WidgetKind::StaticText => [text_w(&w.text) + 4.0, LINE_H * lines(&w.text)], + WidgetKind::EditText => { + if w.multiline { + [(text_w(&w.text) + 12.0).max(160.0), (LINE_H * lines(&w.text) + 8.0).max(60.0)] + } else { + [(text_w(&w.text) + 12.0).max(40.0), 22.0] + } + } + WidgetKind::Checkbox | WidgetKind::RadioButton => [text_w(&w.text) + 24.0, 18.0], + WidgetKind::Slider | WidgetKind::Scrollbar => [100.0, 22.0], + WidgetKind::Progressbar => [100.0, 10.0], + WidgetKind::DropDownList => [w.items.iter().map(|i| text_w(i)).fold(0.0, f64::max) + 36.0, 22.0], + WidgetKind::ListBox => [(w.items.iter().map(|i| text_w(i)).fold(0.0, f64::max) + 24.0).max(80.0), (w.items.len().clamp(3, 10) as f64 * 18.0 + 4.0)], + _ => [20.0, 20.0], + } + }; + [if ps[0] > 0.0 { ps[0] } else { content[0] }, if ps[1] > 0.0 { ps[1] } else { content[1] }] +} + +fn place(w: &mut Widget, x: f64, y: f64, width: f64, height: f64) { + w.bounds = [x, y, width, height]; + if !w.kind.is_container() { + return; + } + let m = margins(w); + let sp = spacing(w); + let (ah, av) = align_children(w); + let inner = [m[0], m[1], (width - m[0] - m[2]).max(0.0), (height - m[1] - m[3]).max(0.0)]; + let orient = orientation(w).to_string(); + let mut cursor = 0.0; + for c in w.children.iter_mut() { + if !c.visible { + c.bounds = [0.0; 4]; + continue; + } + if let Some(b) = c.fixed_bounds { + let (cx, cy, cw, ch) = (x + b[0], y + b[1], b[2] - b[0], b[3] - b[1]); + place(c, cx, cy, cw, ch); + continue; + } + let pref = preferred(c); + let h_align = c.alignment.first().filter(|s| !s.is_empty()).cloned().unwrap_or_else(|| ah.clone()); + let v_align = c.alignment.get(1).filter(|s| !s.is_empty()).cloned().unwrap_or_else(|| av.clone()); + let along = |a: &str, free: f64| match a { + "center" => free / 2.0, + "right" | "bottom" => free, + _ => 0.0, + }; + let (cx, cy, cw, ch) = match orient.as_str() { + "row" => { + let ch = if v_align == "fill" { inner[3] } else { pref[1] }; + let cy = inner[1] + along(&v_align, inner[3] - ch); + let r = (inner[0] + cursor, cy, pref[0], ch); + cursor += pref[0] + sp; + r + } + "stack" => { + let cw = if h_align == "fill" { inner[2] } else { pref[0] }; + let ch = if v_align == "fill" { inner[3] } else { pref[1] }; + (inner[0] + along(&h_align, inner[2] - cw), inner[1] + along(&v_align, inner[3] - ch), cw, ch) + } + _ => { + let cw = if h_align == "fill" { inner[2] } else { pref[0] }; + let cx = inner[0] + along(&h_align, inner[2] - cw); + let r = (cx, inner[1] + cursor, cw, pref[1]); + cursor += pref[1] + sp; + r + } + }; + place(c, x + cx, y + cy, cw, ch); + } + // Rows centre their content run as a whole when it is narrower than the row. + if orient == "row" && (ah == "center" || ah == "right") { + let used = (cursor - sp).max(0.0); + let shift = along_shift(&ah, inner[2] - used); + if shift > 0.0 { + for c in w.children.iter_mut().filter(|c| c.visible && c.fixed_bounds.is_none()) { + shift_all(c, shift, 0.0); + } + } + } + // Columns aligned to the bottom / centre move the whole run. + if orient == "column" && (av == "center" || av == "bottom") { + let used = (cursor - sp).max(0.0); + let shift = along_shift(&av, inner[3] - used); + if shift > 0.0 { + for c in w.children.iter_mut().filter(|c| c.visible && c.fixed_bounds.is_none()) { + shift_all(c, 0.0, shift); + } + } + } +} + +fn along_shift(a: &str, free: f64) -> f64 { + match a { + "center" => (free / 2.0).max(0.0), + "right" | "bottom" => free.max(0.0), + _ => 0.0, + } +} + +fn shift_all(w: &mut Widget, dx: f64, dy: f64) { + w.bounds[0] += dx; + w.bounds[1] += dy; + for c in &mut w.children { + shift_all(c, dx, dy); + } +} + +/// ScriptUI automatic layout: size the window to its preferred size (or the size the script +/// gave it) and place every control (`bounds` relative to the window's top-left). +pub fn layout(w: &mut ScriptWindow) { + let pref = preferred(&w.root); + let (width, height) = match w.root.fixed_bounds { + Some(b) if b[2] > b[0] && b[3] > b[1] => (b[2] - b[0], b[3] - b[1]), + _ => (pref[0], pref[1]), + }; + place(&mut w.root, 0.0, 0.0, width, height); +} + +// ---------------------------------------------------------------- commands + +fn window_p(s: &Session, p: &Value, cmd: &str) -> Result { + let key = p.get("window"); + let w = match key { + Some(Value::Number(n)) => n.as_u64().map(|n| n as u32).filter(|n| s.script_ui.window(*n).is_some()), + Some(Value::String(t)) => s.script_ui.windows.iter().find(|w| w.title == *t).map(|w| w.id), + None if s.script_ui.windows.len() == 1 => s.script_ui.windows.first().map(|w| w.id), + _ => None, + }; + w.ok_or_else(|| crate::commands::bad(cmd, "no such script window (give `window`: id or title from scriptui.list)")) +} + +fn widget_p(s: &Session, win: u32, p: &Value, cmd: &str) -> Result { + let w = s.script_ui.window(win).ok_or_else(|| crate::commands::bad(cmd, "window closed"))?; + let key = p.get("widget").ok_or_else(|| crate::commands::bad(cmd, "missing `widget` (id, `#id`, properties.name or text)"))?; + w.root.lookup(key).map(|w| w.id).ok_or_else(|| crate::commands::bad(cmd, format!("no control {key} in window {win}"))) +} + +fn dispatch(s: &mut Session, ev: ScriptUiEvent) -> Result { + let f = s.script_ui.dispatch.ok_or_else(|| EngineError::Other("scripting is not available in this build".into()))?; + f(s, &ev).map_err(EngineError::Other) +} + +/// `scriptui.list`: the open script windows. +pub(crate) fn list(s: &mut Session, _: &Value) -> Result { + Ok(json!( + s.script_ui + .windows + .iter() + .map(|w| json!({"window": w.id, "title": w.title, "kind": w.kind, "script": w.script, "modal": w.modal, "size": [w.root.bounds[2], w.root.bounds[3]]})) + .collect::>() + )) +} + +/// `scriptui.get`: one window's control tree. +pub(crate) fn get(s: &mut Session, p: &Value) -> Result { + let id = window_p(s, p, "scriptui.get")?; + Ok(serde_json::to_value(s.script_ui.window(id)).unwrap_or(Value::Null)) +} + +/// `scriptui.click`: press a button / toggle a checkbox / pick a radio button. +pub(crate) fn click(s: &mut Session, p: &Value) -> Result { + let c = "scriptui.click"; + let window = window_p(s, p, c)?; + let widget = widget_p(s, window, p, c)?; + let w = s.script_ui.window(window).and_then(|w| w.root.find(widget)).cloned().unwrap_or_default(); + if !w.enabled { + return Err(crate::commands::bad(c, format!("control {widget} is disabled"))); + } + dispatch(s, ScriptUiEvent { window, widget, kind: "click".into(), value: Value::Null }) +} + +/// `scriptui.set`: set a control's value (edit text, slider, checkbox, list selection) as the +/// user would, firing onChanging / onChange. +pub(crate) fn set(s: &mut Session, p: &Value) -> Result { + let c = "scriptui.set"; + let window = window_p(s, p, c)?; + let widget = widget_p(s, window, p, c)?; + let w = s.script_ui.window(window).and_then(|w| w.root.find(widget)).cloned().unwrap_or_default(); + if !w.enabled { + return Err(crate::commands::bad(c, format!("control {widget} is disabled"))); + } + let mut value = p.get("value").cloned().ok_or_else(|| crate::commands::bad(c, "missing `value`"))?; + // List selections by item text. + if matches!(w.kind, WidgetKind::DropDownList | WidgetKind::ListBox) + && let Value::String(t) = &value + { + let i = w.items.iter().position(|it| it == t).ok_or_else(|| crate::commands::bad(c, format!("no item `{t}` (items: {:?})", w.items)))?; + value = json!(i); + } + dispatch(s, ScriptUiEvent { window, widget, kind: "change".into(), value }) +} + +/// `scriptui.close`: close a window (a dialog's `show()` returns `result`, default 2 = Cancel). +pub(crate) fn close(s: &mut Session, p: &Value) -> Result { + let window = window_p(s, p, "scriptui.close")?; + let result = p.get("result").cloned().unwrap_or(json!(2)); + dispatch(s, ScriptUiEvent { window, widget: 0, kind: "close".into(), value: result }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn w(kind: WidgetKind, text: &str) -> Widget { + Widget { kind, text: text.into(), enabled: true, visible: true, ..Default::default() } + } + + #[test] + fn column_and_row_layout() { + let mut row = w(WidgetKind::Group, ""); + row.id = 1; + let mut ok = w(WidgetKind::Button, "OK"); + ok.id = 2; + let mut cancel = w(WidgetKind::Button, "Cancel"); + cancel.id = 3; + row.children = vec![ok, cancel]; + let mut label = w(WidgetKind::StaticText, "Name:"); + label.id = 4; + let mut root = w(WidgetKind::Window, "Dialog"); + root.children = vec![label, row]; + let mut win = ScriptWindow { id: 1, root, visible: true, ..Default::default() }; + layout(&mut win); + let r = &win.root; + // Window: 15 px margins, children stacked with 10 px spacing. + let row = r.find(1).unwrap(); + assert_eq!(row.bounds[2], 170.0, "two 80 px buttons + 10 spacing"); + assert_eq!(r.bounds[2], 200.0); + assert_eq!(r.find(4).unwrap().bounds[1], 15.0); + assert_eq!(row.bounds[1], 15.0 + 16.0 + 10.0); + // Centred (the default alignChildren of windows). + assert_eq!(row.bounds[0], 15.0); + let label = r.find(4).unwrap(); + assert!((label.bounds[0] + label.bounds[2] / 2.0 - 100.0).abs() < 1e-9); + assert_eq!(r.find(3).unwrap().bounds[0], 15.0 + 80.0 + 10.0); + // alignChildren left / fill. + win.root.align_children = vec!["fill".into(), "top".into()]; + layout(&mut win); + assert_eq!(win.root.find(4).unwrap().bounds[2], 170.0); + win.root.align_children = vec!["left".into()]; + layout(&mut win); + assert_eq!(win.root.find(4).unwrap().bounds[0], 15.0); + // preferredSize wins. + win.root.find_mut(2).unwrap().preferred_size = Some([120.0, 30.0]); + layout(&mut win); + assert_eq!(win.root.find(2).unwrap().bounds[2..], [120.0, 30.0]); + // Lookup by name / text / id. + win.root.find_mut(2).unwrap().name = "ok".into(); + assert_eq!(win.root.lookup(&json!("ok")).unwrap().id, 2); + assert_eq!(win.root.lookup(&json!("Cancel")).unwrap().id, 3); + assert_eq!(win.root.lookup(&json!("#4")).unwrap().id, 4); + assert_eq!(win.root.lookup(&json!(1)).unwrap().id, 1); + } + + #[test] + fn publish_replaces_a_hosts_windows() { + let mut ui = ScriptUi::default(); + let win = |id: u32, visible: bool| ScriptWindow { id, visible, root: w(WidgetKind::Window, ""), ..Default::default() }; + ui.publish(7, vec![win(1, true), win(2, true)]); + ui.publish(8, vec![win(3, true)]); + assert_eq!(ui.windows.iter().map(|w| (w.id, w.host)).collect::>(), [(1, 7), (2, 7), (3, 8)]); + ui.publish(7, vec![win(2, true), win(1, false)]); + assert_eq!(ui.windows.iter().map(|w| w.id).collect::>(), [2, 3]); + let j = serde_json::to_value(&ui).unwrap(); + assert_eq!(j["windows"][0]["root"]["type"], "window"); + } +} diff --git a/crates/engine/src/session_settings.rs b/crates/engine/src/session_settings.rs index 1a3370e4..155204d4 100644 --- a/crates/engine/src/session_settings.rs +++ b/crates/engine/src/session_settings.rs @@ -30,10 +30,8 @@ impl Session { self.configure_disk_cache(); // Fewer undo levels apply right away. let levels = self.prefs.general.undo_levels.max(1) as usize; - if self.history.undo.len() > levels { - let extra = self.history.undo.len() - levels; - self.history.undo.drain(..extra); - } + let current = self.project.clone(); + self.history.trim(levels, ¤t); self.prefs_revision += 1; } diff --git a/crates/engine/src/tests_history.rs b/crates/engine/src/tests_history.rs new file mode 100644 index 00000000..326c37c0 --- /dev/null +++ b/crates/engine/src/tests_history.rs @@ -0,0 +1,135 @@ +//! Branching history: undo then a new step keeps the undone line as a branch; the tree lists +//! every state; jumping to any state (on any branch) restores it and keeps everything else. + +use serde_json::{Value, json}; + +use crate::Session; + +fn names(s: &Session) -> Vec { + s.project.comps().next().map(|(_, c)| c.layers.iter().map(|l| l.name.clone()).collect()).unwrap_or_default() +} + +fn solid(s: &mut Session, name: &str) { + s.execute("layer.newSolid", json!({"name": name, "color": "#ff0000", "width": 10, "height": 10})).unwrap(); +} + +fn list(s: &mut Session) -> Vec { + s.execute("edit.history.list", json!({})).unwrap()["states"].as_array().cloned().unwrap() +} + +fn index_of(states: &[Value], label: &str, nth: usize) -> u64 { + states.iter().filter(|n| n["label"] == label).nth(nth).unwrap_or_else(|| panic!("no state {label} in {states:?}"))["index"].as_u64().unwrap() +} + +#[test] +fn undo_then_new_step_keeps_a_branch() { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "H", "width": 64, "height": 64, "duration": 2})).unwrap(); + solid(&mut s, "A"); + solid(&mut s, "B"); + solid(&mut s, "C"); + // Linear so far: Original, New Composition, 3 solids. + let st = list(&mut s); + assert_eq!(st.len(), 5, "{st:?}"); + assert!(st.iter().all(|n| n["depth"] == 0)); + assert_eq!(st.last().unwrap()["current"], true); + // Undo twice (B and C undone), then a new step: AE would forget B and C. + assert!(s.undo() && s.undo()); + let st = list(&mut s); + assert_eq!(st.iter().filter(|n| n["future"] == true).count(), 2); + solid(&mut s, "D"); + assert!(s.history.redo.is_empty(), "the new step clears Redo, as in After Effects"); + assert_eq!(s.history.branches.len(), 1); + assert_eq!(names(&s), ["D", "A"]); + let st = list(&mut s); + assert_eq!(st.len(), 6, "the undone states are still listed: {st:?}"); + let branch: Vec<&Value> = st.iter().filter(|n| n["depth"] == 1).collect(); + assert_eq!(branch.len(), 2, "B and C sit on a branch: {st:?}"); + // Jump to C on the old branch. + let c = st.iter().filter(|n| n["depth"] == 1).last().unwrap()["index"].as_u64().unwrap(); + let r = s.execute("edit.history.goto", json!({"index": c})).unwrap(); + assert_eq!(r["label"], "New Solid", "{r}"); + assert_eq!(names(&s), ["C", "B", "A"]); + // Now D is the branch, and Undo walks the B/C line. + assert_eq!(s.history.branches.len(), 1); + assert_eq!(s.history.branches[0].steps.len(), 1); + assert!(s.undo()); + assert_eq!(names(&s), ["B", "A"]); + assert!(s.redo()); + // ...and back to D by id. + let st = list(&mut s); + let d = st.iter().find(|n| n["depth"] == 1).unwrap()["id"].as_str().unwrap().to_string(); + s.execute("edit.history.goto", json!({"id": d})).unwrap(); + assert_eq!(names(&s), ["D", "A"]); + let st = list(&mut s); + assert_eq!(st.len(), 6); + assert_eq!(st.iter().filter(|n| n["current"] == true).count(), 1); + // The original state. + let root = index_of(&st, "Original", 0); + s.execute("edit.history.goto", json!({"index": root})).unwrap(); + assert!(s.project.comps().next().is_none()); + assert!(s.history.undo.is_empty()); + assert_eq!(list(&mut s).len(), 6, "nothing is lost by jumping"); + // Relative steps along the line. + s.execute("edit.history.goto", json!({"steps": 2})).unwrap(); + assert_eq!(names(&s), ["A"]); + assert!(s.execute("edit.history.goto", json!({"id": "nope"})).is_err()); + assert!(s.execute("edit.history.goto", json!({})).is_err()); +} + +#[test] +fn branches_of_branches_and_purge() { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "H", "width": 64, "height": 64, "duration": 2})).unwrap(); + solid(&mut s, "A"); + solid(&mut s, "B"); + s.undo(); + solid(&mut s, "C"); // branch 1: B + s.undo(); + solid(&mut s, "D"); // branch 2: C + let st = list(&mut s); + assert_eq!(st.len(), 6, "{st:?}"); + // Go to B, undo to A, make E: D's line becomes a branch, and a branch of the new line. + let b = st.iter().find(|n| n["depth"] == 1 && n["label"] == "New Solid").unwrap()["index"].as_u64().unwrap(); + s.execute("edit.history.goto", json!({"index": b})).unwrap(); + let which = names(&s); + assert!(which == ["B", "A"] || which == ["C", "A"], "{which:?}"); + s.undo(); + solid(&mut s, "E"); + let st = list(&mut s); + assert_eq!(st.len(), 7, "{st:?}"); + // Every state is reachable and restores its own layers. + let mut seen = std::collections::BTreeSet::new(); + for i in 0..st.len() { + s.execute("edit.history.goto", json!({"index": i})).unwrap(); + seen.insert(names(&s).join(",")); + assert_eq!(list(&mut s).len(), 7); + } + for want in ["", "A", "B,A", "C,A", "D,A", "E,A"] { + assert!(seen.contains(want), "{want:?} not reachable: {seen:?}"); + } + s.execute("edit.purgeUndo", json!({})).unwrap(); + assert_eq!(list(&mut s).len(), 1); +} + +#[test] +fn undo_levels_trim_branches() { + let mut s = Session::default(); + s.prefs.general.undo_levels = 3; + s.execute("comp.new", json!({"name": "H", "width": 64, "height": 64, "duration": 2})).unwrap(); + for n in ["A", "B", "C", "D", "E"] { + solid(&mut s, n); + } + assert_eq!(s.history.undo.len(), 3); + s.undo(); + s.undo(); + solid(&mut s, "F"); + // The branch (D, E) grew from a state still kept. + assert_eq!(s.history.branch_states(), 2); + for n in ["G", "H", "I"] { + solid(&mut s, n); + } + // Its parent fell off the end of the undo levels: the branch goes too. + assert_eq!(s.history.branch_states(), 0); + assert_eq!(list(&mut s).len(), 4); +} diff --git a/crates/engine/src/tests_paint.rs b/crates/engine/src/tests_paint.rs index 9926e04c..9407ddd1 100644 --- a/crates/engine/src/tests_paint.rs +++ b/crates/engine/src/tests_paint.rs @@ -250,3 +250,56 @@ fn liquify_stroke_adds_the_effect_and_warps() { assert!(layer(&s, id).effects().unwrap().groups().next().is_none()); assert!(px(&s, 0.0, 55, 50)[3] > 0.99); } + +fn pin_keys(s: &Session, id: u64, pin: u64) -> Vec<(f64, [f64; 2])> { + let l = layer(s, id); + let pos = l.props.find_group(pin).unwrap().get("position").unwrap().clone(); + pos.keys.iter().map(|k| (k.time.seconds(), k.value.as_vec2())).collect() +} + +#[test] +fn puppet_pin_recording_makes_keys_at_the_frame_rate() { + let (mut s, id) = setup(); + let a = s.execute("puppet.addPin", json!({"layer": id, "position": [10, 30]})).unwrap(); + s.execute("puppet.addPin", json!({"layer": id, "position": [90, 30]})).unwrap(); + let pin = a["pin"].as_u64().unwrap(); + s.set_time(Tick::from_seconds_f64(0.5)); + // A 0.5 s drag sampled at irregular wall-clock times: right 30 px, linear in time. + let samples: Vec = [0.0, 0.07, 0.2, 0.33, 0.41, 0.5].iter().map(|t| json!([t, 10.0 + 60.0 * t, 30.0])).collect(); + let r = s.execute("puppet.recordPin", json!({"layer": id, "pin": pin, "samples": samples, "smoothing": 0})).unwrap(); + // 0.5 s at 30 fps from 0.5 s: frames 15..=30, one key per frame. + assert_eq!(r["frames"], 16, "{r}"); + let keys = pin_keys(&s, id, pin); + let rec: Vec<&(f64, [f64; 2])> = keys.iter().filter(|k| k.0 >= 0.5 - 1e-9).collect(); + assert_eq!(rec.len(), 16, "{keys:?}"); + for (i, (t, v)) in rec.iter().enumerate() { + assert!((t - (0.5 + i as f64 / 30.0)).abs() < 1e-6, "{t}"); + assert!((v[0] - (10.0 + 60.0 * i as f64 / 30.0)).abs() < 1e-6, "{v:?}"); + } + assert_eq!(keys.len(), 17, "the pin's first key (0 s) is kept"); + assert_eq!(s.time().seconds(), 0.5, "the current time returns to where recording began"); + // The recording deforms the layer: by 1 s the left edge has followed the pin 30 px right. + assert!(px(&s, 0.0, 55, 50)[3] > 0.99); + assert!(px(&s, 1.0, 55, 50)[3] < 0.5); + // Speed 200 %: the same drag plays back over twice the frames; recording again replaces + // the keys in the span. + let r = s.execute("puppet.recordPin", json!({"layer": id, "pin": pin, "samples": samples, "speed": 200, "smoothing": 0})).unwrap(); + assert_eq!(r["frames"], 31); + let keys = pin_keys(&s, id, pin); + assert_eq!(keys.len(), 32); + let k = keys.iter().find(|k| (k.0 - (0.5 + 10.0 / 30.0)).abs() < 1e-6).unwrap(); + assert!((k.1[0] - (10.0 + 60.0 * 5.0 / 30.0)).abs() < 1e-6, "{k:?}"); + // Smoothing removes the extraneous keys of a straight-line drag. + let r = s.execute("puppet.recordPin", json!({"layer": id, "pin": pin, "samples": samples})).unwrap(); + assert!(r["keys"].as_u64().unwrap() < 6, "{r}"); + // One undo step per recording. + s.undo(); + assert_eq!(pin_keys(&s, id, pin).len(), 32); + // Record Options are tool state (serde for agents). + let o = s.execute("puppet.recordOptions", json!({"speed": 50, "smoothing": 0, "useDraftDeformation": true, "showMesh": true})).unwrap(); + assert_eq!(o["speed"], 50.0); + assert!(s.state.puppet.record_draft && s.state.puppet.record_show_mesh); + let r = s.execute("puppet.recordPin", json!({"layer": id, "pin": pin, "samples": samples})).unwrap(); + assert_eq!(r["frames"], 9, "50 %: half the frames"); + assert!(s.execute("puppet.recordPin", json!({"layer": id, "pin": pin, "samples": []})).is_err()); +} diff --git a/crates/engine/src/tracking.rs b/crates/engine/src/tracking.rs index b4863d54..a4386199 100644 --- a/crates/engine/src/tracking.rs +++ b/crates/engine/src/tracking.rs @@ -358,9 +358,8 @@ impl Session { return self.offload_analysis(crate::offload::WorkerJob::Track { comp, layer, tracker, direction, settings, points, times }); } let wait = wait || cfg!(target_arch = "wasm32"); - self.history.undo.push(("Analyze Track".into(), self.project.clone())); - self.history.redo.clear(); - self.history.merge_key = None; + let levels = self.prefs.general.undo_levels as usize; + self.history.record("Analyze Track", self.project.clone(), levels); let shared = Arc::new(TrackShared::default()); let (comp, layer) = (work.comp, work.layer); let thread = if wait { diff --git a/crates/host/Cargo.toml b/crates/host/Cargo.toml index 635e3260..b6e9ff55 100644 --- a/crates/host/Cargo.toml +++ b/crates/host/Cargo.toml @@ -16,6 +16,11 @@ effectcraft-media = { workspace = true } effectcraft-expr = { workspace = true } effectcraft-script = { workspace = true } effectcraft-export = { workspace = true } +effectcraft-plugin = { workspace = true } + +# WebAssembly effect plug-ins run in wasmi on native builds. +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +effectcraft-plugin = { workspace = true, features = ["wasm"] } [dev-dependencies] serde_json = { workspace = true } @@ -23,3 +28,4 @@ effectcraft-psd = { workspace = true } effectcraft-svg = { workspace = true } effectcraft-raster = { workspace = true } effectcraft-time = { workspace = true } +rayon = { workspace = true } diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index b5de7714..c379d091 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -58,6 +58,8 @@ pub fn session() -> Session { expr: Some(Arc::new(effectcraft_expr::Expressions)), expr_check: Some(effectcraft_expr::check_syntax), script: Some(effectcraft_script::runner), + script_ui: effectcraft_engine::scriptui::ScriptUi { dispatch: Some(effectcraft_script::dispatch_ui), ..Default::default() }, + plugin_loader: effectcraft_plugin::wasm_available().then_some(effectcraft_plugin::loader as effectcraft_engine::PluginLoader), ..Default::default() } } diff --git a/crates/host/tests/determinism.rs b/crates/host/tests/determinism.rs new file mode 100644 index 00000000..58a1b933 --- /dev/null +++ b/crates/host/tests/determinism.rs @@ -0,0 +1,74 @@ +//! Deterministic rendering (better than After Effects): the same project renders bit-identical +//! frames every time and whatever the number of render threads. Renders the demo comps (and a +//! comp of seeded-random effects: particles, grain, noise, wiggle expressions, Shatter) twice +//! on one thread and on several, and compares every float of every pixel. + +use effectcraft_engine::Session; +use effectcraft_engine::project::{ItemId, Project}; +use effectcraft_engine::render::{RenderOpts, Renderer}; +use effectcraft_engine::time::Tick; +use serde_json::json; + +fn render(s: &Session, p: &Project, comp: ItemId, t: f64) -> Vec { + let mut r = Renderer::new(p, s.footage.as_ref(), RenderOpts { scale: 0.25, ..Default::default() }); + r.expr = s.expr.as_deref(); + // No layer cache: every render computes everything. + let img = r.comp_frame(comp, Tick::from_seconds_f64(t)); + img.data.iter().flat_map(|px| px.iter().map(|c| c.to_bits())).collect() +} + +fn pool(n: usize) -> rayon::ThreadPool { + rayon::ThreadPoolBuilder::new().num_threads(n).build().expect("thread pool") +} + +/// A comp of effects that draw random numbers (all seeded per instance). +fn random_comp(s: &mut Session) -> ItemId { + s.execute("comp.new", json!({"name": "Random", "width": 480, "height": 270, "frameRate": 24, "duration": 4})).unwrap(); + let cid = s.active_comp_id().unwrap(); + let solid = |s: &mut Session, name: &str, color: &str| { + s.execute("layer.newSolid", json!({"name": name, "color": color, "width": 480, "height": 270})).unwrap()["layer"].as_u64().unwrap() + }; + let a = solid(s, "Particles", "#000000"); + for fx in ["ec.sim.ccparticleworld", "ec.noise.addgrain"] { + s.execute("effect.apply", json!({"layer": a, "effect": fx})).unwrap(); + } + let b = solid(s, "Noise", "#336699"); + for fx in ["ec.noise.fractal", "ec.sim.ccsnowfall", "ec.noise.noisehls"] { + s.execute("effect.apply", json!({"layer": b, "effect": fx})).unwrap(); + } + s.execute("prop.setExpression", json!({"layer": b, "path": "transform/position", "expression": "wiggle(3, 40)"})).unwrap(); + s.execute("prop.setExpression", json!({"layer": b, "path": "transform/opacity", "expression": "random(40, 90)"})).unwrap(); + let c = solid(s, "Shatter", "#cc8844"); + s.execute("layer.setTransform", json!({"layer": c, "prop": "scale", "value": [50, 50]})).unwrap(); + s.execute("effect.apply", json!({"layer": c, "effect": "ec.sim.shatter"})).unwrap(); + cid +} + +#[test] +fn renders_are_bit_identical_across_runs_and_thread_counts() { + let mut s = effectcraft_host::session(); + s.replace_project(effectcraft_engine::demo::demo_project(), None); + let random = random_comp(&mut s); + let p = (*s.project).clone(); + let mut comps: Vec<(ItemId, f64)> = p.comps().map(|(id, c)| (*id, c.duration.seconds())).collect(); + comps.sort_by_key(|c| c.0); + assert!(comps.len() >= 3, "demo comps + the random comp"); + assert!(comps.iter().any(|c| c.0 == random)); + let (one, many) = (pool(1), pool(6)); + let mut frames = 0; + for (comp, dur) in comps { + for t in [0.0, dur * 0.37, dur * 0.81] { + let a = one.install(|| render(&s, &p, comp, t)); + let b = many.install(|| render(&s, &p, comp, t)); + let c = many.install(|| render(&s, &p, comp, t)); + let d = one.install(|| render(&s, &p, comp, t)); + let name = &p.item(comp).unwrap().name; + assert!(!a.is_empty()); + assert!(a == b, "{name} @ {t:.2}s: 1 thread vs 6 threads differ"); + assert!(b == c, "{name} @ {t:.2}s: two 6-thread renders differ"); + assert!(a == d, "{name} @ {t:.2}s: two 1-thread renders differ"); + frames += 1; + } + } + assert!(frames >= 9); +} diff --git a/crates/plugin/Cargo.toml b/crates/plugin/Cargo.toml new file mode 100644 index 00000000..4081b61b --- /dev/null +++ b/crates/plugin/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "effectcraft-plugin" +description = "EffectCraft effect plug-ins: loads sandboxed WebAssembly effect modules (plug-in API v1) into the effect registry" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[features] +# The WebAssembly runtime (wasmi, a pure-Rust interpreter). Hosts enable it on native; without +# it `load_wasm` reports that WebAssembly plug-ins are unavailable. +wasm = ["dep:wasmi"] + +[dependencies] +effectcraft-effects = { workspace = true } +serde_json = { workspace = true } +wasmi = { workspace = true, optional = true } + +[dev-dependencies] +effectcraft-keyframe = { workspace = true } +effectcraft-project = { workspace = true } +effectcraft-raster = { workspace = true } diff --git a/crates/plugin/src/lib.rs b/crates/plugin/src/lib.rs new file mode 100644 index 00000000..1231d171 --- /dev/null +++ b/crates/plugin/src/lib.rs @@ -0,0 +1,66 @@ +//! # effectcraft-plugin (L3) +//! +//! Loads **WebAssembly effect plug-ins** (plug-in API v1, see `docs/plugins.md`) into the effect +//! registry ([`effectcraft_effects::plugin`]). A plug-in is a `.wasm` (or `.wat`) module with +//! no imports that exports: +//! +//! | export | signature | | +//! |---|---|---| +//! | `memory` | memory | linear memory shared with the host | +//! | `ec_api_version` | `() -> i32` | must return 1 | +//! | `ec_manifest_ptr`, `ec_manifest_len` | `() -> i32` | a UTF-8 JSON [`PluginManifest`] in memory | +//! | `ec_alloc` | `(bytes: i32) -> i32` | a buffer of at least `bytes` bytes, 8-byte aligned; it may reuse the previous one | +//! | `ec_render` | `(pixels: i32, width: i32, height: i32, params: i32, nparams: i32, time: f64, scale: f64) -> i32` | process the pixels in place; 0 = ok | +//! +//! For each render the host asks `ec_alloc` for one buffer holding the parameters (`nparams` +//! little-endian `f64`s) followed by the pixels (`width × height × 4` little-endian `f32`s, +//! premultiplied RGBA, row-major), calls `ec_render`, and reads the pixels back. +//! +//! Modules run in the [wasmi](https://github.com/wasmi-labs/wasmi) interpreter (pure Rust, +//! MIT/Apache-2.0) with deterministic floats, no imports (no files, network or clock) and a +//! fuel limit per frame, so a plug-in can't escape its sandbox, hang the app, or render +//! differently on another machine. The runtime is the `wasm` feature (on for the desktop app and +//! the CLI); without it [`load_wasm`] says plug-ins are unavailable. + +use effectcraft_effects::EffectSpec; +pub use effectcraft_effects::plugin::{EffectPlugin, PLUGIN_API_VERSION, PluginFrame, PluginManifest, PluginParams, register_plugin}; + +#[cfg(feature = "wasm")] +mod wasm; + +/// Whether this build can load WebAssembly plug-ins. +pub fn wasm_available() -> bool { + cfg!(feature = "wasm") +} + +/// Load and register a WebAssembly plug-in (`bytes` = a `.wasm` binary or `.wat` text; +/// `source` = where it came from, for listings). Returns the registered effect. +pub fn load_wasm(bytes: &[u8], source: &str) -> Result<&'static EffectSpec, String> { + #[cfg(feature = "wasm")] + { + let p = wasm::WasmPlugin::new(bytes, source)?; + register_plugin(std::sync::Arc::new(p)) + } + #[cfg(not(feature = "wasm"))] + { + let _ = (bytes, source); + Err("WebAssembly plug-ins are not available in this build".into()) + } +} + +/// [`load_wasm`] for the engine's plug-in loader hook: `{id, name, category, version, api}`. +pub fn loader(bytes: &[u8], source: &str) -> Result { + let spec = load_wasm(bytes, source)?; + let m = effectcraft_effects::plugin::plugin(spec.id).map(|p| p.manifest().clone()); + Ok(serde_json::json!({ + "id": spec.id, + "name": spec.name, + "category": spec.category, + "version": m.as_ref().map(|m| m.version.clone()).unwrap_or_default(), + "api": PLUGIN_API_VERSION, + "params": spec.params.len(), + })) +} + +#[cfg(all(test, feature = "wasm"))] +mod tests; diff --git a/crates/plugin/src/tests.rs b/crates/plugin/src/tests.rs new file mode 100644 index 00000000..9363ab3e --- /dev/null +++ b/crates/plugin/src/tests.rs @@ -0,0 +1,164 @@ +//! WebAssembly plug-ins: load, register, render through the effect registry, sandbox limits. + +use effectcraft_effects::{Buf, EffectCtx, EffectEnv, Image}; +use effectcraft_project::build::Ids; + +use crate::load_wasm; + +fn load_err(r: Result<&'static effectcraft_effects::EffectSpec, String>) -> String { + match r { + Err(e) => e, + Ok(s) => panic!("unexpectedly loaded {}", s.id), + } +} + +/// A plug-in in WebAssembly text: multiplies RGB by `gain`, adds `lift` × `tint` when `on`. +fn gain_module(id: &str, render_body: &str) -> String { + let manifest = format!( + r#"{{"api":1,"id":"{id}","name":"Gain {id}","category":"Test Plug-ins","version":"1.0","params":[{{"id":"gain","name":"Gain","type":"slider","default":0.5,"min":0,"max":4}},{{"id":"on","name":"Lift","type":"checkbox","default":false}},{{"id":"tint","name":"Tint","type":"color","default":[0,0,1,1]}}]}}"# + ); + let len = manifest.len(); + let escaped = manifest.replace('\\', "\\\\").replace('"', "\\\""); + format!( + r#"(module + (memory (export "memory") 2) + (data (i32.const 16) "{escaped}") + (func (export "ec_api_version") (result i32) i32.const 1) + (func (export "ec_manifest_ptr") (result i32) i32.const 16) + (func (export "ec_manifest_len") (result i32) i32.const {len}) + (func (export "ec_alloc") (param $n i32) (result i32) + (local $need i32) + local.get $n + i32.const 131071 + i32.add + i32.const 16 + i32.shr_u + memory.size + i32.sub + local.tee $need + i32.const 0 + i32.gt_s + if + local.get $need + memory.grow + drop + end + i32.const 65536) + (func (export "ec_render") (param $px i32) (param $w i32) (param $h i32) (param $params i32) (param $np i32) (param $t f64) (param $scale f64) (result i32) + (local $i i32) (local $end i32) (local $g f32) (local $on f64) + {render_body} + local.get $params + f64.load + f32.demote_f64 + local.set $g + local.get $params + f64.load offset=8 + local.set $on + local.get $px + local.get $w + local.get $h + i32.mul + i32.const 16 + i32.mul + i32.add + local.set $end + local.get $px + local.set $i + block $done + loop $l + local.get $i + local.get $end + i32.ge_u + br_if $done + local.get $i + local.get $i + f32.load + local.get $g + f32.mul + f32.store + local.get $i + local.get $i + f32.load offset=4 + local.get $g + f32.mul + f32.store offset=4 + local.get $i + local.get $i + f32.load offset=8 + local.get $g + f32.mul + local.get $on + local.get $params + f64.load offset=32 + f64.mul + f32.demote_f64 + f32.add + f32.store offset=8 + local.get $i + i32.const 16 + i32.add + local.set $i + br $l + end + end + i32.const 0))"# + ) +} + +fn render(spec: &effectcraft_effects::EffectSpec, gain: f64, on: bool) -> Image { + let mut next = 1; + let g = effectcraft_effects::instantiate(spec, &mut Ids(&mut next), spec.name, [4.0, 2.0]); + let mut params = effectcraft_effects::flatten_params(&g, &mut |p| p.value.clone()); + params.values.insert("gain".into(), effectcraft_keyframe::Value::Scalar(gain)); + params.values.insert("on".into(), effectcraft_keyframe::Value::Bool(on)); + let ctx = EffectCtx { params: ¶ms, time: 0.0, layer_size: [4.0, 2.0], seed: 1, adjustment: false, env: EffectEnv::default() }; + let buf = Buf { img: Image::filled(4, 2, [0.8, 0.6, 0.2, 1.0]), offset: [0.0; 2], scale: 1.0 }; + (spec.render)(&ctx, buf).img +} + +#[test] +fn wasm_plugin_registers_and_renders() { + let spec = load_wasm(gain_module("org.test.gain", "").as_bytes(), "gain.wat").unwrap(); + assert_eq!(spec.id, "org.test.gain"); + assert_eq!(spec.category, "Test Plug-ins"); + assert_eq!(spec.params.len(), 3); + // Found like a built-in, listed under its own category. + assert!(effectcraft_effects::find("org.test.gain").is_some()); + assert!(effectcraft_effects::lookup("Gain org.test.gain").is_some()); + assert!(effectcraft_effects::all().iter().any(|e| e.id == "org.test.gain")); + assert!(effectcraft_effects::categories().contains(&"Test Plug-ins")); + let img = render(spec, 0.5, false); + assert_eq!(img.get(1, 1), [0.4, 0.3, 0.1, 1.0]); + // The checkbox and colour parameters arrive flattened (tint blue = 1 → +1 on blue). + let img = render(spec, 1.0, true); + assert_eq!(img.get(3, 0), [0.8, 0.6, 1.2, 1.0]); + // Deterministic, and safe to run on many threads at once (each takes an instance). + let a: Vec = (0..8).map(|_| render(spec, 0.25, false)).collect(); + std::thread::scope(|sc| { + let hs: Vec<_> = (0..4).map(|_| sc.spawn(|| render(spec, 0.25, false))).collect(); + for h in hs { + assert_eq!(h.join().unwrap(), a[0]); + } + }); + assert!(a.iter().all(|i| *i == a[0])); + // The same id can't be registered twice. + assert!(load_err(load_wasm(gain_module("org.test.gain", "").as_bytes(), "again.wat")).contains("already registered")); +} + +#[test] +fn wasm_plugins_are_sandboxed() { + // A runaway loop runs out of fuel: the frame comes back unchanged instead of hanging. + let spec = load_wasm(gain_module("org.test.spin", "(loop $spin br $spin)").as_bytes(), "spin.wat").unwrap(); + let img = render(spec, 0.5, false); + assert_eq!(img.get(0, 0), [0.8, 0.6, 0.2, 1.0]); + // No imports (files, clock, network…). + let imports = r#"(module (import "env" "now" (func $now (result f64))) (memory (export "memory") 1))"#; + assert!(load_err(load_wasm(imports.as_bytes(), "imports.wat")).contains("import")); + // Garbage, wrong API version, bad manifests. + assert!(load_wasm(b"\0asm nope", "bad.wasm").is_err()); + let v2 = gain_module("org.test.v2", "") + .replace("(func (export \"ec_api_version\") (result i32) i32.const 1)", "(func (export \"ec_api_version\") (result i32) i32.const 2)"); + assert!(load_err(load_wasm(v2.as_bytes(), "v2.wat")).contains("API 2")); + let reserved = gain_module("ec.blur.mine", ""); + assert!(load_err(load_wasm(reserved.as_bytes(), "reserved.wat")).contains("reserved")); +} diff --git a/crates/plugin/src/wasm.rs b/crates/plugin/src/wasm.rs new file mode 100644 index 00000000..8015f7ee --- /dev/null +++ b/crates/plugin/src/wasm.rs @@ -0,0 +1,151 @@ +//! The wasmi-backed [`EffectPlugin`] (plug-in API v1 ABI, see the crate docs). + +use std::sync::Mutex; + +use wasmi::{Config, Engine, Instance, Linker, Memory, Module, Store, TypedFunc, WasmParams, WasmResults}; + +use crate::{EffectPlugin, PLUGIN_API_VERSION, PluginFrame, PluginManifest, PluginParams}; + +/// Instruction budget per frame: a fixed allowance plus some per pixel. Generous for per-pixel +/// work, but a runaway loop stops instead of hanging the render. +const FUEL_BASE: u64 = 50_000_000; +const FUEL_PER_PIXEL: u64 = 20_000; + +type RenderFn = TypedFunc<(i32, i32, i32, i32, i32, f64, f64), i32>; + +/// One instantiated module (renders running on several threads each take their own). +struct Inst { + store: Store<()>, + instance: Instance, + memory: Memory, + alloc: TypedFunc, + render: RenderFn, +} + +impl Inst { + fn func(&self, name: &str) -> Result, String> { + self.instance.get_typed_func::(&self.store, name).map_err(|_| missing(name)) + } +} + +fn missing(name: &str) -> String { + format!("the module exports no `{name}` (or with the wrong signature)") +} + +fn err(e: impl std::fmt::Display) -> String { + e.to_string() +} + +fn instantiate(engine: &Engine, module: &Module) -> Result { + let mut store = Store::new(engine, ()); + store.set_fuel(FUEL_BASE).map_err(err)?; + let linker = Linker::<()>::new(engine); + let instance = linker.instantiate_and_start(&mut store, module).map_err(|e| format!("instantiation failed: {e}"))?; + let memory = instance.get_memory(&store, "memory").ok_or("the module exports no `memory`")?; + let alloc = instance.get_typed_func::(&store, "ec_alloc").map_err(|_| missing("ec_alloc"))?; + let render = instance.get_typed_func::<(i32, i32, i32, i32, i32, f64, f64), i32>(&store, "ec_render").map_err(|_| missing("ec_render"))?; + Ok(Inst { store, instance, memory, alloc, render }) +} + +/// A loaded WebAssembly effect plug-in. +pub struct WasmPlugin { + manifest: PluginManifest, + source: String, + engine: Engine, + module: Module, + /// Idle instances. + pool: Mutex>, +} + +impl WasmPlugin { + pub fn new(bytes: &[u8], source: &str) -> Result { + let mut config = Config::default(); + config.consume_fuel(true); + let engine = Engine::new(&config); + let module = Module::new(&engine, bytes).map_err(|e| format!("not a valid WebAssembly module: {e}"))?; + if let Some(i) = module.imports().next() { + return Err(format!("plug-ins can't import anything (the module imports `{}.{}`)", i.module(), i.name())); + } + let mut inst = instantiate(&engine, &module)?; + let version = inst.func::<(), i32>("ec_api_version")?; + let v = version.call(&mut inst.store, ()).map_err(err)?; + if v != PLUGIN_API_VERSION as i32 { + return Err(format!("the module implements plug-in API {v}, this build implements API {PLUGIN_API_VERSION}")); + } + let ptr = inst.func::<(), i32>("ec_manifest_ptr")?.call(&mut inst.store, ()).map_err(err)?; + let len = inst.func::<(), i32>("ec_manifest_len")?.call(&mut inst.store, ()).map_err(err)?; + let mut text = vec![0u8; len.max(0) as usize]; + inst.memory.read(&inst.store, ptr as u32 as usize, &mut text).map_err(|e| format!("manifest out of bounds: {e}"))?; + let manifest: PluginManifest = serde_json::from_slice(&text).map_err(|e| format!("bad manifest JSON: {e}"))?; + manifest.validate()?; + Ok(WasmPlugin { manifest, source: source.to_string(), engine, module, pool: Mutex::new(vec![inst]) }) + } + + fn take(&self) -> Result { + if let Some(i) = self.pool.lock().ok().and_then(|mut p| p.pop()) { + return Ok(i); + } + instantiate(&self.engine, &self.module) + } + + fn give_back(&self, inst: Inst) { + if let Ok(mut p) = self.pool.lock() { + p.push(inst); + } + } +} + +impl EffectPlugin for WasmPlugin { + fn manifest(&self) -> &PluginManifest { + &self.manifest + } + + fn source(&self) -> String { + self.source.clone() + } + + fn render(&self, frame: &mut PluginFrame, params: &PluginParams, time: f64) -> Result<(), String> { + let n = frame.pixels.len(); + let params_bytes = params.flat.len() * 8; + let total = params_bytes + n * 16; + if total > i32::MAX as usize { + return Err("frame too large for a 32-bit plug-in".into()); + } + let mut inst = self.take()?; + inst.store.set_fuel(FUEL_BASE + FUEL_PER_PIXEL * n as u64).map_err(err)?; + let base = inst.alloc.call(&mut inst.store, total as i32).map_err(err)? as u32 as usize; + // Parameters, then pixels (little-endian, as WebAssembly memory is). + let mut buf: Vec = Vec::with_capacity(total); + for v in params.flat { + buf.extend_from_slice(&v.to_le_bytes()); + } + for px in frame.pixels.iter() { + for c in px { + buf.extend_from_slice(&c.to_le_bytes()); + } + } + inst.memory.write(&mut inst.store, base, &buf).map_err(|e| format!("ec_alloc returned a buffer out of bounds: {e}"))?; + let px_ptr = (base + params_bytes) as i32; + let r = inst + .render + .call(&mut inst.store, (px_ptr, frame.width as i32, frame.height as i32, base as i32, params.flat.len() as i32, time, frame.scale)) + .map_err(|e| format!("ec_render trapped: {e}")); + let r = match r { + Ok(0) => { + inst.memory.read(&inst.store, base + params_bytes, &mut buf[params_bytes..]).map_err(err)?; + for (px, b) in frame.pixels.iter_mut().zip(buf[params_bytes..].chunks_exact(16)) { + for (c, v) in px.iter_mut().zip(b.chunks_exact(4)) { + *c = f32::from_le_bytes([v[0], v[1], v[2], v[3]]); + } + } + Ok(()) + } + Ok(code) => Err(format!("ec_render returned error {code}")), + Err(e) => Err(e), + }; + if r.is_ok() { + self.give_back(inst); + } + r + } +} diff --git a/crates/script/src/lib.rs b/crates/script/src/lib.rs index a1e9e5d4..a7a85213 100644 --- a/crates/script/src/lib.rs +++ b/crates/script/src/lib.rs @@ -30,6 +30,14 @@ //! * **Security**: scripts can import/open/save projects and render, but `File` reads are limited //! to the project's folder and writes (and the network) are off unless Preferences ▸ Scripting //! & Expressions ▸ Allow Scripts to Write Files and Access Network is on. +//! * **ScriptUI** (`scriptui.js`, `ui.rs`): `Window` (dialog / palette / window), `Panel`, +//! `Group`, `Button`, `StaticText`, `EditText`, `Checkbox`, `RadioButton`, `Slider`, +//! `Progressbar`, `DropDownList`, `ListBox`, `TabbedPanel`/`Tab`, with `add()`, `orientation`, +//! `alignChildren`, `alignment`, `margins`, `spacing`, `preferredSize`, `onClick` / +//! `onChange` / `onChanging` / `onClose`, `show()` / `close()` and `layout.layout()`. Windows +//! are published to [`Session::script_ui`](effectcraft_engine::Session::script_ui) for +//! frontends and agents (`scriptui.*` commands); modal dialogs block `show()` until they close. +//! Scripts in the ScriptUI Panels folder run with `this` = a dockable `Panel`. //! //! Entry points: [`install`] sets [`Session::script`](effectcraft_engine::Session::script), which //! the `script.run` command, File ▸ Scripts ▸ Run Script File… (`.jsx`/`.js`), the Script Console @@ -40,19 +48,22 @@ pub mod matchnames; mod model; mod runtime; +mod ui; use effectcraft_engine::{ScriptRequest, Session}; pub use runtime::{Outcome, ScriptError, run}; +pub use ui::dispatch_ui; /// The [`effectcraft_engine::ScriptRunner`] this crate provides. pub fn runner(s: &mut Session, req: &ScriptRequest) -> serde_json::Value { run(s, req).to_json() } -/// Enable scripting on a session. +/// Enable scripting (and ScriptUI) on a session. pub fn install(s: &mut Session) { s.script = Some(runner); + s.script_ui.dispatch = Some(dispatch_ui); } /// Run `code` as a one-off script named `name`. @@ -62,3 +73,5 @@ pub fn run_code(s: &mut Session, code: &str, name: &str) -> Outcome { #[cfg(test)] mod tests; +#[cfg(test)] +mod tests_ui; diff --git a/crates/script/src/model.rs b/crates/script/src/model.rs index 03e80bb5..18270590 100644 --- a/crates/script/src/model.rs +++ b/crates/script/src/model.rs @@ -59,8 +59,8 @@ pub fn query(s: &mut Session, kind: &str, a: &J) -> R { } "labels" => Ok(json!(Label::ALL.iter().map(|l| l.name()).collect::>())), "effects" => Ok(json!( - effectcraft_engine::effects::registry() - .iter() + effectcraft_engine::effects::all() + .into_iter() .map(|e| json!({"id": e.id, "name": e.name, "matchName": matchnames::effect(e.id), "category": e.category})) .collect::>() )), diff --git a/crates/script/src/runtime.rs b/crates/script/src/runtime.rs index 61b49595..750918f4 100644 --- a/crates/script/src/runtime.rs +++ b/crates/script/src/runtime.rs @@ -26,6 +26,9 @@ pub struct Outcome { /// `writeLn` / `$.writeln` / `alert` output, one entry per line. pub output: Vec, pub error: Option, + /// The script is waiting in a modal ScriptUI dialog's `show()` (it goes on when the dialog + /// closes; see [`crate::dispatch_ui`]). + pub waiting: bool, } /// An uncaught script error. @@ -42,36 +45,49 @@ impl Outcome { self.error.is_none() } pub fn to_json(&self) -> J { - json!({ + let mut j = json!({ "ok": self.ok(), "result": self.result, "output": self.output.join("\n"), "error": self.error.as_ref().map(|e| json!({"message": e.message, "line": e.line, "column": e.column, "file": e.file})), - }) + }); + if self.waiting { + j["waiting"] = json!(true); + } + j } } /// The session and per-run state while a script runs (the natives reach it here; nothing /// borrowed ever enters the JS heap). -struct Active { - session: Session, - output: Vec, +pub(crate) struct Active { + pub(crate) session: Session, + pub(crate) output: Vec, /// Open undo groups: (name, project snapshot when the outermost group began). - groups: Vec<(String, Arc)>, + pub(crate) groups: Vec<(String, Arc)>, /// `app.scheduleTask` queue: (id, code, repeat). - tasks: Vec<(u32, String, bool)>, - next_task: u32, - name: String, + pub(crate) tasks: Vec<(u32, String, bool)>, + pub(crate) next_task: u32, + pub(crate) name: String, + /// Running on a script-host thread: modal dialogs hand the session back to the caller + /// through this link while they wait. + pub(crate) link: Option, +} + +impl Active { + pub(crate) fn new(session: Session, name: &str) -> Active { + Active { session, output: vec![], groups: vec![], tasks: vec![], next_task: 0, name: name.to_string(), link: None } + } } thread_local! { - static ACTIVE: RefCell> = const { RefCell::new(None) }; + pub(crate) static ACTIVE: RefCell> = const { RefCell::new(None) }; /// The Script Console's persistent context (never dropped, like the expression runtime: boa's /// own thread-local heap may be torn down first at thread exit). - static CONSOLE: RefCell>> = const { RefCell::new(None) }; + pub(crate) static CONSOLE: RefCell>> = const { RefCell::new(None) }; } -fn with_active(f: impl FnOnce(&mut Active) -> Result) -> Result { +pub(crate) fn with_active(f: impl FnOnce(&mut Active) -> Result) -> Result { ACTIVE.with(|a| match a.try_borrow_mut() { Ok(mut g) => match g.as_mut() { Some(act) => f(act), @@ -81,11 +97,11 @@ fn with_active(f: impl FnOnce(&mut Active) -> Result) -> Result) -> boa_engine::JsError { +pub(crate) fn throw(msg: impl Into) -> boa_engine::JsError { JsNativeError::error().with_message(msg.into()).into() } -fn arg_string(args: &[JsValue], i: usize, ctx: &mut Context) -> JsResult { +pub(crate) fn arg_string(args: &[JsValue], i: usize, ctx: &mut Context) -> JsResult { Ok(args.get(i).cloned().unwrap_or_default().to_string(ctx)?.to_std_string_escaped()) } @@ -97,7 +113,7 @@ fn arg_json(args: &[JsValue], i: usize, ctx: &mut Context) -> JsResult { serde_json::from_str(&s).map_err(|e| throw(format!("bad parameters: {e}"))) } -fn js_str(s: &str) -> JsValue { +pub(crate) fn js_str(s: &str) -> JsValue { JsValue::from(JsString::from(s)) } @@ -205,7 +221,7 @@ fn native_undo(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult) { +pub(crate) fn collapse(s: &mut Session, name: &str, snap: &Arc) { let h = &mut s.history; let Some(start) = h.undo.iter().rposition(|(_, p)| Arc::ptr_eq(p, snap)) else { // Nothing was recorded (or history was reset by New / Open Project). @@ -349,28 +365,32 @@ fn native_file(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult Result { +pub(crate) fn new_context() -> Result { let mut ctx = Context::default(); let limits = ctx.runtime_limits_mut(); limits.set_loop_iteration_limit(200_000_000); limits.set_recursion_limit(2000); - let natives: [(JsString, usize, fn(&JsValue, &[JsValue], &mut Context) -> JsResult); 6] = [ + let natives: [(JsString, usize, fn(&JsValue, &[JsValue], &mut Context) -> JsResult); 9] = [ (js_string!("__exec"), 2, native_exec), (js_string!("__query"), 2, native_query), (js_string!("__print"), 1, native_print), (js_string!("__undo"), 2, native_undo), (js_string!("__task"), 3, native_task), (js_string!("__file"), 3, native_file), + (js_string!("__uiNewId"), 0, crate::ui::native_new_id), + (js_string!("__uiModal"), 1, crate::ui::native_modal), + (js_string!("__uiLayoutNative"), 1, crate::ui::native_layout), ]; for (name, len, f) in natives { ctx.register_global_callable(name, len, NativeFunction::from_fn_ptr(f)).map_err(|e| e.to_string())?; } ctx.eval(Source::from_bytes(PRELUDE)).map_err(|e| format!("script prelude: {e}"))?; + ctx.eval(Source::from_bytes(crate::ui::SCRIPTUI)).map_err(|e| format!("ScriptUI prelude: {e}"))?; Ok(ctx) } /// Turn a boa error into a message with line and column (`code` is the user's script). -fn script_error(e: &boa_engine::JsError, ctx: &mut Context, file: &str, code: &str) -> ScriptError { +pub(crate) fn script_error(e: &boa_engine::JsError, ctx: &mut Context, file: &str, code: &str) -> ScriptError { // Thrown values: `throw new Error("x")`, `throw "x"`, native errors. let display = e.to_string(); let message = match e.as_opaque() { @@ -442,7 +462,7 @@ fn position(msg: &str, file: &str, code: &str) -> (Option, Option) { } /// Evaluate `code` in `ctx` and convert the result. -fn eval_in(ctx: &mut Context, code: &str, file: &str) -> Result { +pub(crate) fn eval_in(ctx: &mut Context, code: &str, file: &str) -> Result { let src = Source::from_bytes(code.as_bytes()).with_path(std::path::Path::new(file)); let script = boa_engine::Script::parse(src, None, ctx).map_err(|e| script_error(&e, ctx, file, code))?; let v = script.evaluate(ctx).map_err(|e| script_error(&e, ctx, file, code))?; @@ -466,8 +486,8 @@ fn result_json(v: &JsValue, ctx: &mut Context) -> J { } /// Restores the caller's session when a run ends (on every exit path). -struct Restore<'a> { - target: &'a mut Session, +pub(crate) struct Restore<'a> { + pub(crate) target: &'a mut Session, } impl Drop for Restore<'_> { @@ -478,8 +498,54 @@ impl Drop for Restore<'_> { } } +/// End of a script step: close undo groups left open, drop selections that no longer exist, +/// and take the output so far. +pub(crate) fn settle() -> Vec { + with_active(|a| { + if let Some((name, snap)) = a.groups.first().cloned() { + collapse(&mut a.session, &name, &snap); + a.groups.clear(); + } + a.session.sanitize_state(); + Ok(std::mem::take(&mut a.output)) + }) + .unwrap_or_default() +} + +/// Evaluate a script file's code, then the one-shot tasks it scheduled (`app.scheduleTask`). +pub(crate) fn eval_with_tasks(ctx: &mut Context, code: &str, file: &str) -> Result { + let mut r = eval_in(ctx, code, file); + let tasks = with_active(|a| Ok(std::mem::take(&mut a.tasks))).unwrap_or_default(); + for (_, code, repeat) in tasks { + if repeat { + continue; + } + if let Err(e) = eval_in(ctx, &code, "scheduled task") + && r.is_ok() + { + r = Err(e); + } + } + r +} + +pub(crate) fn outcome(output: Vec, result: Result) -> Outcome { + let mut out = Outcome { output, ..Default::default() }; + match result { + Ok(v) => out.result = v, + Err(e) => out.error = Some(e), + } + out +} + /// Run a script against `session`: the object model sees and edits that session through engine /// commands. Script errors come back in [`Outcome::error`]. +/// +/// On native builds a script file runs on its own script-host thread, so a ScriptUI dialog's +/// `show()` can wait for the user (the session comes back here meanwhile, and the dialog's +/// events reach the host through [`crate::dispatch_ui`]); palettes and panels keep their host +/// alive for their handlers. The Script Console (and the web build) run inline, where dialogs +/// don't block. pub fn run(session: &mut Session, req: &ScriptRequest) -> Outcome { let running = ACTIVE.with(|a| a.try_borrow().map(|g| g.is_some()).unwrap_or(true)); if running { @@ -488,59 +554,49 @@ pub fn run(session: &mut Session, req: &ScriptRequest) -> Outcome { ..Default::default() }; } + #[cfg(not(target_arch = "wasm32"))] + if !req.console { + return crate::ui::run_threaded(session, req); + } + run_inline(session, req) +} + +/// Run on this thread (the Script Console's persistent context, or a fresh one). +pub(crate) fn run_inline(session: &mut Session, req: &ScriptRequest) -> Outcome { + let host = if req.console { crate::ui::CONSOLE_HOST } else { crate::ui::new_host_id() }; let s = std::mem::take(session); - ACTIVE.with(|a| *a.borrow_mut() = Some(Active { session: s, output: vec![], groups: vec![], tasks: vec![], next_task: 0, name: req.name.to_string() })); + ACTIVE.with(|a| *a.borrow_mut() = Some(Active::new(s, req.name))); let guard = Restore { target: session }; let file = if req.name.is_empty() { "script" } else { req.name }; - let mut result = if req.console { + let (result, windows) = if req.console { CONSOLE.with(|c| { let mut c = c.borrow_mut(); if c.is_none() { match new_context() { Ok(ctx) => *c = Some(ManuallyDrop::new(ctx)), - Err(e) => return Err(ScriptError { message: e, file: file.into(), ..Default::default() }), + Err(e) => return (Err(ScriptError { message: e, file: file.into(), ..Default::default() }), vec![]), } } let ctx = c.as_mut().map(|m| &mut **m).expect("console context"); - eval_in(ctx, req.code, file) + let r = eval_in(ctx, req.code, file); + (r, crate::ui::snapshot(ctx)) }) } else { match new_context() { Ok(mut ctx) => { - let r = eval_in(&mut ctx, req.code, file); - // app.scheduleTask: run the queued one-shot tasks after the script, in order. - let tasks = with_active(|a| Ok(std::mem::take(&mut a.tasks))).unwrap_or_default(); - let mut r = r; - for (_, code, repeat) in tasks { - if repeat { - continue; - } - if let Err(e) = eval_in(&mut ctx, &code, "scheduled task") - && r.is_ok() - { - r = Err(e); - } + let r = eval_with_tasks(&mut ctx, req.code, file); + let windows = crate::ui::snapshot(&mut ctx); + if windows.iter().any(|w| w.visible) { + // Palettes and panels outlive the run: keep their context for the handlers. + crate::ui::keep_inline(host, ctx); } - r + (r, windows) } - Err(e) => Err(ScriptError { message: e, file: file.into(), ..Default::default() }), + Err(e) => (Err(ScriptError { message: e, file: file.into(), ..Default::default() }), vec![]), } }; - // Close undo groups left open; drop selections that no longer exist. - let output = with_active(|a| { - if let Some((name, snap)) = a.groups.first().cloned() { - collapse(&mut a.session, &name, &snap); - a.groups.clear(); - } - a.session.sanitize_state(); - Ok(std::mem::take(&mut a.output)) - }) - .unwrap_or_default(); + let output = settle(); drop(guard); - let mut out = Outcome { output, ..Default::default() }; - match result.as_mut() { - Ok(v) => out.result = std::mem::take(v), - Err(e) => out.error = Some(e.clone()), - } - out + crate::ui::publish(session, host, req.name, windows); + outcome(output, result) } diff --git a/crates/script/src/scriptui.js b/crates/script/src/scriptui.js new file mode 100644 index 00000000..e214be30 --- /dev/null +++ b/crates/script/src/scriptui.js @@ -0,0 +1,544 @@ +// EffectCraft scripting: ScriptUI, the dialog / panel object model After Effects scripts use +// (`new Window("dialog", "Title")`, `win.add("button", undefined, "OK")`, `onClick`, `show()`…). +// +// Written from the behaviour Adobe's public JavaScript Tools Guide documents. Controls are plain +// JavaScript objects; after each script step the host reads `__uiSnapshot()` (a JSON +// description of the open windows) and lays them out, frontends draw them, and user actions come +// back through `__uiDispatch(window, control, kind, valueJson)`. Modal dialogs block in +// `show()` (`__uiModal`) until they close, as in After Effects. + +var __uiWins = []; +// show() results of closed dialogs, by window id. +var __uiResults = {}; + +function __uiModalResult(id) { + var r = __uiResults[id]; + if (r !== undefined) { + delete __uiResults[id]; + return r; + } + return __uiWin(id) ? null : 2; +} + +function __dim(w, h) { + var d = [w, h]; + Object.defineProperty(d, "width", { get: function () { return this[0]; }, set: function (v) { this[0] = v; } }); + Object.defineProperty(d, "height", { get: function () { return this[1]; }, set: function (v) { this[1] = v; } }); + return d; +} +function __bounds4(l, t, r, b) { + var a = [l, t, r, b]; + Object.defineProperty(a, "left", { get: function () { return this[0]; } }); + Object.defineProperty(a, "top", { get: function () { return this[1]; } }); + Object.defineProperty(a, "right", { get: function () { return this[2]; } }); + Object.defineProperty(a, "bottom", { get: function () { return this[3]; } }); + Object.defineProperty(a, "x", { get: function () { return this[0]; } }); + Object.defineProperty(a, "y", { get: function () { return this[1]; } }); + Object.defineProperty(a, "width", { get: function () { return this[2] - this[0]; } }); + Object.defineProperty(a, "height", { get: function () { return this[3] - this[1]; } }); + return a; +} +function __toBounds(b) { + if (b === undefined || b === null) return null; + if (b.length === 4) return [__num(b[0]), __num(b[1]), __num(b[2]), __num(b[3])]; + if (b.width !== undefined) { + var x = b.x || b.left || 0, y = b.y || b.top || 0; + return [x, y, x + __num(b.width), y + __num(b.height)]; + } + return null; +} + +var __uiTypes = { + window: "Window", panel: "Panel", group: "Group", button: "Button", iconbutton: "IconButton", + statictext: "StaticText", edittext: "EditText", checkbox: "Checkbox", radiobutton: "RadioButton", + slider: "Slider", scrollbar: "Scrollbar", progressbar: "Progressbar", dropdownlist: "DropDownList", + listbox: "ListBox", tabbedpanel: "TabbedPanel", tab: "Tab", image: "ScriptUIImage" +}; +var __uiContainers = { window: 1, panel: 1, group: 1, tabbedpanel: 1, tab: 1 }; + +function Window(type, title, bounds, props) { + if (!(this instanceof Window)) return new Window(type, title, bounds, props); + type = String(type === undefined ? "window" : type).toLowerCase(); + if (type.indexOf("{") >= 0) throw __err("ScriptUI resource strings are not supported: build the window with add()"); + if (type !== "dialog" && type !== "palette" && type !== "window") throw __err("Bad window type " + type); + __uiInit(this, "window", null, null, bounds, title, props); + this.__kind = type; + this.__root = true; + __uiWins.push(this); +} +Window.alert = function (s) { alert(s); }; +Window.confirm = function (s) { return confirm(s); }; +Window.prompt = function (m, d) { return prompt(m, d); }; +Window.find = function (type, title) { + for (var i = 0; i < __uiWins.length; i++) if (__uiWins[i].text === title) return __uiWins[i]; + return null; +}; + +function Panel() {} +function Group() {} +function Button() {} +function IconButton() {} +function StaticText() {} +function EditText() {} +function Checkbox() {} +function RadioButton() {} +function Slider() {} +function Scrollbar() {} +function Progressbar() {} +function DropDownList() {} +function ListBox() {} +function TabbedPanel() {} +function Tab() {} +function ScriptUIImage() {} +function ListItem() {} + +var ScriptUI = { + version: "6.2.2", + frameworkName: "EffectCraft", + environment: { keyboardState: { shiftKey: false, ctrlKey: false, altKey: false, metaKey: false, keyName: "" } }, + FontStyle: { REGULAR: 0, BOLD: 1, ITALIC: 2, BOLDITALIC: 3 }, + BrushType: { SOLID_COLOR: 0, THEME_COLOR: 1 }, + PenType: { SOLID_COLOR: 0, THEME_COLOR: 1 }, + Alignment: { LEFT: "left", RIGHT: "right", TOP: "top", BOTTOM: "bottom", CENTER: "center", FILL: "fill" }, + newFont: function (name, style, size) { return { name: name, style: style, size: size }; }, + newImage: function () { return {}; }, + getResourceText: function (t) { return t; }, + events: { createEvent: function (t) { return { type: t }; } } +}; + +function __uiGraphics() { + return { + newBrush: function (t, c) { return { type: t, color: c }; }, + newPen: function (t, c, w) { return { type: t, color: c, lineWidth: w }; }, + newPath: function () {}, moveTo: function () {}, lineTo: function () {}, rectPath: function () {}, + ellipsePath: function () {}, fillPath: function () {}, strokePath: function () {}, closePath: function () {}, + drawString: function () {}, drawOSControl: function () {}, drawImage: function () {}, drawFocusRing: function () {}, + measureString: function (s) { return __dim(String(s).length * 7, 16); }, + font: null, foregroundColor: null, backgroundColor: null, disabledForegroundColor: null, disabledBackgroundColor: null + }; +} + +function __uiInit(o, type, win, parent, bounds, text, props) { + o.type = type; + o.window = win || o; + o.parent = parent || null; + o.__id = win ? win.__next++ : 0; + if (!win) { + o.__next = 1; + o.__wid = __uiNewId(); + } + o.properties = props || {}; + o.text = text === undefined || text === null ? "" : String(text); + o.enabled = true; + o.visible = true; + o.helpTip = ""; + o.children = []; + o.__ub = __toBounds(bounds); + o.__lb = [0, 0, 0, 0]; + o.preferredSize = __dim(-1, -1); + o.minimumSize = __dim(0, 0); + o.maximumSize = __dim(10000, 10000); + o.alignment = null; + o.graphics = __uiGraphics(); + o.__listeners = {}; + if (o.properties.name) o.name = o.properties.name; + if (__uiContainers[type]) { + o.orientation = type === "group" ? "row" : type === "tabbedpanel" ? "stack" : "column"; + o.alignChildren = type === "group" ? ["center", "center"] : ["center", "top"]; + o.spacing = 10; + o.margins = type === "window" ? [15, 15, 15, 15] : type === "panel" ? [10, 15, 10, 10] : type === "tab" ? [10, 10, 10, 10] : [0, 0, 0, 0]; + var self = o; + o.layout = { + layout: function () { __uiLayout(self.window); }, + resize: function () { __uiLayout(self.window); }, + margins: o.margins, + spacing: o.spacing + }; + } + if (type === "checkbox" || type === "radiobutton") o.value = !!o.properties.value; + if (type === "slider" || type === "scrollbar" || type === "progressbar") { + o.value = 0; + o.minvalue = 0; + o.maxvalue = 100; + } + if (type === "edittext") { + o.characters = 0; + o.textselection = ""; + o.active = false; + } + if (type === "dropdownlist" || type === "listbox") { + o.items = []; + o.__sel = []; + var it = o.properties.items; + if (it) for (var i = 0; i < it.length; i++) o.add("item", it[i]); + } + if (type === "tabbedpanel") o.__sel = []; + return o; +} + +function __uiProto() { + return { + add: function (type, bounds, text, props) { + var t = String(type).toLowerCase(); + if (t === "item" || t === "separator") return this.__addItem(t === "separator" ? "-" : bounds); + if (!__uiContainers[this.type]) throw __err("add(): a " + this.type + " can't contain controls"); + var ctor = __uiTypes[t]; + if (!ctor || t === "window") throw __err("add(): unknown control type " + type); + var c = Object.create(__ecGlobal[ctor].prototype); + // Value arguments after `text`: slider (value, min, max), progressbar (value, max), lists (items). + __uiInit(c, t, this.window, this, bounds, typeof text === "object" && text !== null && !(text instanceof Array) ? "" : text, props); + if ((t === "dropdownlist" || t === "listbox") && text instanceof Array) { + for (var i = 0; i < text.length; i++) c.add("item", text[i]); + c.text = ""; + } + if (t === "slider" || t === "scrollbar") { + c.text = ""; + c.value = text === undefined ? 0 : __num(text); + if (arguments.length > 3 && typeof props === "number") { + c.minvalue = props; + c.maxvalue = arguments[4] === undefined ? 100 : __num(arguments[4]); + c.properties = arguments[5] || {}; + } + } + if (t === "progressbar") { + c.text = ""; + c.value = text === undefined ? 0 : __num(text); + if (typeof props === "number") { + c.maxvalue = props; + c.properties = arguments[4] || {}; + } + } + if (t === "checkbox" || t === "radiobutton") c.value = false; + if (t === "edittext" && c.properties.multiline) c.__multiline = true; + if (t === "edittext" && c.properties.readonly) c.__readonly = true; + if (c.properties && c.properties.name) c.name = c.properties.name; + this.children.push(c); + if (t === "tab" && this.type === "tabbedpanel" && this.__sel.length === 0) this.__sel = [this.children.length - 1]; + return c; + }, + remove: function (what) { + var list = this.items && this.type !== "tabbedpanel" ? this.items : this.children; + var i = typeof what === "number" ? what : list.indexOf(what); + if (i < 0 || i >= list.length) return; + list.splice(i, 1); + if (this.items) { + for (var k = 0; k < this.items.length; k++) this.items[k].index = k; + this.__sel = this.__sel.filter(function (s) { return s !== i; }).map(function (s) { return s > i ? s - 1 : s; }); + } + }, + removeAll: function () { + if (this.items) { + this.items = []; + this.__sel = []; + } else this.children = []; + }, + find: function (text) { + var list = this.items || this.children; + for (var i = 0; i < list.length; i++) if (list[i].text === text) return list[i]; + return null; + }, + __addItem: function (text) { + if (!this.items) throw __err("add(\"item\"): only lists have items"); + var it = Object.create(ListItem.prototype); + it.text = String(text === undefined ? "" : text); + it.type = text === "-" ? "separator" : "item"; + it.index = this.items.length; + it.parent = this; + it.image = null; + it.checked = false; + this.items.push(it); + return it; + }, + show: function () { + if (!this.__root) { + this.visible = true; + return; + } + this.__shown = true; + this.__closed = false; + this.__fire("onShow"); + if (this.__kind === "dialog") { + this.__modal = true; + var r = __uiModal(this.__wid); + this.__modal = false; + return r === null ? undefined : r; + } + return undefined; + }, + hide: function () { + if (this.__root) { + this.close(2); + } else this.visible = false; + }, + close: function (result) { + if (!this.__root) return; + if (this.__fire("onClose") === false) return; + this.__closed = true; + this.__shown = false; + this.__result = result === undefined ? 0 : result; + if (this.__modal) __uiResults[this.__wid] = this.__result; + }, + center: function () {}, + notify: function (ev) { + var e = ev || (this.type === "edittext" || this.type === "slider" || this.type === "dropdownlist" || this.type === "listbox" ? "onChange" : "onClick"); + if (e.indexOf("on") !== 0) e = "on" + e.charAt(0).toUpperCase() + e.slice(1); + this.__fire(e); + }, + addEventListener: function (name, fn) { + var k = String(name).toLowerCase(); + (this.__listeners[k] = this.__listeners[k] || []).push(fn); + }, + removeEventListener: function (name, fn) { + var k = String(name).toLowerCase(); + if (this.__listeners[k]) this.__listeners[k] = this.__listeners[k].filter(function (f) { return f !== fn; }); + }, + dispatchEvent: function (e) { this.__fire("on" + String(e.type || e).charAt(0).toUpperCase() + String(e.type || e).slice(1)); }, + __fire: function (name) { + var r; + if (typeof this[name] === "function") r = this[name].call(this); + var l = this.__listeners[name.slice(2).toLowerCase()]; + if (l) for (var i = 0; i < l.length; i++) l[i].call(this, { type: name.slice(2).toLowerCase(), target: this }); + return r; + }, + __handlers: function () { + var out = []; + var names = ["onClick", "onChange", "onChanging", "onDoubleClick", "onClose", "onShow", "onActivate", "onDeactivate", "onResize"]; + for (var i = 0; i < names.length; i++) { + var n = names[i]; + if (typeof this[n] === "function" || (this.__listeners[n.slice(2).toLowerCase()] || []).length) out.push(n); + } + return out; + }, + toString: function () { return "[object " + (__uiTypes[this.type] || "Window") + "]"; } + }; +} + +(function () { + var all = ["Window", "Panel", "Group", "Button", "IconButton", "StaticText", "EditText", "Checkbox", "RadioButton", "Slider", + "Scrollbar", "Progressbar", "DropDownList", "ListBox", "TabbedPanel", "Tab", "ScriptUIImage"]; + for (var i = 0; i < all.length; i++) { + var C = __ecGlobal[all[i]]; + var p = __uiProto(); + for (var k in p) C.prototype[k] = p[k]; + Object.defineProperty(C.prototype, "bounds", { + get: function () { var b = this.__ub || this.__lb; return __bounds4(b[0], b[1], b[2], b[3]); }, + set: function (v) { this.__ub = __toBounds(v); }, + configurable: true + }); + Object.defineProperty(C.prototype, "size", { + get: function () { var b = this.bounds; return __dim(b[2] - b[0], b[3] - b[1]); }, + set: function (v) { + var b = this.bounds; + var w = v.width !== undefined ? v.width : v[0], h = v.height !== undefined ? v.height : v[1]; + this.__ub = [b[0], b[1], b[0] + __num(w), b[1] + __num(h)]; + }, + configurable: true + }); + Object.defineProperty(C.prototype, "location", { + get: function () { var b = this.bounds; return __dim(b[0], b[1]); }, + set: function (v) { + var b = this.bounds; + var x = v.x !== undefined ? v.x : v[0], y = v.y !== undefined ? v.y : v[1]; + this.__ub = [__num(x), __num(y), __num(x) + b[2] - b[0], __num(y) + b[3] - b[1]]; + }, + configurable: true + }); + } + var sel = { + get: function () { + var self = this; + if (this.type === "tabbedpanel") return this.__sel.length ? this.children[this.__sel[0]] : null; + var items = this.__sel.map(function (i) { return self.items[i]; }); + if (this.properties.multiselect) return items.length ? items : null; + return items.length ? items[0] : null; + }, + set: function (v) { + var self = this; + var list = this.type === "tabbedpanel" ? this.children : this.items; + var idx = function (x) { return typeof x === "number" ? x : list.indexOf(x); }; + if (v === null || v === undefined) this.__sel = []; + else if (v instanceof Array) this.__sel = v.map(idx).filter(function (i) { return i >= 0 && i < list.length; }); + else { + var i = idx(v); + this.__sel = i >= 0 && i < list.length ? [i] : []; + } + if (this.items) for (var k = 0; k < this.items.length; k++) this.items[k].selected = this.__sel.indexOf(k) >= 0; + }, + configurable: true + }; + Object.defineProperty(DropDownList.prototype, "selection", sel); + Object.defineProperty(ListBox.prototype, "selection", sel); + Object.defineProperty(TabbedPanel.prototype, "selection", sel); + Object.defineProperty(ListItem.prototype, "toString", { value: function () { return this.text; } }); +})(); + +// A dockable ScriptUI panel (scripts in the ScriptUI Panels folder run with `this` = it). +function __uiDockPanel(title) { + var p = Object.create(Panel.prototype); + __uiInit(p, "panel", null, null, undefined, title, {}); + p.type = "panel"; + p.__kind = "panel"; + p.__root = true; + p.__shown = true; + p.margins = [10, 10, 10, 10]; + __uiWins.push(p); + return p; +} + +function __uiAlign(v) { + if (v === null || v === undefined) return []; + if (v instanceof Array) return v.map(function (x) { return String(x).toLowerCase(); }); + return [String(v).toLowerCase()]; +} +function __uiMargins(m) { + if (m === null || m === undefined) return null; + if (typeof m === "number") return [m, m, m, m]; + if (m.length === 4) return [m[0], m[1], m[2], m[3]]; + return null; +} +function __uiSize(d) { + if (!d) return null; + var w = d.width !== undefined ? d.width : d[0], h = d.height !== undefined ? d.height : d[1]; + w = Number(w); + h = Number(h); + if (!(w > 0) && !(h > 0)) return null; + return [w > 0 ? w : 0, h > 0 ? h : 0]; +} + +function __uiJson(c) { + var ps = __uiSize(c.preferredSize); + if (c.type === "edittext" && c.characters > 0) ps = [c.characters * 7 + 12, ps ? ps[1] : 0]; + var o = { + id: c.__id, + type: c.type, + name: String(c.name || c.properties.name || ""), + text: String(c.text === undefined || c.text === null ? "" : c.text), + enabled: c.enabled !== false, + visible: c.visible !== false, + helpTip: String(c.helpTip || ""), + handlers: c.__handlers(), + children: [] + }; + if (ps) o.preferredSize = ps; + if (c.__ub && !c.__root) o.fixedBounds = c.__ub; + if (c.__root && c.__ub) o.fixedBounds = [0, 0, c.__ub[2] - c.__ub[0], c.__ub[3] - c.__ub[1]]; + if (__uiContainers[c.type]) { + o.orientation = String(c.orientation || "").toLowerCase(); + o.alignChildren = __uiAlign(c.alignChildren); + var m = __uiMargins(c.margins); + if (m) o.margins = m; + if (typeof c.spacing === "number") o.spacing = c.spacing; + var kids = c.children; + for (var i = 0; i < kids.length; i++) o.children.push(__uiJson(kids[i])); + } + if (c.alignment !== null && c.alignment !== undefined) o.alignment = __uiAlign(c.alignment); + if (c.type === "checkbox" || c.type === "radiobutton") o.checked = !!c.value; + if (c.type === "slider" || c.type === "scrollbar" || c.type === "progressbar") { + o.value = Number(c.value) || 0; + o.min = Number(c.minvalue) || 0; + o.max = Number(c.maxvalue); + if (!(o.max === o.max)) o.max = 100; + } + if (c.items) { + o.items = c.items.map(function (it) { return String(it.text); }); + o.selection = c.__sel.slice(); + } + if (c.type === "tabbedpanel") o.activeTab = c.__sel.length ? c.__sel[0] : 0; + if (c.type === "edittext") { + o.multiline = !!c.__multiline; + o.readOnly = !!c.__readonly; + } + return o; +} + +// The open windows of this script, for the host. +function __uiSnapshot() { + var out = []; + for (var i = 0; i < __uiWins.length; i++) { + var w = __uiWins[i]; + out.push({ id: w.__wid, kind: w.__kind, title: String(w.text || ""), visible: !!w.__shown && !w.__closed, modal: !!w.__modal, root: __uiJson(w) }); + } + // Closed windows are forgotten. + __uiWins = __uiWins.filter(function (w) { return !w.__closed; }); + return JSON.stringify(out); +} + +function __uiOpenCount() { + var n = 0; + for (var i = 0; i < __uiWins.length; i++) if (__uiWins[i].__shown && !__uiWins[i].__closed) n++; + return n; +} + +function __uiWin(id) { + for (var i = 0; i < __uiWins.length; i++) if (__uiWins[i].__wid === id) return __uiWins[i]; + return null; +} +function __uiFind(c, id) { + if (c.__id === id) return c; + for (var i = 0; i < (c.children || []).length; i++) { + var r = __uiFind(c.children[i], id); + if (r) return r; + } + return null; +} + +// layout.layout(): lay the window out now (the host does it before drawing anyway) so scripts +// can read bounds and sizes. +function __uiLayout(win) { + var b = JSON.parse(__uiLayoutNative(JSON.stringify({ id: win.__wid, kind: win.__kind, title: String(win.text || ""), visible: true, root: __uiJson(win) }))); + // Bounds are relative to the parent container, as in ScriptUI. + (function apply(c, ox, oy) { + var r = b[c.__id]; + if (!r) return; + c.__lb = [r[0] - ox, r[1] - oy, r[0] - ox + r[2], r[1] - oy + r[3]]; + for (var i = 0; i < (c.children || []).length; i++) apply(c.children[i], r[0], r[1]); + })(win, 0, 0); +} + +// A user action from the frontend. +function __uiDispatch(winId, id, kind, valueJson) { + var w = __uiWin(winId); + if (!w) return; + var v = valueJson === undefined || valueJson === "" ? null : JSON.parse(valueJson); + if (kind === "close") { + w.close(v === null ? 2 : v); + return; + } + var c = __uiFind(w, id); + if (!c || c.enabled === false) return; + var radio = function () { + var sib = c.parent ? c.parent.children : []; + for (var i = 0; i < sib.length; i++) if (sib[i] !== c && sib[i].type === "radiobutton") sib[i].value = false; + c.value = true; + }; + if (kind === "click") { + if (c.type === "checkbox") c.value = !c.value; + else if (c.type === "radiobutton") radio(); + else if (c.type === "tab" && c.parent) { + c.parent.selection = c; + c.parent.__fire("onChange"); + return; + } + var handled = c.__handlers().indexOf("onClick") >= 0; + c.__fire("onClick"); + // Dialog buttons without handlers: OK closes with 1, Cancel with 2. + if (!handled && c.type === "button" && w.__kind === "dialog") { + var n = String(c.name || c.properties.name || c.text).toLowerCase(); + if (c === w.defaultElement || n === "ok") w.close(1); + else if (c === w.cancelElement || n === "cancel") w.close(2); + } + return; + } + // change / changing + if (c.type === "edittext") c.text = v === null ? "" : String(v); + else if (c.type === "statictext") c.text = v === null ? "" : String(v); + else if (c.type === "slider" || c.type === "scrollbar" || c.type === "progressbar") c.value = Number(v); + else if (c.type === "checkbox") c.value = !!v; + else if (c.type === "radiobutton") { if (v) radio(); else c.value = false; } + else if (c.type === "dropdownlist" || c.type === "listbox" || c.type === "tabbedpanel") c.selection = v; + if (c.type === "edittext" || c.type === "slider" || c.type === "scrollbar") c.__fire("onChanging"); + if (kind === "change") { + if (c.type === "checkbox" || c.type === "radiobutton") c.__fire("onClick"); + else c.__fire("onChange"); + } +} diff --git a/crates/script/src/tests_ui.rs b/crates/script/src/tests_ui.rs new file mode 100644 index 00000000..667a3242 --- /dev/null +++ b/crates/script/src/tests_ui.rs @@ -0,0 +1,299 @@ +//! ScriptUI: windows built by scripts are published to the session, drawn by frontends and +//! driven through the `scriptui.*` commands; dialogs block `show()` until they close; dockable +//! panels; File ▸ Scripts (installed and sample scripts). + +use std::sync::Arc; + +use effectcraft_engine::Session; +use effectcraft_engine::scriptui::{WidgetKind, WindowKind}; +use serde_json::{Value, json}; + +use crate::run_code; + +fn session() -> Session { + let mut s = Session { expr: Some(Arc::new(effectcraft_expr::Expressions)), ..Default::default() }; + crate::install(&mut s); + s.config = Some(Arc::new(effectcraft_engine::config::MemoryConfig::default())); + s +} + +fn comp_names(s: &Session) -> Vec { + s.project.items.values().filter(|i| i.as_comp().is_some()).map(|i| i.name.clone()).collect() +} + +fn layer_names(s: &Session) -> Vec { + s.active_comp().map(|c| c.layers.iter().map(|l| l.name.clone()).collect()).unwrap_or_default() +} + +const PALETTE: &str = r#" +var made = 0; +var w = new Window("palette", "Maker"); +w.orientation = "column"; +w.alignChildren = ["fill", "top"]; +var name = w.add("edittext", undefined, "Box"); +name.characters = 12; +var big = w.add("checkbox", undefined, "Big"); +var row = w.add("group"); +var make = row.add("button", undefined, "Make", { name: "make" }); +var label = row.add("statictext", undefined, "none yet"); +var size = w.add("slider", undefined, 50, 10, 200); +var kind = w.add("dropdownlist", undefined, ["Solid", "Null"]); +kind.selection = 0; +var echo = ""; +name.onChange = function () { echo = name.text; }; +make.onClick = function () { + var c = app.project.activeItem; + var s = big.value ? 200 : Math.round(size.value); + if (kind.selection.index === 0) c.layers.addSolid([1, 0, 0], name.text, s, s, 1); + else c.layers.addNull().name = name.text; + made++; + label.text = "made " + made; + writeLn("made " + name.text + " " + s); +}; +w.show(); +"palette up" +"#; + +#[test] +fn palette_controls_run_their_handlers() { + let mut s = session(); + s.execute("comp.new", json!({"name": "Main", "width": 320, "height": 180, "duration": 2})).unwrap(); + let o = run_code(&mut s, PALETTE, "maker.jsx"); + assert!(o.error.is_none(), "{:?}", o.error); + assert_eq!(o.result, json!("palette up")); + assert_eq!(s.script_ui.windows.len(), 1); + let w = s.script_ui.windows[0].clone(); + assert_eq!((w.kind, w.title.as_str(), w.script.as_str()), (WindowKind::Palette, "Maker", "maker.jsx")); + let kinds: Vec = w.root.children.iter().map(|c| c.kind).collect(); + assert_eq!(kinds, [WidgetKind::EditText, WidgetKind::Checkbox, WidgetKind::Group, WidgetKind::Slider, WidgetKind::DropDownList]); + // Laid out: the window has a size, the fill-aligned controls span its width. + assert!(w.root.bounds[2] > 100.0 && w.root.bounds[3] > 100.0, "{:?}", w.root.bounds); + let edit = &w.root.children[0]; + assert_eq!(edit.bounds[2], w.root.bounds[2] - 30.0); + assert_eq!(w.root.children[3].value, 50.0); + assert_eq!((w.root.children[3].min, w.root.children[3].max), (10.0, 200.0)); + assert_eq!(w.root.children[4].items, ["Solid", "Null"]); + assert_eq!(w.root.children[4].selection, [0]); + assert!(w.root.children[2].children[0].handlers.contains(&"onClick".to_string())); + // Agents: list, read, click. + let list = s.execute("scriptui.list", json!({})).unwrap(); + assert_eq!(list[0]["title"], "Maker"); + let tree = s.execute("scriptui.get", json!({"window": w.id})).unwrap(); + assert_eq!(tree["root"]["children"][2]["children"][0]["name"], "make"); + let r = s.execute("scriptui.click", json!({"widget": "make"})).unwrap(); + assert_eq!(r["output"], "made Box 50", "{r}"); + assert_eq!(layer_names(&s), ["Box"]); + // Handlers' edits are undoable engine commands. + assert!(s.undo()); + assert!(layer_names(&s).is_empty()); + // Set values like a user: text (onChange), checkbox, slider, list by item text. + s.execute("scriptui.set", json!({"widget": "#1", "value": "Crate"})).unwrap(); + s.execute("scriptui.click", json!({"widget": "Big"})).unwrap(); + s.execute("scriptui.set", json!({"window": "Maker", "widget": 7, "value": "Null"})).unwrap(); + let r = s.execute("scriptui.click", json!({"widget": "make"})).unwrap(); + assert_eq!(r["output"], "made Crate 200"); + assert_eq!(layer_names(&s), ["Crate"]); + assert!(s.active_comp().unwrap().layers[0].source == effectcraft_engine::project::LayerSource::Null); + // The handler updated a label: the published tree follows. + let w = s.script_ui.windows[0].clone(); + assert_eq!(w.root.children[2].children[1].text, "made 2"); + assert!(w.root.children[1].checked); + assert_eq!(w.root.children[4].selection, [1]); + // Errors. + assert!(s.execute("scriptui.click", json!({"widget": "nope"})).is_err()); + assert!(s.execute("scriptui.get", json!({"window": 999_999})).is_err()); + // Closing the window ends the script host. + s.execute("scriptui.close", json!({})).unwrap(); + assert!(s.script_ui.windows.is_empty()); + assert!(s.execute("scriptui.list", json!({})).unwrap().as_array().unwrap().is_empty()); +} + +const DIALOG: &str = r#" +var d = new Window("dialog", "New Comp"); +d.alignChildren = "left"; +var e = d.add("edittext", undefined, "Untitled"); +var fps = d.add("radiobutton", undefined, "24 fps"); +var fps30 = d.add("radiobutton", undefined, "30 fps"); +fps.value = true; +var g = d.add("group"); +g.add("button", undefined, "Cancel", { name: "cancel" }); +var ok = g.add("button", undefined, "OK", { name: "ok" }); +writeLn("before"); +var r = d.show(); +writeLn("result " + r + " " + e.text + " " + (fps30.value ? 30 : 24)); +if (r === 1) app.project.items.addComp(e.text, 100, 100, 1, 1, fps30.value ? 30 : 24); +r +"#; + +#[test] +fn dialogs_block_show_until_they_close() { + let mut s = session(); + let o = run_code(&mut s, DIALOG, "dialog.jsx"); + // The script waits in show(); the session is back here meanwhile. + assert!(o.waiting, "{o:?}"); + assert_eq!(o.output, ["before"]); + assert_eq!(s.script_ui.windows.len(), 1); + let w = s.script_ui.windows[0].clone(); + assert!(w.modal && w.kind == WindowKind::Dialog); + // The app keeps working while the dialog is up. + s.execute("comp.new", json!({"name": "Other", "width": 64, "height": 64, "duration": 1})).unwrap(); + s.execute("scriptui.set", json!({"widget": "#1", "value": "Shot 010"})).unwrap(); + s.execute("scriptui.click", json!({"widget": "30 fps"})).unwrap(); + let w = s.script_ui.windows[0].clone(); + assert!(!w.root.children[1].checked && w.root.children[2].checked, "radio buttons are exclusive"); + // OK closes the dialog with 1: the script runs on. + let r = s.execute("scriptui.click", json!({"widget": "ok"})).unwrap(); + assert_eq!(r["output"], "result 1 Shot 010 30", "{r}"); + assert_eq!(r["result"], 1); + assert!(r.get("waiting").is_none()); + assert!(s.script_ui.windows.is_empty()); + let names = comp_names(&s); + assert!(names.contains(&"Shot 010".to_string()) && names.contains(&"Other".to_string()), "{names:?}"); + // Cancel (or closing the window) returns 2. + let o = run_code(&mut s, DIALOG, "dialog.jsx"); + assert!(o.waiting); + let r = s.execute("scriptui.close", json!({})).unwrap(); + assert_eq!(r["output"], "result 2 Untitled 24"); + assert_eq!(comp_names(&s).len(), 2); +} + +#[test] +fn layout_and_handler_errors() { + let mut s = session(); + let code = r#" + var w = new Window("palette", "L"); + var g = w.add("group"); + g.orientation = "row"; + var a = g.add("button", undefined, "A"); + var b = g.add("button", undefined, "B"); + b.preferredSize = [120, 30]; + var p = w.add("panel", undefined, "Options"); + p.add("checkbox", undefined, "One"); + var bad = w.add("button", undefined, "Bad"); + bad.onClick = function () { nope(); }; + w.layout.layout(true); + w.show(); + [a.bounds.width, b.size.width, b.size.height, b.bounds[0] - a.bounds[0], w.bounds.width > 200, p.bounds.top > a.bounds.bottom] + "#; + let o = run_code(&mut s, code, "layout.jsx"); + assert!(o.error.is_none(), "{:?}", o.error); + assert_eq!(o.result, json!([80, 120, 30, 90, true, true])); + // An error in a handler is reported, the window stays. + let r = s.execute("scriptui.click", json!({"widget": "Bad"})).unwrap(); + assert_eq!(r["ok"], false); + assert!(r["error"]["message"].as_str().unwrap().contains("nope"), "{r}"); + assert_eq!(s.script_ui.windows.len(), 1); + // Resource strings aren't supported (a clear error). + let o = run_code(&mut s, "new Window(\"dialog { text: 'x' }\")", "res.jsx"); + assert!(o.error.unwrap().message.contains("resource")); + s.execute("scriptui.close", json!({})).unwrap(); +} + +#[test] +fn console_windows_run_inline() { + let mut s = session(); + let req = effectcraft_engine::ScriptRequest { + code: "var cw = new Window('palette', 'Console UI'); var cb = cw.add('button', undefined, 'Hi'); var hits = 0; cb.onClick = function () { hits++; writeLn('hit ' + hits); }; cw.show(); 1", + name: "Script Console", + console: true, + }; + let o = crate::run(&mut s, &req); + assert!(o.error.is_none(), "{:?}", o.error); + assert_eq!(s.script_ui.windows.len(), 1); + let r = s.execute("scriptui.click", json!({"widget": "Hi"})).unwrap(); + assert_eq!(r["output"], "hit 1"); + // The console context sees the handler's state. + let o = crate::run(&mut s, &effectcraft_engine::ScriptRequest { code: "hits", name: "Script Console", console: true }); + assert_eq!(o.result, json!(1)); + // Dialogs don't block in the console. + let o = crate::run( + &mut s, + &effectcraft_engine::ScriptRequest { code: "var dd = new Window('dialog', 'D'); dd.show(); 'went on'", name: "Script Console", console: true }, + ); + assert_eq!(o.result, json!("went on")); + assert_eq!(s.script_ui.windows.len(), 2); + s.execute("scriptui.close", json!({"window": "D"})).unwrap(); + s.execute("scriptui.close", json!({"window": "Console UI"})).unwrap(); + assert!(s.script_ui.windows.is_empty()); +} + +fn solid(s: &mut Session, name: &str, at: f64) -> u64 { + let id = s.execute("layer.newSolid", json!({"name": name, "color": "#808080", "width": 20, "height": 20})).unwrap()["layer"].as_u64().unwrap(); + s.execute("layer.timing", json!({"layers": [id], "in": at})).unwrap(); + id +} + +#[test] +fn sample_scripts_and_installing() { + let mut s = session(); + s.execute("comp.new", json!({"name": "Main", "width": 400, "height": 200, "duration": 10})).unwrap(); + for (n, at) in [("C", 2.0), ("A", 0.0), ("B", 1.0)] { + solid(&mut s, n, at); + } + // File ▸ Scripts lists the samples. + let list = s.execute("file.scripts.list", json!({})).unwrap(); + let names: Vec<&str> = list.as_array().unwrap().iter().map(|e| e["name"].as_str().unwrap()).collect(); + for want in ["Rename Layers.jsx", "Sort Layers by In Point.jsx", "Create Null at Selected Layers.jsx", "Render Queue Batch.jsx", "Layer Tools.jsx"] { + assert!(names.contains(&want), "{names:?}"); + } + // Sort Layers by In Point (all layers when none are selected). + s.execute("edit.deselectAll", json!({})).unwrap(); + s.execute("file.runScript", json!({"name": "Sort Layers by In Point.jsx"})).unwrap(); + assert_eq!(layer_names(&s), ["A", "B", "C"]); + assert!(s.undo()); + assert_eq!(layer_names(&s), ["B", "A", "C"]); + // Create Null at Selected Layers. + let ids: Vec = s.active_comp().unwrap().layers.iter().take(2).map(|l| l.id.0).collect(); + s.execute("layer.select", json!({"layers": ids})).unwrap(); + s.execute("file.runScript", json!({"name": "Create Null at Selected Layers.jsx"})).unwrap(); + let c = s.active_comp().unwrap(); + let null = c.layers.iter().find(|l| l.name == "Centroid").unwrap(); + assert!(c.layers.iter().filter(|l| ids.contains(&l.id.0)).all(|l| l.parent == Some(null.id))); + // Rename Layers: a dialog; fill it in and press OK. + s.execute("layer.select", json!({"layers": ids})).unwrap(); + let r = s.execute("file.runScript", json!({"name": "Rename Layers.jsx"})).unwrap(); + assert_eq!(r["waiting"], true); + s.execute("scriptui.set", json!({"widget": "#11", "value": "Shot_"})).unwrap(); + s.execute("scriptui.click", json!({"widget": "Number the layers"})).unwrap(); + s.execute("scriptui.click", json!({"widget": "ok"})).unwrap(); + let names = layer_names(&s); + assert!(names.contains(&"Shot_B 1".to_string()) && names.contains(&"Shot_A 2".to_string()), "{names:?}"); + // Install a script and a ScriptUI panel (they live in the settings store). + let dir = std::env::temp_dir().join(format!("ec-scripts-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let file = dir.join("Hello.jsx"); + std::fs::write(&file, "writeLn('hello from ' + $.fileName); 42").unwrap(); + s.execute("file.installScript", json!({"path": file.to_string_lossy()})).unwrap(); + let panel = dir.join("Mini Panel.jsx"); + std::fs::write(&panel, "var p = this; p.add('button', undefined, 'Go').onClick = function () { writeLn('go'); }; p.layout.layout(true);").unwrap(); + s.execute("file.installScriptUIPanel", json!({"path": panel.to_string_lossy()})).unwrap(); + let list = s.execute("file.scripts.list", json!({})).unwrap(); + assert!(list.as_array().unwrap().iter().any(|e| e["name"] == "Hello.jsx" && e["source"] == "installed" && e["panel"] == false)); + assert!(list.as_array().unwrap().iter().any(|e| e["name"] == "Mini Panel.jsx" && e["panel"] == true)); + let r = s.execute("file.runScript", json!({"name": "Hello.jsx"})).unwrap(); + assert_eq!(r["result"], 42); + // Window ▸ Mini Panel.jsx: a dockable panel; `this` is the panel. + let r = s.execute("window.scriptPanel", json!({"name": "Mini Panel.jsx"})).unwrap(); + let wid = r["window"].as_u64().unwrap(); + let w = s.script_ui.window(wid as u32).unwrap().clone(); + assert_eq!((w.kind, w.title.as_str()), (WindowKind::Panel, "Mini Panel")); + let r = s.execute("scriptui.click", json!({"window": wid, "widget": "Go"})).unwrap(); + assert_eq!(r["output"], "go"); + // Opening it again brings the same panel forward. + let r = s.execute("window.scriptPanel", json!({"name": "Mini Panel.jsx"})).unwrap(); + assert_eq!(r["window"], wid); + // The sample panel docks too, and works. + let r = s.execute("window.scriptPanel", json!({"name": "Layer Tools.jsx"})).unwrap(); + let tools = r["window"].as_u64().unwrap(); + let before = s.active_comp().unwrap().layers.len(); + s.execute("layer.select", json!({"layers": ids})).unwrap(); + s.execute("scriptui.click", json!({"window": tools, "widget": "Null at Centroid"})).unwrap(); + assert_eq!(s.active_comp().unwrap().layers.len(), before + 1); + s.execute("scriptui.close", json!({"window": tools})).unwrap(); + s.execute("scriptui.close", json!({"window": wid})).unwrap(); + assert!(s.execute("file.installScript", json!({"path": "/nope/x.txt"})).is_err()); + s.execute("file.uninstallScript", json!({"name": "Hello.jsx"})).unwrap(); + assert!(s.execute("file.runScript", json!({"name": "Hello.jsx"})).is_err()); + let _ = std::fs::remove_dir_all(&dir); + let _: Value = json!(null); +} diff --git a/crates/script/src/ui.rs b/crates/script/src/ui.rs new file mode 100644 index 00000000..66574ce0 --- /dev/null +++ b/crates/script/src/ui.rs @@ -0,0 +1,339 @@ +//! ScriptUI hosts: who owns a script window's JavaScript objects, and how user actions reach +//! its handlers. +//! +//! * **Script files** (native builds) run on a *script-host thread*. The caller's session moves +//! to that thread while JavaScript runs and comes back whenever the script stops: at the end, +//! or while a modal dialog waits in `show()`. Each [`ScriptUiEvent`] moves the session there +//! again, runs the handler, and brings it back with the windows' new state. The host ends +//! when its last window closes. Only one side holds the session at a time, so nothing runs +//! concurrently and every edit is still an ordinary undoable engine command. +//! * **The Script Console** (and every run on the web) runs inline: its windows' handlers run in +//! the console's (or a kept) context on the caller's thread, and dialogs don't block. + +use std::cell::RefCell; +use std::collections::BTreeMap; +use std::mem::ManuallyDrop; +use std::sync::atomic::{AtomicU32, Ordering}; + +use boa_engine::{Context, JsResult, JsValue, js_string}; +use effectcraft_engine::Session; +use effectcraft_engine::scriptui::{ScriptUiEvent, ScriptWindow, layout}; +use serde_json::{Value as J, json}; + +use crate::runtime::{ACTIVE, Active, CONSOLE, Restore, ScriptError, arg_string, js_str, outcome, script_error, settle, throw, with_active}; + +pub(crate) const SCRIPTUI: &str = include_str!("scriptui.js"); + +/// The Script Console's host id. +pub(crate) const CONSOLE_HOST: u32 = u32::MAX; + +static NEXT_WINDOW: AtomicU32 = AtomicU32::new(1); +static NEXT_HOST: AtomicU32 = AtomicU32::new(1); + +pub(crate) fn new_host_id() -> u32 { + NEXT_HOST.fetch_add(1, Ordering::Relaxed) +} + +thread_local! { + /// Contexts of inline runs whose palettes / panels are still open (dropped when they close). + static INLINE: RefCell>> = const { RefCell::new(BTreeMap::new()) }; +} + +pub(crate) fn keep_inline(host: u32, ctx: Context) { + INLINE.with(|m| m.borrow_mut().insert(host, ManuallyDrop::new(ctx))); +} + +fn drop_inline(host: u32) { + if let Some(ctx) = INLINE.with(|m| m.borrow_mut().remove(&host)) { + drop(ManuallyDrop::into_inner(ctx)); + } +} + +// ---------------------------------------------------------------- JavaScript side + +fn call_global(ctx: &mut Context, name: &str, args: &[JsValue]) -> JsResult { + let f = ctx.global_object().get(js_string!(name), ctx)?; + let f = f.as_object().ok_or_else(|| throw(format!("{name} is not defined")))?; + f.call(&JsValue::undefined(), args, ctx) +} + +/// The open windows of a context (closed ones are reported once, invisible, then forgotten). +pub(crate) fn snapshot(ctx: &mut Context) -> Vec { + let Ok(v) = call_global(ctx, "__uiSnapshot", &[]) else { return vec![] }; + let s = v.to_string(ctx).map(|s| s.to_std_string_escaped()).unwrap_or_default(); + serde_json::from_str(&s).unwrap_or_default() +} + +/// Run a window event's handlers in `ctx`. +fn dispatch_js(ctx: &mut Context, ev: &ScriptUiEvent) -> Result { + let args = [JsValue::from(ev.window), JsValue::from(ev.widget), js_str(&ev.kind), js_str(&ev.value.to_string())]; + match call_global(ctx, "__uiDispatch", &args) { + Ok(_) => Ok(J::Null), + Err(e) => { + let mut err = script_error(&e, ctx, "ScriptUI handler", ""); + err.message = format!("{} handler: {}", ev.kind, err.message); + Err(err) + } + } +} + +/// `__uiNewId()`: a window id unique in the process. +pub(crate) fn native_new_id(_: &JsValue, _: &[JsValue], _: &mut Context) -> JsResult { + Ok(JsValue::from(NEXT_WINDOW.fetch_add(1, Ordering::Relaxed))) +} + +/// `__uiLayoutNative(windowJson)` → `{controlId: [x, y, width, height]}`: ScriptUI's automatic +/// layout ([`effectcraft_engine::scriptui::layout`]) for `layout.layout()`. +pub(crate) fn native_layout(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult { + let s = arg_string(args, 0, ctx)?; + let mut w: ScriptWindow = serde_json::from_str(&s).map_err(|e| throw(format!("layout: {e}")))?; + layout(&mut w); + let mut all = vec![]; + w.root.walk(&mut all); + let map: serde_json::Map = all.iter().map(|c| (c.id.to_string(), json!(c.bounds))).collect(); + Ok(js_str(&J::Object(map).to_string())) +} + +/// `__uiModal(windowId)`: a dialog's `show()`. On a script-host thread it hands the session back +/// to the caller and waits for the dialog's events until it closes, returning its result. Inline +/// (Script Console, web) it returns `null` at once: the dialog stays open like a palette. +pub(crate) fn native_modal(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult { + let id = args.first().cloned().unwrap_or_default().to_number(ctx)? as u32; + let linked = with_active(|a| Ok(a.link.is_some())).unwrap_or(false); + if !linked { + return Ok(JsValue::null()); + } + loop { + let r = call_global(ctx, "__uiModalResult", &[JsValue::from(id)])?; + if !r.is_null() && !r.is_undefined() { + return Ok(r); + } + let windows = snapshot(ctx); + let out = crate::Outcome { output: settle(), waiting: true, ..Default::default() }; + let Some(ev) = yield_wait(out, windows, false) else { + // The caller went away: cancel the dialog. + return Ok(JsValue::from(2)); + }; + if let Err(e) = dispatch_js(ctx, &ev) { + let line = e.line.map(|l| format!(" (line {l})")).unwrap_or_default(); + let _ = with_active(|a| { + a.output.push(format!("Error{line}: {}", e.message)); + Ok(()) + }); + } + } +} + +// ---------------------------------------------------------------- script-host threads + +pub(crate) enum ToHost { + Start(Session), + Event(Session, ScriptUiEvent), +} + +pub(crate) struct FromHost { + session: Session, + outcome: crate::Outcome, + windows: Vec, + name: String, + /// The host thread has ended (no windows left). + done: bool, +} + +/// The host thread's end of the hand-off. +pub(crate) struct Link { + rx: std::sync::mpsc::Receiver, + tx: std::sync::mpsc::Sender, +} + +/// The caller's end. +struct HostEnd { + tx: std::sync::mpsc::Sender, + rx: std::sync::mpsc::Receiver, +} + +fn hosts() -> &'static std::sync::Mutex> { + static H: std::sync::OnceLock>> = std::sync::OnceLock::new(); + H.get_or_init(Default::default) +} + +// The session crosses to script-host threads and back. +const _: fn() = || { + fn send() {} + send::(); +}; + +/// On the host thread: hand the session (with the windows' state and what the script printed) +/// back to the caller, and wait for the next event. `None` when the caller has gone. +fn yield_wait(outcome: crate::Outcome, windows: Vec, done: bool) -> Option { + let (session, link, name) = ACTIVE.with(|a| { + let mut a = a.borrow_mut(); + let a = a.as_mut()?; + Some((std::mem::take(&mut a.session), a.link.take()?, a.name.clone())) + })?; + if link.tx.send(FromHost { session, outcome, windows, name, done }).is_err() { + return None; + } + if done { + return None; + } + match link.rx.recv() { + Ok(ToHost::Event(s, ev)) => { + ACTIVE.with(|a| { + if let Some(a) = a.borrow_mut().as_mut() { + a.session = s; + a.link = Some(link); + } + }); + Some(ev) + } + _ => None, + } +} + +fn host_main(name: String, code: String, link: Link) { + let Ok(ToHost::Start(session)) = link.rx.recv() else { return }; + let mut a = Active::new(session, &name); + a.link = Some(link); + ACTIVE.with(|c| *c.borrow_mut() = Some(a)); + let file = if name.is_empty() { "script".to_string() } else { name.clone() }; + let body = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let mut ctx = match crate::runtime::new_context() { + Ok(c) => c, + Err(e) => { + yield_wait(outcome(settle(), Err(ScriptError { message: e, file: file.clone(), ..Default::default() })), vec![], true); + return; + } + }; + let r = crate::runtime::eval_with_tasks(&mut ctx, &code, &file); + let mut out = outcome(settle(), r); + loop { + let windows = snapshot(&mut ctx); + let open = windows.iter().any(|w| w.visible); + match yield_wait(out, windows, !open) { + Some(ev) => { + let r = dispatch_js(&mut ctx, &ev); + out = outcome(settle(), r); + } + None => return, + } + } + })); + // A panic: give the session back all the same. + let left = ACTIVE.with(|c| c.borrow_mut().take()); + if let Some(mut a) = left + && let Some(link) = a.link.take() + { + let message = if body.is_err() { "the script engine stopped unexpectedly" } else { "the script ended" }; + let out = crate::Outcome { error: Some(ScriptError { message: message.into(), file, ..Default::default() }), ..Default::default() }; + let _ = link.tx.send(FromHost { session: std::mem::take(&mut a.session), outcome: out, windows: vec![], name, done: true }); + } +} + +/// Wait for the host to hand the session back; keep its link while it has windows open. +fn receive(session: &mut Session, host: u32, end: HostEnd) -> crate::Outcome { + match end.rx.recv() { + Ok(FromHost { session: s, outcome, windows, name, done }) => { + *session = s; + publish(session, host, &name, windows); + if done { + session.script_ui.windows.retain(|w| w.host != host); + session.script_ui.revision += 1; + } else if let Ok(mut h) = hosts().lock() { + h.insert(host, end); + } + outcome + } + Err(_) => crate::Outcome { error: Some(ScriptError { message: "the script host stopped".into(), ..Default::default() }), ..Default::default() }, + } +} + +/// Run a script file on a new script-host thread (see the module docs). +pub(crate) fn run_threaded(session: &mut Session, req: &effectcraft_engine::ScriptRequest) -> crate::Outcome { + let host = new_host_id(); + let (to_tx, to_rx) = std::sync::mpsc::channel(); + let (from_tx, from_rx) = std::sync::mpsc::channel(); + let (name, code) = (req.name.to_string(), req.code.to_string()); + let spawned = + std::thread::Builder::new().name(format!("script-{host}")).stack_size(64 << 20).spawn(move || host_main(name, code, Link { rx: to_rx, tx: from_tx })); + if spawned.is_err() { + return crate::runtime::run_inline(session, req); + } + if let Err(std::sync::mpsc::SendError(ToHost::Start(s) | ToHost::Event(s, _))) = to_tx.send(ToHost::Start(std::mem::take(session))) { + *session = s; + return crate::Outcome { error: Some(ScriptError { message: "the script host did not start".into(), ..Default::default() }), ..Default::default() }; + } + receive(session, host, HostEnd { tx: to_tx, rx: from_rx }) +} + +/// Record the windows a host published (named after the script that made them). +pub(crate) fn publish(session: &mut Session, host: u32, name: &str, mut windows: Vec) { + for w in &mut windows { + if w.script.is_empty() { + w.script = name.to_string(); + } + } + let had = session.script_ui.windows.iter().any(|w| w.host == host); + if windows.is_empty() && !had { + return; + } + session.script_ui.publish(host, windows); + if host != CONSOLE_HOST && !session.script_ui.windows.iter().any(|w| w.host == host) { + drop_inline(host); + } +} + +/// The [`effectcraft_engine::scriptui::ScriptUiDispatch`] this crate provides: run a window +/// event's handlers in the script that made the window, then publish its windows' new state. +/// Returns the handler run's `{ok, output, error, waiting?}`. +pub fn dispatch_ui(session: &mut Session, ev: &ScriptUiEvent) -> Result { + if ACTIVE.with(|a| a.try_borrow().map(|g| g.is_some()).unwrap_or(true)) { + return Err("scripts can't drive script windows".into()); + } + let w = session.script_ui.window(ev.window).ok_or_else(|| format!("no script window {}", ev.window))?; + let (host, name) = (w.host, w.script.clone()); + let inline = host == CONSOLE_HOST || INLINE.with(|m| m.borrow().contains_key(&host)); + if inline { + return Ok(dispatch_inline(session, host, &name, ev).to_json()); + } + let end = hosts().lock().ok().and_then(|mut h| h.remove(&host)); + let Some(end) = end else { + session.script_ui.windows.retain(|w| w.host != host); + session.script_ui.revision += 1; + return Err("the script that made this window has ended".into()); + }; + if let Err(std::sync::mpsc::SendError(ToHost::Event(s, _) | ToHost::Start(s))) = end.tx.send(ToHost::Event(std::mem::take(session), ev.clone())) { + *session = s; + session.script_ui.windows.retain(|w| w.host != host); + session.script_ui.revision += 1; + return Err("the script that made this window has ended".into()); + } + Ok(receive(session, host, end).to_json()) +} + +fn dispatch_inline(session: &mut Session, host: u32, name: &str, ev: &ScriptUiEvent) -> crate::Outcome { + let s = std::mem::take(session); + ACTIVE.with(|a| *a.borrow_mut() = Some(Active::new(s, name))); + let guard = Restore { target: session }; + let (r, windows) = if host == CONSOLE_HOST { + CONSOLE.with(|c| match c.borrow_mut().as_mut() { + Some(ctx) => { + let r = dispatch_js(ctx, ev); + (r, snapshot(ctx)) + } + None => (Ok(J::Null), vec![]), + }) + } else { + INLINE.with(|m| match m.borrow_mut().get_mut(&host) { + Some(ctx) => { + let r = dispatch_js(ctx, ev); + (r, snapshot(ctx)) + } + None => (Ok(J::Null), vec![]), + }) + }; + let output = settle(); + drop(guard); + publish(session, host, name, windows); + outcome(output, r) +} diff --git a/crates/ui-egui/src/header.rs b/crates/ui-egui/src/header.rs index b2fa142e..b3f5908f 100644 --- a/crates/ui-egui/src/header.rs +++ b/crates/ui-egui/src/header.rs @@ -221,7 +221,21 @@ fn puppet_options(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: &egui::Painter } x = r.max.x + 12.0; } - x + 4.0 + // Record Options… (⌘/Ctrl-drag a pin records its motion in real time). + let label = "Record Options..."; + let w = 104.0; + let r = Rect::from_min_size(pos2(x, cy - 10.0), vec2(w, 20.0)); + let resp = ui.interact(r, egui::Id::new("puppet-record-options"), egui::Sense::click()).on_hover_text("⌘/Ctrl-drag a pin to record its motion"); + p.rect_stroke(r, 3.0, egui::Stroke::new(1.0, if resp.hovered() { t.accent } else { t.field_border }), egui::StrokeKind::Inside); + p.text(r.center(), Align2::CENTER_CENTER, label, Tokens::ui(11.5), t.text); + app.auto.add("header.puppet.recordOptions", r, label); + if resp.clicked() { + let ctx = ui.ctx().clone(); + if let Err(e) = crate::menus::invoke(app, &ctx, "puppet.recordOptions", json!({})) { + app.ui.status = e; + } + } + r.max.x + 16.0 } fn ws_menu(app: &mut EffectcraftApp, ui: &mut egui::Ui) { diff --git a/crates/ui-egui/src/panels/effects_presets.rs b/crates/ui-egui/src/panels/effects_presets.rs index 888f9d86..21d63dec 100644 --- a/crates/ui-egui/src/panels/effects_presets.rs +++ b/crates/ui-egui/src/panels/effects_presets.rs @@ -322,7 +322,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let searching = !query.is_empty(); let mut y = list.min.y - scroll; let mut apply: Option = None; - let reg = effectcraft_engine::effects::registry(); + let reg = effectcraft_engine::effects::all(); let keep = |e: &effectcraft_engine::effects::EffectSpec| depth_keeps(&view.depth, e) && (query.is_empty() || e.name.to_lowercase().contains(&query)); let lookup = |id: &str| reg.iter().find(|e| e.id == id); @@ -381,7 +381,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { effect_row(app, ui, &lp, list, &mut y, e, -16.0, &query, "effects.item", &mut apply); } } else { - let mut cats: Vec<&str> = effectcraft_engine::effects::CATEGORIES.to_vec(); + let mut cats: Vec<&str> = effectcraft_engine::effects::categories(); cats.retain(|c| reg.iter().any(|e| e.category == *c)); for cat in cats { let items: Vec<_> = reg.iter().filter(|e| e.category == cat && keep(e)).collect(); diff --git a/crates/ui-egui/src/panels/forms.rs b/crates/ui-egui/src/panels/forms.rs index 39640ef6..50f43f73 100644 --- a/crates/ui-egui/src/panels/forms.rs +++ b/crates/ui-egui/src/panels/forms.rs @@ -136,6 +136,19 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { 1, )], ), + // Puppet tool ▸ Record Options. + "puppet.recordOptions" if !has(p, &["speed", "smoothing", "useDraftDeformation", "showMesh"]) => { + let o = &s.state.puppet; + ( + "Puppet Record Options".into(), + vec![ + Field::num("speed", "Speed (%)", o.record_speed), + Field::num("smoothing", "Smoothing", o.record_smoothing), + Field::bool("useDraftDeformation", "Use Draft Deformation", o.record_draft), + Field::bool("showMesh", "Show Mesh", o.record_show_mesh), + ], + ) + } "layer.setTransform" if !has(p, &["value"]) => { let prop = p.get("prop").and_then(Value::as_str).unwrap_or("position"); let cur = layer.and_then(|l| l.transform()).and_then(|tr| tr.get(if prop == "anchorPoint" { "anchor" } else { prop })).map(|pr| pr.value_at(lt)); diff --git a/crates/ui-egui/src/panels/misc.rs b/crates/ui-egui/src/panels/misc.rs index 69d95f66..ea12591c 100644 --- a/crates/ui-egui/src/panels/misc.rs +++ b/crates/ui-egui/src/panels/misc.rs @@ -172,34 +172,43 @@ pub fn audio(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { pub fn history(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let t = app.tokens; let p = ui.painter().with_clip_rect(rect); + // The branching history (EffectCraft's History panel): every state, branches indented under + // the state they grew from; click any state to go there (`edit.history.goto`). + let nodes = app.session.history_tree(); + let row_h = 22.0; + let cur = nodes.iter().position(|n| n.current).unwrap_or(0); + // Keep the current state in view. + let visible = ((rect.height() - 12.0) / row_h).max(1.0) as usize; + let first = (cur + 2).saturating_sub(visible).min(nodes.len().saturating_sub(visible)); let mut y = rect.min.y + 6.0; - let undo: Vec = app.session.history.undo.iter().map(|u| u.0.clone()).collect(); - let redo: Vec = app.session.history.redo.iter().rev().map(|u| u.0.clone()).collect(); - let n = undo.len(); - let mut jump: Option = None; - for (i, label) in undo.iter().chain(redo.iter()).enumerate() { - let r = Rect::from_min_size(pos2(rect.min.x, y), vec2(rect.width(), 22.0)); - y += 22.0; - let current = i + 1 == n; - let future = i >= n; - let resp = ui.interact(r, egui::Id::new(("hist", i)), Sense::click()); - if current { + let mut jump: Option = None; + for n in nodes.iter().skip(first).take(visible + 1) { + let r = Rect::from_min_size(pos2(rect.min.x, y), vec2(rect.width(), row_h)); + y += row_h; + let resp = ui.interact(r, egui::Id::new(("hist", n.id.as_str())), Sense::click()); + if n.current { p.rect_filled(r, 0.0, t.row_selected); } else if resp.hovered() { p.rect_filled(r, 0.0, t.hover); } - p.text(pos2(r.min.x + 12.0, r.center().y), Align2::LEFT_CENTER, label, Tokens::ui(12.0), if future { t.text_faint } else { t.text }); + let x = r.min.x + 12.0 + n.depth as f32 * 14.0; + if n.depth > 0 { + // Branch marker. + let c = pos2(x - 7.0, r.center().y); + p.line_segment([pos2(c.x, r.min.y), c], egui::Stroke::new(1.0, t.text_faint)); + p.line_segment([c, pos2(c.x + 5.0, c.y)], egui::Stroke::new(1.0, t.text_faint)); + } + let col = if !n.current && (n.future || !n.line) { t.text_faint } else { t.text }; + p.text(pos2(x, r.center().y), Align2::LEFT_CENTER, &n.label, Tokens::ui(12.0), col); + app.auto.add(&format!("history.state.{}", n.index), r, &n.label); if resp.clicked() { - jump = Some(i as i64 + 1 - n as i64); + jump = Some(n.index); } } - if let Some(d) = jump { - for _ in 0..d.unsigned_abs() { - if d < 0 { - app.session.undo(); - } else { - app.session.redo(); - } + if let Some(i) = jump { + let ctx = ui.ctx().clone(); + if let Err(e) = crate::menus::invoke(app, &ctx, "edit.history.goto", serde_json::json!({"index": i})) { + app.ui.status = e; } } } diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index 04999a90..54643bd4 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -121,6 +121,17 @@ enum Gesture { pin: u64, inv: Mat3, }, + /// Puppet tool Record mode (⌘/Ctrl-drag a pin): the drag is sampled in real time (seconds + /// since it began, layer space) and becomes keyframes on release (`puppet.recordPin`); the + /// current time runs along while recording. + PuppetRecord { + layer: LayerId, + pin: u64, + inv: Mat3, + began: f64, + cti: Tick, + samples: Vec<(f64, [f64; 2])>, + }, /// Pen tool: pulling the tangents of the vertex just placed. Pen { layer: LayerId, @@ -861,7 +872,13 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { t if t.is_shape() => Some(Gesture::Create { tool: t, start: cpt }), t if t.puppet_kind().is_some() => pin_hits.iter().rev().find(|h| h.pos.distance(press) < 8.0).and_then(|h| { let l = comp.layer(h.layer)?; - Some(Gesture::PuppetPin { layer: h.layer, pin: h.pin, inv: l2c(&ectx, l).0.inverse()? }) + let inv = l2c(&ectx, l).0.inverse()?; + if mods.command { + let lp = inv.apply(gv2(cpt[0], cpt[1])); + let began = ui.input(|i| i.time); + return Some(Gesture::PuppetRecord { layer: h.layer, pin: h.pin, inv, began, cti: time, samples: vec![(0.0, [lp.x, lp.y])] }); + } + Some(Gesture::PuppetPin { layer: h.layer, pin: h.pin, inv }) }), Tool::Orbit | Tool::PanCamera | Tool::Dolly => Some(Gesture::Camera { tool }), Tool::Selection if super::tracker::hit_at(&track_hits, press).is_some() => { @@ -1097,6 +1114,20 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } let _ = app.session.execute("mask.setVertex", params); } + Gesture::PuppetRecord { layer, pin, inv, began, cti, mut samples } => { + let lp = inv.apply(gv2(cpt[0], cpt[1])); + let now = ui.input(|i| i.time) - began; + if samples.last().is_none_or(|s| now > s.0) { + samples.push((now, [lp.x, lp.y])); + } + // The current time runs with the recording (Record Options ▸ Speed). + let speed = app.session.state.puppet.record_speed.max(1.0) / 100.0; + app.session.set_time(cti + Tick::from_seconds_f64(now / speed)); + painter.circle_filled(pos, 6.0, Color32::from_rgb(0xe0, 0x30, 0x30)); + painter.text(pos + vec2(10.0, -10.0), Align2::LEFT_BOTTOM, "Recording", Tokens::ui(11.0), Color32::from_rgb(0xff, 0x60, 0x60)); + ui.data_mut(|dd| dd.insert_temp(gid, Gesture::PuppetRecord { layer, pin, inv, began, cti, samples })); + ui.ctx().request_repaint(); + } Gesture::PuppetPin { layer, pin, inv } => { let lp = inv.apply(gv2(cpt[0], cpt[1])); if let Err(e) = app.session.execute("puppet.movePin", json!({"layer": layer.0, "pin": pin, "position": [lp.x, lp.y], "merge": merge})) { @@ -1129,6 +1160,12 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { if resp.drag_stopped() { if let (Some(g), Some(pos)) = (gesture, resp.interact_pointer_pos()) { match g { + Gesture::PuppetRecord { layer, pin, cti, samples, .. } => { + let pts: Vec = samples.iter().map(|(t, p)| json!([t, p[0], p[1]])).collect(); + if let Err(e) = app.session.execute("puppet.recordPin", json!({"layer": layer.0, "pin": pin, "samples": pts, "start": cti.seconds()})) { + app.ui.status = e.to_string(); + } + } Gesture::Create { tool, start } => { let end = map.to_comp(pos); create_shape(app, tool, start, end, mods.shift); diff --git a/docs/architecture.md b/docs/architecture.md index a54703b7..6ff400dc 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -34,9 +34,10 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L3 | `export` | Render queue encoding: H.264, ProRes, WebM (VP9 + alpha, Opus), PNG/JPEG/TIFF/EXR sequences, GIF, WAV/AIFF | | L3 | `gpu` | The GPU compositor and GPU effects on wgpu compute shaders (Metal, Vulkan, Direct3D 12, WebGPU), checked against the CPU renderer | | L3 | `lottie` | Lottie JSON / dotLottie import and export (layers, precomps, eased and spatial keyframes, shapes, masks, mattes) with a warnings list for what Lottie cannot express | -| L4 | `engine` | `Session`: project, undo history, editor state, the command registry and menus | -| L4 | `script` | Scripting: JavaScript (boa) with an After Effects-style object model (`app.project`, comps, layers, properties, render queue) whose edits run engine commands; AE match names | -| L4 | `host` | A fully wired `Session` (media, expressions, scripting, exporter) for the frontends | +| L3 | `plugin` | Effect plug-ins: loads sandboxed WebAssembly effect modules (plug-in API v1; the wasmi interpreter behind the `wasm` feature, on for native hosts) into the effect registry ([plugins.md](plugins.md)) | +| L4 | `engine` | `Session`: project, branching undo history, editor state, the command registry and menus, the ScriptUI window model (`scriptui`) | +| L4 | `script` | Scripting: JavaScript (boa) with an After Effects-style object model (`app.project`, comps, layers, properties, render queue) and ScriptUI (dialogs, palettes, dockable panels; script-host threads so dialogs block `show()`), whose edits run engine commands; AE match names | +| L4 | `host` | A fully wired `Session` (media, expressions, scripting, WebAssembly plug-ins, exporter) for the frontends | | L5 | `ui-egui` | The desktop interface: docking, panels, viewer, timeline, graph editor, dialogs, control channel | | L5 | `automation` | The MCP server, headless or bridged to the running app | | L6 | apps `effectcraft`, `effectcraft-cli`, `effectcraft-web` | Desktop app; command-line tool (render, exec, get/set, MCP); the browser app (wasm32, [web.md](web.md)) | diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 21928b90..dfedbea3 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -43,6 +43,7 @@ const LAYERS: &[(&str, u8)] = &[ ("gpu", 3), ("lottie", 3), ("format", 3), + ("plugin", 3), ("engine", 4), ("script", 4), ("host", 4), From 0bce3f2e4938eec54ef0036ef5c8c3a36c542649 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 04:08:23 -0400 Subject: [PATCH 3/9] M13.3: Subspace Warp content-preserving mesh warp, face tracker library Co-Authored-By: Claude Opus 5.5 --- crates/effects/src/warp_stab.rs | 40 +- crates/track/src/camtrack/features.rs | 13 +- crates/track/src/face.rs | 1142 +++++++++++++++++++++++++ crates/track/src/lib.rs | 2 + crates/track/src/solve.rs | 6 + crates/track/src/stabilize.rs | 358 +++++++- crates/track/src/subspace.rs | 447 ++++++++++ 7 files changed, 1971 insertions(+), 37 deletions(-) create mode 100644 crates/track/src/face.rs create mode 100644 crates/track/src/subspace.rs diff --git a/crates/effects/src/warp_stab.rs b/crates/effects/src/warp_stab.rs index c0bbced8..ff535c41 100644 --- a/crates/effects/src/warp_stab.rs +++ b/crates/effects/src/warp_stab.rs @@ -17,8 +17,11 @@ //! Edge Cropping ignores bad edge pixels of the input frames; //! - **Show Track Points** draws the analysed background features. //! -//! *Subspace Warp* is approximated by a per-frame perspective warp (no content-preserving -//! mesh warp yet), and *Rolling Shutter Ripple* is stored but has no effect. +//! *Subspace Warp* warps each frame with a content-preserving mesh fitted to the analysis' +//! subspace-smoothed feature trajectories ([`effectcraft_track::subspace`]); analyses made +//! before trajectories were stored fall back to a perspective warp. *Rolling Shutter Ripple* +//! picks the mesh: Automatic Reduction (12 × 8, stiffer) or Enhanced Reduction (twice the rows, +//! softer), which follows the row-wise wobble of rolling-shutter shake more closely. //! //! Without an analysis (or with one made for another layer size) the input passes through. @@ -30,7 +33,7 @@ use effectcraft_keyframe::Value; use effectcraft_project::ParamUi; use effectcraft_raster::{Image, Px}; use effectcraft_track::Homography; -use effectcraft_track::stabilize::{Framing, Method, Plan, StabResult, StabSettings, WarpAnalysis, plan}; +use effectcraft_track::stabilize::{Framing, Method, Plan, Ripple, StabResult, StabSettings, WarpAnalysis, plan}; use rayon::prelude::*; use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; @@ -145,6 +148,7 @@ pub fn settings(params: &Params, fps: f64) -> StabSettings { // AE's 0…100 slider (50 = balanced) onto the planner's −100…100. crop_less_smooth_more: (f(params, "advanced/cropLessSmoothMore") - 50.0) * 2.0, fps, + ripple: if e(params, "advanced/rollingShutterRipple") == 1 { Ripple::Enhanced } else { Ripple::Automatic }, } } @@ -175,7 +179,7 @@ fn settings_key(s: &StabSettings) -> u64 { for v in [s.smoothness, s.max_scale, s.action_safe, s.additional_scale, s.crop_less_smooth_more, s.fps] { h.write_u64(v.to_bits()); } - h.write(format!("{:?}{:?}{:?}{:?}", s.result, s.method, s.preserve_scale, s.framing).as_bytes()); + h.write(format!("{:?}{:?}{:?}{:?}{:?}", s.result, s.method, s.preserve_scale, s.framing, s.ripple).as_bytes()); h.finish() } @@ -275,6 +279,13 @@ fn render(ctx: &EffectCtx, buf: Buf) -> Buf { } let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; let Some(inv) = pl.warps[k].inverse() else { return buf }; + // Output point → source point of this frame (a mesh lookup for Subspace Warp). + let mesh = pl.meshes.get(k); + let finv = pl.framing.inverse().unwrap_or(Homography::IDENTITY); + let src_of = |p: [f64; 2]| match mesh { + Some(m) => m.sample(finv.apply(p)), + None => inv.apply(p), + }; let (ow, oh) = (((size[0] * sc).ceil() as u32).max(1), ((size[1] * sc).ceil() as u32).max(1)); let synth = matches!(settings(ctx.params, 30.0).framing, Framing::SynthesizeEdges); let crop = [ @@ -306,7 +317,7 @@ fn render(ctx: &EffectCtx, buf: Buf) -> Buf { { continue; } - let q = inv.apply(p); + let q = src_of(p); if !synth { *px = sample(src, q); continue; @@ -337,12 +348,12 @@ fn render(ctx: &EffectCtx, buf: Buf) -> Buf { }); let need = need_m.into_inner().unwrap_or_else(|e| e.into_inner()); if synth && need.iter().any(|w| *w > 0.0) { - synthesize(ctx, &a, k, &inv, &crop, sc, &mut out, &need); + synthesize(ctx, &a, k, &src_of, &crop, sc, &mut out, &need); } if b(ctx.params, "advanced/showTrackPoints") { let r = (2.5 * f(ctx.params, "advanced/trackPointSize") / 100.0 * sc).max(0.5); for q in &a.frames[k].points { - let o = pl.warps[k].apply([q[0] as f64, q[1] as f64]); + let Some(o) = pl.output_of(k, [q[0] as f64, q[1] as f64]) else { continue }; dot(&mut out, [o[0] * sc, o[1] * sc], r, [0.25, 0.75, 1.0, 1.0]); } } @@ -352,7 +363,16 @@ fn render(ctx: &EffectCtx, buf: Buf) -> Buf { /// Fill the pixels flagged in `need` (weight = how much synthesized content they take) from /// neighbouring frames, nearest first. #[allow(clippy::too_many_arguments)] -fn synthesize(ctx: &EffectCtx, a: &WarpAnalysis, k: usize, inv: &Homography, crop: &[f64; 4], sc: f64, out: &mut Image, need: &[f32]) { +fn synthesize( + ctx: &EffectCtx, + a: &WarpAnalysis, + k: usize, + src_of: &(dyn Fn([f64; 2]) -> [f64; 2] + Sync), + crop: &[f64; 4], + sc: f64, + out: &mut Image, + need: &[f32], +) { let Some(host) = ctx.env.host else { return }; let n = a.frames.len(); let range = ((f(ctx.params, "advanced/synthesizeInputRange").max(0.0) / a.frame_duration.max(1e-6)).round() as usize).min(60); @@ -376,7 +396,7 @@ fn synthesize(ctx: &EffectCtx, a: &WarpAnalysis, k: usize, inv: &Homography, cro .filter(|i| !filled[**i]) .filter_map(|i| { let (x, y) = (i % ow, i / ow); - let q = inv.apply([(x as f64 + 0.5) / sc, (y as f64 + 0.5) / sc]); + let q = src_of([(x as f64 + 0.5) / sc, (y as f64 + 0.5) / sc]); let qj = m.apply(q); inside(crop, qj).then(|| (*i, nb.img.sample_bilinear_clamped(qj[0] * nb.scale + nb.offset[0], qj[1] * nb.scale + nb.offset[1]))) }) @@ -412,7 +432,7 @@ mod tests { use effectcraft_track::stabilize::FrameMotion; fn analysis_json(n: usize, w: f64, h: f64, shift: impl Fn(usize) -> [f64; 2]) -> String { - let mut a = WarpAnalysis { version: 1, start: 0.0, frame_duration: 1.0 / 25.0, size: [w, h], detailed: false, frames: vec![] }; + let mut a = WarpAnalysis { version: 1, start: 0.0, frame_duration: 1.0 / 25.0, size: [w, h], detailed: false, frames: vec![], tracks: vec![] }; for k in 0..n { let mut fm = FrameMotion::default(); if k > 0 { diff --git a/crates/track/src/camtrack/features.rs b/crates/track/src/camtrack/features.rs index 30b019e5..1775a304 100644 --- a/crates/track/src/camtrack/features.rs +++ b/crates/track/src/camtrack/features.rs @@ -28,11 +28,19 @@ pub struct TrackAnalyzer { frame: u32, size: [f64; 2], factor: f64, + /// (analysis long side, wanted features) overriding the options' defaults. + budget: Option<(u32, usize)>, } impl TrackAnalyzer { pub fn new(size: [f64; 2], opts: AnalyzeOpts) -> TrackAnalyzer { - TrackAnalyzer { opts, prev: None, active: vec![], tracks: vec![], frame: 0, size, factor: 1.0 } + TrackAnalyzer { opts, prev: None, active: vec![], tracks: vec![], frame: 0, size, factor: 1.0, budget: None } + } + + /// An analyzer working at most at `max_side` pixels with about `want` live features (the + /// Warp Stabilizer's trajectories for Subspace Warp). + pub fn with_budget(size: [f64; 2], max_side: u32, want: usize) -> TrackAnalyzer { + TrackAnalyzer { budget: Some((max_side, want.max(8))), ..TrackAnalyzer::new(size, AnalyzeOpts::default()) } } pub fn frames(&self) -> u32 { @@ -45,6 +53,9 @@ impl TrackAnalyzer { } fn params(&self) -> (u32, usize) { + if let Some(b) = self.budget { + return b; + } if self.opts.detailed { (1280, 900) } else { (720, 500) } } diff --git a/crates/track/src/face.rs b/crates/track/src/face.rs new file mode 100644 index 00000000..78344a14 --- /dev/null +++ b/crates/track/src/face.rs @@ -0,0 +1,1142 @@ +//! Face tracking (Tracker panel ▸ Method ▸ Face Tracking (Outline Only) / (Detailed Features)), +//! a classical pipeline with no learned weights: +//! +//! 1. **Skin model**: on the first frame, the colour of the middle of the mask (luma and +//! normalised r, g chromaticity: median and median absolute deviation) defines skin; every +//! later frame is classified against it inside a search window around the previous face. +//! 2. **Face region and outline**: the skin component containing the face centre is flood-filled; +//! its outer boundary is found along 48 rays from its centroid (so the non-skin holes of eyes, +//! brows, nostrils and mouth don't matter), median-smoothed, and an ellipse is fitted to the +//! outline polygon from its area moments (centre, half-height, roll). The outline is what +//! *Outline Only* keys into the mask. +//! 3. **Facial features**: the non-skin connected components inside the face are matched to the +//! eyes, brows, nostrils and mouth by their position in the face frame (predicted from the +//! previous frame); landmark points are measured on each component (extremes along the face +//! axes for corners and lids, a darkness-weighted centroid for the pupil). Chin and jaw points +//! come from the outline. +//! 4. **Shape model**: an active-shape point distribution model (Cootes, Taylor, Cooper & Graham, +//! "Active Shape Models — Their Training and Application", CVIU 61(1), 1995) regularises the +//! landmarks: a weighted similarity alignment and shape-parameter fit (parameters clamped to +//! ±3 standard deviations) fills landmarks whose feature was not found (a closed eye, a +//! merged component) and replaces detections that disagree with the model. The model is +//! *trained on synthetic shapes* produced by our own parametric face generator +//! ([`synth_shape`]: eye and mouth openness, mouth width, brow raise, head yaw and pitch) by +//! principal component analysis — no external data or weights. +//! +//! Coordinates are layer pixels; the face frame is centred on the face ellipse, with the +//! half-height as unit, `u` right and `v` down (image-left is "Left"). + +use std::sync::OnceLock; + +use effectcraft_raster::Image; + +use crate::Frame; +use crate::camtrack::linalg::{cholesky_solve, sym_eigen}; + +/// Landmarks: (id, display name, canonical position in the face frame). +pub const LANDMARKS: [(&str, &str, [f64; 2]); 26] = [ + ("leftEyebrowInner", "Left Eyebrow Inner", [-0.16, -0.44]), + ("leftEyebrowMiddle", "Left Eyebrow Middle", [-0.32, -0.50]), + ("leftEyebrowOuter", "Left Eyebrow Outer", [-0.48, -0.44]), + ("rightEyebrowInner", "Right Eyebrow Inner", [0.16, -0.44]), + ("rightEyebrowMiddle", "Right Eyebrow Middle", [0.32, -0.50]), + ("rightEyebrowOuter", "Right Eyebrow Outer", [0.48, -0.44]), + ("leftEyeInner", "Left Eye Inner", [-0.17, -0.22]), + ("leftEyeOuter", "Left Eye Outer", [-0.45, -0.22]), + ("leftEyeTop", "Left Eye Top", [-0.31, -0.292]), + ("leftEyeBottom", "Left Eye Bottom", [-0.31, -0.148]), + ("leftPupil", "Left Pupil", [-0.31, -0.22]), + ("rightEyeInner", "Right Eye Inner", [0.17, -0.22]), + ("rightEyeOuter", "Right Eye Outer", [0.45, -0.22]), + ("rightEyeTop", "Right Eye Top", [0.31, -0.292]), + ("rightEyeBottom", "Right Eye Bottom", [0.31, -0.148]), + ("rightPupil", "Right Pupil", [0.31, -0.22]), + ("noseTip", "Nose Tip", [0.0, 0.14]), + ("leftNostril", "Left Nostril", [-0.08, 0.20]), + ("rightNostril", "Right Nostril", [0.08, 0.20]), + ("mouthLeft", "Mouth Left", [-0.24, 0.50]), + ("mouthRight", "Mouth Right", [0.24, 0.50]), + ("mouthTop", "Mouth Top", [0.0, 0.435]), + ("mouthBottom", "Mouth Bottom", [0.0, 0.565]), + ("chin", "Chin", [0.0, 1.0]), + ("leftJaw", "Left Jaw", [-0.614, 0.574]), + ("rightJaw", "Right Jaw", [0.614, 0.574]), +]; + +/// Number of landmarks. +pub const N: usize = LANDMARKS.len(); + +/// Landmark index by id. +pub fn landmark(id: &str) -> Option { + LANDMARKS.iter().position(|l| l.0 == id) +} + +const BROW: [[usize; 3]; 2] = [[0, 1, 2], [3, 4, 5]]; +/// Eye landmarks: inner, outer, top, bottom, pupil. +const EYE: [[usize; 5]; 2] = [[6, 7, 8, 9, 10], [11, 12, 13, 14, 15]]; +const NOSE_TIP: usize = 16; +const NOSTRIL: [usize; 2] = [17, 18]; +/// Mouth: left, right, top, bottom. +const MOUTH: [usize; 4] = [19, 20, 21, 22]; +const CHIN: usize = 23; +const JAW: [usize; 2] = [24, 25]; +/// Half-width of the face relative to its half-height. +const ASPECT: f64 = 0.75; + +// ---------------------------------------------------------------- synthetic faces + +/// Parameters of a synthetic face (the shape model's training data and the tests' footage). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct FaceParams { + /// Face centre (layer pixels) and half-height (pixels). + pub center: [f64; 2], + pub height: f64, + /// Head rotation in degrees: roll (in the image plane), yaw (turn), pitch (nod). + pub roll: f64, + pub yaw: f64, + pub pitch: f64, + /// Eye openness (0 closed … 1 open), left and right. + pub eye_open: [f64; 2], + /// Mouth openness (0 … 1) and width (1 = neutral). + pub mouth_open: f64, + pub mouth_width: f64, + /// Eyebrow raise (−1 … 1), left and right. + pub brow_raise: [f64; 2], +} + +impl Default for FaceParams { + fn default() -> Self { + FaceParams { + center: [0.0; 2], + height: 60.0, + roll: 0.0, + yaw: 0.0, + pitch: 0.0, + eye_open: [1.0; 2], + mouth_open: 0.2, + mouth_width: 1.0, + brow_raise: [0.0; 2], + } + } +} + +fn eye_half_height(open: f64) -> f64 { + 0.012 + 0.06 * open.clamp(0.0, 1.0) +} +fn mouth_half_height(open: f64) -> f64 { + 0.015 + 0.11 * open.clamp(0.0, 1.0) +} + +/// The landmarks of a synthetic face in the face frame (before roll, translation and scale). +pub fn synth_shape(p: &FaceParams) -> [[f64; 2]; N] { + let mut s = [[0.0; 2]; N]; + let (sy, sp) = (p.yaw.to_radians().sin(), p.pitch.to_radians().sin()); + // Inner features slide with the head's turn; the nose (closer to the camera) slides more. + let shift = |depth: f64| [0.35 * sy * depth, 0.3 * sp * depth]; + for (i, l) in LANDMARKS.iter().enumerate() { + s[i] = l.2; + } + for side in 0..2 { + let sg = if side == 0 { -1.0 } else { 1.0 }; + let raise = -0.06 * p.brow_raise[side].clamp(-1.0, 1.0); + for &i in &BROW[side] { + s[i][1] += raise; + } + let h = eye_half_height(p.eye_open[side]); + let e = EYE[side]; + s[e[2]] = [sg * 0.31, -0.22 - h]; + s[e[3]] = [sg * 0.31, -0.22 + h]; + } + let mw = 0.24 * p.mouth_width.clamp(0.5, 1.6); + let mh = mouth_half_height(p.mouth_open); + s[MOUTH[0]] = [-mw, 0.5]; + s[MOUTH[1]] = [mw, 0.5]; + s[MOUTH[2]] = [0.0, 0.5 - mh]; + s[MOUTH[3]] = [0.0, 0.5 + mh]; + for (i, q) in s.iter_mut().enumerate() { + if i == CHIN || JAW.contains(&i) { + continue; + } + let d = if i == NOSE_TIP { + 1.3 + } else if NOSTRIL.contains(&i) { + 1.15 + } else { + 1.0 + }; + let o = shift(d); + q[0] += o[0]; + q[1] += o[1]; + } + s +} + +/// Face frame → layer pixels. +fn to_layer(c: [f64; 2], b: f64, roll_deg: f64, q: [f64; 2]) -> [f64; 2] { + let (sn, cs) = roll_deg.to_radians().sin_cos(); + [c[0] + b * (cs * q[0] - sn * q[1]), c[1] + b * (sn * q[0] + cs * q[1])] +} + +/// Layer pixels → face frame. +fn to_face(c: [f64; 2], b: f64, roll_deg: f64, p: [f64; 2]) -> [f64; 2] { + let (sn, cs) = roll_deg.to_radians().sin_cos(); + let d = [(p[0] - c[0]) / b, (p[1] - c[1]) / b]; + [cs * d[0] + sn * d[1], -sn * d[0] + cs * d[1]] +} + +/// A synthetic face's landmarks in layer pixels. +pub fn synth_landmarks(p: &FaceParams) -> [[f64; 2]; N] { + synth_shape(p).map(|q| to_layer(p.center, p.height, p.roll, q)) +} + +fn hash(i: i64, j: i64) -> f64 { + let mut v = (i.wrapping_mul(73_856_093) ^ j.wrapping_mul(19_349_663)) as u64; + v ^= v >> 13; + v = v.wrapping_mul(0x5bd1_e995); + v ^= v >> 15; + (v % 10_000) as f64 / 10_000.0 +} + +fn seg_dist(p: [f64; 2], a: [f64; 2], b: [f64; 2]) -> f64 { + let d = [b[0] - a[0], b[1] - a[1]]; + let l2 = d[0] * d[0] + d[1] * d[1]; + let t = if l2 > 0.0 { (((p[0] - a[0]) * d[0] + (p[1] - a[1]) * d[1]) / l2).clamp(0.0, 1.0) } else { 0.0 }; + (p[0] - a[0] - t * d[0]).hypot(p[1] - a[1] - t * d[1]) +} + +/// Colour of a synthetic face at face-frame point `q` (`None` outside the face). +fn face_color(p: &FaceParams, s: &[[f64; 2]; N], q: [f64; 2]) -> Option<[f32; 3]> { + if (q[0] / ASPECT).powi(2) + q[1] * q[1] > 1.0 { + return None; + } + let shade = (0.95 - 0.08 * q[1]) as f32; + let mut col = [0.86 * shade, 0.64 * shade, 0.52 * shade]; + for side in 0..2 { + let b = BROW[side]; + let d = seg_dist(q, s[b[0]], s[b[1]]).min(seg_dist(q, s[b[1]], s[b[2]])); + if d < 0.035 { + col = [0.28, 0.18, 0.12]; + } + let e = EYE[side]; + let ec = [(s[e[0]][0] + s[e[1]][0]) / 2.0, (s[e[2]][1] + s[e[3]][1]) / 2.0]; + let (hw, hh) = ((s[e[1]][0] - s[e[0]][0]).abs() / 2.0, (s[e[3]][1] - s[e[2]][1]).abs() / 2.0); + if ((q[0] - ec[0]) / hw).powi(2) + ((q[1] - ec[1]) / hh).powi(2) <= 1.0 { + let pupil = s[e[4]]; + col = if (q[0] - pupil[0]).hypot(q[1] - pupil[1]) < 0.055 { [0.12, 0.08, 0.06] } else { [0.95, 0.95, 0.93] }; + } + } + let _ = p; + for &n in &NOSTRIL { + if ((q[0] - s[n][0]) / 0.035).powi(2) + ((q[1] - s[n][1]) / 0.022).powi(2) <= 1.0 { + col = [0.3, 0.13, 0.1]; + } + } + let m = MOUTH; + let mc = [(s[m[0]][0] + s[m[1]][0]) / 2.0, (s[m[2]][1] + s[m[3]][1]) / 2.0]; + let (mw, mh) = ((s[m[1]][0] - s[m[0]][0]) / 2.0, (s[m[3]][1] - s[m[2]][1]) / 2.0); + if ((q[0] - mc[0]) / mw).powi(2) + ((q[1] - mc[1]) / mh).powi(2) <= 1.0 { + col = [0.42, 0.08, 0.1]; + } + Some(col) +} + +/// Render a synthetic face over a textured blue-green background (2 × 2 supersampled): the +/// test footage of the face tracker. +pub fn render_synthetic(w: u32, h: u32, p: &FaceParams, background_seed: i64) -> Image { + use rayon::prelude::*; + let s = synth_shape(p); + let mut img = Image::new(w, h); + img.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let mut acc = [0.0f32; 3]; + for sy in 0..2 { + for sx in 0..2 { + let l = [x as f64 + 0.25 + 0.5 * sx as f64, y as f64 + 0.25 + 0.5 * sy as f64]; + let q = to_face(p.center, p.height, p.roll, l); + let c = face_color(p, &s, q).unwrap_or_else(|| { + let n = hash((l[0] / 9.0) as i64 + background_seed, (l[1] / 9.0) as i64) as f32; + [0.15 + 0.1 * n, 0.35 + 0.2 * n, 0.45 + 0.25 * n] + }); + for k in 0..3 { + acc[k] += c[k] * 0.25; + } + } + } + *px = [acc[0], acc[1], acc[2], 1.0]; + } + }); + img +} + +// ---------------------------------------------------------------- shape model + +/// Point distribution model: mean shape and principal modes (face frame). +pub struct Pdm { + pub mean: Vec, + /// Modes as rows of length `2N`. + pub modes: Vec>, + /// Variance of each mode. + pub var: Vec, +} + +/// The shape model, trained once on synthetic shapes. +pub fn pdm() -> &'static Pdm { + static P: OnceLock = OnceLock::new(); + P.get_or_init(|| { + let samples = 600; + let dim = 2 * N; + let mut rng = 0x2545_f491_4f6c_dd1du64; + let mut u = || { + rng ^= rng << 13; + rng ^= rng >> 7; + rng ^= rng << 17; + (rng % 1_000_000) as f64 / 1_000_000.0 + }; + let mut data = Vec::with_capacity(samples); + for _ in 0..samples { + let p = FaceParams { + yaw: (u() - 0.5) * 60.0, + pitch: (u() - 0.5) * 40.0, + eye_open: [u(), u()], + mouth_open: u(), + mouth_width: 0.8 + 0.45 * u(), + brow_raise: [u() * 2.0 - 1.0, u() * 2.0 - 1.0], + ..Default::default() + }; + data.push(synth_shape(&p).iter().flat_map(|q| [q[0], q[1]]).collect::>()); + } + let mut mean = vec![0.0; dim]; + for d in &data { + for (m, v) in mean.iter_mut().zip(d) { + *m += v / samples as f64; + } + } + let mut cov = vec![0.0; dim * dim]; + for d in &data { + for a in 0..dim { + let da = d[a] - mean[a]; + for b in 0..dim { + cov[a * dim + b] += da * (d[b] - mean[b]) / samples as f64; + } + } + } + let (vals, vecs) = sym_eigen(&cov, dim); + let total: f64 = vals.iter().map(|v| v.max(0.0)).sum(); + let mut modes = vec![]; + let mut var = vec![]; + let mut acc = 0.0; + for k in (0..dim).rev() { + if acc >= 0.995 * total || vals[k] <= 1e-12 { + break; + } + acc += vals[k]; + modes.push((0..dim).map(|r| vecs[r * dim + k]).collect()); + var.push(vals[k]); + } + Pdm { mean, modes, var } + }) +} + +/// A similarity `p ↦ s R q + t` as `[a, b, tx, ty]` (`x = a u − b v + tx`, `y = b u + a v + ty`). +fn fit_similarity(src: &[[f64; 2]], dst: &[[f64; 2]], w: &[f64]) -> Option<[f64; 4]> { + let ws: f64 = w.iter().sum(); + if ws < 2.0 - 1e-9 { + return None; + } + let mut cs = [0.0; 2]; + let mut cd = [0.0; 2]; + for i in 0..src.len() { + for k in 0..2 { + cs[k] += w[i] * src[i][k] / ws; + cd[k] += w[i] * dst[i][k] / ws; + } + } + let (mut ss, mut dt, mut cr) = (0.0, 0.0, 0.0); + for i in 0..src.len() { + let a = [src[i][0] - cs[0], src[i][1] - cs[1]]; + let b = [dst[i][0] - cd[0], dst[i][1] - cd[1]]; + ss += w[i] * (a[0] * a[0] + a[1] * a[1]); + dt += w[i] * (a[0] * b[0] + a[1] * b[1]); + cr += w[i] * (a[0] * b[1] - a[1] * b[0]); + } + if ss < 1e-12 { + return None; + } + let (a, b) = (dt / ss, cr / ss); + Some([a, b, cd[0] - (a * cs[0] - b * cs[1]), cd[1] - (b * cs[0] + a * cs[1])]) +} + +fn sim_apply(t: &[f64; 4], q: [f64; 2]) -> [f64; 2] { + [t[0] * q[0] - t[1] * q[1] + t[2], t[1] * q[0] + t[0] * q[1] + t[3]] +} + +fn sim_inverse(t: &[f64; 4], p: [f64; 2]) -> [f64; 2] { + let d = [p[0] - t[2], p[1] - t[3]]; + let s2 = t[0] * t[0] + t[1] * t[1]; + [(t[0] * d[0] + t[1] * d[1]) / s2, (-t[1] * d[0] + t[0] * d[1]) / s2] +} + +/// Fit the shape model to detected landmarks (`w` = 0 for missing ones), starting from the +/// face-frame similarity `init`. Returns (model landmarks in layer pixels, similarity). +pub fn fit_shape(detected: &[[f64; 2]; N], w: &[f64; N], init: [f64; 4]) -> ([[f64; 2]; N], [f64; 4]) { + let m = pdm(); + let k = m.modes.len(); + let mut b = vec![0.0; k]; + let mut t = init; + let shape = |b: &[f64]| -> [[f64; 2]; N] { + let mut s = [[0.0; 2]; N]; + for i in 0..N { + for d in 0..2 { + let j = 2 * i + d; + s[i][d] = m.mean[j] + m.modes.iter().zip(b).map(|(row, bk)| row[j] * bk).sum::(); + } + } + s + }; + for _ in 0..10 { + let x = shape(&b); + if let Some(nt) = fit_similarity(&x, detected, w) { + t = nt; + } + // Detections in the model frame. + let y: Vec<[f64; 2]> = detected.iter().map(|p| sim_inverse(&t, *p)).collect(); + let mut ata = vec![0.0; k * k]; + let mut atb = vec![0.0; k]; + for i in 0..N { + if w[i] <= 0.0 { + continue; + } + for d in 0..2 { + let j = 2 * i + d; + let r = y[i][d] - m.mean[j]; + for a in 0..k { + atb[a] += w[i] * m.modes[a][j] * r; + for c in 0..k { + ata[a * k + c] += w[i] * m.modes[a][j] * m.modes[c][j]; + } + } + } + } + for a in 0..k { + // A weak prior towards the mean (Tikhonov, scaled by the mode's variance). + ata[a * k + a] += 1e-3 / m.var[a].max(1e-9) * 1e-3 + 1e-9; + } + if cholesky_solve(&mut ata, k, &mut atb) { + for a in 0..k { + let lim = 3.0 * m.var[a].sqrt(); + b[a] = atb[a].clamp(-lim, lim); + } + } + } + let x = shape(&b); + (x.map(|q| sim_apply(&t, q)), t) +} + +// ---------------------------------------------------------------- tracking + +/// The skin colour model (luma, r and g chromaticity: median and spread). +#[derive(Clone, Copy, Debug, PartialEq)] +struct Skin { + med: [f32; 3], + tol: [f32; 3], +} + +fn features(px: [f32; 4]) -> [f32; 3] { + let s = (px[0] + px[1] + px[2]).max(1e-4); + [0.2126 * px[0] + 0.7152 * px[1] + 0.0722 * px[2], px[0] / s, px[1] / s] +} + +impl Skin { + fn is_skin(&self, px: [f32; 4]) -> bool { + let f = features(px); + (0..3).all(|k| (f[k] - self.med[k]).abs() <= self.tol[k]) + } +} + +/// One frame's face fit. +#[derive(Clone, Debug, PartialEq)] +pub struct FaceFit { + /// Landmarks ([`LANDMARKS`] order), layer pixels. + pub landmarks: [[f64; 2]; N], + /// Which landmarks were measured on this frame (the others come from the shape model). + pub detected: [bool; N], + /// The face outline: a closed polygon (layer pixels). + pub outline: Vec<[f64; 2]>, + /// Face ellipse centre, half-height (pixels) and roll (degrees). + pub center: [f64; 2], + pub height: f64, + pub roll: f64, +} + +/// Follows one face through a clip. +#[derive(Clone, Debug)] +pub struct FaceTracker { + skin: Skin, + prev: FaceFit, + /// Previous landmarks in the face frame (the expected feature positions). + expect: [[f64; 2]; N], +} + +/// A pixel window of a frame in layer coordinates. +struct Window { + x0: i64, + y0: i64, + w: usize, + h: usize, +} + +impl Window { + fn around(frame: &Frame, c: [f64; 2], rx: f64, ry: f64) -> Option { + let (ox, oy) = (frame.offset[0], frame.offset[1]); + let x0 = ((c[0] - rx + ox).floor() as i64).max(0); + let y0 = ((c[1] - ry + oy).floor() as i64).max(0); + let x1 = ((c[0] + rx + ox).ceil() as i64).min(frame.img.width as i64); + let y1 = ((c[1] + ry + oy).ceil() as i64).min(frame.img.height as i64); + (x1 > x0 + 4 && y1 > y0 + 4).then(|| Window { x0, y0, w: (x1 - x0) as usize, h: (y1 - y0) as usize }) + } + fn px(&self, frame: &Frame, i: usize, j: usize) -> [f32; 4] { + frame.img.get(self.x0 + i as i64, self.y0 + j as i64) + } + /// Layer position of the centre of window pixel `(i, j)`. + fn layer(&self, frame: &Frame, i: usize, j: usize) -> [f64; 2] { + [(self.x0 + i as i64) as f64 + 0.5 - frame.offset[0], (self.y0 + j as i64) as f64 + 0.5 - frame.offset[1]] + } + fn cell(&self, frame: &Frame, p: [f64; 2]) -> Option<(usize, usize)> { + let x = (p[0] + frame.offset[0]).floor() as i64 - self.x0; + let y = (p[1] + frame.offset[1]).floor() as i64 - self.y0; + (x >= 0 && y >= 0 && (x as usize) < self.w && (y as usize) < self.h).then_some((x as usize, y as usize)) + } +} + +fn median(v: &mut [f32]) -> f32 { + v.sort_by(f32::total_cmp); + v[v.len() / 2] +} + +/// Area-moment ellipse of a closed polygon: (centroid, half-height along the major axis, +/// half-width, major axis angle from vertical in degrees). +fn polygon_ellipse(poly: &[[f64; 2]]) -> Option<([f64; 2], f64, f64, f64)> { + let n = poly.len(); + let (mut a, mut cx, mut cy) = (0.0, 0.0, 0.0); + let (mut ixx, mut iyy, mut ixy) = (0.0, 0.0, 0.0); + for i in 0..n { + let (p, q) = (poly[i], poly[(i + 1) % n]); + let cr = p[0] * q[1] - q[0] * p[1]; + a += cr; + cx += (p[0] + q[0]) * cr; + cy += (p[1] + q[1]) * cr; + ixx += (p[1] * p[1] + p[1] * q[1] + q[1] * q[1]) * cr; + iyy += (p[0] * p[0] + p[0] * q[0] + q[0] * q[0]) * cr; + ixy += (p[0] * q[1] + 2.0 * p[0] * p[1] + 2.0 * q[0] * q[1] + q[0] * p[1]) * cr; + } + a *= 0.5; + if a.abs() < 1e-9 { + return None; + } + cx /= 6.0 * a; + cy /= 6.0 * a; + // Central second moments (per unit area): var x, var y, cov xy. + let vxx = iyy / (12.0 * a) - cx * cx; + let vyy = ixx / (12.0 * a) - cy * cy; + let vxy = ixy / (24.0 * a) - cx * cy; + let tr = vxx + vyy; + let det = vxx * vyy - vxy * vxy; + let disc = (tr * tr / 4.0 - det).max(0.0).sqrt(); + let (l1, l2) = (tr / 2.0 + disc, (tr / 2.0 - disc).max(0.0)); + // Major axis direction. + let ang = 0.5 * (2.0 * vxy).atan2(vxx - vyy); // from +x + let mut from_vertical = ang.to_degrees() - 90.0; + while from_vertical > 90.0 { + from_vertical -= 180.0; + } + while from_vertical < -90.0 { + from_vertical += 180.0; + } + Some(([cx, cy], 2.0 * l1.sqrt(), 2.0 * l2.sqrt(), from_vertical)) +} + +/// A non-skin component inside the face. +struct Blob { + /// Face-frame positions of its pixels and their darkness (skin luma − luma). + pts: Vec<([f64; 2], f64)>, + centroid: [f64; 2], +} + +impl Blob { + fn extreme(&self, dir: [f64; 2]) -> [f64; 2] { + self.pts.iter().map(|p| p.0).max_by(|a, b| (a[0] * dir[0] + a[1] * dir[1]).total_cmp(&(b[0] * dir[0] + b[1] * dir[1]))).unwrap_or(self.centroid) + } + /// Extreme along `v` (`sign` −1 = top) among the pixels within `band` of the column `u`. + fn lid(&self, u: f64, band: f64, sign: f64) -> [f64; 2] { + let c = self.pts.iter().filter(|p| (p.0[0] - u).abs() <= band).map(|p| p.0); + let best = c.max_by(|a, b| (sign * a[1]).total_cmp(&(sign * b[1]))); + best.map(|p| [u, p[1]]).unwrap_or(self.centroid) + } + fn dark_centroid(&self) -> [f64; 2] { + let (mut s, mut w) = ([0.0; 2], 0.0); + for (q, d) in &self.pts { + let k = d.max(0.0).powi(2); + s[0] += k * q[0]; + s[1] += k * q[1]; + w += k; + } + if w > 1e-12 { [s[0] / w, s[1] / w] } else { self.centroid } + } +} + +impl FaceTracker { + /// Start on `frame` with the face inside `region` (`[x0, y0, x1, y1]`, layer pixels: the + /// mask's bounds). `None` when no face-like skin region is found. + pub fn new(frame: &Frame, region: [f64; 4]) -> Option<(FaceTracker, FaceFit)> { + let c = [(region[0] + region[2]) / 2.0, (region[1] + region[3]) / 2.0]; + let (hw, hh) = ((region[2] - region[0]) / 2.0, (region[3] - region[1]) / 2.0); + if !(hw > 4.0 && hh > 4.0) { + return None; + } + // Skin colour: the middle of the region (cheeks and nose). + let win = Window::around(frame, c, hw * 0.35, hh * 0.3)?; + let mut f: [Vec; 3] = [vec![], vec![], vec![]]; + for j in 0..win.h { + for i in 0..win.w { + let v = features(win.px(frame, i, j)); + for k in 0..3 { + f[k].push(v[k]); + } + } + } + let mut med = [0.0f32; 3]; + let mut tol = [0.0f32; 3]; + let floors = [0.12f32, 0.025, 0.025]; + for k in 0..3 { + med[k] = median(&mut f[k]); + let mut dev: Vec = f[k].iter().map(|v| (v - med[k]).abs()).collect(); + tol[k] = (4.0 * median(&mut dev)).max(floors[k]); + } + let skin = Skin { med, tol }; + let canon: [[f64; 2]; N] = LANDMARKS.map(|l| l.2); + let guess = FaceFit { + landmarks: canon.map(|q| to_layer(c, hh, 0.0, q)), + detected: [false; N], + outline: vec![], + center: c, + height: hh.max(hw / ASPECT), + roll: 0.0, + }; + let mut t = FaceTracker { skin, prev: guess, expect: canon }; + let fit = t.fit(frame)?; + t.accept(&fit); + Some((t, fit)) + } + + /// The fit of the last tracked frame. + pub fn last(&self) -> &FaceFit { + &self.prev + } + + fn accept(&mut self, fit: &FaceFit) { + self.expect = fit.landmarks.map(|p| to_face(fit.center, fit.height, fit.roll, p)); + self.prev = fit.clone(); + } + + /// Track into the next frame (either direction). `None` when the face is lost. + pub fn step(&mut self, frame: &Frame) -> Option { + let fit = self.fit(frame)?; + self.accept(&fit); + Some(fit) + } + + fn fit(&self, frame: &Frame) -> Option { + let pv = &self.prev; + let b0 = pv.height; + let reach = 1.7 * b0; + let win = Window::around(frame, pv.center, reach, reach)?; + let (w, h) = (win.w, win.h); + let mut skin = vec![false; w * h]; + let mut luma = vec![0.0f32; w * h]; + for j in 0..h { + for i in 0..w { + let px = win.px(frame, i, j); + skin[j * w + i] = self.skin.is_skin(px); + luma[j * w + i] = features(px)[0]; + } + } + // Seed: the skin pixel nearest the predicted centre. + let (ci, cj) = win.cell(frame, pv.center)?; + let mut seed = None; + 'search: for r in 0..(0.6 * b0).ceil() as i64 { + for dj in -r..=r { + for di in -r..=r { + if di.abs() != r && dj.abs() != r { + continue; + } + let (i, j) = (ci as i64 + di, cj as i64 + dj); + if i >= 0 && j >= 0 && (i as usize) < w && (j as usize) < h && skin[j as usize * w + i as usize] { + seed = Some((i as usize, j as usize)); + break 'search; + } + } + } + } + let (si, sj) = seed?; + // Flood fill (4-connected). + let mut comp = vec![false; w * h]; + let mut stack = vec![(si, sj)]; + comp[sj * w + si] = true; + let mut count = 0usize; + let (mut mx, mut my) = (0.0, 0.0); + while let Some((i, j)) = stack.pop() { + count += 1; + let p = win.layer(frame, i, j); + mx += p[0]; + my += p[1]; + let nb = [(i.wrapping_sub(1), j), (i + 1, j), (i, j.wrapping_sub(1)), (i, j + 1)]; + for (a, b) in nb { + if a < w && b < h && !comp[b * w + a] && skin[b * w + a] { + comp[b * w + a] = true; + stack.push((a, b)); + } + } + } + if (count as f64) < 0.15 * std::f64::consts::PI * ASPECT * b0 * b0 { + return None; + } + let c0 = [mx / count as f64, my / count as f64]; + // Outline along rays: the farthest component pixel. + let rays = 48; + let mut radii = vec![0.0f64; rays]; + for (k, r) in radii.iter_mut().enumerate() { + let a = k as f64 / rays as f64 * std::f64::consts::TAU; + let d = [a.cos(), a.sin()]; + let mut t = 0.0; + while t < reach * 1.4 { + let p = [c0[0] + d[0] * t, c0[1] + d[1] * t]; + match win.cell(frame, p) { + Some((i, j)) => { + if comp[j * w + i] { + *r = t + 0.5; + } + } + None => break, + } + t += 0.5; + } + } + // Median of 5, circularly. + let sm: Vec = (0..rays) + .map(|k| { + let mut v: Vec = (0..5).map(|d| radii[(k + rays + d - 2) % rays]).collect(); + v.sort_by(f64::total_cmp); + v[2] + }) + .collect(); + let outline: Vec<[f64; 2]> = (0..rays) + .map(|k| { + let a = k as f64 / rays as f64 * std::f64::consts::TAU; + [c0[0] + a.cos() * sm[k], c0[1] + a.sin() * sm[k]] + }) + .collect(); + let (center, major, _minor, ang) = polygon_ellipse(&outline)?; + if !(major > 0.3 * b0 && major < 2.5 * b0) { + return None; + } + // Roll: the ellipse axis nearest the previous roll (it is ambiguous by 180°). + let mut roll = ang; + while roll - pv.roll > 90.0 { + roll -= 180.0; + } + while roll - pv.roll < -90.0 { + roll += 180.0; + } + let b = major; + // Non-skin components inside the face outline (shrunk a little: anti-aliased edges). + let inner: Vec<[f64; 2]> = outline.iter().map(|p| [center[0] + 0.9 * (p[0] - center[0]), center[1] + 0.9 * (p[1] - center[1])]).collect(); + let mut feat = vec![false; w * h]; + for j in 0..h { + for i in 0..w { + if !skin[j * w + i] && crate::mask::inside_polygon(&inner, win.layer(frame, i, j)) { + feat[j * w + i] = true; + } + } + } + let min_area = (0.0006 * b * b).max(2.0) as usize; + let mut seen = vec![false; w * h]; + let mut blobs: Vec = vec![]; + let skin_luma = self.skin.med[0] as f64; + for j0 in 0..h { + for i0 in 0..w { + if !feat[j0 * w + i0] || seen[j0 * w + i0] { + continue; + } + let mut st = vec![(i0, j0)]; + seen[j0 * w + i0] = true; + let mut pts = vec![]; + while let Some((i, j)) = st.pop() { + let q = to_face(center, b, roll, win.layer(frame, i, j)); + pts.push((q, skin_luma - luma[j * w + i] as f64)); + for dj in -1i64..=1 { + for di in -1i64..=1 { + let (a, c) = (i as i64 + di, j as i64 + dj); + if a < 0 || c < 0 || a as usize >= w || c as usize >= h { + continue; + } + let (a, c) = (a as usize, c as usize); + if feat[c * w + a] && !seen[c * w + a] { + seen[c * w + a] = true; + st.push((a, c)); + } + } + } + } + if pts.len() >= min_area { + let n = pts.len() as f64; + let centroid = [pts.iter().map(|p| p.0[0]).sum::() / n, pts.iter().map(|p| p.0[1]).sum::() / n]; + blobs.push(Blob { pts, centroid }); + } + } + } + // Match components to features near their expected positions. + let ex = &self.expect; + let mean_of = |ids: &[usize]| { + let n = ids.len() as f64; + [ids.iter().map(|i| ex[*i][0]).sum::() / n, ids.iter().map(|i| ex[*i][1]).sum::() / n] + }; + let mut used = vec![false; blobs.len()]; + let take = |target: [f64; 2], radius: f64, prefer_large: bool, used: &mut Vec| -> Option { + let mut best: Option<(usize, f64)> = None; + for (k, bl) in blobs.iter().enumerate() { + if used[k] { + continue; + } + let d = (bl.centroid[0] - target[0]).hypot(bl.centroid[1] - target[1]); + if d > radius { + continue; + } + let score = if prefer_large { -(bl.pts.len() as f64) } else { d }; + if best.is_none_or(|b| score < b.1) { + best = Some((k, score)); + } + } + if let Some((k, _)) = best { + used[k] = true; + } + best.map(|b| b.0) + }; + let mut det = [[0.0; 2]; N]; + let mut wgt = [0.0; N]; + let set = |i: usize, q: [f64; 2], det: &mut [[f64; 2]; N], wgt: &mut [f64; N]| { + det[i] = to_layer(center, b, roll, q); + wgt[i] = 1.0; + }; + let eye_ids = [EYE[0].to_vec(), EYE[1].to_vec()]; + let mouth = take(mean_of(&MOUTH), 0.25, true, &mut used); + let eyes = [take(mean_of(&eye_ids[0]), 0.15, false, &mut used), take(mean_of(&eye_ids[1]), 0.15, false, &mut used)]; + let brows = [take(mean_of(&BROW[0]), 0.15, false, &mut used), take(mean_of(&BROW[1]), 0.15, false, &mut used)]; + let nostrils = [take(ex[NOSTRIL[0]], 0.07, false, &mut used), take(ex[NOSTRIL[1]], 0.07, false, &mut used)]; + for side in 0..2 { + let sg = if side == 0 { -1.0 } else { 1.0 }; + if let Some(k) = eyes[side] { + let bl = &blobs[k]; + let e = EYE[side]; + let inner = bl.extreme([-sg, 0.0]); + let outer = bl.extreme([sg, 0.0]); + let mid = 0.5 * (inner[0] + outer[0]); + let band = 0.15 * (outer[0] - inner[0]).abs(); + let px = 0.5 / b; + set(e[0], [inner[0] - sg * px, inner[1]], &mut det, &mut wgt); + set(e[1], [outer[0] + sg * px, outer[1]], &mut det, &mut wgt); + let top = bl.lid(mid, band, -1.0); + let bot = bl.lid(mid, band, 1.0); + set(e[2], [top[0], top[1] - px], &mut det, &mut wgt); + set(e[3], [bot[0], bot[1] + px], &mut det, &mut wgt); + set(e[4], bl.dark_centroid(), &mut det, &mut wgt); + } + if let Some(k) = brows[side] { + let bl = &blobs[k]; + let br = BROW[side]; + let inner = bl.extreme([-sg, 0.0]); + let outer = bl.extreme([sg, 0.0]); + set(br[0], inner, &mut det, &mut wgt); + set(br[2], outer, &mut det, &mut wgt); + // The middle: the brow's centre line at its mid column. + let mid = 0.5 * (inner[0] + outer[0]); + let band = 0.1 * (outer[0] - inner[0]).abs().max(0.05); + let col: Vec = bl.pts.iter().filter(|p| (p.0[0] - mid).abs() <= band).map(|p| p.0[1]).collect(); + if !col.is_empty() { + set(br[1], [mid, col.iter().sum::() / col.len() as f64], &mut det, &mut wgt); + } + } + if let Some(k) = nostrils[side] { + set(NOSTRIL[side], blobs[k].centroid, &mut det, &mut wgt); + } + } + if let (Some(a), Some(c)) = (nostrils[0], nostrils[1]) { + let (p, q) = (blobs[a].centroid, blobs[c].centroid); + set(NOSE_TIP, [0.5 * (p[0] + q[0]), 0.5 * (p[1] + q[1]) - 0.06], &mut det, &mut wgt); + } + if let Some(k) = mouth { + let bl = &blobs[k]; + let l = bl.extreme([-1.0, 0.0]); + let r = bl.extreme([1.0, 0.0]); + let px = 0.5 / b; + set(MOUTH[0], [l[0] - px, l[1]], &mut det, &mut wgt); + set(MOUTH[1], [r[0] + px, r[1]], &mut det, &mut wgt); + let mid = 0.5 * (l[0] + r[0]); + let band = 0.12 * (r[0] - l[0]).abs(); + let top = bl.lid(mid, band, -1.0); + let bot = bl.lid(mid, band, 1.0); + set(MOUTH[2], [top[0], top[1] - px], &mut det, &mut wgt); + set(MOUTH[3], [bot[0], bot[1] + px], &mut det, &mut wgt); + } + // Chin and jaw from the outline, along their canonical directions. + for i in [CHIN, JAW[0], JAW[1]] { + let q = LANDMARKS[i].2; + let dir = to_layer([0.0; 2], 1.0, roll, q); + let ang = dir[1].atan2(dir[0]); + let rad = ray_radius(&outline, center, ang); + let len = dir[0].hypot(dir[1]); + let p = [center[0] + dir[0] / len * rad, center[1] + dir[1] / len * rad]; + det[i] = p; + wgt[i] = 1.0; + } + // Shape model: fill the missing landmarks and replace inconsistent detections. + let (sn, cs) = roll.to_radians().sin_cos(); + let init = [b * cs, b * sn, center[0], center[1]]; + let (model, _) = fit_shape(&det, &wgt, init); + let mut w2 = wgt; + for i in 0..N { + if wgt[i] > 0.0 && (det[i][0] - model[i][0]).hypot(det[i][1] - model[i][1]) > 0.12 * b { + w2[i] = 0.0; + } + } + let (model, sim) = if w2 != wgt { fit_shape(&det, &w2, init) } else { (model, init) }; + let _ = sim; + let mut landmarks = model; + let mut detected = [false; N]; + for i in 0..N { + if w2[i] > 0.0 { + landmarks[i] = det[i]; + detected[i] = true; + } + } + // Roll from the jaw line (it does not move with yaw); fall back to the ellipse. + let jl = landmarks[JAW[0]]; + let jr = landmarks[JAW[1]]; + let roll_jaw = (jr[1] - jl[1]).atan2(jr[0] - jl[0]).to_degrees(); + let roll = if (roll_jaw - roll).abs() < 20.0 { roll_jaw } else { roll }; + Some(FaceFit { landmarks, detected, outline, center, height: b, roll }) + } +} + +/// Distance from `c` to the polygon `poly` (star-shaped around `c`) along angle `ang`. +fn ray_radius(poly: &[[f64; 2]], c: [f64; 2], ang: f64) -> f64 { + let d = [ang.cos(), ang.sin()]; + let mut best = 0.0f64; + let n = poly.len(); + for i in 0..n { + let (p, q) = (poly[i], poly[(i + 1) % n]); + let e = [q[0] - p[0], q[1] - p[1]]; + let den = d[0] * e[1] - d[1] * e[0]; + if den.abs() < 1e-12 { + continue; + } + let w = [p[0] - c[0], p[1] - c[1]]; + let t = (w[0] * e[1] - w[1] * e[0]) / den; + let s = (w[0] * d[1] - w[1] * d[0]) / den; + if t > 0.0 && (-1e-9..=1.0 + 1e-9).contains(&s) { + best = best.max(t); + } + } + best +} + +// ---------------------------------------------------------------- measurements + +/// Face measurements (Extract & Copy Face Measurements): (id, display name). +pub const MEASUREMENTS: [(&str, &str); 14] = [ + ("headPositionX", "Head Position X"), + ("headPositionY", "Head Position Y"), + ("headScale", "Head Scale"), + ("headOrientationX", "Head Orientation X"), + ("headOrientationY", "Head Orientation Y"), + ("headOrientationZ", "Head Orientation Z"), + ("leftEyeOpenness", "Left Eye Openness"), + ("rightEyeOpenness", "Right Eye Openness"), + ("leftEyebrowRaise", "Left Eyebrow Distance from Eye"), + ("rightEyebrowRaise", "Right Eyebrow Distance from Eye"), + ("mouthOpenness", "Mouth Openness"), + ("mouthWidth", "Mouth Width"), + ("mouthOffsetX", "Mouth Offset X"), + ("mouthOffsetY", "Mouth Offset Y"), +]; + +/// Face centre, half-height and roll from the chin and jaw landmarks. +pub fn face_frame(l: &[[f64; 2]; N]) -> ([f64; 2], f64, f64) { + let (ch, jl, jr) = (l[CHIN], l[JAW[0]], l[JAW[1]]); + let jm = [0.5 * (jl[0] + jr[0]), 0.5 * (jl[1] + jr[1])]; + let k = 1.0 - LANDMARKS[JAW[0]].2[1]; + let c = [ch[0] + (jm[0] - ch[0]) / k, ch[1] + (jm[1] - ch[1]) / k]; + // Half-height from the chin and from the jaw width (averaged: less noise). + let jw = (jr[0] - jl[0]).hypot(jr[1] - jl[1]) / (2.0 * LANDMARKS[JAW[1]].2[0]); + let b = (0.5 * ((ch[0] - c[0]).hypot(ch[1] - c[1]) + jw)).max(1e-9); + let roll = (jr[1] - jl[1]).atan2(jr[0] - jl[0]).to_degrees(); + (c, b, roll) +} + +/// The measurements of a frame's landmarks ([`MEASUREMENTS`] order); `reference_height`: the +/// face half-height that is Head Scale 100 %. +pub fn measure(l: &[[f64; 2]; N], reference_height: f64) -> [f64; 14] { + let (c, b, roll) = face_frame(l); + let q = l.map(|p| to_face(c, b, roll, p)); + let canon = LANDMARKS.map(|x| x.2); + // Inner features slide with yaw and pitch (see `synth_shape`). + let inner = [EYE[0][4], EYE[1][4], MOUTH[0], MOUTH[1], EYE[0][0], EYE[1][0], EYE[0][1], EYE[1][1]]; + let n = inner.len() as f64; + let du = inner.iter().map(|i| q[*i][0] - canon[*i][0]).sum::() / n; + let dv = inner.iter().map(|i| q[*i][1] - canon[*i][1]).sum::() / n; + let yaw = (du / 0.35).clamp(-1.0, 1.0).asin().to_degrees(); + let pitch = (dv / 0.3).clamp(-1.0, 1.0).asin().to_degrees(); + let open = |e: [usize; 5]| { + let wdt = (q[e[1]][0] - q[e[0]][0]).abs().max(1e-9); + (q[e[3]][1] - q[e[2]][1]).max(0.0) / wdt * 100.0 + }; + let brow = |side: usize| { + let e = EYE[side]; + let eye_v = 0.5 * (q[e[2]][1] + q[e[3]][1]); + (eye_v - q[BROW[side][1]][1]) * 100.0 + }; + let mw = (q[MOUTH[1]][0] - q[MOUTH[0]][0]).abs(); + let mc = [0.5 * (q[MOUTH[0]][0] + q[MOUTH[1]][0]), 0.5 * (q[MOUTH[2]][1] + q[MOUTH[3]][1])]; + [ + c[0], + c[1], + b / reference_height.max(1e-9) * 100.0, + pitch, + yaw, + roll, + open(EYE[0]), + open(EYE[1]), + brow(0), + brow(1), + (q[MOUTH[3]][1] - q[MOUTH[2]][1]).max(0.0) / mw.max(1e-9) * 100.0, + mw / (2.0 * ASPECT) * 100.0, + (mc[0] - q[NOSE_TIP][0]) * 100.0, + (mc[1] - q[NOSE_TIP][1]) * 100.0, + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + fn frame_params(t: usize) -> FaceParams { + let tf = t as f64; + FaceParams { + center: [160.0 + 30.0 * (tf * 0.15).sin(), 120.0 + 12.0 * (tf * 0.1).cos()], + height: 62.0 * (1.0 + 0.08 * (tf * 0.07).sin()), + roll: 14.0 * (tf * 0.12).sin(), + yaw: 12.0 * (tf * 0.09).sin(), + pitch: 6.0 * (tf * 0.11).cos(), + eye_open: [0.8 + 0.2 * (tf * 0.3).sin(), 0.8 + 0.2 * (tf * 0.3).sin()], + mouth_open: 0.5 + 0.45 * (tf * 0.2).sin(), + mouth_width: 1.0 + 0.1 * (tf * 0.13).cos(), + brow_raise: [0.4 * (tf * 0.17).sin(), 0.4 * (tf * 0.17).sin()], + } + } + + #[test] + fn shape_model_spans_the_synthetic_shapes() { + let m = pdm(); + assert!(m.modes.len() >= 5 && m.modes.len() <= 14, "{} modes", m.modes.len()); + // A synthetic shape is reproduced from half its landmarks. + let p = FaceParams { yaw: 15.0, mouth_open: 0.8, eye_open: [0.3, 0.9], center: [100.0, 80.0], height: 50.0, roll: 10.0, ..Default::default() }; + let truth = synth_landmarks(&p); + let mut w = [0.0; N]; + for (i, wi) in w.iter_mut().enumerate() { + *wi = if i % 2 == 0 || i >= 19 { 1.0 } else { 0.0 }; + } + let (sn, cs) = 10f64.to_radians().sin_cos(); + let (fit, _) = fit_shape(&truth, &w, [50.0 * cs, 50.0 * sn, 100.0, 80.0]); + let err = (0..N).map(|i| (fit[i][0] - truth[i][0]).hypot(fit[i][1] - truth[i][1])).fold(0.0, f64::max); + assert!(err < 0.06 * 50.0, "max error {err}"); + } + + #[test] + fn tracks_a_moving_rotating_synthetic_face() { + let (w, h) = (320, 240); + let p0 = frame_params(0); + let img0 = render_synthetic(w, h, &p0, 3); + // The user's mask: a loose box around the face. + let region = [p0.center[0] - 60.0, p0.center[1] - 75.0, p0.center[0] + 60.0, p0.center[1] + 75.0]; + let (mut tr, fit0) = FaceTracker::new(&Frame { img: &img0, offset: [0.0; 2] }, region).expect("face found"); + let mut fits = vec![fit0]; + let n = 40; + for t in 1..n { + let img = render_synthetic(w, h, &frame_params(t), 3); + fits.push(tr.step(&Frame { img: &img, offset: [0.0; 2] }).unwrap_or_else(|| panic!("lost at {t}"))); + } + let mut per = [0.0f64; N]; + let mut worst = 0.0f64; + let mut outline_err = 0.0; + let mut meas = vec![]; + for (t, f) in fits.iter().enumerate() { + let p = frame_params(t); + let truth = synth_landmarks(&p); + for i in 0..N { + let e = (f.landmarks[i][0] - truth[i][0]).hypot(f.landmarks[i][1] - truth[i][1]) / p.height; + per[i] += e / n as f64; + worst = worst.max(e); + } + // Outline: on the face ellipse. + let mut oe = 0.0; + for q in &f.outline { + let u = to_face(p.center, p.height, p.roll, *q); + oe += ((u[0] / ASPECT).powi(2) + u[1] * u[1]).sqrt() - 1.0; + } + outline_err += (oe / f.outline.len() as f64).abs() / n as f64; + meas.push((p, measure(&f.landmarks, fits[0].height))); + assert!((f.roll - p.roll).abs() < 3.0, "frame {t}: roll {} vs {}", f.roll, p.roll); + } + let mean = per.iter().sum::() / N as f64; + eprintln!("face landmarks: mean error {mean:.4}, worst {worst:.4} face heights; outline {outline_err:.4}"); + for (i, e) in per.iter().enumerate() { + assert!(*e < 0.04, "{}: mean error {e:.3} (face heights)", LANDMARKS[i].0); + } + assert!(mean < 0.02, "mean landmark error {mean:.4}"); + assert!(worst < 0.08, "worst landmark error {worst:.3}"); + assert!(outline_err < 0.04, "outline error {outline_err:.3}"); + // Measurements follow the generator: mouth openness and yaw correlate strongly. + let corr = |a: &[f64], b: &[f64]| { + let n = a.len() as f64; + let (ma, mb) = (a.iter().sum::() / n, b.iter().sum::() / n); + let (mut sab, mut saa, mut sbb) = (0.0, 0.0, 0.0); + for (x, y) in a.iter().zip(b) { + sab += (x - ma) * (y - mb); + saa += (x - ma).powi(2); + sbb += (y - mb).powi(2); + } + sab / (saa * sbb).sqrt() + }; + let mo: Vec = meas.iter().map(|m| m.1[10]).collect(); + let mt: Vec = meas.iter().map(|m| m.0.mouth_open).collect(); + assert!(corr(&mo, &mt) > 0.95, "mouth openness r = {}", corr(&mo, &mt)); + let yw: Vec = meas.iter().map(|m| m.1[4]).collect(); + let yt: Vec = meas.iter().map(|m| m.0.yaw).collect(); + assert!(corr(&yw, &yt) > 0.9, "yaw r = {}", corr(&yw, &yt)); + let rl: Vec = meas.iter().map(|m| m.1[5]).collect(); + let rt: Vec = meas.iter().map(|m| m.0.roll).collect(); + assert!(corr(&rl, &rt) > 0.98, "roll r = {}", corr(&rl, &rt)); + let sc: Vec = meas.iter().map(|m| m.1[2]).collect(); + let st: Vec = meas.iter().map(|m| m.0.height).collect(); + assert!(corr(&sc, &st) > 0.95, "scale r = {}", corr(&sc, &st)); + } + + #[test] + fn measurements_of_the_synthetic_model_are_exact() { + let p = FaceParams { center: [50.0, 60.0], height: 40.0, roll: 20.0, yaw: 10.0, pitch: -5.0, ..Default::default() }; + let m = measure(&synth_landmarks(&p), 40.0); + assert!((m[0] - 50.0).abs() < 1e-6 && (m[1] - 60.0).abs() < 1e-6, "{m:?}"); + assert!((m[2] - 100.0).abs() < 1e-6); + assert!((m[5] - 20.0).abs() < 1e-6); + assert!((m[4] - 10.0).abs() < 0.5 && (m[3] + 5.0).abs() < 0.5, "{m:?}"); + } +} diff --git a/crates/track/src/lib.rs b/crates/track/src/lib.rs index 1c6e6a15..f4a7b88f 100644 --- a/crates/track/src/lib.rs +++ b/crates/track/src/lib.rs @@ -19,6 +19,7 @@ //! after subtracting the frame's `offset` (see [`Frame`]). pub mod camtrack; +pub mod face; pub mod fit; pub mod klt; pub mod mask; @@ -26,6 +27,7 @@ pub mod plane; pub mod roto; pub mod solve; pub mod stabilize; +pub mod subspace; use effectcraft_raster::Image; use rayon::prelude::*; diff --git a/crates/track/src/solve.rs b/crates/track/src/solve.rs index 6c267721..693c52f7 100644 --- a/crates/track/src/solve.rs +++ b/crates/track/src/solve.rs @@ -37,6 +37,12 @@ pub fn solve_linear(mut a: [[f64; N]; N], mut b: [f64; N]) -> Op #[derive(Clone, Copy, Debug, PartialEq)] pub struct Homography(pub [[f64; 3]; 3]); +impl Default for Homography { + fn default() -> Self { + Homography::IDENTITY + } +} + impl Homography { pub const IDENTITY: Homography = Homography([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]); diff --git a/crates/track/src/stabilize.rs b/crates/track/src/stabilize.rs index 1abe79f9..9769fcbd 100644 --- a/crates/track/src/stabilize.rs +++ b/crates/track/src/stabilize.rs @@ -18,14 +18,23 @@ //! Video Stabilization with Robust L1 Optimal Camera Paths" (CVPR 2011), applied here //! per frame rather than through a linear program. //! +//! +//! *Subspace Warp* additionally follows long feature trajectories through the clip +//! ([`crate::camtrack::TrackAnalyzer`]) and stabilises each frame with a content-preserving mesh +//! warp fitted to the subspace-smoothed trajectories ([`crate::subspace`]): scenes with depth +//! (parallax) that one homography per frame cannot hold still are stabilised locally. The +//! framing (crop, auto-scale, Maximum Scale relaxation) works the same on meshes. +//! //! Coordinates are layer pixels; transforms map *source* frame pixels to *stabilized* pixels. use serde::{Deserialize, Serialize}; use crate::Frame; -use crate::fit::{Model, ransac}; +use crate::camtrack::{Track2D, TrackAnalyzer}; +use crate::fit::{Model, least_squares, ransac}; use crate::klt::{CornerOpts, GrayPyramid, LkOpts, analysis_factor, good_features, track}; use crate::solve::Homography; +use crate::subspace::{Mesh, MeshOpts, smooth_trajectories, solve_mesh}; /// Inter-frame motion of one frame (from the previous frame to this one) for each model. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -82,6 +91,10 @@ pub struct WarpAnalysis { pub detailed: bool, /// Per frame; frame 0's motion is the identity. pub frames: Vec, + /// Long feature trajectories (layer pixels, 0.1 px) for Subspace Warp; empty in analyses + /// made before it (which then stabilise like Perspective). + #[serde(skip_serializing_if = "Vec::is_empty")] + pub tracks: Vec, } impl WarpAnalysis { @@ -120,6 +133,7 @@ pub struct Analyzer { prev: Option, frames: Vec, size: [f64; 2], + trajectories: TrackAnalyzer, } fn round_pts(p: &[[f64; 2]], n: usize) -> Vec<[f32; 2]> { @@ -129,7 +143,8 @@ fn round_pts(p: &[[f64; 2]], n: usize) -> Vec<[f32; 2]> { impl Analyzer { pub fn new(size: [f64; 2], opts: AnalyzeOpts) -> Analyzer { - Analyzer { opts, prev: None, frames: vec![], size } + let (side, want) = if opts.detailed { (640, 260) } else { (480, 160) }; + Analyzer { opts, prev: None, frames: vec![], size, trajectories: TrackAnalyzer::with_budget(size, side, want) } } pub fn len(&self) -> usize { @@ -144,6 +159,7 @@ impl Analyzer { let (max_side, max_feat) = if self.opts.detailed { (960, 700) } else { (480, 350) }; let factor = analysis_factor(frame.img.width, frame.img.height, max_side); let pyr = GrayPyramid::from_image(frame.img, frame.offset, factor, 3); + self.trajectories.push(frame); let mut fm = FrameMotion::default(); if let Some(prev) = &self.prev && prev.width() == pyr.width() @@ -184,7 +200,17 @@ impl Analyzer { } pub fn finish(self, start: f64, frame_duration: f64) -> WarpAnalysis { - WarpAnalysis { version: 1, start, frame_duration, size: self.size, detailed: self.opts.detailed, frames: self.frames } + let mut tracks = self.trajectories.finish(start, frame_duration).tracks; + // Short trajectories hardly constrain the subspace: keep those of at least 6 frames, at + // 0.1 px (smaller JSON). + tracks.retain(|t| t.pts.len() >= 6); + for (i, t) in tracks.iter_mut().enumerate() { + t.id = i as u32; + for p in t.pts.iter_mut() { + *p = [(p[0] * 10.0).round() / 10.0, (p[1] * 10.0).round() / 10.0]; + } + } + WarpAnalysis { version: 2, start, frame_duration, size: self.size, detailed: self.opts.detailed, frames: self.frames, tracks } } } @@ -205,11 +231,31 @@ pub enum Method { /// Position, Scale, Rotation. Similarity, Perspective, - /// Approximated by Perspective (see the module docs of the effect). + /// Content-preserving mesh warps fitted to subspace-smoothed trajectories. #[default] SubspaceWarp, } +/// Advanced ▸ Rolling Shutter Ripple: how freely Subspace Warp's mesh may bend row by row (a +/// rolling-shutter camera exposes rows at different times, so shake wobbles the frame). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum Ripple { + /// A 12 × 8 mesh. + #[default] + Automatic, + /// Twice the rows and a softer shape term: follows row-wise wobble more closely. + Enhanced, +} + +impl Ripple { + pub fn mesh(self) -> MeshOpts { + match self { + Ripple::Automatic => MeshOpts { cols: 12, rows: 8, alpha: 1.0, prior: 0.02 }, + Ripple::Enhanced => MeshOpts { cols: 12, rows: 16, alpha: 0.5, prior: 0.02 }, + } + } +} + /// Borders ▸ Framing. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub enum Framing { @@ -239,6 +285,7 @@ pub struct StabSettings { pub crop_less_smooth_more: f64, /// Frames per second (for Smoothness). pub fps: f64, + pub ripple: Ripple, } impl Default for StabSettings { @@ -254,6 +301,7 @@ impl Default for StabSettings { additional_scale: 100.0, crop_less_smooth_more: 0.0, fps: 30.0, + ripple: Ripple::Automatic, } } } @@ -269,8 +317,15 @@ impl StabSettings { /// The per-frame stabilization of a clip. #[derive(Clone, Debug, Default)] pub struct Plan { - /// Final source → output transform per frame (layer pixels), framing scale included. + /// Final source → output transform per frame (layer pixels), framing scale included. For + /// mesh warps, the closest homography (for overlays and edge synthesis). pub warps: Vec, + /// Subspace Warp: per frame, the mesh (stabilised pixels before framing → source pixels); + /// empty for the other methods. + pub meshes: Vec, + /// The framing transform (auto-scale and Additional Scale about the centre) after the + /// stabilisation. + pub framing: Homography, /// The visible region in output pixels `[x0, y0, x1, y1]` (Stabilize, Crop / Auto-scale); /// `None` = everything the warp produces. pub crop: Option<[f64; 4]>, @@ -280,6 +335,24 @@ pub struct Plan { pub valid_fraction: f64, } +impl Plan { + /// The source position (frame `k`'s layer pixels) shown at output point `p`. + pub fn source_of(&self, k: usize, p: [f64; 2]) -> Option<[f64; 2]> { + if let Some(m) = self.meshes.get(k) { + let inv = self.framing.inverse()?; + return Some(m.sample(inv.apply(p))); + } + Some(self.warps.get(k)?.inverse()?.apply(p)) + } + /// Where source point `q` of frame `k` lands in the output. + pub fn output_of(&self, k: usize, q: [f64; 2]) -> Option<[f64; 2]> { + if let Some(m) = self.meshes.get(k) { + return m.forward(q).map(|p| self.framing.apply(p)); + } + Some(self.warps.get(k)?.apply(q)) + } +} + fn convex_contains(q: &[[f64; 2]; 4], p: [f64; 2]) -> bool { let mut sign = 0.0; for i in 0..4 { @@ -418,14 +491,100 @@ pub fn corrections(a: &WarpAnalysis, s: &StabSettings) -> Vec { out } +/// One frame's stabilisation before framing. +#[derive(Clone, Debug)] +enum Warp { + H(Homography), + M(Mesh), +} + +impl Warp { + fn valid(&self, size: [f64; 2]) -> f64 { + match self { + Warp::H(h) => valid_fraction(h, size), + Warp::M(m) => m.valid_fraction(), + } + } + /// Blend towards the identity (`l` = 1 keeps the warp). + fn relax(&self, l: f64) -> Warp { + match self { + Warp::H(h) => Warp::H(Homography::IDENTITY.lerp(h, l)), + Warp::M(m) => Warp::M(m.lerp_identity(l)), + } + } + fn homography(&self) -> Homography { + match self { + Warp::H(h) => *h, + Warp::M(m) => m.homography(), + } + } +} + +/// Remove the uniform scale of the correspondences' best similarity (Preserve Scale): the +/// wanted positions are scaled back about their centroid. +fn unscale(pairs: &mut [([f64; 2], [f64; 2])]) { + let (src, dst): (Vec<[f64; 2]>, Vec<[f64; 2]>) = pairs.iter().copied().unzip(); + let Some(h) = least_squares(Model::Similarity, &src, &dst) else { return }; + let m = h.0; + let k = (m[0][0] * m[1][1] - m[0][1] * m[1][0]).abs().sqrt(); + if k < 1e-6 { + return; + } + let n = dst.len() as f64; + let c = [dst.iter().map(|q| q[0]).sum::() / n, dst.iter().map(|q| q[1]).sum::() / n]; + for (_, d) in pairs.iter_mut() { + *d = [c[0] + (d[0] - c[0]) / k, c[1] + (d[1] - c[1]) / k]; + } +} + +/// Subspace Warp's meshes (stabilised → source, before framing) for every frame; `None` for the +/// other methods and for analyses without trajectories. `base`: the perspective corrections +/// ([`corrections`]), the meshes' prior. +pub fn meshes(a: &WarpAnalysis, s: &StabSettings, base: &[Homography]) -> Option> { + if s.method != Method::SubspaceWarp || a.tracks.is_empty() || base.len() != a.frames.len() { + return None; + } + let n = a.frames.len(); + let o = s.ripple.mesh(); + let size = a.size; + let pairs: Vec> = match s.result { + StabResult::NoMotion => (0..n).map(|t| a.tracks.iter().filter_map(|tr| tr.at(t as u32)).map(|p| (p, base[t].apply(p))).collect()).collect(), + StabResult::SmoothMotion => { + let sigma = s.sigma(); + if sigma < 0.3 { + return Some((0..n).map(|_| Mesh::identity(o.cols, o.rows, size)).collect()); + } + smooth_trajectories(&a.tracks, n, sigma, 9) + } + }; + use rayon::prelude::*; + Some( + pairs + .into_par_iter() + .enumerate() + .map(|(t, mut pr)| { + if s.preserve_scale && pr.len() >= 2 { + unscale(&mut pr); + } + if pr.len() < 6 { Mesh::from_homography(o.cols, o.rows, size, &base[t]) } else { solve_mesh(size, &o, &pr, &base[t]) } + }) + .collect(), + ) +} + /// Stabilization plan: corrections plus framing. pub fn plan(a: &WarpAnalysis, s: &StabSettings) -> Plan { - let mut b = corrections(a, s); + use rayon::prelude::*; + let base = corrections(a, s); + let mut b: Vec = match meshes(a, s, &base) { + Some(m) => m.into_iter().map(Warp::M).collect(), + None => base.into_iter().map(Warp::H).collect(), + }; let size = a.size; let c = [size[0] / 2.0, size[1] / 2.0]; let add = (s.additional_scale / 100.0).max(0.01); let z_add = Homography::scale_about(add, c); - let fracs = |b: &[Homography]| b.iter().map(|h| valid_fraction(h, size)).collect::>(); + let fracs = |b: &[Warp]| b.par_iter().map(|w| w.valid(size)).collect::>(); let mut fr = fracs(&b); let mut vmin = fr.iter().copied().fold(1.0, f64::min); let margin = (s.action_safe / 100.0).clamp(0.0, 0.45); @@ -435,22 +594,24 @@ pub fn plan(a: &WarpAnalysis, s: &StabSettings) -> Plan { // Relax the correction on frames that need more than Maximum Scale. let target = need_cover / max_scale; let n = b.len(); - let mut lam = vec![1.0f64; n]; - for t in 0..n { - if fr[t] >= target { - continue; - } - let (mut lo, mut hi) = (0.0, 1.0); - for _ in 0..24 { - let mid = 0.5 * (lo + hi); - if valid_fraction(&Homography::IDENTITY.lerp(&b[t], mid), size) >= target { - lo = mid; - } else { - hi = mid; + let lam: Vec = (0..n) + .into_par_iter() + .map(|t| { + if fr[t] >= target { + return 1.0; } - } - lam[t] = lo; - } + let (mut lo, mut hi) = (0.0, 1.0); + for _ in 0..24 { + let mid = 0.5 * (lo + hi); + if b[t].relax(mid).valid(size) >= target { + lo = mid; + } else { + hi = mid; + } + } + lo + }) + .collect(); // Spread the relaxation smoothly (min filter, then a Gaussian of the same radius) so // the camera doesn't jump. let r = ((s.sigma() * 0.5).ceil() as usize).max(1); @@ -468,7 +629,7 @@ pub fn plan(a: &WarpAnalysis, s: &StabSettings) -> Plan { (acc / ws).min(lam[t]) }) .collect(); - b = b.iter().zip(&smooth).map(|(h, l)| if *l >= 1.0 { *h } else { Homography::IDENTITY.lerp(h, *l) }).collect(); + b = b.iter().zip(&smooth).map(|(w, l)| if *l >= 1.0 { w.clone() } else { w.relax(*l) }).collect(); fr = fracs(&b); vmin = fr.iter().copied().fold(1.0, f64::min); } @@ -483,13 +644,23 @@ pub fn plan(a: &WarpAnalysis, s: &StabSettings) -> Plan { } }; let z = Homography::scale_about(auto, c); - let warps = b.iter().map(|h| z_add.then_after(&z.then_after(h))).collect(); - Plan { warps, crop, auto_scale: auto, valid_fraction: vmin } + let framing = z_add.then_after(&z); + let warps = b.par_iter().map(|w| framing.then_after(&w.homography())).collect(); + let meshes = b + .into_iter() + .filter_map(|w| match w { + Warp::M(m) => Some(m), + Warp::H(_) => None, + }) + .collect(); + Plan { warps, meshes, framing, crop, auto_scale: auto, valid_fraction: vmin } } #[cfg(test)] mod tests { use super::*; + use effectcraft_raster::Image; + use rayon::prelude::*; /// A synthetic analysis: a slow pan plus per-frame jitter. fn shaky(n: usize) -> (WarpAnalysis, Vec<[f64; 2]>) { @@ -497,7 +668,7 @@ mod tests { let j = [((k * 7919) % 13) as f64 - 6.0, ((k * 104729) % 11) as f64 - 5.0]; [2.0 * k as f64 + j[0], 0.5 * k as f64 + j[1]] }; - let mut a = WarpAnalysis { version: 1, start: 0.0, frame_duration: 1.0 / 30.0, size: [640.0, 360.0], detailed: false, frames: vec![] }; + let mut a = WarpAnalysis { version: 1, start: 0.0, frame_duration: 1.0 / 30.0, size: [640.0, 360.0], detailed: false, frames: vec![], tracks: vec![] }; let path: Vec<[f64; 2]> = (0..n).map(pos).collect(); for k in 0..n { let mut f = FrameMotion::default(); @@ -573,4 +744,139 @@ mod tests { let j = WarpAnalysis::from_json(&a.to_json()).unwrap(); assert_eq!(j, a); } + + // ------------------------------------------------------------ synthetic shaky footage + + fn hash(i: i64, j: i64, k: i64) -> f64 { + let mut v = (i.wrapping_mul(73_856_093) ^ j.wrapping_mul(19_349_663) ^ k.wrapping_mul(83_492_791)) as u64; + v ^= v >> 13; + v = v.wrapping_mul(0x5bd1_e995); + v ^= v >> 15; + (v % 10_000) as f64 / 10_000.0 + } + + fn value_noise(x: f64, y: f64, cell: f64, seed: i64) -> f64 { + let (fx, fy) = (x / cell, y / cell); + let (i, j) = (fx.floor() as i64, fy.floor() as i64); + let (u, v) = (fx - i as f64, fy - j as f64); + let (u, v) = (u * u * (3.0 - 2.0 * u), v * v * (3.0 - 2.0 * v)); + let a = hash(i, j, seed) * (1.0 - u) + hash(i + 1, j, seed) * u; + let b = hash(i, j + 1, seed) * (1.0 - u) + hash(i + 1, j + 1, seed) * u; + a * (1.0 - v) + b * v + } + + fn texture(x: f64, y: f64) -> f32 { + (0.55 * value_noise(x, y, 5.0, 1) + 0.45 * value_noise(x, y, 13.0, 2)) as f32 + } + + /// A scene with depth: a near bump in the middle moves `1 + depth` times as much as the + /// background when the camera shakes (parallax a homography cannot model). + struct Scene { + size: [f64; 2], + cams: Vec<[f64; 2]>, + } + + impl Scene { + fn new(n: usize) -> Scene { + let cams = (0..n) + .map(|t| { + let j = [5.0 * (hash(t as i64, 7, 3) - 0.5), 5.0 * (hash(t as i64, 11, 5) - 0.5)]; + [0.4 * t as f64 + j[0], 0.15 * t as f64 + j[1]] + }) + .collect(); + Scene { size: [320.0, 180.0], cams } + } + fn depth(&self, s: [f64; 2]) -> f64 { + 1.0 + 1.2 * (-((s[0] - 160.0).powi(2) + (s[1] - 90.0).powi(2)) / (2.0 * 38.0f64.powi(2))).exp() + } + /// Where scene point `s` appears on frame `t`. + fn image_of(&self, t: usize, s: [f64; 2]) -> [f64; 2] { + let (c, d) = (self.cams[t], self.depth(s)); + [s[0] + c[0] * d, s[1] + c[1] * d] + } + fn frame(&self, t: usize) -> Image { + let (w, h) = (self.size[0] as u32, self.size[1] as u32); + let mut im = Image::new(w, h); + let c = self.cams[t]; + im.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let p = [x as f64 + 0.5, y as f64 + 0.5]; + let mut s = p; + for _ in 0..10 { + let d = self.depth(s); + s = [p[0] - c[0] * d, p[1] - c[1] * d]; + } + let v = texture(s[0], s[1]); + *px = [v, v, v, 1.0]; + } + }); + im + } + } + + fn residual_jitter(scene: &Scene, out: impl Fn(usize, [f64; 2]) -> Option<[f64; 2]>) -> f64 { + let n = scene.cams.len(); + let mut tot = 0.0; + let mut cnt = 0; + for gy in 0..5 { + for gx in 0..7 { + let s = [70.0 + gx as f64 * 30.0, 45.0 + gy as f64 * 22.0]; + let ys: Vec> = (0..n).map(|t| out(t, scene.image_of(t, s))).collect(); + for t in 6..n - 6 { + if let (Some(a), Some(b), Some(c)) = (ys[t - 1], ys[t], ys[t + 1]) { + tot += (c[0] - 2.0 * b[0] + a[0]).hypot(c[1] - 2.0 * b[1] + a[1]); + cnt += 1; + } + } + } + } + tot / cnt.max(1) as f64 + } + + #[test] + fn subspace_warp_beats_perspective_on_parallax() { + let n = 48; + let scene = Scene::new(n); + let mut an = Analyzer::new(scene.size, AnalyzeOpts::default()); + for t in 0..n { + let img = scene.frame(t); + an.push(&Frame { img: &img, offset: [0.0; 2] }); + } + let a = an.finish(0.0, 1.0 / 24.0); + assert!(a.tracks.len() >= 40, "trajectories: {}", a.tracks.len()); + // The analysis round-trips with its trajectories. + assert_eq!(WarpAnalysis::from_json(&a.to_json()).unwrap(), a); + let base = StabSettings { framing: Framing::StabilizeOnly, fps: 24.0, ..Default::default() }; + let before = residual_jitter(&scene, |_, x| Some(x)); + let ppl = plan(&a, &StabSettings { method: Method::Perspective, ..base }); + assert!(ppl.meshes.is_empty()); + let persp = residual_jitter(&scene, |t, x| ppl.output_of(t, x)); + let spl = plan(&a, &StabSettings { method: Method::SubspaceWarp, ..base }); + assert_eq!(spl.meshes.len(), n); + let sub = residual_jitter(&scene, |t, x| spl.output_of(t, x)); + let epl = plan(&a, &StabSettings { method: Method::SubspaceWarp, ripple: Ripple::Enhanced, ..base }); + let enh = residual_jitter(&scene, |t, x| epl.output_of(t, x)); + eprintln!("jitter: before {before:.3}, perspective {persp:.3}, subspace {sub:.3}, enhanced {enh:.3}"); + assert!(persp < 0.8 * before, "perspective {persp} vs {before}"); + assert!(sub < 0.6 * persp, "subspace {sub} vs perspective {persp}"); + assert!(enh < 0.6 * persp, "enhanced {enh} vs perspective {persp}"); + // source_of and output_of agree. + let q = [150.0, 80.0]; + let o = spl.output_of(10, q).unwrap(); + let back = spl.source_of(10, o).unwrap(); + assert!((back[0] - q[0]).abs() < 1e-3 && (back[1] - q[1]).abs() < 1e-3); + // Auto-scale framing covers the frame with mesh warps too. + let fpl = plan(&a, &StabSettings { method: Method::SubspaceWarp, fps: 24.0, ..Default::default() }); + assert!(fpl.auto_scale >= 1.0); + for k in [0, n / 2, n - 1] { + for p in [[1.0, 1.0], [319.0, 1.0], [319.0, 179.0], [1.0, 179.0]] { + let q = fpl.source_of(k, p).unwrap(); + let inside = q[0] >= -0.5 && q[1] >= -0.5 && q[0] <= 320.5 && q[1] <= 180.5; + assert!(inside || fpl.crop.is_some(), "frame {k}: {p:?} → {q:?}"); + } + } + // Preserve Scale keeps working with meshes. + let psp = plan(&a, &StabSettings { method: Method::SubspaceWarp, preserve_scale: true, ..base }); + assert_eq!(psp.meshes.len(), n); + } } diff --git a/crates/track/src/subspace.rs b/crates/track/src/subspace.rs new file mode 100644 index 00000000..339f9708 --- /dev/null +++ b/crates/track/src/subspace.rs @@ -0,0 +1,447 @@ +//! Warp Stabilizer ▸ Method ▸ **Subspace Warp**: content-preserving warps fitted to smoothed +//! feature trajectories, clean-room from the published method: +//! +//! - Liu, Gleicher, Wang, Jin & Agarwala, "Subspace Video Stabilization", ACM TOG 30(1), 2011: +//! the feature trajectories of a short window form a matrix `M` (two rows per trajectory, one +//! column per frame) that is close to low rank (rank 9 for a moving camera over a static +//! scene); its factorisation `M ≈ W C` gives *eigen-trajectories* `C`, which are smoothed (here +//! with a Gaussian) instead of the trajectories themselves. Every trajectory's smoothed +//! position is `W_i Ĉ_t`; trajectories that do not span the whole window get their +//! coefficients `W_i` by least squares over the frames they cover (the paper's handling of +//! incomplete trajectories in a moving factorisation). Smoothing in the subspace keeps the +//! smoothed trajectories consistent with one another, so the warp does not tear the scene. +//! - Liu, Gleicher, Jin & Agarwala, "Content-Preserving Warps for 3D Video Stabilization", ACM +//! TOG 28(3), 2009: each frame is warped by a grid mesh fitted to the feature correspondences +//! (original → smoothed position) in least squares, with a **similarity term** asking every +//! grid triangle to keep its shape (`V1 = V2 + u (V3 − V2) + v R90 (V3 − V2)` with `u, v` from +//! the undeformed grid) so the warp bends where the scene's depth demands it but stays locally +//! rigid elsewhere. +//! +//! The mesh is stored *inversely*: a regular grid over the stabilised frame whose vertices hold +//! the source position they show ([`Mesh`]), so rendering is a bilinear lookup per output pixel. +//! The data term is the same linear constraint read the other way (the smoothed position's grid +//! cell, bilinearly, must land on the original position). A weak prior ties every vertex to the +//! frame's perspective correction, which fills feature-less regions and keeps the system well +//! posed. + +use rayon::prelude::*; + +use crate::camtrack::Track2D; +use crate::camtrack::linalg::{cholesky_solve, sym_eigen}; +use crate::fit::{Model, least_squares}; +use crate::solve::Homography; + +/// A warp mesh: a regular `cols × rows` grid over the stabilised frame (`size`), each vertex +/// holding the source-frame position it shows. +#[derive(Clone, Debug, PartialEq)] +pub struct Mesh { + pub cols: usize, + pub rows: usize, + pub size: [f64; 2], + /// `(rows + 1) × (cols + 1)` source positions, row-major. + pub verts: Vec<[f64; 2]>, +} + +impl Mesh { + /// The identity mesh. + pub fn identity(cols: usize, rows: usize, size: [f64; 2]) -> Mesh { + let mut m = Mesh { cols: cols.max(1), rows: rows.max(1), size, verts: vec![] }; + m.verts = (0..=m.rows).flat_map(|j| (0..=m.cols).map(move |i| (i, j))).map(|(i, j)| m.grid(i, j)).collect(); + m + } + + /// The mesh of a homography (`h`: source → stabilised). + pub fn from_homography(cols: usize, rows: usize, size: [f64; 2], h: &Homography) -> Mesh { + let inv = h.inverse().unwrap_or(Homography::IDENTITY); + let mut m = Mesh::identity(cols, rows, size); + for v in m.verts.iter_mut() { + *v = inv.apply(*v); + } + m + } + + fn cell(&self) -> [f64; 2] { + [self.size[0] / self.cols as f64, self.size[1] / self.rows as f64] + } + + /// The undeformed position of vertex `(i, j)` (column, row). + pub fn grid(&self, i: usize, j: usize) -> [f64; 2] { + let c = self.cell(); + [i as f64 * c[0], j as f64 * c[1]] + } + + fn idx(&self, i: usize, j: usize) -> usize { + j * (self.cols + 1) + i + } + + /// Cell and bilinear coordinates of a stabilised point (the outer cells extend beyond the + /// frame, so points outside it extrapolate). + fn locate(&self, p: [f64; 2]) -> (usize, usize, f64, f64) { + let c = self.cell(); + let fx = p[0] / c[0]; + let fy = p[1] / c[1]; + let i = (fx.floor().max(0.0) as usize).min(self.cols - 1); + let j = (fy.floor().max(0.0) as usize).min(self.rows - 1); + (i, j, fx - i as f64, fy - j as f64) + } + + /// The source position shown at stabilised point `p`. + pub fn sample(&self, p: [f64; 2]) -> [f64; 2] { + let (i, j, u, v) = self.locate(p); + let a = self.verts[self.idx(i, j)]; + let b = self.verts[self.idx(i + 1, j)]; + let c = self.verts[self.idx(i, j + 1)]; + let d = self.verts[self.idx(i + 1, j + 1)]; + let w = [(1.0 - u) * (1.0 - v), u * (1.0 - v), (1.0 - u) * v, u * v]; + [0, 1].map(|k| w[0] * a[k] + w[1] * b[k] + w[2] * c[k] + w[3] * d[k]) + } + + /// Where source point `q` lands in the stabilised frame (Newton iterations on + /// [`Mesh::sample`]); `None` when it does not converge (a folded mesh). + pub fn forward(&self, q: [f64; 2]) -> Option<[f64; 2]> { + // Start from the mesh's own offset at q. + let s = self.sample(q); + let mut p = [2.0 * q[0] - s[0], 2.0 * q[1] - s[1]]; + for _ in 0..30 { + let f = self.sample(p); + let e = [f[0] - q[0], f[1] - q[1]]; + if e[0].hypot(e[1]) < 1e-6 { + return Some(p); + } + let h = 0.5; + let fx = self.sample([p[0] + h, p[1]]); + let fy = self.sample([p[0], p[1] + h]); + let j = [[(fx[0] - f[0]) / h, (fy[0] - f[0]) / h], [(fx[1] - f[1]) / h, (fy[1] - f[1]) / h]]; + let det = j[0][0] * j[1][1] - j[0][1] * j[1][0]; + if det.abs() < 1e-12 { + return None; + } + p[0] -= (j[1][1] * e[0] - j[0][1] * e[1]) / det; + p[1] -= (-j[1][0] * e[0] + j[0][0] * e[1]) / det; + } + let f = self.sample(p); + ((f[0] - q[0]).hypot(f[1] - q[1]) < 1e-3).then_some(p) + } + + /// `(1 − t) · identity + t · self` per vertex. + pub fn lerp_identity(&self, t: f64) -> Mesh { + let mut m = self.clone(); + for j in 0..=self.rows { + for i in 0..=self.cols { + let g = self.grid(i, j); + let k = self.idx(i, j); + m.verts[k] = [g[0] + (self.verts[k][0] - g[0]) * t, g[1] + (self.verts[k][1] - g[1]) * t]; + } + } + m + } + + /// Largest `s ∈ [0, 1]` such that the centred `s·W × s·H` rectangle of the stabilised frame + /// shows only source pixels (its outline, sampled densely, maps inside the source frame). + pub fn valid_fraction(&self) -> f64 { + let [w, h] = self.size; + let c = [w / 2.0, h / 2.0]; + let tol = 1e-6; + let inside = |q: [f64; 2]| q[0] >= -tol && q[1] >= -tol && q[0] <= w + tol && q[1] <= h + tol; + let fits = |s: f64| { + let (hw, hh) = (s * w / 2.0, s * h / 2.0); + let n = 48; + (0..=n).all(|k| { + let t = k as f64 / n as f64; + let x = c[0] - hw + 2.0 * hw * t; + let y = c[1] - hh + 2.0 * hh * t; + inside(self.sample([x, c[1] - hh])) + && inside(self.sample([x, c[1] + hh])) + && inside(self.sample([c[0] - hw, y])) + && inside(self.sample([c[0] + hw, y])) + }) + }; + if fits(1.0) { + return 1.0; + } + if !fits(0.0) { + return 0.0; + } + let (mut lo, mut hi) = (0.0, 1.0); + for _ in 0..24 { + let mid = 0.5 * (lo + hi); + if fits(mid) { + lo = mid; + } else { + hi = mid; + } + } + lo + } + + /// The homography (source → stabilised) closest to the mesh in least squares. + pub fn homography(&self) -> Homography { + let mut src = vec![]; + let mut dst = vec![]; + for j in 0..=self.rows { + for i in 0..=self.cols { + src.push(self.verts[self.idx(i, j)]); + dst.push(self.grid(i, j)); + } + } + least_squares(Model::Homography, &src, &dst).unwrap_or(Homography::IDENTITY) + } +} + +/// Mesh resolution and rigidity. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct MeshOpts { + pub cols: usize, + pub rows: usize, + /// Weight of the similarity (shape-preserving) term relative to one feature. + pub alpha: f64, + /// Weight of the prior towards the frame's perspective correction, per vertex. + pub prior: f64, +} + +impl Default for MeshOpts { + fn default() -> Self { + MeshOpts { cols: 12, rows: 8, alpha: 1.0, prior: 0.02 } + } +} + +/// Normal equations `AᵀA x = Aᵀb`, accumulated row by row. +struct Normal { + n: usize, + a: Vec, + b: Vec, +} + +impl Normal { + fn row(&mut self, coefs: &[(usize, f64)], rhs: f64, w: f64) { + for (ia, ca) in coefs { + self.b[*ia] += w * ca * rhs; + for (ib, cb) in coefs { + self.a[ia * self.n + ib] += w * ca * cb; + } + } + } +} + +/// Fit the content-preserving warp of one frame: `pairs` are (source position, wanted +/// stabilised position) of its features; `base` is the frame's perspective correction (source → +/// stabilised), the prior. +pub fn solve_mesh(size: [f64; 2], o: &MeshOpts, pairs: &[([f64; 2], [f64; 2])], base: &Homography) -> Mesh { + let mut m = Mesh::from_homography(o.cols, o.rows, size, base); + let nv = (m.cols + 1) * (m.rows + 1); + let n = 2 * nv; + let mut ne = Normal { n, a: vec![0.0; n * n], b: vec![0.0; n] }; + // Prior. + for k in 0..nv { + let v = m.verts[k]; + ne.row(&[(2 * k, 1.0)], v[0], o.prior); + ne.row(&[(2 * k + 1, 1.0)], v[1], o.prior); + } + // Data: the bilinear combination of the wanted position's cell lands on the source position. + let mut used = 0; + for (src, dst) in pairs { + if !(dst[0] >= 0.0 && dst[1] >= 0.0 && dst[0] <= size[0] && dst[1] <= size[1] && src[0].is_finite() && src[1].is_finite()) { + continue; + } + let (i, j, u, v) = m.locate(*dst); + let ks = [m.idx(i, j), m.idx(i + 1, j), m.idx(i, j + 1), m.idx(i + 1, j + 1)]; + let w = [(1.0 - u) * (1.0 - v), u * (1.0 - v), (1.0 - u) * v, u * v]; + for d in 0..2 { + let coefs: Vec<(usize, f64)> = ks.iter().zip(w).map(|(k, w)| (2 * k + d, w)).collect(); + ne.row(&coefs, src[d], 1.0); + } + used += 1; + } + if used == 0 { + return m; + } + // Similarity: four triangles per cell, each corner against its two cell neighbours. + let a2 = o.alpha * o.alpha; + for j in 0..m.rows { + for i in 0..m.cols { + let (a, b, c, d) = ((i, j), (i + 1, j), (i + 1, j + 1), (i, j + 1)); + for (v1, v2, v3) in [(a, b, d), (b, c, a), (c, d, b), (d, a, c)] { + let (g1, g2, g3) = (m.grid(v1.0, v1.1), m.grid(v2.0, v2.1), m.grid(v3.0, v3.1)); + let dv = [g3[0] - g2[0], g3[1] - g2[1]]; + let e = [g1[0] - g2[0], g1[1] - g2[1]]; + let l2 = dv[0] * dv[0] + dv[1] * dv[1]; + // R90(d) = (−d.y, d.x). + let u = (e[0] * dv[0] + e[1] * dv[1]) / l2; + let v = (e[0] * -dv[1] + e[1] * dv[0]) / l2; + let (k1, k2, k3) = (m.idx(v1.0, v1.1), m.idx(v2.0, v2.1), m.idx(v3.0, v3.1)); + // x: V1x − V2x − u (V3x − V2x) + v (V3y − V2y) = 0 + ne.row(&[(2 * k1, 1.0), (2 * k2, u - 1.0), (2 * k3, -u), (2 * k3 + 1, v), (2 * k2 + 1, -v)], 0.0, a2); + // y: V1y − V2y − u (V3y − V2y) − v (V3x − V2x) = 0 + ne.row(&[(2 * k1 + 1, 1.0), (2 * k2 + 1, u - 1.0), (2 * k3 + 1, -u), (2 * k3, -v), (2 * k2, v)], 0.0, a2); + } + } + } + let mut x = ne.b; + if cholesky_solve(&mut ne.a, n, &mut x) { + for k in 0..nv { + m.verts[k] = [x[2 * k], x[2 * k + 1]]; + } + } + m +} + +/// Gaussian weights of the frames `lo..=hi` around `t`. +fn gauss(lo: usize, hi: usize, t: usize, sigma: f64) -> Vec { + let g: Vec = (lo..=hi).map(|j| (-((j as f64 - t as f64).powi(2)) / (2.0 * sigma * sigma)).exp()).collect(); + let s: f64 = g.iter().sum(); + g.into_iter().map(|v| v / s.max(1e-300)).collect() +} + +/// Subspace-smoothed feature positions: for every frame `t < frames`, the (original, smoothed) +/// positions of the trajectories seen on it. `sigma`: Gaussian smoothing in frames; `rank`: the +/// subspace dimension (9 in the paper). Frames without enough trajectories get no pairs. +pub fn smooth_trajectories(tracks: &[Track2D], frames: usize, sigma: f64, rank: usize) -> Vec> { + let max_r = ((3.0 * sigma).ceil() as usize).clamp(1, 30); + (0..frames) + .into_par_iter() + .map(|t| { + let t32 = t as u32; + let mut r = max_r; + // The window around t: shrink it until enough trajectories span it. + let (lo, hi, complete) = loop { + let lo = t.saturating_sub(r); + let hi = (t + r).min(frames.saturating_sub(1)); + let complete: Vec<&Track2D> = tracks.iter().filter(|tr| tr.start as usize <= lo && tr.end() as usize > hi).collect(); + if complete.len() >= 8 || r <= 2 { + break (lo, hi, complete); + } + r = (r * 2 / 3).max(2); + }; + if complete.len() < 4 { + return vec![]; + } + let m = hi - lo + 1; + let rows = 2 * complete.len(); + let k = rank.min(m).min(rows).max(1); + // M: rows × m (row-major). + let mut mat = vec![0.0; rows * m]; + for (i, tr) in complete.iter().enumerate() { + for c in 0..m { + let p = tr.at((lo + c) as u32).unwrap_or([0.0; 2]); + mat[(2 * i) * m + c] = p[0]; + mat[(2 * i + 1) * m + c] = p[1]; + } + } + // Gram MᵀM and its top eigenvectors: the eigen-trajectories (orthonormal rows of C). + let mut gram = vec![0.0; m * m]; + for a in 0..m { + for b in a..m { + let s: f64 = (0..rows).map(|r| mat[r * m + a] * mat[r * m + b]).sum(); + gram[a * m + b] = s; + gram[b * m + a] = s; + } + } + let (_, vecs) = sym_eigen(&gram, m); + // C[q][c] = vecs[c][m − 1 − q] (largest first). + let cmat: Vec> = (0..k).map(|q| (0..m).map(|c| vecs[c * m + (m - 1 - q)]).collect()).collect(); + let g = gauss(lo, hi, t, sigma.max(1e-6)); + let chat: Vec = cmat.iter().map(|row| row.iter().zip(&g).map(|(a, b)| a * b).sum()).collect(); + let tc = t - lo; + let mut out = Vec::new(); + // Complete trajectories: W = M Cᵀ. + for (i, _) in complete.iter().enumerate() { + let src = [mat[(2 * i) * m + tc], mat[(2 * i + 1) * m + tc]]; + let mut dst = [0.0; 2]; + for (d, slot) in dst.iter_mut().enumerate() { + let row = &mat[(2 * i + d) * m..(2 * i + d + 1) * m]; + for q in 0..k { + let w: f64 = row.iter().zip(&cmat[q]).map(|(a, b)| a * b).sum(); + *slot += w * chat[q]; + } + } + out.push((src, dst)); + } + // Incomplete trajectories seen on t: coefficients by least squares over the frames + // they cover in the window. + let need = (k + 3).max(m / 2); + for tr in tracks.iter().filter(|tr| tr.at(t32).is_some() && !(tr.start as usize <= lo && tr.end() as usize > hi)) { + let cols: Vec = (0..m).filter(|c| tr.at((lo + c) as u32).is_some()).collect(); + if cols.len() < need { + continue; + } + let mut ata = vec![0.0; k * k]; + let mut atb = [vec![0.0; k], vec![0.0; k]]; + for &c in &cols { + let p = tr.at((lo + c) as u32).expect("covered"); + for a in 0..k { + for d in 0..2 { + atb[d][a] += cmat[a][c] * p[d]; + } + for b in 0..k { + ata[a * k + b] += cmat[a][c] * cmat[b][c]; + } + } + } + for a in 0..k { + ata[a * k + a] += 1e-9; + } + let mut dst = [0.0; 2]; + let mut ok = true; + for d in 0..2 { + let mut x = atb[d].clone(); + let mut aa = ata.clone(); + if !cholesky_solve(&mut aa, k, &mut x) { + ok = false; + break; + } + dst[d] = x.iter().zip(&chat).map(|(a, b)| a * b).sum(); + } + if ok { + out.push((tr.at(t32).expect("seen"), dst)); + } + } + out + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mesh_identity_sample_and_forward() { + let m = Mesh::identity(4, 3, [100.0, 60.0]); + assert_eq!(m.sample([37.0, 21.0]), [37.0, 21.0]); + assert_eq!(m.valid_fraction(), 1.0); + let h = Homography([[1.0, 0.02, 5.0], [-0.01, 1.0, -3.0], [0.0, 0.0, 1.0]]); + let mh = Mesh::from_homography(4, 3, [100.0, 60.0], &h); + // Affine maps are exact on a bilinear mesh. + let q = [40.0, 30.0]; + let p = mh.forward(q).unwrap(); + let e = h.apply(q); + assert!((p[0] - e[0]).abs() < 1e-6 && (p[1] - e[1]).abs() < 1e-6, "{p:?} {e:?}"); + assert!(mh.homography().max_diff(&h) < 1e-6); + assert!(mh.valid_fraction() < 1.0); + } + + #[test] + fn mesh_fits_a_translation_and_bends_for_local_motion() { + let size = [200.0, 120.0]; + let mut pairs = vec![]; + for y in 0..12 { + for x in 0..20 { + let p = [5.0 + x as f64 * 10.0, 5.0 + y as f64 * 10.0]; + // Everything moves by (3, −2); a blob near the centre by (8, −2). + let bump = (-((p[0] - 100.0).powi(2) + (p[1] - 60.0).powi(2)) / (2.0 * 20.0f64.powi(2))).exp(); + pairs.push((p, [p[0] + 3.0 + 5.0 * bump, p[1] - 2.0])); + } + } + let base = Homography::translation([3.0, -2.0]); + let m = solve_mesh(size, &MeshOpts { cols: 10, rows: 6, alpha: 0.5, prior: 0.01 }, &pairs, &base); + let mut err_mesh = 0.0; + let mut err_h = 0.0; + for (s, d) in &pairs { + let q = m.sample(*d); + err_mesh += (q[0] - s[0]).hypot(q[1] - s[1]); + let qh = base.inverse().unwrap().apply(*d); + err_h += (qh[0] - s[0]).hypot(qh[1] - s[1]); + } + assert!(err_mesh < 0.35 * err_h, "{err_mesh} vs {err_h}"); + } +} From 00eb951711f38459fb8e36d4e874c5b1cd82fc45 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 04:24:01 -0400 Subject: [PATCH 4/9] M13.1: ScriptUI frontend (dialogs, palettes, dockable panels), script menus, example plug-in, docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ui-egui: scriptui_view draws script windows at engine layout bounds; PanelKind::ScriptPanel; File ▸ Scripts and Window menu list scripts/panels (egui + native menus) - examples/plugins/posterize-bands (WebAssembly, API v1); host plug-in tests; desktop loads /Plug-ins - MCP: script_ui and history tools; docs/plugins.md, agents.md, parity, architecture, ATTRIBUTION Co-Authored-By: Claude Opus 5.5 --- ATTRIBUTION.md | 5 + apps/effectcraft/src/main.rs | 5 + crates/automation/src/tests.rs | 26 ++ crates/automation/src/tools.rs | 32 ++ ...te Null at Selected Layers.jsx.attribution | 7 + .../scripts/Rename Layers.jsx.attribution | 7 + .../Render Queue Batch.jsx.attribution | 7 + .../Layer Tools.jsx.attribution | 7 + .../Sort Layers by In Point.jsx.attribution | 7 + crates/engine/src/commands/file_more.rs | 2 +- crates/engine/src/menus.rs | 1 + crates/host/tests/plugins.rs | 106 ++++++ crates/ui-egui/src/dock.rs | 3 + crates/ui-egui/src/dock_ui.rs | 13 + crates/ui-egui/src/lib.rs | 2 + crates/ui-egui/src/menus.rs | 57 +++- crates/ui-egui/src/native_menu.rs | 24 ++ crates/ui-egui/src/panels/mod.rs | 2 + crates/ui-egui/src/panels/scriptui_view.rs | 321 ++++++++++++++++++ crates/ui-egui/tests/ui_scriptui.rs | 112 ++++++ docs/agents.md | 29 ++ docs/parity.md | 14 +- docs/plugins.md | 126 +++++++ examples/plugins/posterize-bands/Cargo.lock | 7 + examples/plugins/posterize-bands/Cargo.toml | 18 + examples/plugins/posterize-bands/src/lib.rs | 93 +++++ 26 files changed, 1026 insertions(+), 7 deletions(-) create mode 100644 crates/engine/scripts/Create Null at Selected Layers.jsx.attribution create mode 100644 crates/engine/scripts/Rename Layers.jsx.attribution create mode 100644 crates/engine/scripts/Render Queue Batch.jsx.attribution create mode 100644 crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx.attribution create mode 100644 crates/engine/scripts/Sort Layers by In Point.jsx.attribution create mode 100644 crates/host/tests/plugins.rs create mode 100644 crates/ui-egui/src/panels/scriptui_view.rs create mode 100644 crates/ui-egui/tests/ui_scriptui.rs create mode 100644 docs/plugins.md create mode 100644 examples/plugins/posterize-bands/Cargo.lock create mode 100644 examples/plugins/posterize-bands/Cargo.toml create mode 100644 examples/plugins/posterize-bands/src/lib.rs diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index d52b19c7..71c54885 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -61,6 +61,11 @@ Screenshots of EffectCraft itself, rendered headlessly, showing only procedurall - Procedural demo content (solids, shapes, gradients, text animations): `crates/engine/src/demo.rs`. - Text animator presets (Typewriter, Fade Up Characters, Bounce In Words, Tracking In, Scramble, Blur In, Jitter, Drop In Lines): `crates/engine/presets/text_animators.json` (sidecar `.attribution`). +- Sample scripts (File ▸ Scripts) and ScriptUI panel: `crates/engine/scripts/Rename Layers.jsx`, + `crates/engine/scripts/Sort Layers by In Point.jsx`, `crates/engine/scripts/Create Null at Selected Layers.jsx`, + `crates/engine/scripts/Render Queue Batch.jsx`, `crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx` + (sidecars `.attribution`). +- Example effect plug-in: `examples/plugins/posterize-bands` (plug-in API v1, WebAssembly). ## First-party brand marks diff --git a/apps/effectcraft/src/main.rs b/apps/effectcraft/src/main.rs index b6426f5f..ac7f1be6 100644 --- a/apps/effectcraft/src/main.rs +++ b/apps/effectcraft/src/main.rs @@ -64,6 +64,11 @@ fn main() -> eframe::Result { // Settings, shortcut presets and the crash-recovery sentinel live in the platform // config directory. Agent-driven runs (`--control`) skip crash recovery. if let Some(dir) = config_dir() { + // WebAssembly effect plug-ins in /Plug-ins load before the menus are built. + let plugins = dir.join("Plug-ins"); + if plugins.is_dir() { + let _ = session.execute("effect.plugins.load", json!({"folder": plugins.to_string_lossy()})); + } session.config = Some(std::sync::Arc::new(effectcraft_engine::config::DirConfig::new(dir))); } session.load_settings(); diff --git a/crates/automation/src/tests.rs b/crates/automation/src/tests.rs index 848d3ee0..9504094a 100644 --- a/crates/automation/src/tests.rs +++ b/crates/automation/src/tests.rs @@ -265,6 +265,32 @@ fn bridge_rejects_non_loopback() { assert!(Backend::bridge("localhost:9877").is_ok()); } +#[test] +fn script_ui_and_history_tools() { + let mut sess = Session::default(); + effectcraft_script::install(&mut sess); + let mut s = McpServer::new(Backend::headless(sess)); + let code = "var d = new Window('dialog', 'Ask'); var g = d.add('group'); g.add('button', undefined, 'OK', {name: 'ok'}); \ + if (d.show() == 1) app.project.items.addComp('Answered', 64, 64, 1, 1, 24); 'done'"; + let r = call_json(&mut s, "run_script", json!({"code": code})); + assert_eq!(r["waiting"], json!(true), "{r}"); + let list = call_json(&mut s, "script_ui", json!({})); + assert_eq!(list[0]["title"], "Ask"); + let tree = call_json(&mut s, "script_ui", json!({"action": "get"})); + assert_eq!(tree["root"]["children"][0]["children"][0]["name"], "ok"); + let r = call_json(&mut s, "script_ui", json!({"action": "click", "widget": "ok"})); + assert_eq!(r["result"], json!("done"), "{r}"); + // History: undo the comp, make another, and jump back to the first on its branch. + call_json(&mut s, "undo", json!({})); + call_json(&mut s, "execute_command", json!({"command": "comp.new", "params": {"name": "Other", "width": 64, "height": 64}})); + let h = call_json(&mut s, "history", json!({})); + let branch = h["states"].as_array().unwrap().iter().find(|n| n["depth"] == 1).unwrap()["index"].clone(); + let h = call_json(&mut s, "history", json!({"goto": branch})); + let cur = h["current"].as_u64().unwrap() as usize; + assert_eq!(h["states"][cur]["depth"], 0, "the jumped-to state is now the working line"); + assert!(call_json(&mut s, "get_project", json!({})).to_string().contains("Answered")); +} + #[test] fn run_script_round_trip() { let mut sess = Session::default(); diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index f9034a38..9e1bfee5 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -315,6 +315,20 @@ fn redo(b: &mut Backend, a: &Value) -> Result { history(b, a, "edit.redo") } +fn script_ui_tool(b: &mut Backend, a: &Value) -> Result { + let action = get(a, "action").and_then(Value::as_str).unwrap_or("list"); + let id = match action { + "list" => "scriptui.list", + "get" => "scriptui.get", + "click" => "scriptui.click", + "set" => "scriptui.set", + "close" => "scriptui.close", + other => return Err(Error::BadArgs(format!("unknown action `{other}` (list | get | click | set | close)"))), + }; + let p = obj(&[("window", get(a, "window")), ("widget", get(a, "widget")), ("value", get(a, "value")), ("result", get(a, "result"))]); + json_reply(b.exec(id, p)?) +} + fn history_tool(b: &mut Backend, a: &Value) -> Result { if let Some(g) = get(a, "goto") { let p = match g { @@ -554,6 +568,24 @@ static TOOLS: &[ToolDef] = &[ schema: || schema(json!({"goto": {"type": ["integer", "string"], "description": "State index or id to jump to (edit.history.goto)."}}), &[]), run: history_tool, }, + ToolDef { + name: "script_ui", + description: "Drive ScriptUI windows that scripts opened (dialogs, palettes, dockable panels). action `list` (default): open windows; `get`: a window's control tree (type, name, text, value, checked, items, selection, bounds); `click`: press a button / toggle a checkbox / pick a radio button; `set`: type text, move a slider, pick a list item (index or text); `close`: close a window (`result` for a dialog's show(), default 2 = Cancel). Controls by id, \"#id\", properties.name or text. A dialog's script resumes when the dialog closes.", + bridge_only: false, + schema: || { + schema( + json!({ + "action": {"type": "string", "enum": ["list", "get", "click", "set", "close"]}, + "window": {"type": ["integer", "string"], "description": "Window id or title (optional when one is open)."}, + "widget": {"type": ["integer", "string"], "description": "Control id, \"#id\", properties.name or text."}, + "value": {"description": "For `set`: text, number, bool, item index or item text."}, + "result": {"type": "integer", "description": "For `close`: what the dialog's show() returns."} + }), + &[], + ) + }, + run: script_ui_tool, + }, ToolDef { name: "screenshot", description: "Screenshot of the live app window, or one panel (`Composition`, `Timeline`, `EffectControls`, `Project`, ...) or element id, as PNG (longest side `max_side`, default 1600). Use it to check what the UI looks like after an action.", diff --git a/crates/engine/scripts/Create Null at Selected Layers.jsx.attribution b/crates/engine/scripts/Create Null at Selected Layers.jsx.attribution new file mode 100644 index 00000000..ea166b70 --- /dev/null +++ b/crates/engine/scripts/Create Null at Selected Layers.jsx.attribution @@ -0,0 +1,7 @@ +asset: Create Null at Selected Layers.jsx +title: EffectCraft sample script (Create Null at Selected Layers) +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-03 by EffectCraft contributors (M13.1) +notes: Written from scratch for EffectCraft against the documented scripting behaviour; not derived from any Adobe sample. diff --git a/crates/engine/scripts/Rename Layers.jsx.attribution b/crates/engine/scripts/Rename Layers.jsx.attribution new file mode 100644 index 00000000..bc07b909 --- /dev/null +++ b/crates/engine/scripts/Rename Layers.jsx.attribution @@ -0,0 +1,7 @@ +asset: Rename Layers.jsx +title: EffectCraft sample script (Rename Layers) +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-03 by EffectCraft contributors (M13.1) +notes: Written from scratch for EffectCraft against the documented scripting behaviour; not derived from any Adobe sample. diff --git a/crates/engine/scripts/Render Queue Batch.jsx.attribution b/crates/engine/scripts/Render Queue Batch.jsx.attribution new file mode 100644 index 00000000..5ab7be71 --- /dev/null +++ b/crates/engine/scripts/Render Queue Batch.jsx.attribution @@ -0,0 +1,7 @@ +asset: Render Queue Batch.jsx +title: EffectCraft sample script (Render Queue Batch) +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-03 by EffectCraft contributors (M13.1) +notes: Written from scratch for EffectCraft against the documented scripting behaviour; not derived from any Adobe sample. diff --git a/crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx.attribution b/crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx.attribution new file mode 100644 index 00000000..6d4d572f --- /dev/null +++ b/crates/engine/scripts/ScriptUI Panels/Layer Tools.jsx.attribution @@ -0,0 +1,7 @@ +asset: Layer Tools.jsx +title: EffectCraft sample script (Layer Tools) +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-03 by EffectCraft contributors (M13.1) +notes: Written from scratch for EffectCraft against the documented scripting behaviour; not derived from any Adobe sample. diff --git a/crates/engine/scripts/Sort Layers by In Point.jsx.attribution b/crates/engine/scripts/Sort Layers by In Point.jsx.attribution new file mode 100644 index 00000000..e744d662 --- /dev/null +++ b/crates/engine/scripts/Sort Layers by In Point.jsx.attribution @@ -0,0 +1,7 @@ +asset: Sort Layers by In Point.jsx +title: EffectCraft sample script (Sort Layers by In Point) +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-03 by EffectCraft contributors (M13.1) +notes: Written from scratch for EffectCraft against the documented scripting behaviour; not derived from any Adobe sample. diff --git a/crates/engine/src/commands/file_more.rs b/crates/engine/src/commands/file_more.rs index 9175af59..2041d400 100644 --- a/crates/engine/src/commands/file_more.rs +++ b/crates/engine/src/commands/file_more.rs @@ -828,7 +828,7 @@ pub fn specs() -> Vec { "Run Script File...", ["File", "Scripts"], None, - "{path (.jsx/.js JavaScript, or a .jsonl/.json command script) | steps: [{command, params}] | name (an installed or sample script, see file.scripts.list)}", + "{path (.jsx/.js JavaScript, or a .jsonl/.json command script) | name (an installed or sample script, see file.scripts.list) | steps: [{command, params}]}", always, run_script ), diff --git a/crates/engine/src/menus.rs b/crates/engine/src/menus.rs index 7f14a988..84522c19 100644 --- a/crates/engine/src/menus.rs +++ b/crates/engine/src/menus.rs @@ -757,6 +757,7 @@ Effect Remove All | effect.removeAll --- Manage Effects... | effect.manage + Load Effect Plug-in... | effect.plugins.load --- @effects Animation diff --git a/crates/host/tests/plugins.rs b/crates/host/tests/plugins.rs new file mode 100644 index 00000000..c8a9b368 --- /dev/null +++ b/crates/host/tests/plugins.rs @@ -0,0 +1,106 @@ +//! Effect plug-ins in a full session: a Rust plug-in registered through the trait and the +//! example WebAssembly plug-in (examples/plugins/posterize-bands) loaded with +//! `effect.plugins.load`, both applied by id like built-ins and rendered. + +use std::sync::Arc; + +use effectcraft_engine::Session; +use effectcraft_engine::effects::plugin::{EffectPlugin, PluginFrame, PluginManifest, PluginParam, PluginParamKind, PluginParams, register_plugin}; +use effectcraft_engine::time::Tick; +use serde_json::json; + +struct Swap { + m: PluginManifest, +} + +impl EffectPlugin for Swap { + fn manifest(&self) -> &PluginManifest { + &self.m + } + fn render(&self, frame: &mut PluginFrame, params: &PluginParams, _time: f64) -> Result<(), String> { + if params.b("on") { + for p in frame.pixels.iter_mut() { + p.swap(0, 2); + } + } + Ok(()) + } +} + +fn setup() -> (Session, u64) { + let mut s = effectcraft_host::session(); + s.execute("comp.new", json!({"name": "P", "width": 40, "height": 20, "duration": 1})).unwrap(); + let l = s.execute("layer.newSolid", json!({"color": [1.0, 0.5, 0.0], "width": 40, "height": 20})).unwrap()["layer"].as_u64().unwrap(); + (s, l) +} + +fn px(s: &Session) -> [f32; 4] { + let cid = s.active_comp_id().unwrap(); + s.render(cid, Tick::ZERO, Default::default()).get(20, 10) +} + +#[test] +fn rust_plugins_apply_by_id() { + let m = PluginManifest { + api: 1, + id: "org.test.swap".into(), + name: "Swap Red Blue".into(), + category: "Channel".into(), + version: "1".into(), + author: String::new(), + description: String::new(), + params: vec![PluginParam { id: "on".into(), name: "On".into(), kind: PluginParamKind::Checkbox { default: true } }], + }; + register_plugin(Arc::new(Swap { m })).unwrap(); + let (mut s, l) = setup(); + s.execute("effect.apply", json!({"layer": l, "effect": "org.test.swap"})).unwrap(); + let p = px(&s); + assert!(p[0] < 0.01 && (p[1] - 0.5).abs() < 0.01 && p[2] > 0.99, "{p:?}"); + // Its parameters are ordinary properties: set (and animate) them like any effect's. + s.execute("prop.set", json!({"layer": l, "path": "effects/#1/on", "value": false})).unwrap(); + let p = px(&s); + assert!(p[0] > 0.99 && p[2] < 0.01, "{p:?}"); + let list = s.execute("effect.plugins.list", json!({})).unwrap(); + assert!(list["plugins"].as_array().unwrap().iter().any(|p| p["id"] == "org.test.swap" && p["category"] == "Channel")); + assert_eq!(list["api"], 1); + assert!(s.execute("effect.list", json!({"filter": "swap red"})).unwrap().as_array().unwrap().len() == 1); +} + +/// Build the example plug-in (skipped when the wasm32 target isn't installed). +fn build_example() -> Option { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../examples/plugins/posterize-bands"); + let target = std::env::var_os("CARGO_TARGET_DIR").map(std::path::PathBuf::from).unwrap_or_else(|| root.join("../../../target")).join("plugin-example"); + let target = if target.is_absolute() { target } else { root.join("../../..").join(target) }; + let st = std::process::Command::new(env!("CARGO")) + .current_dir(&root) + .args(["build", "--release", "--target", "wasm32-unknown-unknown", "--target-dir"]) + .arg(&target) + .status() + .ok()?; + if !st.success() { + eprintln!("skipping: the example plug-in didn't build (rustup target add wasm32-unknown-unknown)"); + return None; + } + Some(target.join("wasm32-unknown-unknown/release/posterize_bands.wasm")) +} + +#[test] +fn example_wasm_plugin_loads_and_renders() { + let Some(wasm) = build_example() else { return }; + let (mut s, l) = setup(); + let r = s.execute("effect.plugins.load", json!({"path": wasm.to_string_lossy()})).unwrap(); + assert_eq!(r["loaded"][0]["id"], "org.effectcraft.example.posterize-bands", "{r}"); + s.execute("effect.apply", json!({"layer": l, "effect": "Posterize Bands"})).unwrap(); + s.execute("prop.set", json!({"layer": l, "path": "effects/#1/mix", "value": 0})).unwrap(); + s.execute("prop.set", json!({"layer": l, "path": "effects/#1/levels", "value": 2})).unwrap(); + // Luminance of (1, .5, 0) is ~0.57: with 2 levels it bands to 1, scaling the colour by 1/y + // (8 bpc: red clips at 1). + let p = px(&s); + let y = 0.2126 + 0.7152 * 0.5; + assert!((p[1] - 0.5 / y as f32).abs() < 0.01 && p[0] > 0.99 && p[2] < 0.01, "{p:?}"); + // Invert: band 0. + s.execute("prop.set", json!({"layer": l, "path": "effects/#1/invert", "value": true})).unwrap(); + assert_eq!(px(&s)[..3], [0.0, 0.0, 0.0]); + let list = s.execute("effect.plugins.list", json!({})).unwrap(); + assert!(list["plugins"].as_array().unwrap().iter().any(|p| p["source"].as_str().unwrap_or("").ends_with(".wasm"))); +} diff --git a/crates/ui-egui/src/dock.rs b/crates/ui-egui/src/dock.rs index 91d7c335..6f045101 100644 --- a/crates/ui-egui/src/dock.rs +++ b/crates/ui-egui/src/dock.rs @@ -36,6 +36,8 @@ pub enum PanelKind { MaskInterpolation, ScriptConsole, EssentialGraphics, + /// A dockable ScriptUI panel (a script from the ScriptUI Panels folder), by script window id. + ScriptPanel(u32), } impl PanelKind { @@ -95,6 +97,7 @@ impl PanelKind { PanelKind::MaskInterpolation => "Mask Interpolation", PanelKind::ScriptConsole => "Script Console", PanelKind::EssentialGraphics => "Essential Graphics", + PanelKind::ScriptPanel(_) => "ScriptUI Panel", } } pub fn id(self) -> String { diff --git a/crates/ui-egui/src/dock_ui.rs b/crates/ui-egui/src/dock_ui.rs index 44900918..e72d3989 100644 --- a/crates/ui-egui/src/dock_ui.rs +++ b/crates/ui-egui/src/dock_ui.rs @@ -18,6 +18,12 @@ impl EffectcraftApp { /// "Effect Controls Title", "Properties: Title", and the Timeline tab named after its comp. fn tab_titles(&self) -> Vec<(PanelKind, String)> { let mut out = Vec::new(); + // ScriptUI panels are named after their script. + for w in self.session.script_ui.windows.iter().filter(|w| w.kind == effectcraft_engine::scriptui::WindowKind::Panel) { + if let Some(t) = panels::scriptui_view::panel_title(self, w.id) { + out.push((PanelKind::ScriptPanel(w.id), t)); + } + } let Some(comp) = self.session.active_comp() else { return out }; let cname = self.session.active_comp_id().and_then(|id| self.session.project.item(id)).map(|i| i.name.clone()).unwrap_or_default(); out.push((PanelKind::Composition, format!("Composition {cname}"))); @@ -45,6 +51,13 @@ impl EffectcraftApp { /// Close a panel wherever it is (docked or floating). pub fn close_panel(&mut self, p: PanelKind) { + // Closing a ScriptUI panel closes its script window (its script's onClose runs). + if let PanelKind::ScriptPanel(id) = p + && self.session.script_ui.window(id).is_some() + && let Err(e) = self.session.execute("scriptui.close", serde_json::json!({"window": id})) + { + self.ui.status = e.to_string(); + } self.ui.dock.close(p); self.edit_layout(|l| l.unfloat(p)); if self.ui.maximized == Some(p) { diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 912f2e68..9cc2113a 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -834,6 +834,8 @@ impl EffectcraftApp { panels::precomp::mini_flowchart(self, &ctx); panels::home::capture_thumbnail(self); panels::dialogs::show(self, &ctx); + panels::scriptui_view::sync_panels(self); + panels::scriptui_view::show_windows(self, &ctx); self.draw_toast(ui, full); } diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index eed7e639..a6ad8542 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -456,6 +456,12 @@ pub fn frontend(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, p: Valu app.dialog = Some(crate::Dialog::About); Value::Null } + // Window ▸ : dock (or bring forward) the panel the script built. + "window.scriptPanel" => { + let id = p.get("window").and_then(Value::as_u64).ok_or("no ScriptUI panel window")? as u32; + app.show_panel(PanelKind::ScriptPanel(id)); + json!({"window": id}) + } "layer.style.options" => { crate::panels::layer_styles_dialog::open(app, &p)?; Value::Null @@ -791,7 +797,9 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option ("path", Ask::Save("Animation.json")), "file.importLottie" => ("path", Ask::Open(&["json", "lottie"])), "render.saveCurrentPreview" => ("path", Ask::Save("Preview.mp4")), - "file.runScript" => ("path", Ask::Open(&["jsx", "js", "jsonl", "json", "txt"])), + "file.runScript" if params.get("name").is_none() => ("path", Ask::Open(&["jsx", "js", "jsonl", "json", "txt"])), + "file.installScript" | "file.installScriptUIPanel" => ("path", Ask::Open(&["jsx", "js"])), + "effect.plugins.load" if params.get("folder").is_none() => ("path", Ask::Open(&["wasm", "wat"])), "file.replaceFootage" => ("path", Ask::Import), "file.collectFiles" => ("folder", Ask::Save("Collected Files")), "essential.exportTemplate" => ("path", Ask::Save("Template.ectemplate")), @@ -1106,6 +1114,39 @@ pub fn handle_shortcuts(app: &mut EffectcraftApp, ctx: &egui::Context) { } } +/// Menu entries listed from the session: (label, command, params). File ▸ Scripts starts with +/// the installed and sample scripts; the Window menu ends with the ScriptUI panels (as in After +/// Effects, by file name). +pub fn dynamic_entries(app: &EffectcraftApp, menu: &str) -> Vec<(String, String, Value)> { + let panels = match menu { + "Scripts" => false, + "Window" => true, + _ => return vec![], + }; + effectcraft_engine::commands::scripts::scripts(&app.session) + .into_iter() + .filter(|e| e.panel == panels) + .map(|e| { + let cmd = if panels { "window.scriptPanel" } else { "file.runScript" }; + let params = json!({"name": e.name}); + (e.name, cmd.to_string(), params) + }) + .collect() +} + +fn dynamic_rows(app: &mut EffectcraftApp, ui: &mut egui::Ui, menu: &str, clicked: &mut Option<(String, Value)>) -> usize { + let rows = dynamic_entries(app, menu); + for (i, (label, cmd, params)) in rows.iter().enumerate() { + let r = ui.add(egui::Button::new((gutter(false), label.as_str()))); + app.auto.add(&format!("menu.{}.{i}", if menu == "Window" { "scriptPanels" } else { "scripts" }), r.rect, label); + if r.clicked() { + *clicked = Some((cmd.clone(), params.clone())); + ui.close(); + } + } + rows.len() +} + /// Draw the in-window menu bar (the engine's AE menu tree). pub fn menu_bar(app: &mut EffectcraftApp, ui: &mut egui::Ui) { let ctx = ui.ctx().clone(); @@ -1118,6 +1159,10 @@ pub fn menu_bar(app: &mut EffectcraftApp, ui: &mut egui::Ui) { let r = ui.menu_button(label, |ui| { ui.set_min_width(if label == "Effect" { 200.0 } else { 280.0 }); menu_nodes(app, ui, children, &mut clicked); + if label == "Window" && !dynamic_entries(app, "Window").is_empty() { + ui.separator(); + dynamic_rows(app, ui, "Window", &mut clicked); + } }); app.auto.add(&format!("menu.{label}"), r.response.rect, label); } @@ -1158,6 +1203,16 @@ fn menu_nodes(app: &mut EffectcraftApp, ui: &mut egui::Ui, nodes: &[MenuNode], c menu_nodes(app, ui, children, clicked); }); } + MenuNode::Submenu { label, children } if label == "Scripts" => { + // File ▸ Scripts: installed and sample scripts, then Install… / Run Script File…. + ui.menu_button((gutter(false), label.as_str()), |ui| { + ui.set_min_width(280.0); + if dynamic_rows(app, ui, "Scripts", clicked) > 0 { + ui.separator(); + } + menu_nodes(app, ui, children, clicked); + }); + } MenuNode::Submenu { label, children } => { ui.menu_button((gutter(false), label.as_str()), |ui| { ui.set_min_width(if children.len() > 30 { 200.0 } else { 240.0 }); diff --git a/crates/ui-egui/src/native_menu.rs b/crates/ui-egui/src/native_menu.rs index ef5070b3..4558b3e2 100644 --- a/crates/ui-egui/src/native_menu.rs +++ b/crates/ui-egui/src/native_menu.rs @@ -154,7 +154,31 @@ pub fn build(app: &EffectcraftApp) -> NativeMenu { kids.push(NativeNode::Separator); } } + let dynamic = |kids: &mut Vec, menu: &str| { + for (k, (label, command, params)) in crate::menus::dynamic_entries(app, menu).into_iter().enumerate() { + kids.push(NativeNode::Item(NativeItem { + id: format!("{id}.d{k}"), + label, + command, + params, + shortcut: None, + accelerator: None, + enabled: true, + checked: None, + })); + } + }; + if label == "Scripts" { + dynamic(&mut kids, "Scripts"); + if !kids.is_empty() { + kids.push(NativeNode::Separator); + } + } rec(app, children, &id, &mut kids); + if label == "Window" && !crate::menus::dynamic_entries(app, "Window").is_empty() { + kids.push(NativeNode::Separator); + dynamic(&mut kids, "Window"); + } out.push(NativeNode::Submenu { label: label.clone(), children: kids }); } MenuNode::Item(e) => { diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index d0fb3a3e..16cc4d82 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -29,6 +29,7 @@ pub mod puppet_tool; pub mod render_queue; pub mod roto_tool; pub mod script_console; +pub mod scriptui_view; pub mod settings; pub mod shortcut_editor; pub mod text_panels; @@ -98,6 +99,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: PanelKind, rect: Rec PanelKind::Paint => paint_panels::paint(app, ui, rect), PanelKind::Brushes => paint_panels::brushes(app, ui, rect), PanelKind::EssentialGraphics => essential::show(app, ui, rect), + PanelKind::ScriptPanel(id) => scriptui_view::panel(app, ui, rect, id), PanelKind::Flowchart => flowchart::show(app, ui, rect), } } diff --git a/crates/ui-egui/src/panels/scriptui_view.rs b/crates/ui-egui/src/panels/scriptui_view.rs new file mode 100644 index 00000000..580e287b --- /dev/null +++ b/crates/ui-egui/src/panels/scriptui_view.rs @@ -0,0 +1,321 @@ +//! ScriptUI windows: the dialogs, palettes and dockable panels scripts build +//! (`Session::script_ui`). Each control is drawn at the bounds the engine's ScriptUI layout +//! computed; what the user does goes back through the `scriptui.click` / `scriptui.set` / +//! `scriptui.close` commands, which run the script's handlers. Every control registers an +//! automation id `scriptui..` (and `scriptui..` when the script +//! named it). + +use effectcraft_engine::scriptui::{ScriptWindow, Widget, WidgetKind, WindowKind, layout}; +use egui::{Align2, Color32, Rect, Sense, Stroke, StrokeKind, UiBuilder, pos2, vec2}; +use serde_json::{Value, json}; + +use crate::EffectcraftApp; +use crate::dock::PanelKind; +use crate::theme::Tokens; + +enum Act { + Click(u32, u32), + Set(u32, u32, Value), + Close(u32), +} + +fn rect_of(origin: egui::Pos2, b: [f64; 4]) -> Rect { + Rect::from_min_size(origin + vec2(b[0] as f32, b[1] as f32), vec2(b[2] as f32, b[3] as f32)) +} + +fn register(app: &mut EffectcraftApp, win: u32, w: &Widget, rect: Rect) { + let label = if w.text.is_empty() { format!("{:?}", w.kind) } else { w.text.clone() }; + app.auto.add(&format!("scriptui.{win}.{}", w.id), rect, &label); + if !w.name.is_empty() { + app.auto.add(&format!("scriptui.{win}.{}", w.name), rect, &label); + } +} + +/// Draw `w`'s children (and, for containers, their children) at `origin` + bounds. +fn draw_children(app: &mut EffectcraftApp, ui: &mut egui::Ui, origin: egui::Pos2, win: u32, w: &Widget, acts: &mut Vec) { + let t = app.tokens; + if w.kind == WidgetKind::TabbedPanel { + // Tab headers, then the active tab only. + let r = rect_of(origin, w.bounds); + let mut x = r.min.x; + for (i, tab) in w.children.iter().enumerate() { + let tw = (tab.text.chars().count() as f32 * 7.0 + 20.0).max(48.0); + let hr = Rect::from_min_size(pos2(x, r.min.y), vec2(tw, 22.0)); + let active = i == w.active_tab; + let resp = ui.interact(hr, egui::Id::new(("sui-tab", win, tab.id)), Sense::click()); + ui.painter().rect_filled( + hr, + 3.0, + if active { + t.row_selected + } else if resp.hovered() { + t.hover + } else { + t.field_bg + }, + ); + ui.painter().text(hr.center(), Align2::CENTER_CENTER, &tab.text, Tokens::ui(12.0), t.text); + register(app, win, tab, hr); + if resp.clicked() && !active && tab.enabled { + acts.push(Act::Click(win, tab.id)); + } + x += tw + 2.0; + } + ui.painter().rect_stroke(Rect::from_min_max(pos2(r.min.x, r.min.y + 22.0), r.max), 3.0, Stroke::new(1.0, t.field_border), StrokeKind::Inside); + if let Some(tab) = w.children.get(w.active_tab) { + draw_children(app, ui, origin, win, tab, acts); + } + return; + } + for c in w.children.iter().filter(|c| c.visible) { + draw_widget(app, ui, origin, win, c, acts); + } +} + +fn draw_widget(app: &mut EffectcraftApp, ui: &mut egui::Ui, origin: egui::Pos2, win: u32, w: &Widget, acts: &mut Vec) { + let t = app.tokens; + let rect = rect_of(origin, w.bounds); + register(app, win, w, rect); + let id = egui::Id::new(("sui", win, w.id)); + let tip = |r: egui::Response| if w.help_tip.is_empty() { r } else { r.on_hover_text(&w.help_tip) }; + match w.kind { + WidgetKind::Group | WidgetKind::Tab | WidgetKind::Window | WidgetKind::TabbedPanel => draw_children(app, ui, origin, win, w, acts), + WidgetKind::Panel => { + let p = ui.painter(); + p.rect_stroke(rect.shrink(1.0), 4.0, Stroke::new(1.0, t.field_border), StrokeKind::Inside); + if !w.text.is_empty() { + let g = p.layout_no_wrap(w.text.clone(), Tokens::ui(11.5), t.text_dim); + let at = pos2(rect.min.x + 10.0, rect.min.y - 1.0); + p.rect_filled(Rect::from_min_size(at - vec2(3.0, 0.0), g.size() + vec2(6.0, 0.0)), 0.0, t.panel_bg); + p.galley(at, g, t.text_dim); + } + draw_children(app, ui, origin, win, w, acts); + } + WidgetKind::Button | WidgetKind::IconButton => { + let r = ui.add_enabled_ui(w.enabled, |ui| ui.put(rect, egui::Button::new(&w.text))).inner; + if tip(r).clicked() { + acts.push(Act::Click(win, w.id)); + } + } + WidgetKind::StaticText => { + let col = if w.enabled { t.text } else { t.text_faint }; + ui.painter().with_clip_rect(rect.expand(2.0)).text(pos2(rect.min.x, rect.min.y + 1.0), Align2::LEFT_TOP, &w.text, Tokens::ui(12.0), col); + } + WidgetKind::EditText => { + // The text being typed lives here until the field loses focus (then onChange). + let mut buf: String = ui.data(|d| d.get_temp::(id)).unwrap_or_else(|| w.text.clone()); + let focused = ui.memory(|m| m.has_focus(id)); + if !focused { + buf = w.text.clone(); + } + let te = if w.multiline { egui::TextEdit::multiline(&mut buf) } else { egui::TextEdit::singleline(&mut buf) }; + let te = te.id(id).interactive(!w.read_only).font(Tokens::ui(12.0)); + let r = ui.add_enabled_ui(w.enabled, |ui| ui.put(rect, te)).inner; + let r = tip(r); + if r.lost_focus() && buf != w.text { + acts.push(Act::Set(win, w.id, json!(buf))); + } + ui.data_mut(|d| d.insert_temp(id, buf)); + } + WidgetKind::Checkbox => { + let mut v = w.checked; + let r = ui.add_enabled_ui(w.enabled, |ui| ui.put(snug(rect, &w.text), egui::Checkbox::new(&mut v, &w.text))).inner; + if tip(r).clicked() { + acts.push(Act::Click(win, w.id)); + } + } + WidgetKind::RadioButton => { + let r = ui.add_enabled_ui(w.enabled, |ui| ui.put(snug(rect, &w.text), egui::RadioButton::new(w.checked, &w.text))).inner; + if tip(r).clicked() && !w.checked { + acts.push(Act::Click(win, w.id)); + } + } + WidgetKind::Slider | WidgetKind::Scrollbar => { + // While dragging, the value lives here; the release sends it (onChanging / onChange). + let mut v: f64 = ui.data(|d| d.get_temp::(id)).unwrap_or(w.value); + let (lo, hi) = if w.max > w.min { (w.min, w.max) } else { (0.0, 100.0) }; + let r = ui + .add_enabled_ui(w.enabled, |ui| { + ui.scope_builder(UiBuilder::new().max_rect(rect), |ui| { + ui.spacing_mut().slider_width = rect.width() - 4.0; + ui.add(egui::Slider::new(&mut v, lo..=hi).show_value(false)) + }) + .inner + }) + .inner; + let r = tip(r); + if r.dragged() { + ui.data_mut(|d| d.insert_temp(id, v)); + } else { + ui.data_mut(|d| d.remove::(id)); + if r.drag_stopped() || r.changed() { + acts.push(Act::Set(win, w.id, json!(v))); + } + } + } + WidgetKind::Progressbar => { + let span = if w.max > w.min { w.max - w.min } else { 100.0 }; + let f = ((w.value - w.min) / span).clamp(0.0, 1.0) as f32; + ui.put(rect, egui::ProgressBar::new(f).desired_height(rect.height())); + } + WidgetKind::DropDownList => { + let cur = w.selection.first().and_then(|i| w.items.get(*i)).cloned().unwrap_or_default(); + let mut pick: Option = None; + ui.add_enabled_ui(w.enabled, |ui| { + ui.scope_builder(UiBuilder::new().max_rect(rect), |ui| { + egui::ComboBox::from_id_salt(id).selected_text(cur).width(rect.width() - 8.0).show_ui(ui, |ui| { + for (i, it) in w.items.iter().enumerate() { + if it == "-" { + ui.separator(); + } else if ui.selectable_label(w.selection.contains(&i), it).clicked() { + pick = Some(i); + } + } + }); + }); + }); + if let Some(i) = pick + && !w.selection.contains(&i) + { + acts.push(Act::Set(win, w.id, json!(i))); + } + } + WidgetKind::ListBox => { + let mut pick: Option = None; + ui.painter().rect_filled(rect, 2.0, t.field_bg); + ui.add_enabled_ui(w.enabled, |ui| { + ui.scope_builder(UiBuilder::new().max_rect(rect.shrink(2.0)), |ui| { + egui::ScrollArea::vertical().id_salt(id).max_height(rect.height() - 4.0).show(ui, |ui| { + ui.set_width(rect.width() - 8.0); + for (i, it) in w.items.iter().enumerate() { + if ui.selectable_label(w.selection.contains(&i), it).clicked() { + pick = Some(i); + } + } + }); + }); + }); + if let Some(i) = pick { + acts.push(Act::Set(win, w.id, json!(i))); + } + } + WidgetKind::Image | WidgetKind::Unknown => { + ui.painter().rect_stroke(rect, 2.0, Stroke::new(1.0, t.field_border), StrokeKind::Inside); + } + } +} + +/// Checkboxes and radio buttons sit at the left of their (possibly filled) bounds. +fn snug(r: Rect, text: &str) -> Rect { + let w = (text.chars().count() as f32 * 7.0 + 26.0).min(r.width()); + Rect::from_min_size(r.min, vec2(w, r.height())) +} + +fn perform(app: &mut EffectcraftApp, ctx: &egui::Context, acts: Vec) { + for a in acts { + let (cmd, p) = match a { + Act::Click(w, id) => ("scriptui.click", json!({"window": w, "widget": id})), + Act::Set(w, id, v) => ("scriptui.set", json!({"window": w, "widget": id, "value": v})), + Act::Close(w) => ("scriptui.close", json!({"window": w})), + }; + match crate::menus::invoke(app, ctx, cmd, p) { + Ok(r) => { + if let Some(e) = r.get("error").filter(|e| !e.is_null()) { + let msg = e["message"].as_str().unwrap_or("script error").to_string(); + app.toast = Some((msg, ctx.input(|i| i.time))); + } else if let Some(o) = r["output"].as_str().filter(|o| !o.is_empty()) { + app.ui.status = o.lines().last().unwrap_or_default().to_string(); + } + } + Err(e) => app.ui.status = e, + } + } +} + +/// The floating script windows (dialogs, palettes, windows), drawn over the app. +pub fn show_windows(app: &mut EffectcraftApp, ctx: &egui::Context) { + let windows: Vec = app.session.script_ui.windows.iter().filter(|w| w.kind != WindowKind::Panel).cloned().collect(); + if windows.is_empty() { + return; + } + let t = app.tokens; + let mut acts = vec![]; + let screen = ctx.content_rect(); + if windows.iter().any(|w| w.modal) { + // A modal dialog blocks the app behind it, as in After Effects. + let r = egui::Area::new(egui::Id::new("scriptui-modal-block")).order(egui::Order::Middle).fixed_pos(screen.min).show(ctx, |ui| { + ui.allocate_exact_size(screen.size(), Sense::click_and_drag()); + ui.painter().rect_filled(screen, 0.0, Color32::from_black_alpha(90)); + }); + let _ = r; + } + for w in &windows { + let size = vec2(w.root.bounds[2] as f32, w.root.bounds[3] as f32); + let mut open = true; + let order = if w.modal { egui::Order::Foreground } else { egui::Order::Middle }; + let resp = egui::Window::new(if w.title.is_empty() { &w.script } else { &w.title }) + .id(egui::Id::new(("scriptui-window", w.id))) + .order(order) + .collapsible(false) + .resizable(false) + .open(&mut open) + .default_pos(screen.center() - size / 2.0) + .frame(egui::Frame::window(&ctx.global_style()).fill(t.panel_bg)) + .show(ctx, |ui| { + let (rect, _) = ui.allocate_exact_size(size, Sense::hover()); + draw_children(app, ui, rect.min, w.id, &w.root, &mut acts); + rect + }); + if let Some(r) = resp { + app.auto.add(&format!("scriptui.{}", w.id), r.response.rect, &w.title); + } + if !open { + acts.push(Act::Close(w.id)); + } + } + perform(app, ctx, acts); +} + +/// A dockable ScriptUI panel's content (Window ▸ .jsx). +pub fn panel(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect, id: u32) { + let t = app.tokens; + let Some(mut w) = app.session.script_ui.window(id).cloned() else { + ui.painter().text(rect.center(), Align2::CENTER_CENTER, "This ScriptUI panel has closed", Tokens::ui(12.0), t.text_faint); + return; + }; + // Docked panels fill their frame (ScriptUI lays them out at the panel's size). + let (pw, ph) = (rect.width() as f64, rect.height() as f64); + w.root.fixed_bounds = Some([0.0, 0.0, pw.max(w.root.bounds[2]), ph.max(w.root.bounds[3])]); + layout(&mut w); + let mut acts = vec![]; + let content = rect; + egui::ScrollArea::both().id_salt(("scriptui-panel", id)).max_width(content.width()).max_height(content.height()).show(ui, |ui| { + let (r, _) = ui.allocate_exact_size(vec2(w.root.bounds[2] as f32, w.root.bounds[3] as f32), Sense::hover()); + draw_children(app, ui, r.min, id, &w.root, &mut acts); + }); + app.auto.add(&format!("scriptui.{id}"), rect, &w.title); + let ctx = ui.ctx().clone(); + perform(app, &ctx, acts); +} + +/// Tab title of a script panel. +pub fn panel_title(app: &EffectcraftApp, id: u32) -> Option { + app.session.script_ui.window(id).map(|w| if w.title.is_empty() { w.script.clone() } else { w.title.clone() }) +} + +/// Close the dock panels whose script window has gone (the script closed it, or a saved +/// workspace from an earlier run). +pub fn sync_panels(app: &mut EffectcraftApp) { + let mut all = vec![]; + app.ui.dock.panels(&mut all); + for f in &app.ui.floating { + all.extend(f.panels.iter().copied()); + } + for p in all { + if let PanelKind::ScriptPanel(id) = p + && app.session.script_ui.window(id).is_none() + { + app.close_panel(p); + } + } +} diff --git a/crates/ui-egui/tests/ui_scriptui.rs b/crates/ui-egui/tests/ui_scriptui.rs new file mode 100644 index 00000000..62102904 --- /dev/null +++ b/crates/ui-egui/tests/ui_scriptui.rs @@ -0,0 +1,112 @@ +//! M13.1 UI: ScriptUI dialogs, palettes and dockable panels drawn from the session's script +//! windows (clicks reach the script's handlers), the branching History panel, and the Window / +//! File ▸ Scripts menus listing scripts (egui_kittest, UI logic only). + +use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::dock::PanelKind; +use egui::{Event, Rect, pos2, vec2}; +use egui_kittest::Harness; +use serde_json::json; + +fn harness() -> Harness<'static, EffectcraftApp> { + let mut s = effectcraft_host::session(); + s.execute("comp.new", json!({"name": "Main", "width": 320, "height": 180, "duration": 4})).unwrap(); + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_| EffectcraftApp::new(s)); + h.run_steps(3); + h +} + +fn rect(h: &Harness<'_, EffectcraftApp>, id: &str) -> Rect { + let e = h.state().auto.find(id).unwrap_or_else(|| panic!("no {id}")); + Rect::from_min_size(pos2(e.rect[0], e.rect[1]), vec2(e.rect[2], e.rect[3])) +} + +fn click(h: &mut Harness<'_, EffectcraftApp>, id: &str) { + let c = rect(h, id).center(); + h.input_mut().events.push(Event::PointerMoved(c)); + for pressed in [true, false] { + h.input_mut().events.push(Event::PointerButton { pos: c, button: egui::PointerButton::Primary, pressed, modifiers: Default::default() }); + } + h.run_steps(3); +} + +fn layer_names(h: &Harness<'_, EffectcraftApp>) -> Vec { + h.state().session.active_comp().map(|c| c.layers.iter().map(|l| l.name.clone()).collect()).unwrap_or_default() +} + +#[test] +fn script_windows_are_drawn_and_clickable() { + let mut h = harness(); + let code = r#" + var w = new Window("palette", "Quick Solid"); + var n = w.add("edittext", undefined, "Card"); + n.characters = 10; + var go = w.add("button", undefined, "Add", { name: "add" }); + go.onClick = function () { app.project.activeItem.layers.addSolid([0, 1, 0], n.text, 50, 50, 1); }; + w.show(); + "#; + let r = h.state_mut().session.execute("script.run", json!({"code": code, "name": "quick.jsx"})).unwrap(); + assert_eq!(r["ok"], true, "{r}"); + h.run_steps(3); + let win = h.state().session.script_ui.windows[0].id; + // Every control has an automation id (by id and by name). + for id in [format!("scriptui.{win}"), format!("scriptui.{win}.1"), format!("scriptui.{win}.2"), format!("scriptui.{win}.add")] { + assert!(h.state().auto.find(&id).is_some(), "missing {id}"); + } + click(&mut h, &format!("scriptui.{win}.add")); + assert_eq!(layer_names(&h), ["Card"]); + // A modal dialog: the script waits in show() until OK is clicked. + let code = r#" + var d = new Window("dialog", "Confirm"); + d.add("statictext", undefined, "Make a null?"); + var g = d.add("group"); + g.add("button", undefined, "Cancel", { name: "cancel" }); + g.add("button", undefined, "OK", { name: "ok" }); + if (d.show() === 1) app.project.activeItem.layers.addNull().name = "Yes"; + "#; + let r = h.state_mut().session.execute("script.run", json!({"code": code, "name": "confirm.jsx"})).unwrap(); + assert_eq!(r["waiting"], true, "{r}"); + h.run_steps(3); + let d = h.state().session.script_ui.windows.iter().find(|w| w.modal).unwrap().id; + click(&mut h, &format!("scriptui.{d}.ok")); + assert_eq!(layer_names(&h), ["Yes", "Card"]); + assert!(h.state().session.script_ui.windows.iter().all(|w| !w.modal)); + // Window ▸ : the sample ScriptUI panel docks as its own tab. + h.state_mut().session.execute("window.scriptPanel", json!({"name": "Layer Tools.jsx"})).unwrap(); + h.run_steps(3); + let panel = h.state().session.script_ui.windows.iter().find(|w| w.script == "Layer Tools.jsx").unwrap().id; + assert!(h.state().ui.dock.contains(PanelKind::ScriptPanel(panel)), "the panel docks"); + assert!(h.state().auto.find(&format!("scriptui.{panel}")).is_some()); + // The menus list scripts and panels. + let window_menu = effectcraft_ui_egui::menus::dynamic_entries(h.state(), "Window"); + assert!(window_menu.iter().any(|(l, c, _)| l == "Layer Tools.jsx" && c == "window.scriptPanel")); + let scripts = effectcraft_ui_egui::menus::dynamic_entries(h.state(), "Scripts"); + assert!(scripts.iter().any(|(l, c, _)| l == "Rename Layers.jsx" && c == "file.runScript")); + // Closing the panel's tab closes its script window. + h.state_mut().close_panel(PanelKind::ScriptPanel(panel)); + h.run_steps(2); + assert!(h.state().session.script_ui.window(panel).is_none()); + h.state_mut().session.execute("scriptui.close", json!({})).unwrap(); +} + +#[test] +fn history_panel_jumps_between_branches() { + let mut h = harness(); + for n in ["A", "B"] { + h.state_mut().session.execute("layer.newSolid", json!({"name": n, "color": "#ffffff", "width": 10, "height": 10})).unwrap(); + } + h.state_mut().session.undo(); + h.state_mut().session.execute("layer.newSolid", json!({"name": "C", "color": "#ffffff", "width": 10, "height": 10})).unwrap(); + assert_eq!(layer_names(&h), ["C", "A"]); + h.state_mut().show_panel(PanelKind::History); + h.run_steps(3); + // Original, New Composition, A, C, and B on its branch. + let states = h.state().session.history_tree(); + assert_eq!(states.len(), 5); + let b = states.iter().find(|n| n.depth == 1).unwrap().index; + click(&mut h, &format!("history.state.{b}")); + assert_eq!(layer_names(&h), ["B", "A"]); + let c = h.state().session.history_tree().iter().find(|n| n.depth == 1).unwrap().index; + click(&mut h, &format!("history.state.{c}")); + assert_eq!(layer_names(&h), ["C", "A"]); +} diff --git a/docs/agents.md b/docs/agents.md index 8689b9da..da0318c4 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -220,6 +220,35 @@ In expressions, `text.sourceText.style` / `getStyleAt(i, t)` read styles and the (`setFontSize(v, start?, count?)`, `setFillColor`, `setText`, `setJustification`…) return a styled document. +### Scripts and ScriptUI windows + +Scripts that build ScriptUI windows publish them to the session; agents drive them like a user: + +1. `scriptui.list` → `[{window, title, kind: dialog|palette|window|panel, script, modal, size}]`. +2. `scriptui.get {"window": id}` → the control tree (`type`, `name`, `text`, `value`, `checked`, + `items`, `selection`, laid-out `bounds`, `handlers`…). +3. `scriptui.click {"widget": "ok"}` presses a button / toggles a checkbox / picks a radio button or + tab; `scriptui.set {"widget": "#4", "value": "Shot_"}` types into edit text, moves a slider or + picks a list item (index or text); `scriptui.close {"result": 2}` closes. Controls are addressed + by id, `#id`, `properties.name` or text; `window` can be omitted when one window is open. Each + returns the handler run's `{ok, output, error}`. + +A dialog's `show()` waits for the user: the `script.run` / `file.runScript` reply carries +`"waiting": true`, and the script continues (its final output arrives in the reply of the click +that closes the dialog). File ▸ Scripts: `file.scripts.list`, `file.runScript {"name": …}`, +`file.installScript` / `file.installScriptUIPanel {"path": …}`, `window.scriptPanel {"name": …}`. + +### History, puppet recording, plug-ins + +* `edit.history.list` lists every undo state as a tree (undoing then editing keeps the undone + states as a branch); `edit.history.goto {"index": n}` (or `id`, or `steps`) jumps to any of + them. MCP: the `history` tool. +* `puppet.recordPin {"layer": "#1", "pin": "Puppet Pin 1", "samples": [[t, x, y]…]}` records a + drag (t = seconds since it began, layer space) into Position keys at the comp frame rate from the + current time; `puppet.recordOptions {speed, smoothing, useDraftDeformation, showMesh}`. +* `effect.plugins.load {"path": "x.wasm"}` / `effect.plugins.list`: WebAssembly effect plug-ins + ([plugins.md](plugins.md)), then `effect.apply` by id like a built-in. + ## CLI Each invocation runs a headless engine with no window. It opens the demo project unless you pass diff --git a/docs/parity.md b/docs/parity.md index 4437bece..a75d37f5 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -22,9 +22,13 @@ Tracking 88%, Effects 85%, Web 65%. What is left, in priority order: the web app's depth (threads, storage, audio, non-blocking renders); stroke taper/wave and multi-segment dashes; variable mask feather points; camera iris/bokeh and focus-link commands; Render Queue field render, crop/resize and templates; approximated effects (Subspace Warp, Key Cleaner) and Liquify's viewer brush; OpenType features and variable -font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; ScriptUI; -Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; face tracking; and -the "better than After Effects" items (a plug-in API, branching history; GPU particles landed in M12.7). +font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; +Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; and face tracking. +ScriptUI (dialogs that block `show()`, palettes, dockable ScriptUI panels, File ▸ Scripts install +and sample scripts), puppet pin recording and the remaining "better than After Effects" items — a +versioned effect plug-in API with sandboxed WebAssembly plug-ins ([plugins.md](plugins.md)), +branching undo history in the History panel, and a bit-identical rendering test across runs and +thread counts — landed in M13.1 (GPU particles landed in M12.7). The sections below are the original audit (morning of 2 October, ≈ 64%) and its updates, kept for history. @@ -112,10 +116,10 @@ the disk cache in the formats wave; the 3D Camera Tracker in M12.6.) | Output | 90% | 3.0 | multiple output modules, pre-render (WebM with VP9 alpha + Opus and WAV/AIFF audio-only landed: VP9 is intra-only, Opus CELT-only) | | Audio | 85% | 0.5 | audio to keyframes | | Import | 88% | 2.0 | AI/EPS/PDF vector footage, PSD smart objects and 3D layers (PSD as footage/composition/retain layer sizes, SVG footage and Create Shapes from Vector Layer landed) | -| Automation | ≈ 90% | 0.8 | scripting covers the documented core object model (AUT-2, M14.4: `app`, project items, comps, layers, properties and keyframes, text documents, markers, render queue, Script Console, `effectcraft-cli script`, MCP `run_script`); still missing: ScriptUI panels/dialogs, `.jsxbin`, sockets | +| Automation | ≈ 97% | 0.3 | `.jsxbin`, sockets, ScriptUI resource strings and custom `onDraw` graphics (the core object model landed in M14.4; ScriptUI windows/dialogs/dockable panels with `scriptui.*` agent commands, File ▸ Scripts install + sample scripts, and the effect plug-in API (EFF-6, WebAssembly) landed in M13.1) | | Shapes | ≈ 80% | 1.5 | Lottie can't carry stroke taper/wave (stroke Taper and Wave, Dash 2/Gap 2/Dash 3/Gap 3 and radial-gradient Highlight Length/Angle landed in M13.5; pen tool for shape paths and vertex editing in M6.5) | | Compositions | ≈ 80% | 3.5 | Mocha-style planar tracks for templates, Essential Graphics' rare controls (font menus, mirrored properties) (the marker dialog, Composition Flowchart, Essential Graphics with master properties, `.ectemplate` templates and Responsive Design — Time landed: CMP-6, CMP-7) | -| Animation | 67% | 10.0 | puppet, Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8; keyframe colour labels and Select Keyframe Label Group, Graph Editor snapping to markers / layer ends in M13.5) | +| Animation | 70% | 9.0 | puppet depth beyond pins and recording (puppet pin recording with Record Options landed in M13.1), Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8; keyframe colour labels and Select Keyframe Label Group, Graph Editor snapping to markers / layer ends in M13.5) | | Text | ≈ 91% | 0.8 | no OpenType feature panel, no extruded strokes, variable-axis animation changes outlines but not advances (vertical Roman / Tate-Chu-Yoko, forced LTR paragraphs, caret on animated and path text, Variable Font Axes and Lottie style runs landed in M13.5; extruded, bevelled text in M7.6; per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API in M9.9–M9.10) | | Web | 85% | 1.0 | viewer frames and Roto Brush propagation still on the page's thread; GPU effects in the browser (browser storage, Web Audio, Web Worker renders/analyses, WebGPU viewer and offline install landed in M15.2) | | 3D | 85% | 6.5 | multi-view layouts, iris shapes in Advanced 3D's depth of field, collapsed 3D precomps in Advanced 3D, the Extended Viewer; stereo rigs, orbit nulls, lights controlled by the camera, cameras/lights from glTF models, environment backgrounds, Advanced 3D motion blur, blend modes and track mattes landed in M7.7; Classic 3D iris-shaped bokeh with highlights, progressive depth of field on tilted layers and the focus-link commands landed in M13.5; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) in M7.4–M7.6 | diff --git a/docs/plugins.md b/docs/plugins.md new file mode 100644 index 00000000..5d5b39c0 --- /dev/null +++ b/docs/plugins.md @@ -0,0 +1,126 @@ +# Effect plug-ins + +EffectCraft loads third-party effects through a small, versioned plug-in API (**API version 1**). +A plug-in effect behaves exactly like a built-in one: it is listed in Effects & Presets (and the +Effect menu) under its category, applied by id (`effect.apply {"effect": "org.example.posterize"}`) +or display name, its parameters are ordinary properties (keyframes, expressions, Effect Controls, +`prop.set`), and it is saved in projects by id. + +There are two ways to write one: + +| | WebAssembly plug-in | Rust plug-in | +|---|---|---| +| What | a `.wasm` (or `.wat`) module | a type implementing `effectcraft_effects::plugin::EffectPlugin` | +| Loaded by | Effect ▸ Load Effect Plug-in…, `effect.plugins.load {path \| folder}`, or the `Plug-ins` folder next to the settings at start-up | `register_plugin(Arc::new(MyEffect))` in an app that links EffectCraft's crates | +| Sandbox | yes: no imports (no files, clock or network), fuel-limited, deterministic floats | none (it is your code) | +| Platforms | desktop and CLI (the wasmi interpreter, crate feature `effectcraft-plugin/wasm`); not the web build | everywhere | + +`effect.plugins.list` lists what is loaded: `{api, wasm, plugins: [{id, name, category, version, +author, source, params}]}`. + +## The contract + +* **Pixels** are premultiplied RGBA `f32` (0–1; values may exceed 1 in 32 bpc projects), row + major, processed **in place**. The buffer is the layer's (after masks and the effects above), + possibly at preview resolution (`scale` = buffer pixels per layer pixel) and padded. +* **Parameters** arrive flattened to `f64`s in the order they are declared: slider, angle, + checkbox (0 / 1) and popup (0-based index) take one value, point two (already in **buffer + pixels**), colour four (RGBA 0–1). +* **Time** is layer time in seconds. +* Rendering must be **deterministic** and may run on several threads at once (a WebAssembly + plug-in gets one instance per thread). +* A failed render (an error code, a trap, running out of fuel) leaves the frame unchanged. + +## The manifest + +Every plug-in describes itself with a JSON manifest: + +```json +{ + "api": 1, + "id": "org.effectcraft.example.posterize-bands", + "name": "Posterize Bands", + "category": "Stylize", + "version": "1.0.0", + "author": "EffectCraft contributors", + "description": "Quantizes luminance into tinted bands.", + "params": [ + {"id": "levels", "name": "Levels", "type": "slider", "default": 4, "min": 2, "max": 64, "sliderMax": 16, "decimals": 0}, + {"id": "mix", "name": "Tint Amount", "type": "slider", "default": 50, "min": 0, "max": 100}, + {"id": "tint", "name": "Tint", "type": "color", "default": [1.0, 0.55, 0.2, 1.0]}, + {"id": "invert", "name": "Invert Bands", "type": "checkbox", "default": false} + ] +} +``` + +* `id`: stable, reverse-domain; letters, digits, `.`, `_`, `-`. `ec.` is reserved for built-ins. + Ids must be unique (a second plug-in with a taken id is refused). +* `category`: one of the Effects & Presets categories (`Blur & Sharpen`, `Stylize`, …) or a new + one, which appears after the built-in categories. +* Parameter types: `slider` (`default`, `min`, `max`, optional `sliderMin`, `sliderMax`, + `decimals`), `angle` (`default` degrees), `checkbox` (`default` bool), `popup` (`options`, + `default` index), `point` (`default` as fractions of the layer size, `[0.5, 0.5]` = centre), + `color` (`default` RGBA 0–1). Parameter ids are letters, digits and `_`. + +## WebAssembly ABI (v1) + +A module with **no imports** that exports: + +| export | signature | | +|---|---|---| +| `memory` | memory | linear memory shared with the host | +| `ec_api_version` | `() -> i32` | must return `1` | +| `ec_manifest_ptr`, `ec_manifest_len` | `() -> i32` | the UTF-8 JSON manifest in memory | +| `ec_alloc` | `(bytes: i32) -> i32` | a buffer of at least `bytes` bytes, 8-byte aligned; it may reuse the previous one | +| `ec_render` | `(pixels: i32, width: i32, height: i32, params: i32, nparams: i32, time: f64, scale: f64) -> i32` | process the pixels in place; return 0 on success | + +For each frame the host calls `ec_alloc` once for a buffer holding the parameters (`nparams` +little-endian `f64`s) followed by the pixels (`width × height × 4` little-endian `f32`s), fills it, +calls `ec_render`, and reads the pixels back. Each call gets a fuel budget proportional to the +frame size; a module that loops forever stops with an error instead of hanging the app. + +### Example + +[`examples/plugins/posterize-bands`](../examples/plugins/posterize-bands) is a complete plug-in in +plain Rust (no dependencies): + +```sh +cd examples/plugins/posterize-bands +cargo build --release --target wasm32-unknown-unknown +# then, in EffectCraft: Effect ▸ Load Effect Plug-in… → target/wasm32-unknown-unknown/release/posterize_bands.wasm +# or headless: +effectcraft-cli exec effect.plugins.load '{"path": "target/wasm32-unknown-unknown/release/posterize_bands.wasm"}' +``` + +Any language that compiles to WebAssembly works (C, Zig, AssemblyScript…) as long as the module +has no imports. `crates/plugin/src/tests.rs` has a plug-in written directly in WebAssembly text. + +## Rust plug-ins + +```rust +use std::sync::Arc; +use effectcraft_effects::plugin::*; + +struct Swap(PluginManifest); + +impl EffectPlugin for Swap { + fn manifest(&self) -> &PluginManifest { &self.0 } + fn render(&self, frame: &mut PluginFrame, params: &PluginParams, _time: f64) -> Result<(), String> { + if params.b("on") { + for p in frame.pixels.iter_mut() { p.swap(0, 2); } + } + Ok(()) + } +} + +register_plugin(Arc::new(Swap(manifest)))?; // before the UI builds its menus +``` + +`PluginParams` reads values by parameter id (`f`, `b`, `point`, `color`). At most 64 plug-in +effects can be registered per process. + +## Compatibility + +The API version is checked when a plug-in loads: a manifest or module for another version is +refused with a message. Within a version the contract above does not change; additions (new +parameter types, new optional exports) bump the version. diff --git a/examples/plugins/posterize-bands/Cargo.lock b/examples/plugins/posterize-bands/Cargo.lock new file mode 100644 index 00000000..c1732e16 --- /dev/null +++ b/examples/plugins/posterize-bands/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "posterize-bands" +version = "1.0.0" diff --git a/examples/plugins/posterize-bands/Cargo.toml b/examples/plugins/posterize-bands/Cargo.toml new file mode 100644 index 00000000..825d704a --- /dev/null +++ b/examples/plugins/posterize-bands/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "posterize-bands" +description = "Example EffectCraft effect plug-in (WebAssembly, plug-in API v1): posterizes luminance into tinted bands" +version = "1.0.0" +edition = "2024" +license = "MIT OR Apache-2.0" +publish = false + +# Standalone: build with `cargo build --release --target wasm32-unknown-unknown` in this folder. +[workspace] + +[lib] +crate-type = ["cdylib"] + +[profile.release] +opt-level = "s" +lto = true +panic = "abort" diff --git a/examples/plugins/posterize-bands/src/lib.rs b/examples/plugins/posterize-bands/src/lib.rs new file mode 100644 index 00000000..9af70943 --- /dev/null +++ b/examples/plugins/posterize-bands/src/lib.rs @@ -0,0 +1,93 @@ +//! Posterize Bands: an example EffectCraft effect plug-in (plug-in API v1, see +//! `docs/plugins.md`). It quantizes each pixel's luminance into `levels` bands and mixes a tint +//! colour in by `mix` percent, optionally only above a `threshold`. +//! +//! Build: `cargo build --release --target wasm32-unknown-unknown`, then load +//! `target/wasm32-unknown-unknown/release/posterize_bands.wasm` with Effect ▸ Load Effect Plug-in +//! (or `effect.plugins.load {"path": …}`), or drop it in the settings folder's `Plug-ins`. + +use std::cell::RefCell; + +/// The manifest: id, name, Effects & Presets category, parameters (flattened in this order: +/// levels, mix, tint r g b a, invert). +const MANIFEST: &str = r#"{ + "api": 1, + "id": "org.effectcraft.example.posterize-bands", + "name": "Posterize Bands", + "category": "Stylize", + "version": "1.0.0", + "author": "EffectCraft contributors", + "description": "Quantizes luminance into tinted bands.", + "params": [ + {"id": "levels", "name": "Levels", "type": "slider", "default": 4, "min": 2, "max": 64, "sliderMax": 16, "decimals": 0}, + {"id": "mix", "name": "Tint Amount", "type": "slider", "default": 50, "min": 0, "max": 100, "decimals": 1}, + {"id": "tint", "name": "Tint", "type": "color", "default": [1.0, 0.55, 0.2, 1.0]}, + {"id": "invert", "name": "Invert Bands", "type": "checkbox", "default": false} + ] +}"#; + +thread_local! { + static BUF: RefCell> = const { RefCell::new(Vec::new()) }; +} + +#[unsafe(no_mangle)] +pub extern "C" fn ec_api_version() -> i32 { + 1 +} + +#[unsafe(no_mangle)] +pub extern "C" fn ec_manifest_ptr() -> i32 { + MANIFEST.as_ptr() as i32 +} + +#[unsafe(no_mangle)] +pub extern "C" fn ec_manifest_len() -> i32 { + MANIFEST.len() as i32 +} + +/// One reusable, 8-byte-aligned buffer for the parameters and pixels. +#[unsafe(no_mangle)] +pub extern "C" fn ec_alloc(bytes: i32) -> i32 { + BUF.with(|b| { + let mut b = b.borrow_mut(); + let words = (bytes.max(0) as usize).div_ceil(8); + if b.len() < words { + b.resize(words, 0); + } + b.as_mut_ptr() as i32 + }) +} + +/// # Safety +/// The host passes pointers into the buffer `ec_alloc` returned. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn ec_render(pixels: i32, width: i32, height: i32, params: i32, nparams: i32, _time: f64, _scale: f64) -> i32 { + if nparams < 7 { + return 1; + } + let p = unsafe { std::slice::from_raw_parts(params as *const f64, nparams as usize) }; + let px = unsafe { std::slice::from_raw_parts_mut(pixels as *mut [f32; 4], (width.max(0) * height.max(0)) as usize) }; + let levels = p[0].round().max(2.0) as f32; + let mix = (p[1] / 100.0).clamp(0.0, 1.0) as f32; + let tint = [p[2] as f32, p[3] as f32, p[4] as f32]; + let invert = p[6] != 0.0; + for c in px.iter_mut() { + let a = c[3]; + if a <= 0.0 { + continue; + } + // Un-premultiply, band the luminance, re-premultiply. + let (r, g, b) = (c[0] / a, c[1] / a, c[2] / a); + let y = 0.2126 * r + 0.7152 * g + 0.0722 * b; + let mut band = (y * levels).floor().min(levels - 1.0) / (levels - 1.0); + if invert { + band = 1.0 - band; + } + for (i, ch) in [r, g, b].into_iter().enumerate() { + let banded = if y > 0.0 { ch * band / y.max(1e-6) } else { band }; + let v = banded * (1.0 - mix) + band * tint[i] * mix; + c[i] = v.max(0.0) * a; + } + } + 0 +} From d209fa001f7361301a31c8aa506b90e234de8c81 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 04:36:12 -0400 Subject: [PATCH 5/9] M13.3: face tracking methods, Face Track Points / Measurements, camera lens distortion, docs Co-Authored-By: Claude Opus 5.5 --- crates/effects/src/camera_tracker.rs | 94 ++++++++++- crates/effects/src/face_track.rs | 45 +++++ crates/effects/src/lib.rs | 2 + crates/engine/src/commands/camera_cmds.rs | 18 +- crates/engine/src/commands/mask_interp.rs | 133 ++++++++++++++- crates/engine/src/lib.rs | 2 + crates/engine/src/mask_track.rs | 159 ++++++++++++++++-- crates/engine/src/tests_face.rs | 106 ++++++++++++ crates/track/src/camtrack/bundle.rs | 46 ++++- crates/track/src/camtrack/mod.rs | 109 +++++++++++- crates/track/src/camtrack/solver.rs | 125 ++++++++++++-- crates/track/src/camtrack/tests.rs | 65 ++++++- crates/track/src/face.rs | 5 + .../ui-egui/src/panels/camera_tracker_ui.rs | 14 +- crates/ui-egui/src/panels/tracker.rs | 28 ++- docs/agents.md | 27 ++- docs/architecture.md | 17 +- docs/effects.md | 10 +- docs/parity.md | 8 +- 19 files changed, 939 insertions(+), 74 deletions(-) create mode 100644 crates/effects/src/face_track.rs create mode 100644 crates/engine/src/tests_face.rs diff --git a/crates/effects/src/camera_tracker.rs b/crates/effects/src/camera_tracker.rs index dd97b585..ff4e51fe 100644 --- a/crates/effects/src/camera_tracker.rs +++ b/crates/effects/src/camera_tracker.rs @@ -8,8 +8,12 @@ //! Shot Type, Angle of View, Solve Method and deleted points it was solved with, so changing //! those re-solves without re-tracking). **Method Used** and **Average Error** report the solve. //! -//! The effect passes its input through; with **Render Track Points** on it draws the solved -//! points (as the viewer shows them) into the frame. The viewer overlay, target and the Create +//! **Solve Lens Distortion** (Advanced; also on with Detailed Analysis) solves radial lens +//! distortion `k1, k2` jointly with the camera; **Undistort Footage** then renders the input +//! undistorted (the pinhole image the solved camera sees) so 3D layers line up with it. +//! +//! The effect passes its input through (undistorted when asked); with **Render Track Points** +//! on it draws the solved points (as the viewer shows them) into the frame. The viewer overlay, target and the Create //! commands live in the engine and the UI. use std::collections::HashMap; @@ -19,6 +23,7 @@ use effectcraft_keyframe::Value; use effectcraft_project::ParamUi; use effectcraft_raster::{Image, Px}; use effectcraft_track::camtrack::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, linalg, point_color}; +use rayon::prelude::*; use crate::warp_stab::param; use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; @@ -58,6 +63,8 @@ pub fn specs() -> Vec { p(METHOD_USED, "Method Used", Value::Str(String::new()), ParamUi::Hidden), p(AVERAGE_ERROR, "Average Error", num(0.0), ParamUi::Hidden), p("advanced/detailedAnalysis", "Detailed Analysis", Value::Bool(false), ParamUi::Checkbox), + p("advanced/lensDistortion", "Solve Lens Distortion", Value::Bool(false), ParamUi::Checkbox), + p("advanced/undistort", "Undistort Footage", Value::Bool(false), ParamUi::Checkbox), p("advanced/autoDeletePoints", "Auto-delete Points Across Time", Value::Bool(true), ParamUi::Checkbox), p("advanced/hideWarningBanner", "Hide Warning Banner", Value::Bool(false), ParamUi::Checkbox), p(TRACKS, "Tracks", Value::Str(String::new()), ParamUi::Hidden), @@ -104,13 +111,15 @@ pub fn settings(params: &Params) -> SolveSettings { hfov: f(params, "horizontalAngleOfView"), method: SolveMethod::ALL.get(e(params, "advanced/solveMethod") as usize).copied().unwrap_or_default(), deleted: del, + lens_distortion: b(params, "advanced/lensDistortion") || b(params, "advanced/detailedAnalysis"), } } /// What a solve depends on besides the tracks. pub fn solve_key(st: &SolveSettings) -> String { let hfov = if st.shot == ShotType::SpecifyAngle { format!("{:.4}", st.hfov) } else { String::new() }; - format!("{:?}|{hfov}|{:?}|{:?}", st.shot, st.method, st.deleted) + let lens = if st.lens_distortion { "|lens" } else { "" }; + format!("{:?}|{hfov}|{:?}|{:?}{lens}", st.shot, st.method, st.deleted) } fn fnv(bytes: &[u8]) -> u64 { @@ -165,18 +174,49 @@ pub fn target_radius(size_pct: f64, z: f64, median_z: f64) -> f64 { (6.0 * size_pct / 100.0 * (median_z / z.max(1e-9))).clamp(1.5, 40.0) } -/// The solved points visible on frame `k`: (id, layer pixel position, camera depth). +/// The solved points visible on frame `k`: (id, layer pixel position in the footage, camera +/// depth). pub fn projected(solve: &CameraSolve, k: usize) -> Vec<(u32, [f64; 2], f64)> { + projected_with(solve, k, false) +} + +/// [`projected`] into the footage, or (`undistorted`) into the undistorted image. +pub fn projected_with(solve: &CameraSolve, k: usize, undistorted: bool) -> Vec<(u32, [f64; 2], f64)> { let Some(cam) = solve.frames.get(k) else { return vec![] }; solve .visible(k) .filter_map(|p| { let c = cam.to_cam(p.pos); - cam.project(solve.size, p.pos).map(|q| (p.id, q, c[2])) + solve.project(k, p.pos, undistorted).map(|q| (p.id, q, c[2])) }) .collect() } +/// Whether the instance renders its input undistorted (Undistort Footage with a solved +/// distortion). +pub fn undistorts(params: &Params, solve: &CameraSolve) -> bool { + b(params, "advanced/undistort") && solve.distortion.is_some_and(|d| !d.is_none()) +} + +/// The input resampled through the solved distortion: every output pixel shows the footage +/// at its distorted position (transparent where that falls outside the layer). +fn undistort(buf: &Buf, sol: &CameraSolve) -> Buf { + let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + let mut out = Image::new(buf.img.width, buf.img.height); + let size = sol.size; + out.rows_mut().for_each(|(y, row)| { + for (x, px) in row.iter_mut().enumerate() { + let p = [(x as f64 + 0.5 - buf.offset[0]) / sc, (y as f64 + 0.5 - buf.offset[1]) / sc]; + let q = sol.to_image(p); + if q[0] < 0.0 || q[1] < 0.0 || q[0] > size[0] || q[1] > size[1] { + continue; + } + *px = buf.img.sample_bilinear_clamped(q[0] * sc + buf.offset[0], q[1] * sc + buf.offset[1]); + } + }); + Buf { img: out, offset: buf.offset, scale: buf.scale } +} + /// Median of the depths. pub fn median_depth(v: &[(u32, [f64; 2], f64)]) -> f64 { let mut z: Vec = v.iter().map(|x| x.2).collect(); @@ -188,15 +228,23 @@ pub fn median_depth(v: &[(u32, [f64; 2], f64)]) -> f64 { } fn render(ctx: &EffectCtx, mut buf: Buf) -> Buf { - if !b(ctx.params, "renderTrackPoints") { + let points = b(ctx.params, "renderTrackPoints"); + if !points && !b(ctx.params, "advanced/undistort") { return buf; } let Some(sol) = solve(ctx.params) else { return buf }; if (sol.size[0] - ctx.layer_size[0]).abs() > 0.5 || (sol.size[1] - ctx.layer_size[1]).abs() > 0.5 { return buf; } + let und = undistorts(ctx.params, &sol); + if und { + buf = undistort(&buf, &sol); + } + if !points { + return buf; + } let Some(k) = sol.frame_at(ctx.time) else { return buf }; - let pts = projected(&sol, k); + let pts = projected_with(&sol, k, und); let med = median_depth(&pts); let size = f(ctx.params, "trackPointSize"); let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; @@ -287,5 +335,37 @@ mod tests { let st = settings(&Params::default()); assert_eq!(st.shot, ShotType::FixedAngle); assert!(solve_key(&st).contains("FixedAngle")); + assert!(!st.lens_distortion && !solve_key(&st).contains("lens")); + } + + #[test] + fn undistort_footage_remaps_through_the_solved_lens() { + use effectcraft_track::camtrack::Distortion; + let sol = CameraSolve { + version: 1, + size: [64.0, 48.0], + start: 0.0, + frame_duration: 1.0 / 30.0, + frames: vec![SolvedFrame { rot: [0.0; 3], center: [0.0; 3], focal: 50.0, solved: true }], + distortion: Some(Distortion { k1: -0.2, k2: 0.0, radius: 40.0 }), + ..Default::default() + }; + let mut params = params_with(&sol, false); + params.values.insert("advanced/undistort".into(), Value::Bool(true)); + assert!(undistorts(¶ms, &sol)); + // A horizontal ramp: undistortion keeps the centre and pulls edge content outwards. + let mut img = Image::new(64, 48); + for (i, px) in img.data.iter_mut().enumerate() { + let x = (i % 64) as f32 / 64.0; + *px = [x, x, x, 1.0]; + } + let ctx = EffectCtx { params: ¶ms, time: 0.0, layer_size: [64.0, 48.0], seed: 0, adjustment: false, env: EffectEnv::default() }; + let out = render(&ctx, Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 }); + let at = |im: &Image, x: u32, y: u32| im.data[im.idx(x, y)][0]; + assert!((at(&out.img, 32, 24) - at(&img, 32, 24)).abs() < 0.02); + // Barrel distortion compresses the edges: the ideal pixel at x = 56 shows footage nearer the centre. + assert!(at(&out.img, 56, 24) < at(&img, 56, 24) - 0.01, "{} vs {}", at(&out.img, 56, 24), at(&img, 56, 24)); + let sp = sol.to_image([56.0, 24.0]); + assert!(sp[0] < 56.0 && (sol.to_ideal(sp)[0] - 56.0).abs() < 1e-6); } } diff --git a/crates/effects/src/face_track.rs b/crates/effects/src/face_track.rs new file mode 100644 index 00000000..7b925165 --- /dev/null +++ b/crates/effects/src/face_track.rs @@ -0,0 +1,45 @@ +//! Face tracking results (Tracker panel ▸ Method ▸ Face Tracking (Detailed Features)): +//! +//! - **Face Track Points** holds one point control per facial landmark (eyebrows, eyes and +//! pupils, nose, nostrils, mouth, chin and jaw), keyed on every tracked frame by the engine's +//! mask tracker ([`effectcraft_track::face`]); expressions and other layers can follow them. +//! - **Face Measurements** holds the measurements derived from those points (head position, +//! scale and orientation, eye openness, eyebrow raise, mouth openness, width and offset), keyed +//! by Extract & Copy Face Measurements (`track.extractFaceMeasurements`). +//! +//! Both pass their input through unchanged: they are data, like expression controls. + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_track::face::{LANDMARKS, MEASUREMENTS}; + +use crate::{Buf, EffectCtx, EffectSpec, ParamSpec, num, p, slider}; + +/// Effect ids. +pub const POINTS_ID: &str = "ec.utility.facetrackpoints"; +pub const MEASUREMENTS_ID: &str = "ec.utility.facemeasurements"; + +fn passthrough(_: &EffectCtx, b: Buf) -> Buf { + b +} + +fn measurement_ui(id: &str) -> ParamUi { + match id { + "headPositionX" | "headPositionY" => slider(-100_000.0, 100_000.0, 0.0, 1920.0, 1), + "headOrientationX" | "headOrientationY" | "headOrientationZ" => slider(-180.0, 180.0, -90.0, 90.0, 1), + _ => slider(-100_000.0, 100_000.0, 0.0, 200.0, 1), + } +} + +pub fn specs() -> Vec { + let points: Vec = LANDMARKS + .iter() + // Point defaults are fractions of the layer size: the canonical face in the middle. + .map(|(id, name, q)| p(id, name, Value::Vec2([0.5 + 0.25 * q[0], 0.5 + 0.25 * q[1]]), ParamUi::Point)) + .collect(); + let measurements: Vec = MEASUREMENTS.iter().map(|(id, name)| p(id, name, num(0.0), measurement_ui(id))).collect(); + vec![ + EffectSpec { id: POINTS_ID, name: "Face Track Points", category: "Utility", params: points, render: passthrough, gpu: false, float: true }, + EffectSpec { id: MEASUREMENTS_ID, name: "Face Measurements", category: "Utility", params: measurements, render: passthrough, gpu: false, float: true }, + ] +} diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index c4a47508..febdd9da 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -22,6 +22,7 @@ mod distort; mod distort2; mod distort3; pub mod distort4; +pub mod face_track; mod generate; mod generate2; mod generate3; @@ -412,6 +413,7 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(puppet::specs()); v.extend(warp_stab::specs()); v.extend(camera_tracker::specs()); + v.extend(face_track::specs()); v.extend(roto::specs()); v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); for s in v.iter_mut() { diff --git a/crates/engine/src/commands/camera_cmds.rs b/crates/engine/src/commands/camera_cmds.rs index 5d454aed..68cd14fb 100644 --- a/crates/engine/src/commands/camera_cmds.rs +++ b/crates/engine/src/commands/camera_cmds.rs @@ -23,7 +23,7 @@ pub fn specs() -> Vec { "Track Camera", ["Animation"], None, - "{layer?, shotType?: fixed|variable|specify, aov?: degrees, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}", + "{layer?, shotType?: fixed|variable|specify, aov?: degrees, solveMethod?: auto|typical|flat|tripod, detailed?, lensDistortion?: solve k1/k2, undistort?: render undistorted, wait?}", can_track, track_camera ), @@ -140,6 +140,8 @@ fn track_camera(s: &mut Session, p: &Value) -> Result { return Err(bad("track.camera", "aov: 1..170 degrees")); } let detailed = b_p(p, "detailed"); + let lens = b_p(p, "lensDistortion"); + let undistort = b_p(p, "undistort"); let existing = s.project.comp(cid).and_then(|c| c.layer(lid)).and_then(Layer::effects).and_then(|fx| fx.groups().find(|g| is_tracker(g)).map(|g| g.uid)); let uid = match existing { Some(u) => u, @@ -148,7 +150,7 @@ fn track_camera(s: &mut Session, p: &Value) -> Result { r["effects"].get(0).and_then(Value::as_u64).ok_or_else(|| EngineError::Other("the effect could not be applied".into()))? } }; - if shot.is_some() || method.is_some() || aov.is_some() || detailed.is_some() { + if shot.is_some() || method.is_some() || aov.is_some() || detailed.is_some() || lens.is_some() || undistort.is_some() { s.edit("3D Camera Tracker Settings", None, |proj, _| { let g = proj .comp_mut(cid) @@ -176,6 +178,12 @@ fn track_camera(s: &mut Session, p: &Value) -> Result { if let Some(d) = detailed { set("advanced/detailedAnalysis", KV::Bool(d)); } + if let Some(d) = lens { + set("advanced/lensDistortion", KV::Bool(d)); + } + if let Some(d) = undistort { + set("advanced/undistort", KV::Bool(d)); + } Ok(()) })?; } @@ -222,6 +230,7 @@ fn status(s: &mut Session, p: &Value) -> Result { out["shotType"] = json!(settings.shot.label()); out["solveMethod"] = json!(settings.method.label()); out["deleted"] = json!(settings.deleted.len()); + out["lensDistortionSolved"] = json!(settings.lens_distortion); let solve = ct::solve(¶ms); out["solved"] = json!(solve.is_some()); if let Some(sv) = &solve { @@ -233,6 +242,11 @@ fn status(s: &mut Session, p: &Value) -> Result { out["focalLength"] = json!(f); out["horizontalAngleOfView"] = json!(sv.hfov(f)); out["groundPlane"] = json!(sv.ground.is_some()); + out["lensDistortion"] = match &sv.distortion { + Some(d) => json!({"k1": d.k1, "k2": d.k2, "radius": d.radius}), + None => Value::Null, + }; + out["undistort"] = json!(ct::undistorts(¶ms, sv)); if let Some(pl) = placed(&s.project, cid, lid, uid) { let k = pl.frame_at(s.time_of(cid)); let (pos, ori, zoom) = pl.camera(k); diff --git a/crates/engine/src/commands/mask_interp.rs b/crates/engine/src/commands/mask_interp.rs index 5df40c6d..a4548944 100644 --- a/crates/engine/src/commands/mask_interp.rs +++ b/crates/engine/src/commands/mask_interp.rs @@ -1,5 +1,6 @@ -//! Mask tracking (Layer ▸ Mask ▸ Track Mask, the Tracker panel in mask mode) and Smart Mask -//! Interpolation (Window ▸ Mask Interpolation). +//! Mask tracking (Layer ▸ Mask ▸ Track Mask, the Tracker panel in mask mode, including the Face +//! Tracking methods and Extract & Copy Face Measurements) and Smart Mask Interpolation +//! (Window ▸ Mask Interpolation). use effectcraft_keyframe::{Keyframe, ShapePath, Value as KV}; use effectcraft_path::interp::{self, AddVertices, InterpOpts, Matching}; @@ -20,7 +21,7 @@ pub fn specs() -> Vec { "Track Mask", ["Animation"], None, - "{layer?, mask?: uid|index|name, method?: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective, direction?: forward|backward|frameForward|frameBackward, start? (s), end? (s), wait?: block until done}", + "{layer?, mask?: uid|index|name, method?: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective|faceOutline|faceDetailed, direction?: forward|backward|frameForward|frameBackward, start? (s), end? (s), wait?: block until done}", has_mask, track_mask ), @@ -29,10 +30,19 @@ pub fn specs() -> Vec { "Mask Tracking Method", [], None, - "{method: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective}", + "{method: position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective|faceOutline|faceDetailed}", super::always, mask_method ), + cmd!( + "track.extractFaceMeasurements", + "Extract & Copy Face Measurements", + [], + None, + "{layer?} → keys a Face Measurements effect from the layer's Face Track Points (one key per tracked frame) and copies those keys", + has_face_points, + extract_face + ), cmd!( "mask.interpolate", "Apply Mask Interpolation", @@ -116,9 +126,12 @@ fn has_mask(s: &Session) -> std::result::Result<(), String> { fn method_p(p: &Value, cmd: &str) -> Result> { match str_p(p, "method") { - Some(m) => MaskMethod::from_name(m) - .map(Some) - .ok_or_else(|| bad(cmd, format!("unknown method `{m}` (position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective)"))), + Some(m) => MaskMethod::from_name(m).map(Some).ok_or_else(|| { + bad( + cmd, + format!("unknown method `{m}` (position|positionScale|positionScaleRotation|positionScaleRotationSkew|perspective|faceOutline|faceDetailed)"), + ) + }), None => Ok(None), } } @@ -190,6 +203,112 @@ fn track_mask(s: &mut Session, p: &Value) -> Result { })) } +// ---------------------------------------------------------------- face measurements + +fn face_points_layer(s: &Session, p: &Value) -> Option<(effectcraft_project::ItemId, LayerId)> { + use effectcraft_effects::face_track::POINTS_ID; + let has = |l: &Layer| l.effects().is_some_and(|fx| fx.groups().any(|g| g.match_id == POINTS_ID)); + if p.get("layer").is_some() { + return layer_p(s, p, "track.extractFaceMeasurements").ok(); + } + let cid = s.active_comp_id()?; + let comp = s.project.comp(cid)?; + let sel = s.state.selected_layers.iter().filter_map(|l| comp.layer(*l)).find(|l| has(l)); + sel.or_else(|| comp.layers.iter().find(|l| has(l))).map(|l| (cid, l.id)) +} + +fn has_face_points(s: &Session) -> std::result::Result<(), String> { + has_comp(s)?; + face_points_layer(s, &json!({})).map(|_| ()).ok_or_else(|| "no Face Track Points: track a mask with Face Tracking (Detailed Features) first".into()) +} + +/// Extract & Copy Face Measurements: measure the Face Track Points at every keyed time, key them +/// into the layer's Face Measurements effect and put those keys on the keyframe clipboard. +fn extract_face(s: &mut Session, p: &Value) -> Result { + use effectcraft_effects::face_track::{MEASUREMENTS_ID, POINTS_ID}; + use effectcraft_track::face::{LANDMARKS, MEASUREMENTS, N, face_frame, measure}; + const CMD: &str = "track.extractFaceMeasurements"; + let (cid, lid) = face_points_layer(s, p).ok_or_else(|| bad(CMD, "no Face Track Points: track a mask with Face Tracking (Detailed Features) first"))?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let g = layer.effects().and_then(|fx| fx.groups().find(|g| g.match_id == POINTS_ID)).ok_or_else(|| bad(CMD, "the layer has no Face Track Points"))?; + let mut times: Vec = LANDMARKS.iter().filter_map(|l| g.get(l.0)).flat_map(|pr| pr.keys.iter().map(|k| k.time)).collect(); + times.sort(); + times.dedup(); + if times.is_empty() { + times.push(layer.layer_time(s.time_of(cid))); + } + let rows: Vec<(Tick, [f64; 14])> = { + let pts_at = |t: Tick| { + let mut l = [[0.0; 2]; N]; + for (i, (id, _, _)) in LANDMARKS.iter().enumerate() { + if let Some(KV::Vec2(v)) = g.get(id).map(|pr| pr.value_at(t)) { + l[i] = v; + } + } + l + }; + let reference = face_frame(&pts_at(times[0])).1; + times.iter().map(|t| (*t, measure(&pts_at(*t), reference))).collect() + }; + let comp_times: Vec = times.iter().map(|t| layer.comp_time(*t)).collect(); + let fd = comp.frame_duration(); + let uid = s.edit("Extract Face Measurements", None, |proj, _| { + let uid = crate::mask_track::face_effect(proj, cid, lid, MEASUREMENTS_ID).ok_or_else(|| bad(CMD, "Face Measurements is not available"))?; + let g = proj.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)).ok_or(EngineError::NoComp)?; + for (k, (id, _)) in MEASUREMENTS.iter().enumerate() { + let Some(pr) = g.get_mut(id) else { continue }; + pr.keys.clear(); + for (t, m) in &rows { + effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*t, KV::Scalar(m[k]))); + } + } + Ok(uid) + })?; + // Copy the keys (Edit ▸ Paste pastes them at the CTI, like copied keyframes). + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let mg = layer.props.find_group(uid).ok_or(EngineError::NoComp)?; + let earliest = comp_times.iter().copied().min().unwrap_or(Tick::ZERO); + let mut clip = vec![]; + for (id, _) in MEASUREMENTS { + let Some(pr) = mg.get(id) else { continue }; + let Some(path) = layer.props.match_path_of(pr.uid) else { continue }; + let keys = pr + .keys + .iter() + .map(|k| { + let mut k = k.clone(); + k.time = layer.comp_time(k.time) - earliest; + k + }) + .collect(); + clip.push(crate::KeyClip { layer: lid, path, keys }); + } + s.state.key_clipboard = clip; + s.state.clip_is_keys = true; + // The same data as text (tab-separated, one row per frame) for other applications. + let mut text = String::from("EffectCraft Face Measurements\n\nFrame\tTime"); + for (_, name) in MEASUREMENTS { + text.push('\t'); + text.push_str(name); + } + text.push('\n'); + let mut table = vec![]; + for ((_, m), ct) in rows.iter().zip(&comp_times) { + let frame = if fd > Tick::ZERO { (ct.seconds() / fd.seconds()).round() as i64 } else { 0 }; + text.push_str(&format!("{frame}\t{:.4}", ct.seconds())); + for v in m { + text.push_str(&format!("\t{v:.3}")); + } + text.push('\n'); + let obj: serde_json::Map = MEASUREMENTS.iter().zip(m).map(|((id, _), v)| (id.to_string(), json!(v))).collect(); + table.push(json!({"time": ct.seconds(), "frame": frame, "values": obj})); + } + s.bump(); + Ok(json!({"layer": lid.0, "effect": uid, "frames": rows.len(), "measurements": table, "clipboard": text})) +} + // ---------------------------------------------------------------- mask interpolation /// Mask Interpolation panel options (kept in the editor state, serde for agents). diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 057c7d10..538e8844 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -753,6 +753,8 @@ mod tests_effects; #[cfg(test)] mod tests_essential; #[cfg(test)] +mod tests_face; +#[cfg(test)] mod tests_fidelity; #[cfg(test)] mod tests_lottie; diff --git a/crates/engine/src/mask_track.rs b/crates/engine/src/mask_track.rs index 00315198..a5741052 100644 --- a/crates/engine/src/mask_track.rs +++ b/crates/engine/src/mask_track.rs @@ -6,6 +6,12 @@ //! algorithm is `effectcraft_track::mask` (KLT features inside the mask + a RANSAC fit of the //! chosen motion model), and every frame's motion is applied to the mask's vertices and //! tangents, so the shape's Bezier structure is kept. +//! +//! The two **Face Tracking** methods use `effectcraft_track::face` instead: the mask (drawn +//! around a face) seeds the tracker, and every frame's face outline becomes the Mask Path +//! (*Outline Only*); *Detailed Features* also keys the facial landmarks into a **Face Track +//! Points** effect on the layer (created on first use), from which Extract & Copy Face +//! Measurements (`track.extractFaceMeasurements`) derives a keyed **Face Measurements** effect. use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; @@ -31,11 +37,22 @@ pub enum MaskMethod { PositionScaleRotation, PositionScaleRotationSkew, Perspective, + /// Face Tracking (Outline Only). + FaceOutline, + /// Face Tracking (Detailed Features). + FaceDetailed, } impl MaskMethod { - pub const ALL: [MaskMethod; 5] = - [MaskMethod::Position, MaskMethod::PositionScale, MaskMethod::PositionScaleRotation, MaskMethod::PositionScaleRotationSkew, MaskMethod::Perspective]; + pub const ALL: [MaskMethod; 7] = [ + MaskMethod::Position, + MaskMethod::PositionScale, + MaskMethod::PositionScaleRotation, + MaskMethod::PositionScaleRotationSkew, + MaskMethod::Perspective, + MaskMethod::FaceOutline, + MaskMethod::FaceDetailed, + ]; pub fn label(self) -> &'static str { match self { MaskMethod::Position => "Position", @@ -43,8 +60,14 @@ impl MaskMethod { MaskMethod::PositionScaleRotation => "Position, Scale & Rotation", MaskMethod::PositionScaleRotationSkew => "Position, Scale, Rotation & Skew", MaskMethod::Perspective => "Perspective", + MaskMethod::FaceOutline => "Face Tracking (Outline Only)", + MaskMethod::FaceDetailed => "Face Tracking (Detailed Features)", } } + /// One of the Face Tracking methods. + pub fn is_face(self) -> bool { + matches!(self, MaskMethod::FaceOutline | MaskMethod::FaceDetailed) + } pub fn id(self) -> &'static str { match self { MaskMethod::Position => "position", @@ -52,6 +75,8 @@ impl MaskMethod { MaskMethod::PositionScaleRotation => "positionScaleRotation", MaskMethod::PositionScaleRotationSkew => "positionScaleRotationSkew", MaskMethod::Perspective => "perspective", + MaskMethod::FaceOutline => "faceOutline", + MaskMethod::FaceDetailed => "faceDetailed", } } pub fn from_name(s: &str) -> Option { @@ -67,6 +92,7 @@ impl MaskMethod { MaskMethod::PositionScaleRotation => Model::Similarity, MaskMethod::PositionScaleRotationSkew => Model::Affine, MaskMethod::Perspective => Model::Homography, + MaskMethod::FaceOutline | MaskMethod::FaceDetailed => Model::Similarity, } } } @@ -109,8 +135,29 @@ pub fn transform_path(p: &ShapePath, h: &trk::Homography) -> ShapePath { } } -/// One tracked frame: (layer time, comp time, mask path). -type MaskFrame = (Tick, Tick, ShapePath); +/// A closed, smooth mask path through a face outline polygon: `n` vertices with Catmull-Rom +/// tangents. `like` lends its feather points. +pub fn outline_path(poly: &[[f64; 2]], n: usize, like: &ShapePath) -> ShapePath { + let m = poly.len(); + let n = n.clamp(3, m.max(3)); + let v: Vec<[f64; 2]> = (0..n).map(|i| poly[(i * m / n) % m.max(1)]).collect(); + let tan: Vec<[f64; 2]> = (0..n) + .map(|i| { + let (a, b) = (v[(i + n - 1) % n], v[(i + 1) % n]); + [(b[0] - a[0]) / 6.0, (b[1] - a[1]) / 6.0] + }) + .collect(); + ShapePath { + vertices: v, + in_tangents: tan.iter().map(|t| [-t[0], -t[1]]).collect(), + out_tangents: tan, + closed: true, + feather: if like.len() == n { like.feather.clone() } else { vec![] }, + } +} + +/// One tracked frame: (layer time, comp time, mask path, face landmarks for Detailed Features). +type MaskFrame = (Tick, Tick, ShapePath, Option<[[f64; 2]; trk::face::N]>); #[derive(Default)] pub struct MaskTrackShared { @@ -169,9 +216,12 @@ fn run_work(w: MaskWork, shared: &MaskTrackShared) { let frame_at = |t: Tick| source_frame(r, &w.project, w.comp, comp, layer, t, w.expr.as_deref()); lock(&shared.state).total = w.times.len().saturating_sub(1) as u64; let Some(first) = frame_at(w.times[0]) else { return fail("the layer has no pixels to track") }; + if w.method.is_face() { + return run_face(&w, shared, first, &frame_at, layer, t0); + } let mut tracker = trk::mask::MaskTracker::new(w.method.model(), flatten(&w.path, 8), &trk::Frame { img: &first.0, offset: first.1 }); let mut path = w.path.clone(); - lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path.clone())); + lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path.clone(), None)); let mut next = if w.times.len() > 1 { frame_at(w.times[1]) } else { None }; for k in 1..w.times.len() { if shared.cancel.load(Ordering::Relaxed) { @@ -190,7 +240,7 @@ fn run_work(w: MaskWork, shared: &MaskTrackShared) { break; }; path = transform_path(&path, &step.motion); - lock(&shared.frames).push((layer.layer_time(w.times[k]), w.times[k], path.clone())); + lock(&shared.frames).push((layer.layer_time(w.times[k]), w.times[k], path.clone(), None)); let mut s = lock(&shared.state); s.done = k as u64; s.elapsed = t0.elapsed().as_secs_f64(); @@ -203,12 +253,99 @@ fn run_work(w: MaskWork, shared: &MaskTrackShared) { s.finished = true; } +/// Face tracking: the face outline keys the mask; Detailed Features adds the landmarks. +fn run_face( + w: &MaskWork, + shared: &MaskTrackShared, + first: (Arc, [f64; 2]), + frame_at: &(dyn Fn(Tick) -> Option<(Arc, [f64; 2])> + Sync), + layer: &effectcraft_project::Layer, + t0: web_time::Instant, +) { + let detailed = w.method == MaskMethod::FaceDetailed; + let b = trk::mask::bounds(&flatten(&w.path, 8)); + let Some((mut tracker, fit)) = trk::face::FaceTracker::new(&trk::Frame { img: &first.0, offset: first.1 }, b) else { + let mut s = lock(&shared.state); + s.error = Some("no face found inside the mask (draw the mask around a face)".into()); + s.finished = true; + return; + }; + let key = |fit: &trk::face::FaceFit| (outline_path(&fit.outline, 16, &w.path), detailed.then_some(fit.landmarks)); + let (path, pts) = key(&fit); + lock(&shared.frames).push((layer.layer_time(w.times[0]), w.times[0], path, pts)); + let mut next = if w.times.len() > 1 { frame_at(w.times[1]) } else { None }; + for k in 1..w.times.len() { + if shared.cancel.load(Ordering::Relaxed) { + lock(&shared.state).cancelled = true; + break; + } + let Some(cur) = next.take() else { + lock(&shared.state).error = Some(format!("no frame at {:.3} s", w.times[k].seconds())); + break; + }; + let (res, nf) = + rayon::join(|| tracker.step(&trk::Frame { img: &cur.0, offset: cur.1 }), || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None }); + next = nf; + let Some(fit) = res else { + lock(&shared.state).error = Some(format!("lost the face at {:.3} s", w.times[k].seconds())); + break; + }; + let (path, pts) = key(&fit); + lock(&shared.frames).push((layer.layer_time(w.times[k]), w.times[k], path, pts)); + let mut s = lock(&shared.state); + s.done = k as u64; + s.elapsed = t0.elapsed().as_secs_f64(); + s.fps = if s.elapsed > 0.0 { k as f64 / s.elapsed } else { 0.0 }; + s.time = w.times[k].seconds(); + crate::offload::report(s.done, s.total); + } + let mut s = lock(&shared.state); + s.elapsed = t0.elapsed().as_secs_f64(); + s.finished = true; +} + +/// The layer's Face Track Points effect (created when missing). Returns its uid. +pub(crate) fn face_points_group(p: &mut Project, comp: ItemId, layer: LayerId) -> Option { + face_effect(p, comp, layer, effectcraft_effects::face_track::POINTS_ID) +} + +/// The layer's effect `id`, applied when missing (data effects of face tracking). +pub(crate) fn face_effect(p: &mut Project, comp: ItemId, layer: LayerId, id: &str) -> Option { + let spec = effectcraft_effects::find(id)?; + let l = p.comp(comp)?.layer(layer)?; + if let Some(g) = l.effects().and_then(|fx| fx.groups().find(|g| g.match_id == spec.id)) { + return Some(g.uid); + } + let (w, h) = effectcraft_render::source_size(p, l); + let size = if w == 0 { p.comp(comp).map(|c| [c.width as f64, c.height as f64]).unwrap_or([1920.0, 1080.0]) } else { [w as f64, h as f64] }; + let mut next = p.next_id; + let g = effectcraft_effects::instantiate(spec, &mut effectcraft_project::build::Ids(&mut next), spec.name, size); + let uid = g.uid; + let fx = p.comp_mut(comp)?.layer_mut(layer)?.props.sub_mut("effects")?; + fx.children.push(g.into()); + p.next_id = next; + Some(uid) +} + fn write_frames(p: &mut Project, job: &MaskTrackJob, frames: &[MaskFrame]) { + let points = if frames.iter().any(|f| f.3.is_some()) { face_points_group(p, job.comp, job.layer) } else { None }; let Some(layer) = p.comp_mut(job.comp).and_then(|c| c.layer_mut(job.layer)) else { return }; - let Some(g) = layer.props.find_group_mut(job.mask) else { return }; - let Some(pr) = g.get_mut("path") else { return }; - for (lt, _, path) in frames { - effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*lt, Value::Path(path.clone()))); + if let Some(g) = layer.props.find_group_mut(job.mask) + && let Some(pr) = g.get_mut("path") + { + for (lt, _, path, _) in frames { + effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*lt, Value::Path(path.clone()))); + } + } + if let Some(g) = points.and_then(|u| layer.props.find_group_mut(u)) { + for (lt, _, _, pts) in frames { + let Some(pts) = pts else { continue }; + for (i, (id, _, _)) in trk::face::LANDMARKS.iter().enumerate() { + if let Some(pr) = g.get_mut(id) { + effectcraft_keyframe::set_key(&mut pr.keys, Keyframe::new(*lt, Value::Vec2(pts[i]))); + } + } + } } } @@ -242,7 +379,7 @@ impl Session { return Err("a track analysis is already running".into()); } self.poll_mask_track(); - if !wait && self.offloads() { + if !wait && self.offloads() && !work.method.is_face() { let MaskWork { comp, layer, path, method, times, .. } = work; return self.offload_analysis(crate::offload::WorkerJob::MaskTrack { comp, layer, mask, direction, path, method, times }); } diff --git a/crates/engine/src/tests_face.rs b/crates/engine/src/tests_face.rs new file mode 100644 index 00000000..dad97935 --- /dev/null +++ b/crates/engine/src/tests_face.rs @@ -0,0 +1,106 @@ +//! Face tracking end to end through the commands, on synthetic face footage generated in-test +//! (a moving, rotating face that opens its mouth and blinks). + +use effectcraft_keyframe::{ShapePath, Value as KV}; +use effectcraft_track::face::{FaceParams, LANDMARKS, MEASUREMENTS, render_synthetic, synth_landmarks}; +use serde_json::json; + +use crate::mask_track::MaskMethod; +use crate::tests_track::{H, W, frame_time, setup}; + +fn face(f: u32) -> FaceParams { + let t = f as f64; + FaceParams { + center: [160.0 + 25.0 * (t * 0.2).sin(), 120.0 + 8.0 * (t * 0.15).cos()], + height: 70.0 * (1.0 + 0.05 * (t * 0.1).sin()), + roll: 12.0 * (t * 0.18).sin(), + yaw: 10.0 * (t * 0.12).sin(), + mouth_open: 0.5 + 0.45 * (t * 0.3).sin(), + eye_open: [0.85, 0.85], + ..Default::default() + } +} + +#[test] +fn face_tracking_keys_outline_points_and_measurements() { + let (mut s, clip, _) = setup(|f| render_synthetic(W, H, &face(f), 5)); + assert_eq!(MaskMethod::from_name("faceDetailed"), Some(MaskMethod::FaceDetailed)); + assert_eq!(MaskMethod::from_name("Face Tracking (Outline Only)"), Some(MaskMethod::FaceOutline)); + // A rough mask around the face. + let c = face(0).center; + let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect(c, 120.0, 160.0).vertices, "closed": true})).unwrap(); + let mask = r["mask"].as_u64().unwrap(); + s.execute("time.set", json!({"time": 0})).unwrap(); + let r = s.execute_checked("track.mask", json!({"layer": clip.0, "mask": mask, "method": "faceDetailed", "direction": "forward", "wait": true})).unwrap(); + assert_eq!(r["frames"], 24, "{r}"); + assert_eq!(s.state.mask_track_method, MaskMethod::FaceDetailed); + let comp = s.active_comp().unwrap(); + let layer = comp.layer(clip).unwrap(); + // The mask follows the face outline. + let path = layer.props.find_group(mask).unwrap().get("path").unwrap(); + assert_eq!(path.keys.len(), 25); + for f in [0, 12, 24] { + let KV::Path(p) = path.value_at(frame_time(f)) else { panic!() }; + let fp = face(f); + let (sn, cs) = fp.roll.to_radians().sin_cos(); + for v in &p.vertices { + let d = [(v[0] - fp.center[0]) / fp.height, (v[1] - fp.center[1]) / fp.height]; + let (u, w) = (cs * d[0] + sn * d[1], -sn * d[0] + cs * d[1]); + let r = ((u / 0.75).powi(2) + w * w).sqrt(); + assert!((r - 1.0).abs() < 0.06, "frame {f}: vertex {v:?} at {r}"); + } + } + // Face Track Points: one keyed point per landmark, on the face. + let fx = layer.effects().unwrap(); + let pts = fx.groups().find(|g| g.match_id == effectcraft_effects::face_track::POINTS_ID).expect("Face Track Points"); + let mut worst: f64 = 0.0; + for f in [0, 6, 12, 18, 24] { + let truth = synth_landmarks(&face(f)); + for (i, (id, _, _)) in LANDMARKS.iter().enumerate() { + let pr = pts.get(id).unwrap(); + assert_eq!(pr.keys.len(), 25, "{id}"); + let KV::Vec2(v) = pr.value_at(frame_time(f)) else { panic!() }; + worst = worst.max((v[0] - truth[i][0]).hypot(v[1] - truth[i][1]) / face(f).height); + } + } + assert!(worst < 0.1, "worst landmark error {worst} face heights"); + // Extract & Copy Face Measurements. + assert!(s.is_enabled("track.extractFaceMeasurements")); + let r = s.execute_checked("track.extractFaceMeasurements", json!({"layer": clip.0})).unwrap(); + assert_eq!(r["frames"], 25); + assert!(r["clipboard"].as_str().unwrap().contains("Mouth Openness")); + assert_eq!(s.state.key_clipboard.len(), MEASUREMENTS.len()); + assert!(s.state.clip_is_keys); + let layer = s.active_comp().unwrap().layer(clip).unwrap(); + let m = layer.effects().unwrap().groups().find(|g| g.match_id == effectcraft_effects::face_track::MEASUREMENTS_ID).expect("Face Measurements"); + let mo = m.get("mouthOpenness").unwrap(); + assert_eq!(mo.keys.len(), 25); + // Mouth openness follows the footage: widest open where the generator opens it most. + let open = |f: u32| mo.value_at(frame_time(f)).as_f64(); + let (fmax, fmin) = ( + (0..25).max_by(|a, b| face(*a).mouth_open.total_cmp(&face(*b).mouth_open)).unwrap(), + (0..25).min_by(|a, b| face(*a).mouth_open.total_cmp(&face(*b).mouth_open)).unwrap(), + ); + assert!(open(fmax) > open(fmin) + 20.0, "{} vs {}", open(fmax), open(fmin)); + let roll = m.get("headOrientationZ").unwrap(); + for f in [0, 10, 20] { + assert!((roll.value_at(frame_time(f)).as_f64() - face(f).roll).abs() < 3.0); + } + // One undo step for the extraction. + assert!(s.undo()); + assert!(!s.active_comp().unwrap().layer(clip).unwrap().effects().unwrap().groups().any(|g| g.match_id == effectcraft_effects::face_track::MEASUREMENTS_ID)); + // Outline Only keys just the mask. + let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect(c, 120.0, 160.0).vertices, "closed": true})).unwrap(); + let m2 = r["mask"].as_u64().unwrap(); + s.execute("time.set", json!({"time": 0})).unwrap(); + s.execute_checked("track.mask", json!({"layer": clip.0, "mask": m2, "method": "faceOutline", "direction": "frameForward", "wait": true})).unwrap(); + let layer = s.active_comp().unwrap().layer(clip).unwrap(); + assert_eq!(layer.props.find_group(m2).unwrap().get("path").unwrap().keys.len(), 2); + // A mask without a face reports it. + let r = s.execute("mask.new", json!({"layer": clip.0, "vertices": ShapePath::rect([20.0, 20.0], 30.0, 30.0).vertices, "closed": true})).unwrap(); + let m3 = r["mask"].as_u64().unwrap(); + s.execute("time.set", json!({"time": 0})).unwrap(); + let _ = s.execute("track.mask", json!({"layer": clip.0, "mask": m3, "method": "faceOutline", "direction": "forward", "wait": true})); + let layer = s.active_comp().unwrap().layer(clip).unwrap(); + assert!(layer.props.find_group(m3).unwrap().get("path").unwrap().keys.is_empty()); +} diff --git a/crates/track/src/camtrack/bundle.rs b/crates/track/src/camtrack/bundle.rs index 72cb2f17..077f04b7 100644 --- a/crates/track/src/camtrack/bundle.rs +++ b/crates/track/src/camtrack/bundle.rs @@ -10,6 +10,7 @@ use rayon::prelude::*; +use super::Distortion; use super::geometry::Pose; use super::linalg::*; @@ -19,6 +20,8 @@ pub struct BaCam { pub pose: Pose, /// Focal length (pixels). pub f: f64, + /// Lens distortion (shared by all cameras; adjusted when [`BaOpts::distortion`]). + pub dist: Distortion, } /// One observation: camera `cam` sees point `pt` at `uv` (pixels relative to the principal @@ -50,11 +53,13 @@ pub struct BaOpts { /// Huber threshold (pixels). pub huber: f64, pub max_iter: usize, + /// Adjust the shared radial distortion `(k1, k2)` too. + pub distortion: bool, } impl Default for BaOpts { fn default() -> Self { - BaOpts { focal: FocalMode::Fixed, fix_points: false, fix_centers: false, fixed: vec![], huber: 2.0, max_iter: 30 } + BaOpts { focal: FocalMode::Fixed, fix_points: false, fix_centers: false, fixed: vec![], huber: 2.0, max_iter: 30, distortion: false } } } @@ -77,7 +82,8 @@ pub fn residual(c: &BaCam, x: V3, uv: [f64; 2]) -> Option<[f64; 2]> { if p[2] <= 1e-9 { return None; } - Some([c.f * p[0] / p[2] - uv[0], c.f * p[1] / p[2] - uv[1]]) + let q = c.dist.distort([c.f * p[0] / p[2], c.f * p[1] / p[2]]); + Some([q[0] - uv[0], q[1] - uv[1]]) } /// Total robust cost. @@ -108,6 +114,8 @@ struct Layout { fidx: Vec, /// Index of the shared focal parameter. shared: Option, + /// Index of `k1` (then `k2`) when the distortion is adjusted. + dist: Option, n: usize, rot_only: bool, } @@ -132,7 +140,11 @@ fn layout(ncam: usize, o: &BaOpts) -> Layout { n += 1; n - 1 }); - Layout { start, fidx, shared, n, rot_only: o.fix_centers } + let dist = o.distortion.then(|| { + n += 2; + n - 2 + }); + Layout { start, fidx, shared, dist, n, rot_only: o.fix_centers } } /// Camera-side Jacobian entries of one observation: (parameter index, d r / d param). @@ -204,12 +216,21 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> continue; } let iz = 1.0 / xc[2]; - let p = [c.f * xc[0] * iz, c.f * xc[1] * iz]; - let r = [p[0] - ob.uv[0], p[1] - ob.uv[1]]; + let p0 = [c.f * xc[0] * iz, c.f * xc[1] * iz]; + let pd = c.dist.distort(p0); + let r = [pd[0] - ob.uv[0], pd[1] - ob.uv[1]]; let e = (r[0] * r[0] + r[1] * r[1]).sqrt(); let w = if e <= o.huber { 1.0 } else { o.huber / e }; - let a = [[c.f * iz, 0.0, -c.f * xc[0] * iz * iz], [0.0, c.f * iz, -c.f * xc[1] * iz * iz]]; + let a0 = [[c.f * iz, 0.0, -c.f * xc[0] * iz * iz], [0.0, c.f * iz, -c.f * xc[1] * iz * iz]]; + // Through the distortion: d r = D d p. + let (dj, dk) = c.dist.jacobian(p0); + let a = [0, 1].map(|i| [0, 1, 2].map(|k| dj[i][0] * a0[0][k] + dj[i][1] * a0[1][k])); + let p = [dj[0][0] * p0[0] + dj[0][1] * p0[1], dj[1][0] * p0[0] + dj[1][1] * p0[1]]; cam_jac(&l, ci, c, xc, &a, p, &mut jc); + if let Some(g) = l.dist { + jc.push((g, [dk[0][0], dk[1][0]])); + jc.push((g + 1, [dk[0][1], dk[1][1]])); + } // Point Jacobian: A R. let jx = if o.fix_points { [[0.0; 3]; 2] @@ -236,7 +257,8 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> [w * (ja[0] * jx[0][0] + ja[1] * jx[1][0]), w * (ja[0] * jx[0][1] + ja[1] * jx[1][1]), w * (ja[0] * jx[0][2] + ja[1] * jx[1][2])]; // Each observation of a point is on another camera: only the shared focal // length repeats. - let found = if Some(*ia) == l.shared { ps.w.iter_mut().find(|e| e.0 == *ia) } else { None }; + let repeats = Some(*ia) == l.shared || l.dist.is_some_and(|g| *ia == g || *ia == g + 1); + let found = if repeats { ps.w.iter_mut().find(|e| e.0 == *ia) } else { None }; match found { Some(e) => e.1 = add(e.1, wv), None => ps.w.push((*ia, wv)), @@ -385,6 +407,10 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> if let Some(g) = l.shared { c.f *= dc[g].clamp(-0.5, 0.5).exp(); } + if let Some(g) = l.dist { + c.dist.k1 += dc[g].clamp(-0.2, 0.2); + c.dist.k2 += dc[g + 1].clamp(-0.2, 0.2); + } } let mut np: Vec = pts.to_vec(); if !o.fix_points { @@ -445,7 +471,11 @@ mod tests { let truth: Vec = (0..80).map(|_| [u() * 6.0, u() * 4.0, 6.0 + u() * 4.0]).collect(); let f = 900.0; let cams_true: Vec = (0..6) - .map(|i| BaCam { pose: Pose { r: rodrigues([0.0, -0.05 * i as f64, 0.01 * i as f64]), c: [0.3 * i as f64, 0.05 * i as f64, 0.0] }, f }) + .map(|i| BaCam { + pose: Pose { r: rodrigues([0.0, -0.05 * i as f64, 0.01 * i as f64]), c: [0.3 * i as f64, 0.05 * i as f64, 0.0] }, + f, + dist: Distortion::NONE, + }) .collect(); let mut obs = vec![]; for (ci, c) in cams_true.iter().enumerate() { diff --git a/crates/track/src/camtrack/mod.rs b/crates/track/src/camtrack/mod.rs index 76c0637f..e8cd28da 100644 --- a/crates/track/src/camtrack/mod.rs +++ b/crates/track/src/camtrack/mod.rs @@ -28,8 +28,15 @@ //! tripod solution when it explains the tracks as well as the general one (a quick rotation fit //! at the general solution's focal length skips the tripod search when it clearly fails). //! -//! The camera model has square pixels, the principal point at the layer centre and no lens -//! distortion. Camera space is x right, y down, z forward (After Effects' convention); a camera +//! 5. **Lens distortion** (Solve Lens Distortion, or Detailed Analysis): radial distortion +//! `p_d = p (1 + k1 ρ² + k2 ρ⁴)` (ρ = distance from the principal point over half the frame +//! diagonal; Brown's model without the tangential terms) is solved jointly with the cameras +//! and points in a final bundle adjustment on the raw tracks; the whole solve is then repeated +//! on tracks undistorted with that estimate (the linear initialisations assume a pinhole +//! camera) and adjusted jointly again. The solve stores `k1, k2`; the effect can undistort +//! its output so composited 3D layers line up. +//! +//! The camera model has square pixels and the principal point at the layer centre. Camera space is x right, y down, z forward (After Effects' convention); a camera //! is `(R, C, f)` with `X_cam = R (X − C)` and pixel `u = f x / z + w / 2`. //! //! The solve is stored in a canonical frame: the first solved frame's camera is at the origin @@ -105,6 +112,69 @@ impl CameraTracks { } } +/// Radial lens distortion around the principal point (see the module docs). +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct Distortion { + pub k1: f64, + pub k2: f64, + /// Normalisation radius ρ = 1 (pixels): half the frame diagonal; 0 = no distortion. + pub radius: f64, +} + +impl Distortion { + pub const NONE: Distortion = Distortion { k1: 0.0, k2: 0.0, radius: 0.0 }; + + /// No distortion for frames of `size` (the radius set, coefficients zero). + pub fn for_size(size: [f64; 2]) -> Distortion { + Distortion { k1: 0.0, k2: 0.0, radius: 0.5 * size[0].hypot(size[1]) } + } + pub fn is_none(&self) -> bool { + self.radius <= 0.0 || (self.k1 == 0.0 && self.k2 == 0.0) + } + fn rho2(&self, p: [f64; 2]) -> f64 { + (p[0] * p[0] + p[1] * p[1]) / (self.radius * self.radius) + } + /// Ideal → distorted (pixels relative to the principal point). + pub fn distort(&self, p: [f64; 2]) -> [f64; 2] { + if self.radius <= 0.0 { + return p; + } + let r2 = self.rho2(p); + let d = 1.0 + self.k1 * r2 + self.k2 * r2 * r2; + [p[0] * d, p[1] * d] + } + /// Distorted → ideal (fixed-point iteration). + pub fn undistort(&self, q: [f64; 2]) -> [f64; 2] { + if self.is_none() { + return q; + } + let mut p = q; + for _ in 0..30 { + let r2 = self.rho2(p); + let d = 1.0 + self.k1 * r2 + self.k2 * r2 * r2; + if d.abs() < 1e-6 { + break; + } + p = [q[0] / d, q[1] / d]; + } + p + } + /// `d distort / d p` (2 × 2, row-major) and `d distort / d (k1, k2)` (rows x, y). + pub fn jacobian(&self, p: [f64; 2]) -> ([[f64; 2]; 2], [[f64; 2]; 2]) { + if self.radius <= 0.0 { + return ([[1.0, 0.0], [0.0, 1.0]], [[0.0; 2]; 2]); + } + let r2 = self.rho2(p); + let d = 1.0 + self.k1 * r2 + self.k2 * r2 * r2; + // d(r2)/dp = 2 p / R². + let dd = (self.k1 + 2.0 * self.k2 * r2) * 2.0 / (self.radius * self.radius); + let j = [[d + p[0] * dd * p[0], p[0] * dd * p[1]], [p[1] * dd * p[0], d + p[1] * dd * p[1]]]; + let k = [[p[0] * r2, p[0] * r2 * r2], [p[1] * r2, p[1] * r2 * r2]]; + (j, k) + } +} + /// 3D Camera Tracker ▸ Shot Type. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -178,6 +248,8 @@ pub struct SolveSettings { pub method: SolveMethod, /// Track ids the user deleted. pub deleted: Vec, + /// Solve radial lens distortion jointly with the cameras. + pub lens_distortion: bool, } /// The solved camera of one frame. @@ -261,6 +333,9 @@ pub struct CameraSolve { pub points: Vec, #[serde(skip_serializing_if = "Option::is_none")] pub ground: Option, + /// The solved lens distortion (Solve Lens Distortion). + #[serde(skip_serializing_if = "Option::is_none")] + pub distortion: Option, } impl CameraSolve { @@ -355,6 +430,36 @@ impl CameraSolve { } } + /// An ideal (pinhole) image point → where it appears in the footage (layer pixels), through + /// the solved lens distortion. + pub fn to_image(&self, p: [f64; 2]) -> [f64; 2] { + match &self.distortion { + Some(d) if !d.is_none() => { + let c = [self.size[0] * 0.5, self.size[1] * 0.5]; + let q = d.distort([p[0] - c[0], p[1] - c[1]]); + [q[0] + c[0], q[1] + c[1]] + } + _ => p, + } + } + /// A footage point → its ideal (undistorted) position. + pub fn to_ideal(&self, p: [f64; 2]) -> [f64; 2] { + match &self.distortion { + Some(d) if !d.is_none() => { + let c = [self.size[0] * 0.5, self.size[1] * 0.5]; + let q = d.undistort([p[0] - c[0], p[1] - c[1]]); + [q[0] + c[0], q[1] + c[1]] + } + _ => p, + } + } + /// Project world point `x` on frame `k`: through the lens distortion into the footage, or + /// (`undistorted`) into the undistorted image. + pub fn project(&self, k: usize, x: V3, undistorted: bool) -> Option<[f64; 2]> { + let q = self.frames.get(k)?.project(self.size, x)?; + Some(if undistorted { q } else { self.to_image(q) }) + } + /// The first solved frame. pub fn first_solved(&self) -> Option { self.frames.iter().position(|c| c.solved) diff --git a/crates/track/src/camtrack/solver.rs b/crates/track/src/camtrack/solver.rs index 85edd2bd..5ed7b81b 100644 --- a/crates/track/src/camtrack/solver.rs +++ b/crates/track/src/camtrack/solver.rs @@ -8,7 +8,7 @@ use rayon::prelude::*; use super::bundle::{self, BaCam, BaObs, BaOpts, FocalMode}; use super::geometry::{self, Pose}; use super::linalg::*; -use super::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, SolvedFrame, SolvedPoint}; +use super::{CameraSolve, CameraTracks, Distortion, ShotType, SolveMethod, SolveSettings, SolvedFrame, SolvedPoint}; use crate::fit::{Model, ransac}; fn debug() -> bool { @@ -142,6 +142,8 @@ struct Cfg { focal: FocalMode, /// Registration estimates a focal length per frame. per_frame_f: bool, + /// The global adjustments also adjust the lens distortion. + distortion: bool, } const F_PRIOR: f64 = 30.0; @@ -183,7 +185,7 @@ impl Recon { &mut cams, &mut pts, &obs, - &BaOpts { focal, fix_points: false, fix_centers: cfg.kind == Kind::Tripod, fixed, huber: d.huber(), max_iter: iters }, + &BaOpts { focal, fix_points: false, fix_centers: cfg.kind == Kind::Tripod, fixed, huber: d.huber(), max_iter: iters, distortion: cfg.distortion }, ); for (k, c) in frames.iter().zip(&cams) { self.cams.insert(*k, *c); @@ -289,6 +291,7 @@ impl Recon { fixed: vec![false], huber, max_iter: 40, + distortion: false, }; let mut best: Option<(usize, BaCam)> = None; // Robust start (large Huber threshold), then tighten; from the prediction and, failing @@ -371,7 +374,11 @@ fn predict(r: &Recon, k: usize) -> Option { match (before, after) { (Some((ka, a)), Some((kb, b))) => { let t = (k - ka) as f64 / (kb - ka) as f64; - Some(BaCam { pose: Pose { r: slerp(&a.pose.r, &b.pose.r, t), c: add(scale(a.pose.c, 1.0 - t), scale(b.pose.c, t)) }, f: a.f * (b.f / a.f).powf(t) }) + Some(BaCam { + pose: Pose { r: slerp(&a.pose.r, &b.pose.r, t), c: add(scale(a.pose.c, 1.0 - t), scale(b.pose.c, t)) }, + f: a.f * (b.f / a.f).powf(t), + dist: a.dist, + }) } (Some((ka, a)), None) => { // Constant velocity from the two previous cameras. @@ -381,7 +388,7 @@ fn predict(r: &Recon, k: usize) -> Option { let dr = log_rot(&mm(&a.pose.r, &mt(&p.pose.r))); let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &a.pose.r); let c = add(a.pose.c, scale(sub(a.pose.c, p.pose.c), t.min(2.0))); - Some(BaCam { pose: Pose { r: rr, c }, f: a.f }) + Some(BaCam { pose: Pose { r: rr, c }, f: a.f, dist: a.dist }) } None => Some(a), } @@ -392,7 +399,7 @@ fn predict(r: &Recon, k: usize) -> Option { let dr = log_rot(&mm(&b.pose.r, &mt(&nx.pose.r))); let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &b.pose.r); let c = add(b.pose.c, scale(sub(b.pose.c, nx.pose.c), t.min(2.0))); - Some(BaCam { pose: Pose { r: rr, c }, f: b.f }) + Some(BaCam { pose: Pose { r: rr, c }, f: b.f, dist: b.dist }) } None => Some(b), }, @@ -459,8 +466,8 @@ fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> fn grow(d: &Data, kfs: &[usize], j: usize, init: &(Pose, HashMap), f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { let (a, b) = (kfs[0], kfs[j]); let mut r = Recon::default(); - r.cams.insert(a, BaCam { pose: Pose::IDENTITY, f }); - r.cams.insert(b, BaCam { pose: init.0, f }); + r.cams.insert(a, BaCam { pose: Pose::IDENTITY, f, dist: Distortion::NONE }); + r.cams.insert(b, BaCam { pose: init.0, f, dist: Distortion::NONE }); r.pts = init.1.clone(); let inner = |r: &mut Recon, frames: &[usize]| { let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed }; @@ -495,7 +502,7 @@ fn grow(d: &Data, kfs: &[usize], j: usize, init: &(Pose, HashMap), f: f /// Tripod reconstruction over keyframes `kfs` at focal length `f`. fn grow_tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { let mut r = Recon::default(); - r.cams.insert(kfs[0], BaCam { pose: Pose::IDENTITY, f }); + r.cams.insert(kfs[0], BaCam { pose: Pose::IDENTITY, f, dist: Distortion::NONE }); let ray = |p: [f64; 2], f: f64| normalize([p[0] / f, p[1] / f, 1.0]); for w in kfs.windows(2) { if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { @@ -519,7 +526,7 @@ fn grow_tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&Atomi rel = geometry::procrustes(&ia, &ib)?; } let pa = r.cams[&a].pose.r; - r.cams.insert(b, BaCam { pose: Pose { r: mm(&rel, &pa), c: [0.0; 3] }, f }); + r.cams.insert(b, BaCam { pose: Pose { r: mm(&rel, &pa), c: [0.0; 3] }, f, dist: Distortion::NONE }); } r.triangulate(d, Kind::Tripod, 0.0, 20.0); let frames: Vec = r.cams.keys().copied().collect(); @@ -640,8 +647,101 @@ fn search_focal(w: f64, eval: &(dyn Fn(f64) -> f64 + Sync)) -> f64 { if best.1 <= errs[i] { best.0.exp() } else { fs[i] } } +/// A solve before its canonical output. +struct Core { + d: Data, + r: Recon, + method: SolveMethod, + kind: Kind, + cfg: Cfg, + focal: FocalMode, + kfs: Vec, +} + +/// `tracks` with every position undistorted by `dist`. +fn undistorted(tracks: &CameraTracks, dist: &Distortion) -> CameraTracks { + let c = [tracks.size[0] * 0.5, tracks.size[1] * 0.5]; + let mut t = tracks.clone(); + for tr in t.tracks.iter_mut() { + for p in tr.pts.iter_mut() { + let q = dist.undistort([p[0] as f64 - c[0], p[1] as f64 - c[1]]); + *p = [(q[0] + c[0]) as f32, (q[1] + c[1]) as f32]; + } + } + t +} + /// Solve the camera of a tracked clip. pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBool>) -> Result { + if !s.lens_distortion { + let c = solve_core(tracks, s, cancel)?; + return Ok(finish(&c.d, tracks, s, c.r, c.method, c.kind)); + } + // Lens distortion: solve, adjust (k1, k2) jointly on the raw tracks, then solve again on + // tracks undistorted with the estimate and adjust jointly once more. + let mut dist = Distortion::for_size(tracks.size); + let mut out = None; + for pass in 0..2 { + let und = (pass > 0).then(|| undistorted(tracks, &dist)); + let c = solve_core(und.as_ref().unwrap_or(tracks), s, cancel)?; + let d = Data::new(tracks, &s.deleted); + d.set_noise(c.d.noise()); + let mut r = c.r; + for cam in r.cams.values_mut() { + cam.dist = dist; + } + let cfg = Cfg { distortion: true, ..c.cfg }; + let all: Vec = r.cams.keys().copied().collect(); + let frames: Vec = if all.len() <= 160 { all.clone() } else { c.kfs.iter().copied().filter(|k| r.cams.contains_key(k)).collect() }; + if frames.len() < 2 { + return err("the camera could not be solved (not enough parallax or features)"); + } + // The pinhole solve's outliers include the distorted frame edges: start from all + // observations (the Huber loss keeps real outliers in check), then reject again. + r.outliers.clear(); + r.adjust(&d, &frames, &cfg, c.focal, 30); + r.outliers.clear(); + r.reject(&d); + r.adjust(&d, &frames, &cfg, c.focal, 30); + dist = r.cams[&frames[0]].dist; + for cam in r.cams.values_mut() { + cam.dist = dist; + } + if frames.len() < all.len() { + // Refine the other frames against the adjusted points and distortion. + let base = r.clone(); + let rcfg = Cfg { distortion: false, ..cfg }; + let refined: Vec<(usize, Option)> = all + .par_iter() + .filter(|k| !frames.contains(k)) + .map(|k| { + let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() }; + let start = local.cams[k]; + let ok = local.register(&d, *k, start, &rcfg); + (*k, if ok { local.cams.get(k).copied() } else { None }) + }) + .collect(); + for (k, cam) in refined { + if let Some(cam) = cam { + r.cams.insert(k, cam); + } + } + r.reject(&d); + } + if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return err("cancelled"); + } + if debug() { + eprintln!("lens distortion pass {pass}: k1 {:.5} k2 {:.5}", dist.k1, dist.k2); + } + let mut sol = finish(&d, tracks, s, r, c.method, c.kind); + sol.distortion = Some(dist); + out = Some(sol); + } + out.ok_or_else(|| SolveError("the camera could not be solved".into())) +} + +fn solve_core(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBool>) -> Result { let d = Data::new(tracks, &s.deleted); if d.n < 2 { return err("the clip must be at least two frames long"); @@ -653,7 +753,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo let cancelled = || cancel.is_some_and(|c| c.load(Ordering::Relaxed)); let fixed_f = (s.shot == ShotType::SpecifyAngle).then(|| focal_of(d.w, if s.hfov > 0.0 { s.hfov } else { 40.0 })); let variable = s.shot == ShotType::VariableZoom; - let gcfg = Cfg { kind: Kind::General, focal: FocalMode::Fixed, per_frame_f: false }; + let gcfg = Cfg { kind: Kind::General, focal: FocalMode::Fixed, per_frame_f: false, distortion: false }; let tcfg = Cfg { kind: Kind::Tripod, ..gcfg }; // The focal-length search runs on the first keyframes only. let mini: Vec = kfs.iter().copied().take(10).collect(); @@ -738,7 +838,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo } let (att, _) = results.swap_remove(pick); let kind = if att.method == SolveMethod::TripodPan { Kind::Tripod } else { Kind::General }; - let cfg = Cfg { kind, focal: FocalMode::Fixed, per_frame_f: variable }; + let cfg = Cfg { kind, focal: FocalMode::Fixed, per_frame_f: variable, distortion: false }; let mut r = att.recon; let focal = match (fixed_f, variable) { (Some(_), _) => FocalMode::Fixed, @@ -845,7 +945,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo (m, _) => m, }; lap("final adjustment"); - Ok(finish(&d, tracks, s, r, method, kind)) + Ok(Core { d, r, method, kind, cfg, focal, kfs: kf_frames }) } /// Thickness of a point cloud: √(smallest / middle eigenvalue of its covariance). @@ -949,5 +1049,6 @@ fn finish(d: &Data, tracks: &CameraTracks, s: &SolveSettings, r: Recon, method: frames, points, ground: None, + distortion: None, } } diff --git a/crates/track/src/camtrack/tests.rs b/crates/track/src/camtrack/tests.rs index 4b5a8346..ff92cfe4 100644 --- a/crates/track/src/camtrack/tests.rs +++ b/crates/track/src/camtrack/tests.rs @@ -105,7 +105,7 @@ fn path_error(s: &CameraSolve, t: &Truth) -> (f64, f64) { } fn settings(shot: ShotType, method: SolveMethod) -> SolveSettings { - SolveSettings { shot, hfov: 0.0, method, deleted: vec![] } + SolveSettings { shot, hfov: 0.0, method, deleted: vec![], lens_distortion: false } } #[test] @@ -149,7 +149,8 @@ fn specified_angle_of_view_is_kept() { let pts = random_points(300, 5, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); let tr = tracks_of(&pts, &t, 6, true); let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); - let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: vec![] }, None).unwrap(); + let s = + solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: vec![], lens_distortion: false }, None).unwrap(); assert!((s.focal() - f).abs() < 1e-6); assert!(s.average_error < 0.5); let (pe, _) = path_error(&s, &t); @@ -226,7 +227,9 @@ fn mostly_flat_scene_and_ground_plane() { } // Ground plane from the solved (coplanar) points: every point lies on it. let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); - let mut s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::MostlyFlat, deleted: vec![] }, None).expect("solve"); + let mut s = + solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::MostlyFlat, deleted: vec![], lens_distortion: false }, None) + .expect("solve"); let ids: Vec = s.visible(0).map(|p| p.id).take(12).collect(); let g = s.ground_from(&ids, 0).expect("ground"); let spread = s.points.iter().map(|p| dot(sub(p.pos, g.origin), g.normal).abs()).fold(0.0, f64::max); @@ -251,7 +254,9 @@ fn deleted_tracks_are_ignored_and_cancel_stops() { let tr = tracks_of(&pts, &t, 14, true); let deleted: Vec = (0..40).collect(); let hfov = 2.0 * (W * 0.5 / 1000.0f64).atan().to_degrees(); - let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: deleted.clone() }, None).unwrap(); + let s = + solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: deleted.clone(), lens_distortion: false }, None) + .unwrap(); assert!(s.points.iter().all(|p| !deleted.contains(&p.id))); let stop = std::sync::atomic::AtomicBool::new(true); let r = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), Some(&stop)); @@ -436,3 +441,55 @@ fn solve_saved_tracks() { t0.elapsed() ); } + +/// Pass every track position through a lens distortion (around the frame centre). +fn distort_tracks(t: &CameraTracks, d: &Distortion) -> CameraTracks { + let mut t = t.clone(); + for tr in t.tracks.iter_mut() { + for p in tr.pts.iter_mut() { + let q = d.distort([p[0] as f64 - W / 2.0, p[1] as f64 - H / 2.0]); + *p = [(q[0] + W / 2.0) as f32, (q[1] + H / 2.0) as f32]; + } + } + t +} + +#[test] +fn lens_distortion_is_recovered_in_the_bundle_adjustment() { + let f = 1000.0; + let t = moving_camera(36, |_| f); + let pts = random_points(450, 7, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]); + let truth = Distortion { k1: -0.12, k2: 0.03, radius: 0.5 * W.hypot(H) }; + let tr = distort_tracks(&tracks_of(&pts, &t, 8, true), &truth); + let plain = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), None).expect("pinhole solve"); + let lens = solve(&tr, &SolveSettings { lens_distortion: true, ..settings(ShotType::FixedAngle, SolveMethod::Typical) }, None).expect("lens solve"); + let d = lens.distortion.expect("distortion solved"); + eprintln!( + "pinhole error {:.3} px (focal {:.1}); with distortion {:.3} px (focal {:.1}), k1 {:.4} k2 {:.4}", + plain.average_error, + plain.focal(), + lens.average_error, + lens.focal(), + d.k1, + d.k2 + ); + assert!(plain.distortion.is_none()); + assert!((d.k1 - truth.k1).abs() < 0.005, "k1 {}", d.k1); + assert!((d.k2 - truth.k2).abs() < 0.01, "k2 {}", d.k2); + // The distortion model explains the tracks: about the noise level, far below the pinhole fit. + assert!(lens.average_error < 0.3, "{}", lens.average_error); + assert!(lens.average_error < 0.5 * plain.average_error, "{} vs {}", lens.average_error, plain.average_error); + assert!((lens.focal() - f).abs() < 0.02 * f, "focal {}", lens.focal()); + let (path, rot) = path_error(&lens, &t); + assert!(path < 0.01 && rot < 0.3, "path {path} rot {rot}"); + // Projection through the lens lands on the (distorted) tracks; undistorting inverts it. + let p = [100.0, 80.0]; + let q = lens.to_image(p); + let back = lens.to_ideal(q); + assert!((back[0] - p[0]).abs() < 1e-6 && (back[1] - p[1]).abs() < 1e-6); + let sp = &lens.points[0]; + let k = sp.first as usize; + let obs = tr.track(sp.id).and_then(|t| t.at(k as u32)).expect("observed"); + let proj = lens.project(k, sp.pos, false).expect("in front"); + assert!((proj[0] - obs[0]).hypot(proj[1] - obs[1]) < 2.0, "{proj:?} vs {obs:?}"); +} diff --git a/crates/track/src/face.rs b/crates/track/src/face.rs index 78344a14..62073d0a 100644 --- a/crates/track/src/face.rs +++ b/crates/track/src/face.rs @@ -624,6 +624,11 @@ impl FaceTracker { }; let mut t = FaceTracker { skin, prev: guess, expect: canon }; let fit = t.fit(frame)?; + // A face shows at least two of: left eye, right eye, mouth. + let found = [EYE[0][4], EYE[1][4], MOUTH[0]].iter().filter(|i| fit.detected[**i]).count(); + if found < 2 { + return None; + } t.accept(&fit); Some((t, fit)) } diff --git a/crates/ui-egui/src/panels/camera_tracker_ui.rs b/crates/ui-egui/src/panels/camera_tracker_ui.rs index 41fb7204..c9c40f05 100644 --- a/crates/ui-egui/src/panels/camera_tracker_ui.rs +++ b/crates/ui-egui/src/panels/camera_tracker_ui.rs @@ -38,6 +38,8 @@ struct Active { show_3d: bool, point_size: f64, target_size: f64, + /// Undistort Footage is on: points project without the lens distortion. + undistort: bool, /// Layer pixels → comp pixels. l2c: Mat3, frame: usize, @@ -66,6 +68,7 @@ fn active(s: &Session, comp: &Comp, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Laye let t = tracks.as_ref()?; (t.frame_duration > 0.0).then(|| ((lt - t.start) / t.frame_duration).round().clamp(0.0, t.frames.saturating_sub(1) as f64) as usize) }); + let undistort = solve.as_ref().is_some_and(|sv| ct::undistorts(¶ms, sv)); return Some(Active { layer: *lid, uid: *uid, @@ -74,6 +77,7 @@ fn active(s: &Session, comp: &Comp, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Laye show_3d: val("showTrackPoints").map(|v| v.as_enum()).unwrap_or(1) == 1, point_size: val("trackPointSize").map(|v| v.as_f64()).unwrap_or(100.0), target_size: val("targetSize").map(|v| v.as_f64()).unwrap_or(100.0), + undistort, l2c: l2c(ectx, l), frame: frame.unwrap_or(0), }); @@ -98,18 +102,20 @@ fn l2c_scale(m: &Mat3) -> f32 { /// Draw the target disc (concentric rings in perspective) of `t` on frame `k`. fn draw_target(p: &egui::Painter, a: &Active, map: &ViewerMap, sv: &CameraSolve, t: &Target) { - let Some(cam) = sv.frames.get(a.frame) else { return }; + if sv.frames.get(a.frame).is_none() { + return; + } let r = t.size * 0.5 * a.target_size / 100.0; for (i, frac) in [1.0, 0.66, 0.33].into_iter().enumerate() { let ring = ct::target_circle(t.center, t.normal, r * frac, 48); - let pts: Vec = ring.iter().filter_map(|x| cam.project(sv.size, *x)).map(|q| to_screen(a, map, q)).collect(); + let pts: Vec = ring.iter().filter_map(|x| sv.project(a.frame, *x, a.undistort)).map(|q| to_screen(a, map, q)).collect(); if pts.len() < 3 { return; } let fill = if i % 2 == 0 { Color32::from_rgba_unmultiplied(0xd0, 0x30, 0x30, 70) } else { Color32::from_rgba_unmultiplied(0xff, 0xff, 0xff, 50) }; p.add(egui::Shape::convex_polygon(pts.clone(), fill, Stroke::new(1.0, Color32::from_rgb(0xe0, 0x40, 0x40)))); } - if let Some(c) = cam.project(sv.size, t.center) { + if let Some(c) = sv.project(a.frame, t.center, a.undistort) { let c = to_screen(a, map, c); p.circle_filled(c, 3.0, Color32::from_rgb(0xff, 0xd0, 0x40)); } @@ -168,7 +174,7 @@ pub fn viewer_hook( let mut hits: Vec = vec![]; match (&a.solve, a.show_3d) { (Some(sv), true) => { - let pts = ct::projected(sv, a.frame); + let pts = ct::projected_with(sv, a.frame, a.undistort); let med = ct::median_depth(&pts); for (id, q, z) in pts { let sp = to_screen(&a, map, q); diff --git a/crates/ui-egui/src/panels/tracker.rs b/crates/ui-egui/src/panels/tracker.rs index 364e114e..02c59636 100644 --- a/crates/ui-egui/src/panels/tracker.rs +++ b/crates/ui-egui/src/panels/tracker.rs @@ -404,6 +404,27 @@ fn mask_mode(app: &mut EffectcraftApp, ui: &mut egui::Ui, ctx: &egui::Context, p bx += 34.0; } y += 34.0; + // Face Tracking (Detailed Features): Extract & Copy Face Measurements. + let has_points = app + .session + .active_comp() + .and_then(|c| c.layer(layer)) + .and_then(|l| l.effects()) + .is_some_and(|fx| fx.groups().any(|g| g.match_id == effectcraft_engine::effects::face_track::POINTS_ID)); + if method == MaskMethod::FaceDetailed || has_points { + let r = Rect::from_min_size(pos2(x0, y), vec2(w.min(260.0), 24.0)); + if button(app, ui, r, "Extract & Copy Face Measurements", has_points && !running, "tracker.mask.extractFace") { + match crate::menus::invoke(app, ctx, "track.extractFaceMeasurements", json!({"layer": layer.0})) { + Ok(v) => { + if let Some(text) = v.get("clipboard").and_then(|t| t.as_str()) { + ctx.copy_text(text.to_string()); + } + } + Err(e) => app.ui.status = e, + } + } + y += 32.0; + } if let Some(pr) = app.session.mask_track_progress() { let bar = Rect::from_min_size(pos2(x0, y), vec2(w, 6.0)); p.rect_filled(bar, 3.0, t.field_bg); @@ -419,7 +440,12 @@ fn mask_mode(app: &mut EffectcraftApp, ui: &mut egui::Ui, ctx: &egui::Context, p ); ctx.request_repaint_after(std::time::Duration::from_millis(50)); } else { - p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, "Tracks the pixels inside the mask and keys its Mask Path.", Tokens::ui(11.0), t.text_faint); + let hint = if method.is_face() { + "Finds the face inside the mask and keys its outline (and, with Detailed Features, Face Track Points)." + } else { + "Tracks the pixels inside the mask and keys its Mask Path." + }; + p.text(pos2(x0, y + 8.0), Align2::LEFT_CENTER, hint, Tokens::ui(11.0), t.text_faint); } } diff --git a/docs/agents.md b/docs/agents.md index 8689b9da..c25aa399 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -105,11 +105,18 @@ effectcraft-cli run clip.ecproj track.motion '{"layer":"#2"}' \ `track.mask` follows the pixels inside a mask and keys its Mask Path on every frame (vertices and tangents move with the fitted motion). `method` is `position`, `positionScale`, -`positionScaleRotation` (the default, also kept as the Tracker panel's Method), `positionScaleRotationSkew` -or `perspective`; `direction` is `forward|backward|frameForward|frameBackward` from the current +`positionScaleRotation` (the default, also kept as the Tracker panel's Method), `positionScaleRotationSkew`, +`perspective`, `faceOutline` (Face Tracking (Outline Only): the mask, drawn around a face, becomes +the face outline on every frame) or `faceDetailed` (Face Tracking (Detailed Features): also keys a +Face Track Points effect with one point per landmark — eyebrows, eyes, pupils, nose tip, +nostrils, mouth corners and lips, chin, jaw); `direction` is `forward|backward|frameForward|frameBackward` from the current time (or `start` / `end` in seconds). Like `track.analyze` it runs in the background in the app (progress in `track.status` under `mask`, `track.stop` cancels and keeps what was tracked) and is -one undo step; `"wait": true` blocks. +one undo step; `"wait": true` blocks. `track.extractFaceMeasurements {layer?}` (Extract & Copy Face +Measurements) keys a Face Measurements effect from the Face Track Points (head position, scale and +orientation X/Y/Z, eye openness, eyebrow distance from eye, mouth openness, width and offset), puts +those keys on the keyframe clipboard and returns them per frame plus a tab-separated `clipboard` +text. `mask.interpolate` (Window ▸ Mask Interpolation ▸ Apply) adds in-between Mask Path keys between each pair of selected Mask Path keys (or the existing keys at `times`, in seconds), giving both @@ -138,7 +145,11 @@ the app re-analyses automatically (headless: run `warp.analyze` again). Settings properties under the instance's groups, e.g. `effects/#1/stabilization/result` (0 Smooth Motion, 1 No Motion), `effects/#1/stabilization/smoothness`, `effects/#1/stabilization/method`, `effects/#1/borders/framing` (0 Stabilize Only … 3 Stabilize, Synthesize Edges), -`effects/#1/borders/autoScale/maximumScale`, `effects/#1/advanced/showTrackPoints`. +`effects/#1/borders/autoScale/maximumScale`, `effects/#1/advanced/showTrackPoints`. Method 3 +(Subspace Warp, the default) bends a mesh per frame to the smoothed feature trajectories, so +scenes with depth stabilise where one perspective transform cannot; +`effects/#1/advanced/rollingShutterRipple` (0 Automatic Reduction, 1 Enhanced Reduction: a finer, +softer mesh) tunes it. ```sh effectcraft-cli run shaky.ecproj track.warpStabilizer '{"layer":"#1","wait":true}' \ @@ -150,13 +161,17 @@ effectcraft-cli exec warp.status '{"layer":"#1"}' shaky.ecproj --json ### 3D Camera Tracker Animation ▸ Track Camera and the Tracker panel's button (`track.camera {layer?, shotType?: -fixed|variable|specify, aov?, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}`) apply +fixed|variable|specify, aov?, solveMethod?: auto|typical|flat|tripod, detailed?, lensDistortion?, +undistort?, wait?}`) apply Effect ▸ Perspective ▸ 3D Camera Tracker (or reuse the layer's) and analyse it in the background: "Analyzing in background (step 1 of 2)" tracks features, "Solving camera" solves. `camera.analyze {layer?, effect?, wait?}` re-runs it, `camera.cancel` stops it without writing anything. `camera.solveStatus` reports `progress`/`banner`, `analyzed`, `solved`, `methodUsed`, `averageError` (pixels), `focalLength`, `horizontalAngleOfView`, the current frame's `camera` -(position, orientation, zoom) and `groundPlane`. `camera.points {time?}` lists the solved points +(position, orientation, zoom), `groundPlane` and the solved `lensDistortion` (`k1`, `k2`, normalised by +half the frame diagonal) when Solve Lens Distortion (`effects/#1/advanced/lensDistortion`, also on +with Detailed Analysis) is on; `effects/#1/advanced/undistort` renders the footage undistorted so +the solved camera's 3D layers line up. `camera.points {time?}` lists the solved points visible now with their `id`, `comp` position, `depth`, `world` position and `error`. Select points with `camera.selectPoints {points, add?, toggle?}` (the viewer's click / Shift-click / diff --git a/docs/architecture.md b/docs/architecture.md index a54703b7..f54c2b9c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -228,6 +228,13 @@ tracks them with pyramidal Lucas–Kanade and fits the chosen model (position RANSAC; the motion moves the Mask Path's vertices and tangents and is keyed per frame. **Mask Interpolation** (`mask.interpolate`) uses `effectcraft-path`'s smart interpolation (arc-length vertex insertion, shape-context matching, rigid in-betweens) to key in-between shapes. +The two **Face Tracking** methods (`effectcraft_track::face`) fit a face inside the mask instead: +a skin colour model from the first frame, the face outline along rays from the skin component's +centre (an area-moment ellipse gives centre, size and roll), facial features as the non-skin +components inside it, and an active-shape point distribution model trained on synthetic face +shapes (our own generator, no external weights) that fills and checks the landmarks. The outline +keys the Mask Path; Detailed Features keys a Face Track Points effect, and +`track.extractFaceMeasurements` derives a keyed Face Measurements effect (and copies its keys). **Warp Stabilizer** (`effects::warp_stab`, `engine::warp`): `warp.analyze` renders the layer's input to the effect (`Renderer::layer_input`: source, masks and the effects above it) for every @@ -236,7 +243,10 @@ translation / similarity / homography fits) as JSON in the effect's hidden Analy with a key of the layer's source, In/Out, start, stretch and Time Remap. `Session::edit` clears analyses whose key no longer matches and queues them; the desktop app re-analyses them in the background. Rendering derives the stabilization plan (smoothed camera path or No Motion, framing, -auto-scale; cached per analysis and settings) and warps each frame; Synthesize Edges fills the +auto-scale; cached per analysis and settings) and warps each frame. Subspace Warp also stores long +feature trajectories in the analysis, smooths them in a low-rank subspace (Liu et al. 2011) and +warps each frame with a content-preserving mesh (Liu et al. 2009) fitted to the smoothed +positions, rendered as a per-pixel mesh lookup; Synthesize Edges fills the borders from neighbouring frames read with `EffectHost::self_at`. The effect lives in the effects crate, hence the `effects → track` edge. @@ -267,7 +277,10 @@ the essential matrix (normalised 8-point in RANSAC) or a planar homography decom resection and triangulation; a sparse Levenberg–Marquardt bundle adjustment (Schur complement over the points, Huber loss) with the focal length fixed, shared, or per frame; a log-spaced and golden-section focal search for Fixed Angle of View / Variable Zoom; a rotation-only model for tripod -pans, chosen by Auto Detect when it explains the tracks as well. Tracks and solve are stored as JSON +pans, chosen by Auto Detect when it explains the tracks as well; with Solve Lens Distortion (or +Detailed Analysis) radial distortion `k1, k2` is adjusted jointly in a final bundle adjustment on +the raw tracks and the solve repeated on tracks undistorted with that estimate; Undistort Footage +renders the input through the inverse model. Tracks and solve are stored as JSON in hidden effect parameters with keys: changing the layer's frames clears both, changing Shot Type, Angle of View, Solve Method or deleting points only re-solves. The solve's canonical frame maps to comp space so the first frame's camera is the default comp camera (or so a chosen ground plane is diff --git a/docs/effects.md b/docs/effects.md index 84735286..fd29541b 100644 --- a/docs/effects.md +++ b/docs/effects.md @@ -3,11 +3,11 @@ -EffectCraft ships 304 effects with 2446 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. +EffectCraft ships 306 effects with 2488 parameters in total, grouped into the same categories as the Effects & Presets panel. Every effect is our own implementation, written from public behaviour descriptions and standard image-processing literature. Parameter names, order, popup options, units, defaults and ranges follow the reference application so that projects, expressions and muscle memory carry over. - **GPU**: 59 effects also run on the GPU compositor with identical results. -- **32**: 304 effects process 32-bit float (HDR, overbright) pixels without clamping. -- **Status**: 262 implemented in full, 42 partial (what is missing is listed). +- **32**: 306 effects process 32-bit float (HDR, overbright) pixels without clamping. +- **Status**: 264 implemented in full, 42 partial (what is missing is listed). Effects are addressed by id (`ec..`) or display name in commands, scripts and the control channel, and every parameter by its id (`effects/#1/blurriness`). @@ -302,7 +302,7 @@ Effects are addressed by id (`ec..`) or display name in commands | Effect | Id | Params | GPU | 32 | Status | |---|---|---:|:-:|:-:|---| -| 3D Camera Tracker | `ec.perspective.cameratracker` | 10 | | 32 | Implemented | +| 3D Camera Tracker | `ec.perspective.cameratracker` | 12 | | 32 | Implemented | | 3D Glasses | `ec.perspective.3dglasses` | 8 | | 32 | Implemented | | Bevel Alpha | `ec.perspective.bevelalpha` | 4 | | 32 | Implemented | | Bevel Edges | `ec.perspective.beveledges` | 4 | | 32 | Implemented | @@ -416,6 +416,8 @@ Effects are addressed by id (`ec..`) or display name in commands | CC Overbrights | `ec.utility.ccoverbrights` | 2 | | 32 | Implemented | | Cineon Converter | `ec.utility.cineon` | 7 | | 32 | Implemented | | Color Profile Converter | `ec.utility.colorprofileconverter` | 7 | | 32 | Implemented | +| Face Measurements | `ec.utility.facemeasurements` | 14 | | 32 | Implemented | +| Face Track Points | `ec.utility.facetrackpoints` | 26 | | 32 | Implemented | | Grow Bounds | `ec.utility.growbounds` | 1 | | 32 | Implemented | | HDR Compander | `ec.utility.hdrcompander` | 3 | | 32 | Implemented | | HDR Highlight Compression | `ec.utility.hdrcompression` | 1 | | 32 | Implemented | diff --git a/docs/parity.md b/docs/parity.md index 4437bece..2d03442b 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -17,13 +17,13 @@ How close EffectCraft is to After Effects 2026, feature by feature, and how much By area: Layers 98%, Output 95%, Compositions 95%, Automation 96%, Paint 95%, Text 95%, Import 96%, Animation 94%, Masks 94%, Preview 94%, Interface 93%, Shapes 93%, 3D 91%, Audio 90%, Project 90%, -Tracking 88%, Effects 85%, Web 65%. +Tracking 96%, Effects 85%, Web 65%. What is left, in priority order: the web app's depth (threads, storage, audio, non-blocking renders); stroke taper/wave and multi-segment dashes; variable mask feather points; camera iris/bokeh and focus-link commands; Render Queue field render, crop/resize and templates; -approximated effects (Subspace Warp, Key Cleaner) and Liquify's viewer brush; OpenType features and variable +approximated effects (Key Cleaner) and Liquify's viewer brush; OpenType features and variable font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; ScriptUI; -Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; face tracking; and +Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; and the "better than After Effects" items (a plug-in API, branching history; GPU particles landed in M12.7). The sections below are the original audit (morning of 2 October, ≈ 64%) and its updates, kept for @@ -124,7 +124,7 @@ the disk cache in the formats wave; the 3D Camera Tracker in M12.6.) | Project | ≈ 68% | 5.5 | auto-save, folder moves, OCIO displays beyond the built-in tone map (Color Engine with OCIO/ACES working spaces, HDR compand/tone mapping, Rec. 2100 PQ/HLG output, Feet + Frames, display colour management, Simulate Output and the locked viewer landed in M7.7; proxies and Interpret Footage fields / pixel aspect / alpha guess landed: PRJ-8, PRJ-3) | | Masks & roto | 74% | 5.0 | Roto Brush's learned (3.0) segmentation model (variable-width mask feather points with the Mask Feather tool landed in M13.5; mask tracking and Mask Interpolation landed in M6.6; Roto Brush & Refine Edge with graph-cut segmentation, flow propagation, edge matting, decontamination and Freeze in M6.7) | | Preview | 66% | 4.0 | GPU bokeh depth of field, wireframes and Advanced 3D compositing (Classic 3D runs and adjustment layers composite on the GPU since M12.7; persistent disk cache with the blue cache bar landed; region of interest, snapshots, exposure and Fast Previews landed in M0.13) | -| Tracking | ≈ 85% | 2.5 | face tracking, Subspace Warp's mesh warp, lens distortion in the camera solve (Rolling Shutter Repair landed in M9.11; point tracker, mask tracking, Warp Stabilizer and the 3D Camera Tracker landed in M6.x / M12.5 / M12.6) | +| Tracking | ≈ 96% | 0.3 | face tracking is a classical (skin model + feature components + shape model) fitter, not a learned detector: profile views and occluded faces are weak; Rolling Shutter Ripple is approximated by Subspace Warp's mesh density (face tracking (Outline Only / Detailed Features with Face Track Points and Extract & Copy Face Measurements), Subspace Warp's content-preserving mesh warp on subspace-smoothed trajectories, and radial lens distortion (k1, k2) in the camera bundle adjustment with Undistort Footage landed in M13.3; Rolling Shutter Repair in M9.11; point tracker, mask tracking, Warp Stabilizer and the 3D Camera Tracker in M6.x / M12.5 / M12.6) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | ## Effects still missing From ca7f7675535522d295ef0572d3a5019dd731dc62 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 04:50:59 -0400 Subject: [PATCH 6/9] M13.1: clippy (history test) Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/tests_history.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/engine/src/tests_history.rs b/crates/engine/src/tests_history.rs index 326c37c0..967c29e3 100644 --- a/crates/engine/src/tests_history.rs +++ b/crates/engine/src/tests_history.rs @@ -46,7 +46,7 @@ fn undo_then_new_step_keeps_a_branch() { let branch: Vec<&Value> = st.iter().filter(|n| n["depth"] == 1).collect(); assert_eq!(branch.len(), 2, "B and C sit on a branch: {st:?}"); // Jump to C on the old branch. - let c = st.iter().filter(|n| n["depth"] == 1).last().unwrap()["index"].as_u64().unwrap(); + let c = st.iter().rfind(|n| n["depth"] == 1).unwrap()["index"].as_u64().unwrap(); let r = s.execute("edit.history.goto", json!({"index": c})).unwrap(); assert_eq!(r["label"], "New Solid", "{r}"); assert_eq!(names(&s), ["C", "B", "A"]); From 81ac7151bc38a6f85022a7e3f7d881fcfbcb446c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 04:53:56 -0400 Subject: [PATCH 7/9] M13.3: CameraSolve distortion field in UI test after merge Co-Authored-By: Claude Opus 5.5 --- crates/ui-egui/tests/ui_camera_tracker.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/ui-egui/tests/ui_camera_tracker.rs b/crates/ui-egui/tests/ui_camera_tracker.rs index 45374fba..963ce3b6 100644 --- a/crates/ui-egui/tests/ui_camera_tracker.rs +++ b/crates/ui-egui/tests/ui_camera_tracker.rs @@ -41,6 +41,7 @@ fn app() -> (EffectcraftApp, u64, u64) { }) .collect(), ground: None, + distortion: None, }; s.execute("prop.set", json!({"layer": plate, "path": "effects/#1/solve", "value": solve.to_json()})).unwrap(); s.camera_pending.clear(); From 1242a775fed8c65893731cf6ee01c38d6f26da4c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 04:55:31 -0400 Subject: [PATCH 8/9] M10.2: Render Queue completeness (render settings, output module, templates, logs, overflow, notify) Render Settings: field render with 3:2 pulldown phases, effects/solo/guide layer/color depth/frame blending/motion blur overrides, custom time sampling (duration), storage overflow through a quota hook. Output Module: crop (edges or region of interest), resize (presets, lock aspect, quality), straight/premultiplied/alpha-only channels, post-render actions, include project link, audio channels and PCM formats, WebM bitrate and key frame interval. Render Settings / Output Module templates with built-ins and defaults per slot (Edit > Templates dialogs), render logs next to the output, Notify on completion. All through renderQueue.* commands and persisted in the project. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + README.md | 7 +- crates/engine/src/commands/comp_more.rs | 40 +- crates/engine/src/commands/proxy.rs | 19 +- crates/engine/src/commands/render_queue.rs | 520 ++++++++++-- crates/engine/src/lib.rs | 3 + crates/engine/src/render_queue.rs | 51 +- crates/engine/src/tests_stubs.rs | 117 +++ crates/export/Cargo.toml | 1 + crates/export/src/encode.rs | 80 +- crates/export/src/lib.rs | 170 ++-- crates/export/src/pipeline.rs | 327 ++++++++ crates/export/src/webm.rs | 79 +- crates/export/tests/render_settings.rs | 310 ++++++++ crates/export/tests/roundtrip.rs | 28 +- crates/host/src/lib.rs | 18 +- crates/project/src/lib.rs | 17 +- crates/project/src/render_queue.rs | 743 +++++++++++++++++- crates/project/src/render_templates.rs | 323 ++++++++ crates/ui-egui/src/lib.rs | 2 + crates/ui-egui/src/menus.rs | 14 +- crates/ui-egui/src/panels/dialogs.rs | 1 + crates/ui-egui/src/panels/mod.rs | 1 + crates/ui-egui/src/panels/render_queue.rs | 259 +++++- crates/ui-egui/src/panels/rq_templates.rs | 166 ++++ crates/ui-egui/tests/render_queue_snapshot.rs | 23 + docs/parity.md | 4 +- docs/web.md | 2 - 28 files changed, 3096 insertions(+), 230 deletions(-) create mode 100644 crates/export/src/pipeline.rs create mode 100644 crates/export/tests/render_settings.rs create mode 100644 crates/project/src/render_templates.rs create mode 100644 crates/ui-egui/src/panels/rq_templates.rs diff --git a/Cargo.lock b/Cargo.lock index c7934100..1dcfbb95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1236,6 +1236,7 @@ name = "effectcraft-export" version = "0.1.0" dependencies = [ "effectcraft-color", + "effectcraft-keyframe", "effectcraft-media", "effectcraft-opusenc", "effectcraft-project", diff --git a/README.md b/README.md index 3bff64fb..3a9cabbc 100644 --- a/README.md +++ b/README.md @@ -136,8 +136,11 @@ tools, and Front, Top, Left and Custom views. ## Export A Render Queue like After Effects', with Render Settings and Output Modules: **H.264** MP4 and -**ProRes** MOV (Proxy to 4444 XQ with alpha) with audio, PNG, JPEG, TIFF and 32-bit EXR -sequences, and animated GIF. The same queue runs from the command line. Every encoder is +**ProRes** MOV (Proxy to 4444 XQ with alpha) with audio, **WebM** (VP9 with inter frames and +alpha, Opus), PNG, JPEG, TIFF and 32-bit EXR sequences, animated GIF and WAV/AIFF. Field +rendering with 3:2 pulldown, effect/solo/guide/depth overrides, crop, region of interest and +resize, Render Settings and Output Module templates with defaults, post-render actions, storage +overflow and render logs. The same queue runs from the command line. Every encoder is FilmCraft's pure-Rust code; there is no FFmpeg inside. **Lottie** goes both ways: File ▸ Export ▸ Lottie JSON… writes a composition (precomps, shape, diff --git a/crates/engine/src/commands/comp_more.rs b/crates/engine/src/commands/comp_more.rs index e3057f67..0f07fd67 100644 --- a/crates/engine/src/commands/comp_more.rs +++ b/crates/engine/src/commands/comp_more.rs @@ -82,7 +82,37 @@ pub(crate) fn encode_png(rgba: &[u8], w: u32, h: u32) -> std::result::Result Result { + use effectcraft_project::render_queue::{RenderQueueItem, TimeSpan}; + use effectcraft_project::render_templates::TemplateSlot; + let cid = comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let t = f_p(p, "time").map(Tick::from_seconds_f64).unwrap_or(s.time()); + let mut it = RenderQueueItem::new(0, cid); + it.settings = s.project.render_templates.default_render_settings(TemplateSlot::Still); + it.settings.time_span = TimeSpan::Custom { start: t, end: t + comp.frame_duration() }; + it.output = s.project.render_templates.default_output_module(TemplateSlot::Still); + if let Some(path) = str_p(p, "path") { + it.output.output = path.to_string(); + } else if !it.output.format.is_sequence() { + it.output.output = "[compName]_frame.[fileExtension]".into(); + } + let id = s.edit("Save Frame As", None, |proj, _| { + it.id = proj.render_queue.iter().map(|i| i.id).max().unwrap_or(0) + 1; + let id = it.id; + proj.render_queue.push(it); + Ok(id) + })?; + let path = s.project.render_queue.last().and_then(|i| s.resolve_output(i)); + Ok(json!({"item": id, "queued": true, "output": path, "time": t.seconds()})) +} + fn save_frame(s: &mut Session, p: &Value) -> Result { + if p.get("queue").and_then(Value::as_bool) == Some(true) { + return queue_frame(s, p); + } let path = str_p(p, "path").ok_or_else(|| bad("comp.saveFrameAs", "missing `path` (.png)"))?.to_string(); let cid = comp_id(s, p)?; let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; @@ -124,7 +154,15 @@ pub fn specs() -> Vec { vec![ cmd!("comp.cropToRegionOfInterest", "Crop Comp to Region of Interest", ["Composition"], None, "{comp?}", has_roi, crop_to_roi), cmd!("comp.cropToLayerBounds", "Crop Comp to Selected Layer(s) Bounds", ["Composition"], None, "{layers?}", has_layers, crop_to_layers), - cmd!("comp.saveFrameAs", "File...", ["Composition", "Save Frame As"], Some("Cmd+Alt+S"), "{path (.png), time?, scale?}", has_comp, save_frame), + cmd!( + "comp.saveFrameAs", + "File...", + ["Composition", "Save Frame As"], + Some("Cmd+Alt+S"), + "{path (.png; with queue: an output path or template), time?, scale?, queue?: bool (add a Render Queue item from the Frame Default templates instead of writing now)}", + has_comp, + save_frame + ), cmd!("comp.responsiveTime", "Responsive Design — Time", [], None, "{op: intro|outro|workArea}", has_comp, responsive_time), ] } diff --git a/crates/engine/src/commands/proxy.rs b/crates/engine/src/commands/proxy.rs index 607dd5dd..3e1126a9 100644 --- a/crates/engine/src/commands/proxy.rs +++ b/crates/engine/src/commands/proxy.rs @@ -4,6 +4,7 @@ //! item's switch). use effectcraft_project::render_queue::{Channels, OutputFormat, OutputModule, PostRenderAction, ProResProfile, RenderQuality, RenderQueueItem, TimeSpan}; +use effectcraft_project::render_templates::TemplateSlot; use effectcraft_project::{Footage, ItemId, ItemKind, Proxy}; use effectcraft_time::Tick; use serde_json::{Value, json}; @@ -159,18 +160,28 @@ fn create_proxy(s: &mut Session, p: &Value) -> Result { k => return Err(bad(C, format!("kind: still|movie, not `{k}`"))), }; let mut it = RenderQueueItem::new(0, cid); + // The Movie / Still Proxy Default templates, at draft quality and the proxy resolution. + let templates = &s.project.render_templates; + let slot = if still { TemplateSlot::StillProxy } else { TemplateSlot::MovieProxy }; + it.settings = templates.default_render_settings(slot); it.settings.quality = RenderQuality::Draft; it.settings.resolution = p.get("resolution").and_then(Value::as_f64).unwrap_or(0.5).clamp(0.05, 1.0); if still { // One frame at the current time (the comp's poster frame when not the active comp). let t = if s.active_comp_id() == Some(cid) { s.time() } else { comp.poster_time }; it.settings.time_span = TimeSpan::Custom { start: t, end: t + comp.frame_duration() }; - it.output = OutputModule::for_format(OutputFormat::PngSequence); - it.output.output = "[compName]_proxy_[#####].png".into(); + it.output = templates.default_output_module(slot); + if !it.output.format.is_sequence() { + it.output = OutputModule::for_format(OutputFormat::PngSequence); + } + it.output.output = "[compName]_proxy_[#####].[fileExtension]".into(); } else { it.settings.time_span = TimeSpan::LengthOfComp; - it.output = OutputModule::for_format(OutputFormat::ProRes); - it.output.prores_profile = ProResProfile::P4444; + it.output = templates.default_output_module(slot); + if !it.output.format.is_movie() { + it.output = OutputModule::for_format(OutputFormat::ProRes); + it.output.prores_profile = ProResProfile::P4444; + } it.output.output = "[compName]_proxy.[fileExtension]".into(); } it.output.channels = Channels::Rgba; diff --git a/crates/engine/src/commands/render_queue.rs b/crates/engine/src/commands/render_queue.rs index 91467ea5..fcf088ac 100644 --- a/crates/engine/src/commands/render_queue.rs +++ b/crates/engine/src/commands/render_queue.rs @@ -1,16 +1,20 @@ //! Render Queue: Composition ▸ Add to Render Queue, the queue's Render Settings / Output Module / -//! Output To, Render and Stop. +//! Output To / Log, Render and Stop, Notify, storage overflow folders, and the Render Settings / +//! Output Module templates (Edit ▸ Templates). //! //! Items are addressed by `item` (the stable id from `renderQueue.add` / `renderQueue.list`) or //! `index` (1-based, the # column). use effectcraft_project::render_queue::{ - AudioOutput, Channels, OutputFormat, OutputModule, PostRenderAction, ProResProfile, RenderQuality, RenderQueueItem, RenderSettings, RenderStatus, TimeSpan, + AlphaMode, AudioFormat, AudioOutput, Channels, ColorDepth, CurrentOrOff, DEFAULT_SEQUENCE_TEMPLATE, DEFAULT_TEMPLATE, EffectsMode, FieldRender, + OutputFormat, OutputModule, PostRenderAction, ProResProfile, Pulldown, RESIZE_PRESETS, RenderLog, RenderQuality, RenderQueueItem, RenderSettings, + RenderStatus, ResizeQuality, SwitchOverride, TimeSpan, post_render_parse, }; +use effectcraft_project::render_templates::{RenderTemplates, TemplateKind, TemplateSlot}; use effectcraft_time::{FrameRate, Tick}; use serde_json::{Value, json}; -use super::{CommandSpec, b_p, bad, comp_id, f_p, has_comp, str_p}; +use super::{CommandSpec, always, b_p, bad, comp_id, f_p, has_comp, str_p}; use crate::{EngineError, Result, Session, cmd, query}; fn can_add(s: &Session) -> std::result::Result<(), String> { @@ -54,16 +58,32 @@ fn time_p(p: &Value, k: &str) -> Option { f_p(p, k).map(Tick::from_seconds_f64) } -/// Apply Render Settings parameters. -fn apply_settings(rs: &mut RenderSettings, p: &Value, cmd: &str) -> Result { +/// Parse an enum parameter. +fn enum_p(p: &Value, k: &str, cmd: &str, parse: fn(&str) -> Option, help: &str) -> Result> { + match p.get(k) { + None | Some(Value::Null) => Ok(None), + Some(Value::String(v)) => parse(v).map(Some).ok_or_else(|| bad(cmd, format!("{k}: {help}"))), + Some(Value::Bool(b)) => parse(if *b { "on" } else { "off" }).map(Some).ok_or_else(|| bad(cmd, format!("{k}: {help}"))), + Some(Value::Number(n)) => parse(&n.to_string()).map(Some).ok_or_else(|| bad(cmd, format!("{k}: {help}"))), + Some(_) => Err(bad(cmd, format!("{k}: {help}"))), + } +} + +/// Apply Render Settings parameters (`template` first, then the individual settings). Any +/// individual change makes the settings "Custom". +fn apply_settings(rs: &mut RenderSettings, templates: &RenderTemplates, p: &Value, cmd: &str) -> Result { let mut any = false; + if let Some(t) = str_p(p, "template") { + *rs = templates.render_settings(t).ok_or_else(|| bad(cmd, format!("no Render Settings template “{t}”")))?; + any = true; + } + let before = rs.clone(); if let Some(q) = str_p(p, "quality") { rs.quality = match q.to_ascii_lowercase().as_str() { "best" => RenderQuality::Best, "draft" => RenderQuality::Draft, _ => return Err(bad(cmd, "quality: best|draft")), }; - any = true; } match p.get("resolution") { Some(Value::String(r)) => { @@ -74,15 +94,21 @@ fn apply_settings(rs: &mut RenderSettings, p: &Value, cmd: &str) -> Result "quarter" => 0.25, _ => return Err(bad(cmd, "resolution: full|half|third|quarter|")), }; - any = true; } Some(Value::Number(n)) => { rs.resolution = n.as_f64().unwrap_or(1.0).clamp(0.01, 4.0); - any = true; } _ => {} } - let (start, end) = (time_p(p, "start"), time_p(p, "end")); + let (start, mut end) = (time_p(p, "start"), time_p(p, "end")); + if let Some(d) = time_p(p, "duration") { + // Time Span ▸ Custom ▸ Duration: from the given (or current custom) start. + let a = start.unwrap_or(match rs.time_span { + TimeSpan::Custom { start, .. } => start, + _ => Tick::ZERO, + }); + end = Some(a + d); + } if let Some(ts) = str_p(p, "timeSpan") { rs.time_span = match ts.to_ascii_lowercase().replace([' ', '_', '-'], "").as_str() { "workarea" | "workareaonly" => TimeSpan::WorkArea, @@ -90,14 +116,12 @@ fn apply_settings(rs: &mut RenderSettings, p: &Value, cmd: &str) -> Result "custom" => TimeSpan::Custom { start: start.unwrap_or(Tick::ZERO), end: end.unwrap_or(Tick::from_seconds_f64(1.0)) }, _ => return Err(bad(cmd, "timeSpan: workArea|comp|custom")), }; - any = true; } else if start.is_some() || end.is_some() { let (a, b) = match rs.time_span { TimeSpan::Custom { start, end } => (start, end), _ => (Tick::ZERO, Tick(i64::MAX / 4)), }; rs.time_span = TimeSpan::Custom { start: start.unwrap_or(a), end: end.unwrap_or(b) }; - any = true; } if let TimeSpan::Custom { start, end } = rs.time_span && end <= start @@ -106,61 +130,193 @@ fn apply_settings(rs: &mut RenderSettings, p: &Value, cmd: &str) -> Result } if let Some(u) = str_p(p, "proxyUse") { rs.proxy_use = effectcraft_project::render_queue::ProxyUse::parse(u).ok_or_else(|| bad(cmd, "proxyUse: current|all|comp|none"))?; - any = true; } match p.get("frameRate") { Some(Value::Null) => { rs.frame_rate = None; - any = true; } Some(v) => { let f = v.as_f64().filter(|f| *f > 0.0 && *f <= 1000.0).ok_or_else(|| bad(cmd, "frameRate: fps > 0 or null (comp rate)"))?; rs.frame_rate = Some(FrameRate::from_f64(f)); - any = true; } None => {} } - if let Some(b) = b_p(p, "motionBlur") { - rs.motion_blur = b; - any = true; + match p.get("motionBlur") { + Some(Value::Bool(b)) => { + rs.motion_blur = *b; + if *b && rs.motion_blur_mode == SwitchOverride::OffForAll { + rs.motion_blur_mode = SwitchOverride::Current; + } + } + Some(_) => { + let m = enum_p(p, "motionBlur", cmd, SwitchOverride::parse, "current|onForChecked|offForAll")?.unwrap_or_default(); + rs.motion_blur = m != SwitchOverride::OffForAll; + rs.motion_blur_mode = m; + } + None => {} + } + if let Some(m) = enum_p(p, "frameBlending", cmd, SwitchOverride::parse, "current|onForChecked|offForAll")? { + rs.frame_blending = m; + } + if let Some(f) = enum_p(p, "fieldRender", cmd, FieldRender::parse, "off|upper|lower")? { + rs.field_render = f; + } + if let Some(f) = enum_p(p, "pulldown", cmd, Pulldown::parse, "off|WSSWW|SSWWW|SWWWS|WWWSS|WWSSW")? { + rs.pulldown = f; + } + if let Some(e) = enum_p(p, "effects", cmd, EffectsMode::parse, "current|allOn|allOff")? { + rs.effects = e; + } + if let Some(e) = enum_p(p, "solo", cmd, CurrentOrOff::parse, "current|allOff")? { + rs.solo = e; + } + if let Some(e) = enum_p(p, "guideLayers", cmd, CurrentOrOff::parse, "current|allOff")? { + rs.guide_layers = e; + } + if let Some(e) = enum_p(p, "colorDepth", cmd, ColorDepth::parse, "current|8|16|32")? { + rs.color_depth = e; } if let Some(b) = b_p(p, "skipExisting") { rs.skip_existing = b; + } + if let Some(b) = b_p(p, "storageOverflow") { + rs.storage_overflow = b; + } + if *rs != before { + rs.name = "Custom".into(); any = true; } Ok(any) } -/// Apply Output Module parameters. -fn apply_output(om: &mut OutputModule, p: &Value, cmd: &str) -> Result { +/// Apply a template to a module, keeping its output path (with the extension fixed). +fn apply_om_template(om: &mut OutputModule, t: &OutputModule) { + let keep = om.output.clone(); + let old = om.format; + *om = t.clone(); + om.output = keep; + let f = om.format; + om.format = old; + om.set_format(f); + om.channels = t.channels; + om.name = t.name.clone(); +} + +/// Apply Output Module parameters (`template` first). `roi`: the viewer's region of interest +/// (captured by `crop.useRoi`). +fn apply_output(om: &mut OutputModule, templates: &RenderTemplates, roi: Option<[f64; 4]>, p: &Value, cmd: &str) -> Result { let mut any = false; + if let Some(t) = str_p(p, "template") { + let t = templates.output_module(t).ok_or_else(|| bad(cmd, format!("no Output Module template “{t}”")))?; + apply_om_template(om, &t); + any = true; + } + let before = om.clone(); if let Some(f) = str_p(p, "format") { let f = OutputFormat::from_name(f).ok_or_else(|| bad(cmd, "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; om.set_format(f); - any = true; } if let Some(c) = str_p(p, "channels") { om.channels = match c.to_ascii_lowercase().replace([' ', '+'], "").as_str() { "rgb" => Channels::Rgb, "rgba" | "rgbalpha" => Channels::Rgba, - _ => return Err(bad(cmd, "channels: rgb|rgba")), + "alpha" | "alphaonly" => Channels::Alpha, + _ => return Err(bad(cmd, "channels: rgb|rgba|alpha")), }; if om.channels == Channels::Rgba && !om.format.supports_alpha() { return Err(bad(cmd, format!("{} has no alpha channel", om.format.label()))); } - any = true; + } + if let Some(c) = str_p(p, "color").or(str_p(p, "alphaMode")) { + om.alpha_mode = match c.to_ascii_lowercase().replace([' ', '(', ')'], "").as_str() { + "straight" | "straightunmatted" => AlphaMode::Straight, + "premultiplied" | "premultipliedmatted" | "premul" => AlphaMode::Premultiplied, + _ => return Err(bad(cmd, "color: straight|premultiplied")), + }; } if let Some(q) = f_p(p, "quality") { om.quality = q.clamp(1.0, 100.0) as u8; - any = true; } if let Some(b) = f_p(p, "bitrate") { - om.bitrate_kbps = b.clamp(100.0, 500_000.0) as u32; - any = true; + om.bitrate_kbps = b.clamp(50.0, 500_000.0) as u32; + } + if let Some(b) = b_p(p, "webmBitrate") { + om.webm_bitrate = b; + } + if let Some(k) = p.get("keyframeInterval").and_then(Value::as_u64) { + om.keyframe_interval = k.min(100_000) as u32; } if let Some(pp) = str_p(p, "proresProfile") { om.prores_profile = ProResProfile::from_name(pp).ok_or_else(|| bad(cmd, "proresProfile: proxy|lt|standard|hq|4444|4444xq"))?; - any = true; + } + match p.get("crop") { + Some(Value::Bool(b)) => om.crop.enabled = *b, + Some(Value::Object(c)) => { + om.crop.enabled = c.get("enabled").and_then(Value::as_bool).unwrap_or(true); + for (k, v) in [("top", &mut om.crop.top), ("left", &mut om.crop.left), ("bottom", &mut om.crop.bottom), ("right", &mut om.crop.right)] { + if let Some(n) = c.get(k).and_then(Value::as_i64) { + *v = n.clamp(-20_000, 20_000) as i32; + } + } + if let Some(u) = c.get("useRoi").and_then(Value::as_bool) { + om.crop.use_roi = u; + if u { + om.crop.roi = c.get("roi").and_then(|r| serde_json::from_value::<[f64; 4]>(r.clone()).ok()).or(roi); + if om.crop.roi.is_none() { + return Err(bad( + cmd, + "Use Region of Interest: the composition has no region of interest (View ▸ Region of Interest) — pass crop.roi [x, y, w, h]", + )); + } + } + } + } + Some(Value::Null) | None => {} + Some(_) => return Err(bad(cmd, "crop: bool or {enabled?, useRoi?, roi?: [x,y,w,h], top?, left?, bottom?, right?}")), + } + // Flat keys (dialog forms): cropTop/cropLeft/cropBottom/cropRight, resizeWidth/resizeHeight. + for (k, v) in [("cropTop", &mut om.crop.top), ("cropLeft", &mut om.crop.left), ("cropBottom", &mut om.crop.bottom), ("cropRight", &mut om.crop.right)] { + if let Some(n) = p.get(k).and_then(Value::as_f64) { + *v = (n.round() as i64).clamp(-20_000, 20_000) as i32; + om.crop.enabled = true; + om.crop.use_roi = false; + } + } + for (k, v) in [("resizeWidth", &mut om.resize.width), ("resizeHeight", &mut om.resize.height)] { + if let Some(n) = p.get(k).and_then(Value::as_f64) { + *v = (n.round() as u32).clamp(1, 30_000); + om.resize.enabled = true; + } + } + match p.get("resize") { + Some(Value::Bool(b)) => om.resize.enabled = *b, + Some(Value::Object(r)) => { + om.resize.enabled = r.get("enabled").and_then(Value::as_bool).unwrap_or(true); + if let Some(name) = r.get("preset").and_then(Value::as_str) { + let (_, w, h) = + RESIZE_PRESETS.iter().find(|(n, _, _)| n.eq_ignore_ascii_case(name)).ok_or_else(|| bad(cmd, format!("no resize preset “{name}”")))?; + (om.resize.width, om.resize.height) = (*w, *h); + om.resize.lock_aspect = false; + } + if let Some(l) = r.get("lockAspect").and_then(Value::as_bool) { + om.resize.lock_aspect = l; + } + if let Some(w) = r.get("width").and_then(Value::as_u64) { + om.resize.width = (w as u32).clamp(1, 30_000); + } + if let Some(h) = r.get("height").and_then(Value::as_u64) { + om.resize.height = (h as u32).clamp(1, 30_000); + } + if let Some(q) = r.get("quality").and_then(Value::as_str) { + om.resize.quality = match q.to_ascii_lowercase().as_str() { + "low" | "bilinear" => ResizeQuality::Low, + "high" | "bicubic" => ResizeQuality::High, + _ => return Err(bad(cmd, "resize.quality: low|high")), + }; + } + } + Some(Value::Null) | None => {} + Some(_) => return Err(bad(cmd, "resize: bool or {enabled?, preset?, width?, height?, lockAspect?, quality?: low|high}")), } if let Some(a) = p.get("audio") { om.audio = match a { @@ -174,15 +330,29 @@ fn apply_output(om: &mut OutputModule, p: &Value, cmd: &str) -> Result { }, _ => return Err(bad(cmd, "audio: auto|on|off")), }; - any = true; } if let Some(r) = p.get("sampleRate").and_then(Value::as_u64) { om.audio_sample_rate = (r as u32).clamp(8_000, 192_000); - any = true; + } + match p.get("audioChannels") { + Some(Value::Number(n)) => om.audio_channels = if n.as_u64() == Some(1) { 1 } else { 2 }, + Some(Value::String(c)) => { + om.audio_channels = match c.to_ascii_lowercase().as_str() { + "mono" | "1" => 1, + "stereo" | "2" => 2, + _ => return Err(bad(cmd, "audioChannels: mono|stereo")), + } + } + _ => {} + } + if let Some(f) = enum_p(p, "audioFormat", cmd, AudioFormat::parse, "16|24|32 (float)")? { + om.audio_format = f; } if let Some(l) = b_p(p, "loop") { om.gif_loop = l; - any = true; + } + if let Some(l) = b_p(p, "includeProjectLink") { + om.include_project_link = l; } if let Some(o) = str_p(p, "output").or(str_p(p, "path")) { om.output = o.to_string(); @@ -194,6 +364,14 @@ fn apply_output(om: &mut OutputModule, p: &Value, cmd: &str) -> Result { om.set_format(f); om.output = keep; } + } + if *om != before { + // Output To alone doesn't make the module custom. + let mut o = om.clone(); + o.output = before.output.clone(); + if o != before { + om.name = "Custom".into(); + } any = true; } Ok(any) @@ -227,7 +405,7 @@ fn requeue(it: &mut RenderQueueItem) { fn item_json(s: &Session, it: &RenderQueueItem, index: usize) -> Value { let comp = s.project.comp(it.comp); let name = s.project.item(it.comp).map(|i| i.name.clone()); - let (w, h) = comp.map(|c| it.settings.output_size(c)).map(|(w, h)| it.output.format.coded_size(w, h)).unwrap_or((0, 0)); + let (w, h) = comp.map(|c| it.output.output_size(c, &it.settings)).unwrap_or((0, 0)); let frames = comp.map(|c| it.settings.frame_count(c)).unwrap_or(0); let mut v = serde_json::to_value(it).unwrap_or(Value::Null); if let Some(o) = v.as_object_mut() { @@ -249,6 +427,10 @@ fn item_json(s: &Session, it: &RenderQueueItem, index: usize) -> Value { o.insert("outputModules".into(), json!(it.extra_outputs.len() + 1)); o.insert("extraOutputs".into(), json!(extra)); o.insert("postRenderAction".into(), json!(it.post_render.label())); + o.insert("logLabel".into(), json!(it.log.label())); + if let Some(p) = s.resolve_output(it) { + o.insert("logPath".into(), json!(effectcraft_project::render_queue::log_path(&p))); + } o.insert("width".into(), json!(w)); o.insert("height".into(), json!(h)); o.insert("frames".into(), json!(frames)); @@ -259,16 +441,25 @@ fn item_json(s: &Session, it: &RenderQueueItem, index: usize) -> Value { fn add(s: &mut Session, p: &Value) -> Result { let cid = s.comp_for_queue(p).ok_or(EngineError::NoComp)?; let mut it = RenderQueueItem::new(0, cid); + // New items start from the Movie Default templates (Edit ▸ Templates). + let templates = s.project.render_templates.clone(); + it.settings = templates.default_render_settings(TemplateSlot::Movie); + it.output = templates.default_output_module(TemplateSlot::Movie); if let Some(f) = str_p(p, "format") { let f = OutputFormat::from_name(f).ok_or_else(|| bad("renderQueue.add", "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; - it.output = OutputModule::for_format(f); + if f != it.output.format { + it.output = OutputModule::for_format(f); + } } - apply_settings(&mut it.settings, p, "renderQueue.add")?; + apply_settings(&mut it.settings, &templates, p, "renderQueue.add")?; let mut op = p.clone(); if let Some(o) = op.as_object_mut() { o.remove("format"); } - apply_output(&mut it.output, &op, "renderQueue.add")?; + apply_output(&mut it.output, &templates, s.state.region_of_interest, &op, "renderQueue.add")?; + if let Some(l) = enum_p(p, "log", "renderQueue.add", RenderLog::parse, "errorsOnly|plusSettings|plusPerFrameInfo")? { + it.log = l; + } let id = s.edit("Add to Render Queue", None, |proj, _| { it.id = proj.render_queue.iter().map(|i| i.id).max().unwrap_or(0) + 1; let id = it.id; @@ -305,7 +496,7 @@ fn set_render(s: &mut Session, p: &Value) -> Result { fn set_render_settings(s: &mut Session, p: &Value) -> Result { let i = item_index(s, p, "renderQueue.setRenderSettings")?; let mut rs = s.project.render_queue[i].settings.clone(); - if !apply_settings(&mut rs, p, "renderQueue.setRenderSettings")? { + if !apply_settings(&mut rs, &s.project.render_templates, p, "renderQueue.setRenderSettings")? { return Err(bad("renderQueue.setRenderSettings", "nothing to change")); } s.edit("Render Settings", None, |proj, _| { @@ -321,12 +512,20 @@ fn set_output_module(s: &mut Session, p: &Value) -> Result { let i = item_index(s, p, "renderQueue.setOutputModule")?; let m = module_p(s, i, p, "renderQueue.setOutputModule")?; let mut om = module_ref(&s.project.render_queue[i], m).clone(); - if !apply_output(&mut om, p, "renderQueue.setOutputModule")? { + let changed = apply_output(&mut om, &s.project.render_templates, s.state.region_of_interest, p, "renderQueue.setOutputModule")?; + let post = match str_p(p, "postRenderAction") { + Some(a) => Some(post_render_parse(a).ok_or_else(|| bad("renderQueue.setOutputModule", "postRenderAction: none|import|importAndReplace|setProxy"))?), + None => None, + }; + if !changed && post.is_none() { return Err(bad("renderQueue.setOutputModule", "nothing to change")); } s.edit("Output Module Settings", None, |proj, _| { let it = &mut proj.render_queue[i]; *module_mut(it, m) = om; + if let Some(a) = post { + it.post_render = a; + } requeue(it); Ok(()) })?; @@ -338,7 +537,7 @@ fn set_output(s: &mut Session, p: &Value) -> Result { let path = str_p(p, "path").or(str_p(p, "output")).ok_or_else(|| bad("renderQueue.setOutput", "pass `path` (file path or template)"))?; let m = module_p(s, i, p, "renderQueue.setOutput")?; let mut om = module_ref(&s.project.render_queue[i], m).clone(); - apply_output(&mut om, &json!({"output": path}), "renderQueue.setOutput")?; + apply_output(&mut om, &s.project.render_templates, None, &json!({"output": path}), "renderQueue.setOutput")?; s.edit("Output To", None, |proj, _| { let it = &mut proj.render_queue[i]; *module_mut(it, m) = om; @@ -452,7 +651,7 @@ fn add_output_module(s: &mut Session, p: &Value) -> Result { if let Some(o) = op.as_object_mut() { o.remove("format"); } - apply_output(&mut om, &op, "render.addOutputModule")?; + apply_output(&mut om, &s.project.render_templates, s.state.region_of_interest, &op, "render.addOutputModule")?; s.edit("Add Output Module", None, |proj, _| { let it = &mut proj.render_queue[i]; it.extra_outputs.push(om); @@ -469,11 +668,11 @@ fn add_output_module(s: &mut Session, p: &Value) -> Result { fn pre_render(s: &mut Session, p: &Value) -> Result { let cid = s.comp_for_queue(p).ok_or(EngineError::NoComp)?; let mut it = RenderQueueItem::new(0, cid); - it.output = OutputModule::for_format(OutputFormat::ProRes); - it.output.prores_profile = ProResProfile::P4444; - it.output.channels = Channels::Rgba; + let templates = s.project.render_templates.clone(); + it.settings = templates.default_render_settings(TemplateSlot::PreRender); + it.output = templates.default_output_module(TemplateSlot::PreRender); it.output.output = "[compName]_prerender.[fileExtension]".into(); - apply_output(&mut it.output, p, "render.preRender")?; + apply_output(&mut it.output, &templates, s.state.region_of_interest, p, "render.preRender")?; it.post_render = PostRenderAction::ImportAndReplace; let id = s.edit("Pre-render", None, |proj, _| { it.id = proj.render_queue.iter().map(|i| i.id).max().unwrap_or(0) + 1; @@ -499,13 +698,202 @@ fn save_current_preview(s: &mut Session, p: &Value) -> Result { let mut item = RenderQueueItem::new(0, cid); item.output = OutputModule::for_format(format); item.output.output = path.clone(); - let job = - crate::ExportJob { project: &s.project, footage: s.footage.as_ref(), expr: s.expr.as_deref(), accel: s.accel.as_deref(), item: &item, path: &path }; + let job = crate::ExportJob { + project: &s.project, + footage: s.footage.as_ref(), + expr: s.expr.as_deref(), + accel: s.accel.as_deref(), + item: &item, + path: &path, + storage: s.storage_quota.as_deref(), + label: "Save Current Preview".into(), + }; let r = exporter.export(&job, &mut |_, _| true).map_err(EngineError::Other)?; s.toast(format!("Saved preview to {}", r.path)); Ok(serde_json::to_value(&r).unwrap_or_default()) } +// ---------------------------------------------------------------- templates (Edit ▸ Templates) + +fn kind_p(p: &Value, cmd: &str) -> Result { + str_p(p, "kind").and_then(TemplateKind::parse).ok_or_else(|| bad(cmd, "kind: renderSettings|outputModule")) +} + +fn name_p<'a>(p: &'a Value, cmd: &str) -> Result<&'a str> { + str_p(p, "name").map(str::trim).filter(|n| !n.is_empty()).ok_or_else(|| bad(cmd, "pass `name`")) +} + +/// Templates as JSON (lists, built-in flags, defaults per slot). +fn templates_json(t: &RenderTemplates) -> Value { + let rs: Vec = t + .render_settings_list() + .iter() + .map(|r| json!({"name": r.name, "builtin": RenderTemplates::is_builtin(TemplateKind::RenderSettings, &r.name), "settings": r})) + .collect(); + let om: Vec = t + .output_module_list() + .iter() + .map(|m| json!({"name": m.name, "builtin": RenderTemplates::is_builtin(TemplateKind::OutputModule, &m.name), "summary": format!("{} · {}", m.format.label(), m.channels.label()), "module": m})) + .collect(); + let slots = |k: TemplateKind| -> Value { + TemplateSlot::ALL.iter().map(|sl| (format!("{sl:?}"), json!(t.default_name(k, *sl)))).collect::>().into() + }; + json!({ + "renderSettings": rs, + "outputModules": om, + "defaults": {"renderSettings": slots(TemplateKind::RenderSettings), "outputModules": slots(TemplateKind::OutputModule)}, + "slots": TemplateSlot::ALL.iter().map(|sl| json!({"id": format!("{sl:?}"), "label": sl.label()})).collect::>(), + "resizePresets": RESIZE_PRESETS.iter().map(|(n, w, h)| json!({"name": n, "width": w, "height": h})).collect::>(), + }) +} + +fn templates(s: &mut Session, _: &Value) -> Result { + Ok(templates_json(&s.project.render_templates)) +} + +/// Save a template: from an item's settings (`item`/`index`, `module`), or the slot default, +/// adjusted by `params` (the same keys as setRenderSettings / setOutputModule). +fn save_template(s: &mut Session, p: &Value) -> Result { + const C: &str = "renderQueue.saveTemplate"; + let kind = kind_p(p, C)?; + let name = name_p(p, C)?.to_string(); + if name.eq_ignore_ascii_case("custom") { + return Err(bad(C, "“Custom” is reserved")); + } + let item = if p.get("item").is_some() || p.get("index").is_some() { Some(item_index(s, p, C)?) } else { None }; + let params = p.get("params").cloned().unwrap_or(Value::Null); + let from = str_p(p, "from"); + let mut t = s.project.render_templates.clone(); + match kind { + TemplateKind::RenderSettings => { + let mut rs = match (item, from) { + (Some(i), _) => s.project.render_queue[i].settings.clone(), + (None, Some(f)) => t.render_settings(f).ok_or_else(|| bad(C, format!("no Render Settings template “{f}”")))?, + (None, None) => t.default_render_settings(TemplateSlot::Movie), + }; + if params.is_object() { + apply_settings(&mut rs, &t, ¶ms, C)?; + } + t.save_render_settings(&name, rs); + } + TemplateKind::OutputModule => { + let mut om = match (item, from) { + (Some(i), _) => module_ref(&s.project.render_queue[i], module_p(s, i, p, C)?).clone(), + (None, Some(f)) => t.output_module(f).ok_or_else(|| bad(C, format!("no Output Module template “{f}”")))?, + (None, None) => t.default_output_module(TemplateSlot::Movie), + }; + if params.is_object() { + apply_output(&mut om, &t, s.state.region_of_interest, ¶ms, C)?; + } + // Templates name files after the composition, not a particular path. + om.output = if om.format.is_sequence() { DEFAULT_SEQUENCE_TEMPLATE } else { DEFAULT_TEMPLATE }.into(); + t.save_output_module(&name, om); + } + } + s.edit("Save Template", None, |proj, _| { + proj.render_templates = t; + Ok(()) + })?; + Ok(json!({"saved": name, "templates": templates_json(&s.project.render_templates)})) +} + +fn delete_template(s: &mut Session, p: &Value) -> Result { + const C: &str = "renderQueue.deleteTemplate"; + let kind = kind_p(p, C)?; + let name = name_p(p, C)?.to_string(); + let mut t = s.project.render_templates.clone(); + t.delete(kind, &name).map_err(|e| bad(C, e))?; + s.edit("Delete Template", None, |proj, _| { + proj.render_templates = t; + Ok(()) + })?; + Ok(json!({"deleted": name})) +} + +fn set_template_default(s: &mut Session, p: &Value) -> Result { + const C: &str = "renderQueue.setTemplateDefault"; + let kind = kind_p(p, C)?; + let slot = str_p(p, "slot").and_then(TemplateSlot::parse).ok_or_else(|| bad(C, "slot: movie|still|preRender|movieProxy|stillProxy"))?; + let name = name_p(p, C)?.to_string(); + let mut t = s.project.render_templates.clone(); + t.set_default(kind, slot, &name).map_err(|e| bad(C, e))?; + s.edit("Template Default", None, |proj, _| { + proj.render_templates = t; + Ok(()) + })?; + Ok(json!({"slot": format!("{slot:?}"), "name": name})) +} + +/// Apply templates to an item: `renderSettings` and/or `outputModule` (to `module`). +fn apply_template(s: &mut Session, p: &Value) -> Result { + const C: &str = "renderQueue.applyTemplate"; + let i = item_index(s, p, C)?; + let m = module_p(s, i, p, C)?; + let t = s.project.render_templates.clone(); + let rs = match str_p(p, "renderSettings") { + Some(n) => Some(t.render_settings(n).ok_or_else(|| bad(C, format!("no Render Settings template “{n}”")))?), + None => None, + }; + let om = match str_p(p, "outputModule") { + Some(n) => { + let tpl = t.output_module(n).ok_or_else(|| bad(C, format!("no Output Module template “{n}”")))?; + let mut om = module_ref(&s.project.render_queue[i], m).clone(); + apply_om_template(&mut om, &tpl); + Some(om) + } + None => None, + }; + if rs.is_none() && om.is_none() { + return Err(bad(C, "pass `renderSettings` and/or `outputModule` (template names)")); + } + s.edit("Apply Template", None, |proj, _| { + let it = &mut proj.render_queue[i]; + if let Some(rs) = rs { + it.settings = rs; + } + if let Some(om) = om { + *module_mut(it, m) = om; + } + requeue(it); + Ok(()) + })?; + Ok(item_json(s, &s.project.render_queue[i], i)) +} + +fn set_log(s: &mut Session, p: &Value) -> Result { + const C: &str = "renderQueue.setLog"; + let i = item_index(s, p, C)?; + let log = enum_p(p, "log", C, RenderLog::parse, "errorsOnly|plusSettings|plusPerFrameInfo")?.ok_or_else(|| bad(C, "pass `log`"))?; + s.edit("Render Queue: Log", None, |proj, _| { + proj.render_queue[i].log = log; + Ok(()) + })?; + Ok(item_json(s, &s.project.render_queue[i], i)) +} + +fn set_notify(s: &mut Session, p: &Value) -> Result { + let on = b_p(p, "notify").or(b_p(p, "value")).unwrap_or(!s.project.render_prefs.notify); + s.edit("Render Queue: Notify", None, |proj, _| { + proj.render_prefs.notify = on; + Ok(()) + })?; + Ok(json!({"notify": on})) +} + +fn set_overflow(s: &mut Session, p: &Value) -> Result { + const C: &str = "renderQueue.setOverflowFolders"; + let folders: Vec = match p.get("folders") { + Some(Value::Array(a)) => a.iter().filter_map(Value::as_str).map(str::trim).filter(|f| !f.is_empty()).map(String::from).collect(), + Some(Value::Null) | None => vec![], + _ => return Err(bad(C, "folders: [path, …]")), + }; + s.edit("Storage Overflow Folders", None, |proj, _| { + proj.render_prefs.overflow_folders = folders.clone(); + Ok(()) + })?; + Ok(json!({"folders": s.project.render_prefs.overflow_folders})) +} + pub fn specs() -> Vec { vec![ cmd!( @@ -513,7 +901,7 @@ pub fn specs() -> Vec { "Add to Render Queue", ["Composition"], Some("Cmd+M"), - "{comp?: id|name, format?: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff, output?: path|template, quality?: best|draft|1-100 (jpeg, webm), resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, proxyUse?: current|all|comp|none, channels?: rgb|rgba, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool}", + "{comp?: id|name, template?: Render Settings template, format?: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff, output?: path|template, log?: errorsOnly|plusSettings|plusPerFrameInfo, quality?: best|draft|1-100 (jpeg, webm), resolution?: full|half|third|quarter|scale, proxyUse?, effects?, solo?, guideLayers?, colorDepth?, frameBlending?, fieldRender?, pulldown?, motionBlur?, timeSpan?: workArea|comp|custom, start?: s, end?: s, duration?: s, frameRate?: fps|null, skipExisting?: bool, storageOverflow?: bool, channels?: rgb|rgba|alpha, color?, bitrate?: kbps, webmBitrate?, keyframeInterval?, proresProfile?, crop?, resize?, includeProjectLink?, audio?: auto|on|off, sampleRate?, audioChannels?, audioFormat?, loop?: bool (values as in setRenderSettings / setOutputModule)}", can_add, add ), @@ -524,7 +912,7 @@ pub fn specs() -> Vec { "Render Settings...", [], None, - "{item?|index?, quality?: best|draft, resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, proxyUse?: current|all|comp|none}", + "{item?|index?, template?: name, quality?: best|draft, resolution?: full|half|third|quarter|scale, proxyUse?: current|all|comp|none, effects?: current|allOn|allOff, solo?: current|allOff, guideLayers?: current|allOff, colorDepth?: current|8|16|32, frameBlending?: current|onForChecked|offForAll, fieldRender?: off|upper|lower, pulldown?: off|WSSWW|SSWWW|SWWWS|WWWSS|WWSSW, motionBlur?: bool|current|onForChecked|offForAll, timeSpan?: workArea|comp|custom, start?: s, end?: s, duration?: s, frameRate?: fps|null, skipExisting?: bool, storageOverflow?: bool}", has_items, set_render_settings ), @@ -533,7 +921,7 @@ pub fn specs() -> Vec { "Output Module Settings...", [], None, - "{item?|index?, module?: n (1 = first), format?, channels?: rgb|rgba, quality?: 1-100, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool, output?}", + "{item?|index?, module?: n (1 = first), template?: name, format?, channels?: rgb|rgba|alpha, color?: straight|premultiplied, quality?: 1-100, bitrate?: kbps, webmBitrate?: bool, keyframeInterval?: frames, proresProfile?: proxy|lt|standard|hq|4444|4444xq, crop?: bool|{useRoi?, roi?, top?, left?, bottom?, right?}, cropTop?, cropLeft?, cropBottom?, cropRight?, resize?: bool|{preset?, width?, height?, lockAspect?, quality?: low|high}, resizeWidth?, resizeHeight?, postRenderAction?: none|import|importAndReplace|setProxy, includeProjectLink?: bool, audio?: auto|on|off, sampleRate?, audioChannels?: mono|stereo, audioFormat?: 16|24|32, loop?: bool, output?}", has_items, set_output_module ), @@ -562,6 +950,46 @@ pub fn specs() -> Vec { cmd!("renderQueue.render", "Render", [], None, "{wait?: bool (default true; the UI renders in the background)}", has_queued, render), cmd!("renderQueue.stop", "Stop Rendering", [], None, "{}", rendering, stop), query!("renderQueue.list", "Render Queue Items", "{}", list), + query!("renderQueue.templates", "Render Templates", "{}", templates), + cmd!( + "renderQueue.saveTemplate", + "Save Template...", + [], + None, + "{kind: renderSettings|outputModule, name, item?|index? (save from this item), from?: template to copy, module?, params?: {setRenderSettings / setOutputModule keys}}", + always, + save_template + ), + cmd!("renderQueue.deleteTemplate", "Delete Template", [], None, "{kind: renderSettings|outputModule, name}", always, delete_template), + cmd!( + "renderQueue.setTemplateDefault", + "Set Template Default", + [], + None, + "{kind: renderSettings|outputModule, slot: movie|still|preRender|movieProxy|stillProxy, name}", + always, + set_template_default + ), + cmd!( + "renderQueue.applyTemplate", + "Apply Template", + [], + None, + "{item?|index?, renderSettings?: template name, outputModule?: template name, module?: n}", + has_items, + apply_template + ), + cmd!("renderQueue.setLog", "Render Queue: Log", [], None, "{item?|index?, log: errorsOnly|plusSettings|plusPerFrameInfo}", has_items, set_log), + cmd!("renderQueue.setNotify", "Notify When Done", [], None, "{notify?: bool (toggles)}", always, set_notify), + cmd!( + "renderQueue.setOverflowFolders", + "Storage Overflow Folders", + [], + None, + "{folders: [path, …] (used in order when the output volume is full)}", + always, + set_overflow + ), query!("renderQueue.formats", "Output Formats", "{}", formats), ] } diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 9333329a..d7a89160 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -275,6 +275,8 @@ pub struct Session { pub importer: Option>, /// Render Queue encoder (the export layer); `None` = export unavailable. pub exporter: Option>, + /// Free-space hook for Render Settings ▸ Use Storage Overflow (`None`: never full). + pub storage_quota: Option>, /// The running (or finished, not yet polled) render. pub render_job: Option, /// The running (or finished, not yet polled) track analysis. @@ -357,6 +359,7 @@ impl Default for Session { expr_check: None, importer: None, exporter: None, + storage_quota: None, render_job: None, track_job: None, mask_job: None, diff --git a/crates/engine/src/render_queue.rs b/crates/engine/src/render_queue.rs index 9dd20e8d..3de9d8e5 100644 --- a/crates/engine/src/render_queue.rs +++ b/crates/engine/src/render_queue.rs @@ -27,6 +27,10 @@ pub struct ExportJob<'a> { pub accel: Option<&'a dyn effectcraft_render::Accelerator>, /// Resolved output path (templates expanded except the `#` frame-number run). pub path: &'a str, + /// Free-space hook (Use Storage Overflow). + pub storage: Option<&'a dyn effectcraft_project::render_queue::StorageQuota>, + /// "#n Comp Name", for the render log. + pub label: String, } /// What an export wrote. @@ -39,6 +43,12 @@ pub struct ExportResult { pub bytes: u64, pub seconds: f64, pub audio: bool, + /// The render log written next to the output, if any. + #[serde(skip_serializing_if = "Option::is_none")] + pub log: Option, + /// Files that went to storage overflow folders. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub overflow: Vec, } /// Encodes render queue items (implemented by the export layer). @@ -127,6 +137,7 @@ struct Work { expr: Option>, accel: Option>, exporter: Arc, + storage: Option>, /// Per queue item: one (item with that output module, resolved path) per output module. items: Vec>, } @@ -143,7 +154,15 @@ pub(crate) fn run_items( shared: &JobShared, notify: &mut dyn FnMut(&JobShared), ) { - let work = Work { project: s.project.clone(), footage: s.footage.clone(), expr: s.expr.clone(), accel: s.accel.clone(), exporter, items }; + let work = Work { + project: s.project.clone(), + footage: s.footage.clone(), + expr: s.expr.clone(), + accel: s.accel.clone(), + exporter, + storage: s.storage_quota.clone(), + items, + }; run_work(work, shared, notify); } @@ -174,7 +193,18 @@ fn run_work(w: Work, shared: &JobShared, notify: &mut dyn FnMut(&JobShared)) { // first module's file is the item's output. let mut r: Result = Err("no output module".into()); for (mi, (mitem, path)) in modules.iter().enumerate() { - let job = ExportJob { project: &w.project, footage: w.footage.as_ref(), expr: w.expr.as_deref(), accel: w.accel.as_deref(), item: mitem, path }; + let name = w.project.item(mitem.comp).map(|i| i.name.clone()).unwrap_or_default(); + let index = w.project.render_queue.iter().position(|i| i.id == mitem.id).map_or(0, |i| i + 1); + let job = ExportJob { + project: &w.project, + footage: w.footage.as_ref(), + expr: w.expr.as_deref(), + accel: w.accel.as_deref(), + item: mitem, + path, + storage: w.storage.as_deref(), + label: format!("#{index} {name} (output module {})", mi + 1), + }; let rr = w.exporter.export(&job, &mut |done, total| { { let mut s = lock(&shared.state); @@ -235,8 +265,7 @@ impl Session { if tpl.is_empty() { return None; } - let (w, h) = item.settings.output_size(comp); - let (w, h) = item.output.format.coded_size(w, h); + let (w, h) = item.output.output_size(comp, &item.settings); let rate = item.settings.rate(comp); let n = item.settings.frame_count(comp) as i64; let f0 = item.settings.first_frame(comp); @@ -329,7 +358,15 @@ impl Session { self.poll_render(); return Ok(ids); } - let work = Work { project: self.project.clone(), footage: self.footage.clone(), expr: self.expr.clone(), accel: self.accel.clone(), exporter, items }; + let work = Work { + project: self.project.clone(), + footage: self.footage.clone(), + expr: self.expr.clone(), + accel: self.accel.clone(), + exporter, + storage: self.storage_quota.clone(), + items, + }; let shared = Arc::new(JobShared::default()); if wait { run_work(work, &shared, &mut |_| {}); @@ -426,6 +463,10 @@ impl Session { } else { format!("Rendered {} item(s) in {:.1} s", st.items_total, st.elapsed) }; + if self.project.render_prefs.notify { + // Notify: the frontend plays a sound and shows a system notification. + self.events.push(Event::Frontend { command: "renderQueue.notify".into(), params: serde_json::json!({"message": msg, "error": failed > 0}) }); + } self.events.push(Event::Toast { message: msg, error: failed > 0 }); } for (id, path) in post { diff --git a/crates/engine/src/tests_stubs.rs b/crates/engine/src/tests_stubs.rs index 997552bc..18e65cb0 100644 --- a/crates/engine/src/tests_stubs.rs +++ b/crates/engine/src/tests_stubs.rs @@ -312,6 +312,123 @@ fn pre_render_imports_and_replaces() { assert!(matches!(s.project.comp(outer).unwrap().layers[0].source, LayerSource::Comp { .. })); } +#[test] +fn post_render_actions_import_and_set_proxy() { + let (mut s, _) = rq(); + let cid = s.active_comp_id().unwrap(); + let items_before = s.project.items.len(); + // Import: the rendered file becomes a footage item. + let a = s.execute("renderQueue.add", json!({"output": "/tmp/post/a.mov"})).unwrap(); + s.execute("renderQueue.setOutputModule", json!({"item": a["item"], "postRenderAction": "import"})).unwrap(); + assert_eq!(s.project.render_queue[0].post_render, PostRenderAction::Import); + s.execute("renderQueue.render", json!({})).unwrap(); + assert_eq!(s.project.items.len(), items_before + 1); + assert!(s.project.items.values().any(|i| matches!(&i.kind, effectcraft_project::ItemKind::Footage(f) if f.path == "/tmp/post/a.mov"))); + // Set Proxy: the comp gets the render as its proxy. + let b = s.execute("renderQueue.add", json!({"output": "/tmp/post/b.mov"})).unwrap(); + s.execute("renderQueue.setOutputModule", json!({"item": b["item"], "postRenderAction": "Set Proxy"})).unwrap(); + s.execute("renderQueue.render", json!({})).unwrap(); + let px = s.project.item(cid).unwrap().proxy.clone().expect("proxy set"); + assert_eq!(px.footage.path, "/tmp/post/b.mov"); + // None: nothing happens. + let n = s.project.items.len(); + let c = s.execute("renderQueue.add", json!({"output": "/tmp/post/c.mov"})).unwrap(); + s.execute("renderQueue.setOutputModule", json!({"item": c["item"], "postRenderAction": "none"})).unwrap(); + s.execute("renderQueue.render", json!({})).unwrap(); + assert_eq!(s.project.items.len(), n); + assert!(s.execute("renderQueue.setOutputModule", json!({"item": c["item"], "postRenderAction": "explode"})).is_err()); +} + +#[test] +fn templates_save_apply_default_and_persist() { + let (mut s, _) = rq(); + let t = s.execute("renderQueue.templates", json!({})).unwrap(); + let names: Vec<&str> = t["renderSettings"].as_array().unwrap().iter().map(|r| r["name"].as_str().unwrap()).collect(); + assert_eq!(names, ["Best Settings", "Draft Settings", "DV Settings", "Multi-Machine Settings", "Current Settings"]); + assert!(t["outputModules"].as_array().unwrap().iter().any(|m| m["name"] == "High Quality with Alpha")); + assert_eq!(t["defaults"]["renderSettings"]["Movie"], "Best Settings"); + // A new item starts from the Movie defaults. + let a = s.execute("renderQueue.add", json!({"output": "/tmp/t/a.mp4"})).unwrap(); + assert_eq!(a["renderSettingsSummary"], "Best Settings"); + assert_eq!(a["outputModuleSummary"], "H.264 - Match Render Settings - 15 Mbps"); + // Edit it: Custom; save templates from it; make them the defaults. + let r = s + .execute( + "renderQueue.setRenderSettings", + json!({"item": a["item"], "fieldRender": "upper", "pulldown": "WSSWW", "colorDepth": 16, "effects": "allOff", "solo": "allOff", "guideLayers": "current", "frameBlending": "offForAll", "motionBlur": "onForChecked", "duration": 0.5, "storageOverflow": false}), + ) + .unwrap(); + assert_eq!(r["renderSettingsSummary"], "Custom"); + assert_eq!(r["settings"]["field_render"], "UpperFirst"); + s.execute( + "renderQueue.setOutputModule", + json!({"item": a["item"], "format": "png", "channels": "rgba", "color": "premultiplied", "crop": {"top": 2, "bottom": 2}, "resize": {"width": 320, "height": 200, "lockAspect": false, "quality": "low"}}), + ) + .unwrap(); + s.execute("renderQueue.saveTemplate", json!({"kind": "renderSettings", "name": "Interlaced 16", "item": a["item"]})).unwrap(); + s.execute("renderQueue.saveTemplate", json!({"kind": "outputModule", "name": "Small PNG", "item": a["item"], "params": {"quality": 50}})).unwrap(); + s.execute("renderQueue.setTemplateDefault", json!({"kind": "renderSettings", "slot": "movie", "name": "Interlaced 16"})).unwrap(); + s.execute("renderQueue.setTemplateDefault", json!({"kind": "outputModule", "slot": "movie", "name": "Small PNG"})).unwrap(); + assert!(s.execute("renderQueue.setTemplateDefault", json!({"kind": "outputModule", "slot": "movie", "name": "Nope"})).is_err()); + let b = s.execute("renderQueue.add", json!({"output": "/tmp/t/b_[####].png"})).unwrap(); + assert_eq!(b["renderSettingsSummary"], "Interlaced 16"); + assert_eq!(b["outputModuleSummary"], "Small PNG"); + let it = s.project.render_queue.last().unwrap().clone(); + assert_eq!(it.output.resize.width, 320); + assert_eq!(it.output.output, "/tmp/t/b_[####].png"); + assert_eq!((b["width"].as_u64(), b["height"].as_u64()), (Some(320), Some(200))); + // Apply built-ins to an item; the output path keeps its name with the new extension. + let r = s.execute("renderQueue.applyTemplate", json!({"item": a["item"], "renderSettings": "DV Settings", "outputModule": "High Quality"})).unwrap(); + assert_eq!(r["renderSettingsSummary"], "DV Settings"); + assert_eq!(r["outputModuleSummary"], "High Quality"); + assert!(r["outputPath"].as_str().unwrap().ends_with("a.mov"), "{r}"); + // Persisted with the project. + let p = effectcraft_project::Project::from_json(&s.project.to_json()).unwrap(); + assert_eq!(p.render_templates, s.project.render_templates); + assert_eq!(p.render_queue, s.project.render_queue); + // Delete: saved ones go; built-ins stay. Undo restores. + s.execute("renderQueue.deleteTemplate", json!({"kind": "outputModule", "name": "Small PNG"})).unwrap(); + assert!(s.project.render_templates.output_module("Small PNG").is_none()); + assert!(s.execute("renderQueue.deleteTemplate", json!({"kind": "renderSettings", "name": "Best Settings"})).is_err()); + s.execute("edit.undo", json!({})).unwrap(); + assert!(s.project.render_templates.output_module("Small PNG").is_some()); +} + +#[test] +fn save_frame_as_queues_with_the_frame_defaults() { + let (mut s, _) = rq(); + s.execute("renderQueue.setTemplateDefault", json!({"kind": "outputModule", "slot": "still", "name": "TIFF Sequence with Alpha"})).unwrap(); + let r = s.execute("comp.saveFrameAs", json!({"queue": true, "time": 1.0})).unwrap(); + assert_eq!(r["queued"], true); + let it = s.project.render_queue.last().unwrap(); + assert_eq!(it.output.format, OutputFormat::TiffSequence); + assert_eq!(it.settings.name, "Current Settings"); + let comp = s.project.comp(it.comp).unwrap(); + assert_eq!(it.settings.frame_count(comp), 1); + assert_eq!(it.settings.frame_time(comp, 0), Tick::from_seconds_f64(1.0)); +} + +#[test] +fn log_notify_and_overflow_settings() { + let (mut s, _) = rq(); + let a = s.execute("renderQueue.add", json!({"output": "/tmp/n/a.mp4", "log": "plusPerFrameInfo"})).unwrap(); + assert_eq!(a["logLabel"], "Plus Per Frame Info"); + assert_eq!(a["logPath"], "/tmp/n/a_RenderLog.txt"); + s.execute("renderQueue.setLog", json!({"item": a["item"], "log": "plusSettings"})).unwrap(); + assert_eq!(s.project.render_queue[0].log, effectcraft_project::render_queue::RenderLog::PlusSettings); + s.execute("renderQueue.setOverflowFolders", json!({"folders": ["/tmp/o1", " ", "/tmp/o2"]})).unwrap(); + assert_eq!(s.project.render_prefs.overflow_folders, ["/tmp/o1", "/tmp/o2"]); + assert_eq!(s.execute("renderQueue.setNotify", json!({})).unwrap()["notify"], true); + s.drain_events(); + s.execute("renderQueue.render", json!({})).unwrap(); + let ev = s.drain_events(); + assert!(ev.iter().any(|e| matches!(e, Event::Frontend { command, .. } if command == "renderQueue.notify")), "{ev:?}"); + s.execute("renderQueue.setNotify", json!({"notify": false})).unwrap(); + s.execute("renderQueue.setRender", json!({"index": 1, "render": true})).unwrap(); + s.execute("renderQueue.render", json!({})).unwrap(); + assert!(!s.drain_events().iter().any(|e| matches!(e, Event::Frontend { command, .. } if command == "renderQueue.notify"))); +} + #[test] fn save_current_preview_writes_a_movie() { let (mut s, ex) = rq(); diff --git a/crates/export/Cargo.toml b/crates/export/Cargo.toml index 15f9b17d..6900c204 100644 --- a/crates/export/Cargo.toml +++ b/crates/export/Cargo.toml @@ -32,3 +32,4 @@ effectcraft-media = { workspace = true } effectcraft-color = { workspace = true } filmcraft-opus = { workspace = true } filmcraft-vp9 = { workspace = true } +effectcraft-keyframe = { workspace = true } diff --git a/crates/export/src/encode.rs b/crates/export/src/encode.rs index d6cf3150..b0566d9b 100644 --- a/crates/export/src/encode.rs +++ b/crates/export/src/encode.rs @@ -4,12 +4,12 @@ use std::io::Write; use effectcraft_project::Comp; -use effectcraft_project::render_queue::{Channels, OutputFormat, ProResProfile}; +use effectcraft_project::render_queue::{AudioFormat, Channels, OutputFormat, ProResProfile}; use effectcraft_time::{FrameRate, TICKS_PER_SECOND, Tick}; use filmcraft_isobmff::{Brand, FourCc, Mp4Writer, PcmConfig, SampleEntry, TrackConfig, WriteSample, WriterOptions}; use rayon::prelude::*; -use crate::{ExportError, Job, Report, Result, State, batch_size, render_frame, rgba8, wants_audio}; +use crate::{Cx, ExportError, Report, Result, State, batch_size, wants_audio}; fn enc(e: impl std::fmt::Display) -> ExportError { ExportError::Encode(e.to_string()) @@ -208,29 +208,60 @@ enum Audio { Pcm, } -const AUDIO_CHANNELS: u32 = 2; +fn deinterleave(buf: &[f32], channels: usize) -> Vec> { + (0..channels).map(|c| buf.chunks_exact(channels).map(|p| p[c]).collect()).collect() +} -fn deinterleave(buf: &[f32]) -> [Vec; 2] { - let l = buf.chunks_exact(2).map(|p| p[0]).collect(); - let r = buf.chunks_exact(2).map(|p| p[1]).collect(); - [l, r] +/// The comp's audio over `n` samples from `start`, interleaved with the module's channel count +/// (mono = the average of left and right). +pub(crate) fn mix(cx: &Cx, start: effectcraft_time::Tick, n: usize, sr: u32) -> Vec { + let st = effectcraft_render::audio::mix_comp(&cx.project, cx.footage, cx.expr, cx.comp, start, n, sr); + if cx.output.audio_channels == 1 { st.chunks_exact(2).map(|p| (p[0] + p[1]) * 0.5).collect() } else { st } +} + +/// PCM bytes of samples in `fmt` (little or big endian). +pub(crate) fn pcm_bytes(buf: &[f32], fmt: AudioFormat, big_endian: bool) -> Vec { + let mut out = Vec::with_capacity(buf.len() * 4); + for &s in buf { + let s = s.clamp(-1.0, 1.0); + match fmt { + AudioFormat::S16 => { + let v = (s * 32767.0).round() as i16; + out.extend_from_slice(&if big_endian { v.to_be_bytes() } else { v.to_le_bytes() }); + } + AudioFormat::S24 => { + let v = (s * 8_388_607.0).round() as i32; + let b = v.to_le_bytes(); + if big_endian { + out.extend_from_slice(&[b[2], b[1], b[0]]); + } else { + out.extend_from_slice(&b[..3]); + } + } + AudioFormat::F32 => out.extend_from_slice(&if big_endian { s.to_be_bytes() } else { s.to_le_bytes() }), + } + } + out } // ---------------------------------------------------------------- the movie type Writer<'f, 's> = Mp4Writer<&'f mut crate::out::Out<'s>>; -pub(crate) fn movie(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { +pub(crate) fn movie(job: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { let rate = job.settings.rate(comp); let fmt = job.output.format; - let channels = if fmt == OutputFormat::ProRes { job.output.channels } else { Channels::Rgb }; + let channels = match job.output.channels { + Channels::Rgba if fmt != OutputFormat::ProRes => Channels::Rgb, + c => c, + }; let mut venc: Box = match fmt { OutputFormat::H264 => Box::new(H264::new(w, h, rate, job.output.bitrate_kbps)?), - _ => Box::new(ProRes::new(w, h, job.output.prores_profile, channels)), + _ => Box::new(ProRes::new(w, h, job.output.prores_profile, if channels == Channels::Alpha { Channels::Rgb } else { channels })), }; let brand = if fmt == OutputFormat::H264 { Brand::Mp4 } else { Brand::Mov }; let batch = batch_size(); - let render = |k: u64| -> Vec { rgba8(&render_frame(job, comp, k), comp, channels, w, h) }; + let render = |k: u64| -> Vec { job.pixels(&job.frame(comp, k), comp, channels, w, h) }; // Encode the first batch before creating tracks: encoders finalise their config on frame 1. let first_end = batch.min(st.total); @@ -242,7 +273,8 @@ pub(crate) fn movie(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> R drop(first); st.advance(first_end)?; - let mut file = crate::out::create(job.sink, job.path)?; + let path = job.place(job.path, 1); + let mut file = crate::out::create(job.sink, &path)?; let opts = WriterOptions::new(brand); let movie_ts = opts.movie_timescale.max(1) as i128; let mut mux: Writer = Mp4Writer::new(&mut file, opts).map_err(mux_err)?; @@ -257,15 +289,22 @@ pub(crate) fn movie(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> R let vt = mux.add_track(vcfg).map_err(mux_err)?; let sr = job.output.audio_sample_rate.clamp(8_000, 192_000); let with_audio = wants_audio(job); + let chans = if job.output.audio_channels == 1 { 1u32 } else { 2 }; + let afmt = job.output.audio_format; let mut audio = None; if with_audio { let (a, entry, start) = if brand == Brand::Mp4 { - let e = filmcraft_aac::Encoder::new(filmcraft_aac::EncoderConfig::cbr(sr, AUDIO_CHANNELS as usize, 320_000)).map_err(enc)?; - let entry = SampleEntry::aac(e.audio_specific_config(), AUDIO_CHANNELS, sr); + let e = filmcraft_aac::Encoder::new(filmcraft_aac::EncoderConfig::cbr(sr, chans as usize, 160_000 * chans)).map_err(enc)?; + let entry = SampleEntry::aac(e.audio_specific_config(), chans, sr); let priming = e.priming_samples() as i64; (Audio::Aac(Box::new(e)), entry, Some(priming)) } else { - let pcm = PcmConfig { bits: 16, float: false, big_endian: false, signed: true, channels: AUDIO_CHANNELS, sample_rate: sr as f64 }; + let (bits, float) = match afmt { + AudioFormat::S16 => (16, false), + AudioFormat::S24 => (24, false), + AudioFormat::F32 => (32, true), + }; + let pcm = PcmConfig { bits, float, big_endian: false, signed: true, channels: chans, sample_rate: sr as f64 }; (Audio::Pcm, SampleEntry::pcm(pcm), None) }; let mut c = TrackConfig::new(entry, sr); @@ -290,21 +329,18 @@ pub(crate) fn movie(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> R } let n = (end - *cursor) as usize; let start = Tick(((*cursor as i128 * TICKS_PER_SECOND as i128) / sr as i128) as i64); - let buf = effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, start, n, sr); + let buf = mix(job, start, n, sr); *cursor = end; match a { Audio::Aac(e) => { - let planar = deinterleave(&buf); + let planar = deinterleave(&buf, chans as usize); let refs: Vec<&[f32]> = planar.iter().map(Vec::as_slice).collect(); for au in e.encode(&refs) { mux.write_sample(*at, WriteSample { data: &au, duration: 1024, composition_offset: 0, is_sync: true }).map_err(mux_err)?; } } Audio::Pcm => { - let mut pcm = Vec::with_capacity(n * 4); - for s in buf { - pcm.extend_from_slice(&((s.clamp(-1.0, 1.0) * 32767.0).round() as i16).to_le_bytes()); - } + let pcm = pcm_bytes(&buf, afmt, false); mux.write_sample(*at, WriteSample { data: &pcm, duration: n as u32, composition_offset: 0, is_sync: true }).map_err(mux_err)?; } } @@ -336,7 +372,7 @@ pub(crate) fn movie(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> R let out = mux.finish().map_err(mux_err)?; out.flush().map_err(mux_err)?; let bytes = file.finish()?; - Ok(Report { path: job.path.to_string(), frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: with_audio }) + Ok(Report { path, frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: with_audio, log: None, overflow: vec![] }) } #[cfg(test)] diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index c9196625..74b12c76 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -2,13 +2,21 @@ //! //! | Format | Container / files | Video | Alpha | Audio | //! |---|---|---|---|---| -//! | H.264 | MP4 | FilmCraft H.264 High, 8-bit 4:2:0 BT.709 limited, VBR | no | AAC-LC (FilmCraft), stereo | -//! | ProRes | QuickTime MOV | FilmCraft ProRes 422 Proxy/LT/422/HQ, 10-bit; 4444/4444 XQ 4:4:4 | 4444 (16-bit alpha coding) | 16-bit PCM, stereo | +//! | H.264 | MP4 | FilmCraft H.264 High, 8-bit 4:2:0 BT.709 limited, VBR | no | AAC-LC (FilmCraft), mono/stereo | +//! | ProRes | QuickTime MOV | FilmCraft ProRes 422 Proxy/LT/422/HQ, 10-bit; 4444/4444 XQ 4:4:4 | 4444 (16-bit alpha coding) | PCM 16/24-bit or 32-bit float, mono/stereo | //! | PNG / JPEG / TIFF sequence | one file per frame | 8-bit | PNG, TIFF | — | //! | OpenEXR sequence | one file per frame | 32-bit float, linear light, premultiplied | yes | — | //! | Animated GIF | GIF89a | 256-colour palette per frame (NeuQuant) | 1-bit | — | //! | WebM | WebM (Matroska) | VP9 profile 0 (`effectcraft-vp9enc`: key + inter frames, a key frame every 2 s, loop filter), 8-bit 4:2:0 | VP9 alpha (BlockAdditional) | Opus (`effectcraft-opusenc`), 48 kHz stereo | -//! | WAV / AIFF | RIFF WAVE / AIFF | — | — | 16-bit PCM stereo (audio only) | +//! | WAV / AIFF | RIFF WAVE / AIFF | — | — | PCM 16/24-bit (WAV also 32-bit float), mono/stereo (audio only) | +//! +//! Every frame goes through [`pipeline`]: the Render Settings overrides (Effects, Solo Switches, +//! Guide Layers, Frame Blending, Motion Blur, Color Depth), field rendering (two fields half a +//! frame apart woven line by line, with optional 3:2 pulldown from film frames at 4/5 of the +//! rate), then the Output Module's Crop (edges or the region of interest) and Resize. Channels +//! may be RGB, RGB + Alpha (straight or premultiplied) or the alpha matte alone. With Use Storage +//! Overflow and a [`StorageQuota`] hook, files that don't fit go to the overflow folders. A +//! render log (Errors Only / Plus Settings / Plus Per Frame Info) is written next to the output. //! //! Frames are rendered in parallel batches (rayon; one batch ≈ one frame per core) and handed to //! the encoder in order. Progress is reported after each batch through a callback that can cancel @@ -20,19 +28,21 @@ mod encode; mod out; +mod pipeline; mod webm; use web_time::Instant; -use effectcraft_project::render_queue::{AudioOutput, Channels, OutputFormat, OutputModule, RenderQuality, RenderSettings, sequence_path}; +use effectcraft_project::render_queue::{AudioOutput, Channels, OutputFormat, OutputModule, RenderLog, RenderQuality, RenderSettings, sequence_path}; use effectcraft_project::{Comp, ItemId, Project}; use effectcraft_raster::Image; -use effectcraft_render::{ExprHost, FootageSource, RenderOpts, Renderer}; -use effectcraft_time::Tick; +use effectcraft_render::{ExprHost, FootageSource}; use rayon::prelude::*; pub use effectcraft_project::render_queue; +pub use effectcraft_project::render_queue::StorageQuota; pub use out::Sink; +pub(crate) use pipeline::Cx; #[derive(Debug, thiserror::Error)] pub enum ExportError { @@ -69,6 +79,21 @@ pub struct Job<'a> { pub path: &'a str, /// Hand finished files to this instead of writing them to disk (web downloads, tests). pub sink: Option<&'a Sink>, + /// Render log, storage overflow. + pub options: JobOptions<'a>, +} + +/// Job options beyond the Render Settings and Output Module. +#[derive(Default)] +pub struct JobOptions<'a> { + /// The render log written next to the output. + pub log: RenderLog, + /// What the log calls the job (e.g. "#2 Main Comp"). + pub label: String, + /// Free-space hook for Use Storage Overflow (`None`: never full). + pub storage: Option<&'a dyn StorageQuota>, + /// Overflow folders, in order. + pub overflow: Vec, } /// Progress after a batch of frames. @@ -100,6 +125,10 @@ pub struct Report { pub seconds: f64, pub bytes: u64, pub audio: bool, + /// The render log, when one was written. + pub log: Option, + /// Files that went to an overflow folder. + pub overflow: Vec, } /// Formats this build can write (all of them; kept as the hook for optional encoders). @@ -107,10 +136,9 @@ pub fn available_formats() -> Vec { OutputFormat::ALL.to_vec() } -/// The size of the written frames for a job (H.264 rounds to even). +/// The size of the written frames for a job (crop, resize; H.264 rounds to even). pub fn output_size(comp: &Comp, settings: &RenderSettings, output: &OutputModule) -> (u32, u32) { - let (w, h) = settings.output_size(comp); - output.format.coded_size(w, h) + output.output_size(comp, settings) } /// Whether the job writes an audio track. @@ -125,19 +153,30 @@ pub fn wants_audio(job: &Job) -> bool { /// Run the export (blocking). `progress` is called after every batch; returning `false` cancels. pub fn export(job: &Job, progress: &mut dyn FnMut(&Progress) -> bool) -> Result { - let comp = job.project.comp(job.comp).ok_or(ExportError::NoComp)?; let t0 = Instant::now(); - let total = job.settings.frame_count(comp); + let cx = Cx::new(job); + let mut r = export_with(&cx, progress, t0); + let log = cx.write_log(&r, t0.elapsed().as_secs_f64()); + if let Ok(rep) = &mut r { + rep.log = log; + rep.overflow = cx.overflowed.lock().unwrap_or_else(|e| e.into_inner()).clone(); + } + r +} + +fn export_with(cx: &Cx, progress: &mut dyn FnMut(&Progress) -> bool, t0: Instant) -> Result { + let comp = cx.comp().ok_or(ExportError::NoComp)?; + let total = cx.settings.frame_count(comp); let mut st = State { t0, total, done: 0, progress }; st.advance(0)?; - let (w, h) = output_size(comp, job.settings, job.output); - let mut report = match job.output.format { - OutputFormat::H264 | OutputFormat::ProRes => encode::movie(job, comp, w, h, &mut st)?, - OutputFormat::Gif => gif_export(job, comp, w, h, &mut st)?, - OutputFormat::WebM => webm::webm(job, comp, w, h, &mut st)?, - OutputFormat::Wav => webm::audio_file(job, comp, false, &mut st)?, - OutputFormat::Aiff => webm::audio_file(job, comp, true, &mut st)?, - f if f.is_sequence() => sequence(job, comp, w, h, &mut st)?, + let (w, h) = output_size(comp, cx.settings, cx.output); + let mut report = match cx.output.format { + OutputFormat::H264 | OutputFormat::ProRes => encode::movie(cx, comp, w, h, &mut st)?, + OutputFormat::Gif => gif_export(cx, comp, w, h, &mut st)?, + OutputFormat::WebM => webm::webm(cx, comp, w, h, &mut st)?, + OutputFormat::Wav => webm::audio_file(cx, comp, false, &mut st)?, + OutputFormat::Aiff => webm::audio_file(cx, comp, true, &mut st)?, + f if f.is_sequence() => sequence(cx, comp, w, h, &mut st)?, f => return Err(ExportError::Unsupported(f.label().into())), }; report.seconds = t0.elapsed().as_secs_f64(); @@ -170,37 +209,8 @@ pub(crate) fn batch_size() -> u64 { rayon::current_num_threads().clamp(2, 16) as u64 } -/// Render output frame `i` of the job at the settings' resolution. -pub(crate) fn render_frame(job: &Job, comp: &Comp, i: u64) -> Image { - let t: Tick = job.settings.frame_time(comp, i); - let opts = RenderOpts { - scale: job.settings.resolution.clamp(0.01, 4.0), - motion_blur: job.settings.motion_blur, - guides: false, - draft: job.settings.quality == RenderQuality::Draft, - // Output renders always look through the comp's active camera. - view: None, - backend: effectcraft_render::Backend::Auto, - roi: None, - proxy: job.settings.proxy_use, - }; - let mut r = Renderer::new(job.project, job.footage, opts); - r.expr = job.expr; - r.accel = job.accel; - r.comp_frame(job.comp, t) -} - -/// 8-bit RGBA of the requested channels (RGB: over the comp background, opaque; RGBA: straight), -/// cropped/padded to `w`×`h`. -pub(crate) fn rgba8(img: &Image, comp: &Comp, channels: Channels, w: u32, h: u32) -> Vec { - let px = match channels { - Channels::Rgb => img.to_rgba8_over(comp.background), - Channels::Rgba => img.to_rgba8(), - }; - fit(px, img.width, img.height, w, h) -} - -fn fit(px: Vec, iw: u32, ih: u32, w: u32, h: u32) -> Vec { +/// Crop/pad 8-bit RGBA to `w`×`h` (codec rounding). +pub(crate) fn fit(px: Vec, iw: u32, ih: u32, w: u32, h: u32) -> Vec { if iw == w && ih == h { return px; } @@ -217,10 +227,13 @@ fn first_frame_number(job: &Job, comp: &Comp) -> i64 { job.settings.first_frame(comp) } -fn sequence(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { - let f0 = first_frame_number(job, comp); - let fmt = job.output.format; - let channels = if fmt.supports_alpha() { job.output.channels } else { Channels::Rgb }; +fn sequence(cx: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { + let f0 = first_frame_number(cx, comp); + let fmt = cx.output.format; + let channels = match cx.output.channels { + Channels::Rgba if !fmt.supports_alpha() => Channels::Rgb, + c => c, + }; let batch = batch_size(); let mut bytes = 0u64; let mut i = 0; @@ -229,13 +242,18 @@ fn sequence(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result> = (i..end) .into_par_iter() .map(|k| { - let path = sequence_path(job.path, f0 + k as i64); - if job.settings.skip_existing && out::exists(job.sink, &path) { + let path = sequence_path(cx.path, f0 + k as i64); + if cx.settings.skip_existing && out::exists(cx.sink, &path) { return Ok(0); } - let img = render_frame(job, comp, k); - let mut f = out::create(job.sink, &path)?; - write_still(&mut f, fmt, &img, comp, channels, w, h, job.output.quality)?; + let img = cx.frame(comp, k); + // Encode in memory first: the storage hook decides where it fits. + let mut buf = std::io::Cursor::new(Vec::new()); + write_still(cx, &mut buf, fmt, &img, comp, channels, w, h, cx.output.quality)?; + let data = buf.into_inner(); + let path = cx.place(&path, data.len() as u64); + let mut f = out::create(cx.sink, &path)?; + std::io::Write::write_all(&mut f, &data).map_err(io)?; f.finish() }) .collect(); @@ -245,11 +263,21 @@ fn sequence(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result Result<()> { +fn write_still( + cx: &Cx, + file: &mut W, + fmt: OutputFormat, + img: &Image, + comp: &Comp, + channels: Channels, + w: u32, + h: u32, + quality: u8, +) -> Result<()> { use image::{ExtendedColorType, ImageEncoder, ImageFormat}; match fmt { OutputFormat::ExrSequence => { @@ -260,6 +288,10 @@ fn write_still(file: &mut out::Out, fmt: OutputFormat, img: &Image, comp: &Comp, for x in 0..w { let p = if x < img.width && y < img.height { img.data[img.idx(x, y)] } else { [0.0; 4] }; let (c, a) = match channels { + Channels::Alpha => { + let a = p[3].clamp(0.0, 1.0); + ([lin(a); 3], 1.0) + } Channels::Rgba => { let a = p[3].clamp(0.0, 1.0); let s = |c: f32| if a > 0.0 { lin((c / a).clamp(0.0, 1.0)) * a } else { 0.0 }; @@ -278,16 +310,17 @@ fn write_still(file: &mut out::Out, fmt: OutputFormat, img: &Image, comp: &Comp, image::DynamicImage::ImageRgba32F(buf).write_to(file, ImageFormat::OpenExr).map_err(|e| ExportError::Encode(e.to_string())) } OutputFormat::JpegSequence => { - let px = rgba8(img, comp, Channels::Rgb, w, h); + let px = cx.pixels(img, comp, if channels == Channels::Alpha { Channels::Alpha } else { Channels::Rgb }, w, h); let rgb: Vec = px.chunks_exact(4).flat_map(|p| [p[0], p[1], p[2]]).collect(); image::codecs::jpeg::JpegEncoder::new_with_quality(file, quality.clamp(1, 100)) .write_image(&rgb, w, h, ExtendedColorType::Rgb8) .map_err(|e| ExportError::Encode(e.to_string())) } OutputFormat::PngSequence | OutputFormat::TiffSequence => { - let px = rgba8(img, comp, channels, w, h); + let px = cx.pixels(img, comp, channels, w, h); let (data, ct) = match channels { Channels::Rgba => (px, ExtendedColorType::Rgba8), + Channels::Alpha => (px.chunks_exact(4).map(|p| p[0]).collect(), ExtendedColorType::L8), Channels::Rgb => (px.chunks_exact(4).flat_map(|p| [p[0], p[1], p[2]]).collect(), ExtendedColorType::Rgb8), }; if fmt == OutputFormat::PngSequence { @@ -301,11 +334,12 @@ fn write_still(file: &mut out::Out, fmt: OutputFormat, img: &Image, comp: &Comp, } } -fn gif_export(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { +fn gif_export(job: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { if w > u16::MAX as u32 || h > u16::MAX as u32 { return Err(ExportError::Unsupported("GIF frames are limited to 65535 px".into())); } - let mut file = out::create(job.sink, job.path)?; + let path = job.place(job.path, 1); + let mut file = out::create(job.sink, &path)?; let mut enc = gif::Encoder::new(&mut file, w as u16, h as u16, &[]).map_err(|e| ExportError::Encode(e.to_string()))?; enc.set_repeat(if job.output.gif_loop { gif::Repeat::Infinite } else { gif::Repeat::Finite(0) }).map_err(|e| ExportError::Encode(e.to_string()))?; let rate = job.settings.rate(comp); @@ -320,8 +354,8 @@ fn gif_export(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result< let frames: Vec> = (i..end) .into_par_iter() .map(|k| { - let img = render_frame(job, comp, k); - let mut px = rgba8(&img, comp, job.output.channels, w, h); + let img = job.frame(comp, k); + let mut px = job.pixels(&img, comp, job.output.channels, w, h); gif::Frame::from_rgba_speed(w as u16, h as u16, &mut px, 10) }) .collect(); @@ -340,7 +374,7 @@ fn gif_export(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result< } drop(enc); let bytes = file.finish()?; - Ok(Report { path: job.path.to_string(), frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: false }) + Ok(Report { path, frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: false, log: None, overflow: vec![] }) } #[cfg(test)] diff --git a/crates/export/src/pipeline.rs b/crates/export/src/pipeline.rs new file mode 100644 index 00000000..76087f2a --- /dev/null +++ b/crates/export/src/pipeline.rs @@ -0,0 +1,327 @@ +//! The frame pipeline shared by every output format: Render Settings overrides (effects, solo, +//! guide layers, frame blending, motion blur), field rendering with 3:2 pulldown, colour depth, +//! the Output Module's crop and resize, alpha handling, storage overflow and the render log. + +use std::sync::Mutex; + +use effectcraft_project::render_queue::{ + AlphaMode, Channels, ColorDepth, CurrentOrOff, EffectsMode, FrameSample, RenderLog, ResizeQuality, SwitchOverride, log_path, +}; +use effectcraft_project::{Comp, Project}; +use effectcraft_raster::Image; +use effectcraft_render::{RenderOpts, Renderer}; +use effectcraft_time::Tick; +use web_time::Instant; + +use crate::{Job, RenderQuality, out}; + +/// A job being exported: the job, the project with the Render Settings overrides applied, and +/// per-frame timing for the render log. +pub(crate) struct Cx<'a> { + pub job: &'a Job<'a>, + pub project: Project, + /// (output frame, seconds) of every rendered frame (Plus Per Frame Info logs). + pub frame_times: Option>>, + /// Files written to an overflow folder. + pub overflowed: Mutex>, +} + +impl<'a> std::ops::Deref for Cx<'a> { + type Target = Job<'a>; + fn deref(&self) -> &Job<'a> { + self.job + } +} + +/// The project as the Render Settings render it: Effects All On / All Off, Solo Switches All +/// Off, Frame Blending and Motion Blur "On for Checked Layers" (the composition switches on). +pub(crate) fn apply_overrides(p: &mut Project, job: &Job) { + let s = job.settings; + let ids: Vec<_> = p.comps().map(|(i, _)| *i).collect(); + for id in ids { + let Some(c) = p.comp_mut(id) else { continue }; + match s.frame_blending { + SwitchOverride::Current => {} + SwitchOverride::OnForChecked => c.enable_frame_blending = true, + SwitchOverride::OffForAll => c.enable_frame_blending = false, + } + match s.motion_blur_override() { + SwitchOverride::Current => {} + SwitchOverride::OnForChecked => c.enable_motion_blur = true, + SwitchOverride::OffForAll => c.enable_motion_blur = false, + } + for l in c.layers.iter_mut() { + match s.effects { + EffectsMode::Current => {} + EffectsMode::AllOn => l.switches.effects = true, + EffectsMode::AllOff => l.switches.effects = false, + } + if s.solo == CurrentOrOff::AllOff { + l.switches.solo = false; + } + } + } +} + +impl<'a> Cx<'a> { + pub fn new(job: &'a Job<'a>) -> Cx<'a> { + let mut project = job.project.clone(); + apply_overrides(&mut project, job); + let frame_times = (job.options.log == RenderLog::PlusPerFrameInfo).then(|| Mutex::new(Vec::new())); + Cx { job, project, frame_times, overflowed: Mutex::new(vec![]) } + } + + pub fn comp(&self) -> Option<&Comp> { + self.project.comp(self.job.comp) + } + + fn render_at(&self, t: Tick) -> Image { + let s = self.job.settings; + let opts = RenderOpts { + scale: s.resolution.clamp(0.01, 4.0), + motion_blur: s.motion_blur_override() != SwitchOverride::OffForAll, + guides: s.guide_layers == CurrentOrOff::Current, + draft: s.quality == RenderQuality::Draft, + // Output renders always look through the comp's active camera. + view: None, + backend: effectcraft_render::Backend::Auto, + roi: None, + proxy: s.proxy_use, + }; + let mut r = Renderer::new(&self.project, self.job.footage, opts); + r.expr = self.job.expr; + r.accel = self.job.accel; + r.comp_frame(self.job.comp, t) + } + + /// Output frame `i`: sampled (fields, pulldown), at the colour depth, cropped and resized. + pub fn frame(&self, comp: &Comp, i: u64) -> Image { + let t0 = Instant::now(); + let img = match self.job.settings.sample(comp, i) { + FrameSample::Progressive(t) => self.render_at(t), + FrameSample::Fields { first, second, upper_first } => { + let a = self.render_at(first); + if second == first { a } else { interleave(&a, &self.render_at(second), upper_first) } + } + }; + let mut img = img; + // Color Depth: renders run at the project's depth; another depth quantises the frame. + let depth = self.job.settings.color_depth; + if depth != ColorDepth::Current + && let Some(levels) = depth.resolve(self.project.settings.bit_depth).levels() + { + img.quantize(levels); + } + let img = self.crop_resize(img); + if let Some(ft) = &self.frame_times { + ft.lock().unwrap_or_else(|e| e.into_inner()).push((i, t0.elapsed().as_secs_f64())); + } + img + } + + /// Output Module ▸ Crop, then Resize. + fn crop_resize(&self, img: Image) -> Image { + let om = self.job.output; + let scale = self.job.settings.resolution.clamp(0.01, 4.0); + let (t, l, b, r) = om.crop.edges(img.width, img.height, scale); + let img = if (t, l, b, r) == (0, 0, 0, 0) { img } else { crop(&img, t, l, b, r) }; + if !om.resize.enabled { + return img; + } + let (w, h) = om.frame_size(img.width, img.height, 1.0); + if (w, h) == (img.width, img.height) { + return img; + } + match om.resize.quality { + ResizeQuality::Low => effectcraft_raster::warp::resample(&img, w, h), + ResizeQuality::High => bicubic(&img, w, h), + } + } + + /// 8-bit RGBA of the requested channels cropped/padded to `w`×`h` (codec rounding): RGB over + /// the comp background (opaque), RGB + Alpha straight or premultiplied, or the alpha matte. + pub fn pixels(&self, img: &Image, comp: &Comp, channels: Channels, w: u32, h: u32) -> Vec { + let px = match channels { + Channels::Rgb => img.to_rgba8_over(comp.background), + Channels::Rgba => match self.job.output.alpha_mode { + AlphaMode::Straight => img.to_rgba8(), + AlphaMode::Premultiplied => premultiplied8(img), + }, + Channels::Alpha => img + .data + .iter() + .flat_map(|p| { + let a = (p[3].clamp(0.0, 1.0) * 255.0 + 0.5) as u8; + [a, a, a, 255] + }) + .collect(), + }; + crate::fit(px, img.width, img.height, w, h) + } + + /// Where a file goes: `path`, or (Use Storage Overflow) the first overflow folder with room + /// for `bytes` when the quota hook says the output volume is full. + pub fn place(&self, path: &str, bytes: u64) -> String { + let Some(q) = self.job.options.storage else { return path.to_string() }; + if !self.job.settings.storage_overflow || q.has_room(path, bytes) { + return path.to_string(); + } + let name = std::path::Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.to_string()); + for dir in &self.job.options.overflow { + let p = std::path::Path::new(dir).join(&name).to_string_lossy().to_string(); + if q.has_room(&p, bytes) { + self.overflowed.lock().unwrap_or_else(|e| e.into_inner()).push(p.clone()); + return p; + } + } + path.to_string() + } + + /// Write the render log next to the output, as the Log setting asks. Returns its path. + pub fn write_log(&self, result: &Result, seconds: f64) -> Option { + let level = self.job.options.log; + // Errors Only: a log for failures (a user stop is not an error). + if level == RenderLog::ErrorsOnly && matches!(result, Ok(_) | Err(crate::ExportError::Cancelled)) { + return None; + } + let comp = self.comp(); + let path = log_path(self.job.path); + let mut s = String::new(); + s.push_str("EffectCraft Render Log\n"); + s.push_str(&format!("Item: {}\n", if self.job.options.label.is_empty() { "(render)" } else { &self.job.options.label })); + s.push_str(&format!("Output: {}\n", self.job.path)); + match result { + Ok(r) => s.push_str(&format!("Result: Done ({} frames, {}×{}, {} bytes) in {seconds:.2} s\n", r.frames, r.width, r.height, r.bytes)), + Err(e) => s.push_str(&format!("Result: Failed\nError: {e}\n")), + } + let over = self.overflowed.lock().unwrap_or_else(|e| e.into_inner()); + if !over.is_empty() { + s.push_str(&format!("Storage overflow: {} file(s) written to overflow folders (first: {})\n", over.len(), over[0])); + } + if level != RenderLog::ErrorsOnly { + s.push_str("\nRender Settings:\n"); + for (k, v) in self.job.settings.describe(comp) { + s.push_str(&format!(" {k}: {v}\n")); + } + s.push_str("\nOutput Module:\n"); + for (k, v) in self.job.output.describe() { + s.push_str(&format!(" {k}: {v}\n")); + } + } + if let (RenderLog::PlusPerFrameInfo, Some(ft), Some(c)) = (level, &self.frame_times, comp) { + let mut v = ft.lock().unwrap_or_else(|e| e.into_inner()).clone(); + v.sort_by_key(|(i, _)| *i); + s.push_str("\nFrames:\n"); + let f0 = self.job.settings.first_frame(c); + for (i, secs) in v { + let t = self.job.settings.frame_time(c, i); + s.push_str(&format!(" Frame {} ({:.3} s): rendered in {:.1} ms\n", f0 + i as i64, t.seconds(), secs * 1000.0)); + } + } + let mut f = out::create(self.job.sink, &path).ok()?; + std::io::Write::write_all(&mut f, s.as_bytes()).ok()?; + f.finish().ok()?; + Some(path) + } +} + +/// Weave two fields: the dominant field's lines (even lines when `upper_first`) from `first`. +pub(crate) fn interleave(first: &Image, second: &Image, upper_first: bool) -> Image { + let mut out = first.clone(); + let w = out.width as usize; + for y in 0..out.height as usize { + let from_first = (y % 2 == 0) == upper_first; + if !from_first && y < second.height as usize && second.width as usize == w { + out.data[y * w..(y + 1) * w].copy_from_slice(&second.data[y * w..(y + 1) * w]); + } + } + out +} + +/// Remove `t`/`l`/`b`/`r` pixels from the edges (negative values add transparent pixels). +pub(crate) fn crop(img: &Image, t: i32, l: i32, b: i32, r: i32) -> Image { + let w = (img.width as i32 - l - r).max(1) as u32; + let h = (img.height as i32 - t - b).max(1) as u32; + let mut out = Image::new(w, h); + for y in 0..h as i32 { + let sy = y + t; + if sy < 0 || sy >= img.height as i32 { + continue; + } + for x in 0..w as i32 { + let sx = x + l; + if sx >= 0 && sx < img.width as i32 { + let o = out.idx(x as u32, y as u32); + out.data[o] = img.data[img.idx(sx as u32, sy as u32)]; + } + } + } + out +} + +/// Bicubic resize (Resize Quality: High). +fn bicubic(src: &Image, w: u32, h: u32) -> Image { + // Shrinking a lot: bilinear with its box pre-filter first, then bicubic for the last step. + if src.width > w * 2 && src.height > h * 2 { + let mid = effectcraft_raster::warp::resample(src, w * 2, h * 2); + return bicubic(&mid, w, h); + } + let mut out = Image::new(w, h); + let (sx, sy) = (src.width as f64 / w as f64, src.height as f64 / h as f64); + for y in 0..h { + for x in 0..w { + let o = out.idx(x, y); + // Clamp to the edge so borders stay opaque. + let fx = ((x as f64 + 0.5) * sx).clamp(0.5, src.width as f64 - 0.5); + let fy = ((y as f64 + 0.5) * sy).clamp(0.5, src.height as f64 - 0.5); + let mut p = src.sample_bicubic(fx, fy); + p[3] = p[3].min(1.0); + out.data[o] = p; + } + } + out +} + +/// Premultiplied 8-bit RGBA (colour matted with black). +fn premultiplied8(img: &Image) -> Vec { + img.data + .iter() + .flat_map(|p| { + let a = p[3].clamp(0.0, 1.0); + [0, 1, 2].map(|c| (p[c].clamp(0.0, a) * 255.0 + 0.5) as u8).into_iter().chain([(a * 255.0 + 0.5) as u8]) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn interleave_takes_alternate_lines() { + let a = Image::filled(3, 4, [1.0, 0.0, 0.0, 1.0]); + let b = Image::filled(3, 4, [0.0, 0.0, 1.0, 1.0]); + let up = interleave(&a, &b, true); + let lo = interleave(&a, &b, false); + for y in 0..4u32 { + assert_eq!(up.data[up.idx(1, y)][0], if y % 2 == 0 { 1.0 } else { 0.0 }, "upper first: line {y}"); + assert_eq!(lo.data[lo.idx(1, y)][0], if y % 2 == 1 { 1.0 } else { 0.0 }, "lower first: line {y}"); + } + } + + #[test] + fn crop_and_pad() { + let mut img = Image::new(4, 3); + for (i, p) in img.data.iter_mut().enumerate() { + *p = [i as f32, 0.0, 0.0, 1.0]; + } + let c = crop(&img, 1, 1, 0, 1); + assert_eq!((c.width, c.height), (2, 2)); + assert_eq!(c.data[0][0], 5.0); + let p = crop(&img, -1, 0, 0, 0); + assert_eq!((p.width, p.height), (4, 4)); + assert_eq!(p.data[0], [0.0; 4]); + assert_eq!(p.data[4][0], 0.0); + assert_eq!(p.data[5][0], 1.0); + } +} diff --git a/crates/export/src/webm.rs b/crates/export/src/webm.rs index 65ca1189..af38eaeb 100644 --- a/crates/export/src/webm.rs +++ b/crates/export/src/webm.rs @@ -15,11 +15,12 @@ use std::io::{Seek, SeekFrom, Write}; use effectcraft_project::Comp; -use effectcraft_project::render_queue::Channels; +use effectcraft_project::render_queue::{AudioFormat, Channels}; use effectcraft_time::{TICKS_PER_SECOND, Tick}; use rayon::prelude::*; -use crate::{Job, Report, Result, State, batch_size, io, render_frame, rgba8, wants_audio}; +use crate::encode::{mix, pcm_bytes}; +use crate::{Cx, Report, Result, State, batch_size, io, wants_audio}; const OPUS_RATE: u32 = 48_000; const OPUS_BITRATE: u32 = 192_000; @@ -124,9 +125,10 @@ fn cluster(frames: &mut [Frame]) -> Vec { out } -pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { +pub(crate) fn webm(job: &Cx, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { let rate = job.settings.rate(comp); let alpha = job.output.channels == Channels::Rgba; + let colour_channels = if job.output.channels == Channels::Alpha { Channels::Alpha } else { Channels::Rgb }; let quality = job.output.quality.clamp(1, 100); let fps = rate.as_f64().max(1.0); let cfg = effectcraft_vp9enc::EncoderConfig { @@ -134,20 +136,22 @@ pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re height: h, quality, full_range: false, - keyframe_interval: (fps * 2.0).round().max(1.0) as u32, - target_kbps: None, + keyframe_interval: if job.output.keyframe_interval > 0 { job.output.keyframe_interval } else { (fps * 2.0).round().max(1.0) as u32 }, + target_kbps: job.output.webm_bitrate.then_some(job.output.bitrate_kbps.max(50)), frame_rate: fps, loop_filter: true, }; let mut color_enc = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone()); let mut alpha_enc = alpha.then(|| effectcraft_vp9enc::Vp9Encoder::new(cfg.clone())); let with_audio = wants_audio(job); - let mut opus = with_audio.then(|| effectcraft_opusenc::OpusEncoder::new(2, OPUS_BITRATE)); + let opus_channels = if job.output.audio_channels == 1 { 1usize } else { 2 }; + let mut opus = with_audio.then(|| effectcraft_opusenc::OpusEncoder::new(opus_channels as _, OPUS_BITRATE)); let total = st.total; let frame_ms = |k: u64| -> i64 { ((k as i128 * 1000 * rate.den as i128 + rate.num as i128 / 2) / rate.num as i128) as i64 }; let duration_ms = frame_ms(total) as f64; - let mut file = crate::out::create(job.sink, job.path)?; + let path = job.place(job.path, 1); + let mut file = crate::out::create(job.sink, &path)?; // EBML header. let mut head = vec![]; let mut eb = vec![]; @@ -201,7 +205,7 @@ pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re el_uint(&mut a, 0x56BB, 80_000_000); let mut au = vec![]; el_float(&mut au, 0xB5, OPUS_RATE as f64); - el_uint(&mut au, 0x9F, 2); + el_uint(&mut au, 0x9F, opus_channels as u64); el(&mut a, 0xE1, &au); el(&mut tracks, TRACK_ENTRY, &a); } @@ -220,10 +224,10 @@ pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re if end > cursor { let n = (end - cursor) as usize; let start = Tick(((cursor as i128 * TICKS_PER_SECOND as i128) / OPUS_RATE as i128) as i64); - pcm_left.extend(effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, start, n, OPUS_RATE)); + pcm_left.extend(mix(job, start, n, OPUS_RATE)); cursor = end; } - let fs = effectcraft_opusenc::OpusEncoder::FRAME_SIZE * 2; + let fs = effectcraft_opusenc::OpusEncoder::FRAME_SIZE * opus_channels; if finish && !pcm_left.is_empty() { // Pad the last packet (pre-skip + padding cover the encoder delay). let pad = (fs - pcm_left.len() % fs) % fs + fs; @@ -244,7 +248,7 @@ pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re // Frames render in parallel; the VP9 streams (inter-coded) encode in order, colour and // alpha side by side. let pixels: Vec> = - (i..end).into_par_iter().map(|k| rgba8(&render_frame(job, comp, k), comp, if alpha { Channels::Rgba } else { Channels::Rgb }, w, h)).collect(); + (i..end).into_par_iter().map(|k| job.pixels(&job.frame(comp, k), comp, if alpha { Channels::Rgba } else { colour_channels }, w, h)).collect(); let mut frames: Vec = Vec::with_capacity(pixels.len()); for (j, px) in pixels.iter().enumerate() { let (color, a) = rayon::join( @@ -288,7 +292,7 @@ pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Re file.write_all(&size).map_err(io)?; file.seek(SeekFrom::End(0)).map_err(io)?; let bytes = file.finish()?; - Ok(Report { path: job.path.to_string(), frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: with_audio }) + Ok(Report { path, frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: with_audio, log: None, overflow: vec![] }) } // ---------------------------------------------------------------- audio-only outputs @@ -306,14 +310,25 @@ fn ieee_extended(v: f64) -> [u8; 10] { out } -/// WAV or AIFF: the comp's audio over the render span, 16-bit stereo. -pub(crate) fn audio_file(job: &Job, comp: &Comp, aiff: bool, st: &mut State) -> Result { +/// WAV or AIFF: the comp's audio over the render span (Audio Output: sample rate, mono/stereo, +/// 16/24-bit integer or 32-bit float; AIFF has no float in plain AIFF, so it writes 24-bit). +pub(crate) fn audio_file(job: &Cx, comp: &Comp, aiff: bool, st: &mut State) -> Result { let sr = job.output.audio_sample_rate.clamp(8_000, 192_000); + let channels = if job.output.audio_channels == 1 { 1u16 } else { 2 }; + let fmt = match job.output.audio_format { + AudioFormat::F32 if aiff => AudioFormat::S24, + f => f, + }; + let bytes_per_sample: u16 = match fmt { + AudioFormat::S16 => 2, + AudioFormat::S24 => 3, + AudioFormat::F32 => 4, + }; let (span_start, span_end) = job.settings.span(comp); let start = span_start.to_units_floor(sr as i64); let end = span_end.to_units_floor(sr as i64); let n = (end - start).max(0) as usize; - let mut pcm = Vec::with_capacity(n * 4); + let mut pcm = Vec::with_capacity(n * (channels * bytes_per_sample) as usize); // Mix in one-second blocks, reporting progress in frames. let block = sr as usize; let mut done = 0usize; @@ -322,10 +337,7 @@ pub(crate) fn audio_file(job: &Job, comp: &Comp, aiff: bool, st: &mut State) -> while done < n { let k = block.min(n - done); let t = Tick((((start + done as i64) as i128 * TICKS_PER_SECOND as i128) / sr as i128) as i64); - for s in effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, t, k, sr) { - let v = (s.clamp(-1.0, 1.0) * 32767.0).round() as i16; - pcm.extend_from_slice(&if aiff { v.to_be_bytes() } else { v.to_le_bytes() }); - } + pcm.extend(pcm_bytes(&mix(job, t, k, sr), fmt, aiff)); done += k; let now = (done as u64 * total_frames / n.max(1) as u64).min(total_frames); if now > reported && st.total > 0 { @@ -336,13 +348,14 @@ pub(crate) fn audio_file(job: &Job, comp: &Comp, aiff: bool, st: &mut State) -> if reported < st.total { st.advance(st.total - reported)?; } + let block_align = channels * bytes_per_sample; + let frames = (pcm.len() / block_align as usize) as u32; let mut out = vec![]; - let frames = (pcm.len() / 4) as u32; if aiff { let mut comm = vec![]; - comm.extend_from_slice(&2u16.to_be_bytes()); + comm.extend_from_slice(&channels.to_be_bytes()); comm.extend_from_slice(&frames.to_be_bytes()); - comm.extend_from_slice(&16u16.to_be_bytes()); + comm.extend_from_slice(&(bytes_per_sample * 8).to_be_bytes()); comm.extend_from_slice(&ieee_extended(sr as f64)); let ssnd_len = 8 + pcm.len(); out.extend_from_slice(b"FORM"); @@ -359,24 +372,30 @@ pub(crate) fn audio_file(job: &Job, comp: &Comp, aiff: bool, st: &mut State) -> out.push(0); } } else { + // WAVE_FORMAT_PCM (1) or WAVE_FORMAT_IEEE_FLOAT (3). + let tag: u16 = if fmt == AudioFormat::F32 { 3 } else { 1 }; out.extend_from_slice(b"RIFF"); - out.extend_from_slice(&((36 + pcm.len()) as u32).to_le_bytes()); + out.extend_from_slice(&((36 + pcm.len() + pcm.len() % 2) as u32).to_le_bytes()); out.extend_from_slice(b"WAVEfmt "); out.extend_from_slice(&16u32.to_le_bytes()); - out.extend_from_slice(&1u16.to_le_bytes()); - out.extend_from_slice(&2u16.to_le_bytes()); + out.extend_from_slice(&tag.to_le_bytes()); + out.extend_from_slice(&channels.to_le_bytes()); out.extend_from_slice(&sr.to_le_bytes()); - out.extend_from_slice(&(sr * 4).to_le_bytes()); - out.extend_from_slice(&4u16.to_le_bytes()); - out.extend_from_slice(&16u16.to_le_bytes()); + out.extend_from_slice(&(sr * block_align as u32).to_le_bytes()); + out.extend_from_slice(&block_align.to_le_bytes()); + out.extend_from_slice(&(bytes_per_sample * 8).to_le_bytes()); out.extend_from_slice(b"data"); out.extend_from_slice(&(pcm.len() as u32).to_le_bytes()); out.extend_from_slice(&pcm); + if pcm.len() % 2 == 1 { + out.push(0); + } } - let mut file = crate::out::create(job.sink, job.path)?; + let path = job.place(job.path, out.len() as u64); + let mut file = crate::out::create(job.sink, &path)?; file.write_all(&out).map_err(io)?; let bytes = file.finish()?; - Ok(Report { path: job.path.to_string(), frames: 0, width: 0, height: 0, seconds: 0.0, bytes, audio: true }) + Ok(Report { path, frames: 0, width: 0, height: 0, seconds: 0.0, bytes, audio: true, log: None, overflow: vec![] }) } #[cfg(test)] diff --git a/crates/export/tests/render_settings.rs b/crates/export/tests/render_settings.rs new file mode 100644 index 00000000..3347efe1 --- /dev/null +++ b/crates/export/tests/render_settings.rs @@ -0,0 +1,310 @@ +//! Render Settings and Output Module features end to end (through a [`Sink`], so nothing touches +//! the disk): field rendering and 3:2 pulldown, crop / region of interest / resize, alpha-only +//! and premultiplied output, the solo and guide-layer overrides, the render log, storage +//! overflow and PCM audio formats. + +use std::sync::{Arc, Mutex}; + +use effectcraft_color::Label; +use effectcraft_export::render_queue::*; +use effectcraft_export::{Job, JobOptions, Report, Sink, StorageQuota, export}; +use effectcraft_keyframe::{Keyframe, Value}; +use effectcraft_project::{Comp, ItemId, ItemKind, LayerSource, Project, Solid, build}; +use effectcraft_render::NoFootage; +use effectcraft_time::{FrameRate, Tick}; + +const W: u32 = 64; +const H: u32 = 48; + +/// A comp at `rate`, black, with a white bar 8 px wide sweeping left to right (64 px/s) and a +/// static red block at the bottom left; `extra` adds layers. +fn project(rate: FrameRate, extra: impl FnOnce(&mut Project, &mut Comp)) -> (Project, ItemId) { + let mut p = Project::default(); + let mut comp = Comp::new(W, H, rate, Tick::from_seconds_f64(2.0)); + comp.background = [0.0, 0.0, 0.0]; + let bar = p.add_item("Bar", Label::Red, None, ItemKind::Solid(Solid { color: [1.0, 1.0, 1.0], width: 8, height: 32, pixel_aspect: 1.0 })); + let mut l = build::layer(&mut p, &comp, "Bar", LayerSource::Solid { item: bar }, (8, 32), None); + l.props.prop_mut("transform/position").unwrap().keys = + vec![Keyframe::new(Tick::ZERO, Value::Vec3([0.0, 16.0, 0.0])), Keyframe::new(Tick::from_seconds_f64(1.0), Value::Vec3([64.0, 16.0, 0.0]))]; + comp.layers.push(l); + let red = p.add_item("Block", Label::Red, None, ItemKind::Solid(Solid { color: [1.0, 0.0, 0.0], width: 16, height: 8, pixel_aspect: 1.0 })); + let mut b = build::layer(&mut p, &comp, "Block", LayerSource::Solid { item: red }, (16, 8), None); + b.props.prop_mut("transform/position").unwrap().value = Value::Vec3([8.0, 44.0, 0.0]); + comp.layers.push(b); + extra(&mut p, &mut comp); + let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.into())); + (p, cid) +} + +type Files = Arc)>>>; + +fn custom(start: f64, end: f64) -> RenderSettings { + RenderSettings { time_span: TimeSpan::Custom { start: Tick::from_seconds_f64(start), end: Tick::from_seconds_f64(end) }, ..Default::default() } +} + +fn run_with(p: &Project, cid: ItemId, s: &RenderSettings, om: &OutputModule, path: &str, options: JobOptions, cancel: bool) -> (Result, Files) { + let files: Files = Default::default(); + let f = files.clone(); + let sink: Box = Box::new(move |path: &str, data: Vec| f.lock().unwrap().push((path.to_string(), data))); + let job = Job { project: p, footage: &NoFootage, expr: None, accel: None, comp: cid, settings: s, output: om, path, sink: Some(&*sink), options }; + let r = export(&job, &mut |pr| !(cancel && pr.done > 0)).map_err(|e| e.to_string()); + (r, files) +} + +fn run(p: &Project, cid: ItemId, s: &RenderSettings, om: &OutputModule, path: &str) -> Files { + let (r, files) = run_with(p, cid, s, om, path, JobOptions::default(), false); + r.expect("export"); + files +} + +fn png(files: &Files, i: usize) -> image::RgbaImage { + let f = files.lock().unwrap(); + let pngs: Vec<&(String, Vec)> = f.iter().filter(|(p, _)| p.ends_with(".png")).collect(); + let mut pngs = pngs; + pngs.sort_by(|a, b| a.0.cmp(&b.0)); + image::load_from_memory(&pngs[i].1).expect("png").to_rgba8() +} + +fn row(img: &image::RgbaImage, y: u32) -> Vec { + (0..img.width()).flat_map(|x| img.get_pixel(x, y).0).collect() +} + +#[test] +fn field_render_interleaves_two_times() { + let (p, cid) = project(FrameRate::new(25, 1), |_, _| {}); + let om = OutputModule::for_format(OutputFormat::PngSequence); + // Frame 10 at 25 fps with fields: 0.40 s and 0.42 s. + for (fr, upper) in [(FieldRender::UpperFirst, true), (FieldRender::LowerFirst, false)] { + let s = RenderSettings { field_render: fr, ..custom(0.4, 0.44) }; + let fields = png(&run(&p, cid, &s, &om, "/f/field_[#####].png"), 0); + // Progressive references at 50 fps: frames 20 (0.40 s) and 21 (0.42 s). + let r = RenderSettings { frame_rate: Some(FrameRate::new(50, 1)), ..custom(0.4, 0.44) }; + let refs = run(&p, cid, &r, &om, "/f/ref_[#####].png"); + let (a, b) = (png(&refs, 0), png(&refs, 1)); + assert_ne!(row(&a, 10), row(&b, 10), "the bar moved between the fields"); + for y in 0..H { + let first = (y % 2 == 0) == upper; + assert_eq!(row(&fields, y), row(if first { &a } else { &b }, y), "{fr:?}: line {y}"); + } + } +} + +#[test] +fn pulldown_follows_the_cadence() { + let ntsc = FrameRate::new(30000, 1001); + let (p, cid) = project(ntsc, |_, _| {}); + let om = OutputModule::for_format(OutputFormat::PngSequence); + let five = 5.0 * 1001.0 / 30000.0; + let s = RenderSettings { field_render: FieldRender::UpperFirst, pulldown: Pulldown::Wssww, ..custom(0.0, five) }; + let out = run(&p, cid, &s, &om, "/p/pd_[#####].png"); + assert_eq!(out.lock().unwrap().len(), 5); + // Film frames at 23.976 fps. + let r = RenderSettings { frame_rate: Some(FrameRate::new(24000, 1001)), ..custom(0.0, 0.2) }; + let film = run(&p, cid, &r, &om, "/p/film_[#####].png"); + let mut pattern = String::new(); + for k in 0..5 { + let (a, b) = Pulldown::Wssww.fields(k as u64).unwrap(); + pattern.push(if a == b { 'W' } else { 'S' }); + let img = png(&out, k); + let (fa, fb) = (png(&film, a as usize), png(&film, b as usize)); + for y in 0..H { + assert_eq!(row(&img, y), row(if y % 2 == 0 { &fa } else { &fb }, y), "frame {k} line {y}"); + } + } + assert_eq!(pattern, "WSSWW"); +} + +#[test] +fn crop_roi_and_resize() { + let (p, cid) = project(FrameRate::new(25, 1), |_, _| {}); + let s = custom(0.4, 0.44); + let full = png(&run(&p, cid, &s, &OutputModule::for_format(OutputFormat::PngSequence), "/c/full_[#####].png"), 0); + let mut om = OutputModule::for_format(OutputFormat::PngSequence); + om.crop = Crop { enabled: true, top: 4, left: 10, bottom: 6, right: 2, ..Default::default() }; + let c = png(&run(&p, cid, &s, &om, "/c/crop_[#####].png"), 0); + assert_eq!((c.width(), c.height()), (52, 38)); + for (x, y) in [(0, 0), (20, 11), (51, 37), (3, 30)] { + assert_eq!(c.get_pixel(x, y), full.get_pixel(x + 10, y + 4), "({x}, {y})"); + } + // Region of interest at half resolution. + om.crop = Crop { enabled: true, use_roi: true, roi: Some([16.0, 8.0, 32.0, 32.0]), ..Default::default() }; + let half = RenderSettings { resolution: 0.5, ..s.clone() }; + let r = png(&run(&p, cid, &half, &om, "/c/roi_[#####].png"), 0); + assert_eq!((r.width(), r.height()), (16, 16)); + // Resize (fixed size, low and high quality) after the crop. + om.crop = Crop::default(); + for q in [ResizeQuality::Low, ResizeQuality::High] { + om.resize = Resize { enabled: true, width: 32, height: 20, lock_aspect: false, quality: q }; + let z = png(&run(&p, cid, &s, &om, "/c/resize_[#####].png"), 0); + assert_eq!((z.width(), z.height()), (32, 20)); + // The red block (bottom left) survives the resize. + let px = z.get_pixel(4, 18).0; + assert!(px[0] > 150 && px[1] < 80, "{q:?}: {px:?}"); + } + om.resize.lock_aspect = true; + let z = png(&run(&p, cid, &s, &om, "/c/aspect_[#####].png"), 0); + assert_eq!((z.width(), z.height()), (32, 24)); +} + +#[test] +fn alpha_only_and_premultiplied() { + let (p, cid) = project(FrameRate::new(25, 1), |p, comp| { + // A half-transparent green layer over the right half. + let g = p.add_item("Green", Label::Green, None, ItemKind::Solid(Solid { color: [0.0, 1.0, 0.0], width: 32, height: 48, pixel_aspect: 1.0 })); + let mut l = build::layer(p, comp, "Green", LayerSource::Solid { item: g }, (32, 48), None); + l.props.prop_mut("transform/position").unwrap().value = Value::Vec3([48.0, 24.0, 0.0]); + l.props.prop_mut("transform/opacity").unwrap().value = Value::Scalar(50.0); + comp.layers.insert(0, l); + }); + let s = custom(0.0, 0.04); + let mut om = OutputModule::for_format(OutputFormat::PngSequence); + om.channels = Channels::Alpha; + let a = png(&run(&p, cid, &s, &om, "/a/alpha_[#####].png"), 0); + assert_eq!(a.get_pixel(60, 2).0[0], 128, "half-opaque green"); + assert_eq!(a.get_pixel(20, 2).0[0], 0, "transparent"); + assert_eq!(a.get_pixel(8, 44).0[0], 255, "opaque block"); + om.channels = Channels::Rgba; + let straight = png(&run(&p, cid, &s, &om, "/a/s_[#####].png"), 0); + om.alpha_mode = AlphaMode::Premultiplied; + let premul = png(&run(&p, cid, &s, &om, "/a/p_[#####].png"), 0); + assert_eq!(straight.get_pixel(60, 2).0, [0, 255, 0, 128]); + assert_eq!(premul.get_pixel(60, 2).0, [0, 128, 0, 128]); +} + +#[test] +fn solo_and_guide_layer_overrides() { + let (p, cid) = project(FrameRate::new(25, 1), |p, comp| { + let g = p.add_item("Guide", Label::Green, None, ItemKind::Solid(Solid { color: [0.0, 0.0, 1.0], width: 8, height: 8, pixel_aspect: 1.0 })); + let mut l = build::layer(p, comp, "Guide", LayerSource::Solid { item: g }, (8, 8), None); + l.props.prop_mut("transform/position").unwrap().value = Value::Vec3([56.0, 44.0, 0.0]); + l.switches.guide = true; + comp.layers.insert(0, l); + // Solo the moving bar. + comp.layers[1].switches.solo = true; + }); + let om = OutputModule::for_format(OutputFormat::PngSequence); + let s = custom(0.4, 0.44); + let cur = png(&run(&p, cid, &s, &om, "/s/cur_[#####].png"), 0); + assert_eq!(cur.get_pixel(8, 44).0[0], 0, "solo: the red block is hidden"); + assert_eq!(cur.get_pixel(56, 44).0[2], 0, "guide layers off by default"); + let off = RenderSettings { solo: CurrentOrOff::AllOff, guide_layers: CurrentOrOff::Current, ..s.clone() }; + let all = png(&run(&p, cid, &off, &om, "/s/off_[#####].png"), 0); + assert_eq!(all.get_pixel(8, 44).0[0], 255, "Solo Switches: All Off"); + // (with solo off the guide renders too: Guide Layers: Current Settings) + assert_eq!(all.get_pixel(56, 44).0[2], 255); +} + +#[test] +fn render_logs() { + let (p, cid) = project(FrameRate::new(25, 1), |_, _| {}); + let om = OutputModule::for_format(OutputFormat::PngSequence); + let s = RenderSettings { field_render: FieldRender::UpperFirst, name: "DV-ish".into(), ..custom(0.0, 0.2) }; + let log_of = |files: &Files| { + files.lock().unwrap().iter().find(|(p, _)| p.ends_with("_RenderLog.txt")).map(|(p, d)| (p.clone(), String::from_utf8(d.clone()).unwrap())) + }; + // Errors Only + success: no log. + let (r, files) = run_with(&p, cid, &s, &om, "/l/a_[#####].png", JobOptions::default(), false); + assert!(r.unwrap().log.is_none()); + assert!(log_of(&files).is_none()); + // Plus Per Frame Info: settings and one line per frame. + let opts = JobOptions { log: RenderLog::PlusPerFrameInfo, label: "#1 Comp".into(), ..Default::default() }; + let (r, files) = run_with(&p, cid, &s, &om, "/l/b_[#####].png", opts, false); + let r = r.unwrap(); + let (path, text) = log_of(&files).expect("log"); + assert_eq!(r.log.as_deref(), Some(path.as_str())); + assert_eq!(path, "/l/b_RenderLog.txt"); + assert!(text.contains("Item: #1 Comp") && text.contains("Result: Done (5 frames"), "{text}"); + assert!(text.contains("Field Render: Upper Field First") && text.contains("Template: DV-ish") && text.contains("Format: PNG Sequence"), "{text}"); + assert_eq!(text.lines().filter(|l| l.contains("rendered in")).count(), 5, "{text}"); + assert!(text.contains("Frame 0 (0.000 s): rendered in") && text.contains("Frame 4 (0.160 s)"), "{text}"); + // Plus Settings: no per-frame lines. + let opts = JobOptions { log: RenderLog::PlusSettings, ..Default::default() }; + let (_, files) = run_with(&p, cid, &s, &om, "/l/c.gif", opts, false); + let (_, text) = log_of(&files).expect("log"); + assert!(text.contains("Render Settings:") && !text.contains("rendered in"), "{text}"); + // Errors Only + failure (the composition is gone): the error is logged. + let gif = OutputModule::for_format(OutputFormat::Gif); + let (r, files) = run_with(&p, ItemId(424_242), &s, &gif, "/l/d.gif", JobOptions::default(), false); + assert!(r.is_err()); + let (path, text) = log_of(&files).expect("error log"); + assert_eq!(path, "/l/d_RenderLog.txt"); + assert!(text.contains("Result: Failed") && text.contains("Error: composition not found") && !text.contains("Render Settings:"), "{text}"); + // A user stop is not an error: no log. + let (r, files) = run_with(&p, cid, &s, &gif, "/l/e.gif", JobOptions::default(), true); + assert!(r.is_err()); + assert!(log_of(&files).is_none()); +} + +/// The primary folder holds `limit` bytes; overflow folders are unlimited. +struct Quota { + limit: u64, + used: Mutex, +} + +impl StorageQuota for Quota { + fn has_room(&self, path: &str, bytes: u64) -> bool { + if !path.starts_with("/primary/") { + return true; + } + let mut u = self.used.lock().unwrap(); + if *u + bytes > self.limit { + return false; + } + *u += bytes; + true + } +} + +#[test] +fn storage_overflow_moves_files_to_the_overflow_folder() { + let (p, cid) = project(FrameRate::new(25, 1), |_, _| {}); + let om = OutputModule::for_format(OutputFormat::PngSequence); + let s = RenderSettings { skip_existing: false, ..custom(0.0, 0.4) }; + let q = Quota { limit: 1, used: Mutex::new(0) }; + let opts = JobOptions { storage: Some(&q), overflow: vec!["/overflow".into()], log: RenderLog::PlusSettings, ..Default::default() }; + let (r, files) = run_with(&p, cid, &s, &om, "/primary/seq_[#####].png", opts, false); + let r = r.unwrap(); + assert_eq!(r.overflow.len(), 10, "{:?}", r.overflow); + let f = files.lock().unwrap(); + assert_eq!(f.iter().filter(|(p, _)| p.starts_with("/overflow/seq_") && p.ends_with(".png")).count(), 10); + let log = f.iter().find(|(p, _)| p.ends_with("_RenderLog.txt")).map(|(_, d)| String::from_utf8_lossy(d).to_string()).unwrap(); + assert!(log.contains("Storage overflow: 10 file(s)"), "{log}"); + drop(f); + // Use Storage Overflow off: everything stays in place. + let off = RenderSettings { storage_overflow: false, ..s }; + let q = Quota { limit: 1, used: Mutex::new(0) }; + let opts = JobOptions { storage: Some(&q), overflow: vec!["/overflow".into()], ..Default::default() }; + let (r, files) = run_with(&p, cid, &off, &om, "/primary/seq_[#####].png", opts, false); + assert!(r.unwrap().overflow.is_empty()); + assert!(files.lock().unwrap().iter().all(|(p, _)| p.starts_with("/primary/"))); +} + +#[test] +fn wav_and_aiff_channels_and_formats() { + let (p, cid) = project(FrameRate::new(25, 1), |_, _| {}); + let s = custom(0.0, 0.5); + let mut om = OutputModule::for_format(OutputFormat::Wav); + om.audio_channels = 1; + om.audio_format = AudioFormat::S24; + om.audio_sample_rate = 44_100; + let files = run(&p, cid, &s, &om, "/w/a.wav"); + let d = files.lock().unwrap()[0].1.clone(); + assert_eq!(&d[..4], b"RIFF"); + let u16_at = |o: usize| u16::from_le_bytes([d[o], d[o + 1]]); + let u32_at = |o: usize| u32::from_le_bytes([d[o], d[o + 1], d[o + 2], d[o + 3]]); + assert_eq!((u16_at(20), u16_at(22), u32_at(24), u16_at(32), u16_at(34)), (1, 1, 44_100, 3, 24)); + assert_eq!(u32_at(40), 22_050 * 3, "half a second of mono 24-bit"); + om.audio_channels = 2; + om.audio_format = AudioFormat::F32; + let files = run(&p, cid, &s, &om, "/w/b.wav"); + let d = files.lock().unwrap()[0].1.clone(); + let u16_at = |o: usize| u16::from_le_bytes([d[o], d[o + 1]]); + assert_eq!((u16_at(20), u16_at(22), u16_at(32), u16_at(34)), (3, 2, 8, 32), "IEEE float stereo"); + // AIFF has no float: 24-bit. + let mut aiff = OutputModule::for_format(OutputFormat::Aiff); + aiff.audio_format = AudioFormat::F32; + let files = run(&p, cid, &s, &aiff, "/w/c.aif"); + let d = files.lock().unwrap()[0].1.clone(); + assert_eq!(&d[8..12], b"AIFF"); + assert_eq!(u16::from_be_bytes([d[26], d[27]]), 24, "sample size"); +} diff --git a/crates/export/tests/roundtrip.rs b/crates/export/tests/roundtrip.rs index 1ade594f..c6810cb0 100644 --- a/crates/export/tests/roundtrip.rs +++ b/crates/export/tests/roundtrip.rs @@ -55,7 +55,7 @@ fn settings() -> RenderSettings { fn run(p: &Project, cid: ItemId, footage: &dyn FootageSource, om: &OutputModule, path: &Path) -> effectcraft_export::Report { let s = settings(); let path = path.to_string_lossy().to_string(); - let job = Job { project: p, footage, expr: None, accel: None, comp: cid, settings: &s, output: om, path: &path, sink: None }; + let job = Job { project: p, footage, expr: None, accel: None, comp: cid, settings: &s, output: om, path: &path, sink: None, options: Default::default() }; let mut last = Progress::default(); let mut calls = 0; let r = export(&job, &mut |pr| { @@ -167,7 +167,18 @@ fn h264_odd_size_is_cropped_even() { let comp = p.comp(cid).expect("comp"); assert_eq!(s.output_size(comp).0 % 2, 1); let path = d.join("odd.mp4").to_string_lossy().to_string(); - let job = Job { project: &p, footage: &NoFootage, expr: None, accel: None, comp: cid, settings: &s, output: &om, path: &path, sink: None }; + let job = Job { + project: &p, + footage: &NoFootage, + expr: None, + accel: None, + comp: cid, + settings: &s, + output: &om, + path: &path, + sink: None, + options: Default::default(), + }; let r = export(&job, &mut |_| true).expect("export"); assert_eq!((r.width % 2, r.height % 2), (0, 0)); } @@ -287,7 +298,18 @@ fn cancel_stops_early() { let om = OutputModule::for_format(OutputFormat::PngSequence); let s = settings(); let path = d.join("c_[#####].png").to_string_lossy().to_string(); - let job = Job { project: &p, footage: &NoFootage, expr: None, accel: None, comp: cid, settings: &s, output: &om, path: &path, sink: None }; + let job = Job { + project: &p, + footage: &NoFootage, + expr: None, + accel: None, + comp: cid, + settings: &s, + output: &om, + path: &path, + sink: None, + options: Default::default(), + }; let r = export(&job, &mut |_| false); assert!(matches!(r, Err(effectcraft_export::ExportError::Cancelled))); let written = std::fs::read_dir(&d).map(|r| r.count()).unwrap_or(0); diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index b5de7714..4161abc7 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -40,9 +40,25 @@ impl Exporter for FileExporter { output: &job.item.output, path: job.path, sink: self.sink.as_deref(), + options: effectcraft_export::JobOptions { + log: job.item.log, + label: job.label.clone(), + storage: job.storage, + overflow: job.project.render_prefs.overflow_folders.clone(), + }, }; match effectcraft_export::export(&j, &mut |p| progress(p.done, p.total)) { - Ok(r) => Ok(ExportResult { path: r.path, frames: r.frames, width: r.width, height: r.height, bytes: r.bytes, seconds: r.seconds, audio: r.audio }), + Ok(r) => Ok(ExportResult { + path: r.path, + frames: r.frames, + width: r.width, + height: r.height, + bytes: r.bytes, + seconds: r.seconds, + audio: r.audio, + log: r.log, + overflow: r.overflow, + }), Err(effectcraft_export::ExportError::Cancelled) => Err(effectcraft_engine::render_queue::CANCELLED.into()), Err(e) => Err(e.to_string()), } diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 470c4edc..bf8fcc2a 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -8,6 +8,7 @@ pub mod build; pub mod essential; pub mod props; pub mod render_queue; +pub mod render_templates; pub mod styles; pub mod tracking; @@ -931,11 +932,25 @@ pub struct Project { /// The Render Queue (Composition ▸ Add to Render Queue). #[serde(default, skip_serializing_if = "Vec::is_empty")] pub render_queue: Vec, + /// Render Settings / Output Module templates and their defaults. + #[serde(default, skip_serializing_if = "render_templates::RenderTemplates::is_default")] + pub render_templates: render_templates::RenderTemplates, + /// Render Queue preferences (Notify, storage overflow folders). + #[serde(default, skip_serializing_if = "render_queue::RenderQueuePrefs::is_default")] + pub render_prefs: render_queue::RenderQueuePrefs, } impl Default for Project { fn default() -> Self { - Project { schema: SCHEMA_VERSION, settings: ProjectSettings::default(), items: BTreeMap::new(), next_id: 1, render_queue: Vec::new() } + Project { + schema: SCHEMA_VERSION, + settings: ProjectSettings::default(), + items: BTreeMap::new(), + next_id: 1, + render_queue: Vec::new(), + render_templates: Default::default(), + render_prefs: Default::default(), + } } } diff --git a/crates/project/src/render_queue.rs b/crates/project/src/render_queue.rs index d0f0c91f..35aed0be 100644 --- a/crates/project/src/render_queue.rs +++ b/crates/project/src/render_queue.rs @@ -84,38 +84,329 @@ impl ProxyUse { } } +/// Render Settings ▸ Field Render: interlaced output, each frame woven from two fields rendered +/// half a frame apart. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum FieldRender { + #[default] + Off, + /// The upper field (lines 0, 2, 4…) holds the earlier time. + UpperFirst, + LowerFirst, +} + +impl FieldRender { + pub const ALL: [FieldRender; 3] = [FieldRender::Off, FieldRender::UpperFirst, FieldRender::LowerFirst]; + pub fn label(self) -> &'static str { + match self { + FieldRender::Off => "Off", + FieldRender::UpperFirst => "Upper Field First", + FieldRender::LowerFirst => "Lower Field First", + } + } + pub fn parse(s: &str) -> Option { + match norm(s).as_str() { + "off" | "none" | "progressive" => Some(FieldRender::Off), + "upper" | "upperfirst" | "upperfieldfirst" => Some(FieldRender::UpperFirst), + "lower" | "lowerfirst" | "lowerfieldfirst" => Some(FieldRender::LowerFirst), + _ => None, + } + } +} + +fn norm(s: &str) -> String { + s.to_ascii_lowercase().replace([' ', '_', '-', '.', ':', '&'], "") +} + +/// Render Settings ▸ 3:2 Pulldown: with field rendering, four film frames (at 4/5 of the output +/// rate) spread over five interlaced frames in 2:3 cadence. The phase names which of the five +/// frames are Whole (both fields from one film frame) or Split. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum Pulldown { + #[default] + Off, + Wssww, + Sswww, + Swwws, + Wwwss, + Wwssw, +} + +/// Film frame (within a group of four) of each of the ten fields of the 2:3 cadence. +const CADENCE: [i64; 10] = [0, 0, 1, 1, 1, 2, 2, 3, 3, 3]; + +impl Pulldown { + pub const ALL: [Pulldown; 6] = [Pulldown::Off, Pulldown::Wssww, Pulldown::Sswww, Pulldown::Swwws, Pulldown::Wwwss, Pulldown::Wwssw]; + pub fn label(self) -> &'static str { + match self { + Pulldown::Off => "Off", + Pulldown::Wssww => "WSSWW", + Pulldown::Sswww => "SSWWW", + Pulldown::Swwws => "SWWWS", + Pulldown::Wwwss => "WWWSS", + Pulldown::Wwssw => "WWSSW", + } + } + pub fn parse(s: &str) -> Option { + let n = norm(s); + if matches!(n.as_str(), "off" | "none") { + return Some(Pulldown::Off); + } + Pulldown::ALL.into_iter().find(|p| p.label().eq_ignore_ascii_case(&n)) + } + /// Offset (in frames) into the WWSSW base cadence. + fn rotation(self) -> Option { + Some(match self { + Pulldown::Off => return None, + Pulldown::Wwssw => 0, + Pulldown::Wssww => 1, + Pulldown::Sswww => 2, + Pulldown::Swwws => 3, + Pulldown::Wwwss => 4, + }) + } + /// The film frames (counted from the first film frame of the render) of the first and + /// second field of output frame `k`. + pub fn fields(self, k: u64) -> Option<(i64, i64)> { + let r = self.rotation()?; + let film = |f: i64| f.div_euclid(10) * 4 + CADENCE[f.rem_euclid(10) as usize]; + let f0 = 2 * (k as i64 + r); + let base = film(2 * r); + Some((film(f0) - base, film(f0 + 1) - base)) + } + /// The W/S pattern of five consecutive output frames. + pub fn pattern(self) -> String { + (0..5).filter_map(|k| self.fields(k).map(|(a, b)| if a == b { 'W' } else { 'S' })).collect() + } +} + +/// A layer switch override (Render Settings ▸ Frame Blending / Motion Blur). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum SwitchOverride { + /// The layer switches and the composition's enable switch. + #[default] + Current, + /// Every layer whose switch is on, whatever the composition's enable switch. + OnForChecked, + OffForAll, +} + +impl SwitchOverride { + pub const ALL: [SwitchOverride; 3] = [SwitchOverride::Current, SwitchOverride::OnForChecked, SwitchOverride::OffForAll]; + pub fn label(self) -> &'static str { + match self { + SwitchOverride::Current => "Current Settings", + SwitchOverride::OnForChecked => "On for Checked Layers", + SwitchOverride::OffForAll => "Off for All Layers", + } + } + pub fn parse(s: &str) -> Option { + match norm(s).as_str() { + "current" | "currentsettings" => Some(SwitchOverride::Current), + "on" | "onforchecked" | "onforcheckedlayers" | "true" => Some(SwitchOverride::OnForChecked), + "off" | "offforall" | "offforalllayers" | "false" => Some(SwitchOverride::OffForAll), + _ => None, + } + } +} + +/// Render Settings ▸ Effects. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum EffectsMode { + #[default] + Current, + AllOn, + AllOff, +} + +impl EffectsMode { + pub const ALL: [EffectsMode; 3] = [EffectsMode::Current, EffectsMode::AllOn, EffectsMode::AllOff]; + pub fn label(self) -> &'static str { + match self { + EffectsMode::Current => "Current Settings", + EffectsMode::AllOn => "All On", + EffectsMode::AllOff => "All Off", + } + } + pub fn parse(s: &str) -> Option { + match norm(s).as_str() { + "current" | "currentsettings" => Some(EffectsMode::Current), + "allon" | "on" => Some(EffectsMode::AllOn), + "alloff" | "off" => Some(EffectsMode::AllOff), + _ => None, + } + } +} + +/// Render Settings ▸ Solo Switches / Guide Layers: the current switches, or all off. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum CurrentOrOff { + #[default] + Current, + AllOff, +} + +impl CurrentOrOff { + pub fn label(self) -> &'static str { + match self { + CurrentOrOff::Current => "Current Settings", + CurrentOrOff::AllOff => "All Off", + } + } + pub fn parse(s: &str) -> Option { + match norm(s).as_str() { + "current" | "currentsettings" | "on" => Some(CurrentOrOff::Current), + "alloff" | "off" => Some(CurrentOrOff::AllOff), + _ => None, + } + } +} + +/// Render Settings ▸ Color Depth. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum ColorDepth { + /// The project's bit depth. + #[default] + Current, + Bpc8, + Bpc16, + Bpc32, +} + +impl ColorDepth { + pub const ALL: [ColorDepth; 4] = [ColorDepth::Current, ColorDepth::Bpc8, ColorDepth::Bpc16, ColorDepth::Bpc32]; + pub fn label(self) -> &'static str { + match self { + ColorDepth::Current => "Current Settings", + ColorDepth::Bpc8 => "8 bits per channel", + ColorDepth::Bpc16 => "16 bits per channel", + ColorDepth::Bpc32 => "32 bits per channel", + } + } + pub fn parse(s: &str) -> Option { + match norm(s).as_str() { + "current" | "currentsettings" => Some(ColorDepth::Current), + "8" | "8bpc" | "8bitsperchannel" => Some(ColorDepth::Bpc8), + "16" | "16bpc" | "16bitsperchannel" => Some(ColorDepth::Bpc16), + "32" | "32bpc" | "32bitsperchannel" | "float" => Some(ColorDepth::Bpc32), + _ => None, + } + } + /// The depth renders use in a project of depth `project`. + pub fn resolve(self, project: crate::BitDepth) -> crate::BitDepth { + match self { + ColorDepth::Current => project, + ColorDepth::Bpc8 => crate::BitDepth::Bpc8, + ColorDepth::Bpc16 => crate::BitDepth::Bpc16, + ColorDepth::Bpc32 => crate::BitDepth::Bpc32, + } + } +} + +/// How one output frame is sampled in time. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FrameSample { + Progressive(Tick), + /// Two fields woven together: `first` goes to the dominant field. + Fields { + first: Tick, + second: Tick, + upper_first: bool, + }, +} + +fn default_true() -> bool { + true +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(default)] pub struct RenderSettings { + /// The template these settings came from ("" or "Custom" once edited). + #[serde(skip_serializing_if = "String::is_empty")] + pub name: String, pub quality: RenderQuality, /// Output scale (1 = Full, 0.5 = Half, 1/3 = Third, 0.25 = Quarter). pub resolution: f64, pub time_span: TimeSpan, - /// `None` = use the comp's frame rate. + /// `None` = use the comp's frame rate (Time Sampling ▸ Use comp's frame rate). pub frame_rate: Option, - /// Motion blur for layers with the switch on (and the comp's Enable Motion Blur). + /// Motion blur allowed at all (false = Off for All Layers; see `motion_blur_mode`). pub motion_blur: bool, + /// Motion Blur: Current Settings / On for Checked Layers / Off for All Layers. + pub motion_blur_mode: SwitchOverride, + /// Frame Blending: Current Settings / On for Checked Layers / Off for All Layers. + pub frame_blending: SwitchOverride, + /// Field Render. + pub field_render: FieldRender, + /// 3:2 Pulldown phase (with field rendering). + pub pulldown: Pulldown, + /// Effects: Current Settings / All On / All Off. + pub effects: EffectsMode, + /// Solo Switches: Current Settings / All Off. + pub solo: CurrentOrOff, + /// Guide Layers: Current Settings / All Off. + pub guide_layers: CurrentOrOff, + /// Color Depth. + pub color_depth: ColorDepth, /// Image sequences: skip frames whose file already exists. pub skip_existing: bool, /// Proxy Use (Best Settings: Use No Proxies). pub proxy_use: ProxyUse, + /// Use Storage Overflow: when the output volume is full, continue in the project's overflow + /// folders. + #[serde(default = "default_true")] + pub storage_overflow: bool, } impl Default for RenderSettings { fn default() -> Self { RenderSettings { + name: String::new(), quality: RenderQuality::Best, resolution: 1.0, time_span: TimeSpan::WorkArea, frame_rate: None, motion_blur: true, + motion_blur_mode: SwitchOverride::Current, + frame_blending: SwitchOverride::Current, + field_render: FieldRender::Off, + pulldown: Pulldown::Off, + effects: EffectsMode::Current, + solo: CurrentOrOff::Current, + guide_layers: CurrentOrOff::AllOff, + color_depth: ColorDepth::Current, skip_existing: false, proxy_use: ProxyUse::UseNone, + storage_overflow: true, } } } impl RenderSettings { + /// Effective Motion Blur override. + pub fn motion_blur_override(&self) -> SwitchOverride { + if self.motion_blur { self.motion_blur_mode } else { SwitchOverride::OffForAll } + } + /// How output frame `i` is sampled (field rendering and 3:2 pulldown). + pub fn sample(&self, comp: &Comp, i: u64) -> FrameSample { + let upper_first = match self.field_render { + FieldRender::Off => return FrameSample::Progressive(self.frame_time(comp, i)), + FieldRender::UpperFirst => true, + FieldRender::LowerFirst => false, + }; + let r = self.rate(comp); + if let Some((a, b)) = self.pulldown.fields(i) { + // Film frames at 4/5 of the output rate, from the first one starting in the span. + let film = FrameRate::new(r.num * 4, r.den * 5); + let f0 = Self::ceil_frame(film, self.span(comp).0); + return FrameSample::Fields { first: film.tick_of(f0 + a), second: film.tick_of(f0 + b), upper_first }; + } + let f = self.first_frame(comp) + i as i64; + let t = r.tick_of(f); + let half = Tick((r.tick_of(f + 1).0 - t.0) / 2); + FrameSample::Fields { first: t, second: Tick(t.0 + half.0), upper_first } + } /// The comp-time span `[start, end)` to render. pub fn span(&self, comp: &Comp) -> (Tick, Tick) { let (a, b) = match self.time_span { @@ -162,11 +453,45 @@ impl RenderSettings { _ => "Custom", } } - /// One-line summary shown next to "Render Settings:" (AE shows the template name). + /// One-line summary shown next to "Render Settings:" (the template name, like AE). pub fn summary(&self) -> String { + if !self.name.is_empty() { + return self.name.clone(); + } let q = if self.quality == RenderQuality::Best { "Best Settings" } else { "Draft Settings" }; format!("{q} · {}", self.resolution_label()) } + /// Every setting as `(label, value)` lines (render logs, the settings dialog). + pub fn describe(&self, comp: Option<&Comp>) -> Vec<(&'static str, String)> { + let span = match self.time_span { + TimeSpan::WorkArea => "Work Area Only".to_string(), + TimeSpan::LengthOfComp => "Length of Comp".to_string(), + TimeSpan::Custom { start, end } => format!("Custom ({:.3}s – {:.3}s)", start.seconds(), end.seconds()), + }; + let rate = match (self.frame_rate, comp) { + (Some(r), _) => format!("{:.3} fps", r.as_f64()), + (None, Some(c)) => format!("Use comp's frame rate ({:.3})", c.frame_rate.as_f64()), + (None, None) => "Use comp's frame rate".into(), + }; + vec![ + ("Template", if self.name.is_empty() { "Custom".into() } else { self.name.clone() }), + ("Quality", format!("{:?}", self.quality)), + ("Resolution", format!("{} ({:.3})", self.resolution_label(), self.resolution)), + ("Proxy Use", self.proxy_use.label().into()), + ("Effects", self.effects.label().into()), + ("Solo Switches", self.solo.label().into()), + ("Guide Layers", self.guide_layers.label().into()), + ("Color Depth", self.color_depth.label().into()), + ("Frame Blending", self.frame_blending.label().into()), + ("Field Render", self.field_render.label().into()), + ("3:2 Pulldown", self.pulldown.label().into()), + ("Motion Blur", self.motion_blur_override().label().into()), + ("Time Span", span), + ("Frame Rate", rate), + ("Skip Existing Files", if self.skip_existing { "On" } else { "Off" }.into()), + ("Use Storage Overflow", if self.storage_overflow { "On" } else { "Off" }.into()), + ] + } } /// Output Module ▸ Format. @@ -287,8 +612,30 @@ pub enum Channels { /// Opaque, composited over the comp background colour. #[default] Rgb, - /// With straight (unmatted) alpha. + /// With alpha (straight or premultiplied: [`AlphaMode`]). Rgba, + /// The alpha channel alone, as an opaque greyscale matte. + Alpha, +} + +impl Channels { + pub fn label(self) -> &'static str { + match self { + Channels::Rgb => "RGB", + Channels::Rgba => "RGB + Alpha", + Channels::Alpha => "Alpha", + } + } +} + +/// Output Module ▸ Color: how RGB is stored with alpha. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum AlphaMode { + /// Unmatted colour. + #[default] + Straight, + /// Colour multiplied by alpha (matted with black). + Premultiplied, } /// Output Module ▸ Audio Output. @@ -301,6 +648,121 @@ pub enum AudioOutput { Off, } +/// Output Module ▸ Audio Output ▸ sample format (PCM outputs: WAV, AIFF, QuickTime). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum AudioFormat { + #[default] + S16, + S24, + /// 32-bit float (WAV, QuickTime; AIFF writes 24-bit). + F32, +} + +impl AudioFormat { + pub fn label(self) -> &'static str { + match self { + AudioFormat::S16 => "16 Bit", + AudioFormat::S24 => "24 Bit", + AudioFormat::F32 => "32 Bit Float", + } + } + pub fn parse(s: &str) -> Option { + match norm(s).as_str() { + "16" | "16bit" | "s16" => Some(AudioFormat::S16), + "24" | "24bit" | "s24" => Some(AudioFormat::S24), + "32" | "32bit" | "32bitfloat" | "f32" | "float" => Some(AudioFormat::F32), + _ => None, + } + } +} + +/// Output Module ▸ Crop (pixels of the rendered frame; negative values add transparent pixels). +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct Crop { + pub enabled: bool, + /// Use Region of Interest: crop to `roi` (comp pixels, captured from the viewer's region of + /// interest when the option is turned on) instead of the edge values. + pub use_roi: bool, + pub roi: Option<[f64; 4]>, + pub top: i32, + pub left: i32, + pub bottom: i32, + pub right: i32, +} + +impl Crop { + /// Edges `(top, left, bottom, right)` in pixels of a `w`×`h` frame rendered at `scale`. + pub fn edges(&self, w: u32, h: u32, scale: f64) -> (i32, i32, i32, i32) { + if !self.enabled { + return (0, 0, 0, 0); + } + if self.use_roi + && let Some([x, y, rw, rh]) = self.roi + { + let l = (x * scale).round() as i32; + let t = (y * scale).round() as i32; + let r = w as i32 - ((x + rw) * scale).round() as i32; + let b = h as i32 - ((y + rh) * scale).round() as i32; + return (t, l, b, r); + } + (self.top, self.left, self.bottom, self.right) + } +} + +/// Output Module ▸ Resize quality. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum ResizeQuality { + /// Bilinear. + Low, + /// Bicubic. + #[default] + High, +} + +/// Output Module ▸ Resize. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct Resize { + pub enabled: bool, + pub width: u32, + pub height: u32, + /// Lock Aspect Ratio: setting one dimension derives the other from the rendered frame. + pub lock_aspect: bool, + pub quality: ResizeQuality, +} + +impl Default for Resize { + fn default() -> Self { + Resize { enabled: false, width: 1920, height: 1080, lock_aspect: true, quality: ResizeQuality::High } + } +} + +/// Output Module ▸ Resize ▸ Resize to: presets. +pub const RESIZE_PRESETS: [(&str, u32, u32); 10] = [ + ("HDTV 1080", 1920, 1080), + ("HDTV 720", 1280, 720), + ("UHD 4K", 3840, 2160), + ("DCI 2K", 2048, 1080), + ("DCI 4K", 4096, 2160), + ("NTSC DV", 720, 480), + ("PAL D1/DV", 720, 576), + ("Square 1080", 1080, 1080), + ("Vertical 1080×1920", 1080, 1920), + ("Web 640×360", 640, 360), +]; + +/// Output Module ▸ Post-Render Action. +pub fn post_render_parse(s: &str) -> Option { + match norm(s).as_str() { + "none" => Some(PostRenderAction::None), + "import" => Some(PostRenderAction::Import), + "importreplace" | "importandreplace" | "importandreplaceusage" | "importreplaceusage" | "replace" => Some(PostRenderAction::ImportAndReplace), + "setproxy" | "proxy" => Some(PostRenderAction::SetProxy), + _ => None, + } +} + /// ProRes flavour. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub enum ProResProfile { @@ -341,38 +803,133 @@ impl ProResProfile { #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(default)] pub struct OutputModule { + /// The template this module came from ("" or "Custom" once edited). + #[serde(skip_serializing_if = "String::is_empty")] + pub name: String, pub format: OutputFormat, pub channels: Channels, + /// Color: straight or premultiplied RGB with alpha. + pub alpha_mode: AlphaMode, /// Output path or template (see the module docs). pub output: String, - /// JPEG quality 1–100. + /// JPEG / WebM quality 1–100. pub quality: u8, - /// H.264 target bitrate. + /// H.264 target bitrate (and WebM's when `webm_bitrate`). pub bitrate_kbps: u32, + /// WebM: rate control by `bitrate_kbps` instead of `quality`. + pub webm_bitrate: bool, + /// Movies: frames between key frames (0 = automatic, two seconds). + pub keyframe_interval: u32, pub prores_profile: ProResProfile, + pub crop: Crop, + pub resize: Resize, pub audio: AudioOutput, pub audio_sample_rate: u32, + /// 1 (mono) or 2 (stereo). + pub audio_channels: u8, + pub audio_format: AudioFormat, /// GIF: loop forever. pub gif_loop: bool, + /// Include Project Link: kept for parity; EffectCraft's writers have no place to store a + /// link back to the project, so it changes nothing in the file. + pub include_project_link: bool, } impl Default for OutputModule { fn default() -> Self { OutputModule { + name: String::new(), format: OutputFormat::H264, channels: Channels::Rgb, + alpha_mode: AlphaMode::Straight, output: DEFAULT_TEMPLATE.into(), quality: 90, bitrate_kbps: 10_000, + webm_bitrate: false, + keyframe_interval: 0, prores_profile: ProResProfile::Hq, + crop: Crop::default(), + resize: Resize::default(), audio: AudioOutput::Auto, audio_sample_rate: 48_000, + audio_channels: 2, + audio_format: AudioFormat::S16, gif_loop: true, + include_project_link: true, } } } impl OutputModule { + /// The size of the frames this module writes for frames rendered at `w`×`h` with render + /// scale `scale`: after Crop and Resize (before codec rounding). + pub fn frame_size(&self, w: u32, h: u32, scale: f64) -> (u32, u32) { + let (t, l, b, r) = self.crop.edges(w, h, scale); + let cw = (w as i32 - l - r).max(1) as u32; + let ch = (h as i32 - t - b).max(1) as u32; + if !self.resize.enabled { + return (cw, ch); + } + let (mut rw, mut rh) = (self.resize.width.max(1), self.resize.height.max(1)); + if self.resize.lock_aspect { + // Width rules; height follows the cropped frame's aspect. + rh = ((rw as f64 * ch as f64 / cw as f64).round() as u32).max(1); + } + rw = rw.min(30_000); + rh = rh.min(30_000); + (rw, rh) + } + /// The written frame size of an item (Render Settings resolution, Crop, Resize, codec + /// rounding). + pub fn output_size(&self, comp: &Comp, settings: &RenderSettings) -> (u32, u32) { + let (w, h) = settings.output_size(comp); + let (w, h) = self.frame_size(w, h, settings.resolution.clamp(0.01, 4.0)); + self.format.coded_size(w, h) + } + /// Every setting as `(label, value)` lines (render logs). + pub fn describe(&self) -> Vec<(&'static str, String)> { + let mut v = vec![ + ("Template", if self.name.is_empty() { "Custom".into() } else { self.name.clone() }), + ("Format", self.format.label().into()), + ("Channels", self.channels.label().into()), + ("Output To", self.output.clone()), + ]; + if self.channels == Channels::Rgba { + v.insert(3, ("Color", format!("{:?}", self.alpha_mode))); + } + match self.format { + OutputFormat::H264 => v.push(("Bitrate", format!("{} kbps", self.bitrate_kbps))), + OutputFormat::ProRes => v.push(("Codec", self.prores_profile.label().into())), + OutputFormat::WebM if self.webm_bitrate => v.push(("Bitrate", format!("{} kbps", self.bitrate_kbps))), + OutputFormat::WebM | OutputFormat::JpegSequence => v.push(("Quality", self.quality.to_string())), + _ => {} + } + if self.crop.enabled { + let c = &self.crop; + v.push(( + "Crop", + if c.use_roi { format!("Region of Interest {:?}", c.roi) } else { format!("T {} L {} B {} R {}", c.top, c.left, c.bottom, c.right) }, + )); + } + if self.resize.enabled { + let r = &self.resize; + v.push(("Resize", format!("{}×{} ({:?} quality{})", r.width, r.height, r.quality, if r.lock_aspect { ", aspect locked" } else { "" }))); + } + if self.format.supports_audio() { + v.push(( + "Audio Output", + format!( + "{:?} · {} Hz · {} · {}", + self.audio, + self.audio_sample_rate, + if self.audio_channels == 1 { "Mono" } else { "Stereo" }, + self.audio_format.label() + ), + )); + } + v.push(("Include Project Link", if self.include_project_link { "On" } else { "Off" }.into())); + v + } pub fn for_format(format: OutputFormat) -> OutputModule { let mut m = OutputModule { format, ..Default::default() }; if format.is_sequence() { @@ -384,7 +941,7 @@ impl OutputModule { pub fn set_format(&mut self, format: OutputFormat) { let old = self.format; self.format = format; - if !format.supports_alpha() { + if !format.supports_alpha() && self.channels == Channels::Rgba { self.channels = Channels::Rgb; } if self.output.contains("[fileExtension]") { @@ -406,10 +963,10 @@ impl OutputModule { } /// One-line summary shown next to "Output Module:". pub fn summary(&self) -> String { - let ch = match self.channels { - Channels::Rgb => "RGB", - Channels::Rgba => "RGB + Alpha", - }; + if !self.name.is_empty() { + return self.name.clone(); + } + let ch = self.channels.label(); match self.format { OutputFormat::H264 => format!("H.264 · {} kbps", self.bitrate_kbps), OutputFormat::ProRes => format!( @@ -478,6 +1035,85 @@ pub struct RenderQueueItem { /// Post-Render Action of the first output module. #[serde(default, skip_serializing_if = "PostRenderAction::is_none")] pub post_render: PostRenderAction, + /// The Log menu: what the render log next to the output records. + #[serde(default, skip_serializing_if = "RenderLog::is_default")] + pub log: RenderLog, +} + +/// Render Queue item ▸ Log. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum RenderLog { + /// A log only when the render fails. + #[default] + ErrorsOnly, + /// Always: the result plus every Render Settings and Output Module setting. + PlusSettings, + /// Also one line per rendered frame. + PlusPerFrameInfo, +} + +impl RenderLog { + pub const ALL: [RenderLog; 3] = [RenderLog::ErrorsOnly, RenderLog::PlusSettings, RenderLog::PlusPerFrameInfo]; + pub fn is_default(&self) -> bool { + *self == RenderLog::ErrorsOnly + } + pub fn label(self) -> &'static str { + match self { + RenderLog::ErrorsOnly => "Errors Only", + RenderLog::PlusSettings => "Plus Settings", + RenderLog::PlusPerFrameInfo => "Plus Per Frame Info", + } + } + pub fn parse(s: &str) -> Option { + match norm(s).as_str() { + "errors" | "errorsonly" => Some(RenderLog::ErrorsOnly), + "settings" | "plussettings" => Some(RenderLog::PlusSettings), + "perframe" | "plusperframe" | "plusperframeinfo" | "frames" => Some(RenderLog::PlusPerFrameInfo), + _ => None, + } + } +} + +/// The render log's path for an output: next to it, `_RenderLog.txt` (a sequence's +/// frame-number run is dropped). +pub fn log_path(output: &str) -> String { + let p = std::path::Path::new(output); + let stem = p.file_stem().map(|s| s.to_string_lossy().to_string()).unwrap_or_default(); + let mut stem: String = stem.replace("_[#####]", "").replace("[#####]", ""); + if let Some(i) = stem.find('#') { + let n = stem[i..].chars().take_while(|c| *c == '#').count(); + stem.replace_range(i..i + n, ""); + } + let stem = stem.trim_end_matches(['_', '.', ' ']); + let name = format!("{}_RenderLog.txt", if stem.is_empty() { "Render" } else { stem }); + match p.parent().filter(|d| !d.as_os_str().is_empty()) { + Some(d) => d.join(name).to_string_lossy().to_string(), + None => name, + } +} + +/// Storage hook (Render Settings ▸ Use Storage Overflow): whether `bytes` more fit on the volume +/// holding `path`. The desktop app has no quota by default; tests and hosts with quotas supply +/// one. +pub trait StorageQuota: Send + Sync { + fn has_room(&self, path: &str, bytes: u64) -> bool; +} + +/// Project-wide Render Queue preferences. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct RenderQueuePrefs { + /// Notify when the render finishes (a sound and a notification in the desktop app). + pub notify: bool, + /// Storage overflow folders, used in order when the output volume is full. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub overflow_folders: Vec, +} + +impl RenderQueuePrefs { + pub fn is_default(&self) -> bool { + *self == RenderQueuePrefs::default() + } } /// What happens after an item renders (Output Module ▸ Post-Render Action). @@ -521,6 +1157,7 @@ impl RenderQueueItem { last_output: None, extra_outputs: vec![], post_render: PostRenderAction::None, + log: RenderLog::ErrorsOnly, } } /// Every output module, the first one first. @@ -612,6 +1249,90 @@ mod tests { assert_eq!(ProResProfile::from_name("hq"), Some(ProResProfile::Hq)); } + #[test] + fn pulldown_cadence() { + assert_eq!(Pulldown::Wwssw.pattern(), "WWSSW"); + for p in &Pulldown::ALL[1..] { + assert_eq!(p.pattern(), p.label(), "{p:?}"); + } + // 2:3 cadence: ten fields carry four film frames; every output frame starts on film 0. + for p in &Pulldown::ALL[1..] { + let mut fields = vec![]; + for k in 0..10 { + let (a, b) = p.fields(k).unwrap(); + fields.extend([a, b]); + } + assert_eq!(fields[0], 0, "{p:?}"); + assert!(fields.windows(2).all(|w| w[1] == w[0] || w[1] == w[0] + 1), "{p:?}: {fields:?}"); + // 20 fields = 8 film frames' worth of cadence (a phase may start or end mid-frame). + assert!((7..=8).contains(&(*fields.last().unwrap() - fields[0])), "{p:?} {fields:?}"); + // Each film frame appears in 2 or 3 consecutive fields, alternating. + let mut runs = vec![]; + let mut n = 1; + for w in fields.windows(2) { + if w[1] == w[0] { + n += 1; + } else { + runs.push(n); + n = 1; + } + } + assert!(runs[1..].iter().all(|r| *r == 2 || *r == 3), "{p:?}: {runs:?}"); + assert!(runs[1..].windows(2).all(|w| w[0] != w[1]), "2 and 3 alternate: {p:?}: {runs:?}"); + } + assert_eq!(Pulldown::parse("sswww"), Some(Pulldown::Sswww)); + // With field rendering at 29.97, film frames are 23.976 fps. + let c = Comp::new(64, 48, FrameRate::new(30000, 1001), Tick::from_seconds_f64(2.0)); + let s = RenderSettings { field_render: FieldRender::UpperFirst, pulldown: Pulldown::Wssww, time_span: TimeSpan::LengthOfComp, ..Default::default() }; + let film = FrameRate::new(24000, 1001); + match s.sample(&c, 1) { + FrameSample::Fields { first, second, upper_first } => { + assert!(upper_first); + assert_eq!((first, second), (film.tick_of(0), film.tick_of(1)), "WSSWW: frame 1 is split between film frames 0 and 1"); + } + f => panic!("{f:?}"), + } + } + + #[test] + fn field_samples_are_half_a_frame_apart() { + let c = Comp::new(64, 48, FrameRate::new(25, 1), Tick::from_seconds_f64(2.0)); + let s = RenderSettings { field_render: FieldRender::LowerFirst, time_span: TimeSpan::LengthOfComp, ..Default::default() }; + assert_eq!(s.sample(&c, 3), FrameSample::Fields { first: Tick::from_seconds_f64(0.12), second: Tick::from_seconds_f64(0.14), upper_first: false }); + let p = RenderSettings { time_span: TimeSpan::LengthOfComp, ..Default::default() }; + assert_eq!(p.sample(&c, 3), FrameSample::Progressive(Tick::from_seconds_f64(0.12))); + } + + #[test] + fn crop_and_resize_sizes() { + let c = Comp::new(1920, 1080, FrameRate::new(25, 1), Tick::from_seconds_f64(1.0)); + let rs = RenderSettings::default(); + let mut m = OutputModule::default(); + assert_eq!(m.output_size(&c, &rs), (1920, 1080)); + m.crop = Crop { enabled: true, top: 10, left: 20, bottom: 30, right: 41, ..Default::default() }; + assert_eq!(m.output_size(&c, &rs), (1858, 1040), "H.264 rounds the odd width down"); + m.format = OutputFormat::PngSequence; + assert_eq!(m.output_size(&c, &rs), (1859, 1040)); + // Region of interest, at half resolution. + m.crop = Crop { enabled: true, use_roi: true, roi: Some([100.0, 50.0, 800.0, 400.0]), ..Default::default() }; + let half = RenderSettings { resolution: 0.5, ..Default::default() }; + assert_eq!(m.output_size(&c, &half), (400, 200)); + // Negative crop pads. + m.crop = Crop { enabled: true, top: -10, bottom: -10, ..Default::default() }; + assert_eq!(m.output_size(&c, &rs), (1920, 1100)); + // Resize: fixed, and with the aspect locked to the (cropped) frame. + m.crop = Crop::default(); + m.resize = Resize { enabled: true, width: 1280, height: 999, lock_aspect: false, quality: ResizeQuality::High }; + assert_eq!(m.output_size(&c, &rs), (1280, 999)); + m.resize.lock_aspect = true; + assert_eq!(m.output_size(&c, &rs), (1280, 720)); + m.crop = Crop { enabled: true, left: 480, right: 480, ..Default::default() }; + assert_eq!(m.output_size(&c, &rs), (1280, 1440), "960×1080 cropped, then 1280 wide at 8:9"); + assert_eq!(log_path("/out/Main_[#####].png"), "/out/Main_RenderLog.txt"); + assert_eq!(log_path("/out/a.mov"), "/out/a_RenderLog.txt"); + assert_eq!(log_path("seq###.tif"), "seq_RenderLog.txt"); + } + #[test] fn spans() { let mut c = Comp::new(100, 50, FrameRate::new(25, 1), Tick::from_seconds_f64(4.0)); diff --git a/crates/project/src/render_templates.rs b/crates/project/src/render_templates.rs new file mode 100644 index 00000000..cf52b000 --- /dev/null +++ b/crates/project/src/render_templates.rs @@ -0,0 +1,323 @@ +//! Render Settings and Output Module templates (Edit ▸ Templates ▸ Render Settings… / Output +//! Module…): the built-in presets, templates saved with the project, and the defaults new +//! Render Queue items, pre-renders and proxies start from. +//! +//! Built-ins always exist; a saved template with a built-in's name overrides it. Templates +//! carry their name in [`RenderSettings::name`] / [`OutputModule::name`], which the Render Queue +//! shows like After Effects does. + +use serde::{Deserialize, Serialize}; + +use crate::render_queue::*; +use effectcraft_time::FrameRate; + +/// A default slot (the Defaults section of the Templates dialogs). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum TemplateSlot { + /// New Render Queue items rendering a movie. + Movie, + /// Single-frame renders (Composition ▸ Save Frame As ▸ File…). + Still, + /// Composition ▸ Pre-render…. + PreRender, + /// File ▸ Create Proxy ▸ Movie…. + MovieProxy, + /// File ▸ Create Proxy ▸ Still…. + StillProxy, +} + +impl TemplateSlot { + pub const ALL: [TemplateSlot; 5] = [TemplateSlot::Movie, TemplateSlot::Still, TemplateSlot::PreRender, TemplateSlot::MovieProxy, TemplateSlot::StillProxy]; + pub fn label(self) -> &'static str { + match self { + TemplateSlot::Movie => "Movie Default", + TemplateSlot::Still => "Frame Default", + TemplateSlot::PreRender => "Pre-Render Default", + TemplateSlot::MovieProxy => "Movie Proxy Default", + TemplateSlot::StillProxy => "Still Proxy Default", + } + } + pub fn parse(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '_', '-'], "").as_str() { + "movie" | "moviedefault" => Some(TemplateSlot::Movie), + "still" | "frame" | "framedefault" | "stilldefault" => Some(TemplateSlot::Still), + "prerender" | "prerenderdefault" => Some(TemplateSlot::PreRender), + "movieproxy" | "movieproxydefault" => Some(TemplateSlot::MovieProxy), + "stillproxy" | "stillproxydefault" => Some(TemplateSlot::StillProxy), + _ => None, + } + } +} + +/// Which template list. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum TemplateKind { + RenderSettings, + OutputModule, +} + +impl TemplateKind { + pub fn parse(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '_', '-'], "").as_str() { + "rendersettings" | "settings" | "render" => Some(TemplateKind::RenderSettings), + "outputmodule" | "output" | "module" => Some(TemplateKind::OutputModule), + _ => None, + } + } +} + +/// Defaults per slot (template names). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct TemplateDefaults { + pub render_settings: Vec<(TemplateSlot, String)>, + pub output_modules: Vec<(TemplateSlot, String)>, +} + +impl Default for TemplateDefaults { + fn default() -> Self { + use TemplateSlot::*; + let s = |v: &[(TemplateSlot, &str)]| v.iter().map(|(k, n)| (*k, n.to_string())).collect(); + TemplateDefaults { + render_settings: s(&[ + (Movie, "Best Settings"), + (Still, "Current Settings"), + (PreRender, "Best Settings"), + (MovieProxy, "Draft Settings"), + (StillProxy, "Best Settings"), + ]), + output_modules: s(&[ + (Movie, "H.264 - Match Render Settings - 15 Mbps"), + (Still, "PNG Sequence"), + (PreRender, "Lossless with Alpha"), + (MovieProxy, "Lossless with Alpha"), + (StillProxy, "PNG Sequence with Alpha"), + ]), + } + } +} + +/// Templates saved with the project, plus the defaults. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct RenderTemplates { + #[serde(skip_serializing_if = "Vec::is_empty")] + pub render_settings: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub output_modules: Vec, + pub defaults: TemplateDefaults, +} + +/// The built-in Render Settings templates. +pub fn builtin_render_settings() -> Vec { + let best = RenderSettings { + name: "Best Settings".into(), + frame_blending: SwitchOverride::OnForChecked, + motion_blur_mode: SwitchOverride::OnForChecked, + ..RenderSettings::default() + }; + let draft = RenderSettings { + name: "Draft Settings".into(), + quality: RenderQuality::Draft, + resolution: 0.5, + proxy_use: ProxyUse::CurrentSettings, + frame_blending: SwitchOverride::Current, + motion_blur_mode: SwitchOverride::Current, + ..RenderSettings::default() + }; + let dv = + RenderSettings { name: "DV Settings".into(), field_render: FieldRender::LowerFirst, frame_rate: Some(FrameRate::new(30000, 1001)), ..best.clone() }; + let multi = RenderSettings { name: "Multi-Machine Settings".into(), skip_existing: true, ..best.clone() }; + let current = RenderSettings { + name: "Current Settings".into(), + proxy_use: ProxyUse::CurrentSettings, + guide_layers: CurrentOrOff::Current, + ..RenderSettings::default() + }; + vec![best, draft, dv, multi, current] +} + +fn om(name: &str, format: OutputFormat) -> OutputModule { + let mut m = OutputModule::for_format(format); + m.name = name.into(); + m +} + +/// The built-in Output Module templates. +pub fn builtin_output_modules() -> Vec { + let mut v = vec![]; + let mut hq = om("High Quality", OutputFormat::ProRes); + hq.prores_profile = ProResProfile::Hq; + v.push(hq); + let mut hqa = om("High Quality with Alpha", OutputFormat::ProRes); + hqa.prores_profile = ProResProfile::P4444; + hqa.channels = Channels::Rgba; + hqa.alpha_mode = AlphaMode::Premultiplied; + v.push(hqa); + // Lossless: the highest-fidelity movie codec this build writes (ProRes 4444 XQ). + let mut ll = om("Lossless", OutputFormat::ProRes); + ll.prores_profile = ProResProfile::P4444Xq; + v.push(ll); + let mut lla = om("Lossless with Alpha", OutputFormat::ProRes); + lla.prores_profile = ProResProfile::P4444Xq; + lla.channels = Channels::Rgba; + v.push(lla); + for mbps in [5u32, 15, 40] { + let mut h = om(&format!("H.264 - Match Render Settings - {mbps} Mbps"), OutputFormat::H264); + h.bitrate_kbps = mbps * 1000; + v.push(h); + } + let mut alpha = om("Alpha Only", OutputFormat::ProRes); + alpha.channels = Channels::Alpha; + v.push(alpha); + v.push(om("AIFF 48kHz", OutputFormat::Aiff)); + v.push(om("WAV 48kHz", OutputFormat::Wav)); + v.push(om("PNG Sequence", OutputFormat::PngSequence)); + let mut pa = om("PNG Sequence with Alpha", OutputFormat::PngSequence); + pa.channels = Channels::Rgba; + v.push(pa); + let mut ta = om("TIFF Sequence with Alpha", OutputFormat::TiffSequence); + ta.channels = Channels::Rgba; + v.push(ta); + v.push(om("JPEG Sequence", OutputFormat::JpegSequence)); + let mut exr = om("OpenEXR Sequence with Alpha", OutputFormat::ExrSequence); + exr.channels = Channels::Rgba; + v.push(exr); + v.push(om("Animated GIF", OutputFormat::Gif)); + v.push(om("WebM (VP9 + Opus)", OutputFormat::WebM)); + let mut wa = om("WebM with Alpha", OutputFormat::WebM); + wa.channels = Channels::Rgba; + v.push(wa); + v +} + +fn same(a: &str, b: &str) -> bool { + a.trim().eq_ignore_ascii_case(b.trim()) +} + +impl RenderTemplates { + pub fn is_default(&self) -> bool { + *self == RenderTemplates::default() + } + /// Every Render Settings template: built-ins (unless overridden), then saved ones. + pub fn render_settings_list(&self) -> Vec { + let mut v: Vec = + builtin_render_settings().into_iter().filter(|b| !self.render_settings.iter().any(|u| same(&u.name, &b.name))).collect(); + v.extend(self.render_settings.iter().cloned()); + v + } + pub fn output_module_list(&self) -> Vec { + let mut v: Vec = builtin_output_modules().into_iter().filter(|b| !self.output_modules.iter().any(|u| same(&u.name, &b.name))).collect(); + v.extend(self.output_modules.iter().cloned()); + v + } + pub fn render_settings(&self, name: &str) -> Option { + self.render_settings_list().into_iter().find(|t| same(&t.name, name)) + } + pub fn output_module(&self, name: &str) -> Option { + self.output_module_list().into_iter().find(|t| same(&t.name, name)) + } + pub fn is_builtin(kind: TemplateKind, name: &str) -> bool { + match kind { + TemplateKind::RenderSettings => builtin_render_settings().iter().any(|t| same(&t.name, name)), + TemplateKind::OutputModule => builtin_output_modules().iter().any(|t| same(&t.name, name)), + } + } + /// Save (or replace) a template under `name`. + pub fn save_render_settings(&mut self, name: &str, mut s: RenderSettings) { + s.name = name.trim().to_string(); + self.render_settings.retain(|t| !same(&t.name, name)); + self.render_settings.push(s); + } + pub fn save_output_module(&mut self, name: &str, mut m: OutputModule) { + m.name = name.trim().to_string(); + self.output_modules.retain(|t| !same(&t.name, name)); + self.output_modules.push(m); + } + /// Delete a saved template (a built-in it overrode comes back). Built-ins themselves can't + /// be deleted. + pub fn delete(&mut self, kind: TemplateKind, name: &str) -> Result<(), String> { + let before = (self.render_settings.len(), self.output_modules.len()); + match kind { + TemplateKind::RenderSettings => self.render_settings.retain(|t| !same(&t.name, name)), + TemplateKind::OutputModule => self.output_modules.retain(|t| !same(&t.name, name)), + } + if before != (self.render_settings.len(), self.output_modules.len()) { + return Ok(()); + } + if Self::is_builtin(kind, name) { Err(format!("“{name}” is a built-in template")) } else { Err(format!("no template “{name}”")) } + } + /// The default template name of a slot. + pub fn default_name(&self, kind: TemplateKind, slot: TemplateSlot) -> String { + let list = match kind { + TemplateKind::RenderSettings => &self.defaults.render_settings, + TemplateKind::OutputModule => &self.defaults.output_modules, + }; + list.iter().find(|(s, _)| *s == slot).map(|(_, n)| n.clone()).unwrap_or_else(|| { + let d = TemplateDefaults::default(); + let l = if kind == TemplateKind::RenderSettings { d.render_settings } else { d.output_modules }; + l.into_iter().find(|(s, _)| *s == slot).map(|(_, n)| n).unwrap_or_default() + }) + } + pub fn set_default(&mut self, kind: TemplateKind, slot: TemplateSlot, name: &str) -> Result<(), String> { + let exists = match kind { + TemplateKind::RenderSettings => self.render_settings(name).is_some(), + TemplateKind::OutputModule => self.output_module(name).is_some(), + }; + if !exists { + return Err(format!("no template “{name}”")); + } + let list = match kind { + TemplateKind::RenderSettings => &mut self.defaults.render_settings, + TemplateKind::OutputModule => &mut self.defaults.output_modules, + }; + list.retain(|(s, _)| *s != slot); + list.push((slot, name.trim().to_string())); + Ok(()) + } + /// The Render Settings a slot starts from (a missing template falls back to the built-in + /// default). + pub fn default_render_settings(&self, slot: TemplateSlot) -> RenderSettings { + self.render_settings(&self.default_name(TemplateKind::RenderSettings, slot)).unwrap_or_else(|| builtin_render_settings().remove(0)) + } + pub fn default_output_module(&self, slot: TemplateSlot) -> OutputModule { + self.output_module(&self.default_name(TemplateKind::OutputModule, slot)).unwrap_or_default() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn builtins_defaults_and_round_trip() { + let mut t = RenderTemplates::default(); + let names: Vec = t.render_settings_list().into_iter().map(|s| s.name).collect(); + assert_eq!(names, ["Best Settings", "Draft Settings", "DV Settings", "Multi-Machine Settings", "Current Settings"]); + assert_eq!(t.default_render_settings(TemplateSlot::Movie).name, "Best Settings"); + assert_eq!(t.default_output_module(TemplateSlot::PreRender).channels, Channels::Rgba); + let dv = t.render_settings("dv settings").unwrap(); + assert_eq!(dv.field_render, FieldRender::LowerFirst); + // Save a custom template, make it the movie default, round-trip through JSON. + let mut s = RenderSettings { resolution: 0.25, ..Default::default() }; + s.color_depth = ColorDepth::Bpc16; + t.save_render_settings("Quarter 16", s); + t.set_default(TemplateKind::RenderSettings, TemplateSlot::Movie, "Quarter 16").unwrap(); + let mut m = OutputModule::for_format(OutputFormat::WebM); + m.crop = Crop { enabled: true, top: 4, left: 2, ..Default::default() }; + t.save_output_module("Web Crop", m); + let json = serde_json::to_string(&t).unwrap(); + let back: RenderTemplates = serde_json::from_str(&json).unwrap(); + assert_eq!(back, t); + assert_eq!(back.default_render_settings(TemplateSlot::Movie).color_depth, ColorDepth::Bpc16); + assert_eq!(back.output_module("web crop").unwrap().crop.top, 4); + assert!(t.delete(TemplateKind::RenderSettings, "Best Settings").is_err()); + t.delete(TemplateKind::OutputModule, "Web Crop").unwrap(); + assert!(t.output_module("Web Crop").is_none()); + // Overriding a built-in, then deleting the override, restores it. + t.save_output_module("High Quality", OutputModule::for_format(OutputFormat::Gif)); + assert_eq!(t.output_module("High Quality").unwrap().format, OutputFormat::Gif); + t.delete(TemplateKind::OutputModule, "High Quality").unwrap(); + assert_eq!(t.output_module("High Quality").unwrap().format, OutputFormat::ProRes); + } +} diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 912f2e68..77dfc39f 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -73,6 +73,8 @@ pub enum Dialog { Precompose, /// Layer ▸ Layer Styles ▸ Layer Style Options… LayerStyles, + /// Edit ▸ Templates ▸ Render Settings… / Output Module…. + RenderTemplates, } /// Host hooks provided by the native app (file pickers etc.). diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index eed7e639..18add8dc 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -506,8 +506,18 @@ pub fn frontend(app: &mut EffectcraftApp, ctx: &egui::Context, id: &str, p: Valu } "app.templates" => { let kind = p.get("kind").and_then(Value::as_str).unwrap_or("renderSettings"); - let title = if kind == "outputModule" { "Output Module Templates" } else { "Render Settings Templates" }; - crate::panels::dialogs::info(app, title, "Render and output templates are managed from the Render Queue panel (Window ▸ Render Queue)."); + crate::panels::rq_templates::open(app, ctx, kind); + Value::Null + } + "renderQueue.notify" => { + // Render finished with Notify on: a toast, the dock/taskbar attention request, and the + // host's system sound (the desktop app handles `renderQueue.notify`). + let msg = p.get("message").and_then(Value::as_str).unwrap_or("Render finished").to_string(); + app.toast = Some((msg, now)); + ctx.send_viewport_cmd(egui::ViewportCommand::RequestUserAttention(egui::UserAttentionType::Informational)); + if let Some(f) = app.hooks.app_action.as_ref() { + f("renderQueue.notify"); + } Value::Null } "app.find" => { diff --git a/crates/ui-egui/src/panels/dialogs.rs b/crates/ui-egui/src/panels/dialogs.rs index cabc8703..8992b69a 100644 --- a/crates/ui-egui/src/panels/dialogs.rs +++ b/crates/ui-egui/src/panels/dialogs.rs @@ -129,6 +129,7 @@ pub fn show(app: &mut EffectcraftApp, ctx: &egui::Context) { Dialog::LayerStyles => super::layer_styles_dialog::show(app, ctx, &t), Dialog::TrackTarget => super::tracker::target_dialog(app, ctx, &t), Dialog::TrackApply => super::tracker::apply_dialog(app, ctx, &t), + Dialog::RenderTemplates => super::rq_templates::show(app, ctx, &t), } } diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index d0fb3a3e..740ced10 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -28,6 +28,7 @@ pub mod properties; pub mod puppet_tool; pub mod render_queue; pub mod roto_tool; +pub mod rq_templates; pub mod script_console; pub mod settings; pub mod shortcut_editor; diff --git a/crates/ui-egui/src/panels/render_queue.rs b/crates/ui-egui/src/panels/render_queue.rs index c02090bc..ec0853a1 100644 --- a/crates/ui-egui/src/panels/render_queue.rs +++ b/crates/ui-egui/src/panels/render_queue.rs @@ -4,10 +4,18 @@ //! Output Module and Output To rows, and a status footer. //! //! Every control dispatches an engine `renderQueue.*` command (so agents can do the same) and -//! registers an automation id: `renderQueue.render`, `renderQueue.stop`, -//! `renderQueue.item..{row,twirl,render,renderSettings,outputModule,outputTo,outputPath}`. +//! registers an automation id: `renderQueue.render`, `renderQueue.stop`, `renderQueue.notify`, +//! `renderQueue.item..{row,twirl,render,renderSettings,outputModule,outputTo,outputPath,log}`. +//! +//! The Render Settings and Output Module menus list the templates first (Edit ▸ Templates), then +//! every setting; entries needing numbers (custom time span and frame rate, crop, resize, bitrate, +//! key frame interval, Make Template) open a form that runs the same command. -use effectcraft_engine::project::render_queue::{AudioOutput, Channels, OutputFormat, ProResProfile, RenderQuality, RenderQueueItem, RenderStatus, TimeSpan}; +use effectcraft_engine::project::render_queue::{ + AlphaMode, AudioFormat, AudioOutput, Channels, ColorDepth, CurrentOrOff, EffectsMode, FieldRender, OutputFormat, PostRenderAction, ProResProfile, Pulldown, + RESIZE_PRESETS, RenderLog, RenderQuality, RenderQueueItem, RenderStatus, ResizeQuality, SwitchOverride, TimeSpan, +}; +use effectcraft_engine::project::render_templates::RenderTemplates; use egui::{Align2, Color32, Rect, Sense, Stroke, StrokeKind, pos2, vec2}; use serde_json::{Value, json}; @@ -53,42 +61,82 @@ fn status_color(t: &Tokens, s: &RenderStatus) -> Color32 { } } -/// Render Settings menu: (label, params). -fn settings_menu(it: &RenderQueueItem) -> Vec<(String, Value)> { +fn mark(on: bool, l: &str) -> String { + if on { format!("✓ {l}") } else { format!(" {l}") } +} + +/// Render Settings menu: (label, params). `{"form": …}` entries open a form. +fn settings_menu(it: &RenderQueueItem, templates: &RenderTemplates) -> Vec<(String, Value)> { let s = &it.settings; - let mark = |on: bool, l: &str| if on { format!("✓ {l}") } else { format!(" {l}") }; - let mut v = vec![ - (mark(s.quality == RenderQuality::Best, "Best Settings"), json!({"quality": "best"})), - (mark(s.quality == RenderQuality::Draft, "Draft Settings"), json!({"quality": "draft"})), - ("-".into(), Value::Null), - ]; + let mut v = vec![]; + for t in templates.render_settings_list() { + v.push((mark(s.name == t.name, &t.name), json!({"template": t.name}))); + } + v.push((" Make Template…".into(), json!({"form": "makeRenderTemplate"}))); + v.push(("-".into(), Value::Null)); + v.push((mark(s.quality == RenderQuality::Best, "Quality: Best"), json!({"quality": "best"}))); + v.push((mark(s.quality == RenderQuality::Draft, "Quality: Draft"), json!({"quality": "draft"}))); for (l, k) in [("Full", "full"), ("Half", "half"), ("Third", "third"), ("Quarter", "quarter")] { v.push((mark(s.resolution_label() == l, &format!("Resolution: {l}")), json!({"resolution": k}))); } v.push(("-".into(), Value::Null)); + for (u, k) in effectcraft_engine::project::render_queue::ProxyUse::ALL.iter().zip(["current", "all", "comp", "none"]) { + v.push((mark(s.proxy_use == *u, &format!("Proxy Use: {}", u.label())), json!({"proxyUse": k}))); + } + for (e, k) in EffectsMode::ALL.iter().zip(["current", "allOn", "allOff"]) { + v.push((mark(s.effects == *e, &format!("Effects: {}", e.label())), json!({"effects": k}))); + } + for (e, k) in [(CurrentOrOff::Current, "current"), (CurrentOrOff::AllOff, "allOff")] { + v.push((mark(s.solo == e, &format!("Solo Switches: {}", e.label())), json!({"solo": k}))); + } + for (e, k) in [(CurrentOrOff::Current, "current"), (CurrentOrOff::AllOff, "allOff")] { + v.push((mark(s.guide_layers == e, &format!("Guide Layers: {}", e.label())), json!({"guideLayers": k}))); + } + for (d, k) in ColorDepth::ALL.iter().zip(["current", "8", "16", "32"]) { + v.push((mark(s.color_depth == *d, &format!("Color Depth: {}", d.label())), json!({"colorDepth": k}))); + } + v.push(("-".into(), Value::Null)); + for (m, k) in SwitchOverride::ALL.iter().zip(["current", "onForChecked", "offForAll"]) { + v.push((mark(s.frame_blending == *m, &format!("Frame Blending: {}", m.label())), json!({"frameBlending": k}))); + } + for (m, k) in SwitchOverride::ALL.iter().zip(["current", "onForChecked", "offForAll"]) { + v.push((mark(s.motion_blur_override() == *m, &format!("Motion Blur: {}", m.label())), json!({"motionBlur": k}))); + } + for (f, k) in FieldRender::ALL.iter().zip(["off", "upper", "lower"]) { + v.push((mark(s.field_render == *f, &format!("Field Render: {}", f.label())), json!({"fieldRender": k}))); + } + if s.field_render != FieldRender::Off { + for pd in Pulldown::ALL { + v.push((mark(s.pulldown == pd, &format!("3:2 Pulldown: {}", pd.label())), json!({"pulldown": pd.label()}))); + } + } + v.push(("-".into(), Value::Null)); v.push((mark(s.time_span == TimeSpan::WorkArea, "Time Span: Work Area Only"), json!({"timeSpan": "workArea"}))); v.push((mark(s.time_span == TimeSpan::LengthOfComp, "Time Span: Length of Comp"), json!({"timeSpan": "comp"}))); - v.push(("-".into(), Value::Null)); + v.push((mark(matches!(s.time_span, TimeSpan::Custom { .. }), "Time Span: Custom…"), json!({"form": "timeSpan"}))); v.push((mark(s.frame_rate.is_none(), "Frame Rate: Use comp's frame rate"), json!({"frameRate": null}))); for fps in [12.0, 15.0, 23.976, 24.0, 25.0, 29.97, 30.0, 50.0, 60.0] { let on = s.frame_rate.is_some_and(|r| (r.as_f64() - fps).abs() < 0.01); v.push((mark(on, &format!("Frame Rate: {fps}")), json!({"frameRate": fps}))); } + v.push((" Frame Rate: Custom…".into(), json!({"form": "frameRate"}))); v.push(("-".into(), Value::Null)); - v.push((mark(s.motion_blur, "Motion Blur: On for Checked Layers"), json!({"motionBlur": !s.motion_blur}))); v.push((mark(s.skip_existing, "Skip Existing Files"), json!({"skipExisting": !s.skip_existing}))); - v.push(("-".into(), Value::Null)); - for (u, k) in effectcraft_engine::project::render_queue::ProxyUse::ALL.iter().zip(["current", "all", "comp", "none"]) { - v.push((mark(s.proxy_use == *u, &format!("Proxy Use: {}", u.label())), json!({"proxyUse": k}))); - } + v.push((mark(s.storage_overflow, "Use Storage Overflow"), json!({"storageOverflow": !s.storage_overflow}))); v } -/// Output Module menu: (label, params). -fn output_menu(it: &RenderQueueItem, available: &[OutputFormat]) -> Vec<(String, Value)> { +/// Output Module menu: (label, params). `{"form": …}` entries open a form. +fn output_menu(it: &RenderQueueItem, available: &[OutputFormat], templates: &RenderTemplates) -> Vec<(String, Value)> { let o = &it.output; - let mark = |on: bool, l: &str| if on { format!("✓ {l}") } else { format!(" {l}") }; let mut v = Vec::new(); + for t in templates.output_module_list() { + if available.contains(&t.format) { + v.push((mark(o.name == t.name, &t.name), json!({"template": t.name}))); + } + } + v.push((" Make Template…".into(), json!({"form": "makeOutputTemplate"}))); + v.push(("-".into(), Value::Null)); for f in OutputFormat::ALL { if available.contains(&f) { v.push((mark(o.format == f, &format!("Format: {}", f.label())), json!({"format": format!("{f:?}")}))); @@ -99,6 +147,13 @@ fn output_menu(it: &RenderQueueItem, available: &[OutputFormat]) -> Vec<(String, if o.format.supports_alpha() { v.push((mark(o.channels == Channels::Rgba, "Channels: RGB + Alpha"), json!({"channels": "rgba"}))); } + if !o.format.is_audio_only() { + v.push((mark(o.channels == Channels::Alpha, "Channels: Alpha"), json!({"channels": "alpha"}))); + } + if o.channels == Channels::Rgba { + v.push((mark(o.alpha_mode == AlphaMode::Straight, "Color: Straight (Unmatted)"), json!({"color": "straight"}))); + v.push((mark(o.alpha_mode == AlphaMode::Premultiplied, "Color: Premultiplied (Matted)"), json!({"color": "premultiplied"}))); + } match o.format { OutputFormat::ProRes => { v.push(("-".into(), Value::Null)); @@ -108,14 +163,18 @@ fn output_menu(it: &RenderQueueItem, available: &[OutputFormat]) -> Vec<(String, } OutputFormat::H264 => { v.push(("-".into(), Value::Null)); - for kbps in [2_000, 5_000, 10_000, 20_000, 40_000] { + for kbps in [2_000, 5_000, 10_000, 15_000, 20_000, 40_000] { v.push((mark(o.bitrate_kbps == kbps, &format!("Bitrate: {} Mbps", kbps / 1000)), json!({"bitrate": kbps}))); } } OutputFormat::JpegSequence | OutputFormat::WebM => { v.push(("-".into(), Value::Null)); for q in [60u8, 80, 90, 100] { - v.push((mark(o.quality == q, &format!("Quality: {q}")), json!({"quality": q}))); + v.push((mark(o.quality == q && !o.webm_bitrate, &format!("Quality: {q}")), json!({"quality": q, "webmBitrate": false}))); + } + if o.format == OutputFormat::WebM { + v.push((mark(o.webm_bitrate, &format!("Target Bitrate ({} kbps)…", o.bitrate_kbps)), json!({"form": "bitrate"}))); + v.push((mark(o.keyframe_interval > 0, "Key Frame Interval…"), json!({"form": "keyframes"}))); } } OutputFormat::Gif => { @@ -124,20 +183,131 @@ fn output_menu(it: &RenderQueueItem, available: &[OutputFormat]) -> Vec<(String, } _ => {} } + if !o.format.is_audio_only() { + v.push(("-".into(), Value::Null)); + v.push((mark(!o.crop.enabled, "Crop: Off"), json!({"crop": false}))); + v.push((mark(o.crop.enabled && o.crop.use_roi, "Crop: Use Region of Interest"), json!({"crop": {"useRoi": true}}))); + v.push((mark(o.crop.enabled && !o.crop.use_roi, "Crop: Edges…"), json!({"form": "crop"}))); + v.push((mark(!o.resize.enabled, "Resize: Off"), json!({"resize": false}))); + for (n, w, h) in RESIZE_PRESETS { + let on = o.resize.enabled && o.resize.width == w && o.resize.height == h && !o.resize.lock_aspect; + v.push((mark(on, &format!("Resize to: {n} ({w}×{h})")), json!({"resize": {"preset": n}}))); + } + v.push((mark(o.resize.enabled, "Resize to: Custom…"), json!({"form": "resize"}))); + v.push(( + mark(o.resize.lock_aspect, "Resize: Lock Aspect Ratio"), + json!({"resize": {"enabled": o.resize.enabled, "lockAspect": !o.resize.lock_aspect}}), + )); + for (q, k) in [(ResizeQuality::Low, "low"), (ResizeQuality::High, "high")] { + v.push((mark(o.resize.quality == q, &format!("Resize Quality: {q:?}")), json!({"resize": {"enabled": o.resize.enabled, "quality": k}}))); + } + } + v.push(("-".into(), Value::Null)); + for (a, k) in [ + (PostRenderAction::None, "none"), + (PostRenderAction::Import, "import"), + (PostRenderAction::ImportAndReplace, "importAndReplace"), + (PostRenderAction::SetProxy, "setProxy"), + ] { + v.push((mark(it.post_render == a, &format!("Post-Render Action: {}", a.label())), json!({"postRenderAction": k}))); + } + v.push((mark(o.include_project_link, "Include Project Link"), json!({"includeProjectLink": !o.include_project_link}))); if o.format.supports_audio() { v.push(("-".into(), Value::Null)); for (a, l) in [(AudioOutput::Auto, "auto"), (AudioOutput::On, "on"), (AudioOutput::Off, "off")] { let label = match a { AudioOutput::Auto => "Audio Output: Auto", - AudioOutput::On => "Audio Output: On (48 kHz stereo)", + AudioOutput::On => "Audio Output: On", AudioOutput::Off => "Audio Output: Off", }; v.push((mark(o.audio == a, label), json!({"audio": l}))); } + for sr in [44_100u32, 48_000, 96_000] { + v.push((mark(o.audio_sample_rate == sr, &format!("Sample Rate: {:.1} kHz", sr as f64 / 1000.0)), json!({"sampleRate": sr}))); + } + v.push((mark(o.audio_channels == 1, "Audio Channels: Mono"), json!({"audioChannels": "mono"}))); + v.push((mark(o.audio_channels != 1, "Audio Channels: Stereo"), json!({"audioChannels": "stereo"}))); + if matches!(o.format, OutputFormat::ProRes | OutputFormat::Wav | OutputFormat::Aiff) { + for f in [AudioFormat::S16, AudioFormat::S24, AudioFormat::F32] { + v.push((mark(o.audio_format == f, &format!("Sample Format: {}", f.label())), json!({"audioFormat": f.label()}))); + } + } } v } +/// Open the form behind a `{"form": …}` menu entry. +fn open_menu_form(app: &mut EffectcraftApp, it: &RenderQueueItem, form: &str) { + use crate::panels::forms::{Field, form as open}; + let id = it.id; + let o = &it.output; + match form { + "makeRenderTemplate" => open( + app, + "Make Render Settings Template", + "renderQueue.saveTemplate", + json!({"kind": "renderSettings", "item": id}), + vec![Field::text("name", "Template name", "")], + ), + "makeOutputTemplate" => open( + app, + "Make Output Module Template", + "renderQueue.saveTemplate", + json!({"kind": "outputModule", "item": id}), + vec![Field::text("name", "Template name", "")], + ), + "timeSpan" => { + let comp = app.session.project.comp(it.comp); + let (a, b) = comp.map(|c| it.settings.span(c)).unwrap_or_default(); + open( + app, + "Custom Time Span", + "renderQueue.setRenderSettings", + json!({"item": id, "timeSpan": "custom"}), + vec![Field::num("start", "Start (s)", a.seconds()), Field::num("duration", "Duration (s)", (b.seconds() - a.seconds()).max(0.04))], + ) + } + "frameRate" => { + let r = app.session.project.comp(it.comp).map(|c| it.settings.rate(c).as_f64()).unwrap_or(30.0); + open(app, "Use This Frame Rate", "renderQueue.setRenderSettings", json!({"item": id}), vec![Field::num("frameRate", "Frames per second", r)]) + } + "crop" => open( + app, + "Crop", + "renderQueue.setOutputModule", + json!({"item": id}), + vec![ + Field::num("cropTop", "Top", o.crop.top as f64), + Field::num("cropLeft", "Left", o.crop.left as f64), + Field::num("cropBottom", "Bottom", o.crop.bottom as f64), + Field::num("cropRight", "Right", o.crop.right as f64), + ], + ), + "resize" => open( + app, + "Resize to", + "renderQueue.setOutputModule", + json!({"item": id}), + vec![Field::num("resizeWidth", "Width", o.resize.width as f64), Field::num("resizeHeight", "Height", o.resize.height as f64)], + ), + "bitrate" => open( + app, + "WebM Target Bitrate", + "renderQueue.setOutputModule", + json!({"item": id, "webmBitrate": true}), + vec![Field::num("bitrate", "Bitrate (kbps)", o.bitrate_kbps as f64)], + ), + "keyframes" => open( + app, + "Key Frame Interval", + "renderQueue.setOutputModule", + json!({"item": id}), + vec![Field::num("keyframeInterval", "Frames between key frames (0 = every 2 s)", o.keyframe_interval as f64)], + ), + _ => {} + } +} + const OUTPUT_TEMPLATES: [&str; 4] = [ "[compName].[fileExtension]", "[compName]_[width]x[height].[fileExtension]", @@ -166,6 +336,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let rendering = app.session.is_rendering(); let available = app.session.exporter.as_ref().map(|e| e.formats()).unwrap_or_default(); let mut actions: Vec<(&'static str, Value)> = Vec::new(); + let mut forms: Vec<(RenderQueueItem, String)> = Vec::new(); let state_id = egui::Id::new("rq-ui"); let (mut open, mut selected): (Vec, Option) = ui.data(|d| d.get_temp(state_id).unwrap_or_default()); @@ -194,6 +365,14 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { actions.push(("renderQueue.render", json!({"wait": false}))); } app.auto.add("renderQueue.render", render_b, "Render"); + // Notify (sound + notification when the queue finishes). + let notify = app.session.project.render_prefs.notify; + let nb = Rect::from_min_size(pos2(render_b.min.x - 8.0 - 2.0 * bw - 8.0, render_b.min.y + 4.0), vec2(18.0, 18.0)); + if widgets::checkbox(ui, nb, notify, &t, egui::Id::new("rq-notify")).clicked() { + actions.push(("renderQueue.setNotify", json!({"notify": !notify}))); + } + p.text(pos2(nb.max.x + 5.0, nb.center().y), Align2::LEFT_CENTER, "Notify", Tokens::ui(11.5), t.text_dim); + app.auto.add("renderQueue.notify", nb, if notify { "Notify: on" } else { "Notify: off" }); if rendering { if widgets::text_button(ui, stop_b, "Stop", false, &t, egui::Id::new("rq-stop")).clicked() { actions.push(("renderQueue.stop", json!({}))); @@ -201,8 +380,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { app.auto.add("renderQueue.stop", stop_b, "Stop"); } // Progress bar. - let bar = - Rect::from_min_max(pos2(x0, render_b.min.y + 4.0), pos2(if rendering { stop_b.min.x - 16.0 } else { render_b.min.x - 16.0 }, render_b.max.y - 4.0)); + let bar = Rect::from_min_max(pos2(x0, render_b.min.y + 4.0), pos2(nb.min.x - 16.0, render_b.max.y - 4.0)); p.rect_filled(bar, 3.0, t.field_bg); p.rect_stroke(bar, 3.0, Stroke::new(1.0, t.field_border), StrokeKind::Inside); let frac = progress.as_ref().filter(|p| p.total > 0).map(|p| (p.done as f32 / p.total as f32).clamp(0.0, 1.0)).unwrap_or(0.0); @@ -355,22 +533,28 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let r = Rect::from_min_size(pos2(list.min.x, y), vec2(list.width(), ROW_H)); y += ROW_H; lp.rect_filled(r, 0.0, t.panel_bg); + let templates = &app.session.project.render_templates; let (label, summary, key, menu): (&str, String, &str, Vec<(String, Value)>) = if sub == 0 { - ("Render Settings:", it.settings.summary(), "renderSettings", settings_menu(it)) + ("Render Settings:", it.settings.summary(), "renderSettings", settings_menu(it, templates)) } else { - ("Output Module:", it.output.summary(), "outputModule", output_menu(it, &available)) + ("Output Module:", it.output.summary(), "outputModule", output_menu(it, &available, templates)) }; lp.text(pos2(cx_num - 6.0, r.center().y), Align2::LEFT_CENTER, label, Tokens::ui(11.5), t.text_dim); let dd = Rect::from_min_size(pos2(ix + 70.0, r.min.y + 3.0), vec2(220.0, ROW_H - 6.0)); let did = egui::Id::new(("rq-dd", it.id, sub)); - if widgets::dropdown(ui, dd, &summary, &t, did).clicked() && !rendering { + let shown: String = if summary.chars().count() > 31 { summary.chars().take(30).chain(std::iter::once('…')).collect() } else { summary.clone() }; + if widgets::dropdown(ui, dd, &shown, &t, did).clicked() && !rendering { widgets::open_popup(ui, did); } let labels: Vec = menu.iter().map(|(l, _)| l.clone()).collect(); if let Some(i) = widgets::popup_menu(ui, did, dd.left_bottom(), &labels, None) { let mut params = menu[i].1.clone(); - params["item"] = json!(it.id); - actions.push((if sub == 0 { "renderQueue.setRenderSettings" } else { "renderQueue.setOutputModule" }, params)); + if let Some(f) = params.get("form").and_then(Value::as_str) { + forms.push((it.clone(), f.to_string())); + } else { + params["item"] = json!(it.id); + actions.push((if sub == 0 { "renderQueue.setRenderSettings" } else { "renderQueue.setOutputModule" }, params)); + } } app.auto.add(&aid(key), dd, &summary); if sub == 0 { @@ -387,6 +571,18 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { ); lp.text(pos2(dd.max.x + 14.0, r.center().y), Align2::LEFT_CENTER, info, Tokens::ui(11.0), t.text_faint); } + // Log: Errors Only / Plus Settings / Plus Per Frame Info. + let ld = Rect::from_min_size(pos2(r.max.x - 190.0, r.min.y + 3.0), vec2(150.0, ROW_H - 6.0)); + lp.text(pos2(ld.min.x - 6.0, r.center().y), Align2::RIGHT_CENTER, "Log:", Tokens::ui(11.5), t.text_dim); + let lid = egui::Id::new(("rq-log", it.id)); + if widgets::dropdown(ui, ld, it.log.label(), &t, lid).clicked() && !rendering { + widgets::open_popup(ui, lid); + } + let llabels: Vec = RenderLog::ALL.iter().map(|l| mark(it.log == *l, l.label())).collect(); + if let Some(i) = widgets::popup_menu(ui, lid, ld.left_bottom(), &llabels, None) { + actions.push(("renderQueue.setLog", json!({"item": it.id, "log": RenderLog::ALL[i].label()}))); + } + app.auto.add(&aid("log"), ld, it.log.label()); continue; } // Output To. @@ -490,6 +686,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { open.retain(|id| queue.iter().any(|i| i.id == *id)); ui.data_mut(|d| d.insert_temp(state_id, (open, selected))); let ctx = ui.ctx().clone(); + for (it, f) in forms { + open_menu_form(app, &it, &f); + } for (id, params) in actions { if let Err(e) = crate::menus::invoke(app, &ctx, id, params) { app.ui.status = e; diff --git a/crates/ui-egui/src/panels/rq_templates.rs b/crates/ui-egui/src/panels/rq_templates.rs new file mode 100644 index 00000000..c60c3009 --- /dev/null +++ b/crates/ui-egui/src/panels/rq_templates.rs @@ -0,0 +1,166 @@ +//! Edit ▸ Templates ▸ Render Settings… / Output Module…: the Defaults (which template new +//! Render Queue items, frames, pre-renders and proxies start from) and the template list +//! (settings of the selected template; Duplicate, Delete, New from a Render Queue item). +//! +//! Every action runs a `renderQueue.*Template*` engine command. Automation ids: +//! `templates.default.`, `templates.select`, `templates.duplicate`, `templates.delete`, +//! `templates.newName`, `templates.saveFromItem`, `templates.ok`. + +use effectcraft_engine::project::render_templates::{RenderTemplates, TemplateKind, TemplateSlot}; +use egui::{Color32, vec2}; +use serde_json::{Value, json}; + +use crate::theme::Tokens; +use crate::{Dialog, EffectcraftApp}; + +#[derive(Clone, Default)] +struct State { + output: bool, + selected: String, + new_name: String, +} + +fn state_id() -> egui::Id { + egui::Id::new("rq-templates-dialog") +} + +/// Open the dialog for `kind` (`renderSettings` | `outputModule`). +pub fn open(app: &mut EffectcraftApp, ctx: &egui::Context, kind: &str) { + let output = TemplateKind::parse(kind) == Some(TemplateKind::OutputModule); + let t = &app.session.project.render_templates; + let k = if output { TemplateKind::OutputModule } else { TemplateKind::RenderSettings }; + let selected = t.default_name(k, TemplateSlot::Movie); + ctx.data_mut(|d| d.insert_temp(state_id(), State { output, selected, new_name: String::new() })); + app.dialog = Some(Dialog::RenderTemplates); +} + +fn names(t: &RenderTemplates, output: bool) -> Vec { + if output { t.output_module_list().into_iter().map(|m| m.name).collect() } else { t.render_settings_list().into_iter().map(|r| r.name).collect() } +} + +pub fn show(app: &mut EffectcraftApp, ctx: &egui::Context, t: &Tokens) { + let mut st: State = ctx.data(|d| d.get_temp(state_id())).unwrap_or_default(); + let kind = if st.output { "outputModule" } else { "renderSettings" }; + let k = if st.output { TemplateKind::OutputModule } else { TemplateKind::RenderSettings }; + let templates = app.session.project.render_templates.clone(); + let list = names(&templates, st.output); + if !list.contains(&st.selected) { + st.selected = list.first().cloned().unwrap_or_default(); + } + let mut actions: Vec<(&'static str, Value)> = vec![]; + let mut close = false; + let title = if st.output { "Output Module Templates" } else { "Render Settings Templates" }; + super::dialogs::modal(ctx, title, vec2(600.0, 560.0), t, |ui| { + ui.label(egui::RichText::new("Defaults").font(Tokens::semibold(12.5)).color(t.text)); + egui::Grid::new("tpl-defaults").num_columns(2).spacing(vec2(14.0, 6.0)).show(ui, |ui| { + for slot in TemplateSlot::ALL { + ui.label(slot.label()); + let cur = templates.default_name(k, slot); + let mut pick = cur.clone(); + let r = egui::ComboBox::from_id_salt(("tpl-def", slot.label())).selected_text(&cur).width(300.0).show_ui(ui, |ui| { + for n in &list { + ui.selectable_value(&mut pick, n.clone(), n); + } + }); + app.auto.add(&format!("templates.default.{slot:?}"), r.response.rect, &cur); + if pick != cur { + actions.push(("renderQueue.setTemplateDefault", json!({"kind": kind, "slot": format!("{slot:?}"), "name": pick}))); + } + ui.end_row(); + } + }); + ui.add_space(12.0); + ui.separator(); + ui.label(egui::RichText::new("Settings").font(Tokens::semibold(12.5)).color(t.text)); + ui.horizontal(|ui| { + ui.label("Settings Name:"); + let r = egui::ComboBox::from_id_salt("tpl-select").selected_text(&st.selected).width(300.0).show_ui(ui, |ui| { + for n in &list { + ui.selectable_value(&mut st.selected, n.clone(), n); + } + }); + app.auto.add("templates.select", r.response.rect, &st.selected); + }); + let builtin = RenderTemplates::is_builtin(k, &st.selected) + && !match k { + TemplateKind::RenderSettings => templates.render_settings.iter().any(|r| r.name.eq_ignore_ascii_case(&st.selected)), + TemplateKind::OutputModule => templates.output_modules.iter().any(|m| m.name.eq_ignore_ascii_case(&st.selected)), + }; + // The selected template's settings. + let lines: Vec<(&'static str, String)> = if st.output { + templates.output_module(&st.selected).map(|m| m.describe()).unwrap_or_default() + } else { + templates.render_settings(&st.selected).map(|r| r.describe(None)).unwrap_or_default() + }; + egui::Frame::new().fill(t.field_bg).inner_margin(egui::Margin::same(8)).show(ui, |ui| { + ui.set_min_height(180.0); + ui.set_width(ui.available_width()); + egui::ScrollArea::vertical().max_height(200.0).show(ui, |ui| { + egui::Grid::new("tpl-lines").num_columns(2).spacing(vec2(12.0, 2.0)).show(ui, |ui| { + for (k, v) in &lines { + ui.label(egui::RichText::new(*k).color(t.text_dim).size(11.5)); + ui.label(egui::RichText::new(v).size(11.5)); + ui.end_row(); + } + }); + }); + }); + ui.horizontal(|ui| { + let r = ui.button("Duplicate"); + app.auto.add("templates.duplicate", r.rect, "Duplicate"); + if r.clicked() { + let mut n = format!("{} copy", st.selected); + let mut i = 2; + while list.iter().any(|x| x.eq_ignore_ascii_case(&n)) { + n = format!("{} copy {i}", st.selected); + i += 1; + } + actions.push(("renderQueue.saveTemplate", json!({"kind": kind, "name": n, "from": st.selected}))); + st.selected = n; + } + let r = ui.add_enabled(!builtin, egui::Button::new("Delete")); + app.auto.add("templates.delete", r.rect, "Delete"); + if r.clicked() { + actions.push(("renderQueue.deleteTemplate", json!({"kind": kind, "name": st.selected}))); + } + if builtin { + ui.label(egui::RichText::new("built-in").color(t.text_faint).size(11.0)); + } + }); + ui.add_space(8.0); + ui.separator(); + // New template from the selected (or only) Render Queue item. + let queue = &app.session.project.render_queue; + ui.horizontal(|ui| { + ui.label("New from Render Queue item:"); + let r = ui.add(egui::TextEdit::singleline(&mut st.new_name).hint_text("template name").desired_width(180.0)); + app.auto.add("templates.newName", r.rect, &st.new_name); + let item = queue.last().map(|i| i.id); + let ok = item.is_some() && !st.new_name.trim().is_empty(); + let r = ui.add_enabled(ok, egui::Button::new(format!("Save from #{}", queue.len().max(1)))); + app.auto.add("templates.saveFromItem", r.rect, "Save from item"); + if r.clicked() + && let Some(id) = item + { + actions.push(("renderQueue.saveTemplate", json!({"kind": kind, "name": st.new_name.trim(), "item": id}))); + st.selected = st.new_name.trim().to_string(); + st.new_name.clear(); + } + }); + ui.add_space(12.0); + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + let r = ui.add(egui::Button::new(egui::RichText::new(" OK ").color(Color32::WHITE)).fill(t.accent)); + app.auto.add("templates.ok", r.rect, "OK"); + close = r.clicked(); + }); + }); + for (id, p) in actions { + if let Err(e) = app.session.execute(id, p) { + app.ui.status = e.to_string(); + } + } + ctx.data_mut(|d| d.insert_temp(state_id(), st)); + if close || ctx.input(|i| i.key_pressed(egui::Key::Escape)) { + app.dialog = None; + } +} diff --git a/crates/ui-egui/tests/render_queue_snapshot.rs b/crates/ui-egui/tests/render_queue_snapshot.rs index 5029a5f2..41ce13d2 100644 --- a/crates/ui-egui/tests/render_queue_snapshot.rs +++ b/crates/ui-egui/tests/render_queue_snapshot.rs @@ -41,3 +41,26 @@ fn render_queue_panel_snapshot() { } img.save(&out).unwrap(); } + +/// Edit ▸ Templates ▸ Output Module… (`RQ_TEMPLATES_SNAPSHOT=/abs/out.png`). +#[test] +#[ignore] +fn templates_dialog_snapshot() { + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + s.execute("renderQueue.add", json!({"output": "/renders/[compName].[fileExtension]"})).unwrap(); + s.execute("renderQueue.saveTemplate", json!({"kind": "outputModule", "name": "Web Delivery", "item": s.project.render_queue[0].id, "params": {"format": "webm", "resize": {"preset": "HDTV 720"}}})) + .unwrap(); + let kind = std::env::var("RQ_TEMPLATES_KIND").unwrap_or_else(|_| "outputModule".into()); + s.execute("app.templates", json!({"kind": kind})).unwrap(); + let mut app = Some(EffectcraftApp::new(s)); + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_| app.take().expect("app")); + h.run_steps(4); + assert!(h.state().dialog.is_some(), "the Templates dialog is open"); + let img = h.render().expect("render"); + let out = std::env::var("RQ_TEMPLATES_SNAPSHOT").unwrap_or_else(|_| concat!(env!("CARGO_MANIFEST_DIR"), "/../../target/test-out/rq_templates.png").into()); + if let Some(d) = std::path::Path::new(&out).parent() { + std::fs::create_dir_all(d).unwrap(); + } + img.save(&out).unwrap(); +} diff --git a/docs/parity.md b/docs/parity.md index 8fc67b9a..2b30c576 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -21,7 +21,7 @@ Tracking 88%, Effects 85%, Web 65%. What is left, in priority order: the web app's depth (threads, storage, audio, non-blocking renders); stroke taper/wave and multi-segment dashes; variable mask feather points; camera iris/bokeh and focus-link commands; GPU -rendering of 3D runs and adjustment layers; Render Queue field render, crop/resize and templates; +rendering of 3D runs and adjustment layers; approximated effects (Subspace Warp, Key Cleaner) and Liquify's viewer brush; OpenType features and variable font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; ScriptUI; Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; face tracking; and @@ -110,7 +110,7 @@ the disk cache in the formats wave; the 3D Camera Tracker in M12.6.) | Area | Weighted parity | Remaining (agent-hours) | Biggest gaps | |---|---|---|---| | Layers | 88% | 4.8 | frame blending, collapse transformations, slip edit | -| Output | 90% | 3.0 | multiple output modules, pre-render (WebM with VP9 alpha + Opus and WAV/AIFF audio-only landed: VP9 key + inter frames with motion search, loop filter and rate control; Opus CELT-only) | +| Output | 97% | 0.8 | Render Settings complete (field render + 3:2 pulldown, effects/solo/guide/depth/blending/blur overrides, time sampling, storage overflow), Output Module crop/ROI/resize, alpha modes, post-render actions, PCM formats, templates with defaults, render logs, Notify (M10.2); WebM VP9 key + inter frames with motion search, loop filter and rate control. Left: Opus is CELT-only, Photoshop sequence output, overflow for movies only checks at file creation | | Audio | 85% | 0.5 | audio to keyframes | | Import | 88% | 2.0 | AI/EPS/PDF vector footage, PSD smart objects and 3D layers (PSD as footage/composition/retain layer sizes, SVG footage and Create Shapes from Vector Layer landed) | | Automation | ≈ 90% | 0.8 | scripting covers the documented core object model (AUT-2, M14.4: `app`, project items, comps, layers, properties and keyframes, text documents, markers, render queue, Script Console, `effectcraft-cli script`, MCP `run_script`); still missing: ScriptUI panels/dialogs, `.jsxbin`, sockets | diff --git a/docs/web.md b/docs/web.md index 041ef76c..72eb75ff 100644 --- a/docs/web.md +++ b/docs/web.md @@ -207,5 +207,3 @@ every request and reply, a render through an in-process offload). - Cancelling an analysis drops its partial result (the worker is terminated). - Stored media count against the origin's storage quota; `effectcraft.listStored()` shows usage, `removeStored` frees it (there is no storage manager in the UI yet). -- `engine.execute` rejects the spread Render Settings / Output Module keys of `renderQueue.add` - (`resolution`, `timeSpan`…) through its strict parameter check; use `ui.menu.invoke` as above. From da6cc851b40a090912e1015f296de914efd38f76 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 3 Oct 2026 05:31:59 -0400 Subject: [PATCH 9/9] =?UTF-8?q?Parity=20audit=20at=20978e8d7:=20=E2=89=889?= =?UTF-8?q?4%=20weighted=20(98%=20with=20fractions),=2080/12/0,=20?= =?UTF-8?q?=E2=89=884=E2=80=935=20h=20wall=20clock=20left?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 14 ++++++++------ docs/parity.md | 28 +++++++++++++++++++--------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 7c7b337b..c02dd556 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -25,12 +25,14 @@ once. ## How far from full parity -Measured feature by feature in [docs/parity.md](docs/parity.md) (late 2 October 2026): +Measured feature by feature in [docs/parity.md](docs/parity.md) (3 October 2026): -- **≈ 93% of After Effects' features, weighted by importance** (96% of the essentials, 90% of - professional daily-use features, 65% of the long tail); all 298 After Effects effects. -- **≈ 50–55 agent-hours of work remain**: about **11–14 hours of wall-clock time** with five Claude - Opus 5.5 agents in parallel, ≈ 6–8 hours for the essentials and daily-use features. -- Hardest remaining pieces: the web app's depth, GPU 3D, Content-Aware Fill, a plug-in API. +- **≈ 94% of After Effects' features, weighted by importance**, counting partial features as half + done (≈ 98% with per-feature fractions): 80 of 92 features done, 12 partial, none missing; all + 298 After Effects effects (42 still simplified). +- **≈ 19–22 agent-hours of work remain**: about **4–5 hours of wall-clock time** with five Claude + Opus 5.5 agents in parallel, ≈ 3 hours for the essentials and daily-use features. +- Hardest remaining pieces: the web app's threading, wider GPU effect coverage, Advanced 3D + completion, and learned-model quality for Roto Brush and face tracking. Come tell us what matters most to you on [Discord](https://discord.gg/artcraft). diff --git a/docs/parity.md b/docs/parity.md index fce24ea3..54be9809 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -2,18 +2,28 @@ How close EffectCraft is to After Effects 2026, feature by feature, and how much work is left. -## Current status (audit at commit `fa26ad9`, 2 October 2026, late) +## Current status (audit at commit `978e8d7`, 3 October 2026) | Measure | Value | |---|---| -| **Feature parity, weighted by tier** (P0 ×3, P1 ×2, P2 ×1) | **≈ 93%** | -| Unweighted | ≈ 91% | -| P0 / P1 / P2 | ≈ 96% / 90% / 65% | -| Features done / partial / missing | 41 / 50 / 1 of 92 (missing: a plug-in API) | -| **Effects** | **298 of 298** After Effects 2026 effects exist (some still simplified) | -| Disabled menu entries left | 26 commands (31 menu items) | -| Remaining work | ≈ 50–55 agent-hours at the pace measured so far (≈ 150 by the original conservative audit scale) | -| **Wall-clock estimate** | **≈ 11–14 hours** with five agents in parallel; ≈ 6–8 hours for 100% of P0 + P1 | +| **Feature parity, weighted by tier** (P0 ×3, P1 ×2, P2 ×1) | **≈ 94%** counting every partial feature as half done; ≈ 98% using per-feature fractions | +| Unweighted | ≈ 94% (half-credit) / 98% (fractions) | +| P0 / P1 / P2 | ≈ 96% / 92% / 85% (half-credit); 99% / 97% / 96% (fractions) | +| Features done / partial / missing | 80 / 12 / 0 of 92 | +| **Effects** | **298 of 298** After Effects 2026 effects exist; 42 still simplified ([effects.md](effects.md)); 58 run on the GPU | +| Disabled menu entries left | 0 (the stub list is empty) | +| Remaining work | ≈ 19–22 agent-hours at the pace measured so far (≈ 58 on the conservative audit scale) | +| **Wall-clock estimate** | **≈ 4–5 hours** with five agents in parallel; ≈ 3 hours for 100% of P0 + P1 | + +Partial features: effect implementations (42 simplified effects), the Preview panel's settings +(range, play from, skip, frame rate, full screen), GPU coverage (Advanced 3D, more GPU effects), +the web app (no threads for viewer frames and Roto Brush), the Learn panel and Align to Selection / +Distribute, Advanced 3D (iris shapes, collapsed precomps, extruded strokes), ScriptUI resource +strings and `onDraw`, Essential Graphics font and scale controls, and the learned-model quality of +Roto Brush and face tracking (both are classical). Menu items still absent: Save Frame As ▸ +Photoshop Layers / ProEXR, Open in Essential Graphics, Vanishing Point import, Watch Folder. + +### Previous audit (commit `fa26ad9`, 2 October 2026, late): ≈ 93% By area: Layers 98%, Output 95%, Compositions 95%, Automation 96%, Paint 95%, Text 95%, Import 96%, Animation 94%, Masks 94%, Preview 94%, Interface 93%, Shapes 93%, 3D 91%, Audio 90%, Project 90%,