From b936ed0055e364ed652dc7def086124e90035455 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 21:41:29 -0400 Subject: [PATCH 01/13] OUT-2: Opus (CELT) encoder (effectcraft-opusenc) New L0 crate effectcraft-opusenc: clean-room RFC 6716 CELT-only encoder (fullband 48 kHz, 20 ms packets, mono/stereo as dual L/R stereo, constant packet size from the bitrate) with the RFC 7845 OpusHead and pre-skip. Forward low-overlap MDCT, inter/intra coarse energy, fine/final energy, the decoder's exact allocation, recursive band splitting, PVQ search and codeword indexing, silence frames. crates/export depends on it; filmcraft-opus (dev) is the round-trip oracle in tests/opus_roundtrip.rs, with ffmpeg reading an in-test Ogg Opus file as a second oracle when installed. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 6 + Cargo.toml | 6 + crates/export/Cargo.toml | 2 + crates/export/tests/opus_roundtrip.rs | 364 +++++++++++++++++ crates/opusenc/Cargo.toml | 12 + crates/opusenc/README.md | 33 ++ crates/opusenc/src/bands.rs | 497 +++++++++++++++++++++++ crates/opusenc/src/energy.rs | 230 +++++++++++ crates/opusenc/src/lib.rs | 311 ++++++++++++++ crates/opusenc/src/mdct.rs | 293 ++++++++++++++ crates/opusenc/src/range.rs | 472 ++++++++++++++++++++++ crates/opusenc/src/rate.rs | 561 ++++++++++++++++++++++++++ crates/opusenc/src/tables.rs | 122 ++++++ docs/architecture.md | 1 + xtask/src/main.rs | 1 + 15 files changed, 2911 insertions(+) create mode 100644 crates/export/tests/opus_roundtrip.rs create mode 100644 crates/opusenc/Cargo.toml create mode 100644 crates/opusenc/README.md create mode 100644 crates/opusenc/src/bands.rs create mode 100644 crates/opusenc/src/energy.rs create mode 100644 crates/opusenc/src/lib.rs create mode 100644 crates/opusenc/src/mdct.rs create mode 100644 crates/opusenc/src/range.rs create mode 100644 crates/opusenc/src/rate.rs create mode 100644 crates/opusenc/src/tables.rs diff --git a/Cargo.lock b/Cargo.lock index 46f34e63..1a1f0f60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1218,6 +1218,7 @@ version = "0.1.0" dependencies = [ "effectcraft-color", "effectcraft-media", + "effectcraft-opusenc", "effectcraft-project", "effectcraft-raster", "effectcraft-render", @@ -1225,6 +1226,7 @@ dependencies = [ "filmcraft-aac", "filmcraft-h264enc", "filmcraft-isobmff", + "filmcraft-opus", "filmcraft-prores", "gif", "image", @@ -1338,6 +1340,10 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "effectcraft-opusenc" +version = "0.1.0" + [[package]] name = "effectcraft-path" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index a1941cf4..da4f7047 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ effectcraft-engine = { path = "crates/engine" } effectcraft-host = { path = "crates/host" } effectcraft-ui-egui = { path = "crates/ui-egui" } effectcraft-automation = { path = "crates/automation" } +effectcraft-opusenc = { path = "crates/opusenc" } # foundation thiserror = "2" @@ -66,6 +67,8 @@ filmcraft-h264enc = { git = "https://github.com/storytold/filmcraft", rev = "89c filmcraft-prores = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } +# decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency) +filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } # gpu (the same wgpu eframe / egui-wgpu use) wgpu = { version = "30.0.1", features = ["fragile-send-sync-non-atomic-wasm"] } @@ -129,6 +132,9 @@ opt-level = 3 [profile.dev.package.effectcraft-media] opt-level = 3 +[profile.dev.package.effectcraft-opusenc] +opt-level = 3 + [profile.dev.package.effectcraft-export] opt-level = 3 diff --git a/crates/export/Cargo.toml b/crates/export/Cargo.toml index 34c83aab..5bdd0fee 100644 --- a/crates/export/Cargo.toml +++ b/crates/export/Cargo.toml @@ -18,6 +18,7 @@ filmcraft-h264enc = { workspace = true } filmcraft-prores = { workspace = true } filmcraft-aac = { workspace = true } filmcraft-isobmff = { workspace = true } +effectcraft-opusenc = { workspace = true } image = { workspace = true, features = ["png", "jpeg", "tiff", "exr"] } gif = { workspace = true } rayon = { workspace = true } @@ -28,3 +29,4 @@ log = { workspace = true } [dev-dependencies] effectcraft-media = { workspace = true } effectcraft-color = { workspace = true } +filmcraft-opus = { workspace = true } diff --git a/crates/export/tests/opus_roundtrip.rs b/crates/export/tests/opus_roundtrip.rs new file mode 100644 index 00000000..5ea099f0 --- /dev/null +++ b/crates/export/tests/opus_roundtrip.rs @@ -0,0 +1,364 @@ +//! Round trip of `effectcraft-opusenc` through FilmCraft's spec-derived Opus decoder, plus an +//! ffmpeg oracle (skipped when ffmpeg is not installed) reading an Ogg Opus file written here. + +use std::process::Command; + +use effectcraft_opusenc::OpusEncoder; + +const SR: f32 = 48_000.0; +const FRAME: usize = OpusEncoder::FRAME_SIZE; + +/// Encodes interleaved `pcm`, padding the tail so the encoder's look-ahead is flushed. +fn encode(pcm: &[f32], channels: u8, bitrate: u32) -> (OpusEncoder, Vec>) { + let c = channels as usize; + let mut enc = OpusEncoder::new(channels, bitrate); + let total = pcm.len() / c + enc.pre_skip() as usize; + let frames = total.div_ceil(FRAME); + let mut padded = pcm.to_vec(); + padded.resize(frames * FRAME * c, 0.0); + let packets = padded.chunks(FRAME * c).map(|f| enc.encode_float(f)).collect(); + (enc, packets) +} + +/// Decodes with filmcraft-opus, drops the pre-skip and returns planar channels of `len` samples. +fn decode(head: &[u8], packets: &[Vec], len: usize) -> Vec> { + let mut dec = filmcraft_opus::Decoder::new(head).expect("OpusHead"); + dec.set_trim_pre_skip(true); + let mut out: Vec> = vec![Vec::new(); dec.channels()]; + for p in packets { + let planes = dec.decode(Some(p)).expect("decode"); + for (o, p) in out.iter_mut().zip(planes) { + o.extend_from_slice(&p); + } + } + for o in &mut out { + assert!(o.len() >= len, "decoded {} < {len}", o.len()); + o.truncate(len); + } + out +} + +fn roundtrip(planar: &[Vec], bitrate: u32) -> (Vec>, Vec>) { + let c = planar.len(); + let len = planar[0].len(); + let mut pcm = vec![0f32; len * c]; + for (ch, p) in planar.iter().enumerate() { + for (i, &v) in p.iter().enumerate() { + pcm[i * c + ch] = v; + } + } + let (enc, packets) = encode(&pcm, c as u8, bitrate); + let decoded = decode(&enc.opus_head(), &packets, len); + (packets, decoded) +} + +/// SNR in dB over the steady part (the first and last 20 ms are excluded). +fn snr(orig: &[f32], dec: &[f32]) -> f64 { + let (a, b) = (FRAME, orig.len() - FRAME); + let mut s = 0f64; + let mut e = 0f64; + for i in a..b { + s += (orig[i] as f64).powi(2); + e += (orig[i] as f64 - dec[i] as f64).powi(2); + } + 10.0 * (s / e.max(1e-30)).log10() +} + +fn sine(freq: f32, amp: f32, len: usize) -> Vec { + (0..len).map(|i| amp * (2.0 * std::f32::consts::PI * freq * i as f32 / SR).sin()).collect() +} + +fn chirp(f0: f32, f1: f32, amp: f32, len: usize) -> Vec { + let dur = len as f32 / SR; + let k = (f1 - f0) / dur; + (0..len) + .map(|i| { + let t = i as f32 / SR; + amp * (2.0 * std::f32::consts::PI * (f0 * t + 0.5 * k * t * t)).sin() + }) + .collect() +} + +fn noise(amp: f32, len: usize, seed: u32) -> Vec { + let mut s = seed; + (0..len) + .map(|_| { + s = s.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + amp * ((s >> 8) as f32 / (1u32 << 24) as f32 * 2.0 - 1.0) + }) + .collect() +} + +/// Energy (dB) of `x` in each of a few octave bands up to 16 kHz, from a Hann-windowed FFT +/// averaged over 2048-sample blocks. +fn band_energies(x: &[f32]) -> Vec { + const N: usize = 2048; + let edges = [100.0, 250.0, 500.0, 1000.0, 2000.0, 4000.0, 8000.0, 16000.0]; + let mut acc = vec![0f64; edges.len() - 1]; + let mut start = N; + while start + N <= x.len() - N { + let mut re: Vec = (0..N).map(|i| x[start + i] as f64 * (0.5 - 0.5 * (2.0 * std::f64::consts::PI * i as f64 / N as f64).cos())).collect(); + let mut im = vec![0f64; N]; + fft(&mut re, &mut im); + for k in 1..N / 2 { + let f = k as f64 * SR as f64 / N as f64; + if let Some(b) = edges.windows(2).position(|w| f >= w[0] && f < w[1]) { + acc[b] += re[k] * re[k] + im[k] * im[k]; + } + } + start += N; + } + acc.iter().map(|e| 10.0 * (e + 1e-20).log10()).collect() +} + +fn fft(re: &mut [f64], im: &mut [f64]) { + let n = re.len(); + let mut j = 0; + for i in 1..n { + let mut bit = n >> 1; + while j & bit != 0 { + j ^= bit; + bit >>= 1; + } + j |= bit; + if i < j { + re.swap(i, j); + im.swap(i, j); + } + } + let mut len = 2; + while len <= n { + let ang = -2.0 * std::f64::consts::PI / len as f64; + for s in (0..n).step_by(len) { + for k in 0..len / 2 { + let (wr, wi) = ((ang * k as f64).cos(), (ang * k as f64).sin()); + let (a, b) = (s + k, s + k + len / 2); + let tr = re[b] * wr - im[b] * wi; + let ti = re[b] * wi + im[b] * wr; + re[b] = re[a] - tr; + im[b] = im[a] - ti; + re[a] += tr; + im[a] += ti; + } + } + len <<= 1; + } +} + +#[test] +fn sine_440_mono() { + let x = sine(440.0, 0.5, 48_000); + for rate in [64_000, 128_000] { + let (packets, d) = roundtrip(std::slice::from_ref(&x), rate); + assert!(packets.iter().all(|p| p.len() == rate as usize / 400)); + let s = snr(&x, &d[0]); + eprintln!("sine 440 Hz mono @ {} kb/s: SNR {s:.1} dB", rate / 1000); + assert!(s > 20.0, "sine SNR {s:.1} dB at {rate}"); + } +} + +#[test] +fn chirp_mono() { + let x = chirp(100.0, 10_000.0, 0.4, 96_000); + for rate in [64_000, 128_000] { + let (_, d) = roundtrip(std::slice::from_ref(&x), rate); + let s = snr(&x, &d[0]); + eprintln!("chirp 100 Hz-10 kHz mono @ {} kb/s: SNR {s:.1} dB", rate / 1000); + assert!(s > 15.0, "chirp SNR {s:.1} dB at {rate}"); + } +} + +#[test] +fn white_noise_mono() { + let x = noise(0.3, 48_000, 12345); + for rate in [64_000, 128_000] { + let (_, d) = roundtrip(std::slice::from_ref(&x), rate); + let s = snr(&x, &d[0]); + let (eo, ed) = (band_energies(&x), band_energies(&d[0])); + let err: Vec = x.iter().zip(&d[0]).map(|(a, b)| a - b).collect(); + let ee = band_energies(&err); + let octave_snr: Vec = eo.iter().zip(&ee).map(|(a, b)| a - b).collect(); + let worst = eo.iter().zip(&ed).map(|(a, b)| (a - b).abs()).fold(0.0, f64::max); + let min_octave = octave_snr.iter().copied().fold(f64::MAX, f64::min); + eprintln!( + "white noise mono @ {} kb/s: full-band SNR {s:.1} dB, octave SNRs 100 Hz-16 kHz {:.1?} dB, worst octave energy error {worst:.2} dB", + rate / 1000, + octave_snr + ); + // Full-band noise loses everything above 20 kHz (CELT's last band ends there) and spends + // few bits per coefficient on 16-20 kHz, so the full-band waveform SNR is low; the band + // energies must match closely and the octaves below 16 kHz must be waveform-coded. + assert!(worst < 1.5, "band energy error {worst:.2} dB at {rate}: {eo:?} vs {ed:?}"); + assert!(s > 0.5, "noise SNR {s:.1} dB at {rate}"); + assert!(min_octave > if rate >= 128_000 { 10.0 } else { 3.0 }, "octave SNRs {octave_snr:?} at {rate}"); + } +} + +#[test] +fn lowpass_noise_mono() { + // White noise through a 4-pole low-pass at about 2 kHz: a dense spectrum the encoder can + // afford to code as a waveform. + let w = noise(0.8, 48_000, 999); + let a = (-2.0 * std::f32::consts::PI * 2_000.0 / SR).exp(); + let mut st = [0f32; 4]; + let x: Vec = w + .iter() + .map(|&v| { + let mut y = v; + for s in st.iter_mut() { + *s = (1.0 - a) * y + a * *s; + y = *s; + } + y + }) + .collect(); + for rate in [64_000, 128_000] { + let (_, d) = roundtrip(std::slice::from_ref(&x), rate); + let s = snr(&x, &d[0]); + eprintln!("low-pass noise mono @ {} kb/s: SNR {s:.1} dB", rate / 1000); + assert!(s > 15.0, "low-pass noise SNR {s:.1} dB at {rate}"); + } +} + +#[test] +fn silence_stays_silent() { + let x = vec![0f32; 48_000]; + let (packets, d) = roundtrip(&[x.clone(), x], 128_000); + assert!(packets.iter().all(|p| p.len() == 3), "silence frames are 3-byte packets"); + for ch in &d { + let peak = ch.iter().fold(0f32, |m, v| m.max(v.abs())); + assert!(peak < 1e-6, "peak {peak}"); + } +} + +#[test] +fn stereo_distinct_channels() { + let l = sine(440.0, 0.5, 48_000); + let r = chirp(200.0, 6_000.0, 0.3, 48_000); + let (packets, d) = roundtrip(&[l.clone(), r.clone()], 256_000); + assert!(packets.iter().all(|p| p[0] == 0xFC && p.len() == 640)); + let (sl, sr) = (snr(&l, &d[0]), snr(&r, &d[1])); + eprintln!("stereo @ 256 kb/s: L (sine) SNR {sl:.1} dB, R (chirp) SNR {sr:.1} dB"); + assert!(sl > 20.0 && sr > 15.0, "stereo SNR L {sl:.1} R {sr:.1}"); + // No leakage of the chirp into the left channel (2-4 kHz octave: chirp only). + let (el, er) = (band_energies(&d[0]), band_energies(&r)); + assert!(el[4] < er[4] - 30.0, "chirp energy leaked into L: {} vs {}", el[4], er[4]); +} + +#[test] +fn stereo_128k_total() { + let l = sine(1000.0, 0.4, 48_000); + let r = sine(330.0, 0.4, 48_000); + let (_, d) = roundtrip(&[l.clone(), r.clone()], 128_000); + let (sl, sr) = (snr(&l, &d[0]), snr(&r, &d[1])); + eprintln!("stereo @ 128 kb/s: L SNR {sl:.1} dB, R SNR {sr:.1} dB"); + assert!(sl > 15.0 && sr > 15.0, "stereo SNR L {sl:.1} R {sr:.1}"); +} + +#[test] +fn signal_after_silence_and_loud_transient() { + // Silence, then a burst at full scale, then silence again: energy prediction must recover. + let mut x = vec![0f32; 24_000]; + x.extend(sine(2_000.0, 0.99, 24_000)); + x.extend(vec![0f32; 24_000]); + let (_, d) = roundtrip(std::slice::from_ref(&x), 96_000); + let s = snr(&x[24_000 + 2 * FRAME..48_000], &d[0][24_000 + 2 * FRAME..48_000]); + eprintln!("burst after silence @ 96 kb/s: SNR {s:.1} dB"); + assert!(s > 15.0, "burst SNR {s:.1}"); + let tail_peak = d[0][48_000 + 2 * FRAME..].iter().fold(0f32, |m, v| m.max(v.abs())); + assert!(tail_peak < 1e-3, "tail peak {tail_peak}"); +} + +// ---- Ogg Opus (RFC 3533 + RFC 7845) for the ffmpeg oracle ---- + +fn ogg_crc(data: &[u8]) -> u32 { + let mut crc = 0u32; + for &b in data { + crc ^= (b as u32) << 24; + for _ in 0..8 { + crc = if crc & 0x8000_0000 != 0 { (crc << 1) ^ 0x04C1_1DB7 } else { crc << 1 }; + } + } + crc +} + +fn ogg_page(out: &mut Vec, packet: &[u8], header_type: u8, granule: u64, seq: u32) { + let mut lacing = vec![255u8; packet.len() / 255]; + lacing.push((packet.len() % 255) as u8); + let start = out.len(); + out.extend_from_slice(b"OggS"); + out.push(0); + out.push(header_type); + out.extend_from_slice(&granule.to_le_bytes()); + out.extend_from_slice(&0x4543_4F50u32.to_le_bytes()); // serial + out.extend_from_slice(&seq.to_le_bytes()); + out.extend_from_slice(&0u32.to_le_bytes()); // CRC placeholder + out.push(lacing.len() as u8); + out.extend_from_slice(&lacing); + out.extend_from_slice(packet); + let crc = ogg_crc(&out[start..]); + out[start + 22..start + 26].copy_from_slice(&crc.to_le_bytes()); +} + +fn ogg_opus(enc: &OpusEncoder, packets: &[Vec], len: usize) -> Vec { + let mut out = Vec::new(); + ogg_page(&mut out, &enc.opus_head(), 0x02, 0, 0); + let vendor = b"effectcraft-opusenc"; + let mut tags = b"OpusTags".to_vec(); + tags.extend_from_slice(&(vendor.len() as u32).to_le_bytes()); + tags.extend_from_slice(vendor); + tags.extend_from_slice(&0u32.to_le_bytes()); + ogg_page(&mut out, &tags, 0, 0, 1); + let pre_skip = enc.pre_skip() as u64; + for (i, p) in packets.iter().enumerate() { + let last = i + 1 == packets.len(); + let granule = if last { pre_skip + len as u64 } else { ((i + 1) * FRAME) as u64 }; + ogg_page(&mut out, p, if last { 0x04 } else { 0 }, granule, i as u32 + 2); + } + out +} + +#[test] +fn ogg_crc_known_value() { + // CRC-32 (poly 0x04C11DB7, no reflection, init 0) of "123456789". + assert_eq!(ogg_crc(b"123456789"), 0x89A1_897F); +} + +#[test] +fn ffmpeg_oracle_decodes_ogg_opus() { + if Command::new("ffmpeg").arg("-version").output().is_err() { + eprintln!("ffmpeg not installed; skipping the oracle test"); + return; + } + for (channels, rate) in [(1u8, 96_000u32), (2, 192_000)] { + let len = 48_000; + let planar: Vec> = + if channels == 1 { vec![chirp(150.0, 9_000.0, 0.4, len)] } else { vec![sine(440.0, 0.5, len), chirp(150.0, 9_000.0, 0.3, len)] }; + let c = channels as usize; + let mut pcm = vec![0f32; len * c]; + for (ch, p) in planar.iter().enumerate() { + for (i, &v) in p.iter().enumerate() { + pcm[i * c + ch] = v; + } + } + let (enc, packets) = encode(&pcm, channels, rate); + let path = std::env::temp_dir().join(format!("effectcraft-opusenc-{}-{channels}.opus", std::process::id())); + std::fs::write(&path, ogg_opus(&enc, &packets, len)).unwrap(); + let out = Command::new("ffmpeg") + .args(["-v", "error", "-i"]) + .arg(&path) + .args(["-f", "f32le", "-ac", &channels.to_string(), "-ar", "48000", "-"]) + .output() + .expect("run ffmpeg"); + let _ = std::fs::remove_file(&path); + assert!(out.status.success(), "ffmpeg failed: {}", String::from_utf8_lossy(&out.stderr)); + let dec: Vec = out.stdout.chunks_exact(4).map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])).collect(); + assert_eq!(dec.len(), len * c, "ffmpeg must honour pre-skip and the end granule"); + for ch in 0..c { + let d: Vec = (0..len).map(|i| dec[i * c + ch]).collect(); + let s = snr(&planar[ch], &d); + eprintln!("ffmpeg {channels} ch @ {} kb/s, channel {ch}: SNR {s:.1} dB", rate / 1000); + assert!(s > 15.0, "ffmpeg-decoded SNR {s:.1} dB (channel {ch})"); + } + } +} diff --git a/crates/opusenc/Cargo.toml b/crates/opusenc/Cargo.toml new file mode 100644 index 00000000..90b0fd02 --- /dev/null +++ b/crates/opusenc/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "effectcraft-opusenc" +description = "Clean-room pure-Rust Opus encoder (RFC 6716 CELT-only, fullband 48 kHz, 20 ms; RFC 7845 OpusHead) for WebM export audio" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] diff --git a/crates/opusenc/README.md b/crates/opusenc/README.md new file mode 100644 index 00000000..0c76d49f --- /dev/null +++ b/crates/opusenc/README.md @@ -0,0 +1,33 @@ +# effectcraft-opusenc + +Clean-room, pure-Rust Opus encoder for EffectCraft's WebM export audio (layer L0, `std` only, +no `unsafe`). + +- **CELT-only** packets (TOC configuration 31: fullband 20 kHz, 20 ms, one frame per packet), + 48 kHz input, mono or stereo (stereo coded as dual L/R stereo), constant packet size from the + requested bitrate; all-zero frames become 3-byte silence packets. +- Forward MDCT with the low-overlap window, band energies with inter/intra coarse prediction + (cheaper of the two per frame), fine and final energy bits, the decoder's exact bit allocation, + spreading and recursive band splitting, greedy PVQ search and codeword indexing. +- Not implemented: SILK/hybrid modes, transient (short-block) frames, TF changes, pitch + post-filter, intensity stereo, dynamic allocation boosts, VBR rate control. +- `OpusEncoder::pre_skip()` is 120 samples (the MDCT overlap); `opus_head()` returns the RFC 7845 + identification header. + +```rust +let mut enc = effectcraft_opusenc::OpusEncoder::new(2, 128_000); +let packet = enc.encode_float(&vec![0.0; effectcraft_opusenc::OpusEncoder::FRAME_SIZE * 2]); +``` + +## Specifications + +- RFC 6716, *Definition of the Opus Audio Codec* (September 2012), with the RFC 8251 updates + (October 2017) as implemented by the decoder we round-trip against. +- RFC 7845, *Ogg Encapsulation for the Opus Audio Codec* (April 2016), §5.1 `OpusHead` and + §4.2 pre-skip. + +Implemented from the RFC text. Constant tables (band edges, `eMeans`, the coarse-energy Laplace +model, the allocation table, ICDFs) and the decoder-side allocation arithmetic are mirrored from +FilmCraft's first-party, spec-derived `filmcraft-opus` decoder (MIT OR Apache-2.0), which is also +the round-trip oracle in `crates/export/tests/opus_roundtrip.rs` (ffmpeg is a second, external +oracle when installed). No third-party encoder or decoder source was consulted. diff --git a/crates/opusenc/src/bands.rs b/crates/opusenc/src/bands.rs new file mode 100644 index 00000000..744de3fe --- /dev/null +++ b/crates/opusenc/src/bands.rs @@ -0,0 +1,497 @@ +//! Band shape coding: PVQ search and codeword indexing, spreading, recursive band splitting +//! (RFC 6716 §4.3.4, encoder direction). +//! +//! The encoder codes 20 ms long-block frames only (no transients, no TF changes), and stereo +//! as dual (independent L/R) stereo, so a band is always coded by `quant_partition` with a single +//! block. Every bit-budget computation mirrors the decoder so both sides take the same +//! split/pulse decisions. + +use crate::range::{BITRES, RangeEncoder, ilog}; +use crate::rate::{Mode, QTHETA_OFFSET, get_pulses}; +use crate::tables::*; + +pub const SPREAD_NORMAL: usize = 2; + +#[inline] +fn frac_mul16(a: i32, b: i32) -> i32 { + (16384 + (a as i16 as i32) * (b as i16 as i32)) >> 15 +} + +fn bitexact_cos(x: i32) -> i32 { + let tmp = (4096 + x * x) >> 13; + let x2 = tmp; + let x2 = (32767 - x2) + frac_mul16(x2, -7651 + frac_mul16(x2, 8277 + frac_mul16(-626, x2))); + 1 + x2 +} + +fn bitexact_log2tan(isin: i32, icos: i32) -> i32 { + let lc = ilog(icos as u32); + let ls = ilog(isin as u32); + let icos = icos << (15 - lc); + let isin = isin << (15 - ls); + (ls - lc) * (1 << 11) + frac_mul16(isin, frac_mul16(isin, -2597) + 7932) - frac_mul16(icos, frac_mul16(icos, -2597) + 7932) +} + +fn compute_qn(n: i32, b: i32, offset: i32, pulse_cap: i32, stereo: bool) -> i32 { + const EXP2_TABLE8: [i32; 8] = [16384, 17866, 19483, 21247, 23170, 25267, 27554, 30048]; + let mut n2 = 2 * n - 1; + if stereo && n == 2 { + n2 -= 1; + } + let mut qb = (b + n2 * offset) / n2; + qb = qb.min(b - pulse_cap - (4 << BITRES)); + qb = qb.min(8 << BITRES); + if qb < (1 << BITRES >> 1) { + 1 + } else { + let qn = EXP2_TABLE8[(qb & 0x7) as usize] >> (14 - (qb >> BITRES)); + (qn + 1) >> 1 << 1 + } +} + +/// One elementary pass of the spreading rotation, inverted (the decoder applies the forward +/// pass; the encoder undoes it before the pulse search). +fn exp_rotation1_inv(x: &mut [f32], len: usize, stride: usize, c: f32, s: f32) { + if len <= stride { + return; + } + if len >= 2 * stride + 1 { + for i in 0..=len - 2 * stride - 1 { + let x1 = x[i]; + let x2 = x[i + stride]; + x[i + stride] = c * x2 - s * x1; + x[i] = c * x1 + s * x2; + } + } + for i in (0..len - stride).rev() { + let x1 = x[i]; + let x2 = x[i + stride]; + x[i + stride] = c * x2 - s * x1; + x[i] = c * x1 + s * x2; + } +} + +/// Inverse of the decoder's spreading rotation (§4.3.4.3) for one block of `len` samples. +pub fn exp_rotation_inv(x: &mut [f32], len: usize, k: usize, spread: usize) { + const SPREAD_FACTOR: [usize; 3] = [15, 10, 5]; + if 2 * k >= len || spread == 0 { + return; + } + let factor = SPREAD_FACTOR[spread - 1]; + let gain = len as f32 / (len + factor * k) as f32; + let theta = 0.5 * gain * gain; + let c = (0.5 * std::f32::consts::PI * theta).cos(); + let s = (0.5 * std::f32::consts::PI * (1.0 - theta)).cos(); + let mut stride2 = 0; + if len >= 8 { + stride2 = 1; + while stride2 * stride2 + stride2 < len { + stride2 += 1; + } + } + // The decoder applies (stride2: s, c) then (1: c, s); undo in reverse order. + exp_rotation1_inv(x, len, 1, c, s); + if stride2 != 0 { + exp_rotation1_inv(x, len, stride2, s, c); + } +} + +/// Greedy PVQ search: the integer vector `y` with `sum |y| = k` closest in direction to `x`. +pub fn pvq_search(x: &[f32], k: usize, y: &mut [i32]) { + let n = x.len(); + y[..n].fill(0); + let abs: Vec = x.iter().map(|v| v.abs()).collect(); + let sum: f32 = abs.iter().sum(); + if !sum.is_finite() || sum <= 1e-15 { + y[0] = k as i32; + return; + } + let mut placed = 0usize; + let mut xy = 0f32; + let mut yy = 0f32; + if k > n / 2 { + // Pre-project onto the pyramid (rounding down) and place the rest greedily. + let r = (k as f32 - 1.0) / sum; + for j in 0..n { + let p = (abs[j] * r).floor() as i32; + y[j] = p; + placed += p as usize; + xy += abs[j] * p as f32; + yy += (p * p) as f32; + } + } + while placed < k { + let mut best = 0usize; + let mut best_num = -1f32; + let mut best_den = 1f32; + for j in 0..n { + let num = xy + abs[j]; + let num = num * num; + let den = yy + 2.0 * y[j] as f32 + 1.0; + if num * best_den > best_num * den { + best_num = num; + best_den = den; + best = j; + } + } + xy += abs[best]; + yy += 2.0 * y[best] as f32 + 1.0; + y[best] += 1; + placed += 1; + } + for j in 0..n { + if x[j] < 0.0 { + y[j] = -y[j]; + } + } +} + +/// Encodes the PVQ codeword `y` (`sum |y| = k`) as the index the RFC 6716 §4.3.4.2 decoder maps +/// back to `y`. +pub fn encode_pulses(y: &[i32], k: usize, enc: &mut RangeEncoder, v: &mut Vec) { + let n = y.len(); + let w = k + 1; + v.clear(); + v.resize((n + 1) * w, 0); + v[0] = 1; + for nn in 1..=n { + v[nn * w] = 1; + for kk in 1..=k { + let a = v[(nn - 1) * w + kk]; + let b = v[nn * w + kk - 1]; + let c = v[(nn - 1) * w + kk - 1]; + v[nn * w + kk] = a.saturating_add(b).saturating_add(c); + } + } + let total = v[n * w + k]; + let vv = |nn: usize, kx: usize| v[nn * w + kx]; + let mut index = 0u64; + let mut kk = k; + for j in 0..n { + if kk == 0 { + break; + } + let rem = n - j; + let t = y[j].unsigned_abs() as usize; + // Codewords with |y_j| = t start at sum_{m < kk - t} V(rem-1, m) within their sign half. + let mut base: u64 = (0..kk - t).map(|m| vv(rem - 1, m)).sum(); + if y[j] < 0 { + base += (vv(rem - 1, kk) + vv(rem, kk)) / 2; + } + index += base; + kk -= t; + } + debug_assert!(total <= u32::MAX as u64); + if total >= 2 { + enc.uint(index as u32, total as u32); + } +} + +pub struct BandEnc<'m, 'e> { + pub m: &'m Mode, + pub enc: &'e mut RangeEncoder, + pub i: usize, + pub spread: usize, + pub remaining_bits: i32, + pvq_v: Vec, + iy: Vec, + tmp: Vec, +} + +impl BandEnc<'_, '_> { + /// `compute_theta` for a mono split (single block): measures the energy split between the + /// two halves and codes the quantised angle with the triangular pdf. + fn compute_theta_split(&mut self, xa: &[f32], xb: &[f32], b: &mut i32, lm: i32) -> (i32, i32, i32) { + let n = xa.len() as i32; + let i = self.i; + let pulse_cap = self.m.log_n[i] + lm * (1 << BITRES); + let offset = (pulse_cap >> 1) - QTHETA_OFFSET; + let qn = compute_qn(n, *b, offset, pulse_cap, false); + let tell = self.enc.tell_frac(); + let mut itheta = 0i32; + if qn != 1 { + let mid = xa.iter().map(|v| v * v).sum::().sqrt(); + let side = xb.iter().map(|v| v * v).sum::().sqrt(); + let ang = (side as f64).atan2(mid as f64) * std::f64::consts::FRAC_2_PI; + let it = ((ang * qn as f64 + 0.5).floor() as i32).clamp(0, qn) as u32; + let q = qn as u32; + let half = q >> 1; + let ft = (half + 1) * (half + 1); + let (fl, fs) = if it <= half { (it * (it + 1) >> 1, it + 1) } else { (ft - ((q + 1 - it) * (q + 2 - it) >> 1), q + 1 - it) }; + self.enc.encode(fl, fl + fs, ft); + itheta = ((it * 16384) / q) as i32; + } + let qalloc = self.enc.tell_frac() - tell; + *b -= qalloc; + let delta = if itheta == 0 { + -16384 + } else if itheta == 16384 { + 16384 + } else { + let imid = bitexact_cos(itheta); + let iside = bitexact_cos(16384 - itheta); + frac_mul16((n - 1) << 7, bitexact_log2tan(iside, imid)) + }; + (itheta, delta, qalloc) + } + + /// `quant_partition` (single block, encoder side). + pub fn quant_partition(&mut self, x: &[f32], mut b: i32, mut lm: i32) { + let n = x.len(); + let i = self.i; + let cache = self.m.cache(lm, i); + let cache_max = cache[cache[0] as usize] as i32; + if lm != -1 && b > cache_max + 12 && n > 2 { + let half = n >> 1; + let (xa, xb) = x.split_at(half); + lm -= 1; + let (itheta, delta, qalloc) = self.compute_theta_split(xa, xb, &mut b, lm); + let mut mbits = 0.max(b.min((b - delta) / 2)); + let mut sbits = b - mbits; + self.remaining_bits -= qalloc; + let mut rebalance = self.remaining_bits; + if mbits >= sbits { + self.quant_partition(xa, mbits, lm); + rebalance = mbits - (rebalance - self.remaining_bits); + if rebalance > 3 << BITRES && itheta != 0 { + sbits += rebalance - (3 << BITRES); + } + self.quant_partition(xb, sbits, lm); + } else { + self.quant_partition(xb, sbits, lm); + rebalance = sbits - (rebalance - self.remaining_bits); + if rebalance > 3 << BITRES && itheta != 16384 { + mbits += rebalance - (3 << BITRES); + } + self.quant_partition(xa, mbits, lm); + } + return; + } + let m = self.m; + let mut q = m.bits2pulses(i, lm, b); + let mut curr_bits = m.pulses2bits(i, lm, q); + self.remaining_bits -= curr_bits; + while self.remaining_bits < 0 && q > 0 { + self.remaining_bits += curr_bits; + q -= 1; + curr_bits = m.pulses2bits(i, lm, q); + self.remaining_bits -= curr_bits; + } + if q != 0 { + let k = get_pulses(q) as usize; + let mut tmp = std::mem::take(&mut self.tmp); + tmp.clear(); + tmp.extend_from_slice(x); + exp_rotation_inv(&mut tmp, n, k, self.spread); + let mut iy = std::mem::take(&mut self.iy); + iy.clear(); + iy.resize(n, 0); + pvq_search(&tmp, k, &mut iy); + encode_pulses(&iy, k, self.enc, &mut self.pvq_v); + self.iy = iy; + self.tmp = tmp; + } + } +} + +/// Codes all band shapes (`quant_all_bands`, encoder side, long blocks). `x` holds `c` channels +/// of `n` normalised MDCT coefficients each. +#[allow(clippy::too_many_arguments)] +pub fn quant_all_bands( + m: &Mode, + start: usize, + end: usize, + x: &[f32], + n_total: usize, + c: usize, + pulses: &[i32; NB_EBANDS], + spread: usize, + dual_stereo: bool, + intensity: usize, + total_bits: i32, + mut balance: i32, + enc: &mut RangeEncoder, + lm: usize, + coded_bands: usize, +) { + let mm = 1usize << lm; + debug_assert!(c == 1 || dual_stereo || intensity <= start, "stereo frames are coded as dual stereo"); + let mut ctx = BandEnc { m, enc, i: 0, spread, remaining_bits: 0, pvq_v: Vec::new(), iy: Vec::new(), tmp: Vec::new() }; + for i in start..end { + ctx.i = i; + let xo = mm * EBANDS[i] as usize; + let n = mm * EBANDS[i + 1] as usize - xo; + let tell = ctx.enc.tell_frac(); + if i != start { + balance -= tell; + } + let remaining_bits = total_bits - tell - 1; + ctx.remaining_bits = remaining_bits; + let b = if i < coded_bands { + let curr_balance = balance / 3.min(coded_bands as i32 - i as i32); + 0.max(16383.min((remaining_bits + 1).min(pulses[i] + curr_balance))) + } else { + 0 + }; + // Bands at and above `intensity` (= coded_bands) carry no bits, so they emit no symbols. + if i < coded_bands || c == 1 { + if c == 2 && dual_stereo && i < intensity { + ctx.quant_partition(&x[xo..xo + n], b / 2, lm as i32); + ctx.quant_partition(&x[n_total + xo..n_total + xo + n], b / 2, lm as i32); + } else { + ctx.quant_partition(&x[xo..xo + n], b, lm as i32); + } + } + balance += pulses[i] + tell; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::range::dec::RangeDecoder; + + fn exp_rotation1(x: &mut [f32], len: usize, stride: usize, c: f32, s: f32) { + let ms = -s; + if len <= stride { + return; + } + for i in 0..len - stride { + let x1 = x[i]; + let x2 = x[i + stride]; + x[i + stride] = c * x2 + s * x1; + x[i] = c * x1 + ms * x2; + } + if len >= 2 * stride + 1 { + for i in (0..=len - 2 * stride - 1).rev() { + let x1 = x[i]; + let x2 = x[i + stride]; + x[i + stride] = c * x2 + s * x1; + x[i] = c * x1 + ms * x2; + } + } + } + + /// The decoder's spreading rotation (single block). + fn exp_rotation(x: &mut [f32], len: usize, k: usize, spread: usize) { + const SPREAD_FACTOR: [usize; 3] = [15, 10, 5]; + if 2 * k >= len || spread == 0 { + return; + } + let factor = SPREAD_FACTOR[spread - 1]; + let gain = len as f32 / (len + factor * k) as f32; + let theta = 0.5 * gain * gain; + let c = (0.5 * std::f32::consts::PI * theta).cos(); + let s = (0.5 * std::f32::consts::PI * (1.0 - theta)).cos(); + let mut stride2 = 0; + if len >= 8 { + stride2 = 1; + while stride2 * stride2 + stride2 < len { + stride2 += 1; + } + } + if stride2 != 0 { + exp_rotation1(x, len, stride2, s, c); + } + exp_rotation1(x, len, 1, c, s); + } + + #[test] + fn spreading_inverse_is_exact() { + for &len in &[8usize, 16, 24, 48, 96, 176] { + for k in [1usize, 2, 3, 5] { + let orig: Vec = (0..len).map(|i| ((i * 31 % 17) as f32 - 8.0) / 8.0).collect(); + let mut x = orig.clone(); + exp_rotation_inv(&mut x, len, k, SPREAD_NORMAL); + exp_rotation(&mut x, len, k, SPREAD_NORMAL); + for (a, b) in x.iter().zip(&orig) { + assert!((a - b).abs() < 1e-4, "len={len} k={k}"); + } + } + } + } + + /// PVQ decoder of RFC 6716 §4.3.4.2 (as in FilmCraft's decoder). + fn decode_pulses(n: usize, k: usize, dec: &mut RangeDecoder) -> Vec { + let w = k + 1; + let mut v = vec![0u64; (n + 1) * w]; + v[0] = 1; + for nn in 1..=n { + v[nn * w] = 1; + for kk in 1..=k { + v[nn * w + kk] = v[(nn - 1) * w + kk].saturating_add(v[nn * w + kk - 1]).saturating_add(v[(nn - 1) * w + kk - 1]); + } + } + let total = v[n * w + k]; + let mut i = if total >= 2 { dec.uint(total as u32) as u64 } else { 0 }; + let mut y = vec![0i32; n]; + let mut kk = k; + for j in 0..n { + let rem = n - j; + if kk == 0 { + break; + } + let vv = |nn: usize, kx: usize| v[nn * w + kx]; + let mut p = (vv(rem - 1, kk) + vv(rem, kk)) / 2; + let neg = i >= p; + if neg { + i -= p; + } + let k0 = kk; + p -= vv(rem - 1, kk); + while p > i && kk > 0 { + kk -= 1; + p -= vv(rem - 1, kk); + } + let mag = (k0 - kk) as i32; + y[j] = if neg { -mag } else { mag }; + i -= p.min(i); + } + y + } + + fn all_codewords(n: usize, k: usize) -> Vec> { + if n == 0 { + return if k == 0 { vec![vec![]] } else { vec![] }; + } + let mut out = Vec::new(); + for t in -(k as i32)..=(k as i32) { + for mut rest in all_codewords(n - 1, k - t.unsigned_abs() as usize) { + rest.insert(0, t); + out.push(rest); + } + } + out + } + + #[test] + fn pvq_index_roundtrip_exhaustive() { + let mut v = Vec::new(); + for n in 1..6usize { + for k in 1..6usize { + for y in all_codewords(n, k) { + let mut e = RangeEncoder::new(); + encode_pulses(&y, k, &mut e, &mut v); + let buf = e.done(16); + let mut d = RangeDecoder::new(&buf); + assert_eq!(decode_pulses(n, k, &mut d), y, "n={n} k={k}"); + } + } + } + } + + #[test] + fn pvq_search_finds_good_direction() { + let x: Vec = (0..32).map(|i| ((i as f32) * 0.7).sin()).collect(); + let mut y = vec![0i32; 32]; + for k in [1usize, 4, 16, 64] { + pvq_search(&x, k, &mut y); + assert_eq!(y.iter().map(|v| v.unsigned_abs() as usize).sum::(), k); + let xy: f32 = x.iter().zip(&y).map(|(a, &b)| a * b as f32).sum(); + let yy: f32 = y.iter().map(|&b| (b * b) as f32).sum(); + let xx: f32 = x.iter().map(|a| a * a).sum(); + let corr = xy / (xx * yy).sqrt(); + assert!(corr > if k >= 16 { 0.9 } else { 0.2 }, "k={k} corr={corr}"); + } + } +} diff --git a/crates/opusenc/src/energy.rs b/crates/opusenc/src/energy.rs new file mode 100644 index 00000000..0057ae33 --- /dev/null +++ b/crates/opusenc/src/energy.rs @@ -0,0 +1,230 @@ +//! Band energy quantisation: Laplace-coded coarse energy with inter/intra prediction, fine +//! energy and the final refinement bits (RFC 6716 §4.3.2, encoder direction). +//! +//! The quantised energies are updated with exactly the decoder's arithmetic so the prediction +//! state never drifts between encoder and decoder. + +use crate::range::RangeEncoder; +use crate::rate::MAX_FINE_BITS; +use crate::tables::*; + +const LAPLACE_MINP: u32 = 1; +const LAPLACE_NMIN: u32 = 16; + +fn laplace_get_freq1(fs0: u32, decay: u32) -> u32 { + let ft = 32768 - LAPLACE_MINP * (2 * LAPLACE_NMIN) - fs0; + (ft * (16384 - decay)) >> 15 +} + +/// Encodes a Laplace-distributed integer (`fs` = probability of 0 in Q15, `decay` in Q14). +/// Returns the value actually coded (very large magnitudes are clamped to the representable +/// range). +pub fn laplace_encode(enc: &mut RangeEncoder, value: i32, mut fs: u32, decay: u32) -> i32 { + let mut fl = 0u32; + let mut coded = value; + if value != 0 { + let s: i32 = if value < 0 { -1 } else { 0 }; + let val = value.abs(); + fl = fs; + fs = laplace_get_freq1(fs, decay); + let mut i = 1; + while fs > 0 && i < val { + fs *= 2; + fl += fs + 2 * LAPLACE_MINP; + fs = (fs * decay) >> 15; + i += 1; + } + if fs == 0 { + let ndi_max = (32768 - fl + LAPLACE_MINP - 1) as i32; + let ndi_max = (ndi_max - s) >> 1; + let di = (val - i).min(ndi_max - 1); + fl = (fl as i32 + (2 * di + 1 + s) * LAPLACE_MINP as i32) as u32; + fs = LAPLACE_MINP.min(32768 - fl); + let mag = i + di; + coded = if s < 0 { -mag } else { mag }; + } else { + fs += LAPLACE_MINP; + if s == 0 { + fl += fs; + } + } + } + enc.encode_bin(fl, fl + fs, 15); + coded +} + +/// `quant_coarse_energy` for one prediction mode. `e` holds the target log2 energies (relative +/// to `E_MEANS`), `old_e` the quantised energies, updated in place. `budget` is the frame size in +/// bits. +#[allow(clippy::too_many_arguments)] +pub fn quant_coarse_energy( + start: usize, + end: usize, + e: &[f32; 2 * NB_EBANDS], + old_e: &mut [f32; 2 * NB_EBANDS], + intra: bool, + enc: &mut RangeEncoder, + c: usize, + lm: usize, + budget: i32, +) { + let prob = &E_PROB_MODEL[lm][intra as usize]; + let (coef, beta) = if intra { (0.0, BETA_INTRA) } else { (PRED_COEF[lm], BETA_COEF[lm]) }; + let mut prev = [0f32; 2]; + for i in start..end { + for ch in 0..c { + let idx = i + ch * NB_EBANDS; + let oldc = old_e[idx].max(-9.0); + let x = e[idx]; + let f = x - coef * oldc - prev[ch]; + let mut qi = (f + 0.5).floor().clamp(-64.0, 64.0) as i32; + let tell = enc.tell(); + // Keep a few bits per remaining band in reserve when the budget runs out. + let bits_left = budget - tell - 3 * c as i32 * (end - i) as i32; + if i != start && bits_left < 30 { + if bits_left < 24 { + qi = qi.min(1); + } + if bits_left < 16 { + qi = qi.max(-1); + } + } + if budget - tell >= 15 { + let pi = 2 * i.min(20); + qi = laplace_encode(enc, qi, (prob[pi] as u32) << 7, (prob[pi + 1] as u32) << 6); + } else if budget - tell >= 2 { + qi = qi.clamp(-1, 1); + enc.icdf(if qi < 0 { (-2 * qi - 1) as usize } else { (2 * qi) as usize }, &SMALL_ENERGY_ICDF, 2); + } else if budget - tell >= 1 { + qi = qi.clamp(-1, 0); + enc.bit_logp(qi == -1, 1); + } else { + qi = -1; + } + let q = qi as f32; + old_e[idx] = oldc; + let tmp = coef * old_e[idx] + prev[ch] + q; + old_e[idx] = tmp; + prev[ch] = prev[ch] + q - beta * q; + } + } +} + +/// `quant_fine_energy`: raw bits refining each band's quantised energy. +pub fn quant_fine_energy( + start: usize, + end: usize, + e: &[f32; 2 * NB_EBANDS], + old_e: &mut [f32; 2 * NB_EBANDS], + fine_quant: &[i32; NB_EBANDS], + enc: &mut RangeEncoder, + c: usize, +) { + for i in start..end { + let fq = fine_quant[i]; + if fq <= 0 { + continue; + } + for ch in 0..c { + let idx = i + ch * NB_EBANDS; + let err = e[idx] - old_e[idx]; + let levels = 1i32 << fq; + let q2 = (((err + 0.5) * levels as f32).floor() as i32).clamp(0, levels - 1); + enc.bits(q2 as u32, fq as u32); + let offset = (q2 as f32 + 0.5) * (1 << (14 - fq)) as f32 / 16384.0 - 0.5; + old_e[idx] += offset; + } + } +} + +/// `quant_energy_finalise`: spends the bits left at the end of the frame on one more bit of +/// energy resolution per band, in priority order. +#[allow(clippy::too_many_arguments)] +pub fn quant_energy_finalise( + start: usize, + end: usize, + e: &[f32; 2 * NB_EBANDS], + old_e: &mut [f32; 2 * NB_EBANDS], + fine_quant: &[i32; NB_EBANDS], + fine_priority: &[i32; NB_EBANDS], + mut bits_left: i32, + enc: &mut RangeEncoder, + c: usize, +) { + for prio in 0..2 { + let mut i = start; + while i < end && bits_left >= c as i32 { + if fine_quant[i] >= MAX_FINE_BITS || fine_priority[i] != prio { + i += 1; + continue; + } + for ch in 0..c { + let idx = i + ch * NB_EBANDS; + let q2 = (e[idx] - old_e[idx] >= 0.0) as u32; + enc.bits(q2, 1); + let offset = (q2 as f32 - 0.5) * (1 << (14 - fine_quant[i] - 1)) as f32 / 16384.0; + old_e[idx] += offset; + bits_left -= 1; + } + i += 1; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::range::dec::RangeDecoder; + + /// Laplace decoder of RFC 6716 §4.3.2.1 (as in FilmCraft's decoder). + fn laplace_decode(dec: &mut RangeDecoder, mut fs: u32, decay: u32) -> i32 { + let mut val = 0i32; + let fm = dec.decode(32768); + let mut fl = 0u32; + if fm >= fs { + val += 1; + fl = fs; + fs = laplace_get_freq1(fs, decay) + LAPLACE_MINP; + while fs > LAPLACE_MINP && fm >= fl + 2 * fs { + fs *= 2; + fl += fs; + fs = (((fs - 2 * LAPLACE_MINP) * decay) >> 15) + LAPLACE_MINP; + val += 1; + } + if fs <= LAPLACE_MINP { + let di = (fm - fl) >> 1; + val += di as i32; + fl += 2 * di * LAPLACE_MINP; + } + if fm < fl + fs { + val = -val; + } else { + fl += fs; + } + } + dec.update(fl, (fl + fs).min(32768), 32768); + val + } + + #[test] + fn laplace_roundtrip_all_models() { + for lm in 0..4 { + for intra in 0..2 { + for band in 0..21 { + let prob = &E_PROB_MODEL[lm][intra]; + let (fs, decay) = ((prob[2 * band] as u32) << 7, (prob[2 * band + 1] as u32) << 6); + let vals: Vec = (-64..=64).collect(); + let mut e = RangeEncoder::new(); + let coded: Vec = vals.iter().map(|&v| laplace_encode(&mut e, v, fs, decay)).collect(); + let size = (e.tell() as usize).div_ceil(8); + let buf = e.done(size); + let mut d = RangeDecoder::new(&buf); + for (&v, &c) in vals.iter().zip(&coded) { + assert!(c == v || (c.signum() == v.signum() && c.abs() < v.abs()), "clamp {v} -> {c}"); + assert_eq!(laplace_decode(&mut d, fs, decay), c, "lm={lm} intra={intra} band={band} v={v}"); + } + } + } + } + } +} diff --git a/crates/opusenc/src/lib.rs b/crates/opusenc/src/lib.rs new file mode 100644 index 00000000..1e7a2cec --- /dev/null +++ b/crates/opusenc/src/lib.rs @@ -0,0 +1,311 @@ +//! Clean-room pure-Rust Opus encoder (RFC 6716 CELT-only mode; RFC 7845 `OpusHead`). +//! +//! Produces fullband (20 kHz) 48 kHz Opus packets of 20 ms, mono or stereo, at a constant +//! packet size derived from the requested bitrate. Each packet is a single CELT frame (TOC +//! configuration 31, frame-count code 0). Stereo is coded as dual (independent L/R) stereo. +//! +//! Layer L0: no dependencies beyond `std`; no `unsafe`; builds for `wasm32-unknown-unknown`. +//! +//! ``` +//! use effectcraft_opusenc::OpusEncoder; +//! let mut enc = OpusEncoder::new(2, 128_000); +//! let pcm = vec![0.0f32; OpusEncoder::FRAME_SIZE * 2]; +//! let packet = enc.encode_float(&pcm); +//! assert_eq!(packet[0] >> 3, 31); // CELT-only, fullband, 20 ms +//! assert_eq!(enc.opus_head().len(), 19); +//! ``` + +// The bit-allocation code mirrors the normative integer arithmetic; keep C-like shift expressions. +#![allow(clippy::precedence, clippy::int_plus_one)] + +mod bands; +mod energy; +mod mdct; +mod range; +mod rate; +mod tables; + +use std::sync::OnceLock; + +use bands::{SPREAD_NORMAL, quant_all_bands}; +use energy::{quant_coarse_energy, quant_energy_finalise, quant_fine_energy}; +use mdct::Mdct; +use range::{BITRES, RangeEncoder}; +use rate::{Mode, compute_allocation}; +use tables::*; + +/// Samples per channel in one 20 ms frame at 48 kHz. +const N: usize = 960; +/// LM of a 20 ms frame (8 short blocks). +const LM: usize = 3; +/// Smallest CELT payload the encoder produces (bytes). +const MIN_FRAME_BYTES: usize = 16; +/// Largest CELT payload allowed in one frame (RFC 6716 §3.4, R2). +const MAX_FRAME_BYTES: usize = 1275; + +fn mode() -> &'static Mode { + static MODE: OnceLock = OnceLock::new(); + MODE.get_or_init(Mode::new) +} + +/// Opus encoder for one mono or stereo stream at 48 kHz. +pub struct OpusEncoder { + channels: usize, + /// CELT payload bytes per packet (the packet is one byte longer: the TOC). + frame_bytes: usize, + mdct: Mdct, + /// Last input sample of each channel (pre-emphasis filter memory, ×32768). + preemph_mem: [f32; 2], + /// Last `OVERLAP` pre-emphasised samples of each channel. + hist: Vec, + /// Quantised band energies, exactly as the decoder holds them. + old_band_e: [f32; 2 * NB_EBANDS], + block: Vec, + freq: Vec, +} + +impl OpusEncoder { + /// Samples per channel per packet (20 ms at 48 kHz). + pub const FRAME_SIZE: usize = N; + + /// Creates an encoder for `channels` (1 or 2) channels of 48 kHz audio at `bitrate_bps` + /// (total for all channels; packets are constant-size, clamped to 6.8–510 kb/s). + pub fn new(channels: u8, bitrate_bps: u32) -> Self { + assert!(channels == 1 || channels == 2, "Opus family-0 streams have 1 or 2 channels"); + let c = channels as usize; + let packet_bytes = (bitrate_bps as usize).div_ceil(400); + let frame_bytes = packet_bytes.saturating_sub(1).clamp(MIN_FRAME_BYTES, MAX_FRAME_BYTES); + OpusEncoder { + channels: c, + frame_bytes, + mdct: Mdct::new(N), + preemph_mem: [0.0; 2], + hist: vec![0.0; c * OVERLAP], + old_band_e: [-28.0; 2 * NB_EBANDS], + block: vec![0.0; N + OVERLAP], + freq: vec![0.0; c * N], + } + } + + /// Number of channels. + pub fn channels(&self) -> u8 { + self.channels as u8 + } + + /// Actual bitrate in bits per second (packet size × 50 packets per second). + pub fn bitrate(&self) -> u32 { + ((self.frame_bytes + 1) * 8 * 50) as u32 + } + + /// Samples at 48 kHz the decoder must discard from the start of the stream (the MDCT + /// overlap); this is the `OpusHead` pre-skip. + pub fn pre_skip(&self) -> u16 { + OVERLAP as u16 + } + + /// The 19-byte `OpusHead` identification header (RFC 7845 §5.1): version 1, channel count, + /// pre-skip, input sample rate 48000, output gain 0, channel mapping family 0. + pub fn opus_head(&self) -> Vec { + let mut h = Vec::with_capacity(19); + h.extend_from_slice(b"OpusHead"); + h.push(1); + h.push(self.channels as u8); + h.extend_from_slice(&self.pre_skip().to_le_bytes()); + h.extend_from_slice(&48_000u32.to_le_bytes()); + h.extend_from_slice(&0i16.to_le_bytes()); + h.push(0); + h + } + + /// Encodes one 20 ms frame of interleaved f32 samples in [-1, 1] (exactly + /// `FRAME_SIZE * channels` values) and returns one Opus packet (TOC byte + CELT frame). + pub fn encode_float(&mut self, pcm: &[f32]) -> Vec { + let c = self.channels; + assert_eq!(pcm.len(), N * c, "encode_float takes exactly FRAME_SIZE samples per channel"); + + // Pre-emphasis, the MDCT and the band energies. + let mut band_e = [0f32; 2 * NB_EBANDS]; + let mut log_e = [0f32; 2 * NB_EBANDS]; + let mut peak = 0f32; + for ch in 0..c { + self.block[..OVERLAP].copy_from_slice(&self.hist[ch * OVERLAP..(ch + 1) * OVERLAP]); + let mut mem = self.preemph_mem[ch]; + for j in 0..N { + let s = pcm[j * c + ch]; + let s = if s.is_finite() { s.clamp(-2.0, 2.0) * 32768.0 } else { 0.0 }; + let v = s - PREEMPH * mem; + mem = s; + self.block[OVERLAP + j] = v; + } + self.preemph_mem[ch] = mem; + self.hist[ch * OVERLAP..(ch + 1) * OVERLAP].copy_from_slice(&self.block[N..]); + peak = self.block.iter().fold(peak, |p, v| p.max(v.abs())); + let freq = &mut self.freq[ch * N..(ch + 1) * N]; + self.mdct.forward(&self.block, freq); + for i in 0..NB_EBANDS { + let lo = (EBANDS[i] as usize) << LM; + let hi = (EBANDS[i + 1] as usize) << LM; + let e = (1e-27f32 + freq[lo..hi].iter().map(|v| v * v).sum::()).sqrt(); + band_e[ch * NB_EBANDS + i] = e; + log_e[ch * NB_EBANDS + i] = e.log2() - E_MEANS[i]; + let g = 1.0 / e; + for v in freq[lo..hi].iter_mut() { + *v *= g; + } + } + } + let silence = peak < 1e-4; + let toc = (31u8 << 3) | if c == 2 { 4 } else { 0 }; + let mut packet = vec![toc]; + packet.extend_from_slice(&self.encode_celt(&log_e, silence)); + packet + } + + /// Codes one CELT frame (RFC 6716 §4.3, in bitstream order). + fn encode_celt(&mut self, log_e: &[f32; 2 * NB_EBANDS], silence: bool) -> Vec { + let m = mode(); + let c = self.channels; + let (start, end) = (0usize, NB_EBANDS); + let mut enc = RangeEncoder::new(); + if c == 1 { + for i in 0..NB_EBANDS { + self.old_band_e[i] = self.old_band_e[i].max(self.old_band_e[NB_EBANDS + i]); + } + } + if silence { + // The silence flag; the decoder treats every remaining bit as consumed. + enc.bit_logp(true, 15); + self.old_band_e = [-28.0; 2 * NB_EBANDS]; + return enc.done(2); + } + let len = self.frame_bytes; + let total_bits_raw = len as i32 * 8; + enc.bit_logp(false, 15); + // No pitch post-filter. + if enc.tell() + 16 <= total_bits_raw { + enc.bit_logp(false, 1); + } + // Long blocks only. + if enc.tell() + 3 <= total_bits_raw { + enc.bit_logp(false, 3); + } + // Coarse energy, choosing intra or inter prediction by trial (whichever is cheaper). + if enc.tell() + 3 <= total_bits_raw { + let mut enc_intra = enc.clone(); + let mut old_intra = self.old_band_e; + enc_intra.bit_logp(true, 3); + quant_coarse_energy(start, end, log_e, &mut old_intra, true, &mut enc_intra, c, LM, total_bits_raw); + enc.bit_logp(false, 3); + let mut old_inter = self.old_band_e; + quant_coarse_energy(start, end, log_e, &mut old_inter, false, &mut enc, c, LM, total_bits_raw); + if enc_intra.tell_frac() < enc.tell_frac() { + enc = enc_intra; + self.old_band_e = old_intra; + } else { + self.old_band_e = old_inter; + } + } else { + quant_coarse_energy(start, end, log_e, &mut self.old_band_e, false, &mut enc, c, LM, total_bits_raw); + } + // TF resolution: no changes. + { + let mut budget = total_bits_raw; + let mut tell = enc.tell(); + let mut logp = 4; + let tf_select_rsv = LM > 0 && tell + logp + 1 <= budget; + budget -= tf_select_rsv as i32; + for _ in start..end { + if tell + logp <= budget { + enc.bit_logp(false, logp as u32); + tell = enc.tell(); + } + logp = 5; + } + if tf_select_rsv && TF_SELECT_TABLE[LM][0] != TF_SELECT_TABLE[LM][2] { + enc.bit_logp(false, 1); + } + } + if enc.tell() + 4 <= total_bits_raw { + enc.icdf(SPREAD_NORMAL, &SPREAD_ICDF, 5); + } + // Dynamic allocation: no boosts. + let cap = m.init_caps(LM, c); + let offsets = [0i32; NB_EBANDS]; + let total_bits = total_bits_raw << BITRES; + let mut tellf = enc.tell_frac(); + for &cp in cap.iter().take(end).skip(start) { + if tellf + (6 << BITRES) < total_bits && 0 < cp { + enc.bit_logp(false, 6); + tellf = enc.tell_frac(); + } + } + let alloc_trim = 5; + if tellf + (6 << BITRES) <= total_bits { + enc.icdf(alloc_trim as usize, &TRIM_ICDF, 7); + } + let bits = (total_bits_raw << BITRES) - enc.tell_frac() - 1; + let alloc = compute_allocation(m, start, end, &offsets, &cap, alloc_trim, bits, c, LM, &mut enc); + quant_fine_energy(start, end, log_e, &mut self.old_band_e, &alloc.fine_quant, &mut enc, c); + quant_all_bands( + m, + start, + end, + &self.freq, + N, + c, + &alloc.pulses, + SPREAD_NORMAL, + alloc.dual_stereo, + alloc.intensity, + total_bits_raw << BITRES, + alloc.balance, + &mut enc, + LM, + alloc.coded_bands, + ); + let bits_left = total_bits_raw - enc.tell(); + quant_energy_finalise(start, end, log_e, &mut self.old_band_e, &alloc.fine_quant, &alloc.fine_priority, bits_left, &mut enc, c); + debug_assert!(enc.tell() <= total_bits_raw, "frame overflow: {} > {}", enc.tell(), total_bits_raw); + if c == 1 { + let (a, b) = self.old_band_e.split_at_mut(NB_EBANDS); + b.copy_from_slice(a); + } + enc.done(len) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn opus_head_layout() { + let e = OpusEncoder::new(2, 96_000); + let h = e.opus_head(); + assert_eq!(&h[..8], b"OpusHead"); + assert_eq!(h[8], 1); + assert_eq!(h[9], 2); + assert_eq!(u16::from_le_bytes([h[10], h[11]]), e.pre_skip()); + assert_eq!(u32::from_le_bytes([h[12], h[13], h[14], h[15]]), 48_000); + assert_eq!(&h[16..], &[0, 0, 0]); + } + + #[test] + fn packet_sizes_follow_bitrate() { + for (ch, rate) in [(1u8, 64_000u32), (2, 128_000), (2, 256_000)] { + let mut e = OpusEncoder::new(ch, rate); + let pcm: Vec = (0..N * ch as usize).map(|i| (i as f32 * 0.05).sin() * 0.5).collect(); + let p = e.encode_float(&pcm); + assert_eq!(p.len(), rate as usize / 400); + assert_eq!(p[0], 0xF8 | if ch == 2 { 4 } else { 0 }); + assert_eq!(e.bitrate(), rate); + } + } + + #[test] + fn silence_is_a_tiny_packet() { + let mut e = OpusEncoder::new(1, 64_000); + let p = e.encode_float(&vec![0.0; N]); + assert_eq!(p.len(), 3); + } +} diff --git a/crates/opusenc/src/mdct.rs b/crates/opusenc/src/mdct.rs new file mode 100644 index 00000000..814de037 --- /dev/null +++ b/crates/opusenc/src/mdct.rs @@ -0,0 +1,293 @@ +//! Forward MDCT with the CELT low-overlap window (RFC 6716 §4.3.7, run in the analysis +//! direction). +//! +//! A 20 ms frame is a 1920-point MDCT (960 coefficients) whose window is zero for 420 samples, +//! rises over the 120-sample overlap, is flat for 840 samples, falls over the next 120 samples +//! and is zero again for the last 420. The input block is therefore the 120 overlap samples kept +//! from the previous frame followed by the 960 new samples. The MDCT is folded into a DCT-IV, +//! which is computed with an N/2-point complex FFT. + +use crate::tables::OVERLAP; + +#[derive(Clone, Copy, Default, Debug)] +pub struct Cpx { + pub re: f32, + pub im: f32, +} + +impl Cpx { + #[inline] + fn mul(self, o: Cpx) -> Cpx { + Cpx { re: self.re * o.re - self.im * o.im, im: self.re * o.im + self.im * o.re } + } + #[inline] + fn add(self, o: Cpx) -> Cpx { + Cpx { re: self.re + o.re, im: self.im + o.im } + } +} + +/// Mixed-radix (2, 3, 4, 5) forward complex FFT, unscaled (decimation in time, recursive). +pub struct Fft { + n: usize, + factors: Vec, + twiddle: Vec, +} + +impl Fft { + pub fn new(n: usize) -> Fft { + let mut factors = Vec::new(); + let mut m = n; + for p in [4usize, 2, 3, 5] { + while m.is_multiple_of(p) { + factors.push(p); + m /= p; + } + } + assert_eq!(m, 1, "unsupported FFT size {n}"); + let twiddle = (0..n) + .map(|k| { + let ph = -2.0 * std::f64::consts::PI * k as f64 / n as f64; + Cpx { re: ph.cos() as f32, im: ph.sin() as f32 } + }) + .collect(); + Fft { n, factors, twiddle } + } + + pub fn process(&self, data: &mut [Cpx], scratch: &mut Vec) { + scratch.clear(); + scratch.extend_from_slice(&data[..self.n]); + self.rec(&mut data[..self.n], scratch, 1, self.n, 0, 1); + } + + fn rec(&self, out: &mut [Cpx], inp: &[Cpx], in_stride: usize, n: usize, fi: usize, tw_stride: usize) { + let p = self.factors[fi]; + let m = n / p; + if m == 1 { + for (i, o) in out.iter_mut().enumerate().take(p) { + *o = inp[i * in_stride]; + } + } else { + for q in 0..p { + self.rec(&mut out[q * m..(q + 1) * m], &inp[q * in_stride..], in_stride * p, m, fi + 1, tw_stride * p); + } + } + let big = self.n; + let mut t = [Cpx::default(); 5]; + for k in 0..m { + for (q, tq) in t.iter_mut().enumerate().take(p) { + *tq = out[q * m + k].mul(self.twiddle[(q * k * tw_stride) % big]); + } + for q2 in 0..p { + let mut acc = Cpx::default(); + for (q, tq) in t.iter().enumerate().take(p) { + acc = acc.add(tq.mul(self.twiddle[((q * q2) % p) * (big / p)])); + } + out[q2 * m + k] = acc; + } + } + } +} + +/// Forward MDCT producing `n` coefficients from a block of `n + OVERLAP` samples. +pub struct Mdct { + /// Number of coefficients (half the MDCT length). + n: usize, + fft: Fft, + pre: Vec, + post: Vec, + /// Rising half of the low-overlap window (RFC 6716 §4.3.7). + pub window: [f32; OVERLAP], + scale: f32, + z: Vec, + scratch: Vec, + v: Vec, +} + +impl Mdct { + pub fn new(n: usize) -> Mdct { + let n2 = n / 2; + let pi = std::f64::consts::PI; + let pre = (0..n2) + .map(|m| { + let ph = -pi * m as f64 / n as f64; + Cpx { re: ph.cos() as f32, im: ph.sin() as f32 } + }) + .collect(); + let post = (0..n2) + .map(|p| { + let ph = -pi * (p as f64 + 0.25) / n as f64; + Cpx { re: ph.cos() as f32, im: ph.sin() as f32 } + }) + .collect(); + let mut window = [0f32; OVERLAP]; + for (i, w) in window.iter_mut().enumerate() { + let x = std::f64::consts::FRAC_PI_2 * (i as f64 + 0.5) / OVERLAP as f64; + let s = x.sin(); + *w = (std::f64::consts::FRAC_PI_2 * s * s).sin() as f32; + } + Mdct { n, fft: Fft::new(n2), pre, post, window, scale: 2.0 / n as f32, z: Vec::new(), scratch: Vec::new(), v: vec![0.0; n] } + } + + /// Transforms `block` (`n + OVERLAP` samples: the previous frame's last `OVERLAP` samples, then + /// `n` new ones) into `out[..n]`, scaled so that the RFC 6716 decoder's inverse MDCT and + /// overlap-add reproduce the input. + pub fn forward(&mut self, block: &[f32], out: &mut [f32]) { + let n = self.n; + let n2 = n / 2; + debug_assert_eq!(block.len(), n + OVERLAP); + // Place the windowed block into the (virtual) 2n-sample MDCT input u at offset `pad`. + let pad = (n - OVERLAP) / 2; + let w = &self.window; + let u = |t: usize| -> f32 { + if t < pad || t >= pad + n + OVERLAP { + return 0.0; + } + let j = t - pad; + let g = if j < OVERLAP { + w[j] + } else if j >= n { + w[OVERLAP - 1 - (j - n)] + } else { + 1.0 + }; + block[j] * g + }; + // Fold to a DCT-IV input: v = (-c_r - d, a - b_r) over the four quarters of u. + for m in 0..n2 { + self.v[m] = -u(n + n2 - 1 - m) - u(n + n2 + m); + self.v[n2 + m] = u(m) - u(n2 + n2 - 1 - m); + } + // DCT-IV via an n/2-point complex FFT. + self.z.clear(); + for m in 0..n2 { + let wv = Cpx { re: self.v[2 * m], im: self.v[n - 1 - 2 * m] }; + self.z.push(wv.mul(self.pre[m])); + } + self.fft.process(&mut self.z, &mut self.scratch); + for p in 0..n2 { + let y = self.z[p].mul(self.post[p]); + out[2 * p] = y.re * self.scale; + out[n - 1 - 2 * p] = -y.im * self.scale; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Direct MDCT definition X[k] = sum u[t] cos(pi/N (t + 1/2 + N/2)(k + 1/2)). + fn direct(block: &[f32], n: usize, window: &[f32; OVERLAP]) -> Vec { + let pad = (n - OVERLAP) / 2; + let mut u = vec![0f64; 2 * n]; + for (j, &b) in block.iter().enumerate() { + let g = if j < OVERLAP { + window[j] + } else if j >= n { + window[OVERLAP - 1 - (j - n)] + } else { + 1.0 + }; + u[pad + j] = (b * g) as f64; + } + (0..n) + .map(|k| { + (0..2 * n).map(|t| u[t] * (std::f64::consts::PI / n as f64 * (t as f64 + 0.5 + n as f64 / 2.0) * (k as f64 + 0.5)).cos()).sum::() * 2.0 + / n as f64 + }) + .collect() + } + + #[test] + fn matches_direct_formula() { + for &n in &[240usize, 960] { + let block: Vec = (0..n + OVERLAP).map(|i| ((i * 37 % 101) as f32 - 50.0) / 7.0).collect(); + let mut m = Mdct::new(n); + let mut out = vec![0f32; n]; + m.forward(&block, &mut out); + let d = direct(&block, n, &m.window); + for k in 0..n { + assert!((out[k] as f64 - d[k]).abs() < 1e-2, "n={n} k={k}: {} vs {}", out[k], d[k]); + } + } + } + + // ---- Replica of the RFC 6716 decoder inverse MDCT (as in FilmCraft's decoder) ---- + + fn imdct_backward(n_full: usize, input: &[f32], out: &mut [f32], window: &[f32]) { + let n2 = n_full >> 1; + let n4 = n_full >> 2; + let t: Vec = (0..n2).map(|i| (2.0 * std::f64::consts::PI * (i as f64 + 0.125) / n_full as f64).cos() as f32).collect(); + let fft = Fft::new(n4); + let mut z = Vec::new(); + for i in 0..n4 { + let x1 = input[2 * i]; + let x2 = input[n2 - 1 - 2 * i]; + let yr = x2 * t[i] + x1 * t[n4 + i]; + let yi = x1 * t[i] - x2 * t[n4 + i]; + z.push(Cpx { re: yi, im: yr }); + } + fft.process(&mut z, &mut Vec::new()); + let overlap = OVERLAP; + let base = overlap >> 1; + let y = &mut out[base..base + n2]; + for i in 0..n4.div_ceil(2) { + let front = z[i]; + let back = z[n4 - 1 - i]; + let (re, im) = (front.im, front.re); + let (t0, t1) = (t[i], t[n4 + i]); + let yr0 = re * t0 + im * t1; + let yi0 = re * t1 - im * t0; + let (re, im) = (back.im, back.re); + let (t0, t1) = (t[n4 - i - 1], t[n2 - i - 1]); + let yr1 = re * t0 + im * t1; + let yi1 = re * t1 - im * t0; + y[2 * i] = yr0; + y[n2 - 1 - 2 * i] = yi0; + y[n2 - 2 - 2 * i] = yr1; + y[2 * i + 1] = yi1; + } + for i in 0..overlap / 2 { + let x1 = out[overlap - 1 - i]; + let x2 = out[i]; + let w1 = window[i]; + let w2 = window[overlap - 1 - i]; + out[i] = x2 * w2 - x1 * w1; + out[overlap - 1 - i] = x2 * w1 + x1 * w2; + } + } + + /// MDCT analysis followed by the decoder's IMDCT + TDAC overlap-add reproduces the input + /// delayed by exactly `OVERLAP` samples. + #[test] + fn perfect_reconstruction_through_decoder_imdct() { + let n = 960; + let frames = 5; + let sig: Vec = (0..n * frames).map(|i| (i as f32 * 0.013).sin() * 1000.0 + ((i * 7919) % 211) as f32 - 105.0).collect(); + let mut m = Mdct::new(n); + let mut hist = vec![0f32; OVERLAP]; + // Decoder memory: [frame output n][folded tail OVERLAP/2] laid out like decode_mem. + let mut mem = vec![0f32; n + OVERLAP]; + let mut decoded = Vec::new(); + for f in 0..frames { + let mut block = hist.clone(); + block.extend_from_slice(&sig[f * n..(f + 1) * n]); + hist.copy_from_slice(&block[n..]); + let mut coefs = vec![0f32; n]; + m.forward(&block, &mut coefs); + // Shift: the previous folded tail (at n..n+OVERLAP/2) moves to 0..OVERLAP/2. + mem.copy_within(n..n + OVERLAP / 2, 0); + imdct_backward(2 * n, &coefs, &mut mem, &m.window); + decoded.extend_from_slice(&mem[..n]); + } + let mut err = 0f64; + let mut sum = 0f64; + for t in OVERLAP..n * frames { + let d = decoded[t] as f64 - sig[t - OVERLAP] as f64; + err += d * d; + sum += (sig[t - OVERLAP] as f64).powi(2); + } + let snr = 10.0 * (sum / err).log10(); + assert!(snr > 80.0, "reconstruction SNR {snr:.1} dB"); + } +} diff --git a/crates/opusenc/src/range.rs b/crates/opusenc/src/range.rs new file mode 100644 index 00000000..daedd58a --- /dev/null +++ b/crates/opusenc/src/range.rs @@ -0,0 +1,472 @@ +//! Range encoder (RFC 6716 §5.1), the exact inverse of the §4.1 range decoder. +//! +//! Range-coded symbols are written from the front of the buffer and raw bits from the back; the +//! bit accounting ([`RangeEncoder::tell`], [`RangeEncoder::tell_frac`]) matches the decoder's, so +//! every budget decision the decoder takes can be mirrored exactly. + +const SYM_BITS: u32 = 8; +const CODE_BITS: u32 = 32; +const SYM_MAX: u32 = (1 << SYM_BITS) - 1; +const CODE_SHIFT: u32 = CODE_BITS - SYM_BITS - 1; +const CODE_TOP: u32 = 1 << (CODE_BITS - 1); +const CODE_BOT: u32 = CODE_TOP >> SYM_BITS; +/// Bit resolution of [`RangeEncoder::tell_frac`] (1/8 bit). +pub const BITRES: u32 = 3; + +/// Number of bits needed to represent `x` (0 for 0). +#[inline] +pub fn ilog(x: u32) -> i32 { + (32 - x.leading_zeros()) as i32 +} + +#[derive(Clone, Debug)] +pub struct RangeEncoder { + out: Vec, + end: Vec, + end_window: u32, + nend_bits: i32, + nbits_total: i32, + rng: u32, + low: u32, + rem: i32, + ext: u32, +} + +impl Default for RangeEncoder { + fn default() -> Self { + Self::new() + } +} + +impl RangeEncoder { + pub fn new() -> Self { + RangeEncoder { + out: Vec::new(), + end: Vec::new(), + end_window: 0, + nend_bits: 0, + nbits_total: (CODE_BITS + 1) as i32, + rng: CODE_TOP, + low: 0, + rem: -1, + ext: 0, + } + } + + /// Current range (equals the decoder's range after the same symbols). + #[cfg(test)] + pub fn rng(&self) -> u32 { + self.rng + } + + fn carry_out(&mut self, c: u32) { + if c != SYM_MAX { + let carry = c >> SYM_BITS; + if self.rem >= 0 { + self.out.push((self.rem as u32 + carry) as u8); + } + if self.ext > 0 { + let sym = (SYM_MAX + carry) & SYM_MAX; + for _ in 0..self.ext { + self.out.push(sym as u8); + } + self.ext = 0; + } + self.rem = (c & SYM_MAX) as i32; + } else { + self.ext += 1; + } + } + + #[inline] + fn normalize(&mut self) { + while self.rng <= CODE_BOT { + self.carry_out(self.low >> CODE_SHIFT); + self.low = (self.low << SYM_BITS) & (CODE_TOP - 1); + self.rng <<= SYM_BITS; + self.nbits_total += SYM_BITS as i32; + } + } + + /// Encodes the symbol with cumulative range `[fl, fh)` of total `ft`. + pub fn encode(&mut self, fl: u32, fh: u32, ft: u32) { + let r = self.rng / ft; + if fl > 0 { + self.low += self.rng - r * (ft - fl); + self.rng = r * (fh - fl); + } else { + self.rng -= r * (ft - fh); + } + self.normalize(); + } + + /// Like [`Self::encode`] with `ft = 1 << bits`. + pub fn encode_bin(&mut self, fl: u32, fh: u32, bits: u32) { + let r = self.rng >> bits; + let ft = 1u32 << bits; + if fl > 0 { + self.low += self.rng - r * (ft - fl); + self.rng = r * (fh - fl); + } else { + self.rng -= r * (ft - fh); + } + self.normalize(); + } + + /// Encodes a binary symbol whose probability of being 1 is `1/2^logp`. + pub fn bit_logp(&mut self, val: bool, logp: u32) { + let r = self.rng; + let s = r >> logp; + let r2 = r - s; + if val { + self.low += r2; + self.rng = s; + } else { + self.rng = r2; + } + self.normalize(); + } + + /// Encodes symbol `s` with an "inverse CDF" table (`ft = 1 << ftb`). + pub fn icdf(&mut self, s: usize, icdf: &[u8], ftb: u32) { + let r = self.rng >> ftb; + if s > 0 { + self.low += self.rng - r * icdf[s - 1] as u32; + self.rng = r * (icdf[s - 1] as u32 - icdf[s] as u32); + } else { + self.rng -= r * icdf[s] as u32; + } + self.normalize(); + } + + /// Encodes a uniformly distributed integer `fl` in `[0, ft)` (`ft >= 2`). + pub fn uint(&mut self, fl: u32, ft: u32) { + debug_assert!(ft > 1 && fl < ft); + let ft1 = ft - 1; + let ftb = ilog(ft1); + if ftb > 8 { + let ftb = (ftb - 8) as u32; + let top = (ft1 >> ftb) + 1; + let v = fl >> ftb; + self.encode(v, v + 1, top); + self.bits(fl & ((1 << ftb) - 1), ftb); + } else { + self.encode(fl, fl + 1, ft); + } + } + + /// Writes `bits` raw bits (0..=25) to the end of the buffer. + pub fn bits(&mut self, val: u32, bits: u32) { + if bits == 0 { + return; + } + let mut window = self.end_window; + let mut used = self.nend_bits; + if used as u32 + bits > CODE_BITS { + while used >= SYM_BITS as i32 { + self.end.push((window & SYM_MAX) as u8); + window >>= SYM_BITS; + used -= SYM_BITS as i32; + } + } + window |= val << used; + used += bits as i32; + self.end_window = window; + self.nend_bits = used; + self.nbits_total += bits as i32; + } + + /// Number of bits written so far (rounded up), identical to the decoder's `tell`. + #[inline] + pub fn tell(&self) -> i32 { + self.nbits_total - ilog(self.rng) + } + + /// Number of 1/8 bits written so far (rounded up), identical to the decoder's `tell_frac`. + pub fn tell_frac(&self) -> i32 { + let nbits = (self.nbits_total as u32) << BITRES; + let mut l = ilog(self.rng) as u32; + let mut r = self.rng >> (l - 16); + for _ in 0..BITRES { + r = (r * r) >> 15; + let b = r >> 16; + l = (l << 1) | b; + r >>= b; + } + nbits.wrapping_sub(l) as i32 + } + + /// Finalises the stream into a buffer of exactly `size` bytes (range data at the front, raw + /// bits at the back, zero padding in between; a partial raw-bit byte is merged into the byte + /// just before the full raw-bit bytes). + pub fn done(mut self, size: usize) -> Vec { + let mut l = CODE_BITS as i32 - ilog(self.rng); + let mut msk = (CODE_TOP - 1) >> l; + let mut end = (self.low.wrapping_add(msk)) & !msk; + if (end | msk) >= self.low.wrapping_add(self.rng) { + l += 1; + msk >>= 1; + end = (self.low.wrapping_add(msk)) & !msk; + } + while l > 0 { + self.carry_out(end >> CODE_SHIFT); + end = (end << SYM_BITS) & (CODE_TOP - 1); + l -= SYM_BITS as i32; + } + if self.rem >= 0 || self.ext > 0 { + self.carry_out(0); + } + let mut window = self.end_window; + let mut used = self.nend_bits; + while used >= SYM_BITS as i32 { + self.end.push((window & SYM_MAX) as u8); + window >>= SYM_BITS; + used -= SYM_BITS as i32; + } + let mut buf = vec![0u8; size]; + let full = self.end.len().min(size); + let room = size - full; + let n = self.out.len().min(room); + buf[..n].copy_from_slice(&self.out[..n]); + for (k, &b) in self.end.iter().take(full).enumerate() { + buf[size - 1 - k] = b; + } + if used > 0 && room > 0 { + buf[room - 1] |= window as u8; + } + buf + } +} + +#[cfg(test)] +pub(crate) mod dec { + //! Minimal range decoder (RFC 6716 §4.1) used to verify the encoder. + use super::*; + const CODE_EXTRA: u32 = (CODE_BITS - 2) % SYM_BITS + 1; + + pub struct RangeDecoder<'a> { + buf: &'a [u8], + offs: usize, + end_offs: usize, + end_window: u32, + nend_bits: i32, + nbits_total: i32, + rng: u32, + val: u32, + rem: u32, + } + + impl<'a> RangeDecoder<'a> { + pub fn new(buf: &'a [u8]) -> Self { + let mut d = RangeDecoder { + buf, + offs: 0, + end_offs: 0, + end_window: 0, + nend_bits: 0, + nbits_total: (CODE_BITS + 1 - ((CODE_BITS - CODE_EXTRA) / SYM_BITS) * SYM_BITS) as i32, + rng: 1 << CODE_EXTRA, + val: 0, + rem: 0, + }; + d.rem = d.read_byte(); + d.val = d.rng - 1 - (d.rem >> (SYM_BITS - CODE_EXTRA)); + d.normalize(); + d + } + fn read_byte(&mut self) -> u32 { + if self.offs < self.buf.len() { + self.offs += 1; + self.buf[self.offs - 1] as u32 + } else { + 0 + } + } + fn read_byte_from_end(&mut self) -> u32 { + if self.end_offs < self.buf.len() { + self.end_offs += 1; + self.buf[self.buf.len() - self.end_offs] as u32 + } else { + 0 + } + } + fn normalize(&mut self) { + while self.rng <= CODE_BOT { + self.nbits_total += SYM_BITS as i32; + self.rng <<= SYM_BITS; + let sym = self.rem; + self.rem = self.read_byte(); + let sym = ((sym << SYM_BITS) | self.rem) >> (SYM_BITS - CODE_EXTRA); + self.val = ((self.val << SYM_BITS).wrapping_add(SYM_MAX & !sym)) & (CODE_TOP - 1); + } + } + pub fn rng(&self) -> u32 { + self.rng + } + pub fn decode(&mut self, ft: u32) -> u32 { + let ext = self.rng / ft; + let s = self.val / ext; + ft - (s + 1).min(ft) + } + pub fn update(&mut self, fl: u32, fh: u32, ft: u32) { + let ext = self.rng / ft; + let s = ext * (ft - fh); + self.val = self.val.wrapping_sub(s); + self.rng = if fl > 0 { ext * (fh - fl) } else { self.rng - s }; + self.normalize(); + } + pub fn bit_logp(&mut self, logp: u32) -> bool { + let r = self.rng; + let d = self.val; + let s = r >> logp; + let ret = d < s; + if !ret { + self.val = d - s; + } + self.rng = if ret { s } else { r - s }; + self.normalize(); + ret + } + pub fn icdf(&mut self, icdf: &[u8], ftb: u32) -> usize { + let mut s = self.rng; + let d = self.val; + let r = s >> ftb; + let mut ret = 0usize; + let mut t; + loop { + t = s; + s = r * icdf[ret] as u32; + if d >= s { + break; + } + ret += 1; + } + self.val = d - s; + self.rng = t - s; + self.normalize(); + ret + } + pub fn uint(&mut self, ft: u32) -> u32 { + let ft1 = ft - 1; + let ftb = ilog(ft1); + if ftb > 8 { + let ftb = (ftb - 8) as u32; + let top = (ft1 >> ftb) + 1; + let s = self.decode(top); + self.update(s, s + 1, top); + (s << ftb) | self.bits(ftb) + } else { + let s = self.decode(ft); + self.update(s, s + 1, ft); + s + } + } + pub fn bits(&mut self, bits: u32) -> u32 { + let mut window = self.end_window; + let mut available = self.nend_bits; + if (available as u32) < bits { + loop { + window |= self.read_byte_from_end() << available; + available += SYM_BITS as i32; + if available > (CODE_BITS - SYM_BITS) as i32 { + break; + } + } + } + let ret = window & ((1u32 << bits) - 1); + self.end_window = window >> bits; + self.nend_bits = available - bits as i32; + self.nbits_total += bits as i32; + ret + } + pub fn tell(&self) -> i32 { + self.nbits_total - ilog(self.rng) + } + } +} + +#[cfg(test)] +mod tests { + use super::dec::RangeDecoder; + use super::*; + + struct Lcg(u32); + impl Lcg { + fn next(&mut self) -> u32 { + self.0 = self.0.wrapping_mul(1664525).wrapping_add(1013904223); + self.0 >> 8 + } + } + + #[test] + fn starts_at_one_bit() { + let e = RangeEncoder::new(); + assert_eq!(e.tell(), 1); + assert_eq!(e.tell_frac(), 8); + } + + #[test] + fn roundtrip_mixed_symbols_and_tell_matches() { + let mut rng = Lcg(7); + let icdf: [u8; 4] = [200, 120, 30, 0]; + for trial in 0..300 { + let n = 1 + (rng.next() % 300) as usize; + let mut ops = Vec::new(); + for _ in 0..n { + match rng.next() % 5 { + 0 => { + let ft = 2 + rng.next() % 1000; + ops.push((0u8, rng.next() % ft, ft)); + } + 1 => { + let logp = 1 + rng.next() % 15; + ops.push((1, rng.next().is_multiple_of(5) as u32, logp)); + } + 2 => { + let bits = 1 + rng.next() % 24; + ops.push((2, rng.next() & ((1 << bits) - 1), bits)); + } + 3 => { + let ft = 2 + rng.next() % 100_000_000; + ops.push((3, rng.next() % ft, ft)); + } + _ => ops.push((4, rng.next() % 4, 0)), + } + } + let mut e = RangeEncoder::new(); + let mut tells = Vec::new(); + for &(k, v, p) in &ops { + match k { + 0 => e.encode(v, v + 1, p), + 1 => e.bit_logp(v != 0, p), + 2 => e.bits(v, p), + 3 => e.uint(v, p), + _ => e.icdf(v as usize, &icdf, 8), + } + tells.push(e.tell()); + } + // Exactly the number of bytes `tell` says were used must suffice. + let size = (e.tell() as usize).div_ceil(8); + let final_rng = e.rng(); + let buf = e.done(size); + assert_eq!(buf.len(), size); + let mut d = RangeDecoder::new(&buf); + for (i, &(k, v, p)) in ops.iter().enumerate() { + let got = match k { + 0 => { + let s = d.decode(p); + d.update(s, s + 1, p); + s + } + 1 => d.bit_logp(p) as u32, + 2 => d.bits(p), + 3 => d.uint(p), + _ => d.icdf(&icdf, 8) as u32, + }; + assert_eq!(got, v, "trial {trial} op {i} kind {k}"); + assert_eq!(d.tell(), tells[i], "tell mismatch trial {trial} op {i}"); + } + assert_eq!(d.rng(), final_rng); + } + } +} diff --git a/crates/opusenc/src/rate.rs b/crates/opusenc/src/rate.rs new file mode 100644 index 00000000..d8665bf1 --- /dev/null +++ b/crates/opusenc/src/rate.rs @@ -0,0 +1,561 @@ +//! Pulse cache and bit allocation (RFC 6716 §4.3.3, §4.3.4.1). +//! +//! The allocation must be bit-identical to the decoder's, so this mirrors FilmCraft's +//! spec-derived `filmcraft-opus` decoder (first-party, MIT OR Apache-2.0) step by step; the only +//! difference is that the encoder *writes* the band-skip, intensity and dual-stereo symbols. + +use crate::range::{BITRES, RangeEncoder, ilog}; +use crate::tables::*; + +const MAX_PSEUDO: i32 = 40; +const LOG_MAX_PSEUDO: i32 = 6; +pub const MAX_FINE_BITS: i32 = 8; +const FINE_OFFSET: i32 = 21; +pub const QTHETA_OFFSET: i32 = 4; +pub const QTHETA_OFFSET_TWOPHASE: i32 = 16; +const ALLOC_STEPS: i32 = 6; + +/// Number of pulses represented by pseudo-pulse index `i`. +pub fn get_pulses(i: i32) -> i32 { + if i < 8 { i } else { (8 + (i & 7)) << ((i >> 3) - 1) } +} + +/// Conservative (rounded up) log2 of `val` with `frac` fractional bits. +pub fn log2_frac(mut val: u32, mut frac: u32) -> i32 { + let mut l = ilog(val); + if val & val.wrapping_sub(1) != 0 { + if l > 16 { + val = ((val - 1) >> (l - 16)) + 1; + } else { + val <<= 16 - l; + } + let mut l = (l - 1) << frac; + loop { + let b = (val >> 16) as i32; + l += b << frac; + val = (val + b as u32) >> b; + val = (val * val + 0x7FFF) >> 15; + if frac == 0 { + break; + } + frac -= 1; + } + l + (val > 0x8000) as i32 + } else { + l -= 1; + l << frac + } +} + +/// PVQ codebook size V(N, K) (saturating far above 2^32). +pub fn pvq_v(n: usize, k: usize) -> u128 { + if k == 0 { + return 1; + } + if n == 0 { + return 0; + } + let mut row = vec![0u128; k + 1]; + row[0] = 1; + for _ in 0..n { + let mut prev_diag = row[0]; + for kk in 1..=k { + let up = row[kk]; + let v = up.saturating_add(row[kk - 1]).saturating_add(prev_diag); + prev_diag = up; + row[kk] = v.min(u128::MAX >> 8); + } + row[0] = 1; + } + row[k] +} + +fn fits_in32(n: usize, k: i32) -> bool { + pvq_v(n, k as usize) <= u32::MAX as u128 +} + +/// Precomputed per-mode data. +pub struct Mode { + /// log2 of band width in 1/8 bits. + pub log_n: [i32; NB_EBANDS], + cache_index: Vec, + cache_bits: Vec, + caps: Vec, +} + +impl Mode { + pub fn new() -> Mode { + let mut log_n = [0i32; NB_EBANDS]; + for (i, l) in log_n.iter_mut().enumerate() { + *l = log2_frac((EBANDS[i + 1] - EBANDS[i]) as u32, BITRES); + } + let (cache_index, cache_bits) = compute_pulse_cache(MAX_LM as i32); + let mut m = Mode { log_n, cache_index, cache_bits, caps: Vec::new() }; + m.caps = compute_caps(&m); + m + } + + #[inline] + pub fn cache(&self, lm: i32, band: usize) -> &[u8] { + let idx = self.cache_index[((lm + 1) as usize) * NB_EBANDS + band]; + &self.cache_bits[idx.max(0) as usize..] + } + + pub fn bits2pulses(&self, band: usize, lm: i32, bits: i32) -> i32 { + let cache = self.cache(lm, band); + let mut lo = 0i32; + let mut hi = cache[0] as i32; + let bits = bits - 1; + for _ in 0..LOG_MAX_PSEUDO { + let mid = (lo + hi + 1) >> 1; + if cache[mid as usize] as i32 >= bits { + hi = mid; + } else { + lo = mid; + } + } + let lo_bits = if lo == 0 { -1 } else { cache[lo as usize] as i32 }; + if bits - lo_bits <= cache[hi as usize] as i32 - bits { lo } else { hi } + } + + pub fn pulses2bits(&self, band: usize, lm: i32, pulses: i32) -> i32 { + if pulses == 0 { 0 } else { self.cache(lm, band)[pulses as usize] as i32 + 1 } + } + + /// Per-band allocation caps for this frame (`init_caps`). + pub fn init_caps(&self, lm: usize, c: usize) -> [i32; NB_EBANDS] { + let mut cap = [0i32; NB_EBANDS]; + for (i, cp) in cap.iter_mut().enumerate() { + let n = (EBANDS[i + 1] - EBANDS[i]) << lm; + *cp = (self.caps[NB_EBANDS * (2 * lm + c - 1) + i] as i32 + 64) * c as i32 * n >> 2; + } + cap + } +} + +fn compute_pulse_cache(lm_max: i32) -> (Vec, Vec) { + let nb = NB_EBANDS; + let rows = (lm_max + 2) as usize; + let mut index = vec![-1i16; nb * rows]; + let mut entries: Vec<(usize, i32, usize)> = Vec::new(); + let mut curr = 0usize; + for i in 0..rows { + for j in 0..nb { + let n = (((EBANDS[j + 1] - EBANDS[j]) << i) >> 1) as usize; + index[i * nb + j] = -1; + 'search: for k in 0..=i { + for nn in 0..nb { + if k == i && nn >= j { + break; + } + if n == (((EBANDS[nn + 1] - EBANDS[nn]) << k) >> 1) as usize { + index[i * nb + j] = index[k * nb + nn]; + break 'search; + } + } + } + if index[i * nb + j] == -1 && n != 0 { + let mut k = 0; + while k < MAX_PSEUDO && fits_in32(n, get_pulses(k + 1)) { + k += 1; + } + index[i * nb + j] = curr as i16; + entries.push((n, k, curr)); + curr += (k + 1) as usize; + } + } + } + let mut bits = vec![0u8; curr]; + for &(n, k, off) in &entries { + let maxk = get_pulses(k) as usize; + let tmp = required_bits(n, maxk, BITRES); + for j in 1..=k { + bits[off + j as usize] = (tmp[get_pulses(j) as usize] - 1) as u8; + } + bits[off] = k as u8; + } + (index, bits) +} + +fn required_bits(n: usize, maxk: usize, frac: u32) -> Vec { + let mut out = vec![0i32; maxk + 1]; + if n == 1 { + for o in out.iter_mut().skip(1) { + *o = 1 << frac; + } + } else { + for (k, o) in out.iter_mut().enumerate().skip(1) { + *o = log2_frac(pvq_v(n, k) as u32, frac); + } + } + out +} + +fn compute_caps(m: &Mode) -> Vec { + let nb = NB_EBANDS; + let mut caps = vec![0u8; (MAX_LM + 1) * 2 * nb]; + for i in 0..=MAX_LM as i32 { + for c in 1..=2i32 { + for j in 0..nb { + let mut n0 = EBANDS[j + 1] - EBANDS[j]; + let max_bits; + if n0 << i == 1 { + max_bits = c * (1 + MAX_FINE_BITS) << BITRES; + } else { + let mut lm0 = 0; + if n0 > 2 { + n0 >>= 1; + lm0 -= 1; + } else if n0 <= 1 { + lm0 = i.min(1); + n0 <<= lm0; + } + let pcache = m.cache(lm0, j); + let mut mb = pcache[pcache[0] as usize] as i32 + 1; + let mut n = n0; + for k in 0..(i - lm0) { + mb <<= 1; + let offset = ((m.log_n[j] + ((lm0 + k) << BITRES)) >> 1) - QTHETA_OFFSET; + let num = 459 * ((2 * n - 1) * offset + mb); + let den = ((2 * n - 1) << 9) - 459; + let qb = ((num + (den >> 1)) / den).min(57); + mb += qb; + n <<= 1; + } + if c == 2 { + mb <<= 1; + let offset = ((m.log_n[j] + (i << BITRES)) >> 1) - if n == 2 { QTHETA_OFFSET_TWOPHASE } else { QTHETA_OFFSET }; + let ndof = 2 * n - 1 - (n == 2) as i32; + let p = if n == 2 { 512 } else { 487 }; + let num = p * (mb + ndof * offset); + let den = (ndof << 9) - p; + let qb = ((num + (den >> 1)) / den).min(if n == 2 { 64 } else { 61 }); + mb += qb; + } + let ndof = c * n + if c == 2 && n > 2 { 1 } else { 0 }; + let mut offset = ((m.log_n[j] + (i << BITRES)) >> 1) - FINE_OFFSET; + if n == 2 { + offset += (1 << BITRES) >> 2; + } + let num = mb + ndof * offset; + let den = (ndof - 1) << BITRES; + let qb = ((num + (den >> 1)) / den).min(MAX_FINE_BITS); + mb += c * qb << BITRES; + max_bits = mb; + } + let w = c * ((EBANDS[j + 1] - EBANDS[j]) << i); + let v = (4 * max_bits / w) - 64; + caps[((i * 2 + c - 1) as usize) * nb + j] = v.clamp(0, 255) as u8; + } + } + } + caps +} + +/// Result of the allocation process. +pub struct Allocation { + pub coded_bands: usize, + pub intensity: usize, + pub dual_stereo: bool, + pub balance: i32, + pub pulses: [i32; NB_EBANDS], + pub fine_quant: [i32; NB_EBANDS], + pub fine_priority: [i32; NB_EBANDS], +} + +/// `clt_compute_allocation`, encoder side. Stereo frames are always coded as dual (L/R) stereo +/// with no intensity bands, and no band is skipped voluntarily. +#[allow(clippy::too_many_arguments)] +pub fn compute_allocation( + m: &Mode, + start: usize, + end: usize, + offsets: &[i32; NB_EBANDS], + cap: &[i32; NB_EBANDS], + alloc_trim: i32, + total: i32, + c: usize, + lm: usize, + enc: &mut RangeEncoder, +) -> Allocation { + let ci = c as i32; + let lmi = lm as i32; + let mut total = total.max(0); + let mut skip_start = start; + let skip_rsv = if total >= 1 << BITRES { 1 << BITRES } else { 0 }; + total -= skip_rsv; + let mut intensity_rsv = 0; + let mut dual_stereo_rsv = 0; + if c == 2 { + intensity_rsv = LOG2_FRAC_TABLE[end - start]; + if intensity_rsv > total { + intensity_rsv = 0; + } else { + total -= intensity_rsv; + dual_stereo_rsv = if total >= 1 << BITRES { 1 << BITRES } else { 0 }; + total -= dual_stereo_rsv; + } + } + let mut bits1 = [0i32; NB_EBANDS]; + let mut bits2 = [0i32; NB_EBANDS]; + let mut thresh = [0i32; NB_EBANDS]; + let mut trim_offset = [0i32; NB_EBANDS]; + for j in start..end { + let w = EBANDS[j + 1] - EBANDS[j]; + thresh[j] = (ci << BITRES).max((3 * w << lmi << BITRES) >> 4); + trim_offset[j] = ci * w * (alloc_trim - 5 - lmi) * (end as i32 - j as i32 - 1) * (1 << (lmi + BITRES as i32)) >> 6; + if w << lmi == 1 { + trim_offset[j] -= ci << BITRES; + } + } + let nb_alloc = ALLOC.len() as i32; + let mut lo = 1; + let mut hi = nb_alloc - 1; + loop { + let mut done = false; + let mut psum = 0; + let mid = (lo + hi) >> 1; + for j in (start..end).rev() { + let n = EBANDS[j + 1] - EBANDS[j]; + let mut bitsj = ci * n * (ALLOC[mid as usize][j] as i32) << lmi >> 2; + if bitsj > 0 { + bitsj = (bitsj + trim_offset[j]).max(0); + } + bitsj += offsets[j]; + if bitsj >= thresh[j] || done { + done = true; + psum += bitsj.min(cap[j]); + } else if bitsj >= ci << BITRES { + psum += ci << BITRES; + } + } + if psum > total { + hi = mid - 1; + } else { + lo = mid + 1; + } + if lo > hi { + break; + } + } + hi = lo; + lo -= 1; + for j in start..end { + let n = EBANDS[j + 1] - EBANDS[j]; + let mut bits1j = ci * n * (ALLOC[lo as usize][j] as i32) << lmi >> 2; + let mut bits2j = if hi >= nb_alloc { cap[j] } else { ci * n * (ALLOC[hi as usize][j] as i32) << lmi >> 2 }; + if bits1j > 0 { + bits1j = (bits1j + trim_offset[j]).max(0); + } + if bits2j > 0 { + bits2j = (bits2j + trim_offset[j]).max(0); + } + if lo > 0 { + bits1j += offsets[j]; + } + bits2j += offsets[j]; + if offsets[j] > 0 { + skip_start = j; + } + bits2j = (bits2j - bits1j).max(0); + bits1[j] = bits1j; + bits2[j] = bits2j; + } + interp_bits2pulses(m, start, end, skip_start, &bits1, &bits2, &thresh, cap, total, skip_rsv, intensity_rsv, dual_stereo_rsv, c, lm, enc) +} + +#[allow(clippy::too_many_arguments)] +fn interp_bits2pulses( + m: &Mode, + start: usize, + end: usize, + skip_start: usize, + bits1: &[i32; NB_EBANDS], + bits2: &[i32; NB_EBANDS], + thresh: &[i32; NB_EBANDS], + cap: &[i32; NB_EBANDS], + mut total: i32, + skip_rsv: i32, + mut intensity_rsv: i32, + mut dual_stereo_rsv: i32, + c: usize, + lm: usize, + enc: &mut RangeEncoder, +) -> Allocation { + let ci = c as i32; + let alloc_floor = ci << BITRES; + let stereo = (c > 1) as i32; + let log_m = (lm as i32) << BITRES; + let mut lo = 0; + let mut hi = 1 << ALLOC_STEPS; + for _ in 0..ALLOC_STEPS { + let mid = (lo + hi) >> 1; + let mut psum = 0; + let mut done = false; + for j in (start..end).rev() { + let tmp = bits1[j] + (mid * bits2[j] >> ALLOC_STEPS); + if tmp >= thresh[j] || done { + done = true; + psum += tmp.min(cap[j]); + } else if tmp >= alloc_floor { + psum += alloc_floor; + } + } + if psum > total { + hi = mid; + } else { + lo = mid; + } + } + let mut bits = [0i32; NB_EBANDS]; + let mut psum = 0; + let mut done = false; + for j in (start..end).rev() { + let mut tmp = bits1[j] + (lo * bits2[j] >> ALLOC_STEPS); + if tmp < thresh[j] && !done { + tmp = if tmp >= alloc_floor { alloc_floor } else { 0 }; + } else { + done = true; + } + tmp = tmp.min(cap[j]); + bits[j] = tmp; + psum += tmp; + } + let s = start; + let mut coded_bands = end; + loop { + let j = coded_bands - 1; + if j <= skip_start { + total += skip_rsv; + break; + } + let mut left = total - psum; + let span = EBANDS[coded_bands] - EBANDS[s]; + let percoeff = left / span; + left -= span * percoeff; + let rem = (left - (EBANDS[j] - EBANDS[s])).max(0); + let band_width = EBANDS[coded_bands] - EBANDS[j]; + let band_bits = bits[j] + percoeff * band_width + rem; + if band_bits >= thresh[j].max(alloc_floor + (1 << BITRES)) { + // Keep this band (and every band below it). + enc.bit_logp(true, 1); + break; + } + psum -= bits[j] + intensity_rsv; + if intensity_rsv > 0 { + intensity_rsv = LOG2_FRAC_TABLE[j - s]; + } + psum += intensity_rsv; + if band_bits >= alloc_floor { + psum += alloc_floor; + bits[j] = alloc_floor; + } else { + bits[j] = 0; + } + coded_bands -= 1; + } + let intensity = if intensity_rsv > 0 { + // No intensity stereo: every coded band is coded in full. + enc.uint((coded_bands - s) as u32, (coded_bands + 1 - s) as u32); + coded_bands + } else { + 0 + }; + if intensity <= s { + total += dual_stereo_rsv; + dual_stereo_rsv = 0; + } + let dual_stereo = if dual_stereo_rsv > 0 { + enc.bit_logp(true, 1); + true + } else { + false + }; + let mut left = total - psum; + let span = EBANDS[coded_bands] - EBANDS[s]; + let percoeff = left / span; + left -= span * percoeff; + for j in s..coded_bands { + bits[j] += percoeff * (EBANDS[j + 1] - EBANDS[j]); + } + for j in s..coded_bands { + let tmp = left.min(EBANDS[j + 1] - EBANDS[j]); + bits[j] += tmp; + left -= tmp; + } + let mut ebits = [0i32; NB_EBANDS]; + let mut fine_priority = [0i32; NB_EBANDS]; + let mut balance = 0; + for j in s..coded_bands { + let n0 = EBANDS[j + 1] - EBANDS[j]; + let n = n0 << lm; + let bit = bits[j] + balance; + let mut excess; + if n > 1 { + excess = (bit - cap[j]).max(0); + bits[j] = bit - excess; + let den = ci * n + if c == 2 && n > 2 && !dual_stereo && j < intensity { 1 } else { 0 }; + let nclogn = den * (m.log_n[j] + log_m); + let mut offset = (nclogn >> 1) - den * FINE_OFFSET; + if n == 2 { + offset += (den << BITRES) >> 2; + } + if bits[j] + offset < den * 2 << BITRES { + offset += nclogn >> 2; + } else if bits[j] + offset < den * 3 << BITRES { + offset += nclogn >> 3; + } + let mut eb = (bits[j] + offset + (den << (BITRES - 1))).max(0); + eb = (eb / den) >> BITRES; + if ci * eb > (bits[j] >> BITRES) { + eb = bits[j] >> stereo >> BITRES; + } + eb = eb.min(MAX_FINE_BITS); + ebits[j] = eb; + fine_priority[j] = (eb * (den << BITRES) >= bits[j] + offset) as i32; + bits[j] -= ci * eb << BITRES; + } else { + excess = (bit - (ci << BITRES)).max(0); + bits[j] = bit - excess; + ebits[j] = 0; + fine_priority[j] = 1; + } + if excess > 0 { + let extra_fine = (excess >> (stereo + BITRES as i32)).min(MAX_FINE_BITS - ebits[j]); + ebits[j] += extra_fine; + let extra_bits = extra_fine * ci << BITRES; + fine_priority[j] = (extra_bits >= excess - balance) as i32; + excess -= extra_bits; + } + balance = excess; + } + for j in coded_bands..end { + ebits[j] = bits[j] >> stereo >> BITRES; + bits[j] = 0; + fine_priority[j] = (ebits[j] < 1) as i32; + } + Allocation { coded_bands, intensity, dual_stereo, balance, pulses: bits, fine_quant: ebits, fine_priority } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn caps_match_decoder_reference() { + let m = Mode::new(); + let first_rows: [u8; 42] = [ + 224, 224, 224, 224, 224, 224, 224, 224, 160, 160, 160, 160, 185, 185, 185, 178, 178, 168, 134, 61, 37, 224, 224, 224, 224, 224, 224, 224, 224, 240, + 240, 240, 240, 207, 207, 207, 198, 198, 183, 144, 66, 40, + ]; + assert_eq!(&m.caps[..42], &first_rows[..]); + } + + #[test] + fn pvq_counts() { + assert_eq!(pvq_v(5, 1), 10); + assert_eq!(pvq_v(1, 7), 2); + assert_eq!(pvq_v(2, 9), 36); + assert_eq!(pvq_v(3, 2), 18); + } +} diff --git a/crates/opusenc/src/tables.rs b/crates/opusenc/src/tables.rs new file mode 100644 index 00000000..00d623af --- /dev/null +++ b/crates/opusenc/src/tables.rs @@ -0,0 +1,122 @@ +//! Constant tables of the 48 kHz CELT mode (RFC 6716 §4.3, Tables 55–58 and §4.3.2). +//! +//! Transcribed from RFC 6716; copied from FilmCraft's spec-derived `filmcraft-opus` decoder +//! (first-party, MIT OR Apache-2.0) so both sides share the exact same values. + +/// Number of energy bands. +pub const NB_EBANDS: usize = 21; +/// Band edges in units of 2.5 ms MDCT bins (Table 55). +pub const EBANDS: [i32; NB_EBANDS + 1] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 34, 40, 48, 60, 78, 100]; +/// Overlap of the low-overlap window (samples at 48 kHz). +pub const OVERLAP: usize = 120; +/// Largest LM (20 ms frames). +pub const MAX_LM: usize = 3; +/// Pre-emphasis coefficient. +pub const PREEMPH: f32 = 0.850_006_1; + +/// Static allocation table (Table 57), indexed `[quality][band]`, in 1/32 bit per MDCT bin. +pub const ALLOC: [[u8; NB_EBANDS]; 11] = transpose([ + [0, 90, 110, 118, 126, 134, 144, 152, 162, 172, 200], + [0, 80, 100, 110, 119, 127, 137, 145, 155, 165, 200], + [0, 75, 90, 103, 112, 120, 130, 138, 148, 158, 200], + [0, 69, 84, 93, 104, 114, 124, 132, 142, 152, 200], + [0, 63, 78, 86, 95, 103, 113, 123, 133, 143, 200], + [0, 56, 71, 80, 89, 97, 107, 117, 127, 137, 200], + [0, 49, 65, 75, 83, 91, 101, 111, 121, 131, 200], + [0, 40, 58, 70, 78, 85, 95, 105, 115, 125, 200], + [0, 34, 51, 65, 72, 78, 88, 98, 108, 118, 198], + [0, 29, 45, 59, 66, 72, 82, 92, 102, 112, 193], + [0, 20, 39, 53, 60, 66, 76, 86, 96, 106, 188], + [0, 18, 32, 47, 54, 60, 70, 80, 90, 100, 183], + [0, 10, 26, 40, 47, 54, 64, 74, 84, 94, 178], + [0, 0, 20, 31, 39, 47, 57, 67, 77, 87, 173], + [0, 0, 12, 23, 32, 41, 51, 61, 71, 81, 168], + [0, 0, 0, 15, 25, 35, 45, 55, 65, 75, 163], + [0, 0, 0, 4, 17, 29, 39, 49, 59, 69, 158], + [0, 0, 0, 0, 12, 23, 33, 43, 53, 63, 153], + [0, 0, 0, 0, 1, 16, 26, 36, 46, 56, 148], + [0, 0, 0, 0, 0, 10, 15, 20, 30, 45, 129], + [0, 0, 0, 0, 0, 1, 1, 1, 1, 20, 104], +]); + +const fn transpose(t: [[u8; 11]; NB_EBANDS]) -> [[u8; NB_EBANDS]; 11] { + let mut out = [[0u8; NB_EBANDS]; 11]; + let mut b = 0; + while b < NB_EBANDS { + let mut q = 0; + while q < 11 { + out[q][b] = t[b][q]; + q += 1; + } + b += 1; + } + out +} + +/// Mean band energy (log2 domain) removed before coarse quantisation. +pub const E_MEANS: [f32; 25] = [ + 6.4375, 6.25, 5.75, 5.3125, 5.0625, 4.8125, 4.5, 4.375, 4.875, 4.6875, 4.5625, 4.4375, 4.875, 4.625, 4.3125, 4.5, 4.375, 4.625, 4.75, 4.4375, 3.75, 3.75, + 3.75, 3.75, 3.75, +]; + +/// Inter-frame prediction coefficient (alpha) per LM, Q15. +pub const PRED_COEF: [f32; 4] = [29440.0 / 32768.0, 26112.0 / 32768.0, 21248.0 / 32768.0, 16384.0 / 32768.0]; +/// Inter-band prediction coefficient (beta) per LM for inter frames, Q15. +pub const BETA_COEF: [f32; 4] = [30147.0 / 32768.0, 22282.0 / 32768.0, 12124.0 / 32768.0, 6554.0 / 32768.0]; +/// Beta for intra frames. +pub const BETA_INTRA: f32 = 4915.0 / 32768.0; + +/// Laplace model parameters for coarse energy: `[LM][intra][2*band + {0: prob, 1: decay}]`. +pub const E_PROB_MODEL: [[[u8; 42]; 2]; 4] = [ + [ + [ + 72, 127, 65, 129, 66, 128, 65, 128, 64, 128, 62, 128, 64, 128, 64, 128, 92, 78, 92, 79, 92, 78, 90, 79, 116, 41, 115, 40, 114, 40, 132, 26, 132, + 26, 145, 17, 161, 12, 176, 10, 177, 11, + ], + [ + 24, 179, 48, 138, 54, 135, 54, 132, 53, 134, 56, 133, 55, 132, 55, 132, 61, 114, 70, 96, 74, 88, 75, 88, 87, 74, 89, 66, 91, 67, 100, 59, 108, 50, + 120, 40, 122, 37, 97, 43, 78, 50, + ], + ], + [ + [ + 83, 78, 84, 81, 88, 75, 86, 74, 87, 71, 90, 73, 93, 74, 93, 74, 109, 40, 114, 36, 117, 34, 117, 34, 143, 17, 145, 18, 146, 19, 162, 12, 165, 10, + 178, 7, 189, 6, 190, 8, 177, 9, + ], + [ + 23, 178, 54, 115, 63, 102, 66, 98, 69, 99, 74, 89, 71, 91, 73, 91, 78, 89, 86, 80, 92, 66, 93, 64, 102, 59, 103, 60, 104, 60, 117, 52, 123, 44, + 138, 35, 133, 31, 97, 38, 77, 45, + ], + ], + [ + [ + 61, 90, 93, 60, 105, 42, 107, 41, 110, 45, 116, 38, 113, 38, 112, 38, 124, 26, 132, 27, 136, 19, 140, 20, 155, 14, 159, 16, 158, 18, 170, 13, 177, + 10, 187, 8, 192, 6, 175, 9, 159, 10, + ], + [ + 21, 178, 59, 110, 71, 86, 75, 85, 84, 83, 91, 66, 88, 73, 87, 72, 92, 75, 98, 72, 105, 58, 107, 54, 115, 52, 114, 55, 112, 56, 129, 51, 132, 40, + 150, 33, 140, 29, 98, 35, 77, 42, + ], + ], + [ + [ + 42, 121, 96, 66, 108, 43, 111, 40, 117, 44, 123, 32, 120, 36, 119, 33, 127, 33, 134, 34, 139, 21, 147, 23, 152, 20, 158, 25, 154, 26, 166, 21, 173, + 16, 184, 13, 184, 10, 150, 13, 139, 15, + ], + [ + 22, 178, 63, 114, 74, 82, 84, 83, 92, 82, 103, 62, 96, 72, 96, 67, 101, 73, 107, 72, 113, 55, 118, 52, 125, 52, 118, 52, 117, 55, 135, 49, 137, 39, + 157, 32, 145, 29, 97, 33, 77, 40, + ], + ], +]; + +/// Inverse CDF for coarse energy when only a few bits remain. +pub const SMALL_ENERGY_ICDF: [u8; 3] = [2, 1, 0]; +/// Conservative log2 in 1/8 bits of the number of coded bands (intensity reservation). +pub const LOG2_FRAC_TABLE: [i32; 24] = [0, 8, 13, 16, 19, 21, 23, 24, 26, 27, 28, 29, 30, 31, 32, 32, 33, 34, 34, 35, 36, 36, 37, 37]; +/// TF resolution changes: `[LM][4*transient + 2*tf_select + tf_flag]`. +pub const TF_SELECT_TABLE: [[i8; 8]; 4] = [[0, -1, 0, -1, 0, -1, 0, -1], [0, -1, 0, -2, 1, 0, 1, -1], [0, -2, 0, -3, 2, 0, 1, -1], [0, -2, 0, -3, 3, 0, 1, -1]]; +/// Spreading decision `{7, 2, 21, 2}/32`. +pub const SPREAD_ICDF: [u8; 4] = [25, 23, 2, 0]; +/// Allocation trim (Table 58). +pub const TRIM_ICDF: [u8; 11] = [126, 124, 119, 109, 87, 41, 19, 9, 4, 2, 0]; diff --git a/docs/architecture.md b/docs/architecture.md index 033ed6cb..c3d869c0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -16,6 +16,7 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L0 | `time` | `Tick` (254 016 000 000 per second), rational frame rates incl. NTSC, SMPTE and drop-frame timecode | | L0 | `geom` | Vectors, matrices, quaternions, the layer transform (anchor, position, scale, orientation, rotation) | | L0 | `color` | sRGB and linear, HSL/HSV, luminance, the 38 blend modes, label colors | +| L0 | `opusenc` | Opus (CELT) encoder: RFC 6716 CELT-only fullband 48 kHz 20 ms packets, mono/stereo, and the RFC 7845 `OpusHead`, for WebM export audio | | L1 | `raster` | Premultiplied float images, sampling, affine and projective warps, blurs, compositing (parallel with rayon) | | L1 | `keyframe` | Animated values, keyframes with temporal ease and spatial Bezier, roving, hold, velocity | | L1 | `path` | Bezier paths, path operators (trim, offset, round corners, zig zag, twist, merge…), stroking, coverage masks | diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 76ff6fa3..0cef3307 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -24,6 +24,7 @@ const LAYERS: &[(&str, u8)] = &[ ("geom", 0), ("color", 0), ("testkit", 0), + ("opusenc", 0), ("raster", 1), ("keyframe", 1), ("path", 1), From bab4f88c22e525ac69474a8293b155fe70fccab4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 21:44:51 -0400 Subject: [PATCH 02/13] M12.6: camera tracker solve Structure from motion for the 3D Camera Tracker in effectcraft-track (camtrack): long KLT feature tracks with re-detection, parallax keyframes, two-view initialisation (normalised 8-point essential matrix in RANSAC or planar homography decomposition), incremental resection + triangulation, sparse Levenberg-Marquardt bundle adjustment with the Schur complement and Huber loss, focal length search (Fixed / Variable Zoom / Specify Angle of View), tripod pan solve, Auto Detect, ground plane / target / world mapping. Synthetic-scene tests (path, focal, error, tripod, flat, rendered frames). Co-Authored-By: Claude Opus 5.5 --- crates/track/Cargo.toml | 2 +- crates/track/src/camtrack/bundle.rs | 416 ++++++++++++ crates/track/src/camtrack/features.rs | 115 ++++ crates/track/src/camtrack/geometry.rs | 432 +++++++++++++ crates/track/src/camtrack/linalg.rs | 476 ++++++++++++++ crates/track/src/camtrack/mod.rs | 376 +++++++++++ crates/track/src/camtrack/solver.rs | 894 ++++++++++++++++++++++++++ crates/track/src/camtrack/tests.rs | 360 +++++++++++ crates/track/src/lib.rs | 1 + 9 files changed, 3071 insertions(+), 1 deletion(-) create mode 100644 crates/track/src/camtrack/bundle.rs create mode 100644 crates/track/src/camtrack/features.rs create mode 100644 crates/track/src/camtrack/geometry.rs create mode 100644 crates/track/src/camtrack/linalg.rs create mode 100644 crates/track/src/camtrack/mod.rs create mode 100644 crates/track/src/camtrack/solver.rs create mode 100644 crates/track/src/camtrack/tests.rs diff --git a/crates/track/Cargo.toml b/crates/track/Cargo.toml index ba449c92..9bd46e26 100644 --- a/crates/track/Cargo.toml +++ b/crates/track/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-track" -description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, and corner-pin / transform solves" +description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, corner-pin / transform solves, stabilization and 3D camera solving (structure from motion)" version.workspace = true edition.workspace = true license.workspace = true diff --git a/crates/track/src/camtrack/bundle.rs b/crates/track/src/camtrack/bundle.rs new file mode 100644 index 00000000..84e8a5ad --- /dev/null +++ b/crates/track/src/camtrack/bundle.rs @@ -0,0 +1,416 @@ +//! Sparse bundle adjustment: Levenberg–Marquardt over cameras and points with the **Schur +//! complement** eliminating the points (Triggs, McLauchlan, Hartley & Fitzgibbon, "Bundle +//! Adjustment — A Modern Synthesis", 2000, §6; H&Z Appendix 6). Residuals are reprojection +//! errors in pixels with a **Huber** robust loss (applied by iteratively reweighted least +//! squares, Triggs et al. §3.3). +//! +//! Cameras are `(R, C, f)`; rotations are updated multiplicatively (`R ← exp([δ]×) R`), focal +//! lengths in log space. The focal length is held fixed, shared by all cameras, or per camera +//! (with a smoothness prior between consecutive cameras). Tripod solves hold the centres fixed. + +use rayon::prelude::*; + +use super::geometry::Pose; +use super::linalg::*; + +/// A camera in the adjustment. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct BaCam { + pub pose: Pose, + /// Focal length (pixels). + pub f: f64, +} + +/// One observation: camera `cam` sees point `pt` at `uv` (pixels relative to the principal +/// point). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct BaObs { + pub cam: u32, + pub pt: u32, + pub uv: [f64; 2], +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum FocalMode { + Fixed, + Shared, + /// Per camera, with prior weight `k` on the log-focal difference of consecutive cameras. + PerCamera(f64), +} + +#[derive(Clone, Debug)] +pub struct BaOpts { + pub focal: FocalMode, + /// Motion only: the points stay fixed. + pub fix_points: bool, + /// The camera centres stay fixed (tripod). + pub fix_centers: bool, + /// Cameras that stay fixed (gauge), by index; missing entries are free. + pub fixed: Vec, + /// Huber threshold (pixels). + pub huber: f64, + pub max_iter: usize, +} + +impl Default for BaOpts { + fn default() -> Self { + BaOpts { focal: FocalMode::Fixed, fix_points: false, fix_centers: false, fixed: vec![], huber: 2.0, max_iter: 30 } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct BaReport { + pub initial_cost: f64, + pub cost: f64, + pub iterations: usize, +} + +const BEHIND: f64 = 1e6; + +fn huber(e2: f64, k: f64) -> f64 { + if e2 <= k * k { e2 } else { 2.0 * k * e2.sqrt() - k * k } +} + +/// Residual (pixels) of one observation, `None` behind the camera. +pub fn residual(c: &BaCam, x: V3, uv: [f64; 2]) -> Option<[f64; 2]> { + let p = c.pose.to_cam(x); + if p[2] <= 1e-9 { + return None; + } + Some([c.f * p[0] / p[2] - uv[0], c.f * p[1] / p[2] - uv[1]]) +} + +/// Total robust cost. +pub fn cost(cams: &[BaCam], pts: &[V3], obs: &[BaObs], o: &BaOpts) -> f64 { + let c: f64 = obs + .par_iter() + .map(|ob| match residual(&cams[ob.cam as usize], pts[ob.pt as usize], ob.uv) { + Some(r) => huber(r[0] * r[0] + r[1] * r[1], o.huber), + None => BEHIND, + }) + .sum(); + c + prior_cost(cams, o) +} + +fn prior_cost(cams: &[BaCam], o: &BaOpts) -> f64 { + match o.focal { + FocalMode::PerCamera(k) => cams.windows(2).map(|w| (k * (w[0].f.ln() - w[1].f.ln())).powi(2)).sum(), + _ => 0.0, + } +} + +/// Parameter layout. +struct Layout { + /// First pose parameter of each camera (`usize::MAX` when its pose is fixed). + start: Vec, + /// Per-camera focal parameter (`usize::MAX` when none). Gauge-fixed cameras keep a free + /// focal length. + fidx: Vec, + /// Index of the shared focal parameter. + shared: Option, + n: usize, + rot_only: bool, +} + +fn layout(ncam: usize, o: &BaOpts) -> Layout { + let per_cam_f = matches!(o.focal, FocalMode::PerCamera(_)); + let d = if o.fix_centers { 3 } else { 6 }; + let mut start = vec![usize::MAX; ncam]; + let mut fidx = vec![usize::MAX; ncam]; + let mut n = 0; + for i in 0..ncam { + if !o.fixed.get(i).copied().unwrap_or(false) { + start[i] = n; + n += d; + } + if per_cam_f { + fidx[i] = n; + n += 1; + } + } + let shared = matches!(o.focal, FocalMode::Shared).then(|| { + n += 1; + n - 1 + }); + Layout { start, fidx, shared, n, rot_only: o.fix_centers } +} + +/// Camera-side Jacobian entries of one observation: (parameter index, d r / d param). +fn cam_jac(l: &Layout, ci: usize, c: &BaCam, xc: V3, a: &[[f64; 3]; 2], p: [f64; 2], out: &mut Vec<(usize, [f64; 2])>) { + out.clear(); + let s = l.start[ci]; + if s != usize::MAX { + // dXc/dδ = −[Xc]× + let sk = skew(xc); + for k in 0..3 { + let g = [-sk[0][k], -sk[1][k], -sk[2][k]]; + out.push((s + k, [dot(a[0], g), dot(a[1], g)])); + } + if !l.rot_only { + // dXc/dC = −R + for k in 0..3 { + let g = [-c.pose.r[0][k], -c.pose.r[1][k], -c.pose.r[2][k]]; + out.push((s + 3 + k, [dot(a[0], g), dot(a[1], g)])); + } + } + } + if l.fidx[ci] != usize::MAX { + out.push((l.fidx[ci], p)); + } + if let Some(g) = l.shared { + out.push((g, p)); + } +} + +/// Run Levenberg–Marquardt. `cams` and `pts` are updated in place. +pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> BaReport { + let l = layout(cams.len(), o); + let npts = pts.len(); + // Observations by point. + let mut by_pt: Vec> = vec![vec![]; npts]; + for (i, ob) in obs.iter().enumerate() { + by_pt[ob.pt as usize].push(i as u32); + } + let mut cur = cost(cams, pts, obs, o); + let initial = cur; + let mut lambda = 1e-4; + let mut iters = 0; + if l.n == 0 && o.fix_points { + return BaReport { initial_cost: initial, cost: cur, iterations: 0 }; + } + let n = l.n; + let parallel = n <= 700; + for _ in 0..o.max_iter { + iters += 1; + // ---- Normal equations (Gauss–Newton with IRLS weights). + // Per point: V (3×3), bp (3), W entries (cam-side index → 3). + struct PtSys { + v: [[f64; 3]; 3], + bp: V3, + w: Vec<(usize, V3)>, + } + let build_point = |j: usize, s_acc: &mut Vec, bc: &mut Vec| -> PtSys { + let mut ps = PtSys { v: [[0.0; 3]; 3], bp: [0.0; 3], w: vec![] }; + let mut jc: Vec<(usize, [f64; 2])> = Vec::with_capacity(8); + for &oi in &by_pt[j] { + let ob = &obs[oi as usize]; + let ci = ob.cam as usize; + let c = &cams[ci]; + let xc = c.pose.to_cam(pts[j]); + if xc[2] <= 1e-9 { + continue; + } + let iz = 1.0 / xc[2]; + let p = [c.f * xc[0] * iz, c.f * xc[1] * iz]; + let r = [p[0] - ob.uv[0], p[1] - ob.uv[1]]; + let e = (r[0] * r[0] + r[1] * r[1]).sqrt(); + let w = if e <= o.huber { 1.0 } else { o.huber / e }; + let a = [[c.f * iz, 0.0, -c.f * xc[0] * iz * iz], [0.0, c.f * iz, -c.f * xc[1] * iz * iz]]; + cam_jac(&l, ci, c, xc, &a, p, &mut jc); + // Point Jacobian: A R. + let jx = if o.fix_points { + [[0.0; 3]; 2] + } else { + let r0 = mtv(&c.pose.r, a[0]); + let r1 = mtv(&c.pose.r, a[1]); + [r0, r1] + }; + for (ia, ja) in &jc { + bc[*ia] -= w * (ja[0] * r[0] + ja[1] * r[1]); + for (ib, jb) in &jc { + s_acc[ia * n + ib] += w * (ja[0] * jb[0] + ja[1] * jb[1]); + } + } + if !o.fix_points { + for a1 in 0..3 { + ps.bp[a1] -= w * (jx[0][a1] * r[0] + jx[1][a1] * r[1]); + for b1 in 0..3 { + ps.v[a1][b1] += w * (jx[0][a1] * jx[0][b1] + jx[1][a1] * jx[1][b1]); + } + } + for (ia, ja) in &jc { + let wv = + [w * (ja[0] * jx[0][0] + ja[1] * jx[1][0]), w * (ja[0] * jx[0][1] + ja[1] * jx[1][1]), w * (ja[0] * jx[0][2] + ja[1] * jx[1][2])]; + match ps.w.iter_mut().find(|e| e.0 == *ia) { + Some(e) => e.1 = add(e.1, wv), + None => ps.w.push((*ia, wv)), + } + } + } + } + ps + }; + let (mut s_mat, mut bc, systems): (Vec, Vec, Vec) = if parallel && npts > 64 { + let chunks = rayon::current_num_threads().clamp(1, 16); + let per = npts.div_ceil(chunks); + let parts: Vec<(Vec, Vec, Vec)> = (0..chunks) + .into_par_iter() + .map(|t| { + let (mut s, mut b) = (vec![0.0; n * n], vec![0.0; n]); + let v = (t * per..((t + 1) * per).min(npts)).map(|j| build_point(j, &mut s, &mut b)).collect(); + (s, b, v) + }) + .collect(); + let mut s = vec![0.0; n * n]; + let mut b = vec![0.0; n]; + let mut sys = Vec::with_capacity(npts); + for (ps, pb, pv) in parts { + for (x, y) in s.iter_mut().zip(ps) { + *x += y; + } + for (x, y) in b.iter_mut().zip(pb) { + *x += y; + } + sys.extend(pv); + } + (s, b, sys) + } else { + let mut s = vec![0.0; n * n]; + let mut b = vec![0.0; n]; + let sys = (0..npts).map(|j| build_point(j, &mut s, &mut b)).collect(); + (s, b, sys) + }; + // Focal prior. + if let FocalMode::PerCamera(k) = o.focal { + for i in 0..cams.len().saturating_sub(1) { + let r = k * (cams[i].f.ln() - cams[i + 1].f.ln()); + let (a, b) = (l.fidx[i], l.fidx[i + 1]); + let ia = (a != usize::MAX).then_some(a); + let ib = (b != usize::MAX).then_some(b); + for (idx, g) in [(ia, k), (ib, -k)] { + let Some(x) = idx else { continue }; + bc[x] -= g * r; + for (idy, h) in [(ia, k), (ib, -k)] { + if let Some(y) = idy { + s_mat[x * n + y] += g * h; + } + } + } + } + } + let base_s = s_mat.clone(); + let base_bc = bc.clone(); + let mean_diag = ((0..n).map(|i| base_s[i * n + i]).sum::() / n.max(1) as f64).max(1e-12); + // ---- Try steps with increasing damping. + let mut accepted = false; + for _try in 0..12 { + let mut s = base_s.clone(); + let mut rhs = base_bc.clone(); + for i in 0..n { + let d = s[i * n + i]; + s[i * n + i] = d + lambda * d.max(1e-6 * mean_diag); + } + // Schur complement over the points. + let mut vinv: Vec> = vec![None; npts]; + if !o.fix_points { + for (j, ps) in systems.iter().enumerate() { + let mut v = ps.v; + let vd = (v[0][0] + v[1][1] + v[2][2]) / 3.0; + for k in 0..3 { + v[k][k] += lambda * v[k][k].max(1e-6 * vd.max(1e-12)) + 1e-12; + } + let Some(vi) = inv3(&v) else { continue }; + vinv[j] = Some(vi); + let vb = mv(&vi, ps.bp); + let wv: Vec = ps.w.iter().map(|(_, w)| mv(&vi, *w)).collect(); + for (a, (ia, wa)) in ps.w.iter().enumerate() { + rhs[*ia] -= dot(*wa, vb); + for (ib, wb) in &ps.w { + s[ia * n + ib] -= dot(wv[a], *wb); + } + } + } + } + let mut dc = rhs; + if n > 0 && !cholesky_solve(&mut s, n, &mut dc) { + lambda *= 10.0; + continue; + } + // Apply. + let mut nc: Vec = cams.to_vec(); + for (i, c) in nc.iter_mut().enumerate() { + let st = l.start[i]; + if st != usize::MAX { + let dr = [dc[st], dc[st + 1], dc[st + 2]]; + c.pose.r = mm(&rodrigues(dr), &c.pose.r); + if !l.rot_only { + c.pose.c = add(c.pose.c, [dc[st + 3], dc[st + 4], dc[st + 5]]); + } + } + if l.fidx[i] != usize::MAX { + c.f *= dc[l.fidx[i]].clamp(-0.5, 0.5).exp(); + } + if let Some(g) = l.shared { + c.f *= dc[g].clamp(-0.5, 0.5).exp(); + } + } + let mut np: Vec = pts.to_vec(); + if !o.fix_points { + np.par_iter_mut().enumerate().for_each(|(j, x)| { + let Some(vi) = &vinv[j] else { return }; + let ps = &systems[j]; + let mut b = ps.bp; + for (ia, w) in &ps.w { + b = sub(b, scale(*w, dc[*ia])); + } + *x = add(*x, mv(vi, b)); + }); + } + let c2 = cost(&nc, &np, obs, o); + if c2.is_finite() && c2 < cur { + cams.copy_from_slice(&nc); + pts.copy_from_slice(&np); + let gain = cur - c2; + cur = c2; + lambda = (lambda / 3.0).max(1e-9); + accepted = gain > 1e-10 * cur.max(1e-12); + if !accepted { + // Converged (tiny improvement). + return BaReport { initial_cost: initial, cost: cur, iterations: iters }; + } + break; + } + lambda *= 4.0; + } + if !accepted { + break; + } + } + BaReport { initial_cost: initial, cost: cur, iterations: iters } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::camtrack::geometry::Rng; + + #[test] + fn bundle_adjustment_converges_from_noise() { + let mut rng = Rng(99); + let mut u = || (rng.next_u64() % 100_000) as f64 / 100_000.0 - 0.5; + let truth: Vec = (0..80).map(|_| [u() * 6.0, u() * 4.0, 6.0 + u() * 4.0]).collect(); + let f = 900.0; + let cams_true: Vec = (0..6) + .map(|i| BaCam { pose: Pose { r: rodrigues([0.0, -0.05 * i as f64, 0.01 * i as f64]), c: [0.3 * i as f64, 0.05 * i as f64, 0.0] }, f }) + .collect(); + let mut obs = vec![]; + for (ci, c) in cams_true.iter().enumerate() { + for (pi, x) in truth.iter().enumerate() { + let r = residual(c, *x, [0.0, 0.0]).unwrap(); + obs.push(BaObs { cam: ci as u32, pt: pi as u32, uv: r }); + } + } + let mut cams = cams_true.clone(); + for (i, c) in cams.iter_mut().enumerate().skip(2) { + c.pose.r = mm(&rodrigues([0.01, -0.01 * i as f64 * 0.3, 0.005]), &c.pose.r); + c.pose.c = add(c.pose.c, [0.02, -0.03, 0.01]); + } + let mut pts: Vec = truth.iter().map(|x| add(*x, [u() * 0.1, u() * 0.1, u() * 0.2])).collect(); + let mut fixed = vec![false; 6]; + fixed[0] = true; + fixed[1] = true; + let rep = adjust(&mut cams, &mut pts, &obs, &BaOpts { fixed, max_iter: 50, ..Default::default() }); + let rms = (rep.cost / obs.len() as f64).sqrt(); + assert!(rms < 1e-4, "{rep:?}"); + } +} diff --git a/crates/track/src/camtrack/features.rs b/crates/track/src/camtrack/features.rs new file mode 100644 index 00000000..311ec711 --- /dev/null +++ b/crates/track/src/camtrack/features.rs @@ -0,0 +1,115 @@ +//! Step 1 of the camera analysis: long feature tracks through the clip. +//! +//! Every frame, the active features are followed from the previous frame with pyramidal +//! Lucas–Kanade and a forward–backward check ([`crate::klt::track`]); a feature that fails ends +//! its track. When fewer than ~85 % of the wanted features remain, new Shi–Tomasi corners are +//! detected away from the active ones and start new tracks. Tracks shorter than three frames are +//! dropped at the end. + +use crate::Frame; +use crate::klt::{CornerOpts, GrayPyramid, LkOpts, analysis_factor, good_features, track}; + +use super::{CameraTracks, Track2D}; + +/// Analysis options. +#[derive(Clone, Copy, Debug, Default)] +pub struct AnalyzeOpts { + /// Detailed Analysis: twice the resolution and more features. + pub detailed: bool, +} + +/// Incremental feature tracking: push the clip's frames in order. +pub struct TrackAnalyzer { + opts: AnalyzeOpts, + prev: Option, + /// (track index, level-0 plane position) of the live features. + active: Vec<(usize, [f64; 2])>, + tracks: Vec, + frame: u32, + size: [f64; 2], + factor: f64, +} + +impl TrackAnalyzer { + pub fn new(size: [f64; 2], opts: AnalyzeOpts) -> TrackAnalyzer { + TrackAnalyzer { opts, prev: None, active: vec![], tracks: vec![], frame: 0, size, factor: 1.0 } + } + + pub fn frames(&self) -> u32 { + self.frame + } + + /// Features currently followed. + pub fn active(&self) -> usize { + self.active.len() + } + + fn params(&self) -> (u32, usize) { + if self.opts.detailed { (1280, 900) } else { (720, 500) } + } + + /// Analyse the next frame. + pub fn push(&mut self, frame: &Frame) { + let (max_side, want) = self.params(); + let factor = analysis_factor(frame.img.width, frame.img.height, max_side); + self.factor = factor as f64; + let pyr = GrayPyramid::from_image(frame.img, frame.offset, factor, 4); + let k = self.frame; + match &self.prev { + Some(prev) if prev.width() == pyr.width() && prev.height() == pyr.height() && prev.offset == pyr.offset => { + let pts: Vec<[f64; 2]> = self.active.iter().map(|a| a.1).collect(); + let q = track(prev, &pyr, &pts, None, &LkOpts { radius: 6, iterations: 30, epsilon: 0.005, fb_max: 0.5, ..Default::default() }); + let mut next = Vec::with_capacity(self.active.len()); + for ((ti, _), q) in self.active.iter().zip(q) { + if let Some(q) = q { + let l = pyr.to_layer(q); + self.tracks[*ti].pts.push([l[0] as f32, l[1] as f32]); + next.push((*ti, q)); + } + } + self.active = next; + } + _ => self.active.clear(), + } + // Re-detect where there are no features. + if (self.active.len() as f64) < want as f64 * 0.85 { + let md = (pyr.width().max(pyr.height()) as f64 / (want as f64).sqrt() * 0.6).clamp(5.0, 40.0); + let cell = md; + let (gw, gh) = ((pyr.width() as f64 / cell).ceil() as usize + 1, (pyr.height() as f64 / cell).ceil() as usize + 1); + let mut occ = vec![false; gw * gh]; + for (_, p) in &self.active { + let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize); + for yy in cy.saturating_sub(1)..=(cy + 1).min(gh - 1) { + for xx in cx.saturating_sub(1)..=(cx + 1).min(gw - 1) { + occ[yy * gw + xx] = true; + } + } + } + let free = |p: [f64; 2]| { + let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize); + !occ.get(cy * gw + cx).copied().unwrap_or(true) + }; + let opts = CornerOpts { max_features: want - self.active.len().min(want), min_distance: md, quality: 0.005, window: 2, border: 8 }; + for p in good_features(&pyr, &opts, Some(&free)) { + let l = pyr.to_layer(p); + let id = self.tracks.len() as u32; + self.tracks.push(Track2D { id, start: k, pts: vec![[l[0] as f32, l[1] as f32]] }); + self.active.push((id as usize, p)); + } + } + self.prev = Some(pyr); + self.frame += 1; + } + + pub fn finish(self, start: f64, frame_duration: f64) -> CameraTracks { + let mut tracks: Vec = self.tracks.into_iter().filter(|t| t.pts.len() >= 3).collect(); + // Round to 1/100 pixel (smaller JSON) and renumber densely. + for (i, t) in tracks.iter_mut().enumerate() { + t.id = i as u32; + for p in t.pts.iter_mut() { + *p = [(p[0] * 100.0).round() / 100.0, (p[1] * 100.0).round() / 100.0]; + } + } + CameraTracks { version: 1, size: self.size, start, frame_duration, frames: self.frame, factor: self.factor, detailed: self.opts.detailed, tracks } + } +} diff --git a/crates/track/src/camtrack/geometry.rs b/crates/track/src/camtrack/geometry.rs new file mode 100644 index 00000000..6956ac2b --- /dev/null +++ b/crates/track/src/camtrack/geometry.rs @@ -0,0 +1,432 @@ +//! Two-view and multi-view geometry for the camera solver, from Hartley & Zisserman (H&Z), +//! *Multiple View Geometry in Computer Vision* (2nd ed.) and Ma et al., *An Invitation to 3-D +//! Vision* (2004). +//! +//! All image points here are **normalised** (calibrated) coordinates `x = (u − c) / f`. +//! A camera is `(R, C)` with `X_cam = R (X − C)`. + +use super::linalg::*; + +/// A deterministic xorshift generator for RANSAC samples. +pub struct Rng(pub u64); +impl Rng { + pub fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + pub fn below(&mut self, n: usize) -> usize { + (self.next_u64() % n.max(1) as u64) as usize + } + /// `k` distinct indices below `n`. + pub fn sample(&mut self, n: usize, k: usize, out: &mut Vec) { + out.clear(); + let mut guard = 0; + while out.len() < k && guard < 1000 { + guard += 1; + let c = self.below(n); + if !out.contains(&c) { + out.push(c); + } + } + } +} + +/// Hartley normalisation (H&Z §4.4.4): centroid to the origin, mean distance √2. +fn normalizer(p: &[[f64; 2]]) -> M3 { + let n = p.len().max(1) as f64; + let c = [p.iter().map(|q| q[0]).sum::() / n, p.iter().map(|q| q[1]).sum::() / n]; + let d = p.iter().map(|q| (q[0] - c[0]).hypot(q[1] - c[1])).sum::() / n; + let s = if d > 1e-12 { std::f64::consts::SQRT_2 / d } else { 1.0 }; + [[s, 0.0, -s * c[0]], [0.0, s, -s * c[1]], [0.0, 0.0, 1.0]] +} + +fn h2(m: &M3, p: [f64; 2]) -> V3 { + mv(m, [p[0], p[1], 1.0]) +} + +/// The essential matrix from ≥ 8 correspondences (`x2ᵀ E x1 = 0`) by the normalised 8-point +/// algorithm (H&Z Alg. 11.1), with the essential constraint (two equal singular values, one +/// zero) enforced (H&Z §9.6.1 / Result 9.18). +pub fn essential_8pt(x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Option { + let n = x1.len().min(x2.len()); + if n < 8 { + return None; + } + let (t1, t2) = (normalizer(&x1[..n]), normalizer(&x2[..n])); + let mut a = Vec::with_capacity(n * 9); + for i in 0..n { + let p = h2(&t1, x1[i]); + let q = h2(&t2, x2[i]); + a.extend_from_slice(&[q[0] * p[0], q[0] * p[1], q[0], q[1] * p[0], q[1] * p[1], q[1], p[0], p[1], 1.0]); + } + let e = null_vector(&a, n, 9); + let en = [[e[0], e[1], e[2]], [e[3], e[4], e[5]], [e[6], e[7], e[8]]]; + // Denormalise: E = T2ᵀ Ê T1. + let e = mm(&mt(&t2), &mm(&en, &t1)); + let (u, s, v) = svd3(&e); + let m = (s[0] + s[1]) * 0.5; + if m <= 0.0 || !m.is_finite() { + return None; + } + let d = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]]; + Some(mm(&u, &mm(&d, &mt(&v)))) +} + +/// Squared Sampson distance of a correspondence to `x2ᵀ E x1 = 0` (H&Z §11.4.3), in the units of +/// the coordinates. +pub fn sampson2(e: &M3, p: [f64; 2], q: [f64; 2]) -> f64 { + let x1 = [p[0], p[1], 1.0]; + let x2 = [q[0], q[1], 1.0]; + let ex1 = mv(e, x1); + let etx2 = mtv(e, x2); + let num = dot(x2, ex1); + let den = ex1[0] * ex1[0] + ex1[1] * ex1[1] + etx2[0] * etx2[0] + etx2[1] * etx2[1]; + if den < 1e-300 { f64::INFINITY } else { num * num / den } +} + +/// RANSAC essential matrix (8-point samples, Sampson distance below `thr`), refitted on the +/// consensus set. Returns the matrix and the inlier mask. +pub fn ransac_essential(x1: &[[f64; 2]], x2: &[[f64; 2]], thr: f64, iterations: usize, seed: u64) -> Option<(M3, Vec)> { + let n = x1.len().min(x2.len()); + if n < 8 { + return None; + } + let t2 = thr * thr; + let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1); + let mut best: Option<(usize, f64, M3)> = None; + let mut idx = vec![]; + let mut max_iter = iterations; + let mut it = 0; + let (mut s1, mut s2) = (vec![[0.0; 2]; 8], vec![[0.0; 2]; 8]); + while it < max_iter { + it += 1; + rng.sample(n, 8, &mut idx); + if idx.len() < 8 { + break; + } + for (k, &i) in idx.iter().enumerate() { + s1[k] = x1[i]; + s2[k] = x2[i]; + } + let Some(e) = essential_8pt(&s1, &s2) else { continue }; + let (mut c, mut err) = (0, 0.0); + for i in 0..n { + let d = sampson2(&e, x1[i], x2[i]); + if d < t2 { + c += 1; + err += d; + } + } + if best.as_ref().is_none_or(|b| c > b.0 || (c == b.0 && err < b.1)) { + best = Some((c, err, e)); + // Adaptive iteration count for 99 % confidence. + let w = c as f64 / n as f64; + let p_good = w.powi(8); + if p_good > 1e-9 { + let need = ((1.0 - 0.99f64).ln() / (1.0 - p_good).max(1e-12).ln()).ceil(); + if need.is_finite() { + max_iter = max_iter.min((need as usize).max(30)); + } + } + } + } + let (_, _, mut e) = best?; + // Refit on the inliers (twice: the consensus may grow). + let mut inl = vec![false; n]; + for _ in 0..2 { + for i in 0..n { + inl[i] = sampson2(&e, x1[i], x2[i]) < t2; + } + let (a, b): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..n).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip(); + match essential_8pt(&a, &b) { + Some(e2) => e = e2, + None => break, + } + } + for i in 0..n { + inl[i] = sampson2(&e, x1[i], x2[i]) < t2; + } + Some((e, inl)) +} + +/// The four `(R, t)` with `x2 ~ R x1 + t` (|t| = 1) of an essential matrix (H&Z Result 9.19). +pub fn decompose_essential(e: &M3) -> [(M3, V3); 4] { + let (mut u, _, mut v) = svd3(e); + if det3(&u) < 0.0 { + for row in u.iter_mut() { + row[2] = -row[2]; + } + } + if det3(&v) < 0.0 { + for row in v.iter_mut() { + row[2] = -row[2]; + } + } + let w = [[0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]]; + let r1 = mm(&u, &mm(&w, &mt(&v))); + let r2 = mm(&u, &mm(&mt(&w), &mt(&v))); + let t = col(&u, 2); + [(r1, t), (r1, scale(t, -1.0)), (r2, t), (r2, scale(t, -1.0))] +} + +/// A camera `(R, C)`. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Pose { + pub r: M3, + pub c: V3, +} + +impl Pose { + pub const IDENTITY: Pose = Pose { r: I3, c: [0.0; 3] }; + /// From `x_cam = R X + t`. + pub fn from_rt(r: M3, t: V3) -> Pose { + Pose { r, c: scale(mtv(&r, t), -1.0) } + } + pub fn t(&self) -> V3 { + scale(mv(&self.r, self.c), -1.0) + } + pub fn to_cam(&self, x: V3) -> V3 { + mv(&self.r, sub(x, self.c)) + } +} + +/// Linear (DLT) triangulation from normalised observations (H&Z §12.2), minimising the +/// algebraic error over all views. +pub fn triangulate(views: &[(Pose, [f64; 2])]) -> Option { + if views.len() < 2 { + return None; + } + let mut ata = [0.0f64; 16]; + for (p, x) in views { + let t = p.t(); + let rows = [[p.r[0][0], p.r[0][1], p.r[0][2], t[0]], [p.r[1][0], p.r[1][1], p.r[1][2], t[1]], [p.r[2][0], p.r[2][1], p.r[2][2], t[2]]]; + let a0: Vec = (0..4).map(|j| x[0] * rows[2][j] - rows[0][j]).collect(); + let a1: Vec = (0..4).map(|j| x[1] * rows[2][j] - rows[1][j]).collect(); + for a in [a0, a1] { + let nrm = a.iter().map(|v| v * v).sum::().sqrt().max(1e-300); + for i in 0..4 { + for j in 0..4 { + ata[i * 4 + j] += a[i] * a[j] / (nrm * nrm); + } + } + } + } + let (_, vecs) = sym_eigen(&ata, 4); + let h = [vecs[0], vecs[4], vecs[8], vecs[12]]; + if h[3].abs() < 1e-12 { + return None; + } + let x = [h[0] / h[3], h[1] / h[3], h[2] / h[3]]; + x.iter().all(|v| v.is_finite()).then_some(x) +} + +/// Angle (radians) between the rays from two camera centres to `x`. +pub fn ray_angle(a: V3, b: V3, x: V3) -> f64 { + let (u, v) = (normalize(sub(x, a)), normalize(sub(x, b))); + dot(u, v).clamp(-1.0, 1.0).acos() +} + +/// Squared reprojection error (normalised units) of `x` in `pose` against observation `o` +/// (`None` behind the camera). +pub fn reproj2(pose: &Pose, x: V3, o: [f64; 2]) -> Option { + let c = pose.to_cam(x); + if c[2] <= 1e-9 { + return None; + } + let (u, v) = (c[0] / c[2] - o[0], c[1] / c[2] - o[1]); + Some(u * u + v * v) +} + +/// Pick the essential decomposition with the most points in front of both cameras; returns +/// the second camera (the first is the identity) and that count. +pub fn select_pose(cands: &[(M3, V3)], x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Option<(Pose, usize)> { + let mut best: Option<(Pose, usize)> = None; + for (r, t) in cands { + let p2 = Pose::from_rt(*r, *t); + let mut c = 0; + for i in 0..x1.len() { + if let Some(x) = triangulate(&[(Pose::IDENTITY, x1[i]), (p2, x2[i])]) + && x[2] > 0.0 + && p2.to_cam(x)[2] > 0.0 + { + c += 1; + } + } + if best.as_ref().is_none_or(|b| c > b.1) { + best = Some((p2, c)); + } + } + best +} + +/// Decompose a calibrated planar homography `H ~ R + t nᵀ / d` (normalised coordinates, +/// `x2 ~ H x1`) into its motions `(R, t/d, n)` (Ma et al. §5.3.3). Up to four solutions; those +/// that put the plane behind the first camera are dropped. +pub fn decompose_homography(h: &M3, x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Vec<(M3, V3, V3)> { + let (_, s, _) = svd3(h); + if s[1] <= 1e-12 { + return vec![]; + } + let mut hn = mscale(h, 1.0 / s[1]); + // Sign: x2ᵀ H x1 > 0 for the correspondences (positive depth). + let pos = x1.iter().zip(x2).filter(|(p, q)| dot([q[0], q[1], 1.0], mv(&hn, [p[0], p[1], 1.0])) > 0.0).count(); + if pos * 2 < x1.len() { + hn = mscale(&hn, -1.0); + } + let hth = mm(&mt(&hn), &hn); + let a: Vec = hth.iter().flatten().copied().collect(); + let (vals, vecs) = sym_eigen(&a, 3); + // Descending: σ1² ≥ σ2² = 1 ≥ σ3². + let v1 = [vecs[2], vecs[5], vecs[8]]; + let v2 = [vecs[1], vecs[4], vecs[7]]; + let v3 = [vecs[0], vecs[3], vecs[6]]; + let (s1, s3) = (vals[2].max(1.0), vals[0].clamp(0.0, 1.0)); + let mut out = vec![]; + if s1 - s3 < 1e-10 { + // H is a rotation: no translation (pure rotation or plane at infinity). + return vec![(nearest_rotation(&hn), [0.0; 3], [0.0, 0.0, 1.0])]; + } + let k = (s1 - s3).sqrt(); + let (a1, a3) = ((1.0 - s3).sqrt(), (s1 - 1.0).sqrt()); + let u1 = scale(add(scale(v1, a1), scale(v3, a3)), 1.0 / k); + let u2 = scale(sub(scale(v1, a1), scale(v3, a3)), 1.0 / k); + for u in [u1, u2] { + let uu = from_cols(v2, u, cross(v2, u)); + let hv2 = mv(&hn, v2); + let hu = mv(&hn, u); + let ww = from_cols(hv2, hu, cross(hv2, hu)); + let r = mm(&ww, &mt(&uu)); + let n = cross(v2, u); + let t = mv(&madd(&hn, &r, -1.0), n); + for sgn in [1.0, -1.0] { + let (n2, t2) = (scale(n, sgn), scale(t, sgn)); + // The plane must be in front of the first camera: nᵀ x1 > 0 (with n pointing away). + let front = x1.iter().filter(|p| dot(n2, [p[0], p[1], 1.0]) > 0.0).count(); + if front * 2 >= x1.len() { + out.push((r, t2, n2)); + } + } + } + out +} + +/// Least-squares plane through points: (centroid, unit normal, RMS in-plane spread). +pub fn fit_plane(p: &[V3]) -> Option<(V3, V3, f64)> { + if p.len() < 3 { + return None; + } + let k = 1.0 / p.len() as f64; + let c = scale(p.iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [0.0; 9]; + for q in p { + let d = sub(*q, c); + for i in 0..3 { + for j in 0..3 { + cov[i * 3 + j] += d[i] * d[j] * k; + } + } + } + let (vals, vecs) = sym_eigen(&cov, 3); + if vals[1] <= 1e-18 { + return None; + } + let n = normalize([vecs[0], vecs[3], vecs[6]]); + Some((c, n, (vals[1] + vals[2]).max(0.0).sqrt())) +} + +/// The rotation `R` minimising `Σ |b − R a|²` over unit rays (orthogonal Procrustes / Kabsch). +pub fn procrustes(a: &[V3], b: &[V3]) -> Option { + if a.len() < 2 { + return None; + } + let mut m = [[0.0; 3]; 3]; + for (x, y) in a.iter().zip(b) { + m = madd(&m, &outer(*y, *x), 1.0); + } + let (u, _, v) = svd3(&m); + let mut r = mm(&u, &mt(&v)); + if det3(&r) < 0.0 { + let mut u2 = u; + for row in u2.iter_mut() { + row[2] = -row[2]; + } + r = mm(&u2, &mt(&v)); + } + Some(r) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scene() -> (Vec, Pose) { + let mut rng = Rng(12345); + let pts: Vec = (0..60) + .map(|_| { + let f = |r: &mut Rng| (r.next_u64() % 10_000) as f64 / 10_000.0 - 0.5; + [f(&mut rng) * 4.0, f(&mut rng) * 3.0, 5.0 + f(&mut rng) * 3.0] + }) + .collect(); + let p2 = Pose { r: rodrigues([0.02, -0.15, 0.03]), c: [0.6, 0.1, -0.2] }; + (pts, p2) + } + + fn proj(p: &Pose, x: V3) -> [f64; 2] { + let c = p.to_cam(x); + [c[0] / c[2], c[1] / c[2]] + } + + #[test] + fn essential_recovers_relative_pose() { + let (pts, p2) = scene(); + let x1: Vec<[f64; 2]> = pts.iter().map(|x| proj(&Pose::IDENTITY, *x)).collect(); + let mut x2: Vec<[f64; 2]> = pts.iter().map(|x| proj(&p2, *x)).collect(); + // Outliers. + x2[3] = [0.3, -0.2]; + x2[17] = [-0.1, 0.25]; + let (e, inl) = ransac_essential(&x1, &x2, 1e-3, 500, 7).unwrap(); + assert!(!inl[3] && !inl[17]); + assert!(inl.iter().filter(|b| **b).count() >= 57); + let (pose, n) = select_pose(&decompose_essential(&e), &x1, &x2).unwrap(); + assert!(n >= 57); + let r_err = norm(log_rot(&mm(&pose.r, &mt(&p2.r)))); + assert!(r_err < 1e-6, "{r_err}"); + let dir = normalize(p2.c); + assert!(norm(sub(normalize(pose.c), dir)) < 1e-5, "{:?} {:?}", pose.c, dir); + } + + #[test] + fn homography_decomposition_contains_the_motion() { + // Plane z = 4 (normal (0,0,1) in camera 1), seen after a motion. + let p2 = Pose { r: rodrigues([0.05, 0.2, -0.04]), c: [0.8, -0.3, 0.4] }; + let pts: Vec = (0..30).map(|i| [((i * 37) % 17) as f64 / 8.0 - 1.0, ((i * 11) % 13) as f64 / 6.0 - 1.0, 4.0]).collect(); + let x1: Vec<[f64; 2]> = pts.iter().map(|x| proj(&Pose::IDENTITY, *x)).collect(); + let x2: Vec<[f64; 2]> = pts.iter().map(|x| proj(&p2, *x)).collect(); + // H = R + t nᵀ / d with t = −R C, n = (0,0,1), d = 4. + let t = p2.t(); + let h = madd(&p2.r, &outer(t, [0.0, 0.0, 0.25]), 1.0); + let sols = decompose_homography(&mscale(&h, 3.7), &x1, &x2); + assert!(!sols.is_empty()); + let ok = sols + .iter() + .any(|(r, td, n)| norm(log_rot(&mm(r, &mt(&p2.r)))) < 1e-6 && norm(sub(*td, scale(t, 0.25))) < 1e-6 && norm(sub(*n, [0.0, 0.0, 1.0])) < 1e-6); + assert!(ok, "{sols:?}"); + } + + #[test] + fn triangulation_and_plane() { + let (pts, p2) = scene(); + for x in &pts[..5] { + let y = triangulate(&[(Pose::IDENTITY, proj(&Pose::IDENTITY, *x)), (p2, proj(&p2, *x))]).unwrap(); + assert!(norm(sub(*x, y)) < 1e-8); + } + let plane: Vec = (0..10).map(|i| [i as f64, (i * i % 7) as f64, 2.0 * i as f64 + 1.0]).collect(); + let (_, n, _) = fit_plane(&plane).unwrap(); + // All points satisfy z = 2x + 1: normal ∝ (2, 0, −1). + assert!(norm(cross(n, normalize([2.0, 0.0, -1.0]))) < 1e-9); + } +} diff --git a/crates/track/src/camtrack/linalg.rs b/crates/track/src/camtrack/linalg.rs new file mode 100644 index 00000000..a7067f26 --- /dev/null +++ b/crates/track/src/camtrack/linalg.rs @@ -0,0 +1,476 @@ +//! Small dense linear algebra for the camera solver, written from the textbook methods: +//! +//! - 3-vectors / 3 × 3 matrices and rotations (Rodrigues' formula and its inverse); +//! - the **one-sided Jacobi SVD** (Hestenes 1958; Golub & Van Loan, *Matrix Computations*, +//! §8.6.4), accurate for the small, badly scaled systems of the 8-point algorithm; +//! - the **cyclic Jacobi eigenvalue method** for symmetric matrices (Golub & Van Loan §8.5); +//! - **Cholesky** factorisation for the bundle adjustment's normal equations; +//! - the closed-form **similarity alignment** of two point sets (Umeyama, "Least-squares +//! estimation of transformation parameters between two point patterns", PAMI 1991). + +pub type V3 = [f64; 3]; +pub type M3 = [[f64; 3]; 3]; + +pub const I3: M3 = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]; + +pub fn add(a: V3, b: V3) -> V3 { + [a[0] + b[0], a[1] + b[1], a[2] + b[2]] +} +pub fn sub(a: V3, b: V3) -> V3 { + [a[0] - b[0], a[1] - b[1], a[2] - b[2]] +} +pub fn scale(a: V3, k: f64) -> V3 { + [a[0] * k, a[1] * k, a[2] * k] +} +pub fn dot(a: V3, b: V3) -> f64 { + a[0] * b[0] + a[1] * b[1] + a[2] * b[2] +} +pub fn cross(a: V3, b: V3) -> V3 { + [a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]] +} +pub fn norm(a: V3) -> f64 { + dot(a, a).sqrt() +} +pub fn normalize(a: V3) -> V3 { + let n = norm(a); + if n > 0.0 { scale(a, 1.0 / n) } else { a } +} +pub fn mv(m: &M3, v: V3) -> V3 { + [dot(m[0], v), dot(m[1], v), dot(m[2], v)] +} +/// `mᵀ v`. +pub fn mtv(m: &M3, v: V3) -> V3 { + [m[0][0] * v[0] + m[1][0] * v[1] + m[2][0] * v[2], m[0][1] * v[0] + m[1][1] * v[1] + m[2][1] * v[2], m[0][2] * v[0] + m[1][2] * v[1] + m[2][2] * v[2]] +} +pub fn mm(a: &M3, b: &M3) -> M3 { + let mut o = [[0.0; 3]; 3]; + for (i, row) in o.iter_mut().enumerate() { + for (j, v) in row.iter_mut().enumerate() { + *v = a[i][0] * b[0][j] + a[i][1] * b[1][j] + a[i][2] * b[2][j]; + } + } + o +} +pub fn mt(a: &M3) -> M3 { + [[a[0][0], a[1][0], a[2][0]], [a[0][1], a[1][1], a[2][1]], [a[0][2], a[1][2], a[2][2]]] +} +pub fn madd(a: &M3, b: &M3, k: f64) -> M3 { + let mut o = *a; + for i in 0..3 { + for j in 0..3 { + o[i][j] += k * b[i][j]; + } + } + o +} +pub fn mscale(a: &M3, k: f64) -> M3 { + madd(&[[0.0; 3]; 3], a, k) +} +pub fn det3(m: &M3) -> f64 { + m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0]) + m[0][2] * (m[1][0] * m[2][1] - m[1][1] * m[2][0]) +} +pub fn skew(v: V3) -> M3 { + [[0.0, -v[2], v[1]], [v[2], 0.0, -v[0]], [-v[1], v[0], 0.0]] +} +pub fn outer(a: V3, b: V3) -> M3 { + [[a[0] * b[0], a[0] * b[1], a[0] * b[2]], [a[1] * b[0], a[1] * b[1], a[1] * b[2]], [a[2] * b[0], a[2] * b[1], a[2] * b[2]]] +} +pub fn col(m: &M3, j: usize) -> V3 { + [m[0][j], m[1][j], m[2][j]] +} +pub fn from_cols(a: V3, b: V3, c: V3) -> M3 { + [[a[0], b[0], c[0]], [a[1], b[1], c[1]], [a[2], b[2], c[2]]] +} +pub fn inv3(m: &M3) -> Option { + let d = det3(m); + if d.abs() < 1e-300 { + return None; + } + let c = |r0: usize, c0: usize, r1: usize, c1: usize| m[r0][c0] * m[r1][c1] - m[r0][c1] * m[r1][c0]; + let adj = [[c(1, 1, 2, 2), -c(0, 1, 2, 2), c(0, 1, 1, 2)], [-c(1, 0, 2, 2), c(0, 0, 2, 2), -c(0, 0, 1, 2)], [c(1, 0, 2, 1), -c(0, 0, 2, 1), c(0, 0, 1, 1)]]; + Some(mscale(&adj, 1.0 / d)) +} + +/// Rotation matrix of an axis-angle vector (Rodrigues' formula). +pub fn rodrigues(w: V3) -> M3 { + let th = norm(w); + let k = skew(w); + if th < 1e-12 { + return madd(&I3, &k, 1.0); + } + let a = th.sin() / th; + let b = (1.0 - th.cos()) / (th * th); + madd(&madd(&I3, &k, a), &mm(&k, &k), b) +} + +/// Axis-angle vector of a rotation matrix (inverse of [`rodrigues`]). +pub fn log_rot(r: &M3) -> V3 { + let c = ((r[0][0] + r[1][1] + r[2][2] - 1.0) * 0.5).clamp(-1.0, 1.0); + let th = c.acos(); + let v = [r[2][1] - r[1][2], r[0][2] - r[2][0], r[1][0] - r[0][1]]; + if th < 1e-9 { + return scale(v, 0.5); + } + if std::f64::consts::PI - th < 1e-6 { + // Near π: the axis is the dominant column of R + I. + let b = madd(r, &I3, 1.0); + let j = (0..3).max_by(|&a, &bb| b[a][a].total_cmp(&b[bb][bb])).unwrap_or(0); + let axis = normalize(col(&b, j)); + return scale(axis, th); + } + scale(v, th / (2.0 * th.sin())) +} + +/// The rotation closest to `m` (Frobenius norm), via the SVD. +pub fn nearest_rotation(m: &M3) -> M3 { + let (u, _, v) = svd3(m); + let mut r = mm(&u, &mt(&v)); + if det3(&r) < 0.0 { + let mut u2 = u; + for row in u2.iter_mut() { + row[2] = -row[2]; + } + r = mm(&u2, &mt(&v)); + } + r +} + +/// Spherical interpolation between two rotations. +pub fn slerp(a: &M3, b: &M3, t: f64) -> M3 { + let d = log_rot(&mm(b, &mt(a))); + mm(&rodrigues(scale(d, t)), a) +} + +/// Thin SVD `A = U diag(s) Vᵀ` of an `m × n` row-major matrix (one-sided Jacobi). `U` is +/// `max(m, n) × n` (rows beyond `m` belong to zero padding), `s` is sorted descending and `V` +/// is `n × n` (columns are the right singular vectors). +pub fn svd(a: &[f64], m: usize, n: usize) -> (Vec, Vec, Vec) { + let rows = m.max(n); + let mut u = vec![0.0; rows * n]; + u[..m * n].copy_from_slice(&a[..m * n]); + let mut v = vec![0.0; n * n]; + for i in 0..n { + v[i * n + i] = 1.0; + } + for _sweep in 0..60 { + let mut off = 0.0f64; + for p in 0..n { + for q in p + 1..n { + let (mut alpha, mut beta, mut gamma) = (0.0, 0.0, 0.0); + for r in 0..rows { + let (x, y) = (u[r * n + p], u[r * n + q]); + alpha += x * x; + beta += y * y; + gamma += x * y; + } + if gamma == 0.0 { + continue; + } + let c0 = gamma.abs() / (alpha * beta).sqrt().max(1e-300); + off = off.max(c0); + if c0 < 1e-15 { + continue; + } + let zeta = (beta - alpha) / (2.0 * gamma); + let t = zeta.signum() / (zeta.abs() + (1.0 + zeta * zeta).sqrt()); + let t = if zeta == 0.0 { 1.0 } else { t }; + let c = 1.0 / (1.0 + t * t).sqrt(); + let s = c * t; + for r in 0..rows { + let (x, y) = (u[r * n + p], u[r * n + q]); + u[r * n + p] = c * x - s * y; + u[r * n + q] = s * x + c * y; + } + for r in 0..n { + let (x, y) = (v[r * n + p], v[r * n + q]); + v[r * n + p] = c * x - s * y; + v[r * n + q] = s * x + c * y; + } + } + } + if off < 1e-15 { + break; + } + } + let mut s: Vec = (0..n).map(|j| (0..rows).map(|r| u[r * n + j] * u[r * n + j]).sum::().sqrt()).collect(); + for (j, sj) in s.iter().enumerate() { + if *sj > 1e-300 { + for r in 0..rows { + u[r * n + j] /= sj; + } + } + } + // Sort descending. + let mut order: Vec = (0..n).collect(); + order.sort_by(|&a, &b| s[b].total_cmp(&s[a])); + let (mut u2, mut v2) = (vec![0.0; rows * n], vec![0.0; n * n]); + for (k, &j) in order.iter().enumerate() { + for r in 0..rows { + u2[r * n + k] = u[r * n + j]; + } + for r in 0..n { + v2[r * n + k] = v[r * n + j]; + } + } + s = order.iter().map(|&j| s[j]).collect(); + (u2, s, v2) +} + +/// SVD of a 3 × 3 matrix: (U, singular values descending, V) with `m = U diag(s) Vᵀ`. +pub fn svd3(m: &M3) -> (M3, V3, M3) { + let a: Vec = m.iter().flatten().copied().collect(); + let (u, s, v) = svd(&a, 3, 3); + let g = |x: &[f64]| [[x[0], x[1], x[2]], [x[3], x[4], x[5]], [x[6], x[7], x[8]]]; + let mut um = g(&u); + // Columns of U for zero singular values are undefined: complete them to an orthonormal basis. + if s[2] < 1e-300 { + let c = cross(col(&um, 0), col(&um, 1)); + for (r, row) in um.iter_mut().enumerate() { + row[2] = c[r]; + } + } + (um, [s[0], s[1], s[2]], g(&v)) +} + +/// The unit vector `x` minimising `|A x|` (`m × n` row-major): the right singular vector of +/// the smallest singular value. +pub fn null_vector(a: &[f64], m: usize, n: usize) -> Vec { + let (_, _, v) = svd(a, m, n); + (0..n).map(|r| v[r * n + n - 1]).collect() +} + +/// Eigen-decomposition of a symmetric `n × n` matrix (cyclic Jacobi): eigenvalues ascending +/// and the matching eigenvectors as the columns of the returned row-major matrix. +pub fn sym_eigen(a: &[f64], n: usize) -> (Vec, Vec) { + let mut a = a.to_vec(); + let mut v = vec![0.0; n * n]; + for i in 0..n { + v[i * n + i] = 1.0; + } + for _ in 0..100 { + let mut off = 0.0; + for p in 0..n { + for q in p + 1..n { + off += a[p * n + q] * a[p * n + q]; + } + } + if off < 1e-30 { + break; + } + for p in 0..n { + for q in p + 1..n { + let apq = a[p * n + q]; + if apq.abs() < 1e-300 { + continue; + } + let theta = (a[q * n + q] - a[p * n + p]) / (2.0 * apq); + let t = theta.signum() / (theta.abs() + (theta * theta + 1.0).sqrt()); + let t = if theta == 0.0 { 1.0 } else { t }; + let c = 1.0 / (t * t + 1.0).sqrt(); + let s = t * c; + for k in 0..n { + let (akp, akq) = (a[k * n + p], a[k * n + q]); + a[k * n + p] = c * akp - s * akq; + a[k * n + q] = s * akp + c * akq; + } + for k in 0..n { + let (apk, aqk) = (a[p * n + k], a[q * n + k]); + a[p * n + k] = c * apk - s * aqk; + a[q * n + k] = s * apk + c * aqk; + } + for k in 0..n { + let (vkp, vkq) = (v[k * n + p], v[k * n + q]); + v[k * n + p] = c * vkp - s * vkq; + v[k * n + q] = s * vkp + c * vkq; + } + } + } + } + let mut order: Vec = (0..n).collect(); + order.sort_by(|&x, &y| a[x * n + x].total_cmp(&a[y * n + y])); + let vals = order.iter().map(|&j| a[j * n + j]).collect(); + let mut vecs = vec![0.0; n * n]; + for (k, &j) in order.iter().enumerate() { + for r in 0..n { + vecs[r * n + k] = v[r * n + j]; + } + } + (vals, vecs) +} + +/// Solve `A x = b` for a symmetric positive definite `A` (`n × n`, row-major) in place by +/// Cholesky factorisation. Returns false when `A` is not positive definite. +pub fn cholesky_solve(a: &mut [f64], n: usize, b: &mut [f64]) -> bool { + for j in 0..n { + let mut d = a[j * n + j]; + for k in 0..j { + d -= a[j * n + k] * a[j * n + k]; + } + if d <= 0.0 || !d.is_finite() { + return false; + } + let d = d.sqrt(); + a[j * n + j] = d; + let (head, tail) = a.split_at_mut((j + 1) * n); + let rowj = &head[j * n..j * n + j]; + // Rows below j are independent. + use rayon::prelude::*; + let work = |row: &mut [f64]| { + let s: f64 = row[j] - row[..j].iter().zip(rowj).map(|(a, b)| a * b).sum::(); + row[j] = s / d; + }; + if n - j > 192 { + tail.par_chunks_mut(n).for_each(work); + } else { + tail.chunks_mut(n).for_each(work); + } + } + // Forward: L y = b. + for i in 0..n { + let mut s = b[i]; + for k in 0..i { + s -= a[i * n + k] * b[k]; + } + b[i] = s / a[i * n + i]; + } + // Back: Lᵀ x = y. + for i in (0..n).rev() { + let mut s = b[i]; + for k in i + 1..n { + s -= a[k * n + i] * b[k]; + } + b[i] = s / a[i * n + i]; + } + b.iter().all(|v| v.is_finite()) +} + +/// A similarity transform `x ↦ s R x + t`. +#[derive(Clone, Copy, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct Similarity { + pub s: f64, + pub r: M3, + pub t: V3, +} + +impl Default for Similarity { + fn default() -> Self { + Similarity { s: 1.0, r: I3, t: [0.0; 3] } + } +} + +impl Similarity { + pub fn apply(&self, x: V3) -> V3 { + add(scale(mv(&self.r, x), self.s), self.t) + } +} + +/// The similarity minimising `Σ |dst − (s R src + t)|²` (Umeyama 1991). +pub fn umeyama(src: &[V3], dst: &[V3]) -> Option { + let n = src.len().min(dst.len()); + if n < 3 { + return None; + } + let k = 1.0 / n as f64; + let ms = scale(src[..n].iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let md = scale(dst[..n].iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [[0.0; 3]; 3]; + let mut var = 0.0; + for i in 0..n { + let (a, b) = (sub(src[i], ms), sub(dst[i], md)); + cov = madd(&cov, &outer(b, a), k); + var += dot(a, a) * k; + } + if var < 1e-300 { + return None; + } + let (u, d, v) = svd3(&cov); + let mut sgn = [1.0, 1.0, 1.0]; + if det3(&u) * det3(&v) < 0.0 { + sgn[2] = -1.0; + } + let ud = from_cols(scale(col(&u, 0), sgn[0]), scale(col(&u, 1), sgn[1]), scale(col(&u, 2), sgn[2])); + let r = mm(&ud, &mt(&v)); + let s = (d[0] * sgn[0] + d[1] * sgn[1] + d[2] * sgn[2]) / var; + let t = sub(md, scale(mv(&r, ms), s)); + Some(Similarity { s, r, t }) +} + +/// Euler angles (degrees) `[x, y, z]` with `m = Rz(z) Ry(y) Rx(x)` (After Effects' Orientation +/// order: X, then Y, then Z applied to the vector). +pub fn euler_xyz(m: &M3) -> V3 { + let sy = (-m[2][0]).clamp(-1.0, 1.0); + let y = sy.asin(); + let (x, z) = if sy.abs() < 0.999_999 { (m[2][1].atan2(m[2][2]), m[1][0].atan2(m[0][0])) } else { ((-m[1][2]).atan2(m[1][1]), 0.0) }; + [x.to_degrees(), y.to_degrees(), z.to_degrees()] +} + +/// `Rz(z) Ry(y) Rx(x)` from degrees. +pub fn from_euler_xyz(e: V3) -> M3 { + let (sx, cx) = e[0].to_radians().sin_cos(); + let (sy, cy) = e[1].to_radians().sin_cos(); + let (sz, cz) = e[2].to_radians().sin_cos(); + let rx = [[1.0, 0.0, 0.0], [0.0, cx, -sx], [0.0, sx, cx]]; + let ry = [[cy, 0.0, sy], [0.0, 1.0, 0.0], [-sy, 0.0, cy]]; + let rz = [[cz, -sz, 0.0], [sz, cz, 0.0], [0.0, 0.0, 1.0]]; + mm(&rz, &mm(&ry, &rx)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn svd_reconstructs() { + let a = [3.0, 1.0, 2.0, -1.0, 4.0, 0.5, 2.0, 2.0, 7.0, 0.1, -3.0, 1.0]; + let (u, s, v) = svd(&a, 4, 3); + for r in 0..4 { + for c in 0..3 { + let x: f64 = (0..3).map(|k| u[r * 3 + k] * s[k] * v[c * 3 + k]).sum(); + assert!((x - a[r * 3 + c]).abs() < 1e-10); + } + } + assert!(s[0] >= s[1] && s[1] >= s[2]); + } + + #[test] + fn rotations_round_trip() { + let w = [0.3, -1.2, 0.7]; + let r = rodrigues(w); + let w2 = log_rot(&r); + assert!(norm(sub(w, w2)) < 1e-12); + let e = euler_xyz(&r); + let r2 = from_euler_xyz(e); + for i in 0..3 { + for j in 0..3 { + assert!((r[i][j] - r2[i][j]).abs() < 1e-12); + } + } + assert!((det3(&r) - 1.0).abs() < 1e-12); + } + + #[test] + fn eigen_and_cholesky() { + let a = [4.0, 1.0, 0.5, 1.0, 3.0, 0.2, 0.5, 0.2, 2.0]; + let (vals, vecs) = sym_eigen(&a, 3); + for k in 0..3 { + let v = [vecs[k], vecs[3 + k], vecs[6 + k]]; + let av = [dot([a[0], a[1], a[2]], v), dot([a[3], a[4], a[5]], v), dot([a[6], a[7], a[8]], v)]; + assert!(norm(sub(av, scale(v, vals[k]))) < 1e-10); + } + let mut m = a; + let mut b = [1.0, 2.0, 3.0]; + assert!(cholesky_solve(&mut m, 3, &mut b)); + let r = [dot([a[0], a[1], a[2]], b), dot([a[3], a[4], a[5]], b), dot([a[6], a[7], a[8]], b)]; + assert!(norm(sub(r, [1.0, 2.0, 3.0])) < 1e-12); + } + + #[test] + fn umeyama_recovers_a_similarity() { + let t = Similarity { s: 2.5, r: rodrigues([0.1, 0.4, -0.3]), t: [1.0, -2.0, 5.0] }; + let src = [[0.0, 0.0, 0.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [1.0, 2.0, 3.0]]; + let dst: Vec = src.iter().map(|p| t.apply(*p)).collect(); + let e = umeyama(&src, &dst).unwrap(); + assert!((e.s - 2.5).abs() < 1e-10); + assert!(norm(sub(e.t, t.t)) < 1e-9); + } +} diff --git a/crates/track/src/camtrack/mod.rs b/crates/track/src/camtrack/mod.rs new file mode 100644 index 00000000..056d7ad0 --- /dev/null +++ b/crates/track/src/camtrack/mod.rs @@ -0,0 +1,376 @@ +//! The 3D Camera Tracker's analysis: structure from motion on a single clip, clean-room from the +//! textbook pipeline (Hartley & Zisserman, *Multiple View Geometry in Computer Vision*, 2nd ed., +//! 2004; Triggs et al., "Bundle Adjustment — A Modern Synthesis", 2000): +//! +//! 1. **Feature tracks** ([`features::TrackAnalyzer`]): Shi–Tomasi corners followed through the +//! clip with pyramidal Lucas–Kanade and a forward–backward check ([`crate::klt`]); lost +//! features end their track and new ones are detected where the frame has none, so tracks +//! are as long as possible. +//! 2. **Solve** ([`solve`]): keyframes are selected by parallax; a two-view reconstruction +//! initialises the scene from either the **essential matrix** (normalised 8-point algorithm +//! inside RANSAC, H&Z §11.2 and §9.6) or, for mostly flat scenes, a **homography +//! decomposition** (Ma, Soatto, Košecká & Sastry, *An Invitation to 3-D Vision*, 2004, §5.3); +//! the other keyframes are added incrementally (**resection** by robust pose refinement, +//! **triangulation** of new tracks by the linear method of H&Z §12.2); a sparse +//! **Levenberg–Marquardt bundle adjustment** with the Schur complement over the points +//! ([`bundle`]) refines everything; the frames between keyframes are resected last. +//! 3. **Focal length**: *Specify Angle of View* fixes it; *Fixed Angle of View* searches it +//! (the reprojection error of the keyframe reconstruction over a log-spaced range of focal +//! lengths, refined by golden-section search) and then adjusts it in the bundle adjustment +//! as one shared parameter; *Variable Zoom* starts from the same estimate and adjusts a focal +//! length per frame, with a weak smoothness prior between neighbouring frames. +//! 4. **Tripod pan** shots (no camera translation) are solved as a pure rotation: frame-to-frame +//! rotations from the tracked rays (orthogonal Procrustes) refined by the same bundle +//! adjustment with the camera centres fixed. *Auto Detect* solves both ways and keeps the +//! tripod solution when it explains the tracks as well as the general one. +//! +//! The camera model has square pixels, the principal point at the layer centre and no lens +//! distortion. Camera space is x right, y down, z forward (After Effects' convention); a camera +//! is `(R, C, f)` with `X_cam = R (X − C)` and pixel `u = f x / z + w / 2`. +//! +//! The solve is stored in a canonical frame: the first solved frame's camera is at the origin +//! looking along +z, scaled so the median depth of the points it sees is 1. [`CameraSolve::world`] +//! maps it into composition space for a camera zoom, optionally re-based on a ground plane. + +pub mod bundle; +pub mod features; +pub mod geometry; +pub mod linalg; +mod solver; +#[cfg(test)] +mod tests; + +use serde::{Deserialize, Serialize}; + +pub use features::{AnalyzeOpts, TrackAnalyzer}; +use linalg::{M3, Similarity, V3}; +pub use solver::{SolveError, solve}; + +/// One feature track: positions (layer pixels) on consecutive frames from `start`. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Track2D { + pub id: u32, + pub start: u32, + pub pts: Vec<[f32; 2]>, +} + +impl Track2D { + pub fn end(&self) -> u32 { + self.start + self.pts.len() as u32 + } + /// Position on frame `k` when the track covers it. + pub fn at(&self, k: u32) -> Option<[f64; 2]> { + let i = k.checked_sub(self.start)? as usize; + self.pts.get(i).map(|p| [p[0] as f64, p[1] as f64]) + } +} + +/// The tracked features of a clip (step 1 of the analysis). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct CameraTracks { + pub version: u32, + /// Layer size (pixels). + pub size: [f64; 2], + /// Layer time (seconds) of frame 0, and the frame duration. + pub start: f64, + pub frame_duration: f64, + pub frames: u32, + /// Analysis downsampling factor (image pixels per analysis pixel): the tracking noise scale. + pub factor: f64, + pub detailed: bool, + pub tracks: Vec, +} + +impl CameraTracks { + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } + pub fn from_json(s: &str) -> Option { + if s.trim().is_empty() { + return None; + } + serde_json::from_str(s).ok() + } + pub fn is_empty(&self) -> bool { + self.frames == 0 + } + pub fn track(&self, id: u32) -> Option<&Track2D> { + self.tracks.binary_search_by_key(&id, |t| t.id).ok().map(|i| &self.tracks[i]) + } +} + +/// 3D Camera Tracker ▸ Shot Type. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ShotType { + #[default] + FixedAngle, + VariableZoom, + SpecifyAngle, +} + +impl ShotType { + pub const ALL: [ShotType; 3] = [ShotType::FixedAngle, ShotType::VariableZoom, ShotType::SpecifyAngle]; + pub fn label(self) -> &'static str { + match self { + ShotType::FixedAngle => "Fixed Angle of View", + ShotType::VariableZoom => "Variable Zoom", + ShotType::SpecifyAngle => "Specify Angle of View", + } + } + pub fn from_name(s: &str) -> Option { + let k = s.to_ascii_lowercase().replace([' ', '-', '_'], ""); + match k.as_str() { + "fixed" | "fixedangle" | "fixedangleofview" => Some(ShotType::FixedAngle), + "variable" | "variablezoom" | "zoom" => Some(ShotType::VariableZoom), + "specify" | "specifyangle" | "specifyangleofview" => Some(ShotType::SpecifyAngle), + _ => None, + } + } +} + +/// 3D Camera Tracker ▸ Advanced ▸ Solve Method (and Method Used). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum SolveMethod { + #[default] + Auto, + Typical, + MostlyFlat, + TripodPan, +} + +impl SolveMethod { + pub const ALL: [SolveMethod; 4] = [SolveMethod::Auto, SolveMethod::Typical, SolveMethod::MostlyFlat, SolveMethod::TripodPan]; + pub fn label(self) -> &'static str { + match self { + SolveMethod::Auto => "Auto Detect", + SolveMethod::Typical => "Typical", + SolveMethod::MostlyFlat => "Mostly Flat Scene", + SolveMethod::TripodPan => "Tripod Pan", + } + } + pub fn from_name(s: &str) -> Option { + let k = s.to_ascii_lowercase().replace([' ', '-', '_'], ""); + match k.as_str() { + "auto" | "autodetect" => Some(SolveMethod::Auto), + "typical" => Some(SolveMethod::Typical), + "flat" | "mostlyflat" | "mostlyflatscene" => Some(SolveMethod::MostlyFlat), + "tripod" | "tripodpan" => Some(SolveMethod::TripodPan), + _ => None, + } + } +} + +/// What a solve is made with. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolveSettings { + pub shot: ShotType, + /// Horizontal angle of view (degrees) for [`ShotType::SpecifyAngle`]. + pub hfov: f64, + pub method: SolveMethod, + /// Track ids the user deleted. + pub deleted: Vec, +} + +/// The solved camera of one frame. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolvedFrame { + /// World → camera rotation (axis-angle). + pub rot: V3, + /// Camera centre. + pub center: V3, + /// Focal length (layer pixels). + pub focal: f64, + pub solved: bool, +} + +impl SolvedFrame { + pub fn r(&self) -> M3 { + linalg::rodrigues(self.rot) + } + /// Camera-space position of a world point. + pub fn to_cam(&self, x: V3) -> V3 { + linalg::mv(&self.r(), linalg::sub(x, self.center)) + } + /// Project a world point to layer pixels (`None` behind the camera). + pub fn project(&self, size: [f64; 2], x: V3) -> Option<[f64; 2]> { + let c = self.to_cam(x); + (c[2] > 1e-9).then(|| [self.focal * c[0] / c[2] + size[0] * 0.5, self.focal * c[1] / c[2] + size[1] * 0.5]) + } +} + +/// A solved 3D track point. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolvedPoint { + /// The feature track it was made from. + pub id: u32, + pub pos: V3, + /// Average reprojection error (pixels). + pub error: f32, + /// Frames `[first, last]` the point was tracked on. + pub first: u32, + pub last: u32, +} + +/// A ground plane and origin (canonical solve coordinates). +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct Ground { + pub origin: V3, + /// Unit normal on the cameras' side ("up"). + pub normal: V3, + /// Unit in-plane direction that becomes world +x. + pub x_axis: V3, +} + +/// A plane through solved points: After Effects' target. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct Target { + pub center: V3, + /// Unit normal facing the camera. + pub normal: V3, + /// Diameter in canonical units. + pub size: f64, +} + +/// The camera solve (step 2 of the analysis). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct CameraSolve { + pub version: u32, + pub size: [f64; 2], + pub start: f64, + pub frame_duration: f64, + pub shot: ShotType, + /// The model that was used (Auto Detect resolves to one of the others). + pub method_used: SolveMethod, + /// Average reprojection error (pixels) over all solved observations. + pub average_error: f64, + pub frames: Vec, + pub points: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub ground: Option, +} + +impl CameraSolve { + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } + pub fn from_json(s: &str) -> Option { + if s.trim().is_empty() { + return None; + } + serde_json::from_str(s).ok() + } + pub fn is_empty(&self) -> bool { + self.frames.is_empty() + } + /// Frame index at a layer time (held at the ends). + pub fn frame_at(&self, layer_time: f64) -> Option { + if self.frames.is_empty() || self.frame_duration <= 0.0 { + return None; + } + let i = ((layer_time - self.start) / self.frame_duration).round(); + Some(i.clamp(0.0, (self.frames.len() - 1) as f64) as usize) + } + pub fn point(&self, id: u32) -> Option<&SolvedPoint> { + self.points.iter().find(|p| p.id == id) + } + /// Points tracked on frame `k`. + pub fn visible(&self, k: usize) -> impl Iterator { + let k = k as u32; + self.points.iter().filter(move |p| p.first <= k && k <= p.last) + } + /// Median focal length of the solved frames (layer pixels). + pub fn focal(&self) -> f64 { + let mut f: Vec = self.frames.iter().filter(|c| c.solved).map(|c| c.focal).collect(); + if f.is_empty() { + return 0.0; + } + f.sort_by(f64::total_cmp); + f[f.len() / 2] + } + /// Horizontal angle of view (degrees) of focal length `f`. + pub fn hfov(&self, f: f64) -> f64 { + 2.0 * (self.size[0] * 0.5 / f.max(1e-9)).atan().to_degrees() + } + + /// The plane through points `ids` seen from frame `k` (needs three points; with more the + /// plane is a least-squares fit). + pub fn target(&self, ids: &[u32], k: usize) -> Option { + let pts: Vec = ids.iter().filter_map(|i| self.point(*i)).map(|p| p.pos).collect(); + let cam = self.frames.get(k).filter(|c| c.solved).or_else(|| self.frames.iter().find(|c| c.solved))?; + geometry::fit_plane(&pts).map(|(c, mut n, spread)| { + if linalg::dot(n, linalg::sub(cam.center, c)) < 0.0 { + n = linalg::scale(n, -1.0); + } + // After Effects sizes the target from the camera distance when the points are close. + let depth = linalg::norm(linalg::sub(cam.center, c)); + Target { center: c, normal: n, size: (spread * 2.0).max(depth * 0.12) } + }) + } + + /// A ground plane through `ids` (origin at their centroid, normal towards frame `k`'s + /// camera, +x along that camera's right vector projected on the plane). + pub fn ground_from(&self, ids: &[u32], k: usize) -> Option { + let t = self.target(ids, k)?; + let cam = self.frames.get(k).filter(|c| c.solved).or_else(|| self.frames.iter().find(|c| c.solved))?; + let right = linalg::mtv(&cam.r(), [1.0, 0.0, 0.0]); + let mut x = linalg::sub(right, linalg::scale(t.normal, linalg::dot(right, t.normal))); + if linalg::norm(x) < 1e-6 { + let fwd = linalg::mtv(&cam.r(), [0.0, 0.0, 1.0]); + x = linalg::cross(fwd, t.normal); + } + Some(Ground { origin: t.center, normal: t.normal, x_axis: linalg::normalize(x) }) + } + + /// Canonical solve coordinates → composition world, for a camera whose zoom on the first + /// solved frame is `zoom0` (comp pixels) in a `comp` sized composition. Without a ground + /// plane, the first solved frame's camera becomes After Effects' default camera (at + /// `(w/2, h/2, −zoom)` looking down +z, so points at the median depth land near z = 0). With + /// one, the ground plane becomes the X-Z plane through the origin, normal along −y (up). + pub fn world(&self, comp: [f64; 2], zoom0: f64) -> Similarity { + match &self.ground { + Some(g) => { + // Rows: canonical directions that map to world x, y, z. + let y = linalg::scale(g.normal, -1.0); + let x = g.x_axis; + let z = linalg::cross(x, y); + let r = [x, y, z]; + let t = linalg::scale(linalg::mv(&r, g.origin), -zoom0); + Similarity { s: zoom0, r, t } + } + None => Similarity { s: zoom0, r: linalg::I3, t: [comp[0] * 0.5, comp[1] * 0.5, -zoom0] }, + } + } + + /// The first solved frame. + pub fn first_solved(&self) -> Option { + self.frames.iter().position(|c| c.solved) + } +} + +/// Deterministic colour of a track point (After Effects draws each target in its own colour). +pub fn point_color(id: u32) -> [f32; 3] { + let mut h = id.wrapping_mul(0x9E37_79B9) ^ 0x5bd1_e995; + h ^= h >> 15; + let hue = (h % 360) as f32 / 60.0; + let x = 1.0 - (hue % 2.0 - 1.0).abs(); + let (r, g, b) = match hue as u32 { + 0 => (1.0, x, 0.0), + 1 => (x, 1.0, 0.0), + 2 => (0.0, 1.0, x), + 3 => (0.0, x, 1.0), + 4 => (x, 0.0, 1.0), + _ => (1.0, 0.0, x), + }; + // Bright, slightly desaturated. + [0.25 + 0.75 * r, 0.25 + 0.75 * g, 0.25 + 0.75 * b] +} diff --git a/crates/track/src/camtrack/solver.rs b/crates/track/src/camtrack/solver.rs new file mode 100644 index 00000000..da3a9290 --- /dev/null +++ b/crates/track/src/camtrack/solver.rs @@ -0,0 +1,894 @@ +//! The camera solve pipeline (see the module docs of [`super`]). + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::atomic::{AtomicBool, Ordering}; + +use rayon::prelude::*; + +use super::bundle::{self, BaCam, BaObs, BaOpts, FocalMode}; +use super::geometry::{self, Pose}; +use super::linalg::*; +use super::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, SolvedFrame, SolvedPoint}; +use crate::fit::{Model, ransac}; + +fn debug() -> bool { + std::env::var_os("EC_CAMTRACK_DEBUG").is_some() +} + +/// Why a solve failed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SolveError(pub String); + +impl std::fmt::Display for SolveError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +fn err(m: &str) -> Result { + Err(SolveError(m.to_string())) +} + +/// Tracks in solver form: pixels relative to the principal point. +struct Data { + n: usize, + w: f64, + /// (track id, first frame, positions). + tracks: Vec<(u32, usize, Vec<[f64; 2]>)>, + /// Per frame: (track index, position). + by_frame: Vec>, + /// Tracking noise scale (pixels). + noise: f64, +} + +impl Data { + fn new(t: &CameraTracks, deleted: &[u32]) -> Data { + let n = t.frames as usize; + let c = [t.size[0] * 0.5, t.size[1] * 0.5]; + let del: HashSet = deleted.iter().copied().collect(); + let mut tracks = vec![]; + let mut by_frame = vec![vec![]; n]; + for tr in t.tracks.iter().filter(|tr| !del.contains(&tr.id) && tr.pts.len() >= 3) { + let ti = tracks.len() as u32; + let pts: Vec<[f64; 2]> = tr.pts.iter().map(|p| [p[0] as f64 - c[0], p[1] as f64 - c[1]]).collect(); + for (k, p) in pts.iter().enumerate() { + if let Some(v) = by_frame.get_mut(tr.start as usize + k) { + v.push((ti, *p)); + } + } + tracks.push((tr.id, tr.start as usize, pts)); + } + Data { n, w: t.size[0], tracks, by_frame, noise: t.factor.max(1.0) } + } + fn at(&self, ti: u32, k: usize) -> Option<[f64; 2]> { + let (_, s, p) = &self.tracks[ti as usize]; + k.checked_sub(*s).and_then(|i| p.get(i).copied()) + } + fn inlier_thr(&self) -> f64 { + 4.0 * self.noise + } + fn huber(&self) -> f64 { + 1.0 * self.noise + } +} + +/// Parallax-based keyframes: a new keyframe when the median feature motion since the last +/// exceeds 3 % of the width, when fewer than 60 % of its features survive, or before the shared +/// features drop below 24. The first and last frames with features are keyframes. +fn keyframes(d: &Data) -> Vec { + let Some(first) = (0..d.n).find(|k| d.by_frame[*k].len() >= 8) else { return vec![] }; + let mut kf = vec![first]; + let mut last = first; + let mut j = first + 1; + while j < d.n { + let base = &d.by_frame[last]; + let mut disp = vec![]; + for (ti, p) in base { + if let Some(q) = d.at(*ti, j) { + disp.push((q[0] - p[0]).hypot(q[1] - p[1])); + } + } + if disp.len() < 24 && j > last + 1 { + // Too few shared features: the previous frame becomes a keyframe. + last = j - 1; + kf.push(last); + continue; + } + disp.sort_by(f64::total_cmp); + let med = disp.get(disp.len() / 2).copied().unwrap_or(0.0); + let ratio = disp.len() as f64 / base.len().max(1) as f64; + if med > 0.03 * d.w || ratio < 0.6 || disp.len() < 24 { + last = j; + kf.push(j); + } + j += 1; + } + if let Some(lastf) = (0..d.n).rev().find(|k| d.by_frame[*k].len() >= 8) + && *kf.last().expect("one") < lastf + { + kf.push(lastf); + } + kf +} + +/// A (partial) reconstruction. +#[derive(Clone, Default)] +struct Recon { + cams: BTreeMap, + pts: HashMap, + /// Rejected observations (track index, frame). + outliers: HashSet<(u32, usize)>, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Kind { + General, + Tripod, +} + +#[derive(Clone, Copy, PartialEq, Debug)] +struct Cfg { + kind: Kind, + /// Focal mode of the global adjustments. + focal: FocalMode, + /// Registration estimates a focal length per frame. + per_frame_f: bool, +} + +const F_PRIOR: f64 = 30.0; + +impl Recon { + /// Observations among `frames` (inliers, with a 3D point). + fn observations(&self, d: &Data, frames: &[usize]) -> (Vec, Vec, Vec, Vec) { + let cams: Vec = frames.iter().map(|k| self.cams[k]).collect(); + let mut pidx: HashMap = HashMap::new(); + let mut pts = vec![]; + let mut ids = vec![]; + let mut obs = vec![]; + for (ci, k) in frames.iter().enumerate() { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + let pi = *pidx.entry(*ti).or_insert_with(|| { + pts.push(*x); + ids.push(*ti); + (pts.len() - 1) as u32 + }); + obs.push(BaObs { cam: ci as u32, pt: pi, uv: *uv }); + } + } + (cams, pts, obs, ids) + } + + /// Bundle-adjust the cameras of `frames` (the first stays fixed) and their points. + fn adjust(&mut self, d: &Data, frames: &[usize], cfg: &Cfg, focal: FocalMode, iters: usize) -> f64 { + let (mut cams, mut pts, obs, ids) = self.observations(d, frames); + if obs.is_empty() { + return f64::INFINITY; + } + let mut fixed = vec![false; cams.len()]; + fixed[0] = true; + let rep = bundle::adjust( + &mut cams, + &mut pts, + &obs, + &BaOpts { focal, fix_points: false, fix_centers: cfg.kind == Kind::Tripod, fixed, huber: d.huber(), max_iter: iters }, + ); + for (k, c) in frames.iter().zip(&cams) { + self.cams.insert(*k, *c); + } + for (ti, x) in ids.iter().zip(&pts) { + self.pts.insert(*ti, *x); + } + rep.cost + } + + /// Mark observations above the inlier threshold as outliers; drop points left with fewer + /// than two observations among the registered frames. + fn reject(&mut self, d: &Data) { + let thr2 = d.inlier_thr().powi(2); + let mut count: HashMap = HashMap::new(); + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + match bundle::residual(c, *x, *uv) { + Some(r) if r[0] * r[0] + r[1] * r[1] <= thr2 => *count.entry(*ti).or_default() += 1, + _ => { + self.outliers.insert((*ti, *k)); + } + } + } + } + self.pts.retain(|ti, _| count.get(ti).copied().unwrap_or(0) >= 2); + } + + /// Triangulate tracks seen on ≥ 2 registered frames that have no point yet. + fn triangulate(&mut self, d: &Data, kind: Kind, min_angle_deg: f64, check: f64) { + let mut seen: HashMap> = HashMap::new(); + for k in self.cams.keys() { + for (ti, _) in &d.by_frame[*k] { + if !self.pts.contains_key(ti) { + seen.entry(*ti).or_default().push(*k); + } + } + } + let thr2 = (d.inlier_thr() * check).powi(2); + let new: Vec<(u32, V3)> = seen + .par_iter() + .filter(|(_, ks)| ks.len() >= 2) + .filter_map(|(ti, ks)| { + let views: Vec<(Pose, [f64; 2], &BaCam)> = ks + .iter() + .map(|k| { + let c = &self.cams[k]; + let uv = d.at(*ti, *k).expect("seen"); + (c.pose, [uv[0] / c.f, uv[1] / c.f], c) + }) + .collect(); + let x = match kind { + Kind::Tripod => normalize(views.iter().fold([0.0; 3], |a, (p, o, _)| add(a, normalize(mtv(&p.r, [o[0], o[1], 1.0]))))), + Kind::General => { + let v: Vec<(Pose, [f64; 2])> = views.iter().map(|(p, o, _)| (*p, *o)).collect(); + let x = geometry::triangulate(&v)?; + // Enough parallax between the most separated views. + let (a, b) = (views[0].0.c, views[views.len() - 1].0.c); + if geometry::ray_angle(a, b, x).to_degrees() < min_angle_deg { + return None; + } + x + } + }; + for (k, (_, _, c)) in ks.iter().zip(&views) { + let uv = d.at(*ti, *k).expect("seen"); + let r = bundle::residual(c, x, uv)?; + if r[0] * r[0] + r[1] * r[1] > thr2 { + return None; + } + } + Some((*ti, x)) + }) + .collect(); + self.pts.extend(new); + } + + /// Resect frame `k` from the existing points starting at `init` (pose, and the focal + /// length when `per_frame_f`). Returns false when too few points agree. + fn register(&mut self, d: &Data, k: usize, init: BaCam, cfg: &Cfg) -> bool { + let mut pts = vec![]; + let mut obs = vec![]; + let mut ids = vec![]; + for (ti, uv) in &d.by_frame[k] { + if let Some(x) = self.pts.get(ti) { + obs.push(BaObs { cam: 0, pt: pts.len() as u32, uv: *uv }); + pts.push(*x); + ids.push(*ti); + } + } + if obs.len() < 6 { + return false; + } + let thr2 = d.inlier_thr().powi(2); + let opts = |huber: f64| BaOpts { + focal: if cfg.per_frame_f { FocalMode::PerCamera(0.0) } else { FocalMode::Fixed }, + fix_points: true, + fix_centers: cfg.kind == Kind::Tripod, + fixed: vec![false], + huber, + max_iter: 40, + }; + let mut best: Option<(usize, BaCam)> = None; + // Robust start (large Huber threshold), then tighten; from the prediction and, failing + // that, from the nearest registered camera. + let nearest = self.cams.iter().min_by_key(|(c, _)| c.abs_diff(k)).map(|(_, c)| *c); + for start in std::iter::once(init).chain(nearest) { + if best.as_ref().is_some_and(|b| b.0 as f64 >= 0.8 * obs.len() as f64) { + break; + } + let mut cams = vec![start]; + bundle::adjust(&mut cams, &mut pts.clone(), &obs, &opts(d.inlier_thr() * 4.0)); + bundle::adjust(&mut cams, &mut pts.clone(), &obs, &opts(d.huber())); + let inl = obs.iter().filter(|o| bundle::residual(&cams[0], pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2)).count(); + if best.as_ref().is_none_or(|b| inl > b.0) { + best = Some((inl, cams[0])); + } + } + let Some((inl, cam)) = best else { return false }; + if inl < 6 || (inl as f64) < 0.5 * obs.len() as f64 { + return false; + } + // Refine on the inliers only. + let keep: Vec = + obs.iter().filter(|o| bundle::residual(&cam, pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2)).copied().collect(); + let mut cams = vec![cam]; + bundle::adjust(&mut cams, &mut pts.clone(), &keep, &opts(d.huber())); + for o in &obs { + let ok = bundle::residual(&cams[0], pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2); + if !ok { + self.outliers.insert((ids[o.pt as usize], k)); + } + } + self.cams.insert(k, cams[0]); + true + } + + /// How well the reconstruction explains every observation on its frames: the mean of the + /// reprojection error truncated at the inlier threshold, an observation without a point + /// counting as the threshold. + fn score(&self, d: &Data) -> f64 { + let thr = d.inlier_thr(); + let mut s = 0.0; + let mut n = 0usize; + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + n += 1; + s += match self.pts.get(ti).and_then(|x| bundle::residual(c, *x, *uv)) { + Some(r) => r[0].hypot(r[1]).min(thr), + None => thr, + }; + } + } + if n > 0 { s / n as f64 } else { f64::INFINITY } + } + + /// Average reprojection error (pixels) and observation count over the inliers. + fn error(&self, d: &Data) -> (f64, usize) { + let mut s = 0.0; + let mut n = 0; + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + if let Some(r) = bundle::residual(c, *x, *uv) { + s += r[0].hypot(r[1]); + n += 1; + } + } + } + (if n > 0 { s / n as f64 } else { f64::INFINITY }, n) + } +} + +/// Interpolated / extrapolated starting camera for frame `k` from the registered ones. +fn predict(r: &Recon, k: usize) -> Option { + let before = r.cams.range(..k).next_back().map(|(a, b)| (*a, *b)); + let after = r.cams.range(k + 1..).next().map(|(a, b)| (*a, *b)); + match (before, after) { + (Some((ka, a)), Some((kb, b))) => { + let t = (k - ka) as f64 / (kb - ka) as f64; + Some(BaCam { pose: Pose { r: slerp(&a.pose.r, &b.pose.r, t), c: add(scale(a.pose.c, 1.0 - t), scale(b.pose.c, t)) }, f: a.f * (b.f / a.f).powf(t) }) + } + (Some((ka, a)), None) => { + // Constant velocity from the two previous cameras. + match r.cams.range(..ka).next_back() { + Some((kp, p)) => { + let t = (k - ka) as f64 / (ka - kp) as f64; + let dr = log_rot(&mm(&a.pose.r, &mt(&p.pose.r))); + let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &a.pose.r); + let c = add(a.pose.c, scale(sub(a.pose.c, p.pose.c), t.min(2.0))); + Some(BaCam { pose: Pose { r: rr, c }, f: a.f }) + } + None => Some(a), + } + } + (None, Some((kb, b))) => match r.cams.range(kb + 1..).next() { + Some((kn, nx)) => { + let t = (kb - k) as f64 / (kn - kb) as f64; + let dr = log_rot(&mm(&b.pose.r, &mt(&nx.pose.r))); + let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &b.pose.r); + let c = add(b.pose.c, scale(sub(b.pose.c, nx.pose.c), t.min(2.0))); + Some(BaCam { pose: Pose { r: rr, c }, f: b.f }) + } + None => Some(b), + }, + (None, None) => None, + } +} + +/// Shared tracks of frames `a` and `b`: (track index, pixels in a, pixels in b). +fn shared(d: &Data, a: usize, b: usize) -> Vec<(u32, [f64; 2], [f64; 2])> { + d.by_frame[a].iter().filter_map(|(ti, p)| d.at(*ti, b).map(|q| (*ti, *p, q))).collect() +} + +/// Two-view initialisations of frames `a`, `b` at focal length `f`: the second camera and the +/// triangulated points (the first camera is the identity). +fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> Vec<(Pose, HashMap)> { + let sh = shared(d, a, b); + if sh.len() < 12 { + return vec![]; + } + let x1: Vec<[f64; 2]> = sh.iter().map(|s| [s.1[0] / f, s.1[1] / f]).collect(); + let x2: Vec<[f64; 2]> = sh.iter().map(|s| [s.2[0] / f, s.2[1] / f]).collect(); + let thr = 2.0 * d.noise / f; + let mut cands: Vec = vec![]; + if use_e && let Some((e, inl)) = geometry::ransac_essential(&x1, &x2, thr, 400, (a * 7919 + b) as u64) { + let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip(); + if let Some((pose, _)) = geometry::select_pose(&geometry::decompose_essential(&e), &p1, &p2) { + cands.push(pose); + } + } + if use_h { + let (s1, s2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = sh.iter().map(|s| (s.1, s.2)).unzip(); + if let Some(fit) = ransac(Model::Homography, &s1, &s2, 2.0 * d.noise, 500, (a * 31 + b) as u64) { + let h = fit.h.0; + // Normalised: K⁻¹ H K with K = diag(f, f, 1). + let hn = [[h[0][0], h[0][1], h[0][2] / f], [h[1][0], h[1][1], h[1][2] / f], [h[2][0] * f, h[2][1] * f, h[2][2]]]; + let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| fit.inliers[*i]).map(|i| (x1[i], x2[i])).unzip(); + for (r, td, _) in geometry::decompose_homography(&hn, &p1, &p2) { + if norm(td) > 1e-6 { + cands.push(Pose::from_rt(r, normalize(td))); + } + } + } + } + let thr2 = (d.inlier_thr() / f).powi(2); + cands + .into_iter() + .filter_map(|pose| { + let mut pts = HashMap::new(); + for (i, s) in sh.iter().enumerate() { + let Some(x) = geometry::triangulate(&[(Pose::IDENTITY, x1[i]), (pose, x2[i])]) else { continue }; + let ok = geometry::reproj2(&Pose::IDENTITY, x, x1[i]).is_some_and(|e| e < thr2) + && geometry::reproj2(&pose, x, x2[i]).is_some_and(|e| e < thr2) + && geometry::ray_angle([0.0; 3], pose.c, x).to_degrees() > 0.3; + if ok { + pts.insert(s.0, x); + } + } + (pts.len() >= 10 && pts.len() * 3 >= sh.len()).then_some((pose, pts)) + }) + .collect() +} + +/// Grow a reconstruction over keyframes `kfs` from an initial pair (`kfs[0]`, `kfs[j]`). +fn grow(d: &Data, kfs: &[usize], j: usize, init: &(Pose, HashMap), f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let (a, b) = (kfs[0], kfs[j]); + let mut r = Recon::default(); + r.cams.insert(a, BaCam { pose: Pose::IDENTITY, f }); + r.cams.insert(b, BaCam { pose: init.0, f }); + r.pts = init.1.clone(); + let inner = |r: &mut Recon, frames: &[usize]| { + let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed }; + r.adjust(d, frames, cfg, focal, 10); + r.reject(d); + }; + inner(&mut r, &[a, b]); + let order: Vec = kfs[1..j].iter().chain(&kfs[j + 1..]).copied().collect(); + let mut done = 0; + for k in order { + if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return None; + } + let start = predict(&r, k)?; + if !r.register(d, k, start, cfg) { + // A failed keyframe ends the reconstruction (a later frame is too far from the known + // points); the caller resects the rest from what is known. + break; + } + r.triangulate(d, cfg.kind, 1.0, 1.0); + done += 1; + let frames: Vec = r.cams.keys().copied().collect(); + if done % 3 == 0 || frames.len() <= 6 { + inner(&mut r, &frames); + } + } + let frames: Vec = r.cams.keys().copied().collect(); + inner(&mut r, &frames); + (r.cams.len() >= 2 && r.pts.len() >= 8).then_some(r) +} + +/// Tripod reconstruction over keyframes `kfs` at focal length `f`. +fn grow_tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let mut r = Recon::default(); + r.cams.insert(kfs[0], BaCam { pose: Pose::IDENTITY, f }); + let ray = |p: [f64; 2], f: f64| normalize([p[0] / f, p[1] / f, 1.0]); + for w in kfs.windows(2) { + if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return None; + } + let (a, b) = (w[0], w[1]); + let sh = shared(d, a, b); + if sh.len() < 6 { + break; + } + let ra: Vec = sh.iter().map(|s| ray(s.1, f)).collect(); + let rb: Vec = sh.iter().map(|s| ray(s.2, f)).collect(); + let mut rel = geometry::procrustes(&ra, &rb)?; + // One round of outlier trimming. + let errs: Vec = ra.iter().zip(&rb).map(|(x, y)| norm(sub(*y, mv(&rel, *x)))).collect(); + let mut sorted = errs.clone(); + sorted.sort_by(f64::total_cmp); + let lim = (sorted[sorted.len() / 2] * 3.0).max(d.inlier_thr() / f); + let (ia, ib): (Vec, Vec) = ra.iter().zip(&rb).zip(&errs).filter(|(_, e)| **e <= lim).map(|((x, y), _)| (*x, *y)).unzip(); + if ia.len() >= 4 { + rel = geometry::procrustes(&ia, &ib)?; + } + let pa = r.cams[&a].pose.r; + r.cams.insert(b, BaCam { pose: Pose { r: mm(&rel, &pa), c: [0.0; 3] }, f }); + } + r.triangulate(d, Kind::Tripod, 0.0, 20.0); + let frames: Vec = r.cams.keys().copied().collect(); + let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed }; + r.adjust(d, &frames, cfg, focal, 15); + r.reject(d); + r.triangulate(d, Kind::Tripod, 0.0, 1.0); + r.adjust(d, &frames, cfg, focal, 15); + (r.cams.len() >= 2 && r.pts.len() >= 8).then_some(r) +} + +/// Focal length (pixels) of a horizontal angle of view. +fn focal_of(w: f64, hfov: f64) -> f64 { + w * 0.5 / (hfov.clamp(1.0, 170.0).to_radians() * 0.5).tan() +} + +struct Attempt { + recon: Recon, + err: f64, + method: SolveMethod, +} + +/// Choose the initial pair: the first keyframe and the later keyframe (among the next few) +/// that keeps the most points while having enough parallax. +fn best_init(d: &Data, kfs: &[usize], f: f64, method: SolveMethod) -> Option<(usize, (Pose, HashMap), bool)> { + let (use_e, use_h) = match method { + SolveMethod::Typical => (true, false), + SolveMethod::MostlyFlat => (false, true), + _ => (true, true), + }; + let mut best: Option<(f64, usize, (Pose, HashMap), bool)> = None; + for j in 1..kfs.len().min(6) { + let cands = init_pair(d, kfs[0], kfs[j], f, use_e, false); + let hc = if use_h { init_pair(d, kfs[0], kfs[j], f, false, true) } else { vec![] }; + for (c, is_h) in cands.into_iter().map(|c| (c, false)).chain(hc.into_iter().map(|c| (c, true))) { + // Score: points × median triangulation angle (capped). + let mut ang: Vec = c.1.values().map(|x| geometry::ray_angle([0.0; 3], c.0.c, *x).to_degrees()).collect(); + ang.sort_by(f64::total_cmp); + let med = ang.get(ang.len() / 2).copied().unwrap_or(0.0); + let score = c.1.len() as f64 * med.min(4.0); + if best.as_ref().is_none_or(|b| score > b.0) { + best = Some((score, j, c, is_h)); + } + } + } + best.map(|b| (b.1, b.2, b.3)) +} + +/// All initial candidates for the pair chosen by [`best_init`] (E and H decompositions). +fn init_candidates(d: &Data, kfs: &[usize], f: f64, method: SolveMethod) -> Vec<(usize, (Pose, HashMap), bool)> { + let Some((j, first, is_h)) = best_init(d, kfs, f, method) else { return vec![] }; + let mut out = vec![(j, first, is_h)]; + if matches!(method, SolveMethod::Auto | SolveMethod::MostlyFlat) { + for c in init_pair(d, kfs[0], kfs[j], f, false, true) { + if out.iter().all(|o| norm(log_rot(&mm(&o.1.0.r, &mt(&c.0.r)))) > 1e-3 || norm(sub(o.1.0.c, c.0.c)) > 1e-3) { + out.push((j, c, true)); + } + } + } + out +} + +/// A general (translating camera) reconstruction of keyframes `kfs` at focal length `f`. +fn general(d: &Data, kfs: &[usize], f: f64, method: SolveMethod, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let cands = init_candidates(d, kfs, f, method); + let res: Vec = cands + .par_iter() + .filter_map(|(j, init, is_h)| { + let r = grow(d, kfs, *j, init, f, cfg, cancel)?; + let (_, n) = r.error(d); + // Prefer reconstructions covering more keyframes. + let cover = r.cams.len() as f64 / kfs.len() as f64; + (n > 0).then(|| Attempt { err: r.score(d) / cover.max(0.05), recon: r, method: if *is_h { SolveMethod::MostlyFlat } else { SolveMethod::Typical } }) + }) + .collect(); + res.into_iter().min_by(|a, b| a.err.total_cmp(&b.err)) +} + +fn tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let r = grow_tripod(d, kfs, f, cfg, cancel)?; + let cover = r.cams.len() as f64 / kfs.len() as f64; + Some(Attempt { err: r.score(d) / cover.max(0.05), recon: r, method: SolveMethod::TripodPan }) +} + +/// Search the focal length minimising `eval(f)` (a reprojection error): a log-spaced scan of +/// horizontal angles of view, then golden-section refinement around the best. +fn search_focal(w: f64, eval: &(dyn Fn(f64) -> f64 + Sync)) -> f64 { + let fovs = [8.0, 12.0, 16.0, 20.0, 25.0, 30.0, 36.0, 43.0, 51.0, 60.0, 70.0, 82.0, 96.0, 112.0]; + let fs: Vec = fovs.iter().map(|a| focal_of(w, *a)).collect(); + let errs: Vec = fs.par_iter().map(|f| eval(*f)).collect(); + if debug() { + eprintln!("focal scan: {:?}", fs.iter().zip(&errs).map(|(f, e)| format!("{f:.0}:{e:.3}")).collect::>()); + } + let i = (0..fs.len()).min_by(|a, b| errs[*a].total_cmp(&errs[*b])).unwrap_or(5); + let (mut lo, mut hi) = (fs[(i + 1).min(fs.len() - 1)].ln(), fs[i.saturating_sub(1)].ln()); + if lo > hi { + std::mem::swap(&mut lo, &mut hi); + } + let g = 0.618_033_988_75; + let (mut x1, mut x2) = (hi - g * (hi - lo), lo + g * (hi - lo)); + let (mut e1, mut e2) = rayon::join(|| eval(x1.exp()), || eval(x2.exp())); + for _ in 0..7 { + if e1 < e2 { + hi = x2; + x2 = x1; + e2 = e1; + x1 = hi - g * (hi - lo); + e1 = eval(x1.exp()); + } else { + lo = x1; + x1 = x2; + e1 = e2; + x2 = lo + g * (hi - lo); + e2 = eval(x2.exp()); + } + } + let best = if e1 < e2 { (x1, e1) } else { (x2, e2) }; + if best.1 <= errs[i] { best.0.exp() } else { fs[i] } +} + +/// Solve the camera of a tracked clip. +pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBool>) -> Result { + let d = Data::new(tracks, &s.deleted); + if d.n < 2 { + return err("the clip must be at least two frames long"); + } + let kfs = keyframes(&d); + if kfs.len() < 2 { + return err("not enough features were tracked to solve the camera"); + } + let cancelled = || cancel.is_some_and(|c| c.load(Ordering::Relaxed)); + let fixed_f = (s.shot == ShotType::SpecifyAngle).then(|| focal_of(d.w, if s.hfov > 0.0 { s.hfov } else { 40.0 })); + let variable = s.shot == ShotType::VariableZoom; + let gcfg = Cfg { kind: Kind::General, focal: FocalMode::Fixed, per_frame_f: false }; + let tcfg = Cfg { kind: Kind::Tripod, ..gcfg }; + // The focal-length search runs on the first keyframes only. + let mini: Vec = kfs.iter().copied().take(10).collect(); + let try_general = matches!(s.method, SolveMethod::Auto | SolveMethod::Typical | SolveMethod::MostlyFlat); + let try_tripod = matches!(s.method, SolveMethod::Auto | SolveMethod::TripodPan); + + let mut results: Vec<(Attempt, f64)> = vec![]; + if try_general { + let f = match fixed_f { + Some(f) => f, + None => { + search_focal(d.w, &|f| general(&d, &mini, f, s.method, &Cfg { per_frame_f: variable, ..gcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)) + } + }; + if cancelled() { + return err("cancelled"); + } + if let Some(a) = general(&d, &kfs, f, s.method, &Cfg { per_frame_f: variable, ..gcfg }, cancel) { + results.push((a, f)); + } + } + if try_tripod && !cancelled() { + let f = match fixed_f { + Some(f) => f, + None => search_focal(d.w, &|f| tripod(&d, &mini, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)), + }; + if let Some(a) = tripod(&d, &kfs, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel) { + results.push((a, f)); + } + } + if cancelled() { + return err("cancelled"); + } + // Auto Detect keeps the tripod model when it explains the tracks about as well. + let pick = { + let g = results.iter().position(|r| r.0.method != SolveMethod::TripodPan); + let t = results.iter().position(|r| r.0.method == SolveMethod::TripodPan); + match (g, t) { + (Some(g), Some(t)) => { + let (eg, et) = (results[g].0.err, results[t].0.err); + // A general solve whose camera hardly moves relative to the scene depth is a pan. + let rg = &results[g].0.recon; + let cs: Vec = rg.cams.values().map(|c| c.pose.c).collect(); + let base = cs.iter().flat_map(|a| cs.iter().map(move |b| norm(sub(*a, *b)))).fold(0.0, f64::max); + let c0 = rg.cams.values().next().expect("camera"); + let mut depth: Vec = rg.pts.values().map(|x| norm(sub(*x, c0.pose.c))).collect(); + depth.sort_by(f64::total_cmp); + let med = depth.get(depth.len() / 2).copied().unwrap_or(1.0); + if debug() { + eprintln!("auto: general {eg:.4} tripod {et:.4} baseline/depth {:.4}", base / med); + } + if et <= (eg * 1.25).max(eg + 0.05 * d.noise) || base / med < 0.01 { t } else { g } + } + (Some(g), None) => g, + (None, Some(t)) => t, + (None, None) => return err("the camera could not be solved (not enough parallax or features)"), + } + }; + if debug() { + for (a, f) in &results { + eprintln!("candidate {:?}: err {:.4} f {f:.1} cams {} pts {}", a.method, a.err, a.recon.cams.len(), a.recon.pts.len()); + } + } + let (att, _) = results.swap_remove(pick); + let kind = if att.method == SolveMethod::TripodPan { Kind::Tripod } else { Kind::General }; + let cfg = Cfg { kind, focal: FocalMode::Fixed, per_frame_f: variable }; + let mut r = att.recon; + let focal = match (fixed_f, variable) { + (Some(_), _) => FocalMode::Fixed, + (None, true) => FocalMode::PerCamera(F_PRIOR), + (None, false) => FocalMode::Shared, + }; + // Keyframes: adjust with the focal length free, reject, re-triangulate, adjust again. + let kf_frames: Vec = r.cams.keys().copied().collect(); + r.adjust(&d, &kf_frames, &cfg, focal, 40); + r.reject(&d); + r.triangulate(&d, kind, 1.0, 1.0); + r.adjust(&d, &kf_frames, &cfg, focal, 40); + if cancelled() { + return err("cancelled"); + } + // Every other frame: resect from the points (nearest registered camera first). + let mut todo: Vec = (0..d.n).filter(|k| !r.cams.contains_key(k)).collect(); + todo.sort_by_key(|k| r.cams.keys().map(|c| c.abs_diff(*k)).min().unwrap_or(usize::MAX)); + let base = r.clone(); + let resected: Vec<(usize, Option)> = todo + .par_iter() + .map(|k| { + let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() }; + let ok = predict(&local, *k).is_some_and(|start| local.register(&d, *k, start, &cfg)); + (*k, if ok { local.cams.get(k).copied() } else { None }) + }) + .collect(); + for (k, c) in resected { + if let Some(c) = c { + r.cams.insert(k, c); + } + } + if cancelled() { + return err("cancelled"); + } + // Points for the tracks that only lived between keyframes, then the final adjustment over + // every frame (when that stays affordable). + r.triangulate(&d, kind, 1.0, 1.0); + let all: Vec = r.cams.keys().copied().collect(); + r.reject(&d); + if all.len() <= 160 { + r.adjust(&d, &all, &cfg, focal, 25); + r.reject(&d); + } else { + // Points only (cameras fixed) would be cheap but the keyframe solution is already + // adjusted; refine each non-keyframe once more against the final points. + let base = r.clone(); + let refined: Vec<(usize, Option)> = all + .par_iter() + .filter(|k| !kf_frames.contains(k)) + .map(|k| { + let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() }; + let start = local.cams[k]; + let ok = local.register(&d, *k, start, &cfg); + (*k, if ok { local.cams.get(k).copied() } else { None }) + }) + .collect(); + for (k, c) in refined.into_iter() { + if let Some(c) = c { + r.cams.insert(k, c); + } + } + r.reject(&d); + } + // Method Used: the chosen model; Auto Detect calls a general solve "Mostly Flat Scene" when + // its points are close to one plane. + let method = match (s.method, att.method) { + (_, SolveMethod::TripodPan) => SolveMethod::TripodPan, + (SolveMethod::Auto, _) => { + let p: Vec = r.pts.values().copied().collect(); + let flat = cov_ratio(&p) < 0.08; + if flat { SolveMethod::MostlyFlat } else { SolveMethod::Typical } + } + (m, _) => m, + }; + Ok(finish(&d, tracks, s, r, method, kind)) +} + +/// Thickness of a point cloud: √(smallest / middle eigenvalue of its covariance). +fn cov_ratio(p: &[V3]) -> f64 { + if p.len() < 3 { + return 1.0; + } + let k = 1.0 / p.len() as f64; + let c = scale(p.iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [0.0; 9]; + for q in p { + let d = sub(*q, c); + for i in 0..3 { + for j in 0..3 { + cov[i * 3 + j] += d[i] * d[j] * k; + } + } + } + let (vals, _) = sym_eigen(&cov, 3); + (vals[0].max(0.0) / vals[1].max(1e-300)).sqrt() +} + +/// Canonical frame, per-frame output (unsolved frames interpolated), points and errors. +fn finish(d: &Data, tracks: &CameraTracks, s: &SolveSettings, r: Recon, method: SolveMethod, kind: Kind) -> CameraSolve { + let (k0, c0) = r.cams.iter().next().map(|(k, c)| (*k, *c)).expect("solved"); + // Scale: median depth of the points seen by the first solved camera is 1. + let mut depths: Vec = d.by_frame[k0].iter().filter_map(|(ti, _)| r.pts.get(ti)).map(|x| c0.pose.to_cam(*x)[2]).filter(|z| *z > 0.0).collect(); + depths.sort_by(f64::total_cmp); + let sc = match kind { + Kind::Tripod => 1.0, + Kind::General => depths.get(depths.len() / 2).copied().filter(|v| *v > 1e-12).unwrap_or(1.0), + }; + let r0 = c0.pose.r; + let cc = c0.pose.c; + let to_world = |x: V3| scale(mv(&r0, sub(x, cc)), 1.0 / sc); + let mut frames = vec![SolvedFrame::default(); d.n]; + for (k, c) in &r.cams { + let rr = mm(&c.pose.r, &mt(&r0)); + frames[*k] = SolvedFrame { rot: log_rot(&rr), center: to_world(c.pose.c), focal: c.f, solved: true }; + } + // Hold / interpolate unsolved frames so the camera path stays continuous. + let solved: Vec = (0..d.n).filter(|k| frames[*k].solved).collect(); + for k in 0..d.n { + if frames[k].solved { + continue; + } + let a = solved.iter().rev().find(|x| **x < k).copied(); + let b = solved.iter().find(|x| **x > k).copied(); + frames[k] = match (a, b) { + (Some(a), Some(b)) => { + let t = (k - a) as f64 / (b - a) as f64; + let (fa, fb) = (frames[a], frames[b]); + let rr = slerp(&fa.r(), &fb.r(), t); + SolvedFrame { + rot: log_rot(&rr), + center: add(scale(fa.center, 1.0 - t), scale(fb.center, t)), + focal: fa.focal + (fb.focal - fa.focal) * t, + solved: false, + } + } + (Some(a), None) => SolvedFrame { solved: false, ..frames[a] }, + (None, Some(b)) => SolvedFrame { solved: false, ..frames[b] }, + (None, None) => frames[k], + }; + } + let mut points = vec![]; + let (mut esum, mut en) = (0.0, 0usize); + let mut ids: Vec<&u32> = r.pts.keys().collect(); + ids.sort(); + for ti in ids { + let x = r.pts[ti]; + let (id, start, pts) = &d.tracks[*ti as usize]; + let (mut s1, mut n1) = (0.0, 0); + for (i, uv) in pts.iter().enumerate() { + let k = start + i; + if r.outliers.contains(&(*ti, k)) { + continue; + } + if let Some(c) = r.cams.get(&k) + && let Some(res) = bundle::residual(c, x, *uv) + { + s1 += res[0].hypot(res[1]); + n1 += 1; + } + } + if n1 < 2 { + continue; + } + esum += s1; + en += n1; + points.push(SolvedPoint { id: *id, pos: to_world(x), error: (s1 / n1 as f64) as f32, first: *start as u32, last: (start + pts.len() - 1) as u32 }); + } + CameraSolve { + version: 1, + size: tracks.size, + start: tracks.start, + frame_duration: tracks.frame_duration, + shot: s.shot, + method_used: method, + average_error: if en > 0 { esum / en as f64 } else { 0.0 }, + frames, + points, + ground: None, + } +} diff --git a/crates/track/src/camtrack/tests.rs b/crates/track/src/camtrack/tests.rs new file mode 100644 index 00000000..258161f9 --- /dev/null +++ b/crates/track/src/camtrack/tests.rs @@ -0,0 +1,360 @@ +//! Synthetic scenes: random 3D points (or a plane) seen by a known moving camera with a known +//! focal length, projected directly (with sub-pixel noise) into feature tracks. + +use super::geometry::Rng; +use super::linalg::*; +use super::*; + +const W: f64 = 1280.0; +const H: f64 = 720.0; + +struct Truth { + /// Per frame: (R world→cam, centre, focal). + cams: Vec<(M3, V3, f64)>, +} + +fn noise(rng: &mut Rng) -> f64 { + // Sum of uniforms ≈ Gaussian, σ ≈ 0.1 px. + let u: f64 = (0..4).map(|_| (rng.next_u64() % 10_000) as f64 / 10_000.0 - 0.5).sum(); + u * 0.17 +} + +/// Project `pts` through the cameras into tracks (a track ends when its point leaves the frame; +/// re-entering starts a new track). +fn tracks_of(pts: &[V3], t: &Truth, seed: u64, noise_on: bool) -> CameraTracks { + let mut rng = Rng(seed); + let mut tracks: Vec = vec![]; + for x in pts { + let mut cur: Option = None; + for (k, (r, c, f)) in t.cams.iter().enumerate() { + let p = mv(r, sub(*x, *c)); + let uv = (p[2] > 0.1).then(|| [f * p[0] / p[2] + W / 2.0, f * p[1] / p[2] + H / 2.0]); + match uv.filter(|u| u[0] >= 2.0 && u[1] >= 2.0 && u[0] < W - 2.0 && u[1] < H - 2.0) { + Some(u) => { + let (nx, ny) = if noise_on { (noise(&mut rng), noise(&mut rng)) } else { (0.0, 0.0) }; + let q = [(u[0] + nx) as f32, (u[1] + ny) as f32]; + match &mut cur { + Some(tr) => tr.pts.push(q), + None => cur = Some(Track2D { id: 0, start: k as u32, pts: vec![q] }), + } + } + None => { + if let Some(tr) = cur.take() { + tracks.push(tr); + } + } + } + } + if let Some(tr) = cur.take() { + tracks.push(tr); + } + } + tracks.retain(|t| t.pts.len() >= 3); + for (i, tr) in tracks.iter_mut().enumerate() { + tr.id = i as u32; + } + CameraTracks { version: 1, size: [W, H], start: 0.0, frame_duration: 1.0 / 30.0, frames: t.cams.len() as u32, factor: 1.0, detailed: false, tracks } +} + +fn random_points(n: usize, seed: u64, center: V3, ext: V3) -> Vec { + let mut rng = Rng(seed); + (0..n) + .map(|_| { + let mut u = || (rng.next_u64() % 100_000) as f64 / 100_000.0 - 0.5; + [center[0] + u() * ext[0], center[1] + u() * ext[1], center[2] + u() * ext[2]] + }) + .collect() +} + +/// A camera dollying sideways and forward while panning to keep the scene in view. +fn moving_camera(frames: usize, f: impl Fn(usize) -> f64) -> Truth { + let cams = (0..frames) + .map(|k| { + let t = k as f64 / (frames - 1) as f64; + let c = [-3.0 + 6.0 * t, -0.6 + 0.8 * t + 0.5 * (std::f64::consts::PI * t).sin(), -1.0 + 2.0 * t - 1.5 * (t - 0.5) * (t - 0.5)]; + // Look at the scene centre (0, 0, 10). + let fwd = normalize(sub([0.3 * t, 0.2, 10.0], c)); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + let down = cross(fwd, right); + let r = [right, down, fwd]; + // A little roll. + let roll = rodrigues([0.0, 0.0, 0.05 * (t * 6.0).sin()]); + (mm(&roll, &r), c, f(k)) + }) + .collect(); + Truth { cams } +} + +/// Align the solved centres to the true ones (similarity) and return the RMS path error +/// relative to the path length, plus the worst rotation error (degrees). +fn path_error(s: &CameraSolve, t: &Truth) -> (f64, f64) { + let solved: Vec = (0..s.frames.len()).filter(|k| s.frames[*k].solved).collect(); + let src: Vec = solved.iter().map(|k| s.frames[*k].center).collect(); + let dst: Vec = solved.iter().map(|k| t.cams[*k].1).collect(); + let sim = umeyama(&src, &dst).expect("alignment"); + let len = norm(sub(dst[0], dst[dst.len() - 1])).max(1e-9); + let rms = (src.iter().zip(&dst).map(|(a, b)| dot(sub(sim.apply(*a), *b), sub(sim.apply(*a), *b))).sum::() / src.len() as f64).sqrt(); + // Rotation: R_true ≈ R_solved · simᵀ. + let mut worst: f64 = 0.0; + for k in &solved { + let rs = mm(&s.frames[*k].r(), &mt(&sim.r)); + let e = norm(log_rot(&mm(&rs, &mt(&t.cams[*k].0)))).to_degrees(); + worst = worst.max(e); + } + (rms / len, worst) +} + +fn settings(shot: ShotType, method: SolveMethod) -> SolveSettings { + SolveSettings { shot, hfov: 0.0, method, deleted: vec![] } +} + +#[test] +fn fixed_angle_solve_recovers_path_and_focal() { + let f = 1100.0; + let t = moving_camera(40, |_| f); + let pts = random_points(400, 1, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 2, true); + let t0 = std::time::Instant::now(); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), None).expect("solve"); + eprintln!("fixed: {:?} err {:.3} px focal {:.1} in {:?}", s.method_used, s.average_error, s.focal(), t0.elapsed()); + assert!(s.frames.iter().all(|c| c.solved)); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.02, "focal {}", s.focal()); + let (pe, re) = path_error(&s, &t); + assert!(pe < 0.01, "path {pe}"); + assert!(re < 0.5, "rotation {re}"); + // Canonical frame: first camera at the origin. + assert!(norm(s.frames[0].center) < 1e-9 && norm(s.frames[0].rot) < 1e-9); + // Serde round trip. + let j = s.to_json(); + assert_eq!(CameraSolve::from_json(&j).unwrap().points.len(), s.points.len()); +} + +#[test] +fn auto_detect_picks_a_general_model_for_a_translating_camera() { + let f = 900.0; + let t = moving_camera(30, |_| f); + let pts = random_points(300, 3, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 4, true); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + assert_eq!(s.method_used, SolveMethod::Typical); + assert!(s.average_error < 0.5); + assert!((s.focal() - f).abs() / f < 0.02, "focal {}", s.focal()); +} + +#[test] +fn specified_angle_of_view_is_kept() { + let f = 1000.0; + let t = moving_camera(25, |_| f); + let pts = random_points(300, 5, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 6, true); + let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); + let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: vec![] }, None).unwrap(); + assert!((s.focal() - f).abs() < 1e-6); + assert!(s.average_error < 0.5); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.01, "{pe}"); +} + +#[test] +fn variable_zoom_recovers_per_frame_focal() { + let t = moving_camera(40, |k| 950.0 + 6.0 * k as f64); + let pts = random_points(450, 7, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]); + let tr = tracks_of(&pts, &t, 8, true); + let s = solve(&tr, &settings(ShotType::VariableZoom, SolveMethod::Typical), None).expect("solve"); + eprintln!("variable: err {:.3}", s.average_error); + assert!(s.average_error < 0.5); + for (k, c) in s.frames.iter().enumerate() { + let f = t.cams[k].2; + assert!((c.focal - f).abs() / f < 0.02, "frame {k}: {} vs {f}", c.focal); + } +} + +#[test] +fn tripod_pan_solve() { + let f = 1200.0; + let cams = (0..40) + .map(|k| { + let a = -0.4 + 0.8 * k as f64 / 39.0; + (mm(&rodrigues([0.0, 0.0, 0.02]), &mm(&rodrigues([0.05 * a, 0.0, 0.0]), &rodrigues([0.0, a, 0.0]))), [0.0; 3], f) + }) + .collect(); + let t = Truth { cams }; + // Points all around at varying distances (rotation only: depth is unobservable). + let pts = random_points(600, 9, [0.0, 0.0, 0.0], [60.0, 20.0, 60.0]).into_iter().filter(|p| p[2] > 3.0).collect::>(); + let tr = tracks_of(&pts, &t, 10, true); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("tripod: {:?} err {:.3} focal {:.1}", s.method_used, s.average_error, s.focal()); + assert_eq!(s.method_used, SolveMethod::TripodPan); + assert!(s.average_error < 0.5); + assert!((s.focal() - f).abs() / f < 0.02, "{}", s.focal()); + for (k, c) in s.frames.iter().enumerate() { + assert!(norm(c.center) < 1e-9); + // Relative rotation to frame 0 matches. + let rel_s = mm(&c.r(), &mt(&s.frames[0].r())); + let rel_t = mm(&t.cams[k].0, &mt(&t.cams[0].0)); + assert!(norm(log_rot(&mm(&rel_s, &mt(&rel_t)))).to_degrees() < 0.2); + } +} + +#[test] +fn mostly_flat_scene_and_ground_plane() { + let f = 1000.0; + // A textured floor y = 2 (below the camera), seen by a camera moving forward and sideways. + let cams = (0..40) + .map(|k| { + let t = k as f64 / 39.0; + let c = [-2.0 + 4.0 * t, -0.3 * (std::f64::consts::PI * t).sin(), 2.0 * t]; + let fwd = normalize([0.1 - 0.2 * t, 0.45, 1.0]); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + let down = cross(fwd, right); + ([right, down, fwd], c, f) + }) + .collect(); + let t = Truth { cams }; + let mut rng = Rng(11); + let pts: Vec = (0..500).map(|_| [((rng.next_u64() % 2000) as f64 / 100.0) - 10.0, 2.0, 2.0 + (rng.next_u64() % 2000) as f64 / 100.0]).collect(); + let tr = tracks_of(&pts, &t, 12, true); + for method in [SolveMethod::MostlyFlat, SolveMethod::Auto] { + let s = solve(&tr, &settings(ShotType::FixedAngle, method), None).expect("solve"); + eprintln!("flat {method:?}: {:?} err {:.3} focal {:.1}", s.method_used, s.average_error, s.focal()); + assert_eq!(s.method_used, SolveMethod::MostlyFlat); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.02, "{}", s.focal()); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.01, "{pe}"); + } + // Ground plane from the solved (coplanar) points: every point lies on it. + let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); + let mut s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::MostlyFlat, deleted: vec![] }, None).expect("solve"); + let ids: Vec = s.visible(0).map(|p| p.id).take(12).collect(); + let g = s.ground_from(&ids, 0).expect("ground"); + let spread = s.points.iter().map(|p| dot(sub(p.pos, g.origin), g.normal).abs()).fold(0.0, f64::max); + let depth = norm(sub(s.frames[0].center, g.origin)); + assert!(spread < 0.01 * depth, "{spread}"); + // Up points towards the camera. + assert!(dot(sub(s.frames[0].center, g.origin), g.normal) > 0.0); + s.ground = Some(g); + let w = s.world([W, H], 1000.0); + // In the world, ground points have y ≈ 0 and the cameras are above (y < 0). + for p in s.points.iter().take(20) { + assert!(w.apply(p.pos)[1].abs() < 0.01 * 1000.0 * depth, "{:?}", w.apply(p.pos)); + } + assert!(w.apply(s.frames[0].center)[1] < 0.0); + assert!(norm(w.apply(g.origin)) < 1e-6); +} + +#[test] +fn deleted_tracks_are_ignored_and_cancel_stops() { + let t = moving_camera(20, |_| 1000.0); + let pts = random_points(250, 13, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 14, true); + let deleted: Vec = (0..40).collect(); + let hfov = 2.0 * (W * 0.5 / 1000.0f64).atan().to_degrees(); + let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: deleted.clone() }, None).unwrap(); + assert!(s.points.iter().all(|p| !deleted.contains(&p.id))); + let stop = std::sync::atomic::AtomicBool::new(true); + let r = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), Some(&stop)); + assert!(r.is_err()); +} + +/// End to end: rendered frames of a textured scene through the KLT feature tracker. +#[test] +fn rendered_scene_end_to_end() { + use effectcraft_raster::Image; + let (w, h) = (480u32, 270u32); + let f = 420.0; + // Three textured planes at different depths (a floor, a back wall and a box face). + fn tex(u: f64, v: f64) -> f32 { + fn hsh(i: i64, j: i64) -> f64 { + let mut x = (i.wrapping_mul(73856093) ^ j.wrapping_mul(19349663)) as u64; + x ^= x >> 13; + x = x.wrapping_mul(0x5bd1e995); + x ^= x >> 15; + (x % 1000) as f64 / 1000.0 + } + fn noise(x: f64, y: f64) -> f64 { + let (i, j) = (x.floor(), y.floor()); + let (tx, ty) = (x - i, y - j); + let (sx, sy) = (tx * tx * (3.0 - 2.0 * tx), ty * ty * (3.0 - 2.0 * ty)); + let (i, j) = (i as i64, j as i64); + let a = hsh(i, j) + (hsh(i + 1, j) - hsh(i, j)) * sx; + let b = hsh(i, j + 1) + (hsh(i + 1, j + 1) - hsh(i, j + 1)) * sx; + a + (b - a) * sy + } + (0.15 + 0.5 * noise(u * 4.0, v * 4.0) + 0.3 * noise(u * 11.0 + 7.0, v * 11.0 + 3.0)) as f32 + } + let frames = 24; + let t = Truth { + cams: (0..frames) + .map(|k| { + let s = k as f64 / (frames - 1) as f64; + let c = [-1.0 + 2.0 * s, -0.2, 0.5 * s]; + let fwd = normalize(sub([0.0, 0.3, 8.0], c)); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + ([right, cross(fwd, right), fwd], c, f) + }) + .collect(), + }; + // Planes: (point, normal, u axis, v axis). + let planes: [(V3, V3, V3, V3); 3] = [ + ([0.0, 1.5, 0.0], [0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]), + ([0.0, 0.0, 10.0], [0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0]), + ([-1.0, 0.0, 6.0], [0.3, 0.0, -1.0], [1.0, 0.0, 0.3], [0.0, 1.0, 0.0]), + ]; + let render = |k: usize| { + let (r, c, f) = t.cams[k]; + let mut img = Image::new(w, h); + for y in 0..h { + for x in 0..w { + let d = mtv(&r, [(x as f64 + 0.5 - w as f64 / 2.0) / f, (y as f64 + 0.5 - h as f64 / 2.0) / f, 1.0]); + let mut best = (f64::INFINITY, 0.0f32); + for (i, (p0, n, ua, va)) in planes.iter().enumerate() { + let den = dot(*n, d); + if den.abs() < 1e-9 { + continue; + } + let tt = dot(*n, sub(*p0, c)) / den; + let hit = add(c, scale(d, tt)); + // The box face is limited in extent. + if i == 2 && (dot(sub(hit, *p0), *ua).abs() > 1.2 || dot(sub(hit, *p0), *va).abs() > 1.0) { + continue; + } + if tt > 0.1 && tt < best.0 { + best = (tt, tex(dot(sub(hit, *p0), *ua) + 10.0 * i as f64, dot(sub(hit, *p0), *va))); + } + } + let v = best.1; + img.set(x, y, [v, v, v, 1.0]); + } + } + img + }; + let mut an = TrackAnalyzer::new([w as f64, h as f64], AnalyzeOpts::default()); + for k in 0..frames { + let img = render(k); + an.push(&crate::Frame::new(&img)); + } + let tr = an.finish(0.0, 1.0 / 24.0); + assert!(tr.tracks.len() > 100, "{}", tr.tracks.len()); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("rendered: {:?} err {:.3} focal {:.1} points {}", s.method_used, s.average_error, s.focal(), s.points.len()); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.05, "{}", s.focal()); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.03, "{pe}"); +} + +/// A long clip (300 frames, ~900 points): `cargo test -p effectcraft-track --release -- --ignored`. +#[test] +#[ignore] +fn perf_long_clip() { + let f = 1100.0; + let t = moving_camera(300, |_| f); + let pts = random_points(900, 21, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]); + let tr = tracks_of(&pts, &t, 22, true); + let t0 = std::time::Instant::now(); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("300 frames, {} tracks: {:?} err {:.3} focal {:.1} in {:?}", tr.tracks.len(), s.method_used, s.average_error, s.focal(), t0.elapsed()); + assert!(s.average_error < 0.5); +} diff --git a/crates/track/src/lib.rs b/crates/track/src/lib.rs index d7b239f5..bd4e21dc 100644 --- a/crates/track/src/lib.rs +++ b/crates/track/src/lib.rs @@ -18,6 +18,7 @@ //! Coordinates are *layer* pixels: the frame image's pixel `(i, j)` covers `[i, i+1) × [j, j+1)` //! after subtracting the frame's `offset` (see [`Frame`]). +pub mod camtrack; pub mod fit; pub mod klt; pub mod mask; From 94e1c86d89a866771f80ad1610afda9aa12e85d8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:10:48 -0400 Subject: [PATCH 03/13] IO-2/SHP-5/PRV-2: PSD import, SVG import + vector/text conversions, persistent disk cache - effectcraft-psd (L1): PSD/PSB reader from Adobe's published file format spec (layers, groups, masks, blend modes, text engine data, lfx2 effects, fill/adjustment layers, vector masks; 1/8/16/32-bit, RGB/CMYK/Gray/Lab/Indexed, raw/RLE/ZIP) and a minimal writer for fixtures. - File > Import: importAs footage|composition|compositionLayerSizes, layer (Choose Layer); groups -> precomps (pass-through collapses), text -> editable text, styles -> Layer Styles, adjustments -> adjustment layers with effects; UI asks the Import Kind for Photoshop files. - effectcraft-svg (L2): SVG parser (shapes, paths, transforms, use, CSS, gradients) and rasteriser; .svg imports as footage, continuously rasterised when the switch is on. - layer.create: shapesFromVector, shapesFromText, masksFromText, editableText (stub removed). - Disk cache (render::disk_cache): LZ4 content-keyed frames and layer buffers, LRU size limit from prefs, atomic writes, restart persistence; edit.purge clears it, cache.diskStats query; viewer frames read/write it and the timeline shows disk-only frames in blue. Co-Authored-By: Claude Opus 5.5 --- ATTRIBUTION.md | 2 + Cargo.lock | 44 + Cargo.toml | 2 + crates/engine/Cargo.toml | 3 + crates/engine/src/commands/create.rs | 139 +++ crates/engine/src/commands/edit.rs | 32 +- crates/engine/src/commands/file.rs | 98 +- crates/engine/src/commands/file_more.rs | 1 + crates/engine/src/commands/mod.rs | 2 + crates/engine/src/commands/stubs.rs | 1 - crates/engine/src/lib.rs | 12 + crates/engine/src/psd_import.rs | 634 ++++++++++++ crates/engine/src/session_settings.rs | 37 + crates/engine/src/tests_disk_cache.rs | 121 +++ crates/engine/src/tests_stubs.rs | 1 + crates/engine/src/tests_track.rs | 2 + crates/engine/src/tests_vector_import.rs | 237 +++++ crates/engine/src/vector.rs | 250 +++++ crates/gpu/src/tests.rs | 1 + crates/host/Cargo.toml | 4 + crates/host/tests/vector_import.rs | 163 +++ crates/lottie/src/import.rs | 1 + crates/lottie/src/tests.rs | 1 + crates/media/Cargo.toml | 2 + crates/media/src/convert.rs | 17 + crates/media/src/layered.rs | 62 ++ crates/media/src/lib.rs | 6 +- crates/media/src/pool.rs | 16 + crates/media/src/probe.rs | 7 +- crates/project/src/lib.rs | 15 + crates/psd/Cargo.toml | 13 + crates/psd/README.md | 17 + crates/psd/src/descriptor.rs | 430 ++++++++ crates/psd/src/engine_data.rs | 288 ++++++ crates/psd/src/lib.rs | 965 ++++++++++++++++++ crates/psd/src/read.rs | 86 ++ crates/psd/src/tests.rs | 189 ++++ crates/psd/src/write.rs | 491 +++++++++ crates/render/Cargo.toml | 1 + crates/render/src/audio.rs | 1 + crates/render/src/cache.rs | 62 +- crates/render/src/disk_cache.rs | 829 +++++++++++++++ crates/render/src/lib.rs | 29 +- crates/render/src/tests_audio_fx.rs | 1 + crates/render/src/tests_color.rs | 1 + crates/render/src/tests_frame_blend.rs | 1 + crates/svg/Cargo.toml | 18 + crates/svg/README.md | 26 + crates/svg/src/color.rs | 254 +++++ crates/svg/src/lib.rs | 949 +++++++++++++++++ crates/svg/src/pathdata.rs | 293 ++++++ crates/svg/src/raster.rs | 220 ++++ crates/svg/src/tests.rs | 108 ++ crates/svg/tests/fixtures/basic-shapes.svg | 15 + .../fixtures/basic-shapes.svg.attribution | 7 + crates/svg/tests/fixtures/paths-gradients.svg | 26 + .../fixtures/paths-gradients.svg.attribution | 7 + crates/ui-egui/src/frames.rs | 108 +- crates/ui-egui/src/lib.rs | 18 +- crates/ui-egui/src/menus.rs | 6 +- crates/ui-egui/src/panels/forms.rs | 20 + crates/ui-egui/src/panels/timeline.rs | 25 + crates/ui-egui/src/theme.rs | 3 + xtask/src/main.rs | 2 + 64 files changed, 7388 insertions(+), 34 deletions(-) create mode 100644 crates/engine/src/commands/create.rs create mode 100644 crates/engine/src/psd_import.rs create mode 100644 crates/engine/src/tests_disk_cache.rs create mode 100644 crates/engine/src/tests_vector_import.rs create mode 100644 crates/engine/src/vector.rs create mode 100644 crates/host/tests/vector_import.rs create mode 100644 crates/media/src/layered.rs create mode 100644 crates/psd/Cargo.toml create mode 100644 crates/psd/README.md create mode 100644 crates/psd/src/descriptor.rs create mode 100644 crates/psd/src/engine_data.rs create mode 100644 crates/psd/src/lib.rs create mode 100644 crates/psd/src/read.rs create mode 100644 crates/psd/src/tests.rs create mode 100644 crates/psd/src/write.rs create mode 100644 crates/render/src/disk_cache.rs create mode 100644 crates/svg/Cargo.toml create mode 100644 crates/svg/README.md create mode 100644 crates/svg/src/color.rs create mode 100644 crates/svg/src/lib.rs create mode 100644 crates/svg/src/pathdata.rs create mode 100644 crates/svg/src/raster.rs create mode 100644 crates/svg/src/tests.rs create mode 100644 crates/svg/tests/fixtures/basic-shapes.svg create mode 100644 crates/svg/tests/fixtures/basic-shapes.svg.attribution create mode 100644 crates/svg/tests/fixtures/paths-gradients.svg create mode 100644 crates/svg/tests/fixtures/paths-gradients.svg.attribution diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 9caf1c27..d0b936b8 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -36,6 +36,8 @@ iconography, images or other Adobe assets; open licences only; every asset attri | `crates/lottie/tests/fixtures/shapes-gradient.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (star, rectangle, gradient fill, dashed stroke, trim paths) | MIT OR Apache-2.0 | | `crates/lottie/tests/fixtures/precomp-matte-text.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (precomp, track matte, parented text) | MIT OR Apache-2.0 | | `crates/lottie/tests/fixtures/legacy-masks.json` | EffectCraft contributors | Original work: hand-written Lottie test fixture (legacy keyframes, masks) | MIT OR Apache-2.0 | +| `crates/svg/tests/fixtures/basic-shapes.svg` | EffectCraft contributors | Original work: hand-written SVG test fixture (basic shapes, groups, transforms) | MIT OR Apache-2.0 | +| `crates/svg/tests/fixtures/paths-gradients.svg` | EffectCraft contributors | Original work: hand-written SVG test fixture (path data, gradients, CSS, use) | MIT OR Apache-2.0 | ## Screenshots diff --git a/Cargo.lock b/Cargo.lock index 46f34e63..1d14bab9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1198,12 +1198,15 @@ dependencies = [ "effectcraft-lottie", "effectcraft-path", "effectcraft-project", + "effectcraft-psd", "effectcraft-raster", "effectcraft-render", + "effectcraft-svg", "effectcraft-text", "effectcraft-time", "effectcraft-track", "image", + "kurbo", "log", "rayon", "serde", @@ -1287,6 +1290,10 @@ dependencies = [ "effectcraft-expr", "effectcraft-media", "effectcraft-project", + "effectcraft-psd", + "effectcraft-raster", + "effectcraft-svg", + "effectcraft-time", "serde_json", ] @@ -1324,8 +1331,10 @@ name = "effectcraft-media" version = "0.1.0" dependencies = [ "effectcraft-project", + "effectcraft-psd", "effectcraft-raster", "effectcraft-render", + "effectcraft-svg", "effectcraft-time", "filmcraft-codecs", "filmcraft-color", @@ -1363,6 +1372,13 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "effectcraft-psd" +version = "0.1.0" +dependencies = [ + "miniz_oxide 0.8.9", +] + [[package]] name = "effectcraft-raster" version = "0.1.0" @@ -1387,11 +1403,24 @@ dependencies = [ "effectcraft-time", "kurbo", "log", + "lz4_flex", "rayon", "serde", "web-time", ] +[[package]] +name = "effectcraft-svg" +version = "0.1.0" +dependencies = [ + "effectcraft-geom", + "effectcraft-path", + "effectcraft-raster", + "kurbo", + "rayon", + "roxmltree", +] + [[package]] name = "effectcraft-text" version = "0.1.0" @@ -2749,6 +2778,12 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" + [[package]] name = "mach2" version = "0.4.3" @@ -3970,6 +4005,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "roxmltree" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb" +dependencies = [ + "memchr", +] + [[package]] name = "rustc-hash" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index a1941cf4..7aabc4af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,8 @@ effectcraft-media = { path = "crates/media" } effectcraft-export = { path = "crates/export" } effectcraft-expr = { path = "crates/expr" } effectcraft-lottie = { path = "crates/lottie" } +effectcraft-psd = { path = "crates/psd" } +effectcraft-svg = { path = "crates/svg" } effectcraft-engine = { path = "crates/engine" } effectcraft-host = { path = "crates/host" } effectcraft-ui-egui = { path = "crates/ui-egui" } diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 132bdcac..0830c78d 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -22,6 +22,9 @@ effectcraft-effects = { workspace = true } effectcraft-render = { workspace = true } effectcraft-track = { workspace = true } effectcraft-lottie = { workspace = true } +effectcraft-psd = { workspace = true } +kurbo = { workspace = true } +effectcraft-svg = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } diff --git a/crates/engine/src/commands/create.rs b/crates/engine/src/commands/create.rs new file mode 100644 index 00000000..72cb2420 --- /dev/null +++ b/crates/engine/src/commands/create.rs @@ -0,0 +1,139 @@ +//! Layer ▸ Create: Convert to Editable Text (Photoshop type layers), Create Shapes from Text, +//! Create Masks from Text and Create Shapes from Vector Layer (SVG footage). + +use effectcraft_color::Label; +use effectcraft_project::{ItemKind, LayerSource, Solid}; +use serde_json::{Value, json}; + +use super::{CommandSpec, bad, has_layers, layers_p, str_p}; +use crate::{EngineError, Result, Session, cmd}; + +const OPS: &str = "editableText|shapesFromText|masksFromText|shapesFromVector"; + +fn create(s: &mut Session, p: &Value) -> Result { + let op = str_p(p, "op").ok_or_else(|| bad("layer.create", format!("missing `op` ({OPS})")))?; + let (cid, lids) = layers_p(s, p)?; + if lids.is_empty() { + return Err(bad("layer.create", "select a layer first")); + } + let time = s.time(); + let mut created = vec![]; + let mut skipped = vec![]; + for lid in lids { + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let Some(src) = comp.layer(lid).cloned() else { continue }; + let footage = src.source.item().and_then(|i| match &s.project.item(i)?.kind { + ItemKind::Footage(f) => Some(f.clone()), + _ => None, + }); + match op { + "shapesFromVector" => { + let Some(f) = footage.filter(|f| f.codec == "SVG" || f.path.to_ascii_lowercase().ends_with(".svg")) else { + skipped.push(format!("{}: not a vector (SVG) footage layer", src.name)); + continue; + }; + let bytes = s.services.read_file(&f.path).map_err(|e| EngineError::Other(format!("cannot read {}: {e}", f.path)))?; + let doc = effectcraft_svg::parse(&bytes).map_err(|e| EngineError::Other(format!("{}: {e}", f.path)))?; + let id = s.edit("Create Shapes from Vector Layer", None, |proj, st| { + let l = crate::vector::shapes_from_vector(proj, &comp, &src, &doc); + let id = l.id; + insert_above(proj, cid, lid, l, true)?; + st.selected_layers = vec![id]; + Ok(id) + })?; + created.push(id.0); + } + "shapesFromText" => { + if !matches!(src.source, LayerSource::Text) { + skipped.push(format!("{}: not a text layer", src.name)); + continue; + } + let snapshot = s.project.clone(); + let id = s.edit("Create Shapes from Text", None, |proj, st| { + let l = crate::vector::shapes_from_text(proj, &comp, &snapshot, cid, &src, time) + .ok_or_else(|| bad("layer.create", "the text layer has no glyphs"))?; + let id = l.id; + insert_above(proj, cid, lid, l, true)?; + st.selected_layers = vec![id]; + Ok(id) + })?; + created.push(id.0); + } + "masksFromText" => { + if !matches!(src.source, LayerSource::Text) { + skipped.push(format!("{}: not a text layer", src.name)); + continue; + } + let outlines = crate::vector::text_outlines_in_comp(&s.project, cid, &comp, &src, time); + if outlines.is_empty() { + skipped.push(format!("{}: no glyphs", src.name)); + continue; + } + let id = s.edit("Create Masks from Text", None, |proj, st| { + let name = comp.unique_layer_name(&format!("{} Outlines", src.name)); + let folder = proj.folder_named("Solids").unwrap_or_else(|| proj.add_item("Solids", Label::Yellow, None, ItemKind::Folder)); + let sid = proj.add_item( + &name, + Label::Red, + Some(folder), + ItemKind::Solid(Solid { color: [1.0, 1.0, 1.0], width: comp.width, height: comp.height, pixel_aspect: 1.0 }), + ); + let mut l = crate::vector::masks_layer(proj, &comp, &name, outlines, sid); + l.in_point = src.in_point; + l.out_point = src.out_point; + let id = l.id; + insert_above(proj, cid, lid, l, true)?; + st.selected_layers = vec![id]; + Ok(id) + })?; + created.push(id.0); + } + "editableText" => { + let Some((f, sl)) = footage.and_then(|f| f.layer.clone().map(|l| (f, l))) else { + skipped.push(format!("{}: not a Photoshop layer", src.name)); + continue; + }; + let bytes = s.services.read_file(&f.path).map_err(|e| EngineError::Other(format!("cannot read {}: {e}", f.path)))?; + let psd = effectcraft_psd::Psd::parse(bytes).map_err(|e| EngineError::Other(e.to_string()))?; + let id = s.edit("Convert to Editable Text", None, |proj, st| { + let l = crate::psd_import::editable_text(proj, &comp, &src, &psd, sl.index as usize) + .ok_or_else(|| bad("layer.create", format!("{}: the Photoshop layer has no text", src.name)))?; + let id = l.id; + let c = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let at = c.layers.iter().position(|x| x.id == lid).ok_or(EngineError::NoComp)?; + c.layers[at] = l; + st.selected_layers = vec![id]; + Ok(id) + })?; + created.push(id.0); + } + other => return Err(bad("layer.create", format!("`{other}` is not available yet (supported: {OPS})"))), + } + } + if created.is_empty() && !skipped.is_empty() { + return Err(bad("layer.create", skipped.join("; "))); + } + Ok(json!({"layers": created, "skipped": skipped})) +} + +/// Insert `l` directly above layer `below`; optionally switch the source's video off (as After +/// Effects does after converting). +fn insert_above( + proj: &mut effectcraft_project::Project, + cid: effectcraft_project::ItemId, + below: effectcraft_project::LayerId, + l: effectcraft_project::Layer, + hide_source: bool, +) -> Result<()> { + let c = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let at = c.layers.iter().position(|x| x.id == below).unwrap_or(0); + if hide_source && let Some(src) = c.layers.get_mut(at) { + src.switches.video = false; + } + c.layers.insert(at, l); + Ok(()) +} + +pub fn specs() -> Vec { + vec![cmd!("layer.create", "Create", [], None, "{op: editableText|shapesFromText|masksFromText|shapesFromVector, layers?}", has_layers, create)] +} diff --git a/crates/engine/src/commands/edit.rs b/crates/engine/src/commands/edit.rs index e7ca4bd7..9a134b05 100644 --- a/crates/engine/src/commands/edit.rs +++ b/crates/engine/src/commands/edit.rs @@ -525,9 +525,38 @@ fn purge_caches(s: &mut Session, p: &Value) -> Result { s.snapshot = None; s.state.viewer.show_snapshot = false; } + let memory = matches!(what.as_str(), "all" | "memoryAndDisk" | "memory" | "image"); + let disk = matches!(what.as_str(), "all" | "memoryAndDisk" | "disk"); + if memory { + s.layer_cache.clear(); + } + let mut disk_entries = 0; + if disk && let Some(dc) = &s.disk_cache { + disk_entries = dc.stats().entries; + dc.clear(); + } s.events.push(crate::Event::PurgeCaches); s.toast(format!("Purged {what} cache")); - Ok(json!({"purged": what})) + Ok(json!({"purged": what, "diskEntries": disk_entries})) +} + +/// `cache.diskStats`: the disk cache's folder, limit and contents. +pub(crate) fn disk_stats(s: &mut Session, _: &Value) -> Result { + let Some(dc) = &s.disk_cache else { return Ok(json!({"enabled": false})) }; + let st = dc.stats(); + Ok(json!({ + "enabled": true, + "folder": dc.folder().to_string_lossy(), + "entries": st.entries, + "frames": dc.keys(effectcraft_render::disk_cache::Kind::Frame).len(), + "layers": dc.keys(effectcraft_render::disk_cache::Kind::Layer).len(), + "bytes": st.bytes, + "maxBytes": st.max_bytes, + "hits": st.hits, + "misses": st.misses, + "writes": st.writes, + "evictions": st.evictions, + })) } /// Path of the footage behind the selected layer or project item. @@ -596,6 +625,7 @@ pub fn specs() -> Vec { cmd!("edit.selectLabelGroup", "Select Label Group", ["Edit", "Label"], None, "{}", has_props_or_layers_or_items, select_label_group), cmd!("edit.purgeUndo", "Undo", ["Edit", "Purge"], None, "{}", always_ok, purge), cmd!("edit.purge", "Purge", [], None, "{what?: all|memoryAndDisk|memory|disk|3d|image|snapshot}", always_ok, purge_caches), + crate::query!("cache.diskStats", "Disk Cache Statistics", "{}", disk_stats), cmd!("edit.editOriginal", "Edit Original...", ["Edit"], Some("Cmd+E"), "{}", has_selected_footage, edit_original), ] } diff --git a/crates/engine/src/commands/file.rs b/crates/engine/src/commands/file.rs index a85a391f..a176c400 100644 --- a/crates/engine/src/commands/file.rs +++ b/crates/engine/src/commands/file.rs @@ -84,10 +84,48 @@ fn import(s: &mut Session, p: &Value) -> Result { Some(Value::String(x)) => vec![x.clone()], _ => return Err(bad("file.import", "missing `paths`")), }; - let importer = s.importer.clone().ok_or_else(|| EngineError::Other("media import is not available in this build".into()))?; + // Photoshop documents as compositions (Import As: Composition / – Retain Layer Sizes). + let import_as = str_p(p, "importAs").unwrap_or("footage"); + let retain = match import_as { + "footage" => None, + "composition" | "comp" => Some(false), + "compositionLayerSizes" | "compositionRetainLayerSizes" | "layerSizes" => Some(true), + other => return Err(bad("file.import", format!("importAs: footage|composition|compositionLayerSizes, not `{other}`"))), + }; + let mut paths = paths; + let mut out_comps = vec![]; let mut ids = vec![]; let mut errors = vec![]; + if let Some(retain) = retain { + let mut rest = vec![]; + for path in paths { + let bytes = match s.services.read_file(&path) { + Ok(b) if effectcraft_psd::is_psd(&b) => b, + _ => { + rest.push(path); + continue; + } + }; + match import_psd_comp(s, &path, bytes, retain) { + Ok((comp, items, warnings)) => { + out_comps.push(comp); + ids.extend(items); + errors.extend(warnings); + } + Err(e) => errors.push(format!("{path}: {e}")), + } + } + paths = rest; + if paths.is_empty() { + if let Some(c) = out_comps.first() { + s.open_comp(effectcraft_project::ItemId(*c)); + } + return Ok(json!({"items": ids, "comps": out_comps, "errors": errors})); + } + } + let importer = s.importer.clone().ok_or_else(|| EngineError::Other("media import is not available in this build".into()))?; let mut probed = vec![]; + let psd_layer = p.get("layer").cloned(); let seq_rate = effectcraft_time::FrameRate::from_f64(s.prefs.import.sequence_fps); for path in &paths { match importer.probe(path) { @@ -99,6 +137,18 @@ fn import(s: &mut Session, p: &Value) -> Result { f.frame_rate = r; f.duration = r.tick_of(frames.max(1)); } + // Choose Layer: one layer of a Photoshop document (document-sized). + if let Some(sel) = &psd_layer + && f.codec == "PSD" + { + match s.services.read_file(path).ok().and_then(|b| effectcraft_psd::Psd::parse(b).ok()).and_then(|d| find_psd_layer(&d, sel)) { + Some((index, name)) => f.layer = Some(effectcraft_project::SourceLayer { index: index as u32, name, layer_size: false }), + None => { + errors.push(format!("{path}: no layer {sel}")); + continue; + } + } + } probed.push((path.clone(), f)) } Err(e) => errors.push(format!("{path}: {e}")), @@ -107,6 +157,10 @@ fn import(s: &mut Session, p: &Value) -> Result { s.edit("Import", None, |proj, st| { for (path, f) in probed { let name = std::path::Path::new(&path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or(path.clone()); + let name = match &f.layer { + Some(l) => format!("{}/{name}", l.name), + None => name, + }; let label = match f.kind { FootageKind::Still | FootageKind::Sequence => Label::Lavender, FootageKind::Audio => Label::SeaFoam, @@ -118,7 +172,37 @@ fn import(s: &mut Session, p: &Value) -> Result { } Ok(()) })?; - Ok(json!({"items": ids, "errors": errors})) + if let Some(c) = out_comps.first() { + s.open_comp(effectcraft_project::ItemId(*c)); + } + Ok(json!({"items": ids, "comps": out_comps, "errors": errors})) +} + +/// A Photoshop layer by index or name (pixel layers only). +fn find_psd_layer(d: &effectcraft_psd::Psd, sel: &Value) -> Option<(usize, String)> { + let l = match sel { + Value::Number(n) => d.layers.get(n.as_u64()? as usize)?, + Value::String(name) => d.layers.iter().find(|l| &l.name == name)?, + _ => return None, + }; + Some((l.index, l.name.clone())) +} + +/// Import a Photoshop document as a composition (one undo step). Returns (comp, items, warnings). +fn import_psd_comp(s: &mut Session, path: &str, bytes: Vec, retain: bool) -> Result<(u64, Vec, Vec)> { + let psd = effectcraft_psd::Psd::parse(bytes).map_err(|e| EngineError::Other(e.to_string()))?; + let name = std::path::Path::new(path).file_stem().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "Photoshop".into()); + let rate = effectcraft_time::FrameRate::FPS_29_97; + let secs = if s.prefs.import.still_footage == "seconds" { s.prefs.import.still_seconds } else { 10.0 }; + let duration = rate.snap_nearest(effectcraft_time::Tick::from_seconds_f64(secs)); + let r = s.edit("Import", None, |proj, st| { + let r = crate::psd_import::import(proj, &psd, path, &name, retain, rate, duration); + st.project_selection = vec![r.comp]; + Ok(r) + })?; + let mut items = vec![r.folder.0]; + items.extend(r.items.iter().map(|i| i.0)); + Ok((r.comp.0, items, r.warnings)) } fn project_settings(s: &mut Session, p: &Value) -> Result { @@ -224,7 +308,15 @@ pub fn specs() -> Vec { cmd!("file.saveAs", "Save As...", ["File", "Save As"], Some("Cmd+Shift+S"), "{path}", always, save_as), cmd!("file.incrementAndSave", "Increment and Save", ["File"], Some("Cmd+Alt+Shift+S"), "{}", has_path, increment_save), cmd!("file.revert", "Revert", ["File"], None, "{}", has_path, revert), - cmd!("file.import", "File...", ["File", "Import"], Some("Cmd+I"), "{paths: [string]}", always, import), + cmd!( + "file.import", + "File...", + ["File", "Import"], + Some("Cmd+I"), + "{paths: [string], importAs?: footage|composition|compositionLayerSizes (Photoshop files), layer?: name|index (footage of one Photoshop layer)}", + always, + import + ), cmd!( "file.projectSettings", "Project Settings...", diff --git a/crates/engine/src/commands/file_more.rs b/crates/engine/src/commands/file_more.rs index 87c1e2df..b62664af 100644 --- a/crates/engine/src/commands/file_more.rs +++ b/crates/engine/src/commands/file_more.rs @@ -107,6 +107,7 @@ fn placeholder_footage(p: &Value) -> Footage { missing: true, sequence: vec![], color_profile: None, + layer: None, } } diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 4537752f..0791d1a3 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -6,6 +6,7 @@ pub mod anim_tools; mod animation; mod comp; mod comp_more; +mod create; mod edit; mod effect; mod file; @@ -118,6 +119,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(help::specs()); v.extend(query::specs()); v.extend(layer_menu::specs()); + v.extend(create::specs()); v.extend(animation::specs()); v.extend(view::specs()); v.extend(viewer_cmds::specs()); diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index f670ffd7..f8bca154 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -34,7 +34,6 @@ pub fn specs() -> Vec { // Time (timeline milestone). stub!("layer.alignVideoToData", "Align Video to Data", ["Layer", "Time"], "{}"), stub!("layer.environment", "Environment Layer", ["Layer"], "{}"), - stub!("layer.create", "Create", [], "{op}"), stub!("layer.autoTrace", "Auto-trace...", ["Layer"], "{}"), stub!("layer.sceneEditDetection", "Scene Edit Detection...", ["Layer"], "{}"), // Cameras / lights / materials / 3D views (3D milestone). diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index ba34488a..9d6afcc0 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -16,10 +16,12 @@ pub mod links; pub mod mask_track; pub mod menus; pub mod prefs; +pub mod psd_import; pub mod render_queue; mod session_settings; pub mod shortcuts; pub mod tracking; +pub mod vector; pub mod viewer; pub mod warp; @@ -268,6 +270,8 @@ pub struct Session { pub journal: Vec<(String, Value)>, /// Processed-layer pixels reused across frames and edits (content-keyed, never stale). pub layer_cache: Arc, + /// The persistent disk cache (Settings ▸ Media & Disk Cache), when enabled. + pub disk_cache: Option>, /// Settings (Preferences). pub prefs: prefs::Prefs, /// Bumped whenever settings change (frontends re-apply theme, labels…). @@ -311,6 +315,7 @@ impl Default for Session { events: vec![], journal: vec![], layer_cache: Arc::new(LayerCache::default()), + disk_cache: None, prefs: prefs::Prefs::default(), prefs_revision: 0, config: None, @@ -393,6 +398,9 @@ impl Session { pub fn bump(&mut self) { self.revision += 1; + if self.disk_cache.is_some() { + self.layer_cache.set_disk_salt(effectcraft_render::disk_cache::footage_salt(&self.project)); + } self.events.push(Event::ProjectChanged { revision: self.revision }); } @@ -574,6 +582,8 @@ mod tests_3d; #[cfg(test)] mod tests_anim_tools; #[cfg(test)] +mod tests_disk_cache; +#[cfg(test)] mod tests_effects; #[cfg(test)] mod tests_fidelity; @@ -602,6 +612,8 @@ mod tests_timeline; #[cfg(test)] mod tests_track; #[cfg(test)] +mod tests_vector_import; +#[cfg(test)] mod tests_viewer; /// Font families available to text layers (bundled + scanned system fonts). diff --git a/crates/engine/src/psd_import.rs b/crates/engine/src/psd_import.rs new file mode 100644 index 00000000..c14888ea --- /dev/null +++ b/crates/engine/src/psd_import.rs @@ -0,0 +1,634 @@ +//! Photoshop import as a composition (File ▸ Import ▸ File… with Import As: Composition or +//! Composition – Retain Layer Sizes), and Layer ▸ Create ▸ Convert to Editable Text. +//! +//! Every pixel layer becomes a footage item that names its layer (`Footage::layer`), placed in +//! a composition the size of the document. Groups become precomps (pass-through groups collapse +//! so their blend modes act on the layers below, like Photoshop); type layers become editable +//! text layers; layer masks are baked into the layer's alpha and vector masks become masks; +//! layer effects map to Layer Styles; adjustment layers become adjustment layers with the +//! matching effect (best effort); clipped layers preserve the transparency underneath. + +use std::sync::Arc; + +use effectcraft_color::BlendMode; +use effectcraft_color::Label; +use effectcraft_keyframe::{Gradient, Justify, ShapePath, TextDoc, Value}; +use effectcraft_project::build::{self, Ids}; +use effectcraft_project::styles::{self, STYLE_BLEND_MODES}; +use effectcraft_project::{AlphaMode, Comp, Footage, FootageKind, ItemId, ItemKind, Layer, LayerSource, MaskMode, Project, PropGroup, Solid, SourceLayer}; +use effectcraft_psd::{Adjustment, DValue, Descriptor, Node, Psd}; +use effectcraft_time::{FrameRate, Tick}; + +/// Photoshop blend mode key → compositor mode (`pass` = pass through, handled by collapsing). +pub fn blend_mode(key: &str) -> BlendMode { + match key { + "diss" => BlendMode::Dissolve, + "dark" => BlendMode::Darken, + "mul " => BlendMode::Multiply, + "idiv" => BlendMode::ColorBurn, + "lbrn" => BlendMode::LinearBurn, + "dkCl" => BlendMode::DarkerColor, + "lite" => BlendMode::Lighten, + "scrn" => BlendMode::Screen, + "div " => BlendMode::ColorDodge, + "lddg" => BlendMode::LinearDodge, + "lgCl" => BlendMode::LighterColor, + "over" => BlendMode::Overlay, + "sLit" => BlendMode::SoftLight, + "hLit" => BlendMode::HardLight, + "vLit" => BlendMode::VividLight, + "lLit" => BlendMode::LinearLight, + "pLit" => BlendMode::PinLight, + "hMix" => BlendMode::HardMix, + "diff" => BlendMode::Difference, + "smud" => BlendMode::Exclusion, + "fsub" => BlendMode::Subtract, + "fdiv" => BlendMode::Divide, + "hue " => BlendMode::Hue, + "sat " => BlendMode::Saturation, + "colr" => BlendMode::Color, + "lum " => BlendMode::Luminosity, + _ => BlendMode::Normal, + } +} + +/// Descriptor blend mode enum (`BlnM`) → compositor mode. +fn desc_blend(v: Option<&str>) -> Option { + Some(match v? { + "Nrml" => BlendMode::Normal, + "Dslv" => BlendMode::Dissolve, + "Drkn" => BlendMode::Darken, + "Mltp" => BlendMode::Multiply, + "CBrn" => BlendMode::ColorBurn, + "linearBurn" => BlendMode::LinearBurn, + "darkerColor" => BlendMode::DarkerColor, + "Lghn" => BlendMode::Lighten, + "Scrn" => BlendMode::Screen, + "CDdg" => BlendMode::ColorDodge, + "linearDodge" => BlendMode::LinearDodge, + "lighterColor" => BlendMode::LighterColor, + "Ovrl" => BlendMode::Overlay, + "SftL" => BlendMode::SoftLight, + "HrdL" => BlendMode::HardLight, + "vividLight" => BlendMode::VividLight, + "linearLight" => BlendMode::LinearLight, + "pinLight" => BlendMode::PinLight, + "hardMix" => BlendMode::HardMix, + "Dfrn" => BlendMode::Difference, + "Xclu" => BlendMode::Exclusion, + "blendSubtraction" => BlendMode::Subtract, + "blendDivide" => BlendMode::Divide, + "H " => BlendMode::Hue, + "Strt" => BlendMode::Saturation, + "Clr " => BlendMode::Color, + "Lmns" => BlendMode::Luminosity, + _ => return None, + }) +} + +/// What the import created. +pub struct PsdImport { + pub comp: ItemId, + pub folder: ItemId, + pub items: Vec, + pub warnings: Vec, +} + +struct Ctx<'a> { + psd: &'a Psd, + path: &'a str, + retain: bool, + folder: ItemId, + rate: FrameRate, + duration: Tick, + items: Vec, + warnings: Vec, +} + +/// Import `psd` (read from `path`) as a composition named `name`. +pub fn import(proj: &mut Project, psd: &Psd, path: &str, name: &str, retain: bool, rate: FrameRate, duration: Tick) -> PsdImport { + let folder = proj.add_item(&format!("{name} Layers"), Label::Yellow, None, ItemKind::Folder); + let mut cx = Ctx { psd, path, retain, folder, rate, duration, items: vec![], warnings: vec![] }; + let comp = build_comp(proj, &mut cx, name, &psd.tree(), None); + PsdImport { comp, folder, items: cx.items, warnings: cx.warnings } +} + +fn new_comp(cx: &Ctx) -> Comp { + Comp::new(cx.psd.width, cx.psd.height, cx.rate, cx.duration) +} + +fn build_comp(proj: &mut Project, cx: &mut Ctx, name: &str, nodes: &[Node], parent: Option) -> ItemId { + let cid = proj.add_item(name, Label::Sandstone, parent, ItemKind::Comp(Arc::new(new_comp(cx)))); + cx.items.push(cid); + let mut comp = new_comp(cx); + for n in nodes { + if let Some(l) = build_layer(proj, cx, &comp, n) { + comp.layers.push(l); + } + } + styles::sync_comp(&mut comp); + if let Some(item) = proj.item_mut(cid) { + item.kind = ItemKind::Comp(Arc::new(comp)); + } + cid +} + +fn footage_item(proj: &mut Project, cx: &mut Ctx, index: usize, name: &str) -> (ItemId, (u32, u32)) { + let l = &cx.psd.layers[index]; + let retain = cx.retain; + let (w, h) = if retain && !l.rect.is_empty() { (l.rect.width(), l.rect.height()) } else { (cx.psd.width, cx.psd.height) }; + let file = std::path::Path::new(cx.path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); + let f = Footage { + path: cx.path.to_string(), + kind: FootageKind::Still, + width: w, + height: h, + pixel_aspect: 1.0, + frame_rate: cx.rate, + native_rate: None, + duration: Tick::ZERO, + has_video: true, + has_audio: false, + alpha: AlphaMode::Straight, + premul_color: [0.0; 3], + loop_count: 1, + codec: "PSD".into(), + missing: false, + sequence: vec![], + color_profile: None, + layer: Some(SourceLayer { index: index as u32, name: name.to_string(), layer_size: retain && !l.rect.is_empty() }), + }; + let id = proj.add_item(&format!("{name}/{file}"), Label::Lavender, Some(cx.folder), ItemKind::Footage(f)); + cx.items.push(id); + (id, (w, h)) +} + +fn build_layer(proj: &mut Project, cx: &mut Ctx, comp: &Comp, n: &Node) -> Option { + let (index, children) = match n { + Node::Layer(i) => (*i, None), + Node::Group { layer, children } => (*layer, Some(children)), + }; + let pl = cx.psd.layers[index].clone(); + let name = if pl.name.is_empty() { format!("Layer {}", index + 1) } else { pl.name.clone() }; + let mut layer = if let Some(children) = children { + let sub = build_comp(proj, cx, &name, children, Some(cx.folder)); + let mut l = build::layer(proj, comp, &name, LayerSource::Comp { item: sub }, (comp.width, comp.height), None); + // Pass-through groups composite their layers straight into the parent. + l.switches.collapse = pl.blend_key() == "pass"; + l + } else if let Some(t) = &pl.text { + text_layer(proj, comp, &name, t) + } else if let Some(adj) = &pl.adjustment { + let mut l = adjustment_layer(proj, comp, &name); + add_adjustment_effect(proj, &mut l, adj, comp, &mut cx.warnings, &name); + l + } else if pl.has_pixels() || pl.fill.is_some() { + let (item, size) = footage_item(proj, cx, index, &name); + let mut l = build::layer(proj, comp, &name, LayerSource::Footage { item }, size, None); + if cx.retain && !pl.rect.is_empty() { + set_vec2(&mut l, "transform/position", [pl.rect.left as f64 + size.0 as f64 / 2.0, pl.rect.top as f64 + size.1 as f64 / 2.0]); + } + l + } else { + // Empty layers have nothing to show. + return None; + }; + layer.name = comp.unique_layer_name(&name); + layer.blend_mode = blend_mode(pl.blend_key()); + layer.switches.video = !pl.hidden; + layer.preserve_transparency = pl.clipping; + set_scalar(&mut layer, "transform/opacity", pl.opacity as f64 / 255.0 * 100.0); + // Vector mask → a mask in layer space. + let origin = layer_origin(&pl, cx.retain, &layer); + if let Some(vm) = pl.vector_mask.as_ref().filter(|v| !v.disabled) + && let Some(masks) = layer.props.sub_mut("masks") + { + let mut ids = Ids(&mut proj.next_id); + for (k, sp) in vm.subpaths.iter().enumerate() { + let n = sp.knots.len(); + let mut path = ShapePath { vertices: vec![], in_tangents: vec![], out_tangents: vec![], closed: sp.closed }; + for kn in &sp.knots { + let v = [kn[1][0] - origin[0], kn[1][1] - origin[1]]; + path.vertices.push(v); + path.in_tangents.push([kn[0][0] - kn[1][0], kn[0][1] - kn[1][1]]); + path.out_tangents.push([kn[2][0] - kn[1][0], kn[2][1] - kn[1][1]]); + } + if n == 0 { + continue; + } + let mode = if vm.inverted { MaskMode::Subtract } else { MaskMode::Add }; + masks.children.push(build::mask(&mut ids, &format!("Mask {}", k + 1), path, mode, [255, 255, 0]).into()); + } + } + // Layer effects → Layer Styles; Fill Opacity → Advanced Blending. + let gl = comp.global_light.clone(); + if let Some(fx) = &pl.effects + && fx.bool("masterFXSwitch") != Some(false) + && layer.can_have_styles() + { + apply_effects(proj, &mut layer, fx, &gl); + } + if pl.fill_opacity < 255 && layer.can_have_styles() { + let mut ids = Ids(&mut proj.next_id); + if layer.props.sub(styles::GROUP).is_none() { + let g = styles::layer_styles(&mut ids, &gl); + let at = layer.props.children.iter().position(|c| c.match_id() == "transform").map(|i| i + 1).unwrap_or(layer.props.children.len()); + layer.props.children.insert(at, g.into()); + } + if let Some(p) = layer.props.prop_mut("layerStyles/blendingOptions/advancedBlending/fillOpacity") { + p.value = Value::Scalar(pl.fill_opacity as f64 / 255.0 * 100.0); + } + } + Some(layer) +} + +/// Where layer space's origin sits in document pixels. +fn layer_origin(pl: &effectcraft_psd::Layer, retain: bool, layer: &Layer) -> [f64; 2] { + if let Some(t) = &pl.text { + return [t.transform[4], t.transform[5]]; + } + if matches!(layer.source, LayerSource::Footage { .. }) && retain && !pl.rect.is_empty() { + return [pl.rect.left as f64, pl.rect.top as f64]; + } + [0.0, 0.0] +} + +fn set_scalar(l: &mut Layer, path: &str, v: f64) { + if let Some(p) = l.props.prop_mut(path) { + p.value = Value::Scalar(v); + } +} + +fn set_vec2(l: &mut Layer, path: &str, v: [f64; 2]) { + if let Some(p) = l.props.prop_mut(path) { + p.value = match p.value { + Value::Vec3(o) => Value::Vec3([v[0], v[1], o[2]]), + _ => Value::Vec2(v), + }; + } +} + +/// Split a PostScript font name into family and style (`Inter-Bold` → `Inter`, `Bold`). +fn font_names(ps: &str) -> (String, String) { + match ps.split_once('-') { + Some((fam, style)) => (fam.to_string(), style.to_string()), + None => (ps.to_string(), "Regular".to_string()), + } +} + +fn text_doc(t: &effectcraft_psd::TextInfo) -> TextDoc { + let mut d = TextDoc::default(); + d.set_text(&t.text); + let [xx, xy, yx, yy, _, _] = t.transform; + let scale = (xx * yy - xy * yx).abs().sqrt().max(1e-6); + if let Some(s) = t.style.size { + d.size = s * scale; + } + if let Some(f) = &t.style.font { + let (fam, style) = font_names(f); + d.font = fam; + d.style = style; + } + if let Some(c) = t.style.color { + d.fill = [c[0] as f32, c[1] as f32, c[2] as f32, 1.0]; + } + d.justify = match t.style.justification { + Some(1) => Justify::Right, + Some(2) => Justify::Center, + Some(3) => Justify::JustifyLastLeft, + Some(4) => Justify::JustifyLastRight, + Some(5) => Justify::JustifyLastCenter, + Some(6) => Justify::JustifyAll, + _ => Justify::Left, + }; + if let Some(tr) = t.style.tracking { + d.tracking = tr; + } + if let Some(l) = t.style.leading { + d.leading = Some(l * scale); + } + d.faux_bold = t.style.faux_bold; + d.faux_italic = t.style.faux_italic; + if t.style.box_text + && let Some([l, top, r, b]) = t.bounds + { + d.box_size = Some([(r - l) * scale, (b - top) * scale]); + d.box_pos = [l * scale, top * scale]; + } + d +} + +fn text_layer(proj: &mut Project, comp: &Comp, name: &str, t: &effectcraft_psd::TextInfo) -> Layer { + let mut l = build::layer(proj, comp, name, LayerSource::Text, (0, 0), None); + if let Some(p) = l.props.prop_mut("text/sourceText") { + p.value = Value::Text(Box::new(text_doc(t))); + } + set_vec2(&mut l, "transform/position", [t.transform[4], t.transform[5]]); + let [xx, xy, _, _, _, _] = t.transform; + let rot = xy.atan2(xx).to_degrees(); + if rot.abs() > 1e-6 { + set_scalar(&mut l, "transform/rotation", rot); + } + l +} + +fn adjustment_layer(proj: &mut Project, comp: &Comp, name: &str) -> Layer { + let folder = proj.folder_named("Solids").unwrap_or_else(|| proj.add_item("Solids", Label::Yellow, None, ItemKind::Folder)); + let sid = proj.add_item( + name, + Label::Red, + Some(folder), + ItemKind::Solid(Solid { color: [1.0, 1.0, 1.0], width: comp.width, height: comp.height, pixel_aspect: 1.0 }), + ); + let mut l = build::layer(proj, comp, name, LayerSource::Solid { item: sid }, (comp.width, comp.height), None); + l.switches.adjustment = true; + l.label = Label::Purple; + l +} + +fn add_effect(proj: &mut Project, l: &mut Layer, effect: &str, comp: &Comp, params: &[(&str, Value)]) { + let Some(spec) = effectcraft_effects::lookup(effect) else { return }; + let mut ids = Ids(&mut proj.next_id); + let mut g = effectcraft_effects::instantiate(spec, &mut ids, spec.name, [comp.width as f64, comp.height as f64]); + for (k, v) in params { + if let Some(p) = g.get_mut(k) { + p.value = v.clone(); + } + } + if let Some(fx) = l.props.sub_mut("effects") { + fx.children.push(g.into()); + } +} + +fn add_adjustment_effect(proj: &mut Project, l: &mut Layer, adj: &Adjustment, comp: &Comp, warnings: &mut Vec, name: &str) { + match adj { + Adjustment::Invert => add_effect(proj, l, "Invert", comp, &[]), + Adjustment::BrightnessContrast { brightness, contrast, .. } => { + add_effect(proj, l, "Brightness & Contrast", comp, &[("brightness", Value::Scalar(*brightness)), ("contrast", Value::Scalar(*contrast))]) + } + Adjustment::HueSaturation { hue, saturation, lightness, colorize } => { + let mut p = vec![("hue", Value::Scalar(*hue)), ("saturation", Value::Scalar(*saturation)), ("lightness", Value::Scalar(*lightness))]; + if *colorize { + p = vec![ + ("colorize", Value::Bool(true)), + ("colorizeHue", Value::Scalar(*hue)), + ("colorizeSaturation", Value::Scalar(*saturation)), + ("colorizeLightness", Value::Scalar(*lightness)), + ]; + } + add_effect(proj, l, "Hue/Saturation", comp, &p) + } + Adjustment::Levels { in_black, in_white, gamma, out_black, out_white } => add_effect( + proj, + l, + "Levels", + comp, + &[ + ("inBlack", Value::Scalar(in_black / 255.0)), + ("inWhite", Value::Scalar(in_white / 255.0)), + ("gamma", Value::Scalar(*gamma)), + ("outBlack", Value::Scalar(out_black / 255.0)), + ("outWhite", Value::Scalar(out_white / 255.0)), + ], + ), + Adjustment::Exposure { exposure, offset, gamma } => add_effect( + proj, + l, + "Exposure", + comp, + &[("exposure", Value::Scalar(*exposure)), ("offset", Value::Scalar(*offset)), ("gamma", Value::Scalar(*gamma))], + ), + Adjustment::Other(k) => warnings.push(format!("{name}: adjustment `{k}` is not converted (layer kept as an empty adjustment layer)")), + } +} + +// ---------------------------------------------------------------- layer effects → styles + +fn style_mode(m: BlendMode) -> Value { + Value::Enum(STYLE_BLEND_MODES.iter().position(|x| x.1 == m).unwrap_or(0) as u32) +} + +fn set(g: &mut PropGroup, k: &str, v: Value) { + if let Some(p) = g.get_mut(k) { + p.value = v; + } +} + +fn color4(c: [f64; 3]) -> Value { + Value::Color([c[0], c[1], c[2], 1.0]) +} + +/// Copy the common style settings (`Md `, `Clr `, `Opct`, angle, distance, size…). +fn common(g: &mut PropGroup, d: &Descriptor) { + if let Some(m) = desc_blend(d.enum_value("Md ")) { + set(g, "blendMode", style_mode(m)); + } + if let Some(c) = d.color("Clr ") { + set(g, "color", color4(c)); + } + for (k, prop) in [("Opct", "opacity"), ("Nose", "noise"), ("Dstn", "distance"), ("blur", "size"), ("lagl", "angle"), ("ShdN", "jitter"), ("Inpr", "range")] + { + if let Some(v) = d.num(k) { + set(g, prop, Value::Scalar(v)); + } + } + if let Some(b) = d.bool("uglg") { + set(g, "useGlobalLight", Value::Bool(b)); + } +} + +fn gradient_of(d: &Descriptor) -> Option { + let g = d.obj("Grad")?; + let list = |k: &str| match g.get(k) { + Some(DValue::List(v)) => v.clone(), + _ => vec![], + }; + let colors: Vec<(f64, [f32; 4])> = list("Clrs") + .iter() + .filter_map(|c| match c { + DValue::Descriptor(s) => { + let col = s.color("Clr ")?; + Some((s.num("Lctn").unwrap_or(0.0) / 4096.0, [col[0] as f32, col[1] as f32, col[2] as f32, 1.0])) + } + _ => None, + }) + .collect(); + let opacities: Vec<(f64, f32)> = list("Trns") + .iter() + .filter_map(|c| match c { + DValue::Descriptor(s) => Some((s.num("Lctn").unwrap_or(0.0) / 4096.0, (s.num("Opct").unwrap_or(100.0) / 100.0) as f32)), + _ => None, + }) + .collect(); + (!colors.is_empty()).then(|| Gradient { colors, opacities: if opacities.is_empty() { vec![(0.0, 1.0), (1.0, 1.0)] } else { opacities } }) +} + +/// The effect descriptor for `key`, or the first enabled one of its `…Multi` list. +fn effect<'a>(fx: &'a Descriptor, key: &str, multi: &str) -> Option<&'a Descriptor> { + fx.obj(key).or_else(|| match fx.get(multi) { + Some(DValue::List(v)) => v.iter().find_map(|x| match x { + DValue::Descriptor(d) if d.bool("enab") != Some(false) => Some(d), + _ => None, + }), + _ => None, + }) +} + +fn apply_effects(proj: &mut Project, layer: &mut Layer, fx: &Descriptor, gl: &styles::GlobalLight) { + const MAP: &[(&str, &str, &str)] = &[ + ("DrSh", "dropShadowMulti", "dropShadow"), + ("IrSh", "innerShadowMulti", "innerShadow"), + ("OrGl", "outerGlowMulti", "outerGlow"), + ("IrGl", "innerGlowMulti", "innerGlow"), + ("ebbl", "bevelEmbossMulti", "bevelEmboss"), + ("ChFX", "satinMulti", "satin"), + ("SoFi", "solidFillMulti", "colorOverlay"), + ("GrFl", "gradientFillMulti", "gradientOverlay"), + ("FrFX", "frameFXMulti", "stroke"), + ]; + for (key, multi, id) in MAP { + let Some(d) = effect(fx, key, multi) else { continue }; + let enabled = d.bool("enab").unwrap_or(true); + let mut ids = Ids(&mut proj.next_id); + let Some(uid) = styles::add_style(layer, &mut ids, id, gl, enabled) else { continue }; + let Some(g) = layer.props.find_group_mut(uid) else { continue }; + common(g, d); + match *id { + "dropShadow" => { + if let Some(v) = d.num("Ckmt") { + set(g, "spread", Value::Scalar(v)); + } + if let Some(b) = d.bool("layerConceals") { + set(g, "knocksOut", Value::Bool(b)); + } + } + "innerShadow" | "innerGlow" => { + if let Some(v) = d.num("Ckmt") { + set(g, "choke", Value::Scalar(v)); + } + if d.enum_value("glwS") == Some("SrcC") { + set(g, "source", Value::Enum(0)); + } + } + "outerGlow" => { + if let Some(v) = d.num("Ckmt") { + set(g, "spread", Value::Scalar(v)); + } + } + "bevelEmboss" => { + let style = match d.enum_value("bvlS") { + Some("OtrB") => 0, + Some("Embs") => 2, + Some("PlEb") => 3, + Some("strokeEmboss") => 4, + _ => 1, + }; + set(g, "style", Value::Enum(style)); + let tech = match d.enum_value("bvlT") { + Some("PrBL") => 1, + Some("Slmt") => 2, + _ => 0, + }; + set(g, "technique", Value::Enum(tech)); + if let Some(v) = d.num("srgR") { + set(g, "depth", Value::Scalar(v)); + } + set(g, "direction", Value::Enum(u32::from(d.enum_value("bvlD") == Some("Out ")))); + if let Some(v) = d.num("Sftn") { + set(g, "soften", Value::Scalar(v)); + } + if let Some(v) = d.num("Lald") { + set(g, "altitude", Value::Scalar(v)); + } + for (k, p) in [("hglM", "highlightMode"), ("sdwM", "shadowMode")] { + if let Some(m) = desc_blend(d.enum_value(k)) { + set(g, p, style_mode(m)); + } + } + for (k, p) in [("hglC", "highlightColor"), ("sdwC", "shadowColor")] { + if let Some(c) = d.color(k) { + set(g, p, color4(c)); + } + } + for (k, p) in [("hglO", "highlightOpacity"), ("sdwO", "shadowOpacity")] { + if let Some(v) = d.num(k) { + set(g, p, Value::Scalar(v)); + } + } + } + "satin" => { + if let Some(b) = d.bool("Invr") { + set(g, "invert", Value::Bool(b)); + } + } + "gradientOverlay" => { + if let Some(gr) = gradient_of(d) { + set(g, "colors", Value::Gradient(gr)); + } + if let Some(a) = d.num("Angl") { + set(g, "angle", Value::Scalar(a)); + } + let st = match d.enum_value("Type") { + Some("Rdl ") => 1, + Some("Angl") => 2, + Some("Rflc") => 3, + Some("Dmnd") => 4, + _ => 0, + }; + set(g, "style", Value::Enum(st)); + if let Some(b) = d.bool("Rvrs") { + set(g, "reverse", Value::Bool(b)); + } + if let Some(b) = d.bool("Algn") { + set(g, "alignWithLayer", Value::Bool(b)); + } + if let Some(v) = d.num("Scl ") { + set(g, "scale", Value::Scalar(v)); + } + } + "stroke" => { + if let Some(v) = d.num("Sz ") { + set(g, "size", Value::Scalar(v)); + } + let pos = match d.enum_value("Styl") { + Some("InsF") => 1, + Some("CtrF") => 2, + _ => 0, + }; + set(g, "position", Value::Enum(pos)); + } + _ => {} + } + } +} + +/// Layer ▸ Create ▸ Convert to Editable Text: a footage layer showing a Photoshop type layer +/// becomes a text layer (`None` when the layer's source has no text data). +pub fn editable_text(proj: &mut Project, comp: &Comp, layer: &Layer, psd: &Psd, index: usize) -> Option { + let t = psd.layers.get(index)?.text.as_ref()?; + let mut l = text_layer(proj, comp, &layer.name, t); + l.name = layer.name.clone(); + l.blend_mode = layer.blend_mode; + l.switches.video = layer.switches.video; + l.in_point = layer.in_point; + l.out_point = layer.out_point; + l.start_time = layer.start_time; + if let (Some(o), Some(p)) = (layer.props.prop("transform/opacity"), l.props.prop_mut("transform/opacity")) { + *p = effectcraft_project::Property { uid: p.uid, ..o.clone() }; + } + Some(l) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn blend_keys() { + assert_eq!(blend_mode("mul "), BlendMode::Multiply); + assert_eq!(blend_mode("smud"), BlendMode::Exclusion); + assert_eq!(blend_mode("pass"), BlendMode::Normal); + assert_eq!(desc_blend(Some("linearDodge")), Some(BlendMode::LinearDodge)); + assert_eq!(font_names("Inter-Bold"), ("Inter".into(), "Bold".into())); + } +} diff --git a/crates/engine/src/session_settings.rs b/crates/engine/src/session_settings.rs index b0512a6a..e54eb93b 100644 --- a/crates/engine/src/session_settings.rs +++ b/crates/engine/src/session_settings.rs @@ -27,6 +27,7 @@ impl Session { self.prefs.normalize(); self.layer_cache.set_budget(self.prefs.layer_cache_bytes()); self.footage.set_cache_budget(self.prefs.media_cache_bytes()); + self.configure_disk_cache(); // Fewer undo levels apply right away. let levels = self.prefs.general.undo_levels.max(1) as usize; if self.history.undo.len() > levels { @@ -36,6 +37,42 @@ impl Session { self.prefs_revision += 1; } + /// Apply Settings ▸ Media & Disk Cache ▸ Disk Cache: open (or close) the persistent cache + /// and attach it to the layer cache. Without a folder setting the platform cache folder is + /// used, but only by frontends with a settings store (headless sessions stay off disk). + pub fn configure_disk_cache(&mut self) { + use effectcraft_render::disk_cache::{DiskCache, default_folder, footage_salt}; + let d = &self.prefs.disk; + let folder = if d.disk_cache_folder.trim().is_empty() { + self.config.is_some().then(default_folder) + } else { + Some(std::path::PathBuf::from(d.disk_cache_folder.trim())) + }; + let max = d.disk_cache_max_gb.max(1) as u64 * (1 << 30); + let Some(folder) = folder.filter(|_| d.disk_cache_enabled) else { + self.disk_cache = None; + self.layer_cache.set_disk(None); + return; + }; + let dc = match &self.disk_cache { + Some(c) if c.folder() == folder.as_path() => { + c.set_max_bytes(max); + c.clone() + } + _ => match DiskCache::open(&folder, max) { + Ok(c) => c, + Err(e) => { + log::warn!("disk cache at {}: {e}", folder.display()); + self.disk_cache = None; + self.layer_cache.set_disk(None); + return; + } + }, + }; + self.layer_cache.set_disk(Some((dc.clone(), footage_salt(&self.project)))); + self.disk_cache = Some(dc); + } + /// Write settings to the config store. pub fn save_prefs(&self) { if let Some(c) = &self.config diff --git a/crates/engine/src/tests_disk_cache.rs b/crates/engine/src/tests_disk_cache.rs new file mode 100644 index 00000000..65180ba8 --- /dev/null +++ b/crates/engine/src/tests_disk_cache.rs @@ -0,0 +1,121 @@ +//! The persistent disk cache (PRV-2) through the session: settings, layer cache persistence +//! across restarts, invalidation on edits, purging. + +use serde_json::json; + +use crate::Session; +use crate::render::RenderOpts; + +fn folder(name: &str) -> String { + let d = std::env::temp_dir().join(format!("effectcraft-session-disk-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + d.to_string_lossy().to_string() +} + +fn session(dir: &str) -> Session { + let mut s = Session::default(); + s.execute_checked("prefs.set", json!({"values": {"disk.diskCacheFolder": dir, "disk.diskCacheEnabled": true}})).unwrap(); + // Write every layer buffer, however quick to render. + s.disk_cache.as_ref().unwrap().set_min_layer_ms(0); + s +} + +fn project(s: &mut Session) -> effectcraft_project::ItemId { + s.execute("comp.new", json!({"name": "D", "width": 160, "height": 90, "frameRate": 30, "duration": 1})).unwrap(); + s.execute("layer.newSolid", json!({"color": "#3080ff", "width": 80, "height": 40})).unwrap(); + s.execute("effect.apply", json!({"layer": "#1", "effect": "Gaussian Blur"})).unwrap(); + s.execute("prop.set", json!({"layer": "#1", "path": "effects/#1/blurriness", "value": 12})).unwrap(); + s.active_comp_id().unwrap() +} + +#[test] +fn disk_cache_settings_persistence_and_invalidation() { + let dir = folder("persist"); + let mut s = session(&dir); + let st = s.execute_checked("cache.diskStats", json!({})).unwrap(); + assert_eq!(st["enabled"], json!(true)); + assert_eq!(st["maxBytes"], json!(100u64 << 30)); + let cid = project(&mut s); + let t = s.time(); + let first = s.render(cid, t, RenderOpts::default()); + let dc = s.disk_cache.clone().unwrap(); + dc.flush(); + let written = dc.stats().entries; + assert!(written > 0, "layer buffers written"); + // "Restart": a new session on the same folder and project renders from disk. + let saved = s.project.clone(); + drop(s); + let mut s2 = session(&dir); + s2.project = saved; + s2.bump(); + s2.open_comp(cid); + let dc2 = s2.disk_cache.clone().unwrap(); + assert_eq!(dc2.stats().entries, written); + let again = s2.render(cid, t, RenderOpts::default()); + assert!(dc2.stats().hits > 0, "{:?}", dc2.stats()); + assert_eq!(again.data, first.data, "disk-cached pixels are exact"); + // An edit changes the keys: a miss, and new entries. + let hits = dc2.stats().hits; + s2.execute("prop.set", json!({"layer": "#1", "path": "effects/#1/blurriness", "value": 20})).unwrap(); + let edited = s2.render(cid, t, RenderOpts::default()); + assert_ne!(edited.data, first.data); + assert_eq!(dc2.stats().hits, hits); + dc2.flush(); + assert!(dc2.stats().entries > written); + // Purge ▸ All Disk Cache empties it; memory purge leaves the disk alone. + let r = s2.execute_checked("edit.purge", json!({"what": "memory"})).unwrap(); + assert_eq!(r["diskEntries"], json!(0)); + assert!(dc2.stats().entries > 0); + let r = s2.execute_checked("edit.purge", json!({"what": "disk"})).unwrap(); + assert!(r["diskEntries"].as_u64().unwrap() > 0); + assert_eq!(dc2.stats().entries, 0); + // Disabling detaches it. + s2.execute_checked("prefs.set", json!({"key": "disk.diskCacheEnabled", "value": false})).unwrap(); + assert!(s2.disk_cache.is_none()); + assert!(s2.layer_cache.disk().is_none()); +} + +#[test] +fn footage_changes_change_the_salt() { + let dir = folder("salt"); + let mut s = session(&dir); + let foot = std::env::temp_dir().join(format!("effectcraft-salt-{}.png", std::process::id())); + std::fs::write(&foot, b"one").unwrap(); + s.edit("add", None, |p, _| { + let f = effectcraft_project::Footage { + path: foot.to_string_lossy().to_string(), + kind: effectcraft_project::FootageKind::Still, + width: 1, + height: 1, + pixel_aspect: 1.0, + frame_rate: effectcraft_time::FrameRate::FPS_30, + native_rate: None, + duration: effectcraft_time::Tick::ZERO, + has_video: true, + has_audio: false, + alpha: effectcraft_project::AlphaMode::Ignore, + premul_color: [0.0; 3], + loop_count: 1, + codec: "PNG".into(), + missing: false, + sequence: vec![], + color_profile: None, + layer: None, + }; + p.add_item("f.png", effectcraft_color::Label::Lavender, None, effectcraft_project::ItemKind::Footage(f)); + Ok(()) + }) + .unwrap(); + let a = effectcraft_render::disk_cache::footage_salt(&s.project); + std::fs::write(&foot, b"two, longer").unwrap(); + let b = effectcraft_render::disk_cache::footage_salt(&s.project); + assert_ne!(a, b); + let cid = project(&mut s); + let k1 = effectcraft_render::disk_cache::comp_content_key(&s.project, cid); + s.execute("prop.set", json!({"layer": "#1", "path": "transform/opacity", "value": 50})).unwrap(); + let k2 = effectcraft_render::disk_cache::comp_content_key(&s.project, cid); + assert_ne!(k1, k2); + // Unrelated comps do not change it. + s.execute("comp.new", json!({"name": "Other", "width": 10, "height": 10, "frameRate": 30, "duration": 1})).unwrap(); + assert_eq!(effectcraft_render::disk_cache::comp_content_key(&s.project, cid), k2); +} diff --git a/crates/engine/src/tests_stubs.rs b/crates/engine/src/tests_stubs.rs index 141858ca..17ffda7f 100644 --- a/crates/engine/src/tests_stubs.rs +++ b/crates/engine/src/tests_stubs.rs @@ -257,6 +257,7 @@ impl Importer for MockImporter { missing: false, sequence: vec![], color_profile: None, + layer: None, }) } } diff --git a/crates/engine/src/tests_track.rs b/crates/engine/src/tests_track.rs index adcf7df0..8cf020f9 100644 --- a/crates/engine/src/tests_track.rs +++ b/crates/engine/src/tests_track.rs @@ -101,6 +101,7 @@ pub(crate) fn setup(make: impl Fn(u32) -> Image + Send + Sync + 'static) -> (Ses missing: false, sequence: vec![], color_profile: None, + layer: None, }; let item = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(footage)); let comp = p.comp(cid).unwrap().clone(); @@ -410,6 +411,7 @@ fn analyze_1080p_single_point_speed() { missing: false, sequence: vec![], color_profile: None, + layer: None, }; let item = p.add_item("hd", Label::Aqua, None, ItemKind::Footage(footage)); let comp = p.comp(cid).unwrap().clone(); diff --git a/crates/engine/src/tests_vector_import.rs b/crates/engine/src/tests_vector_import.rs new file mode 100644 index 00000000..d205630c --- /dev/null +++ b/crates/engine/src/tests_vector_import.rs @@ -0,0 +1,237 @@ +//! Photoshop import as a composition (IO-2) and Layer ▸ Create text conversions (SHP-5). + +use effectcraft_color::BlendMode; +use effectcraft_keyframe::Value as KV; +use effectcraft_project::{ItemKind, LayerSource, MaskMode}; +use effectcraft_psd::write::*; +use effectcraft_psd::{DValue, Descriptor, Rect}; +use serde_json::json; + +use crate::Session; +use crate::render::RenderOpts; + +fn tmp(name: &str) -> String { + let d = std::env::temp_dir().join(format!("effectcraft-psd-tests-{}", std::process::id())); + std::fs::create_dir_all(&d).unwrap(); + d.join(name).to_string_lossy().to_string() +} + +/// A layered document exercising groups, masks, blend modes, text, styles and adjustments. +pub(crate) fn sample_psd() -> Vec { + let mut d = WDoc::new(120, 80); + let shadow = Descriptor::new("null").with("masterFXSwitch", DValue::Bool(true)).with( + "DrSh", + DValue::Descriptor( + Descriptor::new("DrSh") + .with("enab", DValue::Bool(true)) + .with("Md ", DValue::Enum("BlnM".into(), "Mltp".into())) + .with("Clr ", DValue::Descriptor(Descriptor::rgb([0.0, 0.0, 0.5]))) + .with("Opct", DValue::UnitFloat("#Prc".into(), 60.0)) + .with("uglg", DValue::Bool(false)) + .with("lagl", DValue::UnitFloat("#Ang".into(), 45.0)) + .with("Dstn", DValue::UnitFloat("#Pxl".into(), 9.0)) + .with("blur", DValue::UnitFloat("#Pxl".into(), 4.0)), + ), + ); + let stroke = Descriptor::new("null").with( + "FrFX", + DValue::Descriptor( + Descriptor::new("FrFX") + .with("enab", DValue::Bool(true)) + .with("Styl", DValue::Enum("FStl".into(), "InsF".into())) + .with("Md ", DValue::Enum("BlnM".into(), "Nrml".into())) + .with("Opct", DValue::UnitFloat("#Prc".into(), 100.0)) + .with("Sz ", DValue::UnitFloat("#Pxl".into(), 2.0)) + .with("Clr ", DValue::Descriptor(Descriptor::rgb([1.0, 1.0, 0.0]))), + ), + ); + let mut masked = WLayer::solid("Masked Square", Rect::new(10, 10, 40, 40), [0.0, 0.8, 0.2, 1.0]).with_blend(b"mul "); + masked.mask = Some(WMask { + rect: Rect::new(10, 10, 40, 40), + data: (0..1600).map(|i| if i % 40 < 20 { 1.0 } else { 0.0 }).collect(), + default_color: 0, + disabled: false, + }); + let mut hidden = WLayer::solid("Hidden", Rect::new(0, 0, 10, 10), [1.0, 0.0, 1.0, 1.0]); + hidden.hidden = true; + let mut clipped = WLayer::solid("Clipped", Rect::new(60, 30, 20, 20), [1.0, 1.0, 1.0, 1.0]).with_blend(b"scrn").with_opacity(128); + clipped.clipping = true; + let tri = vec![[[70.0, 10.0]; 3], [[110.0, 10.0]; 3], [[90.0, 40.0]; 3]]; + d.layers = vec![ + WLayer::solid("Background", Rect::new(0, 0, 120, 80), [0.2, 0.3, 0.4, 1.0]), + WLayer::group_end(), + masked, + hidden, + WLayer::group("Group A", true, *b"pass"), + WLayer::solid("Base", Rect::new(55, 25, 30, 30), [0.9, 0.1, 0.1, 1.0]).with_block(effects_block(&shadow)), + clipped, + WLayer::solid("Vector Masked", Rect::new(60, 0, 60, 50), [0.1, 0.1, 0.9, 1.0]) + .with_block(vector_mask_block(&[tri], 120, 80, false)) + .with_block(effects_block(&stroke)), + WLayer::empty("Hue Shift", vec![hue_sat_block(40, 10, -5)]), + WLayer::empty("Title", vec![text_block("Hello PSD", [8.0, 70.0], "Inter-Bold", 14.0, [1.0, 1.0, 1.0], 0)]), + ]; + write(&d) +} + +#[test] +fn psd_imports_as_composition() { + let path = tmp("sample.psd"); + std::fs::write(&path, sample_psd()).unwrap(); + let mut s = Session::default(); + let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": "composition"})).unwrap(); + assert_eq!(r["errors"], json!([]), "{r}"); + let cid = effectcraft_project::ItemId(r["comps"][0].as_u64().unwrap()); + assert_eq!(s.state.active_comp, Some(cid)); + assert_eq!(s.project.item(cid).unwrap().name, "sample"); + let c = s.project.comp(cid).unwrap().clone(); + assert_eq!((c.width, c.height), (120, 80)); + let names: Vec<&str> = c.layers.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(names, ["Title", "Hue Shift", "Vector Masked", "Clipped", "Base", "Group A", "Background"]); + // Text layer: editable text, positioned at the type origin. + let title = &c.layers[0]; + assert!(matches!(title.source, LayerSource::Text)); + let Some(KV::Text(doc)) = title.props.prop("text/sourceText").map(|p| p.value.clone()) else { panic!("text") }; + assert_eq!(doc.text, "Hello PSD"); + assert_eq!(doc.size, 14.0); + assert_eq!(doc.font, "Inter"); + assert_eq!(doc.style, "Bold"); + assert_eq!(title.props.prop("transform/position").unwrap().value.as_vec2(), [8.0, 70.0]); + // Adjustment layer with Hue/Saturation. + let hue = &c.layers[1]; + assert!(hue.switches.adjustment); + let fx = hue.effects().unwrap().groups().next().unwrap(); + assert_eq!(fx.get("hue").unwrap().value.as_f64(), 40.0); + assert_eq!(fx.get("saturation").unwrap().value.as_f64(), 10.0); + // Vector mask → mask; Stroke style. + let vm = &c.layers[2]; + let m = vm.masks().unwrap().groups().next().unwrap(); + assert!(matches!(m.kind, effectcraft_project::GroupKind::Mask { mode: MaskMode::Add, .. })); + let Some(KV::Path(p)) = m.get("path").map(|p| p.value.clone()) else { panic!("mask path") }; + assert_eq!(p.vertices.len(), 3); + assert!((p.vertices[1][0] - 110.0).abs() < 1e-3 && (p.vertices[2][1] - 40.0).abs() < 1e-3); + let st = vm.props.group("layerStyles/stroke").unwrap(); + assert_eq!(st.get("size").unwrap().value.as_f64(), 2.0); + assert_eq!(st.get("position").unwrap().value, KV::Enum(1)); + // Clipped → preserve transparency; opacity and blend mode. + let clipped = &c.layers[3]; + assert!(clipped.preserve_transparency); + assert_eq!(clipped.blend_mode, BlendMode::Screen); + assert!((clipped.props.prop("transform/opacity").unwrap().value.as_f64() - 128.0 / 255.0 * 100.0).abs() < 1e-9); + // Drop shadow. + let ds = c.layers[4].props.group("layerStyles/dropShadow").unwrap(); + assert_eq!(ds.get("distance").unwrap().value.as_f64(), 9.0); + assert_eq!(ds.get("angle").unwrap().value.as_f64(), 45.0); + assert_eq!(ds.get("useGlobalLight").unwrap().value, KV::Bool(false)); + assert_eq!(ds.get("opacity").unwrap().value.as_f64(), 60.0); + // Group → collapsed precomp (pass through) with its layers. + let g = &c.layers[5]; + assert!(g.switches.collapse); + let LayerSource::Comp { item } = g.source else { panic!("precomp") }; + let inner = s.project.comp(item).unwrap(); + let inner_names: Vec<&str> = inner.layers.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(inner_names, ["Hidden", "Masked Square"]); + assert!(!inner.layers[0].switches.video); + assert_eq!(inner.layers[1].blend_mode, BlendMode::Multiply); + // Footage items name their Photoshop layer; document-sized footage. + let LayerSource::Footage { item } = inner.layers[1].source else { panic!("footage") }; + let ItemKind::Footage(f) = &s.project.item(item).unwrap().kind else { panic!() }; + let sl = f.layer.as_ref().unwrap(); + assert_eq!((sl.name.as_str(), sl.layer_size), ("Masked Square", false)); + assert_eq!((f.width, f.height), (120, 80)); + assert_eq!(s.project.item(item).unwrap().name, "Masked Square/sample.psd"); + // One undo step removes everything. + s.execute("edit.undo", json!({})).unwrap(); + assert!(s.project.comp(cid).is_none()); +} + +#[test] +fn psd_retain_layer_sizes() { + let path = tmp("retain.psd"); + std::fs::write(&path, sample_psd()).unwrap(); + let mut s = Session::default(); + let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": "compositionLayerSizes"})).unwrap(); + let cid = effectcraft_project::ItemId(r["comps"][0].as_u64().unwrap()); + let c = s.project.comp(cid).unwrap(); + let base = c.layer_by_name("Base").unwrap(); + let LayerSource::Footage { item } = base.source else { panic!() }; + let ItemKind::Footage(f) = &s.project.item(item).unwrap().kind else { panic!() }; + assert_eq!((f.width, f.height), (30, 30)); + assert!(f.layer.as_ref().unwrap().layer_size); + assert_eq!(base.props.prop("transform/position").unwrap().value.as_vec2(), [70.0, 40.0]); + assert_eq!(base.props.prop("transform/anchor").unwrap().value.as_vec2(), [15.0, 15.0]); + // Vector mask vertices are in the layer's own space. + let vm = c.layer_by_name("Vector Masked").unwrap(); + let Some(KV::Path(p)) = vm.masks().unwrap().groups().next().unwrap().get("path").map(|p| p.value.clone()) else { panic!() }; + assert!((p.vertices[0][0] - 10.0).abs() < 1e-3, "{:?}", p.vertices); +} + +#[test] +fn import_rejects_unknown_kind() { + let mut s = Session::default(); + assert!(s.execute_checked("file.import", json!({"paths": ["x.psd"], "importAs": "banana"})).is_err()); +} + +fn text_session() -> (Session, u64) { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "T", "width": 320, "height": 120, "frameRate": 30, "duration": 2})).unwrap(); + let r = s.execute_checked("layer.newText", json!({"text": "Ab8", "size": 64, "position": [160, 80], "fill": "#ffcc00"})).unwrap(); + (s, r["layer"].as_u64().unwrap()) +} + +fn alpha_diff(a: &effectcraft_raster::Image, b: &effectcraft_raster::Image) -> (f32, f32) { + let mut sum = 0.0; + let mut mass = 0.0; + for (p, q) in a.data.iter().zip(&b.data) { + sum += (p[3] - q[3]).abs(); + mass += p[3]; + } + (sum / a.data.len() as f32, mass) +} + +#[test] +fn create_shapes_from_text_matches_text_render() { + let (mut s, lid) = text_session(); + let cid = s.active_comp_id().unwrap(); + let opts = RenderOpts::default(); + let text_img = s.render(cid, s.time(), opts); + s.execute("layer.select", json!({"layers": [lid]})).unwrap(); + let r = s.execute_checked("layer.create", json!({"op": "shapesFromText"})).unwrap(); + let sid = r["layers"][0].as_u64().unwrap(); + let c = s.active_comp().unwrap(); + assert_eq!(c.layers[0].id.0, sid); + assert!(matches!(c.layers[0].source, LayerSource::Shape)); + assert_eq!(c.layers[0].name, "Ab8 Outlines"); + assert!(!c.layer(effectcraft_project::LayerId(lid)).unwrap().switches.video, "source hidden"); + let groups: Vec = c.layers[0].props.sub("contents").unwrap().groups().map(|g| g.name.clone()).collect(); + assert_eq!(groups, ["A", "b", "8"]); + let shape_img = s.render(cid, s.time(), opts); + let (d, mass) = alpha_diff(&text_img, &shape_img); + assert!(mass > 500.0, "text drew something"); + assert!(d < 0.01, "mean alpha difference {d}"); + // Fill colour carried over. + let i = shape_img.data.iter().position(|p| p[3] > 0.99).unwrap(); + let p = shape_img.data[i]; + assert!((p[0] - 1.0).abs() < 0.02 && (p[1] - 0.8).abs() < 0.02 && p[2] < 0.02, "{p:?}"); +} + +#[test] +fn create_masks_from_text() { + let (mut s, lid) = text_session(); + let cid = s.active_comp_id().unwrap(); + let text_img = s.render(cid, s.time(), RenderOpts::default()); + s.execute("layer.select", json!({"layers": [lid]})).unwrap(); + let r = s.execute_checked("layer.create", json!({"op": "masksFromText"})).unwrap(); + let mid = effectcraft_project::LayerId(r["layers"][0].as_u64().unwrap()); + let c = s.active_comp().unwrap(); + let l = c.layer(mid).unwrap(); + assert!(matches!(l.source, LayerSource::Solid { .. })); + // "A", "b" and "8" have 2, 2 and 3 contours. + assert_eq!(l.masks().unwrap().groups().count(), 7); + let img = s.render(cid, s.time(), RenderOpts::default()); + let (d, _) = alpha_diff(&text_img, &img); + assert!(d < 0.01, "mean alpha difference {d}"); + // Other ops need the right layer type. + assert!(s.execute_checked("layer.create", json!({"op": "shapesFromVector", "layers": [lid]})).is_err()); + assert!(s.execute_checked("layer.create", json!({"op": "nullTracing", "layers": [lid]})).is_err()); +} diff --git a/crates/engine/src/vector.rs b/crates/engine/src/vector.rs new file mode 100644 index 00000000..a3f3e040 --- /dev/null +++ b/crates/engine/src/vector.rs @@ -0,0 +1,250 @@ +//! Vector conversions behind Layer ▸ Create: Create Shapes from Vector Layer (SVG footage → +//! shape layer), Create Shapes from Text and Create Masks from Text (glyph outlines). + +use effectcraft_keyframe::{Gradient, ShapePath, Value}; +use effectcraft_project::build::{self, Ids}; +use effectcraft_project::{Comp, ItemId, Layer, LayerSource, MaskMode, Project, PropGroup}; +use effectcraft_svg::{Affine, Doc, Geom, GradientKind, Node, Paint}; +use kurbo::{BezPath, Point}; + +fn shape_paths(ids: &mut Ids, path: &BezPath) -> Vec { + effectcraft_path::from_kurbo(path) + .into_iter() + .enumerate() + .map(|(i, sp)| { + let mut g = build::shape_path(ids, sp); + g.name = format!("Path {}", i + 1); + g + }) + .collect() +} + +fn det_scale(m: Affine) -> f64 { + let c = m.as_coeffs(); + (c[0] * c[3] - c[1] * c[2]).abs().sqrt() +} + +/// Axis-aligned scale + translation only (basic shapes stay parametric). +fn axis_aligned(m: Affine) -> Option<(f64, f64, f64, f64)> { + let c = m.as_coeffs(); + (c[1].abs() < 1e-9 && c[2].abs() < 1e-9 && c[0] > 0.0 && c[3] > 0.0).then_some((c[0], c[3], c[4], c[5])) +} + +fn gradient_value(g: &effectcraft_svg::Gradient) -> Gradient { + Gradient { + colors: g.stops.iter().map(|(o, c)| (*o, [c[0] as f32, c[1] as f32, c[2] as f32, 1.0])).collect(), + opacities: g.stops.iter().map(|(o, c)| (*o, c[3] as f32)).collect(), + } +} + +/// Gradient start/end points in shape-layer space (`m`: shape user space → layer space). +fn gradient_points(g: &effectcraft_svg::Gradient, m: Affine, bbox: kurbo::Rect) -> (bool, [f64; 2], [f64; 2]) { + let unit = if g.bbox_units { Affine::new([bbox.width(), 0.0, 0.0, bbox.height(), bbox.x0, bbox.y0]) } else { Affine::IDENTITY }; + let full = m * unit * g.transform; + let p = |x: f64, y: f64| { + let q = full * Point::new(x, y); + [q.x, q.y] + }; + match g.kind { + GradientKind::Linear { x1, y1, x2, y2 } => (false, p(x1, y1), p(x2, y2)), + GradientKind::Radial { cx, cy, r, .. } => (true, p(cx, cy), p(cx + r, cy)), + } +} + +fn shape_items(ids: &mut Ids, s: &effectcraft_svg::Shape, m: Affine) -> Vec { + use kurbo::Shape as _; + let mut items = vec![]; + // Geometry: parametric when the transform keeps rectangles/ellipses axis-aligned. + match (&s.geom, axis_aligned(m)) { + (Geom::Rect { x, y, w, h, rx, ry }, Some((sx, sy, tx, ty))) if (rx - ry).abs() < 1e-9 && (*rx == 0.0 || (sx - sy).abs() < 1e-9) => { + let (w2, h2) = (w * sx, h * sy); + let pos = [tx + (x + w / 2.0) * sx, ty + (y + h / 2.0) * sy]; + let mut g = build::shape_rect(ids, [w2, h2], pos, rx * sx); + g.name = "Rectangle Path 1".into(); + items.push(g); + } + (Geom::Ellipse { cx, cy, rx, ry }, Some((sx, sy, tx, ty))) => { + items.push(build::shape_ellipse(ids, [2.0 * rx * sx, 2.0 * ry * sy], [tx + cx * sx, ty + cy * sy])); + } + (g, _) => items.extend(shape_paths(ids, &(m * g.to_path()))), + } + let bbox = s.geom.to_path().bounding_box(); + if let Some(st) = &s.stroke { + let width = st.width * det_scale(m); + let mut g = match &st.paint { + Paint::Color(c) => build::shape_stroke(ids, [c[0], c[1], c[2], 1.0], width), + Paint::Gradient(gr) => { + let (radial, a, b) = gradient_points(gr, m, bbox); + build::shape_gradient_stroke(ids, radial, a, b, gradient_value(gr), width) + } + }; + set(&mut g, "opacity", Value::Scalar(st.opacity * 100.0)); + set(&mut g, "cap", Value::Enum(st.cap as u32)); + set(&mut g, "join", Value::Enum(st.join as u32)); + set(&mut g, "miter", Value::Scalar(st.miter)); + if let Some((d, off)) = &st.dash + && let Some(dg) = g.sub_mut("dashes") + { + let k = det_scale(m); + set(dg, "dash", Value::Scalar(d.first().copied().unwrap_or(0.0) * k)); + set(dg, "gap", Value::Scalar(d.get(1).or(d.first()).copied().unwrap_or(0.0) * k)); + set(dg, "offset", Value::Scalar(off * k)); + } + items.push(g); + } + if let Some(f) = &s.fill { + let mut g = match f { + Paint::Color(c) => build::shape_fill(ids, [c[0], c[1], c[2], 1.0]), + Paint::Gradient(gr) => { + let (radial, a, b) = gradient_points(gr, m, bbox); + build::shape_gradient_fill(ids, radial, a, b, gradient_value(gr)) + } + }; + set(&mut g, "opacity", Value::Scalar(s.fill_opacity * 100.0)); + set(&mut g, "rule", Value::Enum(u32::from(s.fill_rule == effectcraft_svg::FillRule::EvenOdd))); + items.push(g); + } + items +} + +fn set(g: &mut PropGroup, k: &str, v: Value) { + if let Some(p) = g.get_mut(k) { + p.value = v; + } +} + +fn group(ids: &mut Ids, name: &str, items: Vec, opacity: f64) -> PropGroup { + let mut g = build::shape_group(ids, name, items); + if let Some(p) = g.prop_mut("transform/opacity") { + p.value = Value::Scalar(opacity * 100.0); + } + g +} + +/// Shape layer contents for an SVG document, in document pixels (transforms are baked into the +/// geometry; group and element opacity go to the group transforms). Top of the paint order first. +pub fn svg_contents(ids: &mut Ids, doc: &Doc) -> Vec { + fn walk(ids: &mut Ids, nodes: &[Node], m: Affine) -> Vec { + let mut out = vec![]; + for n in nodes.iter().rev() { + match n { + Node::Group(g) => { + let items = walk(ids, &g.children, m * g.transform); + if !items.is_empty() { + out.push(group(ids, &g.name, items, g.opacity)); + } + } + Node::Shape(s) => { + let items = shape_items(ids, s, m * s.transform); + if !items.is_empty() { + out.push(group(ids, &s.name, items, s.opacity)); + } + } + } + } + out + } + let root = &doc.root; + let items = walk(ids, &root.children, root.transform); + if (root.opacity - 1.0).abs() > 1e-9 { vec![group(ids, "svg", items, root.opacity)] } else { items } +} + +/// A copy of `src`'s transform group with fresh uids. +fn copy_transform(proj: &mut Project, src: &Layer, dst: &mut Layer) { + let Some(tr) = src.props.sub("transform") else { return }; + let mut tr = tr.clone(); + let mut n = proj.next_id; + tr.reassign_uids(&mut n); + proj.next_id = n + 1; + if let Some(slot) = dst.props.children.iter_mut().find(|c| c.match_id() == "transform") { + *slot = tr.into(); + } +} + +fn like_source(dst: &mut Layer, src: &Layer) { + dst.start_time = src.start_time; + dst.in_point = src.in_point; + dst.out_point = src.out_point; + dst.stretch = src.stretch; + dst.parent = src.parent; + dst.switches.three_d = src.switches.three_d; +} + +/// Layer ▸ Create ▸ Create Shapes from Vector Layer: `doc` is the SVG shown by `src`. +pub fn shapes_from_vector(proj: &mut Project, comp: &Comp, src: &Layer, doc: &Doc) -> Layer { + let mut l = build::layer(proj, comp, &format!("{} Outlines", src.name), LayerSource::Shape, (0, 0), None); + let items = svg_contents(&mut Ids(&mut proj.next_id), doc); + if let Some(c) = l.props.sub_mut("contents") { + c.children.extend(items.into_iter().map(Into::into)); + } + copy_transform(proj, src, &mut l); + like_source(&mut l, src); + l.name = comp.unique_layer_name(&format!("{} Outlines", src.name)); + l +} + +/// Layer ▸ Create ▸ Create Shapes from Text: one group per character with its outline, fill and +/// stroke (as laid out at the comp time `ctx` was made for). +pub fn shapes_from_text(proj: &mut Project, comp: &Comp, ctx_project: &Project, cid: ItemId, src: &Layer, time: effectcraft_time::Tick) -> Option { + let ctx = effectcraft_render::EvalCtx::new(ctx_project, cid, comp, time); + let geom = effectcraft_render::text::text_geom(&ctx, src)?; + let chars: Vec = geom.doc.text.chars().filter(|c| !c.is_whitespace()).collect(); + let mut l = build::layer(proj, comp, &format!("{} Outlines", src.name), LayerSource::Shape, (0, 0), None); + let mut groups = vec![]; + { + let mut ids = Ids(&mut proj.next_id); + for (i, g) in geom.glyphs.iter().enumerate() { + let path = g.path(); + if path.elements().is_empty() { + continue; + } + let mut items = shape_paths(&mut ids, &path); + if g.stroke_width > 0.0 && g.stroke[3] > 0.0 { + let c = g.stroke; + items.push(build::shape_stroke(&mut ids, [c[0] as f64, c[1] as f64, c[2] as f64, 1.0], g.stroke_width)); + } + if g.apply_fill { + let c = g.fill; + let mut f = build::shape_fill(&mut ids, [c[0] as f64, c[1] as f64, c[2] as f64, 1.0]); + set(&mut f, "opacity", Value::Scalar(c[3] as f64 * 100.0)); + items.push(f); + } + let name = chars.get(i).map(|c| c.to_string()).unwrap_or_else(|| format!("Glyph {}", i + 1)); + groups.push(group(&mut ids, &name, items, 1.0)); + } + } + // First character on top, as in After Effects. + if let Some(c) = l.props.sub_mut("contents") { + c.children.extend(groups.into_iter().map(Into::into)); + } + copy_transform(proj, src, &mut l); + like_source(&mut l, src); + l.name = comp.unique_layer_name(&format!("{} Outlines", src.name)); + Some(l) +} + +/// Glyph outlines of a text layer in comp space (for Create Masks from Text). +pub fn text_outlines_in_comp(project: &Project, cid: ItemId, comp: &Comp, src: &Layer, time: effectcraft_time::Tick) -> Vec { + let ctx = effectcraft_render::EvalCtx::new(project, cid, comp, time); + let Some(geom) = effectcraft_render::text::text_geom(&ctx, src) else { return vec![] }; + let (m, _) = ctx.layer_to_comp(src); + let mut out = vec![]; + for g in &geom.glyphs { + let p = g.path(); + let p = effectcraft_path::transform(std::slice::from_ref(&p), &m).remove(0); + out.extend(effectcraft_path::from_kurbo(&p)); + } + out +} + +/// A comp-sized white solid with one mask per glyph contour (Difference, so counters stay open). +pub fn masks_layer(proj: &mut Project, comp: &Comp, name: &str, outlines: Vec, solid: ItemId) -> Layer { + let mut l = build::layer(proj, comp, name, LayerSource::Solid { item: solid }, (comp.width, comp.height), None); + let mut ids = Ids(&mut proj.next_id); + if let Some(masks) = l.props.sub_mut("masks") { + for (i, sp) in outlines.into_iter().enumerate() { + masks.children.push(build::mask(&mut ids, &format!("Mask {}", i + 1), sp, MaskMode::Difference, [255, 255, 0]).into()); + } + } + l +} diff --git a/crates/gpu/src/tests.rs b/crates/gpu/src/tests.rs index ca2fb1b0..cf63e593 100644 --- a/crates/gpu/src/tests.rs +++ b/crates/gpu/src/tests.rs @@ -100,6 +100,7 @@ impl Scene { missing: false, sequence: vec![], color_profile: None, + layer: None, }; let fid = self.p.add_item("Pattern", Label::Lavender, None, ItemKind::Footage(f)); build::layer(&mut self.p, &self.comp, "Pattern", LayerSource::Footage { item: fid }, (w, h), None) diff --git a/crates/host/Cargo.toml b/crates/host/Cargo.toml index b43271c7..7143510a 100644 --- a/crates/host/Cargo.toml +++ b/crates/host/Cargo.toml @@ -18,3 +18,7 @@ effectcraft-export = { workspace = true } [dev-dependencies] serde_json = { workspace = true } +effectcraft-psd = { workspace = true } +effectcraft-svg = { workspace = true } +effectcraft-raster = { workspace = true } +effectcraft-time = { workspace = true } diff --git a/crates/host/tests/vector_import.rs b/crates/host/tests/vector_import.rs new file mode 100644 index 00000000..eb5be076 --- /dev/null +++ b/crates/host/tests/vector_import.rs @@ -0,0 +1,163 @@ +//! Photoshop and SVG import through the fully wired session (media decoding included): pixels +//! of imported compositions, SVG footage at any scale, and Create Shapes from Vector Layer. + +use effectcraft_engine::render::RenderOpts; +use effectcraft_project::ItemId; +use effectcraft_psd::Rect; +use effectcraft_psd::write::*; +use effectcraft_raster::Image; +use serde_json::json; + +fn tmp(name: &str) -> String { + let d = std::env::temp_dir().join(format!("effectcraft-host-vector-{}", std::process::id())); + std::fs::create_dir_all(&d).unwrap(); + d.join(name).to_string_lossy().to_string() +} + +fn mean_diff(a: &Image, b: &Image) -> f32 { + assert_eq!((a.width, a.height), (b.width, b.height)); + let mut s = 0.0; + for (p, q) in a.data.iter().zip(&b.data) { + for c in 0..4 { + s += (p[c] - q[c]).abs(); + } + } + s / (a.data.len() * 4) as f32 +} + +fn psd_doc() -> WDoc { + let mut d = WDoc::new(64, 48); + let mut masked = WLayer::solid("Masked", Rect::new(30, 8, 24, 24), [0.1, 0.9, 0.3, 1.0]); + masked.mask = Some(WMask { + rect: Rect::new(30, 8, 24, 24), + data: (0..576).map(|i| if (i / 24) < 12 { 1.0 } else { 0.25 }).collect(), + default_color: 0, + disabled: false, + }); + d.layers = vec![ + WLayer::solid("Background", Rect::new(0, 0, 64, 48), [0.2, 0.25, 0.3, 1.0]), + WLayer::solid("Half Red", Rect::new(4, 4, 30, 20), [1.0, 0.0, 0.0, 1.0]).with_opacity(128), + masked, + WLayer::group_end(), + WLayer::solid("In Group", Rect::new(10, 30, 20, 12), [0.9, 0.9, 0.1, 1.0]), + WLayer::group("Group", true, *b"norm"), + ]; + d +} + +/// Straight-alpha over of the layers as Photoshop composites them (normal blending), as the +/// merged image to compare the imported composition with. +fn reference(bytes: &[u8]) -> Vec<[f32; 4]> { + let p = effectcraft_psd::Psd::parse(bytes.to_vec()).unwrap(); + let mut acc = vec![[0.0f32; 4]; (p.width * p.height) as usize]; + for l in p.layers.iter().filter(|l| l.has_pixels() && !l.hidden) { + let px = p.layer_pixels(l.index, true).unwrap(); + let o = l.opacity as f32 / 255.0; + for (d, s) in acc.iter_mut().zip(&px.data) { + let a = s[3] * o; + let oa = a + d[3] * (1.0 - a); + for c in 0..3 { + d[c] = if oa > 0.0 { (s[c] * a + d[c] * d[3] * (1.0 - a)) / oa } else { 0.0 }; + } + d[3] = oa; + } + } + acc +} + +#[test] +fn psd_composition_renders_like_the_document() { + let mut doc = psd_doc(); + let bytes = write(&doc); + let merged = reference(&bytes); + doc.composite = Some(merged.clone()); + let bytes = write(&doc); + let path = tmp("layers.psd"); + std::fs::write(&path, &bytes).unwrap(); + let mut s = effectcraft_host::session(); + // Footage (merged image). + let r = s.execute_checked("file.import", json!({"paths": [path]})).unwrap(); + let merged_item = r["items"][0].as_u64().unwrap(); + // Composition and Composition – Retain Layer Sizes. + let mut renders = vec![]; + for kind in ["composition", "compositionLayerSizes"] { + let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": kind})).unwrap(); + let cid = ItemId(r["comps"][0].as_u64().unwrap()); + renders.push(s.render(cid, effectcraft_time::Tick::ZERO, RenderOpts::default())); + } + // The merged footage in a comp of the same size. + s.execute("comp.new", json!({"name": "Merged", "width": 64, "height": 48, "frameRate": 30, "duration": 1})).unwrap(); + s.execute("layer.addItem", json!({"item": merged_item})).unwrap(); + let mcid = s.active_comp_id().unwrap(); + let merged_img = s.render(mcid, effectcraft_time::Tick::ZERO, RenderOpts::default()); + let expected = Image { width: 64, height: 48, data: merged.iter().map(|p| [p[0] * p[3], p[1] * p[3], p[2] * p[3], p[3]]).collect() }; + assert!(mean_diff(&merged_img, &expected) < 2.0 / 255.0, "merged footage"); + for (i, img) in renders.iter().enumerate() { + let d = mean_diff(img, &expected); + assert!(d < 2.0 / 255.0, "import kind {i}: mean diff {d}"); + } + // A single layer as footage (Choose Layer). + let r = s.execute_checked("file.import", json!({"paths": [path], "layer": "Masked"})).unwrap(); + let it = s.project.item(ItemId(r["items"][0].as_u64().unwrap())).unwrap(); + assert_eq!(it.name, "Masked/layers.psd"); +} + +#[test] +fn psd_16bit_cmyk_composition_imports() { + let mut doc = psd_doc(); + doc.depth = 16; + doc.mode = effectcraft_psd::ColorMode::Cmyk; + doc.rle = false; + let bytes = write(&doc); + let merged = reference(&bytes); + let path = tmp("layers16.psd"); + std::fs::write(&path, &bytes).unwrap(); + let mut s = effectcraft_host::session(); + let r = s.execute_checked("file.import", json!({"paths": [path], "importAs": "composition"})).unwrap(); + let cid = ItemId(r["comps"][0].as_u64().unwrap()); + let img = s.render(cid, effectcraft_time::Tick::ZERO, RenderOpts::default()); + let expected = Image { width: 64, height: 48, data: merged.iter().map(|p| [p[0] * p[3], p[1] * p[3], p[2] * p[3], p[3]]).collect() }; + let d = mean_diff(&img, &expected); + assert!(d < 2.0 / 255.0, "mean diff {d}"); +} + +const SVG: &[u8] = include_bytes!("../../svg/tests/fixtures/basic-shapes.svg"); +const SVG2: &[u8] = include_bytes!("../../svg/tests/fixtures/paths-gradients.svg"); + +#[test] +fn svg_footage_and_shapes_from_vector_layer() { + for (name, bytes) in [("basic-shapes.svg", SVG), ("paths-gradients.svg", SVG2)] { + let path = tmp(name); + std::fs::write(&path, bytes).unwrap(); + let mut s = effectcraft_host::session(); + let r = s.execute_checked("file.import", json!({"paths": [path]})).unwrap(); + assert_eq!(r["errors"], json!([]), "{r}"); + let item = r["items"][0].as_u64().unwrap(); + let doc = effectcraft_svg::parse(bytes).unwrap(); + let (w, h) = doc.pixel_size(); + s.execute("comp.new", json!({"name": "V", "width": w, "height": h, "frameRate": 30, "duration": 1})).unwrap(); + let lid = s.execute_checked("layer.addItem", json!({"item": item})).unwrap()["layer"].as_u64().unwrap(); + let cid = s.active_comp_id().unwrap(); + let t = effectcraft_time::Tick::ZERO; + let foot = s.render(cid, t, RenderOpts::default()); + let direct = effectcraft_svg::rasterize(&doc, w, h, 1.0); + assert!(mean_diff(&foot, &direct) < 1.0 / 255.0, "{name}: footage = rasterised SVG"); + s.execute("layer.select", json!({"layers": [lid]})).unwrap(); + let r = s.execute_checked("layer.create", json!({"op": "shapesFromVector"})).unwrap(); + let sid = r["layers"][0].as_u64().unwrap(); + let c = s.active_comp().unwrap(); + assert_eq!(c.layers[0].id.0, sid); + assert!(!c.layers[1].switches.video); + let shapes = s.render(cid, t, RenderOpts::default()); + let d = mean_diff(&shapes, &direct); + assert!(d < 0.02, "{name}: shapes vs SVG mean diff {d}"); + // Scaled up 3×, the shape layer stays sharp and matches the SVG rasterised at 3×. + s.execute("prop.set", json!({"layer": sid, "path": "transform/scale", "value": [300, 300]})).unwrap(); + s.execute("prop.set", json!({"layer": sid, "path": "transform/position", "value": [0, 0]})).unwrap(); + s.execute("prop.set", json!({"layer": sid, "path": "transform/anchor", "value": [0, 0]})).unwrap(); + let big = s.render(cid, t, RenderOpts::default()); + let direct3 = effectcraft_svg::rasterize(&doc, w, h, 3.0); + let d3 = mean_diff(&big, &direct3); + assert!(d3 < 0.03, "{name}: ×3 mean diff {d3}"); + } +} diff --git a/crates/lottie/src/import.rs b/crates/lottie/src/import.rs index c87dc187..22bc5023 100644 --- a/crates/lottie/src/import.rs +++ b/crates/lottie/src/import.rs @@ -171,6 +171,7 @@ pub fn import( missing: false, sequence: vec![], color_profile: None, + layer: None, }; let item = project.add_item(&nm, Label::Lavender, Some(folder), ItemKind::Footage(foot)); cx.items.push(item); diff --git a/crates/lottie/src/tests.rs b/crates/lottie/src/tests.rs index 5140133c..01c724d7 100644 --- a/crates/lottie/src/tests.rs +++ b/crates/lottie/src/tests.rs @@ -592,6 +592,7 @@ fn images_embed_as_base64_and_come_back() { missing: false, sequence: vec![], color_profile: None, + layer: None, }), ); let c = comp(&p, cid); diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 89992569..6e83c949 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -14,6 +14,8 @@ effectcraft-time = { workspace = true } effectcraft-project = { workspace = true } effectcraft-raster = { workspace = true } effectcraft-render = { workspace = true } +effectcraft-psd = { workspace = true } +effectcraft-svg = { workspace = true } filmcraft-codecs = { workspace = true } filmcraft-media = { workspace = true } filmcraft-frame = { workspace = true } diff --git a/crates/media/src/convert.rs b/crates/media/src/convert.rs index fa5df238..dce33d8a 100644 --- a/crates/media/src/convert.rs +++ b/crates/media/src/convert.rs @@ -284,6 +284,22 @@ pub(crate) fn dynamic_to_image(img: &image::DynamicImage, op: AlphaOp) -> Image out } +/// Straight RGBA `f32` pixels (layered stills: Photoshop, SVG) to an [`Image`]. +pub(crate) fn straight_to_image(w: u32, h: u32, px: &[[f32; 4]], op: AlphaOp) -> Image { + let mut out = Image::new(w, h); + let wu = w as usize; + if wu == 0 || h == 0 { + return out; + } + out.data.par_chunks_mut(wu).enumerate().for_each(|(y, row)| { + for (x, o) in row.iter_mut().enumerate() { + let s = px[y * wu + x]; + *o = op.apply([s[0], s[1], s[2]], s[3].clamp(0.0, 1.0)); + } + }); + out +} + #[cfg(test)] mod tests { use super::*; @@ -376,3 +392,4 @@ mod tests { } } } + diff --git a/crates/media/src/layered.rs b/crates/media/src/layered.rs new file mode 100644 index 00000000..204312a5 --- /dev/null +++ b/crates/media/src/layered.rs @@ -0,0 +1,62 @@ +//! Layered and vector stills: Photoshop documents (`effectcraft-psd`: the merged image, or one +//! layer for footage imported as a composition) and SVG (`effectcraft-svg`, rasterised). + +use effectcraft_project::{AlphaMode, Footage}; +use effectcraft_raster::Image; + +use crate::convert::{AlphaOp, straight_to_image}; +use crate::{MediaError, Result}; + +fn is_svg(path: &str, bytes: &[u8]) -> bool { + path.to_ascii_lowercase().ends_with(".svg") || effectcraft_svg::looks_like_svg(bytes) +} + +/// Footage for a Photoshop or SVG file (`None` for other formats). +pub(crate) fn probe(path: &str, bytes: &[u8]) -> Result> { + if effectcraft_psd::is_psd(bytes) { + let psd = effectcraft_psd::Psd::parse(bytes.to_vec()).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + let mut f = crate::probe::still_footage(path, psd.width, psd.height, image::ImageFormat::Png, true); + f.codec = if psd.psb { "PSB".into() } else { "PSD".into() }; + f.alpha = if psd.merged_alpha || !psd.layers.is_empty() { AlphaMode::Straight } else { AlphaMode::Ignore }; + return Ok(Some(f)); + } + if is_svg(path, bytes) { + let doc = effectcraft_svg::parse(bytes).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + let (w, h) = doc.pixel_size(); + let mut f = crate::probe::still_footage(path, w, h, image::ImageFormat::Png, true); + f.codec = "SVG".into(); + f.alpha = AlphaMode::Straight; + return Ok(Some(f)); + } + Ok(None) +} + +/// Decode a Photoshop or SVG still (`None` for other formats). +pub(crate) fn decode(path: &str, bytes: &[u8], footage: &Footage, op: AlphaOp) -> Result> { + if effectcraft_psd::is_psd(bytes) { + let psd = effectcraft_psd::Psd::parse(bytes.to_vec()).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + let px = match &footage.layer { + Some(l) => psd.layer_pixels(l.index as usize, !l.layer_size), + None => psd.composite(), + } + .map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + return Ok(Some(straight_to_image(px.width, px.height, &px.data, op))); + } + if is_svg(path, bytes) { + let doc = effectcraft_svg::parse(bytes).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; + let (w, h) = doc.pixel_size(); + let img = effectcraft_svg::rasterize(&doc, w, h, 1.0); + // The rasteriser produces premultiplied pixels already. + let _ = op; + return Ok(Some(img)); + } + Ok(None) +} + +/// An SVG rasterised at `scale` × its pixel size (Continuously Rasterize). +pub(crate) fn rasterize_svg(bytes: &[u8], scale: f64) -> Option { + let doc = effectcraft_svg::parse(bytes).ok()?; + let (w, h) = doc.pixel_size(); + let (sw, sh) = (((w as f64 * scale).ceil() as u32).clamp(1, 16384), ((h as f64 * scale).ceil() as u32).clamp(1, 16384)); + Some(effectcraft_svg::rasterize(&doc, sw, sh, scale)) +} diff --git a/crates/media/src/lib.rs b/crates/media/src/lib.rs index 29546f7e..246054b3 100644 --- a/crates/media/src/lib.rs +++ b/crates/media/src/lib.rs @@ -10,9 +10,11 @@ //! Video containers (MP4/MOV, Matroska/WebM) and codecs (H.264, HEVC, VP9, AV1, ProRes, DNxHD, //! MJPEG; AAC, Opus, PCM, MP3/FLAC/Vorbis) come from FilmCraft's pure-Rust crates (git dependency, //! pinned; `plan/adr/0001`). FilmCraft types never leave this crate. Stills use the `image` crate -//! (PNG, JPEG, GIF, WebP, TIFF, BMP, OpenEXR). +//! (PNG, JPEG, GIF, WebP, TIFF, BMP, OpenEXR); Photoshop documents (merged image or one layer) come +//! from `effectcraft-psd` and SVG from `effectcraft-svg` (rasterised at any scale). mod convert; +mod layered; mod pool; mod probe; @@ -44,7 +46,7 @@ impl From for MediaError { } /// File extensions recognised as stills. -pub const STILL_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "exr"]; +pub const STILL_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "exr", "psd", "psb", "svg"]; /// File extensions recognised as audio-only files. pub const AUDIO_EXTENSIONS: &[&str] = filmcraft_media::AUDIO_EXTENSIONS; /// File extensions recognised as movies. diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index b4cc51ef..b5d94299 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -359,6 +359,11 @@ impl Inner { let i = Self::frame_index(footage.frame_rate, t, footage.sequence.len() as i64, footage.loop_count); Loc { key: key(&footage.sequence[i as usize], 0), media_t: None } } + // A layer of a layered still is keyed by its layer index and size mode. + FootageKind::Still if footage.layer.is_some() => { + let l = footage.layer.as_ref().map_or(0, |l| 1 + l.index as i64 * 2 + l.layer_size as i64); + Loc { key: key(&footage.path, l), media_t: None } + } FootageKind::Still | FootageKind::Sequence => Loc { key: key(&footage.path, 0), media_t: None }, FootageKind::Video | FootageKind::Audio => { let rate = footage.frame_rate; @@ -437,6 +442,9 @@ impl Inner { match loc.media_t { None => { let bytes = self.read(path)?; + if let Some(img) = crate::layered::decode(path, &bytes, footage, op)? { + return Ok(img); + } let img = image::load_from_memory(&bytes).map_err(|e| MediaError::Decode(format!("{path}: {e}")))?; Ok(dynamic_to_image(&img, op)) } @@ -470,6 +478,14 @@ impl FootageSource for MediaPool { } } + fn vector_frame(&self, _item: ItemId, footage: &Footage, scale: f64) -> Option> { + if footage.missing || !effectcraft_render::is_vector_footage(footage) { + return None; + } + let bytes = self.inner.read(&footage.path).ok()?; + crate::layered::rasterize_svg(&bytes, scale).map(Arc::new) + } + fn audio(&self, _item: ItemId, footage: &Footage, t: Tick, frames: usize, rate: u32) -> Option> { if !footage.has_audio || footage.missing { return None; diff --git a/crates/media/src/probe.rs b/crates/media/src/probe.rs index 4e1c5611..23432cc6 100644 --- a/crates/media/src/probe.rs +++ b/crates/media/src/probe.rs @@ -33,6 +33,9 @@ pub fn probe_single(path: impl AsRef) -> Result { /// Probe in-memory file contents; `path` names the footage (and is stored as `Footage::path`). pub fn probe_bytes(path: &str, bytes: Arc<[u8]>) -> Result { + if let Some(f) = crate::layered::probe(path, &bytes)? { + return Ok(f); + } if let Ok(fmt) = image::guess_format(&bytes) { return probe_still_bytes(path, &bytes, fmt); } @@ -72,6 +75,7 @@ pub(crate) fn footage_from_info(path: &str, info: &filmcraft_media::MediaInfo) - missing: false, sequence: Vec::new(), color_profile: v.and_then(|v| profile_of(&v.color)), + layer: None, } } @@ -88,7 +92,7 @@ fn profile_of(c: &filmcraft_color::ColorInfo) -> Option Footage { +pub(crate) fn still_footage(path: &str, w: u32, h: u32, fmt: image::ImageFormat, has_alpha: bool) -> Footage { let codec = fmt.extensions_str().first().map_or("image", |s| s).to_ascii_uppercase(); // OpenEXR stores premultiplied colour by convention; other formats straight alpha. let alpha = match (has_alpha, fmt) { @@ -114,6 +118,7 @@ fn still_footage(path: &str, w: u32, h: u32, fmt: image::ImageFormat, has_alpha: missing: false, sequence: Vec::new(), color_profile: None, + layer: None, } } diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 1ed35c09..b4854f54 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -579,6 +579,21 @@ pub struct Footage { /// Colour profile (from the file's metadata, or Interpret Footage). `None` = sRGB. #[serde(default, skip_serializing_if = "Option::is_none")] pub color_profile: Option, + /// One layer of a layered file (Photoshop) instead of its merged image. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub layer: Option, +} + +/// A layer of a layered still (a Photoshop document) used as footage +/// (File ▸ Import ▸ Composition / Composition – Retain Layer Sizes, or Choose Layer). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SourceLayer { + /// Layer record index in the file (bottom of the stack = 0). + pub index: u32, + pub name: String, + /// The footage is the layer's own bounds (Retain Layer Sizes) rather than the document size. + #[serde(default)] + pub layer_size: bool, } fn one() -> u32 { diff --git a/crates/psd/Cargo.toml b/crates/psd/Cargo.toml new file mode 100644 index 00000000..49dda6a3 --- /dev/null +++ b/crates/psd/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "effectcraft-psd" +description = "Photoshop PSD/PSB reader (layers, groups, masks, blend modes, text, layer effects, adjustment layers; 8/16/32-bit, RGB/CMYK/Gray/Lab/Indexed, raw/RLE/ZIP) and a minimal writer, from Adobe's published file format specification" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +miniz_oxide = "0.8" diff --git a/crates/psd/README.md b/crates/psd/README.md new file mode 100644 index 00000000..e238b9b2 --- /dev/null +++ b/crates/psd/README.md @@ -0,0 +1,17 @@ +# effectcraft-psd + +Reads Photoshop documents (`.psd`, and `.psb` large documents) for File ▸ Import (as footage, as a +composition, or as a composition with layer sizes retained), and writes a minimal subset (used to +generate test fixtures). + +Implemented from Adobe's public **Adobe Photoshop File Formats Specification** (November 2019 +edition): file header, colour mode data, image resources, layer and mask information (layer +records, channel image data, layer mask data, additional layer information: `luni`, `lsct`/`lsdk`, +`lyid`, `iOpa`, `TySh`, `lfx2`, `SoCo`, `nvrt`, `brit`, `hue2`, `levl`, `expA`, `vmsk`/`vsms`, +`Lr16`/`Lr32`), the descriptor (action) structure, and the merged image data. Raw, RLE (PackBits) +and ZIP (with and without prediction, inflated with `miniz_oxide`) compression; 1/8/16/32-bit; +Bitmap, Grayscale, Indexed, RGB, CMYK (naive conversion) and Lab colour modes. + +Text engine data (`EngineData`) is read best-effort for the font, size, fill colour and +justification of the first style run. No Adobe code, sample files or assets were used; test +fixtures are generated in-test by the writer in this crate ([`write`](src/write.rs)). diff --git a/crates/psd/src/descriptor.rs b/crates/psd/src/descriptor.rs new file mode 100644 index 00000000..e121f823 --- /dev/null +++ b/crates/psd/src/descriptor.rs @@ -0,0 +1,430 @@ +//! The descriptor ("action descriptor") structure: typed key/value trees used by text layers, +//! layer effects, fill layers and newer adjustment layers (spec: "Descriptor structure"). + +use crate::read::Reader; +use crate::{Error, Result}; + +/// A descriptor: a class and a list of keyed items (in file order). +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Descriptor { + pub class_name: String, + pub class_id: String, + pub items: Vec<(String, DValue)>, +} + +/// One descriptor value. +#[derive(Clone, Debug, PartialEq)] +pub enum DValue { + Descriptor(Descriptor), + List(Vec), + Double(f64), + /// Unit (`#Pxl`, `#Prc`, `#Ang`, `#Pnt`…) and value. + UnitFloat(String, f64), + UnitFloats(String, Vec), + Text(String), + /// Enumerated value: type and value ids. + Enum(String, String), + Integer(i32), + LargeInteger(i64), + Bool(bool), + Class(String, String), + Raw(Vec), + Reference(Vec), +} + +/// An item of a reference (`obj `). +#[derive(Clone, Debug, PartialEq)] +pub enum RefItem { + Property(String, String), + Class(String), + Enum(String, String, String), + Offset(String, i32), + Identifier(i32), + Index(i32), + Name(String, String), +} + +impl Descriptor { + pub fn get(&self, key: &str) -> Option<&DValue> { + self.items.iter().find(|(k, _)| k == key).map(|(_, v)| v) + } + pub fn obj(&self, key: &str) -> Option<&Descriptor> { + match self.get(key)? { + DValue::Descriptor(d) => Some(d), + _ => None, + } + } + /// A number: doubles, unit floats, integers. + pub fn num(&self, key: &str) -> Option { + match self.get(key)? { + DValue::Double(v) | DValue::UnitFloat(_, v) => Some(*v), + DValue::Integer(v) => Some(*v as f64), + DValue::LargeInteger(v) => Some(*v as f64), + _ => None, + } + } + pub fn bool(&self, key: &str) -> Option { + match self.get(key)? { + DValue::Bool(b) => Some(*b), + _ => None, + } + } + pub fn text(&self, key: &str) -> Option<&str> { + match self.get(key)? { + DValue::Text(s) => Some(s), + _ => None, + } + } + /// The value id of an enumerated item. + pub fn enum_value(&self, key: &str) -> Option<&str> { + match self.get(key)? { + DValue::Enum(_, v) => Some(v), + _ => None, + } + } + pub fn raw(&self, key: &str) -> Option<&[u8]> { + match self.get(key)? { + DValue::Raw(v) => Some(v), + _ => None, + } + } + /// An RGB colour object (`RGBC` with `Rd `/`Grn `/`Bl ` in 0..255) as 0..1 values; also + /// reads grayscale (`Gry `) colours. + pub fn color(&self, key: &str) -> Option<[f64; 3]> { + let c = self.obj(key)?; + if let (Some(r), Some(g), Some(b)) = (c.num("Rd "), c.num("Grn "), c.num("Bl ")) { + return Some([r / 255.0, g / 255.0, b / 255.0]); + } + if let (Some(r), Some(g), Some(b)) = (c.num("redFloat"), c.num("greenFloat"), c.num("blueFloat")) { + return Some([r, g, b]); + } + c.num("Gry ").map(|g| { + let v = 1.0 - g / 100.0; + [v, v, v] + }) + } +} + +/// A key or class id: length-prefixed string, or a 4-character code when the length is 0. +pub(crate) fn read_id(r: &mut Reader) -> Result { + let n = r.u32()? as usize; + let n = if n == 0 { 4 } else { n }; + let b = r.bytes(n)?; + Ok(String::from_utf8_lossy(b).into_owned()) +} + +pub(crate) fn read_unicode(r: &mut Reader) -> Result { + let n = r.u32()? as usize; + let mut v = Vec::with_capacity(n); + for _ in 0..n { + v.push(r.u16()?); + } + while v.last() == Some(&0) { + v.pop(); + } + Ok(String::from_utf16_lossy(&v)) +} + +/// Read a descriptor (after its version field). +pub fn read_descriptor(r: &mut Reader) -> Result { + let class_name = read_unicode(r)?; + let class_id = read_id(r)?; + let n = r.u32()? as usize; + if n > 100_000 { + return Err(Error::Invalid("descriptor item count".into())); + } + let mut items = Vec::with_capacity(n.min(64)); + for _ in 0..n { + let key = read_id(r)?; + let ty = r.tag()?; + let v = read_value(r, &ty, 0)?; + items.push((key, v)); + } + Ok(Descriptor { class_name, class_id, items }) +} + +fn read_value(r: &mut Reader, ty: &[u8; 4], depth: usize) -> Result { + if depth > 64 { + return Err(Error::Invalid("descriptor nesting".into())); + } + Ok(match ty { + b"Objc" | b"GlbO" => DValue::Descriptor(read_descriptor(r)?), + b"VlLs" => { + let n = r.u32()? as usize; + if n > 1_000_000 { + return Err(Error::Invalid("descriptor list length".into())); + } + let mut v = Vec::with_capacity(n.min(256)); + for _ in 0..n { + let t = r.tag()?; + v.push(read_value(r, &t, depth + 1)?); + } + DValue::List(v) + } + b"doub" => DValue::Double(r.f64()?), + b"UntF" => { + let unit = String::from_utf8_lossy(&r.tag()?).into_owned(); + DValue::UnitFloat(unit, r.f64()?) + } + b"UnFl" => { + let unit = String::from_utf8_lossy(&r.tag()?).into_owned(); + let n = r.u32()? as usize; + let mut v = Vec::with_capacity(n.min(1024)); + for _ in 0..n { + v.push(r.f64()?); + } + DValue::UnitFloats(unit, v) + } + b"TEXT" => DValue::Text(read_unicode(r)?), + b"enum" => { + let t = read_id(r)?; + DValue::Enum(t, read_id(r)?) + } + b"long" => DValue::Integer(r.u32()? as i32), + b"comp" => DValue::LargeInteger(r.u64()? as i64), + b"bool" => DValue::Bool(r.u8()? != 0), + b"type" | b"GlbC" => { + let name = read_unicode(r)?; + DValue::Class(name, read_id(r)?) + } + b"alis" | b"tdta" | b"Pth " => { + let n = r.u32()? as usize; + DValue::Raw(r.bytes(n)?.to_vec()) + } + b"obj " => { + let n = r.u32()? as usize; + let mut items = Vec::with_capacity(n.min(16)); + for _ in 0..n { + let t = r.tag()?; + items.push(match &t { + b"prop" => { + let _name = read_unicode(r)?; + let class = read_id(r)?; + RefItem::Property(class, read_id(r)?) + } + b"Clss" => { + let _name = read_unicode(r)?; + RefItem::Class(read_id(r)?) + } + b"Enmr" => { + let _name = read_unicode(r)?; + let class = read_id(r)?; + let ty = read_id(r)?; + RefItem::Enum(class, ty, read_id(r)?) + } + b"rele" => { + let _name = read_unicode(r)?; + let class = read_id(r)?; + RefItem::Offset(class, r.u32()? as i32) + } + b"Idnt" => RefItem::Identifier(r.u32()? as i32), + b"indx" => RefItem::Index(r.u32()? as i32), + b"name" => { + let _name = read_unicode(r)?; + let class = read_id(r)?; + RefItem::Name(class, read_unicode(r)?) + } + other => return Err(Error::Invalid(format!("reference item `{}`", String::from_utf8_lossy(other)))), + }); + } + DValue::Reference(items) + } + other => return Err(Error::Invalid(format!("descriptor type `{}`", String::from_utf8_lossy(other)))), + }) +} + +// ---------------------------------------------------------------- writing + +fn put_u32(out: &mut Vec, v: u32) { + out.extend_from_slice(&v.to_be_bytes()); +} + +fn put_id(out: &mut Vec, id: &str) { + if id.len() == 4 { + put_u32(out, 0); + } else { + put_u32(out, id.len() as u32); + } + out.extend_from_slice(id.as_bytes()); +} + +pub(crate) fn put_unicode(out: &mut Vec, s: &str) { + let v: Vec = s.encode_utf16().chain(std::iter::once(0)).collect(); + put_u32(out, v.len() as u32); + for c in v { + out.extend_from_slice(&c.to_be_bytes()); + } +} + +/// Serialise a descriptor (without the version field). +pub fn write_descriptor(out: &mut Vec, d: &Descriptor) { + put_unicode(out, &d.class_name); + put_id(out, &d.class_id); + put_u32(out, d.items.len() as u32); + for (k, v) in &d.items { + put_id(out, k); + write_value(out, v); + } +} + +fn write_value(out: &mut Vec, v: &DValue) { + match v { + DValue::Descriptor(d) => { + out.extend_from_slice(b"Objc"); + write_descriptor(out, d); + } + DValue::List(l) => { + out.extend_from_slice(b"VlLs"); + put_u32(out, l.len() as u32); + for x in l { + write_value(out, x); + } + } + DValue::Double(x) => { + out.extend_from_slice(b"doub"); + out.extend_from_slice(&x.to_be_bytes()); + } + DValue::UnitFloat(u, x) => { + out.extend_from_slice(b"UntF"); + out.extend_from_slice(&tag4(u)); + out.extend_from_slice(&x.to_be_bytes()); + } + DValue::UnitFloats(u, xs) => { + out.extend_from_slice(b"UnFl"); + out.extend_from_slice(&tag4(u)); + put_u32(out, xs.len() as u32); + for x in xs { + out.extend_from_slice(&x.to_be_bytes()); + } + } + DValue::Text(s) => { + out.extend_from_slice(b"TEXT"); + put_unicode(out, s); + } + DValue::Enum(t, e) => { + out.extend_from_slice(b"enum"); + put_id(out, t); + put_id(out, e); + } + DValue::Integer(i) => { + out.extend_from_slice(b"long"); + out.extend_from_slice(&i.to_be_bytes()); + } + DValue::LargeInteger(i) => { + out.extend_from_slice(b"comp"); + out.extend_from_slice(&i.to_be_bytes()); + } + DValue::Bool(b) => { + out.extend_from_slice(b"bool"); + out.push(*b as u8); + } + DValue::Class(n, c) => { + out.extend_from_slice(b"type"); + put_unicode(out, n); + put_id(out, c); + } + DValue::Raw(b) => { + out.extend_from_slice(b"tdta"); + put_u32(out, b.len() as u32); + out.extend_from_slice(b); + } + DValue::Reference(items) => { + out.extend_from_slice(b"obj "); + put_u32(out, items.len() as u32); + for it in items { + match it { + RefItem::Property(c, k) => { + out.extend_from_slice(b"prop"); + put_unicode(out, ""); + put_id(out, c); + put_id(out, k); + } + RefItem::Class(c) => { + out.extend_from_slice(b"Clss"); + put_unicode(out, ""); + put_id(out, c); + } + RefItem::Enum(c, t, e) => { + out.extend_from_slice(b"Enmr"); + put_unicode(out, ""); + put_id(out, c); + put_id(out, t); + put_id(out, e); + } + RefItem::Offset(c, o) => { + out.extend_from_slice(b"rele"); + put_unicode(out, ""); + put_id(out, c); + out.extend_from_slice(&o.to_be_bytes()); + } + RefItem::Identifier(i) => { + out.extend_from_slice(b"Idnt"); + out.extend_from_slice(&i.to_be_bytes()); + } + RefItem::Index(i) => { + out.extend_from_slice(b"indx"); + out.extend_from_slice(&i.to_be_bytes()); + } + RefItem::Name(c, n) => { + out.extend_from_slice(b"name"); + put_unicode(out, ""); + put_id(out, c); + put_unicode(out, n); + } + } + } + } + } +} + +fn tag4(s: &str) -> [u8; 4] { + let mut t = *b" "; + for (i, b) in s.bytes().take(4).enumerate() { + t[i] = b; + } + t +} + +/// Builder helpers for descriptors (tests and the writer). +impl Descriptor { + pub fn new(class_id: &str) -> Descriptor { + Descriptor { class_name: String::new(), class_id: class_id.into(), items: vec![] } + } + pub fn with(mut self, key: &str, v: DValue) -> Descriptor { + self.items.push((key.into(), v)); + self + } + /// An `RGBC` colour object from 0..1 values. + pub fn rgb(c: [f64; 3]) -> Descriptor { + Descriptor::new("RGBC").with("Rd ", DValue::Double(c[0] * 255.0)).with("Grn ", DValue::Double(c[1] * 255.0)).with("Bl ", DValue::Double(c[2] * 255.0)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trip() { + let d = Descriptor::new("null") + .with("Txt ", DValue::Text("Héllo".into())) + .with("Clr ", DValue::Descriptor(Descriptor::rgb([1.0, 0.5, 0.0]))) + .with("Opct", DValue::UnitFloat("#Prc".into(), 75.0)) + .with("Md ", DValue::Enum("BlnM".into(), "Mltp".into())) + .with("enab", DValue::Bool(true)) + .with("longKeyName", DValue::List(vec![DValue::Integer(3), DValue::Double(1.5)])) + .with("EngineData", DValue::Raw(b"<< >>".to_vec())) + .with("ref", DValue::Reference(vec![RefItem::Enum("Lyr ".into(), "Ordn".into(), "Trgt".into()), RefItem::Index(2)])); + let mut out = vec![]; + write_descriptor(&mut out, &d); + let back = read_descriptor(&mut Reader::new(&out)).unwrap(); + assert_eq!(back.class_id, "null"); + assert_eq!(back.text("Txt "), Some("Héllo")); + assert_eq!(back.color("Clr "), Some([1.0, 0.5, 0.0])); + assert_eq!(back.num("Opct"), Some(75.0)); + assert_eq!(back.enum_value("Md "), Some("Mltp")); + assert_eq!(back.bool("enab"), Some(true)); + assert_eq!(back.raw("EngineData"), Some(&b"<< >>"[..])); + assert_eq!(back, d); + } +} diff --git a/crates/psd/src/engine_data.rs b/crates/psd/src/engine_data.rs new file mode 100644 index 00000000..c4f23038 --- /dev/null +++ b/crates/psd/src/engine_data.rs @@ -0,0 +1,288 @@ +//! The text engine data of type layers (`EngineData` in the `TySh` descriptor): a PostScript-like +//! tree of dictionaries (`<< /Key value >>`), arrays (`[ ]`), numbers, names, booleans and +//! strings (`(þÿ` + UTF-16BE with backslash escapes `)`). Read best-effort. + +#[derive(Clone, Debug, PartialEq)] +pub enum EValue { + Dict(Vec<(String, EValue)>), + Array(Vec), + Num(f64), + Bool(bool), + Name(String), + Str(String), +} + +impl EValue { + pub fn get(&self, key: &str) -> Option<&EValue> { + match self { + EValue::Dict(v) => v.iter().find(|(k, _)| k == key).map(|(_, v)| v), + _ => None, + } + } + pub fn idx(&self, i: usize) -> Option<&EValue> { + match self { + EValue::Array(v) => v.get(i), + _ => None, + } + } + /// Follow a `/`-separated path; numeric segments index arrays. + pub fn path(&self, p: &str) -> Option<&EValue> { + let mut cur = self; + for seg in p.split('/').filter(|s| !s.is_empty()) { + cur = match seg.parse::() { + Ok(i) => cur.idx(i)?, + Err(_) => cur.get(seg)?, + }; + } + Some(cur) + } + pub fn num(&self) -> Option { + match self { + EValue::Num(n) => Some(*n), + _ => None, + } + } + pub fn str(&self) -> Option<&str> { + match self { + EValue::Str(s) | EValue::Name(s) => Some(s), + _ => None, + } + } +} + +struct P<'a> { + b: &'a [u8], + i: usize, +} + +impl P<'_> { + fn ws(&mut self) { + while self.i < self.b.len() && (self.b[self.i].is_ascii_whitespace() || self.b[self.i] == 0) { + self.i += 1; + } + } + fn peek(&self, s: &[u8]) -> bool { + self.b[self.i..].starts_with(s) + } + fn value(&mut self, depth: usize) -> Option { + if depth > 128 { + return None; + } + self.ws(); + if self.i >= self.b.len() { + return None; + } + if self.peek(b"<<") { + self.i += 2; + let mut v = vec![]; + loop { + self.ws(); + if self.i >= self.b.len() { + return Some(EValue::Dict(v)); + } + if self.peek(b">>") { + self.i += 2; + return Some(EValue::Dict(v)); + } + if self.b[self.i] != b'/' { + // Unexpected token: skip it. + self.i += 1; + continue; + } + let key = self.name(); + let val = self.value(depth + 1)?; + v.push((key, val)); + } + } + match self.b[self.i] { + b'[' => { + self.i += 1; + let mut v = vec![]; + loop { + self.ws(); + if self.i >= self.b.len() { + return Some(EValue::Array(v)); + } + if self.b[self.i] == b']' { + self.i += 1; + return Some(EValue::Array(v)); + } + let start = self.i; + match self.value(depth + 1) { + Some(x) => v.push(x), + None => return Some(EValue::Array(v)), + } + if self.i == start { + self.i += 1; + } + } + } + b'/' => Some(EValue::Name(self.name())), + b'(' => Some(EValue::Str(self.string())), + _ => { + let start = self.i; + while self.i < self.b.len() && !self.b[self.i].is_ascii_whitespace() && !b"[]<>/(".contains(&self.b[self.i]) { + self.i += 1; + } + let tok = std::str::from_utf8(&self.b[start..self.i]).unwrap_or(""); + if self.i == start { + self.i += 1; + return Some(EValue::Name(String::new())); + } + Some(match tok { + "true" => EValue::Bool(true), + "false" => EValue::Bool(false), + t => t.parse::().map(EValue::Num).unwrap_or_else(|_| EValue::Name(t.to_string())), + }) + } + } + } + fn name(&mut self) -> String { + self.i += 1; // '/' + let start = self.i; + while self.i < self.b.len() && !self.b[self.i].is_ascii_whitespace() && !b"[]<>/(".contains(&self.b[self.i]) { + self.i += 1; + } + String::from_utf8_lossy(&self.b[start..self.i]).into_owned() + } + fn string(&mut self) -> String { + self.i += 1; // '(' + let mut raw = vec![]; + while self.i < self.b.len() { + let c = self.b[self.i]; + self.i += 1; + match c { + b'\\' if self.i < self.b.len() => { + let e = self.b[self.i]; + self.i += 1; + raw.push(match e { + b'n' => b'\n', + b'r' => b'\r', + b't' => b'\t', + other => other, + }); + } + b')' => break, + _ => raw.push(c), + } + } + if raw.starts_with(&[0xFE, 0xFF]) { + let u: Vec = raw[2..].chunks_exact(2).map(|c| u16::from_be_bytes([c[0], c[1]])).collect(); + String::from_utf16_lossy(&u).trim_end_matches('\0').to_string() + } else { + String::from_utf8_lossy(&raw).into_owned() + } + } +} + +/// Parse engine data (returns an empty dictionary on garbage). +pub fn parse(b: &[u8]) -> EValue { + let mut p = P { b, i: 0 }; + p.value(0).unwrap_or(EValue::Dict(vec![])) +} + +/// Encode a string the way engine data stores it (UTF-16BE with a BOM, escaped). +pub fn encode_string(s: &str) -> Vec { + let mut out = vec![b'(', 0xFE, 0xFF]; + for u in s.encode_utf16() { + for b in u.to_be_bytes() { + if matches!(b, b'(' | b')' | b'\\') { + out.push(b'\\'); + } + out.push(b); + } + } + out.push(b')'); + out +} + +/// The first style run's character attributes. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct TextStyle { + pub font: Option, + pub size: Option, + /// RGB 0..1. + pub color: Option<[f64; 3]>, + /// 0 left, 1 right, 2 centre, 3+ justified variants. + pub justification: Option, + pub tracking: Option, + pub leading: Option, + pub faux_bold: bool, + pub faux_italic: bool, + /// Paragraph (box) text rather than point text. + pub box_text: bool, +} + +pub fn text_style(ed: &EValue) -> TextStyle { + let engine = ed.get("EngineDict").unwrap_or(ed); + let resources = ed.get("ResourceDict").or_else(|| ed.get("DocumentResources")); + let run = engine.path("StyleRun/RunArray/0/StyleSheet/StyleSheetData"); + let normal = resources.and_then(|r| r.path("StyleSheetSet/0/StyleSheetData")); + let attr = |k: &str| run.and_then(|r| r.get(k)).or_else(|| normal.and_then(|n| n.get(k))); + let font = + attr("Font").and_then(EValue::num).and_then(|i| resources?.path(&format!("FontSet/{}/Name", i as usize))).and_then(EValue::str).map(str::to_string); + let color = attr("FillColor").and_then(|c| c.get("Values")).and_then(|v| match v { + EValue::Array(a) if a.len() >= 4 => Some([a[1].num()?, a[2].num()?, a[3].num()?]), + _ => None, + }); + let para = engine.path("ParagraphRun/RunArray/0/ParagraphSheet/Properties"); + let para_normal = resources.and_then(|r| r.path("ParagraphSheetSet/0/Properties")); + let justification = + para.and_then(|p| p.get("Justification")).or_else(|| para_normal.and_then(|p| p.get("Justification"))).and_then(EValue::num).map(|n| n as u8); + let auto_leading = attr("AutoLeading").is_none_or(|v| *v != EValue::Bool(false)); + let box_text = engine.path("Rendered/Shapes/Children/0/ShapeType").and_then(EValue::num) == Some(1.0); + TextStyle { + font, + size: attr("FontSize").and_then(EValue::num), + color, + justification, + tracking: attr("Tracking").and_then(EValue::num), + leading: if auto_leading { None } else { attr("Leading").and_then(EValue::num) }, + faux_bold: attr("FauxBold") == Some(&EValue::Bool(true)), + faux_italic: attr("FauxItalic") == Some(&EValue::Bool(true)), + box_text, + } +} + +/// Minimal engine data for a type layer with one style run (fixtures, and what a writer needs). +pub fn minimal(text: &str, font: &str, size: f64, color: [f64; 3], justification: u8) -> Vec { + let mut out = Vec::new(); + out.extend_from_slice(b"\n\n<<\n\t/EngineDict\n\t<<\n\t\t/Editor << /Text "); + out.extend_from_slice(&encode_string(&format!("{text}\r"))); + out.extend_from_slice(b" >>\n\t\t/ParagraphRun << /RunArray [ << /ParagraphSheet << /Properties << /Justification "); + out.extend_from_slice(justification.to_string().as_bytes()); + out.extend_from_slice(b" >> >> >> ] >>\n\t\t/StyleRun << /RunArray [ << /StyleSheet << /StyleSheetData << /Font 0 /FontSize "); + out.extend_from_slice(format!("{size:.1}").as_bytes()); + out.extend_from_slice( + format!(" /FillColor << /Type 1 /Values [ 1.0 {:.5} {:.5} {:.5} ] >> /AutoLeading true >> >> >> ] >>\n\t>>\n", color[0], color[1], color[2]).as_bytes(), + ); + out.extend_from_slice(b"\t/ResourceDict << /FontSet [ << /Name "); + out.extend_from_slice(&encode_string(font)); + out.extend_from_slice(b" /Type 0 >> ] >>\n>>"); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_minimal() { + let b = minimal("Hi (there)", "Arial-BoldMT", 36.0, [1.0, 0.5, 0.0], 2); + let v = parse(&b); + assert_eq!(v.path("EngineDict/Editor/Text").and_then(EValue::str), Some("Hi (there)\r")); + let st = text_style(&v); + assert_eq!(st.font.as_deref(), Some("Arial-BoldMT")); + assert_eq!(st.size, Some(36.0)); + assert_eq!(st.color, Some([1.0, 0.5, 0.0])); + assert_eq!(st.justification, Some(2)); + assert!(!st.box_text); + } + + #[test] + fn garbage_is_harmless() { + let _ = parse(b"<< /A [ 1 2 << /B (unterminated"); + let _ = parse(b"]]]>>>> /x"); + let _ = parse(&[0xFF; 64]); + } +} diff --git a/crates/psd/src/lib.rs b/crates/psd/src/lib.rs new file mode 100644 index 00000000..885ab3c2 --- /dev/null +++ b/crates/psd/src/lib.rs @@ -0,0 +1,965 @@ +//! Photoshop document (PSD / PSB) reader, implemented from Adobe's public *Photoshop File Formats +//! Specification* (see README.md), and a minimal writer ([`write`]) for fixtures. +//! +//! [`Psd::parse`] reads the structure (header, layer records and their additional information, +//! the layer tree) without decoding pixels; [`Psd::composite`] decodes the merged image and +//! [`Psd::layer_pixels`] one layer (with its layer mask applied to alpha) on demand. Pixels come +//! out as straight (unpremultiplied) RGBA `f32` in 0..1, display-encoded (32-bit documents, +//! which are linear, are encoded to sRGB). + +pub mod descriptor; +pub mod engine_data; +mod read; +pub mod write; + +use std::sync::Arc; + +pub use descriptor::{DValue, Descriptor}; +use read::Reader; + +#[derive(Debug, Clone, PartialEq)] +pub enum Error { + NotPsd, + Truncated, + Unsupported(String), + Invalid(String), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::NotPsd => write!(f, "not a Photoshop document"), + Error::Truncated => write!(f, "truncated Photoshop document"), + Error::Unsupported(s) => write!(f, "unsupported Photoshop feature: {s}"), + Error::Invalid(s) => write!(f, "invalid Photoshop document: {s}"), + } + } +} + +impl std::error::Error for Error {} + +pub type Result = std::result::Result; + +/// Whether `bytes` start like a Photoshop document. +pub fn is_psd(bytes: &[u8]) -> bool { + bytes.len() >= 6 && &bytes[..4] == b"8BPS" && matches!(u16::from_be_bytes([bytes[4], bytes[5]]), 1 | 2) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ColorMode { + Bitmap, + Grayscale, + Indexed, + Rgb, + Cmyk, + Multichannel, + Duotone, + Lab, +} + +impl ColorMode { + pub fn from_u16(v: u16) -> Option { + Some(match v { + 0 => ColorMode::Bitmap, + 1 => ColorMode::Grayscale, + 2 => ColorMode::Indexed, + 3 => ColorMode::Rgb, + 4 => ColorMode::Cmyk, + 7 => ColorMode::Multichannel, + 8 => ColorMode::Duotone, + 9 => ColorMode::Lab, + _ => return None, + }) + } + pub fn to_u16(self) -> u16 { + match self { + ColorMode::Bitmap => 0, + ColorMode::Grayscale => 1, + ColorMode::Indexed => 2, + ColorMode::Rgb => 3, + ColorMode::Cmyk => 4, + ColorMode::Multichannel => 7, + ColorMode::Duotone => 8, + ColorMode::Lab => 9, + } + } + /// Number of colour channels (before alpha). + pub fn color_channels(self) -> usize { + match self { + ColorMode::Rgb | ColorMode::Lab => 3, + ColorMode::Cmyk => 4, + _ => 1, + } + } +} + +/// A rectangle in document pixels (`right`/`bottom` exclusive). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Rect { + pub top: i32, + pub left: i32, + pub bottom: i32, + pub right: i32, +} + +impl Rect { + pub fn new(left: i32, top: i32, width: i32, height: i32) -> Rect { + Rect { top, left, bottom: top + height, right: left + width } + } + pub fn width(&self) -> u32 { + (self.right - self.left).max(0) as u32 + } + pub fn height(&self) -> u32 { + (self.bottom - self.top).max(0) as u32 + } + pub fn is_empty(&self) -> bool { + self.width() == 0 || self.height() == 0 + } +} + +/// One channel of a layer: id (0.. colour, -1 transparency, -2 layer mask, -3 real user mask) and +/// where its data (compression + image data) sits in the file. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ChannelInfo { + pub id: i16, + pub offset: usize, + pub len: usize, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LayerMask { + pub rect: Rect, + /// Value outside the mask rectangle (0 or 255). + pub default_color: u8, + pub disabled: bool, + pub inverted: bool, +} + +/// Layer kind from the section divider setting (`lsct`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Section { + #[default] + Layer, + OpenFolder, + ClosedFolder, + /// The hidden "" record that closes a group (below its children). + Divider, +} + +/// A type layer (`TySh`). +#[derive(Clone, Debug, Default, PartialEq)] +pub struct TextInfo { + /// Text with `\n` line breaks. + pub text: String, + /// Affine transform (xx, xy, yx, yy, tx, ty): the text origin sits at (tx, ty). + pub transform: [f64; 6], + pub style: engine_data::TextStyle, + /// Text bounds (left, top, right, bottom) relative to the origin, when stored. + pub bounds: Option<[f64; 4]>, +} + +/// Adjustment layers (best effort). +#[derive(Clone, Debug, PartialEq)] +pub enum Adjustment { + Invert, + BrightnessContrast { + brightness: f64, + contrast: f64, + legacy: bool, + }, + HueSaturation { + hue: f64, + saturation: f64, + lightness: f64, + colorize: bool, + }, + Levels { + in_black: f64, + in_white: f64, + gamma: f64, + out_black: f64, + out_white: f64, + }, + Exposure { + exposure: f64, + offset: f64, + gamma: f64, + }, + /// Another adjustment (its tagged-block key, e.g. `curv`, `blnc`). + Other(String), +} + +/// A vector mask path: sub-paths of (in tangent, vertex, out tangent) points in document pixels. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct VectorMask { + pub inverted: bool, + pub disabled: bool, + pub subpaths: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct SubPath { + pub closed: bool, + /// Per knot: [control point preceding, anchor, control point leaving] as (x, y). + pub knots: Vec<[[f64; 2]; 3]>, +} + +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Layer { + /// Index in [`Psd::layers`] (file order: bottom of the stack first). + pub index: usize, + pub name: String, + pub id: Option, + pub rect: Rect, + /// Blend mode key (`norm`, `mul `, `pass`…). + pub blend: [u8; 4], + pub opacity: u8, + /// Fill opacity (`iOpa`), 255 when absent. + pub fill_opacity: u8, + /// Clipped to the layer below. + pub clipping: bool, + pub hidden: bool, + pub protect_transparency: bool, + pub section: Section, + pub channels: Vec, + pub mask: Option, + pub text: Option, + pub adjustment: Option, + /// Solid colour fill layer (`SoCo`), RGB 0..1. + pub fill: Option<[f64; 3]>, + /// Layer effects (`lfx2` descriptor). + pub effects: Option, + pub vector_mask: Option, + /// Keys of every additional-information block the layer carries. + pub keys: Vec<[u8; 4]>, +} + +impl Layer { + pub fn is_group(&self) -> bool { + matches!(self.section, Section::OpenFolder | Section::ClosedFolder) + } + pub fn blend_key(&self) -> &str { + std::str::from_utf8(&self.blend).unwrap_or("norm") + } + pub fn has_pixels(&self) -> bool { + !self.rect.is_empty() && self.channels.iter().any(|c| c.id >= 0) + } +} + +/// The layer tree, top of the stack first. +#[derive(Clone, Debug, PartialEq)] +pub enum Node { + Layer(usize), + Group { layer: usize, children: Vec }, +} + +/// Decoded pixels: straight RGBA, row-major. +#[derive(Clone, Debug, PartialEq)] +pub struct Pixels { + pub width: u32, + pub height: u32, + pub data: Vec<[f32; 4]>, +} + +/// A parsed Photoshop document. +#[derive(Clone, Debug)] +pub struct Psd { + /// PSB (large document format). + pub psb: bool, + pub width: u32, + pub height: u32, + pub channels: u16, + pub depth: u16, + pub mode: ColorMode, + /// Bottom of the stack first (file order). + pub layers: Vec, + /// The merged image's first extra channel is transparency. + pub merged_alpha: bool, + /// Indexed colour palette (256 R, then G, then B). + pub palette: Vec, + data: Arc<[u8]>, + merged_offset: usize, +} + +impl Psd { + pub fn parse(data: impl Into>) -> Result { + let data: Arc<[u8]> = data.into(); + let mut r = Reader::new(&data); + if r.bytes(4).ok() != Some(b"8BPS") { + return Err(Error::NotPsd); + } + let version = r.u16()?; + let psb = match version { + 1 => false, + 2 => true, + _ => return Err(Error::NotPsd), + }; + r.skip(6)?; + let channels = r.u16()?; + let height = r.u32()?; + let width = r.u32()?; + let depth = r.u16()?; + let mode_v = r.u16()?; + let mode = ColorMode::from_u16(mode_v).ok_or_else(|| Error::Unsupported(format!("colour mode {mode_v}")))?; + if !matches!(depth, 1 | 8 | 16 | 32) { + return Err(Error::Unsupported(format!("{depth}-bit"))); + } + if width == 0 || height == 0 || width > 300_000 || height > 300_000 { + return Err(Error::Invalid(format!("size {width}x{height}"))); + } + // Colour mode data. + let n = r.u32()? as usize; + let palette = r.bytes(n)?.to_vec(); + // Image resources. + let n = r.u32()? as usize; + r.skip(n)?; + // Layer and mask information. + let mut psd = Psd { psb, width, height, channels, depth, mode, layers: vec![], merged_alpha: false, palette, data: data.clone(), merged_offset: 0 }; + let lm_len = r.length(psb)?; + let lm_end = r.pos.checked_add(lm_len).filter(|&e| e <= data.len()).ok_or(Error::Truncated)?; + if lm_len > 0 { + let li_len = r.length(psb)?; + let li_end = r.pos.checked_add(li_len).filter(|&e| e <= lm_end).ok_or(Error::Truncated)?; + if li_len > 0 { + psd.layer_info(&data, r.pos, li_end)?; + } + r.pos = li_end; + if r.pos + 4 <= lm_end { + let g = r.u32()? as usize; + r.pos = (r.pos + g).min(lm_end); + } + // Global additional layer information (16/32-bit layer info lives here). + while let Some((key, start, end)) = next_block(&data, &mut r.pos, lm_end, psb) { + if matches!(&key, b"Lr16" | b"Lr32" | b"Layr") && psd.layers.is_empty() && end > start { + // The block holds the layer info, normally without its length field; accept + // both layouts. + if psd.layer_info(&data, start, end).is_err() || psd.layers.is_empty() { + psd.layers.clear(); + let skip = if psb { 8 } else { 4 }; + if start + skip < end && psd.layer_info(&data, start + skip, end).is_err() { + psd.layers.clear(); + } + } + } + r.pos = end; + } + } + psd.merged_offset = lm_end; + Ok(psd) + } + + fn layer_info(&mut self, data: &[u8], start: usize, end: usize) -> Result<()> { + let mut r = Reader::at(&data[..end], start); + let count = r.i16()?; + self.merged_alpha = count < 0; + let n = count.unsigned_abs() as usize; + let mut layers = Vec::with_capacity(n); + for i in 0..n { + layers.push(self.layer_record(&mut r, i)?); + } + // Channel image data follows in record order. + for l in &mut layers { + for c in &mut l.channels { + c.offset = r.pos; + r.skip(c.len)?; + } + } + self.layers = layers; + Ok(()) + } + + fn layer_record(&self, r: &mut Reader, index: usize) -> Result { + let psb = self.psb; + let (top, left, bottom, right) = (r.i32()?, r.i32()?, r.i32()?, r.i32()?); + let nch = r.u16()? as usize; + if nch > 64 { + return Err(Error::Invalid("channel count".into())); + } + let mut channels = Vec::with_capacity(nch); + for _ in 0..nch { + let id = r.i16()?; + let len = r.length(psb)?; + channels.push(ChannelInfo { id, offset: 0, len }); + } + if r.bytes(4)? != b"8BIM" { + return Err(Error::Invalid("layer record signature".into())); + } + let blend = r.tag()?; + let opacity = r.u8()?; + let clipping = r.u8()? != 0; + let flags = r.u8()?; + r.skip(1)?; + let extra = r.u32()? as usize; + let extra_end = r.pos.checked_add(extra).filter(|&e| e <= r.data.len()).ok_or(Error::Truncated)?; + let mut l = Layer { + index, + rect: Rect { top, left, bottom, right }, + blend, + opacity, + fill_opacity: 255, + clipping, + hidden: flags & 2 != 0, + protect_transparency: flags & 1 != 0, + channels, + ..Default::default() + }; + // Layer mask data. + let mlen = r.u32()? as usize; + let mstart = r.pos; + if mlen >= 18 { + let rect = Rect { top: r.i32()?, left: r.i32()?, bottom: r.i32()?, right: r.i32()? }; + let default_color = r.u8()?; + let mflags = r.u8()?; + l.mask = Some(LayerMask { rect, default_color, disabled: mflags & 2 != 0, inverted: mflags & 4 != 0 }); + } + r.pos = mstart + mlen; + // Blending ranges. + let blen = r.u32()? as usize; + r.skip(blen)?; + l.name = r.pascal(4)?; + let mut pos = r.pos; + let data = r.data; + while let Some((key, start, end)) = next_block(data, &mut pos, extra_end, psb) { + l.keys.push(key); + let block = &data[start..end]; + // A malformed block only loses that block's information. + let _ = self.layer_block(&mut l, &key, block); + pos = end; + } + r.pos = extra_end; + Ok(l) + } + + fn layer_block(&self, l: &mut Layer, key: &[u8; 4], b: &[u8]) -> Result<()> { + let mut r = Reader::new(b); + match key { + b"luni" => { + let name = descriptor::read_unicode(&mut r)?; + if !name.is_empty() { + l.name = name; + } + } + b"lyid" => l.id = Some(r.u32()?), + b"iOpa" => l.fill_opacity = r.u8()?, + b"lsct" | b"lsdk" => { + l.section = match r.u32()? { + 1 => Section::OpenFolder, + 2 => Section::ClosedFolder, + 3 => Section::Divider, + _ => Section::Layer, + }; + if b.len() >= 12 && &b[4..8] == b"8BIM" { + l.blend.copy_from_slice(&b[8..12]); + } + } + b"TySh" => { + let _ver = r.u16()?; + let mut t = [0.0; 6]; + for v in &mut t { + *v = r.f64()?; + } + let _text_ver = r.u16()?; + let _desc_ver = r.u32()?; + let d = descriptor::read_descriptor(&mut r)?; + let text = d.text("Txt ").unwrap_or_default().replace("\r\n", "\n").replace('\r', "\n"); + let text = text.strip_suffix('\n').map(str::to_string).unwrap_or(text); + let style = d.raw("EngineData").map(|e| engine_data::text_style(&engine_data::parse(e))).unwrap_or_default(); + let bounds = + d.obj("bounds").or_else(|| d.obj("boundingBox")).and_then(|o| Some([o.num("Left")?, o.num("Top ")?, o.num("Rght")?, o.num("Btom")?])); + l.text = Some(TextInfo { text, transform: t, style, bounds }); + } + b"lfx2" => { + let _ver = r.u32()?; + let _dver = r.u32()?; + l.effects = Some(descriptor::read_descriptor(&mut r)?); + } + b"SoCo" => { + let _ver = r.u32()?; + let d = descriptor::read_descriptor(&mut r)?; + l.fill = d.color("Clr "); + } + b"nvrt" => l.adjustment = Some(Adjustment::Invert), + b"brit" => { + let brightness = r.i16()? as f64; + let contrast = r.i16()? as f64; + if l.adjustment.is_none() { + l.adjustment = Some(Adjustment::BrightnessContrast { brightness, contrast, legacy: true }); + } + } + b"CgEd" => { + let _ver = r.u32()?; + let d = descriptor::read_descriptor(&mut r)?; + l.adjustment = Some(Adjustment::BrightnessContrast { + brightness: d.num("Brgh").unwrap_or(0.0), + contrast: d.num("Cntr").unwrap_or(0.0), + legacy: d.bool("useLegacy").unwrap_or(false), + }); + } + b"hue2" | b"hue " => { + let _ver = r.u16()?; + let colorize = r.u8()? != 0; + r.skip(1)?; + let ch = (r.i16()? as f64, r.i16()? as f64, r.i16()? as f64); + let m = (r.i16()? as f64, r.i16()? as f64, r.i16()? as f64); + let (hue, saturation, lightness) = if colorize { ch } else { m }; + l.adjustment = Some(Adjustment::HueSaturation { hue, saturation, lightness, colorize }); + } + b"levl" => { + let _ver = r.u16()?; + let (ib, iw, ob, ow, g) = (r.i16()?, r.i16()?, r.i16()?, r.i16()?, r.i16()?); + l.adjustment = + Some(Adjustment::Levels { in_black: ib as f64, in_white: iw as f64, gamma: g as f64 / 100.0, out_black: ob as f64, out_white: ow as f64 }); + } + b"expA" => { + let _ver = r.u16()?; + let exposure = r.f32()? as f64; + let offset = r.f32()? as f64; + let gamma = r.f32()? as f64; + l.adjustment = Some(Adjustment::Exposure { exposure, offset, gamma }); + } + b"curv" | b"blnc" | b"post" | b"thrs" | b"selc" | b"mixr" | b"phfl" | b"vibA" | b"blwh" | b"grdm" | b"clrL" => { + if l.adjustment.is_none() { + l.adjustment = Some(Adjustment::Other(String::from_utf8_lossy(key).into_owned())); + } + } + b"vmsk" | b"vsms" => { + let _ver = r.u32()?; + let flags = r.u32()?; + l.vector_mask = Some(self.vector_paths(&mut r, flags)?); + } + _ => {} + } + Ok(()) + } + + fn vector_paths(&self, r: &mut Reader, flags: u32) -> Result { + let (w, h) = (self.width as f64, self.height as f64); + let mut vm = VectorMask { inverted: flags & 1 != 0, disabled: flags & 4 != 0, subpaths: vec![] }; + let fixed = |r: &mut Reader| -> Result { Ok(r.i32()? as f64 / (1 << 24) as f64) }; + while r.remaining() >= 26 { + let sel = r.u16()?; + match sel { + 0 | 3 => { + let _n = r.u16()?; + r.skip(22)?; + vm.subpaths.push(SubPath { closed: sel == 0, knots: vec![] }); + } + 1 | 2 | 4 | 5 => { + let mut pts = [[0.0; 2]; 3]; + for p in &mut pts { + let y = fixed(r)?; + let x = fixed(r)?; + *p = [x * w, y * h]; + } + if let Some(sp) = vm.subpaths.last_mut() { + sp.knots.push(pts); + } + } + _ => r.skip(24)?, + } + } + vm.subpaths.retain(|s| !s.knots.is_empty()); + Ok(vm) + } + + /// The layer tree, top of the stack first. + pub fn tree(&self) -> Vec { + let mut stack: Vec> = vec![vec![]]; + for l in &self.layers { + match l.section { + Section::Divider => stack.push(vec![]), + Section::OpenFolder | Section::ClosedFolder => { + let children = if stack.len() > 1 { stack.pop().unwrap_or_default() } else { vec![] }; + if let Some(top) = stack.last_mut() { + top.push(Node::Group { layer: l.index, children }); + } + } + Section::Layer => { + if let Some(top) = stack.last_mut() { + top.push(Node::Layer(l.index)); + } + } + } + } + while stack.len() > 1 { + let c = stack.pop().unwrap_or_default(); + if let Some(top) = stack.last_mut() { + top.extend(c); + } + } + fn rev(v: &mut Vec) { + v.reverse(); + for n in v { + if let Node::Group { children, .. } = n { + rev(children); + } + } + } + let mut out = stack.pop().unwrap_or_default(); + rev(&mut out); + out + } + + fn bytes_per_sample(&self) -> usize { + match self.depth { + 16 => 2, + 32 => 4, + _ => 1, + } + } + + /// Decode one channel of `w`×`h` samples at `offset` (compression field first). Values 0..1. + fn decode_channel(&self, offset: usize, len: usize, w: usize, h: usize) -> Result> { + let end = offset.checked_add(len).filter(|&e| e <= self.data.len()).ok_or(Error::Truncated)?; + let mut r = Reader::at(&self.data[..end], offset); + let comp = r.u16()?; + let rowb = self.row_bytes(w); + let raw = match comp { + 0 => r.bytes(rowb * h)?.to_vec(), + 1 => { + let mut counts = Vec::with_capacity(h); + for _ in 0..h { + counts.push(if self.psb { r.u32()? as usize } else { r.u16()? as usize }); + } + let mut out = Vec::with_capacity(rowb * h); + for c in counts { + unpack_bits(r.bytes(c)?, rowb, &mut out)?; + } + out + } + 2 | 3 => { + let rest = r.bytes(r.remaining())?; + let mut v = miniz_oxide::inflate::decompress_to_vec_zlib(rest).map_err(|e| Error::Invalid(format!("zip: {e:?}")))?; + v.resize(rowb * h, 0); + if comp == 3 { + self.unpredict(&mut v, w, h); + } + v + } + c => return Err(Error::Unsupported(format!("compression {c}"))), + }; + Ok(self.samples(&raw, w, h)) + } + + fn row_bytes(&self, w: usize) -> usize { + if self.depth == 1 { w.div_ceil(8) } else { w * self.bytes_per_sample() } + } + + fn unpredict(&self, v: &mut [u8], w: usize, h: usize) { + match self.depth { + 16 => { + for row in v.chunks_exact_mut(w * 2).take(h) { + let mut prev = 0u16; + for c in row.chunks_exact_mut(2) { + let x = u16::from_be_bytes([c[0], c[1]]).wrapping_add(prev); + c.copy_from_slice(&x.to_be_bytes()); + prev = x; + } + } + } + 32 => { + for row in v.chunks_exact_mut(w * 4).take(h) { + for i in 1..row.len() { + row[i] = row[i].wrapping_add(row[i - 1]); + } + // Bytes are stored plane by plane (all first bytes, then all second bytes…). + let planar = row.to_vec(); + for x in 0..w { + for b in 0..4 { + row[x * 4 + b] = planar[b * w + x]; + } + } + } + } + _ => { + for row in v.chunks_exact_mut(w).take(h) { + for i in 1..row.len() { + row[i] = row[i].wrapping_add(row[i - 1]); + } + } + } + } + } + + fn samples(&self, raw: &[u8], w: usize, h: usize) -> Vec { + match self.depth { + 1 => { + let rowb = w.div_ceil(8); + let mut out = Vec::with_capacity(w * h); + for y in 0..h { + for x in 0..w { + let b = raw.get(y * rowb + x / 8).copied().unwrap_or(0); + // 1 = black. + out.push(if b & (0x80 >> (x % 8)) != 0 { 0.0 } else { 1.0 }); + } + } + out + } + 16 => raw.chunks_exact(2).take(w * h).map(|c| u16::from_be_bytes([c[0], c[1]]) as f32 / 65535.0).collect(), + 32 => raw.chunks_exact(4).take(w * h).map(|c| f32::from_be_bytes([c[0], c[1], c[2], c[3]])).collect(), + _ => raw.iter().take(w * h).map(|&b| b as f32 / 255.0).collect(), + } + } + + /// Convert colour channels (in the document's mode) to RGB. + fn to_rgb(&self, ch: &[Option>], i: usize) -> [f32; 3] { + let g = |k: usize| ch.get(k).and_then(|c| c.as_ref()).map_or(0.0, |c| c[i]); + let rgb = match self.mode { + ColorMode::Rgb => [g(0), g(1), g(2)], + ColorMode::Cmyk => { + // Stored inverted (1 = no ink). + let k = g(3); + [g(0) * k, g(1) * k, g(2) * k] + } + ColorMode::Indexed if self.palette.len() >= 768 => { + let idx = (g(0) * 255.0).round() as usize & 255; + [self.palette[idx] as f32 / 255.0, self.palette[256 + idx] as f32 / 255.0, self.palette[512 + idx] as f32 / 255.0] + } + ColorMode::Lab => lab_to_rgb(g(0) * 100.0, g(1) * 255.0 - 128.0, g(2) * 255.0 - 128.0), + _ => [g(0); 3], + }; + if self.depth == 32 && self.mode != ColorMode::Lab { rgb.map(srgb_encode) } else { rgb } + } + + /// The merged (flattened) image. + pub fn composite(&self) -> Result { + let (w, h) = (self.width as usize, self.height as usize); + let mut r = Reader::at(&self.data, self.merged_offset); + let comp = r.u16()?; + let nch = self.channels as usize; + let cc = self.mode.color_channels(); + let want = (cc + usize::from(self.merged_alpha)).min(nch); + let rowb = self.row_bytes(w); + let mut planes: Vec>> = vec![None; nch]; + match comp { + 0 => { + for p in planes.iter_mut().take(want) { + *p = Some(self.samples(r.bytes(rowb * h)?, w, h)); + } + } + 1 => { + let mut counts = Vec::with_capacity(nch * h); + for _ in 0..nch * h { + counts.push(if self.psb { r.u32()? as usize } else { r.u16()? as usize }); + } + for (c, p) in planes.iter_mut().enumerate().take(want) { + let mut out = Vec::with_capacity(rowb * h); + for row in 0..h { + unpack_bits(r.bytes(counts[c * h + row])?, rowb, &mut out)?; + } + *p = Some(self.samples(&out, w, h)); + } + } + 2 | 3 => { + let rest = r.bytes(r.remaining())?; + let v = miniz_oxide::inflate::decompress_to_vec_zlib(rest).map_err(|e| Error::Invalid(format!("zip: {e:?}")))?; + for (c, p) in planes.iter_mut().enumerate().take(want) { + let mut plane = v.get(c * rowb * h..(c + 1) * rowb * h).ok_or(Error::Truncated)?.to_vec(); + if comp == 3 { + self.unpredict(&mut plane, w, h); + } + *p = Some(self.samples(&plane, w, h)); + } + } + c => return Err(Error::Unsupported(format!("compression {c}"))), + } + let alpha = if self.merged_alpha && nch > cc { planes[cc].take() } else { None }; + let mut data = Vec::with_capacity(w * h); + for i in 0..w * h { + let [r, g, b] = self.to_rgb(&planes, i); + let a = alpha.as_ref().map_or(1.0, |a| a[i]); + data.push([r, g, b, a]); + } + Ok(Pixels { width: w as u32, height: h as u32, data }) + } + + /// One layer's pixels with its layer mask (and vector-less fill colour) applied to alpha. + /// `canvas`: document-sized with the layer in place; otherwise the layer's own bounds. + pub fn layer_pixels(&self, index: usize, canvas: bool) -> Result { + let l = self.layers.get(index).ok_or_else(|| Error::Invalid(format!("no layer {index}")))?; + let doc = Rect { top: 0, left: 0, bottom: self.height as i32, right: self.width as i32 }; + // Fill layers often have no pixel data: they cover the canvas (or the mask). + let fill_rect = + if l.fill.is_some() && !l.has_pixels() { l.mask.filter(|m| !m.rect.is_empty() && m.default_color == 0).map_or(doc, |m| m.rect) } else { l.rect }; + let out_rect = if canvas { + doc + } else if fill_rect.is_empty() { + Rect { top: 0, left: 0, bottom: 1, right: 1 } + } else { + fill_rect + }; + let (ow, oh) = (out_rect.width() as usize, out_rect.height() as usize); + let mut out = vec![[0.0f32; 4]; ow * oh]; + let src = fill_rect; + let (sw, sh) = (src.width() as usize, src.height() as usize); + if sw > 0 && sh > 0 { + let mut planes: Vec>> = vec![None; self.mode.color_channels()]; + let mut alpha = None; + if l.has_pixels() { + for c in &l.channels { + match c.id { + -1 => alpha = Some(self.decode_channel(c.offset, c.len, sw, sh)?), + id if id >= 0 && (id as usize) < planes.len() => planes[id as usize] = Some(self.decode_channel(c.offset, c.len, sw, sh)?), + _ => {} + } + } + } + let fill = l.fill.map(|c| c.map(|v| v as f32)); + for y in 0..sh { + let dy = src.top + y as i32 - out_rect.top; + if dy < 0 || dy >= oh as i32 { + continue; + } + for x in 0..sw { + let dx = src.left + x as i32 - out_rect.left; + if dx < 0 || dx >= ow as i32 { + continue; + } + let i = y * sw + x; + let (rgb, a) = match fill { + Some(f) if !l.has_pixels() => (f, 1.0), + _ => (self.to_rgb(&planes, i), alpha.as_ref().map_or(1.0, |a| a[i])), + }; + out[dy as usize * ow + dx as usize] = [rgb[0], rgb[1], rgb[2], a]; + } + } + } + // Layer mask: multiplies alpha (also outside the mask rect, by its default colour). + if let Some(m) = l.mask.filter(|m| !m.disabled) { + let mc = l.channels.iter().find(|c| c.id == -2); + let (mw, mh) = (m.rect.width() as usize, m.rect.height() as usize); + let mask = match mc { + Some(c) if mw > 0 && mh > 0 => Some(self.decode_channel(c.offset, c.len, mw, mh)?), + _ => None, + }; + let def = m.default_color as f32 / 255.0; + for y in 0..oh { + let py = out_rect.top + y as i32; + for x in 0..ow { + let px = out_rect.left + x as i32; + let inside = py >= m.rect.top && py < m.rect.bottom && px >= m.rect.left && px < m.rect.right; + let mut v = match (&mask, inside) { + (Some(mk), true) => mk[(py - m.rect.top) as usize * mw + (px - m.rect.left) as usize], + _ => def, + }; + if m.inverted { + v = 1.0 - v; + } + out[y * ow + x][3] *= v; + } + } + } + Ok(Pixels { width: ow as u32, height: oh as u32, data: out }) + } +} + +/// The next additional-information block at `*pos` (signature `8BIM`/`8B64`, key, length): +/// returns (key, data start, data end). Skips up to 3 padding bytes before a signature. +fn next_block(data: &[u8], pos: &mut usize, end: usize, psb: bool) -> Option<([u8; 4], usize, usize)> { + let mut p = *pos; + for _ in 0..4 { + if p + 12 > end { + return None; + } + if &data[p..p + 4] == b"8BIM" || &data[p..p + 4] == b"8B64" { + break; + } + p += 1; + } + if p + 12 > end || !(&data[p..p + 4] == b"8BIM" || &data[p..p + 4] == b"8B64") { + return None; + } + let mut key = [0u8; 4]; + key.copy_from_slice(&data[p + 4..p + 8]); + let wide = + psb && matches!(&key, b"LMsk" | b"Lr16" | b"Lr32" | b"Layr" | b"Mt16" | b"Mt32" | b"Mtrn" | b"Alph" | b"FMsk" | b"lnk2" | b"FEid" | b"FXid" | b"PxSD"); + let mut r = Reader::at(&data[..end], p + 8); + let len = r.length(wide).ok()?; + let start = r.pos; + let stop = start.checked_add(len)?; + if stop > end { + return None; + } + *pos = stop; + Some((key, start, stop)) +} + +/// PackBits decoding of one row (appends exactly `row` bytes, zero-padded). +fn unpack_bits(src: &[u8], row: usize, out: &mut Vec) -> Result<()> { + let start = out.len(); + let mut i = 0; + while i < src.len() && out.len() - start < row { + let n = src[i] as i8; + i += 1; + if n >= 0 { + let k = n as usize + 1; + let s = src.get(i..i + k).ok_or(Error::Truncated)?; + out.extend_from_slice(s); + i += k; + } else if n != -128 { + let k = (1 - n as isize) as usize; + let v = *src.get(i).ok_or(Error::Truncated)?; + out.extend(std::iter::repeat_n(v, k)); + i += 1; + } + } + out.resize(start + row, 0); + Ok(()) +} + +/// PackBits encoding of one row. +pub(crate) fn pack_bits(src: &[u8]) -> Vec { + let mut out = vec![]; + let mut i = 0; + while i < src.len() { + // run of identical bytes + let mut run = 1; + while i + run < src.len() && src[i + run] == src[i] && run < 128 { + run += 1; + } + if run >= 2 { + out.push((1 - run as isize) as i8 as u8); + out.push(src[i]); + i += run; + continue; + } + // literal run + let start = i; + let mut n = 0; + while i < src.len() && n < 128 { + if i + 1 < src.len() && src[i + 1] == src[i] { + break; + } + i += 1; + n += 1; + } + if n == 0 { + i += 1; + n = 1; + } + out.push((n - 1) as u8); + out.extend_from_slice(&src[start..start + n]); + } + out +} + +fn srgb_encode(v: f32) -> f32 { + if v <= 0.003_130_8 { v * 12.92 } else { 1.055 * v.powf(1.0 / 2.4) - 0.055 } +} + +/// CIE L*a*b* (D50) to sRGB (D65 via the Bradford-adapted matrix), clamped. +fn lab_to_rgb(l: f32, a: f32, b: f32) -> [f32; 3] { + let fy = (l + 16.0) / 116.0; + let fx = fy + a / 500.0; + let fz = fy - b / 200.0; + let f = |t: f32| if t > 6.0 / 29.0 { t * t * t } else { 3.0 * (6.0f32 / 29.0).powi(2) * (t - 4.0 / 29.0) }; + let (x, y, z) = (0.964_22 * f(fx), f(fy), 0.825_21 * f(fz)); + // XYZ (D50) → linear sRGB. + let r = 3.133_856 * x - 1.616_867 * y - 0.490_615 * z; + let g = -0.978_768 * x + 1.916_142 * y + 0.033_454 * z; + let bb = 0.071_945 * x - 0.228_991 * y + 1.405_243 * z; + [r, g, bb].map(|v| srgb_encode(v.clamp(0.0, 1.0))) +} + +#[cfg(test)] +mod tests; diff --git a/crates/psd/src/read.rs b/crates/psd/src/read.rs new file mode 100644 index 00000000..97651b2b --- /dev/null +++ b/crates/psd/src/read.rs @@ -0,0 +1,86 @@ +//! Big-endian byte reader. + +use crate::{Error, Result}; + +#[derive(Clone)] +pub struct Reader<'a> { + pub data: &'a [u8], + pub pos: usize, +} + +impl<'a> Reader<'a> { + pub fn new(data: &'a [u8]) -> Self { + Reader { data, pos: 0 } + } + pub fn at(data: &'a [u8], pos: usize) -> Self { + Reader { data, pos } + } + pub fn remaining(&self) -> usize { + self.data.len().saturating_sub(self.pos) + } + pub fn bytes(&mut self, n: usize) -> Result<&'a [u8]> { + if self.remaining() < n { + return Err(Error::Truncated); + } + let s = &self.data[self.pos..self.pos + n]; + self.pos += n; + Ok(s) + } + pub fn skip(&mut self, n: usize) -> Result<()> { + self.bytes(n).map(|_| ()) + } + pub fn u8(&mut self) -> Result { + Ok(self.bytes(1)?[0]) + } + pub fn u16(&mut self) -> Result { + let b = self.bytes(2)?; + Ok(u16::from_be_bytes([b[0], b[1]])) + } + pub fn i16(&mut self) -> Result { + Ok(self.u16()? as i16) + } + pub fn u32(&mut self) -> Result { + let b = self.bytes(4)?; + Ok(u32::from_be_bytes([b[0], b[1], b[2], b[3]])) + } + pub fn i32(&mut self) -> Result { + Ok(self.u32()? as i32) + } + pub fn u64(&mut self) -> Result { + let b = self.bytes(8)?; + let mut a = [0u8; 8]; + a.copy_from_slice(b); + Ok(u64::from_be_bytes(a)) + } + pub fn f32(&mut self) -> Result { + Ok(f32::from_bits(self.u32()?)) + } + pub fn f64(&mut self) -> Result { + Ok(f64::from_bits(self.u64()?)) + } + pub fn tag(&mut self) -> Result<[u8; 4]> { + let b = self.bytes(4)?; + Ok([b[0], b[1], b[2], b[3]]) + } + /// A length field: 4 bytes in PSD, 8 bytes in PSB (`wide`). + pub fn length(&mut self, wide: bool) -> Result { + let v = if wide { self.u64()? } else { self.u32()? as u64 }; + usize::try_from(v).map_err(|_| Error::Invalid("length".into())) + } + /// A Pascal string padded so that length byte + text is a multiple of `pad`. + pub fn pascal(&mut self, pad: usize) -> Result { + let n = self.u8()? as usize; + let s = self.bytes(n)?; + let total = 1 + n; + let padded = total.div_ceil(pad) * pad; + self.skip(padded - total)?; + // Pascal strings are in the system (Mac Roman) encoding; ASCII covers the common case. + Ok(s.iter().map(|&b| if b < 0x80 { b as char } else { mac_roman(b) }).collect()) + } +} + +/// Mac OS Roman upper half (0x80..=0xFF), as published by the Unicode Consortium mapping table. +fn mac_roman(b: u8) -> char { + const T: &str = "ÄÅÇÉÑÖÜáàâäãåçéèêëíìîïñóòôöõúùûü†°¢£§•¶ß®©™´¨≠ÆØ∞±≤≥¥µ∂∑∏π∫ªºΩæø¿¡¬√ƒ≈∆«»… ÀÃÕŒœ–—“”‘’÷◊ÿŸ⁄€‹›fifl‡·‚„‰ÂÊÁËÈÍÎÏÌÓÔ\u{F8FF}ÒÚÛÙıˆ˜¯˘˙˚¸˝˛ˇ"; + T.chars().nth((b - 0x80) as usize).unwrap_or('?') +} diff --git a/crates/psd/src/tests.rs b/crates/psd/src/tests.rs new file mode 100644 index 00000000..68680fe7 --- /dev/null +++ b/crates/psd/src/tests.rs @@ -0,0 +1,189 @@ +use super::write::*; +use super::*; + +fn gradient(w: u32, h: u32) -> Vec<[f32; 4]> { + (0..w * h).map(|i| [(i % w) as f32 / (w - 1).max(1) as f32, (i / w) as f32 / (h - 1).max(1) as f32, 0.25, 1.0]).collect() +} + +fn close(a: &[[f32; 4]], b: &[[f32; 4]], tol: f32) -> bool { + a.len() == b.len() && a.iter().zip(b).all(|(x, y)| (0..4).all(|c| (x[c] - y[c]).abs() <= tol)) +} + +fn doc_with_layers(depth: u16, mode: ColorMode, rle: bool) -> WDoc { + let mut d = WDoc::new(40, 30); + d.depth = depth; + d.mode = mode; + d.rle = rle; + let mut bg = WLayer::pixels("Background", Rect::new(0, 0, 40, 30), gradient(40, 30)); + bg.opacity = 255; + let mut red = WLayer::solid("Red box", Rect::new(5, 6, 10, 8), [1.0, 0.0, 0.0, 1.0]).with_blend(b"mul ").with_opacity(128); + red.hidden = true; + let mut masked = WLayer::solid("Masked", Rect::new(20, 10, 12, 12), [0.0, 0.5, 1.0, 1.0]).with_blend(b"scrn"); + // Mask: left half opaque, right half transparent; default (outside) black. + let mrect = Rect::new(20, 10, 12, 12); + masked.mask = Some(WMask { rect: mrect, data: (0..144).map(|i| if i % 12 < 6 { 1.0 } else { 0.0 }).collect(), default_color: 0, disabled: false }); + d.layers = vec![bg, red, masked]; + d +} + +#[test] +fn layers_round_trip_8bit_rle() { + let bytes = write(&doc_with_layers(8, ColorMode::Rgb, true)); + assert!(is_psd(&bytes)); + let p = Psd::parse(bytes).unwrap(); + assert_eq!((p.width, p.height, p.depth, p.mode), (40, 30, 8, ColorMode::Rgb)); + assert_eq!(p.layers.len(), 3); + let names: Vec<&str> = p.layers.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(names, ["Background", "Red box", "Masked"]); + assert_eq!(p.layers[1].blend_key(), "mul "); + assert_eq!(p.layers[1].opacity, 128); + assert!(p.layers[1].hidden && !p.layers[0].hidden); + assert_eq!(p.layers[2].blend_key(), "scrn"); + assert_eq!(p.layers[1].rect, Rect::new(5, 6, 10, 8)); + // Pixels (8-bit quantisation). + let bg = p.layer_pixels(0, false).unwrap(); + assert!(close(&bg.data, &gradient(40, 30), 1.0 / 255.0)); + // Retain layer size vs canvas. + let red = p.layer_pixels(1, false).unwrap(); + assert_eq!((red.width, red.height), (10, 8)); + let red_c = p.layer_pixels(1, true).unwrap(); + assert_eq!((red_c.width, red_c.height), (40, 30)); + assert_eq!(red_c.data[6 * 40 + 5], [1.0, 0.0, 0.0, 1.0]); + assert_eq!(red_c.data[0][3], 0.0); + // Layer mask becomes alpha. + let m = p.layer_pixels(2, false).unwrap(); + assert_eq!(m.data[0][3], 1.0); + assert_eq!(m.data[11][3], 0.0); + // Merged composite. + let c = p.composite().unwrap(); + assert_eq!((c.width, c.height), (40, 30)); + assert!(p.merged_alpha); +} + +#[test] +fn layers_round_trip_16bit_raw_and_cmyk_gray() { + for (depth, mode, rle) in + [(16, ColorMode::Rgb, false), (16, ColorMode::Rgb, true), (8, ColorMode::Cmyk, true), (16, ColorMode::Cmyk, false), (8, ColorMode::Grayscale, false)] + { + let bytes = write(&doc_with_layers(depth, mode, rle)); + let p = Psd::parse(bytes).unwrap_or_else(|e| panic!("{depth} {mode:?} {rle}: {e}")); + assert_eq!(p.layers.len(), 3, "{depth} {mode:?}"); + assert_eq!(p.depth, depth); + let bg = p.layer_pixels(0, true).unwrap(); + let tol = if depth == 16 { 1.0 / 30000.0 } else { 1.5 / 255.0 }; + let want: Vec<[f32; 4]> = + if mode == ColorMode::Grayscale { gradient(40, 30).iter().map(|p| [p[0], p[0], p[0], 1.0]).collect() } else { gradient(40, 30) }; + assert!(close(&bg.data, &want, tol), "{depth} {mode:?} {rle}: {:?} vs {:?}", bg.data[41], want[41]); + let c = p.composite().unwrap(); + assert_eq!(c.data.len(), 1200); + } +} + +#[test] +fn groups_build_a_tree() { + let mut d = WDoc::new(16, 16); + d.layers = vec![ + WLayer::solid("Bottom", Rect::new(0, 0, 16, 16), [1.0; 4]), + WLayer::group_end(), + WLayer::solid("Inner A", Rect::new(0, 0, 4, 4), [1.0, 0.0, 0.0, 1.0]), + WLayer::group_end(), + WLayer::solid("Deep", Rect::new(2, 2, 4, 4), [0.0, 1.0, 0.0, 1.0]), + WLayer::group("Nested", false, *b"norm"), + WLayer::group("Group 1", true, *b"pass"), + WLayer::solid("Top", Rect::new(8, 8, 4, 4), [0.0, 0.0, 1.0, 1.0]), + ]; + let p = Psd::parse(write(&d)).unwrap(); + assert_eq!(p.layers[6].section, Section::OpenFolder); + assert_eq!(p.layers[6].blend_key(), "pass"); + assert_eq!(p.layers[5].section, Section::ClosedFolder); + let t = p.tree(); + assert_eq!( + t, + vec![ + Node::Layer(7), + Node::Group { layer: 6, children: vec![Node::Group { layer: 5, children: vec![Node::Layer(4)] }, Node::Layer(2)] }, + Node::Layer(0) + ] + ); +} + +#[test] +fn text_effects_fill_and_adjustments() { + let mut d = WDoc::new(64, 32); + let fx = Descriptor::new("null").with("masterFXSwitch", DValue::Bool(true)).with( + "DrSh", + DValue::Descriptor( + Descriptor::new("DrSh") + .with("enab", DValue::Bool(true)) + .with("Md ", DValue::Enum("BlnM".into(), "Mltp".into())) + .with("Clr ", DValue::Descriptor(Descriptor::rgb([0.0, 0.0, 1.0]))) + .with("Opct", DValue::UnitFloat("#Prc".into(), 60.0)) + .with("Dstn", DValue::UnitFloat("#Pxl".into(), 7.0)), + ), + ); + d.layers = vec![ + WLayer::empty("Fill", vec![solid_color_block([0.0, 1.0, 0.0])]), + WLayer::empty("Title", vec![text_block("Hello\nWorld", [10.0, 20.0], "Inter-Bold", 18.0, [1.0, 0.0, 0.0], 2)]).with_block(effects_block(&fx)), + WLayer::empty("Hue", vec![hue_sat_block(30, -20, 10)]), + WLayer::empty("Lev", vec![levels_block(10, 240, 5, 250, 1.2)]), + WLayer::empty("Inv", vec![invert_block()]), + WLayer::empty("BC", vec![brightness_block(20, -10)]), + WLayer::empty("Exp", vec![exposure_block(1.5, 0.0, 1.0)]), + WLayer::solid("Vec", Rect::new(0, 0, 64, 32), [1.0; 4]).with_block(vector_mask_block( + &[vec![[[8.0, 4.0]; 3], [[40.0, 4.0]; 3], [[40.0, 20.0]; 3]]], + 64, + 32, + false, + )), + ]; + let p = Psd::parse(write(&d)).unwrap(); + assert_eq!(p.layers[0].fill, Some([0.0, 1.0, 0.0])); + let fill = p.layer_pixels(0, false).unwrap(); + assert_eq!((fill.width, fill.height), (64, 32)); + assert_eq!(fill.data[5], [0.0, 1.0, 0.0, 1.0]); + let t = p.layers[1].text.as_ref().unwrap(); + assert_eq!(t.text, "Hello\nWorld"); + assert_eq!(t.transform[4..], [10.0, 20.0]); + assert_eq!(t.style.font.as_deref(), Some("Inter-Bold")); + assert_eq!(t.style.size, Some(18.0)); + assert_eq!(t.style.color, Some([1.0, 0.0, 0.0])); + assert_eq!(t.style.justification, Some(2)); + let e = p.layers[1].effects.as_ref().unwrap(); + let ds = e.obj("DrSh").unwrap(); + assert_eq!(ds.enum_value("Md "), Some("Mltp")); + assert_eq!(ds.num("Dstn"), Some(7.0)); + assert_eq!(p.layers[2].adjustment, Some(Adjustment::HueSaturation { hue: 30.0, saturation: -20.0, lightness: 10.0, colorize: false })); + assert_eq!(p.layers[3].adjustment, Some(Adjustment::Levels { in_black: 10.0, in_white: 240.0, gamma: 1.2, out_black: 5.0, out_white: 250.0 })); + assert_eq!(p.layers[4].adjustment, Some(Adjustment::Invert)); + assert_eq!(p.layers[5].adjustment, Some(Adjustment::BrightnessContrast { brightness: 20.0, contrast: -10.0, legacy: true })); + assert_eq!(p.layers[6].adjustment, Some(Adjustment::Exposure { exposure: 1.5, offset: 0.0, gamma: 1.0 })); + let vm = p.layers[7].vector_mask.as_ref().unwrap(); + assert_eq!(vm.subpaths.len(), 1); + assert!(vm.subpaths[0].closed); + let k = &vm.subpaths[0].knots; + assert_eq!(k.len(), 3); + assert!((k[1][1][0] - 40.0).abs() < 1e-4 && (k[2][1][1] - 20.0).abs() < 1e-4); +} + +#[test] +fn packbits_round_trip() { + for src in [vec![], vec![1u8], vec![5; 300], (0..=255).collect::>(), vec![1, 1, 2, 3, 3, 3, 4, 5, 5]] { + let enc = pack_bits(&src); + let mut out = vec![]; + unpack_bits(&enc, src.len(), &mut out).unwrap(); + assert_eq!(out, src); + } +} + +#[test] +fn truncated_and_garbage_input_errors() { + let bytes = write(&doc_with_layers(8, ColorMode::Rgb, true)); + for n in [0, 3, 10, 26, 40, 100, bytes.len() / 2] { + let r = Psd::parse(bytes[..n].to_vec()); + if let Ok(p) = r { + let _ = p.composite(); + } + } + assert!(Psd::parse(b"8BPS\0\x09garbage".to_vec()).is_err()); + assert!(Psd::parse(b"PNG".to_vec()).is_err()); +} diff --git a/crates/psd/src/write.rs b/crates/psd/src/write.rs new file mode 100644 index 00000000..fb78cc78 --- /dev/null +++ b/crates/psd/src/write.rs @@ -0,0 +1,491 @@ +//! A minimal Photoshop document writer, per the same specification as the reader: RGB, CMYK or +//! Grayscale, 8 or 16 bits, raw or RLE channels, pixel layers with layer masks, groups and any +//! additional-information blocks (helpers below build text, effects, fill and adjustment +//! blocks). Used to generate test fixtures; not a full exporter. + +use crate::descriptor::{self, DValue, Descriptor}; +use crate::{ColorMode, Rect, Section, engine_data, pack_bits}; + +/// A layer mask for [`WLayer`]. +#[derive(Clone, Debug)] +pub struct WMask { + pub rect: Rect, + /// `rect.width() * rect.height()` values in 0..1. + pub data: Vec, + pub default_color: u8, + pub disabled: bool, +} + +/// A layer to write. +#[derive(Clone, Debug)] +pub struct WLayer { + pub name: String, + pub rect: Rect, + /// Straight RGBA, `rect.width() * rect.height()` pixels. + pub pixels: Vec<[f32; 4]>, + pub blend: [u8; 4], + pub opacity: u8, + pub hidden: bool, + pub clipping: bool, + pub mask: Option, + pub section: Section, + /// Extra additional-information blocks (key, data). + pub blocks: Vec<([u8; 4], Vec)>, +} + +impl WLayer { + /// A pixel layer. + pub fn pixels(name: &str, rect: Rect, pixels: Vec<[f32; 4]>) -> WLayer { + WLayer { + name: name.into(), + rect, + pixels, + blend: *b"norm", + opacity: 255, + hidden: false, + clipping: false, + mask: None, + section: Section::Layer, + blocks: vec![], + } + } + /// A layer filled with one colour. + pub fn solid(name: &str, rect: Rect, rgba: [f32; 4]) -> WLayer { + let n = (rect.width() * rect.height()) as usize; + WLayer::pixels(name, rect, vec![rgba; n]) + } + /// The top record of a group (its children come *before* it in file order, after a + /// [`WLayer::group_end`]). + pub fn group(name: &str, open: bool, blend: [u8; 4]) -> WLayer { + let mut l = WLayer::pixels(name, Rect::default(), vec![]); + l.section = if open { Section::OpenFolder } else { Section::ClosedFolder }; + l.blend = blend; + l + } + /// The hidden record that closes a group (written below its children). + pub fn group_end() -> WLayer { + let mut l = WLayer::pixels("", Rect::default(), vec![]); + l.section = Section::Divider; + l + } + /// A layer without pixels carrying `blocks` (adjustment and fill layers, type layers). + pub fn empty(name: &str, blocks: Vec<([u8; 4], Vec)>) -> WLayer { + let mut l = WLayer::pixels(name, Rect::default(), vec![]); + l.blocks = blocks; + l + } + pub fn with_blend(mut self, key: &[u8; 4]) -> WLayer { + self.blend = *key; + self + } + pub fn with_opacity(mut self, o: u8) -> WLayer { + self.opacity = o; + self + } + pub fn with_block(mut self, b: ([u8; 4], Vec)) -> WLayer { + self.blocks.push(b); + self + } +} + +/// A document to write. +#[derive(Clone, Debug)] +pub struct WDoc { + pub width: u32, + pub height: u32, + /// 8 or 16. + pub depth: u16, + /// Rgb, Cmyk or Grayscale. + pub mode: ColorMode, + pub rle: bool, + /// Bottom of the stack first (file order). + pub layers: Vec, + /// Merged image (straight RGBA, document-sized); `None` composites visible pixel layers + /// with Normal blending. + pub composite: Option>, +} + +impl WDoc { + pub fn new(width: u32, height: u32) -> WDoc { + WDoc { width, height, depth: 8, mode: ColorMode::Rgb, rle: true, layers: vec![], composite: None } + } +} + +fn u16b(out: &mut Vec, v: u16) { + out.extend_from_slice(&v.to_be_bytes()); +} +fn u32b(out: &mut Vec, v: u32) { + out.extend_from_slice(&v.to_be_bytes()); +} + +/// Colour channels for a straight RGB pixel in the document mode. +fn mode_channels(mode: ColorMode, p: [f32; 4]) -> Vec { + match mode { + ColorMode::Cmyk => { + // Stored values are inverted ink amounts: k = max, c = r / max (reader: r = c * k). + let m = p[0].max(p[1]).max(p[2]); + if m <= 0.0 { vec![0.0, 0.0, 0.0, 0.0] } else { vec![p[0] / m, p[1] / m, p[2] / m, m] } + } + ColorMode::Grayscale => vec![p[0]], + _ => vec![p[0], p[1], p[2]], + } +} + +fn encode_samples(v: &[f32], depth: u16) -> Vec { + let mut out = Vec::with_capacity(v.len() * if depth == 16 { 2 } else { 1 }); + for &x in v { + let x = x.clamp(0.0, 1.0); + if depth == 16 { + u16b(&mut out, (x * 65535.0).round() as u16); + } else { + out.push((x * 255.0).round() as u8); + } + } + out +} + +/// One channel's data (compression field + data) for a `w`×`h` plane. +fn channel_data(v: &[f32], w: usize, h: usize, depth: u16, rle: bool) -> Vec { + let raw = encode_samples(v, depth); + let mut out = vec![]; + let rowb = raw.len() / h.max(1); + if !rle || w == 0 || h == 0 { + u16b(&mut out, 0); + out.extend_from_slice(&raw); + return out; + } + u16b(&mut out, 1); + let rows: Vec> = raw.chunks(rowb.max(1)).map(pack_bits).collect(); + for r in &rows { + u16b(&mut out, r.len() as u16); + } + for r in rows { + out.extend_from_slice(&r); + } + out +} + +fn pascal4(out: &mut Vec, s: &str) { + let b: Vec = s.bytes().map(|c| if c < 0x80 { c } else { b'?' }).take(255).collect(); + out.push(b.len() as u8); + out.extend_from_slice(&b); + let total = 1 + b.len(); + out.resize(out.len() + total.div_ceil(4) * 4 - total, 0); +} + +fn block(out: &mut Vec, key: &[u8; 4], data: &[u8]) { + out.extend_from_slice(b"8BIM"); + out.extend_from_slice(key); + let padded = data.len().div_ceil(2) * 2; + u32b(out, padded as u32); + out.extend_from_slice(data); + out.resize(out.len() + padded - data.len(), 0); +} + +fn layer_info(doc: &WDoc) -> Vec { + let mut out = vec![]; + out.extend_from_slice(&(-(doc.layers.len() as i16)).to_be_bytes()); + let mut chan_data: Vec)>> = vec![]; + for l in &doc.layers { + let (w, h) = (l.rect.width() as usize, l.rect.height() as usize); + let mut chans: Vec<(i16, Vec)> = vec![]; + let n = w * h; + let alpha: Vec = l.pixels.iter().take(n).map(|p| p[3]).collect(); + chans.push((-1, channel_data(&alpha, w, h, doc.depth, doc.rle))); + let cc = doc.mode.color_channels(); + let planes: Vec> = l.pixels.iter().take(n).map(|p| mode_channels(doc.mode, *p)).collect(); + for c in 0..cc { + let plane: Vec = planes.iter().map(|v| v[c]).collect(); + chans.push((c as i16, channel_data(&plane, w, h, doc.depth, doc.rle))); + } + if let Some(m) = &l.mask { + chans.push((-2, channel_data(&m.data, m.rect.width() as usize, m.rect.height() as usize, doc.depth, doc.rle))); + } + // record + for v in [l.rect.top, l.rect.left, l.rect.bottom, l.rect.right] { + out.extend_from_slice(&v.to_be_bytes()); + } + u16b(&mut out, chans.len() as u16); + for (id, d) in &chans { + out.extend_from_slice(&id.to_be_bytes()); + u32b(&mut out, d.len() as u32); + } + out.extend_from_slice(b"8BIM"); + out.extend_from_slice(&l.blend); + out.push(l.opacity); + out.push(l.clipping as u8); + let mut flags = 8u8; + if l.hidden { + flags |= 2; + } + out.push(flags); + out.push(0); + let mut extra = vec![]; + match &l.mask { + Some(m) => { + u32b(&mut extra, 20); + for v in [m.rect.top, m.rect.left, m.rect.bottom, m.rect.right] { + extra.extend_from_slice(&v.to_be_bytes()); + } + extra.push(m.default_color); + extra.push(if m.disabled { 2 } else { 0 }); + extra.extend_from_slice(&[0, 0]); + } + None => u32b(&mut extra, 0), + } + u32b(&mut extra, 0); // blending ranges + pascal4(&mut extra, &l.name); + let luni = unicode_plain(&l.name); + block(&mut extra, b"luni", &luni); + match l.section { + Section::Layer => {} + s => { + let mut d = vec![]; + u32b( + &mut d, + match s { + Section::OpenFolder => 1, + Section::ClosedFolder => 2, + _ => 3, + }, + ); + d.extend_from_slice(b"8BIM"); + d.extend_from_slice(&l.blend); + block(&mut extra, b"lsct", &d); + } + } + for (k, d) in &l.blocks { + block(&mut extra, k, d); + } + u32b(&mut out, extra.len() as u32); + out.extend_from_slice(&extra); + chan_data.push(chans); + } + for chans in chan_data { + for (_, d) in chans { + out.extend_from_slice(&d); + } + } + if out.len() % 2 == 1 { + out.push(0); + } + out +} + +fn unicode_plain(s: &str) -> Vec { + let v: Vec = s.encode_utf16().collect(); + let mut out = vec![]; + u32b(&mut out, v.len() as u32); + for c in v { + u16b(&mut out, c); + } + out +} + +/// Normal "over" of the visible pixel layers (groups ignored), straight alpha. +fn flatten(doc: &WDoc) -> Vec<[f32; 4]> { + let (w, h) = (doc.width as i32, doc.height as i32); + let mut acc = vec![[0.0f32; 4]; (w * h) as usize]; + for l in doc.layers.iter().filter(|l| !l.hidden && l.section == Section::Layer) { + let lw = l.rect.width() as i32; + for (i, p) in l.pixels.iter().enumerate() { + let (x, y) = (l.rect.left + i as i32 % lw.max(1), l.rect.top + i as i32 / lw.max(1)); + if x < 0 || y < 0 || x >= w || y >= h { + continue; + } + let a = p[3] * l.opacity as f32 / 255.0; + let d = &mut acc[(y * w + x) as usize]; + let oa = a + d[3] * (1.0 - a); + for c in 0..3 { + d[c] = if oa > 0.0 { (p[c] * a + d[c] * d[3] * (1.0 - a)) / oa } else { 0.0 }; + } + d[3] = oa; + } + } + acc +} + +/// Serialise a document. +pub fn write(doc: &WDoc) -> Vec { + let cc = doc.mode.color_channels(); + let mut out = vec![]; + out.extend_from_slice(b"8BPS"); + u16b(&mut out, 1); + out.extend_from_slice(&[0; 6]); + u16b(&mut out, (cc + 1) as u16); + u32b(&mut out, doc.height); + u32b(&mut out, doc.width); + u16b(&mut out, doc.depth); + u16b(&mut out, doc.mode.to_u16()); + u32b(&mut out, 0); // colour mode data + u32b(&mut out, 0); // image resources + // Layer and mask information. + let mut lm = vec![]; + if !doc.layers.is_empty() { + let li = layer_info(doc); + if doc.depth == 16 { + u32b(&mut lm, 0); + u32b(&mut lm, 0); // global layer mask info + let mut padded = li; + while !padded.len().is_multiple_of(4) { + padded.push(0); + } + lm.extend_from_slice(b"8BIM"); + lm.extend_from_slice(b"Lr16"); + u32b(&mut lm, padded.len() as u32); + lm.extend_from_slice(&padded); + } else { + u32b(&mut lm, li.len() as u32); + lm.extend_from_slice(&li); + u32b(&mut lm, 0); + } + } + u32b(&mut out, lm.len() as u32); + out.extend_from_slice(&lm); + // Merged image. + let comp = doc.composite.clone().unwrap_or_else(|| flatten(doc)); + let (w, h) = (doc.width as usize, doc.height as usize); + let planes: Vec> = comp.iter().map(|p| mode_channels(doc.mode, *p)).collect(); + let mut chans: Vec> = (0..cc).map(|c| planes.iter().map(|v| v[c]).collect()).collect(); + chans.push(comp.iter().map(|p| p[3]).collect()); + if doc.rle { + u16b(&mut out, 1); + let rows: Vec>> = + chans.iter().map(|c| encode_samples(c, doc.depth).chunks((w * if doc.depth == 16 { 2 } else { 1 }).max(1)).map(pack_bits).collect()).collect(); + for c in &rows { + for r in c { + u16b(&mut out, r.len() as u16); + } + } + for c in rows { + for r in c { + out.extend_from_slice(&r); + } + } + } else { + u16b(&mut out, 0); + for c in &chans { + out.extend_from_slice(&encode_samples(c, doc.depth)); + } + } + let _ = h; + out +} + +// ---------------------------------------------------------------- block helpers + +/// `TySh`: a type layer with one style run; the text origin sits at `origin`. +pub fn text_block(text: &str, origin: [f64; 2], font: &str, size: f64, color: [f64; 3], justification: u8) -> ([u8; 4], Vec) { + let mut d = vec![]; + u16b(&mut d, 1); + for v in [1.0, 0.0, 0.0, 1.0, origin[0], origin[1]] { + d.extend_from_slice(&f64::to_be_bytes(v)); + } + u16b(&mut d, 50); + u32b(&mut d, 16); + let desc = Descriptor::new("TxLr") + .with("Txt ", DValue::Text(text.replace('\n', "\r"))) + .with("textGridding", DValue::Enum("textGridding".into(), "None".into())) + .with("Ornt", DValue::Enum("Ornt".into(), "Hrzn".into())) + .with("EngineData", DValue::Raw(engine_data::minimal(text, font, size, color, justification))); + descriptor::write_descriptor(&mut d, &desc); + // Warp: version, descriptor version, an empty warp descriptor. + u16b(&mut d, 1); + u32b(&mut d, 16); + descriptor::write_descriptor(&mut d, &Descriptor::new("warp").with("warpStyle", DValue::Enum("warpStyle".into(), "warpNone".into()))); + for _ in 0..4 { + u32b(&mut d, 0); + } + (*b"TySh", d) +} + +/// `lfx2`: layer effects from an effects descriptor (`null` class with `DrSh`, `FrFX`…). +pub fn effects_block(desc: &Descriptor) -> ([u8; 4], Vec) { + let mut d = vec![]; + u32b(&mut d, 0); + u32b(&mut d, 16); + descriptor::write_descriptor(&mut d, desc); + (*b"lfx2", d) +} + +/// `SoCo`: solid colour fill. +pub fn solid_color_block(rgb: [f64; 3]) -> ([u8; 4], Vec) { + let mut d = vec![]; + u32b(&mut d, 16); + descriptor::write_descriptor(&mut d, &Descriptor::new("null").with("Clr ", DValue::Descriptor(Descriptor::rgb(rgb)))); + (*b"SoCo", d) +} + +/// `hue2`: Hue/Saturation (master). +pub fn hue_sat_block(hue: i16, sat: i16, light: i16) -> ([u8; 4], Vec) { + let mut d = vec![]; + u16b(&mut d, 2); + d.extend_from_slice(&[0, 0]); + for v in [0i16, 0, 0, hue, sat, light] { + d.extend_from_slice(&v.to_be_bytes()); + } + // Six colour ranges (unused). + d.resize(d.len() + 6 * 24, 0); + (*b"hue2", d) +} + +/// `levl`: Levels (composite record). +pub fn levels_block(in_black: i16, in_white: i16, out_black: i16, out_white: i16, gamma: f64) -> ([u8; 4], Vec) { + let mut d = vec![]; + u16b(&mut d, 2); + for v in [in_black, in_white, out_black, out_white, (gamma * 100.0).round() as i16] { + d.extend_from_slice(&v.to_be_bytes()); + } + for _ in 0..28 { + for v in [0i16, 255, 0, 255, 100] { + d.extend_from_slice(&v.to_be_bytes()); + } + } + (*b"levl", d) +} + +pub fn invert_block() -> ([u8; 4], Vec) { + (*b"nvrt", vec![]) +} + +pub fn brightness_block(brightness: i16, contrast: i16) -> ([u8; 4], Vec) { + let mut d = vec![]; + for v in [brightness, contrast, 127] { + d.extend_from_slice(&v.to_be_bytes()); + } + d.push(0); + (*b"brit", d) +} + +pub fn exposure_block(exposure: f32, offset: f32, gamma: f32) -> ([u8; 4], Vec) { + let mut d = vec![]; + u16b(&mut d, 1); + for v in [exposure, offset, gamma] { + d.extend_from_slice(&v.to_be_bytes()); + } + (*b"expA", d) +} + +/// `vmsk`: a vector mask from closed polygons/curves given as (in, anchor, out) knots in pixels. +pub fn vector_mask_block(subpaths: &[Vec<[[f64; 2]; 3]>], width: u32, height: u32, inverted: bool) -> ([u8; 4], Vec) { + let mut d = vec![]; + u32b(&mut d, 3); + u32b(&mut d, inverted as u32); + // Path fill rule record. + u16b(&mut d, 6); + d.resize(d.len() + 24, 0); + let fx = |v: f64| ((v * (1 << 24) as f64).round() as i32).to_be_bytes(); + for sp in subpaths { + u16b(&mut d, 0); + u16b(&mut d, sp.len() as u16); + d.resize(d.len() + 22, 0); + for k in sp { + u16b(&mut d, 1); + for p in k { + d.extend_from_slice(&fx(p[1] / height as f64)); + d.extend_from_slice(&fx(p[0] / width as f64)); + } + } + } + (*b"vmsk", d) +} diff --git a/crates/render/Cargo.toml b/crates/render/Cargo.toml index 4c8a9787..cdaa85c1 100644 --- a/crates/render/Cargo.toml +++ b/crates/render/Cargo.toml @@ -19,6 +19,7 @@ effectcraft-raster = { workspace = true } effectcraft-path = { workspace = true } effectcraft-text = { workspace = true } effectcraft-effects = { workspace = true } +lz4_flex = { version = "0.11", default-features = false, features = ["std", "safe-encode", "safe-decode", "checked-decode"] } kurbo = { workspace = true } rayon = { workspace = true } web-time = { workspace = true } diff --git a/crates/render/src/audio.rs b/crates/render/src/audio.rs index 4884411f..4b872ad2 100644 --- a/crates/render/src/audio.rs +++ b/crates/render/src/audio.rs @@ -330,6 +330,7 @@ mod tests { missing: false, sequence: vec![], color_profile: None, + layer: None, }; let fid = p.add_item("x.wav", effectcraft_color::Label::SeaFoam, None, ItemKind::Footage(f)); let comp = Comp::new(64, 64, FrameRate::new(25, 1), Tick::from_seconds_f64(2.0)); diff --git a/crates/render/src/cache.rs b/crates/render/src/cache.rs index 8cd27d59..23702fd6 100644 --- a/crates/render/src/cache.rs +++ b/crates/render/src/cache.rs @@ -25,9 +25,13 @@ use effectcraft_project::{ItemKind, Layer, LayerSource, Node, PropGroup}; use crate::eval::EvalCtx; -/// Thread-safe, memory-budgeted cache of processed layer buffers. +/// Thread-safe, memory-budgeted cache of processed layer buffers, optionally backed by the +/// persistent [`DiskCache`](crate::disk_cache::DiskCache): a memory miss reads the disk, and +/// buffers that were slow to render are written there (keys salted per project, see +/// [`LayerCache::set_disk`]). pub struct LayerCache { inner: Mutex, + disk: Mutex, u64)>>, } struct Entry { @@ -43,6 +47,8 @@ struct Inner { clock: u64, hits: u64, misses: u64, + /// When each recent miss happened (to time the render that follows it). + missed_at: HashMap, } /// Hit/miss counters and memory use (for perf readouts and tests). @@ -63,7 +69,37 @@ impl Default for LayerCache { impl LayerCache { /// A cache holding at most `budget` bytes of pixels. pub fn new(budget: usize) -> LayerCache { - LayerCache { inner: Mutex::new(Inner { map: HashMap::new(), bytes: 0, budget, clock: 0, hits: 0, misses: 0 }) } + LayerCache { + inner: Mutex::new(Inner { map: HashMap::new(), bytes: 0, budget, clock: 0, hits: 0, misses: 0, missed_at: HashMap::new() }), + disk: Mutex::new(None), + } + } + + /// Attach (or detach) the disk cache. `salt` separates projects and footage versions (see + /// `footage_salt`); keys on disk are `(salt, key)`. + pub fn set_disk(&self, disk: Option<(Arc, u64)>) { + if let Ok(mut d) = self.disk.lock() { + *d = disk; + } + } + + /// Change the disk salt (after footage changed) keeping the attached disk cache. + pub fn set_disk_salt(&self, salt: u64) { + if let Ok(mut d) = self.disk.lock() + && let Some((_, s)) = d.as_mut() + { + *s = salt; + } + } + + pub fn disk(&self) -> Option> { + self.disk.lock().ok()?.as_ref().map(|d| d.0.clone()) + } + + fn disk_key(&self, key: u64) -> Option<(Arc, u128)> { + let d = self.disk.lock().ok()?; + let (dc, salt) = d.as_ref()?; + Some((dc.clone(), ((*salt as u128) << 64) | key as u128)) } pub fn get(&self, key: u64) -> Option> { @@ -79,12 +115,32 @@ impl LayerCache { } None => { g.misses += 1; - None + if g.missed_at.len() > 4096 { + g.missed_at.clear(); + } + g.missed_at.insert(key, web_time::Instant::now()); + drop(g); + // Disk cache: a hit comes back into memory. + let (dc, dk) = self.disk_key(key)?; + let buf = Arc::new(dc.get_layer(dk)?); + self.insert_mem(key, buf.clone()); + Some(buf) } } } pub fn insert(&self, key: u64, buf: Arc) { + let started = self.inner.lock().ok().and_then(|mut g| g.missed_at.remove(&key)); + if let Some((dc, dk)) = self.disk_key(key) { + let slow = started.is_some_and(|t| t.elapsed().as_secs_f64() * 1e3 >= dc.min_layer_ms() as f64); + if slow { + dc.put_layer(dk, &buf); + } + } + self.insert_mem(key, buf); + } + + fn insert_mem(&self, key: u64, buf: Arc) { let bytes = buf.img.data.len() * std::mem::size_of::(); let Ok(mut g) = self.inner.lock() else { return }; if bytes > g.budget / 4 { diff --git a/crates/render/src/disk_cache.rs b/crates/render/src/disk_cache.rs new file mode 100644 index 00000000..1afc9bbb --- /dev/null +++ b/crates/render/src/disk_cache.rs @@ -0,0 +1,829 @@ +//! The persistent disk cache (Settings ▸ Media & Disk Cache ▸ Disk Cache): RAM-preview frames +//! and layer cache entries kept on disk between sessions, like After Effects' disk cache. +//! +//! * Entries are files keyed by a 128-bit content hash (`/v1//<2 hex>/<32 hex>.ecc`): +//! frames by the hash of the composition's content (the comp and everything it uses, footage +//! files' size and modification time included) plus frame, scale and view; layer buffers by +//! the layer cache's content key salted with the project's footage fingerprint. A changed +//! project therefore never reads stale pixels: its keys simply change. +//! * Payloads are LZ4-compressed (`lz4_flex`, MIT): frames as 8-bit premultiplied RGBA, layer +//! buffers as `f32` RGBA with the bytes shuffled into planes (exact, so cached and fresh +//! renders are bit-identical). Every file ends in a checksum; damaged files read as misses and +//! are deleted. +//! * Size limit with least-recently-used eviction. The in-memory index is rebuilt from the +//! folder when the cache opens (file modification times order it), so the cache survives +//! restarts; reads refresh a file's time. +//! * Safe concurrent access: writes go to a unique temporary file that is renamed into place +//! (atomic on every desktop OS), so readers in this or another process never see a partial +//! entry; a missing or damaged file is just a miss. Writes run on a background thread with a +//! bounded queue (a full queue drops the write rather than stalling rendering). + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; + +use effectcraft_effects::Buf; +use effectcraft_raster::Image; + +/// File format version (part of the folder layout). +const VERSION: &str = "v1"; +const MAGIC: &[u8; 4] = b"ECDC"; +/// Pending writes before new ones are dropped. +const QUEUE_LIMIT: usize = 64; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Kind { + Frame, + Layer, +} + +impl Kind { + fn dir(self) -> &'static str { + match self { + Kind::Frame => "frames", + Kind::Layer => "layers", + } + } + fn tag(self) -> u8 { + match self { + Kind::Frame => 1, + Kind::Layer => 2, + } + } +} + +/// A cached viewer frame: 8-bit premultiplied RGBA. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Frame8 { + pub width: u32, + pub height: u32, + pub rgba: Vec, +} + +/// Counters for tests, the Info panel and `cache.diskStats`. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct DiskStats { + pub entries: usize, + pub bytes: u64, + pub max_bytes: u64, + pub hits: u64, + pub misses: u64, + pub writes: u64, + pub evictions: u64, + pub dropped: u64, +} + +struct Meta { + bytes: u64, + last_use: u64, +} + +#[derive(Default)] +struct Index { + map: HashMap<(Kind, u128), Meta>, + total: u64, + clock: u64, +} + +enum Job { + Write(Kind, u128, Vec), +} + +struct Queue { + jobs: std::collections::VecDeque, + busy: bool, +} + +pub struct DiskCache { + dir: PathBuf, + index: Mutex, + max_bytes: AtomicU64, + hits: AtomicU64, + misses: AtomicU64, + writes: AtomicU64, + evictions: AtomicU64, + dropped: AtomicU64, + tmp_counter: AtomicU64, + queue: Arc<(Mutex, Condvar)>, + /// Layer buffers that took less than this to render are not written (ms). + min_layer_ms: AtomicU64, +} + +/// FNV-1a 128-bit (content keys). +#[derive(Clone, Copy)] +pub struct Hash128(u128); + +impl Default for Hash128 { + fn default() -> Self { + Hash128(0x6c62_272e_07bb_0142_62b8_2175_6295_c58d) + } +} + +impl Hash128 { + pub fn write(&mut self, bytes: &[u8]) { + for &b in bytes { + self.0 ^= b as u128; + self.0 = self.0.wrapping_mul(0x0000_0000_0100_0000_0000_0000_0000_013b); + } + } + pub fn write_u64(&mut self, v: u64) { + self.write(&v.to_le_bytes()); + } + pub fn finish(&self) -> u128 { + self.0 + } +} + +fn checksum(data: &[u8]) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for &b in data { + h ^= b as u64; + h = h.wrapping_mul(0x0000_0100_0000_01b3); + } + h +} + +/// The platform's default disk cache folder. +pub fn default_folder() -> PathBuf { + let home = std::env::var_os("HOME").map(PathBuf::from); + if cfg!(target_os = "macos") + && let Some(h) = &home + { + return h.join("Library/Caches/EffectCraft/Disk Cache"); + } + if cfg!(windows) + && let Some(l) = std::env::var_os("LOCALAPPDATA") + { + return PathBuf::from(l).join("EffectCraft").join("Disk Cache"); + } + if let Some(x) = std::env::var_os("XDG_CACHE_HOME") { + return PathBuf::from(x).join("effectcraft/disk-cache"); + } + home.map(|h| h.join(".cache/effectcraft/disk-cache")).unwrap_or_else(|| std::env::temp_dir().join("effectcraft-disk-cache")) +} + +impl DiskCache { + /// Open (or create) the cache in `folder`, holding at most `max_bytes`. + pub fn open(folder: impl AsRef, max_bytes: u64) -> std::io::Result> { + let dir = folder.as_ref().join(VERSION); + std::fs::create_dir_all(&dir)?; + let dc = Arc::new(DiskCache { + dir, + index: Mutex::new(Index::default()), + max_bytes: AtomicU64::new(max_bytes), + hits: AtomicU64::new(0), + misses: AtomicU64::new(0), + writes: AtomicU64::new(0), + evictions: AtomicU64::new(0), + dropped: AtomicU64::new(0), + tmp_counter: AtomicU64::new(0), + queue: Arc::new((Mutex::new(Queue { jobs: Default::default(), busy: false }), Condvar::new())), + min_layer_ms: AtomicU64::new(30), + }); + dc.rescan(); + dc.evict(); + #[cfg(not(target_arch = "wasm32"))] + { + let weak = Arc::downgrade(&dc); + let q = dc.queue.clone(); + std::thread::Builder::new().name("ec-disk-cache".into()).spawn(move || writer(weak, q))?; + } + Ok(dc) + } + + pub fn folder(&self) -> &Path { + self.dir.parent().unwrap_or(&self.dir) + } + + /// Rebuild the index from the folder (oldest files first). + pub fn rescan(&self) { + let mut found: Vec<(std::time::SystemTime, Kind, u128, u64)> = vec![]; + for kind in [Kind::Frame, Kind::Layer] { + let Ok(shards) = std::fs::read_dir(self.dir.join(kind.dir())) else { continue }; + for shard in shards.flatten() { + let Ok(files) = std::fs::read_dir(shard.path()) else { continue }; + for f in files.flatten() { + let p = f.path(); + let name = p.file_name().and_then(|n| n.to_str()).unwrap_or(""); + if name.contains(".tmp") { + // A writer that died mid-write; old temporaries are removed. + if f.metadata().and_then(|m| m.modified()).is_ok_and(|t| t.elapsed().is_ok_and(|e| e.as_secs() > 600)) { + let _ = std::fs::remove_file(&p); + } + continue; + } + let Some(hex) = name.strip_suffix(".ecc") else { continue }; + let Ok(key) = u128::from_str_radix(hex, 16) else { continue }; + let Ok(md) = f.metadata() else { continue }; + found.push((md.modified().unwrap_or(std::time::UNIX_EPOCH), kind, key, md.len())); + } + } + } + found.sort_by_key(|e| e.0); + let mut idx = Index::default(); + for (_, kind, key, bytes) in found { + idx.clock += 1; + let clock = idx.clock; + idx.total += bytes; + idx.map.insert((kind, key), Meta { bytes, last_use: clock }); + } + if let Ok(mut g) = self.index.lock() { + *g = idx; + } + } + + fn path(&self, kind: Kind, key: u128) -> PathBuf { + let hex = format!("{key:032x}"); + self.dir.join(kind.dir()).join(&hex[..2]).join(format!("{hex}.ecc")) + } + + pub fn max_bytes(&self) -> u64 { + self.max_bytes.load(Ordering::Relaxed) + } + + /// Change the size limit (evicts the least recently used entries when shrinking). + pub fn set_max_bytes(&self, b: u64) { + self.max_bytes.store(b, Ordering::Relaxed); + self.evict(); + } + + /// Layer buffers faster to render than this are not worth a disk write. + pub fn set_min_layer_ms(&self, ms: u64) { + self.min_layer_ms.store(ms, Ordering::Relaxed); + } + + pub fn min_layer_ms(&self) -> u64 { + self.min_layer_ms.load(Ordering::Relaxed) + } + + pub fn contains(&self, kind: Kind, key: u128) -> bool { + self.index.lock().is_ok_and(|g| g.map.contains_key(&(kind, key))) + } + + /// Keys of one kind (unordered). + pub fn keys(&self, kind: Kind) -> Vec { + self.index.lock().map(|g| g.map.keys().filter(|k| k.0 == kind).map(|k| k.1).collect()).unwrap_or_default() + } + + pub fn stats(&self) -> DiskStats { + let (entries, bytes) = self.index.lock().map(|g| (g.map.len(), g.total)).unwrap_or_default(); + DiskStats { + entries, + bytes, + max_bytes: self.max_bytes(), + hits: self.hits.load(Ordering::Relaxed), + misses: self.misses.load(Ordering::Relaxed), + writes: self.writes.load(Ordering::Relaxed), + evictions: self.evictions.load(Ordering::Relaxed), + dropped: self.dropped.load(Ordering::Relaxed), + } + } + + /// Delete every entry (Edit ▸ Purge ▸ All Disk Cache / Empty Disk Cache). + pub fn clear(&self) { + self.flush(); + if let Ok(mut g) = self.index.lock() { + *g = Index::default(); + } + for kind in [Kind::Frame, Kind::Layer] { + let _ = std::fs::remove_dir_all(self.dir.join(kind.dir())); + } + } + + /// Wait until queued writes are on disk. + pub fn flush(&self) { + let (m, cv) = &*self.queue; + let Ok(mut q) = m.lock() else { return }; + #[cfg(target_arch = "wasm32")] + { + q.jobs.clear(); + } + while !q.jobs.is_empty() || q.busy { + q = match cv.wait(q) { + Ok(q) => q, + Err(_) => return, + }; + } + } + + fn read(&self, kind: Kind, key: u128) -> Option> { + if !self.contains(kind, key) { + self.misses.fetch_add(1, Ordering::Relaxed); + return None; + } + let path = self.path(kind, key); + let data = std::fs::read(&path).ok(); + let ok = data.as_deref().and_then(|d| verify(d, kind)); + match ok { + Some(payload) => { + let payload = payload.to_vec(); + self.hits.fetch_add(1, Ordering::Relaxed); + if let Ok(mut g) = self.index.lock() { + g.clock += 1; + let c = g.clock; + if let Some(m) = g.map.get_mut(&(kind, key)) { + m.last_use = c; + } + } + // Persist the recency for the next session (best effort). + if let Ok(f) = std::fs::File::options().append(true).open(&path) { + let _ = f.set_modified(std::time::SystemTime::now()); + } + Some(payload) + } + None => { + self.misses.fetch_add(1, Ordering::Relaxed); + // Gone (another process evicted it) or damaged: forget it. + if data.is_some() { + let _ = std::fs::remove_file(&path); + } + self.forget(kind, key); + None + } + } + } + + fn forget(&self, kind: Kind, key: u128) { + if let Ok(mut g) = self.index.lock() + && let Some(m) = g.map.remove(&(kind, key)) + { + g.total -= m.bytes; + } + } + + fn enqueue(&self, kind: Kind, key: u128, payload: Vec) { + #[cfg(target_arch = "wasm32")] + { + self.write_now(kind, key, &payload); + } + #[cfg(not(target_arch = "wasm32"))] + { + let (m, cv) = &*self.queue; + let Ok(mut q) = m.lock() else { return }; + if q.jobs.len() >= QUEUE_LIMIT { + self.dropped.fetch_add(1, Ordering::Relaxed); + return; + } + q.jobs.push_back(Job::Write(kind, key, payload)); + cv.notify_all(); + } + } + + /// Write an entry now (atomic rename), then evict down to the limit. + fn write_now(&self, kind: Kind, key: u128, payload: &[u8]) { + if self.contains(kind, key) { + return; + } + let path = self.path(kind, key); + let Some(parent) = path.parent() else { return }; + if std::fs::create_dir_all(parent).is_err() { + return; + } + let mut data = Vec::with_capacity(payload.len() + 16); + data.extend_from_slice(MAGIC); + data.push(kind.tag()); + data.extend_from_slice(&[0, 0, 0]); + data.extend_from_slice(payload); + let sum = checksum(&data); + data.extend_from_slice(&sum.to_le_bytes()); + let n = self.tmp_counter.fetch_add(1, Ordering::Relaxed); + let tmp = parent.join(format!("{key:032x}.tmp.{}.{n}", std::process::id())); + if std::fs::write(&tmp, &data).is_err() { + let _ = std::fs::remove_file(&tmp); + return; + } + if std::fs::rename(&tmp, &path).is_err() { + let _ = std::fs::remove_file(&tmp); + return; + } + self.writes.fetch_add(1, Ordering::Relaxed); + if let Ok(mut g) = self.index.lock() { + g.clock += 1; + let c = g.clock; + let bytes = data.len() as u64; + if let Some(old) = g.map.insert((kind, key), Meta { bytes, last_use: c }) { + g.total -= old.bytes; + } + g.total += bytes; + } + self.evict(); + } + + /// Remove least recently used entries until the cache fits its limit. + fn evict(&self) { + let max = self.max_bytes(); + let victims: Vec<(Kind, u128)> = { + let Ok(mut g) = self.index.lock() else { return }; + if g.total <= max { + return; + } + let mut by_age: Vec<(u64, (Kind, u128), u64)> = g.map.iter().map(|(k, m)| (m.last_use, *k, m.bytes)).collect(); + by_age.sort_unstable_by_key(|e| e.0); + let mut out = vec![]; + for (_, k, bytes) in by_age { + if g.total <= max { + break; + } + g.map.remove(&k); + g.total -= bytes; + out.push(k); + } + out + }; + for (kind, key) in victims { + let _ = std::fs::remove_file(self.path(kind, key)); + self.evictions.fetch_add(1, Ordering::Relaxed); + } + } + + // ------------------------------------------------------------ typed entries + + pub fn get_frame(&self, key: u128) -> Option { + let p = self.read(Kind::Frame, key)?; + decode_frame(&p) + } + + /// Queue a frame for writing (8-bit premultiplied RGBA, `width * height * 4` bytes). + pub fn put_frame(&self, key: u128, width: u32, height: u32, rgba: &[u8]) { + if self.contains(Kind::Frame, key) || rgba.len() != width as usize * height as usize * 4 { + return; + } + self.enqueue(Kind::Frame, key, encode_frame(width, height, rgba)); + } + + pub fn get_layer(&self, key: u128) -> Option { + let p = self.read(Kind::Layer, key)?; + decode_layer(&p) + } + + /// Queue a layer buffer for writing. + pub fn put_layer(&self, key: u128, buf: &Buf) { + if self.contains(Kind::Layer, key) { + return; + } + self.enqueue(Kind::Layer, key, encode_layer(buf)); + } + + /// Write a layer buffer synchronously (tests). + pub fn put_layer_now(&self, key: u128, buf: &Buf) { + self.write_now(Kind::Layer, key, &encode_layer(buf)); + } + + /// Write a frame synchronously (tests). + pub fn put_frame_now(&self, key: u128, width: u32, height: u32, rgba: &[u8]) { + self.write_now(Kind::Frame, key, &encode_frame(width, height, rgba)); + } +} + +#[cfg(not(target_arch = "wasm32"))] +fn writer(dc: std::sync::Weak, q: Arc<(Mutex, Condvar)>) { + let (m, cv) = &*q; + loop { + let job = { + let Ok(mut g) = m.lock() else { return }; + loop { + if let Some(j) = g.jobs.pop_front() { + g.busy = true; + break j; + } + if dc.strong_count() == 0 { + return; + } + g = match cv.wait_timeout(g, std::time::Duration::from_millis(500)) { + Ok((g, _)) => g, + Err(_) => return, + }; + } + }; + let Some(cache) = dc.upgrade() else { return }; + match job { + Job::Write(kind, key, payload) => cache.write_now(kind, key, &payload), + } + drop(cache); + if let Ok(mut g) = m.lock() { + g.busy = false; + } + cv.notify_all(); + } +} + +/// Check magic, kind and checksum; returns the payload. +fn verify(d: &[u8], kind: Kind) -> Option<&[u8]> { + if d.len() < 16 || &d[..4] != MAGIC || d[4] != kind.tag() { + return None; + } + let (body, sum) = d.split_at(d.len() - 8); + let mut s = [0u8; 8]; + s.copy_from_slice(sum); + (checksum(body) == u64::from_le_bytes(s)).then_some(&body[8..]) +} + +fn encode_frame(w: u32, h: u32, rgba: &[u8]) -> Vec { + let mut out = Vec::with_capacity(rgba.len() / 2 + 16); + out.extend_from_slice(&w.to_le_bytes()); + out.extend_from_slice(&h.to_le_bytes()); + out.extend_from_slice(&lz4_flex::compress_prepend_size(rgba)); + out +} + +fn decode_frame(p: &[u8]) -> Option { + if p.len() < 8 { + return None; + } + let w = u32::from_le_bytes(p[0..4].try_into().ok()?); + let h = u32::from_le_bytes(p[4..8].try_into().ok()?); + let rgba = lz4_flex::decompress_size_prepended(&p[8..]).ok()?; + (rgba.len() == w as usize * h as usize * 4).then_some(Frame8 { width: w, height: h, rgba }) +} + +fn encode_layer(b: &Buf) -> Vec { + let (w, h) = (b.img.width, b.img.height); + let n = b.img.data.len(); + // Byte planes: all first bytes of every float, then all second bytes… (compresses well). + let mut planes = vec![0u8; n * 16]; + for (i, px) in b.img.data.iter().enumerate() { + for c in 0..4 { + let bytes = px[c].to_le_bytes(); + for (k, byte) in bytes.iter().enumerate() { + planes[(k * 4 + c) * n + i] = *byte; + } + } + } + let mut out = Vec::with_capacity(planes.len() / 4 + 40); + out.extend_from_slice(&w.to_le_bytes()); + out.extend_from_slice(&h.to_le_bytes()); + out.extend_from_slice(&b.offset[0].to_le_bytes()); + out.extend_from_slice(&b.offset[1].to_le_bytes()); + out.extend_from_slice(&b.scale.to_le_bytes()); + out.extend_from_slice(&lz4_flex::compress_prepend_size(&planes)); + out +} + +fn decode_layer(p: &[u8]) -> Option { + if p.len() < 32 { + return None; + } + let w = u32::from_le_bytes(p[0..4].try_into().ok()?); + let h = u32::from_le_bytes(p[4..8].try_into().ok()?); + let f = |i: usize| -> Option { Some(f64::from_le_bytes(p[i..i + 8].try_into().ok()?)) }; + let offset = [f(8)?, f(16)?]; + let scale = f(24)?; + let planes = lz4_flex::decompress_size_prepended(&p[32..]).ok()?; + let n = w as usize * h as usize; + if planes.len() != n * 16 { + return None; + } + let mut img = Image::new(w, h); + for (i, px) in img.data.iter_mut().enumerate() { + for c in 0..4 { + let mut bytes = [0u8; 4]; + for (k, byte) in bytes.iter_mut().enumerate() { + *byte = planes[(k * 4 + c) * n + i]; + } + px[c] = f32::from_le_bytes(bytes); + } + } + Some(Buf { img, offset, scale }) +} + +// ---------------------------------------------------------------- content keys + +/// Bump when rendering changes in a way that makes old disk entries wrong. +const RENDER_VERSION: &str = "effectcraft-render-1"; + +fn hash_debug(h: &mut Hash128, v: &impl std::fmt::Debug) { + h.write(format!("{v:?}").as_bytes()); + h.write(&[0xff]); +} + +/// A file's size and modification time (so edited footage gets new keys). +fn file_stamp(h: &mut Hash128, path: &str) { + if path.is_empty() { + return; + } + if let Ok(md) = std::fs::metadata(path) { + h.write_u64(md.len()); + if let Ok(t) = md.modified() + && let Ok(d) = t.duration_since(std::time::UNIX_EPOCH) + { + h.write_u64(d.as_nanos() as u64); + } + } +} + +fn footage_stamp(h: &mut Hash128, f: &effectcraft_project::Footage) { + file_stamp(h, &f.path); + for p in &f.sequence { + file_stamp(h, p); + } +} + +/// Salt for layer cache keys on disk: the renderer version and every footage file's identity +/// (layer keys name footage items, not file contents). +pub fn footage_salt(project: &effectcraft_project::Project) -> u64 { + let mut h = Hash128::default(); + h.write(RENDER_VERSION.as_bytes()); + for (id, it) in &project.items { + if let effectcraft_project::ItemKind::Footage(f) = &it.kind { + h.write_u64(id.0); + hash_debug(&mut h, f); + footage_stamp(&mut h, f); + } + } + let v = h.finish(); + (v as u64) ^ ((v >> 64) as u64) +} + +/// Content key of a composition's pixels: project render settings, the comp and everything it +/// uses (nested comps, footage, solids, footage files' size and time). With expressions in play +/// (which can read any comp) the whole project counts. +pub fn comp_content_key(project: &effectcraft_project::Project, comp: effectcraft_project::ItemId) -> u128 { + use effectcraft_project::{ItemKind, LayerSource}; + let mut h = Hash128::default(); + h.write(RENDER_VERSION.as_bytes()); + hash_debug(&mut h, &project.settings); + let mut seen = std::collections::BTreeSet::new(); + let mut stack = vec![comp]; + let mut expressions = false; + while let Some(id) = stack.pop() { + if !seen.insert(id) { + continue; + } + let Some(it) = project.item(id) else { continue }; + if let ItemKind::Comp(c) = &it.kind { + for l in &c.layers { + if let LayerSource::Comp { item } | LayerSource::Footage { item } | LayerSource::Solid { item } = l.source { + stack.push(item); + } + if !expressions { + l.props.walk("", &mut |_, p| expressions |= p.expr.is_some()); + } + } + } + } + let ids: Vec = if expressions { project.items.keys().copied().collect() } else { seen.into_iter().collect() }; + for id in ids { + let Some(it) = project.item(id) else { continue }; + h.write_u64(id.0); + h.write(it.name.as_bytes()); + hash_debug(&mut h, &it.kind); + if let ItemKind::Footage(f) = &it.kind { + footage_stamp(&mut h, f); + } + } + h.write_u64(comp.0); + h.finish() +} + +/// Disk key of one viewer frame. +pub fn frame_key(content: u128, frame: i64, scale_milli: u32, view: u64) -> u128 { + let mut h = Hash128::default(); + h.write(&content.to_le_bytes()); + h.write_u64(frame as u64); + h.write_u64(scale_milli as u64); + h.write_u64(view); + h.finish() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tmpdir(name: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("effectcraft-disk-cache-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + d + } + + fn buf(seed: u32, w: u32, h: u32) -> Buf { + let mut img = Image::new(w, h); + for (i, p) in img.data.iter_mut().enumerate() { + let v = ((i as u32).wrapping_mul(2654435761u32).wrapping_add(seed) % 1000) as f32 / 999.0; + *p = [v, v * 0.5, 1.0 - v, if i % 7 == 0 { 0.0 } else { 1.0 }]; + } + Buf { img, offset: [1.5, -2.25], scale: 0.5 } + } + + #[test] + fn hit_miss_and_exact_round_trip() { + let dir = tmpdir("hit"); + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + assert!(dc.get_layer(42).is_none()); + let b = buf(1, 33, 17); + dc.put_layer(42, &b); + dc.flush(); + let back = dc.get_layer(42).unwrap(); + assert_eq!(back.img, b.img); + assert_eq!((back.offset, back.scale), (b.offset, b.scale)); + let rgba: Vec = (0..20 * 10 * 4).map(|i| (i % 251) as u8).collect(); + dc.put_frame(7, 20, 10, &rgba); + dc.flush(); + assert_eq!(dc.get_frame(7).unwrap().rgba, rgba); + let st = dc.stats(); + assert_eq!((st.entries, st.hits, st.misses, st.writes), (2, 2, 1, 2)); + // Wrong kind is a miss. + assert!(dc.get_frame(42).is_none()); + } + + #[test] + fn persists_across_restarts_and_clears() { + let dir = tmpdir("persist"); + { + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + dc.put_layer_now(1, &buf(1, 8, 8)); + dc.put_frame_now(2, 2, 2, &[9; 16]); + } + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + assert_eq!(dc.stats().entries, 2); + assert!(dc.contains(Kind::Layer, 1) && dc.contains(Kind::Frame, 2)); + assert_eq!(dc.get_frame(2).unwrap().rgba, vec![9; 16]); + dc.clear(); + assert_eq!(dc.stats().entries, 0); + assert!(dc.get_frame(2).is_none()); + let again = DiskCache::open(&dir, 1 << 30).unwrap(); + assert_eq!(again.stats().entries, 0); + } + + #[test] + fn lru_eviction_respects_the_limit() { + let dir = tmpdir("lru"); + let dc = DiskCache::open(&dir, u64::MAX).unwrap(); + for k in 0..6u128 { + dc.put_layer_now(k, &buf(k as u32, 64, 64)); + } + let per = dc.stats().bytes / 6; + // Touch 0 so it is the most recently used. + assert!(dc.get_layer(0).is_some()); + dc.set_max_bytes(per * 3 + per / 2); + let st = dc.stats(); + assert!(st.bytes <= per * 3 + per / 2, "{st:?}"); + assert_eq!(st.entries, 3); + assert!(dc.contains(Kind::Layer, 0), "recently used survives"); + assert!(!dc.contains(Kind::Layer, 1) && !dc.contains(Kind::Layer, 2) && !dc.contains(Kind::Layer, 3)); + assert_eq!(st.evictions, 3); + // Files are really gone. + let files = walk(&dir); + assert_eq!(files, 3); + // New writes keep evicting. + dc.put_layer_now(99, &buf(9, 64, 64)); + assert!(dc.stats().bytes <= per * 3 + per / 2); + } + + fn walk(d: &Path) -> usize { + let mut n = 0; + if let Ok(rd) = std::fs::read_dir(d) { + for e in rd.flatten() { + let p = e.path(); + if p.is_dir() { + n += walk(&p); + } else if p.extension().is_some_and(|x| x == "ecc") { + n += 1; + } + } + } + n + } + + #[test] + fn damaged_files_are_misses() { + let dir = tmpdir("damaged"); + let dc = DiskCache::open(&dir, 1 << 30).unwrap(); + dc.put_frame_now(5, 4, 4, &[1; 64]); + let p = dc.path(Kind::Frame, 5); + let mut d = std::fs::read(&p).unwrap(); + let n = d.len(); + d[n / 2] ^= 0xff; + std::fs::write(&p, d).unwrap(); + assert!(dc.get_frame(5).is_none()); + assert!(!dc.contains(Kind::Frame, 5)); + assert!(!p.exists()); + } + + #[test] + fn concurrent_readers_and_writers() { + let dir = tmpdir("concurrent"); + let a = DiskCache::open(&dir, 1 << 30).unwrap(); + // A second instance on the same folder (another process, in effect). + let b = DiskCache::open(&dir, 1 << 30).unwrap(); + std::thread::scope(|s| { + for t in 0..4u32 { + let (a, b) = (a.clone(), b.clone()); + s.spawn(move || { + for i in 0..20u128 { + let k = i % 5; + let w = if t % 2 == 0 { &a } else { &b }; + w.put_layer_now(k, &buf(k as u32, 16, 16)); + if let Some(x) = w.get_layer(k) { + assert_eq!(x.img, buf(k as u32, 16, 16).img); + } + } + }); + } + }); + b.rescan(); + for k in 0..5u128 { + assert_eq!(b.get_layer(k).unwrap().img, buf(k as u32, 16, 16).img); + } + } +} diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 14a81575..a4e718a3 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -9,6 +9,7 @@ pub mod audio; pub mod cache; pub mod color; +pub mod disk_cache; use color::Region; pub mod eval; pub mod masks; @@ -46,6 +47,16 @@ pub trait FootageSource: Send + Sync { fn cache_budget(&self) -> Option { None } + /// Vector footage (SVG) rasterised at `scale` × its pixel size, for Continuously Rasterize. + /// `None` when the footage is not vector or cannot be read. + fn vector_frame(&self, _item: ItemId, _footage: &Footage, _scale: f64) -> Option> { + None + } +} + +/// Footage that can be rasterised at any scale (SVG). +pub fn is_vector_footage(f: &Footage) -> bool { + f.codec == "SVG" } /// Layer parameters and audio for effects (see [`effectcraft_effects::EffectHost`]). @@ -348,7 +359,12 @@ impl<'a> Renderer<'a> { /// stays sharp when scaled up). pub fn raster_scale(&self, ctx: &EvalCtx, layer: &Layer) -> f64 { let s = self.opts.scale; - if !layer.switches.collapse || !matches!(layer.source, LayerSource::Text | LayerSource::Shape) { + let vector = match layer.source { + LayerSource::Text | LayerSource::Shape => true, + LayerSource::Footage { item } => matches!(self.project.item(item).map(|i| &i.kind), Some(ItemKind::Footage(f)) if is_vector_footage(f)), + _ => false, + }; + if !layer.switches.collapse || !vector { return s; } let (l2c, _) = ctx.layer_to_comp(layer); @@ -631,6 +647,17 @@ impl<'a> Renderer<'a> { if !f.has_video { return None; } + // Continuously rasterised vector footage: drawn at the on-screen scale. + if layer.switches.collapse && is_vector_footage(f) { + let k = self.raster_scale(ctx, layer); + if let Some(img) = self.footage.vector_frame(*item, f, k) { + let mut buf = Buf { img: (*img).clone(), offset: [0.0; 2], scale: k }; + if let Some(c) = self.pipe.media_in(f.color_profile) { + color::convert(&mut buf.img, &c); + } + return Some(buf); + } + } let lt = ctx.source_time(layer); let img = self.footage_frame(ctx, layer, *item, f, lt)?; let k = if img.width > 0 { f.width.max(1) as f64 / img.width as f64 } else { 1.0 }; diff --git a/crates/render/src/tests_audio_fx.rs b/crates/render/src/tests_audio_fx.rs index dfd49ccc..991fac32 100644 --- a/crates/render/src/tests_audio_fx.rs +++ b/crates/render/src/tests_audio_fx.rs @@ -51,6 +51,7 @@ fn audio_footage() -> Footage { missing: false, sequence: vec![], color_profile: None, + layer: None, } } diff --git a/crates/render/src/tests_color.rs b/crates/render/src/tests_color.rs index 87f9dd1a..016cdc93 100644 --- a/crates/render/src/tests_color.rs +++ b/crates/render/src/tests_color.rs @@ -172,6 +172,7 @@ fn footage(profile: Option) -> Footage { missing: false, sequence: vec![], color_profile: profile, + layer: None, } } diff --git a/crates/render/src/tests_frame_blend.rs b/crates/render/src/tests_frame_blend.rs index c172b81c..3c301d38 100644 --- a/crates/render/src/tests_frame_blend.rs +++ b/crates/render/src/tests_frame_blend.rs @@ -80,6 +80,7 @@ fn setup(mode: FrameBlend, size: (u32, u32)) -> (Project, ItemId) { missing: false, sequence: vec![], color_profile: None, + layer: None, }; let fid = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(f)); let mut l = build::layer(&mut p, &comp, "clip", LayerSource::Footage { item: fid }, size, None); diff --git a/crates/svg/Cargo.toml b/crates/svg/Cargo.toml new file mode 100644 index 00000000..0e1af771 --- /dev/null +++ b/crates/svg/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "effectcraft-svg" +description = "SVG import: parses SVG 1.1 / SVG 2 static vector content (paths, basic shapes, groups, transforms, use, CSS styles, solid and gradient fills/strokes, opacity) and rasterises it at any scale" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +effectcraft-geom = { workspace = true } +effectcraft-raster = { workspace = true } +effectcraft-path = { workspace = true } +kurbo = { workspace = true } +roxmltree = "0.21" +rayon = { workspace = true } diff --git a/crates/svg/README.md b/crates/svg/README.md new file mode 100644 index 00000000..862bd01d --- /dev/null +++ b/crates/svg/README.md @@ -0,0 +1,26 @@ +# effectcraft-svg + +SVG import for EffectCraft: `.svg` files import as footage (rasterised at any scale, so +Continuously Rasterize stays sharp) and Layer ▸ Create ▸ Create Shapes from Vector Layer turns +them into editable shape layers. + +Implemented from the W3C **Scalable Vector Graphics (SVG) 1.1 (Second Edition)** recommendation +(16 August 2011) and the **SVG 2** candidate recommendation (4 October 2018) for the parts listed +here, plus **CSS Color Module Level 3** for colour keywords and functions: + +- document: `svg` (width/height, `viewBox`, `preserveAspectRatio`), nested `svg`, `g`, `defs`, + `use` (x/y, `href` / `xlink:href`), `symbol` (through `use`), `title`/`desc`/`metadata` ignored; +- shapes: `path` (all path data commands incl. elliptical arcs), `rect` (rx/ry), `circle`, + `ellipse`, `line`, `polyline`, `polygon`; +- `transform` lists (matrix, translate, scale, rotate with centre, skewX, skewY); +- painting: `fill`, `fill-rule`, `fill-opacity`, `stroke`, `stroke-width`, `stroke-linecap`, + `stroke-linejoin`, `stroke-miterlimit`, `stroke-dasharray`, `stroke-dashoffset`, + `stroke-opacity`, `opacity`, `display`, `visibility`, `color`/`currentColor`; +- `linearGradient` and `radialGradient` (with `fx`/`fy` focal point), `gradientUnits`, + `gradientTransform`, `spreadMethod`, `href` inheritance of stops and attributes; +- styling through presentation attributes, the `style` attribute and ` + + + + + + + + + + + + + + + + + diff --git a/crates/svg/tests/fixtures/paths-gradients.svg.attribution b/crates/svg/tests/fixtures/paths-gradients.svg.attribution new file mode 100644 index 00000000..d4b8758f --- /dev/null +++ b/crates/svg/tests/fixtures/paths-gradients.svg.attribution @@ -0,0 +1,7 @@ +asset: paths-gradients.svg +title: SVG test fixture: path data, gradients, CSS styles and use +author: EffectCraft contributors +source: original work +license: MIT OR Apache-2.0 +added: 2026-10-02 by EffectCraft contributors (Claude agent, SVG import) +notes: Hand-written for the effectcraft-svg parser and Create Shapes from Vector Layer tests. diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index 62239f19..d0e5a6d9 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -7,6 +7,11 @@ //! wasm32 has no threads: jobs wait in the same priority queue and [`Frames::pump`] renders them //! on the UI thread between egui frames, within a time budget. //! +//! With the disk cache on (Settings ▸ Media & Disk Cache), every CPU-rendered frame is also +//! written to disk under a content key (the comp's content hash, frame, scale, view and render +//! options), and a job first looks there: frames survive restarts and project reopenings. The +//! timeline shows disk-only frames as a blue cache bar. +//! //! With Mercury GPU Acceleration (a [`Gpu`] on egui-wgpu's device and the project's renderer set //! to the GPU) frames stay on the GPU: the compositor writes an RGBA8 texture that the viewer //! registers with egui-wgpu and draws directly; pixels are read back only when something needs @@ -16,6 +21,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::{Arc, Mutex}; use effectcraft_engine::project::{ItemId, Project}; +use effectcraft_engine::render::disk_cache::{self, DiskCache}; use effectcraft_engine::render::{ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; use effectcraft_engine::time::Tick; use effectcraft_gpu::{DisplayFrame, Gpu}; @@ -122,6 +128,37 @@ pub struct RenderSource { /// Show Channel / exposure / region-of-interest drawing read the CPU frame texture, so GPU /// frames are read back (the compositing still runs on the GPU). pub gpu_display: bool, + /// The persistent disk cache, when enabled. + pub disk: Option>, +} + +/// Content keys of (project revision, comp), computed once per revision. +type ContentKeys = Arc>>; + +fn content_key(keys: &ContentKeys, project: &Project, revision: u64, comp: u64) -> u128 { + if let Some(k) = keys.lock().ok().and_then(|m| m.get(&(revision, comp)).copied()) { + return k; + } + let k = disk_cache::comp_content_key(project, ItemId(comp)); + if let Ok(mut m) = keys.lock() { + if m.len() > 256 { + m.clear(); + } + m.insert((revision, comp), k); + } + k +} + +fn opts_hash(o: &RenderOpts) -> u64 { + let mut h = disk_cache::Hash128::default(); + h.write(format!("{o:?}").as_bytes()); + h.finish() as u64 +} + +/// Disk key of a viewer frame (render options included: region of interest, draft…). +fn frame_disk_key(keys: &ContentKeys, project: &Project, key: &FrameKey, opts_hash: u64) -> u128 { + let content = content_key(keys, project, key.revision, key.comp); + disk_cache::frame_key(content, key.frame, key.scale, key.view ^ opts_hash) } pub struct Frames { @@ -135,6 +172,7 @@ pub struct Frames { ctx: Option, /// Render time of the last viewer (urgent) frame in ms, for the Info panel / perf readout. pub last_ms: Arc>, + content_keys: ContentKeys, } impl Default for Frames { @@ -149,6 +187,7 @@ impl Default for Frames { pool: rayon::ThreadPoolBuilder::new().num_threads(threads).thread_name(|i| format!("ec-frame-{i}")).build().expect("frame pool"), ctx: None, last_ms: Arc::new(Mutex::new(0.0)), + content_keys: Arc::default(), } } } @@ -195,6 +234,21 @@ impl Frames { v } + /// Frames of (revision, comp, scale) in the disk cache but not in RAM — the blue cache bar. + /// `opts` are the viewer's render options (part of the disk key). + pub fn disk_frames(&self, src: &RenderSource, revision: u64, comp: u64, scale: u32, view: u64, frames: i64, opts: &RenderOpts) -> Vec { + let Some(dc) = &src.disk else { return vec![] }; + let ram: HashSet = self.cached_frames(revision, comp, scale).into_iter().collect(); + let oh = opts_hash(opts); + (0..frames.clamp(0, 100_000)) + .filter(|f| !ram.contains(f)) + .filter(|f| { + let key = FrameKey { revision, comp, frame: *f, scale, view }; + dc.contains(disk_cache::Kind::Frame, frame_disk_key(&self.content_keys, &src.project, &key, oh)) + }) + .collect() + } + /// Change the RAM preview cache budget (Settings ▸ Memory & CPU), evicting the oldest /// frames when it shrinks. pub fn set_budget(&self, bytes: usize) { @@ -278,7 +332,14 @@ impl Frames { } fn worker(&self) -> Worker { - Worker { queue: self.queue.clone(), cache: self.cache.clone(), inflight: self.inflight.clone(), ctx: self.ctx.clone(), last: self.last_ms.clone() } + Worker { + queue: self.queue.clone(), + cache: self.cache.clone(), + inflight: self.inflight.clone(), + ctx: self.ctx.clone(), + last: self.last_ms.clone(), + content_keys: self.content_keys.clone(), + } } /// Render queued frames on this thread, most important first, until the queue is empty or @@ -306,12 +367,13 @@ struct Worker { inflight: Arc>>, ctx: Option, last: Arc>, + content_keys: ContentKeys, } impl Worker { /// Render the queued job that matters most right now. `false` when the queue was empty. fn run_next(&self) -> bool { - let Worker { queue, cache, inflight, ctx, last } = self; + let Worker { queue, cache, inflight, ctx, last, content_keys } = self; let job = { let Ok(mut q) = queue.lock() else { return false }; // Urgent first (newest), then prefetch in request order. @@ -325,19 +387,18 @@ impl Worker { job }; let t0 = web_time::Instant::now(); - let mut r = Renderer::new(&job.src.project, job.src.footage.as_ref(), job.opts); - r.expr = job.src.expr.as_deref(); - r.cache = Some(&job.src.layer_cache); - r.accel = job.src.gpu.as_ref().map(|g| g as &dyn effectcraft_engine::render::Accelerator); - // The GPU leaves the frame in a texture for the viewer; otherwise (Software Only, no - // adapter, or a frame the GPU cannot finish here) the CPU renders it. - let gpu_frame = match (&job.src.gpu, r.active_accel()) { - (Some(g), Some(_)) if job.src.gpu_display => g.render_display(&r, job.comp, job.t), - _ => None, - }; - let ci = match gpu_frame { - Some(f) => FrameImage::Gpu(Arc::new(f)), - None => FrameImage::Cpu(Arc::new(to_color_image(&r.comp_frame(job.comp, job.t)))), + let disk = job.src.disk.as_ref().map(|dc| (dc.clone(), frame_disk_key(content_keys, &job.src.project, &job.key, opts_hash(&job.opts)))); + let from_disk = disk.as_ref().and_then(|(dc, k)| dc.get_frame(*k)); + let ci = match from_disk { + Some(f) => FrameImage::Cpu(Arc::new(egui::ColorImage::from_rgba_premultiplied([f.width as usize, f.height as usize], &f.rgba))), + None => { + let ci = self.render_job(&job); + if let (Some((dc, k)), FrameImage::Cpu(img)) = (&disk, &ci) { + let rgba: Vec = img.pixels.iter().flat_map(|c| c.to_array()).collect(); + dc.put_frame(*k, img.size[0] as u32, img.size[1] as u32, &rgba); + } + ci + } }; if job.urgent && let Ok(mut l) = last.lock() @@ -361,6 +422,23 @@ impl Worker { } true } + + fn render_job(&self, job: &Job) -> FrameImage { + let mut r = Renderer::new(&job.src.project, job.src.footage.as_ref(), job.opts); + r.expr = job.src.expr.as_deref(); + r.cache = Some(&job.src.layer_cache); + r.accel = job.src.gpu.as_ref().map(|g| g as &dyn effectcraft_engine::render::Accelerator); + // The GPU leaves the frame in a texture for the viewer; otherwise (Software Only, no + // adapter, or a frame the GPU cannot finish here) the CPU renders it. + let gpu_frame = match (&job.src.gpu, r.active_accel()) { + (Some(g), Some(_)) if job.src.gpu_display => g.render_display(&r, job.comp, job.t), + _ => None, + }; + match gpu_frame { + Some(f) => FrameImage::Gpu(Arc::new(f)), + None => FrameImage::Cpu(Arc::new(to_color_image(&r.comp_frame(job.comp, job.t)))), + } + } } struct Job { diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 071c1948..ec6a2278 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -308,6 +308,7 @@ impl EffectcraftApp { layer_cache: self.session.layer_cache.clone(), gpu: self.gpu.clone(), gpu_display: false, + disk: self.session.disk_cache.clone(), } } @@ -405,13 +406,11 @@ impl EffectcraftApp { self.request_frame_with(comp, frame, scale, true); } - fn request_frame_with(&self, comp: ItemId, frame: i64, scale: f64, urgent: bool) { - let Some(c) = self.session.project.comp(comp) else { return }; - let key = self.frame_key(comp, frame, scale); - let t = c.frame_rate.tick_of(frame); + /// Render options of viewer frames of `comp` at `scale`. + pub fn frame_opts(&self, comp: ItemId, scale: f64) -> RenderOpts { let (draft, _) = self.session.state.viewer.fast_previews.render(self.ui.viewer.interacting); let roi = self.session.state.region_of_interest.filter(|_| self.session.active_comp_id() == Some(comp)); - let opts = RenderOpts { + RenderOpts { scale, motion_blur: true, guides: true, @@ -419,7 +418,14 @@ impl EffectcraftApp { view: self.session.view_camera(comp), roi, backend: effectcraft_engine::render::Backend::Auto, - }; + } + } + + fn request_frame_with(&self, comp: ItemId, frame: i64, scale: f64, urgent: bool) { + let Some(c) = self.session.project.comp(comp) else { return }; + let key = self.frame_key(comp, frame, scale); + let t = c.frame_rate.tick_of(frame); + let opts = self.frame_opts(comp, scale); if urgent { self.frames.request_urgent(&self.render_source(), key, comp, t, opts); } else { diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index da5eaec2..7ebadbf4 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -716,7 +716,7 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option None, @@ -739,6 +739,10 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option bool { } /// If `id` is a dialog command invoked without its parameters, open its form and return true. +/// The import includes a Photoshop document. +fn is_layered(p: &Value) -> bool { + let psd = |v: &Value| v.as_str().is_some_and(|s| matches!(s.rsplit('.').next().map(str::to_ascii_lowercase).as_deref(), Some("psd" | "psb"))); + match p.get("paths").or(p.get("path")) { + Some(Value::Array(a)) => a.iter().any(psd), + Some(v) => psd(v), + None => false, + } +} + pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { let s = &app.session; let comp = s.active_comp(); @@ -116,6 +126,16 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { let lt = layer.map(|l| l.layer_time(t)).unwrap_or(t); let base = p.clone(); let (title, fields): (String, Vec) = match id { + // Photoshop files: Import Kind (Footage / Composition / – Retain Layer Sizes). + "file.import" if !has(p, &["importAs"]) && is_layered(p) => ( + "Import Photoshop File".into(), + vec![Field::choice( + "importAs", + "Import Kind", + &[("Footage", json!("footage")), ("Composition", json!("composition")), ("Composition - Retain Layer Sizes", json!("compositionLayerSizes"))], + 1, + )], + ), "layer.setTransform" if !has(p, &["value"]) => { let prop = p.get("prop").and_then(Value::as_str).unwrap_or("position"); let cur = layer.and_then(|l| l.transform()).and_then(|tr| tr.get(if prop == "anchorPoint" { "anchor" } else { prop })).map(|pr| pr.value_at(lt)); diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index 5b6c085f..785491d7 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -602,6 +602,31 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { if let Some((a, b)) = run { draw_run(a, b); } + // Disk-cached frames not in RAM (blue), like After Effects. + if app.session.disk_cache.is_some() { + let frames = comp.frame_rate.frame_at(comp.duration); + let opts = app.frame_opts(cid, scale_key as f64 / 1000.0); + let disk = app.frames.disk_frames(&app.render_source(), app.session.revision, cid.0, scale_key, app.view_hash(cid), frames, &opts); + let draw_blue = |a: i64, b: i64| { + let x0 = tm.x(a as f64 * fd); + let x1 = tm.x((b + 1) as f64 * fd); + p.rect_filled(Rect::from_min_max(pos2(x0, cy0), pos2(x1, cy0 + 2.5)), 0.0, t.cache_blue); + }; + let mut run: Option<(i64, i64)> = None; + for f in disk { + run = match run { + Some((a, b)) if f == b + 1 => Some((a, f)), + Some((a, b)) => { + draw_blue(a, b); + Some((f, f)) + } + None => Some((f, f)), + }; + } + if let Some((a, b)) = run { + draw_blue(a, b); + } + } // Ticks + labels. // Label spacing in whole frames (AE: `00:15f`-style seconds:frames labels). let fps_i = fr.as_f64().round().max(1.0) as i64; diff --git a/crates/ui-egui/src/theme.rs b/crates/ui-egui/src/theme.rs index 5c82e916..c32ecb24 100644 --- a/crates/ui-egui/src/theme.rs +++ b/crates/ui-egui/src/theme.rs @@ -60,6 +60,8 @@ pub struct Tokens { pub cti: Color32, pub work_area: Color32, pub cache_green: Color32, + /// Timeline cache bar for frames held in the disk cache (not in RAM). + pub cache_blue: Color32, pub keyframe: Color32, pub keyframe_selected: Color32, pub pasteboard: Color32, @@ -108,6 +110,7 @@ impl Tokens { cti: Color32::from_rgb(0x2d, 0x8c, 0xeb), work_area: Color32::from_rgb(0x4a, 0x4a, 0x4a), cache_green: Color32::from_rgb(0x3c, 0xa6, 0x4c), + cache_blue: Color32::from_rgb(0x3a, 0x6f, 0xd8), keyframe: Color32::from_rgb(0xa8, 0xa8, 0xa8), keyframe_selected: Color32::from_rgb(0x3d, 0x8f, 0xf5), pasteboard: Color32::from_rgb(0x1a, 0x1a, 0x1a), diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 76ff6fa3..09e15047 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -27,10 +27,12 @@ const LAYERS: &[(&str, u8)] = &[ ("raster", 1), ("keyframe", 1), ("path", 1), + ("psd", 1), ("project", 2), ("text", 2), ("effects", 2), ("track", 2), + ("svg", 2), ("render", 3), ("media", 3), ("expr", 3), From 406706e3d55b47f6addf342d1ddc08db12ff2b09 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:28:15 -0400 Subject: [PATCH 04/13] OUT-2: WebM export (VP9 intra encoder + alpha, Opus), WAV/AIFF audio-only outputs - effectcraft-vp9enc (L0): VP9 profile 0 key-frame encoder from the VP9 bitstream spec (8x8 intra blocks, DC/V/H/TM, 4x4 DCT/ADST, lossless WHT at quality 100, tile columns); round-trips exactly through FilmCraft's VP9 decoder and ffmpeg. - export: WebM muxer (EBML/Matroska spec) with VP9, VP9 alpha in BlockAdditional (AlphaMode 1), Opus (CodecDelay = pre-skip); WAV and AIFF outputs. - OutputFormat WebM/Wav/Aiff in the Render Queue, renderQueue.add formats, CLI --format. - Tests: FilmCraft decode-back (frames, size, pixels, audio RMS), ffprobe/ffmpeg oracles. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 6 + Cargo.toml | 5 + apps/effectcraft-cli/src/main.rs | 2 +- crates/engine/src/commands/render_queue.rs | 8 +- crates/engine/src/rq_tests.rs | 2 +- crates/export/Cargo.toml | 4 +- crates/export/src/lib.rs | 6 + crates/export/src/webm.rs | 368 +++++++++ crates/export/tests/roundtrip.rs | 92 +++ crates/export/tests/vp9_roundtrip.rs | 149 ++++ crates/host/tests/vector_import.rs | 23 + crates/media/src/convert.rs | 1 - crates/project/src/render_queue.rs | 30 +- crates/ui-egui/src/panels/render_queue.rs | 2 +- crates/vp9enc/Cargo.toml | 12 + crates/vp9enc/README.md | 15 + crates/vp9enc/src/bool.rs | 193 +++++ crates/vp9enc/src/lib.rs | 857 +++++++++++++++++++++ crates/vp9enc/src/tables.rs | 130 ++++ docs/architecture.md | 15 +- docs/parity.md | 10 +- xtask/src/main.rs | 1 + 22 files changed, 1911 insertions(+), 20 deletions(-) create mode 100644 crates/export/src/webm.rs create mode 100644 crates/export/tests/vp9_roundtrip.rs create mode 100644 crates/vp9enc/Cargo.toml create mode 100644 crates/vp9enc/README.md create mode 100644 crates/vp9enc/src/bool.rs create mode 100644 crates/vp9enc/src/lib.rs create mode 100644 crates/vp9enc/src/tables.rs diff --git a/Cargo.lock b/Cargo.lock index 866ae15f..9b28b8ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1226,11 +1226,13 @@ dependencies = [ "effectcraft-raster", "effectcraft-render", "effectcraft-time", + "effectcraft-vp9enc", "filmcraft-aac", "filmcraft-h264enc", "filmcraft-isobmff", "filmcraft-opus", "filmcraft-prores", + "filmcraft-vp9", "gif", "image", "log", @@ -1485,6 +1487,10 @@ dependencies = [ "web-time", ] +[[package]] +name = "effectcraft-vp9enc" +version = "0.1.0" + [[package]] name = "effectcraft-web" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 08a7d19d..dc2ce073 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ effectcraft-export = { path = "crates/export" } effectcraft-expr = { path = "crates/expr" } effectcraft-lottie = { path = "crates/lottie" } effectcraft-psd = { path = "crates/psd" } +effectcraft-vp9enc = { path = "crates/vp9enc" } effectcraft-svg = { path = "crates/svg" } effectcraft-engine = { path = "crates/engine" } effectcraft-host = { path = "crates/host" } @@ -71,6 +72,7 @@ filmcraft-aac = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf2 filmcraft-isobmff = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } # decoder used as the round-trip oracle of effectcraft-opusenc (crates/export dev-dependency) filmcraft-opus = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } +filmcraft-vp9 = { git = "https://github.com/storytold/filmcraft", rev = "89cfbf24f698b0f0e538eb7795929f19bbfb1dbb" } # gpu (the same wgpu eframe / egui-wgpu use) wgpu = { version = "30.0.1", features = ["fragile-send-sync-non-atomic-wasm"] } @@ -140,6 +142,9 @@ opt-level = 3 [profile.dev.package.effectcraft-export] opt-level = 3 +[profile.dev.package.effectcraft-vp9enc] +opt-level = 3 + # FilmCraft codec hot loops (same settings as FilmCraft's own workspace). [profile.dev.package.filmcraft-h264] opt-level = 3 diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rs index 90d7ef66..3a04e15c 100644 --- a/apps/effectcraft-cli/src/main.rs +++ b/apps/effectcraft-cli/src/main.rs @@ -9,7 +9,7 @@ //! effectcraft-cli get [--time S] read a property //! effectcraft-cli set [--time S] [--expression E] //! effectcraft-cli render-frame [--comp C] [--time S|--frame N] [--max-side PX|--scale K] [--out F.png] -//! effectcraft-cli render [--comp C] --out FILE [--format h264|prores|png|jpeg|tiff|exr|gif] [--start S] [--end S] +//! effectcraft-cli render [--comp C] --out FILE [--format h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff] [--start S] [--end S] //! [--work-area] [--fps N] [--resolution full|half|third|quarter|K] [--quality best|draft] [--channels rgb|rgba] //! [--jpeg-quality N] [--bitrate KBPS] [--prores proxy|lt|standard|hq|4444|4444xq] [--audio auto|on|off] //! effectcraft-cli render F.ecproj --queue render the project's Render Queue diff --git a/crates/engine/src/commands/render_queue.rs b/crates/engine/src/commands/render_queue.rs index c905ce6c..be6cebfb 100644 --- a/crates/engine/src/commands/render_queue.rs +++ b/crates/engine/src/commands/render_queue.rs @@ -131,7 +131,7 @@ fn apply_settings(rs: &mut RenderSettings, p: &Value, cmd: &str) -> Result fn apply_output(om: &mut OutputModule, p: &Value, cmd: &str) -> Result { let mut any = false; if let Some(f) = str_p(p, "format") { - let f = OutputFormat::from_name(f).ok_or_else(|| bad(cmd, "format: h264|prores|png|jpeg|tiff|exr|gif"))?; + let f = OutputFormat::from_name(f).ok_or_else(|| bad(cmd, "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; om.set_format(f); any = true; } @@ -256,7 +256,7 @@ fn add(s: &mut Session, p: &Value) -> Result { let cid = s.comp_for_queue(p).ok_or(EngineError::NoComp)?; let mut it = RenderQueueItem::new(0, cid); if let Some(f) = str_p(p, "format") { - let f = OutputFormat::from_name(f).ok_or_else(|| bad("renderQueue.add", "format: h264|prores|png|jpeg|tiff|exr|gif"))?; + let f = OutputFormat::from_name(f).ok_or_else(|| bad("renderQueue.add", "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; it.output = OutputModule::for_format(f); } apply_settings(&mut it.settings, p, "renderQueue.add")?; @@ -434,7 +434,7 @@ fn add_output_module(s: &mut Session, p: &Value) -> Result { let n = it.extra_outputs.len() + 2; let mut om = it.output.clone(); if let Some(f) = str_p(p, "format") { - let f = OutputFormat::from_name(f).ok_or_else(|| bad("render.addOutputModule", "format: h264|prores|png|jpeg|tiff|exr|gif"))?; + let f = OutputFormat::from_name(f).ok_or_else(|| bad("render.addOutputModule", "format: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff"))?; om = OutputModule::for_format(f); } if str_p(p, "output").is_none() && str_p(p, "path").is_none() { @@ -509,7 +509,7 @@ pub fn specs() -> Vec { "Add to Render Queue", ["Composition"], Some("Cmd+M"), - "{comp?: id|name, format?: h264|prores|png|jpeg|tiff|exr|gif, output?: path|template, quality?: best|draft|1-100 (jpeg), resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, channels?: rgb|rgba, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool}", + "{comp?: id|name, format?: h264|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff, output?: path|template, quality?: best|draft|1-100 (jpeg, webm), resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, channels?: rgb|rgba, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool}", can_add, add ), diff --git a/crates/engine/src/rq_tests.rs b/crates/engine/src/rq_tests.rs index c4aa08b8..f172b5e9 100644 --- a/crates/engine/src/rq_tests.rs +++ b/crates/engine/src/rq_tests.rs @@ -159,6 +159,6 @@ fn without_exporter() { s.execute("renderQueue.add", json!({})).unwrap(); assert!(!s.is_enabled("renderQueue.render")); let f = s.execute("renderQueue.formats", json!({})).unwrap(); - assert_eq!(f["formats"].as_array().unwrap().len(), 7); + assert_eq!(f["formats"].as_array().unwrap().len(), 10); assert_eq!(f["formats"][0]["available"], false); } diff --git a/crates/export/Cargo.toml b/crates/export/Cargo.toml index 5bdd0fee..15f9b17d 100644 --- a/crates/export/Cargo.toml +++ b/crates/export/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-export" -description = "Render Queue jobs and output modules: H.264 MP4 (+AAC) and ProRes MOV (+PCM) via FilmCraft's pure-Rust encoders, PNG/JPEG/TIFF/EXR sequences, animated GIF" +description = "Render Queue jobs and output modules: H.264 MP4 (+AAC) and ProRes MOV (+PCM) via FilmCraft's pure-Rust encoders, WebM (VP9 + Opus), PNG/JPEG/TIFF/EXR sequences, animated GIF, WAV/AIFF" version.workspace = true edition.workspace = true license.workspace = true @@ -19,6 +19,7 @@ filmcraft-prores = { workspace = true } filmcraft-aac = { workspace = true } filmcraft-isobmff = { workspace = true } effectcraft-opusenc = { workspace = true } +effectcraft-vp9enc = { workspace = true } image = { workspace = true, features = ["png", "jpeg", "tiff", "exr"] } gif = { workspace = true } rayon = { workspace = true } @@ -30,3 +31,4 @@ log = { workspace = true } effectcraft-media = { workspace = true } effectcraft-color = { workspace = true } filmcraft-opus = { workspace = true } +filmcraft-vp9 = { workspace = true } diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index dc4c35c8..a2af6a4f 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -7,6 +7,8 @@ //! | PNG / JPEG / TIFF sequence | one file per frame | 8-bit | PNG, TIFF | — | //! | OpenEXR sequence | one file per frame | 32-bit float, linear light, premultiplied | yes | — | //! | Animated GIF | GIF89a | 256-colour palette per frame (NeuQuant) | 1-bit | — | +//! | WebM | WebM (Matroska) | VP9 profile 0 intra frames (`effectcraft-vp9enc`), 8-bit 4:2:0 | VP9 alpha (BlockAdditional) | Opus (`effectcraft-opusenc`), 48 kHz stereo | +//! | WAV / AIFF | RIFF WAVE / AIFF | — | — | 16-bit PCM stereo (audio only) | //! //! Frames are rendered in parallel batches (rayon; one batch ≈ one frame per core) and handed to //! the encoder in order. Progress is reported after each batch through a callback that can cancel @@ -18,6 +20,7 @@ mod encode; mod out; +mod webm; use web_time::Instant; @@ -131,6 +134,9 @@ pub fn export(job: &Job, progress: &mut dyn FnMut(&Progress) -> bool) -> Result< let mut report = match job.output.format { OutputFormat::H264 | OutputFormat::ProRes => encode::movie(job, comp, w, h, &mut st)?, OutputFormat::Gif => gif_export(job, comp, w, h, &mut st)?, + OutputFormat::WebM => webm::webm(job, comp, w, h, &mut st)?, + OutputFormat::Wav => webm::audio_file(job, comp, false, &mut st)?, + OutputFormat::Aiff => webm::audio_file(job, comp, true, &mut st)?, f if f.is_sequence() => sequence(job, comp, w, h, &mut st)?, f => return Err(ExportError::Unsupported(f.label().into())), }; diff --git a/crates/export/src/webm.rs b/crates/export/src/webm.rs new file mode 100644 index 00000000..eb99cd22 --- /dev/null +++ b/crates/export/src/webm.rs @@ -0,0 +1,368 @@ +//! WebM output (VP9 video, optional VP9 alpha, Opus audio) and audio-only WAV / AIFF. +//! +//! The WebM container is written from the Matroska / WebM specifications (IETF RFC 8794 EBML, +//! the Matroska element registry and the WebM container guidelines): EBML header, Segment +//! (size patched at the end), Info, Tracks and one Cluster per render batch. Video frames are +//! SimpleBlocks; with alpha (Channels: RGB + Alpha) each frame is a BlockGroup whose +//! BlockAdditional (BlockAddID 1) carries a second VP9 frame coding the alpha channel as luma, +//! with `AlphaMode` 1 on the track, as WebM defines. VP9 comes from `effectcraft-vp9enc` (every +//! frame a key frame), Opus from `effectcraft-opusenc` (CELT, 48 kHz stereo, 20 ms packets, +//! `CodecDelay` = pre-skip). WAV (RIFF, 16-bit PCM) and AIFF (big-endian 16-bit, 80-bit +//! extended sample rate) follow their published file layouts. + +use std::io::{Seek, SeekFrom, Write}; + +use effectcraft_project::Comp; +use effectcraft_project::render_queue::Channels; +use effectcraft_time::{TICKS_PER_SECOND, Tick}; +use rayon::prelude::*; + +use crate::{Job, Report, Result, State, batch_size, io, render_frame, rgba8, wants_audio}; + +const OPUS_RATE: u32 = 48_000; +const OPUS_BITRATE: u32 = 192_000; + +// Element ids (with their length markers). +const EBML: u32 = 0x1A45_DFA3; +const SEGMENT: u32 = 0x1853_8067; +const INFO: u32 = 0x1549_A966; +const TRACKS: u32 = 0x1654_AE6B; +const TRACK_ENTRY: u32 = 0xAE; +const CLUSTER: u32 = 0x1F43_B675; + +fn id_bytes(id: u32) -> Vec { + let b = id.to_be_bytes(); + let skip = b.iter().position(|x| *x != 0).unwrap_or(3); + b[skip..].to_vec() +} + +/// EBML variable-size integer for an element data size (shortest form). +fn vint(n: u64) -> Vec { + for len in 1..=8u32 { + let max = (1u64 << (7 * len)) - 2; + if n <= max { + let marked = n | (1u64 << (7 * len)); + return marked.to_be_bytes()[8 - len as usize..].to_vec(); + } + } + let mut v = vec![0x01]; + v.extend_from_slice(&n.to_be_bytes()[1..]); + v +} + +fn el(out: &mut Vec, id: u32, data: &[u8]) { + out.extend(id_bytes(id)); + out.extend(vint(data.len() as u64)); + out.extend_from_slice(data); +} + +fn el_uint(out: &mut Vec, id: u32, v: u64) { + let b = v.to_be_bytes(); + let skip = b.iter().position(|x| *x != 0).unwrap_or(7); + el(out, id, &b[skip..]); +} + +fn el_float(out: &mut Vec, id: u32, v: f64) { + el(out, id, &v.to_be_bytes()); +} + +fn el_str(out: &mut Vec, id: u32, s: &str) { + el(out, id, s.as_bytes()); +} + +/// A block payload: track number, timestamp relative to the cluster, flags, frame. +fn block(track: u64, rel: i16, flags: u8, data: &[u8]) -> Vec { + let mut b = vint(track); + b.extend_from_slice(&rel.to_be_bytes()); + b.push(flags); + b.extend_from_slice(data); + b +} + +struct Frame { + ms: i64, + track: u64, + data: Vec, + alpha: Option>, +} + +fn cluster(frames: &mut [Frame]) -> Vec { + frames.sort_by_key(|f| (f.ms, f.track)); + let base = frames.first().map_or(0, |f| f.ms); + let mut body = vec![]; + el_uint(&mut body, 0xE7, base.max(0) as u64); + for f in frames.iter() { + let rel = (f.ms - base).clamp(i16::MIN as i64, i16::MAX as i64) as i16; + match &f.alpha { + None => el(&mut body, 0xA3, &block(f.track, rel, 0x80, &f.data)), + Some(a) => { + let mut g = vec![]; + el(&mut g, 0xA1, &block(f.track, rel, 0x00, &f.data)); + let mut more = vec![]; + el_uint(&mut more, 0xEE, 1); + el(&mut more, 0xA5, a); + let mut adds = vec![]; + el(&mut adds, 0xA6, &more); + el(&mut g, 0x75A1, &adds); + el(&mut body, 0xA0, &g); + } + } + } + let mut out = vec![]; + el(&mut out, CLUSTER, &body); + out +} + +pub(crate) fn webm(job: &Job, comp: &Comp, w: u32, h: u32, st: &mut State) -> Result { + let rate = job.settings.rate(comp); + let alpha = job.output.channels == Channels::Rgba; + let quality = job.output.quality.clamp(1, 100); + let cfg = effectcraft_vp9enc::EncoderConfig { width: w, height: h, quality, full_range: false }; + let with_audio = wants_audio(job); + let mut opus = with_audio.then(|| effectcraft_opusenc::OpusEncoder::new(2, OPUS_BITRATE)); + let total = st.total; + let frame_ms = |k: u64| -> i64 { ((k as i128 * 1000 * rate.den as i128 + rate.num as i128 / 2) / rate.num as i128) as i64 }; + let duration_ms = frame_ms(total) as f64; + + let mut file = crate::out::create(job.sink, job.path)?; + // EBML header. + let mut head = vec![]; + let mut eb = vec![]; + el_uint(&mut eb, 0x4286, 1); + el_uint(&mut eb, 0x42F7, 1); + el_uint(&mut eb, 0x42F2, 4); + el_uint(&mut eb, 0x42F3, 8); + el_str(&mut eb, 0x4282, "webm"); + el_uint(&mut eb, 0x4287, 4); + el_uint(&mut eb, 0x4285, 2); + el(&mut head, EBML, &eb); + // Segment with an 8-byte size patched at the end. + head.extend(id_bytes(SEGMENT)); + let size_pos = head.len() as u64; + head.extend_from_slice(&[0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]); + let seg_start = head.len() as u64; + let mut info = vec![]; + el_uint(&mut info, 0x2AD7B1, 1_000_000); + el_float(&mut info, 0x4489, duration_ms); + el_str(&mut info, 0x4D80, "EffectCraft"); + el_str(&mut info, 0x5741, "EffectCraft"); + el(&mut head, INFO, &info); + let mut tracks = vec![]; + let mut v = vec![]; + el_uint(&mut v, 0xD7, 1); + el_uint(&mut v, 0x73C5, 1); + el_uint(&mut v, 0x83, 1); + el_uint(&mut v, 0x9C, 0); + el_str(&mut v, 0x86, "V_VP9"); + el_uint(&mut v, 0x23E383, (1_000_000_000u128 * rate.den as u128 / rate.num.max(1) as u128) as u64); + if alpha { + el_uint(&mut v, 0x55EE, 1); + } + let mut video = vec![]; + el_uint(&mut video, 0xB0, w as u64); + el_uint(&mut video, 0xBA, h as u64); + if alpha { + el_uint(&mut video, 0x53C0, 1); + } + el(&mut v, 0xE0, &video); + el(&mut tracks, TRACK_ENTRY, &v); + if let Some(o) = &opus { + let mut a = vec![]; + el_uint(&mut a, 0xD7, 2); + el_uint(&mut a, 0x73C5, 2); + el_uint(&mut a, 0x83, 2); + el_uint(&mut a, 0x9C, 0); + el_str(&mut a, 0x86, "A_OPUS"); + el(&mut a, 0x63A2, &o.opus_head()); + el_uint(&mut a, 0x56AA, o.pre_skip() as u64 * 1_000_000_000 / OPUS_RATE as u64); + el_uint(&mut a, 0x56BB, 80_000_000); + let mut au = vec![]; + el_float(&mut au, 0xB5, OPUS_RATE as f64); + el_uint(&mut au, 0x9F, 2); + el(&mut a, 0xE1, &au); + el(&mut tracks, TRACK_ENTRY, &a); + } + el(&mut head, TRACKS, &tracks); + file.write_all(&head).map_err(io)?; + let mut seg_len = head.len() as u64 - seg_start; + + // Audio: mixed in blocks, encoded in 20 ms packets. + let (span_start, span_end) = job.settings.span(comp); + let mut cursor = span_start.to_units_floor(OPUS_RATE as i64); + let mut pcm_left: Vec = vec![]; + let mut packets = 0u64; + let mut audio_until = |until: Tick, frames: &mut Vec, finish: bool| { + let Some(enc) = opus.as_mut() else { return }; + let end = until.to_units_floor(OPUS_RATE as i64); + if end > cursor { + let n = (end - cursor) as usize; + let start = Tick(((cursor as i128 * TICKS_PER_SECOND as i128) / OPUS_RATE as i128) as i64); + pcm_left.extend(effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, start, n, OPUS_RATE)); + cursor = end; + } + let fs = effectcraft_opusenc::OpusEncoder::FRAME_SIZE * 2; + if finish && !pcm_left.is_empty() { + // Pad the last packet (pre-skip + padding cover the encoder delay). + let pad = (fs - pcm_left.len() % fs) % fs + fs; + pcm_left.extend(std::iter::repeat_n(0.0, pad)); + } + while pcm_left.len() >= fs { + let pkt = enc.encode_float(&pcm_left[..fs]); + pcm_left.drain(..fs); + frames.push(Frame { ms: packets as i64 * 20, track: 2, data: pkt, alpha: None }); + packets += 1; + } + }; + + let batch = batch_size(); + let mut i = 0; + while i < total { + let end = (i + batch).min(total); + let encoded: Vec<(Vec, Option>)> = (i..end) + .into_par_iter() + .map(|k| { + let px = rgba8(&render_frame(job, comp, k), comp, if alpha { Channels::Rgba } else { Channels::Rgb }, w, h); + let mut enc = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone()); + let (y, u, vv) = effectcraft_vp9enc::rgba_to_yuv420(&px, w, h, false); + let color = enc.encode_yuv420(&y, &u, &vv); + let a = alpha.then(|| { + let mut ea = effectcraft_vp9enc::Vp9Encoder::new(cfg.clone()); + let (y, u, vv) = effectcraft_vp9enc::alpha_to_yuv420(&px, w, h); + ea.encode_yuv420(&y, &u, &vv) + }); + (color, a) + }) + .collect(); + let mut frames: Vec = + encoded.into_iter().enumerate().map(|(j, (data, alpha))| Frame { ms: frame_ms(i + j as u64), track: 1, data, alpha }).collect(); + let t_end = if end >= total { span_end } else { job.settings.frame_time(comp, end) }; + audio_until(t_end, &mut frames, end >= total); + let c = cluster(&mut frames); + file.write_all(&c).map_err(io)?; + seg_len += c.len() as u64; + st.advance(end - i)?; + i = end; + } + if total == 0 { + let mut frames = vec![]; + audio_until(span_end, &mut frames, true); + if !frames.is_empty() { + let c = cluster(&mut frames); + file.write_all(&c).map_err(io)?; + seg_len += c.len() as u64; + } + } + // Patch the Segment size. + let mut size = [0u8; 8]; + size[0] = 0x01; + size[1..].copy_from_slice(&seg_len.to_be_bytes()[1..]); + file.seek(SeekFrom::Start(size_pos)).map_err(io)?; + file.write_all(&size).map_err(io)?; + file.seek(SeekFrom::End(0)).map_err(io)?; + let bytes = file.finish()?; + Ok(Report { path: job.path.to_string(), frames: 0, width: w, height: h, seconds: 0.0, bytes, audio: with_audio }) +} + +// ---------------------------------------------------------------- audio-only outputs + +fn ieee_extended(v: f64) -> [u8; 10] { + // 80-bit IEEE 754 extended precision (AIFF COMM sample rate). + let mut out = [0u8; 10]; + if v <= 0.0 { + return out; + } + let e = v.log2().floor() as i32; + let mant = (v / 2f64.powi(e) * (1u64 << 63) as f64) as u64; + out[..2].copy_from_slice(&((e + 16383) as u16).to_be_bytes()); + out[2..].copy_from_slice(&mant.to_be_bytes()); + out +} + +/// WAV or AIFF: the comp's audio over the render span, 16-bit stereo. +pub(crate) fn audio_file(job: &Job, comp: &Comp, aiff: bool, st: &mut State) -> Result { + let sr = job.output.audio_sample_rate.clamp(8_000, 192_000); + let (span_start, span_end) = job.settings.span(comp); + let start = span_start.to_units_floor(sr as i64); + let end = span_end.to_units_floor(sr as i64); + let n = (end - start).max(0) as usize; + let mut pcm = Vec::with_capacity(n * 4); + // Mix in one-second blocks, reporting progress in frames. + let block = sr as usize; + let mut done = 0usize; + let total_frames = st.total.max(1); + let mut reported = 0u64; + while done < n { + let k = block.min(n - done); + let t = Tick((((start + done as i64) as i128 * TICKS_PER_SECOND as i128) / sr as i128) as i64); + for s in effectcraft_render::audio::mix_comp(job.project, job.footage, job.expr, job.comp, t, k, sr) { + let v = (s.clamp(-1.0, 1.0) * 32767.0).round() as i16; + pcm.extend_from_slice(&if aiff { v.to_be_bytes() } else { v.to_le_bytes() }); + } + done += k; + let now = (done as u64 * total_frames / n.max(1) as u64).min(total_frames); + if now > reported && st.total > 0 { + st.advance(now - reported)?; + reported = now; + } + } + if reported < st.total { + st.advance(st.total - reported)?; + } + let mut out = vec![]; + let frames = (pcm.len() / 4) as u32; + if aiff { + let mut comm = vec![]; + comm.extend_from_slice(&2u16.to_be_bytes()); + comm.extend_from_slice(&frames.to_be_bytes()); + comm.extend_from_slice(&16u16.to_be_bytes()); + comm.extend_from_slice(&ieee_extended(sr as f64)); + let ssnd_len = 8 + pcm.len(); + out.extend_from_slice(b"FORM"); + out.extend_from_slice(&((4 + 8 + comm.len() + 8 + ssnd_len + ssnd_len % 2) as u32).to_be_bytes()); + out.extend_from_slice(b"AIFF"); + out.extend_from_slice(b"COMM"); + out.extend_from_slice(&(comm.len() as u32).to_be_bytes()); + out.extend_from_slice(&comm); + out.extend_from_slice(b"SSND"); + out.extend_from_slice(&(ssnd_len as u32).to_be_bytes()); + out.extend_from_slice(&[0; 8]); + out.extend_from_slice(&pcm); + if ssnd_len % 2 == 1 { + out.push(0); + } + } else { + out.extend_from_slice(b"RIFF"); + out.extend_from_slice(&((36 + pcm.len()) as u32).to_le_bytes()); + out.extend_from_slice(b"WAVEfmt "); + out.extend_from_slice(&16u32.to_le_bytes()); + out.extend_from_slice(&1u16.to_le_bytes()); + out.extend_from_slice(&2u16.to_le_bytes()); + out.extend_from_slice(&sr.to_le_bytes()); + out.extend_from_slice(&(sr * 4).to_le_bytes()); + out.extend_from_slice(&4u16.to_le_bytes()); + out.extend_from_slice(&16u16.to_le_bytes()); + out.extend_from_slice(b"data"); + out.extend_from_slice(&(pcm.len() as u32).to_le_bytes()); + out.extend_from_slice(&pcm); + } + let mut file = crate::out::create(job.sink, job.path)?; + file.write_all(&out).map_err(io)?; + let bytes = file.finish()?; + Ok(Report { path: job.path.to_string(), frames: 0, width: 0, height: 0, seconds: 0.0, bytes, audio: true }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn vints_and_ids() { + assert_eq!(vint(0), vec![0x80]); + assert_eq!(vint(126), vec![0xFE]); + assert_eq!(vint(127), vec![0x40, 0x7F]); + assert_eq!(id_bytes(0xA3), vec![0xA3]); + assert_eq!(id_bytes(EBML), vec![0x1A, 0x45, 0xDF, 0xA3]); + assert_eq!(ieee_extended(44100.0), [0x40, 0x0E, 0xAC, 0x44, 0, 0, 0, 0, 0, 0]); + assert_eq!(ieee_extended(48000.0)[..4], [0x40, 0x0E, 0xBB, 0x80]); + } +} diff --git a/crates/export/tests/roundtrip.rs b/crates/export/tests/roundtrip.rs index 186aa1a9..1ade594f 100644 --- a/crates/export/tests/roundtrip.rs +++ b/crates/export/tests/roundtrip.rs @@ -363,3 +363,95 @@ fn movies_carry_audio() { assert!(!effectcraft_media::probe(&path).expect("probe").has_audio); } } + +#[test] +fn webm_vp9_opus_roundtrip() { + let d = out_dir("webm"); + let wav = d.join("tone.wav"); + write_tone(&wav, 2.0); + let (p, cid) = project(Some(&wav)); + let pool = MediaPool::new(); + let om = OutputModule { quality: 90, audio: AudioOutput::Auto, ..OutputModule::for_format(OutputFormat::WebM) }; + let path = d.join("comp.webm"); + let r = run(&p, cid, &pool, &om, &path); + assert!(r.audio && r.bytes > 200, "{r:?}"); + // Decoded back through FilmCraft (Matroska demuxer, VP9 and Opus decoders). + let f = decode_movie(&path, FRAMES, 0.08); + assert!(f.has_audio); + let s = MediaPool::new().audio_samples(&f, Tick::from_seconds_f64(0.3), 24_000, 48_000); + let l = rms(s.iter().step_by(2).copied()); + assert!((l - 0.3536).abs() < 0.06, "left RMS {l}"); + ffmpeg_check(&path, 0.08); + if let Some(s) = ffprobe(&path) { + assert!(s.contains("vp9|64|48|yuv420p"), "{s}"); + assert!(s.contains("opus"), "{s}"); + } +} + +#[test] +fn webm_with_alpha() { + let d = out_dir("webm-alpha"); + let (p, cid) = project(None); + // Transparent outside the red solid. + let om = OutputModule { channels: Channels::Rgba, ..OutputModule::for_format(OutputFormat::WebM) }; + let path = d.join("alpha.webm"); + run(&p, cid, &NoFootage, &om, &path); + let f = effectcraft_media::probe(&path).expect("probe our output"); + assert_eq!((f.width, f.height), (W, H)); + let bytes = std::fs::read(&path).unwrap(); + // AlphaMode element (0x53C0) and BlockAdditions (0x75A1). + assert!(bytes.windows(3).any(|w| w == [0x53, 0xC0, 0x81]), "AlphaMode 1"); + assert!(bytes.windows(2).any(|w| w == [0x75, 0xA1]), "BlockAdditions"); + // ffmpeg's libvpx decoder reads the alpha channel. + let png = path.with_extension("alpha.png"); + let ok = Command::new("ffmpeg") + .args(["-hide_banner", "-loglevel", "error", "-y", "-c:v", "libvpx-vp9", "-i"]) + .arg(&path) + .args(["-vf", "select=eq(n\\,8)", "-frames:v", "1", "-pix_fmt", "rgba"]) + .arg(&png) + .status() + .map(|s| s.success()) + .unwrap_or(false); + if ok { + let img = image::open(&png).unwrap().to_rgba8(); + assert!(img.get_pixel(2, 2)[3] < 20, "corner transparent"); + assert!(img.get_pixel(W / 2, H / 2)[3] > 235, "centre opaque"); + } else { + eprintln!("ffmpeg with libvpx not available: skipping alpha oracle"); + } +} + +#[test] +fn wav_and_aiff_audio_only() { + let d = out_dir("audio-only"); + let wav = d.join("tone.wav"); + write_tone(&wav, 2.0); + let (p, cid) = project(Some(&wav)); + let pool = MediaPool::new(); + for (fmt, name, magic) in [(OutputFormat::Wav, "out.wav", &b"RIFF"[..]), (OutputFormat::Aiff, "out.aif", &b"FORM"[..])] { + let om = OutputModule::for_format(fmt); + let path = d.join(name); + let r = run(&p, cid, &pool, &om, &path); + assert!(r.audio); + let bytes = std::fs::read(&path).unwrap(); + assert_eq!(&bytes[..4], magic); + // 1.2 s of 48 kHz 16-bit stereo. + assert!((bytes.len() as i64 - 57_600 * 4).abs() < 200, "{name}: {}", bytes.len()); + let l = if fmt == OutputFormat::Wav { + // Decoded by FilmCraft. + let f = effectcraft_media::probe(&path).expect("probe audio"); + assert!(f.has_audio && !f.has_video, "{name}"); + let s = MediaPool::new().audio_samples(&f, Tick::from_seconds_f64(0.3), 24_000, 48_000); + rms(s.iter().step_by(2).copied()) + } else { + // AIFF: big-endian samples after the SSND chunk's offset/block size. + let at = bytes.windows(4).position(|w| w == b"SSND").expect("SSND") + 16; + let left = bytes[at..].chunks_exact(4).skip(14_400).take(24_000).map(|c| i16::from_be_bytes([c[0], c[1]]) as f32 / 32767.0); + rms(left) + }; + assert!((l - 0.3536).abs() < 0.01, "{name}: left RMS {l}"); + if let Some(s) = ffprobe(&path) { + assert!(s.contains("pcm_s16"), "{s}"); + } + } +} diff --git a/crates/export/tests/vp9_roundtrip.rs b/crates/export/tests/vp9_roundtrip.rs new file mode 100644 index 00000000..8156d850 --- /dev/null +++ b/crates/export/tests/vp9_roundtrip.rs @@ -0,0 +1,149 @@ +//! The VP9 intra encoder (`effectcraft-vp9enc`) round-trips through FilmCraft's VP9 decoder (and +//! ffmpeg as an external oracle when installed): sizes, lossless exactness, PSNR. + +use effectcraft_vp9enc::{EncoderConfig, Vp9Encoder}; +use filmcraft_vp9::{Decoder, Plane}; + +fn plane_u8(p: &Plane) -> Vec { + match p { + Plane::U8(v) => v.clone(), + Plane::U16(v) => v.iter().map(|x| *x as u8).collect(), + } +} + +/// Synthetic planes: gradients, noise and sharp edges. +fn source(w: usize, h: usize, kind: u32) -> (Vec, Vec, Vec) { + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let mut s = 99u32 + kind; + let mut rnd = || { + s = s.wrapping_mul(1664525).wrapping_add(1013904223); + (s >> 24) as u8 + }; + let y: Vec = (0..w * h) + .map(|i| { + let (x, yy) = (i % w, i / w); + match kind { + 0 => ((x * 255) / w.max(1)) as u8 / 2 + ((yy * 255) / h.max(1)) as u8 / 2, + 1 => rnd(), + _ => { + if (x / 7 + yy / 5) % 2 == 0 { + 30 + } else { + 220 + } + } + } + }) + .collect(); + let u: Vec = (0..cw * ch).map(|i| (64 + (i % cw) * 128 / cw.max(1)) as u8).collect(); + let v: Vec = (0..cw * ch).map(|i| (200 - (i / cw) * 100 / ch.max(1)) as u8).collect(); + (y, u, v) +} + +fn decode(frame: &[u8]) -> filmcraft_vp9::Picture { + let mut d = Decoder::new(); + let mut pics = d.decode(frame, 0).expect("decode"); + assert_eq!(pics.len(), 1); + pics.remove(0) +} + +fn crop(p: &[u8], stride: usize, w: usize, h: usize) -> Vec { + (0..h).flat_map(|y| p[y * stride..y * stride + w].to_vec()).collect() +} + +fn psnr(a: &[u8], b: &[u8]) -> f64 { + let mse: f64 = a.iter().zip(b).map(|(x, y)| (*x as f64 - *y as f64).powi(2)).sum::() / a.len() as f64; + if mse == 0.0 { 99.0 } else { 10.0 * (255.0f64 * 255.0 / mse).log10() } +} + +#[test] +fn lossless_is_exact_for_odd_sizes() { + for (w, h) in [(1, 1), (33, 17), (64, 64), (200, 120), (7, 70)] { + for kind in 0..3 { + let (y, u, v) = source(w, h, kind); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false }); + let f = e.encode_yuv420(&y, &u, &v); + let p = decode(&f); + assert_eq!((p.width, p.height), (w as u32, h as u32)); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + assert_eq!(crop(&plane_u8(&p.y), p.y_stride, w, h), y, "{w}x{h} kind {kind}: luma"); + assert_eq!(crop(&plane_u8(&p.u), p.uv_stride, cw, ch), u, "{w}x{h} kind {kind}: u"); + assert_eq!(crop(&plane_u8(&p.v), p.uv_stride, cw, ch), v, "{w}x{h} kind {kind}: v"); + } + } +} + +#[test] +fn lossy_quality_and_size() { + let (w, h) = (200, 120); + for kind in [0, 2] { + let (y, u, v) = source(w, h, kind); + let mut last = 0usize; + for q in [30u8, 60, 80, 95] { + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: q, full_range: false }); + let f = e.encode_yuv420(&y, &u, &v); + let p = decode(&f); + let got = crop(&plane_u8(&p.y), p.y_stride, w, h); + let db = psnr(&got, &y); + let min = match q { + 30 => 20.0, + 60 => 28.0, + 80 => 32.0, + _ => 38.0, + }; + assert!(db > min, "kind {kind} q{q}: PSNR {db:.1} dB, {} bytes", f.len()); + assert!(f.len() >= last, "higher quality, more bytes"); + last = f.len(); + } + } +} + +#[test] +fn wide_frames_use_tile_columns() { + // 4160 px > 64 superblocks: two tile columns are required. + let (w, h) = (4160, 16); + let (y, u, v) = source(w, h, 0); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 90, full_range: false }); + let f = e.encode_yuv420(&y, &u, &v); + let p = decode(&f); + assert!(psnr(&crop(&plane_u8(&p.y), p.y_stride, w, h), &y) > 35.0); +} + +#[test] +fn ffmpeg_decodes_our_stream() { + let (w, h) = (96, 64); + let (y, u, v) = source(w, h, 0); + let mut e = Vp9Encoder::new(EncoderConfig { width: w as u32, height: h as u32, quality: 100, full_range: false }); + let frame = e.encode_yuv420(&y, &u, &v); + // IVF container (one frame). + let mut ivf = Vec::new(); + ivf.extend_from_slice(b"DKIF"); + ivf.extend_from_slice(&0u16.to_le_bytes()); + ivf.extend_from_slice(&32u16.to_le_bytes()); + ivf.extend_from_slice(b"VP90"); + ivf.extend_from_slice(&(w as u16).to_le_bytes()); + ivf.extend_from_slice(&(h as u16).to_le_bytes()); + ivf.extend_from_slice(&30u32.to_le_bytes()); + ivf.extend_from_slice(&1u32.to_le_bytes()); + ivf.extend_from_slice(&1u32.to_le_bytes()); + ivf.extend_from_slice(&0u32.to_le_bytes()); + ivf.extend_from_slice(&(frame.len() as u32).to_le_bytes()); + ivf.extend_from_slice(&0u64.to_le_bytes()); + ivf.extend_from_slice(&frame); + let dir = std::env::temp_dir().join(format!("effectcraft-vp9-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("a.ivf"); + std::fs::write(&path, &ivf).unwrap(); + let out = std::process::Command::new("ffmpeg") + .args(["-hide_banner", "-loglevel", "error", "-i"]) + .arg(&path) + .args(["-f", "rawvideo", "-pix_fmt", "yuv420p", "-"]) + .output(); + let Ok(out) = out else { + eprintln!("ffmpeg not found: skipping oracle"); + return; + }; + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + assert_eq!(&out.stdout[..w * h], &y[..], "ffmpeg luma matches (lossless)"); + assert_eq!(&out.stdout[w * h..w * h + u.len()], &u[..]); +} diff --git a/crates/host/tests/vector_import.rs b/crates/host/tests/vector_import.rs index eb5be076..590112c9 100644 --- a/crates/host/tests/vector_import.rs +++ b/crates/host/tests/vector_import.rs @@ -161,3 +161,26 @@ fn svg_footage_and_shapes_from_vector_layer() { assert!(d3 < 0.03, "{name}: ×3 mean diff {d3}"); } } + +#[test] +fn svg_footage_continuously_rasterizes() { + let path = tmp("cr.svg"); + std::fs::write(&path, SVG).unwrap(); + let doc = effectcraft_svg::parse(SVG).unwrap(); + let (w, h) = doc.pixel_size(); + let mut s = effectcraft_host::session(); + let item = s.execute_checked("file.import", json!({"paths": [path]})).unwrap()["items"][0].as_u64().unwrap(); + s.execute("comp.new", json!({"name": "C", "width": w, "height": h, "frameRate": 30, "duration": 1})).unwrap(); + let lid = s.execute_checked("layer.addItem", json!({"item": item})).unwrap()["layer"].as_u64().unwrap(); + for (k, v) in [("transform/anchor", json!([0, 0])), ("transform/position", json!([0, 0])), ("transform/scale", json!([300, 300]))] { + s.execute("prop.set", json!({"layer": lid, "path": k, "value": v})).unwrap(); + } + let cid = s.active_comp_id().unwrap(); + let t = effectcraft_time::Tick::ZERO; + let want = effectcraft_svg::rasterize(&doc, w, h, 3.0); + let soft = mean_diff(&s.render(cid, t, RenderOpts::default()), &want); + s.execute("layer.setSwitch", json!({"layers": [lid], "switch": "collapse", "value": true})).unwrap(); + let sharp = mean_diff(&s.render(cid, t, RenderOpts::default()), &want); + assert!(sharp < 0.004, "continuously rasterised diff {sharp}"); + assert!(sharp < soft, "sharper than the upscaled pixels ({sharp} vs {soft})"); +} diff --git a/crates/media/src/convert.rs b/crates/media/src/convert.rs index dce33d8a..1edcf5fd 100644 --- a/crates/media/src/convert.rs +++ b/crates/media/src/convert.rs @@ -392,4 +392,3 @@ mod tests { } } } - diff --git a/crates/project/src/render_queue.rs b/crates/project/src/render_queue.rs index c41462b5..d1e6683c 100644 --- a/crates/project/src/render_queue.rs +++ b/crates/project/src/render_queue.rs @@ -136,17 +136,26 @@ pub enum OutputFormat { ExrSequence, /// Animated GIF. Gif, + /// WebM: VP9 video (alpha with RGB + Alpha) + Opus audio. + WebM, + /// Audio only: WAV (16-bit PCM). + Wav, + /// Audio only: AIFF (16-bit PCM). + Aiff, } impl OutputFormat { - pub const ALL: [OutputFormat; 7] = [ + pub const ALL: [OutputFormat; 10] = [ OutputFormat::H264, OutputFormat::ProRes, + OutputFormat::WebM, OutputFormat::PngSequence, OutputFormat::JpegSequence, OutputFormat::TiffSequence, OutputFormat::ExrSequence, OutputFormat::Gif, + OutputFormat::Wav, + OutputFormat::Aiff, ]; pub fn label(self) -> &'static str { @@ -158,6 +167,9 @@ impl OutputFormat { OutputFormat::TiffSequence => "TIFF Sequence", OutputFormat::ExrSequence => "OpenEXR Sequence", OutputFormat::Gif => "Animated GIF", + OutputFormat::WebM => "WebM (VP9 + Opus)", + OutputFormat::Wav => "WAV", + OutputFormat::Aiff => "AIFF", } } pub fn extension(self) -> &'static str { @@ -169,19 +181,26 @@ impl OutputFormat { OutputFormat::TiffSequence => "tif", OutputFormat::ExrSequence => "exr", OutputFormat::Gif => "gif", + OutputFormat::WebM => "webm", + OutputFormat::Wav => "wav", + OutputFormat::Aiff => "aif", } } pub fn is_sequence(self) -> bool { matches!(self, OutputFormat::PngSequence | OutputFormat::JpegSequence | OutputFormat::TiffSequence | OutputFormat::ExrSequence) } pub fn is_movie(self) -> bool { - matches!(self, OutputFormat::H264 | OutputFormat::ProRes) + matches!(self, OutputFormat::H264 | OutputFormat::ProRes | OutputFormat::WebM) + } + /// Audio-only outputs (no video is rendered). + pub fn is_audio_only(self) -> bool { + matches!(self, OutputFormat::Wav | OutputFormat::Aiff) } pub fn supports_alpha(self) -> bool { - !matches!(self, OutputFormat::H264 | OutputFormat::JpegSequence) + !matches!(self, OutputFormat::H264 | OutputFormat::JpegSequence | OutputFormat::Wav | OutputFormat::Aiff) } pub fn supports_audio(self) -> bool { - self.is_movie() + self.is_movie() || self.is_audio_only() } /// The written frame size for a rendered size: H.264 needs even dimensions (rounded down, /// at least 2). @@ -201,6 +220,9 @@ impl OutputFormat { "tif" | "tiff" | "tiffsequence" => OutputFormat::TiffSequence, "exr" | "openexr" | "openexrsequence" => OutputFormat::ExrSequence, "gif" | "animatedgif" => OutputFormat::Gif, + "webm" | "vp9" | "webmvp9opus" => OutputFormat::WebM, + "wav" | "wave" => OutputFormat::Wav, + "aif" | "aiff" | "aifc" => OutputFormat::Aiff, _ => return None, }) } diff --git a/crates/ui-egui/src/panels/render_queue.rs b/crates/ui-egui/src/panels/render_queue.rs index 154dd4c2..06403181 100644 --- a/crates/ui-egui/src/panels/render_queue.rs +++ b/crates/ui-egui/src/panels/render_queue.rs @@ -108,7 +108,7 @@ fn output_menu(it: &RenderQueueItem, available: &[OutputFormat]) -> Vec<(String, v.push((mark(o.bitrate_kbps == kbps, &format!("Bitrate: {} Mbps", kbps / 1000)), json!({"bitrate": kbps}))); } } - OutputFormat::JpegSequence => { + OutputFormat::JpegSequence | OutputFormat::WebM => { v.push(("-".into(), Value::Null)); for q in [60u8, 80, 90, 100] { v.push((mark(o.quality == q, &format!("Quality: {q}")), json!({"quality": q}))); diff --git a/crates/vp9enc/Cargo.toml b/crates/vp9enc/Cargo.toml new file mode 100644 index 00000000..62144b25 --- /dev/null +++ b/crates/vp9enc/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "effectcraft-vp9enc" +description = "VP9 intra-frame encoder (profile 0, 8-bit 4:2:0) for WebM export, from the VP9 Bitstream & Decoding Process Specification" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] diff --git a/crates/vp9enc/README.md b/crates/vp9enc/README.md new file mode 100644 index 00000000..c8f781b8 --- /dev/null +++ b/crates/vp9enc/README.md @@ -0,0 +1,15 @@ +# effectcraft-vp9enc + +A small VP9 encoder for WebM export: every frame is a key frame (profile 0, 8-bit 4:2:0, +BT.709), coded with 8×8 intra blocks (DC / V / H / TM prediction chosen per block), 4×4 +DCT/ADST transforms with default probabilities, or the Walsh–Hadamard transform at quality 100 +(mathematically lossless). + +Written from the **VP9 Bitstream & Decoding Process Specification, version 0.6 (31 March +2016)**: the encoder emits the uncompressed and compressed headers (§6.2, §6.3), partition, mode +info and token syntax (§6.4) and reconstructs with the specification's intra prediction (§8.5.1) +and integer inverse transforms (§8.7) so it stays in step with any conformant decoder. The +boolean encoder is the inverse of the §9.2 decoder in the form given in RFC 6386 §7.3. Spec +tables (§10) were transcribed from FilmCraft's first-party decoder `filmcraft-vp9` (MIT OR +Apache-2.0), which is also the decoder the tests round-trip through; ffmpeg is used only as an +external test oracle. No libvpx or other third-party encoder/decoder source was consulted. diff --git a/crates/vp9enc/src/bool.rs b/crates/vp9enc/src/bool.rs new file mode 100644 index 00000000..45db92dd --- /dev/null +++ b/crates/vp9enc/src/bool.rs @@ -0,0 +1,193 @@ +//! Boolean (arithmetic) encoder: the inverse of the VP9 boolean decoder (spec §9.2), written as +//! the classic carry-propagating range coder described in RFC 6386 §7.3 (VP8 and VP9 share the +//! bool coder). + +pub struct BoolEncoder { + out: Vec, + range: u32, + bottom: u32, + bit_count: i32, +} + +impl Default for BoolEncoder { + fn default() -> Self { + BoolEncoder::new() + } +} + +impl BoolEncoder { + pub fn new() -> Self { + BoolEncoder { out: Vec::new(), range: 255, bottom: 0, bit_count: 24 } + } + + fn add_one(&mut self) { + let mut i = self.out.len(); + while i > 0 { + i -= 1; + if self.out[i] == 255 { + self.out[i] = 0; + } else { + self.out[i] += 1; + return; + } + } + } + + /// Write `bit` with probability `prob`/256 of it being 0. + #[inline] + pub fn write(&mut self, bit: bool, prob: u8) { + let split = 1 + (((self.range - 1) * prob as u32) >> 8); + if bit { + self.bottom = self.bottom.wrapping_add(split); + self.range -= split; + } else { + self.range = split; + } + while self.range < 128 { + self.range <<= 1; + if self.bottom & (1 << 31) != 0 { + self.add_one(); + } + self.bottom <<= 1; + self.bit_count -= 1; + if self.bit_count == 0 { + self.out.push((self.bottom >> 24) as u8); + self.bottom &= (1 << 24) - 1; + self.bit_count = 8; + } + } + } + + pub fn literal(&mut self, v: u32, bits: u32) { + for i in (0..bits).rev() { + self.write((v >> i) & 1 != 0, 128); + } + } + + /// Write `value` with a tree (leaves stored negated, as in the decoder's tables). + pub fn tree(&mut self, tree: &[i8], probs: &[u8], value: u8) { + // Find the path to the leaf. + fn path(tree: &[i8], node: usize, value: u8, out: &mut Vec<(usize, bool)>) -> bool { + for b in 0..2 { + let t = tree[node + b]; + out.push((node >> 1, b == 1)); + if t <= 0 { + if (-t) as u8 == value { + return true; + } + } else if path(tree, t as usize, value, out) { + return true; + } + out.pop(); + } + false + } + let mut p = Vec::with_capacity(8); + path(tree, 0, value, &mut p); + for (i, bit) in p { + self.write(bit, probs[i]); + } + } + + /// Flush (RFC 6386 §7.3) and return the bytes. + pub fn finish(mut self) -> Vec { + let mut c = self.bit_count; + let mut v = self.bottom; + if v & (1u32 << (32 - c)) != 0 { + self.add_one(); + } + v <<= c & 7; + c >>= 3; + while { + c -= 1; + c >= 0 + } { + v <<= 8; + } + for _ in 0..4 { + self.out.push((v >> 24) as u8); + v <<= 8; + } + self.out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A straightforward decoder per VP9 spec §9.2. + struct Dec<'a> { + d: &'a [u8], + pos: usize, + value: u64, + bits: i32, + range: u32, + } + impl<'a> Dec<'a> { + fn new(d: &'a [u8]) -> Self { + let mut x = Dec { d, pos: 0, value: 0, bits: 0, range: 255 }; + x.fill(); + x + } + fn fill(&mut self) { + while self.bits <= 48 { + let b = if self.pos < self.d.len() { self.d[self.pos] } else { 0 }; + self.pos += 1; + self.value |= (b as u64) << (56 - self.bits); + self.bits += 8; + } + } + fn read(&mut self, p: u8) -> bool { + let split = 1 + (((self.range - 1) * p as u32) >> 8); + let big = (split as u64) << 56; + let bit = if self.value >= big { + self.range -= split; + self.value -= big; + true + } else { + self.range = split; + false + }; + while self.range < 128 { + self.range <<= 1; + self.value <<= 1; + self.bits -= 1; + } + self.fill(); + bit + } + } + + #[test] + fn round_trip() { + let mut e = BoolEncoder::new(); + let mut seq = vec![]; + let mut s = 12345u32; + for i in 0..20000 { + s = s.wrapping_mul(1103515245).wrapping_add(12345); + let p = ((s >> 8) % 255 + 1) as u8; + let bit = (s >> 20) % 256 >= p as u32 || i % 97 == 0; + seq.push((bit, p)); + e.write(bit, p); + } + let d = e.finish(); + let mut dec = Dec::new(&d); + for (i, (b, p)) in seq.into_iter().enumerate() { + assert_eq!(dec.read(p), b, "bit {i}"); + } + } + + #[test] + fn all_ones_carry() { + let mut e = BoolEncoder::new(); + for _ in 0..5000 { + e.write(true, 1); + } + let d = e.finish(); + let mut dec = Dec::new(&d); + for _ in 0..5000 { + assert!(dec.read(1)); + } + } +} diff --git a/crates/vp9enc/src/lib.rs b/crates/vp9enc/src/lib.rs new file mode 100644 index 00000000..02cbb0e2 --- /dev/null +++ b/crates/vp9enc/src/lib.rs @@ -0,0 +1,857 @@ +//! A VP9 intra-frame encoder (profile 0, 8-bit 4:2:0), written from the *VP9 Bitstream & +//! Decoding Process Specification* v0.6 by mirroring its decoding process in reverse (see +//! README.md). Every frame is a key frame: one tile row (tile columns as the frame width +//! requires), 64×64 superblocks split to 8×8 blocks, 4×4 transforms (the Walsh–Hadamard +//! transform at quality 100, which is lossless), intra modes DC / V / H / TM chosen per block by +//! prediction error, default probabilities (no probability updates). The encoder reconstructs +//! exactly what a decoder reconstructs (the specification's integer inverse transforms and +//! intra edge rules), so prediction never drifts. + +mod bool; +mod tables; + +use bool::BoolEncoder; +use tables::*; + +const DC_PRED: u8 = 0; +const V_PRED: u8 = 1; +const H_PRED: u8 = 2; +const TM_PRED: u8 = 9; +const MODES: [u8; 4] = [DC_PRED, V_PRED, H_PRED, TM_PRED]; + +const DCT_DCT: u8 = 0; +const ADST_DCT: u8 = 1; +const DCT_ADST: u8 = 2; +const ADST_ADST: u8 = 3; + +/// mode2txfm_map for the modes used here. +fn mode_tx_type(mode: u8) -> u8 { + match mode { + V_PRED => ADST_DCT, + H_PRED => DCT_ADST, + TM_PRED => ADST_ADST, + _ => DCT_DCT, + } +} + +const PARTITION_TREE: [i8; 6] = [0, 2, -1, 4, -2, -3]; +const INTRA_MODE_TREE: [i8; 18] = [0, 2, -9, 4, -1, 6, 8, 12, -2, 10, -4, -5, -3, 14, -8, 16, -6, -7]; + +/// extra_bits[token] = (cat, numExtra, base). +const EXTRA_BITS: [(u8, u8, u16); 11] = + [(0, 0, 0), (0, 0, 1), (0, 0, 2), (0, 0, 3), (0, 0, 4), (1, 1, 5), (2, 2, 7), (3, 3, 11), (4, 4, 19), (5, 5, 35), (6, 14, 67)]; +const CAT_PROBS: [&[u8]; 7] = [ + &[], + &[159], + &[165, 145], + &[173, 148, 140], + &[176, 155, 140, 135], + &[180, 157, 141, 134, 130], + &[254, 254, 254, 252, 249, 243, 230, 196, 177, 153, 140, 133, 130, 129], +]; + +/// Encoder settings. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EncoderConfig { + pub width: u32, + pub height: u32, + /// 1..=100; 100 is lossless. + pub quality: u8, + /// Full-range (0–255) rather than limited (16–235) YUV; only signalled. + pub full_range: bool, +} + +/// Probabilities of the full token tree (9.3.2) for model probability `p`. +fn pareto(p: u8) -> [u8; 8] { + let p = p.max(1) as usize; + let x = (p - 1) / 2; + let mut t = [0u8; 8]; + for n in 0..8 { + t[n] = if p & 1 == 1 { PARETO_TABLE[x * 8 + n] } else { ((PARETO_TABLE[x * 8 + n] as u32 + PARETO_TABLE[(x + 1) * 8 + n] as u32) >> 1) as u8 }; + } + t +} + +fn scan(tx_type: u8) -> &'static [u16; 16] { + match tx_type { + ADST_DCT => &ROW_SCAN_4X4, + DCT_ADST => &COL_SCAN_4X4, + _ => &DEFAULT_SCAN_4X4, + } +} + +/// Coefficient context neighbours (9.3.2) of each scan position. +fn neighbors(tx_type: u8) -> [(usize, usize); 16] { + let sc = scan(tx_type); + let mut nb = [(0, 0); 16]; + for (c, &pos) in sc.iter().enumerate().skip(1) { + let pos = pos as usize; + let (i, j) = (pos / 4, pos % 4); + nb[c] = if i > 0 && j > 0 { + let a = (i - 1) * 4 + j; + let b = i * 4 + j - 1; + match tx_type { + DCT_ADST => (a, a), + ADST_DCT => (b, b), + _ => (a, b), + } + } else if i > 0 { + ((i - 1) * 4 + j, (i - 1) * 4 + j) + } else { + (j - 1, j - 1) + }; + } + nb +} + +// ---------------------------------------------------------------- transforms (8.7) + +fn r14(v: i32) -> i32 { + (v + (1 << 13)) >> 14 +} + +fn idct4(x: [i32; 4]) -> [i32; 4] { + let v0 = r14(x[0] * 11585 - x[2] * 11585); + let v1 = r14(x[0] * 11585 + x[2] * 11585); + let v2 = r14(x[1] * 6270 - x[3] * 15137); + let v3 = r14(x[1] * 15137 + x[3] * 6270); + [v1 + v3, v0 + v2, v0 - v2, v1 - v3] +} + +fn iadst4(x: [i32; 4]) -> [i32; 4] { + let (s19, s29, s39, s49) = (5283i64, 9929i64, 13377i64, 15212i64); + let x: [i64; 4] = x.map(|v| v as i64); + let s0 = s19 * x[0]; + let s1 = s29 * x[0]; + let s2 = s39 * x[1]; + let s3 = s49 * x[2]; + let s4 = s19 * x[2]; + let s5 = s29 * x[3]; + let s6 = s49 * x[3]; + let s7 = s39 * (x[0] - x[2] + x[3]); + let x0 = s0 + s3 + s5; + let x1 = s1 - s4 - s6; + let x2 = s7; + let x3 = s2; + let r = |v: i64| ((v + (1 << 13)) >> 14) as i32; + [r(x0 + x3), r(x1 + x3), r(x2), r(x0 + x1 - x3)] +} + +/// The decoder's 4×4 inverse transform of dequantised coefficients (row-major) to a residual. +fn inverse4(coefs: &[i32; 16], tx_type: u8) -> [i32; 16] { + let row_adst = matches!(tx_type, DCT_ADST | ADST_ADST); + let col_adst = matches!(tx_type, ADST_DCT | ADST_ADST); + let mut t = [[0i32; 4]; 4]; + for i in 0..4 { + let r = [coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]]; + t[i] = if row_adst { iadst4(r) } else { idct4(r) }; + } + let mut out = [0i32; 16]; + for j in 0..4 { + let c = [t[0][j], t[1][j], t[2][j], t[3][j]]; + let c = if col_adst { iadst4(c) } else { idct4(c) }; + for i in 0..4 { + out[i * 4 + j] = (c[i] + 8) >> 4; + } + } + out +} + +/// Inverse Walsh–Hadamard (8.7.1.10) on one line. +fn iwht(x: [i32; 4], shift: u32) -> [i32; 4] { + let mut a = x[0] >> shift; + let mut c = x[1] >> shift; + let mut d = x[2] >> shift; + let mut b = x[3] >> shift; + a += c; + d -= b; + let e = (a - d) >> 1; + b = e - b; + c = e - c; + a -= b; + d += c; + [a, b, c, d] +} + +/// The exact inverse of [`iwht`] (with shift 0): inputs that produce outputs `y`. +fn fwht(y: [i32; 4]) -> [i32; 4] { + let (aa, bb, cc, dd) = (y[0], y[1], y[2], y[3]); + let d1 = dd - cc; + let a1 = aa + bb; + let e = (a1 - d1) >> 1; + let b0 = e - bb; + let c0 = e - cc; + let a0 = a1 - c0; + let d0 = d1 + b0; + [a0, c0, d0, b0] +} + +fn inverse_wht(coefs: &[i32; 16]) -> [i32; 16] { + let mut t = [[0i32; 4]; 4]; + for i in 0..4 { + t[i] = iwht([coefs[i * 4], coefs[i * 4 + 1], coefs[i * 4 + 2], coefs[i * 4 + 3]], 2); + } + let mut out = [0i32; 16]; + for j in 0..4 { + let r = iwht([t[0][j], t[1][j], t[2][j], t[3][j]], 0); + for i in 0..4 { + out[i * 4 + j] = r[i]; + } + } + out +} + +/// Lossless forward transform: levels (dequantised by 4) whose inverse is exactly `res`. +fn forward_wht(res: &[i32; 16]) -> [i32; 16] { + let mut t = [[0i32; 4]; 4]; + for j in 0..4 { + let col = fwht([res[j], res[4 + j], res[8 + j], res[12 + j]]); + for i in 0..4 { + t[i][j] = col[i]; + } + } + let mut lv = [0i32; 16]; + for i in 0..4 { + let r = fwht(t[i]); + lv[i * 4..i * 4 + 4].copy_from_slice(&r); + } + lv +} + +/// Forward transform matrices: the inverses of the decoder's (linear part of the) inverse +/// transforms, so `inverse4(forward(res)) ≈ res`. +fn forward_matrices() -> &'static [[[f64; 16]; 16]; 4] { + use std::sync::OnceLock; + static M: OnceLock<[[[f64; 16]; 16]; 4]> = OnceLock::new(); + M.get_or_init(|| { + let mut out = [[[0.0; 16]; 16]; 4]; + for (ty, o) in out.iter_mut().enumerate() { + // Columns of the inverse: response to a scaled unit coefficient. + let k = 1 << 16; + let mut inv = [[0.0f64; 16]; 16]; + for c in 0..16 { + let mut e = [0i32; 16]; + e[c] = k; + let r = inverse4(&e, ty as u8); + for p in 0..16 { + inv[p][c] = r[p] as f64 / k as f64; + } + } + *o = invert16(inv); + } + out + }) +} + +fn invert16(m: [[f64; 16]; 16]) -> [[f64; 16]; 16] { + let mut a = m; + let mut inv = [[0.0; 16]; 16]; + for (i, r) in inv.iter_mut().enumerate() { + r[i] = 1.0; + } + for col in 0..16 { + let piv = (col..16).max_by(|&x, &y| a[x][col].abs().total_cmp(&a[y][col].abs())).unwrap_or(col); + a.swap(col, piv); + inv.swap(col, piv); + let d = a[col][col]; + for j in 0..16 { + a[col][j] /= d; + inv[col][j] /= d; + } + for r in 0..16 { + if r != col { + let f = a[r][col]; + if f != 0.0 { + for j in 0..16 { + a[r][j] -= f * a[col][j]; + inv[r][j] -= f * inv[col][j]; + } + } + } + } + } + inv +} + +// ---------------------------------------------------------------- planes and prediction + +struct Plane { + w: usize, + /// Last valid column / row for prediction edges (the 8-aligned mode-info grid). + max_x: usize, + max_y: usize, + src: Vec, + rec: Vec, +} + +impl Plane { + fn new(data: &[u8], vw: usize, vh: usize, w: usize, h: usize, max_x: usize, max_y: usize) -> Plane { + let mut src = vec![0u8; w * h]; + for y in 0..h { + let sy = y.min(vh.saturating_sub(1)); + for x in 0..w { + let sx = x.min(vw.saturating_sub(1)); + src[y * w + x] = data.get(sy * vw + sx).copied().unwrap_or(128); + } + } + Plane { w, max_x, max_y, src, rec: vec![0; w * h] } + } +} + +/// Intra prediction (8.5.1) of a square block for DC / V / H / TM. +fn predict(p: &Plane, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> [i32; 64] { + let size = 1usize << log2; + let base = 128i32; + let buf = &p.rec; + let w = p.w; + let mut above = [0i32; 8]; + let mut left = [0i32; 8]; + if have_above { + for i in 0..size { + above[i] = buf[(y - 1) * w + (x + i).min(p.max_x)] as i32; + } + } else { + above[..size].fill(base - 1); + } + let top_left = if have_above && have_left { + buf[(y - 1) * w + (x - 1).min(p.max_x)] as i32 + } else if have_above { + base + 1 + } else { + base - 1 + }; + if have_left { + for i in 0..size { + left[i] = buf[(y + i).min(p.max_y) * w + x - 1] as i32; + } + } else { + left[..size].fill(base + 1); + } + let mut out = [0i32; 64]; + for i in 0..size { + for j in 0..size { + out[i * size + j] = match mode { + V_PRED => above[j], + H_PRED => left[i], + TM_PRED => (above[j] + left[i] - top_left).clamp(0, 255), + _ => 0, + }; + } + } + if mode == DC_PRED { + let v = match (have_left, have_above) { + (true, true) => (left[..size].iter().sum::() + above[..size].iter().sum::() + size as i32) >> (log2 + 1), + (true, false) => (left[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, + (false, true) => (above[..size].iter().sum::() + (1 << (log2 - 1))) >> log2, + (false, false) => base, + }; + out[..size * size].fill(v); + } + out +} + +// ---------------------------------------------------------------- the encoder + +pub struct Vp9Encoder { + cfg: EncoderConfig, + qidx: u8, + lossless: bool, +} + +/// Per-tile coding state. +struct Tile<'a> { + enc: &'a Vp9Encoder, + bc: BoolEncoder, + planes: &'a mut [Plane; 3], + mi_cols: usize, + mi_rows: usize, + col_start: usize, + above_nz: [Vec; 3], + left_nz: [Vec; 3], + above_part: Vec, + left_part: Vec, + /// y mode of every 8×8 block (for mode contexts). + modes: &'a mut [u8], +} + +impl Vp9Encoder { + pub fn new(cfg: EncoderConfig) -> Self { + let q = cfg.quality.clamp(1, 100); + let lossless = q >= 100; + // Quality → base_q_idx: 99 ≈ 3, 90 ≈ 30, 50 ≈ 150, 1 ≈ 255. + let qidx = if lossless { 0 } else { ((100 - q as i32) * 255 / 85).clamp(1, 255) as u8 }; + Vp9Encoder { cfg, qidx, lossless } + } + + pub fn config(&self) -> &EncoderConfig { + &self.cfg + } + + /// Encode one key frame from planar 8-bit 4:2:0 (`y`: width×height, `u`/`v`: + /// ((w+1)/2)×((h+1)/2), tightly packed). + pub fn encode_yuv420(&mut self, y: &[u8], u: &[u8], v: &[u8]) -> Vec { + let (w, h) = (self.cfg.width.max(1) as usize, self.cfg.height.max(1) as usize); + let mi_cols = w.div_ceil(8); + let mi_rows = h.div_ceil(8); + let sb_cols = mi_cols.div_ceil(8); + let sb_rows = mi_rows.div_ceil(8); + let (pw, ph) = (sb_cols * 64, sb_rows * 64); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let mut planes = [ + Plane::new(y, w, h, pw, ph, mi_cols * 8 - 1, mi_rows * 8 - 1), + Plane::new(u, cw, ch, pw / 2, ph / 2, mi_cols * 4 - 1, mi_rows * 4 - 1), + Plane::new(v, cw, ch, pw / 2, ph / 2, mi_cols * 4 - 1, mi_rows * 4 - 1), + ]; + // Tile columns: as few as the width allows (tile_info, 6.2.13). + let mut min_log2 = 0; + while (64usize << min_log2) < sb_cols { + min_log2 += 1; + } + let mut max_log2 = 1; + while (sb_cols >> max_log2) >= 4 { + max_log2 += 1; + } + max_log2 -= 1; + let tile_cols = 1usize << min_log2; + let mut modes = vec![0u8; mi_cols * mi_rows]; + let mut tiles = vec![]; + for t in 0..tile_cols { + let start = (((t * sb_cols) >> min_log2) * 8).min(mi_cols); + let end = ((((t + 1) * sb_cols) >> min_log2) * 8).min(mi_cols); + let mut tile = Tile { + enc: self, + bc: BoolEncoder::new(), + planes: &mut planes, + mi_cols, + mi_rows, + col_start: start, + above_nz: [vec![0; mi_cols * 2 + 16], vec![0; mi_cols * 2 + 16], vec![0; mi_cols * 2 + 16]], + left_nz: [vec![0; 16], vec![0; 16], vec![0; 16]], + above_part: vec![0; mi_cols + 8], + left_part: vec![0; 8], + modes: &mut modes, + }; + tile.bc.write(false, 128); + for r in (0..mi_rows).step_by(8) { + for p in 0..3 { + tile.left_nz[p].fill(0); + } + tile.left_part.fill(0); + for c in (start..end).step_by(8) { + tile.partition(r, c, 12); + } + } + tiles.push(tile.bc.finish()); + } + // Compressed header. + let mut ch_enc = BoolEncoder::new(); + ch_enc.write(false, 128); + if !self.lossless { + ch_enc.literal(0, 2); // tx_mode ONLY_4X4 + } + ch_enc.literal(0, 1); // no coefficient probability updates (TX_4X4) + for _ in 0..3 { + ch_enc.write(false, 252); // skip probabilities unchanged + } + let compressed = ch_enc.finish(); + // Uncompressed header. + let mut bw = BitWriter::default(); + bw.put(2, 2); // frame_marker + bw.put(0, 1); // profile low bit + bw.put(0, 1); // profile high bit + bw.put(0, 1); // show_existing_frame + bw.put(0, 1); // frame_type = KEY_FRAME + bw.put(1, 1); // show_frame + bw.put(0, 1); // error_resilient_mode + bw.put(0x49, 8); + bw.put(0x83, 8); + bw.put(0x42, 8); + bw.put(2, 3); // color_space CS_BT_709 + bw.put(self.cfg.full_range as u32, 1); + bw.put(w as u32 - 1, 16); + bw.put(h as u32 - 1, 16); + bw.put(0, 1); // render_and_frame_size_different + bw.put(0, 1); // refresh_frame_context + bw.put(1, 1); // frame_parallel_decoding_mode + bw.put(0, 2); // frame_context_idx + bw.put(0, 6); // loop_filter_level + bw.put(0, 3); // sharpness + bw.put(0, 1); // mode_ref_delta_enabled + bw.put(self.qidx as u32, 8); + bw.put(0, 1); // delta_q_y_dc + bw.put(0, 1); // delta_q_uv_dc + bw.put(0, 1); // delta_q_uv_ac + bw.put(0, 1); // segmentation_enabled + if min_log2 < max_log2 { + bw.put(0, 1); // no extra tile columns + } + bw.put(0, 1); // tile_rows_log2 = 0 + bw.put(compressed.len() as u32, 16); + let mut out = bw.finish(); + out.extend_from_slice(&compressed); + let n = tiles.len(); + for (i, t) in tiles.into_iter().enumerate() { + if i + 1 < n { + out.extend_from_slice(&(t.len() as u32).to_be_bytes()); + } + out.extend_from_slice(&t); + } + out + } +} + +#[derive(Default)] +struct BitWriter { + out: Vec, + acc: u32, + n: u32, +} + +impl BitWriter { + fn put(&mut self, v: u32, bits: u32) { + for i in (0..bits).rev() { + self.acc = (self.acc << 1) | ((v >> i) & 1); + self.n += 1; + if self.n == 8 { + self.out.push(self.acc as u8); + self.acc = 0; + self.n = 0; + } + } + } + fn finish(mut self) -> Vec { + if self.n > 0 { + self.out.push((self.acc << (8 - self.n)) as u8); + } + self.out + } +} + +const BLOCK_8X8: u8 = 3; +const NUM_8X8: [usize; 13] = [1, 1, 1, 1, 1, 2, 2, 2, 4, 4, 4, 8, 8]; +const MI_WIDTH_LOG2: [usize; 13] = [0, 0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3]; + +impl Tile<'_> { + fn partition(&mut self, r: usize, c: usize, bsize: u8) { + if r >= self.mi_rows || c >= self.mi_cols { + return; + } + let num8 = NUM_8X8[bsize as usize]; + let half = num8 >> 1; + let has_rows = (r + half) < self.mi_rows; + let has_cols = (c + half) < self.mi_cols; + let bsl = MI_WIDTH_LOG2[bsize as usize]; + let boffset = 3 - bsl; + let mut above = 0u8; + let mut left = 0u8; + for i in 0..num8 { + above |= self.above_part[c + i]; + left |= self.left_part[(r & 7) + i]; + } + let above = (above >> boffset) & 1; + let left = (left >> boffset) & 1; + let ctx = bsl * 4 + left as usize * 2 + above as usize; + let probs = &KF_PARTITION_PROBS[ctx * 3..ctx * 3 + 3]; + if bsize == BLOCK_8X8 { + // PARTITION_NONE. + self.bc.write(false, probs[0]); + self.block(r, c); + self.above_part[c] = 15 >> 1; + self.left_part[r & 7] = 15 >> 1; + return; + } + // PARTITION_SPLIT. + if has_rows && has_cols { + self.bc.tree(&PARTITION_TREE, probs, 3); + } else if has_cols { + self.bc.write(true, probs[1]); + } else if has_rows { + self.bc.write(true, probs[2]); + } + let sub = bsize - 3; + self.partition(r, c, sub); + self.partition(r, c + half, sub); + self.partition(r + half, c, sub); + self.partition(r + half, c + half, sub); + } + + fn block(&mut self, r: usize, c: usize) { + let avail_u = r > 0; + let avail_l = c > self.col_start; + // Mode decision on the 8×8 luma block and the 4×4 chroma blocks. + let y_mode = self.choose(0, c * 8, r * 8, 3, avail_l, avail_u); + let uv_mode = { + let mut best = (i64::MAX, DC_PRED); + for m in MODES { + let cost: i64 = (1..3).map(|p| self.cost(p, c * 4, r * 4, 2, m, avail_l, avail_u)).sum(); + if cost < best.0 { + best = (cost, m); + } + } + best.1 + }; + // skip = 0 (context: neighbours' skip, always 0 here); tx_size implied (4×4). + self.bc.write(false, DEFAULT_SKIP_PROB[0]); + let above_mode = if avail_u { self.modes[(r - 1) * self.mi_cols + c] } else { DC_PRED }; + let left_mode = if avail_l { self.modes[r * self.mi_cols + c - 1] } else { DC_PRED }; + let o = (above_mode as usize * 10 + left_mode as usize) * 9; + self.bc.tree(&INTRA_MODE_TREE, &KF_Y_MODE_PROBS[o..o + 9], y_mode); + let o = y_mode as usize * 9; + self.bc.tree(&INTRA_MODE_TREE, &KF_UV_MODE_PROBS[o..o + 9], uv_mode); + self.modes[r * self.mi_cols + c] = y_mode; + // Residual: luma 2×2 transform blocks, then one 4×4 block per chroma plane. + for (dy, dx) in [(0, 0), (0, 1), (1, 0), (1, 1)] { + let tx = if self.enc.lossless { DCT_DCT } else { mode_tx_type(y_mode) }; + self.tx_block(0, c * 8 + dx * 4, r * 8 + dy * 4, y_mode, tx, avail_l || dx > 0, avail_u || dy > 0); + } + for p in 1..3 { + self.tx_block(p, c * 4, r * 4, uv_mode, DCT_DCT, avail_l, avail_u); + } + } + + fn cost(&self, plane: usize, x: usize, y: usize, log2: usize, mode: u8, have_left: bool, have_above: bool) -> i64 { + let p = &self.planes[plane]; + let pred = predict(p, x, y, log2, mode, have_left, have_above); + let size = 1 << log2; + let mut s = 0i64; + for i in 0..size { + for j in 0..size { + s += (p.src[(y + i) * p.w + x + j] as i32 - pred[i * size + j]).abs() as i64; + } + } + s + } + + fn choose(&self, plane: usize, x: usize, y: usize, log2: usize, have_left: bool, have_above: bool) -> u8 { + let mut best = (i64::MAX, DC_PRED); + for m in MODES { + let cost = self.cost(plane, x, y, log2, m, have_left, have_above); + if cost < best.0 { + best = (cost, m); + } + } + best.1 + } + + #[allow(clippy::too_many_arguments)] + fn tx_block(&mut self, plane: usize, x: usize, y: usize, mode: u8, tx_type: u8, have_left: bool, have_above: bool) { + let ss = (plane > 0) as usize; + let max_x = (self.mi_cols * 8) >> ss; + let max_y = (self.mi_rows * 8) >> ss; + let (x4, y4) = (x >> 2, y >> 2); + let ly = y4 & (15 >> ss); + if x >= max_x || y >= max_y { + self.above_nz[plane][x4] = 0; + self.left_nz[plane][ly] = 0; + return; + } + let p = &self.planes[plane]; + let pred = predict(p, x, y, 2, mode, have_left, have_above); + let mut res = [0i32; 16]; + for i in 0..4 { + for j in 0..4 { + res[i * 4 + j] = p.src[(y + i) * p.w + x + j] as i32 - pred[i * 4 + j]; + } + } + let (dcq, acq) = (DC_QLOOKUP[self.enc.qidx as usize], AC_QLOOKUP[self.enc.qidx as usize]); + // Quantised levels (raster order). + let levels: [i32; 16] = if self.enc.lossless { + forward_wht(&res) + } else { + let f = &forward_matrices()[tx_type as usize]; + let mut lv = [0i32; 16]; + for k in 0..16 { + let coef: f64 = (0..16).map(|q| f[k][q] * res[q] as f64).sum(); + let qs = if k == 0 { dcq } else { acq } as f64; + let a = coef.abs() / qs; + // Rounding with a small dead zone for AC coefficients. + let l = if k == 0 { (a + 0.5).floor() } else { (a + 0.38).floor() }; + lv[k] = if coef < 0.0 { -(l as i32) } else { l as i32 }; + } + lv + }; + let sc = scan(tx_type); + let eob = (0..16).rev().find(|&i| levels[sc[i] as usize] != 0).map_or(0, |i| i + 1); + // Tokens. + let ctx0 = (self.above_nz[plane][x4] + self.left_nz[plane][ly]) as usize; + self.tokens(plane, tx_type, &levels, eob, ctx0); + // Reconstruct exactly as the decoder does. + let mut deq = [0i32; 16]; + for k in 0..16 { + deq[k] = levels[k] * if k == 0 { dcq } else { acq }; + } + let resid = if eob == 0 { + [0i32; 16] + } else if self.enc.lossless { + inverse_wht(&deq) + } else { + inverse4(&deq, tx_type) + }; + let p = &mut self.planes[plane]; + for i in 0..4 { + for j in 0..4 { + p.rec[(y + i) * p.w + x + j] = (pred[i * 4 + j] + resid[i * 4 + j]).clamp(0, 255) as u8; + } + } + let nz = (eob > 0) as u8; + self.above_nz[plane][x4] = nz; + self.left_nz[plane][ly] = nz; + } + + fn tokens(&mut self, plane: usize, tx_type: u8, levels: &[i32; 16], eob: usize, ctx0: usize) { + let sc = scan(tx_type); + let nb = neighbors(tx_type); + let ptype = (plane > 0) as usize; + // coef_probs[TX_4X4][ptype][intra]: 6 bands × 6 contexts × 3. + let base = (ptype * 2) * 6 * 6 * 3; + let mut tc = [0u8; 16]; + let mut check_eob = true; + let mut c = 0; + while c < 16 { + let pos = sc[c] as usize; + let band = COEFBAND_4X4[c] as usize; + let ctx = if c == 0 { ctx0 } else { ((1 + tc[nb[c].0] + tc[nb[c].1]) >> 1) as usize }; + let o = base + (band * 6 + ctx) * 3; + let p = [DEFAULT_COEF_PROBS[o], DEFAULT_COEF_PROBS[o + 1], DEFAULT_COEF_PROBS[o + 2]]; + if check_eob { + let more = c < eob; + self.bc.write(more, p[0]); + if !more { + break; + } + } + let v = levels[pos].unsigned_abs(); + if v == 0 { + self.bc.write(false, p[1]); + tc[pos] = 0; + check_eob = false; + c += 1; + continue; + } + self.bc.write(true, p[1]); + check_eob = true; + let token = match v { + 1 => 1, + 2 => 2, + 3 => 3, + 4 => 4, + 5..=6 => 5, + 7..=10 => 6, + 11..=18 => 7, + 19..=34 => 8, + 35..=66 => 9, + _ => 10, + }; + if token == 1 { + self.bc.write(false, p[2]); + } else { + self.bc.write(true, p[2]); + let pp = pareto(p[2]); + let path: &[(usize, bool)] = match token { + 2 => &[(0, false), (1, false)], + 3 => &[(0, false), (1, true), (2, false)], + 4 => &[(0, false), (1, true), (2, true)], + 5 => &[(0, true), (3, false), (4, false)], + 6 => &[(0, true), (3, false), (4, true)], + 7 => &[(0, true), (3, true), (5, false), (6, false)], + 8 => &[(0, true), (3, true), (5, false), (6, true)], + 9 => &[(0, true), (3, true), (5, true), (7, false)], + _ => &[(0, true), (3, true), (5, true), (7, true)], + }; + for &(n, b) in path { + self.bc.write(b, pp[n]); + } + let (cat, num_extra, base_v) = EXTRA_BITS[token]; + let extra = (v - base_v as u32).min((1 << num_extra) - 1); + let cp = CAT_PROBS[cat as usize]; + for e in 0..num_extra as usize { + self.bc.write((extra >> (num_extra as usize - 1 - e)) & 1 != 0, cp[e]); + } + } + tc[pos] = ENERGY_CLASS[token]; + self.bc.write(levels[pos] < 0, 128); + c += 1; + } + } +} + +// ---------------------------------------------------------------- colour conversion + +/// BT.709 RGB(A) 8-bit → planar YUV 4:2:0 (limited or full range). +pub fn rgba_to_yuv420(rgba: &[u8], w: u32, h: u32, full_range: bool) -> (Vec, Vec, Vec) { + let (w, h) = (w as usize, h as usize); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let mut y = vec![0u8; w * h]; + let mut u = vec![0u8; cw * ch]; + let mut v = vec![0u8; cw * ch]; + let (ys, yo, cs) = if full_range { (255.0, 0.0, 255.0) } else { (219.0, 16.0, 224.0) }; + let px = |x: usize, yy: usize| -> [f32; 3] { + let i = (yy.min(h - 1) * w + x.min(w - 1)) * 4; + [rgba[i] as f32 / 255.0, rgba[i + 1] as f32 / 255.0, rgba[i + 2] as f32 / 255.0] + }; + let luma = |c: [f32; 3]| 0.2126 * c[0] + 0.7152 * c[1] + 0.0722 * c[2]; + for yy in 0..h { + for x in 0..w { + y[yy * w + x] = (luma(px(x, yy)) * ys + yo).round().clamp(0.0, 255.0) as u8; + } + } + for cy in 0..ch { + for cx in 0..cw { + let mut acc = [0.0f32; 3]; + for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] { + let c = px(cx * 2 + dx, cy * 2 + dy); + for k in 0..3 { + acc[k] += c[k] / 4.0; + } + } + let l = luma(acc); + let cb = (acc[2] - l) / 1.8556; + let cr = (acc[0] - l) / 1.5748; + u[cy * cw + cx] = (cb * cs + 128.0).round().clamp(0.0, 255.0) as u8; + v[cy * cw + cx] = (cr * cs + 128.0).round().clamp(0.0, 255.0) as u8; + } + } + (y, u, v) +} + +/// The alpha channel as a luma plane (full range) with neutral chroma — the second VP9 stream of +/// WebM alpha. +pub fn alpha_to_yuv420(rgba: &[u8], w: u32, h: u32) -> (Vec, Vec, Vec) { + let (w, h) = (w as usize, h as usize); + let (cw, ch) = (w.div_ceil(2), h.div_ceil(2)); + let y: Vec = rgba.chunks_exact(4).take(w * h).map(|p| p[3]).collect(); + (y, vec![128; cw * ch], vec![128; cw * ch]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wht_is_exactly_invertible() { + let mut s = 7u32; + for _ in 0..2000 { + let mut r = [0i32; 16]; + for v in &mut r { + s = s.wrapping_mul(1664525).wrapping_add(1013904223); + *v = (s >> 16) as i32 % 511 - 255; + } + let lv = forward_wht(&r); + let deq = lv.map(|l| l * 4); + assert_eq!(inverse_wht(&deq), r); + } + } + + #[test] + fn forward_transforms_invert_the_decoder() { + for ty in 0..4u8 { + let f = &forward_matrices()[ty as usize]; + let r: [i32; 16] = std::array::from_fn(|i| (i as i32 * 37 % 61) - 30); + let c: [i32; 16] = std::array::from_fn(|k| (0..16).map(|q| f[k][q] * r[q] as f64).sum::().round() as i32); + let back = inverse4(&c, ty); + for i in 0..16 { + assert!((back[i] - r[i]).abs() <= 1, "type {ty}: {back:?} vs {r:?}"); + } + } + } +} diff --git a/crates/vp9enc/src/tables.rs b/crates/vp9enc/src/tables.rs new file mode 100644 index 00000000..dc8027a6 --- /dev/null +++ b/crates/vp9enc/src/tables.rs @@ -0,0 +1,130 @@ +//! Tables from the VP9 Bitstream & Decoding Process Specification v0.6 (section 10), as +//! transcribed in FilmCraft's first-party decoder `filmcraft-vp9` (MIT OR Apache-2.0, the same +//! licence as this crate). + +#![allow(clippy::unreadable_literal)] + +pub const DC_QLOOKUP: [i32; 256] = [ + 4, 8, 8, 9, 10, 11, 12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 23, 24, 25, 26, 26, 27, 28, 29, 30, 31, 32, 32, 33, 34, 35, 36, 37, 38, 38, 39, 40, + 41, 42, 43, 43, 44, 45, 46, 47, 48, 48, 49, 50, 51, 52, 53, 53, 54, 55, 56, 57, 57, 58, 59, 60, 61, 62, 62, 63, 64, 65, 66, 66, 67, 68, 69, 70, 70, 71, 72, + 73, 74, 74, 75, 76, 77, 78, 78, 79, 80, 81, 81, 82, 83, 84, 85, 85, 87, 88, 90, 92, 93, 95, 96, 98, 99, 101, 102, 104, 105, 107, 108, 110, 111, 113, 114, + 116, 117, 118, 120, 121, 123, 125, 127, 129, 131, 134, 136, 138, 140, 142, 144, 146, 148, 150, 152, 154, 156, 158, 161, 164, 166, 169, 172, 174, 177, 180, + 182, 185, 187, 190, 192, 195, 199, 202, 205, 208, 211, 214, 217, 220, 223, 226, 230, 233, 237, 240, 243, 247, 250, 253, 257, 261, 265, 269, 272, 276, 280, + 284, 288, 292, 296, 300, 304, 309, 313, 317, 322, 326, 330, 335, 340, 344, 349, 354, 359, 364, 369, 374, 379, 384, 389, 395, 400, 406, 411, 417, 423, 429, + 435, 441, 447, 454, 461, 467, 475, 482, 489, 497, 505, 513, 522, 530, 539, 549, 559, 569, 579, 590, 602, 614, 626, 640, 654, 668, 684, 700, 717, 736, 755, + 775, 796, 819, 843, 869, 896, 925, 955, 988, 1022, 1058, 1098, 1139, 1184, 1232, 1282, 1336, +]; + +pub const AC_QLOOKUP: [i32; 256] = [ + 4, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, + 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, + 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 132, 134, + 136, 138, 140, 142, 144, 146, 148, 150, 152, 155, 158, 161, 164, 167, 170, 173, 176, 179, 182, 185, 188, 191, 194, 197, 200, 203, 207, 211, 215, 219, 223, + 227, 231, 235, 239, 243, 247, 251, 255, 260, 265, 270, 275, 280, 285, 290, 295, 300, 305, 311, 317, 323, 329, 335, 341, 347, 353, 359, 366, 373, 380, 387, + 394, 401, 408, 416, 424, 432, 440, 448, 456, 465, 474, 483, 492, 501, 510, 520, 530, 540, 550, 560, 571, 582, 593, 604, 615, 627, 639, 651, 663, 676, 689, + 702, 715, 729, 743, 757, 771, 786, 801, 816, 832, 848, 864, 881, 898, 915, 933, 951, 969, 988, 1007, 1026, 1046, 1066, 1087, 1108, 1129, 1151, 1173, 1196, + 1219, 1243, 1267, 1292, 1317, 1343, 1369, 1396, 1423, 1451, 1479, 1508, 1537, 1567, 1597, 1628, 1660, 1692, 1725, 1759, 1793, 1828, +]; + +pub const DEFAULT_SCAN_4X4: [u16; 16] = [0, 4, 1, 5, 8, 2, 12, 9, 3, 6, 13, 10, 7, 14, 11, 15]; + +pub const COL_SCAN_4X4: [u16; 16] = [0, 4, 8, 1, 12, 5, 9, 2, 13, 6, 10, 3, 7, 14, 11, 15]; + +pub const ROW_SCAN_4X4: [u16; 16] = [0, 1, 4, 2, 5, 3, 6, 8, 9, 7, 12, 10, 13, 11, 14, 15]; + +pub const COEFBAND_4X4: [u8; 16] = [0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5]; + +pub const ENERGY_CLASS: [u8; 12] = [0, 1, 2, 3, 3, 4, 4, 5, 5, 5, 5, 5]; + +pub const PARETO_TABLE: [u8; 1024] = [ + 3, 86, 128, 6, 86, 23, 88, 29, 9, 86, 129, 17, 88, 61, 94, 76, 15, 87, 129, 28, 89, 93, 100, 110, 20, 88, 130, 38, 91, 118, 106, 136, 26, 89, 131, 48, 92, + 139, 111, 156, 31, 90, 131, 58, 94, 156, 117, 171, 37, 90, 132, 66, 95, 171, 122, 184, 42, 91, 132, 75, 97, 183, 127, 194, 47, 92, 133, 83, 98, 193, 132, + 202, 52, 93, 133, 90, 100, 201, 137, 208, 57, 94, 134, 98, 101, 208, 142, 214, 62, 94, 135, 105, 103, 214, 146, 218, 66, 95, 135, 111, 104, 219, 151, 222, + 71, 96, 136, 117, 106, 224, 155, 225, 76, 97, 136, 123, 107, 227, 159, 228, 80, 98, 137, 129, 109, 231, 162, 231, 84, 98, 138, 134, 110, 234, 166, 233, 89, + 99, 138, 140, 112, 236, 170, 235, 93, 100, 139, 145, 113, 238, 173, 236, 97, 101, 140, 149, 115, 240, 176, 238, 101, 102, 140, 154, 116, 242, 179, 239, + 105, 103, 141, 158, 118, 243, 182, 240, 109, 104, 141, 162, 119, 244, 185, 241, 113, 104, 142, 166, 120, 245, 187, 242, 116, 105, 143, 170, 122, 246, 190, + 243, 120, 106, 143, 173, 123, 247, 192, 244, 123, 107, 144, 177, 125, 248, 195, 244, 127, 108, 145, 180, 126, 249, 197, 245, 130, 109, 145, 183, 128, 249, + 199, 245, 134, 110, 146, 186, 129, 250, 201, 246, 137, 111, 147, 189, 131, 251, 203, 246, 140, 112, 147, 192, 132, 251, 205, 247, 143, 113, 148, 194, 133, + 251, 207, 247, 146, 114, 149, 197, 135, 252, 208, 248, 149, 115, 149, 199, 136, 252, 210, 248, 152, 115, 150, 201, 138, 252, 211, 248, 155, 116, 151, 204, + 139, 253, 213, 249, 158, 117, 151, 206, 140, 253, 214, 249, 161, 118, 152, 208, 142, 253, 216, 249, 163, 119, 153, 210, 143, 253, 217, 249, 166, 120, 153, + 212, 144, 254, 218, 250, 168, 121, 154, 213, 146, 254, 220, 250, 171, 122, 155, 215, 147, 254, 221, 250, 173, 123, 155, 217, 148, 254, 222, 250, 176, 124, + 156, 218, 150, 254, 223, 250, 178, 125, 157, 220, 151, 254, 224, 251, 180, 126, 157, 221, 152, 254, 225, 251, 183, 127, 158, 222, 153, 254, 226, 251, 185, + 128, 159, 224, 155, 255, 227, 251, 187, 129, 160, 225, 156, 255, 228, 251, 189, 131, 160, 226, 157, 255, 228, 251, 191, 132, 161, 227, 159, 255, 229, 251, + 193, 133, 162, 228, 160, 255, 230, 252, 195, 134, 163, 230, 161, 255, 231, 252, 197, 135, 163, 231, 162, 255, 231, 252, 199, 136, 164, 232, 163, 255, 232, + 252, 201, 137, 165, 233, 165, 255, 233, 252, 202, 138, 166, 233, 166, 255, 233, 252, 204, 139, 166, 234, 167, 255, 234, 252, 206, 140, 167, 235, 168, 255, + 235, 252, 207, 141, 168, 236, 169, 255, 235, 252, 209, 142, 169, 237, 171, 255, 236, 252, 210, 144, 169, 237, 172, 255, 236, 252, 212, 145, 170, 238, 173, + 255, 237, 252, 214, 146, 171, 239, 174, 255, 237, 253, 215, 147, 172, 240, 175, 255, 238, 253, 216, 148, 173, 240, 176, 255, 238, 253, 218, 149, 173, 241, + 177, 255, 239, 253, 219, 150, 174, 241, 179, 255, 239, 253, 220, 152, 175, 242, 180, 255, 240, 253, 222, 153, 176, 242, 181, 255, 240, 253, 223, 154, 177, + 243, 182, 255, 240, 253, 224, 155, 178, 244, 183, 255, 241, 253, 225, 156, 178, 244, 184, 255, 241, 253, 226, 158, 179, 244, 185, 255, 242, 253, 228, 159, + 180, 245, 186, 255, 242, 253, 229, 160, 181, 245, 187, 255, 242, 253, 230, 161, 182, 246, 188, 255, 243, 253, 231, 163, 183, 246, 189, 255, 243, 253, 232, + 164, 184, 247, 190, 255, 243, 253, 233, 165, 185, 247, 191, 255, 244, 253, 234, 166, 185, 247, 192, 255, 244, 253, 235, 168, 186, 248, 193, 255, 244, 253, + 236, 169, 187, 248, 194, 255, 244, 253, 236, 170, 188, 248, 195, 255, 245, 253, 237, 171, 189, 249, 196, 255, 245, 254, 238, 173, 190, 249, 197, 255, 245, + 254, 239, 174, 191, 249, 198, 255, 245, 254, 240, 175, 192, 249, 199, 255, 246, 254, 240, 177, 193, 250, 200, 255, 246, 254, 241, 178, 194, 250, 201, 255, + 246, 254, 242, 179, 195, 250, 202, 255, 246, 254, 242, 181, 196, 250, 203, 255, 247, 254, 243, 182, 197, 251, 204, 255, 247, 254, 244, 184, 198, 251, 205, + 255, 247, 254, 244, 185, 199, 251, 206, 255, 247, 254, 245, 186, 200, 251, 207, 255, 247, 254, 246, 188, 201, 252, 207, 255, 248, 254, 246, 189, 202, 252, + 208, 255, 248, 254, 247, 191, 203, 252, 209, 255, 248, 254, 247, 192, 204, 252, 210, 255, 248, 254, 248, 194, 205, 252, 211, 255, 248, 254, 248, 195, 206, + 252, 212, 255, 249, 254, 249, 197, 207, 253, 213, 255, 249, 254, 249, 198, 208, 253, 214, 255, 249, 254, 250, 200, 210, 253, 215, 255, 249, 254, 250, 201, + 211, 253, 215, 255, 249, 254, 250, 203, 212, 253, 216, 255, 249, 254, 251, 204, 213, 253, 217, 255, 250, 254, 251, 206, 214, 254, 218, 255, 250, 254, 252, + 207, 216, 254, 219, 255, 250, 254, 252, 209, 217, 254, 220, 255, 250, 254, 252, 211, 218, 254, 221, 255, 250, 254, 253, 213, 219, 254, 222, 255, 250, 254, + 253, 214, 221, 254, 223, 255, 250, 254, 253, 216, 222, 254, 224, 255, 251, 254, 253, 218, 224, 254, 225, 255, 251, 254, 254, 220, 225, 254, 225, 255, 251, + 254, 254, 222, 227, 255, 226, 255, 251, 254, 254, 224, 228, 255, 227, 255, 251, 254, 254, 226, 230, 255, 228, 255, 251, 254, 255, 228, 231, 255, 230, 255, + 251, 254, 255, 230, 233, 255, 231, 255, 252, 254, 255, 232, 235, 255, 232, 255, 252, 254, 255, 235, 237, 255, 233, 255, 252, 254, 255, 238, 240, 255, 235, + 255, 252, 255, 255, 241, 243, 255, 236, 255, 252, 254, 255, 246, 247, 255, 239, 255, 253, 255, +]; + +pub const KF_PARTITION_PROBS: [u8; 48] = [ + 158, 97, 94, 93, 24, 99, 85, 119, 44, 62, 59, 67, 149, 53, 53, 94, 20, 48, 83, 53, 24, 52, 18, 18, 150, 40, 39, 78, 12, 26, 67, 33, 11, 24, 7, 5, 174, 35, + 49, 68, 11, 27, 57, 15, 9, 12, 3, 3, +]; + +pub const KF_Y_MODE_PROBS: [u8; 900] = [ + 137, 30, 42, 148, 151, 207, 70, 52, 91, 92, 45, 102, 136, 116, 180, 74, 90, 100, 73, 32, 19, 187, 222, 215, 46, 34, 100, 91, 30, 32, 116, 121, 186, 93, 86, + 94, 72, 35, 36, 149, 68, 206, 68, 63, 105, 73, 31, 28, 138, 57, 124, 55, 122, 151, 67, 23, 21, 140, 126, 197, 40, 37, 171, 86, 27, 28, 128, 154, 212, 45, + 43, 53, 74, 32, 27, 107, 86, 160, 63, 134, 102, 59, 67, 44, 140, 161, 202, 78, 67, 119, 63, 36, 126, 146, 123, 158, 60, 90, 96, 43, 46, 168, 134, 107, 128, + 69, 142, 92, 44, 29, 68, 159, 201, 177, 50, 57, 77, 58, 38, 76, 114, 97, 172, 78, 133, 92, 46, 41, 76, 140, 63, 184, 69, 112, 57, 38, 32, 85, 140, 46, 112, + 54, 151, 133, 39, 27, 61, 131, 110, 175, 44, 75, 136, 52, 30, 74, 113, 130, 175, 51, 64, 58, 47, 35, 80, 100, 74, 143, 64, 163, 74, 36, 61, 116, 114, 128, + 162, 80, 125, 82, 82, 26, 26, 171, 208, 204, 44, 32, 105, 55, 44, 68, 166, 179, 192, 57, 57, 108, 42, 26, 11, 199, 241, 228, 23, 15, 85, 68, 42, 19, 131, + 160, 199, 55, 52, 83, 58, 50, 25, 139, 115, 232, 39, 52, 118, 50, 35, 33, 153, 104, 162, 64, 59, 131, 44, 24, 16, 150, 177, 202, 33, 19, 156, 55, 27, 12, + 153, 203, 218, 26, 27, 49, 53, 49, 21, 110, 116, 168, 59, 80, 76, 38, 72, 19, 168, 203, 212, 50, 50, 107, 103, 26, 36, 129, 132, 201, 83, 80, 93, 59, 38, + 83, 112, 103, 162, 98, 136, 90, 62, 30, 23, 158, 200, 207, 59, 57, 50, 67, 30, 29, 84, 86, 191, 102, 91, 59, 60, 32, 33, 112, 71, 220, 64, 89, 104, 53, 26, + 34, 130, 56, 149, 84, 120, 103, 53, 21, 23, 133, 109, 210, 56, 77, 172, 77, 19, 29, 112, 142, 228, 55, 66, 36, 61, 29, 29, 93, 97, 165, 83, 175, 162, 47, + 47, 43, 114, 137, 181, 100, 99, 95, 69, 23, 29, 128, 83, 199, 46, 44, 101, 53, 40, 55, 139, 69, 183, 61, 80, 110, 40, 29, 19, 161, 180, 207, 43, 24, 91, + 60, 34, 19, 105, 61, 198, 53, 64, 89, 52, 31, 22, 158, 40, 209, 58, 62, 89, 44, 31, 29, 147, 46, 158, 56, 102, 198, 35, 19, 12, 135, 87, 209, 41, 45, 167, + 55, 25, 21, 118, 95, 215, 38, 39, 66, 51, 38, 25, 113, 58, 164, 70, 93, 97, 47, 54, 34, 146, 108, 203, 72, 103, 151, 64, 19, 37, 156, 66, 138, 49, 95, 133, + 46, 27, 80, 150, 55, 124, 55, 121, 135, 36, 23, 27, 165, 149, 166, 54, 64, 118, 53, 21, 36, 131, 63, 163, 60, 109, 81, 40, 26, 35, 154, 40, 185, 51, 97, + 123, 35, 19, 34, 179, 19, 97, 48, 129, 124, 36, 20, 26, 136, 62, 164, 33, 77, 154, 45, 18, 32, 130, 90, 157, 40, 79, 91, 45, 26, 28, 129, 45, 129, 49, 147, + 123, 38, 44, 51, 136, 74, 162, 57, 97, 121, 75, 17, 22, 136, 138, 185, 32, 34, 166, 56, 39, 58, 133, 117, 173, 48, 53, 187, 35, 21, 12, 161, 212, 207, 20, + 23, 145, 56, 29, 19, 117, 109, 181, 55, 68, 112, 47, 29, 17, 153, 64, 220, 59, 51, 114, 46, 16, 24, 136, 76, 147, 41, 64, 172, 34, 17, 11, 108, 152, 187, + 13, 15, 209, 51, 24, 14, 115, 133, 209, 32, 26, 104, 55, 30, 18, 122, 79, 179, 44, 88, 116, 37, 49, 25, 129, 168, 164, 41, 54, 148, 82, 22, 32, 127, 143, + 213, 39, 41, 70, 62, 44, 61, 123, 105, 189, 48, 57, 64, 47, 25, 17, 175, 222, 220, 24, 30, 86, 68, 36, 17, 106, 102, 206, 59, 74, 74, 57, 39, 23, 151, 68, + 216, 55, 63, 58, 49, 30, 35, 141, 70, 168, 82, 40, 115, 51, 25, 15, 136, 129, 202, 38, 35, 139, 68, 26, 16, 111, 141, 215, 29, 28, 28, 59, 39, 19, 114, 75, + 180, 77, 104, 42, 40, 61, 26, 126, 152, 206, 61, 59, 93, 78, 23, 39, 111, 117, 170, 74, 124, 94, 48, 34, 86, 101, 92, 146, 78, 179, 134, 47, 22, 24, 138, + 187, 178, 68, 69, 59, 56, 25, 33, 105, 112, 187, 95, 177, 129, 48, 31, 27, 114, 63, 183, 82, 116, 56, 43, 28, 37, 121, 63, 123, 61, 192, 169, 42, 17, 24, + 109, 97, 177, 56, 76, 122, 58, 18, 28, 105, 139, 182, 70, 92, 63, 46, 23, 32, 74, 86, 150, 67, 183, 88, 36, 38, 48, 92, 122, 165, 88, 137, 91, 65, 70, 60, + 155, 159, 199, 61, 60, 81, 44, 78, 115, 132, 119, 173, 71, 112, 93, 39, 38, 21, 184, 227, 206, 42, 32, 64, 58, 47, 36, 124, 137, 193, 80, 82, 78, 49, 50, + 35, 144, 95, 205, 63, 78, 59, 41, 53, 52, 148, 71, 142, 65, 128, 51, 40, 36, 28, 143, 143, 202, 40, 55, 137, 52, 34, 29, 129, 183, 227, 42, 35, 43, 42, 44, + 44, 104, 105, 164, 64, 130, 80, 43, 81, 53, 140, 169, 204, 68, 84, 72, +]; + +pub const KF_UV_MODE_PROBS: [u8; 90] = [ + 144, 11, 54, 157, 195, 130, 46, 58, 108, 118, 15, 123, 148, 131, 101, 44, 93, 131, 113, 12, 23, 188, 226, 142, 26, 32, 125, 120, 11, 50, 123, 163, 135, 64, + 77, 103, 113, 9, 36, 155, 111, 157, 32, 44, 161, 116, 9, 55, 176, 76, 96, 37, 61, 149, 115, 9, 28, 141, 161, 167, 21, 25, 193, 120, 12, 32, 145, 195, 142, + 32, 38, 86, 116, 12, 64, 120, 140, 125, 49, 115, 121, 102, 19, 66, 162, 182, 122, 35, 59, 128, +]; + +pub const DEFAULT_SKIP_PROB: [u8; 3] = [192, 128, 64]; + +pub const DEFAULT_COEF_PROBS: [u8; 432] = [ + 195, 29, 183, 84, 49, 136, 8, 42, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 107, 169, 35, 99, 159, 17, 82, 140, 8, 66, 114, 2, 44, 76, 1, 19, 32, 40, 132, 201, + 29, 114, 187, 13, 91, 157, 7, 75, 127, 3, 58, 95, 1, 28, 47, 69, 142, 221, 42, 122, 201, 15, 91, 159, 6, 67, 121, 1, 42, 77, 1, 17, 31, 102, 148, 228, 67, + 117, 204, 17, 82, 154, 6, 59, 114, 2, 39, 75, 1, 15, 29, 156, 57, 233, 119, 57, 212, 58, 48, 163, 29, 40, 124, 12, 30, 81, 3, 12, 31, 191, 107, 226, 124, + 117, 204, 25, 99, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 148, 210, 37, 126, 194, 8, 93, 157, 2, 68, 118, 1, 39, 69, 1, 17, 33, 41, 151, 213, 27, 123, 193, 3, + 82, 144, 1, 58, 105, 1, 32, 60, 1, 13, 26, 59, 159, 220, 23, 126, 198, 4, 88, 151, 1, 66, 114, 1, 38, 71, 1, 18, 34, 114, 136, 232, 51, 114, 207, 11, 83, + 155, 3, 56, 105, 1, 33, 65, 1, 17, 34, 149, 65, 234, 121, 57, 215, 61, 49, 166, 28, 36, 114, 12, 25, 76, 3, 16, 42, 214, 49, 220, 132, 63, 188, 42, 65, + 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 137, 221, 104, 131, 216, 49, 111, 192, 21, 87, 155, 2, 49, 87, 1, 16, 28, 89, 163, 230, 90, 137, 220, 29, 100, 183, 10, + 70, 135, 2, 42, 81, 1, 17, 33, 108, 167, 237, 55, 133, 222, 15, 97, 179, 4, 72, 135, 1, 45, 85, 1, 19, 38, 124, 146, 240, 66, 124, 224, 17, 88, 175, 4, 58, + 122, 1, 36, 75, 1, 18, 37, 141, 79, 241, 126, 70, 227, 66, 58, 182, 30, 44, 136, 12, 34, 96, 2, 20, 47, 229, 99, 249, 143, 111, 235, 46, 109, 192, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 82, 158, 236, 94, 146, 224, 25, 117, 191, 9, 87, 149, 3, 56, 99, 1, 33, 57, 83, 167, 237, 68, 145, 222, 10, 103, 177, 2, 72, 131, 1, 41, + 79, 1, 20, 39, 99, 167, 239, 47, 141, 224, 10, 104, 178, 2, 73, 133, 1, 44, 85, 1, 22, 47, 127, 145, 243, 71, 129, 228, 17, 93, 177, 3, 61, 124, 1, 41, 84, + 1, 21, 52, 157, 78, 244, 140, 72, 231, 69, 58, 184, 31, 44, 137, 14, 38, 105, 8, 23, 61, +]; diff --git a/docs/architecture.md b/docs/architecture.md index c3d869c0..096c75bd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -16,18 +16,21 @@ cpal or muda. Everything in L0 to L4, the egui UI and the web app also build for | L0 | `time` | `Tick` (254 016 000 000 per second), rational frame rates incl. NTSC, SMPTE and drop-frame timecode | | L0 | `geom` | Vectors, matrices, quaternions, the layer transform (anchor, position, scale, orientation, rotation) | | L0 | `color` | sRGB and linear, HSL/HSV, luminance, the 38 blend modes, label colors | +| L0 | `vp9enc` | VP9 intra-frame encoder (profile 0, 8-bit 4:2:0; lossless at quality 100) for WebM export, from the VP9 bitstream specification | | L0 | `opusenc` | Opus (CELT) encoder: RFC 6716 CELT-only fullband 48 kHz 20 ms packets, mono/stereo, and the RFC 7845 `OpusHead`, for WebM export audio | | L1 | `raster` | Premultiplied float images, sampling, affine and projective warps, blurs, compositing (parallel with rayon) | | L1 | `keyframe` | Animated values, keyframes with temporal ease and spatial Bezier, roving, hold, velocity | +| L1 | `psd` | Photoshop PSD/PSB reader (layers, groups, masks, blend modes, text, layer effects, adjustment layers; 8/16/32-bit; RGB/CMYK/Gray/Lab) and a minimal writer, from Adobe's published format specification | | L1 | `path` | Bezier paths, path operators (trim, offset, round corners, zig zag, twist, merge…), stroking, coverage masks | | L2 | `project` | The document: items, compositions, layers, the property tree, render queue model, `.ecproj` serde | | L2 | `text` | Fonts, shaping, layout, per-glyph geometry, text animators and selectors | | L2 | `effects` | The effect registry (241 effects) and their CPU implementations | +| L2 | `svg` | SVG import (W3C SVG 1.1/2 static subset: shapes, paths, transforms, `use`, CSS, gradients) and rasterisation at any scale | | L2 | `track` | Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation, Lucas–Kanade sub-pixel refinement), confidence, homography/affine/similarity solves | | L3 | `render` | Evaluation and compositing: sources, masks, effects, transforms, 3D, motion blur, mattes, blending, layer cache, audio mixdown | -| L3 | `media` | Footage decoding (FilmCraft's pure-Rust codecs), image sequences, frame cache | +| L3 | `media` | Footage decoding (FilmCraft's pure-Rust codecs), image sequences, Photoshop and SVG stills, frame cache | | L3 | `expr` | The expression engine (JavaScript via boa) with the After Effects object model | -| L3 | `export` | Render queue encoding: H.264, ProRes, PNG/JPEG/TIFF/EXR sequences, GIF, audio | +| L3 | `export` | Render queue encoding: H.264, ProRes, WebM (VP9 + alpha, Opus), PNG/JPEG/TIFF/EXR sequences, GIF, WAV/AIFF | | L3 | `gpu` | The GPU compositor and GPU effects on wgpu compute shaders (Metal, Vulkan, Direct3D 12, WebGPU), checked against the CPU renderer | | L3 | `lottie` | Lottie JSON / dotLottie import and export (layers, precomps, eased and spatial keyframes, shapes, masks, mattes) with a warnings list for what Lottie cannot express | | L4 | `engine` | `Session`: project, undo history, editor state, the command registry and menus | @@ -76,6 +79,14 @@ its evaluated inputs, excluding the transform. Static and transform-only layers editing one layer re-renders only that layer. Effects that read the clock directly are declared in `effects::TIME_DEPENDENT`, and a test checks every registered effect against that list. +A persistent **disk cache** (`render::disk_cache`, Settings ▸ Media & Disk Cache) backs both +the layer cache and the viewer's RAM preview: layer buffers that were slow to render and every +preview frame are written (LZ4, atomic rename, checksummed) under 128-bit content keys — the +comp's content hash (the comp, what it uses, footage file size and time) plus frame, scale and +view for frames; the layer key salted with the footage fingerprint for layers — so entries +survive restarts and never go stale. LRU eviction keeps it under the size limit; Edit ▸ Purge +empties it, `cache.diskStats` reports it, and the timeline draws disk-only frames in blue. + **Time effects** (Echo, Posterize Time, Timewarp…) read the layer at other times through `EffectHost::self_at`: the renderer renders the layer's source and masks (plus, optionally, the effects before it) at that layer time via `Renderer::layer_input`, cached under separate *input* diff --git a/docs/parity.md b/docs/parity.md index d259fec2..5d291ca7 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -75,7 +75,7 @@ library groundwork for mask tracking and Warp Stabilizer. What is left is concentrated in large systems: Roto Brush and Refine Edge, the 3D camera tracker, Advanced 3D (models, PBR, image-based lighting), the Warp Stabilizer and mask-tracking UI on top of -the new tracking library, a JavaScript scripting object model, SVG/PSD import, WebM, a disk cache, +the new tracking library, a JavaScript scripting object model, the missing effect categories (3D Channel, Immersive Video, OCIO), native macOS menus, and wasm threads. @@ -84,9 +84,9 @@ wasm threads. | Area | Weighted parity | Remaining (agent-hours) | Biggest gaps | |---|---|---|---| | Layers | 88% | 4.8 | frame blending, collapse transformations, slip edit | -| Output | 85% | 4.2 | multiple output modules, pre-render, WebM, audio-only | +| Output | 90% | 3.0 | multiple output modules, pre-render (WebM with VP9 alpha + Opus and WAV/AIFF audio-only landed: VP9 is intra-only, Opus CELT-only) | | Audio | 85% | 0.5 | audio to keyframes | -| Import | 78% | 4.5 | PSD, SVG as shapes, Lottie | +| Import | 88% | 2.0 | AI/EPS/PDF vector footage, PSD smart objects and 3D layers (PSD as footage/composition/retain layer sizes, SVG footage and Create Shapes from Vector Layer landed) | | Automation | 76% | 3.3 | a JavaScript scripting object model | | Shapes | 68% | 3.9 | taper and wave strokes (pen tool for shape paths and vertex editing landed in M6.5) | | Compositions | 67% | 7.4 | marker dialog, flowchart, Essential Graphics | @@ -98,7 +98,7 @@ wasm threads. | Interface | 55% | 10.4 | drag-to-dock and floating panels, preferences, native macOS menus (viewer rulers/snapping/channels/snapshots landed in M0.13) | | Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves | | Masks & roto | 52% | 10.0 | Roto Brush, variable-width mask feather points (mask tracking and Mask Interpolation landed in M6.6) | -| Preview | 55% | 6.1 | GPU 3D and adjustment layers, disk cache (region of interest, snapshots, exposure and Fast Previews landed in M0.13) | +| Preview | 62% | 5.0 | GPU 3D and adjustment layers (persistent disk cache with the blue cache bar landed; region of interest, snapshots, exposure and Fast Previews landed in M0.13) | | Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking, Subspace Warp's mesh warp and rolling-shutter repair (point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | @@ -125,7 +125,7 @@ Mocha and Cineware are third-party and not counted. 10. Drag-to-dock and floating panels, saved workspaces, native macOS menus. 11. Auto-save, crash recovery, recent projects (in progress). 12. Expression gaps: `sampleImage`, `footage()` (the `sourceText` style API landed in M9.9). -13. Lottie (in progress), WebM, SVG and PSD import. +13. ~~Lottie, WebM, SVG and PSD import~~ (landed; a disk cache too). 14. Puppet and paint tools (in progress). 15. Preferences and a shortcut editor that can rebind (in progress); real Wiggler, Smoother and Motion Sketch; the marker dialog. diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 7e7616a5..d822a1b9 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -23,6 +23,7 @@ const LAYERS: &[(&str, u8)] = &[ ("time", 0), ("geom", 0), ("color", 0), + ("vp9enc", 0), ("testkit", 0), ("opusenc", 0), ("raster", 1), From abd39f5fd87472d5c6343c2298774416beeecb2b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:44:06 -0400 Subject: [PATCH 05/13] =?UTF-8?q?CMP-7:=20Essential=20Graphics=20controls,?= =?UTF-8?q?=20master=20properties,=20.ectemplate=20templates,=20Responsive?= =?UTF-8?q?=20Design=20=E2=80=94=20Time?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comps expose controls (properties typed as text/color/slider/checkbox/point/angle/dropdown, Media Replacement, groups, comments); precomp layers get an Essential Properties group whose edits override per instance (Push to Comp / Revert); templates export/import as an open ZIP (manifest.json + project subset + media + poster). Stretched precomps keep protected regions at their original speed. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/commands/essential.rs | 773 ++++++++++++++++++++++++ crates/engine/src/commands/file.rs | 1 + crates/engine/src/commands/file_more.rs | 1 + crates/engine/src/commands/mod.rs | 2 + crates/engine/src/lib.rs | 11 + crates/engine/src/menus.rs | 7 +- crates/engine/src/tests_essential.rs | 289 +++++++++ crates/engine/src/tests_model3d.rs | 1 + crates/engine/src/tests_stubs.rs | 1 + crates/engine/src/tests_track.rs | 2 + crates/gpu/src/tests.rs | 1 + crates/lottie/src/import.rs | 1 + crates/lottie/src/lib.rs | 2 +- crates/lottie/src/tests.rs | 1 + crates/media/src/pool.rs | 2 +- crates/media/src/probe.rs | 3 + crates/project/src/essential.rs | 538 +++++++++++++++++ crates/project/src/lib.rs | 114 +++- crates/project/src/props.rs | 7 + crates/render/src/audio.rs | 1 + crates/render/src/eval.rs | 76 +++ crates/render/src/lib.rs | 30 +- crates/render/src/tests_audio_fx.rs | 1 + crates/render/src/tests_color.rs | 1 + crates/render/src/tests_frame_blend.rs | 1 + 25 files changed, 1862 insertions(+), 5 deletions(-) create mode 100644 crates/engine/src/commands/essential.rs create mode 100644 crates/engine/src/tests_essential.rs create mode 100644 crates/project/src/essential.rs diff --git a/crates/engine/src/commands/essential.rs b/crates/engine/src/commands/essential.rs new file mode 100644 index 00000000..65e93d4e --- /dev/null +++ b/crates/engine/src/commands/essential.rs @@ -0,0 +1,773 @@ +//! Essential Graphics (Window ▸ Essential Graphics) and master properties. +//! +//! - **Authoring**: a comp's controls ([`EssentialGraphics`]): Add Property to Essential +//! Graphics, Media Replacement for footage layers, groups, comments, renaming, reordering, the +//! template name and the panel's Primary composition. +//! - **Instances**: every precomp layer of such a comp shows the controls under **Essential +//! Properties**; changing one overrides it for that instance only (`essential.set`, or any +//! property edit of the master property); Push to Comp writes the override into the source +//! comp, Revert drops it. +//! - **Templates**: `essential.exportTemplate` writes an open `.ectemplate` (a ZIP holding +//! `manifest.json` with the exposed controls, `project.ecproj` with the comp and everything it +//! uses, the media files and a poster frame); `essential.importTemplate` merges one into the +//! project and adds an instance to the active comp. + +use effectcraft_keyframe::Value as KV; +use effectcraft_project::essential::{self, ControlType, EgControl, EgKind, EssentialGraphics}; +use effectcraft_project::{GroupKind, ItemId, ItemKind, LayerId, LayerSource, ParamUi, Project, Uid}; +use effectcraft_time::Tick; +use serde_json::{Value, json}; + +use super::{CommandSpec, b_p, bad, comp_id, has_comp, layer_p, selected_leaf_props, str_p}; +use crate::{EngineError, Result, Session, cmd, query}; + +/// Format id written into template manifests. +pub const TEMPLATE_FORMAT: &str = "effectcraft-template"; +pub const TEMPLATE_VERSION: u32 = 1; + +fn control_p(p: &Value, cmd: &str) -> Result { + p.get("control").and_then(Value::as_u64).ok_or_else(|| bad(cmd, "missing `control` (control id)")) +} + +/// Comp whose Essential Graphics a command edits: `comp`, else the panel's Primary comp, else +/// the active comp. +fn eg_comp(s: &Session, p: &Value) -> Result { + if p.get("comp").is_some() { + return comp_id(s, p); + } + if let Some(c) = s.state.essential_primary.filter(|c| s.project.comp(*c).is_some()) { + return Ok(c); + } + comp_id(s, p) +} + +fn eg_of(proj: &mut Project, cid: ItemId) -> Result<&mut EssentialGraphics> { + let name = proj.item(cid).map(|i| i.name.clone()).unwrap_or_default(); + let c = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + Ok(c.essential.get_or_insert_with(|| EssentialGraphics { name, controls: vec![] })) +} + +/// Insert `ctl` into `group` (or the top level) at `index` (or the end). +fn insert(eg: &mut EssentialGraphics, ctl: EgControl, group: Option, index: Option, cmd: &str) -> Result<()> { + let list = eg.list_mut(group).ok_or_else(|| bad(cmd, "`group` is not a group control"))?; + let i = index.unwrap_or(list.len()).min(list.len()); + list.insert(i, ctl); + Ok(()) +} + +fn set_primary(s: &mut Session, p: &Value) -> Result { + let cid = comp_id(s, p)?; + s.state.essential_primary = Some(cid); + s.bump(); + Ok(json!({"comp": cid.0})) +} + +fn set_name(s: &mut Session, p: &Value) -> Result { + let cid = eg_comp(s, p)?; + let name = str_p(p, "name").ok_or_else(|| bad("essential.setName", "missing `name`"))?.to_string(); + s.edit("Essential Graphics Name", None, |proj, _| { + eg_of(proj, cid)?.name = name.clone(); + Ok(()) + })?; + Ok(json!({"comp": cid.0, "name": name})) +} + +/// Add Property to Essential Graphics: `{layer, path|prop}` or the selected properties. +fn add_property(s: &mut Session, p: &Value) -> Result { + const C: &str = "essential.addProperty"; + let targets: Vec<(ItemId, LayerId, Uid)> = if p.get("path").is_some() || p.get("prop").is_some() { + vec![super::prop::prop_ref(s, p, C)?] + } else { + let cid = comp_id(s, p)?; + selected_leaf_props(s).into_iter().map(|(l, u)| (cid, l, u)).collect() + }; + if targets.is_empty() { + return Err(bad(C, "select a property in the timeline (or give `layer` and `path`)")); + } + let group = p.get("group").and_then(Value::as_u64); + let index = p.get("index").and_then(Value::as_u64).map(|i| i as usize); + let rename = str_p(p, "name").map(str::to_string); + let mut added = vec![]; + for (cid, lid, uid) in &targets { + let l = s.project.comp(*cid).and_then(|c| c.layer(*lid)).ok_or(EngineError::NoComp)?; + let pr = l.props.find(*uid).ok_or_else(|| bad(C, "no such property"))?; + if control_type(pr).is_none() { + return Err(bad(C, format!("`{}` can't be added to Essential Graphics (unsupported property type)", pr.name))); + } + if s.project.comp(*cid).and_then(|c| c.essential.as_ref()).is_some_and(|e| e.control_for(*lid, *uid).is_some()) { + continue; + } + added.push((*cid, *lid, *uid, rename.clone().unwrap_or_else(|| pr.name.clone()))); + } + let ids = s.edit("Add Property to Essential Graphics", None, |proj, _| { + let mut ids = vec![]; + for (k, (cid, lid, uid, name)) in added.iter().enumerate() { + let id = proj.alloc(); + let eg = eg_of(proj, *cid)?; + insert(eg, EgControl { id, name: name.clone(), kind: EgKind::Property { layer: *lid, prop: *uid } }, group, index.map(|i| i + k), C)?; + ids.push(id); + } + Ok(ids) + })?; + Ok(json!({"controls": ids})) +} + +fn control_type(pr: &effectcraft_project::Property) -> Option { + essential::control_type(pr) +} + +/// Media Replacement for a footage layer. +fn add_media(s: &mut Session, p: &Value) -> Result { + const C: &str = "essential.addMedia"; + let (cid, lid) = layer_p(s, p, C)?; + let l = s.project.comp(cid).and_then(|c| c.layer(lid)).ok_or(EngineError::NoComp)?; + if !matches!(l.source, LayerSource::Footage { .. }) { + return Err(bad(C, "Media Replacement needs a footage layer")); + } + let name = str_p(p, "name").map(str::to_string).unwrap_or_else(|| format!("{} (Media)", l.name)); + let group = p.get("group").and_then(Value::as_u64); + let id = s.edit("Add Media Replacement", None, |proj, _| { + let id = proj.alloc(); + let eg = eg_of(proj, cid)?; + if eg.flat().iter().any(|c| matches!(c.kind, EgKind::Media { layer } if layer == lid)) { + return Err(bad(C, "this layer already has a Media Replacement control")); + } + insert(eg, EgControl { id, name, kind: EgKind::Media { layer: lid } }, group, None, C)?; + Ok(id) + })?; + Ok(json!({"control": id})) +} + +fn add_group(s: &mut Session, p: &Value) -> Result { + let cid = eg_comp(s, p)?; + let name = str_p(p, "name").unwrap_or("Group").to_string(); + let index = p.get("index").and_then(Value::as_u64).map(|i| i as usize); + let id = s.edit("Add Essential Graphics Group", None, |proj, _| { + let id = proj.alloc(); + insert(eg_of(proj, cid)?, EgControl { id, name, kind: EgKind::Group { children: vec![] } }, None, index, "essential.addGroup")?; + Ok(id) + })?; + Ok(json!({"control": id})) +} + +fn add_comment(s: &mut Session, p: &Value) -> Result { + let cid = eg_comp(s, p)?; + let text = str_p(p, "text").unwrap_or("Comment").to_string(); + let group = p.get("group").and_then(Value::as_u64); + let id = s.edit("Add Essential Graphics Comment", None, |proj, _| { + let id = proj.alloc(); + insert(eg_of(proj, cid)?, EgControl { id, name: "Comment".into(), kind: EgKind::Comment { text } }, group, None, "essential.addComment")?; + Ok(id) + })?; + Ok(json!({"control": id})) +} + +fn rename(s: &mut Session, p: &Value) -> Result { + const C: &str = "essential.rename"; + let cid = eg_comp(s, p)?; + let id = control_p(p, C)?; + let name = str_p(p, "name").or(str_p(p, "text")).ok_or_else(|| bad(C, "missing `name`"))?.to_string(); + s.edit("Rename Essential Graphics Control", None, |proj, _| { + let c = eg_of(proj, cid)?.find_mut(id).ok_or_else(|| bad(C, format!("no control {id}")))?; + match &mut c.kind { + // A comment's name is its text. + EgKind::Comment { text } => *text = name.clone(), + _ => c.name = name.clone(), + } + Ok(()) + })?; + Ok(json!({"control": id, "name": name})) +} + +fn remove(s: &mut Session, p: &Value) -> Result { + const C: &str = "essential.remove"; + let cid = eg_comp(s, p)?; + let id = control_p(p, C)?; + s.edit("Remove Essential Graphics Control", None, |proj, _| { + eg_of(proj, cid)?.remove(id).ok_or_else(|| bad(C, format!("no control {id}")))?; + Ok(()) + })?; + Ok(json!({"control": id})) +} + +fn move_control(s: &mut Session, p: &Value) -> Result { + const C: &str = "essential.move"; + let cid = eg_comp(s, p)?; + let id = control_p(p, C)?; + let group = p.get("group").and_then(Value::as_u64); + let index = p.get("index").and_then(Value::as_u64).map(|i| i as usize); + if group == Some(id) { + return Err(bad(C, "a group can't contain itself")); + } + s.edit("Move Essential Graphics Control", None, |proj, _| { + let eg = eg_of(proj, cid)?; + let ctl = eg.remove(id).ok_or_else(|| bad(C, format!("no control {id}")))?; + if matches!(ctl.kind, EgKind::Group { .. }) && group.is_some() { + return Err(bad(C, "groups can't be nested")); + } + insert(eg, ctl, group, index, C) + })?; + Ok(json!({"control": id})) +} + +/// Dropdown Menu Control ▸ Edit: the menu items of a dropdown (popup) property. +fn edit_dropdown(s: &mut Session, p: &Value) -> Result { + const C: &str = "effect.editDropdown"; + let items: Vec = match p.get("items") { + Some(Value::Array(a)) => a.iter().filter_map(|v| v.as_str().map(str::to_string)).filter(|x| !x.is_empty()).collect(), + _ => return Err(bad(C, "missing `items` (array of strings)")), + }; + if items.is_empty() { + return Err(bad(C, "a dropdown needs at least one item")); + } + let q = if p.get("path").is_none() && p.get("prop").is_none() { + // The first Dropdown Menu Control of the layer. + let (cid, lid) = layer_p(s, p, C)?; + let l = s.project.comp(cid).and_then(|c| c.layer(lid)).ok_or(EngineError::NoComp)?; + let fx = l.effects().and_then(|fx| fx.groups().find(|g| matches!(&g.kind, GroupKind::Effect { effect } if effect == "ec.control.dropdown"))); + let uid = fx.and_then(|g| g.get("menu")).map(|pr| pr.uid).ok_or_else(|| bad(C, "the layer has no Dropdown Menu Control"))?; + let mut q = p.clone(); + q["prop"] = json!(uid); + q["layer"] = json!(lid.0); + q + } else { + p.clone() + }; + let (cid, lid, uid) = super::prop::prop_ref(s, &q, C)?; + let n = items.len() as u32; + s.edit("Edit Dropdown", None, |proj, _| { + let pr = super::layer_mut(proj, cid, lid)?.props.find_mut(uid).ok_or_else(|| bad(C, "no such property"))?; + if !matches!(pr.ui, ParamUi::Popup { .. }) { + return Err(bad(C, "not a dropdown property")); + } + pr.ui = ParamUi::Popup { options: items.clone() }; + let clamp = |v: &mut KV| { + if let KV::Enum(i) = v { + *i = (*i).min(n - 1); + } + }; + clamp(&mut pr.value); + for k in &mut pr.keys { + clamp(&mut k.value); + } + Ok(()) + })?; + Ok(json!({"prop": uid, "items": items})) +} + +fn solo_supported(s: &mut Session, p: &Value) -> Result { + let on = b_p(p, "on").unwrap_or(!s.state.essential_solo); + s.state.essential_solo = on; + s.bump(); + Ok(json!({"on": on})) +} + +/// Properties of a comp's layers that Essential Graphics can expose (Solo Supported +/// Properties): `[{layer, uid, path, name, type}]`. +pub fn supported_properties(project: &Project, cid: ItemId) -> Vec { + let Some(c) = project.comp(cid) else { return vec![] }; + let mut out = vec![]; + for l in &c.layers { + l.props.walk("", &mut |_, pr| { + if pr.three_d_only && !l.is_3d() { + return; + } + if let Some(t) = essential::control_type(pr) { + let path = super::match_path_of(&l.props, pr.uid).unwrap_or_default(); + if path.starts_with(essential::GROUP) { + return; + } + out.push(json!({"layer": l.id.0, "layerName": l.name, "uid": pr.uid, "path": path, "name": l.props.name_path_of(pr.uid), "type": t})); + } + }); + } + out +} + +fn control_json(project: &Project, cid: ItemId, c: &EgControl) -> Value { + match &c.kind { + EgKind::Property { layer, prop } => { + let src = essential::source_prop(project, cid, c); + json!({ + "id": c.id, "name": c.name, "kind": "property", "layer": layer.0, "prop": prop, + "layerName": src.map(|(l, _)| l.name.clone()), + "path": src.and_then(|(l, _)| super::match_path_of(&l.props, *prop)), + "type": src.and_then(|(_, pr)| essential::control_type(pr)), + "value": src.map(|(_, pr)| pr.value_at(Tick::ZERO).to_json()), + "missing": src.is_none(), + }) + } + EgKind::Media { layer } => json!({ + "id": c.id, "name": c.name, "kind": "media", "type": "media", "layer": layer.0, + "item": essential::source_media(project, cid, c).map(|i| i.0), + }), + EgKind::Comment { text } => json!({"id": c.id, "name": c.name, "kind": "comment", "text": text}), + EgKind::Group { children } => json!({ + "id": c.id, "name": c.name, "kind": "group", + "children": children.iter().map(|x| control_json(project, cid, x)).collect::>(), + }), + } +} + +fn list(s: &mut Session, p: &Value) -> Result { + let cid = eg_comp(s, p)?; + let eg = s.project.comp(cid).and_then(|c| c.essential.clone()).unwrap_or_default(); + let name = if eg.name.is_empty() { s.project.item(cid).map(|i| i.name.clone()).unwrap_or_default() } else { eg.name.clone() }; + Ok(json!({ + "comp": cid.0, + "primary": s.state.essential_primary.map(|c| c.0), + "name": name, + "solo": s.state.essential_solo, + "controls": eg.controls.iter().map(|c| control_json(&s.project, cid, c)).collect::>(), + "supported": if b_p(p, "supported").unwrap_or(false) { json!(supported_properties(&s.project, cid)) } else { Value::Null }, + })) +} + +// ------------------------------------------------------------------------------ instances + +/// The precomp layer (an instance) and its source comp. +fn instance(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, ItemId)> { + let (cid, lid) = layer_p(s, p, cmd)?; + let l = s.project.comp(cid).and_then(|c| c.layer(lid)).ok_or(EngineError::NoComp)?; + match l.source { + LayerSource::Comp { item } if essential::group(l).is_some() => Ok((cid, lid, item)), + _ => Err(bad(cmd, "the layer is not a precomp with Essential Properties")), + } +} + +/// The master property (instance child) of a control. +fn master_uid(s: &Session, cid: ItemId, lid: LayerId, control: u64) -> Option { + let l = s.project.comp(cid)?.layer(lid)?; + let g = essential::group(l)?; + let m = essential::match_id(control); + let mut found = None; + g.walk("", &mut |_, pr| { + if pr.match_id == m { + found = Some(pr.uid); + } + }); + found +} + +fn instance_json(s: &mut Session, p: &Value) -> Result { + let (cid, lid, src) = instance(s, p, "essential.instance")?; + let l = s.project.comp(cid).and_then(|c| c.layer(lid)).ok_or(EngineError::NoComp)?; + let over = essential::overridden(l); + let eg = s.project.comp(src).and_then(|c| c.essential.clone()).unwrap_or_default(); + let t = s.time(); + let g = essential::group(l).cloned(); + let mut out = vec![]; + for c in eg.flat() { + let m = essential::match_id(c.id); + let mut pr = None; + if let Some(g) = &g { + g.walk("", &mut |_, x| { + if x.match_id == m { + pr = Some(x.clone()); + } + }); + } + let kind = match c.kind { + EgKind::Property { .. } => "property", + EgKind::Media { .. } => "media", + EgKind::Comment { .. } => "comment", + EgKind::Group { .. } => "group", + }; + out.push(json!({ + "control": c.id, "name": c.name, "kind": kind, + "uid": pr.as_ref().map(|x| x.uid), + "path": pr.as_ref().and_then(|x| super::match_path_of(&l.props, x.uid)), + "value": pr.as_ref().map(|x| x.value_at(l.layer_time(t)).to_json()), + "type": pr.as_ref().and_then(essential::control_type).or((kind == "media").then_some(ControlType::Media)), + "overridden": pr.as_ref().is_some_and(|x| over.contains(&x.uid)), + })); + } + Ok(json!({"layer": lid.0, "comp": src.0, "name": eg.name, "controls": out})) +} + +/// Set an instance's value for a control (`value`, or `item` for Media Replacement). +fn set_override(s: &mut Session, p: &Value) -> Result { + const C: &str = "essential.set"; + let (cid, lid, _) = instance(s, p, C)?; + let control = control_p(p, C)?; + let uid = master_uid(s, cid, lid, control).ok_or_else(|| bad(C, format!("no control {control} on this instance")))?; + if let Some(item) = p.get("item") { + let item = match item { + Value::Number(n) => ItemId(n.as_u64().unwrap_or(0)), + Value::String(name) => s.project.find_by_name(name).map(|i| i.id).ok_or_else(|| bad(C, format!("no item `{name}`")))?, + _ => return Err(bad(C, "`item` is an item id or name")), + }; + if !matches!(s.project.item(item).map(|i| &i.kind), Some(ItemKind::Footage(f)) if f.has_video) { + return Err(bad(C, "replacement media must be footage with video")); + } + s.edit("Replace Media", None, |proj, _| { + let pr = super::layer_mut(proj, cid, lid)?.props.find_mut(uid).ok_or_else(|| bad(C, "no such property"))?; + pr.value = KV::Scalar(item.0 as f64); + Ok(()) + })?; + return Ok(json!({"control": control, "item": item.0})); + } + let mut q = json!({"comp": cid.0, "layer": lid.0, "prop": uid, "value": p.get("value").cloned().ok_or_else(|| bad(C, "missing `value` (or `item`)"))?}); + if let Some(m) = p.get("merge") { + q["merge"] = m.clone(); + } + s.execute("prop.set", q) +} + +/// Controls of an instance to act on: `control` or every overridden one. +fn targets(s: &Session, cid: ItemId, lid: LayerId, p: &Value) -> Vec<(u64, Uid)> { + let Some(l) = s.project.comp(cid).and_then(|c| c.layer(lid)) else { return vec![] }; + let over = essential::overridden(l); + let Some(g) = essential::group(l) else { return vec![] }; + let want = p.get("control").and_then(Value::as_u64); + let mut out = vec![]; + g.walk("", &mut |_, pr| { + if let Some(c) = essential::control_of(&pr.match_id) + && want.is_none_or(|w| w == c) + && (want.is_some() || over.contains(&pr.uid)) + { + out.push((c, pr.uid)); + } + }); + out +} + +fn revert(s: &mut Session, p: &Value) -> Result { + const C: &str = "essential.revert"; + let (cid, lid, _) = instance(s, p, C)?; + let ts = targets(s, cid, lid, p); + if ts.is_empty() { + return Ok(json!({"reverted": []})); + } + s.edit("Revert Essential Property", None, |proj, _| { + let l = super::layer_mut(proj, cid, lid)?; + let g = l.props.sub_mut(essential::GROUP).ok_or_else(|| bad(C, "no Essential Properties"))?; + if let GroupKind::Essential { overridden } = &mut g.kind { + overridden.retain(|u| !ts.iter().any(|(_, x)| x == u)); + } + for (_, uid) in &ts { + if let Some(pr) = g.find_mut(*uid) { + // The sync below brings back the source value. + pr.keys.clear(); + pr.expr = None; + } + } + Ok(()) + })?; + // Edits that only clear an override must not be taken for a new override: values are + // refreshed by the sync inside `edit`, which ran after the override list was trimmed. + Ok(json!({"reverted": ts.iter().map(|(c, _)| c).collect::>()})) +} + +/// Push to Comp: write the instance's override values into the source comp's properties (and +/// replaced media into the source layer), then drop the overrides. +fn push_to_comp(s: &mut Session, p: &Value) -> Result { + const C: &str = "essential.pushToComp"; + let (cid, lid, src) = instance(s, p, C)?; + let ts = targets(s, cid, lid, p); + let eg = s.project.comp(src).and_then(|c| c.essential.clone()).unwrap_or_default(); + let l = s.project.comp(cid).and_then(|c| c.layer(lid)).cloned().ok_or(EngineError::NoComp)?; + let mut writes = vec![]; + for (control, uid) in &ts { + let (Some(ctl), Some(pr)) = (eg.find(*control), l.props.find(*uid)) else { continue }; + writes.push((ctl.kind.clone(), pr.clone())); + } + s.edit("Push Override Values to Source", None, |proj, _| { + let sc = proj.comp_mut(src).ok_or(EngineError::NoComp)?; + for (kind, pr) in &writes { + match kind { + EgKind::Property { layer, prop } => { + let Some(inner) = sc.layer_mut(*layer) else { continue }; + // Instance keys are in the precomp layer's time (= the source comp's time); + // the source property's keys are in the inner layer's time. + let keys: Vec<_> = pr + .keys + .iter() + .map(|k| { + let mut k = k.clone(); + k.time = inner.layer_time(k.time); + k + }) + .collect(); + if let Some(target) = inner.props.find_mut(*prop) { + target.value = pr.value.clone(); + target.keys = keys; + } + } + EgKind::Media { layer } => { + if let Some(inner) = sc.layer_mut(*layer) { + inner.source = LayerSource::Footage { item: ItemId(pr.value.as_f64().max(0.0) as u64) }; + } + } + _ => {} + } + } + let g = super::layer_mut(proj, cid, lid)?.props.sub_mut(essential::GROUP).ok_or_else(|| bad(C, "no Essential Properties"))?; + if let GroupKind::Essential { overridden } = &mut g.kind { + overridden.retain(|u| !ts.iter().any(|(_, x)| x == u)); + } + for (_, uid) in &ts { + if let Some(pr) = g.find_mut(*uid) { + pr.keys.clear(); + pr.expr = None; + } + } + Ok(()) + })?; + Ok(json!({"pushed": ts.iter().map(|(c, _)| c).collect::>()})) +} + +// ------------------------------------------------------------------------------ templates + +fn file_name(path: &str) -> String { + std::path::Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.to_string()) +} + +/// The manifest's control list (flattened, with the group each belongs to). +fn manifest_controls(project: &Project, cid: ItemId, eg: &EssentialGraphics) -> Vec { + fn go(project: &Project, cid: ItemId, v: &[EgControl], group: Option, out: &mut Vec) { + for c in v { + let mut j = control_json(project, cid, c); + if let Some(o) = j.as_object_mut() { + o.remove("children"); + o.insert("group".into(), json!(group)); + } + out.push(j); + if let EgKind::Group { children } = &c.kind { + go(project, cid, children, Some(c.id), out); + } + } + } + let mut out = vec![]; + go(project, cid, &eg.controls, None, &mut out); + out +} + +fn export_template(s: &mut Session, p: &Value) -> Result { + const C: &str = "essential.exportTemplate"; + let cid = eg_comp(s, p)?; + let path = str_p(p, "path").ok_or_else(|| bad(C, "missing `path` (.ectemplate)"))?.to_string(); + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let eg = comp.essential.clone().filter(|e| !e.controls.is_empty()).ok_or_else(|| bad(C, "the composition has no Essential Graphics controls"))?; + let comp_name = s.project.item(cid).map(|i| i.name.clone()).unwrap_or_default(); + let name = str_p(p, "name").map(str::to_string).unwrap_or_else(|| if eg.name.is_empty() { comp_name.clone() } else { eg.name.clone() }); + // The project subset: the comp and everything it uses, at the root. + let deps = essential::dependencies(&s.project, cid); + let mut sub = Project { settings: s.project.settings.clone(), next_id: s.project.next_id, ..Project::default() }; + let mut entries: Vec<(String, Vec)> = vec![]; + let mut media = vec![]; + for id in &deps { + let Some(mut it) = s.project.item(*id).cloned() else { continue }; + it.parent = None; + it.proxy = None; + if let ItemKind::Footage(f) = &mut it.kind + && f.data.is_none() + { + let files: Vec = if f.sequence.is_empty() { vec![f.path.clone()] } else { f.sequence.clone() }; + let mut stored = vec![]; + for (k, file) in files.iter().enumerate() { + if let Ok(bytes) = s.services.read_file(file) { + let entry = format!("media/{}/{}", id.0, file_name(file)); + entries.push((entry.clone(), bytes)); + if k == 0 { + f.path = entry.clone(); + } + stored.push(entry); + } + } + if !stored.is_empty() { + if !f.sequence.is_empty() { + f.sequence = stored.clone(); + } + media.push(json!({"item": id.0, "files": stored})); + } + } + sub.items.insert(*id, it); + } + let manifest = json!({ + "format": TEMPLATE_FORMAT, + "version": TEMPLATE_VERSION, + "generator": format!("EffectCraft {}", env!("CARGO_PKG_VERSION")), + "name": name, + "comp": cid.0, + "compName": comp_name, + "width": comp.width, + "height": comp.height, + "frameRate": comp.frame_rate.as_f64(), + "duration": comp.duration.seconds(), + "controls": manifest_controls(&s.project, cid, &eg), + "media": media, + "poster": "poster.png", + }); + // A poster frame (longest side ≤ 320 px). + if let Ok((w, h, rgba)) = s.render_rgba8(cid, comp.poster_time, 320) + && let Ok(png) = super::comp_more::encode_png(&rgba, w, h) + { + entries.push(("poster.png".into(), png)); + } + entries.insert(0, ("project.ecproj".into(), sub.to_json().into_bytes())); + entries.insert(0, ("manifest.json".into(), serde_json::to_vec_pretty(&manifest).unwrap_or_default())); + let bytes = effectcraft_lottie::zip::store(&entries); + s.services.write_file(&path, &bytes).map_err(|e| EngineError::Other(format!("cannot write {path}: {e}")))?; + s.toast(format!("Exported template “{name}” to {path}")); + Ok(json!({"path": path, "name": name, "bytes": bytes.len(), "controls": eg.flat().len(), "items": deps.len()})) +} + +/// Read a template's manifest and project (with media paths still archive-relative). +pub fn read_template(bytes: &[u8]) -> std::result::Result<(Value, Project, Vec<(String, Vec)>), String> { + let entries = effectcraft_lottie::zip::read_stored(bytes); + let get = |n: &str| entries.iter().find(|(k, _)| k == n).map(|(_, v)| v.clone()); + let manifest: Value = + serde_json::from_slice(&get("manifest.json").ok_or("not an EffectCraft template (no manifest.json)")?).map_err(|e| format!("manifest.json: {e}"))?; + if manifest.get("format").and_then(Value::as_str) != Some(TEMPLATE_FORMAT) { + return Err("not an EffectCraft template".into()); + } + if manifest.get("version").and_then(Value::as_u64).unwrap_or(0) > TEMPLATE_VERSION as u64 { + return Err("the template was made by a newer EffectCraft".into()); + } + let text = String::from_utf8(get("project.ecproj").ok_or("template has no project.ecproj")?).map_err(|e| e.to_string())?; + let project = Project::from_json(&text).map_err(|e| e.to_string())?; + Ok((manifest, project, entries)) +} + +fn import_template(s: &mut Session, p: &Value) -> Result { + const C: &str = "essential.importTemplate"; + let path = str_p(p, "path").ok_or_else(|| bad(C, "missing `path` (.ectemplate)"))?.to_string(); + let bytes = s.services.read_file(&path).map_err(|e| EngineError::Other(format!("cannot read {path}: {e}")))?; + let (manifest, mut sub, entries) = read_template(&bytes).map_err(|e| bad(C, e))?; + let name = manifest.get("name").and_then(Value::as_str).unwrap_or("Template").to_string(); + let tcomp = manifest.get("comp").and_then(Value::as_u64).map(ItemId).ok_or_else(|| bad(C, "manifest has no `comp`"))?; + // Media next to the template: `/ Media//`. + let pp = std::path::Path::new(&path); + let dir = pp.parent().map(|d| d.to_path_buf()).unwrap_or_default(); + let stem = pp.file_stem().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "Template".into()); + let media_dir = dir.join(format!("{stem} Media")); + let mut written = std::collections::HashMap::new(); + for (k, v) in entries.iter().filter(|(k, _)| k.starts_with("media/")) { + let rel = k.trim_start_matches("media/"); + let out = media_dir.join(rel); + if let Some(parent) = out.parent() { + let _ = std::fs::create_dir_all(parent); + } + let out_s = out.to_string_lossy().to_string(); + if s.services.write_file(&out_s, v).is_ok() { + written.insert(k.clone(), out_s); + } + } + for it in sub.items.values_mut() { + if let ItemKind::Footage(f) = &mut it.kind { + if let Some(w) = written.get(&f.path) { + f.path = w.clone(); + } + for x in &mut f.sequence { + if let Some(w) = written.get(x) { + *x = w.clone(); + } + } + } + } + let target = if b_p(p, "addToComp") == Some(false) { None } else { comp_id(s, p).ok() }; + let base = s.project.next_id; + essential::offset_ids(&mut sub, base); + let new_comp = ItemId(tcomp.0 + base); + let ids = s.edit("Import Template", None, |proj, st| { + let folder = proj.add_item(&name, effectcraft_color::Label::Yellow, None, ItemKind::Folder); + let mut ids = vec![]; + for (id, mut it) in std::mem::take(&mut sub.items) { + if it.parent.is_none() { + it.parent = Some(folder); + } + ids.push(id.0); + proj.items.insert(id, it); + } + proj.next_id = proj.next_id.max(sub.next_id); + proj.fix_next_id(); + st.project_selection = vec![new_comp]; + Ok(ids) + })?; + let mut layer = Value::Null; + if let Some(cid) = target + && s.project.comp(cid).is_some() + { + let r = s.execute("layer.addItem", json!({"comp": cid.0, "item": new_comp.0, "time": 0.0}))?; + layer = r["layer"].clone(); + } + s.toast(format!("Imported template “{name}”")); + Ok(json!({"comp": new_comp.0, "items": ids, "layer": layer, "name": name})) +} + +/// Read a template's manifest without importing it (Essential Graphics / agents). +fn template_info(s: &mut Session, p: &Value) -> Result { + let path = str_p(p, "path").ok_or_else(|| bad("essential.templateInfo", "missing `path`"))?; + let bytes = s.services.read_file(path).map_err(|e| EngineError::Other(format!("cannot read {path}: {e}")))?; + let (manifest, _, _) = read_template(&bytes).map_err(|e| bad("essential.templateInfo", e))?; + Ok(manifest) +} + +fn has_instance(s: &Session) -> std::result::Result<(), String> { + has_comp(s)?; + let ok = s.active_comp().is_some_and(|c| s.state.selected_layers.iter().any(|l| c.layer(*l).is_some_and(|l| essential::group(l).is_some()))); + if ok { Ok(()) } else { Err("select a precomp layer with Essential Properties".into()) } +} + +fn has_eg(s: &Session) -> std::result::Result<(), String> { + let cid = s.state.essential_primary.or(s.state.active_comp).ok_or("no composition is open")?; + if s.project.comp(cid).and_then(|c| c.essential.as_ref()).is_some_and(|e| !e.controls.is_empty()) { + Ok(()) + } else { + Err("add properties to Essential Graphics first".into()) + } +} + +fn has_selected_props(s: &Session) -> std::result::Result<(), String> { + has_comp(s)?; + if s.state.selected_props.is_empty() { Err("select a property in the timeline".into()) } else { Ok(()) } +} + +pub fn specs() -> Vec { + vec![ + cmd!("essential.setPrimary", "Primary Composition", [], None, "{comp}", has_comp, set_primary), + cmd!("essential.setName", "Essential Graphics Name", [], None, "{comp?, name}", has_comp, set_name), + cmd!( + "essential.addProperty", + "Add Property to Essential Graphics", + ["Animation"], + None, + "{comp?, layer?, path?|prop?: uid (default: the selected properties), name?, group?: control id, index?}", + has_selected_props, + add_property + ), + cmd!("essential.addMedia", "Add Media Replacement", [], None, "{layer, name?, group?}", has_comp, add_media), + cmd!("essential.addGroup", "Add Group", [], None, "{comp?, name?, index?}", has_comp, add_group), + cmd!("essential.addComment", "Add Comment", [], None, "{comp?, text?, group?}", has_comp, add_comment), + cmd!("essential.rename", "Rename Control", [], None, "{comp?, control, name}", has_comp, rename), + cmd!("essential.remove", "Remove Control", [], None, "{comp?, control}", has_comp, remove), + cmd!("essential.move", "Move Control", [], None, "{comp?, control, group?: id (null = top level), index?}", has_comp, move_control), + cmd!("essential.soloSupported", "Solo Supported Properties", [], None, "{on?}", has_comp, solo_supported), + cmd!( + "essential.exportTemplate", + "Essential Graphics Template...", + ["File", "Export"], + None, + "{comp?, path (.ectemplate), name?}", + has_eg, + export_template + ), + cmd!( + "essential.importTemplate", + "Essential Graphics Template...", + ["File", "Import"], + None, + "{path (.ectemplate), comp?, addToComp?: bool}", + super::always, + import_template + ), + cmd!("essential.set", "Set Essential Property", [], None, "{layer, control, value | item (media)}", has_instance, set_override), + cmd!("essential.pushToComp", "Push Override Values to Source", [], None, "{layer, control? (default: all overridden)}", has_instance, push_to_comp), + cmd!("essential.revert", "Revert", [], None, "{layer, control? (default: all overridden)}", has_instance, revert), + cmd!("effect.editDropdown", "Edit Dropdown Menu", [], None, "{layer, path?|prop?, items: [string]}", has_comp, edit_dropdown), + query!("essential.list", "Essential Graphics controls", "{comp?, supported?: bool}", list), + query!("essential.instance", "Essential Properties of a precomp layer", "{layer}", instance_json), + query!("essential.templateInfo", "Read a template's manifest", "{path}", template_info), + ] +} diff --git a/crates/engine/src/commands/file.rs b/crates/engine/src/commands/file.rs index 11e8a387..9e81db54 100644 --- a/crates/engine/src/commands/file.rs +++ b/crates/engine/src/commands/file.rs @@ -111,6 +111,7 @@ fn import(s: &mut Session, p: &Value) -> Result { FootageKind::Still | FootageKind::Sequence => Label::Lavender, FootageKind::Audio => Label::SeaFoam, FootageKind::Video | FootageKind::Model => Label::Aqua, + FootageKind::Data => Label::Sandstone, }; let id = proj.add_item(&name, label, None, ItemKind::Footage(f)); ids.push(id.0); diff --git a/crates/engine/src/commands/file_more.rs b/crates/engine/src/commands/file_more.rs index 87c1e2df..f5f4d370 100644 --- a/crates/engine/src/commands/file_more.rs +++ b/crates/engine/src/commands/file_more.rs @@ -107,6 +107,7 @@ fn placeholder_footage(p: &Value) -> Footage { missing: true, sequence: vec![], color_profile: None, + ..Default::default() } } diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 1a144723..7a053c77 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -8,6 +8,7 @@ mod comp; mod comp_more; mod edit; mod effect; +pub mod essential; mod file; mod file_more; mod frontend; @@ -136,6 +137,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(puppet::specs()); v.extend(liquify::specs()); v.extend(settings::specs()); + v.extend(essential::specs()); v.extend(stubs::specs()); v }) diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 61a6f9c3..d0a4ba1a 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -197,6 +197,13 @@ pub struct EditorState { /// Composition viewer display options (Show Channel, exposure, snapshot, Fast Previews). #[serde(default)] pub viewer: commands::viewer_cmds::ViewOptions, + /// Essential Graphics panel ▸ Primary composition (`None` = the active comp). + #[serde(default)] + pub essential_primary: Option, + /// Essential Graphics panel ▸ Solo Supported Properties (the timeline shows only properties + /// Essential Graphics can expose). + #[serde(default)] + pub essential_solo: bool, } fn one_view() -> u8 { @@ -368,6 +375,8 @@ impl Session { let r = f(&mut p, &mut st)?; // Layer styles: one Global Light per comp, whichever layer edited it. effectcraft_project::styles::sync_global_light(&before, &mut p); + // Essential Properties of precomp layers follow their comps' Essential Graphics. + effectcraft_project::essential::sync_project(&before, &mut p); // Warp Stabilizer analyses made from other frames are cleared (and queued again). for w in warp::invalidate(&before, &mut p) { if !self.warp_pending.contains(&w) { @@ -576,6 +585,8 @@ mod tests_anim_tools; #[cfg(test)] mod tests_effects; #[cfg(test)] +mod tests_essential; +#[cfg(test)] mod tests_fidelity; #[cfg(test)] mod tests_lottie; diff --git a/crates/engine/src/menus.rs b/crates/engine/src/menus.rs index 38252a88..6177e637 100644 --- a/crates/engine/src/menus.rs +++ b/crates/engine/src/menus.rs @@ -17,7 +17,8 @@ //! //! An entry's shortcut is the command's default shortcut unless the line gives one (needed for //! entries with bound parameters). Adobe-service entries (Team Projects, Libraries, Bridge, Media -//! Encoder, Essential Graphics, Behance, Creative Cloud…) are intentionally absent. +//! Encoder, Behance, Creative Cloud…) are intentionally absent; Essential Graphics templates use +//! EffectCraft's own open `.ectemplate` format. use std::sync::OnceLock; @@ -337,9 +338,11 @@ File Placeholder... | file.importPlaceholder Solid... | file.importSolid Lottie... | file.importLottie + Essential Graphics Template... | essential.importTemplate Export Add to Render Queue | renderQueue.add Lottie JSON... | file.exportLottie + Essential Graphics Template... | essential.exportTemplate --- Find | app.find --- @@ -816,6 +819,7 @@ Animation Remove All Text Animators | text.removeAllAnimators --- Add Expression | prop.setExpression + Add Property to Essential Graphics | essential.addProperty Separate Dimensions | prop.separateDimensions Track Camera | track.camera Warp Stabilizer VFX | track.warpStabilizer @@ -931,6 +935,7 @@ Window Character | window.panel {"panel":"character"} | Cmd+6 Content-Aware Fill | window.unavailablePanel {"panel":"contentAwareFill"} Effects & Presets | window.panel {"panel":"effectsPresets"} | Cmd+5 + Essential Graphics | window.panel {"panel":"essentialGraphics"} Info | window.panel {"panel":"info"} | Cmd+2 Lumetri Scopes | window.unavailablePanel {"panel":"lumetriScopes"} Mask Interpolation | window.panel {"panel":"maskInterpolation"} diff --git a/crates/engine/src/tests_essential.rs b/crates/engine/src/tests_essential.rs new file mode 100644 index 00000000..33473857 --- /dev/null +++ b/crates/engine/src/tests_essential.rs @@ -0,0 +1,289 @@ +//! Essential Graphics: authoring controls, master properties (per-instance overrides, Push to +//! Comp, Revert), `.ectemplate` export/import, and Responsive Design — Time. + +use effectcraft_project::essential; +use effectcraft_project::{ItemId, LayerId}; +use effectcraft_time::Tick; +use serde_json::{Value, json}; + +use crate::Session; +use crate::render::RenderOpts; + +fn out_dir(name: &str) -> std::path::PathBuf { + let d = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/test-out").join(name); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +/// A "Card" comp (64×64): a white solid with a Fill effect (red) and a text layer; its Fill +/// colour, opacity and text are exposed. Returns (session, card comp, solid layer). +fn card() -> (Session, ItemId, u64, Value) { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Card", "width": 64, "height": 64, "frameRate": 30, "duration": 2})).unwrap(); + let card = s.active_comp_id().unwrap(); + let solid = s.execute("layer.newSolid", json!({"name": "BG", "color": "#ffffff"})).unwrap()["layer"].as_u64().unwrap(); + s.execute("effect.apply", json!({"layer": solid, "effect": "Fill"})).unwrap(); + let text = s.execute("layer.newText", json!({"text": "Hello"})).unwrap()["layer"].as_u64().unwrap(); + let color = s.execute("essential.addProperty", json!({"layer": solid, "path": "effects/#1/color", "name": "Card Color"})).unwrap(); + let opacity = s.execute("essential.addProperty", json!({"layer": solid, "path": "transform/opacity"})).unwrap(); + let g = s.execute("essential.addGroup", json!({"name": "Words"})).unwrap()["control"].as_u64().unwrap(); + let title = s.execute("essential.addProperty", json!({"layer": text, "path": "text/sourceText", "group": g})).unwrap(); + s.execute("essential.addComment", json!({"text": "Pick a colour"})).unwrap(); + s.execute("essential.setName", json!({"name": "Lower Card"})).unwrap(); + let ids = json!({ + "color": color["controls"][0], "opacity": opacity["controls"][0], "group": g, "title": title["controls"][0], + }); + (s, card, solid, ids) +} + +fn center(s: &Session, comp: ItemId) -> [f32; 4] { + let img = s.render(comp, Tick::ZERO, RenderOpts::default()); + img.get(img.width as i64 / 2, img.height as i64 / 2) +} + +#[test] +fn controls_are_listed_typed_and_editable() { + let (mut s, card, solid, ids) = card(); + let l = s.execute("essential.list", json!({"supported": true})).unwrap(); + assert_eq!(l["name"], "Lower Card"); + let top: Vec<&str> = l["controls"].as_array().unwrap().iter().map(|c| c["kind"].as_str().unwrap()).collect(); + assert_eq!(top, ["property", "property", "group", "comment"]); + assert_eq!(l["controls"][0]["name"], "Card Color"); + assert_eq!(l["controls"][0]["type"], "color"); + assert_eq!(l["controls"][1]["type"], "slider"); + assert_eq!(l["controls"][2]["children"][0]["type"], "text"); + assert!(l["supported"].as_array().unwrap().iter().any(|p| p["path"] == "transform/position" && p["type"] == "point")); + assert!(l["supported"].as_array().unwrap().iter().any(|p| p["path"] == "transform/rotation" && p["type"] == "angle")); + // Rename, move into the group, remove; all undoable. + let opacity = ids["opacity"].as_u64().unwrap(); + s.execute("essential.rename", json!({"control": opacity, "name": "Fade"})).unwrap(); + s.execute("essential.move", json!({"control": opacity, "group": ids["group"], "index": 0})).unwrap(); + let l = s.execute("essential.list", json!({})).unwrap(); + assert_eq!(l["controls"][1]["children"][0]["name"], "Fade"); + s.execute("edit.undo", json!({})).unwrap(); + s.execute("edit.undo", json!({})).unwrap(); + let l = s.execute("essential.list", json!({})).unwrap(); + assert_eq!(l["controls"][1]["name"], "Opacity"); + s.execute("essential.remove", json!({"control": opacity})).unwrap(); + assert_eq!(s.execute("essential.list", json!({})).unwrap()["controls"].as_array().unwrap().len(), 3); + // Unsupported properties are refused; adding the same property twice adds nothing. + s.execute("layer.addMask", json!({"layer": solid})).unwrap(); + assert!(s.execute("essential.addProperty", json!({"layer": solid, "path": "masks/#1/path"})).is_err()); + let again = s.execute("essential.addProperty", json!({"layer": solid, "path": "effects/#1/color"})).unwrap(); + assert_eq!(again["controls"], json!([])); + // Dropdown Menu Control: its items are editable and it becomes a dropdown control. + s.execute("effect.apply", json!({"layer": solid, "effect": "Dropdown Menu Control"})).unwrap(); + s.execute("effect.editDropdown", json!({"layer": solid, "items": ["Small", "Medium", "Large", "Huge"]})).unwrap(); + let d = s.execute("essential.addProperty", json!({"layer": solid, "path": "effects/#2/menu"})).unwrap()["controls"][0].as_u64().unwrap(); + let l = s.execute("essential.list", json!({})).unwrap(); + let c = l["controls"].as_array().unwrap().iter().find(|c| c["id"] == d).unwrap().clone(); + assert_eq!(c["type"], "dropdown"); + let comp = s.project.comp(card).unwrap(); + let menu = comp.layer(LayerId(solid)).unwrap().props.prop("effects/#2/menu").unwrap(); + assert!(matches!(&menu.ui, effectcraft_project::ParamUi::Popup { options } if options.len() == 4 && options[3] == "Huge")); +} + +#[test] +fn master_properties_override_per_instance_push_and_revert() { + let (mut s, card, solid, ids) = card(); + s.execute("comp.new", json!({"name": "Main", "width": 64, "height": 64, "frameRate": 30, "duration": 2})).unwrap(); + let main = s.active_comp_id().unwrap(); + let a = s.execute("layer.addItem", json!({"item": card.0})).unwrap()["layer"].as_u64().unwrap(); + let b = s.execute("layer.addItem", json!({"item": card.0})).unwrap()["layer"].as_u64().unwrap(); + // Both instances show the controls under Essential Properties. + let inst = s.execute("essential.instance", json!({"layer": a})).unwrap(); + let names: Vec<&str> = inst["controls"].as_array().unwrap().iter().map(|c| c["name"].as_str().unwrap()).collect(); + assert_eq!(names, ["Card Color", "Opacity", "Words", "Source Text", "Comment"]); + assert_eq!(inst["controls"][0]["value"], json!([1.0, 0.0, 0.0, 1.0])); + // Override the colour of the top instance (a = layer #2, b on top is #1). + s.execute("layer.select", json!({"layers": [b]})).unwrap(); + s.execute("essential.set", json!({"layer": b, "control": ids["color"], "value": [0.0, 0.0, 1.0, 1.0]})).unwrap(); + let px = center(&s, main); + assert!(px[2] > 0.99 && px[0] < 0.01, "instance b is blue: {px:?}"); + let inst_b = s.execute("essential.instance", json!({"layer": b})).unwrap(); + assert_eq!(inst_b["controls"][0]["overridden"], true); + assert_eq!(s.execute("essential.instance", json!({"layer": a})).unwrap()["controls"][0]["overridden"], false); + // The source comp is untouched; hiding b shows a still red. + assert!(center(&s, card)[0] > 0.99); + s.execute("layer.setSwitch", json!({"layer": b, "switch": "video", "value": false})).unwrap(); + let px = center(&s, main); + assert!(px[0] > 0.99 && px[2] < 0.01, "instance a stays red: {px:?}"); + s.execute("edit.undo", json!({})).unwrap(); + // Editing the master property in the timeline (prop.set) also overrides. + let path = inst_b["controls"][1]["path"].as_str().unwrap().to_string(); + assert!(path.starts_with("essential/"), "{path}"); + s.execute("prop.set", json!({"layer": a, "path": path, "value": 50})).unwrap(); + assert_eq!(s.execute("essential.instance", json!({"layer": a})).unwrap()["controls"][1]["overridden"], true); + // A change to the source shows in instances that don't override it. + s.execute("prop.set", json!({"comp": card.0, "layer": solid, "path": "transform/opacity", "value": 80})).unwrap(); + assert_eq!(s.execute("essential.instance", json!({"layer": b})).unwrap()["controls"][1]["value"], json!(80.0)); + assert_eq!(s.execute("essential.instance", json!({"layer": a})).unwrap()["controls"][1]["value"], json!(50.0)); + // Revert a's opacity. + s.execute("essential.revert", json!({"layer": a, "control": ids["opacity"]})).unwrap(); + let inst = s.execute("essential.instance", json!({"layer": a})).unwrap(); + assert_eq!(inst["controls"][1]["overridden"], false); + assert_eq!(inst["controls"][1]["value"], json!(80.0)); + // Push b's colour to the source comp: the override goes away and every instance is blue. + s.execute("essential.pushToComp", json!({"layer": b})).unwrap(); + let src = s.project.comp(card).unwrap().layer(LayerId(solid)).unwrap().props.prop("effects/#1/color").unwrap().value.clone(); + assert_eq!(src.to_json(), json!([0.0, 0.0, 1.0, 1.0])); + assert_eq!(s.execute("essential.instance", json!({"layer": b})).unwrap()["controls"][0]["overridden"], false); + assert_eq!(s.execute("essential.instance", json!({"layer": a})).unwrap()["controls"][0]["value"], json!([0.0, 0.0, 1.0, 1.0])); + // Undo restores the override and the red source. + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(s.execute("essential.instance", json!({"layer": b})).unwrap()["controls"][0]["overridden"], true); + let src = s.project.comp(card).unwrap().layer(LayerId(solid)).unwrap().props.prop("effects/#1/color").unwrap().value.clone(); + assert_eq!(src.to_json(), json!([1.0, 0.0, 0.0, 1.0])); + // Removing a control removes it from instances. + s.execute("essential.remove", json!({"comp": card.0, "control": ids["color"]})).unwrap(); + let l = s.project.comp(main).unwrap().layer(LayerId(b)).unwrap().clone(); + assert!(essential::group(&l).unwrap().children.iter().all(|n| n.name() != "Card Color")); +} + +#[test] +fn template_export_import_round_trip_with_overrides() { + let dir = out_dir("ectemplate"); + let (mut s, card, _solid, ids) = card(); + let path = dir.join("Lower Card.ectemplate").to_string_lossy().to_string(); + let r = s.execute("essential.exportTemplate", json!({"comp": card.0, "path": path})).unwrap(); + assert_eq!(r["name"], "Lower Card"); + let info = s.execute("essential.templateInfo", json!({"path": path})).unwrap(); + assert_eq!(info["format"], "effectcraft-template"); + assert_eq!(info["width"], 64); + let kinds: Vec<&str> = info["controls"].as_array().unwrap().iter().map(|c| c["kind"].as_str().unwrap()).collect(); + assert_eq!(kinds, ["property", "property", "group", "property", "comment"]); + assert_eq!(info["controls"][3]["group"], ids["group"]); + let entries = effectcraft_lottie::zip::read_stored(&std::fs::read(&path).unwrap()); + let names: Vec<&str> = entries.iter().map(|(n, _)| n.as_str()).collect(); + assert!(names.contains(&"manifest.json") && names.contains(&"project.ecproj") && names.contains(&"poster.png"), "{names:?}"); + + // A fresh project imports it and gets an instance in its active comp. + let mut t = Session::default(); + t.execute("comp.new", json!({"name": "Show", "width": 64, "height": 64, "frameRate": 30, "duration": 2})).unwrap(); + let show = t.active_comp_id().unwrap(); + t.execute("layer.newSolid", json!({"name": "Back", "color": "#00ff00"})).unwrap(); + let r = t.execute("essential.importTemplate", json!({"path": path})).unwrap(); + let tcomp = ItemId(r["comp"].as_u64().unwrap()); + let inst = r["layer"].as_u64().unwrap(); + assert_eq!(t.project.item(tcomp).unwrap().name, "Card"); + assert!(t.project.items.values().any(|i| i.is_folder() && i.name == "Lower Card")); + // Ids were remapped consistently: the template comp's controls point at its own layers. + let eg = t.project.comp(tcomp).unwrap().essential.clone().unwrap(); + for c in eg.flat() { + if let essential::EgKind::Property { .. } = c.kind { + assert!(essential::source_prop(&t.project, tcomp, c).is_some(), "{c:?}"); + } + } + let px = center(&t, show); + assert!(px[0] > 0.99 && px[1] < 0.01, "the card renders red over green: {px:?}"); + // Override on the imported instance, then save/load keeps it. + let color_ctl = eg.flat().iter().find(|c| c.name == "Card Color").unwrap().id; + t.execute("essential.set", json!({"layer": inst, "control": color_ctl, "value": "#ffff00"})).unwrap(); + let px = center(&t, show); + assert!(px[0] > 0.99 && px[1] > 0.99 && px[2] < 0.01, "yellow: {px:?}"); + let saved = effectcraft_project::Project::from_json(&t.project.to_json()).unwrap(); + let mut u = Session::default(); + u.replace_project(saved, None); + u.open_comp(show); + let px = center(&u, show); + assert!(px[0] > 0.99 && px[1] > 0.99 && px[2] < 0.01, "override survives save/load: {px:?}"); + // Importing again keeps both copies apart (fresh ids). + let r2 = t.execute("essential.importTemplate", json!({"path": path, "addToComp": false})).unwrap(); + assert_ne!(r2["comp"], r["comp"]); + assert!(r2["layer"].is_null()); + // Not a template. + std::fs::write(dir.join("bad.ectemplate"), b"nope").unwrap(); + assert!(t.execute("essential.importTemplate", json!({"path": dir.join("bad.ectemplate").to_string_lossy()})).is_err()); +} + +#[test] +fn media_replacement_swaps_footage_per_instance() { + use std::sync::Arc; + // Two "footage" items rendered by a fake source: item → solid colour. + struct Colors; + impl crate::render::FootageSource for Colors { + fn frame(&self, _: ItemId, f: &effectcraft_project::Footage, _: Tick) -> Option> { + let c = if f.path.contains("red") { [1.0, 0.0, 0.0, 1.0] } else { [0.0, 0.0, 1.0, 1.0] }; + Some(Arc::new(effectcraft_raster::Image::filled(16, 16, c))) + } + } + let mut s = Session { footage: Arc::new(Colors), ..Default::default() }; + let foot = |p: &str| effectcraft_project::Footage { + path: p.into(), + kind: effectcraft_project::FootageKind::Still, + width: 16, + height: 16, + has_video: true, + ..Default::default() + }; + let (red, blue) = s + .edit("x", None, |proj, _| { + let r = proj.add_item("red.png", Default::default(), None, effectcraft_project::ItemKind::Footage(foot("red.png"))); + let b = proj.add_item("blue.png", Default::default(), None, effectcraft_project::ItemKind::Footage(foot("blue.png"))); + Ok((r, b)) + }) + .unwrap(); + s.execute("comp.new", json!({"name": "Slot", "width": 16, "height": 16, "frameRate": 30, "duration": 1})).unwrap(); + let slot = s.active_comp_id().unwrap(); + let fl = s.execute("layer.addItem", json!({"item": red.0})).unwrap()["layer"].as_u64().unwrap(); + let m = s.execute("essential.addMedia", json!({"layer": fl})).unwrap()["control"].as_u64().unwrap(); + s.execute("comp.new", json!({"name": "Main", "width": 16, "height": 16, "frameRate": 30, "duration": 1})).unwrap(); + let main = s.active_comp_id().unwrap(); + let inst = s.execute("layer.addItem", json!({"item": slot.0})).unwrap()["layer"].as_u64().unwrap(); + assert!(center(&s, main)[0] > 0.99); + s.execute("essential.set", json!({"layer": inst, "control": m, "item": blue.0})).unwrap(); + let px = center(&s, main); + assert!(px[2] > 0.99 && px[0] < 0.01, "{px:?}"); + assert!(center(&s, slot)[0] > 0.99, "the source keeps its media"); + assert_eq!(s.execute("essential.instance", json!({"layer": inst})).unwrap()["controls"][0]["type"], "media"); + s.execute("essential.revert", json!({"layer": inst})).unwrap(); + assert!(center(&s, main)[0] > 0.99); +} + +/// Responsive Design — Time: stretching a precomp keeps its protected regions at their +/// original speed; the unprotected parts absorb the stretch. +#[test] +fn protected_regions_keep_speed_under_time_stretch() { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Inner", "width": 32, "height": 32, "frameRate": 30, "duration": 4})).unwrap(); + let inner = s.active_comp_id().unwrap(); + // Intro 0–1 s and outro 3–4 s. + s.execute("time.set", json!({"time": 1.0})).unwrap(); + s.execute("comp.responsiveTime", json!({"op": "intro"})).unwrap(); + s.execute("time.set", json!({"time": 3.0})).unwrap(); + s.execute("comp.responsiveTime", json!({"op": "outro"})).unwrap(); + let comp = s.project.comp(inner).unwrap().clone(); + assert_eq!(crate::render::eval::protected_regions(&comp), vec![(0.0, 1.0), (3.0, 4.0)]); + s.execute("comp.new", json!({"name": "Outer", "width": 32, "height": 32, "frameRate": 30, "duration": 10})).unwrap(); + let outer = s.active_comp_id().unwrap(); + let l = s.execute("layer.addItem", json!({"item": inner.0})).unwrap()["layer"].as_u64().unwrap(); + s.execute("layer.select", json!({"layers": [l]})).unwrap(); + s.execute("layer.timeStretch", json!({"percent": 200})).unwrap(); + let st = |s: &Session, t: f64| { + let c = s.project.comp(outer).unwrap(); + let layer = c.layer(LayerId(l)).unwrap(); + crate::render::EvalCtx::new(&s.project, outer, c, Tick::from_seconds_f64(t)).source_time(layer).seconds() + }; + // 8 s total: intro 1 s at speed 1, middle 2 s stretched to 6 s, outro 1 s at speed 1. + let near = |a: f64, b: f64| (a - b).abs() < 1e-6; + assert!(near(st(&s, 0.5), 0.5)); + assert!(near(st(&s, 1.0), 1.0)); + assert!(near(st(&s, 4.0), 2.0)); + assert!(near(st(&s, 7.0), 3.0)); + assert!(near(st(&s, 7.5), 3.5), "outro plays at full speed: {}", st(&s, 7.5)); + // Protected regions play at the original speed: one comp second = one source second. + assert!(near(st(&s, 0.8) - st(&s, 0.3), 0.5)); + assert!(near(st(&s, 7.9) - st(&s, 7.4), 0.5)); + assert!(near(st(&s, 5.0) - st(&s, 4.0), 1.0 / 3.0), "the middle is slowed down 3×"); + // Compressing to 75 %: the middle plays faster. + s.execute("layer.timeStretch", json!({"percent": 75})).unwrap(); + assert!(near(st(&s, 0.5), 0.5)); + assert!(near(st(&s, 1.0 + 0.5), 1.0 + 0.5 / 0.5), "{}", st(&s, 1.5)); + // Without protected regions the plain stretch applies. + s.execute("comp.open", json!({"comp": inner.0})).ok(); + let mut p = (*s.project).clone(); + p.comp_mut(inner).unwrap().markers.clear(); + s.project = std::sync::Arc::new(p); + assert!(near(st(&s, 1.5), 2.0)); +} diff --git a/crates/engine/src/tests_model3d.rs b/crates/engine/src/tests_model3d.rs index 51f9173d..02e4e52a 100644 --- a/crates/engine/src/tests_model3d.rs +++ b/crates/engine/src/tests_model3d.rs @@ -52,6 +52,7 @@ impl Importer for ModelImporter { missing: false, sequence: vec![], color_profile: None, + ..Default::default() }) } } diff --git a/crates/engine/src/tests_stubs.rs b/crates/engine/src/tests_stubs.rs index 141858ca..5cb70246 100644 --- a/crates/engine/src/tests_stubs.rs +++ b/crates/engine/src/tests_stubs.rs @@ -257,6 +257,7 @@ impl Importer for MockImporter { missing: false, sequence: vec![], color_profile: None, + ..Default::default() }) } } diff --git a/crates/engine/src/tests_track.rs b/crates/engine/src/tests_track.rs index adcf7df0..8c7037b6 100644 --- a/crates/engine/src/tests_track.rs +++ b/crates/engine/src/tests_track.rs @@ -101,6 +101,7 @@ pub(crate) fn setup(make: impl Fn(u32) -> Image + Send + Sync + 'static) -> (Ses missing: false, sequence: vec![], color_profile: None, + ..Default::default() }; let item = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(footage)); let comp = p.comp(cid).unwrap().clone(); @@ -410,6 +411,7 @@ fn analyze_1080p_single_point_speed() { missing: false, sequence: vec![], color_profile: None, + ..Default::default() }; let item = p.add_item("hd", Label::Aqua, None, ItemKind::Footage(footage)); let comp = p.comp(cid).unwrap().clone(); diff --git a/crates/gpu/src/tests.rs b/crates/gpu/src/tests.rs index ca2fb1b0..698b3fc5 100644 --- a/crates/gpu/src/tests.rs +++ b/crates/gpu/src/tests.rs @@ -100,6 +100,7 @@ impl Scene { missing: false, sequence: vec![], color_profile: None, + ..Default::default() }; let fid = self.p.add_item("Pattern", Label::Lavender, None, ItemKind::Footage(f)); build::layer(&mut self.p, &self.comp, "Pattern", LayerSource::Footage { item: fid }, (w, h), None) diff --git a/crates/lottie/src/import.rs b/crates/lottie/src/import.rs index c87dc187..0fb1bfeb 100644 --- a/crates/lottie/src/import.rs +++ b/crates/lottie/src/import.rs @@ -171,6 +171,7 @@ pub fn import( missing: false, sequence: vec![], color_profile: None, + ..Default::default() }; let item = project.add_item(&nm, Label::Lavender, Some(folder), ItemKind::Footage(foot)); cx.items.push(item); diff --git a/crates/lottie/src/lib.rs b/crates/lottie/src/lib.rs index 55c2d01c..198ed7cf 100644 --- a/crates/lottie/src/lib.rs +++ b/crates/lottie/src/lib.rs @@ -24,7 +24,7 @@ mod export; mod import; mod shapes; mod text; -mod zip; +pub mod zip; use effectcraft_color::BlendMode; use effectcraft_time::{FrameRate, TICKS_PER_SECOND, Tick}; diff --git a/crates/lottie/src/tests.rs b/crates/lottie/src/tests.rs index 5140133c..6bb37d83 100644 --- a/crates/lottie/src/tests.rs +++ b/crates/lottie/src/tests.rs @@ -592,6 +592,7 @@ fn images_embed_as_base64_and_come_back() { missing: false, sequence: vec![], color_profile: None, + ..Default::default() }), ); let c = comp(&p, cid); diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index b6632fba..478ddc09 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -386,7 +386,7 @@ impl Inner { let i = Self::frame_index(footage.frame_rate, t, footage.sequence.len() as i64, footage.loop_count); Loc { key: key(&footage.sequence[i as usize], 0), media_t: None } } - FootageKind::Still | FootageKind::Sequence | FootageKind::Model => Loc { key: key(&footage.path, 0), media_t: None }, + FootageKind::Still | FootageKind::Sequence | FootageKind::Model | FootageKind::Data => Loc { key: key(&footage.path, 0), media_t: None }, FootageKind::Video | FootageKind::Audio => { let rate = footage.frame_rate; let n = Self::frame_count(rate, footage.duration); diff --git a/crates/media/src/probe.rs b/crates/media/src/probe.rs index e97544db..9f1c13c3 100644 --- a/crates/media/src/probe.rs +++ b/crates/media/src/probe.rs @@ -64,6 +64,7 @@ pub fn probe_model(path: &str, bytes: &[u8], resolve: &dyn Fn(&str) -> Option`. Changing a child marks it *overridden* for that instance only; +//! [`sync_project`] keeps the groups in step with the controls (and refreshes the values of +//! controls that are not overridden), [`with_overrides`] builds the project an instance renders. + +use serde::{Deserialize, Serialize}; + +use crate::props::{GroupKind, Node, ParamUi, PropGroup, Property, Uid}; +use crate::{ItemId, ItemKind, Layer, LayerId, LayerSource, Project, Value}; + +/// Match id of the Essential Properties group on precomp layers. +pub const GROUP: &str = "essential"; + +/// A composition's Essential Graphics definition. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct EssentialGraphics { + /// Template name (Essential Graphics panel ▸ name field; the exported template's title). + #[serde(default)] + pub name: String, + #[serde(default)] + pub controls: Vec, +} + +/// One entry of the Essential Graphics panel. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct EgControl { + pub id: u64, + /// Display name (renamable; defaults to the property's name). + pub name: String, + pub kind: EgKind, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum EgKind { + /// A property of a layer of the comp. + Property { layer: LayerId, prop: Uid }, + /// Media Replacement: the footage of a footage layer can be swapped per instance. + Media { layer: LayerId }, + /// A text note shown in the panel. + Comment { text: String }, + /// A named, collapsible group of controls. + Group { + #[serde(default)] + children: Vec, + }, +} + +/// The kind of control a property becomes. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ControlType { + Text, + Color, + Slider, + Checkbox, + Point, + Angle, + Dropdown, + Media, +} + +impl ControlType { + pub fn label(self) -> &'static str { + match self { + ControlType::Text => "Text", + ControlType::Color => "Color", + ControlType::Slider => "Slider", + ControlType::Checkbox => "Checkbox", + ControlType::Point => "Point", + ControlType::Angle => "Angle", + ControlType::Dropdown => "Dropdown", + ControlType::Media => "Media", + } + } +} + +/// The control a property is shown as, `None` when Essential Graphics can't expose it (paths, +/// gradients, layer pickers, hidden data). +pub fn control_type(p: &Property) -> Option { + if matches!(p.ui, ParamUi::Hidden) { + return None; + } + Some(match (&p.value, &p.ui) { + (Value::Text(_) | Value::Str(_), _) => ControlType::Text, + (Value::Color(_), _) => ControlType::Color, + (Value::Bool(_), _) | (_, ParamUi::Checkbox) => ControlType::Checkbox, + (Value::Enum(_), _) | (_, ParamUi::Popup { .. }) => ControlType::Dropdown, + (_, ParamUi::Angle) => ControlType::Angle, + (Value::Vec2(_) | Value::Vec3(_), _) => ControlType::Point, + (Value::Scalar(_), _) => ControlType::Slider, + _ => return None, + }) +} + +impl EssentialGraphics { + /// Every control, depth first (groups before their children). + pub fn flat(&self) -> Vec<&EgControl> { + fn go<'a>(v: &'a [EgControl], out: &mut Vec<&'a EgControl>) { + for c in v { + out.push(c); + if let EgKind::Group { children } = &c.kind { + go(children, out); + } + } + } + let mut out = vec![]; + go(&self.controls, &mut out); + out + } + pub fn find(&self, id: u64) -> Option<&EgControl> { + self.flat().into_iter().find(|c| c.id == id) + } + pub fn find_mut(&mut self, id: u64) -> Option<&mut EgControl> { + fn go(v: &mut [EgControl], id: u64) -> Option<&mut EgControl> { + for c in v { + if c.id == id { + return Some(c); + } + if let EgKind::Group { children } = &mut c.kind + && let Some(x) = go(children, id) + { + return Some(x); + } + } + None + } + go(&mut self.controls, id) + } + /// Remove a control (anywhere) and return it. + pub fn remove(&mut self, id: u64) -> Option { + fn go(v: &mut Vec, id: u64) -> Option { + if let Some(i) = v.iter().position(|c| c.id == id) { + return Some(v.remove(i)); + } + v.iter_mut().find_map(|c| if let EgKind::Group { children } = &mut c.kind { go(children, id) } else { None }) + } + go(&mut self.controls, id) + } + /// The list a control is inserted into: a group's children or the top level. + pub fn list_mut(&mut self, group: Option) -> Option<&mut Vec> { + match group { + None => Some(&mut self.controls), + Some(g) => match &mut self.find_mut(g)?.kind { + EgKind::Group { children } => Some(children), + _ => None, + }, + } + } + /// The control exposing property `prop` of `layer`, if any. + pub fn control_for(&self, layer: LayerId, prop: Uid) -> Option<&EgControl> { + self.flat().into_iter().find(|c| matches!(c.kind, EgKind::Property { layer: l, prop: p } if l == layer && p == prop)) + } + /// Controls an instance can override (properties and media; not comments or groups). + pub fn has_instance_controls(&self) -> bool { + self.flat().iter().any(|c| matches!(c.kind, EgKind::Property { .. } | EgKind::Media { .. })) + } +} + +/// Match id of an instance property / group for a control. +pub fn match_id(control: u64) -> String { + format!("eg{control}") +} + +/// Control id of an Essential Properties child (`eg42` → 42). +pub fn control_of(match_id: &str) -> Option { + match_id.strip_prefix("eg")?.parse().ok() +} + +/// The Essential Properties group of a layer. +pub fn group(layer: &Layer) -> Option<&PropGroup> { + layer.props.sub(GROUP) +} + +/// Uids of the overridden master properties of a layer. +pub fn overridden(layer: &Layer) -> Vec { + match group(layer).map(|g| &g.kind) { + Some(GroupKind::Essential { overridden }) => overridden.clone(), + _ => vec![], + } +} + +/// The source property a control exposes, in `project`. +pub fn source_prop<'a>(project: &'a Project, comp: ItemId, c: &EgControl) -> Option<(&'a Layer, &'a Property)> { + let EgKind::Property { layer, prop } = c.kind else { return None }; + let l = project.comp(comp)?.layer(layer)?; + Some((l, l.props.find(prop)?)) +} + +/// The footage item a media control shows by default (the layer's own source). +pub fn source_media(project: &Project, comp: ItemId, c: &EgControl) -> Option { + let EgKind::Media { layer } = c.kind else { return None }; + match project.comp(comp)?.layer(layer)?.source { + LayerSource::Footage { item } => Some(item), + _ => None, + } +} + +/// Build the Essential Properties group a layer nesting `comp` should have, reusing uids and +/// override values from `old`. Uids for new nodes come from `next`. +fn build_group(project: &Project, comp: ItemId, eg: &EssentialGraphics, old: Option<&PropGroup>, next: &mut u64) -> PropGroup { + let over = match old.map(|g| &g.kind) { + Some(GroupKind::Essential { overridden }) => overridden.clone(), + _ => vec![], + }; + let mut alloc = || { + let v = *next; + *next += 1; + v + }; + fn find_old<'a>(old: Option<&'a PropGroup>, m: &str) -> Option<&'a Node> { + let g = old?; + g.children.iter().find(|c| c.match_id() == m).or_else(|| g.groups().find_map(|sub| find_old(Some(sub), m))) + } + fn build( + project: &Project, + comp: ItemId, + list: &[EgControl], + old: Option<&PropGroup>, + over: &[Uid], + alloc: &mut dyn FnMut() -> u64, + keep: &mut Vec, + ) -> Vec { + let mut out = vec![]; + for c in list { + let m = match_id(c.id); + let prev = find_old(old, &m); + match &c.kind { + EgKind::Comment { .. } => {} + EgKind::Group { children } => { + let uid = prev.map(Node::uid).unwrap_or_else(&mut *alloc); + let mut g = PropGroup::new(uid, &m, &c.name); + g.children = build(project, comp, children, old, over, alloc, keep); + out.push(Node::Group(g)); + } + EgKind::Property { .. } => { + let Some((_, src)) = source_prop(project, comp, c) else { continue }; + let prev = prev.and_then(Node::as_prop); + let uid = prev.map(|p| p.uid).unwrap_or_else(&mut *alloc); + let mut p = Property { uid, match_id: m, name: c.name.clone(), keys: vec![], expr: None, ..src.clone() }; + // Instance properties follow the source's look but are always editable. + p.static_only = false; + if let Some(prev) = prev.filter(|pp| over.contains(&pp.uid)) { + p.value = prev.value.clone(); + p.keys = prev.keys.clone(); + p.expr = prev.expr.clone(); + keep.push(uid); + } else { + p.value = src.value_at(effectcraft_time::Tick::ZERO); + } + out.push(Node::Prop(p)); + } + EgKind::Media { .. } => { + let Some(item) = source_media(project, comp, c) else { continue }; + let prev = prev.and_then(Node::as_prop); + let uid = prev.map(|p| p.uid).unwrap_or_else(&mut *alloc); + let mut p = Property::new(uid, &m, &c.name, Value::Scalar(item.0 as f64)).with_ui(ParamUi::Hidden); + p.static_only = true; + if let Some(prev) = prev.filter(|pp| over.contains(&pp.uid)) { + p.value = prev.value.clone(); + keep.push(uid); + } + out.push(Node::Prop(p)); + } + } + } + out + } + let uid = old.map(|g| g.uid).unwrap_or_else(&mut alloc); + let mut keep = vec![]; + let children = build(project, comp, &eg.controls, old, &over, &mut alloc, &mut keep); + PropGroup { uid, match_id: GROUP.into(), name: "Essential Properties".into(), kind: GroupKind::Essential { overridden: keep }, enabled: true, children } +} + +/// Master properties the user changed in this edit (`before` → `after`) become overridden. +fn mark_overrides(before: &Project, after: &mut Project) { + let ids: Vec = after.comps().map(|(i, _)| *i).collect(); + for cid in ids { + let Some(comp) = after.comp(cid) else { continue }; + let Some(old_comp) = before.comp(cid) else { continue }; + let mut changes: Vec<(LayerId, Vec)> = vec![]; + for l in &comp.layers { + let Some(g) = group(l) else { continue }; + let Some(ol) = old_comp.layer(l.id) else { continue }; + let Some(og) = group(ol) else { continue }; + // Properties overridden before this edit stay as the edit left them (Revert and + // Push to Comp clear overrides by editing them). + let mut over = overridden(l); + over.extend(overridden(ol)); + let mut add = vec![]; + g.walk("", &mut |_, p| { + if over.contains(&p.uid) { + return; + } + if let Some(op) = og.find(p.uid) + && (op.value != p.value || op.keys != p.keys || op.expr != p.expr) + { + add.push(p.uid); + } + }); + if !add.is_empty() { + changes.push((l.id, add)); + } + } + if changes.is_empty() { + continue; + } + let Some(c) = after.comp_mut(cid) else { continue }; + for (lid, add) in changes { + if let Some(g) = c.layer_mut(lid).and_then(|l| l.props.sub_mut(GROUP)) + && let GroupKind::Essential { overridden } = &mut g.kind + { + overridden.extend(add); + } + } + } +} + +/// After an edit: mark changed master properties as overridden, then add, update or remove the +/// Essential Properties groups of every precomp layer so they match their comp's controls. +pub fn sync_project(before: &Project, after: &mut Project) { + mark_overrides(before, after); + let ids: Vec = after.comps().map(|(i, _)| *i).collect(); + let mut next = after.next_id; + for cid in ids { + let Some(comp) = after.comp(cid) else { continue }; + let mut updates: Vec<(usize, Option)> = vec![]; + for (i, l) in comp.layers.iter().enumerate() { + let want = match l.source { + LayerSource::Comp { item } => after.comp(item).and_then(|c| c.essential.as_ref()).filter(|e| e.has_instance_controls()).map(|e| (item, e)), + _ => None, + }; + let old = group(l); + match want { + Some((item, eg)) => { + let g = build_group(after, item, eg, old, &mut next); + if old != Some(&g) { + updates.push((i, Some(g))); + } + } + None if old.is_some() => updates.push((i, None)), + None => {} + } + } + if updates.is_empty() { + continue; + } + let Some(c) = after.comp_mut(cid) else { continue }; + for (i, g) in updates { + let l = &mut c.layers[i]; + let pos = l.props.children.iter().position(|n| n.match_id() == GROUP); + match (pos, g) { + (Some(p), Some(g)) => l.props.children[p] = Node::Group(g), + (None, Some(g)) => l.props.children.insert(0, Node::Group(g)), + (Some(p), None) => { + l.props.children.remove(p); + } + (None, None) => {} + } + } + } + after.next_id = after.next_id.max(next); +} + +/// One instance override: the control and the value it takes in this instance. +#[derive(Clone, Debug, PartialEq)] +pub struct Override { + pub control: u64, + pub value: Value, +} + +/// The project as an instance of `comp` renders it: each overridden control's source property +/// set to the instance value (keyframes dropped; its expression, if any, still applies) and +/// replaced media swapped in. `None` when nothing applies. +pub fn with_overrides(project: &Project, comp: ItemId, overrides: &[Override]) -> Option { + let eg = project.comp(comp)?.essential.clone()?; + let mut p = project.clone(); + let c = p.comp_mut(comp)?; + let mut any = false; + for o in overrides { + let Some(ctl) = eg.find(o.control) else { continue }; + match ctl.kind { + EgKind::Property { layer, prop } => { + if let Some(pr) = c.layer_mut(layer).and_then(|l| l.props.find_mut(prop)) { + pr.keys.clear(); + pr.value = o.value.clone(); + any = true; + } + } + EgKind::Media { layer } => { + let item = ItemId(o.value.as_f64().max(0.0) as u64); + // Replacement media must be footage. + if !matches!(project.item(item).map(|i| &i.kind), Some(ItemKind::Footage(_))) { + continue; + } + if let Some(l) = c.layer_mut(layer) { + l.source = LayerSource::Footage { item }; + any = true; + } + } + _ => {} + } + } + any.then_some(p) +} + +// ------------------------------------------------------------------------------ id remapping + +/// Shift every id (items, layers, property uids, Essential Graphics controls) of `p` by +/// `offset`, keeping all references consistent: used when merging a template project into +/// another project. +pub fn offset_ids(p: &mut Project, offset: u64) { + let items = std::mem::take(&mut p.items); + for (_, mut it) in items { + it.id = ItemId(it.id.0 + offset); + it.parent = it.parent.map(|f| ItemId(f.0 + offset)); + if let ItemKind::Comp(c) = &mut it.kind { + let c = std::sync::Arc::make_mut(c); + for l in &mut c.layers { + offset_layer(l, offset); + } + if let Some(eg) = &mut c.essential { + fn go(v: &mut [EgControl], o: u64) { + for c in v { + c.id += o; + match &mut c.kind { + EgKind::Property { layer, prop } => { + *layer = LayerId(layer.0 + o); + *prop += o; + } + EgKind::Media { layer } => *layer = LayerId(layer.0 + o), + EgKind::Group { children } => go(children, o), + EgKind::Comment { .. } => {} + } + } + } + go(&mut eg.controls, offset); + } + } + p.items.insert(it.id, it); + } + p.render_queue.clear(); + p.next_id += offset; +} + +fn offset_layer(l: &mut Layer, o: u64) { + l.id = LayerId(l.id.0 + o); + l.parent = l.parent.map(|x| LayerId(x.0 + o)); + if let Some(tm) = &mut l.track_matte { + tm.layer = LayerId(tm.layer.0 + o); + } + match &mut l.source { + LayerSource::Footage { item } | LayerSource::Comp { item } | LayerSource::Solid { item } | LayerSource::Model { item } => *item = ItemId(item.0 + o), + _ => {} + } + fn group(g: &mut PropGroup, o: u64, essential_parent: bool) { + g.uid += o; + if let GroupKind::Essential { overridden } = &mut g.kind { + for u in overridden.iter_mut() { + *u += o; + } + } + let is_essential = matches!(g.kind, GroupKind::Essential { .. }) || essential_parent; + for c in &mut g.children { + match c { + Node::Prop(p) => { + p.uid += o; + let fix = |v: &mut Value| { + if let Value::Layer(Some(id)) = v { + *id += o; + } + }; + fix(&mut p.value); + for k in &mut p.keys { + fix(&mut k.value); + } + if is_essential && control_of(&p.match_id).is_some() { + p.match_id = match_id(control_of(&p.match_id).unwrap_or(0) + o); + // Media replacement values are item ids. + if matches!(p.ui, ParamUi::Hidden) + && let Value::Scalar(x) = &mut p.value + { + *x += o as f64; + } + } + } + Node::Group(sub) => { + if is_essential && let Some(id) = control_of(&sub.match_id) { + sub.match_id = match_id(id + o); + } + group(sub, o, is_essential); + } + } + } + } + group(&mut l.props, o, false); +} + +/// The items `comp` needs (itself, nested comps, footage, solids), transitively. +pub fn dependencies(project: &Project, comp: ItemId) -> Vec { + let mut out = vec![]; + let mut stack = vec![comp]; + while let Some(id) = stack.pop() { + if out.contains(&id) { + continue; + } + let Some(it) = project.item(id) else { continue }; + out.push(id); + if let ItemKind::Comp(c) = &it.kind { + for l in &c.layers { + if let Some(i) = l.source.item() { + stack.push(i); + } + // Media replacement candidates referenced by instances inside. + if let Some(g) = group(l) { + g.walk("", &mut |_, p| { + if matches!(p.ui, ParamUi::Hidden) + && control_of(&p.match_id).is_some() + && let Value::Scalar(x) = p.value + { + stack.push(ItemId(x as u64)); + } + }); + } + } + } + } + out.sort(); + out +} diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 4a00c413..0e9c6947 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -5,6 +5,7 @@ //! change. pub mod build; +pub mod essential; pub mod props; pub mod render_queue; pub mod styles; @@ -214,6 +215,9 @@ pub struct Comp { /// Viewer guides (View ▸ Add Guide…), in comp pixels. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub guides: Vec, + /// Essential Graphics: the controls this comp exposes to instances and templates. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub essential: Option, } /// A viewer guide line: vertical guides sit at an x position, horizontal ones at a y position. @@ -251,6 +255,7 @@ impl Comp { poster_time: Tick::ZERO, guides: vec![], global_light: styles::GlobalLight::default(), + essential: None, } } pub fn layer(&self, id: LayerId) -> Option<&Layer> { @@ -590,6 +595,38 @@ pub enum FootageKind { Audio, /// A 3D model (glTF 2.0 / OBJ), placed as a model layer. Model, + /// A data file (JSON, CSV, TSV) for data-driven animation: read by expressions through + /// `footage(name).sourceData` / `sourceText` / `dataValue`; it has no pixels. + Data, +} + +/// Interpret Footage ▸ Fields and Pulldown ▸ Separate Fields. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum FieldOrder { + /// Progressive frames. + #[default] + Off, + /// Interlaced, upper (odd) field first: each field becomes a frame at twice the rate. + UpperFirst, + LowerFirst, +} + +impl FieldOrder { + pub fn label(self) -> &'static str { + match self { + FieldOrder::Off => "Off", + FieldOrder::UpperFirst => "Upper Field First", + FieldOrder::LowerFirst => "Lower Field First", + } + } + pub fn parse(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '_', '-'], "").as_str() { + "off" | "none" | "progressive" => Some(FieldOrder::Off), + "upper" | "upperfirst" | "upperfieldfirst" => Some(FieldOrder::UpperFirst), + "lower" | "lowerfirst" | "lowerfieldfirst" => Some(FieldOrder::LowerFirst), + _ => None, + } + } } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -629,6 +666,77 @@ pub struct Footage { /// Colour profile (from the file's metadata, or Interpret Footage). `None` = sRGB. #[serde(default, skip_serializing_if = "Option::is_none")] pub color_profile: Option, + /// Interpret Footage ▸ Fields and Pulldown ▸ Separate Fields. + #[serde(default, skip_serializing_if = "is_default")] + pub fields: FieldOrder, + /// Interpret Footage ▸ Alpha ▸ Invert Alpha. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub invert_alpha: bool, + /// Interpret Footage ▸ Color ▸ Interpret As Linear Light: the file's values are linear. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub linear_light: bool, + /// Data footage ([`FootageKind::Data`]): the file's text (JSON, CSV or TSV), kept in the + /// project so expressions read it everywhere (and on the web). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub data: Option, +} + +fn is_default(v: &T) -> bool { + *v == T::default() +} + +impl Default for Footage { + fn default() -> Self { + Footage { + path: String::new(), + kind: FootageKind::Still, + width: 0, + height: 0, + pixel_aspect: 1.0, + frame_rate: FrameRate::FPS_30, + native_rate: None, + duration: Tick::ZERO, + has_video: false, + has_audio: false, + alpha: AlphaMode::Straight, + premul_color: [0.0; 3], + loop_count: 1, + codec: String::new(), + missing: false, + sequence: vec![], + color_profile: None, + fields: FieldOrder::Off, + invert_alpha: false, + linear_light: false, + data: None, + } + } +} + +impl Footage { + /// The data format of data footage: `json`, `csv` or `tsv` (from the file extension). + pub fn data_format(&self) -> Option<&'static str> { + if self.kind != FootageKind::Data { + return None; + } + let ext = std::path::Path::new(&self.path).extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); + Some(match ext.as_str() { + "csv" => "csv", + "tsv" | "tab" => "tsv", + _ => "json", + }) + } +} + +/// A proxy: a stand-in file used instead of a footage item or composition (File ▸ Set Proxy / +/// Create Proxy), when its Use Proxy switch is on and the render's Proxy Use allows it. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Proxy { + /// The proxy's footage (path, size, rate…; File ▸ Interpret Footage ▸ Proxy edits it). + pub footage: Footage, + /// Use Proxy (the Project panel's proxy indicator: filled = in use, hollow = set but off). + #[serde(default = "yes")] + pub enabled: bool, } fn one() -> u32 { @@ -663,6 +771,9 @@ pub struct Item { #[serde(default)] pub parent: Option, pub kind: ItemKind, + /// The item's proxy (footage and compositions). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub proxy: Option>, } impl Item { @@ -676,6 +787,7 @@ impl Item { FootageKind::Sequence => "Image Sequence", FootageKind::Audio => "Audio", FootageKind::Model => "3D Model", + FootageKind::Data => "Data", }, ItemKind::Solid(_) => "Solid", } @@ -753,7 +865,7 @@ impl Project { } pub fn add_item(&mut self, name: &str, label: Label, parent: Option, kind: ItemKind) -> ItemId { let id = ItemId(self.alloc()); - self.items.insert(id, Item { id, name: name.into(), label, comment: String::new(), parent, kind }); + self.items.insert(id, Item { id, name: name.into(), label, comment: String::new(), parent, kind, proxy: None }); id } /// Children of a folder (None = root), folders first then by name. diff --git a/crates/project/src/props.rs b/crates/project/src/props.rs index c1a3e3b9..c81a96b0 100644 --- a/crates/project/src/props.rs +++ b/crates/project/src/props.rs @@ -181,6 +181,13 @@ pub enum GroupKind { }, /// A motion tracker (Motion Trackers ▸ Tracker n) with its Tracker panel settings. Tracker { settings: Box }, + /// The Essential Properties of a precomp layer (master properties): one child per control + /// of the nested comp's Essential Graphics; `overridden` lists the children (uids) whose + /// values this instance overrides. + Essential { + #[serde(default)] + overridden: Vec, + }, } /// Mask motion blur (sub-frame samples of an animated mask path). diff --git a/crates/render/src/audio.rs b/crates/render/src/audio.rs index 4884411f..98bb9666 100644 --- a/crates/render/src/audio.rs +++ b/crates/render/src/audio.rs @@ -330,6 +330,7 @@ mod tests { missing: false, sequence: vec![], color_profile: None, + ..Default::default() }; let fid = p.add_item("x.wav", effectcraft_color::Label::SeaFoam, None, ItemKind::Footage(f)); let comp = Comp::new(64, 64, FrameRate::new(25, 1), Tick::from_seconds_f64(2.0)); diff --git a/crates/render/src/eval.rs b/crates/render/src/eval.rs index eefe77c3..7bf67774 100644 --- a/crates/render/src/eval.rs +++ b/crates/render/src/eval.rs @@ -100,6 +100,15 @@ impl<'a> EvalCtx<'a> { if let Some(tr) = layer.props.get("timeRemap") { return Tick::from_seconds_f64(self.value(layer, tr).as_f64()); } + // Responsive Design — Time: a time-stretched precomp keeps its protected regions at + // their original speed. + if let LayerSource::Comp { item } = layer.source + && (layer.stretch - 100.0).abs() > 1e-9 + && let Some(nc) = self.project.comp(item) + && let Some(t) = responsive_source_time(nc, layer.stretch / 100.0, (self.time - layer.start_time).seconds()) + { + return t; + } layer.layer_time(self.time) } pub fn layer(&self, id: LayerId) -> Option<&'a Layer> { @@ -185,6 +194,73 @@ impl<'a> EvalCtx<'a> { } } +/// The protected regions of a comp (Responsive Design — Time markers), merged, sorted and +/// clamped to the comp, in seconds. +pub fn protected_regions(comp: &Comp) -> Vec<(f64, f64)> { + let d = comp.duration.seconds(); + let mut v: Vec<(f64, f64)> = comp + .markers + .iter() + .filter(|m| m.protected && m.duration > Tick::ZERO) + .map(|m| (m.time.seconds().clamp(0.0, d), (m.time + m.duration).seconds().clamp(0.0, d))) + .filter(|(a, b)| b > a) + .collect(); + v.sort_by(|a, b| a.0.total_cmp(&b.0)); + let mut out: Vec<(f64, f64)> = vec![]; + for (a, b) in v { + match out.last_mut() { + Some(last) if a <= last.1 => last.1 = last.1.max(b), + _ => out.push((a, b)), + } + } + out +} + +/// Source time of a precomp time-stretched by `stretch` (1 = 100%), `elapsed` comp seconds +/// after the layer's start, with Responsive Design — Time: protected regions play at their +/// original speed and the unprotected parts absorb the whole stretch. `None` when the comp has +/// no protected regions or the stretch can't be absorbed (reversed, or shorter than the +/// protected regions), so the plain stretch applies. +pub fn responsive_source_time(comp: &Comp, stretch: f64, elapsed: f64) -> Option { + let regions = protected_regions(comp); + if regions.is_empty() || stretch <= 0.0 { + return None; + } + let d = comp.duration.seconds(); + let lp: f64 = regions.iter().map(|(a, b)| b - a).sum(); + let lu = d - lp; + let k = if lu > 1e-9 { (d * stretch - lp) / lu } else { 1.0 }; + if k <= 1e-9 { + return None; + } + if elapsed < 0.0 { + return Some(Tick::from_seconds_f64(elapsed / k)); + } + // Segments in source order: (start, end, protected). + let mut segs = vec![]; + let mut at = 0.0; + for (a, b) in ®ions { + if *a > at { + segs.push((at, *a, false)); + } + segs.push((*a, *b, true)); + at = *b; + } + if d > at { + segs.push((at, d, false)); + } + let mut pos = 0.0; + for (a, b, prot) in segs { + let speed = if prot { 1.0 } else { k }; + let len = (b - a) * speed; + if elapsed < pos + len { + return Some(Tick::from_seconds_f64(a + (elapsed - pos) / speed)); + } + pos += len; + } + Some(Tick::from_seconds_f64(d + (elapsed - pos) / k)) +} + /// Size of a layer's source in layer pixels. pub fn source_size(project: &Project, layer: &Layer) -> (u32, u32) { match &layer.source { diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 64e2684b..5e7425bb 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -154,6 +154,22 @@ impl EffectHost for FxHost<'_, '_, '_> { } } +/// The Essential Properties overrides of a precomp layer, evaluated at the context time. +pub fn essential_overrides(ctx: &EvalCtx, layer: &Layer) -> Vec { + use effectcraft_project::essential; + let over = essential::overridden(layer); + let Some(g) = essential::group(layer).filter(|_| !over.is_empty()) else { return vec![] }; + let mut out = vec![]; + g.walk("", &mut |_, p| { + if over.contains(&p.uid) + && let Some(control) = essential::control_of(&p.match_id) + { + out.push(essential::Override { control, value: ctx.value(layer, p) }); + } + }); + out +} + /// No footage available (renders footage layers as transparent). pub struct NoFootage; impl FootageSource for NoFootage { @@ -332,6 +348,10 @@ impl<'a> Renderer<'a> { if styles::active(ctx, layer) { return None; } + // Instance overrides (Essential Properties) need the nested comp rendered on its own. + if !effectcraft_project::essential::overridden(layer).is_empty() { + return None; + } Some(*item) } @@ -655,7 +675,15 @@ impl<'a> Renderer<'a> { if self.project.comp_contains(*item, ctx.comp_id) { return None; } - let sub = self.nested(); + // Essential Properties overrides render the nested comp with this instance's + // values. + let ov = essential_overrides(ctx, layer); + let tmp = if ov.is_empty() { None } else { effectcraft_project::essential::with_overrides(self.project, *item, &ov) }; + let base = self.nested(); + let sub = match &tmp { + Some(p) => Renderer { project: p, ..base }, + None => base, + }; let lt = ctx.source_time(layer); // Frame blending between the nested comp's frames (time-stretched/remapped). let mode = frame_blend_mode(ctx, layer); diff --git a/crates/render/src/tests_audio_fx.rs b/crates/render/src/tests_audio_fx.rs index a249955f..4cb4e5e7 100644 --- a/crates/render/src/tests_audio_fx.rs +++ b/crates/render/src/tests_audio_fx.rs @@ -51,6 +51,7 @@ fn audio_footage() -> Footage { missing: false, sequence: vec![], color_profile: None, + ..Default::default() } } diff --git a/crates/render/src/tests_color.rs b/crates/render/src/tests_color.rs index 87f9dd1a..0430a72b 100644 --- a/crates/render/src/tests_color.rs +++ b/crates/render/src/tests_color.rs @@ -172,6 +172,7 @@ fn footage(profile: Option) -> Footage { missing: false, sequence: vec![], color_profile: profile, + ..Default::default() } } diff --git a/crates/render/src/tests_frame_blend.rs b/crates/render/src/tests_frame_blend.rs index c172b81c..6cfbdb11 100644 --- a/crates/render/src/tests_frame_blend.rs +++ b/crates/render/src/tests_frame_blend.rs @@ -80,6 +80,7 @@ fn setup(mode: FrameBlend, size: (u32, u32)) -> (Project, ItemId) { missing: false, sequence: vec![], color_profile: None, + ..Default::default() }; let fid = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(f)); let mut l = build::layer(&mut p, &comp, "clip", LayerSource::Footage { item: fid }, size, None); From 6d0d7de92220e705f5c1150d07ec575ddb8b1b2b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:58:04 -0400 Subject: [PATCH 06/13] ANM-5: sampleImage, data footage (JSON/CSV/TSV) with footage().sourceData, thisProject, error list, Expression Language menu sampleImage renders the sampled layer through the evaluating renderer's footage source (EvalCtx::footage), cached per thread and frame. File > Import reads .json/.csv/.tsv as data footage kept in the project. expr.errors collects failing expressions for the error bar; expr.languageMenu lists the categorised Expression Language menu. Marker keys expose protectedRegion and cue points; colorDepth, timeToFeetAndFrames, comp.activeCamera. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/commands/expr_tools.rs | 375 +++++++++++++++++++ crates/engine/src/commands/file.rs | 32 +- crates/engine/src/commands/layer.rs | 6 +- crates/engine/src/commands/layer_time.rs | 2 +- crates/engine/src/commands/mod.rs | 2 + crates/engine/src/commands/prop.rs | 3 +- crates/engine/src/commands/puppet.rs | 2 +- crates/engine/src/commands/three_d.rs | 4 +- crates/engine/src/tracking.rs | 2 +- crates/expr/src/host.rs | 130 ++++++- crates/expr/src/lib.rs | 3 +- crates/expr/src/prelude.js | 100 ++++- crates/expr/src/runtime.rs | 13 + crates/expr/src/sample.rs | 119 ++++++ crates/expr/src/tests.rs | 78 +++- crates/gpu/src/tests_adv3d.rs | 2 +- crates/host/src/lib.rs | 78 +++- crates/render/src/eval.rs | 5 +- crates/render/src/lib.rs | 2 +- crates/render/src/three_d/adv/tests.rs | 2 +- crates/ui-egui/src/panels/effect_controls.rs | 2 +- crates/ui-egui/src/panels/layer_panel.rs | 2 +- crates/ui-egui/src/panels/properties.rs | 2 +- crates/ui-egui/src/panels/text_panels.rs | 4 +- crates/ui-egui/src/panels/timeline.rs | 2 +- crates/ui-egui/src/panels/viewer.rs | 4 +- 26 files changed, 951 insertions(+), 25 deletions(-) create mode 100644 crates/engine/src/commands/expr_tools.rs create mode 100644 crates/expr/src/sample.rs diff --git a/crates/engine/src/commands/expr_tools.rs b/crates/engine/src/commands/expr_tools.rs new file mode 100644 index 00000000..868be63a --- /dev/null +++ b/crates/engine/src/commands/expr_tools.rs @@ -0,0 +1,375 @@ +//! Expression tools: the error list behind the expression error bar, and the Expression +//! Language menu (the ⓕ button next to an expression), written from the public Expression +//! Language Reference. + +use effectcraft_project::{ItemId, Layer, Property}; +use effectcraft_render::EvalCtx; +use effectcraft_time::Tick; +use serde::Serialize; +use serde_json::{Value, json}; + +use super::{CommandSpec, comp_id, f_p}; +use crate::{EngineError, Result, Session, query}; + +/// One failing expression. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct ExprError { + pub comp: u64, + pub layer: u64, + #[serde(rename = "layerIndex")] + pub layer_index: usize, + #[serde(rename = "layerName")] + pub layer_name: String, + pub prop: u64, + /// Match path (`transform/opacity`). + pub path: String, + /// Property name path as shown in the timeline (`Transform/Opacity`). + pub name: String, + pub message: String, + /// The expression is disabled (it failed to compile). + pub disabled: bool, +} + +impl ExprError { + /// The error bar's text: `Error at line 1 in property 'Opacity' of layer 1 ('Solid') in comp 'Main'. …` + pub fn describe(&self, comp_name: &str) -> String { + let prop = self.name.rsplit('/').next().unwrap_or(&self.name); + format!("{} — property '{}' of layer {} ('{}') in comp '{}'", self.message, prop, self.layer_index, self.layer_name, comp_name) + } +} + +/// Every failing expression of comp `cid` at comp time `t`: evaluation errors of enabled +/// expressions and the syntax errors that disabled others. +pub fn errors(s: &Session, cid: ItemId, t: Tick) -> Vec { + let Some(comp) = s.project.comp(cid) else { return vec![] }; + let ctx = EvalCtx { footage: Some(s.footage.as_ref()), expr: s.expr.as_deref(), ..EvalCtx::new(&s.project, cid, comp, t) }; + let mut out = vec![]; + for (i, l) in comp.layers.iter().enumerate() { + let mut props: Vec<&Property> = vec![]; + l.props.walk("", &mut |_, p| { + if p.expr.as_ref().is_some_and(|e| !e.text.trim().is_empty()) { + props.push(p); + } + }); + for p in props { + let e = p.expr.as_ref().expect("filtered"); + let msg = if e.enabled { + match &ctx.expr { + Some(h) => h.eval(&ctx, l, p, &p.value_at(l.layer_time(t))).err(), + None => None, + } + } else { + s.expr_check.and_then(|check| check(&e.text).err()) + }; + if let Some(message) = msg { + out.push(error_of(cid, i, l, p, message, !e.enabled)); + } + } + } + out +} + +fn error_of(cid: ItemId, i: usize, l: &Layer, p: &Property, message: String, disabled: bool) -> ExprError { + ExprError { + comp: cid.0, + layer: l.id.0, + layer_index: i + 1, + layer_name: l.name.clone(), + prop: p.uid, + path: super::match_path_of(&l.props, p.uid).unwrap_or_default(), + name: l.props.name_path_of(p.uid).unwrap_or_else(|| p.name.clone()), + message, + disabled, + } +} + +fn errors_cmd(s: &mut Session, p: &Value) -> Result { + let cid = comp_id(s, p)?; + let t = f_p(p, "time").map(Tick::from_seconds_f64).unwrap_or_else(|| s.time_of(cid)); + let name = s.project.item(cid).map(|i| i.name.clone()).ok_or(EngineError::NoComp)?; + let list = errors(s, cid, t); + let text: Vec = list.iter().map(|e| e.describe(&name)).collect(); + Ok(json!({"comp": cid.0, "time": t.seconds(), "count": list.len(), "errors": list, "text": text})) +} + +/// The Expression Language menu: (category, [(label, inserted text)]). Submenus are nested +/// with `▸` in the category name. +pub fn language_menu() -> Vec<(&'static str, Vec<(&'static str, &'static str)>)> { + vec![ + ( + "Global", + vec![ + ("comp(name)", "comp(\"name\")"), + ("footage(name)", "footage(\"name\")"), + ("thisComp", "thisComp"), + ("thisLayer", "thisLayer"), + ("thisProperty", "thisProperty"), + ("thisProject", "thisProject"), + ("time", "time"), + ("colorDepth", "colorDepth"), + ("posterizeTime(framesPerSecond)", "posterizeTime(framesPerSecond)"), + ("value", "value"), + ], + ), + ( + "Vector Math", + vec![ + ("add(vec1, vec2)", "add(vec1, vec2)"), + ("sub(vec1, vec2)", "sub(vec1, vec2)"), + ("mul(vec, amount)", "mul(vec, amount)"), + ("div(vec, amount)", "div(vec, amount)"), + ("clamp(value, limit1, limit2)", "clamp(value, limit1, limit2)"), + ("dot(vec1, vec2)", "dot(vec1, vec2)"), + ("cross(vec1, vec2)", "cross(vec1, vec2)"), + ("normalize(vec)", "normalize(vec)"), + ("length(vec)", "length(vec)"), + ("length(point1, point2)", "length(point1, point2)"), + ("lookAt(fromPoint, atPoint)", "lookAt(fromPoint, atPoint)"), + ], + ), + ( + "Random Numbers", + vec![ + ("seedRandom(offset, timeless=false)", "seedRandom(offset, timeless=false)"), + ("random()", "random()"), + ("random(maxValOrArray)", "random(maxValOrArray)"), + ("random(minValOrArray, maxValOrArray)", "random(minValOrArray, maxValOrArray)"), + ("gaussRandom()", "gaussRandom()"), + ("gaussRandom(maxValOrArray)", "gaussRandom(maxValOrArray)"), + ("gaussRandom(minValOrArray, maxValOrArray)", "gaussRandom(minValOrArray, maxValOrArray)"), + ("noise(valOrArray)", "noise(valOrArray)"), + ], + ), + ( + "Interpolation", + vec![ + ("linear(t, value1, value2)", "linear(t, value1, value2)"), + ("linear(t, tMin, tMax, value1, value2)", "linear(t, tMin, tMax, value1, value2)"), + ("ease(t, value1, value2)", "ease(t, value1, value2)"), + ("ease(t, tMin, tMax, value1, value2)", "ease(t, tMin, tMax, value1, value2)"), + ("easeIn(t, value1, value2)", "easeIn(t, value1, value2)"), + ("easeIn(t, tMin, tMax, value1, value2)", "easeIn(t, tMin, tMax, value1, value2)"), + ("easeOut(t, value1, value2)", "easeOut(t, value1, value2)"), + ("easeOut(t, tMin, tMax, value1, value2)", "easeOut(t, tMin, tMax, value1, value2)"), + ], + ), + ( + "Color Conversion", + vec![ + ("rgbToHsl(rgbaArray)", "rgbToHsl(rgbaArray)"), + ("hslToRgb(hslaArray)", "hslToRgb(hslaArray)"), + ("hexToRgb(hexString)", "hexToRgb(hexString)"), + ], + ), + ("Other Math", vec![("degreesToRadians(degrees)", "degreesToRadians(degrees)"), ("radiansToDegrees(radians)", "radiansToDegrees(radians)")]), + ( + "JavaScript Math", + vec![ + ("Math.cos(value)", "Math.cos(value)"), + ("Math.sin(value)", "Math.sin(value)"), + ("Math.atan2(y, x)", "Math.atan2(y, x)"), + ("Math.sqrt(value)", "Math.sqrt(value)"), + ("Math.abs(value)", "Math.abs(value)"), + ("Math.round(value)", "Math.round(value)"), + ("Math.floor(value)", "Math.floor(value)"), + ("Math.min(value1, value2)", "Math.min(value1, value2)"), + ("Math.max(value1, value2)", "Math.max(value1, value2)"), + ("Math.PI", "Math.PI"), + ], + ), + ( + "Comp", + vec![ + ("layer(index)", "layer(index)"), + ("layer(name)", "layer(\"name\")"), + ("layer(otherLayer, relIndex)", "layer(otherLayer, relIndex)"), + ("marker", "marker"), + ("numLayers", "numLayers"), + ("activeCamera", "activeCamera"), + ("width", "width"), + ("height", "height"), + ("duration", "duration"), + ("frameDuration", "frameDuration"), + ("pixelAspect", "pixelAspect"), + ("name", "name"), + ("bgColor", "bgColor"), + ("displayStartTime", "displayStartTime"), + ], + ), + ( + "Footage", + vec![ + ("width", "width"), + ("height", "height"), + ("duration", "duration"), + ("frameDuration", "frameDuration"), + ("pixelAspect", "pixelAspect"), + ("name", "name"), + ("sourceText", "sourceText"), + ("sourceData", "sourceData"), + ("dataValue(dataPath)", "dataValue(dataPath)"), + ("dataKeyCount", "dataKeyCount"), + ], + ), + ( + "Layer ▸ Sub-objects", + vec![ + ("source", "source"), + ("effect(name)", "effect(\"name\")"), + ("effect(index)", "effect(index)"), + ("mask(name)", "mask(\"name\")"), + ("mask(index)", "mask(index)"), + ], + ), + ( + "Layer ▸ General", + vec![ + ("width", "width"), + ("height", "height"), + ("index", "index"), + ("parent", "parent"), + ("hasParent", "hasParent"), + ("inPoint", "inPoint"), + ("outPoint", "outPoint"), + ("startTime", "startTime"), + ("hasVideo", "hasVideo"), + ("enabled", "enabled"), + ("active", "active"), + ], + ), + ( + "Layer ▸ Properties", + vec![ + ("anchorPoint", "anchorPoint"), + ("position", "position"), + ("scale", "scale"), + ("rotation", "rotation"), + ("opacity", "opacity"), + ("marker", "marker"), + ("name", "name"), + ], + ), + ( + "Layer ▸ Space Transforms", + vec![ + ("toComp(point, t = time)", "toComp(point, t = time)"), + ("fromComp(point, t = time)", "fromComp(point, t = time)"), + ("toWorld(point, t = time)", "toWorld(point, t = time)"), + ("fromWorld(point, t = time)", "fromWorld(point, t = time)"), + ("toCompVec(vec, t = time)", "toCompVec(vec, t = time)"), + ("fromCompVec(vec, t = time)", "fromCompVec(vec, t = time)"), + ("toWorldVec(vec, t = time)", "toWorldVec(vec, t = time)"), + ("fromWorldVec(vec, t = time)", "fromWorldVec(vec, t = time)"), + ], + ), + ( + "Layer ▸ Other", + vec![ + ("sourceRectAtTime(t = time, includeExtents = false)", "sourceRectAtTime(t = time, includeExtents = false)"), + ("sampleImage(point, radius = [.5, .5], postEffect = true, t = time)", "sampleImage(point, radius = [.5, .5], postEffect = true, t = time)"), + ], + ), + ( + "Property", + vec![ + ("value", "value"), + ("valueAtTime(t)", "valueAtTime(t)"), + ("velocity", "velocity"), + ("velocityAtTime(t)", "velocityAtTime(t)"), + ("speed", "speed"), + ("speedAtTime(t)", "speedAtTime(t)"), + ("wiggle(freq, amp, octaves = 1, amp_mult = .5, t = time)", "wiggle(freq, amp, octaves = 1, amp_mult = .5, t = time)"), + ("temporalWiggle(freq, amp, octaves = 1, amp_mult = .5, t = time)", "temporalWiggle(freq, amp, octaves = 1, amp_mult = .5, t = time)"), + ("smooth(width = .2, samples = 5, t = time)", "smooth(width = .2, samples = 5, t = time)"), + ("loopIn(type = \"cycle\", numKeyframes = 0)", "loopIn(type = \"cycle\", numKeyframes = 0)"), + ("loopOut(type = \"cycle\", numKeyframes = 0)", "loopOut(type = \"cycle\", numKeyframes = 0)"), + ("loopInDuration(type = \"cycle\", duration = 0)", "loopInDuration(type = \"cycle\", duration = 0)"), + ("loopOutDuration(type = \"cycle\", duration = 0)", "loopOutDuration(type = \"cycle\", duration = 0)"), + ("key(index)", "key(index)"), + ("key(markerName)", "key(markerName)"), + ("nearestKey(t)", "nearestKey(t)"), + ("numKeys", "numKeys"), + ("name", "name"), + ("propertyIndex", "propertyIndex"), + ], + ), + ( + "Path Property", + vec![ + ("points(t = time)", "points(t = time)"), + ("inTangents(t = time)", "inTangents(t = time)"), + ("outTangents(t = time)", "outTangents(t = time)"), + ("isClosed()", "isClosed()"), + ( + "createPath(points, inTangents = [], outTangents = [], is_closed = true)", + "createPath(points, inTangents = [], outTangents = [], is_closed = true)", + ), + ], + ), + ( + "Text ▸ Style", + vec![ + ("text.sourceText.style", "text.sourceText.style"), + ("getStyleAt(charIndex, t = time)", "getStyleAt(charIndex, t = time)"), + ("createStyle()", "createStyle()"), + ("setFontSize(value)", "setFontSize(value)"), + ("setFillColor(value)", "setFillColor(value)"), + ("setText(value)", "setText(value)"), + ], + ), + ("Text ▸ Expression Selector", vec![("textIndex", "textIndex"), ("textTotal", "textTotal"), ("selectorValue", "selectorValue")]), + ("Key", vec![("value", "value"), ("time", "time"), ("index", "index")]), + ( + "Marker Key", + vec![ + ("duration", "duration"), + ("comment", "comment"), + ("chapter", "chapter"), + ("url", "url"), + ("frameTarget", "frameTarget"), + ("eventCuePoint", "eventCuePoint"), + ("cuePointName", "cuePointName"), + ("parameters", "parameters"), + ("protectedRegion", "protectedRegion"), + ], + ), + ( + "Time Conversion", + vec![ + ( + "timeToFrames(t = time + thisComp.displayStartTime, fps = 1.0 / thisComp.frameDuration, isDuration = false)", + "timeToFrames(t = time + thisComp.displayStartTime, fps = 1.0 / thisComp.frameDuration, isDuration = false)", + ), + ("framesToTime(frames, fps = 1.0 / thisComp.frameDuration)", "framesToTime(frames, fps = 1.0 / thisComp.frameDuration)"), + ( + "timeToTimecode(t = time + thisComp.displayStartTime, timecodeBase = 30, isDuration = false)", + "timeToTimecode(t = time + thisComp.displayStartTime, timecodeBase = 30, isDuration = false)", + ), + ( + "timeToFeetAndFrames(t = time + thisComp.displayStartTime, fps = 1.0 / thisComp.frameDuration, framesPerFoot = 16, isDuration = false)", + "timeToFeetAndFrames(t = time + thisComp.displayStartTime, fps = 1.0 / thisComp.frameDuration, framesPerFoot = 16, isDuration = false)", + ), + ( + "timeToCurrentFormat(t = time + thisComp.displayStartTime, fps = 1.0 / thisComp.frameDuration, isDuration = false)", + "timeToCurrentFormat(t = time + thisComp.displayStartTime, fps = 1.0 / thisComp.frameDuration, isDuration = false)", + ), + ], + ), + ] +} + +fn menu_cmd(_s: &mut Session, _p: &Value) -> Result { + Ok(json!( + language_menu() + .into_iter() + .map(|(c, items)| json!({"category": c, "items": items.into_iter().map(|(l, t)| json!({"label": l, "text": t})).collect::>()})) + .collect::>() + )) +} + +pub fn specs() -> Vec { + vec![ + query!("expr.errors", "Expression errors (the error bar)", "{comp?, time? (s)}", errors_cmd), + query!("expr.languageMenu", "Expression Language menu", "{}", menu_cmd), + ] +} diff --git a/crates/engine/src/commands/file.rs b/crates/engine/src/commands/file.rs index 9e81db54..972e00ea 100644 --- a/crates/engine/src/commands/file.rs +++ b/crates/engine/src/commands/file.rs @@ -84,12 +84,23 @@ fn import(s: &mut Session, p: &Value) -> Result { Some(Value::String(x)) => vec![x.clone()], _ => return Err(bad("file.import", "missing `paths`")), }; - let importer = s.importer.clone().ok_or_else(|| EngineError::Other("media import is not available in this build".into()))?; let mut ids = vec![]; let mut errors = vec![]; let mut probed = vec![]; let seq_rate = effectcraft_time::FrameRate::from_f64(s.prefs.import.sequence_fps); for path in &paths { + // Data files (JSON, CSV, TSV) for data-driven animation: kept as text in the project. + if let Some(f) = data_footage(s, path) { + match f { + Ok(f) => probed.push((path.clone(), f)), + Err(e) => errors.push(format!("{path}: {e}")), + } + continue; + } + let Some(importer) = s.importer.clone() else { + errors.push(format!("{path}: media import is not available in this build")); + continue; + }; match importer.probe(path) { Ok(mut f) => { // Settings ▸ Import ▸ Sequence Footage frames per second. @@ -122,6 +133,25 @@ fn import(s: &mut Session, p: &Value) -> Result { Ok(json!({"items": ids, "errors": errors})) } +/// Extensions imported as data footage. +pub const DATA_EXTENSIONS: &[&str] = &["json", "csv", "tsv"]; + +/// A data footage item for `path` (JSON / CSV / TSV), `None` for other files. +pub(crate) fn data_footage(s: &Session, path: &str) -> Option> { + let ext = std::path::Path::new(path).extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); + if !DATA_EXTENSIONS.contains(&ext.as_str()) { + return None; + } + Some((|| { + let bytes = s.services.read_file(path).map_err(|e| e.to_string())?; + let text = String::from_utf8(bytes).map_err(|_| "data files must be UTF-8 text".to_string())?; + if ext == "json" { + serde_json::from_str::(text.trim_start_matches('\u{feff}')).map_err(|e| format!("invalid JSON: {e}"))?; + } + Ok(effectcraft_project::Footage { path: path.to_string(), kind: FootageKind::Data, codec: ext.to_uppercase(), data: Some(text), ..Default::default() }) + })()) +} + fn project_settings(s: &mut Session, p: &Value) -> Result { s.edit("Project Settings", None, |proj, _| { if let Some(b) = str_p(p, "bitDepth") { diff --git a/crates/engine/src/commands/layer.rs b/crates/engine/src/commands/layer.rs index ac728200..e57b1c0a 100644 --- a/crates/engine/src/commands/layer.rs +++ b/crates/engine/src/commands/layer.rs @@ -239,6 +239,9 @@ fn add_item(s: &mut Session, p: &Value) -> Result { let d = (s.prefs.import.still_footage == "seconds").then(|| Tick::from_seconds_f64(s.prefs.import.still_seconds)); (LayerSource::Footage { item }, (f.width, f.height), d) } + ItemKind::Footage(f) if f.kind == effectcraft_project::FootageKind::Data => { + return Err(bad("layer.addItem", "data files can't be layers; read them in expressions with footage(\"name\").sourceData")); + } ItemKind::Footage(f) if f.kind == effectcraft_project::FootageKind::Model => { return super::model3d::new_model(s, &serde_json::json!({"comp": cid.0, "item": item.0, "time": f_p(p, "time")})); } @@ -505,7 +508,8 @@ fn set_parent(s: &mut Session, p: &Value) -> Result { // Like AE's pick-whip, parenting keeps the layer where it is: its transform is re-expressed // in the new parent's space (Position, Rotation, Scale), unless `compensate: false`. let compensate = b_p(p, "compensate").unwrap_or(true); - let ectx = effectcraft_render::EvalCtx { project: &s.project, comp_id: cid, comp, time: s.time(), expr: s.expr.as_deref() }; + let ectx = + effectcraft_render::EvalCtx { project: &s.project, comp_id: cid, comp, time: s.time(), expr: s.expr.as_deref(), footage: Some(s.footage.as_ref()) }; let space = |id: Option| -> effectcraft_geom::Mat3 { id.and_then(|i| comp.layer(i)).filter(|l| !l.is_3d()).map(|l| ectx.layer_to_comp(l).0).unwrap_or(effectcraft_geom::Mat3::IDENTITY) }; diff --git a/crates/engine/src/commands/layer_time.rs b/crates/engine/src/commands/layer_time.rs index d8090352..7a0eb86b 100644 --- a/crates/engine/src/commands/layer_time.rs +++ b/crates/engine/src/commands/layer_time.rs @@ -126,7 +126,7 @@ fn freeze(s: &mut Session, p: &Value, last: bool) -> Result { return Err(bad("layer.freezeFrame", "freezing needs a footage or composition layer")); } // What each layer shows now (honouring an existing remap) — evaluated before the edit. - let ectx = effectcraft_render::EvalCtx { project: &s.project, comp_id: cid, comp, time: cti, expr: s.expr.as_deref() }; + let ectx = effectcraft_render::EvalCtx { project: &s.project, comp_id: cid, comp, time: cti, expr: s.expr.as_deref(), footage: Some(s.footage.as_ref()) }; let now: Vec<(effectcraft_project::LayerId, f64)> = comp .layers .iter() diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 7a053c77..e1b3f15f 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -9,6 +9,7 @@ mod comp_more; mod edit; mod effect; pub mod essential; +pub mod expr_tools; mod file; mod file_more; mod frontend; @@ -138,6 +139,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(liquify::specs()); v.extend(settings::specs()); v.extend(essential::specs()); + v.extend(expr_tools::specs()); v.extend(stubs::specs()); v }) diff --git a/crates/engine/src/commands/prop.rs b/crates/engine/src/commands/prop.rs index 05dc37fa..2f34898c 100644 --- a/crates/engine/src/commands/prop.rs +++ b/crates/engine/src/commands/prop.rs @@ -557,7 +557,8 @@ fn convert_expr_to_keys(s: &mut Session, p: &Value) -> Result { let fd = comp.frame_duration(); let mut t = layer.in_point; while t < layer.out_point { - let ctx = effectcraft_render::EvalCtx { project: &s.project, comp_id: cid, comp: &comp, time: t, expr: s.expr.as_deref() }; + let ctx = + effectcraft_render::EvalCtx { project: &s.project, comp_id: cid, comp: &comp, time: t, expr: s.expr.as_deref(), footage: Some(s.footage.as_ref()) }; keys.push(Keyframe::new(layer.layer_time(t), ctx.value(&layer, &pr))); t += fd; } diff --git a/crates/engine/src/commands/puppet.rs b/crates/engine/src/commands/puppet.rs index 00117dad..4ada1dcd 100644 --- a/crates/engine/src/commands/puppet.rs +++ b/crates/engine/src/commands/puppet.rs @@ -29,7 +29,7 @@ pub fn puppet_eval(s: &Session, cid: ItemId, lid: LayerId, fx_uid: Uid, t: Tick) let fx = layer.effects()?; let index = fx.groups().position(|g| g.uid == fx_uid)?; let g = fx.groups().nth(index)?; - let ctx = EvalCtx { project: &s.project, comp_id: cid, comp, time: t, expr: s.expr.as_deref() }; + let ctx = EvalCtx { project: &s.project, comp_id: cid, comp, time: t, expr: s.expr.as_deref(), footage: None }; let mut r = Renderer::new(&s.project, &*s.footage, RenderOpts::default()); r.expr = s.expr.as_deref(); r.cache = Some(&s.layer_cache); diff --git a/crates/engine/src/commands/three_d.rs b/crates/engine/src/commands/three_d.rs index aea32105..55060d17 100644 --- a/crates/engine/src/commands/three_d.rs +++ b/crates/engine/src/commands/three_d.rs @@ -393,7 +393,7 @@ fn get_3d(s: &mut Session, p: &Value) -> Result { let (w, h) = (comp.width as f64, comp.height as f64); let vs = s.state.views3d.get(&cid).cloned().unwrap_or_default(); let t = s.state.times.get(&cid).copied().unwrap_or(Tick::ZERO); - let ctx = EvalCtx { project: &s.project, comp_id: cid, comp, time: t, expr: s.expr.as_deref() }; + let ctx = EvalCtx { project: &s.project, comp_id: cid, comp, time: t, expr: s.expr.as_deref(), footage: None }; let cam = vs.override_camera(w, h).unwrap_or_else(|| camera::active_camera(&ctx)); let f = cam.forward(); let lights: Vec = comp @@ -535,7 +535,7 @@ fn auto_orient(s: &mut Session, p: &Value) -> Result { /// through each layer's world matrix; cameras and lights contribute their position). fn layer_points(s: &Session, cid: ItemId, ids: &[LayerId], t: Tick) -> Vec { let Some(comp) = s.project.comp(cid) else { return vec![] }; - let ctx = EvalCtx { project: &s.project, comp_id: cid, comp, time: t, expr: s.expr.as_deref() }; + let ctx = EvalCtx { project: &s.project, comp_id: cid, comp, time: t, expr: s.expr.as_deref(), footage: None }; let mut out = vec![]; for l in comp.layers.iter().filter(|l| ids.contains(&l.id)) { let m = ctx.world_matrix(l); diff --git a/crates/engine/src/tracking.rs b/crates/engine/src/tracking.rs index c406edb9..6a22fd81 100644 --- a/crates/engine/src/tracking.rs +++ b/crates/engine/src/tracking.rs @@ -206,7 +206,7 @@ pub(crate) fn source_frame( t: Tick, expr: Option<&dyn ExprHost>, ) -> Option<(Arc, [f64; 2])> { - let ctx = EvalCtx { project, comp_id: cid, comp, time: t, expr }; + let ctx = EvalCtx { project, comp_id: cid, comp, time: t, expr, footage: None }; if let effectcraft_project::LayerSource::Footage { item } = &layer.source && let Some(effectcraft_project::ItemKind::Footage(f)) = project.item(*item).map(|i| &i.kind) && f.has_video diff --git a/crates/expr/src/host.rs b/crates/expr/src/host.rs index 4b92e224..70093790 100644 --- a/crates/expr/src/host.rs +++ b/crates/expr/src/host.rs @@ -52,6 +52,93 @@ pub enum Req { /// A Source Text value as JSON `{doc, text, runs: [{len, style}], paras}` (styles as /// `layer.setText` keys), or null for other properties. Doc { comp: u64, layer: u64, path: String, t: Secs, pre: bool }, + /// `footage(name)`: `[name, width, height, duration, frameDuration, pixelAspect, kind, + /// sourceText|null, dataFormat|null]` or null. + Footage { name: String }, + /// A data footage item's parsed data as JSON `{data, rows, header}` (`rows`/`header` for + /// CSV/TSV), or null. + FootageData { name: String }, + /// `sampleImage`: the average straight `[r, g, b, a]` of the layer's pixels in the box + /// `point ± radius` (layer space) at comp time `t`, after masks and effects when `post`. + Sample { comp: u64, layer: u64, x: Secs, y: Secs, rx: Secs, ry: Secs, post: bool, t: Secs }, + /// `thisProject`: `[bitsPerChannel, linearBlending, workingSpace]`. + Project, +} + +/// Parse CSV/TSV text (RFC 4180 quoting) into rows of cells. +pub fn parse_delimited(text: &str, sep: char) -> Vec> { + let mut rows = vec![]; + let mut row = vec![]; + let mut cell = String::new(); + let mut quoted = false; + let mut chars = text.chars().peekable(); + while let Some(c) = chars.next() { + if quoted { + match c { + '"' if chars.peek() == Some(&'"') => { + chars.next(); + cell.push('"'); + } + '"' => quoted = false, + _ => cell.push(c), + } + continue; + } + match c { + '"' if cell.is_empty() => quoted = true, + c if c == sep => row.push(std::mem::take(&mut cell)), + '\r' => {} + '\n' => { + row.push(std::mem::take(&mut cell)); + rows.push(std::mem::take(&mut row)); + } + _ => cell.push(c), + } + } + if !cell.is_empty() || !row.is_empty() { + row.push(cell); + rows.push(row); + } + rows.retain(|r| !(r.len() == 1 && r[0].trim().is_empty())); + rows +} + +/// A CSV/TSV cell as JSON: numbers become numbers. +fn cell_json(c: &str) -> serde_json::Value { + let t = c.trim(); + match t.parse::() { + Ok(x) if x.is_finite() && !t.is_empty() => serde_json::json!(x), + _ => serde_json::json!(c), + } +} + +/// The data of a data footage item as `{data, rows, header}` JSON (see [`Req::FootageData`]). +pub fn footage_data_json(f: &effectcraft_project::Footage) -> Option { + let text = f.data.as_deref()?; + let text = text.strip_prefix('\u{feff}').unwrap_or(text); + Some(match f.data_format()? { + "json" => { + let v: serde_json::Value = serde_json::from_str(text).ok()?; + serde_json::json!({"data": v, "rows": null, "header": null}).to_string() + } + fmt => { + let rows = parse_delimited(text, if fmt == "tsv" { '\t' } else { ',' }); + let (header, body) = match rows.split_first() { + Some((h, b)) => (h.clone(), b.to_vec()), + None => (vec![], vec![]), + }; + let objects: Vec = body + .iter() + .map(|r| { + serde_json::Value::Object( + header.iter().enumerate().map(|(i, k)| (k.clone(), r.get(i).map(|c| cell_json(c)).unwrap_or(serde_json::Value::Null))).collect(), + ) + }) + .collect(); + let cells: Vec> = body.iter().map(|r| r.iter().map(|c| cell_json(c)).collect()).collect(); + serde_json::json!({"data": objects, "rows": cells, "header": header}).to_string() + } + }) } /// A text document as the style API's JSON. @@ -192,7 +279,7 @@ impl<'a> Resolver<'a> { } fn ctx_at(&self, c: u64, t: f64) -> Option> { let comp = self.comp(c)?; - Some(EvalCtx { project: self.ctx.project, comp_id: ItemId(c), comp, time: Tick::from_seconds_f64(t), expr: self.ctx.expr }) + Some(EvalCtx { project: self.ctx.project, comp_id: ItemId(c), comp, time: Tick::from_seconds_f64(t), expr: self.ctx.expr, footage: self.ctx.footage }) } fn prop(&self, c: u64, l: u64, path: &str) -> Option<(&'a Layer, &'a Property)> { let layer = self.layer(c, l)?; @@ -345,6 +432,36 @@ impl<'a> Resolver<'a> { }; Resp::nums(r) } + Req::Footage { name } => { + let it = project.items.values().find(|i| &i.name == name && matches!(i.kind, ItemKind::Footage(_)))?; + let ItemKind::Footage(f) = &it.kind else { return None }; + Resp::List(vec![ + Resp::Str(it.name.clone()), + Resp::Num(f.width as f64), + Resp::Num(f.height as f64), + Resp::Num(f.duration.seconds()), + Resp::Num(f.frame_rate.frame_duration().seconds()), + Resp::Num(f.pixel_aspect), + Resp::Str(it.type_name().into()), + f.data.clone().map_or(Resp::Null, Resp::Str), + f.data_format().map_or(Resp::Null, |x| Resp::Str(x.into())), + ]) + } + Req::FootageData { name } => { + let it = project.items.values().find(|i| &i.name == name && matches!(i.kind, ItemKind::Footage(_)))?; + let ItemKind::Footage(f) = &it.kind else { return None }; + Resp::Str(footage_data_json(f)?) + } + Req::Project => Resp::List(vec![ + Resp::Num(project.settings.bit_depth.bits() as f64), + Resp::Bool(project.settings.blend_linear || project.settings.linearize), + Resp::Str(project.settings.working_space.map(|w| format!("{w:?}")).unwrap_or_default()), + ]), + Req::Sample { comp, layer, x, y, rx, ry, post, t } => { + let l = self.layer(*comp, *layer)?; + let f = |b: &Secs| f64::from_bits(*b); + Resp::nums(crate::sample::sample_image(self.ctx, ItemId(*comp), l, [f(x), f(y)], [f(rx), f(ry)], *post, f(t))?.map(|v| v as f64)) + } Req::Markers { comp, layer } => { let marks: Vec<(f64, &effectcraft_project::Marker)> = match layer { Some(l) => { @@ -363,6 +480,17 @@ impl<'a> Resolver<'a> { Resp::Str(m.comment.clone()), Resp::Str(m.chapter.clone()), Resp::Str(m.url.clone()), + Resp::Str(m.frame_target.clone()), + Resp::Bool(m.cue_point.as_ref().is_some_and(|c| !c.navigation)), + Resp::Str(m.cue_point.as_ref().map(|c| c.name.clone()).unwrap_or_default()), + Resp::List( + m.cue_point + .iter() + .flat_map(|c| c.params.iter()) + .map(|(k, v)| Resp::List(vec![Resp::Str(k.clone()), Resp::Str(v.clone())])) + .collect(), + ), + Resp::Bool(m.protected), ]) }) .collect(), diff --git a/crates/expr/src/lib.rs b/crates/expr/src/lib.rs index 323e3c7a..88c1b031 100644 --- a/crates/expr/src/lib.rs +++ b/crates/expr/src/lib.rs @@ -28,6 +28,7 @@ pub mod host; mod noise; pub mod rewrite; mod runtime; +mod sample; use effectcraft_keyframe::Value; use effectcraft_project::{ItemId, Layer, LayerId, Project, Property}; @@ -112,7 +113,7 @@ fn evaluate_out( if depth >= MAX_DEPTH { return Err("Error: expressions reference each other too deeply (circular reference?)".into()); } - let ctx = EvalCtx { project: ctx.project, comp_id: ctx.comp_id, comp: ctx.comp, time: ctx.time, expr: Some(host) }; + let ctx = EvalCtx { project: ctx.project, comp_id: ctx.comp_id, comp: ctx.comp, time: ctx.time, expr: Some(host), footage: ctx.footage }; let comp = ctx.comp_id.0; let path = layer.props.path_of(prop.uid).unwrap_or_else(|| format!("@{}", prop.uid)); let resolver = Resolver { ctx, own: Own { comp, layer, prop, path: path.clone() } }; diff --git a/crates/expr/src/prelude.js b/crates/expr/src/prelude.js index 67ddf458..1655f75d 100644 --- a/crates/expr/src/prelude.js +++ b/crates/expr/src/prelude.js @@ -18,6 +18,7 @@ var selectorValue = [100, 100, 100]; // Request ops (must match runtime.rs). var __COMP = 0, __COMPNAME = 1, __LAYER = 2, __LINFO = 3, __CHILD = 4, __PINFO = 5, __VALUE = 6, __XFORM = 7, __RECT = 8, __MARKERS = 9, __DOC = 10; +var __FOOT = 11, __FDATA = 12, __SAMPLE = 13, __PROJ = 14; function __begin(t, v, c, l, path, uid, idx, fd, ti, tt, sv) { time = t; @@ -29,6 +30,7 @@ function __begin(t, v, c, l, path, uid, idx, fd, ti, tt, sv) { __rc = 0; __seed = 0; __timeless = false; + __projectInfo = null; } function __finish(r) { @@ -269,6 +271,18 @@ function timeToTimecode(t, base, isDuration) { return (neg ? '-' : '') + h + ':' + __pad(mm) + ':' + __pad(ss) + ':' + __pad(ff); } +function timeToFeetAndFrames(t, fps, framesPerFoot, isDuration) { + if (t === undefined) t = time + thisComp.displayStartTime; + if (fps === undefined) fps = 1 / __S.fd; + if (framesPerFoot === undefined) framesPerFoot = 16; + var neg = t < 0; + var f = Math.floor(Math.abs(t) * fps + 1e-6); + var feet = Math.floor(f / framesPerFoot), fr = f % framesPerFoot; + var s = String(fr); + while (s.length < 2) s = '0' + s; + return (neg ? '-' : '') + feet + '+' + s; +} + function timeToCurrentFormat(t, fps) { if (fps === undefined) fps = Math.round(1 / __S.fd); return timeToTimecode(t, fps); @@ -371,6 +385,14 @@ Comp.prototype = { get displayStartTime() { return this.__info[7]; }, get bgColor() { return this.__info[8]; }, get marker() { return new Markers(this.__c, -1); }, + // The topmost enabled camera active at the current time. + get activeCamera() { + for (var i = 1; i <= this.numLayers; i++) { + var l = this.layer(i); + if (l.__info[10] === 'Camera' && l.active) return l; + } + __err('Composition "' + this.name + '" has no active camera'); + }, layer: function (a, rel) { if (a !== null && typeof a === 'object' && a.__isLayer) return this.layer(a.index + (rel || 0)); var id = __h(__LAYER, this.__c, a); @@ -386,6 +408,68 @@ function comp(name) { return new Comp(id); } +// Footage (data-driven animation) and the project ------------------------------------------------ + +function Footage(info) { this.__i = info; this.__d = null; } +Footage.prototype = { + __isFootage: true, + get name() { return this.__i[0]; }, + get width() { return this.__i[1]; }, + get height() { return this.__i[2]; }, + get duration() { return this.__i[3]; }, + get frameDuration() { return this.__i[4]; }, + get pixelAspect() { return this.__i[5]; }, + get ntscDropFrame() { return false; }, + // The data file's text (JSON / CSV / TSV), or "" for media. + get sourceText() { return this.__i[7] === null ? '' : this.__i[7]; }, + get __data() { + if (this.__d !== null) return this.__d; + var j = __h(__FDATA, this.__i[0]); + if (j === null) __err('Footage "' + this.name + '" has no data (or it could not be parsed)'); + return (this.__d = JSON.parse(j)); + }, + // JSON: the parsed document; CSV/TSV: one object per row keyed by the header row. + get sourceData() { return this.__data.data; }, + // CSV/TSV cell by [column, row] (0-based, rows after the header). + dataValue: function (idx) { + idx = __v(idx); + var rows = this.__data.rows; + if (rows === null) { + // JSON: a path of keys / indexes. + var v = this.__data.data; + for (var i = 0; i < idx.length; i++) { + if (v === null || v === undefined) __err('dataValue: no value at ' + JSON.stringify(idx)); + v = v[idx[i]]; + } + return v; + } + var r = rows[idx[1]]; + if (r === undefined || r[idx[0]] === undefined) __err('dataValue: no cell at column ' + idx[0] + ', row ' + idx[1]); + return r[idx[0]]; + }, + get dataKeyCount() { var r = this.__data.rows; return r === null ? 0 : r.length; }, + // The column names (CSV/TSV header row). + get dataKeyNames() { return this.__data.header || []; }, + toString: function () { return '[object Footage]'; }, +}; + +function footage(name) { + var info = __h(__FOOT, String(name)); + if (info === null) __err('Footage named "' + name + '" does not exist'); + return new Footage(info); +} + +var __projectInfo = null; +// The project's bits per channel (8, 16 or 32). +__global('colorDepth', function () { return thisProject.bitsPerChannel; }); +var thisProject = { + get __info() { return __projectInfo || (__projectInfo = __h(__PROJ)); }, + get fullPath() { return ''; }, + get bitsPerChannel() { return this.__info[0]; }, + get linearBlending() { return this.__info[1]; }, + toString: function () { return '[object Project]'; }, +}; + // Layer ----------------------------------------------------------------------------------------- function Layer(c, l) { this.__c = c; this.__l = l; this.__i = null; } @@ -464,7 +548,14 @@ Layer.prototype = { var r = __h(__RECT, this.__c, this.__l, t === undefined ? time : t, !!includeExtents); return { top: r[0], left: r[1], width: r[2], height: r[3] }; }, - sampleImage: function () { return [0, 0, 0, 0]; }, + // Average straight RGBA of the layer's pixels in `point ± radius` (layer space). + sampleImage: function (point, radius, postEffect, t) { + point = __v(point); + radius = radius === undefined ? [0.5, 0.5] : __v(radius); + if (!Array.isArray(radius)) radius = [radius, radius]; + return __h(__SAMPLE, this.__c, this.__l, point[0], point[1], radius[0], radius.length > 1 ? radius[1] : radius[0], + postEffect === undefined ? true : !!__v(postEffect), t === undefined ? time : __v(t)); + }, toString: function () { return '[object Layer]'; }, }; @@ -817,7 +908,12 @@ Markers.prototype = { } if (!(n >= 1 && n <= list.length)) __err('Marker index ' + n + ' out of range'); var m = list[n - 1]; - return { time: m[0], duration: m[1], comment: m[2], chapter: m[3], url: m[4], index: n }; + var params = {}; + for (var k = 0; k < m[8].length; k++) params[m[8][k][0]] = m[8][k][1]; + return { + time: m[0], duration: m[1], comment: m[2], chapter: m[3], url: m[4], index: n, + frameTarget: m[5], eventCuePoint: m[6], cuePointName: m[7], parameters: params, protectedRegion: m[9], + }; }, nearestKey: function (t) { var list = this.__list; diff --git a/crates/expr/src/runtime.rs b/crates/expr/src/runtime.rs index af7ed9b6..d6138839 100644 --- a/crates/expr/src/runtime.rs +++ b/crates/expr/src/runtime.rs @@ -145,6 +145,19 @@ fn native_host(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult Req::Footage { name: string(args, 1, ctx)? }, + 12 => Req::FootageData { name: string(args, 1, ctx)? }, + 13 => Req::Sample { + comp: id(args, 1, ctx)?, + layer: id(args, 2, ctx)?, + x: secs(num(args, 3, ctx)?), + y: secs(num(args, 4, ctx)?), + rx: secs(num(args, 5, ctx)?), + ry: secs(num(args, 6, ctx)?), + post: arg(args, 7).to_boolean(), + t: secs(num(args, 8, ctx)?), + }, + 14 => Req::Project, _ => return Err(JsNativeError::typ().with_message("unknown host request").into()), }; let found = FRAMES.with(|f| { diff --git a/crates/expr/src/sample.rs b/crates/expr/src/sample.rs new file mode 100644 index 00000000..bff5a214 --- /dev/null +++ b/crates/expr/src/sample.rs @@ -0,0 +1,119 @@ +//! `sampleImage`: read a layer's rendered pixels from an expression. +//! +//! The layer is rendered at full resolution through the renderer that is evaluating the +//! expression (its footage source; [`EvalCtx::footage`]): source → masks → effects for +//! `postEffect = true`, the bare source otherwise, in layer space (before the transform). Buffers +//! are cached per thread for the frame, so many expressions sampling the same layer at the same +//! time render it once. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::sync::Arc; + +use effectcraft_project::{ItemId, Layer}; +use effectcraft_render::{EvalCtx, NoFootage, RenderOpts, Renderer}; +use effectcraft_time::Tick; + +/// A rendered layer buffer: straight-from-the-renderer premultiplied pixels with the layer +/// space → buffer pixel mapping. +#[derive(Clone)] +struct Sampled { + img: Arc, + offset: [f64; 2], + scale: f64, +} + +type Key = (usize, u64, u64, u64, bool, u64); + +thread_local! { + static CACHE: RefCell>> = RefCell::new(HashMap::new()); + /// Nested sampleImage renders (a sampled layer whose expressions sample again). + static DEPTH: RefCell = const { RefCell::new(0) }; +} + +const MAX_CACHE: usize = 32; +const MAX_DEPTH: usize = 2; + +/// A fingerprint of the layer's definition (so edits never reuse stale pixels). +fn fingerprint(layer: &Layer) -> u64 { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + format!("{layer:?}").hash(&mut h); + h.finish() +} + +fn render(ctx: EvalCtx, comp: ItemId, layer: &Layer, post: bool, t: f64) -> Option { + let depth = DEPTH.with(|d| *d.borrow()); + if depth >= MAX_DEPTH { + return None; + } + let key: Key = (ctx.project as *const _ as usize, comp.0, layer.id.0, t.to_bits(), post, fingerprint(layer)); + if let Some(hit) = CACHE.with(|c| c.borrow().get(&key).cloned()) { + return hit; + } + DEPTH.with(|d| *d.borrow_mut() += 1); + struct Pop; + impl Drop for Pop { + fn drop(&mut self) { + DEPTH.with(|d| *d.borrow_mut() -= 1); + } + } + let _pop = Pop; + let footage: &dyn effectcraft_render::FootageSource = ctx.footage.unwrap_or(&NoFootage); + let mut r = Renderer::new(ctx.project, footage, RenderOpts { motion_blur: false, ..Default::default() }); + r.expr = ctx.expr; + let ectx = r.eval_ctx(comp, Tick::from_seconds_f64(t))?; + let out = if post { + r.content_buf(&ectx, layer).map(|b| Sampled { img: Arc::new(b.img.clone()), offset: b.offset, scale: b.scale }) + } else { + r.layer_source(&ectx, layer).map(|b| Sampled { img: Arc::new(b.img), offset: b.offset, scale: b.scale }) + }; + CACHE.with(|c| { + let mut c = c.borrow_mut(); + if c.len() >= MAX_CACHE { + c.clear(); + } + c.insert(key, out.clone()); + }); + out +} + +/// The average colour (straight RGBA, 0..1) of `layer`'s pixels in the box `point ± radius` +/// (layer pixels) at comp time `t`. Transparent outside the layer. +pub fn sample_image(ctx: EvalCtx, comp: ItemId, layer: &Layer, point: [f64; 2], radius: [f64; 2], post: bool, t: f64) -> Option<[f32; 4]> { + if !layer.source.is_av() { + return Some([0.0; 4]); + } + let Some(b) = render(ctx, comp, layer, post, t) else { return Some([0.0; 4]) }; + let to_px = |x: f64, y: f64| (x * b.scale + b.offset[0], y * b.scale + b.offset[1]); + let (rx, ry) = (radius[0].abs() * b.scale, radius[1].abs() * b.scale); + let (cx, cy) = to_px(point[0], point[1]); + // Pixels whose centres (at +0.5) fall in the box; at least the one under the point. + let (x0, x1, y0, y1) = if rx < 0.5 && ry < 0.5 { + let (px, py) = (cx.floor() as i64, cy.floor() as i64); + (px, px, py, py) + } else { + let lo = |c: f64, r: f64| (c - r - 0.5).ceil() as i64; + let hi = |c: f64, r: f64| (c + r - 0.5).floor() as i64; + (lo(cx, rx), hi(cx, rx).max(lo(cx, rx)), lo(cy, ry), hi(cy, ry).max(lo(cy, ry))) + }; + let mut acc = [0.0f64; 4]; + let mut n = 0.0; + for y in y0..=y1.min(y0 + 4096) { + for x in x0..=x1.min(x0 + 4096) { + let p = b.img.get(x, y); + for k in 0..4 { + acc[k] += p[k] as f64; + } + n += 1.0; + } + } + if n == 0.0 { + return Some([0.0; 4]); + } + let a = acc[3] / n; + if a <= 1e-9 { + return Some([0.0; 4]); + } + Some([(acc[0] / n / a) as f32, (acc[1] / n / a) as f32, (acc[2] / n / a) as f32, a as f32]) +} diff --git a/crates/expr/src/tests.rs b/crates/expr/src/tests.rs index fd02acef..d75972f7 100644 --- a/crates/expr/src/tests.rs +++ b/crates/expr/src/tests.rs @@ -1019,7 +1019,7 @@ fn text_expression_selector_values() { let sel_of = |p: &Project| { let c = p.comp(cid).unwrap(); let l = c.layer(lid).unwrap(); - let ctx = EvalCtx { project: p, comp_id: cid, comp: c, time: Tick::ZERO, expr: Some(&Expressions) }; + let ctx = EvalCtx { project: p, comp_id: cid, comp: c, time: Tick::ZERO, expr: Some(&Expressions), footage: None }; let doc = effectcraft_render::text::source_text(&ctx, l).unwrap(); let lay = effectcraft_text::layout_doc(&doc); let anim = l.props.group("text/animators/#1").unwrap(); @@ -1051,3 +1051,79 @@ fn text_expression_selector_values() { let v = eval_property(&p, cid, lid, "text/animators/#1/selectors/#2/amount", 0.0).unwrap(); assert_eq!(v, Value::Vec3([100.0; 3])); } + +/// `sampleImage` reads the layer's rendered pixels (layer space, straight colour) after masks +/// and effects, or the bare source with `postEffect = false`; the radius averages a box. +#[test] +fn sample_image_reads_rendered_pixels() { + let f = fx(); + let code = |c: &str| format!("var c = {c}; c[0] * 1000 + c[1] * 100 + c[2] * 10 + c[3]"); + // B: 40×40 blue solid. + assert_close!(f.num(&code("thisComp.layer(\"B\").sampleImage([20, 20])"), 0.0), 11.0); + // Half the box is outside the layer: alpha 0.5, colour still pure blue (straight). + assert_close!(f.num(&code("thisComp.layer(\"B\").sampleImage([0, 20], [1, 1])"), 0.0), 10.5); + // A: red with a mask covering 5..15: outside the mask it's transparent after masks, red before. + assert_close!(f.num(&code("thisComp.layer(\"A\").sampleImage([2, 2])"), 0.0), 0.0); + assert_close!(f.num(&code("thisComp.layer(\"A\").sampleImage([2, 2], [0.5, 0.5], false)"), 0.0), 1001.0); + assert_close!(f.num(&code("thisComp.layer(\"A\").sampleImage([10, 10])"), 0.0), 1001.0); + // The result has four values; outside the layer it's transparent. + assert_close!(f.num("thisComp.layer(\"B\").sampleImage([20, 20]).length", 0.0), 4.0); + assert_close!(f.num(&code("thisComp.layer(\"B\").sampleImage([-50, -50])"), 0.0), 0.0); +} + +fn data_item(p: &mut Project, name: &str, text: &str) { + let f = effectcraft_project::Footage { + path: format!("/data/{name}"), + kind: effectcraft_project::FootageKind::Data, + data: Some(text.into()), + ..Default::default() + }; + p.add_item(name, Label::Sandstone, None, ItemKind::Footage(f)); +} + +/// Data-driven animation: `footage(name)` with `sourceData` / `sourceText` / `dataValue` for +/// JSON, CSV and TSV items. +#[test] +fn data_footage_json_csv_tsv() { + let mut f = fx(); + data_item(&mut f.p, "stats.json", r#"{"title": "Q3", "items": [{"v": 3}, {"v": 7.5}]}"#); + data_item(&mut f.p, "scores.csv", "name,score\n\"Smith, J\",12\nLee,30.5\n"); + data_item(&mut f.p, "table.tsv", "a\tb\n1\t2\n3\t4\n"); + assert_close!(f.num("footage(\"stats.json\").sourceData.items[1].v", 0.0), 7.5); + assert_eq!(f.text("footage(\"stats.json\").sourceData.title", 0.0), "Q3"); + assert_close!(f.num("footage(\"stats.json\").dataValue([\"items\", 0, \"v\"])", 0.0), 3.0); + assert_close!(f.num("footage(\"scores.csv\").sourceData[1].score", 0.0), 30.5); + assert_eq!(f.text("footage(\"scores.csv\").sourceData[0].name", 0.0), "Smith, J"); + assert_close!(f.num("footage(\"scores.csv\").dataValue([1, 0])", 0.0), 12.0); + assert_close!(f.num("footage(\"scores.csv\").dataKeyCount", 0.0), 2.0); + assert_eq!(f.text("footage(\"scores.csv\").dataKeyNames.join('|')", 0.0), "name|score"); + assert_close!(f.num("footage(\"table.tsv\").dataValue([1, 1])", 0.0), 4.0); + assert_close!(f.num("footage(\"table.tsv\").sourceText.length", 0.0), 12.0); + assert!(f.err("footage(\"missing.json\").sourceData").contains("does not exist")); + // Animated from data: the value follows the CSV row picked by time. + assert_close!(f.num("footage(\"scores.csv\").dataValue([1, Math.floor(time)])", 1.2), 30.5); +} + +#[test] +fn project_and_time_globals() { + let mut f = fx(); + assert_close!(f.num("thisProject.bitsPerChannel", 0.0), 8.0); + assert_close!(f.num("colorDepth", 0.0), 8.0); + assert_close!(f.num("thisProject.linearBlending ? 1 : 0", 0.0), 0.0); + assert_eq!(f.text("timeToFeetAndFrames(2)", 0.0), "3+12"); + assert_close!(f.num("posterizeTime(4); time", 0.6), 0.5); + // Path API. + assert_close!(f.num("createPath([[0, 0], [10, 0], [10, 10]], [], [], true).points()[2][1]", 0.0), 10.0); + assert_close!(f.num("createPath([[0, 0], [10, 0]], [[1, 1], [2, 2]], [], false).inTangents()[1][0]", 0.0), 2.0); + assert_close!(f.num("mask(\"Mask 1\").maskPath.points().length", 0.0), 4.0); + assert_close!(f.num("mask(\"Mask 1\").maskPath.isClosed() ? 1 : 0", 0.0), 1.0); + // Marker keys carry protected regions and cue points. + { + let a = f.a; + let l = f.layer_mut(a); + l.markers[0].protected = true; + l.markers[0].cue_point = Some(effectcraft_project::CuePoint { name: "go".into(), navigation: false, params: vec![("k".into(), "v".into())] }); + } + assert_close!(f.num("thisLayer.marker.key(1).protectedRegion ? 1 : 0", 0.0), 1.0); + assert_eq!(f.text("thisComp.layer(\"A\").marker.key(1).cuePointName + thisComp.layer(\"A\").marker.key(1).parameters.k", 0.0), "gov"); +} diff --git a/crates/gpu/src/tests_adv3d.rs b/crates/gpu/src/tests_adv3d.rs index 067e4c8a..f0e0f08b 100644 --- a/crates/gpu/src/tests_adv3d.rs +++ b/crates/gpu/src/tests_adv3d.rs @@ -91,7 +91,7 @@ fn scene_project() -> (Project, ItemId) { fn scene(p: &Project, cid: ItemId) -> Scene { let r = Renderer::new(p, &NoFootage, RenderOpts::default()); - let ctx = EvalCtx { project: p, comp_id: cid, comp: p.comp(cid).unwrap(), time: Tick::ZERO, expr: None }; + let ctx = EvalCtx { project: p, comp_id: cid, comp: p.comp(cid).unwrap(), time: Tick::ZERO, expr: None, footage: None }; let run: Vec<&effectcraft_project::Layer> = ctx.comp.layers.iter().rev().filter(|l| l.is_3d() && l.has_video()).collect(); adv::scene_of(&r, &ctx, &run, (ctx.comp.width, ctx.comp.height)) } diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index d8906a1a..dccf1aac 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -183,7 +183,8 @@ mod tests { let cid = s.active_comp_id().unwrap(); let comp = s.project.comp(cid).unwrap(); let layer = comp.layer(effectcraft_engine::project::LayerId(l)).unwrap(); - let ctx = effectcraft_engine::render::EvalCtx { project: &s.project, comp_id: cid, comp, time: Default::default(), expr: s.expr.as_deref() }; + let ctx = + effectcraft_engine::render::EvalCtx { project: &s.project, comp_id: cid, comp, time: Default::default(), expr: s.expr.as_deref(), footage: None }; ctx.value(layer, layer.props.prop(path).unwrap()) } @@ -221,6 +222,81 @@ mod tests { assert_eq!(value(&s, a, "transform/rotation").as_f64(), 12.0); } + /// The expression error bar's list: evaluation errors and syntax errors, with the layer and + /// property they belong to. + #[test] + fn expression_errors_are_collected() { + let mut s = super::session(); + s.execute("comp.new", json!({"name": "C", "width": 100, "height": 100, "frameRate": 30, "duration": 2})).unwrap(); + let a = s.execute("layer.newSolid", json!({"name": "Box", "color": "#ffffff"})).unwrap()["layer"].as_u64().unwrap(); + s.execute("prop.setExpression", json!({"layer": a, "path": "transform/opacity", "expression": "50"})).unwrap(); + assert_eq!(s.execute("expr.errors", json!({})).unwrap()["count"], 0); + s.execute("prop.setExpression", json!({"layer": a, "path": "transform/rotation", "expression": "thisComp.layer(\"Nope\").rotation"})).unwrap(); + s.execute("prop.setExpression", json!({"layer": a, "path": "transform/opacity", "expression": "1 +* ("})).unwrap(); + let r = s.execute("expr.errors", json!({})).unwrap(); + assert_eq!(r["count"], 2, "{r}"); + let errs = r["errors"].as_array().unwrap(); + let rot = errs.iter().find(|e| e["path"] == "transform/rotation").unwrap(); + assert!(rot["message"].as_str().unwrap().contains("Nope"), "{rot}"); + assert_eq!(rot["layerIndex"], 1); + assert_eq!(rot["layerName"], "Box"); + assert_eq!(rot["disabled"], false); + let op = errs.iter().find(|e| e["path"] == "transform/opacity").unwrap(); + assert_eq!(op["disabled"], true); + assert!(r["text"][0].as_str().unwrap().contains("of layer 1 ('Box') in comp 'C'"), "{r}"); + // A user-disabled (but valid) expression is not an error. + s.execute("prop.setExpression", json!({"layer": a, "path": "transform/rotation", "expression": "45"})).unwrap(); + s.execute("prop.setExpression", json!({"layer": a, "path": "transform/rotation", "enabled": false})).unwrap(); + assert_eq!(s.execute("expr.errors", json!({})).unwrap()["count"], 1); + // The Expression Language menu is categorised. + let m = s.execute("expr.languageMenu", json!({})).unwrap(); + assert!(m.as_array().unwrap().iter().any(|c| c["category"] == "Footage" && c["items"].as_array().unwrap().iter().any(|i| i["text"] == "sourceData"))); + } + + /// File ▸ Import of JSON/CSV data files as data footage, read by expressions. + #[test] + fn imported_data_files_drive_expressions() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/test-out/host-data"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("levels.csv"), "label,value\nlow,10\nhigh,90\n").unwrap(); + std::fs::write(dir.join("cfg.json"), r#"{"angle": 33}"#).unwrap(); + std::fs::write(dir.join("broken.json"), "{nope").unwrap(); + let mut s = super::session(); + s.execute("comp.new", json!({"name": "C", "width": 100, "height": 100, "frameRate": 30, "duration": 2})).unwrap(); + let paths: Vec = ["levels.csv", "cfg.json", "broken.json"].iter().map(|n| dir.join(n).to_string_lossy().to_string()).collect(); + let r = s.execute("file.import", json!({"paths": paths})).unwrap(); + assert_eq!(r["items"].as_array().unwrap().len(), 2, "{r}"); + assert_eq!(r["errors"].as_array().unwrap().len(), 1, "{r}"); + let csv = r["items"][0].as_u64().unwrap(); + assert_eq!(s.project.item(effectcraft_engine::project::ItemId(csv)).unwrap().type_name(), "Data"); + assert!(s.execute("layer.addItem", json!({"item": csv})).is_err(), "data files are not layers"); + let a = s.execute("layer.newSolid", json!({"color": "#ffffff"})).unwrap()["layer"].as_u64().unwrap(); + s.execute("prop.setExpression", json!({"layer": a, "path": "transform/opacity", "expression": "footage(\"levels.csv\").sourceData[1].value"})).unwrap(); + s.execute("prop.setExpression", json!({"layer": a, "path": "transform/rotation", "expression": "footage(\"cfg.json\").sourceData.angle"})).unwrap(); + assert_eq!(value(&s, a, "transform/opacity").as_f64(), 90.0); + assert_eq!(value(&s, a, "transform/rotation").as_f64(), 33.0); + } + + /// sampleImage through the renderer: a box's Fill colour follows the solid under it. + #[test] + fn sample_image_drives_a_colour() { + let mut s = super::session(); + s.execute("comp.new", json!({"name": "C", "width": 100, "height": 100, "frameRate": 30, "duration": 2})).unwrap(); + s.execute("layer.newSolid", json!({"name": "Bg", "color": "#00ff00"})).unwrap(); + let a = s.execute("layer.newSolid", json!({"name": "Fg", "color": "#ffffff", "width": 20, "height": 20})).unwrap()["layer"].as_u64().unwrap(); + s.execute("effect.apply", json!({"layer": a, "effect": "Fill"})).unwrap(); + s.execute( + "prop.setExpression", + json!({"layer": a, "path": "effects/#1/color", "expression": "thisComp.layer(\"Bg\").sampleImage(thisComp.layer(\"Bg\").fromComp(toComp(anchorPoint)))"}), + ) + .unwrap(); + let cid = s.active_comp_id().unwrap(); + let img = s.render(cid, s.time(), effectcraft_engine::render::RenderOpts::default()); + let px = img.get(50, 50); + assert!(px[1] > 0.99 && px[0] < 0.01, "the box is filled with the sampled green: {px:?}"); + } + #[test] fn expression_syntax_errors_disable_the_expression() { let mut s = super::session(); diff --git a/crates/render/src/eval.rs b/crates/render/src/eval.rs index 7bf67774..aff4d93c 100644 --- a/crates/render/src/eval.rs +++ b/crates/render/src/eval.rs @@ -36,11 +36,14 @@ pub struct EvalCtx<'a> { /// Comp time. pub time: Tick, pub expr: Option<&'a dyn ExprHost>, + /// Footage frames for expressions that read rendered pixels (`sampleImage`): the renderer's + /// own source while rendering; `None` renders footage as transparent. + pub footage: Option<&'a dyn crate::FootageSource>, } impl<'a> EvalCtx<'a> { pub fn new(project: &'a Project, comp_id: ItemId, comp: &'a Comp, time: Tick) -> EvalCtx<'a> { - EvalCtx { project, comp_id, comp, time, expr: None } + EvalCtx { project, comp_id, comp, time, expr: None, footage: None } } pub fn at(&self, time: Tick) -> EvalCtx<'a> { EvalCtx { time, ..*self } diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 5e7425bb..6699d68d 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -451,7 +451,7 @@ impl<'a> Renderer<'a> { } fn ctx(&self, comp_id: ItemId, comp: &'a Comp, t: Tick) -> EvalCtx<'a> { - EvalCtx { project: self.project, comp_id, comp, time: t, expr: self.expr } + EvalCtx { project: self.project, comp_id, comp, time: t, expr: self.expr, footage: Some(self.footage) } } /// The accelerator to use for this render, if any (see [`Backend`]). diff --git a/crates/render/src/three_d/adv/tests.rs b/crates/render/src/three_d/adv/tests.rs index 47ad6178..b1d94413 100644 --- a/crates/render/src/three_d/adv/tests.rs +++ b/crates/render/src/three_d/adv/tests.rs @@ -303,7 +303,7 @@ fn extruded_text_renders_as_a_mesh() { set(l, "geometryOptions/bevelStyle", Value::Enum(1)); set(l, "transform/rotationY", Value::Scalar(40.0)); }); - let ctx = crate::EvalCtx { project: &p, comp_id: cid, comp: p.comp(cid).unwrap(), time: Tick::ZERO, expr: None }; + let ctx = crate::EvalCtx { project: &p, comp_id: cid, comp: p.comp(cid).unwrap(), time: Tick::ZERO, expr: None, footage: None }; let l = &ctx.comp.layers[0]; let params = scene::extrusion_params(&ctx, l).unwrap(); let meshes = scene::extruded_meshes(&ctx, l, ¶ms); diff --git a/crates/ui-egui/src/panels/effect_controls.rs b/crates/ui-egui/src/panels/effect_controls.rs index 0a19d57f..70d56237 100644 --- a/crates/ui-egui/src/panels/effect_controls.rs +++ b/crates/ui-egui/src/panels/effect_controls.rs @@ -793,7 +793,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let comp_name = app.session.project.item(cid).map(|i| i.name.clone()).unwrap_or_default(); let snap_project = app.session.project.clone(); let snap_expr = app.session.expr.clone(); - let ectx = EvalCtx { project: &snap_project, comp_id: cid, comp: &comp, time: app.session.time(), expr: snap_expr.as_deref() }; + let ectx = EvalCtx { project: &snap_project, comp_id: cid, comp: &comp, time: app.session.time(), expr: snap_expr.as_deref(), footage: None }; let hdr = Rect::from_min_size(rect.min, vec2(rect.width(), 24.0)); p.text(pos2(hdr.min.x + 10.0, hdr.center().y), Align2::LEFT_CENTER, format!("{} • {}", comp_name, layer.name), Tokens::ui(11.5), t.text_dim); p.line_segment([hdr.left_bottom(), hdr.right_bottom()], Stroke::new(1.0, t.separator)); diff --git a/crates/ui-egui/src/panels/layer_panel.rs b/crates/ui-egui/src/panels/layer_panel.rs index e53e268e..5149241e 100644 --- a/crates/ui-egui/src/panels/layer_panel.rs +++ b/crates/ui-egui/src/panels/layer_panel.rs @@ -119,7 +119,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let tex = match cached { Some(c) if c.key == key => Some(c), _ => { - let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp: &comp, time, expr: app.session.expr.as_deref() }; + let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp: &comp, time, expr: app.session.expr.as_deref(), footage: None }; let mut r = Renderer::new(&app.session.project, &*app.session.footage, RenderOpts { scale, ..Default::default() }); r.expr = app.session.expr.as_deref(); r.cache = Some(&app.session.layer_cache); diff --git a/crates/ui-egui/src/panels/properties.rs b/crates/ui-egui/src/panels/properties.rs index fbd4e5e9..c5795685 100644 --- a/crates/ui-egui/src/panels/properties.rs +++ b/crates/ui-egui/src/panels/properties.rs @@ -69,7 +69,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let project = app.session.project.clone(); let expr = app.session.expr.clone(); let now = app.session.time(); - let ectx = EvalCtx { project: &project, comp_id: cid, comp: &comp, time: now, expr: expr.as_deref() }; + let ectx = EvalCtx { project: &project, comp_id: cid, comp: &comp, time: now, expr: expr.as_deref(), footage: None }; let x0 = rect.min.x + PAD; let w = rect.width() - 2.0 * PAD; // Vertical scrolling: content height is known after drawing, so clamp with last frame's. diff --git a/crates/ui-egui/src/panels/text_panels.rs b/crates/ui-egui/src/panels/text_panels.rs index edb1132f..31f98bb0 100644 --- a/crates/ui-egui/src/panels/text_panels.rs +++ b/crates/ui-egui/src/panels/text_panels.rs @@ -56,7 +56,7 @@ pub fn text_target(app: &EffectcraftApp) -> Option { .iter() .filter_map(|id| comp.layer(*id)) .find(|l| matches!(l.source, effectcraft_engine::project::LayerSource::Text))?; - let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp, time: app.session.time(), expr: app.session.expr.as_deref() }; + let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp, time: app.session.time(), expr: app.session.expr.as_deref(), footage: None }; effectcraft_engine::render::text::source_text(&ectx, layer).map(|d| TextTarget { layer: layer.id.0, doc: d, range: None }) } @@ -453,7 +453,7 @@ pub fn align_layers(app: &mut EffectcraftApp, op: &str) { let time = app.session.time(); let mut moves = vec![]; { - let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp: &comp, time, expr: app.session.expr.as_deref() }; + let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp: &comp, time, expr: app.session.expr.as_deref(), footage: None }; for l in comp.layers.iter().filter(|l| app.session.state.selected_layers.contains(&l.id)) { let Some(b) = effectcraft_engine::render::content_bounds(&ectx, l) else { continue }; let (m, _) = ectx.layer_to_comp(l); diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index c9a42698..cd3bf6f9 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -687,7 +687,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { app.ui.timeline.scroll_y = app.ui.timeline.scroll_y.min(max_scroll); let snap_project = app.session.project.clone(); let snap_expr = app.session.expr.clone(); - let ectx = EvalCtx { project: &snap_project, comp_id: cid, comp: &comp, time, expr: snap_expr.as_deref() }; + let ectx = EvalCtx { project: &snap_project, comp_id: cid, comp: &comp, time, expr: snap_expr.as_deref(), footage: None }; let lp = p.with_clip_rect(rows_rect.intersect(Rect::from_min_max(rows_rect.min, pos2(graph_x0 - 1.0, rows_rect.max.y)))); let gp = p.with_clip_rect(rows_rect.intersect(Rect::from_min_max(pos2(graph_x0, rows_rect.min.y), rows_rect.max))); let mut y = rows_rect.min.y - app.ui.timeline.scroll_y; diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index d847edb8..76ef92a9 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -394,7 +394,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } let snap_project = app.session.project.clone(); let snap_expr = app.session.expr.clone(); - let ectx = EvalCtx { project: &snap_project, comp_id: cid, comp: &comp, time, expr: snap_expr.as_deref() }; + let ectx = EvalCtx { project: &snap_project, comp_id: cid, comp: &comp, time, expr: snap_expr.as_deref(), footage: None }; // The frame (or snapshot) through Show Channel and exposure; ROI frames cover the region. vt::draw_frame(app, &ctx, &painter, comp_rect, cid, &ectx); painter.rect_stroke(comp_rect, 0.0, Stroke::new(1.0, Color32::from_black_alpha(160)), StrokeKind::Outside); @@ -891,6 +891,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { comp: app.session.project.comp(cid).unwrap_or(&comp), time, expr: app.session.expr.as_deref(), + footage: None, }; if let Some(m) = l2c(&e2, &l).0.inverse() { *i2 = m; @@ -1396,6 +1397,7 @@ fn create_shape(app: &mut EffectcraftApp, tool: Tool, a: [f64; 2], b: [f64; 2], comp: &comp, time: app.session.time(), expr: None, + footage: None, }; let inv = l2c(&ectx, &l).0.inverse().unwrap_or(Mat3::IDENTITY); let p0 = inv.apply(gv2(cx - w / 2.0, cy - h / 2.0)); From cba8d7e737db9438c9b0e46fd882875df5e7946a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 23:02:39 -0400 Subject: [PATCH 07/13] M12.6: 3D Camera Tracker effect + UI Effect > Perspective > 3D Camera Tracker (Shot Type, Horizontal Angle of View, Show/Render Track Points, Track Point Size, Target Size, Advanced: Solve Method, Method Used, Average Error, Detailed Analysis, Auto-delete Points Across Time, Hide Warning Banner); tracks and solve stored in the effect with keys (frame edits re-track, setting / deleted-point edits only re-solve). Background job (Analyzing in background (step 1 of 2) / Solving camera banner, cancel, one undo step). Commands: track.camera (replaces the stub), camera.analyze/cancel/solveStatus/points/selectPoints/ createFromSolve/create/setGroundPlane/deletePoints; the one-node 3D Tracker Camera keyed per frame and text / solids / nulls / shadow catcher + light on the target plane, Create Multiple. Viewer: depth-sized coloured targets, hover target plane, click / shift / marquee selection, AE right-click menu, Delete; Effect Controls Analyze / Cancel / Create Camera; Tracker panel Track Camera. Solver: adaptive noise thresholds, denser keyframes, quick tripod check, faster Cholesky and parallel Schur complement. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + ROADMAP.md | 4 +- crates/effects/src/camera_tracker.rs | 291 ++++++++ crates/effects/src/lib.rs | 4 + crates/engine/src/camera_track.rs | 640 ++++++++++++++++++ crates/engine/src/commands/camera_cmds.rs | 574 ++++++++++++++++ crates/engine/src/commands/mod.rs | 2 + crates/engine/src/commands/stubs.rs | 1 - crates/engine/src/lib.rs | 22 + crates/engine/src/tests_camera_track.rs | 307 +++++++++ crates/engine/src/tests_stubs.rs | 2 +- crates/track/Cargo.toml | 1 + crates/track/src/camtrack/bundle.rs | 90 ++- crates/track/src/camtrack/features.rs | 2 +- crates/track/src/camtrack/linalg.rs | 30 +- crates/track/src/camtrack/mod.rs | 8 +- crates/track/src/camtrack/solver.rs | 83 ++- crates/track/src/camtrack/tests.rs | 78 +++ crates/ui-egui/src/lib.rs | 5 + .../ui-egui/src/panels/camera_tracker_ui.rs | 438 ++++++++++++ crates/ui-egui/src/panels/effect_controls.rs | 2 + crates/ui-egui/src/panels/mod.rs | 1 + crates/ui-egui/src/panels/tracker.rs | 7 +- crates/ui-egui/src/panels/viewer.rs | 3 + crates/ui-egui/tests/ui_camera_tracker.rs | 175 +++++ docs/agents.md | 28 + docs/architecture.md | 16 + docs/parity.md | 10 +- 28 files changed, 2771 insertions(+), 54 deletions(-) create mode 100644 crates/effects/src/camera_tracker.rs create mode 100644 crates/engine/src/camera_track.rs create mode 100644 crates/engine/src/commands/camera_cmds.rs create mode 100644 crates/engine/src/tests_camera_track.rs create mode 100644 crates/ui-egui/src/panels/camera_tracker_ui.rs create mode 100644 crates/ui-egui/tests/ui_camera_tracker.rs diff --git a/Cargo.lock b/Cargo.lock index 46f34e63..ce792ecb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1420,6 +1420,7 @@ dependencies = [ "rayon", "serde", "serde_json", + "web-time", ] [[package]] diff --git a/ROADMAP.md b/ROADMAP.md index a3ed979c..7780c164 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -28,10 +28,10 @@ once. Measured feature by feature in [docs/parity.md](docs/parity.md) (end of 2 October 2026): - **≈ 79% of After Effects' features, weighted by importance** (87% of the essentials, 67% of - professional daily-use features, 21% of the long tail); 257 of 298 effects, 16 of them on the GPU. + professional daily-use features, 21% of the long tail); 258 of 298 effects, 16 of them on the GPU. - **≈ 100 agent-hours of work remain by the audit's estimate.** With five Claude Opus 5.5 agents in parallel that is **≈ 8–10 hours of wall-clock time at the pace measured so far, up to ≈ 25 hours** by the audit's conservative per-feature figures. -- Hardest remaining pieces: Roto Brush, the 3D camera tracker, Advanced 3D, a scripting object model. +- Hardest remaining pieces: Roto Brush, Advanced 3D, a scripting object model (the 3D Camera Tracker landed). Come tell us what matters most to you on [Discord](https://discord.gg/artcraft). diff --git a/crates/effects/src/camera_tracker.rs b/crates/effects/src/camera_tracker.rs new file mode 100644 index 00000000..dd97b585 --- /dev/null +++ b/crates/effects/src/camera_tracker.rs @@ -0,0 +1,291 @@ +//! 3D Camera Tracker (Effect ▸ Perspective ▸ 3D Camera Tracker; Animation ▸ Track Camera). +//! +//! The analysis runs as a background job in the engine (`camera.analyze`): step 1 tracks +//! features through the clip ([`effectcraft_track::camtrack::TrackAnalyzer`]), step 2 solves +//! the camera ([`effectcraft_track::camtrack::solve`]). Both results are stored as JSON in the +//! instance's hidden parameters — **Tracks** (with **Tracks Key**: what the frames were made +//! from, so source / In-Out / time edits invalidate them) and **Solve** (with **Solve Key**: the +//! Shot Type, Angle of View, Solve Method and deleted points it was solved with, so changing +//! those re-solves without re-tracking). **Method Used** and **Average Error** report the solve. +//! +//! The effect passes its input through; with **Render Track Points** on it draws the solved +//! points (as the viewer shows them) into the frame. The viewer overlay, target and the Create +//! commands live in the engine and the UI. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::{Image, Px}; +use effectcraft_track::camtrack::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, linalg, point_color}; + +use crate::warp_stab::param; +use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; + +/// The effect's id. +pub const ID: &str = "ec.perspective.cameratracker"; + +/// Hidden parameters (see the module docs). +pub const TRACKS: &str = "tracks"; +pub const TRACKS_KEY: &str = "tracksKey"; +pub const SOLVE: &str = "solve"; +pub const SOLVE_KEY: &str = "solveKey"; +/// Deleted track point ids (JSON array). +pub const DELETED: &str = "deleted"; +/// The "3D Tracker Camera" layer the instance created (layer id, 0 = none). +pub const CAMERA: &str = "camera"; +pub const METHOD_USED: &str = "advanced/methodUsed"; +pub const AVERAGE_ERROR: &str = "advanced/averageError"; + +pub const SHOT_TYPES: [&str; 3] = ["Fixed Angle of View", "Variable Zoom", "Specify Angle of View"]; +pub const SHOW_POINTS: [&str; 2] = ["2D Source", "3D Solved"]; +pub const SOLVE_METHODS: [&str; 4] = ["Auto Detect", "Typical", "Mostly Flat Scene", "Tripod Pan"]; + +pub fn specs() -> Vec { + vec![EffectSpec { + id: ID, + name: "3D Camera Tracker", + category: "Perspective", + params: vec![ + p("shotType", "Shot Type", Value::Enum(0), popup(&SHOT_TYPES)), + p("horizontalAngleOfView", "Horizontal Angle of View", num(40.0), slider(1.0, 170.0, 10.0, 120.0, 1)), + p("showTrackPoints", "Show Track Points", Value::Enum(1), popup(&SHOW_POINTS)), + p("renderTrackPoints", "Render Track Points", Value::Bool(false), ParamUi::Checkbox), + p("trackPointSize", "Track Point Size", num(100.0), slider(0.0, 1000.0, 0.0, 300.0, 0)), + p("targetSize", "Target Size", num(100.0), slider(1.0, 1000.0, 10.0, 300.0, 0)), + p("advanced/solveMethod", "Solve Method", Value::Enum(0), popup(&SOLVE_METHODS)), + p(METHOD_USED, "Method Used", Value::Str(String::new()), ParamUi::Hidden), + p(AVERAGE_ERROR, "Average Error", num(0.0), ParamUi::Hidden), + p("advanced/detailedAnalysis", "Detailed Analysis", Value::Bool(false), ParamUi::Checkbox), + p("advanced/autoDeletePoints", "Auto-delete Points Across Time", Value::Bool(true), ParamUi::Checkbox), + p("advanced/hideWarningBanner", "Hide Warning Banner", Value::Bool(false), ParamUi::Checkbox), + p(TRACKS, "Tracks", Value::Str(String::new()), ParamUi::Hidden), + p(TRACKS_KEY, "Tracks Key", Value::Str(String::new()), ParamUi::Hidden), + p(SOLVE, "Solve", Value::Str(String::new()), ParamUi::Hidden), + p(SOLVE_KEY, "Solve Key", Value::Str(String::new()), ParamUi::Hidden), + p(DELETED, "Deleted Points", Value::Str(String::new()), ParamUi::Hidden), + p(CAMERA, "Camera Layer", num(0.0), ParamUi::Hidden), + ], + render, + gpu: false, + float: true, + }] +} + +fn f(params: &Params, id: &str) -> f64 { + param(params, id).map(Value::as_f64).unwrap_or(0.0) +} +fn e(params: &Params, id: &str) -> u32 { + param(params, id).map(Value::as_enum).unwrap_or(0) +} +fn b(params: &Params, id: &str) -> bool { + param(params, id).map(Value::as_bool).unwrap_or(false) +} +fn s<'a>(params: &'a Params, id: &str) -> &'a str { + match param(params, id) { + Some(Value::Str(s)) => s, + _ => "", + } +} + +/// Deleted point ids of an instance (stored as a JSON array of numbers). +pub fn deleted(params: &Params) -> Vec { + s(params, DELETED).trim().trim_start_matches('[').trim_end_matches(']').split(',').filter_map(|v| v.trim().parse().ok()).collect() +} + +/// The solve settings of an instance. +pub fn settings(params: &Params) -> SolveSettings { + let mut del = deleted(params); + del.sort_unstable(); + del.dedup(); + SolveSettings { + shot: ShotType::ALL.get(e(params, "shotType") as usize).copied().unwrap_or_default(), + hfov: f(params, "horizontalAngleOfView"), + method: SolveMethod::ALL.get(e(params, "advanced/solveMethod") as usize).copied().unwrap_or_default(), + deleted: del, + } +} + +/// What a solve depends on besides the tracks. +pub fn solve_key(st: &SolveSettings) -> String { + let hfov = if st.shot == ShotType::SpecifyAngle { format!("{:.4}", st.hfov) } else { String::new() }; + format!("{:?}|{hfov}|{:?}|{:?}", st.shot, st.method, st.deleted) +} + +fn fnv(bytes: &[u8]) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for b in bytes { + h ^= *b as u64; + h = h.wrapping_mul(0x0100_0000_01b3); + } + h +} + +type Cache = Mutex>>; + +fn cached(c: &'static OnceLock>, json: &str, parse: impl Fn(&str) -> Option) -> Option> { + let key = fnv(json.as_bytes()); + let c = c.get_or_init(Default::default); + if let Some(a) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&key) { + return Some(a.clone()); + } + let a = Arc::new(parse(json)?); + let mut m = c.lock().unwrap_or_else(|e| e.into_inner()); + if m.len() > 8 { + m.clear(); + } + m.insert(key, a.clone()); + Some(a) +} + +/// The stored feature tracks of an instance (`None` when not analysed). +pub fn tracks(params: &Params) -> Option> { + static C: OnceLock> = OnceLock::new(); + let j = s(params, TRACKS); + if j.is_empty() { + return None; + } + cached(&C, j, CameraTracks::from_json).filter(|t| !t.is_empty()) +} + +/// The stored camera solve of an instance (`None` when not solved). +pub fn solve(params: &Params) -> Option> { + static C: OnceLock> = OnceLock::new(); + let j = s(params, SOLVE); + if j.is_empty() { + return None; + } + cached(&C, j, CameraSolve::from_json).filter(|t| !t.is_empty()) +} + +/// Screen size (layer pixels) of a target drawn for a point at depth `z`, relative to the +/// median depth of the frame's points. +pub fn target_radius(size_pct: f64, z: f64, median_z: f64) -> f64 { + (6.0 * size_pct / 100.0 * (median_z / z.max(1e-9))).clamp(1.5, 40.0) +} + +/// The solved points visible on frame `k`: (id, layer pixel position, camera depth). +pub fn projected(solve: &CameraSolve, k: usize) -> Vec<(u32, [f64; 2], f64)> { + let Some(cam) = solve.frames.get(k) else { return vec![] }; + solve + .visible(k) + .filter_map(|p| { + let c = cam.to_cam(p.pos); + cam.project(solve.size, p.pos).map(|q| (p.id, q, c[2])) + }) + .collect() +} + +/// Median of the depths. +pub fn median_depth(v: &[(u32, [f64; 2], f64)]) -> f64 { + let mut z: Vec = v.iter().map(|x| x.2).collect(); + if z.is_empty() { + return 1.0; + } + z.sort_by(f64::total_cmp); + z[z.len() / 2] +} + +fn render(ctx: &EffectCtx, mut buf: Buf) -> Buf { + if !b(ctx.params, "renderTrackPoints") { + return buf; + } + let Some(sol) = solve(ctx.params) else { return buf }; + if (sol.size[0] - ctx.layer_size[0]).abs() > 0.5 || (sol.size[1] - ctx.layer_size[1]).abs() > 0.5 { + return buf; + } + let Some(k) = sol.frame_at(ctx.time) else { return buf }; + let pts = projected(&sol, k); + let med = median_depth(&pts); + let size = f(ctx.params, "trackPointSize"); + let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + for (id, q, z) in pts { + let c = point_color(id); + let r = target_radius(size, z, med) * sc; + let at = [q[0] * sc + buf.offset[0], q[1] * sc + buf.offset[1]]; + ring(&mut buf.img, at, r, (r * 0.25).max(1.0), [c[0], c[1], c[2], 1.0]); + disc(&mut buf.img, at, (r * 0.2).max(0.8), [c[0], c[1], c[2], 1.0]); + } + buf +} + +fn lerp_px(a: Px, b: Px, t: f32) -> Px { + [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t, a[2] + (b[2] - a[2]) * t, a[3] + (b[3] - a[3]) * t] +} + +fn cover(img: &mut Image, c: [f64; 2], r_out: f64, cov: impl Fn(f64) -> f64, col: Px) { + let (x0, x1) = ((c[0] - r_out - 1.0).floor().max(0.0) as i64, (c[0] + r_out + 1.0).ceil() as i64); + let (y0, y1) = ((c[1] - r_out - 1.0).floor().max(0.0) as i64, (c[1] + r_out + 1.0).ceil() as i64); + for y in y0..y1.min(img.height as i64) { + for x in x0..x1.min(img.width as i64) { + let d = (x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1]); + let a = cov(d).clamp(0.0, 1.0) as f32; + if a > 0.0 { + let i = img.idx(x as u32, y as u32); + img.data[i] = lerp_px(img.data[i], col, a); + } + } + } +} + +fn disc(img: &mut Image, c: [f64; 2], r: f64, col: Px) { + cover(img, c, r, |d| r + 0.5 - d, col); +} + +fn ring(img: &mut Image, c: [f64; 2], r: f64, w: f64, col: Px) { + cover(img, c, r + w, |d| (w * 0.5 + 0.5 - (d - r).abs()).min(1.0), col); +} + +/// Points of a target disc (canonical coordinates) for drawing: `n` points on the circle of +/// radius `size / 2` in the plane through `center` with normal `normal`. +pub fn target_circle(center: linalg::V3, normal: linalg::V3, radius: f64, n: usize) -> Vec { + let a = if normal[0].abs() < 0.9 { [1.0, 0.0, 0.0] } else { [0.0, 1.0, 0.0] }; + let u = linalg::normalize(linalg::cross(normal, a)); + let v = linalg::cross(normal, u); + (0..n) + .map(|i| { + let t = i as f64 / n as f64 * std::f64::consts::TAU; + linalg::add(center, linalg::add(linalg::scale(u, radius * t.cos()), linalg::scale(v, radius * t.sin()))) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::EffectEnv; + use effectcraft_track::camtrack::{SolvedFrame, SolvedPoint}; + + fn params_with(solve: &CameraSolve, render: bool) -> Params { + let mut p = Params::default(); + p.values.insert(SOLVE.into(), Value::Str(solve.to_json())); + p.values.insert("renderTrackPoints".into(), Value::Bool(render)); + p.values.insert("trackPointSize".into(), Value::Scalar(100.0)); + p + } + + #[test] + fn render_track_points_draws_targets_and_passes_through_otherwise() { + let sol = CameraSolve { + version: 1, + size: [64.0, 48.0], + start: 0.0, + frame_duration: 1.0 / 30.0, + frames: vec![SolvedFrame { rot: [0.0; 3], center: [0.0; 3], focal: 50.0, solved: true }], + points: vec![SolvedPoint { id: 3, pos: [0.0, 0.0, 1.0], error: 0.1, first: 0, last: 0 }], + ..Default::default() + }; + let img = Image::new(64, 48); + for on in [false, true] { + let params = params_with(&sol, on); + let ctx = EffectCtx { params: ¶ms, time: 0.0, layer_size: [64.0, 48.0], seed: 0, adjustment: false, env: EffectEnv::default() }; + let out = render(&ctx, Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 }); + let px = out.img.data[out.img.idx(32, 24)]; + assert_eq!(px[3] > 0.5, on); + } + let st = settings(&Params::default()); + assert_eq!(st.shot, ShotType::FixedAngle); + assert!(solve_key(&st).contains("FixedAngle")); + } +} diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 6d11e19a..9c2aefb5 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -8,6 +8,7 @@ pub mod audio_fx; mod blur2; +pub mod camera_tracker; mod channel; mod channel2; mod color2; @@ -354,6 +355,7 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(paint::specs()); v.extend(puppet::specs()); v.extend(warp_stab::specs()); + v.extend(camera_tracker::specs()); v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); for s in v.iter_mut() { if GPU_EFFECTS.contains(&s.id) { @@ -494,6 +496,8 @@ pub const TIME_DEPENDENT: &[&str] = &[ "ec.time.pixelmotionblur", // Each frame gets its own stabilizing warp. warp_stab::ID, + // Render Track Points draws each frame's solved points. + camera_tracker::ID, ]; /// See [`TIME_DEPENDENT`]. diff --git a/crates/engine/src/camera_track.rs b/crates/engine/src/camera_track.rs new file mode 100644 index 00000000..1b984aac --- /dev/null +++ b/crates/engine/src/camera_track.rs @@ -0,0 +1,640 @@ +//! 3D Camera Tracker runtime: the analysis job (`camera.analyze`, `track.camera`), keeping +//! analyses valid, and turning a solve into composition layers (`camera.createFromSolve`). +//! +//! The job renders the layer's input to the effect (`Renderer::layer_input`) for every frame +//! between the layer's In and Out points on a background thread and tracks features through +//! them (step 1 of 2, "Analyzing in background"), then solves the camera (step 2, "Solving +//! camera"). When only the solve settings changed (Shot Type, Angle of View, Solve Method, +//! deleted points) the stored tracks are re-solved without re-tracking. Results are written into +//! the effect's hidden parameters as one undo step; edits that change the layer's frames clear +//! them and queue the effect again ([`invalidate`], run after every edit), like After Effects. +//! +//! Composition space: the solve's canonical frame maps to comp world by +//! [`CameraSolve::world`] with the zoom of the first solved frame (focal length × the layer's +//! scale in the comp), so the "3D Tracker Camera" reproduces the footage's view and created +//! layers sit on the selected points. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use effectcraft_effects::camera_tracker::{self as ct, AVERAGE_ERROR, METHOD_USED, SOLVE, SOLVE_KEY, TRACKS, TRACKS_KEY}; +use effectcraft_keyframe::{Keyframe, Value}; +use effectcraft_project::{AutoOrient, GroupKind, ItemId, ItemKind, Layer, LayerId, Project, PropGroup, Uid}; +use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; +use effectcraft_time::Tick; +use effectcraft_track::camtrack::linalg::{self, Similarity, V3}; +use effectcraft_track::camtrack::{AnalyzeOpts, CameraSolve, CameraTracks, SolveSettings, Target, TrackAnalyzer}; +use serde::Serialize; + +use crate::tracking::lock; +use crate::warp::{analysis_times, signature}; +use crate::{Event, Session}; + +/// Live analysis progress (serde for agents / the control channel). +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CameraProgress { + /// 1 = tracking features, 2 = solving the camera. + pub step: u8, + pub done: u64, + pub total: u64, + pub elapsed: f64, + pub fps: f64, + pub finished: bool, + pub cancelled: bool, + pub error: Option, + /// Re-solving stored tracks (no tracking step). + pub solve_only: bool, +} + +impl CameraProgress { + /// The viewer banner text, as After Effects words it. + pub fn banner(&self) -> String { + match self.step { + 2 => "Solving camera".into(), + _ => { + let pc = (100 * self.done).checked_div(self.total).unwrap_or(0); + format!("Analyzing in background (step 1 of 2): {pc}%") + } + } + } +} + +struct CamResult { + /// New tracks (None for a re-solve). + tracks: Option, + solve: Result, +} + +#[derive(Default)] +pub struct CameraShared { + pub cancel: AtomicBool, + pub state: Mutex, + result: Mutex>, +} + +/// A running (or finished, not yet polled) camera analysis. +pub struct CameraJob { + pub shared: Arc, + thread: Option>, + pub comp: ItemId, + pub layer: LayerId, + pub effect: Uid, + /// The layer signature the tracks are made from, and the solve settings key. + key: String, + solve_key: String, +} + +impl CameraJob { + pub fn is_finished(&self) -> bool { + lock(&self.shared.state).finished + } + pub fn progress(&self) -> CameraProgress { + lock(&self.shared.state).clone() + } +} + +struct CamWork { + project: Arc, + footage: Arc, + expr: Option>, + cache: Arc, + comp: ItemId, + layer: LayerId, + effect_index: usize, + size: [f64; 2], + detailed: bool, + times: Vec, + /// Stored tracks to re-solve (skips step 1). + tracks: Option>, + settings: SolveSettings, +} + +pub(crate) fn is_tracker(g: &PropGroup) -> bool { + matches!(&g.kind, GroupKind::Effect { effect } if effect == ct::ID) +} + +fn str_of(g: &PropGroup, m: &str) -> String { + match g.prop(m).map(|p| &p.value) { + Some(Value::Str(s)) => s.clone(), + _ => String::new(), + } +} + +/// What a tracker's tracks are made from: the layer's frames and Detailed Analysis. +fn tracks_key(project: &Project, cid: ItemId, l: &Layer, g: &PropGroup) -> String { + let detailed = effectcraft_effects::warp_stab::param(&static_params(g), "advanced/detailedAnalysis").is_some_and(Value::as_bool); + format!("{}{}", signature(project, cid, l), if detailed { "+detailed" } else { "" }) +} + +/// Parameters of an instance from static values (the tracker's settings are not animated). +pub fn static_params(g: &PropGroup) -> effectcraft_effects::Params { + effectcraft_effects::flatten_params(g, &mut |p| p.value.clone()) +} + +fn clear(g: &mut PropGroup, all: bool) { + let mut ids = vec![SOLVE, SOLVE_KEY, METHOD_USED]; + if all { + ids.extend([TRACKS, TRACKS_KEY]); + } + for m in ids { + if let Some(p) = g.prop_mut(m) { + p.value = Value::Str(String::new()); + p.keys.clear(); + } + } + if let Some(p) = g.prop_mut(AVERAGE_ERROR) { + p.value = Value::Scalar(0.0); + } +} + +/// Clear analyses that no longer match, after an edit: tracks made from other frames (and their +/// solve), and solves made with other settings. Returns the trackers that need (re-)analysis: +/// cleared ones and newly applied, unanalysed ones. +pub(crate) fn invalidate(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> { + let mut todo = vec![]; + let mut clears = vec![]; + for (cid, it) in &after.items { + let ItemKind::Comp(c) = &it.kind else { continue }; + for l in &c.layers { + let Some(fx) = l.effects() else { continue }; + if !fx.groups().any(is_tracker) { + continue; + } + let old_layer = before.comp(*cid).and_then(|bc| bc.layer(l.id)); + for g in fx.groups().filter(|g| is_tracker(g)) { + let tracked = !str_of(g, TRACKS).is_empty(); + if tracked && str_of(g, TRACKS_KEY) != tracks_key(after, *cid, l, g) { + clears.push((*cid, l.id, g.uid, true)); + todo.push((*cid, l.id, g.uid)); + } else if tracked && str_of(g, SOLVE_KEY) != ct::solve_key(&ct::settings(&static_params(g))) { + clears.push((*cid, l.id, g.uid, false)); + todo.push((*cid, l.id, g.uid)); + } else if !tracked && old_layer.and_then(|ol| ol.props.find_group(g.uid)).is_none() { + todo.push((*cid, l.id, g.uid)); + } + } + } + } + for (cid, lid, uid, all) in clears { + if let Some(g) = after.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)) { + clear(g, all); + } + } + todo +} + +fn input_frame(r: &Renderer, ctx: &EvalCtx, layer: &Layer, effect_index: usize) -> Option<(Arc, [f64; 2])> { + let buf = r.layer_input(ctx, layer, effect_index)?; + if (buf.scale - 1.0).abs() > 1e-9 && buf.scale > 0.0 { + let w = (buf.img.width as f64 / buf.scale).round().max(1.0) as u32; + let h = (buf.img.height as f64 / buf.scale).round().max(1.0) as u32; + let off = [buf.offset[0] / buf.scale, buf.offset[1] / buf.scale]; + return Some((Arc::new(effectcraft_raster::resample(&buf.img, w, h)), off)); + } + Some((Arc::new(buf.img.clone()), buf.offset)) +} + +fn run_work(w: CamWork, shared: &CameraShared) { + let t0 = web_time::Instant::now(); + let fail = |msg: &str| { + let mut s = lock(&shared.state); + s.error = Some(msg.to_string()); + s.finished = true; + }; + let cancelled = || { + let mut s = lock(&shared.state); + s.cancelled = true; + s.finished = true; + }; + let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") }; + let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") }; + let lt0 = layer.layer_time(w.times[0]).seconds(); + let fd = if w.times.len() > 1 { layer.layer_time(w.times[1]).seconds() - lt0 } else { comp.frame_duration().seconds() }; + let (tracks, fresh) = match w.tracks.clone() { + Some(t) => (t, false), + None => { + let mut r = Renderer::new(&w.project, w.footage.as_ref(), RenderOpts { scale: 1.0, ..Default::default() }); + r.expr = w.expr.as_deref(); + r.cache = Some(&w.cache); + let r = &r; + let ctx0 = EvalCtx::new(&w.project, w.comp, comp, Tick::ZERO); + let ctx0 = EvalCtx { expr: w.expr.as_deref(), ..ctx0 }; + let frame_at = |t: Tick| input_frame(r, &ctx0.at(t), layer, w.effect_index); + { + let mut s = lock(&shared.state); + s.step = 1; + s.total = w.times.len() as u64; + } + let mut an = TrackAnalyzer::new(w.size, AnalyzeOpts { detailed: w.detailed }); + let mut next = frame_at(w.times[0]); + for k in 0..w.times.len() { + if shared.cancel.load(Ordering::Relaxed) { + return cancelled(); + } + let Some(cur) = next.take() else { return fail(&format!("no frame at {:.3} s", w.times[k].seconds())) }; + let ((), nf) = rayon::join( + || an.push(&effectcraft_track::Frame { img: &cur.0, offset: cur.1 }), + || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None }, + ); + next = nf; + let mut s = lock(&shared.state); + s.done = k as u64 + 1; + s.elapsed = t0.elapsed().as_secs_f64(); + s.fps = if s.elapsed > 0.0 { s.done as f64 / s.elapsed } else { 0.0 }; + } + (Arc::new(an.finish(lt0, fd)), true) + } + }; + lock(&shared.state).step = 2; + let solve = effectcraft_track::camtrack::solve(&tracks, &w.settings, Some(&shared.cancel)).map_err(|e| e.0); + if shared.cancel.load(Ordering::Relaxed) { + return cancelled(); + } + *lock(&shared.result) = Some(CamResult { tracks: fresh.then(|| (*tracks).clone()), solve }); + let mut s = lock(&shared.state); + s.elapsed = t0.elapsed().as_secs_f64(); + s.finished = true; +} + +impl Session { + pub fn is_camera_analyzing(&self) -> bool { + self.camera_job.as_ref().is_some_and(|j| !j.is_finished()) + } + + /// Progress of the running (or just finished) analysis. + pub fn camera_progress(&self) -> Option { + self.camera_job.as_ref().map(|j| j.progress()) + } + + /// The tracker being analysed: (comp, layer, effect uid). + pub fn camera_target(&self) -> Option<(ItemId, LayerId, Uid)> { + self.camera_job.as_ref().map(|j| (j.comp, j.layer, j.effect)) + } + + /// Cancel the running analysis (nothing is written). + pub fn stop_camera(&mut self) -> bool { + match &self.camera_job { + Some(j) if !j.is_finished() => { + j.shared.cancel.store(true, Ordering::Relaxed); + true + } + _ => false, + } + } + + /// Start analysing 3D Camera Tracker `effect` on `layer` (re-solving the stored tracks when + /// they are still valid). `wait`: block until done (wasm32 always does). Returns the number + /// of frames. + pub fn start_camera(&mut self, comp: ItemId, layer: LayerId, effect: Uid, wait: bool) -> Result { + if self.is_camera_analyzing() { + return Err("a 3D Camera Tracker analysis is already running".into()); + } + self.poll_camera_job(); + self.camera_pending.retain(|x| *x != (comp, layer, effect)); + let c = self.project.comp(comp).ok_or("no composition")?; + let l = c.layer(layer).ok_or("no layer")?; + let fx = l.effects().ok_or("the layer has no effects")?; + let index = fx.groups().position(|g| g.uid == effect).ok_or("no such effect")?; + let g = fx.groups().nth(index).filter(|g| is_tracker(g)).ok_or("not a 3D Camera Tracker")?; + if l.switches.adjustment { + return Err("3D Camera Tracker needs footage, not an adjustment layer".into()); + } + let times = analysis_times(c, l); + if times.len() < 2 { + return Err("the layer must be at least two frames long".into()); + } + let (w, h) = effectcraft_render::source_size(&self.project, l); + let size = if w == 0 { [c.width as f64, c.height as f64] } else { [w as f64, h as f64] }; + let params = static_params(g); + let key = tracks_key(&self.project, comp, l, g); + let settings = ct::settings(¶ms); + let solve_key = ct::solve_key(&settings); + let tracks = ct::tracks(¶ms).filter(|_| str_of(g, TRACKS_KEY) == key); + let n = times.len(); + let work = CamWork { + project: self.project.clone(), + footage: self.footage.clone(), + expr: self.expr.clone(), + cache: self.layer_cache.clone(), + comp, + layer, + effect_index: index, + size, + detailed: params.values.iter().any(|(k, v)| k.ends_with("detailedAnalysis") && v.as_bool()), + times, + tracks: tracks.clone(), + settings, + }; + let wait = wait || cfg!(target_arch = "wasm32"); + let shared = Arc::new(CameraShared::default()); + { + let mut st = lock(&shared.state); + st.solve_only = tracks.is_some(); + st.step = if tracks.is_some() { 2 } else { 1 }; + st.total = n as u64; + } + let thread = if wait { + run_work(work, &shared); + None + } else { + let sh = shared.clone(); + Some(std::thread::Builder::new().name("camera-track".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?) + }; + self.camera_job = Some(CameraJob { shared, thread, comp, layer, effect, key, solve_key }); + self.events.push(Event::ProjectChanged { revision: self.revision }); + self.poll_camera_job(); + Ok(n) + } + + /// Write a finished analysis into its effect (one undo step) and drop the job. + fn poll_camera_job(&mut self) -> bool { + let Some(job) = &self.camera_job else { return false }; + if !job.is_finished() { + return false; + } + let mut job = self.camera_job.take().expect("job"); + if let Some(t) = job.thread.take() { + let _ = t.join(); + } + let st = job.progress(); + let result = lock(&job.shared.result).take(); + match (result, &st.error) { + (Some(res), None) if !st.cancelled => { + let (cid, lid, uid) = (job.comp, job.layer, job.effect); + let (key, skey) = (job.key.clone(), job.solve_key.clone()); + let tracks_json = res.tracks.as_ref().map(CameraTracks::to_json); + let (solve_json, method, avg, msg, failed) = match &res.solve { + Ok(s) => ( + s.to_json(), + s.method_used.label().to_string(), + s.average_error, + format!( + "3D Camera Tracker: solved {} frame(s), {} points, average error {:.2} px ({:.1} s)", + s.frames.len(), + s.points.len(), + s.average_error, + st.elapsed + ), + false, + ), + Err(e) => (String::new(), String::new(), 0.0, format!("3D Camera Tracker: {e}"), true), + }; + let r = self.edit(if st.solve_only { "3D Camera Tracker Solve" } else { "3D Camera Tracker Analysis" }, None, |p, _| { + let g = p + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .ok_or_else(|| crate::EngineError::Other("the 3D Camera Tracker was removed during analysis".into()))?; + let mut set = |m: &str, v: Value| { + if let Some(pr) = g.prop_mut(m) { + pr.value = v; + } + }; + if let Some(t) = &tracks_json { + set(TRACKS, Value::Str(t.clone())); + set(TRACKS_KEY, Value::Str(key.clone())); + } + set(SOLVE, Value::Str(solve_json.clone())); + set(SOLVE_KEY, Value::Str(skey.clone())); + set(METHOD_USED, Value::Str(method.clone())); + set(AVERAGE_ERROR, Value::Scalar(avg)); + Ok(()) + }); + let (msg, failed) = match r { + Ok(()) => (msg, failed), + Err(e) => (format!("3D Camera Tracker: {e}"), true), + }; + self.events.push(Event::Toast { message: msg, error: failed }); + } + _ => { + let msg = match &st.error { + Some(e) => format!("3D Camera Tracker analysis failed: {e}"), + None => "3D Camera Tracker analysis cancelled".into(), + }; + self.events.push(Event::Toast { message: msg, error: st.error.is_some() }); + self.bump(); + } + } + true + } + + /// Frontends call this every frame: finishes analyses and (with `auto`) starts queued + /// (re-)analyses in the background, like After Effects. + pub fn poll_camera(&mut self, auto: bool) -> bool { + let changed = self.poll_camera_job(); + if auto && self.camera_job.is_none() { + while let Some((c, l, e)) = self.camera_pending.first().copied() { + self.camera_pending.remove(0); + if self.start_camera(c, l, e, false).is_ok() { + return true; + } + } + } + changed + } +} + +// ---------------------------------------------------------------- solve → composition + +/// A tracker's solve placed in its composition. +pub struct Placed { + pub solve: Arc, + /// Canonical → comp world. + pub world: Similarity, + /// Comp pixels per layer pixel. + pub scale: f64, + /// Comp times of the analysed frames. + pub times: Vec, + pub comp_size: [f64; 2], +} + +impl Placed { + /// Comp-world position of a canonical point. + pub fn to_world(&self, x: V3) -> V3 { + self.world.apply(x) + } + /// Canonical position of a comp-world point. + pub fn from_world(&self, x: V3) -> V3 { + let r = &self.world; + linalg::scale(linalg::mtv(&r.r, linalg::sub(x, r.t)), 1.0 / r.s) + } + /// The 3D Tracker Camera on frame `k`: (position, orientation degrees, zoom). + pub fn camera(&self, k: usize) -> (V3, V3, f64) { + let c = &self.solve.frames[k]; + let m = linalg::mm(&self.world.r, &linalg::mt(&c.r())); + (self.to_world(c.center), linalg::euler_xyz(&m), c.focal * self.scale) + } + /// Analysed frame at comp time `t`. + pub fn frame_at(&self, t: Tick) -> usize { + self.times.iter().enumerate().min_by_key(|(_, x)| (x.0 - t.0).abs()).map(|(i, _)| i).unwrap_or(0).min(self.solve.frames.len().saturating_sub(1)) + } +} + +/// Comp pixels per layer pixel of `layer` (its 2D transform at its In point). +fn layer_scale(project: &Project, cid: ItemId, layer: &Layer) -> f64 { + let Some(comp) = project.comp(cid) else { return 1.0 }; + let ctx = EvalCtx::new(project, cid, comp, layer.in_point.max(Tick::ZERO)); + let m = ctx.world_matrix(layer).0; + let s = m[0][0].hypot(m[1][0]); + if s.is_finite() && s > 1e-9 { s } else { 1.0 } +} + +/// The solve of tracker `uid` on `lid`, placed in the comp. +pub fn placed(project: &Project, cid: ItemId, lid: LayerId, uid: Uid) -> Option { + let comp = project.comp(cid)?; + let layer = comp.layer(lid)?; + let g = layer.props.find_group(uid).filter(|g| is_tracker(g))?; + let solve = ct::solve(&static_params(g))?; + let scale = layer_scale(project, cid, layer); + let k0 = solve.first_solved()?; + let comp_size = [comp.width as f64, comp.height as f64]; + let world = solve.world(comp_size, solve.frames[k0].focal * scale); + let mut times = analysis_times(comp, layer); + times.truncate(solve.frames.len()); + Some(Placed { solve, world, scale, times, comp_size }) +} + +/// What to create from a target (`camera.createFromSolve`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CreateKind { + Text, + Solid, + Null, + ShadowCatcher, + /// Only the camera. + Camera, +} + +impl CreateKind { + pub fn from_name(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '-', '_'], "").as_str() { + "text" => Some(CreateKind::Text), + "solid" => Some(CreateKind::Solid), + "null" => Some(CreateKind::Null), + "shadowcatcher" | "shadow" => Some(CreateKind::ShadowCatcher), + "camera" => Some(CreateKind::Camera), + _ => None, + } + } +} + +/// Rotation (layer → world) of a layer lying on a plane with unit `normal` (towards the camera), +/// its x axis along `right` projected on the plane: the layer's front (−z) faces the camera. +pub fn plane_orientation(normal: V3, right: V3) -> V3 { + let z = linalg::scale(normal, -1.0); + let mut x = linalg::sub(right, linalg::scale(z, linalg::dot(right, z))); + if linalg::norm(x) < 1e-9 { + x = if z[0].abs() < 0.9 { [1.0, 0.0, 0.0] } else { [0.0, 1.0, 0.0] }; + x = linalg::sub(x, linalg::scale(z, linalg::dot(x, z))); + } + let x = linalg::normalize(x); + let y = linalg::cross(z, x); + let m = linalg::from_cols(x, y, z); + norm_angles(linalg::euler_xyz(&m)) +} + +fn norm_angles(a: V3) -> V3 { + a.map(|v| { + let r = v.rem_euclid(360.0); + if (r - 360.0).abs() < 1e-9 { 0.0 } else { r } + }) +} + +/// Key `path` on `l` at every `(time, value)`. +pub(crate) fn key_all(l: &mut Layer, path: &str, keys: &[(Tick, Value)]) { + if let Some(pr) = l.props.prop_mut(path) { + pr.keys = keys.iter().map(|(t, v)| Keyframe::new(*t, v.clone())).collect(); + if let Some((_, v)) = keys.first() { + pr.value = v.clone(); + } + } +} + +/// Write the solved camera path into camera layer `l` (one-node, keyed on every analysed frame; +/// Zoom keyed for Variable Zoom). +pub(crate) fn write_camera(l: &mut Layer, pl: &Placed) { + l.auto_orient = AutoOrient::Off; + let n = pl.times.len().min(pl.solve.frames.len()); + let mut pos = Vec::with_capacity(n); + let mut ori = Vec::with_capacity(n); + let mut zoom = Vec::with_capacity(n); + let mut prev: Option = None; + for k in 0..n { + let (p, o, z) = pl.camera(k); + // Unwrap the angles so per-frame keys interpolate the short way. + let o = match prev { + None => norm_angles(o), + Some(q) => [0, 1, 2].map(|i| q[i] + (o[i] - q[i] + 180.0).rem_euclid(360.0) - 180.0), + }; + prev = Some(o); + let t = l.layer_time(pl.times[k]); + pos.push((t, Value::Vec3(p))); + ori.push((t, Value::Vec3(o))); + zoom.push((t, Value::Scalar(z))); + } + key_all(l, "transform/position", &pos); + key_all(l, "transform/orientation", &ori); + let variable = pl.solve.frames.iter().any(|f| (f.focal - pl.solve.frames[0].focal).abs() > 1e-6 * f.focal); + if variable { + key_all(l, "cameraOptions/zoom", &zoom); + } else if let Some((_, z)) = zoom.first() + && let Some(pr) = l.props.prop_mut("cameraOptions/zoom") + { + pr.keys.clear(); + pr.value = z.clone(); + } + if let Some((_, z)) = zoom.first() + && let Some(pr) = l.props.prop_mut("cameraOptions/focusDistance") + { + pr.value = z.clone(); + } + // A one-node camera has no Point of Interest. + if let Some(tr) = l.props.sub_mut("transform") { + tr.children.retain(|c| !matches!(c, effectcraft_project::Node::Prop(p) if p.match_id == "poi")); + } + for rot in ["transform/rotationX", "transform/rotationY", "transform/rotation"] { + if let Some(pr) = l.props.prop_mut(rot) { + pr.keys.clear(); + pr.value = Value::Scalar(0.0); + } + } +} + +/// The target for `ids` on frame `k`: when fewer than three points are given, the nearest +/// solved neighbours (in 3D) complete the plane. +pub fn target_for(solve: &CameraSolve, ids: &[u32], k: usize) -> Option { + let mut use_ids: Vec = ids.iter().copied().filter(|i| solve.point(*i).is_some()).collect(); + if use_ids.is_empty() { + return None; + } + if use_ids.len() < 3 { + let c = solve.point(use_ids[0])?.pos; + let mut others: Vec<(f64, u32)> = solve.visible(k).filter(|p| !use_ids.contains(&p.id)).map(|p| (linalg::norm(linalg::sub(p.pos, c)), p.id)).collect(); + others.sort_by(|a, b| a.0.total_cmp(&b.0)); + for (_, id) in others.into_iter().take(6usize.saturating_sub(use_ids.len())) { + use_ids.push(id); + } + let mut t = solve.target(&use_ids, k)?; + // The target sits on the chosen point(s). + let pts: Vec = ids.iter().filter_map(|i| solve.point(*i)).map(|p| p.pos).collect(); + t.center = linalg::scale(pts.iter().fold([0.0; 3], |a, b| linalg::add(a, *b)), 1.0 / pts.len() as f64); + return Some(t); + } + solve.target(&use_ids, k) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plane_orientation_faces_the_normal() { + let n = linalg::normalize([0.2, -0.9, -0.3]); + let o = plane_orientation(n, [1.0, 0.0, 0.0]); + let m = linalg::from_euler_xyz(o); + // Layer −z (its front) is the normal. + let front = linalg::mv(&m, [0.0, 0.0, -1.0]); + assert!(linalg::norm(linalg::sub(front, n)) < 1e-9); + } +} diff --git a/crates/engine/src/commands/camera_cmds.rs b/crates/engine/src/commands/camera_cmds.rs new file mode 100644 index 00000000..5d454aed --- /dev/null +++ b/crates/engine/src/commands/camera_cmds.rs @@ -0,0 +1,574 @@ +//! 3D Camera Tracker commands: Animation ▸ Track Camera (`track.camera`), the effect's Analyze / +//! Cancel / Create Camera buttons, the viewer's track point selection and right-click menu +//! (Set Ground Plane and Origin, Create Text / Solid / Null / Shadow Catcher … and Camera, +//! Create Multiple …, Delete Selected Points) and the status / points queries. + +use effectcraft_color::Label; +use effectcraft_effects::camera_tracker::{self as ct, CAMERA, DELETED, SOLVE}; +use effectcraft_keyframe::{Justify, TextDoc, Value as KV}; +use effectcraft_project::build; +use effectcraft_project::{ItemId, ItemKind, Layer, LayerId, LayerSource, LightKind, Project, Solid, Uid}; +use effectcraft_track::camtrack::linalg::{self, V3}; +use effectcraft_track::camtrack::{ShotType, SolveMethod, Target, point_color}; +use serde_json::{Value, json}; + +use super::{CommandSpec, b_p, bad, f_p, has_comp, layer_p, str_p}; +use crate::camera_track::{CreateKind, Placed, is_tracker, placed, plane_orientation, static_params, target_for, write_camera}; +use crate::{EngineError, Result, Session, cmd, query}; + +pub fn specs() -> Vec { + vec![ + cmd!( + "track.camera", + "Track Camera", + ["Animation"], + None, + "{layer?, shotType?: fixed|variable|specify, aov?: degrees, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}", + can_track, + track_camera + ), + cmd!("camera.analyze", "Analyze", [], None, "{layer?, effect?: uid|name|index, wait?: block until done}", has_comp, analyze), + cmd!("camera.cancel", "Cancel", [], None, "{}", is_analyzing, |s, _| Ok(json!({"stopped": s.stop_camera()}))), + query!("camera.solveStatus", "3D Camera Tracker Status", "{layer?, effect?}", status), + query!("camera.points", "3D Camera Tracker Points", "{layer?, effect?, time?: comp seconds} → visible solved points", points), + cmd!("camera.selectPoints", "Select Track Points", [], None, "{points: [id], add?, toggle?}", has_comp, select_points), + cmd!( + "camera.createFromSolve", + "Create from Camera Solve", + [], + None, + "{kind: text|solid|null|shadowCatcher|camera, points?: [id] (default: selected), target?: {center, normal, size?} (comp world), multiple?, layer?, effect?}", + has_comp, + create_from_solve + ), + cmd!("camera.create", "Create Camera", [], None, "{layer?, effect?}", has_comp, |s, p| create_from_solve(s, &with_kind(p, "camera"))), + cmd!("camera.setGroundPlane", "Set Ground Plane and Origin", [], None, "{points?: [id] (default: selected), layer?, effect?}", has_comp, set_ground), + cmd!("camera.deletePoints", "Delete Selected Points", [], None, "{points?: [id] (default: selected), layer?, effect?, wait?}", has_comp, delete_points), + ] +} + +fn with_kind(p: &Value, kind: &str) -> Value { + let mut q = p.clone(); + if !q.is_object() { + q = json!({}); + } + q["kind"] = json!(kind); + q +} + +fn can_track(s: &Session) -> std::result::Result<(), String> { + has_comp(s)?; + let c = s.active_comp().ok_or("no composition is open")?; + match s.state.selected_layers.first().and_then(|l| c.layer(*l)) { + Some(l) if l.has_video() && !l.switches.adjustment && !l.is_camera() && !l.is_light() => Ok(()), + Some(_) => Err("Track Camera needs a footage or composition layer".into()), + None => Err("select a layer to track".into()), + } +} + +fn is_analyzing(s: &Session) -> std::result::Result<(), String> { + if s.is_camera_analyzing() { Ok(()) } else { Err("no 3D Camera Tracker analysis is running".into()) } +} + +/// (comp, layer, effect uid) of a 3D Camera Tracker: `layer` / `effect` (uid, name or 1-based +/// index), else a selected one, else the first in the comp. +fn ct_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> { + let cid = super::comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let candidates: Vec = match p.get("layer") { + Some(v) => vec![super::resolve_layer(comp, v).ok_or_else(|| bad(cmd, format!("no layer {v}")))?], + None => { + let mut v: Vec = s.state.selected_props.iter().map(|(l, _)| *l).collect(); + v.extend(s.state.selected_layers.iter().copied()); + v.extend(comp.layers.iter().map(|l| l.id)); + v + } + }; + for lid in candidates { + let Some(fx) = comp.layer(lid).and_then(Layer::effects) else { continue }; + let g = match p.get("effect") { + Some(Value::Number(n)) => { + let n = n.as_u64().unwrap_or(0); + fx.groups().find(|g| g.uid == n).or_else(|| fx.groups().nth((n as usize).saturating_sub(1))) + } + Some(Value::String(name)) => fx.groups().find(|g| &g.name == name), + _ => s + .state + .selected_props + .iter() + .find_map(|(l, u)| (*l == lid).then(|| fx.groups().find(|g| g.uid == *u && is_tracker(g))).flatten()) + .or_else(|| fx.groups().find(|g| is_tracker(g))), + }; + if let Some(g) = g.filter(|g| is_tracker(g)) { + return Ok((cid, lid, g.uid)); + } + if p.get("layer").is_some() { + break; + } + } + Err(bad(cmd, "no 3D Camera Tracker found (apply Animation ▸ Track Camera first)")) +} + +fn shot_p(p: &Value) -> Result> { + match p.get("shotType") { + None => Ok(None), + Some(Value::Number(n)) => ShotType::ALL.get(n.as_u64().unwrap_or(99) as usize).copied().map(Some).ok_or_else(|| bad("track.camera", "shotType: 0..2")), + Some(Value::String(v)) => ShotType::from_name(v).map(Some).ok_or_else(|| bad("track.camera", "shotType: fixed|variable|specify")), + Some(_) => Err(bad("track.camera", "shotType: fixed|variable|specify")), + } +} + +fn method_p(p: &Value) -> Result> { + match p.get("solveMethod") { + None => Ok(None), + Some(Value::Number(n)) => { + SolveMethod::ALL.get(n.as_u64().unwrap_or(99) as usize).copied().map(Some).ok_or_else(|| bad("track.camera", "solveMethod: 0..3")) + } + Some(Value::String(v)) => SolveMethod::from_name(v).map(Some).ok_or_else(|| bad("track.camera", "solveMethod: auto|typical|flat|tripod")), + Some(_) => Err(bad("track.camera", "solveMethod: auto|typical|flat|tripod")), + } +} + +/// Animation ▸ Track Camera / the Tracker panel button: apply the 3D Camera Tracker to the layer +/// (reusing one it already has), set the given options and start the analysis. +fn track_camera(s: &mut Session, p: &Value) -> Result { + let (cid, lid) = layer_p(s, p, "track.camera")?; + let shot = shot_p(p)?; + let method = method_p(p)?; + let aov = f_p(p, "aov").or_else(|| f_p(p, "angleOfView")); + if aov.is_some_and(|a| !(1.0..=170.0).contains(&a)) { + return Err(bad("track.camera", "aov: 1..170 degrees")); + } + let detailed = b_p(p, "detailed"); + let existing = s.project.comp(cid).and_then(|c| c.layer(lid)).and_then(Layer::effects).and_then(|fx| fx.groups().find(|g| is_tracker(g)).map(|g| g.uid)); + let uid = match existing { + Some(u) => u, + None => { + let r = s.execute("effect.apply", json!({"effect": ct::ID, "layers": [lid.0], "comp": cid.0}))?; + r["effects"].get(0).and_then(Value::as_u64).ok_or_else(|| EngineError::Other("the effect could not be applied".into()))? + } + }; + if shot.is_some() || method.is_some() || aov.is_some() || detailed.is_some() { + s.edit("3D Camera Tracker Settings", None, |proj, _| { + let g = proj + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?; + let mut set = |m: &str, v: KV| { + if let Some(pr) = g.prop_mut(m) { + pr.keys.clear(); + pr.value = v; + } + }; + if let Some(sh) = shot { + set("shotType", KV::Enum(ShotType::ALL.iter().position(|x| *x == sh).unwrap_or(0) as u32)); + } + if let Some(a) = aov { + set("horizontalAngleOfView", KV::Scalar(a)); + if shot.is_none() { + set("shotType", KV::Enum(2)); + } + } + if let Some(m) = method { + set("advanced/solveMethod", KV::Enum(SolveMethod::ALL.iter().position(|x| *x == m).unwrap_or(0) as u32)); + } + if let Some(d) = detailed { + set("advanced/detailedAnalysis", KV::Bool(d)); + } + Ok(()) + })?; + } + // Select the effect so the viewer shows its track points. + s.state.selected_props = vec![(lid, uid)]; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?; + let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_camera_analyzing(), "progress": s.camera_progress()}); + if wait { + out["status"] = status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?; + } + Ok(out) +} + +fn analyze(s: &mut Session, p: &Value) -> Result { + let (cid, lid, uid) = ct_p(s, p, "camera.analyze")?; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?; + let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_camera_analyzing(), "progress": s.camera_progress()}); + if wait { + out["status"] = status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?; + } + Ok(out) +} + +fn status(s: &mut Session, p: &Value) -> Result { + s.poll_camera(false); + let mut out = json!({"running": s.is_camera_analyzing(), "progress": s.camera_progress(), "banner": s.camera_progress().map(|p| p.banner())}); + let Ok((cid, lid, uid)) = ct_p(s, p, "camera.solveStatus") else { return Ok(out) }; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let g = layer.props.find_group(uid).ok_or(EngineError::NoComp)?; + let params = static_params(g); + out["layer"] = json!(lid.0); + out["effect"] = json!(uid); + out["pending"] = json!(s.camera_pending.contains(&(cid, lid, uid))); + let tracks = ct::tracks(¶ms); + out["analyzed"] = json!(tracks.is_some()); + if let Some(t) = &tracks { + out["tracks"] = json!(t.tracks.len()); + out["frames"] = json!(t.frames); + } + let settings = ct::settings(¶ms); + out["shotType"] = json!(settings.shot.label()); + out["solveMethod"] = json!(settings.method.label()); + out["deleted"] = json!(settings.deleted.len()); + let solve = ct::solve(¶ms); + out["solved"] = json!(solve.is_some()); + if let Some(sv) = &solve { + let f = sv.focal(); + out["methodUsed"] = json!(sv.method_used.label()); + out["averageError"] = json!(sv.average_error); + out["points"] = json!(sv.points.len()); + out["solvedFrames"] = json!(sv.frames.iter().filter(|c| c.solved).count()); + out["focalLength"] = json!(f); + out["horizontalAngleOfView"] = json!(sv.hfov(f)); + out["groundPlane"] = json!(sv.ground.is_some()); + if let Some(pl) = placed(&s.project, cid, lid, uid) { + let k = pl.frame_at(s.time_of(cid)); + let (pos, ori, zoom) = pl.camera(k); + out["frame"] = json!(k); + out["camera"] = json!({"position": pos, "orientation": ori, "zoom": zoom}); + out["visible"] = json!(sv.visible(k).count()); + } + } + let cam = params.get(CAMERA).map(|v| v.as_f64() as u64).unwrap_or(0); + out["cameraLayer"] = if cam > 0 && comp.layer(LayerId(cam)).is_some_and(Layer::is_camera) { json!(cam) } else { Value::Null }; + out["selected"] = json!(s.state.camera_points); + Ok(out) +} + +/// The visible solved points at the current (or given) time, with their comp positions. +fn points(s: &mut Session, p: &Value) -> Result { + let (cid, lid, uid) = ct_p(s, p, "camera.points")?; + let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad("camera.points", "the camera is not solved yet"))?; + let t = f_p(p, "time").map(effectcraft_time::Tick::from_seconds_f64).unwrap_or_else(|| s.time_of(cid)); + let k = pl.frame_at(t); + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let ctx = effectcraft_render::EvalCtx::new(&s.project, cid, comp, t); + let m = ctx.world_matrix(layer); + let list: Vec = ct::projected(&pl.solve, k) + .into_iter() + .map(|(id, q, z)| { + let c = m.apply(effectcraft_geom::vec3(q[0], q[1], 0.0)); + let sp = pl.solve.point(id).copied().unwrap_or_default(); + let col = point_color(id); + json!({"id": id, "comp": [c.x, c.y], "layer": q, "depth": z, "world": pl.to_world(sp.pos), "error": sp.error, "color": col, "selected": s.state.camera_points.contains(&id)}) + }) + .collect(); + Ok(json!({"frame": k, "points": list})) +} + +fn ids_p(s: &Session, p: &Value) -> Vec { + match p.get("points") { + Some(Value::Array(a)) => a.iter().filter_map(Value::as_u64).map(|v| v as u32).collect(), + Some(Value::Number(n)) => n.as_u64().map(|v| vec![v as u32]).unwrap_or_default(), + _ => s.state.camera_points.clone(), + } +} + +fn select_points(s: &mut Session, p: &Value) -> Result { + let ids: Vec = match p.get("points") { + Some(Value::Array(a)) => a.iter().filter_map(Value::as_u64).map(|v| v as u32).collect(), + _ => vec![], + }; + if b_p(p, "toggle").unwrap_or(false) { + for id in ids { + match s.state.camera_points.iter().position(|x| *x == id) { + Some(i) => { + s.state.camera_points.remove(i); + } + None => s.state.camera_points.push(id), + } + } + } else if b_p(p, "add").unwrap_or(false) { + for id in ids { + if !s.state.camera_points.contains(&id) { + s.state.camera_points.push(id); + } + } + } else { + s.state.camera_points = ids; + } + s.bump(); + Ok(json!({"selected": s.state.camera_points})) +} + +/// The target of a create / ground command in canonical solve coordinates. +fn target_p(s: &Session, p: &Value, pl: &Placed, k: usize, cmd: &str) -> Result { + if let Some(t) = p.get("target") { + let v3 = |k: &str| -> Option { + let a = t.get(k)?.as_array()?; + Some([a.first()?.as_f64()?, a.get(1)?.as_f64()?, a.get(2).and_then(Value::as_f64).unwrap_or(0.0)]) + }; + let (c, n) = (v3("center").ok_or_else(|| bad(cmd, "target.center: [x, y, z]"))?, v3("normal").unwrap_or([0.0, -1.0, 0.0])); + let size = t.get("size").and_then(Value::as_f64).unwrap_or(100.0); + let nc = linalg::normalize(linalg::mtv(&pl.world.r, n)); + return Ok(Target { center: pl.from_world(c), normal: nc, size: size / pl.world.s }); + } + let ids = ids_p(s, p); + if ids.is_empty() { + return Err(bad(cmd, "select track points (or give `points` / `target`)")); + } + target_for(&pl.solve, &ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane")) +} + +fn next_name(c: &effectcraft_project::Comp, base: &str) -> String { + let n = (1..).find(|i| !c.layers.iter().any(|l| l.name == format!("{base} {i}"))).unwrap_or(1); + format!("{base} {n}") +} + +fn place(l: &mut Layer, pos: V3, ori: V3, src: &Layer) { + l.switches.three_d = true; + l.in_point = src.in_point; + l.out_point = src.out_point; + if let Some(pr) = l.props.prop_mut("transform/position") { + pr.value = KV::Vec3(pos); + } + if let Some(pr) = l.props.prop_mut("transform/orientation") { + pr.value = KV::Vec3(ori); + } +} + +/// Insert `l` above the tracked layer. +fn insert_above(proj: &mut Project, cid: ItemId, above: LayerId, l: Layer) -> Result { + let comp = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let at = comp.layers.iter().position(|x| x.id == above).unwrap_or(0); + let id = l.id; + comp.layers.insert(at, l); + Ok(id) +} + +fn add_solid(proj: &mut Project, comp: &effectcraft_project::Comp, name: &str, size: u32, color: [f32; 3]) -> Layer { + let folder = proj.folder_named("Solids").unwrap_or_else(|| proj.add_item("Solids", Label::Yellow, None, ItemKind::Folder)); + let sid = proj.add_item(name, Label::Red, Some(folder), ItemKind::Solid(Solid { color, width: size, height: size, pixel_aspect: 1.0 })); + build::layer(proj, comp, name, LayerSource::Solid { item: sid }, (size, size), None) +} + +/// Create Text / Solid / Null / Shadow Catcher (and Light) on the target, plus the 3D Tracker +/// Camera when the tracker has none yet — one undo step. +fn create_from_solve(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.createFromSolve"; + let (cid, lid, uid) = ct_p(s, p, cmd)?; + let kind = CreateKind::from_name(str_p(p, "kind").unwrap_or("null")).ok_or_else(|| bad(cmd, "kind: text|solid|null|shadowCatcher|camera"))?; + let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad(cmd, "the camera is not solved yet (analysis still running or failed)"))?; + let k = pl.frame_at(s.time_of(cid)); + let multiple = b_p(p, "multiple").unwrap_or(false); + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let src = comp.layer(lid).ok_or(EngineError::NoComp)?.clone(); + let g = src.props.find_group(uid).ok_or(EngineError::NoComp)?; + let params = static_params(g); + let size_pct = effectcraft_effects::warp_stab::param(¶ms, "targetSize").map(|v| v.as_f64()).unwrap_or(100.0) / 100.0; + let existing_cam = params.get(CAMERA).map(|v| v.as_f64() as u64).filter(|id| *id > 0 && comp.layer(LayerId(*id)).is_some_and(Layer::is_camera)); + // Targets: one, or one per selected point (Create Multiple …) sharing the selection's plane. + let targets: Vec = if kind == CreateKind::Camera { + vec![] + } else { + let t = target_p(s, p, &pl, k, cmd)?; + if multiple { + let ids = ids_p(s, p); + ids.iter().filter_map(|i| pl.solve.point(*i)).map(|sp| Target { center: sp.pos, ..t }).collect() + } else { + vec![t] + } + }; + // Camera right vector on this frame (world), to align created layers with the view. + let cam = &pl.solve.frames[k]; + let right = linalg::mv(&pl.world.r, linalg::mtv(&cam.r(), [1.0, 0.0, 0.0])); + let label = match (kind, multiple) { + (CreateKind::Text, false) => "Create Text and Camera", + (CreateKind::Solid, false) => "Create Solid and Camera", + (CreateKind::Null, false) => "Create Null and Camera", + (CreateKind::ShadowCatcher, _) => "Create Shadow Catcher, Camera and Light", + (CreateKind::Text, true) => "Create Multiple Text Layers and Camera", + (CreateKind::Solid, true) => "Create Multiple Solids and Camera", + (CreateKind::Null, true) => "Create Multiple Nulls and Camera", + (CreateKind::Camera, _) => "Create Camera", + }; + let (camera, created) = s.edit(label, None, |proj, st| { + let mut created = vec![]; + // The camera: reuse the tracker's, else create "3D Tracker Camera" at the top. + let camera = match existing_cam { + Some(id) if kind != CreateKind::Camera => LayerId(id), + Some(id) => { + let l = proj.comp_mut(cid).and_then(|c| c.layer_mut(LayerId(id))).ok_or(EngineError::NoComp)?; + write_camera(l, &pl); + LayerId(id) + } + None => { + let mut l = build::layer(proj, &comp, "3D Tracker Camera", LayerSource::Camera, (comp.width, comp.height), None); + l.in_point = src.in_point; + l.out_point = src.out_point; + write_camera(&mut l, &pl); + let id = l.id; + proj.comp_mut(cid).ok_or(EngineError::NoComp)?.layers.insert(0, l); + if let Some(pr) = proj.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)).and_then(|g| g.prop_mut(CAMERA)) { + pr.value = KV::Scalar(id.0 as f64); + } + id + } + }; + for t in &targets { + let center = pl.to_world(t.center); + let normal = linalg::normalize(linalg::mv(&pl.world.r, t.normal)); + let ori = plane_orientation(normal, right); + let size = (t.size * pl.world.s * size_pct).max(1.0); + let cur = proj.comp(cid).ok_or(EngineError::NoComp)?.clone(); + match kind { + CreateKind::Text => { + let mut l = build::layer(proj, &cur, "Text", LayerSource::Text, (comp.width, comp.height), None); + let doc = TextDoc { text: "Text".into(), justify: Justify::Center, size: (size * 0.35).clamp(4.0, 2000.0), ..Default::default() }; + if let Some(pr) = l.props.prop_mut("text/sourceText") { + pr.value = KV::Text(Box::new(doc)); + } + place(&mut l, center, ori, &src); + created.push(insert_above(proj, cid, lid, l)?); + } + CreateKind::Solid => { + let name = next_name(&cur, "Track Solid"); + let c = point_color((created.len() as u32).wrapping_add(7)); + let mut l = add_solid(proj, &cur, &name, size.round().max(1.0) as u32, c); + place(&mut l, center, ori, &src); + created.push(insert_above(proj, cid, lid, l)?); + } + CreateKind::Null => { + let name = next_name(&cur, "Track Null"); + let mut l = build::layer(proj, &cur, &name, LayerSource::Null, (comp.width, comp.height), None); + place(&mut l, center, ori, &src); + if let Some(pr) = l.props.prop_mut("transform/scale") { + pr.value = KV::Vec3([size, size, size]); + } + created.push(insert_above(proj, cid, lid, l)?); + } + CreateKind::ShadowCatcher => { + let name = next_name(&cur, "Shadow Catcher"); + let mut l = add_solid(proj, &cur, &name, (size * 4.0).round().max(1.0) as u32, [1.0, 1.0, 1.0]); + place(&mut l, center, ori, &src); + if let Some(pr) = l.props.prop_mut("materialOptions/acceptsShadows") { + pr.value = KV::Enum(2); + } + if let Some(pr) = l.props.prop_mut("materialOptions/castsShadows") { + pr.value = KV::Enum(0); + } + let sc = created.len(); + created.push(insert_above(proj, cid, lid, l)?); + let cur = proj.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let lname = next_name(&cur, "Light"); + let mut light = build::layer(proj, &cur, &lname, LayerSource::Light { kind: LightKind::Point }, (comp.width, comp.height), None); + light.in_point = src.in_point; + light.out_point = src.out_point; + let lp = linalg::add(center, linalg::add(linalg::scale(normal, size * 2.0), linalg::scale(right, size))); + if let Some(pr) = light.props.prop_mut("transform/position") { + pr.value = KV::Vec3(lp); + } + if let Some(pr) = light.props.prop_mut("lightOptions/castsShadows") { + pr.value = KV::Bool(true); + } + let _ = sc; + created.push(insert_above(proj, cid, created[created.len() - 1], light)?); + } + CreateKind::Camera => {} + } + } + st.selected_layers = if created.is_empty() { vec![camera] } else { created.clone() }; + st.selected_props.clear(); + Ok((camera, created)) + })?; + let t0 = targets.first().map( + |t| json!({"center": pl.to_world(t.center), "normal": linalg::normalize(linalg::mv(&pl.world.r, t.normal)), "size": t.size * pl.world.s * size_pct}), + ); + Ok(json!({"camera": camera.0, "layers": created.iter().map(|l| l.0).collect::>(), "target": t0})) +} + +fn set_ground(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.setGroundPlane"; + let (cid, lid, uid) = ct_p(s, p, cmd)?; + let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad(cmd, "the camera is not solved yet"))?; + let k = pl.frame_at(s.time_of(cid)); + let ids = ids_p(s, p); + if ids.is_empty() { + return Err(bad(cmd, "select track points (or give `points`)")); + } + let t = target_for(&pl.solve, &ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane"))?; + let mut sv = (*pl.solve).clone(); + let mut g = sv.ground_from(&ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane"))?; + g.origin = t.center; + sv.ground = Some(g); + let json = sv.to_json(); + s.edit("Set Ground Plane and Origin", None, |proj, _| { + let pr = proj + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .and_then(|g| g.prop_mut(SOLVE)) + .ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?; + pr.value = KV::Str(json); + Ok(()) + })?; + Ok(json!({"origin": g.origin, "normal": g.normal})) +} + +/// Delete points (and, with Auto-delete Points Across Time, the points of other tracks on the +/// same 3D feature), then re-solve. +fn delete_points(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.deletePoints"; + let (cid, lid, uid) = ct_p(s, p, cmd)?; + let ids = ids_p(s, p); + if ids.is_empty() { + return Err(bad(cmd, "select track points (or give `points`)")); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let g = comp.layer(lid).and_then(|l| l.props.find_group(uid)).ok_or(EngineError::NoComp)?; + let params = static_params(g); + let mut del = ct::deleted(¶ms); + let auto = effectcraft_effects::warp_stab::param(¶ms, "advanced/autoDeletePoints").is_some_and(KV::as_bool); + let mut add = ids.clone(); + if auto && let Some(sv) = ct::solve(¶ms) { + // Same feature tracked again at other times: solved within 1 % of its depth. + let cam0 = sv.first_solved().map(|k| sv.frames[k].center).unwrap_or([0.0; 3]); + for id in &ids { + let Some(pt) = sv.point(*id) else { continue }; + let r = linalg::norm(linalg::sub(pt.pos, cam0)) * 0.01; + for q in &sv.points { + let disjoint = q.last < pt.first || q.first > pt.last; + if q.id != *id && disjoint && linalg::norm(linalg::sub(q.pos, pt.pos)) < r { + add.push(q.id); + } + } + } + } + for id in add { + if !del.contains(&id) { + del.push(id); + } + } + del.sort_unstable(); + let json = serde_json::to_string(&del).unwrap_or_default(); + s.edit("Delete Track Points", None, |proj, _| { + let pr = proj + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .and_then(|g| g.prop_mut(DELETED)) + .ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?; + pr.value = KV::Str(json); + Ok(()) + })?; + s.state.camera_points.retain(|x| !del.contains(x)); + // Re-solve now (After Effects re-solves after deleting points). + let wait = b_p(p, "wait").unwrap_or(false); + if !s.is_camera_analyzing() { + s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?; + } + Ok(json!({"deleted": del, "running": s.is_camera_analyzing()})) +} diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 4537752f..04034281 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -4,6 +4,7 @@ mod anim; pub mod anim_tools; mod animation; +mod camera_cmds; mod comp; mod comp_more; mod edit; @@ -129,6 +130,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(track::specs()); v.extend(mask_interp::specs()); v.extend(warp_cmds::specs()); + v.extend(camera_cmds::specs()); v.extend(paint::specs()); v.extend(puppet::specs()); v.extend(settings::specs()); diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index f670ffd7..74de3aec 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -59,7 +59,6 @@ pub fn specs() -> Vec { stub!("keys.audioToKeyframes", "Convert Audio to Keyframes", ["Animation", "Keyframe Assistant"], "{}"), stub!("keys.rpfCameraImport", "RPF Camera Import", ["Animation", "Keyframe Assistant"], "{}"), stub!("text.animatorFontAxes", "Variable Font Axes", ["Animation", "Animate Text"], "{}"), - stub!("track.camera", "Track Camera", ["Animation"], "{}"), // Workspaces and panels that don't exist yet. stub!("window.unavailablePanel", "Panel", [], "{panel}"), ] diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index ba34488a..fbd2b391 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -9,6 +9,7 @@ //! (compositions are `Arc`s, so untouched comps are shared). pub mod autosave; +pub mod camera_track; pub mod commands; pub mod config; pub mod demo; @@ -41,6 +42,7 @@ pub use effectcraft_project as project; pub use effectcraft_render as render; pub use effectcraft_text as text; pub use effectcraft_time as time; +pub use effectcraft_track as track; pub use render_queue::{ExportJob, ExportResult, Exporter, JobState}; #[derive(Debug, thiserror::Error)] @@ -170,6 +172,9 @@ pub struct EditorState { /// Tracker panel ▸ Method in mask mode (Track Mask). #[serde(default)] pub mask_track_method: mask_track::MaskMethod, + /// 3D Camera Tracker: the selected track points (ids) in the viewer. + #[serde(default)] + pub camera_points: Vec, /// Mask Interpolation panel options. #[serde(default)] pub mask_interp: commands::mask_interp::MaskInterpOptions, @@ -263,6 +268,10 @@ pub struct Session { pub warp_job: Option, /// Warp Stabilizers waiting for (re-)analysis: (comp, layer, effect uid). pub warp_pending: Vec<(ItemId, LayerId, Uid)>, + /// The running (or finished, not yet polled) 3D Camera Tracker analysis. + pub camera_job: Option, + /// 3D Camera Trackers waiting for (re-)analysis: (comp, layer, effect uid). + pub camera_pending: Vec<(ItemId, LayerId, Uid)>, pub events: Vec, /// Commands executed: (id, params). pub journal: Vec<(String, Value)>, @@ -308,6 +317,8 @@ impl Default for Session { mask_job: None, warp_job: None, warp_pending: vec![], + camera_job: None, + camera_pending: vec![], events: vec![], journal: vec![], layer_cache: Arc::new(LayerCache::default()), @@ -374,6 +385,12 @@ impl Session { self.warp_pending.push(w); } } + // So are 3D Camera Tracker tracks (and solves made with other settings). + for w in camera_track::invalidate(&before, &mut p) { + if !self.camera_pending.contains(&w) { + self.camera_pending.push(w); + } + } let same = merge.is_some() && merge.map(str::to_string) == self.history.merge_key; if !same { self.history.undo.push((label.to_string(), before)); @@ -551,6 +568,9 @@ impl Session { self.stop_warp(); self.warp_job = None; self.warp_pending.clear(); + self.stop_camera(); + self.camera_job = None; + self.camera_pending.clear(); p.fix_next_id(); self.project = Arc::new(p); self.history = History::default(); @@ -574,6 +594,8 @@ mod tests_3d; #[cfg(test)] mod tests_anim_tools; #[cfg(test)] +mod tests_camera_track; +#[cfg(test)] mod tests_effects; #[cfg(test)] mod tests_fidelity; diff --git a/crates/engine/src/tests_camera_track.rs b/crates/engine/src/tests_camera_track.rs new file mode 100644 index 00000000..5bd5539c --- /dev/null +++ b/crates/engine/src/tests_camera_track.rs @@ -0,0 +1,307 @@ +//! The 3D Camera Tracker end to end through the commands, on synthetic footage rendered in-test: +//! textured planes (a floor, a back wall and a box face) seen through a known moving camera. + +use effectcraft_effects::camera_tracker as ct; +use effectcraft_geom::vec3; +use effectcraft_project::LayerId; +use effectcraft_raster::Image; +use effectcraft_render::EvalCtx; +use effectcraft_render::three_d::camera::active_camera; +use effectcraft_track::camtrack::linalg::{self, M3, V3}; +use serde_json::{Value, json}; + +use crate::Session; +use crate::tests_track::{H, W, frame_time, setup, texture}; + +const F: f64 = 260.0; + +/// The true camera on frame `f`: (world → camera rotation, centre). +fn true_camera(f: u32) -> (M3, V3) { + let t = f as f64 / 24.0; + let c = [-1.0 + 2.0 * t, -0.3 + 0.2 * (t * 3.0).sin(), 0.5 * t]; + let fwd = linalg::normalize(linalg::sub([0.2 * t, 0.4, 8.0], c)); + let right = linalg::normalize(linalg::cross([0.0, 1.0, 0.0], fwd)); + ([right, linalg::cross(fwd, right), fwd], c) +} + +/// Planes: (point, normal, u axis, v axis, half extent or 0 = infinite). +const PLANES: [(V3, V3, V3, V3, f64); 3] = [ + ([0.0, 1.5, 0.0], [0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0], 0.0), + ([0.0, 0.0, 10.0], [0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0], 0.0), + ([-1.2, 0.2, 6.0], [0.3, 0.0, -1.0], [0.957_826, 0.0, 0.287_348], [0.0, 1.0, 0.0], 1.1), +]; + +fn scene_frame(f: u32) -> Image { + let (r, c) = true_camera(f); + let mut img = Image::new(W, H); + for y in 0..H { + for x in 0..W { + let d = linalg::mtv(&r, [(x as f64 + 0.5 - W as f64 / 2.0) / F, (y as f64 + 0.5 - H as f64 / 2.0) / F, 1.0]); + let mut best = (f64::INFINITY, 0.2f32); + for (i, (p0, n, ua, va, ext)) in PLANES.iter().enumerate() { + let den = linalg::dot(*n, d); + if den.abs() < 1e-9 { + continue; + } + let tt = linalg::dot(*n, linalg::sub(*p0, c)) / den; + if tt <= 0.1 || tt >= best.0 { + continue; + } + let hit = linalg::add(c, linalg::scale(d, tt)); + let (u, v) = (linalg::dot(linalg::sub(hit, *p0), *ua), linalg::dot(linalg::sub(hit, *p0), *va)); + if *ext > 0.0 && (u.abs() > *ext || v.abs() > *ext) { + continue; + } + best = (tt, texture(u * 40.0, v * 40.0, 11 + i as u32)); + } + img.set(x, y, [best.1, best.1, best.1, 1.0]); + } + } + img +} + +/// The comp with the clip tracked (blocking) with `params`. +fn tracked(params: Value) -> (Session, LayerId) { + let (mut s, clip, _) = setup(scene_frame); + s.execute("layer.select", json!({"layers": [clip.0]})).unwrap(); + assert!(s.is_enabled("track.camera")); + let mut p = params; + p["wait"] = json!(true); + let r = s.execute_checked("track.camera", p).unwrap(); + assert_eq!(r["frames"], 25, "{r}"); + assert!(s.camera_job.is_none()); + (s, clip) +} + +fn status(s: &mut Session) -> Value { + s.execute("camera.solveStatus", json!({})).unwrap() +} + +#[test] +fn track_camera_solves_and_creates_layers_on_the_plane() { + let (mut s, clip) = tracked(json!({})); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + let err = st["averageError"].as_f64().unwrap(); + assert!(err < 0.5, "average error {err}"); + let f = st["focalLength"].as_f64().unwrap(); + assert!((f - F).abs() / F < 0.05, "focal {f}"); + assert_ne!(st["methodUsed"], "Tripod Pan"); + // Method Used / Average Error are reported in the effect. + let g = s.active_comp().unwrap().layer(clip).unwrap().effects().unwrap().groups().next().unwrap().clone(); + assert!(matches!(g.prop(ct::METHOD_USED).map(|p| &p.value), Some(effectcraft_keyframe::Value::Str(m)) if !m.is_empty())); + + // Pick floor points on frame 12 (their 3D positions lie on y = 1.5 in the true scene: the + // ones seen below the horizon at the bottom of the frame). + s.execute("time.set", json!({"time": frame_time(12).seconds()})).unwrap(); + let pts = s.execute("camera.points", json!({})).unwrap(); + let list = pts["points"].as_array().unwrap(); + assert!(list.len() > 30, "{}", list.len()); + let floor: Vec = list.iter().filter(|p| p["comp"][1].as_f64().unwrap() > H as f64 * 0.8).map(|p| p["id"].as_u64().unwrap()).take(8).collect(); + assert!(floor.len() >= 3, "{floor:?}"); + s.execute("camera.selectPoints", json!({"points": floor})).unwrap(); + assert_eq!(s.state.camera_points.len(), floor.len()); + + // Create Solid and Camera: one undo step, camera + solid. + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + let cam = LayerId(r["camera"].as_u64().unwrap()); + let solid = LayerId(r["layers"][0].as_u64().unwrap()); + let comp = s.active_comp().unwrap().clone(); + assert!(comp.layer(cam).unwrap().is_camera()); + assert_eq!(comp.layer(cam).unwrap().name, "3D Tracker Camera"); + assert_eq!(comp.layers[0].id, cam); + let sl = comp.layer(solid).unwrap(); + assert!(sl.switches.three_d); + assert!(sl.name.starts_with("Track Solid")); + // The camera is keyed on every frame. + assert_eq!(comp.layer(cam).unwrap().props.prop("transform/position").unwrap().keys.len(), 25); + + // Every solved point projects through the created camera exactly where the solve puts it, + // and onto its tracked position (the footage fills the comp, so layer pixels are comp + // pixels): within 1 px for 95 % of the observations (the rest are the solve's outliers: + // features on occlusion edges), 0.5 px on average. + let cid = s.active_comp_id().unwrap(); + let uid = g.uid; + let pl = crate::camera_track::placed(&s.project, cid, clip, uid).unwrap(); + let tracks = ct::tracks(&crate::camera_track::static_params(&g)).unwrap(); + let mut errs = vec![]; + for k in [0u32, 6, 12, 18, 24] { + let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(k)); + let camst = active_camera(&ctx); + for sp in pl.solve.visible(k as usize) { + let Some(obs) = tracks.track(sp.id).and_then(|t| t.at(k)) else { continue }; + let w = pl.to_world(sp.pos); + let q = camst.project(W as f64, H as f64, vec3(w[0], w[1], w[2])).unwrap(); + let own = pl.solve.frames[k as usize].project(pl.solve.size, sp.pos).unwrap(); + assert!((q.x - own[0]).hypot(q.y - own[1]) < 0.01, "camera layer disagrees with the solve"); + errs.push((q.x - obs[0]).hypot(q.y - obs[1])); + } + } + assert!(errs.len() > 100, "{}", errs.len()); + errs.sort_by(f64::total_cmp); + let p95 = errs[errs.len() * 95 / 100]; + let mean = errs.iter().sum::() / errs.len() as f64; + assert!(p95 < 1.0 && mean < 0.5, "p95 {p95} px, mean {mean} px"); + + // The solid lies on the selected points' plane. + let tr = sl.props.sub("transform").unwrap(); + let pos = tr.get("position").unwrap().value.as_vec3(); + let ori = tr.get("orientation").unwrap().value.as_vec3(); + let normal = linalg::mv(&linalg::from_euler_xyz(ori), [0.0, 0.0, -1.0]); + let size = r["target"]["size"].as_f64().unwrap(); + for id in &s.state.camera_points { + let w = pl.to_world(pl.solve.point(*id).unwrap().pos); + let d = linalg::dot(linalg::sub(w, pos), normal).abs(); + assert!(d < 0.05 * size.max(1.0) + 1.0, "point {id} is {d} px off the plane"); + } + // The solid faces the camera. + let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(12)); + let eye = active_camera(&ctx).eye; + assert!(linalg::dot(normal, linalg::sub([eye.x, eye.y, eye.z], pos)) > 0.0); + + // Undo removes both layers; redo brings them back. + let before = comp.layers.len(); + assert!(s.undo()); + assert_eq!(s.active_comp().unwrap().layers.len(), before - 2); + assert!(s.redo()); + assert_eq!(s.active_comp().unwrap().layers.len(), before); + + // A second create reuses the camera: Create Multiple Nulls and Camera. + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "null", "multiple": true})).unwrap(); + assert_eq!(r["camera"].as_u64().unwrap(), cam.0); + assert_eq!(r["layers"].as_array().unwrap().len(), floor.len()); + // Text and Shadow Catcher (+ light). + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "text"})).unwrap(); + assert_eq!(r["layers"].as_array().unwrap().len(), 1); + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "shadowCatcher"})).unwrap(); + let ids: Vec = r["layers"].as_array().unwrap().iter().map(|v| LayerId(v.as_u64().unwrap())).collect(); + let comp = s.active_comp().unwrap(); + assert!(comp.layer(ids[0]).unwrap().name.starts_with("Shadow Catcher")); + assert_eq!(comp.layer(ids[0]).unwrap().props.prop("materialOptions/acceptsShadows").unwrap().value.as_enum(), 2); + assert!(comp.layer(ids[1]).unwrap().is_light()); + assert!(s.execute_checked("camera.createFromSolve", json!({"kind": "teapot"})).is_err()); + + // Set Ground Plane and Origin: the target becomes the origin; a null created there sits at + // (0, 0, 0) lying on the X-Z plane (its front facing up, −y). + s.execute_checked("camera.setGroundPlane", json!({})).unwrap(); + assert_eq!(status(&mut s)["groundPlane"], true); + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "null"})).unwrap(); + let null = LayerId(r["layers"][0].as_u64().unwrap()); + let tr = s.active_comp().unwrap().layer(null).unwrap().props.sub("transform").unwrap().clone(); + let pos = tr.get("position").unwrap().value.as_vec3(); + assert!(linalg::norm(pos) < 1e-6, "{pos:?}"); + let up = linalg::mv(&linalg::from_euler_xyz(tr.get("orientation").unwrap().value.as_vec3()), [0.0, 0.0, -1.0]); + assert!(linalg::norm(linalg::sub(up, [0.0, -1.0, 0.0])) < 1e-6, "{up:?}"); + + // An explicit target (comp world) places the layer there, its front facing the normal. + let r = s + .execute_checked("camera.createFromSolve", json!({"kind": "solid", "target": {"center": [10.0, 0.0, 20.0], "normal": [0.0, 0.0, -1.0], "size": 80.0}})) + .unwrap(); + let l = s.active_comp().unwrap().layer(LayerId(r["layers"][0].as_u64().unwrap())).unwrap().clone(); + let tr = l.props.sub("transform").unwrap(); + let pos = tr.get("position").unwrap().value.as_vec3(); + assert!(linalg::norm(linalg::sub(pos, [10.0, 0.0, 20.0])) < 1e-6, "{pos:?}"); + let front = linalg::mv(&linalg::from_euler_xyz(tr.get("orientation").unwrap().value.as_vec3()), [0.0, 0.0, -1.0]); + assert!(linalg::norm(linalg::sub(front, [0.0, 0.0, -1.0])) < 1e-6, "{front:?}"); + + // Serde keeps the analysis. + let back = effectcraft_project::Project::from_json(&s.project.to_json()).unwrap(); + let g2 = back.comp(cid).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap(); + let sv = ct::solve(&crate::camera_track::static_params(g2)).unwrap(); + assert!(sv.ground.is_some()); + assert_eq!(sv.points.len(), pl.solve.points.len()); +} + +#[test] +fn invalidation_resolve_cancel_and_undo() { + let (mut s, clip) = tracked(json!({"shotType": "specify", "aov": 2.0 * (W as f64 / 2.0 / F).atan().to_degrees()})); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + assert_eq!(st["shotType"], "Specify Angle of View"); + assert!((st["focalLength"].as_f64().unwrap() - F).abs() < 0.01 * F); + let cid = s.active_comp_id().unwrap(); + let uid = st["effect"].as_u64().unwrap(); + // Undo removes the analysis (one step); redo restores it. + assert!(s.undo()); + assert_eq!(status(&mut s)["analyzed"], false); + assert!(s.redo()); + assert_eq!(status(&mut s)["solved"], true); + + // Changing the Solve Method keeps the tracks and only re-solves. + s.execute_checked("prop.set", json!({"layer": clip.0, "path": "effects/#1/advanced/solveMethod", "value": 1})).unwrap(); + let st = status(&mut s); + assert_eq!(st["analyzed"], true); + assert_eq!(st["solved"], false); + assert_eq!(st["pending"], true); + s.execute_checked("camera.analyze", json!({"wait": true})).unwrap(); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + assert_eq!(st["methodUsed"], "Typical"); + + // Deleting points re-solves without them. + let pts = s.execute("camera.points", json!({})).unwrap(); + let del: Vec = pts["points"].as_array().unwrap().iter().take(3).map(|p| p["id"].as_u64().unwrap()).collect(); + s.execute_checked("camera.deletePoints", json!({"points": del, "wait": true})).unwrap(); + s.poll_camera(false); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + assert!(st["deleted"].as_u64().unwrap() >= 3); + let g = s.project.comp(cid).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().clone(); + let sv = ct::solve(&crate::camera_track::static_params(&g)).unwrap(); + assert!(sv.points.iter().all(|p| !del.contains(&(p.id as u64)))); + + // Trimming the layer changes its frames: tracks and solve are cleared and queued. + s.edit("trim", None, |p, _| { + p.comp_mut(cid).unwrap().layer_mut(clip).unwrap().in_point = frame_time(2); + Ok(()) + }) + .unwrap(); + let st = status(&mut s); + assert_eq!(st["analyzed"], false); + assert_eq!(st["pending"], true); + + // A background analysis can be cancelled: nothing is written. + s.execute_checked("camera.analyze", json!({})).unwrap(); + assert!(s.is_camera_analyzing() || s.camera_job.is_some()); + let banner = status(&mut s)["banner"].as_str().map(str::to_string); + if s.is_camera_analyzing() { + assert!(banner.unwrap().starts_with("Analyzing in background (step 1 of 2)")); + } + s.execute("camera.cancel", json!({})).ok(); + let t0 = std::time::Instant::now(); + while s.camera_job.is_some() { + assert!(t0.elapsed().as_secs() < 120); + std::thread::sleep(std::time::Duration::from_millis(5)); + s.poll_camera(false); + } + // (When the job finished before the cancel landed it wrote a valid analysis.) + let st = status(&mut s); + assert_eq!(st["running"], false); + assert!(s.execute_checked("track.camera", json!({"layer": clip.0, "shotType": "bogus"})).is_err()); + assert!(s.execute_checked("track.camera", json!({"layer": clip.0, "aov": 500})).is_err()); +} + +#[test] +fn effect_parameters_match_after_effects() { + let spec = effectcraft_effects::registry().iter().find(|e| e.id == ct::ID).unwrap(); + assert_eq!(spec.name, "3D Camera Tracker"); + assert_eq!(spec.category, "Perspective"); + let names: Vec<&str> = spec.params.iter().filter(|p| !matches!(p.ui, effectcraft_project::ParamUi::Hidden)).map(|p| p.name).collect(); + for n in [ + "Shot Type", + "Horizontal Angle of View", + "Show Track Points", + "Render Track Points", + "Track Point Size", + "Target Size", + "Solve Method", + "Detailed Analysis", + "Auto-delete Points Across Time", + "Hide Warning Banner", + ] { + assert!(names.contains(&n), "{n}"); + } + // The Animation menu entry is a real command now. + let c = crate::find_command("track.camera").unwrap(); + assert_eq!(c.label, "Track Camera"); +} diff --git a/crates/engine/src/tests_stubs.rs b/crates/engine/src/tests_stubs.rs index 141858ca..bde98add 100644 --- a/crates/engine/src/tests_stubs.rs +++ b/crates/engine/src/tests_stubs.rs @@ -331,7 +331,7 @@ fn implemented_stubs_are_enabled_and_workspaces_are_frontend() { assert!(s.drain_events().iter().any(|e| matches!(e, Event::Frontend { command, .. } if command == id))); } // What remains a stub is still disabled. - for id in ["view.displayColorManagement", "file.createProxy", "track.camera"] { + for id in ["view.displayColorManagement", "file.createProxy"] { assert!(!s.is_enabled(id), "{id}"); } } diff --git a/crates/track/Cargo.toml b/crates/track/Cargo.toml index 9bd46e26..4800e06e 100644 --- a/crates/track/Cargo.toml +++ b/crates/track/Cargo.toml @@ -14,3 +14,4 @@ effectcraft-raster = { workspace = true } rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +web-time = { workspace = true } diff --git a/crates/track/src/camtrack/bundle.rs b/crates/track/src/camtrack/bundle.rs index 84e8a5ad..72cb2f17 100644 --- a/crates/track/src/camtrack/bundle.rs +++ b/crates/track/src/camtrack/bundle.rs @@ -173,6 +173,9 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> } let mut cur = cost(cams, pts, obs, o); let initial = cur; + let t0 = web_time::Instant::now(); + let mut tries = 0usize; + let dbg = std::env::var_os("EC_CAMTRACK_DEBUG").is_some() && l.n > 200; let mut lambda = 1e-4; let mut iters = 0; if l.n == 0 && o.fix_points { @@ -231,7 +234,10 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> for (ia, ja) in &jc { let wv = [w * (ja[0] * jx[0][0] + ja[1] * jx[1][0]), w * (ja[0] * jx[0][1] + ja[1] * jx[1][1]), w * (ja[0] * jx[0][2] + ja[1] * jx[1][2])]; - match ps.w.iter_mut().find(|e| e.0 == *ia) { + // Each observation of a point is on another camera: only the shared focal + // length repeats. + let found = if Some(*ia) == l.shared { ps.w.iter_mut().find(|e| e.0 == *ia) } else { None }; + match found { Some(e) => e.1 = add(e.1, wv), None => ps.w.push((*ia, wv)), } @@ -294,31 +300,67 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> // ---- Try steps with increasing damping. let mut accepted = false; for _try in 0..12 { + tries += 1; let mut s = base_s.clone(); let mut rhs = base_bc.clone(); for i in 0..n { let d = s[i * n + i]; s[i * n + i] = d + lambda * d.max(1e-6 * mean_diag); } - // Schur complement over the points. - let mut vinv: Vec> = vec![None; npts]; + // Schur complement over the points: S −= W V⁻¹ Wᵀ, rhs −= W V⁻¹ bp (in parallel + // chunks of points when the reduced system is small enough to copy per thread). + let vinv: Vec> = if o.fix_points { + vec![None; npts] + } else { + systems + .par_iter() + .map(|ps| { + let mut v = ps.v; + let vd = (v[0][0] + v[1][1] + v[2][2]) / 3.0; + for k in 0..3 { + v[k][k] += lambda * v[k][k].max(1e-6 * vd.max(1e-12)) + 1e-12; + } + inv3(&v) + }) + .collect() + }; if !o.fix_points { - for (j, ps) in systems.iter().enumerate() { - let mut v = ps.v; - let vd = (v[0][0] + v[1][1] + v[2][2]) / 3.0; - for k in 0..3 { - v[k][k] += lambda * v[k][k].max(1e-6 * vd.max(1e-12)) + 1e-12; - } - let Some(vi) = inv3(&v) else { continue }; - vinv[j] = Some(vi); - let vb = mv(&vi, ps.bp); - let wv: Vec = ps.w.iter().map(|(_, w)| mv(&vi, *w)).collect(); - for (a, (ia, wa)) in ps.w.iter().enumerate() { - rhs[*ia] -= dot(*wa, vb); - for (ib, wb) in &ps.w { - s[ia * n + ib] -= dot(wv[a], *wb); + let reduce = |range: std::ops::Range, s: &mut [f64], rhs: &mut [f64]| { + for j in range { + let (Some(vi), ps) = (&vinv[j], &systems[j]) else { continue }; + let vb = mv(vi, ps.bp); + let wv: Vec = ps.w.iter().map(|(_, w)| mv(vi, *w)).collect(); + for (a, (ia, wa)) in ps.w.iter().enumerate() { + rhs[*ia] -= dot(*wa, vb); + for (ib, wb) in &ps.w[a..] { + let d = dot(wv[a], *wb); + s[ia * n + ib] -= d; + if ia != ib { + s[ib * n + ia] -= d; + } + } } } + }; + let chunks = rayon::current_num_threads().clamp(1, 8); + if n <= 1200 && npts >= 64 && chunks > 1 { + let per = npts.div_ceil(chunks); + let parts: Vec<(Vec, Vec)> = (0..chunks) + .into_par_iter() + .map(|t| { + let (mut ds, mut dr) = (vec![0.0; n * n], vec![0.0; n]); + reduce(t * per..((t + 1) * per).min(npts), &mut ds, &mut dr); + (ds, dr) + }) + .collect(); + for (ds, dr) in parts { + s.par_iter_mut().zip(ds.par_iter()).for_each(|(x, y)| *x += y); + for (x, y) in rhs.iter_mut().zip(dr) { + *x += y; + } + } + } else { + reduce(0..npts, &mut s, &mut rhs); } } let mut dc = rhs; @@ -363,9 +405,18 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> let gain = cur - c2; cur = c2; lambda = (lambda / 3.0).max(1e-9); - accepted = gain > 1e-10 * cur.max(1e-12); + accepted = gain > 1e-6 * cur.max(1e-12); if !accepted { // Converged (tiny improvement). + if dbg { + eprintln!( + "ba n={} pts={} obs={} iters={iters} tries={tries} {:.2}s cost {initial:.1} -> {cur:.1}", + l.n, + npts, + obs.len(), + t0.elapsed().as_secs_f64() + ); + } return BaReport { initial_cost: initial, cost: cur, iterations: iters }; } break; @@ -376,6 +427,9 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> break; } } + if dbg { + eprintln!("ba n={} pts={} obs={} iters={iters} tries={tries} {:.2}s cost {initial:.1} -> {cur:.1}", l.n, npts, obs.len(), t0.elapsed().as_secs_f64()); + } BaReport { initial_cost: initial, cost: cur, iterations: iters } } diff --git a/crates/track/src/camtrack/features.rs b/crates/track/src/camtrack/features.rs index 311ec711..30b019e5 100644 --- a/crates/track/src/camtrack/features.rs +++ b/crates/track/src/camtrack/features.rs @@ -89,7 +89,7 @@ impl TrackAnalyzer { let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize); !occ.get(cy * gw + cx).copied().unwrap_or(true) }; - let opts = CornerOpts { max_features: want - self.active.len().min(want), min_distance: md, quality: 0.005, window: 2, border: 8 }; + let opts = CornerOpts { max_features: want - self.active.len().min(want), min_distance: md, quality: 0.001, window: 2, border: 8 }; for p in good_features(&pyr, &opts, Some(&free)) { let l = pyr.to_layer(p); let id = self.tracks.len() as u32; diff --git a/crates/track/src/camtrack/linalg.rs b/crates/track/src/camtrack/linalg.rs index a7067f26..b719cc8e 100644 --- a/crates/track/src/camtrack/linalg.rs +++ b/crates/track/src/camtrack/linalg.rs @@ -298,6 +298,23 @@ pub fn sym_eigen(a: &[f64], n: usize) -> (Vec, Vec) { (vals, vecs) } +/// Dot product, four lanes (lets the compiler vectorise). +fn dot_n(a: &[f64], b: &[f64]) -> f64 { + let mut acc = [0.0f64; 4]; + let (ca, cb) = (a.chunks_exact(4), b.chunks_exact(4)); + let (ra, rb) = (ca.remainder(), cb.remainder()); + for (x, y) in ca.zip(cb) { + for k in 0..4 { + acc[k] += x[k] * y[k]; + } + } + let mut s = acc[0] + acc[1] + acc[2] + acc[3]; + for (x, y) in ra.iter().zip(rb) { + s += x * y; + } + s +} + /// Solve `A x = b` for a symmetric positive definite `A` (`n × n`, row-major) in place by /// Cholesky factorisation. Returns false when `A` is not positive definite. pub fn cholesky_solve(a: &mut [f64], n: usize, b: &mut [f64]) -> bool { @@ -313,16 +330,9 @@ pub fn cholesky_solve(a: &mut [f64], n: usize, b: &mut [f64]) -> bool { a[j * n + j] = d; let (head, tail) = a.split_at_mut((j + 1) * n); let rowj = &head[j * n..j * n + j]; - // Rows below j are independent. - use rayon::prelude::*; - let work = |row: &mut [f64]| { - let s: f64 = row[j] - row[..j].iter().zip(rowj).map(|(a, b)| a * b).sum::(); - row[j] = s / d; - }; - if n - j > 192 { - tail.par_chunks_mut(n).for_each(work); - } else { - tail.chunks_mut(n).for_each(work); + // Rows below j (sequential: per-column parallel dispatch costs more than it saves). + for row in tail.chunks_mut(n) { + row[j] = (row[j] - dot_n(&row[..j], rowj)) / d; } } // Forward: L y = b. diff --git a/crates/track/src/camtrack/mod.rs b/crates/track/src/camtrack/mod.rs index 056d7ad0..76c0637f 100644 --- a/crates/track/src/camtrack/mod.rs +++ b/crates/track/src/camtrack/mod.rs @@ -13,7 +13,10 @@ //! the other keyframes are added incrementally (**resection** by robust pose refinement, //! **triangulation** of new tracks by the linear method of H&Z §12.2); a sparse //! **Levenberg–Marquardt bundle adjustment** with the Schur complement over the points -//! ([`bundle`]) refines everything; the frames between keyframes are resected last. +//! ([`bundle`]) refines everything; the frames between keyframes are resected last. Inlier +//! thresholds start from the analysis resolution and are re-estimated from the residuals +//! (median absolute residual) once the keyframes are solved, so features on moving objects, +//! occlusion edges and screen-locked overlays drop out. //! 3. **Focal length**: *Specify Angle of View* fixes it; *Fixed Angle of View* searches it //! (the reprojection error of the keyframe reconstruction over a log-spaced range of focal //! lengths, refined by golden-section search) and then adjusts it in the bundle adjustment @@ -22,7 +25,8 @@ //! 4. **Tripod pan** shots (no camera translation) are solved as a pure rotation: frame-to-frame //! rotations from the tracked rays (orthogonal Procrustes) refined by the same bundle //! adjustment with the camera centres fixed. *Auto Detect* solves both ways and keeps the -//! tripod solution when it explains the tracks as well as the general one. +//! tripod solution when it explains the tracks as well as the general one (a quick rotation fit +//! at the general solution's focal length skips the tripod search when it clearly fails). //! //! The camera model has square pixels, the principal point at the layer centre and no lens //! distortion. Camera space is x right, y down, z forward (After Effects' convention); a camera diff --git a/crates/track/src/camtrack/solver.rs b/crates/track/src/camtrack/solver.rs index da3a9290..85edd2bd 100644 --- a/crates/track/src/camtrack/solver.rs +++ b/crates/track/src/camtrack/solver.rs @@ -37,8 +37,9 @@ struct Data { tracks: Vec<(u32, usize, Vec<[f64; 2]>)>, /// Per frame: (track index, position). by_frame: Vec>, - /// Tracking noise scale (pixels). - noise: f64, + /// Tracking noise scale (pixels): starts from the analysis resolution and is re-estimated + /// from the residuals once the keyframes are solved. + noise_bits: std::sync::atomic::AtomicU64, } impl Data { @@ -58,23 +59,31 @@ impl Data { } tracks.push((tr.id, tr.start as usize, pts)); } - Data { n, w: t.size[0], tracks, by_frame, noise: t.factor.max(1.0) } + let noise = (0.5 * t.factor).max(0.5); + Data { n, w: t.size[0], tracks, by_frame, noise_bits: std::sync::atomic::AtomicU64::new(noise.to_bits()) } } fn at(&self, ti: u32, k: usize) -> Option<[f64; 2]> { let (_, s, p) = &self.tracks[ti as usize]; k.checked_sub(*s).and_then(|i| p.get(i).copied()) } + fn noise(&self) -> f64 { + f64::from_bits(self.noise_bits.load(Ordering::Relaxed)) + } + fn set_noise(&self, v: f64) { + self.noise_bits.store(v.to_bits(), Ordering::Relaxed); + } fn inlier_thr(&self) -> f64 { - 4.0 * self.noise + 4.0 * self.noise() } fn huber(&self) -> f64 { - 1.0 * self.noise + self.noise() } } /// Parallax-based keyframes: a new keyframe when the median feature motion since the last -/// exceeds 3 % of the width, when fewer than 60 % of its features survive, or before the shared -/// features drop below 24. The first and last frames with features are keyframes. +/// exceeds 1.5 % of the width, when fewer than 70 % of its features survive, after 12 frames, or +/// before the shared features drop below 24. The first and last frames with features are +/// keyframes. fn keyframes(d: &Data) -> Vec { let Some(first) = (0..d.n).find(|k| d.by_frame[*k].len() >= 8) else { return vec![] }; let mut kf = vec![first]; @@ -97,7 +106,7 @@ fn keyframes(d: &Data) -> Vec { disp.sort_by(f64::total_cmp); let med = disp.get(disp.len() / 2).copied().unwrap_or(0.0); let ratio = disp.len() as f64 / base.len().max(1) as f64; - if med > 0.03 * d.w || ratio < 0.6 || disp.len() < 24 { + if med > 0.015 * d.w || ratio < 0.7 || disp.len() < 24 || j - last >= 12 { last = j; kf.push(j); } @@ -405,7 +414,7 @@ fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> } let x1: Vec<[f64; 2]> = sh.iter().map(|s| [s.1[0] / f, s.1[1] / f]).collect(); let x2: Vec<[f64; 2]> = sh.iter().map(|s| [s.2[0] / f, s.2[1] / f]).collect(); - let thr = 2.0 * d.noise / f; + let thr = 2.0 * d.noise() / f; let mut cands: Vec = vec![]; if use_e && let Some((e, inl)) = geometry::ransac_essential(&x1, &x2, thr, 400, (a * 7919 + b) as u64) { let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip(); @@ -415,7 +424,7 @@ fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> } if use_h { let (s1, s2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = sh.iter().map(|s| (s.1, s.2)).unzip(); - if let Some(fit) = ransac(Model::Homography, &s1, &s2, 2.0 * d.noise, 500, (a * 31 + b) as u64) { + if let Some(fit) = ransac(Model::Homography, &s1, &s2, 2.0 * d.noise(), 500, (a * 31 + b) as u64) { let h = fit.h.0; // Normalised: K⁻¹ H K with K = diag(f, f, 1). let hn = [[h[0][0], h[0][1], h[0][2] / f], [h[1][0], h[1][1], h[1][2] / f], [h[2][0] * f, h[2][1] * f, h[2][2]]]; @@ -651,6 +660,12 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo let try_general = matches!(s.method, SolveMethod::Auto | SolveMethod::Typical | SolveMethod::MostlyFlat); let try_tripod = matches!(s.method, SolveMethod::Auto | SolveMethod::TripodPan); + let t0 = web_time::Instant::now(); + let lap = |what: &str| { + if debug() { + eprintln!("{what}: {:.2} s", t0.elapsed().as_secs_f64()); + } + }; let mut results: Vec<(Attempt, f64)> = vec![]; if try_general { let f = match fixed_f { @@ -666,7 +681,20 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo results.push((a, f)); } } - if try_tripod && !cancelled() { + lap("general"); + // Auto Detect: a quick tripod check at the general solve's focal length; when a rotation + // clearly cannot explain the tracks, the tripod search is skipped. + let tripod_plausible = match (s.method, results.first()) { + (SolveMethod::Auto, Some((g, f))) => { + let quick = tripod(&d, &mini, *f, &tcfg, cancel).map(|a| a.err).unwrap_or(f64::INFINITY); + if debug() { + eprintln!("quick tripod check: {quick:.3} vs general {:.3}", g.err); + } + quick <= 3.0 * g.err + d.noise() + } + _ => true, + }; + if try_tripod && tripod_plausible && !cancelled() { let f = match fixed_f { Some(f) => f, None => search_focal(d.w, &|f| tripod(&d, &mini, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)), @@ -696,7 +724,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo if debug() { eprintln!("auto: general {eg:.4} tripod {et:.4} baseline/depth {:.4}", base / med); } - if et <= (eg * 1.25).max(eg + 0.05 * d.noise) || base / med < 0.01 { t } else { g } + if et <= (eg * 1.25).max(eg + 0.05 * d.noise()) || base / med < 0.01 { t } else { g } } (Some(g), None) => g, (None, Some(t)) => t, @@ -717,15 +745,44 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo (None, true) => FocalMode::PerCamera(F_PRIOR), (None, false) => FocalMode::Shared, }; + lap("tripod"); // Keyframes: adjust with the focal length free, reject, re-triangulate, adjust again. let kf_frames: Vec = r.cams.keys().copied().collect(); r.adjust(&d, &kf_frames, &cfg, focal, 40); r.reject(&d); + // Re-estimate the tracking noise from the inlier residuals (robustly, from their median: + // for 2D Gaussian noise of deviation σ the median distance is 1.1774 σ), then tighten the + // thresholds: observations of moving objects and occlusion edges become outliers. + let mut res: Vec = vec![]; + for (k, c) in &r.cams { + for (ti, uv) in &d.by_frame[*k] { + if r.outliers.contains(&(*ti, *k)) { + continue; + } + if let Some(e) = r.pts.get(ti).and_then(|x| bundle::residual(c, *x, *uv)) { + res.push(e[0].hypot(e[1])); + } + } + } + if res.len() > 20 { + res.sort_by(f64::total_cmp); + let sigma = res[res.len() / 2] / 1.1774; + let floor = 0.05 * tracks.factor.max(1.0); + d.set_noise((1.5 * sigma).clamp(floor, d.noise())); + if debug() { + eprintln!("noise: σ {sigma:.3} → {:.3}", d.noise()); + } + r.outliers.clear(); + r.reject(&d); + } r.triangulate(&d, kind, 1.0, 1.0); r.adjust(&d, &kf_frames, &cfg, focal, 40); + r.reject(&d); + r.adjust(&d, &kf_frames, &cfg, focal, 20); if cancelled() { return err("cancelled"); } + lap("keyframe adjustment"); // Every other frame: resect from the points (nearest registered camera first). let mut todo: Vec = (0..d.n).filter(|k| !r.cams.contains_key(k)).collect(); todo.sort_by_key(|k| r.cams.keys().map(|c| c.abs_diff(*k)).min().unwrap_or(usize::MAX)); @@ -746,6 +803,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo if cancelled() { return err("cancelled"); } + lap("resection"); // Points for the tracks that only lived between keyframes, then the final adjustment over // every frame (when that stays affordable). r.triangulate(&d, kind, 1.0, 1.0); @@ -786,6 +844,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo } (m, _) => m, }; + lap("final adjustment"); Ok(finish(&d, tracks, s, r, method, kind)) } diff --git a/crates/track/src/camtrack/tests.rs b/crates/track/src/camtrack/tests.rs index 258161f9..4b5a8346 100644 --- a/crates/track/src/camtrack/tests.rs +++ b/crates/track/src/camtrack/tests.rs @@ -358,3 +358,81 @@ fn perf_long_clip() { eprintln!("300 frames, {} tracks: {:?} err {:.3} focal {:.1} in {:?}", tr.tracks.len(), s.method_used, s.average_error, s.focal(), t0.elapsed()); assert!(s.average_error < 0.5); } + +/// Solve tracks saved from the app (`EC_CAMTRACK_FILE=tracks.json cargo test -p effectcraft-track +/// --release solve_saved_tracks -- --ignored --nocapture`): a debugging aid. +#[test] +#[ignore] +fn solve_saved_tracks() { + let Some(path) = std::env::var_os("EC_CAMTRACK_FILE") else { return }; + let mut tr = CameraTracks::from_json(&std::fs::read_to_string(path).unwrap()).unwrap(); + if let Some(maxy) = std::env::var("EC_CAMTRACK_MAXY").ok().and_then(|v| v.parse::().ok()) { + tr.tracks.retain(|t| t.pts.iter().all(|p| p[1] < maxy)); + } + let shot = if std::env::var_os("EC_CAMTRACK_VARIABLE").is_some() { ShotType::VariableZoom } else { ShotType::FixedAngle }; + let t0 = std::time::Instant::now(); + // With the true cameras (`[{pos, poi, zoom}]` per frame, two-node), how consistent are the + // tracks themselves? + if let Some(tp) = std::env::var_os("EC_CAMTRACK_TRUTH") { + let v: Vec = serde_json::from_str(&std::fs::read_to_string(tp).unwrap()).unwrap(); + let g = |x: &serde_json::Value| -> V3 { [x[0].as_f64().unwrap(), x[1].as_f64().unwrap(), x[2].as_f64().unwrap()] }; + let cams: Vec<(super::geometry::Pose, f64)> = v + .iter() + .map(|c| { + let (pos, poi) = (g(&c["pos"]), g(&c["poi"])); + let fwd = normalize(sub(poi, pos)); + let d0 = [0.0, 1.0, 0.0]; + let down = normalize(sub(d0, scale(fwd, dot(d0, fwd)))); + let right = cross(down, fwd); + (super::geometry::Pose { r: [right, down, fwd], c: pos }, c["zoom"].as_f64().unwrap()) + }) + .collect(); + let (w, h) = (tr.size[0], tr.size[1]); + let mut means = vec![]; + let mut drop_ids = vec![]; + for t in &tr.tracks { + let views: Vec<(super::geometry::Pose, [f64; 2])> = t + .pts + .iter() + .enumerate() + .map(|(i, p)| { + let (pose, z) = cams[t.start as usize + i]; + (pose, [(p[0] as f64 - w / 2.0) / z, (p[1] as f64 - h / 2.0) / z]) + }) + .collect(); + let Some(x) = super::geometry::triangulate(&views) else { continue }; + let e: Vec = views.iter().filter_map(|(p, o)| super::geometry::reproj2(p, x, *o).map(|e| e.sqrt() * cams[0].1)).collect(); + means.push((e.iter().sum::() / e.len().max(1) as f64, t.pts[0])); + if std::env::var_os("EC_CAMTRACK_TRUTH_FILTER").is_some() && means.last().unwrap().0 > 1.0 { + drop_ids.push(t.id); + } + } + tr.tracks.retain(|t| !drop_ids.contains(&t.id)); + means.sort_by(|a, b| a.0.total_cmp(&b.0)); + eprintln!( + "true-camera track errors (px): median {:.2}; quartiles {:?}", + means[means.len() / 2].0, + [means[means.len() / 4].0, means[means.len() * 3 / 4].0] + ); + for m in means.iter().step_by(6) { + eprintln!(" {:.2} at {:?}", m.0, m.1); + } + } + let mut st = settings(shot, SolveMethod::Auto); + if let Some(a) = std::env::var("EC_CAMTRACK_AOV").ok().and_then(|v| v.parse::().ok()) { + st.shot = ShotType::SpecifyAngle; + st.hfov = a; + } + let s = solve(&tr, &st, None).expect("solve"); + eprintln!( + "{} frames, {} tracks: {:?} err {:.3} focal {:.1} (hfov {:.1}) points {} in {:?}", + tr.frames, + tr.tracks.len(), + s.method_used, + s.average_error, + s.focal(), + s.hfov(s.focal()), + s.points.len(), + t0.elapsed() + ); +} diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 071c1948..d581ef35 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -727,6 +727,11 @@ impl EffectcraftApp { self.session.poll_warp(true); ctx.request_repaint_after(std::time::Duration::from_millis(100)); } + // 3D Camera Tracker: likewise (tracking, then solving). + if self.session.camera_job.is_some() || !self.session.camera_pending.is_empty() { + self.session.poll_camera(true); + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } self.apply_prefs(&ctx); self.handle_events(&ctx); self.tick_autosave(&ctx); diff --git a/crates/ui-egui/src/panels/camera_tracker_ui.rs b/crates/ui-egui/src/panels/camera_tracker_ui.rs new file mode 100644 index 00000000..41fb7204 --- /dev/null +++ b/crates/ui-egui/src/panels/camera_tracker_ui.rs @@ -0,0 +1,438 @@ +//! 3D Camera Tracker in the Composition viewer and Effect Controls: +//! +//! - while the effect is selected, the solved 3D track points are drawn as coloured targets +//! sized by depth (or the 2D source tracks with Show Track Points ▸ 2D Source); +//! - hovering between three points shows the target plane they span (a disc in perspective); +//! click a point (Shift adds / toggles) or drag a marquee to select points, click inside a +//! triangle to select its three points; the selection's target is drawn; +//! - right-click: Set Ground Plane and Origin, Create Text / Solid / Null / Shadow Catcher … +//! and Camera, Create Multiple …, Delete Selected Points; Delete removes the selection; +//! - the "Analyzing in background (step 1 of 2)" / "Solving camera" banner; +//! - the Effect Controls block with Analyze / Cancel / Create Camera, Method Used and Average +//! Error. +//! +//! Everything is dispatched to the `camera.*` commands. + +use std::sync::Arc; + +use effectcraft_engine::Session; +use effectcraft_engine::effects::camera_tracker as ct; +use effectcraft_engine::geom::{Mat3, vec2 as gv2}; +use effectcraft_engine::keyframe::Value; +use effectcraft_engine::project::{Comp, Layer, LayerId, PropGroup, Uid}; +use effectcraft_engine::render::EvalCtx; +use effectcraft_engine::track::camtrack::{CameraSolve, CameraTracks, Target, point_color}; +use egui::{Align2, Color32, Pos2, Rect, Sense, Stroke, pos2, vec2}; +use serde_json::json; + +use super::viewer::ViewerMap; +use crate::theme::Tokens; +use crate::{EffectcraftApp, widgets}; + +/// The tracker whose points the viewer shows. +struct Active { + layer: LayerId, + uid: Uid, + solve: Option>, + tracks: Option>, + show_3d: bool, + point_size: f64, + target_size: f64, + /// Layer pixels → comp pixels. + l2c: Mat3, + frame: usize, +} + +/// A drawn point: (id, screen position, screen radius). +type Hit = (u32, Pos2, f32); + +fn tracker_group(l: &Layer, uid: Uid) -> Option<&PropGroup> { + l.props.find_group(uid).filter(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == ct::ID)) +} + +/// The selected 3D Camera Tracker effect on an active layer (After Effects shows the points +/// while the effect is selected). +fn active(s: &Session, comp: &Comp, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3) -> Option { + let time = ectx.time; + for (lid, uid) in &s.state.selected_props { + let Some(l) = comp.layer(*lid).filter(|l| l.is_active_at(time)) else { continue }; + let Some(g) = tracker_group(l, *uid) else { continue }; + let params = effectcraft_engine::camera_track::static_params(g); + let val = |id: &str| effectcraft_engine::effects::warp_stab::param(¶ms, id).cloned(); + let solve = ct::solve(¶ms); + let tracks = ct::tracks(¶ms); + let lt = l.layer_time(time).seconds(); + let frame = solve.as_ref().and_then(|sv| sv.frame_at(lt)).or_else(|| { + let t = tracks.as_ref()?; + (t.frame_duration > 0.0).then(|| ((lt - t.start) / t.frame_duration).round().clamp(0.0, t.frames.saturating_sub(1) as f64) as usize) + }); + return Some(Active { + layer: *lid, + uid: *uid, + solve, + tracks, + show_3d: val("showTrackPoints").map(|v| v.as_enum()).unwrap_or(1) == 1, + point_size: val("trackPointSize").map(|v| v.as_f64()).unwrap_or(100.0), + target_size: val("targetSize").map(|v| v.as_f64()).unwrap_or(100.0), + l2c: l2c(ectx, l), + frame: frame.unwrap_or(0), + }); + } + None +} + +fn to_screen(a: &Active, map: &ViewerMap, q: [f64; 2]) -> Pos2 { + let c = a.l2c.apply(gv2(q[0], q[1])); + map.to_screen([c.x, c.y]) +} + +fn col(id: u32) -> Color32 { + let c = point_color(id); + Color32::from_rgb((c[0] * 255.0) as u8, (c[1] * 255.0) as u8, (c[2] * 255.0) as u8) +} + +/// Comp-pixel scale of the layer → comp mapping. +fn l2c_scale(m: &Mat3) -> f32 { + m.0[0][0].hypot(m.0[1][0]) as f32 +} + +/// Draw the target disc (concentric rings in perspective) of `t` on frame `k`. +fn draw_target(p: &egui::Painter, a: &Active, map: &ViewerMap, sv: &CameraSolve, t: &Target) { + let Some(cam) = sv.frames.get(a.frame) else { return }; + let r = t.size * 0.5 * a.target_size / 100.0; + for (i, frac) in [1.0, 0.66, 0.33].into_iter().enumerate() { + let ring = ct::target_circle(t.center, t.normal, r * frac, 48); + let pts: Vec = ring.iter().filter_map(|x| cam.project(sv.size, *x)).map(|q| to_screen(a, map, q)).collect(); + if pts.len() < 3 { + return; + } + let fill = if i % 2 == 0 { Color32::from_rgba_unmultiplied(0xd0, 0x30, 0x30, 70) } else { Color32::from_rgba_unmultiplied(0xff, 0xff, 0xff, 50) }; + p.add(egui::Shape::convex_polygon(pts.clone(), fill, Stroke::new(1.0, Color32::from_rgb(0xe0, 0x40, 0x40)))); + } + if let Some(c) = cam.project(sv.size, t.center) { + let c = to_screen(a, map, c); + p.circle_filled(c, 3.0, Color32::from_rgb(0xff, 0xd0, 0x40)); + } +} + +/// The three drawn points forming a triangle around `hp` (nearest first). +fn triangle_at(hits: &[Hit], hp: Pos2) -> Option<[u32; 3]> { + let mut near: Vec<&Hit> = hits.iter().collect(); + near.sort_by(|x, y| x.1.distance_sq(hp).total_cmp(&y.1.distance_sq(hp))); + let near: Vec<&Hit> = near.into_iter().take(8).collect(); + let inside = |a: Pos2, b: Pos2, c: Pos2| { + let s = |p: Pos2, q: Pos2| (q.x - p.x) * (hp.y - p.y) - (q.y - p.y) * (hp.x - p.x); + let (d1, d2, d3) = (s(a, b), s(b, c), s(c, a)); + !((d1 < 0.0 || d2 < 0.0 || d3 < 0.0) && (d1 > 0.0 || d2 > 0.0 || d3 > 0.0)) + }; + for i in 0..near.len() { + for j in i + 1..near.len() { + for k in j + 1..near.len() { + let (a, b, c) = (near[i], near[j], near[k]); + // Skip slivers. + let area = ((b.1 - a.1).x * (c.1 - a.1).y - (b.1 - a.1).y * (c.1 - a.1).x).abs(); + if area > 30.0 && inside(a.1, b.1, c.1) { + return Some([a.0, b.0, c.0]); + } + } + } + } + None +} + +fn run(app: &mut EffectcraftApp, ctx: &egui::Context, cmd: &str, p: serde_json::Value) { + if let Err(e) = crate::menus::invoke(app, ctx, cmd, p) { + app.ui.status = e; + } +} + +/// The viewer overlay and its interaction (call after the viewer's own gestures so it takes +/// the clicks while a 3D Camera Tracker is selected). +pub fn viewer_hook( + app: &mut EffectcraftApp, + ui: &mut egui::Ui, + painter: &egui::Painter, + map: &ViewerMap, + ectx: &EvalCtx, + l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3, +) { + let ctx = ui.ctx().clone(); + let comp = ectx.comp; + banner(app, painter, map, ectx, l2c); + if !app.ui.viewer.show_layer_controls { + return; + } + let Some(a) = active(&app.session, comp, ectx, l2c) else { return }; + let selected = app.session.state.camera_points.clone(); + let lscale = l2c_scale(&a.l2c) * map.zoom; + let mut hits: Vec = vec![]; + match (&a.solve, a.show_3d) { + (Some(sv), true) => { + let pts = ct::projected(sv, a.frame); + let med = ct::median_depth(&pts); + for (id, q, z) in pts { + let sp = to_screen(&a, map, q); + if !map.area.expand(20.0).contains(sp) { + continue; + } + let r = (ct::target_radius(a.point_size, z, med) as f32 * lscale).clamp(2.0, 30.0); + let c = col(id); + let sel = selected.contains(&id); + if sel { + painter.circle_filled(sp, r, c); + painter.circle_stroke(sp, r + 1.5, Stroke::new(1.5, Color32::WHITE)); + } else { + painter.circle_stroke(sp, r, Stroke::new((r * 0.3).clamp(1.0, 3.0), c)); + painter.circle_filled(sp, (r * 0.25).max(1.0), c); + } + hits.push((id, sp, r)); + } + } + _ => { + // 2D source tracks (also shown before the solve finishes). + if let Some(tr) = &a.tracks { + for t in &tr.tracks { + let Some(q) = t.at(a.frame as u32) else { continue }; + let sp = to_screen(&a, map, q); + let c = col(t.id); + let r = (3.0 * a.point_size as f32 / 100.0).clamp(1.5, 12.0); + painter.line_segment([sp - vec2(r, r), sp + vec2(r, r)], Stroke::new(1.2, c)); + painter.line_segment([sp - vec2(r, -r), sp + vec2(r, -r)], Stroke::new(1.2, c)); + hits.push((t.id, sp, r)); + } + } + } + } + let Some(sv) = a.solve.clone() else { return }; + if app.ui.tool != crate::state::Tool::Selection { + return; + } + // Interaction over the image. + let resp = ui.interact(map.area, egui::Id::new("viewer-camera-tracker"), Sense::click_and_drag()); + app.auto.add("viewer.cameraTracker", map.area, "3D Camera Tracker points"); + for (id, sp, r) in hits.iter().take(400) { + app.auto.add(&format!("viewer.cameraTracker.point.{id}"), Rect::from_center_size(*sp, vec2(r * 2.0 + 4.0, r * 2.0 + 4.0)), "track point"); + } + let hover = resp.hover_pos(); + let point_at = |p: Pos2| hits.iter().filter(|h| h.1.distance(p) <= h.2 + 3.0).min_by(|x, y| x.1.distance(p).total_cmp(&y.1.distance(p))).map(|h| h.0); + let tri = hover.filter(|p| point_at(*p).is_none()).and_then(|p| triangle_at(&hits, p)); + let mid = egui::Id::new("viewer-camera-tracker-marquee"); + let marquee: Option = ui.data(|d| d.get_temp(mid)); + // Hover target / the selection's target. + if marquee.is_none() { + if let Some(t3) = tri + && let Some(t) = effectcraft_engine::camera_track::target_for(&sv, &t3, a.frame) + { + let ps: Vec = t3.iter().filter_map(|i| hits.iter().find(|h| h.0 == *i)).map(|h| h.1).collect(); + if ps.len() == 3 { + painter.add(egui::Shape::closed_line(ps, Stroke::new(1.0, Color32::from_white_alpha(200)))); + } + draw_target(painter, &a, map, &sv, &t); + } else if !selected.is_empty() + && let Some(t) = effectcraft_engine::camera_track::target_for(&sv, &selected, a.frame) + { + draw_target(painter, &a, map, &sv, &t); + } + } + let mods = ui.input(|i| i.modifiers); + if resp.drag_started() + && let Some(p) = ui.input(|i| i.pointer.press_origin()) + { + ui.data_mut(|d| d.insert_temp(mid, p)); + } + if let (Some(start), Some(p)) = (marquee, resp.interact_pointer_pos()) { + let r = Rect::from_two_pos(start, p); + let t = app.tokens; + painter.rect_filled(r, 0.0, t.accent.gamma_multiply(0.12)); + painter.rect_stroke(r, 0.0, Stroke::new(1.0, t.accent), egui::StrokeKind::Middle); + if resp.drag_stopped() { + let ids: Vec = hits.iter().filter(|h| r.contains(h.1)).map(|h| h.0).collect(); + ui.data_mut(|d| d.remove::(mid)); + run(app, &ctx, "camera.selectPoints", json!({"points": ids, "add": mods.shift})); + } + } else if resp.drag_stopped() { + ui.data_mut(|d| d.remove::(mid)); + } + if resp.clicked() + && let Some(p) = resp.interact_pointer_pos() + { + match (point_at(p), triangle_at(&hits, p)) { + (Some(id), _) => run(app, &ctx, "camera.selectPoints", json!({"points": [id], "toggle": mods.shift})), + (None, Some(t3)) => run(app, &ctx, "camera.selectPoints", json!({"points": t3, "add": mods.shift})), + (None, None) => run(app, &ctx, "camera.selectPoints", json!({"points": []})), + } + } + // Right-click: the points under the cursor (selection, else the hovered triangle). + let cid = egui::Id::new("viewer-camera-tracker-menu-points"); + if resp.secondary_clicked() { + let pts: Vec = if !selected.is_empty() { + selected.clone() + } else if let Some(p) = resp.interact_pointer_pos() { + point_at(p).map(|i| vec![i]).or_else(|| triangle_at(&hits, p).map(|t| t.to_vec())).unwrap_or_default() + } else { + vec![] + }; + ui.data_mut(|d| d.insert_temp(cid, pts)); + } + let base = json!({"layer": a.layer.0, "effect": a.uid}); + resp.context_menu(|ui| { + let pts: Vec = ui.data(|d| d.get_temp(cid)).unwrap_or_default(); + let with = |kind: Option<&str>, multiple: bool| { + let mut q = base.clone(); + q["points"] = json!(pts); + if let Some(k) = kind { + q["kind"] = json!(k); + } + if multiple { + q["multiple"] = json!(true); + } + q + }; + let some = !pts.is_empty(); + let many = pts.len() > 1; + let mut pick: Option<(&str, serde_json::Value)> = None; + if ui.add_enabled(some, egui::Button::new("Set Ground Plane and Origin")).clicked() { + pick = Some(("camera.setGroundPlane", with(None, false))); + } + ui.separator(); + for (label, kind) in [ + ("Create Text and Camera", "text"), + ("Create Solid and Camera", "solid"), + ("Create Null and Camera", "null"), + ("Create Shadow Catcher, Camera and Light", "shadowCatcher"), + ] { + if ui.add_enabled(some, egui::Button::new(label)).clicked() { + pick = Some(("camera.createFromSolve", with(Some(kind), false))); + } + } + ui.separator(); + for (label, kind) in + [("Create Multiple Text Layers and Camera", "text"), ("Create Multiple Solids and Camera", "solid"), ("Create Multiple Nulls and Camera", "null")] + { + if ui.add_enabled(many, egui::Button::new(label)).clicked() { + pick = Some(("camera.createFromSolve", with(Some(kind), true))); + } + } + ui.separator(); + if ui.add_enabled(some, egui::Button::new("Delete Selected Points")).clicked() { + pick = Some(("camera.deletePoints", with(None, false))); + } + if let Some((cmd, q)) = pick { + ui.close(); + run(app, &ctx, cmd, q); + } + }); + if resp.hovered() + && !selected.is_empty() + && ui.input(|i| i.key_pressed(egui::Key::Delete) || i.key_pressed(egui::Key::Backspace)) + && !ctx.egui_wants_keyboard_input() + { + run(app, &ctx, "camera.deletePoints", json!({"layer": a.layer.0, "effect": a.uid, "points": selected})); + } +} + +/// "Analyzing in background (step 1 of 2)" / "Solving camera" over the analysed layer, and a +/// warning when the solve failed (unless Hide Warning Banner). +fn banner(app: &mut EffectcraftApp, painter: &egui::Painter, map: &ViewerMap, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3) { + let comp = ectx.comp; + let time = ectx.time; + let cid = ectx.comp_id; + let draw = |app: &mut EffectcraftApp, layer: &Layer, text: String, color: Color32, id: &str| { + let m = l2c(ectx, layer); + let (w, h) = effectcraft_engine::render::source_size(&app.session.project, layer); + let c = m.apply(gv2(w as f64 / 2.0, h as f64 / 2.0)); + let at = map.to_screen([c.x, c.y]); + let galley = painter.layout_no_wrap(text.clone(), Tokens::medium(13.0), Color32::WHITE); + let r = Rect::from_center_size(at, galley.size() + vec2(28.0, 14.0)).intersect(map.area); + painter.rect_filled(r, 3.0, color); + painter.galley(pos2(r.center().x - galley.size().x / 2.0, r.center().y - galley.size().y / 2.0), galley, Color32::WHITE); + app.auto.add(id, r, &text); + }; + if let Some((cc, cl, _)) = app.session.camera_target() + && cc == cid + && let Some(pr) = app.session.camera_progress() + && let Some(layer) = comp.layer(cl) + && layer.is_active_at(time) + { + draw(app, layer, pr.banner(), Color32::from_rgba_unmultiplied(0x1d, 0x4f, 0x9c, 0xe0), "viewer.cameraBanner"); + return; + } + // A failed solve: tracks without a solve while nothing is running or queued. + for l in comp.layers.iter().filter(|l| l.is_active_at(time)) { + let Some(fx) = l.effects() else { continue }; + for g in fx.groups().filter(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == ct::ID)) { + let params = effectcraft_engine::camera_track::static_params(g); + let hide = effectcraft_engine::effects::warp_stab::param(¶ms, "advanced/hideWarningBanner").is_some_and(Value::as_bool); + let solve_key = matches!(g.prop(ct::SOLVE_KEY).map(|p| &p.value), Some(Value::Str(s)) if !s.is_empty()); + let failed = solve_key && ct::tracks(¶ms).is_some() && ct::solve(¶ms).is_none(); + if failed && !hide && !app.session.camera_pending.iter().any(|(_, ll, u)| *ll == l.id && *u == g.uid) { + draw(app, l, "Unable to solve camera".into(), Color32::from_rgba_unmultiplied(0x9c, 0x2a, 0x1d, 0xe0), "viewer.cameraWarning"); + return; + } + } + } +} + +/// Effect Controls: Analyze / Cancel, Create Camera and the solve report. +#[allow(clippy::too_many_arguments)] +pub fn editor( + app: &mut EffectcraftApp, + ui: &mut egui::Ui, + p: &egui::Painter, + layer: &Layer, + g: &PropGroup, + r: Rect, + actions: &mut Vec<(String, serde_json::Value)>, +) { + let t = app.tokens; + let running = app.session.camera_target().is_some_and(|(_, l, u)| l == layer.id && u == g.uid) && app.session.is_camera_analyzing(); + let busy = app.session.is_camera_analyzing(); + let me = json!({"layer": layer.id.0, "effect": g.uid}); + let b1 = Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 4.0), vec2(84.0, 22.0)); + let b2 = Rect::from_min_size(pos2(b1.max.x + 8.0, b1.min.y), vec2(84.0, 22.0)); + let b3 = Rect::from_min_size(pos2(b2.max.x + 8.0, b1.min.y), vec2(110.0, 22.0)); + if widgets::text_button(ui, b1, "Analyze", false, &t, egui::Id::new(("ct-analyze", g.uid))).clicked() && !busy { + actions.push(("camera.analyze".into(), me.clone())); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.analyze", g.uid), b1, "Analyze"); + if running { + if widgets::text_button(ui, b2, "Cancel", false, &t, egui::Id::new(("ct-cancel", g.uid))).clicked() { + actions.push(("camera.cancel".into(), json!({}))); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.cancel", g.uid), b2, "Cancel"); + } + let params = effectcraft_engine::camera_track::static_params(g); + let solve = ct::solve(¶ms); + if solve.is_some() { + if widgets::text_button(ui, b3, "Create Camera", false, &t, egui::Id::new(("ct-camera", g.uid))).clicked() { + actions.push(("camera.create".into(), me.clone())); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.createCamera", g.uid), b3, "Create Camera"); + } + let status = match app.session.camera_progress() { + Some(pr) if running => pr.banner(), + _ => match &solve { + Some(sv) => format!("Method Used: {} Average Error: {:.2} pixels", sv.method_used.label(), sv.average_error), + None if ct::tracks(¶ms).is_some() && app.session.camera_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => { + "Waiting to solve".into() + } + None if ct::tracks(¶ms).is_some() => "Unable to solve camera: try another Solve Method or Shot Type".into(), + None if app.session.camera_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => "Waiting to analyze".into(), + None => "Not analyzed: click Analyze".into(), + }, + }; + p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim); + if let Some(sv) = &solve { + let f = sv.focal(); + p.text( + pos2(r.min.x + 24.0, r.min.y + 56.0), + Align2::LEFT_CENTER, + format!("{} solved points · Horizontal Angle of View {:.1}°", sv.points.len(), sv.hfov(f)), + Tokens::ui(11.5), + t.text_faint, + ); + } +} + +/// Height of [`editor`]. +pub const EDITOR_HEIGHT: f32 = 68.0; diff --git a/crates/ui-egui/src/panels/effect_controls.rs b/crates/ui-egui/src/panels/effect_controls.rs index 0a19d57f..11640d88 100644 --- a/crates/ui-egui/src/panels/effect_controls.rs +++ b/crates/ui-egui/src/panels/effect_controls.rs @@ -545,6 +545,7 @@ fn editor_height(effect: &str, width: f32) -> f32 { match effect { "ec.color.curves" => 34.0 + curves_size(width) + 26.0, effectcraft_engine::effects::warp_stab::ID => 50.0, + effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::EDITOR_HEIGHT, "ec.color.levels" | "ec.color.levelsic" => 34.0 + 80.0 + 58.0, "ec.color.autolevels" | "ec.color.autocontrast" | "ec.color.autocolor" => 24.0 + 80.0 + 12.0, _ => 0.0, @@ -920,6 +921,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { match effect.as_str() { "ec.color.curves" => curves_editor(app, ui, &bp, &layer, g, &ectx, er, &mut actions), effectcraft_engine::effects::warp_stab::ID => warp_editor(app, ui, &bp, &layer, g, er, &mut actions), + effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::editor(app, ui, &bp, &layer, g, er, &mut actions), "ec.color.levels" | "ec.color.levelsic" => levels_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions), _ => histogram_only(app, ui, &bp, g, er), } diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 3f737502..ead1fe99 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -1,6 +1,7 @@ //! Panel bodies. pub mod anim_tools; +pub mod camera_tracker_ui; pub mod comp_settings; pub mod dialogs; pub mod dialogs_3d; diff --git a/crates/ui-egui/src/panels/tracker.rs b/crates/ui-egui/src/panels/tracker.rs index d1f456b2..364e114e 100644 --- a/crates/ui-egui/src/panels/tracker.rs +++ b/crates/ui-egui/src/panels/tracker.rs @@ -130,11 +130,14 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let cur = current(app); let can_track = has_comp && source.is_some(); - // Track Camera / Warp Stabilizer (not available) and Track Motion / Stabilize Motion. + // Track Camera / Warp Stabilizer and Track Motion / Stabilize Motion. let r1 = Rect::from_min_size(pos2(x0, y), vec2(half, 22.0)); let r2 = Rect::from_min_size(pos2(x0 + half + 6.0, y), vec2(half, 22.0)); let src_p = source.map(|l| json!({"layer": l.0})).unwrap_or_else(|| json!({})); - button(app, ui, r1, "Track Camera", false, "tracker.trackCamera"); + let can_camera = app.session.is_enabled("track.camera") && !app.session.is_camera_analyzing(); + if button(app, ui, r1, "Track Camera", can_camera, "tracker.trackCamera") { + run(app, &ctx, "track.camera", json!({})); + } let can_warp = app.session.is_enabled("track.warpStabilizer") && !app.session.is_warp_analyzing(); if button(app, ui, r2, "Warp Stabilizer", can_warp, "tracker.warpStabilizer") { run(app, &ctx, "track.warpStabilizer", json!({})); diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index d847edb8..c9dd70db 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -1144,6 +1144,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { app.session.open_comp(*item); } } + // 3D Camera Tracker points, targets and their menu (on top of the viewer's gestures while + // the effect is selected), and its analysis banner. + super::camera_tracker_ui::viewer_hook(app, ui, &painter, &map, &ectx, &|c, l| l2c(c, l).0); // Effect point controls and crosshair/eyedropper picks (on top of the viewer's gestures). crate::panels::effect_controls::viewer_hook(app, ui, &painter, &map, &ectx, &|c, l| l2c(c, l).0); diff --git a/crates/ui-egui/tests/ui_camera_tracker.rs b/crates/ui-egui/tests/ui_camera_tracker.rs new file mode 100644 index 00000000..45374fba --- /dev/null +++ b/crates/ui-egui/tests/ui_camera_tracker.rs @@ -0,0 +1,175 @@ +//! Headless checks for the 3D Camera Tracker UI (egui_kittest, UI logic only): the viewer's track +//! points and selection, the Effect Controls buttons and the Tracker panel's Track Camera button. +//! The solve is a synthetic one written into the effect (the analysis itself is tested in the +//! engine and track crates). + +use effectcraft_engine::Session; +use effectcraft_engine::effects::camera_tracker as ct; +use effectcraft_engine::track::camtrack::{CameraSolve, ShotType, SolveMethod, SolvedFrame, SolvedPoint}; +use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::dock::PanelKind; +use egui_kittest::Harness; +use serde_json::json; + +const W: f64 = 640.0; +const H: f64 = 360.0; + +fn app() -> (EffectcraftApp, u64, u64) { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Shot", "width": W, "height": H, "duration": 2})).unwrap(); + let plate = s.execute("layer.newSolid", json!({"name": "Plate", "color": "#406080"})).unwrap()["layer"].as_u64().unwrap(); + let r = s.execute("effect.apply", json!({"effect": ct::ID, "layers": [plate]})).unwrap(); + let uid = r["effects"][0].as_u64().unwrap(); + // A synthetic solve: a static camera (focal 600) looking at a 6 × 4 grid of points on z = 1. + let frames = 60; + let solve = CameraSolve { + version: 1, + size: [W, H], + start: 0.0, + frame_duration: 1.0 / 30.0, + shot: ShotType::FixedAngle, + method_used: SolveMethod::Typical, + average_error: 0.21, + frames: vec![SolvedFrame { rot: [0.0; 3], center: [0.0; 3], focal: 600.0, solved: true }; frames], + points: (0..24) + .map(|i| SolvedPoint { + id: i, + pos: [((i % 6) as f64 - 2.5) * 0.15, ((i / 6) as f64 - 1.5) * 0.12, 1.0 + 0.01 * (i % 3) as f64], + error: 0.2, + first: 0, + last: frames as u32 - 1, + }) + .collect(), + ground: None, + }; + s.execute("prop.set", json!({"layer": plate, "path": "effects/#1/solve", "value": solve.to_json()})).unwrap(); + s.camera_pending.clear(); + s.state.selected_props = vec![(effectcraft_engine::project::LayerId(plate), uid)]; + s.execute("layer.select", json!({"layers": [plate]})).unwrap(); + s.state.selected_props = vec![(effectcraft_engine::project::LayerId(plate), uid)]; + (EffectcraftApp::new(s), plate, uid) +} + +fn ids(h: &Harness<'_, EffectcraftApp>) -> Vec { + h.state().auto.previous.iter().chain(h.state().auto.elements.iter()).map(|e| e.id.clone()).collect() +} + +fn rect(h: &Harness<'_, EffectcraftApp>, id: &str) -> egui::Rect { + let e = h.state().auto.find(id).unwrap_or_else(|| panic!("no {id}")); + egui::Rect::from_min_size(egui::pos2(e.rect[0], e.rect[1]), egui::vec2(e.rect[2], e.rect[3])) +} + +fn click(h: &mut Harness<'_, EffectcraftApp>, at: egui::Pos2, modifiers: egui::Modifiers) { + h.input_mut().events.push(egui::Event::ModifiersChanged(modifiers)); + h.input_mut().events.push(egui::Event::PointerMoved(at)); + h.input_mut().events.push(egui::Event::PointerButton { pos: at, button: egui::PointerButton::Primary, pressed: true, modifiers }); + h.input_mut().events.push(egui::Event::PointerButton { pos: at, button: egui::PointerButton::Primary, pressed: false, modifiers }); + h.run_steps(3); + h.input_mut().events.push(egui::Event::ModifiersChanged(Default::default())); + h.step(); +} + +#[test] +fn viewer_points_selection_and_effect_controls() { + let (a, plate, uid) = app(); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).build_eframe(|_| a); + h.state_mut().show_panel(PanelKind::Composition); + h.run_steps(4); + let all = ids(&h); + assert!(all.iter().any(|i| i == "viewer.cameraTracker"), "track point overlay registered"); + let mut pts: Vec<&String> = all.iter().filter(|i| i.starts_with("viewer.cameraTracker.point.")).collect(); + pts.sort(); + pts.dedup(); + let n = pts.len(); + assert_eq!(n, 24, "every visible point is drawn"); + // Click a point: it is selected; Shift-click another adds it. + let r = rect(&h, "viewer.cameraTracker.point.7"); + click(&mut h, r.center(), Default::default()); + assert_eq!(h.state().session.state.camera_points, vec![7]); + let r = rect(&h, "viewer.cameraTracker.point.9"); + click(&mut h, r.center(), egui::Modifiers::SHIFT); + assert_eq!(h.state().session.state.camera_points, vec![7, 9]); + // Marquee over the first row selects its six points. + let (a0, a5) = (rect(&h, "viewer.cameraTracker.point.0"), rect(&h, "viewer.cameraTracker.point.5")); + let from = a0.left_top() - egui::vec2(6.0, 6.0); + let to = a5.right_bottom() + egui::vec2(6.0, 6.0); + h.input_mut().events.push(egui::Event::PointerMoved(from)); + h.input_mut().events.push(egui::Event::PointerButton { pos: from, button: egui::PointerButton::Primary, pressed: true, modifiers: Default::default() }); + h.step(); + for i in 1..=8 { + h.input_mut().events.push(egui::Event::PointerMoved(from + (to - from) * (i as f32 / 8.0))); + h.step(); + } + h.input_mut().events.push(egui::Event::PointerButton { pos: to, button: egui::PointerButton::Primary, pressed: false, modifiers: Default::default() }); + h.run_steps(3); + let mut sel = h.state().session.state.camera_points.clone(); + sel.sort(); + assert_eq!(sel, vec![0, 1, 2, 3, 4, 5]); + // The right-click menu's Create Solid and Camera (via its command). + let ctx = h.ctx.clone(); + effectcraft_ui_egui::menus::invoke(h.state_mut(), &ctx, "camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + let comp = h.state().session.active_comp().unwrap(); + assert!(comp.layers.iter().any(|l| l.name == "3D Tracker Camera" && l.is_camera())); + assert!(comp.layers.iter().any(|l| l.name.starts_with("Track Solid"))); + + // Effect Controls: Analyze and Create Camera. + h.state_mut().session.execute("layer.select", json!({"layers": [plate]})).unwrap(); + h.state_mut().show_panel(PanelKind::EffectControls); + h.run_steps(4); + let all = ids(&h); + for want in [format!("effectControls.cameraTracker.{uid}.analyze"), format!("effectControls.cameraTracker.{uid}.createCamera")] { + assert!(all.contains(&want), "missing {want}"); + } + // Tracker panel: Track Camera is available for the selected layer. + h.state_mut().show_panel(PanelKind::Tracker); + h.run_steps(3); + assert!(ids(&h).iter().any(|i| i == "tracker.trackCamera")); + assert!(h.state().session.is_enabled("track.camera")); +} + +/// Step the UI until background frame renders have landed. +fn settle(h: &mut Harness<'_, EffectcraftApp>) { + for _ in 0..600 { + h.step(); + if h.state().frames.inflight() == 0 && h.state().frames.last_ms.lock().map(|v| *v > 0.0).unwrap_or(false) { + break; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + for _ in 0..4 { + h.step(); + } +} + +#[test] +#[ignore = "analyses the demo's 3D Showcase and renders with wgpu; run with --ignored (set EC_SNAPSHOT_DIR to keep PNGs)"] +fn snapshot_showcase_track() { + let dir = std::env::var("EC_SNAPSHOT_DIR").ok(); + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + s.execute("comp.new", json!({"name": "Camera Track", "width": 1920, "height": 1080, "duration": 4, "frameRate": 29.97})).unwrap(); + s.execute("layer.addItem", json!({"item": "3D Showcase"})).unwrap(); + s.execute("layer.select", json!({"layers": ["#1"]})).unwrap(); + s.execute("track.camera", json!({"layer": "#1", "wait": true})).unwrap(); + s.execute("time.set", json!({"time": 2.0})).unwrap(); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).with_pixels_per_point(1.0).wgpu().build_eframe(|_| EffectcraftApp::new(s)); + h.state_mut().show_panel(PanelKind::Composition); + settle(&mut h); + let pts: Vec = ids(&h).into_iter().filter(|i| i.starts_with("viewer.cameraTracker.point.")).collect(); + assert!(pts.len() > 10, "{}", pts.len()); + // Hover between points to show a target, then select three and show the selection's target. + let r = rect(&h, &pts[0]); + h.input_mut().events.push(egui::Event::PointerMoved(r.center() + egui::vec2(12.0, 12.0))); + h.run_steps(2); + if let Some(d) = &dir { + h.render().expect("render").save(format!("{d}/camera_tracker_hover.png")).unwrap(); + } + let sel: Vec = pts.iter().take(5).filter_map(|i| i.rsplit('.').next()?.parse().ok()).collect(); + h.state_mut().session.execute("camera.selectPoints", json!({"points": sel})).unwrap(); + h.state_mut().session.execute("camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + h.state_mut().session.execute("layer.select", json!({"layers": ["#3"]})).unwrap(); + settle(&mut h); + if let Some(d) = &dir { + h.render().expect("render").save(format!("{d}/camera_tracker_solid.png")).unwrap(); + } +} diff --git a/docs/agents.md b/docs/agents.md index 510bc356..bda72461 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -146,6 +146,34 @@ effectcraft-cli exec warp.analyze '{"layer":"#1","wait":true}' shaky.ecproj --sa effectcraft-cli exec warp.status '{"layer":"#1"}' shaky.ecproj --json ``` +### 3D Camera Tracker + +Animation ▸ Track Camera and the Tracker panel's button (`track.camera {layer?, shotType?: +fixed|variable|specify, aov?, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}`) apply +Effect ▸ Perspective ▸ 3D Camera Tracker (or reuse the layer's) and analyse it in the background: +"Analyzing in background (step 1 of 2)" tracks features, "Solving camera" solves. +`camera.analyze {layer?, effect?, wait?}` re-runs it, `camera.cancel` stops it without writing +anything. `camera.solveStatus` reports `progress`/`banner`, `analyzed`, `solved`, `methodUsed`, +`averageError` (pixels), `focalLength`, `horizontalAngleOfView`, the current frame's `camera` +(position, orientation, zoom) and `groundPlane`. `camera.points {time?}` lists the solved points +visible now with their `id`, `comp` position, `depth`, `world` position and `error`. + +Select points with `camera.selectPoints {points, add?, toggle?}` (the viewer's click / Shift-click / +marquee), then the right-click menu's commands: `camera.setGroundPlane {points?}` (Set Ground Plane +and Origin), `camera.createFromSolve {kind: text|solid|null|shadowCatcher, points? | target: +{center, normal, size?}, multiple?}` (Create Text / Solid / Null / Shadow Catcher, Camera and Light, +Create Multiple …), `camera.deletePoints {points?, wait?}` (re-solves; with Auto-delete Points +Across Time the same feature's other tracks go too) and `camera.create` (the Create Camera button). +Each is one undo step; the first create adds the one-node "3D Tracker Camera" keyed on every frame. +Changing the layer's frames clears the analysis; changing `effects/#1/shotType`, +`effects/#1/horizontalAngleOfView`, `effects/#1/advanced/solveMethod` re-solves the stored tracks. + +```sh +effectcraft-cli run shot.ecproj track.camera '{"layer":"#1","wait":true}' \ + camera.points '{}' --json +effectcraft-cli run shot.ecproj camera.createFromSolve '{"kind":"solid","points":[12,40,77]}' --save +``` + ### Text: styles and editing Source Text holds character style runs and per-paragraph settings. `layer.setText` changes the diff --git a/docs/architecture.md b/docs/architecture.md index 033ed6cb..f9f0586f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -176,6 +176,22 @@ auto-scale; cached per analysis and settings) and warps each frame; Synthesize E borders from neighbouring frames read with `EffectHost::self_at`. The effect lives in the effects crate, hence the `effects → track` edge. +**3D Camera Tracker** (`effects::camera_tracker`, `engine::camera_track`, `effectcraft-track`'s +`camtrack`): `camera.analyze` / `track.camera` render the layer's input to the effect on a +background thread and run structure from motion in two steps. Step 1 follows Shi–Tomasi features +through the clip with pyramidal Lucas–Kanade (forward–backward checked, re-detected where the frame +has none) into long 2D tracks. Step 2 solves the camera: parallax keyframes; a two-view start from +the essential matrix (normalised 8-point in RANSAC) or a planar homography decomposition; incremental +resection and triangulation; a sparse Levenberg–Marquardt bundle adjustment (Schur complement over +the points, Huber loss) with the focal length fixed, shared, or per frame; a log-spaced and +golden-section focal search for Fixed Angle of View / Variable Zoom; a rotation-only model for tripod +pans, chosen by Auto Detect when it explains the tracks as well. Tracks and solve are stored as JSON +in hidden effect parameters with keys: changing the layer's frames clears both, changing Shot Type, +Angle of View, Solve Method or deleting points only re-solves. The solve's canonical frame maps to +comp space so the first frame's camera is the default comp camera (or so a chosen ground plane is +the X-Z plane at the origin); `camera.createFromSolve` keys a one-node "3D Tracker Camera" on every +frame and places text, solids, nulls or a shadow catcher and light on the target plane. + ## 5. Expressions Expressions are JavaScript, run by boa, with After Effects' object model (`thisComp`, `thisLayer`, diff --git a/docs/parity.md b/docs/parity.md index d259fec2..25e5a080 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -15,7 +15,7 @@ rendered, does not count as done. | P1 (professional daily use) | ≈ 38% | | P2 (long tail) | ≈ 17% | | Features done / partial / missing | 18 / 60 / 14 of 92 | -| **Effects** | **257 of 298** After Effects 2026 effects by name (86%); ≈ 70% allowing for simplified implementations | +| **Effects** | **258 of 298** After Effects 2026 effects by name (87%); ≈ 70% allowing for simplified implementations | | Remaining work | ≈ **167 agent-hours** (139 h of features + ≈ 20% for 1:1 polish against After Effects) | | Wall-clock estimate | ≈ **42–50 hours** with five agents working in parallel and one integrating; ≈ 31–35 h for P0 + P1 only | @@ -73,7 +73,7 @@ library groundwork for mask tracking and Warp Stabilizer. | Remaining (audit-scale agent-hours) | ≈ 167 | ≈ 100 | | Wall clock, five parallel agents | 42–50 h | **≈ 8–10 h at today's measured pace; ≈ 25 h by the audit's conservative figures** | -What is left is concentrated in large systems: Roto Brush and Refine Edge, the 3D camera tracker, +What is left is concentrated in large systems: Roto Brush and Refine Edge, Advanced 3D (models, PBR, image-based lighting), the Warp Stabilizer and mask-tracking UI on top of the new tracking library, a JavaScript scripting object model, SVG/PSD import, WebM, a disk cache, the missing effect categories (3D Channel, Immersive Video, OCIO), native macOS menus, and @@ -93,13 +93,13 @@ wasm threads. | Animation | 65% | 10.4 | puppet, Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8) | | Text | ≈ 85% | 1.5 | vertical type is basic (upright characters, no tate-chu-yoko), no OpenType feature panel, no text on 3D bevels; per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API landed (M9.9–M9.10) | | Web | 60% | 2.5 | threads, browser storage, audio | -| 3D | 58% | 20.5 | Advanced 3D (models, PBR, image-based light), 3D camera tracker, multi-view layouts | +| 3D | 62% | 18.5 | Advanced 3D (models, PBR, image-based light), multi-view layouts (the 3D Camera Tracker landed in M12.6) | | Effects | 58% | 13.6 | Effect Controls widgets (angle dial, point crosshair, eyedropper, curves), GPU versions of more effects, 41 missing effects | | Interface | 55% | 10.4 | drag-to-dock and floating panels, preferences, native macOS menus (viewer rulers/snapping/channels/snapshots landed in M0.13) | | Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves | | Masks & roto | 52% | 10.0 | Roto Brush, variable-width mask feather points (mask tracking and Mask Interpolation landed in M6.6) | | Preview | 55% | 6.1 | GPU 3D and adjustment layers, disk cache (region of interest, snapshots, exposure and Fast Previews landed in M0.13) | -| Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking, Subspace Warp's mesh warp and rolling-shutter repair (point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) | +| Tracking | ≈ 85% | 2.5 | face tracking, Subspace Warp's mesh warp, rolling-shutter repair, lens distortion in the camera solve (point tracker, mask tracking, Warp Stabilizer and the 3D Camera Tracker landed in M6.x / M12.5 / M12.6) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | ## Effects still missing @@ -107,7 +107,7 @@ wasm threads. Immersive Video (all 12), 3D Channel (all 8, including Cryptomatte), Color Stabilizer and the five OCIO transforms, CC Flo Motion, Liquify, Rolling Shutter Repair, Compressor, Distortion and Gate (audio), Camera-Shake Deblur, CC Radial Blur, CC Hair, Particle Playground, -Keylight (our Screen Key stands in), Color Profile Converter and the 3D Camera Tracker. Boris FX +Keylight (our Screen Key stands in) and Color Profile Converter. Boris FX Mocha and Cineware are third-party and not counted. ## Highest-value gaps, in order From c33b80614ace5e598c48e998f59b49aaf96c26ed Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 23:11:03 -0400 Subject: [PATCH 08/13] =?UTF-8?q?PRJ-8:=20proxies=20=E2=80=94=20Set=20Prox?= =?UTF-8?q?y=20/=20None=20/=20Use=20Proxy=20/=20Create=20Proxy=20/=20Inter?= =?UTF-8?q?pret=20Proxy,=20Proxy=20Use;=20PRJ-3=20interpret=20fields,=20PA?= =?UTF-8?q?R,=20alpha=20guess?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Items (footage and comps) carry a proxy rendered in their place at the source's size when its switch and Render Settings > Proxy Use allow it; Create Proxy queues a half-resolution render whose post-render action attaches it. Interpret Footage gains Separate Fields (upper/lower first, fields become frames at twice the rate), Guess alpha with matte colour, explicit matte colour, Invert Alpha, Interpret As Linear Light; non-square footage, solids and precomps are stretched by their pixel aspect in the comp. The four proxy stubs are gone. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/commands/file_more.rs | 207 +++++++++++++--- crates/engine/src/commands/layer.rs | 2 +- crates/engine/src/commands/mod.rs | 9 +- crates/engine/src/commands/proxy.rs | 223 +++++++++++++++++ crates/engine/src/commands/render_queue.rs | 8 +- crates/engine/src/commands/stubs.rs | 6 - crates/engine/src/lib.rs | 5 +- crates/engine/src/render_queue.rs | 5 + crates/engine/src/tests_menu_cmds.rs | 2 +- crates/engine/src/tests_proxy.rs | 275 +++++++++++++++++++++ crates/engine/src/tests_stubs.rs | 2 +- crates/export/src/lib.rs | 1 + crates/host/src/lib.rs | 53 ++++ crates/project/src/render_queue.rs | 51 ++++ crates/render/src/cache.rs | 2 + crates/render/src/eval.rs | 25 +- crates/render/src/lib.rs | 130 ++++++++-- crates/ui-egui/src/lib.rs | 1 + 18 files changed, 938 insertions(+), 69 deletions(-) create mode 100644 crates/engine/src/commands/proxy.rs create mode 100644 crates/engine/src/tests_proxy.rs diff --git a/crates/engine/src/commands/file_more.rs b/crates/engine/src/commands/file_more.rs index f5f4d370..45e6865a 100644 --- a/crates/engine/src/commands/file_more.rs +++ b/crates/engine/src/commands/file_more.rs @@ -400,56 +400,179 @@ fn run_script(s: &mut Session, p: &Value) -> Result { // ---------------------------------------------------------------- footage +/// Interpret Footage settings parsed from command parameters (every field optional). +#[derive(Clone, Debug, Default)] +pub(crate) struct Interpretation { + rate: Option, + /// Back to the file's own rate (Use frame rate from file). + native: bool, + alpha: Option, + /// Alpha ▸ Guess: decide Straight / Premultiplied (and the matte colour) from a frame. + guess: bool, + matte: Option<[f32; 3]>, + invert_alpha: Option, + loops: Option, + par: Option, + fields: Option, + profile: Option>, + linear: Option, +} + +impl Interpretation { + pub(crate) fn parse(p: &Value, cmd: &str) -> Result { + let mut it = Interpretation { rate: f_p(p, "frameRate").map(FrameRate::from_f64), ..Default::default() }; + if str_p(p, "frameRate") == Some("file") { + it.native = true; + } + match str_p(p, "alpha") { + Some("straight") => it.alpha = Some(AlphaMode::Straight), + Some("premultiplied") => it.alpha = Some(AlphaMode::Premultiplied), + Some("ignore") => it.alpha = Some(AlphaMode::Ignore), + Some("guess") => it.guess = true, + Some(a) => return Err(bad(cmd, format!("alpha: straight|premultiplied|ignore|guess, not `{a}`"))), + None => {} + } + it.guess |= super::b_p(p, "guessAlpha").unwrap_or(false); + it.matte = match p.get("matteColor").or(p.get("premulColor")) { + None => None, + Some(v) => Some( + effectcraft_keyframe::Value::Color([0.0, 0.0, 0.0, 1.0]) + .coerce_json(v) + .map(|c| { + let c = c.as_color(); + [c[0], c[1], c[2]] + }) + .ok_or_else(|| bad(cmd, "matteColor: #hex or [r, g, b] (0..1)"))?, + ), + }; + it.invert_alpha = super::b_p(p, "invertAlpha"); + it.loops = p.get("loop").and_then(Value::as_u64).map(|v| v.clamp(1, 9999) as u32); + it.par = f_p(p, "pixelAspect").filter(|x| *x > 0.0); + it.fields = match str_p(p, "fields") { + Some(f) => Some(effectcraft_project::FieldOrder::parse(f).ok_or_else(|| bad(cmd, "fields: off|upper|lower"))?), + None => None, + }; + // Color ▸ Assign Profile: a colour space id, or "auto"/"none" for the file's own (sRGB). + it.profile = match str_p(p, "colorProfile") { + Some("auto" | "none" | "") => Some(None), + Some(c) => match effectcraft_project::ColorSpace::parse(c) { + Some(cs) => Some(Some(cs)), + None => return Err(bad(cmd, format!("colorProfile: srgb|rec709|rec2020|p3|auto, not `{c}`"))), + }, + None => None, + }; + it.linear = super::b_p(p, "linearLight"); + Ok(it) + } + + pub(crate) fn apply(&self, f: &mut Footage, guessed: Option<(AlphaMode, [f32; 3])>) { + if self.native { + if let Some(n) = f.native_rate.take() { + f.frame_rate = n; + } + } else if let Some(r) = self.rate { + if f.native_rate.is_none() { + f.native_rate = Some(f.frame_rate); + } + f.frame_rate = r; + } + if let Some(a) = self.alpha { + f.alpha = a; + } + if let Some((a, m)) = guessed { + f.alpha = a; + f.premul_color = m; + } + if let Some(m) = self.matte { + f.premul_color = m; + } + if let Some(v) = self.invert_alpha { + f.invert_alpha = v; + } + if let Some(l) = self.loops { + f.loop_count = l; + } + if let Some(x) = self.par { + f.pixel_aspect = x; + } + if let Some(x) = self.fields { + f.fields = x; + } + if let Some(c) = self.profile { + f.color_profile = c; + } + if let Some(v) = self.linear { + f.linear_light = v; + } + } +} + +/// Interpret Footage ▸ Alpha ▸ Guess: read the first frame as straight colour and decide +/// whether the file is premultiplied (every partly transparent pixel's colour within its alpha +/// of a black or white matte) or straight. `None` when the frame has no partial alpha. +pub(crate) fn guess_alpha(s: &Session, item: ItemId, f: &Footage) -> Option<(AlphaMode, [f32; 3])> { + let probe = Footage { alpha: AlphaMode::Straight, invert_alpha: false, ..f.clone() }; + let img = s.footage.frame(item, &probe, Tick::ZERO)?; + let (mut partial, mut black, mut white) = (0usize, true, true); + let eps = 1.5 / 255.0; + for px in img.data.iter() { + let a = px[3]; + if a <= eps || a >= 1.0 - eps { + continue; + } + partial += 1; + for c in &px[..3] { + // The file's colour (the decoder premultiplied it as straight). + let file = c / a; + black &= file <= a + eps; + white &= file >= 1.0 - a - eps; + } + } + if partial == 0 { + return None; + } + Some(if black { + (AlphaMode::Premultiplied, [0.0; 3]) + } else if white { + (AlphaMode::Premultiplied, [1.0; 3]) + } else { + (AlphaMode::Straight, f.premul_color) + }) +} + fn interpret(s: &mut Session, p: &Value) -> Result { let items = items_p(s, p); if items.is_empty() { return Err(bad("file.interpretFootage", "select footage")); } - let rate = f_p(p, "frameRate").map(FrameRate::from_f64); - let alpha = match str_p(p, "alpha") { - Some("straight") => Some(AlphaMode::Straight), - Some("premultiplied") => Some(AlphaMode::Premultiplied), - Some("ignore") => Some(AlphaMode::Ignore), - Some(a) => return Err(bad("file.interpretFootage", format!("alpha: straight|premultiplied|ignore, not `{a}`"))), - None => None, - }; - let loops = p.get("loop").and_then(Value::as_u64).map(|v| v.max(1) as u32); - let par = f_p(p, "pixelAspect"); - // Color ▸ Assign Profile: a colour space id, or "auto"/"none" for the file's own (sRGB). - let profile = match str_p(p, "colorProfile") { - Some("auto" | "none" | "") => Some(None), - Some(c) => match effectcraft_project::ColorSpace::parse(c) { - Some(cs) => Some(Some(cs)), - None => return Err(bad("file.interpretFootage", format!("colorProfile: srgb|rec709|rec2020|p3|auto, not `{c}`"))), - }, - None => None, - }; + let it = Interpretation::parse(p, "file.interpretFootage")?; + let guesses: Vec> = items + .iter() + .map(|i| match (it.guess, s.project.item(*i).map(|x| &x.kind)) { + (true, Some(ItemKind::Footage(f))) => guess_alpha(s, *i, f), + _ => None, + }) + .collect(); s.edit("Interpret Footage", None, |proj, _| { - for i in &items { + for (i, g) in items.iter().zip(&guesses) { if let Some(ItemKind::Footage(f)) = proj.item_mut(*i).map(|x| &mut x.kind) { - if let Some(r) = rate { - if f.native_rate.is_none() { - f.native_rate = Some(f.frame_rate); - } - f.frame_rate = r; - } - if let Some(a) = alpha { - f.alpha = a; - } - if let Some(l) = loops { - f.loop_count = l; - } - if let Some(x) = par { - f.pixel_aspect = x; - } - if let Some(c) = profile { - f.color_profile = c; - } + it.apply(f, *g); } } Ok(()) })?; - Ok(Value::Null) + let out: Vec = items + .iter() + .filter_map(|i| match s.project.item(*i).map(|x| &x.kind) { + Some(ItemKind::Footage(f)) => Some(json!({ + "item": i.0, "alpha": format!("{:?}", f.alpha), "matteColor": f.premul_color, "invertAlpha": f.invert_alpha, + "fields": f.fields.label(), "pixelAspect": f.pixel_aspect, "loop": f.loop_count, "frameRate": f.frame_rate.as_f64(), + "linearLight": f.linear_light, + })), + _ => None, + }) + .collect(); + Ok(json!({"items": out})) } fn remember_interpretation(s: &mut Session, p: &Value) -> Result { @@ -470,6 +593,10 @@ fn apply_interpretation(s: &mut Session, p: &Value) -> Result { f.premul_color = src.premul_color; f.pixel_aspect = src.pixel_aspect; f.loop_count = src.loop_count; + f.fields = src.fields; + f.invert_alpha = src.invert_alpha; + f.linear_light = src.linear_light; + f.color_profile = src.color_profile; if src.native_rate.is_some() { if f.native_rate.is_none() { f.native_rate = Some(f.frame_rate); @@ -648,7 +775,7 @@ pub fn specs() -> Vec { "Main...", ["File", "Interpret Footage"], Some("Cmd+Alt+G"), - "{items?, frameRate?, alpha?: straight|premultiplied|ignore, loop?, pixelAspect?, colorProfile?: srgb|rec709|rec2020|p3|auto}", + "{items?, frameRate?: fps|\"file\", alpha?: straight|premultiplied|ignore|guess, guessAlpha?, matteColor?, invertAlpha?, loop?, pixelAspect?, fields?: off|upper|lower, colorProfile?: srgb|rec709|rec2020|p3|auto, linearLight?}", has_footage_selection, interpret ), diff --git a/crates/engine/src/commands/layer.rs b/crates/engine/src/commands/layer.rs index e57b1c0a..6f326b02 100644 --- a/crates/engine/src/commands/layer.rs +++ b/crates/engine/src/commands/layer.rs @@ -53,7 +53,7 @@ fn new_solid_like(s: &mut Session, p: &Value, adjustment: bool) -> Result let name = str_p(p, "name").map(str::to_string).unwrap_or(default_name); let id = s.edit(if adjustment { "New Adjustment Layer" } else { "New Solid" }, None, |proj, st| { let folder = solids_folder(proj); - let sid = proj.add_item(&name, Label::Red, Some(folder), ItemKind::Solid(Solid { color, width: w, height: h, pixel_aspect: 1.0 })); + let sid = proj.add_item(&name, Label::Red, Some(folder), ItemKind::Solid(Solid { color, width: w, height: h, pixel_aspect: comp.pixel_aspect })); let mut l = build::layer(proj, &comp, &name, LayerSource::Solid { item: sid }, (w, h), None); if adjustment { l.switches.adjustment = true; diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index e1b3f15f..7d44f5e5 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -11,7 +11,7 @@ mod effect; pub mod essential; pub mod expr_tools; mod file; -mod file_more; +pub(crate) mod file_more; mod frontend; mod help; mod key_transform; @@ -29,6 +29,7 @@ pub mod paint; mod paths; mod project_items; mod prop; +mod proxy; pub mod puppet; mod query; mod render_queue; @@ -140,11 +141,17 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(settings::specs()); v.extend(essential::specs()); v.extend(expr_tools::specs()); + v.extend(proxy::specs()); v.extend(stubs::specs()); v }) } +/// Set `footage` as the proxy of `item` (File ▸ Create Proxy's post-render action). +pub(crate) fn proxy_set(s: &mut Session, item: ItemId, footage: effectcraft_project::Footage) -> Result<()> { + proxy::set_proxy_footage(s, &[item], footage) +} + /// (id, name) of the text animation presets. pub fn text_preset_list() -> Vec<(String, String)> { text_anim::preset_names() diff --git a/crates/engine/src/commands/proxy.rs b/crates/engine/src/commands/proxy.rs new file mode 100644 index 00000000..607dd5dd --- /dev/null +++ b/crates/engine/src/commands/proxy.rs @@ -0,0 +1,223 @@ +//! Proxies (File ▸ Create Proxy / Set Proxy / Interpret Footage ▸ Proxy): a stand-in file for +//! a footage item or a composition. The Project panel's proxy switch (`file.useProxy`) turns +//! it on or off per item; renders follow Render Settings ▸ Proxy Use (the viewer uses each +//! item's switch). + +use effectcraft_project::render_queue::{Channels, OutputFormat, OutputModule, PostRenderAction, ProResProfile, RenderQuality, RenderQueueItem, TimeSpan}; +use effectcraft_project::{Footage, ItemId, ItemKind, Proxy}; +use effectcraft_time::Tick; +use serde_json::{Value, json}; + +use super::{CommandSpec, b_p, bad, has_project_selection, str_p}; +use crate::{EngineError, Result, Session, cmd}; + +/// Footage items and comps a proxy command acts on: `items`/`item`, else the Project panel +/// selection. +fn targets(s: &Session, p: &Value) -> Vec { + let ids: Vec = match p.get("items").or(p.get("item")) { + Some(Value::Array(a)) => a.iter().filter_map(Value::as_u64).map(ItemId).collect(), + Some(Value::Number(n)) => n.as_u64().map(ItemId).into_iter().collect(), + Some(Value::String(name)) => s.project.find_by_name(name).map(|i| i.id).into_iter().collect(), + _ => s.state.project_selection.clone(), + }; + ids.into_iter().filter(|i| matches!(s.project.item(*i).map(|x| &x.kind), Some(ItemKind::Footage(_) | ItemKind::Comp(_)))).collect() +} + +fn has_targets(s: &Session) -> std::result::Result<(), String> { + has_project_selection(s)?; + if targets(s, &Value::Null).is_empty() { Err("select footage or a composition in the Project panel".into()) } else { Ok(()) } +} + +fn has_proxy(s: &Session) -> std::result::Result<(), String> { + has_targets(s)?; + if targets(s, &Value::Null).iter().any(|i| s.project.item(*i).is_some_and(|x| x.proxy.is_some())) { + Ok(()) + } else { + Err("the selected items have no proxy".into()) + } +} + +/// Probe a proxy file through the importer. +fn probe(s: &Session, path: &str, cmd: &str) -> Result { + let importer = s.importer.clone().ok_or_else(|| EngineError::Other("media import is not available in this build".into()))?; + let f = importer.probe(path).map_err(|e| bad(cmd, format!("{path}: {e}")))?; + if !f.has_video { + return Err(bad(cmd, format!("{path} has no video to use as a proxy"))); + } + Ok(f) +} + +/// Attach `footage` as the proxy of `items` (Use Proxy on). One undo step. +pub(crate) fn set_proxy_footage(s: &mut Session, items: &[ItemId], footage: Footage) -> Result<()> { + s.edit("Set Proxy", None, |proj, _| { + for i in items { + if let Some(it) = proj.item_mut(*i) { + it.proxy = Some(Box::new(Proxy { footage: footage.clone(), enabled: true })); + } + } + Ok(()) + }) +} + +fn set_proxy(s: &mut Session, p: &Value) -> Result { + const C: &str = "file.setProxy"; + let items = targets(s, p); + if items.is_empty() { + return Err(bad(C, "select footage or a composition")); + } + let path = str_p(p, "path").ok_or_else(|| bad(C, "missing `path`"))?.to_string(); + let f = probe(s, &path, C)?; + let (w, h) = (f.width, f.height); + set_proxy_footage(s, &items, f)?; + Ok(json!({"items": items.iter().map(|i| i.0).collect::>(), "path": path, "width": w, "height": h})) +} + +fn set_proxy_none(s: &mut Session, p: &Value) -> Result { + let items = targets(s, p); + s.edit("Set Proxy None", None, |proj, _| { + for i in &items { + if let Some(it) = proj.item_mut(*i) { + it.proxy = None; + } + } + Ok(()) + })?; + Ok(json!({"items": items.iter().map(|i| i.0).collect::>()})) +} + +/// The Project panel's proxy switch: use the proxy (filled box) or the full-resolution source. +fn use_proxy(s: &mut Session, p: &Value) -> Result { + const C: &str = "file.useProxy"; + let items: Vec = targets(s, p).into_iter().filter(|i| s.project.item(*i).is_some_and(|x| x.proxy.is_some())).collect(); + if items.is_empty() { + return Err(bad(C, "the selected items have no proxy")); + } + let first = s.project.item(items[0]).and_then(|x| x.proxy.as_ref()).is_some_and(|px| px.enabled); + let on = b_p(p, "on").unwrap_or(!first); + s.edit(if on { "Use Proxy" } else { "Use Full Resolution" }, None, |proj, _| { + for i in &items { + if let Some(px) = proj.item_mut(*i).and_then(|x| x.proxy.as_mut()) { + px.enabled = on; + } + } + Ok(()) + })?; + Ok(json!({"items": items.iter().map(|i| i.0).collect::>(), "on": on})) +} + +/// File ▸ Interpret Footage ▸ Proxy: Interpret Footage settings for the proxy file. +fn interpret_proxy(s: &mut Session, p: &Value) -> Result { + const C: &str = "file.interpretProxy"; + let items: Vec = targets(s, p).into_iter().filter(|i| s.project.item(*i).is_some_and(|x| x.proxy.is_some())).collect(); + if items.is_empty() { + return Err(bad(C, "the selected items have no proxy")); + } + let it = super::file_more::Interpretation::parse(p, C)?; + let guesses: Vec<_> = items + .iter() + .map(|i| { + let px = s.project.item(*i).and_then(|x| x.proxy.as_ref()); + match (p.get("alpha").and_then(Value::as_str) == Some("guess") || b_p(p, "guessAlpha") == Some(true), px) { + (true, Some(px)) => super::file_more::guess_alpha(s, *i, &px.footage), + _ => None, + } + }) + .collect(); + s.edit("Interpret Proxy", None, |proj, _| { + for (i, g) in items.iter().zip(&guesses) { + if let Some(px) = proj.item_mut(*i).and_then(|x| x.proxy.as_mut()) { + it.apply(&mut px.footage, *g); + } + } + Ok(()) + })?; + Ok(json!({"items": items.iter().map(|i| i.0).collect::>()})) +} + +/// File ▸ Create Proxy ▸ Still / Movie: queue a half-resolution render of the composition whose +/// post-render action sets the result as its proxy. +fn create_proxy(s: &mut Session, p: &Value) -> Result { + const C: &str = "file.createProxy"; + let cid = match p.get("comp").or(p.get("item")) { + Some(_) => { + let q = json!({"comp": p.get("comp").or(p.get("item")).cloned()}); + super::comp_id(s, &q)? + } + None => s + .state + .project_selection + .iter() + .copied() + .find(|i| s.project.comp(*i).is_some()) + .or(s.active_comp_id()) + .ok_or_else(|| bad(C, "select a composition"))?, + }; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let still = match str_p(p, "kind").unwrap_or("movie") { + "still" => true, + "movie" => false, + k => return Err(bad(C, format!("kind: still|movie, not `{k}`"))), + }; + let mut it = RenderQueueItem::new(0, cid); + it.settings.quality = RenderQuality::Draft; + it.settings.resolution = p.get("resolution").and_then(Value::as_f64).unwrap_or(0.5).clamp(0.05, 1.0); + if still { + // One frame at the current time (the comp's poster frame when not the active comp). + let t = if s.active_comp_id() == Some(cid) { s.time() } else { comp.poster_time }; + it.settings.time_span = TimeSpan::Custom { start: t, end: t + comp.frame_duration() }; + it.output = OutputModule::for_format(OutputFormat::PngSequence); + it.output.output = "[compName]_proxy_[#####].png".into(); + } else { + it.settings.time_span = TimeSpan::LengthOfComp; + it.output = OutputModule::for_format(OutputFormat::ProRes); + it.output.prores_profile = ProResProfile::P4444; + it.output.output = "[compName]_proxy.[fileExtension]".into(); + } + it.output.channels = Channels::Rgba; + if let Some(o) = str_p(p, "path").or(str_p(p, "output")) { + it.output.output = o.to_string(); + } + it.post_render = PostRenderAction::SetProxy; + let id = s.edit("Create Proxy", None, |proj, _| { + it.id = proj.render_queue.iter().map(|i| i.id).max().unwrap_or(0) + 1; + let id = it.id; + proj.render_queue.push(it); + Ok(id) + })?; + let path = s.project.render_queue.last().and_then(|i| s.resolve_output(i)); + Ok(json!({"item": id, "comp": cid.0, "kind": if still { "still" } else { "movie" }, "output": path, "time": Tick::ZERO.seconds()})) +} + +fn has_comp_selection(s: &Session) -> std::result::Result<(), String> { + if s.state.project_selection.iter().any(|i| s.project.comp(*i).is_some()) || s.active_comp_id().is_some() { + Ok(()) + } else { + Err("select a composition".into()) + } +} + +pub fn specs() -> Vec { + vec![ + cmd!( + "file.createProxy", + "Create Proxy", + [], + None, + "{kind: still|movie, comp?|item?, path?|output? (template), resolution? (default 0.5)}", + has_comp_selection, + create_proxy + ), + cmd!("file.setProxy", "File...", ["File", "Set Proxy"], None, "{path, items?|item?}", has_targets, set_proxy), + cmd!("file.setProxyNone", "None", ["File", "Set Proxy"], None, "{items?|item?}", has_proxy, set_proxy_none), + cmd!("file.useProxy", "Use Proxy", [], None, "{items?|item?, on?}", has_proxy, use_proxy), + cmd!( + "file.interpretProxy", + "Proxy...", + ["File", "Interpret Footage"], + None, + "{items?|item?, alpha?, matteColor?, invertAlpha?, frameRate?, loop?, pixelAspect?, fields?, colorProfile?, linearLight?}", + has_proxy, + interpret_proxy + ), + ] +} diff --git a/crates/engine/src/commands/render_queue.rs b/crates/engine/src/commands/render_queue.rs index c905ce6c..acdf4912 100644 --- a/crates/engine/src/commands/render_queue.rs +++ b/crates/engine/src/commands/render_queue.rs @@ -104,6 +104,10 @@ fn apply_settings(rs: &mut RenderSettings, p: &Value, cmd: &str) -> Result { return Err(bad(cmd, "end must be after start")); } + if let Some(u) = str_p(p, "proxyUse") { + rs.proxy_use = effectcraft_project::render_queue::ProxyUse::parse(u).ok_or_else(|| bad(cmd, "proxyUse: current|all|comp|none"))?; + any = true; + } match p.get("frameRate") { Some(Value::Null) => { rs.frame_rate = None; @@ -509,7 +513,7 @@ pub fn specs() -> Vec { "Add to Render Queue", ["Composition"], Some("Cmd+M"), - "{comp?: id|name, format?: h264|prores|png|jpeg|tiff|exr|gif, output?: path|template, quality?: best|draft|1-100 (jpeg), resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, channels?: rgb|rgba, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool}", + "{comp?: id|name, format?: h264|prores|png|jpeg|tiff|exr|gif, output?: path|template, quality?: best|draft|1-100 (jpeg), resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, proxyUse?: current|all|comp|none, channels?: rgb|rgba, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool}", can_add, add ), @@ -520,7 +524,7 @@ pub fn specs() -> Vec { "Render Settings...", [], None, - "{item?|index?, quality?: best|draft, resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool}", + "{item?|index?, quality?: best|draft, resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, proxyUse?: current|all|comp|none}", has_items, set_render_settings ), diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index 8b244bf6..b62f3654 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -20,12 +20,6 @@ pub fn specs() -> Vec { // macOS application menu items handled by the OS. stub!("app.hideOthers", "Hide Others", [], "{}"), stub!("app.showAll", "Show All", [], "{}"), - // Render queue / export (export milestone). - stub!("file.createProxy", "Create Proxy", [], "{kind: still|movie}"), - // Proxies (no proxy model yet). - stub!("file.setProxy", "File...", ["File", "Set Proxy"], "{path}"), - stub!("file.setProxyNone", "None", ["File", "Set Proxy"], "{}"), - stub!("file.interpretProxy", "Proxy...", ["File", "Interpret Footage"], "{}"), stub!("keys.selectLabelGroup", "Select Keyframe Label Group", [], "{scope}"), // Layer ▸ New (3D / content-aware fill). stub!("layer.newContentAwareFill", "Content-Aware Fill Layer...", ["Layer", "New"], "{}"), diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index d0a4ba1a..b2c4d342 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -340,7 +340,8 @@ impl Session { let spec = commands::find(id).ok_or_else(|| EngineError::UnknownCommand(id.to_string()))?; if let Err(why) = (spec.enabled)(self) { // Explicit targets (agents, scripts) don't need a UI selection. - let explicit = ["layer", "layers", "prop", "keys"].iter().any(|k| params.get(k).is_some()) && commands::has_comp(self).is_ok(); + let explicit = (["layer", "layers", "prop", "keys"].iter().any(|k| params.get(k).is_some()) && commands::has_comp(self).is_ok()) + || ["item", "items"].iter().any(|k| params.get(k).is_some()); if !explicit { return Err(EngineError::Disabled(id.to_string(), why)); } @@ -601,6 +602,8 @@ mod tests_model3d; #[cfg(test)] mod tests_project_items; #[cfg(test)] +mod tests_proxy; +#[cfg(test)] mod tests_settings; #[cfg(test)] mod tests_stubs; diff --git a/crates/engine/src/render_queue.rs b/crates/engine/src/render_queue.rs index bc490f47..efeb670f 100644 --- a/crates/engine/src/render_queue.rs +++ b/crates/engine/src/render_queue.rs @@ -388,6 +388,11 @@ impl Session { } let importer = self.importer.clone().ok_or("media import is not available in this build")?; let footage = importer.probe(path)?; + if item.post_render == PostRenderAction::SetProxy { + crate::commands::proxy_set(self, item.comp, footage).map_err(|e| e.to_string())?; + self.toast(format!("Proxy set from {path}")); + return Ok(None); + } let name = std::path::Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.to_string()); let replace = item.post_render == PostRenderAction::ImportAndReplace; let comp = item.comp; diff --git a/crates/engine/src/tests_menu_cmds.rs b/crates/engine/src/tests_menu_cmds.rs index 040c494e..13087ca3 100644 --- a/crates/engine/src/tests_menu_cmds.rs +++ b/crates/engine/src/tests_menu_cmds.rs @@ -487,7 +487,7 @@ fn frontend_commands_emit_events_and_stubs_are_disabled() { let ev = frontend_events(&mut s); assert_eq!(ev[0].0, "view.zoomIn"); assert_eq!(ev[1], ("window.panel".to_string(), json!({"panel": "align"}))); - for id in ["file.createProxy", "camera.stereoRig", "view.displayColorManagement", "view.3d.default", "layer.create", "track.motion"] { + for id in ["keys.audioToKeyframes", "camera.stereoRig", "view.displayColorManagement", "view.3d.default", "layer.create", "track.motion"] { assert!(!s.is_enabled(id), "{id}"); assert!(matches!(s.execute(id, json!({})), Err(EngineError::Disabled(..))), "{id}"); } diff --git a/crates/engine/src/tests_proxy.rs b/crates/engine/src/tests_proxy.rs new file mode 100644 index 00000000..83ae59be --- /dev/null +++ b/crates/engine/src/tests_proxy.rs @@ -0,0 +1,275 @@ +//! Proxies (Set Proxy, Use Proxy, Proxy Use render settings, comp proxies, Create Proxy) and +//! Interpret Footage (fields, pixel aspect, alpha guess / invert, linear light), rendered through +//! a synthetic footage source. + +use std::sync::Arc; + +use effectcraft_project::render_queue::ProxyUse; +use effectcraft_project::{AlphaMode, Footage, FootageKind, ItemId, ItemKind}; +use effectcraft_raster::Image; +use effectcraft_time::{FrameRate, Tick}; +use serde_json::json; + +use crate::Session; +use crate::render::{FootageSource, RenderOpts}; + +/// Files by name: `red16.png` (16×16 red), `blue8.png` (8×8 blue), `lines.mov` (interlaced: +/// even rows red, odd rows blue; 4×4, 10 fps), `matte_black.png` / `matte_white.png` (a +/// semi-transparent green premultiplied against black / white, read as straight), `half.png` +/// (white at 25 % alpha), `wide.png` (10×10 white). +struct Synthetic; + +fn file_px(path: &str, x: u32, y: u32) -> [f32; 4] { + let name = std::path::Path::new(path).file_name().and_then(|n| n.to_str()).unwrap_or(""); + match name { + "red16.png" => [1.0, 0.0, 0.0, 1.0], + "blue8.png" => [0.0, 0.0, 1.0, 1.0], + "lines.mov" => { + if y.is_multiple_of(2) { + [1.0, 0.0, 0.0, 1.0] + } else { + [0.0, 0.0, 1.0, 1.0] + } + } + // Straight interpretation of a file whose colour is green·a + matte·(1 − a), a = 0.5. + "matte_black.png" => [0.0, 0.5, 0.0, 0.5], + "matte_white.png" => [0.5, 1.0, 0.5, 0.5], + "half.png" => [1.0, 1.0, 1.0, 0.25], + _ => { + let _ = x; + [1.0, 1.0, 1.0, 1.0] + } + } +} + +fn size_of(path: &str) -> (u32, u32) { + match std::path::Path::new(path).file_name().and_then(|n| n.to_str()).unwrap_or("") { + "red16.png" => (16, 16), + "blue8.png" => (8, 8), + "lines.mov" | "matte_black.png" | "matte_white.png" | "half.png" => (4, 4), + _ => (10, 10), + } +} + +impl FootageSource for Synthetic { + fn frame(&self, _: ItemId, f: &Footage, _t: Tick) -> Option> { + let (w, h) = size_of(&f.path); + let mut img = Image::new(w, h); + for y in 0..h { + for x in 0..w { + // Decoders hand out premultiplied pixels (straight file colour × alpha). + let p = file_px(&f.path, x, y); + img.set(x, y, [p[0] * p[3], p[1] * p[3], p[2] * p[3], p[3]]); + } + } + Some(Arc::new(img)) + } +} + +struct Probe; +impl crate::Importer for Probe { + fn probe(&self, path: &str) -> Result { + let (w, h) = size_of(path); + let video = path.ends_with(".mov"); + Ok(Footage { + path: path.into(), + kind: if video { FootageKind::Video } else { FootageKind::Still }, + width: w, + height: h, + has_video: true, + frame_rate: FrameRate::from_f64(10.0), + duration: Tick::from_seconds_f64(if video { 2.0 } else { 0.0 }), + ..Default::default() + }) + } +} + +fn session() -> Session { + Session { footage: Arc::new(Synthetic), importer: Some(Arc::new(Probe)), ..Default::default() } +} + +fn import(s: &mut Session, path: &str) -> ItemId { + ItemId(s.execute("file.import", json!({"paths": [path]})).unwrap()["items"][0].as_u64().unwrap()) +} + +fn px(s: &Session, comp: ItemId, t: f64, opts: RenderOpts, x: i64, y: i64) -> [f32; 4] { + s.render(comp, Tick::from_seconds_f64(t), opts).get(x, y) +} + +fn red(p: [f32; 4]) -> bool { + p[0] > 0.99 && p[2] < 0.01 && p[3] > 0.99 +} +fn blue(p: [f32; 4]) -> bool { + p[2] > 0.99 && p[0] < 0.01 && p[3] > 0.99 +} + +#[test] +fn footage_proxy_follows_use_proxy_and_proxy_use() { + let mut s = session(); + let full = import(&mut s, "/m/red16.png"); + s.execute("comp.new", json!({"name": "C", "width": 16, "height": 16, "frameRate": 10, "duration": 1})).unwrap(); + let c = s.active_comp_id().unwrap(); + s.execute("layer.addItem", json!({"item": full.0})).unwrap(); + let o = RenderOpts::default(); + assert!(red(px(&s, c, 0.0, o, 8, 8))); + s.execute("file.setProxy", json!({"item": full.0, "path": "/m/blue8.png"})).unwrap(); + // The 8×8 proxy fills the 16×16 footage frame. + for (x, y) in [(1, 1), (8, 8), (14, 14)] { + assert!(blue(px(&s, c, 0.0, o, x, y)), "proxy at {x},{y}"); + } + assert!(s.project.item(full).unwrap().proxy.as_ref().unwrap().enabled); + // Render Settings ▸ Proxy Use. + assert!(red(px(&s, c, 0.0, RenderOpts { proxy: ProxyUse::UseNone, ..o }, 8, 8))); + assert!(red(px(&s, c, 0.0, RenderOpts { proxy: ProxyUse::UseCompOnly, ..o }, 8, 8))); + // Use Proxy off: the viewer shows the source, Use All Proxies still uses it. + s.execute("file.useProxy", json!({"item": full.0})).unwrap(); + assert!(!s.project.item(full).unwrap().proxy.as_ref().unwrap().enabled); + assert!(red(px(&s, c, 0.0, o, 8, 8))); + assert!(blue(px(&s, c, 0.0, RenderOpts { proxy: ProxyUse::UseAll, ..o }, 8, 8))); + // Undo / Set Proxy ▸ None. + s.execute("edit.undo", json!({})).unwrap(); + assert!(blue(px(&s, c, 0.0, o, 8, 8))); + s.execute("file.setProxyNone", json!({"item": full.0})).unwrap(); + assert!(s.project.item(full).unwrap().proxy.is_none()); + assert!(red(px(&s, c, 0.0, RenderOpts { proxy: ProxyUse::UseAll, ..o }, 8, 8))); + // Render queue items carry Proxy Use (Best Settings: none). + let r = s.execute("renderQueue.add", json!({"proxyUse": "all"})).unwrap(); + assert_eq!(s.project.render_queue[0].settings.proxy_use, ProxyUse::UseAll, "{r}"); + assert_eq!(effectcraft_project::render_queue::RenderSettings::default().proxy_use, ProxyUse::UseNone); +} + +#[test] +fn comp_proxy_and_create_proxy() { + let mut s = session(); + s.execute("comp.new", json!({"name": "Inner", "width": 16, "height": 16, "frameRate": 10, "duration": 1})).unwrap(); + let inner = s.active_comp_id().unwrap(); + s.execute("layer.newSolid", json!({"color": "#ff0000"})).unwrap(); + s.execute("comp.new", json!({"name": "Outer", "width": 16, "height": 16, "frameRate": 10, "duration": 1})).unwrap(); + let outer = s.active_comp_id().unwrap(); + s.execute("layer.addItem", json!({"item": inner.0})).unwrap(); + let o = RenderOpts::default(); + assert!(red(px(&s, outer, 0.0, o, 8, 8))); + s.execute("file.setProxy", json!({"item": inner.0, "path": "/m/blue8.png"})).unwrap(); + assert!(blue(px(&s, outer, 0.0, o, 14, 1))); + assert!(blue(px(&s, outer, 0.0, RenderOpts { proxy: ProxyUse::UseCompOnly, ..o }, 8, 8))); + assert!(red(px(&s, outer, 0.0, RenderOpts { proxy: ProxyUse::UseNone, ..o }, 8, 8))); + s.execute("file.setProxyNone", json!({"item": inner.0})).unwrap(); + // Create Proxy queues a half-resolution render whose post-render action sets the proxy. + s.state.project_selection = vec![inner]; + let r = s.execute("file.createProxy", json!({"kind": "still"})).unwrap(); + let it = s.project.render_queue.last().unwrap().clone(); + assert_eq!(it.comp, inner); + assert_eq!(it.post_render, effectcraft_project::render_queue::PostRenderAction::SetProxy); + assert_eq!(it.settings.resolution, 0.5); + assert!(r["output"].as_str().unwrap().contains("Inner_proxy_"), "{r}"); + let m = s.execute("file.createProxy", json!({"kind": "movie", "comp": inner.0})).unwrap(); + assert!(m["output"].as_str().unwrap().ends_with("Inner_proxy.mov"), "{m}"); + // The post-render action (what a finished render runs) attaches the file. + let id = it.id; + s.post_render_action(id, "/m/blue8.png").unwrap(); + assert_eq!(s.project.item(inner).unwrap().proxy.as_ref().unwrap().footage.path, "/m/blue8.png"); + assert!(blue(px(&s, outer, 0.0, o, 8, 8))); +} + +#[test] +fn interpret_proxy_settings_apply_to_the_proxy() { + let mut s = session(); + let full = import(&mut s, "/m/red16.png"); + s.execute("file.setProxy", json!({"item": full.0, "path": "/m/half.png"})).unwrap(); + s.execute("file.interpretProxy", json!({"item": full.0, "alpha": "ignore", "invertAlpha": true})).unwrap(); + let px = s.project.item(full).unwrap().proxy.clone().unwrap(); + assert_eq!(px.footage.alpha, AlphaMode::Ignore); + assert!(px.footage.invert_alpha); + let f = match &s.project.item(full).unwrap().kind { + ItemKind::Footage(f) => f.clone(), + _ => unreachable!(), + }; + assert_eq!(f.alpha, AlphaMode::Straight, "the main footage keeps its interpretation"); +} + +/// Separate Fields: each field becomes a frame at twice the frame rate, the other field's lines +/// interpolated; the dominant field comes first. +#[test] +fn separate_fields_make_frames_from_fields() { + let mut s = session(); + let mov = import(&mut s, "/m/lines.mov"); + s.execute("comp.new", json!({"name": "C", "width": 4, "height": 4, "frameRate": 20, "duration": 1})).unwrap(); + let c = s.active_comp_id().unwrap(); + s.execute("layer.addItem", json!({"item": mov.0})).unwrap(); + let o = RenderOpts::default(); + // Progressive: alternating rows. + assert!(red(px(&s, c, 0.0, o, 1, 0)) && blue(px(&s, c, 0.0, o, 1, 1))); + s.execute("file.interpretFootage", json!({"item": mov.0, "fields": "upper"})).unwrap(); + // Field 1 (upper = even rows, red) then field 2 (odd rows, blue), each 1/20 s. + for y in 0..3 { + assert!(red(px(&s, c, 0.0, o, 1, y)), "upper field at row {y}"); + assert!(blue(px(&s, c, 0.05, o, 1, y + 1)), "lower field at row {}", y + 1); + } + s.execute("file.interpretFootage", json!({"item": mov.0, "fields": "lower"})).unwrap(); + assert!(blue(px(&s, c, 0.0, o, 1, 2))); + assert!(red(px(&s, c, 0.05, o, 1, 1))); + let r = s.execute("file.interpretFootage", json!({"item": mov.0, "fields": "off"})).unwrap(); + assert_eq!(r["items"][0]["fields"], "Off"); +} + +/// Non-square pixels: a 10×10 footage with pixel aspect 2 covers 20×10 comp pixels. +#[test] +fn pixel_aspect_stretches_footage_in_the_comp() { + let mut s = session(); + let w = import(&mut s, "/m/wide.png"); + s.execute("comp.new", json!({"name": "C", "width": 40, "height": 20, "frameRate": 10, "duration": 1})).unwrap(); + let c = s.active_comp_id().unwrap(); + s.execute("layer.addItem", json!({"item": w.0})).unwrap(); + let o = RenderOpts::default(); + let covered = |s: &Session| s.render(c, Tick::ZERO, o).data.iter().filter(|p| p[3] > 0.5).count(); + assert_eq!(covered(&s), 100); + s.execute("file.interpretFootage", json!({"item": w.0, "pixelAspect": 2.0})).unwrap(); + assert_eq!(covered(&s), 200); + // Centered on the comp: x 10..30 (edge pixels are filtered). + assert!(px(&s, c, 0.0, o, 11, 10)[3] > 0.99 && px(&s, c, 0.0, o, 28, 10)[3] > 0.99); + assert!(px(&s, c, 0.0, o, 8, 10)[3] < 0.01 && px(&s, c, 0.0, o, 31, 10)[3] < 0.01); + // A comp with the same pixel aspect shows the footage 1:1. + s.execute("comp.settings", json!({"pixelAspect": 2.0})).unwrap(); + assert_eq!(covered(&s), 100); +} + +/// Alpha ▸ Guess: premultiplied files are recognised with their matte colour; Invert Alpha +/// and Interpret As Linear Light change the decoded pixels. +#[test] +fn guess_alpha_invert_and_linear_light() { + let mut s = session(); + let b = import(&mut s, "/m/matte_black.png"); + let w = import(&mut s, "/m/matte_white.png"); + let h = import(&mut s, "/m/half.png"); + let foot = |s: &Session, i: ItemId| match &s.project.item(i).unwrap().kind { + ItemKind::Footage(f) => f.clone(), + _ => unreachable!(), + }; + s.execute("file.interpretFootage", json!({"items": [b.0, w.0, h.0], "alpha": "guess"})).unwrap(); + assert_eq!(foot(&s, b).alpha, AlphaMode::Premultiplied); + assert_eq!(foot(&s, b).premul_color, [0.0; 3]); + assert_eq!(foot(&s, w).alpha, AlphaMode::Premultiplied); + assert_eq!(foot(&s, w).premul_color, [1.0; 3]); + // White at 25 %: not explained by a black matte (1 > 0.25) but it is by white… either way + // the guess is consistent with the pixels. + assert!(matches!(foot(&s, h).alpha, AlphaMode::Straight | AlphaMode::Premultiplied)); + // An explicit matte colour. + s.execute("file.interpretFootage", json!({"item": b.0, "alpha": "premultiplied", "matteColor": "#808080"})).unwrap(); + assert!((foot(&s, b).premul_color[0] - 128.0 / 255.0).abs() < 1e-3); + // Invert Alpha: 25 % → 75 %. + s.execute("file.interpretFootage", json!({"item": h.0, "alpha": "straight", "invertAlpha": true})).unwrap(); + s.execute("comp.new", json!({"name": "C", "width": 4, "height": 4, "frameRate": 10, "duration": 1})).unwrap(); + let c = s.active_comp_id().unwrap(); + s.execute("layer.addItem", json!({"item": h.0})).unwrap(); + let p = px(&s, c, 0.0, RenderOpts::default(), 1, 1); + assert!((p[3] - 0.75).abs() < 0.01 && p[0] > 0.7, "{p:?}"); + // Interpret As Linear Light keeps alpha and re-encodes the colour. + s.execute("file.interpretFootage", json!({"item": h.0, "invertAlpha": false, "linearLight": true})).unwrap(); + let p = px(&s, c, 0.0, RenderOpts::default(), 1, 1); + assert!((p[3] - 0.25).abs() < 0.01, "{p:?}"); + // Remember / Apply Interpretation carries the new settings. + s.state.project_selection = vec![h]; + s.execute("file.rememberInterpretation", json!({"item": h.0})).unwrap(); + s.execute("file.applyInterpretation", json!({"items": [w.0]})).unwrap(); + assert!(foot(&s, w).linear_light); +} diff --git a/crates/engine/src/tests_stubs.rs b/crates/engine/src/tests_stubs.rs index 5cb70246..924fb579 100644 --- a/crates/engine/src/tests_stubs.rs +++ b/crates/engine/src/tests_stubs.rs @@ -332,7 +332,7 @@ fn implemented_stubs_are_enabled_and_workspaces_are_frontend() { assert!(s.drain_events().iter().any(|e| matches!(e, Event::Frontend { command, .. } if command == id))); } // What remains a stub is still disabled. - for id in ["view.displayColorManagement", "file.createProxy", "track.camera"] { + for id in ["view.displayColorManagement", "keys.audioToKeyframes", "track.camera"] { assert!(!s.is_enabled(id), "{id}"); } } diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index dc4c35c8..86568871 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -176,6 +176,7 @@ pub(crate) fn render_frame(job: &Job, comp: &Comp, i: u64) -> Image { view: None, backend: effectcraft_render::Backend::Auto, roi: None, + proxy: job.settings.proxy_use, }; let mut r = Renderer::new(job.project, job.footage, opts); r.expr = job.expr; diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index dccf1aac..b90a42f0 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -297,6 +297,59 @@ mod tests { assert!(px[1] > 0.99 && px[0] < 0.01, "the box is filled with the sampled green: {px:?}"); } + /// Proxies through the real media and export layers: Render Settings ▸ Proxy Use decides + /// whether an export uses the proxy, and Create Proxy renders and attaches one. + #[test] + fn proxies_in_exports_and_create_proxy() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/test-out/host-proxy"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let p = |n: &str| dir.join(n).to_string_lossy().to_string(); + let mut s = super::session(); + // Source and proxy stills, written by the app itself. + s.execute("comp.new", json!({"name": "Red", "width": 16, "height": 16, "frameRate": 10, "duration": 1})).unwrap(); + let red = s.active_comp_id().unwrap(); + s.execute("layer.newSolid", json!({"color": "#ff0000"})).unwrap(); + s.execute("comp.saveFrameAs", json!({"path": p("red.png")})).unwrap(); + s.execute("comp.new", json!({"name": "Blue", "width": 8, "height": 8, "frameRate": 10, "duration": 1})).unwrap(); + s.execute("layer.newSolid", json!({"color": "#0000ff"})).unwrap(); + s.execute("comp.saveFrameAs", json!({"path": p("blue.png")})).unwrap(); + let foot = s.execute("file.import", json!({"paths": [p("red.png")]})).unwrap()["items"][0].as_u64().unwrap(); + s.execute("file.setProxy", json!({"item": foot, "path": p("blue.png")})).unwrap(); + s.execute("comp.new", json!({"name": "Out", "width": 16, "height": 16, "frameRate": 10, "duration": 1})).unwrap(); + let out = s.active_comp_id().unwrap(); + s.execute("layer.addItem", json!({"item": foot})).unwrap(); + // Export one frame with each Proxy Use. + for (name, using) in [("all", "use_all_[#].png"), ("none", "use_none_[#].png")] { + s.execute( + "renderQueue.add", + json!({"comp": out.0, "format": "png", "output": p(using), "timeSpan": "custom", "start": 0.0, "end": 0.1, "proxyUse": name}), + ) + .unwrap(); + } + s.execute("renderQueue.render", json!({"wait": true})).unwrap(); + s.poll_render(); + let centre = |s: &mut effectcraft_engine::Session, path: &str| { + let id = s.execute("file.import", json!({"paths": [path]})).unwrap()["items"][0].as_u64().unwrap(); + s.execute("comp.new", json!({"name": "Probe", "width": 16, "height": 16, "frameRate": 10, "duration": 1})).unwrap(); + s.execute("layer.addItem", json!({"item": id})).unwrap(); + let c = s.active_comp_id().unwrap(); + s.render(c, Default::default(), Default::default()).get(8, 8) + }; + let all = centre(&mut s, &p("use_all_0.png")); + assert!(all[2] > 0.99 && all[0] < 0.01, "Use All Proxies renders the blue proxy: {all:?}"); + let none = centre(&mut s, &p("use_none_0.png")); + assert!(none[0] > 0.99 && none[2] < 0.01, "Use No Proxies renders the red source: {none:?}"); + // Create Proxy ▸ Still: rendered at half size and attached to the comp. + s.execute("file.createProxy", json!({"kind": "still", "comp": red.0, "path": p("red_proxy_[#####].png")})).unwrap(); + s.execute("renderQueue.render", json!({"wait": true})).unwrap(); + s.poll_render(); + let px = s.project.item(red).unwrap().proxy.clone().expect("Create Proxy attached the rendered still"); + assert!(px.enabled); + assert_eq!((px.footage.width, px.footage.height), (8, 8)); + assert!(std::path::Path::new(&px.footage.path).exists(), "{}", px.footage.path); + } + #[test] fn expression_syntax_errors_disable_the_expression() { let mut s = super::session(); diff --git a/crates/project/src/render_queue.rs b/crates/project/src/render_queue.rs index c41462b5..31537a5b 100644 --- a/crates/project/src/render_queue.rs +++ b/crates/project/src/render_queue.rs @@ -39,6 +39,51 @@ pub enum TimeSpan { Custom { start: Tick, end: Tick }, } +/// Render Settings ▸ Proxy Use: which proxies a render uses. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum ProxyUse { + /// Each item's Use Proxy switch (what the viewer shows). + #[default] + CurrentSettings, + /// Every item that has a proxy uses it. + UseAll, + /// Only compositions' proxies. + UseCompOnly, + /// No proxies (full-resolution sources). + UseNone, +} + +impl ProxyUse { + pub const ALL: [ProxyUse; 4] = [ProxyUse::CurrentSettings, ProxyUse::UseAll, ProxyUse::UseCompOnly, ProxyUse::UseNone]; + pub fn label(self) -> &'static str { + match self { + ProxyUse::CurrentSettings => "Current Settings", + ProxyUse::UseAll => "Use All Proxies", + ProxyUse::UseCompOnly => "Use Comp Proxies Only", + ProxyUse::UseNone => "Use No Proxies", + } + } + pub fn parse(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '_', '-'], "").as_str() { + "current" | "currentsettings" => Some(ProxyUse::CurrentSettings), + "all" | "useall" | "useallproxies" => Some(ProxyUse::UseAll), + "comp" | "componly" | "usecompproxiesonly" | "usecomponly" => Some(ProxyUse::UseCompOnly), + "none" | "usenone" | "usenoproxies" => Some(ProxyUse::UseNone), + _ => None, + } + } + /// Whether an item's proxy is used: `comp` = the item is a composition, `enabled` = its Use + /// Proxy switch. + pub fn uses(self, comp: bool, enabled: bool) -> bool { + match self { + ProxyUse::CurrentSettings => enabled, + ProxyUse::UseAll => true, + ProxyUse::UseCompOnly => comp, + ProxyUse::UseNone => false, + } + } +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(default)] pub struct RenderSettings { @@ -52,6 +97,8 @@ pub struct RenderSettings { pub motion_blur: bool, /// Image sequences: skip frames whose file already exists. pub skip_existing: bool, + /// Proxy Use (Best Settings: Use No Proxies). + pub proxy_use: ProxyUse, } impl Default for RenderSettings { @@ -63,6 +110,7 @@ impl Default for RenderSettings { frame_rate: None, motion_blur: true, skip_existing: false, + proxy_use: ProxyUse::UseNone, } } } @@ -419,6 +467,8 @@ pub enum PostRenderAction { Import, /// Import it and replace every use of the rendered composition (Composition ▸ Pre-render). ImportAndReplace, + /// Set the rendered file as the composition's proxy (File ▸ Create Proxy). + SetProxy, } impl PostRenderAction { @@ -430,6 +480,7 @@ impl PostRenderAction { PostRenderAction::None => "None", PostRenderAction::Import => "Import", PostRenderAction::ImportAndReplace => "Import & Replace Usage", + PostRenderAction::SetProxy => "Set Proxy", } } } diff --git a/crates/render/src/cache.rs b/crates/render/src/cache.rs index 8cd27d59..ca850a86 100644 --- a/crates/render/src/cache.rs +++ b/crates/render/src/cache.rs @@ -286,6 +286,8 @@ fn key_with(ctx: &EvalCtx, layer: &Layer, scale: f64, draft: bool, footage: bool let it = ctx.project.item(*item)?; let ItemKind::Footage(f) = &it.kind else { return None }; hash_debug(&mut h, f); + // A proxy (and its Use Proxy switch) changes the pixels. + hash_debug(&mut h, &it.proxy); item.hash(&mut h); ctx.source_time(layer).0.hash(&mut h); // Frame blending mixes neighbouring source frames. diff --git a/crates/render/src/eval.rs b/crates/render/src/eval.rs index aff4d93c..d8d3f641 100644 --- a/crates/render/src/eval.rs +++ b/crates/render/src/eval.rs @@ -120,6 +120,28 @@ impl<'a> EvalCtx<'a> { /// Local (parent-space) transform of a layer. pub fn local_matrix(&self, layer: &Layer) -> Mat4 { + self.local_matrix_par(layer, 1.0) + } + + /// Source pixel aspect relative to the comp's: non-square footage (or solids, or precomps + /// with another pixel aspect) is stretched horizontally by this factor in the comp. + pub fn par_ratio(&self, layer: &Layer) -> f64 { + let par = match &layer.source { + LayerSource::Footage { item } | LayerSource::Solid { item } | LayerSource::Comp { item } => match self.project.item(*item).map(|i| &i.kind) { + Some(effectcraft_project::ItemKind::Footage(f)) => f.pixel_aspect, + Some(effectcraft_project::ItemKind::Solid(s)) => s.pixel_aspect, + Some(effectcraft_project::ItemKind::Comp(c)) => c.pixel_aspect, + _ => 1.0, + }, + _ => 1.0, + }; + let r = par / if self.comp.pixel_aspect > 0.0 { self.comp.pixel_aspect } else { 1.0 }; + if r.is_finite() && r > 0.0 && (r - 1.0).abs() > 1e-9 { r } else { 1.0 } + } + + /// [`Self::local_matrix`] with the source's pixel aspect folded into the scale (`par`; 1 for + /// a parent's matrix, which children don't inherit). + fn local_matrix_par(&self, layer: &Layer, par: f64) -> Mat4 { let Some(tr) = layer.transform() else { return Mat4::IDENTITY }; let three = layer.is_3d(); let pos = self.position(layer, tr); @@ -130,6 +152,7 @@ impl<'a> EvalCtx<'a> { } let anchor = self.v3(layer, tr, "anchor", [0.0; 3]); let mut scale = self.v3(layer, tr, "scale", [100.0; 3]); + scale[0] *= par; if !three { scale[2] = 100.0; let a = Vec3::from([anchor[0], anchor[1], 0.0]); @@ -154,7 +177,7 @@ impl<'a> EvalCtx<'a> { /// Layer space → comp (world) space, including parents. pub fn world_matrix(&self, layer: &Layer) -> Mat4 { - let mut m = self.local_matrix(layer); + let mut m = self.local_matrix_par(layer, self.par_ratio(layer)); let mut cur = layer.parent; let mut guard = 0; while let Some(pid) = cur { diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 6699d68d..01f5f96a 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -247,11 +247,22 @@ pub struct RenderOpts { /// Region of interest `[x, y, w, h]` in comp pixels (viewer only): the top-level frame covers /// just this rectangle, and only its pixels are composited. pub roi: Option<[f64; 4]>, + /// Which proxies stand in for footage and compositions (Render Settings ▸ Proxy Use). + pub proxy: effectcraft_project::render_queue::ProxyUse, } impl Default for RenderOpts { fn default() -> Self { - RenderOpts { scale: 1.0, motion_blur: true, guides: false, draft: false, view: None, roi: None, backend: Backend::Cpu } + RenderOpts { + scale: 1.0, + motion_blur: true, + guides: false, + draft: false, + view: None, + roi: None, + backend: Backend::Cpu, + proxy: effectcraft_project::render_queue::ProxyUse::CurrentSettings, + } } } @@ -675,6 +686,13 @@ impl<'a> Renderer<'a> { if self.project.comp_contains(*item, ctx.comp_id) { return None; } + // A composition proxy (a rendered still or movie) stands in for the comp. + if let Some(pf) = self.proxy_for(*item) + && let Some(nc) = self.project.comp(*item) + { + let img = self.footage.frame(*item, pf, ctx.source_time(layer))?; + return Some(self.footage_buf(&img, pf, nc.width, nc.height)); + } // Essential Properties overrides render the nested comp with this instance's // values. let ov = essential_overrides(ctx, layer); @@ -706,20 +724,10 @@ impl<'a> Renderer<'a> { return None; } let lt = ctx.source_time(layer); - let img = self.footage_frame(ctx, layer, *item, f, lt)?; - let k = if img.width > 0 { f.width.max(1) as f64 / img.width as f64 } else { 1.0 }; - // Keep native pixels when downsampling is small; resample otherwise. - let mut buf = if s < 0.75 { - let w = ((f.width as f64 * s).round() as u32).max(1); - let h = ((f.height as f64 * s).round() as u32).max(1); - Buf { img: effectcraft_raster::resample(&img, w, h), offset: [0.0; 2], scale: s } - } else { - Buf { img: (*img).clone(), offset: [0.0; 2], scale: 1.0 / k } - }; - if let Some(c) = self.pipe.media_in(f.color_profile) { - color::convert(&mut buf.img, &c); - } - Some(buf) + // The proxy is decoded instead, at its own size, and fills the footage's frame. + let pf = self.proxy_for(*item).unwrap_or(f); + let img = self.footage_frame(ctx, layer, *item, pf, lt)?; + Some(self.footage_buf(&img, pf, f.width, f.height)) } LayerSource::Text => Some(self.authored(text::render(ctx, layer, self.raster_scale(ctx, layer)))), LayerSource::Shape => layer.props.sub("contents").map(|c| self.authored(shapes::render(ctx, layer, c, self.raster_scale(ctx, layer)))), @@ -727,6 +735,32 @@ impl<'a> Renderer<'a> { } } + /// The proxy footage used for `item` under this render's Proxy Use, if any. + pub fn proxy_for(&self, item: ItemId) -> Option<&'a Footage> { + let it = self.project.item(item)?; + let px = it.proxy.as_deref()?; + (self.opts.proxy.uses(matches!(it.kind, ItemKind::Comp(_)), px.enabled) && px.footage.has_video).then_some(&px.footage) + } + + /// A decoded frame of `f` as a layer buffer covering `w`×`h` layer pixels (the footage's + /// nominal size; proxies are scaled up to it), converted from its colour profile. + fn footage_buf(&self, img: &Image, f: &Footage, w: u32, h: u32) -> Buf { + let s = self.opts.scale; + let k = if img.width > 0 { w.max(1) as f64 / img.width as f64 } else { 1.0 }; + // Keep native pixels when downsampling is small; resample otherwise. + let mut buf = if s < 0.75 && k <= 1.0 + 1e-9 { + let rw = ((w as f64 * s).round() as u32).max(1); + let rh = ((h as f64 * s).round() as u32).max(1); + Buf { img: effectcraft_raster::resample(img, rw, rh), offset: [0.0; 2], scale: s } + } else { + Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 / k } + }; + if let Some(c) = self.pipe.media_in(f.color_profile) { + color::convert(&mut buf.img, &c); + } + buf + } + /// Authored colours into the working space (linear working spaces). fn authored(&self, mut buf: Buf) -> Buf { if let Some(c) = self.pipe.authored_in() { @@ -739,6 +773,21 @@ impl<'a> Renderer<'a> { /// when the layer's Frame Blending switch (and the comp's Enable Frame Blending) is on and /// `t` falls between frames (footage rate ≠ comp rate, time stretch, time remapping). fn footage_frame(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick) -> Option> { + // Interpret Footage ▸ Separate Fields: each field is a frame at twice the rate. + if f.fields != effectcraft_project::FieldOrder::Off && matches!(f.kind, FootageKind::Video | FootageKind::Sequence) { + let field_rate = FrameRate::new(f.frame_rate.num * 2, f.frame_rate.den); + let i = field_rate.frame_at(t).max(0); + let img = self.footage.frame(item, f, f.frame_rate.tick_of(i / 2))?; + let dominant_upper = f.fields == effectcraft_project::FieldOrder::UpperFirst; + // Upper field = even lines (0, 2, …). + let parity = if (i % 2 == 0) == dominant_upper { 0 } else { 1 }; + return Some(Arc::new(interpret_pixels(&field_frame(&img, parity), f))); + } + let img = self.footage_frame_raw(ctx, layer, item, f, t)?; + if f.invert_alpha || f.linear_light { Some(Arc::new(interpret_pixels(&img, f))) } else { Some(img) } + } + + fn footage_frame_raw(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick) -> Option> { let mode = frame_blend_mode(ctx, layer); if mode != FrameBlend::Off && matches!(f.kind, FootageKind::Video | FootageKind::Sequence) { let (i, w) = frame_position(t, f.frame_rate); @@ -1307,6 +1356,57 @@ impl<'a> Renderer<'a> { } /// The layer's frame blending mode, if the comp's Enable Frame Blending switch is on. +/// One field of an interlaced frame as a full frame: the field's lines (`parity` 0 = even / +/// upper, 1 = odd / lower) kept, the other lines interpolated from their neighbours. +pub fn field_frame(img: &Image, parity: usize) -> Image { + let (w, h) = (img.width as usize, img.height as usize); + let mut out = img.clone(); + if h < 2 { + return out; + } + for y in 0..h { + if y % 2 == parity { + continue; + } + let above = y.checked_sub(1); + let below = (y + 1 < h).then_some(y + 1); + for x in 0..w { + let px = match (above, below) { + (Some(a), Some(b)) => { + let (p, q) = (img.data[a * w + x], img.data[b * w + x]); + [(p[0] + q[0]) * 0.5, (p[1] + q[1]) * 0.5, (p[2] + q[2]) * 0.5, (p[3] + q[3]) * 0.5] + } + (Some(a), None) => img.data[a * w + x], + (None, Some(b)) => img.data[b * w + x], + (None, None) => img.data[y * w + x], + }; + out.data[y * w + x] = px; + } + } + out +} + +/// Interpret Footage ▸ Invert Alpha and Interpret As Linear Light on decoded (premultiplied) +/// pixels. +pub fn interpret_pixels(img: &Image, f: &Footage) -> Image { + if !f.invert_alpha && !f.linear_light { + return img.clone(); + } + let space = f.color_profile.unwrap_or(effectcraft_color::ColorSpace::Srgb); + let mut out = img.clone(); + for p in out.data.iter_mut() { + let a = p[3]; + let mut c = if a > 1e-6 { [p[0] / a, p[1] / a, p[2] / a] } else { [0.0; 3] }; + if f.linear_light { + // The file's values are linear light: encode them like the rest of the footage. + c = c.map(|v| space.encode(v)); + } + let a = if f.invert_alpha { 1.0 - a } else { a }; + *p = [c[0] * a, c[1] * a, c[2] * a, a]; + } + out +} + fn frame_blend_mode(ctx: &EvalCtx, layer: &Layer) -> FrameBlend { if ctx.comp.enable_frame_blending { layer.switches.frame_blend } else { FrameBlend::Off } } diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 071c1948..cc12a341 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -419,6 +419,7 @@ impl EffectcraftApp { view: self.session.view_camera(comp), roi, backend: effectcraft_engine::render::Backend::Auto, + proxy: Default::default(), }; if urgent { self.frames.request_urgent(&self.render_source(), key, comp, t, opts); From 2cba455175df975f72b9c45d8173265498166805 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 23:31:50 -0400 Subject: [PATCH 09/13] CMP-7: Essential Graphics panel, expression error bar and Expression Language menu, proxy indicator, Interpret Footage dialog Window > Essential Graphics: Primary comp, Solo Supported Properties (timeline filter), template name, typed controls editing the source properties, drag a property name from the timeline onto the list, rename/reorder/remove, Add Formatting (group, comment), Media Replacement, Export Template, and the selected instance's Essential Properties with Revert / Push. The comp viewer shows an expression error bar (click to reveal); the expression editor's f button opens the categorised Expression Language menu. Project panel proxy switch; Interpret Footage dialog gains Guess, matte colour, Invert Alpha, Separate Fields, Linear Light, Proxy. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/commands/essential.rs | 17 +- crates/engine/src/commands/expr_tools.rs | 14 +- crates/ui-egui/src/dock.rs | 5 +- crates/ui-egui/src/menus.rs | 5 +- crates/ui-egui/src/panels/essential.rs | 427 ++++++++++++++++++++++ crates/ui-egui/src/panels/expr_bar.rs | 95 +++++ crates/ui-egui/src/panels/forms.rs | 38 +- crates/ui-egui/src/panels/mod.rs | 5 + crates/ui-egui/src/panels/project.rs | 18 + crates/ui-egui/src/panels/render_queue.rs | 4 + crates/ui-egui/src/panels/timeline.rs | 64 +++- crates/ui-egui/src/panels/viewer.rs | 8 +- 12 files changed, 675 insertions(+), 25 deletions(-) create mode 100644 crates/ui-egui/src/panels/essential.rs create mode 100644 crates/ui-egui/src/panels/expr_bar.rs diff --git a/crates/engine/src/commands/essential.rs b/crates/engine/src/commands/essential.rs index 65e93d4e..a24fcdc0 100644 --- a/crates/engine/src/commands/essential.rs +++ b/crates/engine/src/commands/essential.rs @@ -65,7 +65,8 @@ fn set_primary(s: &mut Session, p: &Value) -> Result { fn set_name(s: &mut Session, p: &Value) -> Result { let cid = eg_comp(s, p)?; let name = str_p(p, "name").ok_or_else(|| bad("essential.setName", "missing `name`"))?.to_string(); - s.edit("Essential Graphics Name", None, |proj, _| { + s.edit("Essential Graphics Name", None, |proj, st| { + st.essential_primary.get_or_insert(cid); eg_of(proj, cid)?.name = name.clone(); Ok(()) })?; @@ -99,7 +100,10 @@ fn add_property(s: &mut Session, p: &Value) -> Result { } added.push((*cid, *lid, *uid, rename.clone().unwrap_or_else(|| pr.name.clone()))); } - let ids = s.edit("Add Property to Essential Graphics", None, |proj, _| { + let ids = s.edit("Add Property to Essential Graphics", None, |proj, st| { + if let Some((cid, ..)) = added.first() { + st.essential_primary.get_or_insert(*cid); + } let mut ids = vec![]; for (k, (cid, lid, uid, name)) in added.iter().enumerate() { let id = proj.alloc(); @@ -126,7 +130,8 @@ fn add_media(s: &mut Session, p: &Value) -> Result { } let name = str_p(p, "name").map(str::to_string).unwrap_or_else(|| format!("{} (Media)", l.name)); let group = p.get("group").and_then(Value::as_u64); - let id = s.edit("Add Media Replacement", None, |proj, _| { + let id = s.edit("Add Media Replacement", None, |proj, st| { + st.essential_primary.get_or_insert(cid); let id = proj.alloc(); let eg = eg_of(proj, cid)?; if eg.flat().iter().any(|c| matches!(c.kind, EgKind::Media { layer } if layer == lid)) { @@ -142,7 +147,8 @@ fn add_group(s: &mut Session, p: &Value) -> Result { let cid = eg_comp(s, p)?; let name = str_p(p, "name").unwrap_or("Group").to_string(); let index = p.get("index").and_then(Value::as_u64).map(|i| i as usize); - let id = s.edit("Add Essential Graphics Group", None, |proj, _| { + let id = s.edit("Add Essential Graphics Group", None, |proj, st| { + st.essential_primary.get_or_insert(cid); let id = proj.alloc(); insert(eg_of(proj, cid)?, EgControl { id, name, kind: EgKind::Group { children: vec![] } }, None, index, "essential.addGroup")?; Ok(id) @@ -154,7 +160,8 @@ fn add_comment(s: &mut Session, p: &Value) -> Result { let cid = eg_comp(s, p)?; let text = str_p(p, "text").unwrap_or("Comment").to_string(); let group = p.get("group").and_then(Value::as_u64); - let id = s.edit("Add Essential Graphics Comment", None, |proj, _| { + let id = s.edit("Add Essential Graphics Comment", None, |proj, st| { + st.essential_primary.get_or_insert(cid); let id = proj.alloc(); insert(eg_of(proj, cid)?, EgControl { id, name: "Comment".into(), kind: EgKind::Comment { text } }, group, None, "essential.addComment")?; Ok(id) diff --git a/crates/engine/src/commands/expr_tools.rs b/crates/engine/src/commands/expr_tools.rs index 868be63a..5648fec1 100644 --- a/crates/engine/src/commands/expr_tools.rs +++ b/crates/engine/src/commands/expr_tools.rs @@ -212,7 +212,7 @@ pub fn language_menu() -> Vec<(&'static str, Vec<(&'static str, &'static str)>)> ], ), ( - "Layer ▸ Sub-objects", + "Layer > Sub-objects", vec![ ("source", "source"), ("effect(name)", "effect(\"name\")"), @@ -222,7 +222,7 @@ pub fn language_menu() -> Vec<(&'static str, Vec<(&'static str, &'static str)>)> ], ), ( - "Layer ▸ General", + "Layer > General", vec![ ("width", "width"), ("height", "height"), @@ -238,7 +238,7 @@ pub fn language_menu() -> Vec<(&'static str, Vec<(&'static str, &'static str)>)> ], ), ( - "Layer ▸ Properties", + "Layer > Properties", vec![ ("anchorPoint", "anchorPoint"), ("position", "position"), @@ -250,7 +250,7 @@ pub fn language_menu() -> Vec<(&'static str, Vec<(&'static str, &'static str)>)> ], ), ( - "Layer ▸ Space Transforms", + "Layer > Space Transforms", vec![ ("toComp(point, t = time)", "toComp(point, t = time)"), ("fromComp(point, t = time)", "fromComp(point, t = time)"), @@ -263,7 +263,7 @@ pub fn language_menu() -> Vec<(&'static str, Vec<(&'static str, &'static str)>)> ], ), ( - "Layer ▸ Other", + "Layer > Other", vec![ ("sourceRectAtTime(t = time, includeExtents = false)", "sourceRectAtTime(t = time, includeExtents = false)"), ("sampleImage(point, radius = [.5, .5], postEffect = true, t = time)", "sampleImage(point, radius = [.5, .5], postEffect = true, t = time)"), @@ -307,7 +307,7 @@ pub fn language_menu() -> Vec<(&'static str, Vec<(&'static str, &'static str)>)> ], ), ( - "Text ▸ Style", + "Text > Style", vec![ ("text.sourceText.style", "text.sourceText.style"), ("getStyleAt(charIndex, t = time)", "getStyleAt(charIndex, t = time)"), @@ -317,7 +317,7 @@ pub fn language_menu() -> Vec<(&'static str, Vec<(&'static str, &'static str)>)> ("setText(value)", "setText(value)"), ], ), - ("Text ▸ Expression Selector", vec![("textIndex", "textIndex"), ("textTotal", "textTotal"), ("selectorValue", "selectorValue")]), + ("Text > Expression Selector", vec![("textIndex", "textIndex"), ("textTotal", "textTotal"), ("selectorValue", "selectorValue")]), ("Key", vec![("value", "value"), ("time", "time"), ("index", "index")]), ( "Marker Key", diff --git a/crates/ui-egui/src/dock.rs b/crates/ui-egui/src/dock.rs index 99bc7c5e..3644d7ac 100644 --- a/crates/ui-egui/src/dock.rs +++ b/crates/ui-egui/src/dock.rs @@ -34,10 +34,11 @@ pub enum PanelKind { History, Markers, MaskInterpolation, + EssentialGraphics, } impl PanelKind { - pub const ALL: [PanelKind; 24] = [ + pub const ALL: [PanelKind; 25] = [ PanelKind::Project, PanelKind::EffectControls, PanelKind::Composition, @@ -62,6 +63,7 @@ impl PanelKind { PanelKind::History, PanelKind::Markers, PanelKind::MaskInterpolation, + PanelKind::EssentialGraphics, ]; pub fn title(self) -> &'static str { match self { @@ -89,6 +91,7 @@ impl PanelKind { PanelKind::History => "History", PanelKind::Markers => "Markers", PanelKind::MaskInterpolation => "Mask Interpolation", + PanelKind::EssentialGraphics => "Essential Graphics", } } pub fn id(self) -> String { diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index 22b2bd40..f97851f8 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -700,6 +700,9 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option ("path", Ask::Open(&["jsonl", "json", "txt"])), "file.replaceFootage" => ("path", Ask::Import), "file.collectFiles" => ("folder", Ask::Save("Collected Files")), + "essential.exportTemplate" => ("path", Ask::Save("Template.ectemplate")), + "essential.importTemplate" => ("path", Ask::Open(&["ectemplate"])), + "file.setProxy" => ("path", Ask::Open(&["mp4", "mov", "m4v", "mkv", "webm", "png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "exr"])), _ => return None, }; if params.get(key).is_some() @@ -716,7 +719,7 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option None, diff --git a/crates/ui-egui/src/panels/essential.rs b/crates/ui-egui/src/panels/essential.rs new file mode 100644 index 00000000..f4c565c5 --- /dev/null +++ b/crates/ui-egui/src/panels/essential.rs @@ -0,0 +1,427 @@ +//! Essential Graphics panel (Window ▸ Essential Graphics). +//! +//! - **Primary** composition, **Solo Supported Properties** (the timeline lists only what can +//! be exposed) and the template name. +//! - The controls: drag a property name from the timeline onto the list (or Animation ▸ Add +//! Property to Essential Graphics); each shows as a text / colour / slider / checkbox / point / +//! angle / dropdown control editing the source property; rename by double-clicking the name; +//! reorder with ▲▼, remove with ✕; groups and comments from Add Formatting; Media +//! Replacement for the selected footage layer. +//! - **Export Template…** writes an open `.ectemplate`. +//! - With a precomp layer that has Essential Properties selected, its instance values: edit them +//! (overrides), Revert or Push to Comp. +//! +//! Every control runs an engine `essential.*` / `prop.set` command and registers an automation +//! id (`essential.primary`, `essential.solo`, `essential.name`, `essential.control..value`, +//! `essential.instance..value`, `essential.export`…). + +use effectcraft_engine::keyframe::Value as KV; +use effectcraft_engine::project::essential::{self, ControlType, EgControl, EgKind}; +use effectcraft_engine::project::{ItemId, ItemKind, LayerSource, ParamUi, Property}; +use egui::{Rect, RichText}; +use serde_json::{Value, json}; + +use crate::EffectcraftApp; +use crate::panels::DragPayload; + +type Actions = Vec<(String, Value)>; + +/// The comp the panel edits: the Primary comp, else the active one. +fn primary(app: &EffectcraftApp) -> Option { + let s = &app.session; + s.state.essential_primary.filter(|c| s.project.comp(*c).is_some()).or(s.active_comp_id()) +} + +pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let ctx = ui.ctx().clone(); + let mut child = ui.new_child(egui::UiBuilder::new().max_rect(rect.shrink(8.0)).layout(egui::Layout::top_down(egui::Align::Min))); + let mut out = None; + egui::ScrollArea::vertical().id_salt("eg-panel").auto_shrink([false, false]).show(&mut child, |ui| out = Some(body(app, ui))); + let Some((actions, invoke)) = out else { return }; + for (id, p) in actions { + if let Err(e) = app.session.execute(&id, p) { + app.ui.status = e.to_string(); + } + } + for (id, p) in invoke { + if let Err(e) = crate::menus::invoke(app, &ctx, &id, p) { + app.ui.status = e; + } + } +} + +fn body(app: &mut EffectcraftApp, ui: &mut egui::Ui) -> (Actions, Actions) { + let t = app.tokens; + let ctx = ui.ctx().clone(); + let mut actions: Actions = vec![]; + let mut invoke: Vec<(String, Value)> = vec![]; + let Some(cid) = primary(app) else { + ui.label(RichText::new("Open a composition to build Essential Graphics").color(t.text_faint)); + return (actions, invoke); + }; + let project = app.session.project.clone(); + let comp_name = project.item(cid).map(|i| i.name.clone()).unwrap_or_default(); + + // ---- Primary + Solo. + ui.horizontal(|ui| { + ui.label(RichText::new("Primary:").color(t.text_dim)); + let r = egui::ComboBox::from_id_salt("eg-primary").selected_text(&comp_name).width(150.0).show_ui(ui, |ui| { + for (id, _) in project.comps() { + let name = project.item(*id).map(|i| i.name.clone()).unwrap_or_default(); + if ui.selectable_label(*id == cid, name).clicked() { + actions.push(("essential.setPrimary".into(), json!({"comp": id.0}))); + } + } + }); + app.auto.add("essential.primary", r.response.rect, &comp_name); + }); + let solo = app.session.state.essential_solo; + let r = ui.add(egui::Button::new("Solo Supported Properties").selected(solo)); + app.auto.add("essential.solo", r.rect, "Solo Supported Properties"); + if r.clicked() { + actions.push(("essential.soloSupported".into(), json!({"on": !solo}))); + } + // ---- Name. + let eg = project.comp(cid).and_then(|c| c.essential.clone()).unwrap_or_default(); + let shown_name = if eg.name.is_empty() { comp_name.clone() } else { eg.name.clone() }; + let name_id = egui::Id::new("eg-name-buf"); + let mut name: String = ctx.data(|d| d.get_temp(name_id)).unwrap_or_else(|| shown_name.clone()); + let r = ui.add(egui::TextEdit::singleline(&mut name).hint_text("Template name").desired_width(f32::INFINITY)); + app.auto.add("essential.name", r.rect, &shown_name); + if r.has_focus() { + ctx.data_mut(|d| d.insert_temp(name_id, name.clone())); + } else { + if r.lost_focus() && name.trim() != shown_name { + actions.push(("essential.setName".into(), json!({"comp": cid.0, "name": name.trim()}))); + } + ctx.data_mut(|d| d.remove::(name_id)); + } + ui.separator(); + + // ---- Controls (drop target for timeline properties). + let list = ui.scope(|ui| { + ui.set_min_height(60.0); + if eg.controls.is_empty() { + ui.label(RichText::new("Drag properties here from the timeline,\nor use Animation > Add Property to Essential Graphics.").color(t.text_faint)); + } + controls(app, ui, cid, &eg.controls, None, &mut actions); + }); + let drop_rect = list.response.rect.expand2(egui::vec2(0.0, 4.0)); + app.auto.add("essential.dropZone", drop_rect, "Drag properties here"); + if let Some(payload) = egui::DragAndDrop::payload::(&ctx) + && let DragPayload::Property { layer, prop } = *payload + && ui.rect_contains_pointer(drop_rect) + { + ui.painter().rect_stroke(drop_rect, 2.0, egui::Stroke::new(2.0, t.accent), egui::StrokeKind::Inside); + if ctx.input(|i| i.pointer.any_released()) { + let comp = app.session.active_comp_id().map(|c| c.0); + actions.push(("essential.addProperty".into(), json!({"comp": comp, "layer": layer, "prop": prop}))); + egui::DragAndDrop::clear_payload(&ctx); + } + } + ui.separator(); + + // ---- Bottom buttons. + ui.horizontal_wrapped(|ui| { + let r = ui.menu_button("Add Formatting", |ui| { + if ui.button("Add Group").clicked() { + actions.push(("essential.addGroup".into(), json!({"comp": cid.0}))); + ui.close(); + } + if ui.button("Add Comment").clicked() { + actions.push(("essential.addComment".into(), json!({"comp": cid.0}))); + ui.close(); + } + }); + app.auto.add("essential.addFormatting", r.response.rect, "Add Formatting"); + let footage_layer = app.session.active_comp().and_then(|c| { + app.session.state.selected_layers.iter().filter_map(|l| c.layer(*l)).find(|l| matches!(l.source, LayerSource::Footage { .. })).map(|l| l.id.0) + }); + let r = ui.add_enabled(footage_layer.is_some(), egui::Button::new("Media Replacement")); + app.auto.add("essential.addMedia", r.rect, "Add Media Replacement"); + if r.clicked() + && let Some(l) = footage_layer + { + actions.push(("essential.addMedia".into(), json!({"layer": l}))); + } + let r = ui.add_enabled(!eg.controls.is_empty(), egui::Button::new("Export Template…")); + app.auto.add("essential.export", r.rect, "Export Template"); + if r.clicked() { + invoke.push(("essential.exportTemplate".into(), json!({"comp": cid.0}))); + } + }); + + // ---- The selected instance's Essential Properties. + instance(app, ui, &mut actions); + (actions, invoke) +} + +/// One list of controls (top level or a group's children). +fn controls(app: &mut EffectcraftApp, ui: &mut egui::Ui, cid: ItemId, list: &[EgControl], group: Option, actions: &mut Actions) { + let t = app.tokens; + let n = list.len(); + for (i, c) in list.iter().enumerate() { + let row = ui.horizontal(|ui| { + // Reorder / remove. + for (label, auto, enabled, to) in [("▲", "up", i > 0, i.saturating_sub(1)), ("▼", "down", i + 1 < n, i + 1)] { + let r = ui.add_enabled(enabled, egui::Button::new(RichText::new(label).size(9.0)).small().frame(false)); + app.auto.add(&format!("essential.control.{}.{auto}", c.id), r.rect, label); + if r.clicked() { + actions.push(("essential.move".into(), json!({"comp": cid.0, "control": c.id, "group": group, "index": to}))); + } + } + match &c.kind { + EgKind::Group { .. } => { + name_label(app, ui, cid, c, RichText::new(&c.name).strong(), actions); + } + EgKind::Comment { text } => { + let shown = EgControl { name: text.clone(), ..c.clone() }; + name_label(app, ui, cid, &shown, RichText::new(text).italics().color(t.text_dim), actions); + } + EgKind::Property { .. } => { + name_label(app, ui, cid, c, RichText::new(&c.name), actions); + let src = essential::source_prop(&app.session.project, cid, c).map(|(l, p)| (l.id.0, l.layer_time(app.session.time()), p.clone())); + match src { + Some((layer, lt, p)) => { + let v = p.value_at(lt); + if let Some(nv) = value_widget(app, ui, &format!("essential.control.{}.value", c.id), &p, &v) { + actions.push(( + "prop.set".into(), + json!({"comp": cid.0, "layer": layer, "prop": p.uid, "value": nv, "merge": format!("eg-{}", c.id)}), + )); + } + } + None => { + ui.label(RichText::new("(missing)").color(t.danger)); + } + } + } + EgKind::Media { .. } => { + name_label(app, ui, cid, c, RichText::new(&c.name), actions); + let item = essential::source_media(&app.session.project, cid, c); + let name = item.and_then(|i| app.session.project.item(i)).map(|i| i.name.clone()).unwrap_or_default(); + ui.label(RichText::new(format!("▣ {name}")).color(t.text_dim)); + } + } + let r = ui.add(egui::Button::new(RichText::new("×").size(10.0)).small().frame(false)).on_hover_text("Remove"); + app.auto.add(&format!("essential.control.{}.remove", c.id), r.rect, "Remove"); + if r.clicked() { + actions.push(("essential.remove".into(), json!({"comp": cid.0, "control": c.id}))); + } + }); + app.auto.add(&format!("essential.control.{}.row", c.id), row.response.rect, &c.name); + if let EgKind::Group { children } = &c.kind { + ui.indent(("eg-group", c.id), |ui| controls(app, ui, cid, children, Some(c.id), actions)); + } + } +} + +/// A control's name; double-click to rename (comments: edit the text). +fn name_label(app: &mut EffectcraftApp, ui: &mut egui::Ui, cid: ItemId, c: &EgControl, text: RichText, actions: &mut Actions) { + let edit_id = egui::Id::new(("eg-rename", c.id)); + let editing: Option = ui.ctx().data(|d| d.get_temp(edit_id)); + match editing { + Some(mut buf) => { + let r = ui.add(egui::TextEdit::singleline(&mut buf).desired_width(120.0)); + app.auto.add(&format!("essential.control.{}.rename", c.id), r.rect, &c.name); + r.request_focus(); + if r.lost_focus() || ui.input(|i| i.key_pressed(egui::Key::Enter)) { + if !buf.trim().is_empty() && buf.trim() != c.name { + actions.push(("essential.rename".into(), json!({"comp": cid.0, "control": c.id, "name": buf.trim()}))); + } + ui.ctx().data_mut(|d| d.remove::(edit_id)); + } else { + ui.ctx().data_mut(|d| d.insert_temp(edit_id, buf)); + } + } + None => { + let r = ui.add(egui::Label::new(text).sense(egui::Sense::click()).truncate()); + app.auto.add(&format!("essential.control.{}.name", c.id), r.rect, &c.name); + if r.double_clicked() { + ui.ctx().data_mut(|d| d.insert_temp(edit_id, c.name.clone())); + } + } + } +} + +/// The editor for a control's value; returns the new value (JSON for `prop.set`). +fn value_widget(app: &mut EffectcraftApp, ui: &mut egui::Ui, auto: &str, p: &Property, v: &KV) -> Option { + let ty = essential::control_type(p)?; + let mut out = None; + let rect; + match ty { + ControlType::Text => { + let cur = match v { + KV::Text(d) => d.text.clone(), + KV::Str(s) => s.clone(), + _ => String::new(), + }; + let id = egui::Id::new(("eg-text", auto)); + let mut buf: String = ui.ctx().data(|d| d.get_temp(id)).unwrap_or_else(|| cur.clone()); + let r = ui.add(egui::TextEdit::singleline(&mut buf).desired_width(140.0)); + if r.has_focus() { + ui.ctx().data_mut(|d| d.insert_temp(id, buf)); + } else { + if r.lost_focus() && buf != cur { + out = Some(json!(buf)); + } + ui.ctx().data_mut(|d| d.remove::(id)); + } + rect = r.rect; + } + ControlType::Color => { + let c = v.as_color(); + let mut rgba = egui::Rgba::from_rgba_unmultiplied(c[0], c[1], c[2], c[3]); + let r = egui::widgets::color_picker::color_edit_button_rgba(ui, &mut rgba, egui::widgets::color_picker::Alpha::Opaque); + if r.changed() { + let u = rgba.to_rgba_unmultiplied(); + out = Some(json!([u[0], u[1], u[2], u[3]])); + } + rect = r.rect; + } + ControlType::Checkbox => { + let mut on = v.as_bool(); + let r = ui.checkbox(&mut on, ""); + if r.changed() { + out = Some(json!(on)); + } + rect = r.rect; + } + ControlType::Dropdown => { + let opts = match &p.ui { + ParamUi::Popup { options } => options.clone(), + _ => vec![], + }; + let cur = v.as_enum() as usize; + let r = egui::ComboBox::from_id_salt(("eg-dd", auto)).selected_text(opts.get(cur).cloned().unwrap_or_default()).show_ui(ui, |ui| { + for (i, o) in opts.iter().enumerate() { + if ui.selectable_label(i == cur, o).clicked() { + out = Some(json!(i)); + } + } + }); + rect = r.response.rect; + } + ControlType::Point => { + let mut a = v.as_vec2(); + let r1 = ui.add(egui::DragValue::new(&mut a[0]).speed(1.0).max_decimals(1)); + let r2 = ui.add(egui::DragValue::new(&mut a[1]).speed(1.0).max_decimals(1)); + if r1.changed() || r2.changed() { + out = Some(json!(a)); + } + rect = r1.rect.union(r2.rect); + } + ControlType::Angle => { + let mut a = v.as_f64(); + let r = ui.add(egui::DragValue::new(&mut a).speed(1.0).suffix("°").max_decimals(1)); + if r.changed() { + out = Some(json!(a)); + } + rect = r.rect; + } + ControlType::Slider | ControlType::Media => { + let mut x = v.as_f64(); + let r = match p.ui { + ParamUi::Slider { slider_min, slider_max, .. } => ui.add(egui::Slider::new(&mut x, slider_min..=slider_max).max_decimals(2)), + _ => ui.add(egui::DragValue::new(&mut x).speed(0.5).max_decimals(2)), + }; + if r.changed() { + out = Some(json!(x)); + } + rect = r.rect; + } + } + app.auto.add(auto, rect, &p.name); + out +} + +/// Essential Properties of the selected precomp layer: per-instance values, overrides marked ●, +/// Revert and Push to Comp. +fn instance(app: &mut EffectcraftApp, ui: &mut egui::Ui, actions: &mut Actions) { + let t = app.tokens; + let Some(comp) = app.session.active_comp().cloned() else { return }; + let Some(layer) = app.session.state.selected_layers.iter().filter_map(|l| comp.layer(*l)).find(|l| essential::group(l).is_some()).cloned() else { return }; + let LayerSource::Comp { item: src } = layer.source else { return }; + let Some(eg) = app.session.project.comp(src).and_then(|c| c.essential.clone()) else { return }; + let over = essential::overridden(&layer); + let lt = layer.layer_time(app.session.time()); + ui.separator(); + ui.label(RichText::new(format!("Essential Properties — {}", layer.name)).strong()); + let group = essential::group(&layer).cloned().unwrap_or_else(|| effectcraft_engine::project::PropGroup::new(0, "", "")); + for c in eg.flat() { + let m = essential::match_id(c.id); + let mut pr = None; + group.walk("", &mut |_, x| { + if x.match_id == m { + pr = Some(x.clone()); + } + }); + let Some(p) = pr else { continue }; + let is_over = over.contains(&p.uid); + ui.horizontal(|ui| { + ui.label(RichText::new(if is_over { "●" } else { "○" }).color(if is_over { t.accent } else { t.text_faint })).on_hover_text(if is_over { + "Overridden in this instance" + } else { + "Follows the source composition" + }); + ui.label(&c.name); + let auto = format!("essential.instance.{}.value", c.id); + match c.kind { + EgKind::Media { .. } => { + let cur = ItemId(p.value.as_f64() as u64); + let cur_name = app.session.project.item(cur).map(|i| i.name.clone()).unwrap_or_default(); + let items: Vec<(ItemId, String)> = app + .session + .project + .items + .values() + .filter(|i| matches!(&i.kind, ItemKind::Footage(f) if f.has_video)) + .map(|i| (i.id, i.name.clone())) + .collect(); + let r = egui::ComboBox::from_id_salt(("eg-media", layer.id.0, c.id)).selected_text(cur_name).show_ui(ui, |ui| { + for (id, name) in &items { + if ui.selectable_label(*id == cur, name).clicked() { + actions.push(("essential.set".into(), json!({"layer": layer.id.0, "control": c.id, "item": id.0}))); + } + } + }); + app.auto.add(&auto, r.response.rect, &c.name); + } + _ => { + let v = p.value_at(lt); + if let Some(nv) = value_widget(app, ui, &auto, &p, &v) { + actions.push(( + "essential.set".into(), + json!({"layer": layer.id.0, "control": c.id, "value": nv, "merge": format!("egi-{}-{}", layer.id.0, c.id)}), + )); + } + } + } + if is_over { + let r = ui.small_button("Revert"); + app.auto.add(&format!("essential.instance.{}.revert", c.id), r.rect, "Revert"); + if r.clicked() { + actions.push(("essential.revert".into(), json!({"layer": layer.id.0, "control": c.id}))); + } + let r = ui.small_button("Push"); + app.auto.add(&format!("essential.instance.{}.push", c.id), r.rect, "Push to Comp"); + if r.clicked() { + actions.push(("essential.pushToComp".into(), json!({"layer": layer.id.0, "control": c.id}))); + } + } + }); + } + ui.horizontal(|ui| { + let any = !over.is_empty(); + let r = ui.add_enabled(any, egui::Button::new("Push All to Comp")); + app.auto.add("essential.instance.pushAll", r.rect, "Push Override Values to Source"); + if r.clicked() { + actions.push(("essential.pushToComp".into(), json!({"layer": layer.id.0}))); + } + let r = ui.add_enabled(any, egui::Button::new("Revert All")); + app.auto.add("essential.instance.revertAll", r.rect, "Revert All"); + if r.clicked() { + actions.push(("essential.revert".into(), json!({"layer": layer.id.0}))); + } + }); +} diff --git a/crates/ui-egui/src/panels/expr_bar.rs b/crates/ui-egui/src/panels/expr_bar.rs new file mode 100644 index 00000000..09d4a608 --- /dev/null +++ b/crates/ui-egui/src/panels/expr_bar.rs @@ -0,0 +1,95 @@ +//! The expression error bar: a strip at the bottom of the Composition viewer listing failing +//! expressions of the comp at the current time (`expr.errors`), with ◀ ▶ to step through them. +//! Clicking the message selects the layer and reveals the property in the timeline. +//! Automation ids: `viewer.exprErrors`, `viewer.exprErrors.prev`, `viewer.exprErrors.next`. + +use effectcraft_engine::commands::expr_tools::{self, ExprError}; +use effectcraft_engine::project::ItemId; +use egui::{Align2, Color32, Rect, Sense, pos2, vec2}; +use serde_json::json; + +use crate::EffectcraftApp; +use crate::theme::Tokens; + +/// Height of the bar. +pub const HEIGHT: f32 = 22.0; + +#[derive(Clone, Default)] +struct Cache { + key: (u64, u64, i64), + errors: Vec, + /// Shown error (index). + current: usize, +} + +fn cache_id() -> egui::Id { + egui::Id::new("expr-error-bar") +} + +/// The comp's failing expressions (recomputed when the project or the time changes; kept while +/// playing). +fn errors(app: &EffectcraftApp, ctx: &egui::Context, cid: ItemId) -> Cache { + let t = app.session.time_of(cid); + let key = (app.session.revision, cid.0, t.0); + let mut c: Cache = ctx.data(|d| d.get_temp(cache_id())).unwrap_or_default(); + let stale = c.key != key && !(app.playback.playing && c.key.1 == cid.0); + if stale { + c.errors = expr_tools::errors(&app.session, cid, t); + c.key = key; + c.current = c.current.min(c.errors.len().saturating_sub(1)); + ctx.data_mut(|d| d.insert_temp(cache_id(), c.clone())); + } + c +} + +/// Height the bar needs for `cid` (0 when every expression evaluates). +pub fn height(app: &EffectcraftApp, ctx: &egui::Context, cid: ItemId) -> f32 { + if app.session.expr.is_none() || errors(app, ctx, cid).errors.is_empty() { 0.0 } else { HEIGHT } +} + +pub fn draw(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect, cid: ItemId) { + let t = app.tokens; + let ctx = ui.ctx().clone(); + let mut c = errors(app, &ctx, cid); + if c.errors.is_empty() { + return; + } + let p = ui.painter().with_clip_rect(rect); + p.rect_filled(rect, 0.0, Color32::from_rgb(0x4a, 0x3a, 0x12)); + let warn = Color32::from_rgb(0xf5, 0xb0, 0x30); + p.text(pos2(rect.min.x + 10.0, rect.center().y), Align2::LEFT_CENTER, "⚠", Tokens::ui(13.0), warn); + let n = c.errors.len(); + let i = c.current.min(n - 1); + // ◀ i/n ▶ on the right. + let next = Rect::from_center_size(pos2(rect.max.x - 14.0, rect.center().y), vec2(16.0, 16.0)); + let count = Rect::from_center_size(pos2(rect.max.x - 44.0, rect.center().y), vec2(40.0, 16.0)); + let prev = Rect::from_center_size(pos2(rect.max.x - 74.0, rect.center().y), vec2(16.0, 16.0)); + for (r, label, auto, d) in [(prev, "◀", "viewer.exprErrors.prev", n - 1), (next, "▶", "viewer.exprErrors.next", 1)] { + let resp = ui.interact(r, egui::Id::new(("expr-bar", auto)), Sense::click()); + p.text(r.center(), Align2::CENTER_CENTER, label, Tokens::ui(11.0), if resp.hovered() { Color32::WHITE } else { t.text }); + app.auto.add(auto, r, label); + if resp.clicked() { + c.current = (i + d) % n; + ctx.data_mut(|dd| dd.insert_temp(cache_id(), c.clone())); + } + } + p.text(count.center(), Align2::CENTER_CENTER, format!("{}/{n}", i + 1), Tokens::ui(11.0), t.text); + let comp_name = app.session.project.item(cid).map(|it| it.name.clone()).unwrap_or_default(); + let e = c.errors[i].clone(); + let msg_rect = Rect::from_min_max(pos2(rect.min.x + 26.0, rect.min.y), pos2(prev.min.x - 6.0, rect.max.y)); + let resp = ui.interact(msg_rect, egui::Id::new("expr-bar-msg"), Sense::click()).on_hover_text("Click to reveal the expression in the timeline"); + p.with_clip_rect(msg_rect).text( + pos2(msg_rect.min.x, rect.center().y), + Align2::LEFT_CENTER, + e.describe(&comp_name), + Tokens::ui(11.5), + if resp.hovered() { Color32::WHITE } else { t.text }, + ); + app.auto.add("viewer.exprErrors", msg_rect, &e.describe(&comp_name)); + if resp.clicked() { + let _ = app.session.execute("layer.select", json!({"comp": cid.0, "layers": [e.layer]})); + let _ = app.session.execute("prop.select", json!({"comp": cid.0, "layer": e.layer, "prop": e.prop, "selectKeys": false})); + let _ = crate::menus::frontend(app, &ctx, "timeline.revealProps", json!({"props": [{"layer": e.layer, "prop": e.prop}]})); + app.ui.timeline.expr_closed.remove(&e.prop); + } +} diff --git a/crates/ui-egui/src/panels/forms.rs b/crates/ui-egui/src/panels/forms.rs index 4b731cca..af117e8e 100644 --- a/crates/ui-egui/src/panels/forms.rs +++ b/crates/ui-egui/src/panels/forms.rs @@ -218,27 +218,48 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { ), ], ), - "file.interpretFootage" if !has(p, &["frameRate", "alpha", "loop", "pixelAspect", "colorProfile"]) => { - let f = s.state.project_selection.first().and_then(|i| s.project.item(*i)).and_then(|it| match &it.kind { - effectcraft_engine::project::ItemKind::Footage(f) => Some(f.clone()), + "file.interpretFootage" | "file.interpretProxy" + if !has(p, &["frameRate", "alpha", "loop", "pixelAspect", "colorProfile", "fields", "invertAlpha", "matteColor", "linearLight"]) => + { + let proxy = id == "file.interpretProxy"; + let f = s.state.project_selection.first().and_then(|i| s.project.item(*i)).and_then(|it| match (&it.kind, &it.proxy) { + (_, Some(px)) if proxy => Some(px.footage.clone()), + (effectcraft_engine::project::ItemKind::Footage(f), _) if !proxy => Some(f.clone()), _ => None, }); let Some(f) = f else { return false }; + let hex = |c: [f32; 3]| format!("#{:02x}{:02x}{:02x}", (c[0] * 255.0).round() as u8, (c[1] * 255.0).round() as u8, (c[2] * 255.0).round() as u8); ( - "Interpret Footage".into(), + if proxy { "Interpret Footage: Proxy".into() } else { "Interpret Footage".into() }, vec![ - Field::num("frameRate", "Frame rate", f.frame_rate.as_f64()), + // Main Options ▸ Alpha. Field::choice( "alpha", "Alpha", - &[("Straight", json!("straight")), ("Premultiplied", json!("premultiplied")), ("Ignore", json!("ignore"))], + &[ + ("Interpret Straight - Unmatted", json!("straight")), + ("Interpret Premultiplied - Matted With Color", json!("premultiplied")), + ("Ignore", json!("ignore")), + ("Guess", json!("guess")), + ], f.alpha as usize, ), + Field::text("matteColor", "Matte color (premultiplied)", &hex(f.premul_color)), + Field::bool("invertAlpha", "Invert Alpha", f.invert_alpha), + // Main Options ▸ Frame Rate, Fields and Pulldown, Other Options. + Field::num("frameRate", "Assume this frame rate", f.frame_rate.as_f64()), + Field::choice( + "fields", + "Separate Fields", + &[("Off", json!("off")), ("Upper Field First", json!("upper")), ("Lower Field First", json!("lower"))], + f.fields as usize, + ), + Field::num("pixelAspect", "Pixel Aspect Ratio", f.pixel_aspect), Field::num("loop", "Loop (times)", f.loop_count as f64), - Field::num("pixelAspect", "Pixel aspect ratio", f.pixel_aspect), + // Color. Field::choice( "colorProfile", - "Color profile", + "Assign Profile", &[ ("Embedded / sRGB", json!("auto")), ("sRGB IEC61966-2.1", json!("srgb")), @@ -248,6 +269,7 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { ], f.color_profile.map_or(0, |c| 1 + effectcraft_engine::project::ColorSpace::ALL.iter().position(|x| *x == c).unwrap_or(0)), ), + Field::bool("linearLight", "Interpret As Linear Light", f.linear_light), ], ) } diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 3f737502..df60beb3 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -6,6 +6,8 @@ pub mod dialogs; pub mod dialogs_3d; pub mod effect_controls; pub mod effects_presets; +pub mod essential; +pub mod expr_bar; pub mod forms; pub mod fx_widgets; pub mod graph; @@ -47,6 +49,8 @@ pub enum DragPayload { Item(u64), /// An effect id from Effects & Presets. Effect(String), + /// A property dragged from the timeline (to the Essential Graphics panel). + Property { layer: u64, prop: u64 }, } /// The current time formatted per project settings (timecode with `;` for drop-frame, or frames). @@ -90,6 +94,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: PanelKind, rect: Rec PanelKind::Layer => layer_panel::show(app, ui, rect), PanelKind::Paint => paint_panels::paint(app, ui, rect), PanelKind::Brushes => paint_panels::brushes(app, ui, rect), + PanelKind::EssentialGraphics => essential::show(app, ui, rect), other => misc::placeholder(app, ui, rect, other), } } diff --git a/crates/ui-egui/src/panels/project.rs b/crates/ui-egui/src/panels/project.rs index d13f3768..6482b555 100644 --- a/crates/ui-egui/src/panels/project.rs +++ b/crates/ui-egui/src/panels/project.rs @@ -322,6 +322,24 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { app.auto.add(&format!("project.item.{}.twirl", id.0), tw, &it.name); } icons::paint(&lp, Rect::from_center_size(pos2(x0 + 18.0, r.center().y), vec2(14.0, 14.0)), item_icon(&it), t.text_dim); + // Proxy indicator: filled = the proxy is used, hollow = set but off. Click to switch. + if let Some(px) = it.proxy.as_ref() { + let pr = Rect::from_center_size(pos2(x0 + 4.0, r.center().y), vec2(9.0, 9.0)); + if px.enabled { + lp.rect_filled(pr, 1.0, t.text); + } else { + lp.rect_stroke(pr, 1.0, Stroke::new(1.0, t.text), egui::StrokeKind::Inside); + } + let presp = ui.interact(pr.expand(2.0).intersect(list), egui::Id::new(("pproxy", id.0)), Sense::click()).on_hover_text(format!( + "Proxy: {} ({})", + px.footage.path, + if px.enabled { "in use" } else { "off" } + )); + app.auto.add(&format!("project.item.{}.proxy", id.0), pr, if px.enabled { "Proxy in use" } else { "Proxy off" }); + if presp.clicked() { + actions.push(("file.useProxy".into(), json!({"item": id.0}))); + } + } let name_clip = Rect::from_min_max(pos2(x0 + 30.0, r.min.y), pos2(r.min.x + name_w + 4.0, r.max.y)).intersect(list); let name_rect = Rect::from_min_max(pos2(x0 + 28.0, r.min.y + 2.0), pos2(r.min.x + name_w + 2.0, r.max.y - 2.0)); // Inline edits (rename / comment). diff --git a/crates/ui-egui/src/panels/render_queue.rs b/crates/ui-egui/src/panels/render_queue.rs index 154dd4c2..b29bcf18 100644 --- a/crates/ui-egui/src/panels/render_queue.rs +++ b/crates/ui-egui/src/panels/render_queue.rs @@ -77,6 +77,10 @@ fn settings_menu(it: &RenderQueueItem) -> Vec<(String, Value)> { v.push(("-".into(), Value::Null)); v.push((mark(s.motion_blur, "Motion Blur: On for Checked Layers"), json!({"motionBlur": !s.motion_blur}))); v.push((mark(s.skip_existing, "Skip Existing Files"), json!({"skipExisting": !s.skip_existing}))); + v.push(("-".into(), Value::Null)); + for (u, k) in effectcraft_engine::project::render_queue::ProxyUse::ALL.iter().zip(["current", "all", "comp", "none"]) { + v.push((mark(s.proxy_use == *u, &format!("Proxy Use: {}", u.label())), json!({"proxyUse": k}))); + } v } diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index cd3bf6f9..b05e523a 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -234,6 +234,19 @@ fn build_rows(app: &EffectcraftApp, comp: &Comp) -> Vec { continue; } rows.push(Row { layer: l.id, depth: 0, kind: RowKind::Layer }); + // Essential Graphics ▸ Solo Supported Properties: every property it can expose. + if app.session.state.essential_solo { + let mut v = vec![]; + for (i, c) in l.props.children.iter().enumerate() { + if c.match_id() == effectcraft_engine::project::essential::GROUP { + continue; + } + let g = PropGroup { children: vec![l.props.children[i].clone()], ..PropGroup::new(0, "", "") }; + collect_props(&g, &mut v, &|p| prop_visible(p, l) && effectcraft_engine::project::essential::control_type(p).is_some()); + } + rows.extend(v.into_iter().map(|uid| Row { layer: l.id, depth: 1, kind: RowKind::Prop { uid } })); + continue; + } if !tl.open_layers.contains(&l.id.0) { continue; } @@ -1143,6 +1156,40 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { lp.text(wr.center(), Align2::CENTER_CENTER, "⚠", Tokens::ui(12.0), Color32::from_rgb(0xf0, 0xa0, 0x30)); let _ = ui.interact(wr, egui::Id::new(("expr-err", uid)), Sense::hover()).on_hover_text(e); } + // ⓕ: the Expression Language menu inserts at the cursor. + let lang_r = Rect::from_center_size(pos2(cw.switches + 70.0, cy), vec2(16.0, 16.0)); + let lang = ui.interact(lang_r, egui::Id::new(("expr-lang", uid)), Sense::click()).on_hover_text("Expression Language Menu"); + lp.circle_stroke(lang_r.center(), 6.5, Stroke::new(1.0, if lang.hovered() { t.text } else { t.text_dim })); + lp.text(lang_r.center(), Align2::CENTER_CENTER, "f", Tokens::semibold(10.0), if lang.hovered() { t.text } else { t.text_dim }); + app.auto.add(&format!("timeline.prop.{uid}.exprLanguage"), lang_r, "Expression Language Menu"); + let mut picked: Option<&'static str> = None; + egui::Popup::menu(&lang).show(|ui| { + for (cat, items) in effectcraft_engine::commands::expr_tools::language_menu() { + ui.menu_button(cat, |ui| { + for (label, text) in items { + if ui.button(label).clicked() { + picked = Some(text); + ui.close(); + } + } + }); + } + }); + if let Some(text) = picked { + let edit_id = egui::Id::new(("expr-edit", uid)); + let buf_id = egui::Id::new(("expr-buf", uid)); + let cur: String = ctx.data(|d| d.get_temp(buf_id)).unwrap_or_else(|| ex.text.clone()); + let at = egui::text_edit::TextEditState::load(&ctx, edit_id) + .and_then(|st| st.cursor.char_range()) + .map(|r| r.primary.index.0) + .unwrap_or(cur.chars().count()) + .min(cur.chars().count()); + let byte = cur.char_indices().nth(at).map(|(b, _)| b).unwrap_or(cur.len()); + let mut next = cur.clone(); + next.insert_str(byte, text); + actions.push(("prop.setExpression".into(), json!({"layer": layer.id.0, "prop": uid, "expression": next}))); + ctx.data_mut(|d| d.remove::(buf_id)); + } // The editor in the time-graph area. ui.set_clip_rect(right_clip); gp.rect_filled(Rect::from_min_max(pos2(graph_x0, r.min.y), r.max), 0.0, Color32::from_rgb(0x1a, 0x1a, 0x1a)); @@ -1253,8 +1300,19 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { Tokens::ui(12.0), if prop.has_expression() { Color32::from_rgb(0xe8, 0x7c, 0x5c) } else { t.text }, ); - let name_resp = - ui.interact(Rect::from_min_max(pos2(name_x, r.min.y), pos2(cw.switches - 2.0, r.max.y)), egui::Id::new(("pname", uid)), Sense::click()); + let name_resp = ui.interact( + Rect::from_min_max(pos2(name_x, r.min.y), pos2(cw.switches - 2.0, r.max.y)), + egui::Id::new(("pname", uid)), + Sense::click_and_drag(), + ); + if name_resp.drag_started() { + // Drop on the Essential Graphics panel to expose the property. + egui::DragAndDrop::set_payload(&ctx, crate::panels::DragPayload::Property { layer: layer.id.0, prop: *uid }); + } + if name_resp.dragged() { + egui::Tooltip::always_open(ctx.clone(), ui.layer_id(), egui::Id::new("prop-drag-tip"), egui::PopupAnchor::Pointer) + .show(|ui| ui.label(&prop.name)); + } if name_resp.clicked() { actions.push(("prop.select".into(), json!({"layer": layer.id.0, "prop": uid, "add": ui.input(|i| i.modifiers.shift)}))); } @@ -1494,6 +1552,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { // Drop targets: footage/comps from the Project panel, effects from Effects & Presets. if let Some(payload) = egui::DragAndDrop::payload::(&ctx) + && !matches!(*payload, crate::panels::DragPayload::Property { .. }) && ui.rect_contains_pointer(rows_rect) { p.rect_stroke(rows_rect, 0.0, Stroke::new(2.0, t.accent), StrokeKind::Inside); @@ -1507,6 +1566,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { actions.push(("effect.apply".into(), json!({"effect": e, "layers": [l.0]}))); } } + crate::panels::DragPayload::Property { .. } => {} } egui::DragAndDrop::clear_payload(&ctx); } diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index 76ef92a9..227e916a 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -347,7 +347,10 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { // Bottom control bar. let bar_h = 30.0; let bar = Rect::from_min_max(pos2(rect.min.x, rect.max.y - bar_h), rect.max); - let full = Rect::from_min_max(pos2(rect.min.x, nav.max.y), pos2(rect.max.x, bar.min.y)); + // The expression error bar sits above the control bar while expressions fail. + let err_h = super::expr_bar::height(app, &ctx, cid); + let err_bar = Rect::from_min_max(pos2(rect.min.x, bar.min.y - err_h), pos2(rect.max.x, bar.min.y)); + let full = Rect::from_min_max(pos2(rect.min.x, nav.max.y), pos2(rect.max.x, err_bar.min.y)); let pasteboard = app.ui.viewer.pasteboard.map(|[r, g, b]| Color32::from_rgb(r, g, b)).unwrap_or(t.pasteboard); // View ▸ Switch View Layout: extra views (Top / Front / Right) beside the main view, which // keeps the overlays and the interaction. @@ -1158,6 +1161,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { vt::draw_snap(&ctx, &painter, &map, &ectx); vt::rulers(app, ui, &map, outer, area); vt::bottom_bar(app, ui, bar, zoom, fit, time, &comp); + if err_h > 0.0 { + super::expr_bar::draw(app, ui, err_bar, cid); + } } /// Parent-space position change per comp pixel of drag (x and y) for a layer: 2D layers map From b927964d983432bb8f817fb921e95bc81170602d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 23:32:13 -0400 Subject: [PATCH 10/13] CMP-7: document Essential Graphics, templates, responsive time, proxies and expression data Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 24 +++++++++++++++++++++++- docs/parity.md | 6 +++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 95bad818..a030e26c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -205,11 +205,33 @@ auto-scale; cached per analysis and settings) and warps each frame; Synthesize E borders from neighbouring frames read with `EffectHost::self_at`. The effect lives in the effects crate, hence the `effects → track` edge. +**Essential Graphics** (`project::essential`, `engine::commands::essential`): a comp's +`essential` field lists its exposed controls (properties, Media Replacement, groups, comments). +Every precomp layer of such a comp gets an Essential Properties group (`GroupKind::Essential`, +children `eg`), kept in step by `Session::edit`; editing a child overrides it for +that instance. The renderer draws an instance with overrides from a copy of the project with the +overridden source properties set (`essential::with_overrides`). Templates are `.ectemplate` +ZIPs: `manifest.json` (controls), `project.ecproj` (the comp and its dependencies), `media/…` and +`poster.png`; importing offsets every id past the project's (`essential::offset_ids`). +**Responsive Design — Time**: a time-stretched precomp maps time piecewise so its protected +marker regions play at 100 % (`eval::responsive_source_time`). + +**Proxies and interpretation**: an item's `proxy` (footage or comp) is decoded in its place at +the source's nominal size when `RenderOpts::proxy` (Render Settings ▸ Proxy Use) allows it. +Interpret Footage's Separate Fields turns each field into a frame at twice the rate (the other +lines interpolated); pixel aspect stretches footage, solids and precomps in the comp +(`EvalCtx::par_ratio`, not inherited by children); Invert Alpha and Interpret As Linear Light +apply after decoding. + ## 5. Expressions Expressions are JavaScript, run by boa, with After Effects' object model (`thisComp`, `thisLayer`, `time`, `value`, `wiggle`, `loopOut`, vector maths on arrays, and so on). A syntax error keeps the -text, disables the expression and shows a warning, like After Effects. +text, disables the expression and shows a warning, like After Effects. `sampleImage` renders the +sampled layer through the evaluating renderer's footage source (`EvalCtx::footage`), cached per +thread and frame; `footage(name)` reads data footage (JSON, CSV, TSV imported with File ▸ Import, +text kept in the project) through `sourceData`, `sourceText` and `dataValue`. `expr.errors` +lists failing expressions for the viewer's error bar. ## 6. Commands, the UI seam and automation diff --git a/docs/parity.md b/docs/parity.md index 419b407c..37b2d670 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -89,14 +89,14 @@ and OCIO, landed in M9.11.) | Import | 78% | 4.5 | PSD, SVG as shapes, Lottie | | Automation | 76% | 3.3 | a JavaScript scripting object model | | Shapes | 68% | 3.9 | taper and wave strokes (pen tool for shape paths and vertex editing landed in M6.5) | -| Compositions | 67% | 7.4 | marker dialog, flowchart, Essential Graphics | +| Compositions | ≈ 75% | 5.0 | marker dialog, flowchart (Essential Graphics with master properties, `.ectemplate` templates and Responsive Design — Time landed: CMP-7) | | Animation | 65% | 10.4 | puppet, Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8) | | Text | ≈ 86% | 1.3 | vertical type is basic (upright characters, no tate-chu-yoko), no OpenType feature panel, no extruded strokes (extruded, bevelled text landed in M7.6); per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API landed (M9.9–M9.10) | | Web | 60% | 2.5 | threads, browser storage, audio | | 3D | 72% | 12.0 | 3D camera tracker, multi-view layouts, Advanced 3D motion blur and blend modes, cameras/lights from models; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) landed in M7.4–M7.6 | | Effects | ≈ 75% | 5.0 | GPU versions of more effects and the 3D Camera Tracker (every other After Effects effect exists since M9.11 and M12.5) | | Interface | 55% | 10.4 | drag-to-dock and floating panels, preferences, native macOS menus (viewer rulers/snapping/channels/snapshots landed in M0.13) | -| Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves | +| Project | ≈ 60% | 6.5 | 8/16/32-bit pipeline and colour management, auto-save, folder moves (proxies and Interpret Footage fields / pixel aspect / alpha guess landed: PRJ-8, PRJ-3) | | Masks & roto | 52% | 10.0 | Roto Brush, variable-width mask feather points (mask tracking and Mask Interpolation landed in M6.6) | | Preview | 55% | 6.1 | GPU 3D and adjustment layers, disk cache (region of interest, snapshots, exposure and Fast Previews landed in M0.13) | | Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking and Subspace Warp's mesh warp (Rolling Shutter Repair landed in M9.11; point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) | @@ -136,7 +136,7 @@ Mocha and Cineware are third-party and not counted. 9. A real 8/16/32-bit pipeline with linear blending and colour management. 10. Drag-to-dock and floating panels, saved workspaces, native macOS menus. 11. Auto-save, crash recovery, recent projects (in progress). -12. Expression gaps: `sampleImage`, `footage()` (the `sourceText` style API landed in M9.9). +12. ~~Expression gaps: `sampleImage`, `footage()`~~ (landed with data footage, the error bar and the Expression Language menu; the `sourceText` style API landed in M9.9). 13. Lottie (in progress), WebM, SVG and PSD import. 14. Puppet and paint tools (in progress). 15. Preferences and a shortcut editor that can rebind (in progress); real Wiggler, Smoother and Motion Sketch; the marker dialog. From 5df98d0982ac963b508631cc0b6a1f56c00a4403 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 23:41:28 -0400 Subject: [PATCH 11/13] =?UTF-8?q?M0.11:=20parity=20audit=20at=20fa26ad9=20?= =?UTF-8?q?(=E2=89=8893%=20weighted;=20298/298=20effects;=20=E2=89=8811?= =?UTF-8?q?=E2=80=9314=20h=20wall=20clock=20left)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 13 ++++++------- docs/parity.md | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 2bf7d8ba..724c9d70 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -25,13 +25,12 @@ once. ## How far from full parity -Measured feature by feature in [docs/parity.md](docs/parity.md) (end of 2 October 2026): +Measured feature by feature in [docs/parity.md](docs/parity.md) (late 2 October 2026): -- **≈ 79% of After Effects' features, weighted by importance** (87% of the essentials, 67% of - professional daily-use features, 21% of the long tail); 298 of 298 effects, 16 of them on the GPU. -- **≈ 100 agent-hours of work remain by the audit's estimate.** With five Claude Opus 5.5 agents in - parallel that is **≈ 8–10 hours of wall-clock time at the pace measured so far, up to ≈ 25 hours** - by the audit's conservative per-feature figures. -- Hardest remaining pieces: Advanced 3D, a scripting object model (Roto Brush & Refine Edge landed in M6.7, the 3D Camera Tracker in M12.6). +- **≈ 93% of After Effects' features, weighted by importance** (96% of the essentials, 90% of + professional daily-use features, 65% of the long tail); all 298 After Effects effects. +- **≈ 50–55 agent-hours of work remain**: about **11–14 hours of wall-clock time** with five Claude + Opus 5.5 agents in parallel, ≈ 6–8 hours for the essentials and daily-use features. +- Hardest remaining pieces: the web app's depth, GPU 3D, Content-Aware Fill, a plug-in API. Come tell us what matters most to you on [Discord](https://discord.gg/artcraft). diff --git a/docs/parity.md b/docs/parity.md index 0f3103ac..32cb65b7 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -1,9 +1,36 @@ # Parity with After Effects How close EffectCraft is to After Effects 2026, feature by feature, and how much work is left. -Audited against the code at commit `5273f1b` (2 October 2026). Every feature in our catalog was -graded from the code itself: a disabled menu entry, or a setting that is stored but never -rendered, does not count as done. + +## Current status (audit at commit `fa26ad9`, 2 October 2026, late) + +| Measure | Value | +|---|---| +| **Feature parity, weighted by tier** (P0 ×3, P1 ×2, P2 ×1) | **≈ 93%** | +| Unweighted | ≈ 91% | +| P0 / P1 / P2 | ≈ 96% / 90% / 65% | +| Features done / partial / missing | 41 / 50 / 1 of 92 (missing: a plug-in API) | +| **Effects** | **298 of 298** After Effects 2026 effects exist (some still simplified) | +| Disabled menu entries left | 26 commands (31 menu items) | +| Remaining work | ≈ 50–55 agent-hours at the pace measured so far (≈ 150 by the original conservative audit scale) | +| **Wall-clock estimate** | **≈ 11–14 hours** with five agents in parallel; ≈ 6–8 hours for 100% of P0 + P1 | + +By area: Layers 98%, Output 95%, Compositions 95%, Automation 96%, Paint 95%, Text 95%, Import 96%, +Animation 94%, Masks 94%, Preview 94%, Interface 93%, Shapes 93%, 3D 91%, Audio 90%, Project 90%, +Tracking 88%, Effects 85%, Web 65%. + +What is left, in priority order: the web app's depth (threads, storage, audio, non-blocking +renders); stroke taper/wave and multi-segment dashes; variable mask feather points; real +`sampleImage` and `footage()` in expressions; camera iris/bokeh and focus-link commands; GPU +rendering of 3D runs and adjustment layers; Render Queue field render, crop/resize and templates; +approximated effects (Subspace Warp, Key Cleaner) and Liquify's viewer brush; Advanced 3D motion blur +and blend modes; a few viewer and colour-management menu items; OpenType features and variable +font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; proxies; ScriptUI; +Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; face tracking; and +the "better than After Effects" items (a plug-in API, branching history, GPU particles). + +The sections below are the original audit (morning of 2 October, ≈ 64%) and its updates, kept for +history. ## Summary From 0bee1ab2daa1fd40f2a031a4d39e97f2d33127a8 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 23:42:00 -0400 Subject: [PATCH 12/13] M15.2: web app depth (WEB-1): browser storage, Web Audio, worker renders, WebGPU viewer, PWA - Persistence: settings, shortcuts, recent projects, imported media, saved projects and auto-saves live in OPFS (IndexedDB fallback) through an in-memory mirror flushed in the background (apps/effectcraft-web/src/store.rs, persist.rs); ConfigStore gains FileOps so auto-save / recovery work in the browser store; the session snapshot restores the last project + editor state on reload; File > Open Recent works from storage. - Audio: Web Audio output (AudioWorklet, ScriptProcessor fallback) fed by AudioPlayback::pump on wasm32; A/V sync from AudioContext.currentTime; meters; unlock on first gesture. - Background jobs: engine::offload (WorkerRequest/WorkerReply protocol, Offload trait, run_request); Render Queue, tracker, mask tracker, Warp Stabilizer and Roto Freeze run in Web Workers (a second engine instance on the shared compiled module), progress streamed back; cancel terminates the worker. - GPU: WebGPU viewer frames drawn straight from the compositor's texture (gpu_display on wasm32 when the viewer draws plainly); async readback (Gpu::read_display_async) for pixels. - PWA: manifest, service worker precaching the build (xtask fills list/version), icons copied from assets/app-icon; COOP/COEP kept from cached responses; offline reload works. - window.effectcraft: saveToBrowser, listStored, removeStored, flush; info() reports storage, audio, gpu, workers. - Tests: native store + protocol + offload (render, track, cancel) tests; smoke.mjs extended (GPU path, worker render responsiveness, audio gesture + playback, reload persistence, Open Recent, offline reload). Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 2 + ROADMAP.md | 2 +- apps/effectcraft-web/Cargo.toml | 15 +- apps/effectcraft-web/js/host.js | 335 ++++++++ apps/effectcraft-web/src/api.rs | 59 ++ apps/effectcraft-web/src/app.rs | 223 ++++++ apps/effectcraft-web/src/audio.rs | 109 +++ apps/effectcraft-web/src/files.rs | 63 +- apps/effectcraft-web/src/lib.rs | 182 +---- apps/effectcraft-web/src/persist.rs | 190 +++++ apps/effectcraft-web/src/store.rs | 399 ++++++++++ apps/effectcraft-web/src/worker.rs | 186 +++++ apps/effectcraft-web/tests/smoke.mjs | 160 +++- apps/effectcraft-web/web/audio-worklet.js | 44 ++ apps/effectcraft-web/web/index.html | 32 +- apps/effectcraft-web/web/manifest.webmanifest | 17 + apps/effectcraft-web/web/sw.js | 45 ++ apps/effectcraft-web/web/worker.js | 19 + crates/engine/src/autosave.rs | 48 +- crates/engine/src/commands/file.rs | 2 +- crates/engine/src/commands/settings.rs | 2 +- crates/engine/src/config.rs | 53 +- crates/engine/src/lib.rs | 8 + crates/engine/src/mask_track.rs | 12 +- crates/engine/src/offload.rs | 741 ++++++++++++++++++ crates/engine/src/render_queue.rs | 94 ++- crates/engine/src/roto.rs | 33 +- crates/engine/src/rq_tests.rs | 4 +- crates/engine/src/session_settings.rs | 10 +- crates/engine/src/tracking.rs | 15 +- crates/engine/src/warp.rs | 23 +- crates/gpu/src/context.rs | 35 + crates/gpu/src/lib.rs | 6 + crates/ui-egui/src/audio.rs | 39 +- crates/ui-egui/src/frames.rs | 13 +- crates/ui-egui/src/lib.rs | 61 +- crates/ui-egui/src/panels/viewer_tools.rs | 40 +- docs/architecture.md | 13 +- docs/parity.md | 2 +- docs/web.md | 167 +++- xtask/src/main.rs | 46 +- 41 files changed, 3230 insertions(+), 319 deletions(-) create mode 100644 apps/effectcraft-web/js/host.js create mode 100644 apps/effectcraft-web/src/app.rs create mode 100644 apps/effectcraft-web/src/audio.rs create mode 100644 apps/effectcraft-web/src/persist.rs create mode 100644 apps/effectcraft-web/src/store.rs create mode 100644 apps/effectcraft-web/src/worker.rs create mode 100644 apps/effectcraft-web/web/audio-worklet.js create mode 100644 apps/effectcraft-web/web/manifest.webmanifest create mode 100644 apps/effectcraft-web/web/sw.js create mode 100644 apps/effectcraft-web/web/worker.js create mode 100644 crates/engine/src/offload.rs diff --git a/Cargo.lock b/Cargo.lock index 0d3ae06b..019ecb4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1511,10 +1511,12 @@ dependencies = [ "egui", "js-sys", "log", + "serde", "serde_json", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "web-time", ] [[package]] diff --git a/ROADMAP.md b/ROADMAP.md index 55b05822..f50e2146 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -20,7 +20,7 @@ once. | M11 | Animation tools: audio playback, meters and waveforms, Lottie import and export (done); presets, Motion Sketch, Wiggler | In progress | | M12 | Performance: layer cache and parallel compositing (done), GPU compositing, disk cache, motion tracking | In progress | | M13+ | Puppet, paint, roto, motion tracking, a plugin API | Planned | -| M15 | The web app (WebAssembly, WebGPU) | Done, single-threaded | +| M15 | The web app (WebAssembly, WebGPU) | Done: browser storage, Web Audio, renders and analyses in Web Workers, offline install ([docs/web.md](docs/web.md)) | | M14 | Built for agents: MCP server, command-line tool, control channel; Settings, keyboard shortcut editor, auto-save and crash recovery ([docs/preferences.md](docs/preferences.md)) | Done | ## How far from full parity diff --git a/apps/effectcraft-web/Cargo.toml b/apps/effectcraft-web/Cargo.toml index d3f8aebb..03663089 100644 --- a/apps/effectcraft-web/Cargo.toml +++ b/apps/effectcraft-web/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-web" -description = "EffectCraft in the browser (wasm32): eframe web runner, in-memory files, browser file pickers and downloads, window.effectcraft API" +description = "EffectCraft in the browser (wasm32): eframe web runner, persistent browser storage (OPFS / IndexedDB), Web Audio, Web Worker renders, PWA, window.effectcraft API" version.workspace = true edition.workspace = true license.workspace = true @@ -12,10 +12,14 @@ workspace = true [lib] crate-type = ["cdylib", "rlib"] -# Everything is wasm32-only: on other targets this crate is empty (so `cargo test --workspace` -# and clippy stay cheap). Build with `cargo xtask web` (see docs/web.md). -[target.'cfg(target_arch = "wasm32")'.dependencies] +# The browser glue is wasm32-only; on other targets the crate holds just the portable storage +# model (`store`), unit-tested natively. Build the app with `cargo xtask web` (see docs/web.md). +[dependencies] effectcraft-engine = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } + +[target.'cfg(target_arch = "wasm32")'.dependencies] effectcraft-host = { workspace = true } effectcraft-media = { workspace = true } effectcraft-export = { workspace = true } @@ -23,8 +27,8 @@ effectcraft-expr = { workspace = true } effectcraft-ui-egui = { workspace = true } egui = { workspace = true } eframe = { workspace = true, features = ["default_fonts", "wgpu"] } -serde_json = { workspace = true } log = { workspace = true } +web-time = { workspace = true } # The wasm-bindgen CLI used by `cargo xtask web` must be exactly this version. wasm-bindgen = "=0.2.129" wasm-bindgen-futures = "0.4" @@ -35,6 +39,7 @@ web-sys = { version = "0.3", features = [ "CssStyleDeclaration", "Document", "Element", + "DedicatedWorkerGlobalScope", "Event", "EventTarget", "File", diff --git a/apps/effectcraft-web/js/host.js b/apps/effectcraft-web/js/host.js new file mode 100644 index 00000000..7b6200eb --- /dev/null +++ b/apps/effectcraft-web/js/host.js @@ -0,0 +1,335 @@ +// Browser services for the EffectCraft web app (imported by the wasm module through +// wasm-bindgen; see src/persist.rs, src/audio.rs and src/worker.rs): +// +// - storage: the Origin Private File System, or IndexedDB where OPFS can't write; +// - audio: a Web Audio output (AudioWorklet, ScriptProcessorNode fallback) fed stereo blocks; +// - workers: a pool of Web Workers, each running its own instance of the engine. +// +// Plain functions only: the Rust side owns the logic. + +// ------------------------------------------------------------------ storage + +const DIR = "effectcraft"; +const DB = "effectcraft"; +const STORE = "entries"; +let backend = null; + +// OPFS file names can't hold "/": keys are percent-encoded. +const enc = (k) => encodeURIComponent(k); +const dec = (n) => decodeURIComponent(n); + +async function opfs() { + if (!navigator.storage || !navigator.storage.getDirectory) throw new Error("no OPFS"); + const root = await navigator.storage.getDirectory(); + const dir = await root.getDirectoryHandle(DIR, { create: true }); + // Probe a write: some browsers expose OPFS but only write it from workers. + const probe = await dir.getFileHandle(".probe", { create: true }); + if (!probe.createWritable) throw new Error("OPFS is read-only on this thread"); + const w = await probe.createWritable(); + await w.write(new Uint8Array([1])); + await w.close(); + await dir.removeEntry(".probe"); + return { + name: "opfs", + async loadAll() { + const out = []; + for await (const [name, h] of dir.entries()) { + if (h.kind !== "file" || name.startsWith(".") || name.endsWith(".crswap")) continue; + const f = await h.getFile(); + out.push({ key: dec(name), data: new Uint8Array(await f.arrayBuffer()), modified: f.lastModified }); + } + return out; + }, + async put(key, data) { + // createWritable writes a swap file and replaces the entry on close: atomic. + const h = await dir.getFileHandle(enc(key), { create: true }); + const w = await h.createWritable(); + await w.write(data); + await w.close(); + }, + async del(key) { + try { await dir.removeEntry(enc(key)); } catch (e) { if (e.name !== "NotFoundError") throw e; } + }, + }; +} + +function req(r) { + return new Promise((res, rej) => { r.onsuccess = () => res(r.result); r.onerror = () => rej(r.error); }); +} + +async function idb() { + if (!self.indexedDB) throw new Error("no IndexedDB"); + const open = indexedDB.open(DB, 1); + open.onupgradeneeded = () => open.result.createObjectStore(STORE, { keyPath: "key" }); + const db = await req(open); + const tx = (mode) => db.transaction(STORE, mode).objectStore(STORE); + return { + name: "indexeddb", + async loadAll() { + const all = await req(tx("readonly").getAll()); + return all.map((r) => ({ key: r.key, data: new Uint8Array(r.data), modified: r.modified })); + }, + async put(key, data, modified) { + await req(tx("readwrite").put({ key, data: data.slice().buffer, modified })); + }, + async del(key) { + await req(tx("readwrite").delete(key)); + }, + }; +} + +function memory() { + const m = new Map(); + return { + name: "memory", + async loadAll() { return [...m.values()]; }, + async put(key, data, modified) { m.set(key, { key, data: data.slice(), modified }); }, + async del(key) { m.delete(key); }, + }; +} + +/// Open the store: OPFS, else IndexedDB, else memory (nothing persists). Resolves with the +/// backend's name. `prefer` ("opfs" | "indexeddb" | "memory") forces one (tests). +export async function storeOpen(prefer) { + const order = { opfs: [opfs, idb, memory], indexeddb: [idb, memory], memory: [memory] }[prefer] ?? [opfs, idb, memory]; + for (const make of order) { + try { backend = await make(); break; } catch (e) { console.info(`storage: ${make.name}: ${e.message ?? e}`); } + } + // Ask the browser not to evict the store under storage pressure (granted for installed apps). + try { if (navigator.storage && navigator.storage.persist) await navigator.storage.persist(); } catch {} + return backend.name; +} + +/// Every stored entry: [{key, data: Uint8Array, modified}]. +export async function storeLoadAll() { return backend.loadAll(); } +export async function storePut(key, data, modified) { return backend.put(key, data, modified); } +export async function storeDelete(key) { return backend.del(key); } + +/// {usage, quota, persisted, backend}. +export async function storeEstimate() { + const out = { backend: backend ? backend.name : null }; + try { + const e = await navigator.storage.estimate(); + out.usage = e.usage; out.quota = e.quota; + out.persisted = await navigator.storage.persisted(); + } catch {} + return out; +} + +// ------------------------------------------------------------------ audio + +// One AudioContext for the page. Browsers start it suspended until the user interacts with the +// page, so the first pointer/key event resumes it ("unlock"). +let ctx = null; +let node = null; +let kind = null; +// Sample frames handed to the output since `audioStart`, and the context time it started at. +let posted = 0; +let startTime = 0; +let underruns = 0; +// ScriptProcessor fallback: the queue lives on this thread. +let spQueue = []; +let spOffset = 0; +// Blocks pushed while the output node is being created. +let starting = false; +let pendingPush = []; +// Bumped by every start/stop: a start superseded while it awaited the worklet gives up. +let generation = 0; + +function context() { + if (!ctx) { + const AC = self.AudioContext || self.webkitAudioContext; + if (!AC) return null; + ctx = new AC({ latencyHint: "interactive" }); + } + return ctx; +} + +function unlock() { + const c = context(); + if (c && c.state !== "running") c.resume().catch(() => {}); +} + +/// Install the user-gesture unlock (called once at startup). +export function audioInstallUnlock() { + for (const ev of ["pointerdown", "keydown", "touchend"]) { + self.addEventListener(ev, unlock, { capture: true, passive: true }); + } +} + +/// {state, sampleRate, backend, posted, played, underruns, baseLatency, outputLatency}. +export function audioState() { + const c = ctx; + return { + state: c ? c.state : "none", + sampleRate: c ? c.sampleRate : 0, + backend: kind, + posted, + played: audioPlayed(), + underruns, + baseLatency: c ? c.baseLatency || 0 : 0, + outputLatency: c ? c.outputLatency || 0 : 0, + }; +} + +/// The output's sample rate (creates the context; 0 without Web Audio). +export function audioSampleRate() { + const c = context(); + return c ? c.sampleRate : 0; +} + +/// Start an output node; resolves when it is ready. Samples pushed before then queue up. +export async function audioStart(workletUrl) { + audioStop(); + const c = context(); + if (!c) throw new Error("Web Audio is not available"); + unlock(); + posted = 0; + underruns = 0; + spQueue = []; + spOffset = 0; + starting = true; + pendingPush = []; + const gen = ++generation; + if (c.audioWorklet && workletUrl) { + try { + if (!c.__ecWorklet) { + await c.audioWorklet.addModule(workletUrl); + c.__ecWorklet = true; + } + if (gen !== generation) return; + node = new AudioWorkletNode(c, "effectcraft-output", { numberOfInputs: 0, outputChannelCount: [2] }); + node.port.onmessage = (e) => { if (e.data && e.data.underrun) underruns += e.data.underrun; }; + kind = "worklet"; + } catch (e) { + console.warn("audio: AudioWorklet unavailable, using ScriptProcessorNode", e); + node = null; + } + } + if (!node) { + node = c.createScriptProcessor(2048, 0, 2); + node.onaudioprocess = (e) => { + const l = e.outputBuffer.getChannelData(0); + const r = e.outputBuffer.getChannelData(1); + let i = 0; + while (i < l.length && spQueue.length) { + const b = spQueue[0]; + const n = Math.min(l.length - i, (b.length - spOffset) >> 1); + for (let k = 0; k < n; k++) { l[i + k] = b[spOffset + 2 * k]; r[i + k] = b[spOffset + 2 * k + 1]; } + i += n; spOffset += 2 * n; + if (spOffset >= b.length) { spQueue.shift(); spOffset = 0; } + } + if (i < l.length) { underruns += l.length - i; l.fill(0, i); r.fill(0, i); } + }; + kind = "scriptProcessor"; + } + starting = false; + if (!node) return; + node.connect(c.destination); + startTime = c.currentTime; + const p = pendingPush; + pendingPush = []; + for (const b of p) send(b); +} + +function send(samples) { + if (kind === "worklet") node.port.postMessage(samples, [samples.buffer]); + else spQueue.push(samples); +} + +/// Queue interleaved stereo samples (Float32Array) for output. +export function audioPush(samples) { + posted += samples.length >> 1; + if (node && !starting) send(samples); + else if (starting) pendingPush.push(samples); +} + +/// Sample frames played since `audioStart`, by the context clock (AudioContext.currentTime). +export function audioPlayed() { + if (!ctx || !node || starting) return 0; + return Math.max(0, Math.round((ctx.currentTime - startTime) * ctx.sampleRate)); +} + +/// Output latency in sample frames (what's handed over but not audible yet comes on top). +export function audioOutputLatency() { + if (!ctx) return 0; + return Math.round(((ctx.outputLatency || 0) + (ctx.baseLatency || 0)) * ctx.sampleRate); +} + +export function audioStop() { + generation++; + starting = false; + pendingPush = []; + if (node) { + try { node.disconnect(); } catch {} + if (kind === "worklet") node.port.postMessage("stop"); + } + node = null; + spQueue = []; +} + +// ------------------------------------------------------------------ workers + +// Idle workers are reused; each keeps the files it was sent (path → size) so a file goes to a +// worker once. +const idle = []; +const running = new Map(); // job id → {worker, done} + +function spawn(base) { + const worker = new Worker(new URL("worker.js", base), { type: "module", name: "effectcraft-worker" }); + const w = { worker, files: new Map(), ready: null }; + w.ready = new Promise((res, rej) => { + worker.onmessage = (e) => { if (e.data && e.data.type === "ready") res(); }; + worker.onerror = (e) => rej(new Error(e.message || "worker failed to start")); + }); + worker.postMessage({ type: "init", module: self.effectcraftModule ?? null, base }); + return w; +} + +/// Run a job in a worker. `files`: [[path, Uint8Array]] the job reads; `onMessage(type, json, +/// path, bytes)` receives "reply" (json), "file" (path, bytes) and "error" (json = message). +export function workerRun(id, json, files, base, onMessage) { + const w = idle.pop() ?? spawn(base); + running.set(id, w); + w.ready.then(() => { + w.worker.onmessage = (e) => { + const m = e.data; + if (m.type === "reply") { + onMessage("reply", m.json, null, null); + if (m.done) { + running.delete(id); + idle.push(w); + } + } else if (m.type === "file") { + onMessage("file", null, m.path, new Uint8Array(m.bytes)); + } + }; + w.worker.onerror = (e) => { + running.delete(id); + onMessage("error", String(e.message || "worker error"), null, null); + w.worker.terminate(); + }; + for (const [path, bytes] of files) { + if (w.files.get(path) === bytes.length) continue; + w.files.set(path, bytes.length); + w.worker.postMessage({ type: "file", path, bytes }); + } + w.worker.postMessage({ type: "job", json }); + }, (e) => { + running.delete(id); + onMessage("error", String(e.message || e), null, null); + }); +} + +/// Stop a job: its worker is terminated (it can't be interrupted mid-job). +export function workerCancel(id) { + const w = running.get(id); + if (!w) return; + running.delete(id); + w.worker.terminate(); +} + +/// {running, idle}. +export function workerStats() { + return { running: running.size, idle: idle.length }; +} diff --git a/apps/effectcraft-web/src/api.rs b/apps/effectcraft-web/src/api.rs index 5579d111..57cfd98d 100644 --- a/apps/effectcraft-web/src/api.rs +++ b/apps/effectcraft-web/src/api.rs @@ -92,9 +92,68 @@ pub fn info() -> JsValue { } v["version"] = json!(env!("CARGO_PKG_VERSION")); v["threads"] = json!(false); + v["workers"] = from_js(&crate::worker::stats()); + v["audio"] = from_js(&crate::audio::state()); + if let Some(e) = crate::persist::last_error() { + v["storage"]["error"] = json!(e); + } to_js(&v) } +/// Save the project to browser storage without downloading it: `path` (default: its current +/// path, or `/.ecproj`). Resolves with `{path, bytes}`. +#[wasm_bindgen(js_name = saveToBrowser)] +pub fn save_to_browser(path: Option) -> js_sys::Promise { + js_sys::Promise::new(&mut |resolve, reject| { + let path = path.clone(); + crate::post(move |app| { + let p = path.or_else(|| app.session.path.clone()).unwrap_or_else(|| format!("/{}.ecproj", app.session.project_name())); + let p = if p.starts_with('/') { p } else { format!("/{p}") }; + crate::files::save_quietly(&p); + match app.session.execute("file.saveAs", json!({"path": p})) { + Ok(v) => { + let _ = resolve.call1(&JsValue::NULL, &to_js(&v)); + } + Err(e) => { + let _ = reject.call1(&JsValue::NULL, &JsValue::from_str(&e.to_string())); + } + } + }); + }) +} + +/// What browser storage holds: `{backend, usage, quota, persisted, files: [{path, size, +/// modified}], config: [name]}` (render outputs, which are not stored, are left out). +#[wasm_bindgen(js_name = listStored)] +pub async fn list_stored() -> JsValue { + let est = wasm_bindgen_futures::JsFuture::from(crate::persist::store_estimate()).await.map(|v| from_js(&v)).unwrap_or(json!({})); + let files: Vec = crate::files::STORE + .files() + .into_iter() + .filter(|f| f.3) + .map(|(path, size, modified, _)| json!({"path": path, "size": size, "modified": modified})) + .collect(); + let config: Vec = + crate::files::STORE.lock().list(crate::store::CONFIG).into_iter().map(|(k, ..)| json!(k.trim_start_matches(crate::store::CONFIG))).collect(); + let mut v = est; + v["files"] = json!(files); + v["config"] = json!(config); + v["pending"] = json!(crate::files::STORE.lock().has_pending()); + to_js(&v) +} + +/// Remove a stored file (media, project, auto-save). Resolves with whether it existed. +#[wasm_bindgen(js_name = removeStored)] +pub fn remove_stored(path: String) -> bool { + crate::files::STORE.remove_file(&path) +} + +/// Resolves once every change is written to browser storage. +#[wasm_bindgen] +pub fn flush() -> js_sys::Promise { + crate::persist::flushed() +} + /// Settle the promises whose replies arrived (called every frame). pub(crate) fn poll_replies() { PENDING.with(|p| { diff --git a/apps/effectcraft-web/src/app.rs b/apps/effectcraft-web/src/app.rs new file mode 100644 index 00000000..0076cd4f --- /dev/null +++ b/apps/effectcraft-web/src/app.rs @@ -0,0 +1,223 @@ +//! The app: startup (storage, session restore, hooks), the eframe app and the page glue. + +use std::cell::RefCell; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use effectcraft_engine::Session; + +use crate::{api, audio, files, persist, worker}; +use effectcraft_ui_egui::EffectcraftApp; +use serde_json::json; +use wasm_bindgen::prelude::*; + +/// Work for the UI thread with access to the app (posted by async tasks: picked files…). +type Action = Box; + +static WORKER: AtomicBool = AtomicBool::new(false); + +/// Running inside a Web Worker (no page, no downloads). +pub fn is_worker() -> bool { + WORKER.load(Ordering::Relaxed) +} + +pub(crate) fn set_worker() { + WORKER.store(true, Ordering::Relaxed); +} + +/// `rel` resolved against the page's URL (`worker.js`, `audio-worklet.js`); `""` = the page's +/// folder. +pub fn page_url(rel: &str) -> String { + let href = web_sys::window().and_then(|w| w.location().href().ok()).unwrap_or_default(); + web_sys::Url::new_with_base(if rel.is_empty() { "./" } else { rel }, &href).map(|u| u.href()).unwrap_or_else(|_| rel.to_string()) +} + +thread_local! { + static ACTIONS: RefCell> = const { RefCell::new(Vec::new()) }; + static CTX: RefCell> = const { RefCell::new(None) }; +} + +/// Run `f` on the UI thread at the next frame. +pub fn post(f: impl FnOnce(&mut EffectcraftApp) + 'static) { + ACTIONS.with(|a| a.borrow_mut().push(Box::new(f))); + repaint(); +} + +/// Ask for a UI frame. +pub fn repaint() { + CTX.with(|c| { + if let Some(c) = c.borrow().as_ref() { + c.request_repaint(); + } + }); +} + +/// Open a project or import media from table paths (picked or dropped files). +pub(crate) fn open_or_import(app: &mut EffectcraftApp, paths: Vec) { + let (projects, media): (Vec, Vec) = paths.into_iter().partition(|p| p.to_ascii_lowercase().ends_with(".ecproj")); + if let Some(p) = projects.last() + && let Err(e) = app.session.execute("file.open", json!({"path": p})) + { + app.ui.status = e.to_string(); + } + if !media.is_empty() { + match app.session.execute("file.import", json!({"paths": media})) { + Ok(r) => { + if let Some(errs) = r["errors"].as_array().filter(|e| !e.is_empty()) { + app.ui.status = errs.iter().filter_map(|e| e.as_str()).collect::>().join("; "); + } + } + Err(e) => app.ui.status = e.to_string(), + } + } +} + +/// The eframe app: the shared EffectCraft UI plus the browser host's per-frame duties. +pub struct WebApp { + pub app: EffectcraftApp, + /// Keep the session snapshot (off with `?empty`). + pub persist: bool, +} + +impl eframe::App for WebApp { + fn raw_input_hook(&mut self, ctx: &egui::Context, raw_input: &mut egui::RawInput) { + self.app.raw_input_hook(ctx, raw_input); + } + + fn logic(&mut self, ctx: &egui::Context, frame: &mut eframe::Frame) { + let actions = ACTIONS.with(|a| std::mem::take(&mut *a.borrow_mut())); + for f in actions { + f(&mut self.app); + } + // Files dropped on the page: read them, then open/import. + let dropped: Vec = ctx.input(|i| i.raw.dropped_files.iter().filter_map(|f| f.web_file().cloned()).collect()); + if !dropped.is_empty() { + files::read_files(dropped, |paths| post(move |app| open_or_import(app, paths))); + } + self.app.logic(ctx, frame); + api::poll_replies(); + // Render Queue outputs written this frame (here or by a worker) → downloads. + files::flush_downloads(); + // What a reload comes back to. + if self.persist { + persist::snapshot(&self.app.session, ctx.input(|i| i.time), 1.0); + } + api::set_info("gpu", json!({"compositor": self.app.gpu_adapter(), "viewerOnGpu": self.app.viewer_on_gpu()})); + } + + fn ui(&mut self, ui: &mut egui::Ui, frame: &mut eframe::Frame) { + self.app.ui(ui, frame); + } +} + +fn query() -> Option { + web_sys::window().and_then(|w| w.location().search().ok()).and_then(|s| web_sys::UrlSearchParams::new_with_str(&s).ok()) +} + +fn query_flag(name: &str) -> bool { + query().is_some_and(|p| p.has(name)) +} + +fn query_value(name: &str) -> Option { + query().and_then(|p| p.get(name)) +} + +/// A session wired for the browser: media decoding from memory, expressions, scripting, Render +/// Queue export to downloads, settings in browser storage, renders and analyses in workers. +pub fn session() -> Session { + let pool = Arc::new(effectcraft_media::MediaPool::new()); + persist::register_media(&pool); + Session { + services: Arc::new(files::WebServices), + footage: pool.clone(), + importer: Some(Arc::new(files::WebImporter { pool })), + expr: Some(Arc::new(effectcraft_expr::Expressions)), + expr_check: Some(effectcraft_expr::check_syntax), + script: Some(effectcraft_host::script::runner), + exporter: Some(Arc::new(effectcraft_host::FileExporter { sink: Some(files::export_sink()) })), + config: Some(Arc::new(persist::config())), + offload: (!query_flag("noworkers")).then(|| Arc::new(worker::WorkerOffload) as Arc), + ..Default::default() + } +} + +/// Browser file pickers in place of the desktop's file dialogs. +fn install_hooks(app: &mut EffectcraftApp) { + app.hooks.pick_files = Some(Box::new(|exts: &[&str]| { + let accept = exts.iter().map(|e| format!(".{e}")).collect::>().join(","); + if let Err(e) = files::pick(&accept, true, |paths| post(move |app| open_or_import(app, paths))) { + log::warn!("file picker: {e:?}"); + } + // the import happens when the browser hands over the files + Vec::new() + })); + app.hooks.pick_open_project = Some(Box::new(|| { + if let Err(e) = files::pick(".ecproj", false, |paths| post(move |app| open_or_import(app, paths))) { + log::warn!("file picker: {e:?}"); + } + None + })); + // Saving downloads the file: the chosen name is all a "save dialog" needs. + app.hooks.pick_save = Some(Box::new(|name: &str| Some(files::path_for(if name.is_empty() { "Untitled Project.ecproj" } else { name })))); + app.hooks.audio_device = Some(Box::new(audio::open)); + app.hooks.audio_devices = Some(Box::new(|| vec!["Browser default output".into()])); +} + +/// Entry point (called by the page's bootstrap script once the wasm module is instantiated). +#[wasm_bindgen] +pub async fn start(canvas_id: String) -> Result<(), JsValue> { + eframe::WebLogger::init(log::LevelFilter::Info).ok(); + let doc = web_sys::window().and_then(|w| w.document()).ok_or("no document")?; + let canvas: web_sys::HtmlCanvasElement = doc.get_element_by_id(&canvas_id).ok_or("no canvas")?.dyn_into()?; + let empty = query_flag("empty"); + let home = query_flag("home"); + // Browser storage: settings, recent projects, imported media, saved projects, auto-saves and + // the last session come back. + let t0 = js_sys::Date::now(); + let stored = match persist::load(&query_value("storage").unwrap_or_default()).await { + Ok(n) => n, + Err(e) => { + log::warn!("browser storage unavailable: {e}"); + 0 + } + }; + api::set_info("storage", json!({"backend": persist::backend(), "entries": stored, "loadMs": js_sys::Date::now() - t0})); + audio::install_unlock(); + let runner = eframe::WebRunner::new(); + runner + .start( + canvas, + eframe::WebOptions::default(), + Box::new(move |cc| { + CTX.with(|c| *c.borrow_mut() = Some(cc.egui_ctx.clone())); + let mut session = session(); + session.load_settings(); + let recovery = session.begin_recovery(); + // `?empty`: a blank project; `?demo`: the demo project; otherwise the last visit's + // project (the demo on a first visit). + let restored = !empty && !query_flag("demo") && persist::restore(&mut session); + if !restored && !empty { + let _ = session.execute("file.openDemoProject", json!({})); + } + api::set_info("restored", json!(restored)); + let mut app = EffectcraftApp::new(session); + app.ui.start_screen = home; + if let Some(r) = recovery.filter(|r| !restored && r.autosave.is_some()) { + app.offer_recovery(r); + } + install_hooks(&mut app); + let (tx, rx) = std::sync::mpsc::channel(); + api::set_sender(tx); + app = app.with_control(rx); + let backend = cc.wgpu_render_state.as_ref().map(|rs| format!("{:?}", rs.adapter.get_info().backend)); + api::set_info("backend", json!(backend)); + Ok(Box::new(WebApp { app, persist: !empty })) + }), + ) + .await?; + // Hide the loading message once the app runs. + if let Some(el) = doc.get_element_by_id("loading") { + el.remove(); + } + Ok(()) +} diff --git a/apps/effectcraft-web/src/audio.rs b/apps/effectcraft-web/src/audio.rs new file mode 100644 index 00000000..c16f3092 --- /dev/null +++ b/apps/effectcraft-web/src/audio.rs @@ -0,0 +1,109 @@ +//! Audio preview through Web Audio (`js/host.js`): an AudioWorklet (ScriptProcessorNode where +//! worklets are unavailable) plays the blocks the UI mixes ahead +//! ([`effectcraft_ui_egui::audio::AudioPlayback::pump`]). +//! +//! Every UI frame [`WebAudioOut::pump`] moves samples from the feed to the output so it stays +//! ~150 ms ahead of what has played. A/V sync follows `AudioContext.currentTime`: frames played +//! = (currentTime − start) × rate, so the device latency the playback clock subtracts is +//! "handed over but not played yet" plus the context's output latency. +//! +//! Browsers keep the AudioContext suspended until the user interacts with the page; the host +//! resumes it on the first pointer or key event. + +use std::sync::Arc; + +use effectcraft_ui_egui::audio::{AudioDevice, AudioFeed}; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen(module = "/js/host.js")] +extern "C" { + #[wasm_bindgen(js_name = audioInstallUnlock)] + pub fn install_unlock(); + #[wasm_bindgen(js_name = audioState)] + pub fn state() -> JsValue; + #[wasm_bindgen(js_name = audioSampleRate)] + fn sample_rate() -> f64; + #[wasm_bindgen(js_name = audioStart)] + fn audio_start(worklet_url: &str) -> js_sys::Promise; + #[wasm_bindgen(js_name = audioPush)] + fn audio_push(samples: js_sys::Float32Array); + #[wasm_bindgen(js_name = audioPlayed)] + fn audio_played() -> f64; + #[wasm_bindgen(js_name = audioOutputLatency)] + fn audio_output_latency() -> f64; + #[wasm_bindgen(js_name = audioStop)] + fn audio_stop(); +} + +/// How far ahead of the playhead the output is kept fed. +const LEAD_SECONDS: f64 = 0.15; + +pub struct WebAudioOut { + rate: u32, + feed: Option>, + /// Sample frames handed to the output since start. + posted: u64, + buf: Vec, +} + +/// The Settings ▸ Audio factory: Web Audio's default output (`None` without Web Audio). +pub fn open(_: &effectcraft_ui_egui::audio::AudioOutput) -> Option> { + let rate = sample_rate(); + (rate > 0.0).then(|| Box::new(WebAudioOut { rate: rate as u32, feed: None, posted: 0, buf: Vec::new() }) as Box) +} + +fn worklet_url() -> String { + crate::page_url("audio-worklet.js") +} + +impl AudioDevice for WebAudioOut { + fn sample_rate(&self) -> u32 { + self.rate + } + + fn start(&mut self, feed: Arc) -> Result<(), String> { + self.feed = Some(feed); + self.posted = 0; + let p = audio_start(&worklet_url()); + wasm_bindgen_futures::spawn_local(async move { + if let Err(e) = wasm_bindgen_futures::JsFuture::from(p).await { + log::warn!("audio output: {e:?}"); + } + }); + Ok(()) + } + + fn stop(&mut self) { + self.feed = None; + audio_stop(); + } + + fn latency_frames(&self) -> u64 { + let played = audio_played() as u64; + self.posted.saturating_sub(played) + audio_output_latency() as u64 + } + + fn pump(&mut self) { + let Some(feed) = &self.feed else { return }; + let played = audio_played() as u64; + let lead = (LEAD_SECONDS * self.rate as f64) as u64; + let want = (played + lead).saturating_sub(self.posted) as usize; + // Only real samples: the output plays silence by itself when it runs dry. + let n = want.min(feed.queued_frames()); + if n == 0 { + return; + } + self.buf.resize(n * 2, 0.0); + feed.pull(&mut self.buf); + audio_push(js_sys::Float32Array::from(&self.buf[..])); + self.posted += n as u64; + } +} + +impl Drop for WebAudioOut { + fn drop(&mut self) { + if self.feed.is_some() { + audio_stop(); + } + } +} diff --git a/apps/effectcraft-web/src/files.rs b/apps/effectcraft-web/src/files.rs index 853a92e6..78700e28 100644 --- a/apps/effectcraft-web/src/files.rs +++ b/apps/effectcraft-web/src/files.rs @@ -1,39 +1,55 @@ -//! Files in the browser: an in-memory file table in place of the file system, `` -//! pickers for Open/Import, and downloads for everything the app writes (projects, renders). +//! Files in the browser: a virtual file table in place of the file system, `` +//! pickers for Open/Import, and downloads for what the user saves and renders. //! //! Picked or dropped files are read into the table under `/`; the engine, the media pool and //! the exporter read and write through it ([`WebServices`], [`WebImporter`], [`export_sink`]). +//! The table is the persistent [`Store`]'s file namespace: imported media, saved projects and +//! auto-saves survive reloads ([`crate::persist`]); render outputs don't (they download). -use std::collections::HashMap; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, LazyLock, Mutex}; use effectcraft_engine::Services; use effectcraft_engine::project::Footage; use wasm_bindgen::JsCast; use wasm_bindgen::prelude::*; -static FILES: Mutex>>> = Mutex::new(None); +use crate::store::Store; + +/// The page's store (each worker has its own, unpersisted). +pub static STORE: LazyLock> = LazyLock::new(|| { + let s = Arc::new(Store::default()); + let _ = s.clock.set(js_sys::Date::now); + s +}); /// Render outputs written since the last [`flush_downloads`]. static OUTPUTS: Mutex> = Mutex::new(Vec::new()); +/// Saves that write to browser storage only (no download): `api.saveToBrowser`. +static QUIET: Mutex> = Mutex::new(Vec::new()); -fn with_files(f: impl FnOnce(&mut HashMap>) -> R) -> R { - f(FILES.lock().unwrap_or_else(|e| e.into_inner()).get_or_insert_default()) -} - -/// Store a file in the table. +/// Store a file in the table (persisted). pub fn put(path: &str, data: Arc<[u8]>) { - with_files(|m| m.insert(path.to_string(), data)); + STORE.put_file(path, data, true); } pub fn get(path: &str) -> Option> { - with_files(|m| m.get(path).cloned()) + STORE.file(path) } /// `(path, size)` of every file in the table. pub fn list() -> Vec<(String, usize)> { - let mut v: Vec<(String, usize)> = with_files(|m| m.iter().map(|(k, d)| (k.clone(), d.len())).collect()); - v.sort(); - v + STORE.files().into_iter().map(|(p, n, _, _)| (p, n)).collect() +} + +/// The next write of `path` goes to browser storage only (no download). +pub fn save_quietly(path: &str) { + QUIET.lock().unwrap_or_else(|e| e.into_inner()).push(path.to_string()); +} + +/// A rendered file (from the page's exporter or a worker): kept in the table unpersisted and +/// downloaded at the next [`flush_downloads`]. +pub fn add_output(path: &str, data: Arc<[u8]>) { + STORE.put_file(path, data, false); + OUTPUTS.lock().unwrap_or_else(|e| e.into_inner()).push(path.to_string()); } /// The table path for a picked/dropped file name. @@ -85,7 +101,8 @@ pub fn download(name: &str, data: &[u8]) -> Result<(), JsValue> { Ok(()) } -/// Engine file access (projects): reads from the table; writes go to the table and download. +/// Engine file access (projects): reads from the table; writes go to the table (persisted) and +/// download, except [`save_quietly`] saves. pub struct WebServices; impl Services for WebServices { @@ -96,6 +113,15 @@ impl Services for WebServices { } fn write_file(&self, path: &str, data: &[u8]) -> std::io::Result<()> { put(path, data.into()); + let quiet = { + let mut q = QUIET.lock().unwrap_or_else(|e| e.into_inner()); + let n = q.len(); + q.retain(|p| p != path); + q.len() != n + }; + if quiet || crate::is_worker() { + return Ok(()); + } download(file_name(path), data).map_err(|e| std::io::Error::other(format!("download failed: {e:?}"))) } } @@ -116,10 +142,7 @@ impl effectcraft_engine::Importer for WebImporter { /// Render Queue output: files land in the table and are downloaded after the render /// ([`flush_downloads`]; several files, e.g. an image sequence, as one `.zip`). pub fn export_sink() -> Arc { - Arc::new(|path: &str, data: Vec| { - put(path, data.into()); - OUTPUTS.lock().unwrap_or_else(|e| e.into_inner()).push(path.to_string()); - }) + Arc::new(|path: &str, data: Vec| add_output(path, data.into())) } /// Download the render outputs written since the last call. diff --git a/apps/effectcraft-web/src/lib.rs b/apps/effectcraft-web/src/lib.rs index edd90e31..59de4433 100644 --- a/apps/effectcraft-web/src/lib.rs +++ b/apps/effectcraft-web/src/lib.rs @@ -6,175 +6,31 @@ //! //! | Desktop | Web | //! |---|---| -//! | file system (`FsServices`, media reads, export writes) | in-memory file table ([`files`]) | +//! | file system (`FsServices`, media reads, export writes) | a virtual file table ([`files`]) persisted to the Origin Private File System / IndexedDB ([`persist`], [`store`]) | +//! | config directory (settings, shortcuts, recent projects, auto-saves) | the same store ([`store::WebConfig`]) | //! | rfd file dialogs | `` pickers; drop files on the page | //! | written files (Save, Render Queue) | browser downloads (several render files as one `.zip`) | //! | frame render threads | `Frames::pump` on the UI thread between egui frames | -//! | Render Queue thread | the render runs inline (the page waits until it is done) | +//! | Render Queue / analysis threads | Web Workers running their own engine instance ([`worker`]) | +//! | cpal audio output | Web Audio: an AudioWorklet fed from the preview mixdown ([`audio`]) | //! | TCP control channel / MCP | `window.effectcraft` JavaScript API ([`api`]) | //! -//! Everything here is `wasm32`-only; on other targets the crate is empty. -#![cfg(target_arch = "wasm32")] +//! [`store`] is portable (unit-tested natively); everything else is `wasm32`-only. +pub mod store; + +#[cfg(target_arch = "wasm32")] pub mod api; +#[cfg(target_arch = "wasm32")] +mod app; +#[cfg(target_arch = "wasm32")] +pub mod audio; +#[cfg(target_arch = "wasm32")] pub mod files; +#[cfg(target_arch = "wasm32")] +pub mod persist; +#[cfg(target_arch = "wasm32")] +pub mod worker; -use std::cell::RefCell; -use std::sync::Arc; - -use effectcraft_engine::Session; -use effectcraft_ui_egui::EffectcraftApp; -use serde_json::json; -use wasm_bindgen::prelude::*; - -/// Work for the UI thread with access to the app (posted by async tasks: picked files…). -type Action = Box; - -thread_local! { - static ACTIONS: RefCell> = const { RefCell::new(Vec::new()) }; - static CTX: RefCell> = const { RefCell::new(None) }; -} - -/// Run `f` on the UI thread at the next frame. -pub fn post(f: impl FnOnce(&mut EffectcraftApp) + 'static) { - ACTIONS.with(|a| a.borrow_mut().push(Box::new(f))); - repaint(); -} - -/// Ask for a UI frame. -pub fn repaint() { - CTX.with(|c| { - if let Some(c) = c.borrow().as_ref() { - c.request_repaint(); - } - }); -} - -/// Open a project or import media from table paths (picked or dropped files). -fn open_or_import(app: &mut EffectcraftApp, paths: Vec) { - let (projects, media): (Vec, Vec) = paths.into_iter().partition(|p| p.to_ascii_lowercase().ends_with(".ecproj")); - if let Some(p) = projects.last() - && let Err(e) = app.session.execute("file.open", json!({"path": p})) - { - app.ui.status = e.to_string(); - } - if !media.is_empty() { - match app.session.execute("file.import", json!({"paths": media})) { - Ok(r) => { - if let Some(errs) = r["errors"].as_array().filter(|e| !e.is_empty()) { - app.ui.status = errs.iter().filter_map(|e| e.as_str()).collect::>().join("; "); - } - } - Err(e) => app.ui.status = e.to_string(), - } - } -} - -/// The eframe app: the shared EffectCraft UI plus the browser host's per-frame duties. -pub struct WebApp { - pub app: EffectcraftApp, -} - -impl eframe::App for WebApp { - fn raw_input_hook(&mut self, ctx: &egui::Context, raw_input: &mut egui::RawInput) { - self.app.raw_input_hook(ctx, raw_input); - } - - fn logic(&mut self, ctx: &egui::Context, frame: &mut eframe::Frame) { - let actions = ACTIONS.with(|a| std::mem::take(&mut *a.borrow_mut())); - for f in actions { - f(&mut self.app); - } - // Files dropped on the page: read them, then open/import. - let dropped: Vec = ctx.input(|i| i.raw.dropped_files.iter().filter_map(|f| f.web_file().cloned()).collect()); - if !dropped.is_empty() { - files::read_files(dropped, |paths| post(move |app| open_or_import(app, paths))); - } - self.app.logic(ctx, frame); - api::poll_replies(); - // Render Queue outputs written this frame → downloads. - files::flush_downloads(); - } - - fn ui(&mut self, ui: &mut egui::Ui, frame: &mut eframe::Frame) { - self.app.ui(ui, frame); - } -} - -fn query_flag(name: &str) -> bool { - web_sys::window().and_then(|w| w.location().search().ok()).and_then(|s| web_sys::UrlSearchParams::new_with_str(&s).ok()).is_some_and(|p| p.has(name)) -} - -/// A session wired for the browser: media decoding from memory, expressions, scripting, Render -/// Queue export to downloads. -pub fn session() -> Session { - let pool = Arc::new(effectcraft_media::MediaPool::new()); - Session { - services: Arc::new(files::WebServices), - footage: pool.clone(), - importer: Some(Arc::new(files::WebImporter { pool })), - expr: Some(Arc::new(effectcraft_expr::Expressions)), - expr_check: Some(effectcraft_expr::check_syntax), - script: Some(effectcraft_host::script::runner), - exporter: Some(Arc::new(effectcraft_host::FileExporter { sink: Some(files::export_sink()) })), - ..Default::default() - } -} - -/// Browser file pickers in place of the desktop's file dialogs. -fn install_hooks(app: &mut EffectcraftApp) { - app.hooks.pick_files = Some(Box::new(|exts: &[&str]| { - let accept = exts.iter().map(|e| format!(".{e}")).collect::>().join(","); - if let Err(e) = files::pick(&accept, true, |paths| post(move |app| open_or_import(app, paths))) { - log::warn!("file picker: {e:?}"); - } - // the import happens when the browser hands over the files - Vec::new() - })); - app.hooks.pick_open_project = Some(Box::new(|| { - if let Err(e) = files::pick(".ecproj", false, |paths| post(move |app| open_or_import(app, paths))) { - log::warn!("file picker: {e:?}"); - } - None - })); - // Saving downloads the file: the chosen name is all a "save dialog" needs. - app.hooks.pick_save = Some(Box::new(|name: &str| Some(files::path_for(if name.is_empty() { "Untitled Project.ecproj" } else { name })))); -} - -/// Entry point (called by the page's bootstrap script once the wasm module is instantiated). -#[wasm_bindgen] -pub async fn start(canvas_id: String) -> Result<(), JsValue> { - eframe::WebLogger::init(log::LevelFilter::Info).ok(); - let doc = web_sys::window().and_then(|w| w.document()).ok_or("no document")?; - let canvas: web_sys::HtmlCanvasElement = doc.get_element_by_id(&canvas_id).ok_or("no canvas")?.dyn_into()?; - let demo = !query_flag("empty"); - let home = query_flag("home"); - let runner = eframe::WebRunner::new(); - runner - .start( - canvas, - eframe::WebOptions::default(), - Box::new(move |cc| { - CTX.with(|c| *c.borrow_mut() = Some(cc.egui_ctx.clone())); - let mut session = session(); - if demo { - let _ = session.execute("file.openDemoProject", json!({})); - } - let mut app = EffectcraftApp::new(session); - app.ui.start_screen = home; - install_hooks(&mut app); - let (tx, rx) = std::sync::mpsc::channel(); - api::set_sender(tx); - app = app.with_control(rx); - let backend = cc.wgpu_render_state.as_ref().map(|rs| format!("{:?}", rs.adapter.get_info().backend)); - api::set_info("backend", json!(backend)); - Ok(Box::new(WebApp { app })) - }), - ) - .await?; - // Hide the loading message once the app runs. - if let Some(el) = doc.get_element_by_id("loading") { - el.remove(); - } - Ok(()) -} +#[cfg(target_arch = "wasm32")] +pub use app::*; diff --git a/apps/effectcraft-web/src/persist.rs b/apps/effectcraft-web/src/persist.rs new file mode 100644 index 00000000..6735d066 --- /dev/null +++ b/apps/effectcraft-web/src/persist.rs @@ -0,0 +1,190 @@ +//! Persistence: the [`crate::store::Store`] mirrored to the browser's storage (the Origin +//! Private File System, IndexedDB where OPFS can't write, memory as a last resort; `js/host.js`). +//! +//! - [`load`] (before the app starts) opens the storage and reads every entry into the mirror. +//! - Changes flush in the background ([`schedule_flush`], wired as the store's change hook): +//! one write per changed key, in order, with nothing written twice concurrently. +//! - The session snapshot ([`snapshot`]) records the open project and editor state every second +//! or so while it changes, so a reload (or a crash) comes back to where the user was +//! ([`restore`]). + +use std::cell::Cell; + +use effectcraft_engine::Session; +use effectcraft_engine::config::ConfigStore; +use effectcraft_engine::project::Project; +use wasm_bindgen::prelude::*; +use wasm_bindgen_futures::JsFuture; + +use crate::files::STORE; +use crate::store::{SNAPSHOT, SNAPSHOT_META, SnapshotMeta, WebConfig}; + +#[wasm_bindgen(module = "/js/host.js")] +extern "C" { + #[wasm_bindgen(js_name = storeOpen)] + fn store_open(prefer: &str) -> js_sys::Promise; + #[wasm_bindgen(js_name = storeLoadAll)] + fn store_load_all() -> js_sys::Promise; + #[wasm_bindgen(js_name = storePut)] + fn store_put(key: &str, data: js_sys::Uint8Array, modified: f64) -> js_sys::Promise; + #[wasm_bindgen(js_name = storeDelete)] + fn store_delete(key: &str) -> js_sys::Promise; + #[wasm_bindgen(js_name = storeEstimate)] + pub(crate) fn store_estimate() -> js_sys::Promise; +} + +thread_local! { + static FLUSHING: Cell = const { Cell::new(false) }; + static BACKEND: std::cell::RefCell = const { std::cell::RefCell::new(String::new()) }; + static LAST_ERROR: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; + static FLUSH_WAITERS: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; + static SNAP: Cell<(u64, f64)> = const { Cell::new((u64::MAX, 0.0)) }; + static LAST_META: std::cell::RefCell = const { std::cell::RefCell::new(String::new()) }; +} + +/// The storage in use: `opfs`, `indexeddb` or `memory`. +pub fn backend() -> String { + BACKEND.with(|b| b.borrow().clone()) +} + +/// The last storage error (quota exceeded…), if any. +pub fn last_error() -> Option { + LAST_ERROR.with(|e| e.borrow().clone()) +} + +fn js_err(e: JsValue) -> String { + e.as_string().or_else(|| js_sys::Reflect::get(&e, &"message".into()).ok().and_then(|m| m.as_string())).unwrap_or_else(|| format!("{e:?}")) +} + +/// Open the browser storage and read everything into the store. `prefer` forces a backend +/// (`?storage=indexeddb` / `memory`; empty = OPFS, then IndexedDB). Returns the entry count. +pub async fn load(prefer: &str) -> Result { + let name = JsFuture::from(store_open(prefer)).await.map_err(js_err)?; + BACKEND.with(|b| *b.borrow_mut() = name.as_string().unwrap_or_default()); + let all = JsFuture::from(store_load_all()).await.map_err(js_err)?; + let all: js_sys::Array = all.dyn_into().map_err(js_err)?; + let n = all.length() as usize; + { + let mut m = STORE.lock(); + for e in all.iter() { + let get = |k: &str| js_sys::Reflect::get(&e, &k.into()).unwrap_or(JsValue::UNDEFINED); + let Some(key) = get("key").as_string() else { continue }; + let data: js_sys::Uint8Array = match get("data").dyn_into() { + Ok(d) => d, + Err(_) => continue, + }; + m.load(&key, data.to_vec().into(), get("modified").as_f64().unwrap_or(0.0)); + } + } + let _ = STORE.on_change.set(schedule_flush); + Ok(n) +} + +/// Write pending changes to the browser storage (in the background; one flush at a time). +pub fn schedule_flush() { + if FLUSHING.with(|f| f.replace(true)) { + return; // the running flush picks the new changes up + } + wasm_bindgen_futures::spawn_local(async { + loop { + let pending = STORE.lock().take_pending(); + if pending.is_empty() { + break; + } + for p in pending { + let r = match &p.data { + Some(d) => JsFuture::from(store_put(&p.key, js_sys::Uint8Array::from(&d[..]), p.modified)).await, + None => JsFuture::from(store_delete(&p.key)).await, + }; + if let Err(e) = r { + let msg = format!("saving {} to browser storage failed: {}", p.key, js_err(e)); + log::warn!("{msg}"); + LAST_ERROR.with(|l| *l.borrow_mut() = Some(msg)); + } + } + } + FLUSHING.with(|f| f.set(false)); + for w in FLUSH_WAITERS.with(|w| std::mem::take(&mut *w.borrow_mut())) { + let _ = w.call0(&JsValue::NULL); + } + }); +} + +/// A promise that resolves once every change so far is in the browser storage. +pub fn flushed() -> js_sys::Promise { + js_sys::Promise::new(&mut |resolve, _| { + if !FLUSHING.with(|f| f.get()) && !STORE.lock().has_pending() { + let _ = resolve.call0(&JsValue::NULL); + } else { + FLUSH_WAITERS.with(|w| w.borrow_mut().push(resolve)); + schedule_flush(); + } + }) +} + +/// The config store for the session. +pub fn config() -> WebConfig { + WebConfig::new(STORE.clone()) +} + +/// Record the open project and editor state when they changed (checked at most every +/// `min_secs`). +pub fn snapshot(s: &Session, now: f64, min_secs: f64) { + let (rev, at) = SNAP.with(|c| c.get()); + if now - at < min_secs { + return; + } + SNAP.with(|c| c.set((rev, now))); + let meta = SnapshotMeta { path: s.path.clone(), dirty: s.is_dirty(), state: serde_json::to_value(&s.state).ok(), saved: 0.0 }; + let key = serde_json::to_string(&meta).unwrap_or_default(); + if rev == s.revision && LAST_META.with(|m| *m.borrow() == key) { + return; + } + let c = config(); + if rev != s.revision { + let _ = c.write(SNAPSHOT, &s.project.to_json()); + } + let _ = c.write(SNAPSHOT_META, &serde_json::to_string(&SnapshotMeta { saved: js_sys::Date::now(), ..meta }).unwrap_or_default()); + SNAP.with(|c| c.set((s.revision, now))); + LAST_META.with(|m| *m.borrow_mut() = key); +} + +/// Reopen the last visit's project and editor state. Returns whether there was one. +pub fn restore(s: &mut Session) -> bool { + let c = config(); + let (Some(json), Some(meta)) = (c.read(SNAPSHOT), c.read(SNAPSHOT_META)) else { return false }; + let meta: SnapshotMeta = serde_json::from_str(&meta).unwrap_or_default(); + let p = match Project::from_json(&json) { + Ok(p) => p, + Err(e) => { + log::warn!("restoring the last session: {e}"); + return false; + } + }; + s.replace_project(p, meta.path.clone()); + if let Some(st) = meta.state.and_then(|v| serde_json::from_value(v).ok()) { + s.state = st; + } + if meta.dirty { + // Unsaved changes stay unsaved. + s.bump(); + } + SNAP.with(|c| c.set((s.revision, 0.0))); + log::info!("restored the last session ({})", meta.path.as_deref().unwrap_or("untitled")); + true +} + +/// Register persisted media with the media pool (so restored projects find their footage). +pub fn register_media(pool: &effectcraft_media::MediaPool) -> usize { + let mut n = 0; + for (path, _, _, persisted) in STORE.files() { + if persisted + && !path.to_ascii_lowercase().ends_with(".ecproj") + && let Some(d) = STORE.file(&path) + { + pool.add_bytes(&path, d); + n += 1; + } + } + n +} diff --git a/apps/effectcraft-web/src/store.rs b/apps/effectcraft-web/src/store.rs new file mode 100644 index 00000000..0e4ea029 --- /dev/null +++ b/apps/effectcraft-web/src/store.rs @@ -0,0 +1,399 @@ +//! What the browser keeps between visits: the in-memory side of the persistent store. +//! +//! The engine's storage traits are synchronous ([`ConfigStore`], [`FileOps`], +//! [`effectcraft_engine::Services`]); the browser's storage (the Origin Private File System, or +//! IndexedDB where OPFS can't write) is asynchronous. So the app works on a [`Mirror`]: every +//! stored entry is loaded into memory before the app starts, reads and writes hit the mirror +//! synchronously, and writes queue up as pending operations that the web host flushes to the +//! browser in the background (coalesced per key, in order). +//! +//! Keys are namespaced: +//! +//! | Key | What | +//! |---|---| +//! | `config/` | settings (`prefs.json`), shortcut presets, the recovery sentinel, the session snapshot | +//! | `files` | the virtual file table: imported media, saved projects, auto-saves (`/EffectCraft Auto-Save/…`) | +//! +//! Render outputs go into the file table unpersisted (they download instead). +//! +//! This module is plain Rust (no browser APIs) so it is unit-tested natively. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_engine::config::{ConfigStore, FileOps}; + +/// Prefix of configuration entries. +pub const CONFIG: &str = "config/"; +/// Prefix of file-table entries (followed by the absolute path, `files/clip.mp4`). +pub const FILES: &str = "files"; +/// The session snapshot (project + editor state), written while the app runs and restored +/// on the next visit. +pub const SNAPSHOT: &str = "session.ecproj"; +/// Its metadata ([`SnapshotMeta`]). +pub const SNAPSHOT_META: &str = "session.json"; + +/// One stored blob. +#[derive(Clone, Debug)] +pub struct Entry { + pub data: Arc<[u8]>, + /// Milliseconds since the Unix epoch. + pub modified: f64, + /// Written to the browser's storage (render outputs are not). + pub persist: bool, +} + +/// A queued write for the browser storage: `data` = `None` deletes the key. +#[derive(Clone, Debug, PartialEq)] +pub struct Pending { + pub key: String, + pub data: Option>, + pub modified: f64, +} + +/// The in-memory copy of the store plus the writes not yet flushed. +#[derive(Default)] +pub struct Mirror { + entries: BTreeMap, + /// Keys changed since the last [`Mirror::take_pending`], in first-change order. + dirty: Vec, +} + +impl Mirror { + /// An entry read from the browser's storage at startup (nothing to flush). + pub fn load(&mut self, key: &str, data: Arc<[u8]>, modified: f64) { + self.entries.insert(key.to_string(), Entry { data, modified, persist: true }); + } + + pub fn put(&mut self, key: &str, data: Arc<[u8]>, modified: f64, persist: bool) { + let was_persisted = self.entries.get(key).is_some_and(|e| e.persist); + self.entries.insert(key.to_string(), Entry { data, modified, persist }); + if persist || was_persisted { + self.touch(key); + } + } + + pub fn remove(&mut self, key: &str) -> bool { + match self.entries.remove(key) { + Some(e) => { + if e.persist { + self.touch(key); + } + true + } + None => false, + } + } + + fn touch(&mut self, key: &str) { + if !self.dirty.iter().any(|k| k == key) { + self.dirty.push(key.to_string()); + } + } + + pub fn get(&self, key: &str) -> Option<&Entry> { + self.entries.get(key) + } + + /// Entries under `prefix`: (key, size, modified, persisted). + pub fn list(&self, prefix: &str) -> Vec<(String, usize, f64, bool)> { + self.entries + .range(prefix.to_string()..) + .take_while(|(k, _)| k.starts_with(prefix)) + .map(|(k, e)| (k.clone(), e.data.len(), e.modified, e.persist)) + .collect() + } + + /// The writes to flush, one per changed key with its latest state (a put, or a delete when + /// the key is gone or no longer persisted). + pub fn take_pending(&mut self) -> Vec { + std::mem::take(&mut self.dirty) + .into_iter() + .map(|key| match self.entries.get(&key) { + Some(e) if e.persist => Pending { data: Some(e.data.clone()), modified: e.modified, key }, + _ => Pending { key, data: None, modified: 0.0 }, + }) + .collect() + } + + pub fn has_pending(&self) -> bool { + !self.dirty.is_empty() + } +} + +/// The key of a file-table path (`/a/b.mov` → `files/a/b.mov`). +pub fn file_key(path: &str) -> String { + if path.starts_with('/') { format!("{FILES}{path}") } else { format!("{FILES}/{path}") } +} + +/// The path of a file-table key. +pub fn key_path(key: &str) -> Option<&str> { + key.strip_prefix(FILES).filter(|p| p.starts_with('/')) +} + +/// The persistent store: a [`Mirror`] and a hook that wakes the flusher after changes. +#[derive(Default)] +pub struct Store { + pub mirror: Mutex, + /// Called after every change that needs flushing (the web host schedules a flush). + pub on_change: OnceLock, + /// Clock in milliseconds since the Unix epoch (tests pin it). + pub clock: OnceLock f64>, +} + +impl Store { + pub fn lock(&self) -> std::sync::MutexGuard<'_, Mirror> { + self.mirror.lock().unwrap_or_else(|e| e.into_inner()) + } + + fn now(&self) -> f64 { + self.clock.get().map_or(0.0, |f| f()) + } + + fn changed(&self) { + if let Some(f) = self.on_change.get() { + f(); + } + } + + // ---- file table + + /// Store a file under `path` (persisted unless `persist` is false: render outputs). + pub fn put_file(&self, path: &str, data: Arc<[u8]>, persist: bool) { + let now = self.now(); + self.lock().put(&file_key(path), data, now, persist); + if persist { + self.changed(); + } + } + + pub fn file(&self, path: &str) -> Option> { + self.lock().get(&file_key(path)).map(|e| e.data.clone()) + } + + pub fn remove_file(&self, path: &str) -> bool { + let r = self.lock().remove(&file_key(path)); + self.changed(); + r + } + + /// Every file: (path, size, modified ms, persisted). + pub fn files(&self) -> Vec<(String, usize, f64, bool)> { + self.lock().list(FILES).into_iter().filter_map(|(k, n, m, p)| Some((key_path(&k)?.to_string(), n, m, p))).collect() + } +} + +/// Settings and friends ([`ConfigStore`]) in the store; auto-saves ([`FileOps`]) in its file +/// table, under `/` (untitled projects auto-save to `/EffectCraft Auto-Save/`). +pub struct WebConfig { + store: Arc, + files: WebFiles, +} + +impl WebConfig { + pub fn new(store: Arc) -> WebConfig { + WebConfig { files: WebFiles(store.clone()), store } + } +} + +/// [`FileOps`] on the store's file table. +pub struct WebFiles(pub Arc); + +impl ConfigStore for WebConfig { + fn read(&self, name: &str) -> Option { + let m = self.store.lock(); + m.get(&format!("{CONFIG}{name}")).map(|e| String::from_utf8_lossy(&e.data).into_owned()) + } + fn write(&self, name: &str, data: &str) -> std::io::Result<()> { + let now = self.store.now(); + self.store.lock().put(&format!("{CONFIG}{name}"), data.as_bytes().into(), now, true); + self.store.changed(); + Ok(()) + } + fn remove(&self, name: &str) -> std::io::Result<()> { + self.store.lock().remove(&format!("{CONFIG}{name}")); + self.store.changed(); + Ok(()) + } + fn dir(&self) -> Option { + Some(PathBuf::from("/")) + } + fn files(&self) -> &dyn FileOps { + &self.files + } +} + +fn path_str(p: &Path) -> String { + p.to_string_lossy().replace('\\', "/") +} + +impl FileOps for WebFiles { + fn list(&self, dir: &Path) -> Vec<(String, std::time::SystemTime)> { + let dir = path_str(dir); + let dir = dir.trim_end_matches('/'); + self.0 + .files() + .into_iter() + .filter_map(|(p, _, m, _)| { + let (parent, name) = p.rsplit_once('/')?; + (parent == dir).then(|| (name.to_string(), std::time::UNIX_EPOCH + std::time::Duration::from_secs_f64(m.max(0.0) / 1000.0))) + }) + .collect() + } + fn write(&self, path: &Path, data: &[u8]) -> std::io::Result<()> { + self.0.put_file(&path_str(path), data.into(), true); + Ok(()) + } + fn remove(&self, path: &Path) -> std::io::Result<()> { + if self.0.remove_file(&path_str(path)) { Ok(()) } else { Err(std::io::Error::new(std::io::ErrorKind::NotFound, path_str(path))) } + } + fn is_file(&self, path: &Path) -> bool { + self.0.file(&path_str(path)).is_some() + } +} + +/// What the session snapshot records besides the project. +#[derive(Clone, Debug, Default, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct SnapshotMeta { + /// The project's file (None = untitled). + pub path: Option, + /// Unsaved changes when the snapshot was taken. + pub dirty: bool, + /// Viewer, timeline and selection state. + pub state: Option, + /// Milliseconds since the Unix epoch. + pub saved: f64, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn store() -> Arc { + let s = Arc::new(Store::default()); + let _ = s.clock.set(|| 1_000.0); + s + } + + #[test] + fn writes_coalesce_per_key_in_order() { + let mut m = Mirror::default(); + m.put("config/a", Arc::from(&b"1"[..]), 1.0, true); + m.put("files/x.png", Arc::from(&b"px"[..]), 2.0, true); + m.put("config/a", Arc::from(&b"2"[..]), 3.0, true); + let p = m.take_pending(); + assert_eq!(p.len(), 2); + assert_eq!((p[0].key.as_str(), p[0].data.as_deref()), ("config/a", Some(&b"2"[..]))); + assert_eq!(p[0].modified, 3.0); + assert_eq!(p[1].key, "files/x.png"); + assert!(!m.has_pending() && m.take_pending().is_empty()); + // Delete after put: one delete. + m.put("files/y", Arc::from(&b"y"[..]), 4.0, true); + m.remove("files/y"); + assert_eq!(m.take_pending(), vec![Pending { key: "files/y".into(), data: None, modified: 0.0 }]); + } + + #[test] + fn unpersisted_entries_never_flush_and_loaded_ones_are_clean() { + let mut m = Mirror::default(); + m.load("files/old.mov", Arc::from(&b"m"[..]), 5.0); + m.put("files/render.gif", Arc::from(&b"g"[..]), 6.0, false); + assert!(!m.has_pending()); + assert!(!m.remove("files/none")); + m.remove("files/render.gif"); + assert!(!m.has_pending()); + // Overwriting a persisted file with an unpersisted one deletes the stored copy. + m.put("files/old.mov", Arc::from(&b"n"[..]), 7.0, false); + assert_eq!(m.take_pending()[0].data, None); + assert_eq!(m.list("files").len(), 1); + } + + #[test] + fn keys_and_paths_round_trip() { + assert_eq!(file_key("/a b/c.mov"), "files/a b/c.mov"); + assert_eq!(file_key("c.mov"), "files/c.mov"); + assert_eq!(key_path("files/a b/c.mov"), Some("/a b/c.mov")); + assert_eq!(key_path("config/prefs.json"), None); + let s = store(); + s.put_file("/clip.mp4", Arc::from(&b"abc"[..]), true); + s.put_file("/out.gif", Arc::from(&b"g"[..]), false); + let f = s.files(); + assert_eq!(f, vec![("/clip.mp4".to_string(), 3, 1000.0, true), ("/out.gif".to_string(), 1, 1000.0, false)]); + assert_eq!(s.file("/clip.mp4").as_deref(), Some(&b"abc"[..])); + } + + #[test] + fn config_store_and_file_ops() { + let s = store(); + let c = WebConfig::new(s.clone()); + assert_eq!(c.read("prefs.json"), None); + c.write("prefs.json", "{\"a\":1}").unwrap(); + assert_eq!(c.read("prefs.json").as_deref(), Some("{\"a\":1}")); + c.remove("prefs.json").unwrap(); + assert_eq!(c.read("prefs.json"), None); + // Config entries are not files. + c.write("session.lock", "{}").unwrap(); + assert!(s.files().is_empty()); + let fs = c.files(); + fs.write(Path::new("/EffectCraft Auto-Save/P auto-save 1.ecproj"), b"{}").unwrap(); + fs.write(Path::new("/EffectCraft Auto-Save/P auto-save 2.ecproj"), b"{}").unwrap(); + fs.write(Path::new("/other.ecproj"), b"{}").unwrap(); + let mut names: Vec = fs.list(Path::new("/EffectCraft Auto-Save")).into_iter().map(|e| e.0).collect(); + names.sort(); + assert_eq!(names, ["P auto-save 1.ecproj", "P auto-save 2.ecproj"]); + assert!(fs.is_file(Path::new("/other.ecproj"))); + fs.remove(Path::new("/other.ecproj")).unwrap(); + assert!(!fs.is_file(Path::new("/other.ecproj"))); + assert!(fs.remove(Path::new("/other.ecproj")).is_err()); + let pending = s.lock().take_pending(); + let keys: Vec<&str> = pending.iter().map(|p| p.key.as_str()).collect(); + assert_eq!( + keys, + [ + "config/prefs.json", + "config/session.lock", + "files/EffectCraft Auto-Save/P auto-save 1.ecproj", + "files/EffectCraft Auto-Save/P auto-save 2.ecproj", + "files/other.ecproj" + ] + ); + assert_eq!(pending[0].data, None, "removed"); + assert_eq!(pending[4].data, None, "removed"); + } + + #[test] + fn autosave_and_recovery_run_in_the_browser_store() { + use effectcraft_engine::Session; + use serde_json::json; + let s = store(); + let mut a = Session { config: Some(Arc::new(WebConfig::new(s.clone()))), ..Default::default() }; + a.load_settings(); + assert!(a.begin_recovery().is_none()); + a.execute("file.newProject", json!({})).unwrap(); + a.execute("comp.new", json!({"name": "C"})).unwrap(); + // Untitled: the auto-save lands in the store's file table. + let p = a.autosave_now().unwrap(); + assert_eq!(p, "/EffectCraft Auto-Save/Untitled Project auto-save 1.ecproj"); + assert!(s.file(&p).is_some()); + // The tab closes without a clean exit: the next visit offers the auto-save. + let mut b = Session { config: Some(Arc::new(WebConfig::new(s.clone()))), ..Default::default() }; + b.load_settings(); + let r = b.begin_recovery().expect("recovery"); + assert!(r.autosave.is_some()); + // Increment and Save sees the store's files. + let fs = WebConfig::new(s.clone()); + fs.files().write(Path::new("/A.ecproj"), b"{}").unwrap(); + let next = effectcraft_engine::autosave::increment_path("/A.ecproj", |p| fs.files().is_file(Path::new(p))); + assert_eq!(next, "/A 2.ecproj"); + } + + #[test] + fn snapshot_meta_round_trips() { + let m = SnapshotMeta { path: Some("/p.ecproj".into()), dirty: true, state: Some(serde_json::json!({"activeComp": 3})), saved: 12.5 }; + let t = serde_json::to_string(&m).unwrap(); + assert_eq!(serde_json::from_str::(&t).unwrap(), m); + assert_eq!(serde_json::from_str::("{}").unwrap(), SnapshotMeta::default()); + } +} diff --git a/apps/effectcraft-web/src/worker.rs b/apps/effectcraft-web/src/worker.rs new file mode 100644 index 00000000..2d96f641 --- /dev/null +++ b/apps/effectcraft-web/src/worker.rs @@ -0,0 +1,186 @@ +//! Renders and analyses in Web Workers ([`effectcraft_engine::offload`]). +//! +//! **Page side** — [`WorkerOffload`], the session's [`Offload`]: serializes the job, ships the +//! footage files it reads and the request to a worker from the host's pool (`js/host.js` +//! `workerRun`), and turns the worker's messages back into [`WorkerReply`]s in the job's +//! [`Inbox`]. Rendered files come back as their bytes and download like any render. +//! +//! **Worker side** — `web/worker.js` instantiates the same wasm module (the page's compiled +//! `WebAssembly.Module`, so nothing is compiled twice) and calls [`worker_init`], then +//! [`worker_file`] per footage file and [`worker_job`] per job: a plain engine session (footage +//! from memory, expressions, the exporter) runs it blocking and posts replies as it goes. Each +//! worker has its own memory: no shared-memory threads, so this works on stable Rust. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::rc::Rc; +use std::sync::Arc; + +use effectcraft_engine::Session; +use effectcraft_engine::offload::{Inbox, Offload, WorkerReply, WorkerRequest}; +use wasm_bindgen::JsCast; +use wasm_bindgen::prelude::*; + +#[wasm_bindgen(module = "/js/host.js")] +extern "C" { + #[wasm_bindgen(js_name = workerRun)] + fn worker_run(id: f64, json: &str, files: js_sys::Array, base: &str, on_message: &js_sys::Function); + #[wasm_bindgen(js_name = workerCancel)] + fn worker_cancel(id: f64); + #[wasm_bindgen(js_name = workerStats)] + pub fn stats() -> JsValue; +} + +type OnMessage = Closure; + +thread_local! { + /// Message handlers of running jobs (dropped when the job ends). + static HANDLERS: RefCell> = RefCell::new(HashMap::new()); + /// The worker's session and its media pool. + static SESSION: RefCell> = const { RefCell::new(None) }; + static POOL: RefCell>> = const { RefCell::new(None) }; +} + +/// The page's offload: one worker per running job (idle workers are reused). +pub struct WorkerOffload; + +impl Offload for WorkerOffload { + fn start(&self, req: WorkerRequest, inbox: Arc) -> Result<(), String> { + let id = req.id; + let files = js_sys::Array::new(); + for p in &req.files { + if let Some(d) = crate::files::get(p) { + files.push(&js_sys::Array::of2(&JsValue::from_str(p), &js_sys::Uint8Array::from(&d[..]))); + } + } + let json = serde_json::to_string(&req).map_err(|e| e.to_string())?; + let on: OnMessage = Closure::new(move |kind: String, json: JsValue, path: JsValue, bytes: JsValue| { + match kind.as_str() { + "reply" => { + let Some(t) = json.as_string() else { return }; + match serde_json::from_str::(&t) { + Ok(r) => { + let done = matches!(r, WorkerReply::Done); + inbox.push(r); + if done { + end(id); + } + } + Err(e) => log::warn!("worker reply: {e}"), + } + } + "file" => { + if let (Some(p), Ok(b)) = (path.as_string(), bytes.dyn_into::()) { + crate::files::add_output(&p, b.to_vec().into()); + } + } + _ => { + inbox.push(WorkerReply::Failed { error: json.as_string().unwrap_or_else(|| "worker failed".into()) }); + inbox.push(WorkerReply::Done); + end(id); + } + } + crate::repaint(); + }); + worker_run(id as f64, &json, files, &crate::page_url(""), on.as_ref().unchecked_ref()); + HANDLERS.with(|h| h.borrow_mut().insert(id, on)); + Ok(()) + } + + fn cancel(&self, id: u64) { + worker_cancel(id as f64); + end(id); + } +} + +/// Drop a job's handler (after the current call returns). +fn end(id: u64) { + wasm_bindgen_futures::spawn_local(async move { + HANDLERS.with(|h| h.borrow_mut().remove(&id)); + }); +} + +// ---------------------------------------------------------------- worker side + +fn post(msg: &JsValue, transfer: Option<&js_sys::Array>) { + let scope: web_sys::DedicatedWorkerGlobalScope = js_sys::global().unchecked_into(); + let r = match transfer { + Some(t) => scope.post_message_with_transfer(msg, t), + None => scope.post_message(msg), + }; + if let Err(e) = r { + log::warn!("worker: postMessage: {e:?}"); + } +} + +fn obj(pairs: &[(&str, JsValue)]) -> JsValue { + let o = js_sys::Object::new(); + for (k, v) in pairs { + let _ = js_sys::Reflect::set(&o, &JsValue::from_str(k), v); + } + o.into() +} + +/// Set up the worker's engine session (called once by `web/worker.js`). +#[wasm_bindgen(js_name = workerInit)] +pub fn worker_init() { + crate::set_worker(); + eframe::WebLogger::init(log::LevelFilter::Info).ok(); + let pool = Arc::new(effectcraft_media::MediaPool::new()); + POOL.with(|p| *p.borrow_mut() = Some(pool.clone())); + // Rendered files go back to the page (transferred, not copied). + let sink: Arc = Arc::new(|path: &str, data: Vec| { + let bytes = js_sys::Uint8Array::from(&data[..]); + let buf = bytes.buffer(); + post(&obj(&[("type", "file".into()), ("path", path.into()), ("bytes", buf.clone().into())]), Some(&js_sys::Array::of1(&buf))); + }); + let s = Session { + services: Arc::new(crate::files::WebServices), + footage: pool.clone(), + importer: Some(Arc::new(crate::files::WebImporter { pool })), + expr: Some(Arc::new(effectcraft_expr::Expressions)), + expr_check: Some(effectcraft_expr::check_syntax), + exporter: Some(Arc::new(effectcraft_host::FileExporter { sink: Some(sink) })), + ..Default::default() + }; + SESSION.with(|c| *c.borrow_mut() = Some(s)); +} + +/// A footage file for the jobs that follow. +#[wasm_bindgen(js_name = workerFile)] +pub fn worker_file(path: String, bytes: js_sys::Uint8Array) { + let data: Arc<[u8]> = bytes.to_vec().into(); + crate::files::STORE.put_file(&path, data.clone(), false); + POOL.with(|p| { + if let Some(p) = p.borrow().as_ref() { + p.add_bytes(&path, data); + } + }); +} + +/// Run one job (a JSON [`WorkerRequest`]) and post its replies. +#[wasm_bindgen(js_name = workerJob)] +pub fn worker_job(json: String) { + let send: effectcraft_engine::offload::Post = Rc::new(|r: WorkerReply| { + let done = matches!(r, WorkerReply::Done); + let t = serde_json::to_string(&r).unwrap_or_default(); + post(&obj(&[("type", "reply".into()), ("json", t.into()), ("done", done.into())]), None); + }); + let req: WorkerRequest = match serde_json::from_str(&json) { + Ok(r) => r, + Err(e) => { + send(WorkerReply::Failed { error: format!("bad request: {e}") }); + send(WorkerReply::Done); + return; + } + }; + SESSION.with(|c| { + let mut c = c.borrow_mut(); + let Some(s) = c.as_mut() else { + send(WorkerReply::Failed { error: "worker not initialised".into() }); + send(WorkerReply::Done); + return; + }; + effectcraft_engine::offload::run_request(s, req, &send); + }); +} diff --git a/apps/effectcraft-web/tests/smoke.mjs b/apps/effectcraft-web/tests/smoke.mjs index de2f7f53..3ba19343 100644 --- a/apps/effectcraft-web/tests/smoke.mjs +++ b/apps/effectcraft-web/tests/smoke.mjs @@ -4,10 +4,15 @@ // cargo xtask web --serve 8765 & # build + serve /web/dist // node apps/effectcraft-web/tests/smoke.mjs --url http://127.0.0.1:8765/ --out /tmp/ec-web // -// Steps: load (timing), demo comp in the viewer (screenshot, viewer pixels), `render.frame` -// through `window.effectcraft`, Render Queue GIF and PNG-sequence (.zip) exports (downloads), File ▸ Save As (download), -// re-opening the saved project through `effectcraft.addFile`. Prints a JSON report; exits non-zero -// on failure. `--headed` shows the browser. +// Steps: load (timing), the GPU path (WebGPU → GPU compositor, viewer frames on the GPU), demo +// comp in the viewer (screenshot, viewer pixels), `render.frame` through `window.effectcraft`, +// Render Queue GIF and PNG-sequence (.zip) exports (downloads), a background Render Queue job in +// a Web Worker while the page stays responsive (event-loop gaps measured), audio (the +// AudioContext starts on a user gesture; preview playback feeds it), File ▸ Save As (download), +// re-opening the saved project through `effectcraft.addFile`, persistence across a reload (the +// session, imported media, settings, a project saved to browser storage, Open Recent) and an +// offline reload through the service worker. Prints a JSON report; exits non-zero on failure. +// `--headed` shows the browser. import { spawn } from "node:child_process"; import { mkdirSync, writeFileSync, readdirSync, statSync, mkdtempSync } from "node:fs"; import { join } from "node:path"; @@ -85,6 +90,22 @@ async function js(expr, timeout = 300000) { return r.result.value; } +const check = (cond, msg) => { + if (!cond) throw new Error(`check failed: ${msg}`); +}; + +// Wait until the app runs (after a navigation or reload). +async function ready(ms = 180000) { + await until("!!(window.effectcraftLoad && (window.effectcraftLoad.readyMs || window.effectcraftLoad.error))", ms); + const load = await js("window.effectcraftLoad"); + if (load.error) throw new Error(load.error); + return load; +} + +async function idle() { + await until(`effectcraft.execute("renderQueue.list", {}).then(r => !r.rendering)`, 300000, 200); +} + async function shot(name) { const r = await send("Page.captureScreenshot", { format: "png" }); const p = join(out, `${name}.png`); @@ -121,11 +142,11 @@ try { await send("Emulation.setDeviceMetricsOverride", { width: 1600, height: 1000, deviceScaleFactor: 1, mobile: false }); const t0 = Date.now(); await send("Page.navigate", { url }); - await until("!!(window.effectcraftLoad && (window.effectcraftLoad.readyMs || window.effectcraftLoad.error))", 180000); - report.load = await js("window.effectcraftLoad"); + report.load = await ready(); report.load.wallMs = Date.now() - t0; - if (report.load.error) throw new Error(report.load.error); report.info = await js("effectcraft.info()"); + // Storage: OPFS where the browser writes it (Chrome does), else IndexedDB. + check(["opfs", "indexeddb"].includes(report.info.storage?.backend), `storage backend ${JSON.stringify(report.info.storage)}`); // The demo comp in the viewer: wait for the first viewer frame (renderMs > 0). const ms = await until("effectcraft.inspect().then(i => i.renderMs > 0)", 120000, 500); @@ -133,6 +154,15 @@ try { const insp = await js("effectcraft.inspect()"); report.steps.viewer = { waitMs: ms, renderMs: insp.renderMs, activeComp: insp.activeComp, window: insp.window, screenshot: await shot("01-demo") }; + // GPU: with WebGPU, eframe runs on it and the compositor renders viewer frames on the GPU + // (no readback); without it, WebGL2 / CPU. + const info = await js("effectcraft.info()"); + report.steps.gpu = { webgpu: info.webgpu, backend: info.backend, ...info.gpu }; + if (info.webgpu && info.backend === "BrowserWebGpu") { + check(info.gpu && info.gpu.compositor, "GPU compositor on WebGPU"); + check(info.gpu.viewerOnGpu, "viewer frames on the GPU"); + } + // Comp pixels through the API. const f = await js("effectcraft.renderFrame({max_side: 480}).then(r => ({w: r.width, h: r.height, pngBytes: atob(r.png).length}))"); report.steps.renderFrame = f; @@ -145,12 +175,74 @@ try { await js(`effectcraft.execute("renderQueue.render", {})`); const gif = await waitDownload((n) => n.endsWith(".gif")); report.steps.gifExport = { ...gif, ms: Date.now() - rqT }; + await idle(); // An image sequence arrives as one .zip. await js(`effectcraft.execute("renderQueue.setRender", {index: 1, render: false})`); await js(`effectcraft.request("ui.menu.invoke", {id: "renderQueue.add", params: {format: "png", output: "seq_[####].png", resolution: 0.125, timeSpan: "custom", start: 0, end: 0.2}})`); await js(`effectcraft.execute("renderQueue.render", {})`); report.steps.pngSequence = await waitDownload((n) => n.endsWith(".zip")); + await idle(); + + // A background render (`wait: false`, what the Render button does) runs in a Web Worker: the + // page keeps its event loop. Measure the longest gap between 10 ms timers while it renders. + await js(`effectcraft.execute("renderQueue.setRender", {index: 2, render: false})`); + await js(`effectcraft.request("ui.menu.invoke", {id: "renderQueue.add", params: {format: "gif", output: "bg_[compName].gif", resolution: 0.5, timeSpan: "custom", start: 0, end: 4}})`); + report.steps.backgroundRender = await js(`(async () => { + const t0 = performance.now(); + const r = await effectcraft.execute("renderQueue.render", {wait: false}); + const callMs = performance.now() - t0; + let maxGapMs = 0, last = performance.now(), ticks = 0, progress = 0; + for (;;) { + await new Promise((res) => setTimeout(res, 10)); + const now = performance.now(); + maxGapMs = Math.max(maxGapMs, now - last); + last = now; + if (++ticks % 5) continue; + const l = await effectcraft.execute("renderQueue.list", {}); + if (l.progress) progress = Math.max(progress, l.progress.done); + if (!l.rendering) break; + } + return {startedRendering: r.rendering, callMs, renderMs: performance.now() - t0, maxGapMs, ticks, progressFrames: progress, workers: effectcraft.info().workers}; + })()`); + const bg = report.steps.backgroundRender; + bg.download = await waitDownload((n) => n.startsWith("bg_") && n.endsWith(".gif")); + check(bg.startedRendering, "render started in the background"); + check(bg.maxGapMs < 400, `event loop blocked ${bg.maxGapMs} ms during the render`); + check(bg.progressFrames > 0, "progress reported while rendering"); + + // Audio: the AudioContext starts on a user gesture. + report.steps.audio = { before: (await js("effectcraft.info()")).audio }; + check(report.steps.audio.before.state !== "running", "audio context not running before a gesture"); + // (a click in the window's bottom-right corner, the status bar: modifier keys alone don't + // count as user activation) + await send("Input.dispatchMouseEvent", { type: "mousePressed", x: 1590, y: 995, button: "left", clickCount: 1 }); + await send("Input.dispatchMouseEvent", { type: "mouseReleased", x: 1590, y: 995, button: "left", clickCount: 1 }); + await until("effectcraft.info().audio.state === 'running'", 10000); + report.steps.audio.afterGesture = (await js("effectcraft.info()")).audio; + // A 2 s tone in the comp; preview playback feeds the output and the meters. + await js(`(async () => { + const rate = 48000, n = 2 * rate, b = new ArrayBuffer(44 + n * 4), v = new DataView(b); + const w = (o, t) => [...t].forEach((c, i) => v.setUint8(o + i, c.charCodeAt(0))); + w(0, "RIFF"); v.setUint32(4, 36 + n * 4, true); w(8, "WAVE"); w(12, "fmt "); v.setUint32(16, 16, true); + v.setUint16(20, 1, true); v.setUint16(22, 2, true); v.setUint32(24, rate, true); v.setUint32(28, rate * 4, true); + v.setUint16(32, 4, true); v.setUint16(34, 16, true); w(36, "data"); v.setUint32(40, n * 4, true); + for (let i = 0; i < n; i++) { const x = Math.round(Math.sin(2 * Math.PI * 440 * i / rate) * 0.5 * 32767); v.setInt16(44 + 4 * i, x, true); v.setInt16(46 + 4 * i, x, true); } + return effectcraft.addFile(new File([b], "tone.wav")); + })()`); + await until(`effectcraft.execute("layer.addItem", {item: "tone.wav", time: 0}).then(() => true, () => false)`, 20000, 300); + await js(`effectcraft.request("ui.playback", {action: "stop"})`); + await js(`effectcraft.execute("time.set", {time: 0})`); + await js(`effectcraft.request("ui.playback", {action: "play"})`); + await sleep(1500); + const pb = await js(`effectcraft.request("ui.playback", {action: "status"})`); + const au = (await js("effectcraft.info()")).audio; + await js(`effectcraft.request("ui.playback", {action: "stop"})`); + report.steps.audio.playing = { playback: pb, output: au }; + check(pb.audio, "audio preview running"); + check(au.posted > 0.5 * au.sampleRate, `audio fed (${au.posted} frames)`); + check(au.played > 0, "audio clock advancing"); + check(Math.max(...(pb.levelsDb || [-99])) > -20, `meters (${JSON.stringify(pb.levelsDb)})`); // Save As → .ecproj download; reopen it. await js(`effectcraft.execute("file.saveAs", {path: "/smoke.ecproj"})`); @@ -162,6 +254,60 @@ try { if (!comps) throw new Error("reopened project has no active comp"); report.steps.reopen = { ...opened, activeComp: comps }; report.steps.final = { screenshot: await shot("02-after") }; + + // Persistence: the session, imported media, settings and a project saved to browser storage + // survive a reload; File ▸ Open Recent reopens the project. + await js(`effectcraft.execute("layer.newSolid", {name: "Persisted Solid", color: "#22aa66"})`); + await js(`effectcraft.execute("prefs.set", {key: "general.recentItems", value: 7})`); + const saved = await js(`effectcraft.saveToBrowser("/persist-test.ecproj")`); + await js(`effectcraft.execute("layer.newSolid", {name: "Unsaved Solid"})`); + await sleep(1500); // the session snapshot is taken at most every second + await js("effectcraft.flush()"); + const before = await js("effectcraft.listStored()"); + const tr = Date.now(); + await send("Page.reload", {}); + await sleep(500); + const load2 = await ready(); + const after = await js("effectcraft.listStored()"); + const info2 = await js("effectcraft.info()"); + const layers = await js(`effectcraft.execute("comp.info", {}).then(c => c.layers.map(l => l.name))`); + const recentItems = await js(`effectcraft.execute("prefs.get", {key: "general.recentItems"})`); + const rec = await js(`effectcraft.execute("file.recoveryInfo", {})`); + const paths = after.files.map((f) => f.path); + report.steps.persistence = { saved, reloadMs: Date.now() - tr, load: load2, restored: info2.restored, storage: info2.storage, layers, recentItems, recent: rec.recentProjects, stored: paths, usage: after.usage, filesBefore: before.files.length }; + check(info2.restored, "session restored after reload"); + check(layers.includes("Persisted Solid") && layers.includes("Unsaved Solid"), `layers after reload: ${layers}`); + check(paths.includes("/tone.wav") && paths.includes("/persist-test.ecproj"), `stored files: ${paths}`); + check(recentItems === 7, `setting after reload: ${recentItems}`); + check(rec.recentProjects[0] === "/persist-test.ecproj", `recent projects: ${rec.recentProjects}`); + await js(`effectcraft.execute("file.openRecent", {index: 0})`); + const reopened = await js(`effectcraft.execute("comp.info", {}).then(c => c.layers.map(l => l.name))`); + check(reopened.includes("Persisted Solid") && !reopened.includes("Unsaved Solid"), `Open Recent: ${reopened}`); + // The imported tone is still in the project and decodes from storage after the reload. + const tone = await js(`effectcraft.execute("layer.addItem", {item: "tone.wav", time: 0}).then(() => "ok", (e) => String(e))`); + check(tone === "ok", `tone.wav after reload: ${tone}`); + report.steps.persistence.openRecent = reopened; + + // PWA: manifest, service worker in control, and an offline reload. + const pwa = await js(`(async () => { + const m = await (await fetch("manifest.webmanifest")).json(); + const reg = await navigator.serviceWorker.getRegistration(); + const keys = await caches.keys(); + const cached = keys.length ? (await (await caches.open(keys[0])).keys()).length : 0; + return {manifest: m.name, icons: m.icons.length, display: m.display, sw: !!(reg && reg.active), controlled: !!navigator.serviceWorker.controller, caches: keys, cached}; + })()`); + check(pwa.sw && pwa.cached > 4, `service worker: ${JSON.stringify(pwa)}`); + await send("Network.enable", {}); + await send("Network.emulateNetworkConditions", { offline: true, latency: 0, downloadThroughput: -1, uploadThroughput: -1 }); + await send("Page.reload", {}); + await sleep(500); + const offline = await ready(); + const info3 = await js("effectcraft.info()"); + await send("Network.emulateNetworkConditions", { offline: false, latency: 0, downloadThroughput: -1, uploadThroughput: -1 }); + report.steps.pwa = { ...pwa, offlineLoad: offline, offlineControlled: info3.serviceWorker, crossOriginIsolated: info3.crossOriginIsolated }; + check(info3.serviceWorker, "offline reload served by the service worker"); + check(info3.crossOriginIsolated, "cross-origin isolated under the service worker"); + report.steps.final2 = { screenshot: await shot("03-offline") }; } catch (e) { failed = true; report.error = String(e && e.stack || e); diff --git a/apps/effectcraft-web/web/audio-worklet.js b/apps/effectcraft-web/web/audio-worklet.js new file mode 100644 index 00000000..97756c52 --- /dev/null +++ b/apps/effectcraft-web/web/audio-worklet.js @@ -0,0 +1,44 @@ +// EffectCraft audio output (AudioWorklet): plays interleaved stereo blocks posted by the page +// (src/audio.rs feeds them from the preview mixdown). Running dry plays silence, reported back +// as underrun frames. +class EffectcraftOutput extends AudioWorkletProcessor { + constructor() { + super(); + this.queue = []; + this.offset = 0; + this.stopped = false; + this.port.onmessage = (e) => { + if (e.data === "stop") this.stopped = true; + else this.queue.push(e.data); + }; + } + + process(_inputs, outputs) { + const out = outputs[0]; + const l = out[0]; + const r = out[1] ?? out[0]; + let i = 0; + while (i < l.length && this.queue.length) { + const b = this.queue[0]; + const n = Math.min(l.length - i, (b.length - this.offset) >> 1); + for (let k = 0; k < n; k++) { + l[i + k] = b[this.offset + 2 * k]; + r[i + k] = b[this.offset + 2 * k + 1]; + } + i += n; + this.offset += 2 * n; + if (this.offset >= b.length) { + this.queue.shift(); + this.offset = 0; + } + } + if (i < l.length) { + l.fill(0, i); + r.fill(0, i); + if (!this.stopped) this.port.postMessage({ underrun: l.length - i }); + } + return !this.stopped; + } +} + +registerProcessor("effectcraft-output", EffectcraftOutput); diff --git a/apps/effectcraft-web/web/index.html b/apps/effectcraft-web/web/index.html index 3e6cbf06..7c6c6c3c 100644 --- a/apps/effectcraft-web/web/index.html +++ b/apps/effectcraft-web/web/index.html @@ -4,8 +4,12 @@ + + EffectCraft + +